Merge pull request #998 from restic/fix-820

fuse: Add cache for blob sizes

CHANGELOG.md

@@ -20,6 +20,10 @@ Important Changes in 0.X.Y
    https://github.com/restic/restic/issues/989
    https://github.com/restic/restic/pull/993
 
+ * Improved performance for the fuse mount: Listing directories which contain
+   large files now is significantly faster.
+   https://github.com/restic/restic/pull/998
+
 Important Changes in 0.6.1
 ==========================
 

src/restic/backend/s3/s3.go

@@ -8,7 +8,6 @@ import (
 	"path"
 	"restic"
 	"strings"
-	"sync"
 	"time"
 
 	"restic/backend"
@@ -23,12 +22,10 @@ const connLimit = 10
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
-	client       *minio.Client
+	client     *minio.Client
-	sem          *backend.Semaphore
+	sem        *backend.Semaphore
-	bucketname   string
+	bucketname string
-	prefix       string
+	prefix     string
-	cacheMutex   sync.RWMutex
-	cacheObjSize map[string]int64
 	backend.Layout
 }
 
@@ -53,11 +50,10 @@ func Open(cfg Config) (restic.Backend, error) {
 	}
 
 	be := &s3{
-		client:       client,
+		client:     client,
-		sem:          sem,
+		sem:        sem,
-		bucketname:   cfg.Bucket,
+		bucketname: cfg.Bucket,
-		prefix:       cfg.Prefix,
+		prefix:     cfg.Prefix,
-		cacheObjSize: make(map[string]int64),
 	}
 
 	client.SetCustomTransport(backend.Transport())

src/restic/fuse/dir.go

@@ -24,9 +24,11 @@ type dir struct {
 	inode       uint64
 	node        *restic.Node
 	ownerIsRoot bool
+
+	blobsize *BlobSizeCache
 }
 
-func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, ownerIsRoot bool) (*dir, error) {
+func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, ownerIsRoot bool, blobsize *BlobSizeCache) (*dir, error) {
 	debug.Log("new dir for %v (%v)", node.Name, node.Subtree.Str())
 	tree, err := repo.LoadTree(ctx, *node.Subtree)
 	if err != nil {
@@ -44,6 +46,7 @@ func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, owne
 		items:       items,
 		inode:       node.Inode,
 		ownerIsRoot: ownerIsRoot,
+		blobsize:    blobsize,
 	}, nil
 }
 
@@ -66,7 +69,7 @@ func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *rest
 	return tree.Nodes, nil
 }
 
-func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot SnapshotWithId, ownerIsRoot bool) (*dir, error) {
+func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot SnapshotWithId, ownerIsRoot bool, blobsize *BlobSizeCache) (*dir, error) {
 	debug.Log("new dir for snapshot %v (%v)", snapshot.ID.Str(), snapshot.Tree.Str())
 	tree, err := repo.LoadTree(ctx, *snapshot.Tree)
 	if err != nil {
@@ -99,6 +102,7 @@ func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot Sn
 		items:       items,
 		inode:       inodeFromBackendID(snapshot.ID),
 		ownerIsRoot: ownerIsRoot,
+		blobsize:    blobsize,
 	}, nil
 }
 
@@ -167,9 +171,9 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 	}
 	switch node.Type {
 	case "dir":
-		return newDir(ctx, d.repo, node, d.ownerIsRoot)
+		return newDir(ctx, d.repo, node, d.ownerIsRoot, d.blobsize)
 	case "file":
-		return newFile(d.repo, node, d.ownerIsRoot)
+		return newFile(d.repo, node, d.ownerIsRoot, d.blobsize)
 	case "symlink":
 		return newLink(d.repo, node, d.ownerIsRoot)
 	default:

src/restic/fuse/file.go

@@ -41,14 +41,17 @@ type file struct {
 
 const defaultBlobSize = 128 * 1024
 
-func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
+func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool, blobsize *BlobSizeCache) (fusefile *file, err error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	var bytes uint64
 	sizes := make([]int, len(node.Content))
 	for i, id := range node.Content {
-		size, err := repo.LookupBlobSize(id, restic.DataBlob)
+		size, ok := blobsize.Lookup(id)
-		if err != nil {
+		if !ok {
-			return nil, err
+			size, err = repo.LookupBlobSize(id, restic.DataBlob)
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		sizes[i] = int(size)

src/restic/fuse/file_test.go

@@ -108,7 +108,7 @@ func TestFuseFile(t *testing.T) {
 		Size:    filesize,
 		Content: content,
 	}
-	f, err := newFile(repo, node, false)
+	f, err := newFile(repo, node, false, nil)
 	OK(t, err)
 
 	attr := fuse.Attr{}

src/restic/fuse/snapshot.go

@@ -14,10 +14,39 @@ import (
 
 	"restic"
 	"restic/debug"
+	"restic/repository"
 
 	"golang.org/x/net/context"
 )
 
+// BlobSizeCache caches the size of blobs in the repo.
+type BlobSizeCache struct {
+	m map[restic.ID]uint
+}
+
+// NewBlobSizeCache returns a new blob size cache containing all entries from midx.
+func NewBlobSizeCache(midx *repository.MasterIndex) *BlobSizeCache {
+	m := make(map[restic.ID]uint, 1000)
+	for _, idx := range midx.All() {
+		for pb := range idx.Each(nil) {
+			m[pb.ID] = pb.Length
+		}
+	}
+	return &BlobSizeCache{
+		m: m,
+	}
+}
+
+// Lookup returns the size of the blob id.
+func (c *BlobSizeCache) Lookup(id restic.ID) (size uint, found bool) {
+	if c == nil {
+		return 0, false
+	}
+
+	size, found = c.m[id]
+	return size, found
+}
+
 type SnapshotWithId struct {
 	*restic.Snapshot
 	restic.ID
@@ -36,6 +65,8 @@ type SnapshotsDir struct {
 	tags        []string
 	host        string
 
+	blobsize *BlobSizeCache
+
 	// knownSnapshots maps snapshot timestamp to the snapshot
 	sync.RWMutex
 	knownSnapshots map[string]SnapshotWithId
@@ -53,6 +84,7 @@ func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, t
 		host:           host,
 		knownSnapshots: make(map[string]SnapshotWithId),
 		processed:      restic.NewIDSet(),
+		blobsize:       NewBlobSizeCache(repo.Index().(*repository.MasterIndex)),
 	}
 }
 
@@ -158,5 +190,5 @@ func (sn *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error
 		}
 	}
 
-	return newDirFromSnapshot(ctx, sn.repo, snapshot, sn.ownerIsRoot)
+	return newDirFromSnapshot(ctx, sn.repo, snapshot, sn.ownerIsRoot, sn.blobsize)
 }