From 2fd8a3865c88163f58bf785f179b459aed85b64f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 12 Feb 2025 21:05:54 +0100 Subject: [PATCH] index: automatically write full indexes in StorePack --- internal/repository/index/associated_data_test.go | 12 ++++++------ internal/repository/index/master_index.go | 15 ++++++++++----- internal/repository/packer_manager.go | 5 +---- internal/repository/repository.go | 6 ++++-- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/internal/repository/index/associated_data_test.go b/internal/repository/index/associated_data_test.go index 82dd9908d..d811a7faa 100644 --- a/internal/repository/index/associated_data_test.go +++ b/internal/repository/index/associated_data_test.go @@ -35,8 +35,8 @@ func TestAssociatedSet(t *testing.T) { bh, blob := makeFakePackedBlob() mi := NewMasterIndex() - mi.StorePack(blob.PackID, []restic.Blob{blob.Blob}) - test.OK(t, mi.SaveIndex(context.TODO(), &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob.PackID, []restic.Blob{blob.Blob}, &noopSaver{})) + test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) bs := NewAssociatedSet[uint8](mi) test.Equals(t, bs.Len(), 0) @@ -118,15 +118,15 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) { _, blob := makeFakePackedBlob() mi := NewMasterIndex() - mi.StorePack(blob.PackID, []restic.Blob{blob.Blob}) - test.OK(t, mi.SaveIndex(context.TODO(), &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob.PackID, []restic.Blob{blob.Blob}, &noopSaver{})) + test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) bs := NewAssociatedSet[uint8](mi) // add new blobs to index after building the set of, blob2 := makeFakePackedBlob() - mi.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) - test.OK(t, mi.SaveIndex(context.TODO(), &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, []restic.Blob{blob2.Blob}, &noopSaver{})) + test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) // non-existent test.Equals(t, false, bs.Has(of)) diff --git a/internal/repository/index/master_index.go b/internal/repository/index/master_index.go index 16923090b..827f8d9f9 100644 --- a/internal/repository/index/master_index.go +++ b/internal/repository/index/master_index.go @@ -153,7 +153,12 @@ func (mi *MasterIndex) Insert(idx *Index) { } // StorePack remembers the id and pack in the index. -func (mi *MasterIndex) StorePack(id restic.ID, blobs []restic.Blob) { +func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs []restic.Blob, r restic.SaverUnpacked[restic.FileType]) error { + mi.storePack(id, blobs) + return mi.saveFullIndex(ctx, r) +} + +func (mi *MasterIndex) storePack(id restic.ID, blobs []restic.Blob) { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() @@ -589,13 +594,13 @@ func (mi *MasterIndex) saveIndex(ctx context.Context, r restic.SaverUnpacked[res return mi.MergeFinalIndexes() } -// SaveIndex saves all new indexes in the backend. -func (mi *MasterIndex) SaveIndex(ctx context.Context, r restic.SaverUnpacked[restic.FileType]) error { +// Flush saves all new indexes in the backend. +func (mi *MasterIndex) Flush(ctx context.Context, r restic.SaverUnpacked[restic.FileType]) error { return mi.saveIndex(ctx, r, mi.finalizeNotFinalIndexes()...) } -// SaveFullIndex saves all full indexes in the backend. -func (mi *MasterIndex) SaveFullIndex(ctx context.Context, r restic.SaverUnpacked[restic.FileType]) error { +// saveFullIndex saves all full indexes in the backend. +func (mi *MasterIndex) saveFullIndex(ctx context.Context, r restic.SaverUnpacked[restic.FileType]) error { return mi.saveIndex(ctx, r, mi.finalizeFullIndexes()...) } diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 9d53c911b..d5c340e39 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -187,8 +187,5 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *packe // update blobs in the index debug.Log(" updating blobs %v to pack %v", p.Packer.Blobs(), id) - r.idx.StorePack(id, p.Packer.Blobs()) - - // Save index if full - return r.idx.SaveFullIndex(ctx, &internalRepository{r}) + return r.idx.StorePack(ctx, id, p.Packer.Blobs(), &internalRepository{r}) } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index aee0db103..8f8131b96 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -542,7 +542,7 @@ func (r *Repository) Flush(ctx context.Context) error { return err } - return r.idx.SaveIndex(ctx, &internalRepository{r}) + return r.idx.Flush(ctx, &internalRepository{r}) } func (r *Repository) StartPackUploader(ctx context.Context, wg *errgroup.Group) { @@ -701,7 +701,9 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest invalid = append(invalid, fi.ID) m.Unlock() } - r.idx.StorePack(fi.ID, entries) + if err := r.idx.StorePack(ctx, fi.ID, entries, &internalRepository{r}); err != nil { + return err + } p.Add(1) }