Issue: 4942: cmd_rewrite: add snapshot summary data to an existing snapshot. (#5185)

Co-authored-by: Michael Eischer <michael.eischer@fau.de>

changelog/unreleased/issue-4942

@@ -0,0 +1,11 @@
+Enhancement: support creating snapshot summary statistics for old snapshots
+
+When `rewrite` is used with the `--snapshot-summary` option, a new snapshot is
+created containing statistics summary data. Only two fields in the summary will
+be non-zero: `TotalFilesProcessed` and `TotalBytesProcessed`.
+
+When rewrite is called with one of the `--exclude` options, `TotalFilesProcessed`
+and `TotalBytesProcessed` will be updated in the snapshot summary.
+
+https://github.com/restic/restic/issues/4942
+https://github.com/restic/restic/pull/5185

cmd/restic/cmd_repair_snapshots.go

@@ -143,8 +143,9 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
 		Verbosef("\n%v\n", sn)
 		changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
-			func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
+			func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
-				return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
+				id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
+				return id, nil, err
 			}, opts.DryRun, opts.Forget, nil, "repaired")
 		if err != nil {
 			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)

cmd/restic/cmd_rewrite.go

@@ -35,6 +35,13 @@ Please note that the --forget option only removes the snapshots and not the actu
 data stored in the repository. In order to delete the no longer referenced data,
 use the "prune" command.
 
+When rewrite is used with the --snapshot-summary option, a new snapshot is
+created containing statistics summary data. Only two fields in the summary will
+be non-zero: TotalFilesProcessed and TotalBytesProcessed.
+
+When rewrite is called with one of the --exclude options, TotalFilesProcessed
+and TotalBytesProcessed will be updated in the snapshot summary.
+
 EXIT STATUS
 ===========
 
@@ -85,6 +92,7 @@ func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) {
 type RewriteOptions struct {
 	Forget          bool
 	DryRun          bool
+	SnapshotSummary bool
 
 	Metadata snapshotMetadataArgs
 	restic.SnapshotFilter
@@ -101,12 +109,15 @@ func init() {
 	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
 	f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname")
 	f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
+	f.BoolVarP(&rewriteOptions.SnapshotSummary, "snapshot-summary", "s", false, "create snapshot summary record if it does not exist")
 
 	initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
 	rewriteOptions.ExcludePatternOptions.Add(f)
 }
 
-type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
+// rewriteFilterFunc returns the filtered tree ID or an error. If a snapshot summary is returned, the snapshot will
+// be updated accordingly.
+type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error)
 
 func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
 	if sn.Tree == nil {
@@ -126,7 +137,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 
 	var filter rewriteFilterFunc
 
-	if len(rejectByNameFuncs) > 0 {
+	if len(rejectByNameFuncs) > 0 || opts.SnapshotSummary {
 		selectByName := func(nodepath string) bool {
 			for _, reject := range rejectByNameFuncs {
 				if reject(nodepath) {
@@ -146,22 +157,24 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 
 		rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode)
 
-		filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
+		filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
 			id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
 			if err != nil {
-				return restic.ID{}, err
+				return restic.ID{}, nil, err
 			}
 			ss := querySize()
+			summary := &restic.SnapshotSummary{}
 			if sn.Summary != nil {
-				sn.Summary.TotalFilesProcessed = ss.FileCount
+				*summary = *sn.Summary
-				sn.Summary.TotalBytesProcessed = ss.FileSize
 			}
-			return id, err
+			summary.TotalFilesProcessed = ss.FileCount
+			summary.TotalBytesProcessed = ss.FileSize
+			return id, summary, err
 		}
 
 	} else {
-		filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, error) {
+		filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
-			return *sn.Tree, nil
+			return *sn.Tree, nil, nil
 		}
 	}
 
@@ -176,9 +189,10 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 	repo.StartPackUploader(wgCtx, wg)
 
 	var filteredTree restic.ID
+	var summary *restic.SnapshotSummary
 	wg.Go(func() error {
 		var err error
-		filteredTree, err = filter(ctx, sn)
+		filteredTree, summary, err = filter(ctx, sn)
 		if err != nil {
 			return err
 		}
@@ -203,7 +217,12 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		return true, nil
 	}
 
-	if filteredTree == *sn.Tree && newMetadata == nil {
+	matchingSummary := true
+	if summary != nil {
+		matchingSummary = sn.Summary != nil && *summary == *sn.Summary
+	}
+
+	if filteredTree == *sn.Tree && newMetadata == nil && matchingSummary {
 		debug.Log("Snapshot %v not modified", sn)
 		return false, nil
 	}
@@ -230,6 +249,9 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 	// Always set the original snapshot id as this essentially a new snapshot.
 	sn.Original = sn.ID()
 	sn.Tree = &filteredTree
+	if summary != nil {
+		sn.Summary = summary
+	}
 
 	if !forget {
 		sn.AddTags([]string{addTag})
@@ -263,7 +285,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 }
 
 func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
-	if opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
+	if !opts.SnapshotSummary && opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
 		return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
 	}
 

cmd/restic/cmd_rewrite_integration_test.go

@@ -139,3 +139,36 @@ func TestRewriteMetadata(t *testing.T) {
 		testRewriteMetadata(t, metadata)
 	}
 }
+
+func TestRewriteSnaphotSummary(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	createBasicRewriteRepo(t, env)
+
+	rtest.OK(t, runRewrite(context.TODO(), RewriteOptions{SnapshotSummary: true}, env.gopts, []string{}))
+	// no new snapshot should be created as the snapshot already has a summary
+	snapshots := testListSnapshots(t, env.gopts, 1)
+
+	// replace snapshot by one without a summary
+	_, repo, unlock, err := openWithExclusiveLock(context.TODO(), env.gopts, false)
+	rtest.OK(t, err)
+	sn, err := restic.LoadSnapshot(context.TODO(), repo, snapshots[0])
+	rtest.OK(t, err)
+	oldSummary := sn.Summary
+	sn.Summary = nil
+	rtest.OK(t, repo.RemoveUnpacked(context.TODO(), restic.WriteableSnapshotFile, snapshots[0]))
+	snapshots[0], err = restic.SaveSnapshot(context.TODO(), repo, sn)
+	rtest.OK(t, err)
+	unlock()
+
+	// rewrite snapshot and lookup ID of new snapshot
+	rtest.OK(t, runRewrite(context.TODO(), RewriteOptions{SnapshotSummary: true}, env.gopts, []string{}))
+	newSnapshots := testListSnapshots(t, env.gopts, 2)
+	newSnapshot := restic.NewIDSet(newSnapshots...).Sub(restic.NewIDSet(snapshots...)).List()[0]
+
+	sn, err = restic.LoadSnapshot(context.TODO(), repo, newSnapshot)
+	rtest.OK(t, err)
+	rtest.Assert(t, sn.Summary != nil, "snapshot should have summary attached")
+	rtest.Equals(t, oldSummary.TotalBytesProcessed, sn.Summary.TotalBytesProcessed, "unexpected TotalBytesProcessed value")
+	rtest.Equals(t, oldSummary.TotalFilesProcessed, sn.Summary.TotalFilesProcessed, "unexpected TotalFilesProcessed value")
+}

doc/045_working_with_repos.rst

@@ -332,6 +332,10 @@ command, see :ref:`backup-excluding-files` for details.
 It is possible to rewrite only a subset of snapshots by filtering them the same
 way as for the ``copy`` command, see :ref:`copy-filtering-snapshots`.
 
+The option ``--snapshot-summary`` can be used to attach summary data to existing
+snapshots that do not have this information. When a snapshot summary is created
+the only fields added are ``TotalFilesProcessed`` and ``TotalBytesProcessed``.
+
 By default, the ``rewrite`` command will keep the original snapshots and create
 new ones for every snapshot which was modified during rewriting. The new
 snapshots are marked with the tag ``rewrite`` to differentiate them from the