MirrorSave
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue: 4942: cmd_rewrite: add snapshot summary data to an existing snapshot. (#5185) 

Co-authored-by: Michael Eischer <michael.eischer@fau.de>
This commit is contained in:
Winfried Plappert 2025-02-05 19:40:20 +00:00 committed by GitHub
parent 060a44202f
commit 4104a8e6a5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 87 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
changelog/unreleased/issue-4942 Normal file
View File

@ -0,0 +1,11 @@
Enhancement: support creating snapshot summary statistics for old snapshots
When `rewrite` is used with the `--snapshot-summary` option, a new snapshot is
created containing statistics summary data. Only two fields in the summary will
be non-zero: `TotalFilesProcessed` and `TotalBytesProcessed`.
When rewrite is called with one of the `--exclude` options, `TotalFilesProcessed`
and `TotalBytesProcessed` will be updated in the snapshot summary.
https://github.com/restic/restic/issues/4942
https://github.com/restic/restic/pull/5185

5
cmd/restic/cmd_repair_snapshots.go
View File

@ -143,8 +143,9 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
Verbosef("\n%v\n", sn)
changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
return id, nil, err
}, opts.DryRun, opts.Forget, nil, "repaired")
if err != nil {
return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)

50
cmd/restic/cmd_rewrite.go
View File

@ -35,6 +35,13 @@ Please note that the --forget option only removes the snapshots and not the actu
data stored in the repository. In order to delete the no longer referenced data,
use the "prune" command.
When rewrite is used with the --snapshot-summary option, a new snapshot is
created containing statistics summary data. Only two fields in the summary will
be non-zero: TotalFilesProcessed and TotalBytesProcessed.
When rewrite is called with one of the --exclude options, TotalFilesProcessed
and TotalBytesProcessed will be updated in the snapshot summary.
EXIT STATUS
===========
@ -83,8 +90,9 @@ func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) {
// RewriteOptions collects all options for the rewrite command.
type RewriteOptions struct {
Forget bool
DryRun bool
Forget bool
DryRun bool
SnapshotSummary bool
Metadata snapshotMetadataArgs
restic.SnapshotFilter
@ -101,12 +109,15 @@ func init() {
f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname")
f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
f.BoolVarP(&rewriteOptions.SnapshotSummary, "snapshot-summary", "s", false, "create snapshot summary record if it does not exist")
initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
rewriteOptions.ExcludePatternOptions.Add(f)
}
type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
// rewriteFilterFunc returns the filtered tree ID or an error. If a snapshot summary is returned, the snapshot will
// be updated accordingly.
type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error)
func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
if sn.Tree == nil {
@ -126,7 +137,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
var filter rewriteFilterFunc
if len(rejectByNameFuncs) > 0 {
if len(rejectByNameFuncs) > 0 || opts.SnapshotSummary {
selectByName := func(nodepath string) bool {
for _, reject := range rejectByNameFuncs {
if reject(nodepath) {
@ -146,22 +157,24 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode)
filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
if err != nil {
return restic.ID{}, err
return restic.ID{}, nil, err
}
ss := querySize()
summary := &restic.SnapshotSummary{}
if sn.Summary != nil {
sn.Summary.TotalFilesProcessed = ss.FileCount
sn.Summary.TotalBytesProcessed = ss.FileSize
*summary = *sn.Summary
}
return id, err
summary.TotalFilesProcessed = ss.FileCount
summary.TotalBytesProcessed = ss.FileSize
return id, summary, err
}
} else {
filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, error) {
return *sn.Tree, nil
filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
return *sn.Tree, nil, nil
}
}
@ -176,9 +189,10 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
repo.StartPackUploader(wgCtx, wg)
var filteredTree restic.ID
var summary *restic.SnapshotSummary
wg.Go(func() error {
var err error
filteredTree, err = filter(ctx, sn)
filteredTree, summary, err = filter(ctx, sn)
if err != nil {
return err
}
@ -203,7 +217,12 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
return true, nil
}
if filteredTree == *sn.Tree && newMetadata == nil {
matchingSummary := true
if summary != nil {
matchingSummary = sn.Summary != nil && *summary == *sn.Summary
}
if filteredTree == *sn.Tree && newMetadata == nil && matchingSummary {
debug.Log("Snapshot %v not modified", sn)
return false, nil
}
@ -230,6 +249,9 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
// Always set the original snapshot id as this essentially a new snapshot.
sn.Original = sn.ID()
sn.Tree = &filteredTree
if summary != nil {
sn.Summary = summary
}
if !forget {
sn.AddTags([]string{addTag})
@ -263,7 +285,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
}
func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
if opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
if !opts.SnapshotSummary && opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
}

33
cmd/restic/cmd_rewrite_integration_test.go
View File

@ -139,3 +139,36 @@ func TestRewriteMetadata(t *testing.T) {
testRewriteMetadata(t, metadata)
}
}
func TestRewriteSnaphotSummary(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
createBasicRewriteRepo(t, env)
rtest.OK(t, runRewrite(context.TODO(), RewriteOptions{SnapshotSummary: true}, env.gopts, []string{}))
// no new snapshot should be created as the snapshot already has a summary
snapshots := testListSnapshots(t, env.gopts, 1)
// replace snapshot by one without a summary
_, repo, unlock, err := openWithExclusiveLock(context.TODO(), env.gopts, false)
rtest.OK(t, err)
sn, err := restic.LoadSnapshot(context.TODO(), repo, snapshots[0])
rtest.OK(t, err)
oldSummary := sn.Summary
sn.Summary = nil
rtest.OK(t, repo.RemoveUnpacked(context.TODO(), restic.WriteableSnapshotFile, snapshots[0]))
snapshots[0], err = restic.SaveSnapshot(context.TODO(), repo, sn)
rtest.OK(t, err)
unlock()
// rewrite snapshot and lookup ID of new snapshot
rtest.OK(t, runRewrite(context.TODO(), RewriteOptions{SnapshotSummary: true}, env.gopts, []string{}))
newSnapshots := testListSnapshots(t, env.gopts, 2)
newSnapshot := restic.NewIDSet(newSnapshots...).Sub(restic.NewIDSet(snapshots...)).List()[0]
sn, err = restic.LoadSnapshot(context.TODO(), repo, newSnapshot)
rtest.OK(t, err)
rtest.Assert(t, sn.Summary != nil, "snapshot should have summary attached")
rtest.Equals(t, oldSummary.TotalBytesProcessed, sn.Summary.TotalBytesProcessed, "unexpected TotalBytesProcessed value")
rtest.Equals(t, oldSummary.TotalFilesProcessed, sn.Summary.TotalFilesProcessed, "unexpected TotalFilesProcessed value")
}

4
doc/045_working_with_repos.rst
View File

@ -332,6 +332,10 @@ command, see :ref:`backup-excluding-files` for details.
It is possible to rewrite only a subset of snapshots by filtering them the same
way as for the ``copy`` command, see :ref:`copy-filtering-snapshots`.
The option ``--snapshot-summary`` can be used to attach summary data to existing
snapshots that do not have this information. When a snapshot summary is created
the only fields added are ``TotalFilesProcessed`` and ``TotalBytesProcessed``.
By default, the ``rewrite`` command will keep the original snapshots and create
new ones for every snapshot which was modified during rewriting. The new
snapshots are marked with the tag ``rewrite`` to differentiate them from the