From af839f9548c3086c29a2b299a90722e849f5de77 Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Mon, 4 Nov 2024 02:14:45 -0500 Subject: [PATCH 1/7] restore: exclude/include xattrs For: https://github.com/restic/restic/issues/5089 Signed-off-by: Tesshu Flower --- cmd/restic/cmd_restore.go | 48 ++++++++++++++++++++++++++---- internal/fs/node.go | 8 ++--- internal/fs/node_test.go | 9 ++++-- internal/fs/node_xattr.go | 20 ++++++++----- internal/fs/node_xattr_all_test.go | 2 +- internal/restorer/restorer.go | 4 ++- 6 files changed, 70 insertions(+), 21 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 82dd408a8..fc3148ce1 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -54,11 +54,13 @@ type RestoreOptions struct { filter.IncludePatternOptions Target string restic.SnapshotFilter - DryRun bool - Sparse bool - Verify bool - Overwrite restorer.OverwriteBehavior - Delete bool + DryRun bool + Sparse bool + Verify bool + Overwrite restorer.OverwriteBehavior + Delete bool + ExcludeXattrPattern []string + IncludeXattrPattern []string } var restoreOptions RestoreOptions @@ -72,6 +74,9 @@ func init() { restoreOptions.ExcludePatternOptions.Add(flags) restoreOptions.IncludePatternOptions.Add(flags) + flags.StringArrayVar(&restoreOptions.ExcludeXattrPattern, "exclude-xattr", nil, "exclude xattr by `pattern` (can be specified multiple times)") + flags.StringArrayVar(&restoreOptions.IncludeXattrPattern, "include-xattr", nil, "include xattr by `pattern` (can be specified multiple times)") + initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter) flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done") flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse") @@ -96,6 +101,9 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, hasExcludes := len(excludePatternFns) > 0 hasIncludes := len(includePatternFns) > 0 + hasXattrExcludes := len(opts.ExcludeXattrPattern) > 0 + hasXattrIncludes := len(opts.IncludeXattrPattern) > 0 + switch { case len(args) == 0: return errors.Fatal("no snapshot ID specified") @@ -110,6 +118,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, if hasExcludes && hasIncludes { return errors.Fatal("exclude and include patterns are mutually exclusive") } + + if hasXattrExcludes && hasXattrIncludes { + return errors.Fatal("exclude and include xattr patterns are mutually exclusive") + } + if opts.DryRun && opts.Verify { return errors.Fatal("--dry-run and --verify are mutually exclusive") } @@ -219,6 +232,31 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, res.SelectFilter = selectIncludeFilter } + if !hasXattrExcludes && !hasXattrIncludes { + // set default of including xattrs from the 'user' namespace + opts.IncludeXattrPattern = []string{"user.*"} + } + if hasXattrExcludes { + if err := filter.ValidatePatterns(opts.ExcludeXattrPattern); err != nil { + return errors.Fatalf("--exclude-xattr: %s", err) + } + + res.XattrSelectFilter = func(xattrName string) bool { + shouldReject := filter.RejectByPattern(opts.ExcludeXattrPattern, Warnf)(xattrName) + return !shouldReject + } + } else { + // User has either input include xattr pattern(s) or we're using our default include pattern + if err := filter.ValidatePatterns(opts.IncludeXattrPattern); err != nil { + return errors.Fatalf("--include-xattr: %s", err) + } + + res.XattrSelectFilter = func(xattrName string) bool { + shouldInclude, _ := filter.IncludeByPattern(opts.IncludeXattrPattern, Warnf)(xattrName) + return shouldInclude + } + } + if !gopts.JSON { msg.P("restoring %s to %s\n", res.Snapshot(), opts.Target) } diff --git a/internal/fs/node.go b/internal/fs/node.go index 058d9cc7b..ab2aca957 100644 --- a/internal/fs/node.go +++ b/internal/fs/node.go @@ -230,8 +230,8 @@ func mkfifo(path string, mode uint32) (err error) { } // NodeRestoreMetadata restores node metadata -func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error { - err := nodeRestoreMetadata(node, path, warn) +func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error { + err := nodeRestoreMetadata(node, path, warn, xattrSelectFilter) if err != nil { // It is common to have permission errors for folders like /home // unless you're running as root, so ignore those. @@ -246,14 +246,14 @@ func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) return err } -func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error { +func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error { var firsterr error if err := lchown(path, int(node.UID), int(node.GID)); err != nil { firsterr = errors.WithStack(err) } - if err := nodeRestoreExtendedAttributes(node, path); err != nil { + if err := nodeRestoreExtendedAttributes(node, path, xattrSelectFilter); err != nil { debug.Log("error restoring extended attributes for %v: %v", path, err) if firsterr == nil { firsterr = err diff --git a/internal/fs/node_test.go b/internal/fs/node_test.go index 15f19adc7..b67295f68 100644 --- a/internal/fs/node_test.go +++ b/internal/fs/node_test.go @@ -13,6 +13,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -216,7 +217,8 @@ func TestNodeRestoreAt(t *testing.T) { nodePath = filepath.Join(tempdir, test.Name) } rtest.OK(t, NodeCreateAt(&test, nodePath)) - rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) + rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }, + func(_ string) bool { return true } /* restore all xattrs */)) fs := &Local{} meta, err := fs.OpenFile(nodePath, O_NOFOLLOW, true) @@ -291,6 +293,7 @@ func TestNodeRestoreMetadataError(t *testing.T) { nodePath := filepath.Join(tempdir, node.Name) // This will fail because the target file does not exist - err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) - rtest.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") + err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }, + func(_ string) bool { return true }) + test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") } diff --git a/internal/fs/node_xattr.go b/internal/fs/node_xattr.go index 1bdafc575..2a2b5c0fb 100644 --- a/internal/fs/node_xattr.go +++ b/internal/fs/node_xattr.go @@ -65,14 +65,17 @@ func handleXattrErr(err error) error { } } -func nodeRestoreExtendedAttributes(node *restic.Node, path string) error { +func nodeRestoreExtendedAttributes(node *restic.Node, path string, xattrSelectFilter func(xattrName string) bool) error { expectedAttrs := map[string]struct{}{} for _, attr := range node.ExtendedAttributes { - err := setxattr(path, attr.Name, attr.Value) - if err != nil { - return err + // Only restore xattrs that match the filter + if xattrSelectFilter(attr.Name) { + err := setxattr(path, attr.Name, attr.Value) + if err != nil { + return err + } + expectedAttrs[attr.Name] = struct{}{} } - expectedAttrs[attr.Name] = struct{}{} } // remove unexpected xattrs @@ -84,8 +87,11 @@ func nodeRestoreExtendedAttributes(node *restic.Node, path string) error { if _, ok := expectedAttrs[name]; ok { continue } - if err := removexattr(path, name); err != nil { - return err + // Only attempt to remove xattrs that match the filter + if xattrSelectFilter(name) { + if err := removexattr(path, name); err != nil { + return err + } } } diff --git a/internal/fs/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go index 81c931e24..469f140d7 100644 --- a/internal/fs/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -26,7 +26,7 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribu Type: restic.NodeTypeFile, ExtendedAttributes: attrs, } - rtest.OK(t, nodeRestoreExtendedAttributes(node, file)) + rtest.OK(t, nodeRestoreExtendedAttributes(node, file, func(_ string) bool { return true } /*restore all xattrs*/)) nodeActual := &restic.Node{ Type: restic.NodeTypeFile, diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 14a8edeac..536958d4f 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -31,6 +31,8 @@ type Restorer struct { // SelectFilter determines whether the item is selectedForRestore or whether a childMayBeSelected. // selectedForRestore must not depend on isDir as `removeUnexpectedFiles` always passes false to isDir. SelectFilter func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) + + XattrSelectFilter func(xattrName string) (xattrSelectedForRestore bool) } var restorerAbortOnAllErrors = func(_ string, err error) error { return err } @@ -288,7 +290,7 @@ func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location s return nil } debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := fs.NodeRestoreMetadata(node, target, res.Warn) + err := fs.NodeRestoreMetadata(node, target, res.Warn, res.XattrSelectFilter) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } From f457b16b23f691c23ce0cabcd04ed257ef790b4e Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Fri, 15 Nov 2024 15:55:29 -0500 Subject: [PATCH 2/7] update nodeRestoreExtendedAttributes() for win - also other platforms - move xattr include/exclude filter parsing into separate func Signed-off-by: Tesshu Flower --- cmd/restic/cmd_restore.go | 68 +++++++++++++++++------------- internal/fs/node_noxattr.go | 2 +- internal/fs/node_test.go | 3 +- internal/fs/node_windows.go | 17 +++++--- internal/fs/node_windows_test.go | 2 +- internal/fs/node_xattr_all_test.go | 3 +- internal/restorer/restorer.go | 13 +++--- 7 files changed, 62 insertions(+), 46 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index fc3148ce1..3bc6ac5c5 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -101,9 +101,6 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, hasExcludes := len(excludePatternFns) > 0 hasIncludes := len(includePatternFns) > 0 - hasXattrExcludes := len(opts.ExcludeXattrPattern) > 0 - hasXattrIncludes := len(opts.IncludeXattrPattern) > 0 - switch { case len(args) == 0: return errors.Fatal("no snapshot ID specified") @@ -119,10 +116,6 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return errors.Fatal("exclude and include patterns are mutually exclusive") } - if hasXattrExcludes && hasXattrIncludes { - return errors.Fatal("exclude and include xattr patterns are mutually exclusive") - } - if opts.DryRun && opts.Verify { return errors.Fatal("--dry-run and --verify are mutually exclusive") } @@ -232,29 +225,9 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, res.SelectFilter = selectIncludeFilter } - if !hasXattrExcludes && !hasXattrIncludes { - // set default of including xattrs from the 'user' namespace - opts.IncludeXattrPattern = []string{"user.*"} - } - if hasXattrExcludes { - if err := filter.ValidatePatterns(opts.ExcludeXattrPattern); err != nil { - return errors.Fatalf("--exclude-xattr: %s", err) - } - - res.XattrSelectFilter = func(xattrName string) bool { - shouldReject := filter.RejectByPattern(opts.ExcludeXattrPattern, Warnf)(xattrName) - return !shouldReject - } - } else { - // User has either input include xattr pattern(s) or we're using our default include pattern - if err := filter.ValidatePatterns(opts.IncludeXattrPattern); err != nil { - return errors.Fatalf("--include-xattr: %s", err) - } - - res.XattrSelectFilter = func(xattrName string) bool { - shouldInclude, _ := filter.IncludeByPattern(opts.IncludeXattrPattern, Warnf)(xattrName) - return shouldInclude - } + res.XattrSelectFilter, err = getXattrSelectFilter(opts) + if err != nil { + return err } if !gopts.JSON { @@ -295,3 +268,38 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return nil } + +func getXattrSelectFilter(opts RestoreOptions) (func(xattrName string) bool, error) { + hasXattrExcludes := len(opts.ExcludeXattrPattern) > 0 + hasXattrIncludes := len(opts.IncludeXattrPattern) > 0 + + if hasXattrExcludes && hasXattrIncludes { + return nil, errors.Fatal("exclude and include xattr patterns are mutually exclusive") + } + + if hasXattrExcludes { + if err := filter.ValidatePatterns(opts.ExcludeXattrPattern); err != nil { + return nil, errors.Fatalf("--exclude-xattr: %s", err) + } + + return func(xattrName string) bool { + shouldReject := filter.RejectByPattern(opts.ExcludeXattrPattern, Warnf)(xattrName) + return !shouldReject + }, nil + } + + if hasXattrIncludes { + // User has either input include xattr pattern(s) or we're using our default include pattern + if err := filter.ValidatePatterns(opts.IncludeXattrPattern); err != nil { + return nil, errors.Fatalf("--include-xattr: %s", err) + } + + return func(xattrName string) bool { + shouldInclude, _ := filter.IncludeByPattern(opts.IncludeXattrPattern, Warnf)(xattrName) + return shouldInclude + }, nil + } + + // no includes or excludes, set default of including all xattrs + return func(_ string) bool { return true }, nil +} diff --git a/internal/fs/node_noxattr.go b/internal/fs/node_noxattr.go index 726827f62..2dbd72c9d 100644 --- a/internal/fs/node_noxattr.go +++ b/internal/fs/node_noxattr.go @@ -8,7 +8,7 @@ import ( ) // nodeRestoreExtendedAttributes is a no-op -func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string, _ func(xattrName string) bool) error { return nil } diff --git a/internal/fs/node_test.go b/internal/fs/node_test.go index b67295f68..490ab7e40 100644 --- a/internal/fs/node_test.go +++ b/internal/fs/node_test.go @@ -217,8 +217,9 @@ func TestNodeRestoreAt(t *testing.T) { nodePath = filepath.Join(tempdir, test.Name) } rtest.OK(t, NodeCreateAt(&test, nodePath)) + // Restore metadata, restoring all xattrs rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }, - func(_ string) bool { return true } /* restore all xattrs */)) + func(_ string) bool { return true })) fs := &Local{} meta, err := fs.OpenFile(nodePath, O_NOFOLLOW, true) diff --git a/internal/fs/node_windows.go b/internal/fs/node_windows.go index 74cf6c0e5..df0a7ea65 100644 --- a/internal/fs/node_windows.go +++ b/internal/fs/node_windows.go @@ -69,15 +69,20 @@ func utimesNano(path string, atime, mtime int64, _ restic.NodeType) error { } // restore extended attributes for windows -func nodeRestoreExtendedAttributes(node *restic.Node, path string) (err error) { +func nodeRestoreExtendedAttributes(node *restic.Node, path string, xattrSelectFilter func(xattrName string) bool) error { count := len(node.ExtendedAttributes) if count > 0 { - eas := make([]extendedAttribute, count) - for i, attr := range node.ExtendedAttributes { - eas[i] = extendedAttribute{Name: attr.Name, Value: attr.Value} + eas := []extendedAttribute{} + for _, attr := range node.ExtendedAttributes { + // Filter for xattrs we want to include/exclude + if xattrSelectFilter(attr.Name) { + eas = append(eas, extendedAttribute{Name: attr.Name, Value: attr.Value}) + } } - if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil { - return errExt + if len(eas) > 0 { + if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil { + return errExt + } } } return nil diff --git a/internal/fs/node_windows_test.go b/internal/fs/node_windows_test.go index f75df54d3..458a7bcb1 100644 --- a/internal/fs/node_windows_test.go +++ b/internal/fs/node_windows_test.go @@ -218,7 +218,7 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode *restic.Node, warn // If warning is not expected, this code should not get triggered. test.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", testPath, msg)) } - }) + }, func(_ string) bool { return true }) test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath)) fs := &Local{} diff --git a/internal/fs/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go index 469f140d7..65ac5c580 100644 --- a/internal/fs/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -26,7 +26,8 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribu Type: restic.NodeTypeFile, ExtendedAttributes: attrs, } - rtest.OK(t, nodeRestoreExtendedAttributes(node, file, func(_ string) bool { return true } /*restore all xattrs*/)) + /* restore all xattrs */ + rtest.OK(t, nodeRestoreExtendedAttributes(node, file, func(_ string) bool { return true })) nodeActual := &restic.Node{ Type: restic.NodeTypeFile, diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 536958d4f..cce175ebc 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -99,12 +99,13 @@ func (c *OverwriteBehavior) Type() string { // NewRestorer creates a restorer preloaded with the content from the snapshot id. func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Restorer { r := &Restorer{ - repo: repo, - opts: opts, - fileList: make(map[string]bool), - Error: restorerAbortOnAllErrors, - SelectFilter: func(string, bool) (bool, bool) { return true, true }, - sn: sn, + repo: repo, + opts: opts, + fileList: make(map[string]bool), + Error: restorerAbortOnAllErrors, + SelectFilter: func(string, bool) (bool, bool) { return true, true }, + XattrSelectFilter: func(string) bool { return true }, + sn: sn, } return r From 24422e20a6abb804f0c132137730285f01026ef4 Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Mon, 2 Dec 2024 19:38:43 -0500 Subject: [PATCH 3/7] restore: xattr restore filter tests Signed-off-by: Tesshu Flower --- internal/fs/node_xattr_all_test.go | 138 +++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/internal/fs/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go index 65ac5c580..cf3738722 100644 --- a/internal/fs/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -4,12 +4,14 @@ package fs import ( + "bytes" "os" "path/filepath" "runtime" "strings" "testing" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -37,6 +39,56 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribu rtest.Assert(t, nodeActual.Equals(*node), "xattr mismatch got %v expected %v", nodeActual.ExtendedAttributes, node.ExtendedAttributes) } +func setAndVerifyXattrWithSelectFilter(t *testing.T, file string, testAttr []testXattrToRestore, xattrSelectFilter func(_ string) bool) { + attrs := make([]restic.ExtendedAttribute, len(testAttr)) + for i := range testAttr { + attrs[i] = testAttr[i].xattr + } + + if runtime.GOOS == "windows" { + // windows seems to convert the xattr name to upper case + for i := range attrs { + attrs[i].Name = strings.ToUpper(attrs[i].Name) + } + } + + node := &restic.Node{ + Type: restic.NodeTypeFile, + ExtendedAttributes: attrs, + } + + rtest.OK(t, nodeRestoreExtendedAttributes(node, file, xattrSelectFilter)) + + nodeActual := &restic.Node{ + Type: restic.NodeTypeFile, + } + rtest.OK(t, nodeFillExtendedAttributes(nodeActual, file, false)) + + // Check nodeActual to make sure only xattrs we expect are there + for _, testAttr := range testAttr { + xattrFound := false + xattrRestored := false + for _, restoredAttr := range nodeActual.ExtendedAttributes { + if restoredAttr.Name == testAttr.xattr.Name { + xattrFound = true + xattrRestored = bytes.Equal(restoredAttr.Value, testAttr.xattr.Value) + break + } + } + if testAttr.shouldRestore { + rtest.Assert(t, xattrFound, "xattr %s not restored", testAttr.xattr.Name) + rtest.Assert(t, xattrRestored, "xattr %v value not restored", testAttr.xattr) + } else { + rtest.Assert(t, !xattrFound, "xattr %v should not have been restored", testAttr.xattr) + } + } +} + +type testXattrToRestore struct { + xattr restic.ExtendedAttribute + shouldRestore bool +} + func TestOverwriteXattr(t *testing.T) { dir := t.TempDir() file := filepath.Join(dir, "file") @@ -47,6 +99,10 @@ func TestOverwriteXattr(t *testing.T) { Name: "user.foo", Value: []byte("bar"), }, + { + Name: "abc.test", + Value: []byte("testxattr"), + }, }) setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ @@ -56,3 +112,85 @@ func TestOverwriteXattr(t *testing.T) { }, }) } + +func TestOverwriteXattrWithSelectFilter(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "file2") + rtest.OK(t, os.WriteFile(file, []byte("hello world"), 0o600)) + + noopWarnf := func(_ string, _ ...interface{}) {} + + // Set a filter as if the user passed in --include-xattr user.* + xattrSelectFilter1 := func(xattrName string) bool { + shouldInclude, _ := filter.IncludeByPattern([]string{"user.*"}, noopWarnf)(xattrName) + return shouldInclude + } + + setAndVerifyXattrWithSelectFilter(t, file, []testXattrToRestore{ + { + xattr: restic.ExtendedAttribute{ + Name: "user.foo", + Value: []byte("bar"), + }, + shouldRestore: true, + }, + { + xattr: restic.ExtendedAttribute{ + Name: "user.test", + Value: []byte("testxattr"), + }, + shouldRestore: true, + }, + { + xattr: restic.ExtendedAttribute{ + Name: "security.other", + Value: []byte("testing"), + }, + shouldRestore: false, + }, + }, xattrSelectFilter1) + + // Set a filter as if the user passed in --include-xattr user.* + xattrSelectFilter2 := func(xattrName string) bool { + shouldInclude, _ := filter.IncludeByPattern([]string{"user.o*", "user.comm*"}, noopWarnf)(xattrName) + return shouldInclude + } + + setAndVerifyXattrWithSelectFilter(t, file, []testXattrToRestore{ + { + xattr: restic.ExtendedAttribute{ + Name: "user.other", + Value: []byte("some"), + }, + shouldRestore: true, + }, + { + xattr: restic.ExtendedAttribute{ + Name: "security.other", + Value: []byte("testing"), + }, + shouldRestore: false, + }, + { + xattr: restic.ExtendedAttribute{ + Name: "user.open", + Value: []byte("door"), + }, + shouldRestore: true, + }, + { + xattr: restic.ExtendedAttribute{ + Name: "user.common", + Value: []byte("testing"), + }, + shouldRestore: true, + }, + { + xattr: restic.ExtendedAttribute{ + Name: "user.bad", + Value: []byte("dontincludeme"), + }, + shouldRestore: false, + }, + }, xattrSelectFilter2) +} From 3ac697d03d8460b774cadf133ef378174c598ea9 Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Mon, 2 Dec 2024 23:33:15 -0500 Subject: [PATCH 4/7] linux default restore only user xattrs, doc update * On Linux restore only user.* xattrs by default * restore all for other OSs * Update docs and changelog about the new restore flags --exclude-xattr and --include-xattr Signed-off-by: Tesshu Flower --- changelog/unreleased/issue-5089 | 22 ++++++++++++++++++++++ cmd/restic/cmd_restore.go | 11 ++++++++++- doc/050_restore.rst | 16 ++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-5089 diff --git a/changelog/unreleased/issue-5089 b/changelog/unreleased/issue-5089 new file mode 100644 index 000000000..51b5a679c --- /dev/null +++ b/changelog/unreleased/issue-5089 @@ -0,0 +1,22 @@ +Enhancement: Allow including or excluding extended file attributes +during restore. + +# Describe the problem in the past tense, the new behavior in the present +# tense. Mention the affected commands, backends, operating systems, etc. +# If the problem description just says that a feature was missing, then +# only explain the new behavior. +# Focus on user-facing behavior, not the implementation. +# Use "Restic now ..." instead of "We have changed ...". +# +Restic restore used to attempt to restore all extended file attributes. +Now two new command line flags are added to restore to control which +extended file attributes will be restored. + +The new flags are `--exclude-xattr` and `--include-xattr`. + +If the flags are not provided, restic will default to restoring +only `user` namespaced extended file attributes on Linux, and all +extended file attributes on other operating systems. + +https://github.com/restic/restic/issues/5089 +https://github.com/restic/restic/pull/5129 diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 3bc6ac5c5..870d496c9 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -3,6 +3,7 @@ package main import ( "context" "path/filepath" + "runtime" "time" "github.com/restic/restic/internal/debug" @@ -300,6 +301,14 @@ func getXattrSelectFilter(opts RestoreOptions) (func(xattrName string) bool, err }, nil } - // no includes or excludes, set default of including all xattrs + // User has not specified any xattr includes or excludes + if runtime.GOOS == "linux" { + // For Linux, set default of including only user.* xattrs + return func(xattrName string) bool { + shouldInclude, _ := filter.IncludeByPattern([]string{"user.*"}, Warnf)(xattrName) + return shouldInclude + }, nil + } + // Not linux, default to including all xattrs return func(_ string) bool { return true }, nil } diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 9558ab1d4..4ca738a3f 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -88,6 +88,22 @@ disk space. Note that the exact location of the holes can differ from those in the original file, as their location is determined while restoring and is not stored explicitly. +Restoring extended file attributes +---------------------------------- + +By default, user namespaced extended attributes for files are restored on Linux, +and all extended attributes are restored for other operating systems. + +Use ``--exclude-xattr`` and ``--include-xattr`` to control which extended +attributes are restored for files in the snapshot. For example, to restore +user and security namespaced extended attributes for files: + +.. code-block:: console + + $ restic -r /srv/restic-repo restore 79766175 --target /tmp/restore-work --include-xattr user.* --include-xattr security.* + enter password for repository: + restoring to /tmp/restore-work + Restoring in-place ------------------ From cd84fe085368785cb2e4c8b2b8f3dfe67141dc3c Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Fri, 10 Jan 2025 15:25:09 -0500 Subject: [PATCH 5/7] xattrs - restore all by default, doc/chglog update Signed-off-by: Tesshu Flower --- changelog/unreleased/issue-5089 | 15 +++------------ cmd/restic/cmd_restore.go | 11 +---------- doc/050_restore.rst | 5 ++--- 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/changelog/unreleased/issue-5089 b/changelog/unreleased/issue-5089 index 51b5a679c..43c5c8366 100644 --- a/changelog/unreleased/issue-5089 +++ b/changelog/unreleased/issue-5089 @@ -1,22 +1,13 @@ -Enhancement: Allow including or excluding extended file attributes -during restore. +Enhancement: Allow including/excluding extended file attributes during restore -# Describe the problem in the past tense, the new behavior in the present -# tense. Mention the affected commands, backends, operating systems, etc. -# If the problem description just says that a feature was missing, then -# only explain the new behavior. -# Focus on user-facing behavior, not the implementation. -# Use "Restic now ..." instead of "We have changed ...". -# -Restic restore used to attempt to restore all extended file attributes. +Restic restore attempts to restore all extended file attributes. Now two new command line flags are added to restore to control which extended file attributes will be restored. The new flags are `--exclude-xattr` and `--include-xattr`. If the flags are not provided, restic will default to restoring -only `user` namespaced extended file attributes on Linux, and all -extended file attributes on other operating systems. +all extended file attributes. https://github.com/restic/restic/issues/5089 https://github.com/restic/restic/pull/5129 diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 870d496c9..7a3b029da 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -3,7 +3,6 @@ package main import ( "context" "path/filepath" - "runtime" "time" "github.com/restic/restic/internal/debug" @@ -301,14 +300,6 @@ func getXattrSelectFilter(opts RestoreOptions) (func(xattrName string) bool, err }, nil } - // User has not specified any xattr includes or excludes - if runtime.GOOS == "linux" { - // For Linux, set default of including only user.* xattrs - return func(xattrName string) bool { - shouldInclude, _ := filter.IncludeByPattern([]string{"user.*"}, Warnf)(xattrName) - return shouldInclude - }, nil - } - // Not linux, default to including all xattrs + // default to including all xattrs return func(_ string) bool { return true }, nil } diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 4ca738a3f..b37f3c4fb 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -91,10 +91,9 @@ stored explicitly. Restoring extended file attributes ---------------------------------- -By default, user namespaced extended attributes for files are restored on Linux, -and all extended attributes are restored for other operating systems. +By default, all extended attributes for files are restored. -Use ``--exclude-xattr`` and ``--include-xattr`` to control which extended +Use only ``--exclude-xattr`` or ``--include-xattr`` to control which extended attributes are restored for files in the snapshot. For example, to restore user and security namespaced extended attributes for files: From 44cef250778a2f57dbf39a1508d7d2f4b56c5f78 Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Fri, 10 Jan 2025 21:12:03 -0500 Subject: [PATCH 6/7] remove bad test xattr Signed-off-by: Tesshu Flower --- internal/fs/node_xattr_all_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/fs/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go index cf3738722..79fde63e1 100644 --- a/internal/fs/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -99,10 +99,6 @@ func TestOverwriteXattr(t *testing.T) { Name: "user.foo", Value: []byte("bar"), }, - { - Name: "abc.test", - Value: []byte("testxattr"), - }, }) setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ From 5e8654c71d7ce4c1f4b09544684ecabde3854c92 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 18 Jan 2025 22:54:47 +0100 Subject: [PATCH 7/7] restore: fix xattr filter test on windows --- internal/fs/node_xattr_all_test.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/fs/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go index 79fde63e1..6a9a2e4bf 100644 --- a/internal/fs/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -42,14 +42,11 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribu func setAndVerifyXattrWithSelectFilter(t *testing.T, file string, testAttr []testXattrToRestore, xattrSelectFilter func(_ string) bool) { attrs := make([]restic.ExtendedAttribute, len(testAttr)) for i := range testAttr { - attrs[i] = testAttr[i].xattr - } - - if runtime.GOOS == "windows" { // windows seems to convert the xattr name to upper case - for i := range attrs { - attrs[i].Name = strings.ToUpper(attrs[i].Name) + if runtime.GOOS == "windows" { + testAttr[i].xattr.Name = strings.ToUpper(testAttr[i].xattr.Name) } + attrs[i] = testAttr[i].xattr } node := &restic.Node{ @@ -109,6 +106,18 @@ func TestOverwriteXattr(t *testing.T) { }) } +func uppercaseOnWindows(patterns []string) []string { + // windows seems to convert the xattr name to upper case + if runtime.GOOS == "windows" { + out := []string{} + for _, pattern := range patterns { + out = append(out, strings.ToUpper(pattern)) + } + return out + } + return patterns +} + func TestOverwriteXattrWithSelectFilter(t *testing.T) { dir := t.TempDir() file := filepath.Join(dir, "file2") @@ -118,7 +127,7 @@ func TestOverwriteXattrWithSelectFilter(t *testing.T) { // Set a filter as if the user passed in --include-xattr user.* xattrSelectFilter1 := func(xattrName string) bool { - shouldInclude, _ := filter.IncludeByPattern([]string{"user.*"}, noopWarnf)(xattrName) + shouldInclude, _ := filter.IncludeByPattern(uppercaseOnWindows([]string{"user.*"}), noopWarnf)(xattrName) return shouldInclude } @@ -148,7 +157,7 @@ func TestOverwriteXattrWithSelectFilter(t *testing.T) { // Set a filter as if the user passed in --include-xattr user.* xattrSelectFilter2 := func(xattrName string) bool { - shouldInclude, _ := filter.IncludeByPattern([]string{"user.o*", "user.comm*"}, noopWarnf)(xattrName) + shouldInclude, _ := filter.IncludeByPattern(uppercaseOnWindows([]string{"user.o*", "user.comm*"}), noopWarnf)(xattrName) return shouldInclude }