Merge pull request #5129 from tesshuflower/5089_exclude_xattrs_on_restore

Allow excluding xattrs at restore time

changelog/unreleased/issue-5089

@@ -0,0 +1,13 @@
+Enhancement: Allow including/excluding extended file attributes during restore
+
+Restic restore attempts to restore all extended file attributes.
+Now two new command line flags are added to restore to control which
+extended file attributes will be restored.
+
+The new flags are `--exclude-xattr` and `--include-xattr`.
+
+If the flags are not provided, restic will default to restoring
+all extended file attributes.
+
+https://github.com/restic/restic/issues/5089
+https://github.com/restic/restic/pull/5129

cmd/restic/cmd_restore.go

@@ -54,11 +54,13 @@ type RestoreOptions struct {
 	filter.IncludePatternOptions
 	Target string
 	restic.SnapshotFilter
-	DryRun    bool
+	DryRun              bool
-	Sparse    bool
+	Sparse              bool
-	Verify    bool
+	Verify              bool
-	Overwrite restorer.OverwriteBehavior
+	Overwrite           restorer.OverwriteBehavior
-	Delete    bool
+	Delete              bool
+	ExcludeXattrPattern []string
+	IncludeXattrPattern []string
 }
 
 var restoreOptions RestoreOptions
@@ -72,6 +74,9 @@ func init() {
 	restoreOptions.ExcludePatternOptions.Add(flags)
 	restoreOptions.IncludePatternOptions.Add(flags)
 
+	flags.StringArrayVar(&restoreOptions.ExcludeXattrPattern, "exclude-xattr", nil, "exclude xattr by `pattern` (can be specified multiple times)")
+	flags.StringArrayVar(&restoreOptions.IncludeXattrPattern, "include-xattr", nil, "include xattr by `pattern` (can be specified multiple times)")
+
 	initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
 	flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done")
 	flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
@@ -110,6 +115,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 	if hasExcludes && hasIncludes {
 		return errors.Fatal("exclude and include patterns are mutually exclusive")
 	}
+
 	if opts.DryRun && opts.Verify {
 		return errors.Fatal("--dry-run and --verify are mutually exclusive")
 	}
@@ -219,6 +225,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		res.SelectFilter = selectIncludeFilter
 	}
 
+	res.XattrSelectFilter, err = getXattrSelectFilter(opts)
+	if err != nil {
+		return err
+	}
+
 	if !gopts.JSON {
 		msg.P("restoring %s to %s\n", res.Snapshot(), opts.Target)
 	}
@@ -257,3 +268,38 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 
 	return nil
 }
+
+func getXattrSelectFilter(opts RestoreOptions) (func(xattrName string) bool, error) {
+	hasXattrExcludes := len(opts.ExcludeXattrPattern) > 0
+	hasXattrIncludes := len(opts.IncludeXattrPattern) > 0
+
+	if hasXattrExcludes && hasXattrIncludes {
+		return nil, errors.Fatal("exclude and include xattr patterns are mutually exclusive")
+	}
+
+	if hasXattrExcludes {
+		if err := filter.ValidatePatterns(opts.ExcludeXattrPattern); err != nil {
+			return nil, errors.Fatalf("--exclude-xattr: %s", err)
+		}
+
+		return func(xattrName string) bool {
+			shouldReject := filter.RejectByPattern(opts.ExcludeXattrPattern, Warnf)(xattrName)
+			return !shouldReject
+		}, nil
+	}
+
+	if hasXattrIncludes {
+		// User has either input include xattr pattern(s) or we're using our default include pattern
+		if err := filter.ValidatePatterns(opts.IncludeXattrPattern); err != nil {
+			return nil, errors.Fatalf("--include-xattr: %s", err)
+		}
+
+		return func(xattrName string) bool {
+			shouldInclude, _ := filter.IncludeByPattern(opts.IncludeXattrPattern, Warnf)(xattrName)
+			return shouldInclude
+		}, nil
+	}
+
+	// default to including all xattrs
+	return func(_ string) bool { return true }, nil
+}

doc/050_restore.rst

@@ -88,6 +88,21 @@ disk space. Note that the exact location of the holes can differ from those in
 the original file, as their location is determined while restoring and is not
 stored explicitly.
 
+Restoring extended file attributes
+----------------------------------
+
+By default, all extended attributes for files are restored.
+
+Use only ``--exclude-xattr`` or ``--include-xattr`` to control which extended
+attributes are restored for files in the snapshot. For example, to restore
+user and security namespaced extended attributes for files:
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo restore 79766175 --target /tmp/restore-work --include-xattr user.* --include-xattr security.*
+    enter password for repository:
+    restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
+
 Restoring in-place
 ------------------
 

internal/fs/node.go

@@ -230,8 +230,8 @@ func mkfifo(path string, mode uint32) (err error) {
 }
 
 // NodeRestoreMetadata restores node metadata
-func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error {
+func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error {
-	err := nodeRestoreMetadata(node, path, warn)
+	err := nodeRestoreMetadata(node, path, warn, xattrSelectFilter)
 	if err != nil {
 		// It is common to have permission errors for folders like /home
 		// unless you're running as root, so ignore those.
@@ -246,14 +246,14 @@ func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string))
 	return err
 }
 
-func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error {
+func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string), xattrSelectFilter func(xattrName string) bool) error {
 	var firsterr error
 
 	if err := lchown(path, int(node.UID), int(node.GID)); err != nil {
 		firsterr = errors.WithStack(err)
 	}
 
-	if err := nodeRestoreExtendedAttributes(node, path); err != nil {
+	if err := nodeRestoreExtendedAttributes(node, path, xattrSelectFilter); err != nil {
 		debug.Log("error restoring extended attributes for %v: %v", path, err)
 		if firsterr == nil {
 			firsterr = err

internal/fs/node_noxattr.go

@@ -8,7 +8,7 @@ import (
 )
 
 // nodeRestoreExtendedAttributes is a no-op
-func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error {
+func nodeRestoreExtendedAttributes(_ *restic.Node, _ string, _ func(xattrName string) bool) error {
 	return nil
 }
 

internal/fs/node_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/test"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -216,7 +217,9 @@ func TestNodeRestoreAt(t *testing.T) {
 				nodePath = filepath.Join(tempdir, test.Name)
 			}
 			rtest.OK(t, NodeCreateAt(&test, nodePath))
-			rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }))
+			// Restore metadata, restoring all xattrs
+			rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) },
+				func(_ string) bool { return true }))
 
 			fs := &Local{}
 			meta, err := fs.OpenFile(nodePath, O_NOFOLLOW, true)
@@ -291,6 +294,7 @@ func TestNodeRestoreMetadataError(t *testing.T) {
 	nodePath := filepath.Join(tempdir, node.Name)
 
 	// This will fail because the target file does not exist
-	err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })
+	err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) },
-	rtest.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason")
+		func(_ string) bool { return true })
+	test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason")
 }

internal/fs/node_windows.go

@@ -69,15 +69,20 @@ func utimesNano(path string, atime, mtime int64, _ restic.NodeType) error {
 }
 
 // restore extended attributes for windows
-func nodeRestoreExtendedAttributes(node *restic.Node, path string) (err error) {
+func nodeRestoreExtendedAttributes(node *restic.Node, path string, xattrSelectFilter func(xattrName string) bool) error {
 	count := len(node.ExtendedAttributes)
 	if count > 0 {
-		eas := make([]extendedAttribute, count)
+		eas := []extendedAttribute{}
-		for i, attr := range node.ExtendedAttributes {
+		for _, attr := range node.ExtendedAttributes {
-			eas[i] = extendedAttribute{Name: attr.Name, Value: attr.Value}
+			// Filter for xattrs we want to include/exclude
+			if xattrSelectFilter(attr.Name) {
+				eas = append(eas, extendedAttribute{Name: attr.Name, Value: attr.Value})
+			}
 		}
-		if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil {
+		if len(eas) > 0 {
-			return errExt
+			if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil {
+				return errExt
+			}
 		}
 	}
 	return nil

internal/fs/node_windows_test.go

@@ -218,7 +218,7 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode *restic.Node, warn
 			// If warning is not expected, this code should not get triggered.
 			test.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", testPath, msg))
 		}
-	})
+	}, func(_ string) bool { return true })
 	test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath))
 
 	fs := &Local{}

internal/fs/node_xattr.go

@@ -65,14 +65,17 @@ func handleXattrErr(err error) error {
 	}
 }
 
-func nodeRestoreExtendedAttributes(node *restic.Node, path string) error {
+func nodeRestoreExtendedAttributes(node *restic.Node, path string, xattrSelectFilter func(xattrName string) bool) error {
 	expectedAttrs := map[string]struct{}{}
 	for _, attr := range node.ExtendedAttributes {
-		err := setxattr(path, attr.Name, attr.Value)
+		// Only restore xattrs that match the filter
-		if err != nil {
+		if xattrSelectFilter(attr.Name) {
-			return err
+			err := setxattr(path, attr.Name, attr.Value)
+			if err != nil {
+				return err
+			}
+			expectedAttrs[attr.Name] = struct{}{}
 		}
-		expectedAttrs[attr.Name] = struct{}{}
 	}
 
 	// remove unexpected xattrs
@@ -84,8 +87,11 @@ func nodeRestoreExtendedAttributes(node *restic.Node, path string) error {
 		if _, ok := expectedAttrs[name]; ok {
 			continue
 		}
-		if err := removexattr(path, name); err != nil {
+		// Only attempt to remove xattrs that match the filter
-			return err
+		if xattrSelectFilter(name) {
+			if err := removexattr(path, name); err != nil {
+				return err
+			}
 		}
 	}
 

internal/fs/node_xattr_all_test.go

@@ -4,12 +4,14 @@
 package fs
 
 import (
+	"bytes"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -26,7 +28,8 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribu
 		Type:               restic.NodeTypeFile,
 		ExtendedAttributes: attrs,
 	}
-	rtest.OK(t, nodeRestoreExtendedAttributes(node, file))
+	/* restore all xattrs */
+	rtest.OK(t, nodeRestoreExtendedAttributes(node, file, func(_ string) bool { return true }))
 
 	nodeActual := &restic.Node{
 		Type: restic.NodeTypeFile,
@@ -36,6 +39,53 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribu
 	rtest.Assert(t, nodeActual.Equals(*node), "xattr mismatch got %v expected %v", nodeActual.ExtendedAttributes, node.ExtendedAttributes)
 }
 
+func setAndVerifyXattrWithSelectFilter(t *testing.T, file string, testAttr []testXattrToRestore, xattrSelectFilter func(_ string) bool) {
+	attrs := make([]restic.ExtendedAttribute, len(testAttr))
+	for i := range testAttr {
+		// windows seems to convert the xattr name to upper case
+		if runtime.GOOS == "windows" {
+			testAttr[i].xattr.Name = strings.ToUpper(testAttr[i].xattr.Name)
+		}
+		attrs[i] = testAttr[i].xattr
+	}
+
+	node := &restic.Node{
+		Type:               restic.NodeTypeFile,
+		ExtendedAttributes: attrs,
+	}
+
+	rtest.OK(t, nodeRestoreExtendedAttributes(node, file, xattrSelectFilter))
+
+	nodeActual := &restic.Node{
+		Type: restic.NodeTypeFile,
+	}
+	rtest.OK(t, nodeFillExtendedAttributes(nodeActual, file, false))
+
+	// Check nodeActual to make sure only xattrs we expect are there
+	for _, testAttr := range testAttr {
+		xattrFound := false
+		xattrRestored := false
+		for _, restoredAttr := range nodeActual.ExtendedAttributes {
+			if restoredAttr.Name == testAttr.xattr.Name {
+				xattrFound = true
+				xattrRestored = bytes.Equal(restoredAttr.Value, testAttr.xattr.Value)
+				break
+			}
+		}
+		if testAttr.shouldRestore {
+			rtest.Assert(t, xattrFound, "xattr %s not restored", testAttr.xattr.Name)
+			rtest.Assert(t, xattrRestored, "xattr %v value not restored", testAttr.xattr)
+		} else {
+			rtest.Assert(t, !xattrFound, "xattr %v should not have been restored", testAttr.xattr)
+		}
+	}
+}
+
+type testXattrToRestore struct {
+	xattr         restic.ExtendedAttribute
+	shouldRestore bool
+}
+
 func TestOverwriteXattr(t *testing.T) {
 	dir := t.TempDir()
 	file := filepath.Join(dir, "file")
@@ -55,3 +105,97 @@ func TestOverwriteXattr(t *testing.T) {
 		},
 	})
 }
+
+func uppercaseOnWindows(patterns []string) []string {
+	// windows seems to convert the xattr name to upper case
+	if runtime.GOOS == "windows" {
+		out := []string{}
+		for _, pattern := range patterns {
+			out = append(out, strings.ToUpper(pattern))
+		}
+		return out
+	}
+	return patterns
+}
+
+func TestOverwriteXattrWithSelectFilter(t *testing.T) {
+	dir := t.TempDir()
+	file := filepath.Join(dir, "file2")
+	rtest.OK(t, os.WriteFile(file, []byte("hello world"), 0o600))
+
+	noopWarnf := func(_ string, _ ...interface{}) {}
+
+	// Set a filter as if the user passed in --include-xattr user.*
+	xattrSelectFilter1 := func(xattrName string) bool {
+		shouldInclude, _ := filter.IncludeByPattern(uppercaseOnWindows([]string{"user.*"}), noopWarnf)(xattrName)
+		return shouldInclude
+	}
+
+	setAndVerifyXattrWithSelectFilter(t, file, []testXattrToRestore{
+		{
+			xattr: restic.ExtendedAttribute{
+				Name:  "user.foo",
+				Value: []byte("bar"),
+			},
+			shouldRestore: true,
+		},
+		{
+			xattr: restic.ExtendedAttribute{
+				Name:  "user.test",
+				Value: []byte("testxattr"),
+			},
+			shouldRestore: true,
+		},
+		{
+			xattr: restic.ExtendedAttribute{
+				Name:  "security.other",
+				Value: []byte("testing"),
+			},
+			shouldRestore: false,
+		},
+	}, xattrSelectFilter1)
+
+	// Set a filter as if the user passed in --include-xattr user.*
+	xattrSelectFilter2 := func(xattrName string) bool {
+		shouldInclude, _ := filter.IncludeByPattern(uppercaseOnWindows([]string{"user.o*", "user.comm*"}), noopWarnf)(xattrName)
+		return shouldInclude
+	}
+
+	setAndVerifyXattrWithSelectFilter(t, file, []testXattrToRestore{
+		{
+			xattr: restic.ExtendedAttribute{
+				Name:  "user.other",
+				Value: []byte("some"),
+			},
+			shouldRestore: true,
+		},
+		{
+			xattr: restic.ExtendedAttribute{
+				Name:  "security.other",
+				Value: []byte("testing"),
+			},
+			shouldRestore: false,
+		},
+		{
+			xattr: restic.ExtendedAttribute{
+				Name:  "user.open",
+				Value: []byte("door"),
+			},
+			shouldRestore: true,
+		},
+		{
+			xattr: restic.ExtendedAttribute{
+				Name:  "user.common",
+				Value: []byte("testing"),
+			},
+			shouldRestore: true,
+		},
+		{
+			xattr: restic.ExtendedAttribute{
+				Name:  "user.bad",
+				Value: []byte("dontincludeme"),
+			},
+			shouldRestore: false,
+		},
+	}, xattrSelectFilter2)
+}

internal/restorer/restorer.go

@@ -31,6 +31,8 @@ type Restorer struct {
 	// SelectFilter determines whether the item is selectedForRestore or whether a childMayBeSelected.
 	// selectedForRestore must not depend on isDir as `removeUnexpectedFiles` always passes false to isDir.
 	SelectFilter func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool)
+
+	XattrSelectFilter func(xattrName string) (xattrSelectedForRestore bool)
 }
 
 var restorerAbortOnAllErrors = func(_ string, err error) error { return err }
@@ -97,12 +99,13 @@ func (c *OverwriteBehavior) Type() string {
 // NewRestorer creates a restorer preloaded with the content from the snapshot id.
 func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Restorer {
 	r := &Restorer{
-		repo:         repo,
+		repo:              repo,
-		opts:         opts,
+		opts:              opts,
-		fileList:     make(map[string]bool),
+		fileList:          make(map[string]bool),
-		Error:        restorerAbortOnAllErrors,
+		Error:             restorerAbortOnAllErrors,
-		SelectFilter: func(string, bool) (bool, bool) { return true, true },
+		SelectFilter:      func(string, bool) (bool, bool) { return true, true },
-		sn:           sn,
+		XattrSelectFilter: func(string) bool { return true },
+		sn:                sn,
 	}
 
 	return r
@@ -288,7 +291,7 @@ func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location s
 		return nil
 	}
 	debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location)
-	err := fs.NodeRestoreMetadata(node, target, res.Warn)
+	err := fs.NodeRestoreMetadata(node, target, res.Warn, res.XattrSelectFilter)
 	if err != nil {
 		debug.Log("node.RestoreMetadata(%s) error %v", target, err)
 	}