diff --git a/changelog/unreleased/issue-4179 b/changelog/unreleased/issue-4179 new file mode 100644 index 000000000..37955b060 --- /dev/null +++ b/changelog/unreleased/issue-4179 @@ -0,0 +1,11 @@ +Enhancement: add sort options for `ls` command + +in the past, the output of ls -l was sorted by name. Now it can be sorted by +one of the specifiers (name|size|time=mtime|atime|ctime|extension). +Use --sort to achieve this. + +Reverse sorting also has been implemtented. Use --reverse to indicate reverse +sorting. + +https://github.com/restic/restic/issues/4179 +https://github.com/restic/restic/pull/5182 diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 06ae6cc20..6e0d230b1 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -1,11 +1,14 @@ package main import ( + "cmp" "context" "encoding/json" "fmt" "io" "os" + "path/filepath" + "slices" "strings" "time" @@ -36,6 +39,10 @@ will allow traversing into matching directories' subfolders. Any directory paths specified must be absolute (starting with a path separator); paths use the forward slash '/' as separator. +File listings can be sorted by specifying --sort followed by one of the +sort specifiers '(name|size|time=mtime|atime|ctime|extension)'. +The sorting can be reversed by specifying --reverse. + EXIT STATUS =========== @@ -59,6 +66,8 @@ type LsOptions struct { Recursive bool HumanReadable bool Ncdu bool + Sort string + Reverse bool } var lsOptions LsOptions @@ -72,6 +81,8 @@ func init() { flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')") + flags.StringVarP(&lsOptions.Sort, "sort", "s", "name", "sort output by (name|size|time=mtime|atime|ctime|extension)") + flags.BoolVar(&lsOptions.Reverse, "reverse", false, "reverse sorted output") } type lsPrinter interface { @@ -277,6 +288,12 @@ func (p *textLsPrinter) Close() error { return nil } +// for ls -l output sorting +type toSortOutput struct { + nodepath string + node *restic.Node +} + func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") @@ -284,6 +301,18 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri if opts.Ncdu && gopts.JSON { return errors.Fatal("only either '--json' or '--ncdu' can be specified") } + if opts.Sort != "name" && opts.Ncdu { + return errors.Fatal("--sort and --ncdu are mutually exclusive") + } + if opts.Reverse && opts.Ncdu { + return errors.Fatal("--reverse and --ncdu are mutually exclusive") + } + + sortMode := SortModeName + err := sortMode.Set(opts.Sort) + if err != nil { + return err + } // extract any specific directories to walk var dirs []string @@ -347,6 +376,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } var printer lsPrinter + collector := []toSortOutput{} + outputSort := sortMode != SortModeName || opts.Reverse if gopts.JSON { printer = &jsonLsPrinter{ @@ -356,6 +387,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri printer = &ncduLsPrinter{ out: globalOptions.stdout, } + outputSort = false } else { printer = &textLsPrinter{ dirs: dirs, @@ -393,8 +425,12 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri printedDir := false if withinDir(nodepath) { // if we're within a target path, print the node - if err := printer.Node(nodepath, node, false); err != nil { - return err + if outputSort { + collector = append(collector, toSortOutput{nodepath, node}) + } else { + if err := printer.Node(nodepath, node, false); err != nil { + return err + } } printedDir = true @@ -409,7 +445,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri // there yet), signal the walker to descend into any subdirs if approachingMatchingTree(nodepath) { // print node leading up to the target paths - if !printedDir { + if !printedDir && !outputSort { return printer.Node(nodepath, node, true) } return nil @@ -444,5 +480,101 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } + if outputSort { + printSortedOutput(printer, opts, sortMode, collector) + } + return printer.Close() } + +func printSortedOutput(printer lsPrinter, opts LsOptions, sortMode SortMode, collector []toSortOutput) { + switch sortMode { + case SortModeName: + case SortModeSize: + slices.SortStableFunc(collector, func(a, b toSortOutput) int { + return cmp.Or( + cmp.Compare(a.node.Size, b.node.Size), + cmp.Compare(a.nodepath, b.nodepath), + ) + }) + case SortModeMtime: + slices.SortStableFunc(collector, func(a, b toSortOutput) int { + return cmp.Or( + a.node.ModTime.Compare(b.node.ModTime), + cmp.Compare(a.nodepath, b.nodepath), + ) + }) + case SortModeAtime: + slices.SortStableFunc(collector, func(a, b toSortOutput) int { + return cmp.Or( + a.node.AccessTime.Compare(b.node.AccessTime), + cmp.Compare(a.nodepath, b.nodepath), + ) + }) + case SortModeCtime: + slices.SortStableFunc(collector, func(a, b toSortOutput) int { + return cmp.Or( + a.node.ChangeTime.Compare(b.node.ChangeTime), + cmp.Compare(a.nodepath, b.nodepath), + ) + }) + case SortModeExt: + // map name to extension + mapExt := make(map[string]string, len(collector)) + for _, item := range collector { + ext := filepath.Ext(item.nodepath) + mapExt[item.nodepath] = ext + } + + slices.SortStableFunc(collector, func(a, b toSortOutput) int { + return cmp.Or( + cmp.Compare(mapExt[a.nodepath], mapExt[b.nodepath]), + cmp.Compare(a.nodepath, b.nodepath), + ) + }) + } + + if opts.Reverse { + slices.Reverse(collector) + } + for _, elem := range collector { + _ = printer.Node(elem.nodepath, elem.node, false) + } +} + +// SortMode defines the allowed sorting modes +type SortMode string + +// Allowed sort modes +const ( + SortModeName SortMode = "name" + SortModeSize SortMode = "size" + SortModeAtime SortMode = "atime" + SortModeCtime SortMode = "ctime" + SortModeMtime SortMode = "mtime" + SortModeExt SortMode = "extension" + SortModeInvalid SortMode = "--invalid--" +) + +// Set implements the method needed for pflag command flag parsing. +func (c *SortMode) Set(s string) error { + switch s { + case "name": + *c = SortModeName + case "size": + *c = SortModeSize + case "atime": + *c = SortModeAtime + case "ctime": + *c = SortModeCtime + case "mtime", "time": + *c = SortModeMtime + case "extension": + *c = SortModeExt + default: + *c = SortModeInvalid + return fmt.Errorf("invalid sort mode %q, must be one of (name|size|atime|ctime|mtime=time|extension)", s) + } + + return nil +} diff --git a/cmd/restic/cmd_ls_integration_test.go b/cmd/restic/cmd_ls_integration_test.go index f5655bdff..29e153419 100644 --- a/cmd/restic/cmd_ls_integration_test.go +++ b/cmd/restic/cmd_ls_integration_test.go @@ -19,7 +19,7 @@ func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args [ } func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { - out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID}) + out := testRunLsWithOpts(t, gopts, LsOptions{Sort: "name"}, []string{snapshotID}) return strings.Split(string(out), "\n") } @@ -45,7 +45,64 @@ func TestRunLsNcdu(t *testing.T) { {"latest", "/0"}, {"latest", "/0", "/0/9"}, } { - ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, paths) + ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true, Sort: "name"}, paths) assertIsValidJSON(t, ncdu) } } + +func TestRunLsSort(t *testing.T) { + compareName := []string{ + "/for_cmd_ls", + "/for_cmd_ls/file1.txt", + "/for_cmd_ls/file2.txt", + "/for_cmd_ls/python.py", + "", // last empty line + } + + compareSize := []string{ + "/for_cmd_ls", + "/for_cmd_ls/file2.txt", + "/for_cmd_ls/file1.txt", + "/for_cmd_ls/python.py", + "", + } + + compareExt := []string{ + "/for_cmd_ls", + "/for_cmd_ls/python.py", + "/for_cmd_ls/file1.txt", + "/for_cmd_ls/file2.txt", + "", + } + + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + testRunBackup(t, env.testdata+"/0", []string{"for_cmd_ls"}, opts, env.gopts) + + // sort by size + out := testRunLsWithOpts(t, env.gopts, LsOptions{Sort: "size"}, []string{"latest"}) + fileList := strings.Split(string(out), "\n") + rtest.Assert(t, len(fileList) == 5, "invalid ls --sort size, expected 5 array elements, got %v", len(fileList)) + for i, item := range compareSize { + rtest.Assert(t, item == fileList[i], "invalid ls --sort size, expected element '%s', got '%s'", item, fileList[i]) + } + + // sort by file extension + out = testRunLsWithOpts(t, env.gopts, LsOptions{Sort: "extension"}, []string{"latest"}) + fileList = strings.Split(string(out), "\n") + rtest.Assert(t, len(fileList) == 5, "invalid ls --sort extension, expected 5 array elements, got %v", len(fileList)) + for i, item := range compareExt { + rtest.Assert(t, item == fileList[i], "invalid ls --sort extension, expected element '%s', got '%s'", item, fileList[i]) + } + + // explicit name sort + out = testRunLsWithOpts(t, env.gopts, LsOptions{Sort: "name"}, []string{"latest"}) + fileList = strings.Split(string(out), "\n") + rtest.Assert(t, len(fileList) == 5, "invalid ls --sort name, expected 5 array elements, got %v", len(fileList)) + for i, item := range compareName { + rtest.Assert(t, item == fileList[i], "invalid ls --sort name, expected element '%s', got '%s'", item, fileList[i]) + } +} diff --git a/cmd/restic/testdata/backup-data.tar.gz b/cmd/restic/testdata/backup-data.tar.gz index 6ba5881ae..5d7cd215c 100644 Binary files a/cmd/restic/testdata/backup-data.tar.gz and b/cmd/restic/testdata/backup-data.tar.gz differ diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index f31e75c84..f8ff390f0 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -121,7 +121,7 @@ as separator. .. code-block:: console $ restic ls latest /home - + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): /home /home/user @@ -153,6 +153,49 @@ outputting information about a snapshot in the NCDU format using the ``--ncdu`` You can use it as follows: ``restic ls latest --ncdu | ncdu -f -`` +You can use the options ``--sort`` and ``--reverse`` to tailor ``ls`` output to your needs. +``--sort`` can be one of ``name | size | time=mtime | atime | ctime | extension``. The default +sorting option is ``name``. The sorting order can be reversed by specifying ``--reverse``. + +.. code-block:: console + + $ restic ls --long latest --sort size --reverse + + snapshot 711b0bb6 of [/tmp/restic] at 2025-02-03 08:16:05.310764668 +0000 UTC filtered by []: + -rw-rw-r-- 1000 1000 16772 2025-02-03 08:09:11 /tmp/restic/cmd_find.go + -rw-rw-r-- 1000 1000 3077 2025-02-03 08:15:46 /tmp/restic/conf.py + -rw-rw-r-- 1000 1000 2834 2025-02-03 08:09:35 /tmp/restic/find.go + -rw-rw-r-- 1000 1000 1473 2025-02-03 08:15:30 /tmp/restic/010_introduction.rst + drwxrwxr-x 1000 1000 0 2025-02-03 08:15:46 /tmp/restic + dtrwxrwxrwx 0 0 0 2025-02-03 08:14:22 /tmp + +.. code-block:: console + + $ restic ls --long latest --sort time + + snapshot 711b0bb6 of [/tmp/restic] at 2025-02-03 08:16:05.310764668 +0000 UTC filtered by []: + -rw-rw-r-- 1000 1000 16772 2025-02-03 08:09:11 /tmp/restic/cmd_find.go + -rw-rw-r-- 1000 1000 2834 2025-02-03 08:09:35 /tmp/restic/find.go + dtrwxrwxrwx 0 0 0 2025-02-03 08:14:22 /tmp + -rw-rw-r-- 1000 1000 1473 2025-02-03 08:15:30 /tmp/restic/010_introduction.rst + drwxrwxr-x 1000 1000 0 2025-02-03 08:15:46 /tmp/restic + -rw-rw-r-- 1000 1000 3077 2025-02-03 08:15:46 /tmp/restic/conf.py + +Sorting works with option ``--json`` as well. Sorting and option ``--ncdu`` are mutually exclusive. +It works also without specifying the option ``--long``. + +.. code-block:: console + + $ restic ls latest --sort extension + + snapshot 711b0bb6 of [/tmp/restic] at 2025-02-03 08:16:05.310764668 +0000 UTC filtered by []: + /tmp + /tmp/restic + /tmp/restic/cmd_find.go + /tmp/restic/find.go + /tmp/restic/conf.py + /tmp/restic/010_introduction.rst + Copying snapshots between repositories ====================================== @@ -317,7 +360,7 @@ Modifying metadata of snapshots =============================== Sometimes it may be desirable to change the metadata of an existing snapshot. -Currently, rewriting the hostname and the time of the backup is supported. +Currently, rewriting the hostname and the time of the backup is supported. This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp. .. code-block:: console