ls: sort output by size, atime, ctime, mtime, time(=mtime), extension (#5182)

Enhancement: create ability to sort output of restic ls -l by
name, size, atime, ctime, mtime, time(=mtime), X(=extension), extension

---------

Co-authored-by: Michael Eischer <michael.eischer@fau.de>

changelog/unreleased/issue-4179

@@ -0,0 +1,11 @@
+Enhancement: add sort options for `ls` command
+
+in the past, the output of ls -l was sorted by name. Now it can be sorted by
+one of the specifiers (name|size|time=mtime|atime|ctime|extension).
+Use --sort <sortable field> to achieve this.
+
+Reverse sorting also has been implemtented. Use --reverse to indicate reverse
+sorting.
+
+https://github.com/restic/restic/issues/4179
+https://github.com/restic/restic/pull/5182

cmd/restic/cmd_ls.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"cmp"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"slices"
 	"strings"
 	"time"
 
@@ -36,6 +39,10 @@ will allow traversing into matching directories' subfolders.
 Any directory paths specified must be absolute (starting with
 a path separator); paths use the forward slash '/' as separator.
 
+File listings can be sorted by specifying --sort followed by one of the
+sort specifiers '(name|size|time=mtime|atime|ctime|extension)'.
+The sorting can be reversed by specifying --reverse.
+
 EXIT STATUS
 ===========
 
@@ -59,6 +66,8 @@ type LsOptions struct {
 	Recursive     bool
 	HumanReadable bool
 	Ncdu          bool
+	Sort          string
+	Reverse       bool
 }
 
 var lsOptions LsOptions
@@ -72,6 +81,8 @@ func init() {
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 	flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
 	flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
+	flags.StringVarP(&lsOptions.Sort, "sort", "s", "name", "sort output by (name|size|time=mtime|atime|ctime|extension)")
+	flags.BoolVar(&lsOptions.Reverse, "reverse", false, "reverse sorted output")
 }
 
 type lsPrinter interface {
@@ -277,6 +288,12 @@ func (p *textLsPrinter) Close() error {
 	return nil
 }
 
+// for ls -l output sorting
+type toSortOutput struct {
+	nodepath string
+	node     *restic.Node
+}
+
 func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
 		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
@@ -284,6 +301,18 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 	if opts.Ncdu && gopts.JSON {
 		return errors.Fatal("only either '--json' or '--ncdu' can be specified")
 	}
+	if opts.Sort != "name" && opts.Ncdu {
+		return errors.Fatal("--sort and --ncdu are mutually exclusive")
+	}
+	if opts.Reverse && opts.Ncdu {
+		return errors.Fatal("--reverse and --ncdu are mutually exclusive")
+	}
+
+	sortMode := SortModeName
+	err := sortMode.Set(opts.Sort)
+	if err != nil {
+		return err
+	}
 
 	// extract any specific directories to walk
 	var dirs []string
@@ -347,6 +376,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 	}
 
 	var printer lsPrinter
+	collector := []toSortOutput{}
+	outputSort := sortMode != SortModeName || opts.Reverse
 
 	if gopts.JSON {
 		printer = &jsonLsPrinter{
@@ -356,6 +387,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		printer = &ncduLsPrinter{
 			out: globalOptions.stdout,
 		}
+		outputSort = false
 	} else {
 		printer = &textLsPrinter{
 			dirs:          dirs,
@@ -393,8 +425,12 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		printedDir := false
 		if withinDir(nodepath) {
 			// if we're within a target path, print the node
-			if err := printer.Node(nodepath, node, false); err != nil {
+			if outputSort {
-				return err
+				collector = append(collector, toSortOutput{nodepath, node})
+			} else {
+				if err := printer.Node(nodepath, node, false); err != nil {
+					return err
+				}
 			}
 			printedDir = true
 
@@ -409,7 +445,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		// there yet), signal the walker to descend into any subdirs
 		if approachingMatchingTree(nodepath) {
 			// print node leading up to the target paths
-			if !printedDir {
+			if !printedDir && !outputSort {
 				return printer.Node(nodepath, node, true)
 			}
 			return nil
@@ -444,5 +480,101 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
+	if outputSort {
+		printSortedOutput(printer, opts, sortMode, collector)
+	}
+
 	return printer.Close()
 }
+
+func printSortedOutput(printer lsPrinter, opts LsOptions, sortMode SortMode, collector []toSortOutput) {
+	switch sortMode {
+	case SortModeName:
+	case SortModeSize:
+		slices.SortStableFunc(collector, func(a, b toSortOutput) int {
+			return cmp.Or(
+				cmp.Compare(a.node.Size, b.node.Size),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		})
+	case SortModeMtime:
+		slices.SortStableFunc(collector, func(a, b toSortOutput) int {
+			return cmp.Or(
+				a.node.ModTime.Compare(b.node.ModTime),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		})
+	case SortModeAtime:
+		slices.SortStableFunc(collector, func(a, b toSortOutput) int {
+			return cmp.Or(
+				a.node.AccessTime.Compare(b.node.AccessTime),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		})
+	case SortModeCtime:
+		slices.SortStableFunc(collector, func(a, b toSortOutput) int {
+			return cmp.Or(
+				a.node.ChangeTime.Compare(b.node.ChangeTime),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		})
+	case SortModeExt:
+		// map name to extension
+		mapExt := make(map[string]string, len(collector))
+		for _, item := range collector {
+			ext := filepath.Ext(item.nodepath)
+			mapExt[item.nodepath] = ext
+		}
+
+		slices.SortStableFunc(collector, func(a, b toSortOutput) int {
+			return cmp.Or(
+				cmp.Compare(mapExt[a.nodepath], mapExt[b.nodepath]),
+				cmp.Compare(a.nodepath, b.nodepath),
+			)
+		})
+	}
+
+	if opts.Reverse {
+		slices.Reverse(collector)
+	}
+	for _, elem := range collector {
+		_ = printer.Node(elem.nodepath, elem.node, false)
+	}
+}
+
+// SortMode defines the allowed sorting modes
+type SortMode string
+
+// Allowed sort modes
+const (
+	SortModeName    SortMode = "name"
+	SortModeSize    SortMode = "size"
+	SortModeAtime   SortMode = "atime"
+	SortModeCtime   SortMode = "ctime"
+	SortModeMtime   SortMode = "mtime"
+	SortModeExt     SortMode = "extension"
+	SortModeInvalid SortMode = "--invalid--"
+)
+
+// Set implements the method needed for pflag command flag parsing.
+func (c *SortMode) Set(s string) error {
+	switch s {
+	case "name":
+		*c = SortModeName
+	case "size":
+		*c = SortModeSize
+	case "atime":
+		*c = SortModeAtime
+	case "ctime":
+		*c = SortModeCtime
+	case "mtime", "time":
+		*c = SortModeMtime
+	case "extension":
+		*c = SortModeExt
+	default:
+		*c = SortModeInvalid
+		return fmt.Errorf("invalid sort mode %q, must be one of (name|size|atime|ctime|mtime=time|extension)", s)
+	}
+
+	return nil
+}

cmd/restic/cmd_ls_integration_test.go

@@ -19,7 +19,7 @@ func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args [
 }
 
 func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
-	out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
+	out := testRunLsWithOpts(t, gopts, LsOptions{Sort: "name"}, []string{snapshotID})
 	return strings.Split(string(out), "\n")
 }
 
@@ -45,7 +45,64 @@ func TestRunLsNcdu(t *testing.T) {
 		{"latest", "/0"},
 		{"latest", "/0", "/0/9"},
 	} {
-		ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, paths)
+		ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true, Sort: "name"}, paths)
 		assertIsValidJSON(t, ncdu)
 	}
 }
+
+func TestRunLsSort(t *testing.T) {
+	compareName := []string{
+		"/for_cmd_ls",
+		"/for_cmd_ls/file1.txt",
+		"/for_cmd_ls/file2.txt",
+		"/for_cmd_ls/python.py",
+		"", // last empty line
+	}
+
+	compareSize := []string{
+		"/for_cmd_ls",
+		"/for_cmd_ls/file2.txt",
+		"/for_cmd_ls/file1.txt",
+		"/for_cmd_ls/python.py",
+		"",
+	}
+
+	compareExt := []string{
+		"/for_cmd_ls",
+		"/for_cmd_ls/python.py",
+		"/for_cmd_ls/file1.txt",
+		"/for_cmd_ls/file2.txt",
+		"",
+	}
+
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	testRunBackup(t, env.testdata+"/0", []string{"for_cmd_ls"}, opts, env.gopts)
+
+	// sort by size
+	out := testRunLsWithOpts(t, env.gopts, LsOptions{Sort: "size"}, []string{"latest"})
+	fileList := strings.Split(string(out), "\n")
+	rtest.Assert(t, len(fileList) == 5, "invalid ls --sort size, expected 5 array elements, got %v", len(fileList))
+	for i, item := range compareSize {
+		rtest.Assert(t, item == fileList[i], "invalid ls --sort size, expected element '%s', got '%s'", item, fileList[i])
+	}
+
+	// sort by file extension
+	out = testRunLsWithOpts(t, env.gopts, LsOptions{Sort: "extension"}, []string{"latest"})
+	fileList = strings.Split(string(out), "\n")
+	rtest.Assert(t, len(fileList) == 5, "invalid ls --sort extension, expected 5 array elements, got %v", len(fileList))
+	for i, item := range compareExt {
+		rtest.Assert(t, item == fileList[i], "invalid ls --sort extension, expected element '%s', got '%s'", item, fileList[i])
+	}
+
+	// explicit name sort
+	out = testRunLsWithOpts(t, env.gopts, LsOptions{Sort: "name"}, []string{"latest"})
+	fileList = strings.Split(string(out), "\n")
+	rtest.Assert(t, len(fileList) == 5, "invalid ls --sort name, expected 5 array elements, got %v", len(fileList))
+	for i, item := range compareName {
+		rtest.Assert(t, item == fileList[i], "invalid ls --sort name, expected element '%s', got '%s'", item, fileList[i])
+	}
+}

doc/045_working_with_repos.rst

@@ -153,6 +153,49 @@ outputting information about a snapshot in the NCDU format using the ``--ncdu``
 
 You can use it as follows: ``restic ls latest --ncdu | ncdu -f -``
 
+You can use the options ``--sort`` and ``--reverse`` to tailor ``ls`` output to your needs.
+``--sort`` can be one of ``name | size | time=mtime | atime | ctime | extension``. The default
+sorting option is ``name``. The sorting order can be reversed by specifying ``--reverse``.
+
+.. code-block:: console
+
+    $ restic ls --long latest --sort size --reverse
+
+    snapshot 711b0bb6 of [/tmp/restic] at 2025-02-03 08:16:05.310764668 +0000 UTC filtered by []:
+    -rw-rw-r--  1000  1000  16772 2025-02-03 08:09:11 /tmp/restic/cmd_find.go
+    -rw-rw-r--  1000  1000   3077 2025-02-03 08:15:46 /tmp/restic/conf.py
+    -rw-rw-r--  1000  1000   2834 2025-02-03 08:09:35 /tmp/restic/find.go
+    -rw-rw-r--  1000  1000   1473 2025-02-03 08:15:30 /tmp/restic/010_introduction.rst
+    drwxrwxr-x  1000  1000      0 2025-02-03 08:15:46 /tmp/restic
+    dtrwxrwxrwx     0     0      0 2025-02-03 08:14:22 /tmp
+
+.. code-block:: console
+
+    $ restic ls --long latest --sort time
+
+    snapshot 711b0bb6 of [/tmp/restic] at 2025-02-03 08:16:05.310764668 +0000 UTC filtered by []:
+    -rw-rw-r--  1000  1000  16772 2025-02-03 08:09:11 /tmp/restic/cmd_find.go
+    -rw-rw-r--  1000  1000   2834 2025-02-03 08:09:35 /tmp/restic/find.go
+    dtrwxrwxrwx     0     0      0 2025-02-03 08:14:22 /tmp
+    -rw-rw-r--  1000  1000   1473 2025-02-03 08:15:30 /tmp/restic/010_introduction.rst
+    drwxrwxr-x  1000  1000      0 2025-02-03 08:15:46 /tmp/restic
+    -rw-rw-r--  1000  1000   3077 2025-02-03 08:15:46 /tmp/restic/conf.py
+
+Sorting works with option ``--json`` as well. Sorting and option ``--ncdu`` are mutually exclusive.
+It works also without specifying the option ``--long``.
+
+.. code-block:: console
+
+    $ restic ls latest --sort extension
+
+    snapshot 711b0bb6 of [/tmp/restic] at 2025-02-03 08:16:05.310764668 +0000 UTC filtered by []:
+    /tmp
+    /tmp/restic
+    /tmp/restic/cmd_find.go
+    /tmp/restic/find.go
+    /tmp/restic/conf.py
+    /tmp/restic/010_introduction.rst
+
 
 Copying snapshots between repositories
 ======================================