From 79d435efb1571f887989118dd3b3df41683a4fcb Mon Sep 17 00:00:00 2001 From: Dark Dragon Date: Sun, 2 Feb 2025 17:30:45 +0100 Subject: [PATCH 1/4] Use printer.NewCounter() instead of newTerminalProgressMax() where possible (max is unknown). --- cmd/restic/cmd_check.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 8788b0caf..3ae79149e 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -316,7 +316,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args wg.Add(1) go func() { defer wg.Done() - bar := newTerminalProgressMax(!gopts.Quiet, 0, "snapshots", term) + bar := printer.NewCounter("snapshots") defer bar.Done() chkr.Structure(ctx, bar, errChan) }() @@ -353,9 +353,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } doReadData := func(packs map[restic.ID]int64) { - packCount := uint64(len(packs)) - - p := newTerminalProgressMax(!gopts.Quiet, packCount, "packs", term) + p := printer.NewCounter("packs") + p.SetMax(uint64(len(packs))) errChan := make(chan error) go chkr.ReadPacks(ctx, packs, p, errChan) From a58a8f2ce0f536bc7c9d4696b3b688fdcebd1cbd Mon Sep 17 00:00:00 2001 From: Dark Dragon Date: Mon, 30 Dec 2024 22:33:01 +0100 Subject: [PATCH 2/4] Add JSON output to check command --- changelog/unreleased/issue-1378 | 7 +++++ cmd/restic/cmd_check.go | 50 ++++++++++++++++++++++++++++++++- doc/075_scripting.rst | 12 ++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-1378 diff --git a/changelog/unreleased/issue-1378 b/changelog/unreleased/issue-1378 new file mode 100644 index 000000000..fd5d379cc --- /dev/null +++ b/changelog/unreleased/issue-1378 @@ -0,0 +1,7 @@ +Enhancement: Add JSON support to check + +Restic `check` now also supports the `--json` option and gives all +statistics in JSON format. + +https://github.com/restic/restic/issues/1378 +https://github.com/restic/restic/pull/5194 diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 3ae79149e..765e93744 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "math/rand" "os" "strconv" @@ -215,7 +216,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags") } - printer := newTerminalProgressPrinter(gopts.verbosity, term) + var printer progress.Printer + if !gopts.JSON { + printer = newTerminalProgressPrinter(gopts.verbosity, term) + } else { + printer = newJSONErrorPrinter(term) + } cleanup := prepareCheckCache(opts, &gopts, printer) defer cleanup() @@ -431,6 +437,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args return errors.Fatal("repository contains errors") } printer.P("no errors were found\n") + if gopts.JSON { + status := checkSuccess{ + MessageType: "checked", + Message: "no errors were found", + } + term.Print(ui.ToJSONString(status)) + } return nil } @@ -478,3 +491,38 @@ func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, packs := selectRandomPacksByPercentage(allPacks, subsetPercentage) return packs } + +type checkSuccess struct { + MessageType string `json:"message_type"` // "checked" + Message string `json:"message"` +} + +type checkError struct { + MessageType string `json:"message_type"` // "error" + Message string `json:"message"` +} + +type jsonErrorPrinter struct { + term ui.Terminal +} + +func newJSONErrorPrinter(term ui.Terminal) *jsonErrorPrinter { + return &jsonErrorPrinter{ + term: term, + } +} + +func (*jsonErrorPrinter) NewCounter(_ string) *progress.Counter { + return nil +} + +func (p *jsonErrorPrinter) E(msg string, args ...interface{}) { + status := checkError{ + MessageType: "error", + Message: fmt.Sprintf(msg, args...), + } + p.term.Print(ui.ToJSONString(status)) +} +func (*jsonErrorPrinter) P(_ string, _ ...interface{}) {} +func (*jsonErrorPrinter) V(_ string, _ ...interface{}) {} +func (*jsonErrorPrinter) VV(_ string, _ ...interface{}) {} diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 57a8e2872..3cc009c15 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -245,6 +245,18 @@ are stored in JSON form. Specifying ``--json`` or ``--quiet`` will suppress any non-JSON messages the command generates. +check +----- + +The ``check`` command outputs JSON messages with the following format: + ++------------------+--------------------------------+ +| ``message_type`` | Either "checked" or "error" | ++------------------+--------------------------------+ +| ``message`` | Descriptive message | ++------------------+--------------------------------+ + + diff ---- From 7cc1aa0cd4b96f0b63a6e07e3d020c8899939833 Mon Sep 17 00:00:00 2001 From: Dark Dragon Date: Mon, 30 Dec 2024 23:44:46 +0100 Subject: [PATCH 3/4] Add check summary --- cmd/restic/cmd_check.go | 53 ++++++++++++++++++++++------------------- doc/075_scripting.rst | 32 ++++++++++++++++++++----- 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 765e93744..0f710d30e 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -212,6 +212,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress } func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error { + summary := checkSummary{MessageType: "summary"} if len(args) != 0 { return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags") } @@ -249,26 +250,24 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } errorsFound := false - suggestIndexRebuild := false - mixedFound := false for _, hint := range hints { switch hint.(type) { case *checker.ErrDuplicatePacks: term.Print(hint.Error()) - suggestIndexRebuild = true + summary.HintRepairIndex = true case *checker.ErrMixedPack: term.Print(hint.Error()) - mixedFound = true + summary.HintPrune = true default: printer.E("error: %v\n", hint) errorsFound = true } } - if suggestIndexRebuild { + if summary.HintRepairIndex { term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n") } - if mixedFound { + if summary.HintPrune { term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n") } @@ -277,6 +276,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args printer.E("error: %v\n", err) } + summary.NumErrors += len(errs) + summary.HintRepairIndex = true printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n") return errors.Fatal("repository contains errors") } @@ -299,6 +300,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args salvagePacks.Insert(packErr.ID) } errorsFound = true + summary.NumErrors++ printer.E("%v\n", err) } } else { @@ -307,9 +309,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } } - if orphanedPacks > 0 && !errorsFound { - // hide notice if repository is damaged - printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks) + if orphanedPacks > 0 { + summary.HintPrune = true + if !errorsFound { + // hide notice if repository is damaged + printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks) + } } if ctx.Err() != nil { return ctx.Err() @@ -332,9 +337,11 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args if e, ok := err.(*checker.TreeError); ok { printer.E("error for tree %v:\n", e.ID.Str()) for _, treeErr := range e.Errors { + summary.NumErrors++ printer.E(" %v\n", treeErr) } } else { + summary.NumErrors++ printer.E("error: %v\n", err) } } @@ -367,6 +374,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args for err := range errChan { errorsFound = true + summary.NumErrors++ printer.E("%v\n", err) if err, ok := err.(*repository.ErrPackData); ok { salvagePacks.Insert(err.PackID) @@ -418,11 +426,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args if len(salvagePacks) > 0 { printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") - var strIDs []string for id := range salvagePacks { - strIDs = append(strIDs, id.String()) + summary.BrokenPacks = append(summary.BrokenPacks, id.String()) } - printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) + printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(summary.BrokenPacks, " ")) printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") } @@ -430,6 +437,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args return ctx.Err() } + if gopts.JSON { + term.Print(ui.ToJSONString(summary)) + } if errorsFound { if len(salvagePacks) == 0 { printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n") @@ -437,14 +447,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args return errors.Fatal("repository contains errors") } printer.P("no errors were found\n") - if gopts.JSON { - status := checkSuccess{ - MessageType: "checked", - Message: "no errors were found", - } - term.Print(ui.ToJSONString(status)) - } - return nil } @@ -492,9 +494,12 @@ func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, return packs } -type checkSuccess struct { - MessageType string `json:"message_type"` // "checked" - Message string `json:"message"` +type checkSummary struct { + MessageType string `json:"message_type"` // "summary" + NumErrors int `json:"num_errors"` + BrokenPacks []string `json:"broken_packs"` // run "restic repair packs ID..." and "restic repair snapshots --forget" to remove damaged files + HintRepairIndex bool `json:"suggest_repair_index"` // run "restic repair index" + HintPrune bool `json:"suggest_prune"` // run "restic prune" } type checkError struct { @@ -521,7 +526,7 @@ func (p *jsonErrorPrinter) E(msg string, args ...interface{}) { MessageType: "error", Message: fmt.Sprintf(msg, args...), } - p.term.Print(ui.ToJSONString(status)) + p.term.Error(ui.ToJSONString(status)) } func (*jsonErrorPrinter) P(_ string, _ ...interface{}) {} func (*jsonErrorPrinter) V(_ string, _ ...interface{}) {} diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 3cc009c15..8e0846cc3 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -248,13 +248,33 @@ non-JSON messages the command generates. check ----- -The ``check`` command outputs JSON messages with the following format: +The ``check`` command uses the JSON lines format with the following message types. -+------------------+--------------------------------+ -| ``message_type`` | Either "checked" or "error" | -+------------------+--------------------------------+ -| ``message`` | Descriptive message | -+------------------+--------------------------------+ +Status +^^^^^^ + ++--------------------------+------------------------------------------------------------------------------------------------+ +| ``message_type`` | Always "summary" | ++--------------------------+------------------------------------------------------------------------------------------------+ +| ``num_errors`` | Number of errors | ++--------------------------+------------------------------------------------------------------------------------------------+ +| ``broken_packs`` | Run "restic repair packs ID..." and "restic repair snapshots --forget" to remove damaged files | ++--------------------------+------------------------------------------------------------------------------------------------+ +| ``suggest_repair_index`` | Run "restic repair index" | ++--------------------------+------------------------------------------------------------------------------------------------+ +| ``suggest_prune`` | Run "restic prune" | ++--------------------------+------------------------------------------------------------------------------------------------+ + +Error +^^^^^ + +These errors are printed on ``stderr``. + ++----------------------+---------------------------------------------------------------------+ +| ``message_type`` | Always "error" | ++----------------------+---------------------------------------------------------------------+ +| ``message`` | Error message. May change in arbitrary ways across restic versions. | ++----------------------+---------------------------------------------------------------------+ diff From 49a411f7acf479df6a03d4fd2f05bbe9ab803a87 Mon Sep 17 00:00:00 2001 From: Dark Dragon Date: Sun, 2 Feb 2025 17:43:54 +0100 Subject: [PATCH 4/4] Print JSON summary in all error cases --- cmd/restic/cmd_check.go | 40 +++++++++++++----------- cmd/restic/cmd_check_integration_test.go | 3 +- cmd/restic/cmd_migrate.go | 2 +- cmd/restic/cmd_prune_integration_test.go | 6 ++-- cmd/restic/integration_test.go | 3 +- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 0f710d30e..99e933af2 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -46,7 +46,14 @@ Exit status is 12 if the password is incorrect. RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() defer cancel() - return runCheck(cmd.Context(), checkOptions, globalOptions, args, term) + summary, err := runCheck(cmd.Context(), checkOptions, globalOptions, args, term) + if globalOptions.JSON { + if err != nil && summary.NumErrors == 0 { + summary.NumErrors = 1 + } + term.Print(ui.ToJSONString(summary)) + } + return err }, PreRunE: func(_ *cobra.Command, _ []string) error { return checkFlags(checkOptions) @@ -211,10 +218,10 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress return cleanup } -func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error { +func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) (checkSummary, error) { summary := checkSummary{MessageType: "summary"} if len(args) != 0 { - return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags") + return summary, errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags") } var printer progress.Printer @@ -232,21 +239,21 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock) if err != nil { - return err + return summary, err } defer unlock() chkr := checker.New(repo, opts.CheckUnused) err = chkr.LoadSnapshots(ctx) if err != nil { - return err + return summary, err } printer.P("load indexes\n") bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) hints, errs := chkr.LoadIndex(ctx, bar) if ctx.Err() != nil { - return ctx.Err() + return summary, ctx.Err() } errorsFound := false @@ -279,7 +286,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args summary.NumErrors += len(errs) summary.HintRepairIndex = true printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n") - return errors.Fatal("repository contains errors") + return summary, errors.Fatal("repository contains errors") } orphanedPacks := 0 @@ -317,7 +324,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } } if ctx.Err() != nil { - return ctx.Err() + return summary, ctx.Err() } printer.P("check snapshots, trees and blobs\n") @@ -351,13 +358,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args // deadlocking in the case of errors. wg.Wait() if ctx.Err() != nil { - return ctx.Err() + return summary, ctx.Err() } if opts.CheckUnused { unused, err := chkr.UnusedBlobs(ctx) if err != nil { - return err + return summary, err } for _, id := range unused { printer.P("unused blob %v\n", id) @@ -409,7 +416,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args repoSize += size } if repoSize == 0 { - return errors.Fatal("Cannot read from a repository having size 0") + return summary, errors.Fatal("Cannot read from a repository having size 0") } subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset) if subsetSize > repoSize { @@ -419,7 +426,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args printer.P("read %d bytes of data packs\n", subsetSize) } if packs == nil { - return errors.Fatal("internal error: failed to select packs to check") + return summary, errors.Fatal("internal error: failed to select packs to check") } doReadData(packs) } @@ -434,20 +441,17 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } if ctx.Err() != nil { - return ctx.Err() + return summary, ctx.Err() } - if gopts.JSON { - term.Print(ui.ToJSONString(summary)) - } if errorsFound { if len(salvagePacks) == 0 { printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n") } - return errors.Fatal("repository contains errors") + return summary, errors.Fatal("repository contains errors") } printer.P("no errors were found\n") - return nil + return summary, nil } // selectPacksByBucket selects subsets of packs by ranges of buckets. diff --git a/cmd/restic/cmd_check_integration_test.go b/cmd/restic/cmd_check_integration_test.go index f1e6517e0..f5a3dc395 100644 --- a/cmd/restic/cmd_check_integration_test.go +++ b/cmd/restic/cmd_check_integration_test.go @@ -32,7 +32,8 @@ func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) { ReadData: true, CheckUnused: checkUnused, } - return runCheck(context.TODO(), opts, gopts, nil, term) + _, err := runCheck(context.TODO(), opts, gopts, nil, term) + return err }) return buf.String(), err } diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index 5c3e425ed..f6c28e383 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -105,7 +105,7 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio // the repository is already locked checkGopts.NoLock = true - err = runCheck(ctx, checkOptions, checkGopts, []string{}, term) + _, err = runCheck(ctx, checkOptions, checkGopts, []string{}, term) if err != nil { return err } diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 536ec40d8..0561f8243 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -112,7 +112,8 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) { createPrunableRepo(t, env) testRunPrune(t, env.gopts, pruneOpts) rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { - return runCheck(context.TODO(), checkOpts, env.gopts, nil, term) + _, err := runCheck(context.TODO(), checkOpts, env.gopts, nil, term) + return err })) } @@ -220,7 +221,8 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o testRunCheck(t, env.gopts) } else { rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { - return runCheck(context.TODO(), optionsCheck, env.gopts, nil, term) + _, err := runCheck(context.TODO(), optionsCheck, env.gopts, nil, term) + return err }) != nil, "check should have reported an error") } diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 777573f26..3ef98a168 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -88,7 +88,8 @@ func TestListOnce(t *testing.T) { createPrunableRepo(t, env) testRunPrune(t, env.gopts, pruneOpts) rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { - return runCheck(context.TODO(), checkOpts, env.gopts, nil, term) + _, err := runCheck(context.TODO(), checkOpts, env.gopts, nil, term) + return err })) rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)