diff --git a/changelog/unreleased/issue-3202 b/changelog/unreleased/issue-3202 new file mode 100644 index 000000000..e6fca05e1 --- /dev/null +++ b/changelog/unreleased/issue-3202 @@ -0,0 +1,10 @@ +Enhancement: Add warmup support on S3 backend before repacks and restores + +Introduce S3 backend options for transitioning pack files from cold to hot +storage on S3 and S3-compatible providers. Note: only works before repacks +(prune/copy) and restore for now, and gated behind a new "s3-restore" feature +flag. + +https://github.com/restic/restic/pull/5173 +https://github.com/restic/restic/issues/3202 +https://github.com/restic/restic/issues/2504 diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index cd92193ac..301e0e180 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -237,7 +237,15 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep } bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied") - _, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar) + _, err = repository.Repack( + ctx, + srcRepo, + dstRepo, + packList, + copyBlobs, + bar, + func(msg string, args ...interface{}) { fmt.Printf(msg+"\n", args...) }, + ) bar.Done() if err != nil { return errors.Fatal(err.Error()) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 7a3b029da..c930abc31 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -179,6 +179,12 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, res.Warn = func(message string) { msg.E("Warning: %s\n", message) } + res.Info = func(message string) { + if gopts.JSON { + return + } + msg.P("Info: %s\n", message) + } selectExcludeFilter := func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) { matched := false diff --git a/doc/faq.rst b/doc/faq.rst index 74dd77d71..3b62f641d 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -242,3 +242,33 @@ collect a list of all files, causing the following error: List(data) returned error, retrying after 1s: [...]: request timeout In this case you can increase the timeout using the ``--stuck-request-timeout`` option. + +Are "cold storages" supported? +------------------------------ + +Generally, restic does not natively support "cold storage" solutions. However, +experimental support for restoring from **S3 Glacier** and **S3 Glacier Deep +Archive** storage classes is available: + +.. code-block:: console + + $ restic backup -o s3.storage-class=GLACIER somedir/ + $ RESTIC_FEATURES=s3-restore restic restore -o s3.enable-restore=1 -o s3.restore-days=7 -o s3.restore-timeout=1d latest + +**Notes:** + +- This feature is still in early alpha stage. Expect arbitrary breaking changes + in the future (although we'll do our best-effort to avoid them). +- Expect restores to hang from 1 up to 42 hours depending on your storage + class, provider and luck. Restores from cold storages are known to be + time-consuming. You may need to adjust the `s3.restore-timeout` if a restore + operation takes more than 24 hours. +- Restic will prevent sending metadata files (such as config files, lock files + or tree blobs) to Glacier or Deep Archive. Standard class is used instead to + ensure normal and fast operations for most tasks. +- Currently, only the following commands are known to work: + + - `backup` + - `copy` + - `prune` + - `restore` diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 27390ee13..b2ef7ec30 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -475,3 +475,9 @@ func (be *Backend) Delete(ctx context.Context) error { // Close does nothing func (be *Backend) Close() error { return nil } + +// Warmup not implemented +func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 3ef2bcbe3..6f66b3673 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -335,3 +335,9 @@ func (be *b2Backend) Delete(ctx context.Context) error { // Close does nothing func (be *b2Backend) Close() error { return nil } + +// Warmup not implemented +func (be *b2Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (be *b2Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/backend.go b/internal/backend/backend.go index f606e1123..2529dfab5 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -75,6 +75,21 @@ type Backend interface { // Delete removes all data in the backend. Delete(ctx context.Context) error + + // Warmup ensures that the specified handles are ready for upcoming reads. + // This is particularly useful for transitioning files from cold to hot + // storage. + // + // The method is non-blocking. WarmupWait can be used to wait for + // completion. + // + // Returns: + // - Handles currently warming up. + // - An error if warmup fails. + Warmup(ctx context.Context, h []Handle) ([]Handle, error) + + // WarmupWait waits until all given handles are warm. + WarmupWait(ctx context.Context, h []Handle) error } type Unwrapper interface { diff --git a/internal/backend/cache/backend.go b/internal/backend/cache/backend.go index 3754266ba..f323b1663 100644 --- a/internal/backend/cache/backend.go +++ b/internal/backend/cache/backend.go @@ -258,3 +258,13 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backen return nil } + +// Warmup delegates to wrapped backend. +func (b *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) { + return b.Backend.Warmup(ctx, h) +} + +// WarmupWait delegates to wrapped backend. +func (b *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error { + return b.Backend.WarmupWait(ctx, h) +} diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go index 8af0ce9ad..fbce41916 100644 --- a/internal/backend/dryrun/dry_backend.go +++ b/internal/backend/dryrun/dry_backend.go @@ -82,3 +82,9 @@ func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offse func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { return be.b.Stat(ctx, h) } + +// Warmup should not occur during dry-runs. +func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index ad50f194b..ab20ca103 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -363,3 +363,9 @@ func (be *Backend) Delete(ctx context.Context) error { // Close does nothing. func (be *Backend) Close() error { return nil } + +// Warmup not implemented +func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index ee87ae5d6..e2065742f 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -371,3 +371,9 @@ func (b *Local) Close() error { // same function. return nil } + +// Warmup not implemented +func (b *Local) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (b *Local) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 981c0a182..e5ee297a1 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -249,3 +249,9 @@ func (be *MemoryBackend) Delete(ctx context.Context) error { func (be *MemoryBackend) Close() error { return nil } + +// Warmup not implemented +func (be *MemoryBackend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (be *MemoryBackend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/mock/backend.go b/internal/backend/mock/backend.go index a03198443..2083f7e88 100644 --- a/internal/backend/mock/backend.go +++ b/internal/backend/mock/backend.go @@ -20,6 +20,8 @@ type Backend struct { ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error RemoveFn func(ctx context.Context, h backend.Handle) error DeleteFn func(ctx context.Context) error + WarmupFn func(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) + WarmupWaitFn func(ctx context.Context, h []backend.Handle) error ConnectionsFn func() uint HasherFn func() hash.Hash HasAtomicReplaceFn func() bool @@ -150,5 +152,21 @@ func (m *Backend) Delete(ctx context.Context) error { return m.DeleteFn(ctx) } +func (m *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) { + if m.WarmupFn == nil { + return []backend.Handle{}, errors.New("not implemented") + } + + return m.WarmupFn(ctx, h) +} + +func (m *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error { + if m.WarmupWaitFn == nil { + return errors.New("not implemented") + } + + return m.WarmupWaitFn(ctx, h) +} + // Make sure that Backend implements the backend interface. var _ backend.Backend = &Backend{} diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index 8294aa8c4..fb5ed34eb 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -340,3 +340,9 @@ func (be *Backend) Close() error { debug.Log("wait for rclone returned: %v", be.waitResult) return be.waitResult } + +// Warmup not implemented +func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 7bdedff39..2c5f59b4e 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -439,3 +439,9 @@ func (b *Backend) Close() error { func (b *Backend) Delete(ctx context.Context) error { return util.DefaultDelete(ctx, b) } + +// Warmup not implemented +func (b *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (b *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/retry/backend_retry.go b/internal/backend/retry/backend_retry.go index de8a520ec..aa48bde77 100644 --- a/internal/backend/retry/backend_retry.go +++ b/internal/backend/retry/backend_retry.go @@ -289,3 +289,11 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend func (be *Backend) Unwrap() backend.Backend { return be.Backend } + +// Warmup delegates to wrapped backend +func (be *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) { + return be.Backend.Warmup(ctx, h) +} +func (be *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error { + return be.Backend.WarmupWait(ctx, h) +} diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index be2a78ce5..77f27408e 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -5,6 +5,7 @@ import ( "os" "path" "strings" + "time" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" @@ -23,6 +24,11 @@ type Config struct { Layout string `option:"layout" help:"use this backend layout (default: auto-detect) (deprecated)"` StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"` + EnableRestore bool `option:"enable-restore" help:"restore objects from GLACIER or DEEP_ARCHIVE storage classes (default: false, requires \"s3-restore\" feature flag)"` + RestoreDays int `option:"restore-days" help:"lifetime in days of restored object (default: 7)"` + RestoreTimeout time.Duration `option:"restore-timeout" help:"maximum time to wait for objects transition (default: 1d)"` + RestoreTier string `option:"restore-tier" help:"Retrieval tier at which the restore will be processed. (Standard, Bulk or Expedited) (default: Standard)"` + Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` MaxRetries uint `option:"retries" help:"set the number of retries attempted"` Region string `option:"region" help:"set region"` @@ -34,8 +40,12 @@ type Config struct { // NewConfig returns a new Config with the default values filled in. func NewConfig() Config { return Config{ - Connections: 5, - ListObjectsV1: false, + Connections: 5, + ListObjectsV1: false, + EnableRestore: false, + RestoreDays: 7, + RestoreTimeout: 24 * time.Hour, + RestoreTier: "Standard", } } diff --git a/internal/backend/s3/config_test.go b/internal/backend/s3/config_test.go index 085dbeedb..74f959a65 100644 --- a/internal/backend/s3/config_test.go +++ b/internal/backend/s3/config_test.go @@ -3,117 +3,117 @@ package s3 import ( "strings" "testing" + "time" "github.com/restic/restic/internal/backend/test" ) +func newTestConfig(cfg Config) Config { + if cfg.Connections == 0 { + cfg.Connections = 5 + } + if cfg.RestoreDays == 0 { + cfg.RestoreDays = 7 + } + if cfg.RestoreTimeout == 0 { + cfg.RestoreTimeout = 24 * time.Hour + } + if cfg.RestoreTier == "" { + cfg.RestoreTier = "Standard" + } + return cfg +} + var configTests = []test.ConfigTestData[Config]{ - {S: "s3://eu-central-1/bucketname", Cfg: Config{ - Endpoint: "eu-central-1", - Bucket: "bucketname", - Prefix: "", - Connections: 5, - }}, - {S: "s3://eu-central-1/bucketname/", Cfg: Config{ - Endpoint: "eu-central-1", - Bucket: "bucketname", - Prefix: "", - Connections: 5, - }}, - {S: "s3://eu-central-1/bucketname/prefix/directory", Cfg: Config{ - Endpoint: "eu-central-1", - Bucket: "bucketname", - Prefix: "prefix/directory", - Connections: 5, - }}, - {S: "s3://eu-central-1/bucketname/prefix/directory/", Cfg: Config{ - Endpoint: "eu-central-1", - Bucket: "bucketname", - Prefix: "prefix/directory", - Connections: 5, - }}, - {S: "s3:eu-central-1/foobar", Cfg: Config{ - Endpoint: "eu-central-1", - Bucket: "foobar", - Prefix: "", - Connections: 5, - }}, - {S: "s3:eu-central-1/foobar/", Cfg: Config{ - Endpoint: "eu-central-1", - Bucket: "foobar", - Prefix: "", - Connections: 5, - }}, - {S: "s3:eu-central-1/foobar/prefix/directory", Cfg: Config{ - Endpoint: "eu-central-1", - Bucket: "foobar", - Prefix: "prefix/directory", - Connections: 5, - }}, - {S: "s3:eu-central-1/foobar/prefix/directory/", Cfg: Config{ - Endpoint: "eu-central-1", - Bucket: "foobar", - Prefix: "prefix/directory", - Connections: 5, - }}, - {S: "s3:hostname.foo/foobar", Cfg: Config{ - Endpoint: "hostname.foo", - Bucket: "foobar", - Prefix: "", - Connections: 5, - }}, - {S: "s3:hostname.foo/foobar/prefix/directory", Cfg: Config{ - Endpoint: "hostname.foo", - Bucket: "foobar", - Prefix: "prefix/directory", - Connections: 5, - }}, - {S: "s3:https://hostname/foobar", Cfg: Config{ - Endpoint: "hostname", - Bucket: "foobar", - Prefix: "", - Connections: 5, - }}, - {S: "s3:https://hostname:9999/foobar", Cfg: Config{ - Endpoint: "hostname:9999", - Bucket: "foobar", - Prefix: "", - Connections: 5, - }}, - {S: "s3:https://hostname:9999/foobar/", Cfg: Config{ - Endpoint: "hostname:9999", - Bucket: "foobar", - Prefix: "", - Connections: 5, - }}, - {S: "s3:http://hostname:9999/foobar", Cfg: Config{ - Endpoint: "hostname:9999", - Bucket: "foobar", - Prefix: "", - UseHTTP: true, - Connections: 5, - }}, - {S: "s3:http://hostname:9999/foobar/", Cfg: Config{ - Endpoint: "hostname:9999", - Bucket: "foobar", - Prefix: "", - UseHTTP: true, - Connections: 5, - }}, - {S: "s3:http://hostname:9999/bucket/prefix/directory", Cfg: Config{ - Endpoint: "hostname:9999", - Bucket: "bucket", - Prefix: "prefix/directory", - UseHTTP: true, - Connections: 5, - }}, - {S: "s3:http://hostname:9999/bucket/prefix/directory/", Cfg: Config{ - Endpoint: "hostname:9999", - Bucket: "bucket", - Prefix: "prefix/directory", - UseHTTP: true, - Connections: 5, - }}, + {S: "s3://eu-central-1/bucketname", Cfg: newTestConfig(Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "", + })}, + {S: "s3://eu-central-1/bucketname/", Cfg: newTestConfig(Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "", + })}, + {S: "s3://eu-central-1/bucketname/prefix/directory", Cfg: newTestConfig(Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "prefix/directory", + })}, + {S: "s3://eu-central-1/bucketname/prefix/directory/", Cfg: newTestConfig(Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "prefix/directory", + })}, + {S: "s3:eu-central-1/foobar", Cfg: newTestConfig(Config{ + Endpoint: "eu-central-1", + Bucket: "foobar", + Prefix: "", + })}, + {S: "s3:eu-central-1/foobar/", Cfg: newTestConfig(Config{ + Endpoint: "eu-central-1", + Bucket: "foobar", + Prefix: "", + })}, + {S: "s3:eu-central-1/foobar/prefix/directory", Cfg: newTestConfig(Config{ + Endpoint: "eu-central-1", + Bucket: "foobar", + Prefix: "prefix/directory", + })}, + {S: "s3:eu-central-1/foobar/prefix/directory/", Cfg: newTestConfig(Config{ + Endpoint: "eu-central-1", + Bucket: "foobar", + Prefix: "prefix/directory", + })}, + {S: "s3:hostname.foo/foobar", Cfg: newTestConfig(Config{ + Endpoint: "hostname.foo", + Bucket: "foobar", + Prefix: "", + })}, + {S: "s3:hostname.foo/foobar/prefix/directory", Cfg: newTestConfig(Config{ + Endpoint: "hostname.foo", + Bucket: "foobar", + Prefix: "prefix/directory", + })}, + {S: "s3:https://hostname/foobar", Cfg: newTestConfig(Config{ + Endpoint: "hostname", + Bucket: "foobar", + Prefix: "", + })}, + {S: "s3:https://hostname:9999/foobar", Cfg: newTestConfig(Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "", + })}, + {S: "s3:https://hostname:9999/foobar/", Cfg: newTestConfig(Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "", + })}, + {S: "s3:http://hostname:9999/foobar", Cfg: newTestConfig(Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "", + UseHTTP: true, + })}, + {S: "s3:http://hostname:9999/foobar/", Cfg: newTestConfig(Config{ + Endpoint: "hostname:9999", + Bucket: "foobar", + Prefix: "", + UseHTTP: true, + })}, + {S: "s3:http://hostname:9999/bucket/prefix/directory", Cfg: newTestConfig(Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", + UseHTTP: true, + })}, + {S: "s3:http://hostname:9999/bucket/prefix/directory/", Cfg: newTestConfig(Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", + UseHTTP: true, + })}, } func TestParseConfig(t *testing.T) { diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index e3d4cc499..e0d8ea623 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -8,8 +8,11 @@ import ( "net/http" "os" "path" + "slices" "strings" + "time" + "github.com/cenkalti/backoff/v4" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" @@ -32,6 +35,17 @@ type Backend struct { // make sure that *Backend implements backend.Backend var _ backend.Backend = &Backend{} +var archiveClasses = []string{"GLACIER", "DEEP_ARCHIVE"} + +type warmupStatus int + +const ( + warmupStatusCold warmupStatus = iota + warmupStatusWarmingUp + warmupStatusWarm + warmupStatusLukewarm +) + func NewFactory() location.Factory { return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open) } @@ -39,6 +53,10 @@ func NewFactory() location.Factory { func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) + if cfg.EnableRestore && !feature.Flag.Enabled(feature.S3Restore) { + return nil, fmt.Errorf("feature flag `s3-restore` is required to use `-o s3.enable-restore=true`") + } + if cfg.KeyID == "" && cfg.Secret.String() != "" { return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty") } else if cfg.KeyID != "" && cfg.Secret.String() == "" { @@ -266,9 +284,9 @@ func (be *Backend) Path() string { // For archive storage classes, only data files are stored using that class; metadata // must remain instantly accessible. func (be *Backend) useStorageClass(h backend.Handle) bool { - notArchiveClass := be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" isDataFile := h.Type == backend.PackFile && !h.IsMetadata - return isDataFile || notArchiveClass + isArchiveClass := slices.Contains(archiveClasses, be.cfg.StorageClass) + return !isArchiveClass || isDataFile } // Save stores data in the backend at the handle. @@ -440,3 +458,148 @@ func (be *Backend) Delete(ctx context.Context) error { // Close does nothing func (be *Backend) Close() error { return nil } + +// Warmup transitions handles from cold to hot storage if needed. +func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) { + handlesWarmingUp := []backend.Handle{} + + if be.cfg.EnableRestore { + for _, h := range handles { + filename := be.Filename(h) + isWarmingUp, err := be.requestRestore(ctx, filename) + if err != nil { + return handlesWarmingUp, err + } + if isWarmingUp { + debug.Log("s3 file is being restored: %s", filename) + handlesWarmingUp = append(handlesWarmingUp, h) + } + } + } + + return handlesWarmingUp, nil +} + +// requestRestore sends a glacier restore request on a given file. +func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, error) { + objectInfo, err := be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{}) + if err != nil { + return false, err + } + + ws := be.getWarmupStatus(objectInfo) + switch ws { + case warmupStatusWarm: + return false, nil + case warmupStatusWarmingUp: + return true, nil + } + + opts := minio.RestoreRequest{} + opts.SetDays(be.cfg.RestoreDays) + opts.SetGlacierJobParameters(minio.GlacierJobParameters{Tier: minio.TierType(be.cfg.RestoreTier)}) + + if err := be.client.RestoreObject(ctx, be.cfg.Bucket, filename, "", opts); err != nil { + var e minio.ErrorResponse + if errors.As(err, &e) { + switch e.Code { + case "InvalidObjectState": + return false, nil + case "RestoreAlreadyInProgress": + return true, nil + } + } + return false, err + } + + isWarmingUp := ws != warmupStatusLukewarm + return isWarmingUp, nil +} + +// getWarmupStatus returns the warmup status of the provided object. +func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus { + // We can't use objectInfo.StorageClass to get the storage class of the + // object because this field is only set during ListObjects operations. + // The response header is the documented way to get the storage class + // for GetObject/StatObject operations. + storageClass := objectInfo.Metadata.Get("X-Amz-Storage-Class") + isArchiveClass := slices.Contains(archiveClasses, storageClass) + if !isArchiveClass { + return warmupStatusWarm + } + + restore := objectInfo.Restore + if restore != nil { + if restore.OngoingRestore { + return warmupStatusWarmingUp + } + + minExpiryTime := time.Now().Add(time.Duration(be.cfg.RestoreDays) * 24 * time.Hour) + expiryTime := restore.ExpiryTime + if !expiryTime.IsZero() { + if minExpiryTime.Before(expiryTime) { + return warmupStatusWarm + } + return warmupStatusLukewarm + } + } + + return warmupStatusCold +} + +// WarmupWait waits until all handles are in hot storage. +func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) error { + timeoutCtx, timeoutCtxCancel := context.WithTimeout(ctx, be.cfg.RestoreTimeout) + defer timeoutCtxCancel() + + if be.cfg.EnableRestore { + for _, h := range handles { + filename := be.Filename(h) + err := be.waitForRestore(timeoutCtx, filename) + if err != nil { + return err + } + debug.Log("s3 file is restored: %s", filename) + } + } + + return nil +} + +// waitForRestore waits for a given file to be restored. +func (be *Backend) waitForRestore(ctx context.Context, filename string) error { + for { + var objectInfo minio.ObjectInfo + + // Restore requests can last many hours, therefore network may fail + // temporarily. We don't need to die in such even. + b := backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 10) + b = backoff.WithContext(b, ctx) + err := backoff.Retry( + func() (err error) { + objectInfo, err = be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{}) + return + }, + b, + ) + if err != nil { + return err + } + + ws := be.getWarmupStatus(objectInfo) + switch ws { + case warmupStatusLukewarm: + fallthrough + case warmupStatusWarm: + return nil + case warmupStatusCold: + return errors.New("waiting on S3 handle that is not warming up") + } + + select { + case <-time.After(1 * time.Minute): + case <-ctx.Done(): + return ctx.Err() + } + } +} diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 14819a2df..df7c3b14a 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -588,3 +588,9 @@ func (r *SFTP) deleteRecursive(ctx context.Context, name string) error { func (r *SFTP) Delete(ctx context.Context) error { return r.deleteRecursive(ctx, r.p) } + +// Warmup not implemented +func (r *SFTP) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (r *SFTP) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index dfa2055cd..090d00512 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -269,3 +269,9 @@ func (be *beSwift) Delete(ctx context.Context) error { // Close does nothing func (be *beSwift) Close() error { return nil } + +// Warmup not implemented +func (be *beSwift) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { + return []backend.Handle{}, nil +} +func (be *beSwift) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/feature/registry.go b/internal/feature/registry.go index 0773ea136..999797271 100644 --- a/internal/feature/registry.go +++ b/internal/feature/registry.go @@ -9,6 +9,7 @@ const ( DeviceIDForHardlinks FlagName = "device-id-for-hardlinks" ExplicitS3AnonymousAuth FlagName = "explicit-s3-anonymous-auth" SafeForgetKeepTags FlagName = "safe-forget-keep-tags" + S3Restore FlagName = "s3-restore" ) func init() { @@ -17,5 +18,6 @@ func init() { DeviceIDForHardlinks: {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"}, ExplicitS3AnonymousAuth: {Type: Stable, Description: "forbid anonymous S3 authentication unless `-o s3.unsafe-anonymous-auth=true` is set"}, SafeForgetKeepTags: {Type: Stable, Description: "prevent deleting all snapshots if the tag passed to `forget --keep-tags tagname` does not exist"}, + S3Restore: {Type: Alpha, Description: "restore S3 objects from cold storage classes when `-o s3.enable-restore=true` is set"}, }) } diff --git a/internal/repository/prune.go b/internal/repository/prune.go index 1f5832239..ba13ba1a3 100644 --- a/internal/repository/prune.go +++ b/internal/repository/prune.go @@ -557,7 +557,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er printer.P("repacking packs\n") bar := printer.NewCounter("packs repacked") bar.SetMax(uint64(len(plan.repackPacks))) - _, err := Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar) + _, err := Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar, printer.P) bar.Done() if err != nil { return errors.Fatal(err.Error()) diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 8c9ca28bb..929191478 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -6,6 +6,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" @@ -18,6 +19,8 @@ type repackBlobSet interface { Len() int } +type LogFunc func(msg string, args ...interface{}) + // Repack takes a list of packs together with a list of blobs contained in // these packs. Each pack is loaded and the blobs listed in keepBlobs is saved // into a new pack. Returned is the list of obsolete packs which can then @@ -25,9 +28,21 @@ type repackBlobSet interface { // // The map keepBlobs is modified by Repack, it is used to keep track of which // blobs have been processed. -func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Repository, packs restic.IDSet, keepBlobs repackBlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) { +func Repack( + ctx context.Context, + repo restic.Repository, + dstRepo restic.Repository, + packs restic.IDSet, + keepBlobs repackBlobSet, + p *progress.Counter, + logf LogFunc, +) (obsoletePacks restic.IDSet, err error) { debug.Log("repacking %d packs while keeping %d blobs", len(packs), keepBlobs.Len()) + if logf == nil { + logf = func(_ string, _ ...interface{}) {} + } + if repo == dstRepo && dstRepo.Connections() < 2 { return nil, errors.New("repack step requires a backend connection limit of at least two") } @@ -37,7 +52,7 @@ func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito dstRepo.StartPackUploader(wgCtx, wg) wg.Go(func() error { var err error - obsoletePacks, err = repack(wgCtx, repo, dstRepo, packs, keepBlobs, p) + obsoletePacks, err = repack(wgCtx, repo, dstRepo, packs, keepBlobs, p, logf) return err }) @@ -47,9 +62,30 @@ func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito return obsoletePacks, nil } -func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Repository, packs restic.IDSet, keepBlobs repackBlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) { +func repack( + ctx context.Context, + repo restic.Repository, + dstRepo restic.Repository, + packs restic.IDSet, + keepBlobs repackBlobSet, + p *progress.Counter, + logf LogFunc, +) (obsoletePacks restic.IDSet, err error) { wg, wgCtx := errgroup.WithContext(ctx) + if feature.Flag.Enabled(feature.S3Restore) { + job, err := repo.StartWarmup(ctx, packs) + if err != nil { + return nil, err + } + if job.HandleCount() != 0 { + logf("warming up %d packs from cold storage, this may take a while...", job.HandleCount()) + if err := job.Wait(ctx); err != nil { + return nil, err + } + } + } + var keepMutex sync.Mutex downloadQueue := make(chan restic.PackBlobs) wg.Go(func() error { diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 0691cdbbb..9248e42c2 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -160,7 +160,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe } func repack(t *testing.T, repo restic.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) { - repackedBlobs, err := repository.Repack(context.TODO(), repo, repo, packs, blobs, nil) + repackedBlobs, err := repository.Repack(context.TODO(), repo, repo, packs, blobs, nil, nil) if err != nil { t.Fatal(err) } @@ -279,7 +279,7 @@ func testRepackCopy(t *testing.T, version uint) { _, keepBlobs := selectBlobs(t, random, repo, 0.2) copyPacks := findPacksForBlobs(t, repo, keepBlobs) - _, err := repository.Repack(context.TODO(), repoWrapped, dstRepoWrapped, copyPacks, keepBlobs, nil) + _, err := repository.Repack(context.TODO(), repoWrapped, dstRepoWrapped, copyPacks, keepBlobs, nil, nil) if err != nil { t.Fatal(err) } @@ -318,7 +318,7 @@ func testRepackWrongBlob(t *testing.T, version uint) { _, keepBlobs := selectBlobs(t, random, repo, 0) rewritePacks := findPacksForBlobs(t, repo, keepBlobs) - _, err := repository.Repack(context.TODO(), repo, repo, rewritePacks, keepBlobs, nil) + _, err := repository.Repack(context.TODO(), repo, repo, rewritePacks, keepBlobs, nil, nil) if err == nil { t.Fatal("expected repack to fail but got no error") } @@ -366,7 +366,7 @@ func testRepackBlobFallback(t *testing.T, version uint) { rtest.OK(t, repo.Flush(context.Background())) // repack must fallback to valid copy - _, err = repository.Repack(context.TODO(), repo, repo, rewritePacks, keepBlobs, nil) + _, err = repository.Repack(context.TODO(), repo, repo, rewritePacks, keepBlobs, nil, nil) rtest.OK(t, err) keepBlobs = restic.NewBlobSet(restic.BlobHandle{Type: restic.DataBlob, ID: id}) diff --git a/internal/repository/warmup.go b/internal/repository/warmup.go new file mode 100644 index 000000000..7d96185a7 --- /dev/null +++ b/internal/repository/warmup.go @@ -0,0 +1,39 @@ +package repository + +import ( + "context" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/restic" +) + +type WarmupJob struct { + repo *Repository + handlesWarmingUp []backend.Handle +} + +// HandleCount returns the number of handles that are currently warming up. +func (job *WarmupJob) HandleCount() int { + return len(job.handlesWarmingUp) +} + +// Wait waits for all handles to be warm. +func (job *WarmupJob) Wait(ctx context.Context) error { + return job.repo.be.WarmupWait(ctx, job.handlesWarmingUp) +} + +// StartWarmup creates a new warmup job, requesting the backend to warmup the specified packs. +func (repo *Repository) StartWarmup(ctx context.Context, packs restic.IDSet) (restic.WarmupJob, error) { + handles := make([]backend.Handle, 0, len(packs)) + for pack := range packs { + handles = append( + handles, + backend.Handle{Type: restic.PackFile, Name: pack.String()}, + ) + } + handlesWarmingUp, err := repo.be.Warmup(ctx, handles) + return &WarmupJob{ + repo: repo, + handlesWarmingUp: handlesWarmingUp, + }, err +} diff --git a/internal/repository/warmup_test.go b/internal/repository/warmup_test.go new file mode 100644 index 000000000..a555a22ae --- /dev/null +++ b/internal/repository/warmup_test.go @@ -0,0 +1,73 @@ +package repository + +import ( + "context" + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/mock" + "github.com/restic/restic/internal/restic" +) + +func TestWarmupRepository(t *testing.T) { + warmupCalls := [][]backend.Handle{} + warmupWaitCalls := [][]backend.Handle{} + simulateWarmingUp := false + + be := mock.NewBackend() + be.WarmupFn = func(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) { + warmupCalls = append(warmupCalls, handles) + if simulateWarmingUp { + return handles, nil + } + return []backend.Handle{}, nil + } + be.WarmupWaitFn = func(ctx context.Context, handles []backend.Handle) error { + warmupWaitCalls = append(warmupWaitCalls, handles) + return nil + } + + repo, _ := New(be, Options{}) + + id1, _ := restic.ParseID("1111111111111111111111111111111111111111111111111111111111111111") + id2, _ := restic.ParseID("2222222222222222222222222222222222222222222222222222222222222222") + id3, _ := restic.ParseID("3333333333333333333333333333333333333333333333333333333333333333") + job, err := repo.StartWarmup(context.TODO(), restic.NewIDSet(id1, id2)) + if err != nil { + t.Fatalf("error when starting warmup: %v", err) + } + if len(warmupCalls) != 1 { + t.Fatalf("expected %d calls to warmup, got %d", 1, len(warmupCalls)) + } + if len(warmupCalls[0]) != 2 { + t.Fatalf("expected warmup on %d handles, got %d", 2, len(warmupCalls[0])) + } + if job.HandleCount() != 0 { + t.Fatalf("expected all files to be warm, got %d cold", job.HandleCount()) + } + + simulateWarmingUp = true + job, err = repo.StartWarmup(context.TODO(), restic.NewIDSet(id3)) + if err != nil { + t.Fatalf("error when starting warmup: %v", err) + } + if len(warmupCalls) != 2 { + t.Fatalf("expected %d calls to warmup, got %d", 2, len(warmupCalls)) + } + if len(warmupCalls[1]) != 1 { + t.Fatalf("expected warmup on %d handles, got %d", 1, len(warmupCalls[1])) + } + if job.HandleCount() != 1 { + t.Fatalf("expected %d file to be warming up, got %d", 1, job.HandleCount()) + } + + if err := job.Wait(context.TODO()); err != nil { + t.Fatalf("error when waiting warmup: %v", err) + } + if len(warmupWaitCalls) != 1 { + t.Fatalf("expected %d calls to warmupWait, got %d", 1, len(warmupCalls)) + } + if len(warmupWaitCalls[0]) != 1 { + t.Fatalf("expected warmupWait to be called with %d handles, got %d", 1, len(warmupWaitCalls[0])) + } +} diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 07ef9cbc0..977950f59 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -60,6 +60,9 @@ type Repository interface { SaveUnpacked(ctx context.Context, t WriteableFileType, buf []byte) (ID, error) // RemoveUnpacked removes a file from the repository. This will eventually be restricted to deleting only snapshots. RemoveUnpacked(ctx context.Context, t WriteableFileType, id ID) error + + // StartWarmup creates a new warmup job, requesting the backend to warmup the specified packs. + StartWarmup(ctx context.Context, packs IDSet) (WarmupJob, error) } type FileType = backend.FileType @@ -157,3 +160,10 @@ type Unpacked[FT FileTypes] interface { type ListBlobser interface { ListBlobs(ctx context.Context, fn func(PackedBlob)) error } + +type WarmupJob interface { + // HandleCount returns the number of handles that are currently warming up. + HandleCount() int + // Wait waits for all handles to be warm. + Wait(ctx context.Context) error +} diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 31234b960..e39115b70 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -2,6 +2,7 @@ package restorer import ( "context" + "fmt" "path/filepath" "sync" @@ -9,6 +10,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/restore" @@ -41,12 +43,15 @@ type packInfo struct { } type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error +type startWarmupFn func(context.Context, restic.IDSet) (restic.WarmupJob, error) // fileRestorer restores set of files type fileRestorer struct { idx func(restic.BlobType, restic.ID) []restic.PackedBlob blobsLoader blobsLoaderFn + startWarmup startWarmupFn + workerCount int filesWriter *filesWriter zeroChunk restic.ID @@ -58,6 +63,7 @@ type fileRestorer struct { dst string files []*fileInfo Error func(string, error) error + Info func(string) } func newFileRestorer(dst string, @@ -66,6 +72,7 @@ func newFileRestorer(dst string, connections uint, sparse bool, allowRecursiveDelete bool, + startWarmup startWarmupFn, progress *restore.Progress) *fileRestorer { // as packs are streamed the concurrency is limited by IO @@ -74,6 +81,7 @@ func newFileRestorer(dst string, return &fileRestorer{ idx: idx, blobsLoader: blobsLoader, + startWarmup: startWarmup, filesWriter: newFilesWriter(workerCount, allowRecursiveDelete), zeroChunk: repository.ZeroChunk(), sparse: sparse, @@ -82,6 +90,7 @@ func newFileRestorer(dst string, workerCount: workerCount, dst: dst, Error: restorerAbortOnAllErrors, + Info: func(_ string) {}, } } @@ -192,6 +201,19 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { // drop no longer necessary file list r.files = nil + if feature.Flag.Enabled(feature.S3Restore) { + warmupJob, err := r.startWarmup(ctx, restic.NewIDSet(packOrder...)) + if err != nil { + return err + } + if warmupJob.HandleCount() != 0 { + r.Info(fmt.Sprintf("warming up %d packs from cold storage, this may take a while...", warmupJob.HandleCount())) + if err := warmupJob.Wait(ctx); err != nil { + return err + } + } + } + wg, ctx := errgroup.WithContext(ctx) downloadCh := make(chan *packInfo) diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index f594760e4..62d93d64d 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -23,6 +24,11 @@ type TestFile struct { blobs []TestBlob } +type TestWarmupJob struct { + handlesCount int + waitCalled bool +} + type TestRepo struct { packsIDToData map[restic.ID][]byte @@ -31,6 +37,8 @@ type TestRepo struct { files []*fileInfo filesPathToContent map[string]string + warmupJobs []*TestWarmupJob + // loader blobsLoaderFn } @@ -44,6 +52,21 @@ func (i *TestRepo) fileContent(file *fileInfo) string { return i.filesPathToContent[file.location] } +func (i *TestRepo) StartWarmup(ctx context.Context, packs restic.IDSet) (restic.WarmupJob, error) { + job := TestWarmupJob{handlesCount: len(packs)} + i.warmupJobs = append(i.warmupJobs, &job) + return &job, nil +} + +func (job *TestWarmupJob) HandleCount() int { + return job.handlesCount +} + +func (job *TestWarmupJob) Wait(_ context.Context) error { + job.waitCalled = true + return nil +} + func newTestRepo(content []TestFile) *TestRepo { type Pack struct { name string @@ -111,6 +134,7 @@ func newTestRepo(content []TestFile) *TestRepo { blobs: blobs, files: files, filesPathToContent: filesPathToContent, + warmupJobs: []*TestWarmupJob{}, } repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { blobs = append([]restic.Blob{}, blobs...) @@ -141,10 +165,12 @@ func newTestRepo(content []TestFile) *TestRepo { } func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool, sparse bool) { + defer feature.TestSetFlag(t, feature.Flag, feature.S3Restore, true)() + t.Helper() repo := newTestRepo(content) - r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, false, repo.StartWarmup, nil) if files == nil { r.files = repo.files @@ -177,6 +203,15 @@ func verifyRestore(t *testing.T, r *fileRestorer, repo *TestRepo) { t.Errorf("file %v has wrong content: want %q, got %q", file.location, content, data) } } + + if len(repo.warmupJobs) == 0 { + t.Errorf("warmup did not occur") + } + for i, warmupJob := range repo.warmupJobs { + if !warmupJob.waitCalled { + t.Errorf("warmup job %d was not waited", i) + } + } } func TestFileRestorerBasic(t *testing.T) { @@ -285,7 +320,7 @@ func TestErrorRestoreFiles(t *testing.T) { return loadError } - r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, repo.StartWarmup, nil) r.files = repo.files err := r.restoreFiles(context.TODO()) @@ -326,7 +361,7 @@ func TestFatalDownloadError(t *testing.T) { }) } - r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, repo.StartWarmup, nil) r.files = repo.files var errors []string diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index cce175ebc..977ed42a6 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -28,6 +28,7 @@ type Restorer struct { Error func(location string, err error) error Warn func(message string) + Info func(message string) // SelectFilter determines whether the item is selectedForRestore or whether a childMayBeSelected. // selectedForRestore must not depend on isDir as `removeUnexpectedFiles` always passes false to isDir. SelectFilter func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) @@ -357,8 +358,9 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) idx := NewHardlinkIndex[string]() filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.LookupBlob, - res.repo.Connections(), res.opts.Sparse, res.opts.Delete, res.opts.Progress) + res.repo.Connections(), res.opts.Sparse, res.opts.Delete, res.repo.StartWarmup, res.opts.Progress) filerestorer.Error = res.Error + filerestorer.Info = res.Info debug.Log("first pass for %q", dst)