2023-10-01 11:40:12 +02:00
|
|
|
package backend
|
2016-08-31 19:10:10 +02:00
|
|
|
|
2017-06-03 17:39:57 +02:00
|
|
|
import (
|
|
|
|
"context"
|
2024-07-10 21:46:26 +02:00
|
|
|
"fmt"
|
2020-12-19 12:39:48 +01:00
|
|
|
"hash"
|
2017-06-03 17:39:57 +02:00
|
|
|
"io"
|
|
|
|
)
|
2017-01-22 12:32:20 +01:00
|
|
|
|
2024-07-10 21:46:26 +02:00
|
|
|
var ErrNoRepository = fmt.Errorf("repository does not exist")
|
|
|
|
|
2016-08-31 19:10:10 +02:00
|
|
|
// Backend is used to store and access data.
|
2020-12-16 13:58:02 +01:00
|
|
|
//
|
|
|
|
// Backend operations that return an error will be retried when a Backend is
|
|
|
|
// wrapped in a RetryBackend. To prevent that from happening, the operations
|
|
|
|
// should return a github.com/cenkalti/backoff/v4.PermanentError. Errors from
|
|
|
|
// the context package need not be wrapped, as context cancellation is checked
|
|
|
|
// separately by the retrying logic.
|
2016-08-31 19:10:10 +02:00
|
|
|
type Backend interface {
|
2023-12-06 13:11:55 +01:00
|
|
|
// Connections returns the maximum number of concurrent backend operations.
|
2021-08-07 22:20:49 +02:00
|
|
|
Connections() uint
|
|
|
|
|
2020-12-19 12:39:48 +01:00
|
|
|
// Hasher may return a hash function for calculating a content hash for the backend
|
|
|
|
Hasher() hash.Hash
|
|
|
|
|
2022-05-01 20:07:29 +02:00
|
|
|
// HasAtomicReplace returns whether Save() can atomically replace files
|
|
|
|
HasAtomicReplace() bool
|
|
|
|
|
2023-10-01 13:05:56 +02:00
|
|
|
// Remove removes a File described by h.
|
2017-06-03 17:39:57 +02:00
|
|
|
Remove(ctx context.Context, h Handle) error
|
2016-08-31 19:10:10 +02:00
|
|
|
|
|
|
|
// Close the backend
|
|
|
|
Close() error
|
|
|
|
|
2018-03-03 14:20:54 +01:00
|
|
|
// Save stores the data from rd under the given handle.
|
|
|
|
Save(ctx context.Context, h Handle, rd RewindReader) error
|
2016-08-31 19:10:10 +02:00
|
|
|
|
2018-01-17 05:59:16 +01:00
|
|
|
// Load runs fn with a reader that yields the contents of the file at h at the
|
2017-01-23 17:20:08 +01:00
|
|
|
// given offset. If length is larger than zero, only a portion of the file
|
2024-05-11 00:12:56 +02:00
|
|
|
// is read. If the length is larger than zero and the file is too short to return
|
|
|
|
// the requested length bytes, then an error MUST be returned that is recognized
|
|
|
|
// by IsPermanentError().
|
2018-01-17 05:59:16 +01:00
|
|
|
//
|
|
|
|
// The function fn may be called multiple times during the same Load invocation
|
|
|
|
// and therefore must be idempotent.
|
|
|
|
//
|
2023-10-01 10:24:33 +02:00
|
|
|
// Implementations are encouraged to use util.DefaultLoad
|
2018-01-17 05:59:16 +01:00
|
|
|
Load(ctx context.Context, h Handle, length int, offset int64, fn func(rd io.Reader) error) error
|
2017-01-22 22:01:12 +01:00
|
|
|
|
2016-08-31 20:29:54 +02:00
|
|
|
// Stat returns information about the File identified by h.
|
2017-06-03 17:39:57 +02:00
|
|
|
Stat(ctx context.Context, h Handle) (FileInfo, error)
|
2016-08-31 19:10:10 +02:00
|
|
|
|
2018-01-20 13:43:07 +01:00
|
|
|
// List runs fn for each file in the backend which has the type t. When an
|
|
|
|
// error occurs (or fn returns an error), List stops and returns it.
|
2018-01-23 21:21:54 +01:00
|
|
|
//
|
2018-01-24 16:25:40 +01:00
|
|
|
// The function fn is called exactly once for each file during successful
|
|
|
|
// execution and at most once in case of an error.
|
|
|
|
//
|
2018-01-23 21:21:54 +01:00
|
|
|
// The function fn is called in the same Goroutine that List() is called
|
|
|
|
// from.
|
2018-01-20 13:43:07 +01:00
|
|
|
List(ctx context.Context, t FileType, fn func(FileInfo) error) error
|
2017-06-15 13:40:27 +02:00
|
|
|
|
|
|
|
// IsNotExist returns true if the error was caused by a non-existing file
|
|
|
|
// in the backend.
|
2022-10-08 12:37:18 +02:00
|
|
|
//
|
|
|
|
// The argument may be a wrapped error. The implementation is responsible
|
|
|
|
// for unwrapping it.
|
2017-06-15 13:40:27 +02:00
|
|
|
IsNotExist(err error) bool
|
2017-10-14 13:38:17 +02:00
|
|
|
|
2024-05-11 00:12:56 +02:00
|
|
|
// IsPermanentError returns true if the error can very likely not be resolved
|
|
|
|
// by retrying the operation. Backends should return true if the file is missing,
|
|
|
|
// the requested range does not (completely) exist in the file or the user is
|
|
|
|
// not authorized to perform the requested operation.
|
|
|
|
IsPermanentError(err error) bool
|
|
|
|
|
2017-10-14 13:38:17 +02:00
|
|
|
// Delete removes all data in the backend.
|
|
|
|
Delete(ctx context.Context) error
|
feat(backends/s3): add warmup support before repacks and restores (#5173)
* feat(backends/s3): add warmup support before repacks and restores
This commit introduces basic support for transitioning pack files stored
in cold storage to hot storage on S3 and S3-compatible providers.
To prevent unexpected behavior for existing users, the feature is gated
behind new flags:
- `s3.enable-restore`: opt-in flag (defaults to false)
- `s3.restore-days`: number of days for the restored objects to remain
in hot storage (defaults to `7`)
- `s3.restore-timeout`: maximum time to wait for a single restoration
(default to `1 day`)
- `s3.restore-tier`: retrieval tier at which the restore will be
processed. (default to `Standard`)
As restoration times can be lengthy, this implementation preemptively
restores selected packs to prevent incessant restore-delays during
downloads. This is slightly sub-optimal as we could process packs
out-of-order (as soon as they're transitioned), but this would really
add too much complexity for a marginal gain in speed.
To maintain simplicity and prevent resources exhautions with lots of
packs, no new concurrency mechanisms or goroutines were added. This just
hooks gracefully into the existing routines.
**Limitations:**
- Tests against the backend were not written due to the lack of cold
storage class support in MinIO. Testing was done manually on
Scaleway's S3-compatible object storage. If necessary, we could
explore testing with LocalStack or mocks, though this requires further
discussion.
- Currently, this feature only warms up before restores and repacks
(prune/copy), as those are the two main use-cases I came across.
Support for other commands may be added in future iterations, as long
as affected packs can be calculated in advance.
- The feature is gated behind a new alpha `s3-restore` feature flag to
make it explicit that the feature is still wet behind the ears.
- There is no explicit user notification for ongoing pack restorations.
While I think it is not necessary because of the opt-in flag, showing
some notice may improve usability (but would probably require major
refactoring in the progress bar which I didn't want to start). Another
possibility would be to add a flag to send restores requests and fail
early.
See https://github.com/restic/restic/issues/3202
* ui: warn user when files are warming up from cold storage
* refactor: remove the PacksWarmer struct
It's easier to handle multiple handles in the backend directly, and it
may open the door to reducing the number of requests made to the backend
in the future.
2025-02-01 19:26:27 +01:00
|
|
|
|
|
|
|
// Warmup ensures that the specified handles are ready for upcoming reads.
|
|
|
|
// This is particularly useful for transitioning files from cold to hot
|
|
|
|
// storage.
|
|
|
|
//
|
|
|
|
// The method is non-blocking. WarmupWait can be used to wait for
|
|
|
|
// completion.
|
|
|
|
//
|
|
|
|
// Returns:
|
|
|
|
// - Handles currently warming up.
|
|
|
|
// - An error if warmup fails.
|
|
|
|
Warmup(ctx context.Context, h []Handle) ([]Handle, error)
|
|
|
|
|
|
|
|
// WarmupWait waits until all given handles are warm.
|
|
|
|
WarmupWait(ctx context.Context, h []Handle) error
|
2016-08-31 19:10:10 +02:00
|
|
|
}
|
|
|
|
|
2023-10-01 11:40:12 +02:00
|
|
|
type Unwrapper interface {
|
2023-04-08 12:53:43 +02:00
|
|
|
// Unwrap returns the underlying backend or nil if there is none.
|
|
|
|
Unwrap() Backend
|
|
|
|
}
|
|
|
|
|
2023-06-17 19:23:55 +02:00
|
|
|
func AsBackend[B Backend](b Backend) B {
|
|
|
|
for b != nil {
|
|
|
|
if be, ok := b.(B); ok {
|
|
|
|
return be
|
|
|
|
}
|
|
|
|
|
2023-10-01 11:40:12 +02:00
|
|
|
if be, ok := b.(Unwrapper); ok {
|
2023-06-17 19:23:55 +02:00
|
|
|
b = be.Unwrap()
|
|
|
|
} else {
|
|
|
|
// not the backend we're looking for
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
var be B
|
|
|
|
return be
|
|
|
|
}
|
|
|
|
|
2023-06-17 19:36:56 +02:00
|
|
|
type FreezeBackend interface {
|
|
|
|
Backend
|
|
|
|
// Freeze blocks all backend operations except those on lock files
|
|
|
|
Freeze()
|
|
|
|
// Unfreeze allows all backend operations to continue
|
|
|
|
Unfreeze()
|
|
|
|
}
|
|
|
|
|
2018-01-20 13:43:07 +01:00
|
|
|
// FileInfo is contains information about a file in the backend.
|
|
|
|
type FileInfo struct {
|
|
|
|
Size int64
|
|
|
|
Name string
|
|
|
|
}
|
2023-04-21 21:51:58 +02:00
|
|
|
|
|
|
|
// ApplyEnvironmenter fills in a backend configuration from the environment
|
|
|
|
type ApplyEnvironmenter interface {
|
2023-06-08 15:28:07 +02:00
|
|
|
ApplyEnvironment(prefix string)
|
2023-04-21 21:51:58 +02:00
|
|
|
}
|