retry/rclone: retry errors up to 5 times

internal/backend/backend.go

@@ -95,6 +95,10 @@ type Properties struct {
 
 	// HasAtomicReplace states whether Save() can atomically replace files
 	HasAtomicReplace bool
+
+	// HasFlakyErrors states whether the backend may temporarily return errors
+	// that are considered as permanent for existing files.
+	HasFlakyErrors bool
 }
 
 type Unwrapper interface {

internal/backend/rclone/backend.go

@@ -341,6 +341,12 @@ func (be *Backend) Close() error {
 	return be.waitResult
 }
 
+func (be *Backend) Properties() backend.Properties {
+	properties := be.Backend.Properties()
+	properties.HasFlakyErrors = true
+	return properties
+}
+
 // Warmup not implemented
 func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
 	return []backend.Handle{}, nil

internal/backend/retry/backend_retry.go

@@ -127,12 +127,20 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error
 		b = backoff.WithMaxRetries(b, 10)
 	}
 
+	permanentErrorAttempts := 1
+	if be.Backend.Properties().HasFlakyErrors {
+		permanentErrorAttempts = 5
+	}
+
 	err := retryNotifyErrorWithSuccess(
 		func() error {
 			err := f()
 			// don't retry permanent errors as those very likely cannot be fixed by retrying
 			// TODO remove IsNotExist(err) special cases when removing the feature flag
 			if feature.Flag.Enabled(feature.BackendErrorRedesign) && !errors.Is(err, &backoff.PermanentError{}) && be.Backend.IsPermanentError(err) {
+				permanentErrorAttempts--
+			}
+			if permanentErrorAttempts <= 0 {
 				return backoff.Permanent(err)
 			}
 			return err