From 8d1185b3b80df6da49685cdd2369ed9c8922f1cb Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sun, 16 Feb 2025 22:40:55 +0100
Subject: [PATCH] retry/rclone: retry errors up to 5 times

---
 internal/backend/backend.go             | 4 ++++
 internal/backend/rclone/backend.go      | 6 ++++++
 internal/backend/retry/backend_retry.go | 8 ++++++++
 3 files changed, 18 insertions(+)

diff --git a/internal/backend/backend.go b/internal/backend/backend.go
index d3e47d0b2..413d8c96a 100644
--- a/internal/backend/backend.go
+++ b/internal/backend/backend.go
@@ -95,6 +95,10 @@ type Properties struct {
 
 	// HasAtomicReplace states whether Save() can atomically replace files
 	HasAtomicReplace bool
+
+	// HasFlakyErrors states whether the backend may temporarily return errors
+	// that are considered as permanent for existing files.
+	HasFlakyErrors bool
 }
 
 type Unwrapper interface {
diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index fb5ed34eb..c9e395b8b 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -341,6 +341,12 @@ func (be *Backend) Close() error {
 	return be.waitResult
 }
 
+func (be *Backend) Properties() backend.Properties {
+	properties := be.Backend.Properties()
+	properties.HasFlakyErrors = true
+	return properties
+}
+
 // Warmup not implemented
 func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
 	return []backend.Handle{}, nil
diff --git a/internal/backend/retry/backend_retry.go b/internal/backend/retry/backend_retry.go
index 869380d8b..14db608a3 100644
--- a/internal/backend/retry/backend_retry.go
+++ b/internal/backend/retry/backend_retry.go
@@ -127,12 +127,20 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error
 		b = backoff.WithMaxRetries(b, 10)
 	}
 
+	permanentErrorAttempts := 1
+	if be.Backend.Properties().HasFlakyErrors {
+		permanentErrorAttempts = 5
+	}
+
 	err := retryNotifyErrorWithSuccess(
 		func() error {
 			err := f()
 			// don't retry permanent errors as those very likely cannot be fixed by retrying
 			// TODO remove IsNotExist(err) special cases when removing the feature flag
 			if feature.Flag.Enabled(feature.BackendErrorRedesign) && !errors.Is(err, &backoff.PermanentError{}) && be.Backend.IsPermanentError(err) {
+				permanentErrorAttempts--
+			}
+			if permanentErrorAttempts <= 0 {
 				return backoff.Permanent(err)
 			}
 			return err