MirrorSave
/
rclone
mirror of https://github.com/rclone/rclone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

oracleobjectstorage: speed up operations by using S3 pacer and setting minsleep to 10ms 

Uploading 100 files of each 1 MB took 20 seconds before. With above fix it takes around 2 seconds now.

10x time improvement in line with pacer's sleep reduction from 100ms to 10ms
This commit is contained in:
Manoj Ghosh 2023-01-25 02:48:16 -08:00 committed by GitHub
parent e2afd00118
commit 8c8ee9905c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/oracleobjectstorage/options.go
View File

@ -17,9 +17,7 @@ const (
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024) defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
defaultUploadConcurrency = 10 defaultUploadConcurrency = 10
maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024) maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
minSleep = 100 * time.Millisecond minSleep = 10 * time.Millisecond
maxSleep = 5 * time.Minute
decayConstant = 1 // bigger for slower decay, exponential
defaultCopyTimeoutDuration = fs.Duration(time.Minute) defaultCopyTimeoutDuration = fs.Duration(time.Minute)
) )

8
backend/oracleobjectstorage/oracleobjectstorage.go
View File

@ -64,14 +64,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil { if err != nil {
return nil, err return nil, err
} }
p := pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)) pc := fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep)))
// Set pacer retries to 2 (1 try and 1 retry) because we are
// relying on SDK retry mechanism, but we allow 2 attempts to
// retry directory listings after XMLSyntaxError
pc.SetRetries(2)
f := &Fs{ f := &Fs{
name: name, name: name,
opt: *opt, opt: *opt,
ci: ci, ci: ci,
srv: objectStorageClient, srv: objectStorageClient,
cache: bucket.NewCache(), cache: bucket.NewCache(),
pacer: fs.NewPacer(ctx, p), pacer: pc,
} }
f.setRoot(root) f.setRoot(root)
f.features = (&fs.Features{ f.features = (&fs.Features{