MirrorSave
/
rclone
mirror of https://github.com/rclone/rclone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

s3: fix hang on aborting multpart upload with iDrive e2 

Apparently the abort multipart upload call doesn't return while
multipart uploads are in progress on iDrive e2.

This means that if we CTRL-C a multpart upload rclone hangs until the
all parts uploading have completed. However since rclone is uploading
multiple parts at once this doesn't happen until after the entire file
is uploaded.

This was fixed by cancelling the upload context which causes all the
uploads to stop instantly.
This commit is contained in:
Nick Craig-Wood 2023-03-22 10:23:17 +00:00
parent 32f71c97ea
commit ddb3b17e96
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/s3/s3.go
View File

@ -5122,7 +5122,9 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
} }
uid := cout.UploadId uid := cout.UploadId
uploadCtx, cancel := context.WithCancel(ctx)
defer atexit.OnError(&err, func() { defer atexit.OnError(&err, func() {
cancel()
if o.fs.opt.LeavePartsOnError { if o.fs.opt.LeavePartsOnError {
return return
} }
@ -5142,7 +5144,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
})() })()
var ( var (
g, gCtx = errgroup.WithContext(ctx) g, gCtx = errgroup.WithContext(uploadCtx)
finished = false finished = false
partsMu sync.Mutex // to protect parts partsMu sync.Mutex // to protect parts
parts []*s3.CompletedPart parts []*s3.CompletedPart
@ -5224,7 +5226,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq) uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq)
if err != nil { if err != nil {
if partNum <= int64(concurrency) { if partNum <= int64(concurrency) {
return f.shouldRetry(ctx, err) return f.shouldRetry(gCtx, err)
} }
// retry all chunks once have done the first batch // retry all chunks once have done the first batch
return true, err return true, err
@ -5256,7 +5258,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
var resp *s3.CompleteMultipartUploadOutput var resp *s3.CompleteMultipartUploadOutput
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ resp, err = f.c.CompleteMultipartUploadWithContext(uploadCtx, &s3.CompleteMultipartUploadInput{
Bucket: req.Bucket, Bucket: req.Bucket,
Key: req.Key, Key: req.Key,
MultipartUpload: &s3.CompletedMultipartUpload{ MultipartUpload: &s3.CompletedMultipartUpload{
@ -5265,7 +5267,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
RequestPayer: req.RequestPayer, RequestPayer: req.RequestPayer,
UploadId: uid, UploadId: uid,
}) })
return f.shouldRetry(ctx, err) return f.shouldRetry(uploadCtx, err)
}) })
if err != nil { if err != nil {
return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err) return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err)