From 46135d830e67fb71508cc0a75af97b0efe2eb9c6 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 17 Jun 2016 17:20:08 +0100 Subject: [PATCH] Add --ignore-size flag - fixes #399 --- docs/content/docs.md | 17 ++++++++++++++--- fs/config.go | 7 +++++++ fs/operations.go | 13 ++++++++----- fs/operations_test.go | 44 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 8 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index d9641d7fa..4d21cee50 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -437,6 +437,20 @@ While this isn't a generally recommended option, it can be useful in cases where your files change due to encryption. However, it cannot correct partial transfers in case a transfer was interrupted. +### --ignore-size ### + +Normally rclone will look at modification time and size of files to +see if they are equal. If you set this flag then rclone will check +only the modification time. If `--checksum` is set then it only +checks the checksum. + +It will also cause rclone to skip verifying the sizes are the same +after transfer. + +This can be useful for transferring files to and from onedrive which +occasionally misreports the size of image files (see +[#399](https://github.com/ncw/rclone/issues/399) for more info). + ### -I, --ignore-times ### Using this option will cause rclone to unconditionally upload all @@ -534,9 +548,6 @@ This can be useful transferring files from dropbox which have been modified by the desktop sync client which doesn't set checksums of modification times in the same way as rclone. -When using this flag, rclone won't update mtimes of remote files if -they are incorrect as it would normally. - ### --stats=TIME ### Rclone will print stats at regular intervals to show its progress. diff --git a/fs/config.go b/fs/config.go index 2369be156..ce3324554 100644 --- a/fs/config.go +++ b/fs/config.go @@ -87,6 +87,7 @@ var ( noGzip = pflag.BoolP("no-gzip-encoding", "", false, "Don't set Accept-Encoding: gzip.") dedupeMode = pflag.StringP("dedupe-mode", "", "interactive", "Dedupe mode interactive|skip|first|newest|oldest|rename.") maxDepth = pflag.IntP("max-depth", "", -1, "If set limits the recursion depth to this.") + ignoreSize = pflag.BoolP("ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.") bwLimit SizeSuffix // Key to use for password en/decryption. @@ -221,6 +222,7 @@ type ConfigInfo struct { NoGzip bool // Disable compression DedupeMode DeduplicateMode MaxDepth int + IgnoreSize bool } // Transport returns an http.RoundTripper with the correct timeouts @@ -324,6 +326,7 @@ func LoadConfig() { Config.UpdateOlder = *updateOlder Config.NoGzip = *noGzip Config.MaxDepth = *maxDepth + Config.IgnoreSize = *ignoreSize ConfigPath = *configFile @@ -358,6 +361,10 @@ func LoadConfig() { Config.DeleteDuring = true } + if Config.IgnoreSize && Config.SizeOnly { + log.Fatalf(`Can't use --size-only and --ignore-size together.`) + } + // Load configuration file. var err error ConfigFile, err = loadConfigFile() diff --git a/fs/operations.go b/fs/operations.go index 3d93c37e2..bd6b04c84 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -95,7 +95,8 @@ func CheckHashes(src, dst Object) (equal bool, hash HashType, err error) { // // If the src and dst size are different then it is considered to be // not equal. If --size-only is in effect then this is the only check -// that is done. +// that is done. If --ignore-size is in effect then this check is +// skipped and the files are considered the same size. // // If the size is the same and the mtime is the same then it is // considered to be equal. This check is skipped if using --checksum. @@ -108,9 +109,11 @@ func CheckHashes(src, dst Object) (equal bool, hash HashType, err error) { // Otherwise the file is considered to be not equal including if there // were errors reading info. func Equal(src, dst Object) bool { - if src.Size() != dst.Size() { - Debug(src, "Sizes differ") - return false + if !Config.IgnoreSize { + if src.Size() != dst.Size() { + Debug(src, "Sizes differ") + return false + } } if Config.SizeOnly { Debug(src, "Sizes identical") @@ -261,7 +264,7 @@ tryAgain: } // Verify sizes are the same after transfer - if src.Size() != dst.Size() { + if !Config.IgnoreSize && src.Size() != dst.Size() { Stats.Error() err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size()) ErrorLog(dst, "%s", err) diff --git a/fs/operations_test.go b/fs/operations_test.go index 4cf521105..c90f0ed1b 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -479,6 +479,50 @@ func TestSyncSizeOnly(t *testing.T) { fstest.CheckItems(t, r.fremote, file1) } +// Create a file and sync it. Keep the last modified date but change +// the size. With --ignore-size we expect nothing to to be +// transferred on the second sync. +func TestSyncIgnoreSize(t *testing.T) { + r := NewRun(t) + defer r.Finalise() + fs.Config.IgnoreSize = true + defer func() { fs.Config.IgnoreSize = false }() + + file1 := r.WriteFile("ignore-size", "contents", t1) + fstest.CheckItems(t, r.flocal, file1) + + fs.Stats.ResetCounters() + err := fs.Sync(r.fremote, r.flocal) + if err != nil { + t.Fatalf("Initial sync failed: %v", err) + } + + // We should have transferred exactly one file. + if fs.Stats.GetTransfers() != 1 { + t.Fatalf("Sync 1: want 1 transfer, got %d", fs.Stats.GetTransfers()) + } + + fstest.CheckItems(t, r.fremote, file1) + + // Update size but not date of file + file2 := r.WriteFile("ignore-size", "longer contents but same date", t1) + fstest.CheckItems(t, r.flocal, file2) + + fs.Stats.ResetCounters() + err = fs.Sync(r.fremote, r.flocal) + if err != nil { + t.Fatalf("Sync failed: %v", err) + } + + // We should have transferred no files + if fs.Stats.GetTransfers() != 0 { + t.Fatalf("Sync 2: want 0 transfers, got %d", fs.Stats.GetTransfers()) + } + + fstest.CheckItems(t, r.flocal, file2) + fstest.CheckItems(t, r.fremote, file1) +} + func TestSyncIgnoreTimes(t *testing.T) { r := NewRun(t) defer r.Finalise()