b2: Add --b2-versions flag so old versions can be listed and retreived. #420

b2/api/types.go

@@ -2,7 +2,9 @@ package api
 
 import (
 	"fmt"
+	"path"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/ncw/rclone/fs"
@@ -62,10 +64,65 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
 	if err != nil {
 		return err
 	}
-	*t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6))
+	*t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6).UTC())
 	return nil
 }
 
+const versionFormat = "-v2006-01-02-150405.000"
+
+// AddVersion adds the timestamp as a version string into the filename passed in.
+func (t Timestamp) AddVersion(remote string) string {
+	ext := path.Ext(remote)
+	base := remote[:len(remote)-len(ext)]
+	s := (time.Time)(t).Format(versionFormat)
+	// Replace the '.' with a '-'
+	s = strings.Replace(s, ".", "-", -1)
+	return base + s + ext
+}
+
+// RemoveVersion removes the timestamp from a filename as a version string.
+//
+// It returns the new file name and a timestamp, or the old filename
+// and a zero timestamp.
+func RemoveVersion(remote string) (t Timestamp, newRemote string) {
+	newRemote = remote
+	ext := path.Ext(remote)
+	base := remote[:len(remote)-len(ext)]
+	if len(base) < len(versionFormat) {
+		return
+	}
+	versionStart := len(base) - len(versionFormat)
+	// Check it ends in -xxx
+	if base[len(base)-4] != '-' {
+		return
+	}
+	// Replace with .xxx for parsing
+	base = base[:len(base)-4] + "." + base[len(base)-3:]
+	newT, err := time.Parse(versionFormat, base[versionStart:])
+	if err != nil {
+		return
+	}
+	return Timestamp(newT), base[:versionStart] + ext
+}
+
+// IsZero returns true if the timestamp is unitialised
+func (t Timestamp) IsZero() bool {
+	return (time.Time)(t).IsZero()
+}
+
+// Equal compares two timestamps
+//
+// If either are !IsZero then it returns false
+func (t Timestamp) Equal(s Timestamp) bool {
+	if (time.Time)(t).IsZero() {
+		return false
+	}
+	if (time.Time)(s).IsZero() {
+		return false
+	}
+	return (time.Time)(t).Equal((time.Time)(s))
+}
+
 // File is info about a file
 type File struct {
 	ID              string            `json:"fileId"`          // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.

b2/api/types_test.go

@@ -0,0 +1,87 @@
+package api_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/b2/api"
+	"github.com/ncw/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	emptyT api.Timestamp
+	t0     = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123456789Z"))
+	t0r    = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123000000Z"))
+	t1     = api.Timestamp(fstest.Time("2001-02-03T04:05:06.123000000Z"))
+)
+
+func TestTimestampMarshalJSON(t *testing.T) {
+	resB, err := t0.MarshalJSON()
+	res := string(resB)
+	require.NoError(t, err)
+	assert.Equal(t, "3661123", res)
+
+	resB, err = t1.MarshalJSON()
+	res = string(resB)
+	require.NoError(t, err)
+	assert.Equal(t, "981173106123", res)
+}
+
+func TestTimestampUnmarshalJSON(t *testing.T) {
+	var tActual api.Timestamp
+	err := tActual.UnmarshalJSON([]byte("981173106123"))
+	require.NoError(t, err)
+	assert.Equal(t, (time.Time)(t1), (time.Time)(tActual))
+}
+
+func TestTimestampAddVersion(t *testing.T) {
+	for _, test := range []struct {
+		t        api.Timestamp
+		in       string
+		expected string
+	}{
+		{t0, "potato.txt", "potato-v1970-01-01-010101-123.txt"},
+		{t1, "potato", "potato-v2001-02-03-040506-123"},
+		{t1, "", "-v2001-02-03-040506-123"},
+	} {
+		actual := test.t.AddVersion(test.in)
+		assert.Equal(t, test.expected, actual, test.in)
+	}
+}
+
+func TestTimestampRemoveVersion(t *testing.T) {
+	for _, test := range []struct {
+		in             string
+		expectedT      api.Timestamp
+		expectedRemote string
+	}{
+		{"potato.txt", emptyT, "potato.txt"},
+		{"potato-v1970-01-01-010101-123.txt", t0r, "potato.txt"},
+		{"potato-v2001-02-03-040506-123", t1, "potato"},
+		{"-v2001-02-03-040506-123", t1, ""},
+		{"potato-v2A01-02-03-040506-123", emptyT, "potato-v2A01-02-03-040506-123"},
+		{"potato-v2001-02-03-040506=123", emptyT, "potato-v2001-02-03-040506=123"},
+	} {
+		actualT, actualRemote := api.RemoveVersion(test.in)
+		assert.Equal(t, test.expectedT, actualT, test.in)
+		assert.Equal(t, test.expectedRemote, actualRemote, test.in)
+	}
+}
+
+func TestTimestampIsZero(t *testing.T) {
+	assert.True(t, emptyT.IsZero())
+	assert.False(t, t0.IsZero())
+	assert.False(t, t1.IsZero())
+}
+
+func TestTimestampEqual(t *testing.T) {
+	assert.False(t, emptyT.Equal(emptyT))
+	assert.False(t, t0.Equal(emptyT))
+	assert.False(t, emptyT.Equal(t0))
+	assert.False(t, t0.Equal(t1))
+	assert.False(t, t1.Equal(t0))
+	assert.True(t, t0.Equal(t0))
+	assert.True(t, t1.Equal(t1))
+}

b2/b2.go

@@ -41,6 +41,7 @@ const (
 	maxSleep         = 2 * time.Second
 	decayConstant    = 1 // bigger for slower decay, exponential
 	maxParts         = 10000
+	maxVersions      = 100 // maximum number of versions we search in --b2-versions mode
 )
 
 // Globals
@@ -49,6 +50,8 @@ var (
 	chunkSize          = fs.SizeSuffix(96 * 1024 * 1024)
 	uploadCutoff       = fs.SizeSuffix(200E6)
 	b2TestMode         = pflag.StringP("b2-test-mode", "", "", "A flag string for X-Bz-Test-Mode header.")
+	b2Versions         = pflag.BoolP("b2-versions", "", false, "Include old versions in directory listings.")
+	errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
 )
 
 // Register with Fs
@@ -528,7 +531,8 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
 func (f *Fs) listFiles(out fs.ListOpts, dir string) {
 	defer out.Finished()
 	// List the objects
-	err := f.list(dir, out.Level(), "", 0, false, func(remote string, object *api.File, isDirectory bool) error {
+	last := ""
+	err := f.list(dir, out.Level(), "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
 		if isDirectory {
 			dir := &fs.Dir{
 				Name:  remote,
@@ -539,6 +543,15 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
 				return fs.ErrorListAborted
 			}
 		} else {
+			if remote == last {
+				remote = object.UploadTimestamp.AddVersion(remote)
+			} else {
+				last = remote
+			}
+			// hide objects represent deleted files which we don't list
+			if object.Action == "hide" {
+				return nil
+			}
 			o, err := f.newObjectWithInfo(remote, object)
 			if err != nil {
 				return err
@@ -914,12 +927,22 @@ func (o *Object) readMetaData() (err error) {
 	if o.id != "" {
 		return nil
 	}
+	maxSearched := 1
+	var timestamp api.Timestamp
+	baseRemote := o.remote
+	if *b2Versions {
+		timestamp, baseRemote = api.RemoveVersion(baseRemote)
+		maxSearched = maxVersions
+	}
 	var info *api.File
-	err = o.fs.list("", fs.MaxLevel, o.remote, 1, false, func(remote string, object *api.File, isDirectory bool) error {
+	err = o.fs.list("", fs.MaxLevel, baseRemote, maxSearched, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
 		if isDirectory {
 			return nil
 		}
-		if remote == o.remote {
+		if remote == baseRemote {
+			if !timestamp.IsZero() && !timestamp.Equal(object.UploadTimestamp) {
+				return nil
+			}
 			info = object
 		}
 		return errEndList // read only 1 item
@@ -1046,7 +1069,13 @@ func (o *Object) Open() (in io.ReadCloser, err error) {
 	opts := rest.Opts{
 		Method:   "GET",
 		Absolute: true,
-		Path:     o.fs.info.DownloadURL + "/file/" + urlEncode(o.fs.bucket) + "/" + urlEncode(o.fs.root+o.remote),
+		Path:     o.fs.info.DownloadURL,
+	}
+	// Download by id if set otherwise by name
+	if o.id != "" {
+		opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
+	} else {
+		opts.Path += "/file/" + urlEncode(o.fs.bucket) + "/" + urlEncode(o.fs.root+o.remote)
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
@@ -1108,6 +1137,9 @@ func urlEncode(in string) string {
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) {
+	if *b2Versions {
+		return errNotWithVersions
+	}
 	size := src.Size()
 
 	// If a large file upload in chunks - see upload.go
@@ -1256,6 +1288,9 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) {
 
 // Remove an object
 func (o *Object) Remove() error {
+	if *b2Versions {
+		return errNotWithVersions
+	}
 	bucketID, err := o.fs.getBucketID()
 	if err != nil {
 		return err

docs/content/b2.md

@@ -109,22 +109,6 @@ will be used in the syncing process. You can use the `--checksum` flag.
 Large files which are uploaded in chunks will store their SHA1 on the
 object as `X-Bz-Info-large_file_sha1` as recommended by Backblaze.
 
-### Versions ###
-
-When rclone uploads a new version of a file it creates a [new version
-of it](https://www.backblaze.com/b2/docs/file_versions.html).
-Likewise when you delete a file, the old version will still be
-available.
-
-The old versions of files are visible in the B2 web interface, but not
-via rclone yet.
-
-If you wish to remove all the old versions then you can use the
-`rclone cleanup remote:bucket` command which will delete all the old
-versions of files, leaving the current ones intact.
-
-When you `purge` a bucket, all the old versions will be deleted also.
-
 ### Transfers ###
 
 Backblaze recommends that you do lots of transfers simultaneously for
@@ -135,6 +119,64 @@ depending on your hardware, how big the files are, how much you want
 to load your computer, etc.  The default of `--transfers 4` is
 definitely too low for Backblaze B2 though.
 
+### Versions ###
+
+When rclone uploads a new version of a file it creates a [new version
+of it](https://www.backblaze.com/b2/docs/file_versions.html).
+Likewise when you delete a file, the old version will still be
+available.
+
+Old versions of files are visible using the `--b2-versions` flag.
+
+If you wish to remove all the old versions then you can use the
+`rclone cleanup remote:bucket` command which will delete all the old
+versions of files, leaving the current ones intact.  You can also
+supply a path and only old versions under that path will be deleted,
+eg `rclone cleanup remote:bucket/path/to/stuff`.
+
+When you `purge` a bucket, the current and the old versions will be
+deleted then the bucket will be deleted.
+
+However `delete` will cause the current versions of the files to
+become hidden old versions.
+
+Here is a session showing the listing and and retreival of an old
+version followed by a `cleanup` of the old versions.
+
+Show current version and all the versions with `--b2-versions` flag.
+
+```
+$ rclone -q ls b2:cleanup-test
+        9 one.txt
+
+$ rclone -q --b2-versions ls b2:cleanup-test
+        9 one.txt
+        8 one-v2016-07-04-141032-000.txt
+       16 one-v2016-07-04-141003-000.txt
+       15 one-v2016-07-02-155621-000.txt
+```
+
+Retreive an old verson
+
+```
+$ rclone -q --b2-versions copy b2:cleanup-test/one-v2016-07-04-141003-000.txt /tmp
+
+$ ls -l /tmp/one-v2016-07-04-141003-000.txt
+-rw-rw-r-- 1 ncw ncw 16 Jul  2 17:46 /tmp/one-v2016-07-04-141003-000.txt
+```
+
+Clean up all the old versions and show that they've gone.
+
+```
+$ rclone -q cleanup b2:cleanup-test
+
+$ rclone -q ls b2:cleanup-test
+        9 one.txt
+
+$ rclone -q --b2-versions ls b2:cleanup-test
+        9 one.txt
+```
+
 ### Specific options ###
 
 Here are the command line options specific to this cloud storage
@@ -167,3 +209,31 @@ specific errors for debugging purposes.
 These will be set in the `X-Bz-Test-Mode` header which is documented
 in the [b2 integrations
 checklist](https://www.backblaze.com/b2/docs/integration_checklist.html).
+
+#### --b2-versions ####
+
+When set rclone will show and act on older versions of files.  For example
+
+Listing without `--b2-versions`
+
+```
+$ rclone -q ls b2:cleanup-test
+        9 one.txt
+```
+
+And with
+
+```
+$ rclone -q --b2-versions ls b2:cleanup-test
+        9 one.txt
+        8 one-v2016-07-04-141032-000.txt
+       16 one-v2016-07-04-141003-000.txt
+       15 one-v2016-07-02-155621-000.txt
+```
+
+Showing that the current version is unchanged but older versions can
+be seen.  These have the UTC date that they were uploaded to the
+server to the nearest millisecond appended to them.
+
+Note that when using `--b2-versions` no file write operations are
+permitted, so you can't upload files or delete them.