diff --git a/fs/features.go b/fs/features.go index 828f3b94b..cb9b69649 100644 --- a/fs/features.go +++ b/fs/features.go @@ -39,6 +39,7 @@ type Features struct { NoMultiThreading bool // set if can't have multiplethreads on one download open Overlay bool // this wraps one or more backends to add functionality ChunkWriterDoesntSeek bool // set if the chunk writer doesn't need to read the data more than once + DoubleSlash bool // set if backend supports double slashes in paths // Purge all files in the directory specified // @@ -383,6 +384,8 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features { ft.PartialUploads = ft.PartialUploads && mask.PartialUploads ft.NoMultiThreading = ft.NoMultiThreading && mask.NoMultiThreading // ft.Overlay = ft.Overlay && mask.Overlay don't propagate Overlay + ft.ChunkWriterDoesntSeek = ft.ChunkWriterDoesntSeek && mask.ChunkWriterDoesntSeek + ft.DoubleSlash = ft.DoubleSlash && mask.DoubleSlash if mask.Purge == nil { ft.Purge = nil diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 2369e7c2d..c6903bc9a 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -2121,6 +2121,144 @@ func Run(t *testing.T, opt *Opt) { } }) + // Run tests for bucket based Fs + // TestIntegration/FsMkdir/FsPutFiles/Bucket + t.Run("Bucket", func(t *testing.T) { + // Test if this Fs is bucket based - this test won't work for wrapped bucket based backends. + if !f.Features().BucketBased { + t.Skip("Not a bucket based backend") + } + if f.Features().CanHaveEmptyDirectories { + t.Skip("Can have empty directories") + } + if !f.Features().DoubleSlash { + t.Skip("Can't have // in paths") + } + // Create some troublesome file names + fileNames := []string{ + file1.Path, + file2.Path, + ".leadingdot", + "/.leadingdot", + "///tripleslash", + "//doubleslash", + "dir/.leadingdot", + "dir///tripleslash", + "dir//doubleslash", + } + dirNames := []string{ + "hello? sausage", + "hello? sausage/êé", + "hello? sausage/êé/Hello, 世界", + "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", + "/", + "//", + "///", + "dir", + "dir/", + "dir//", + } + t1 := fstest.Time("2003-02-03T04:05:06.499999999Z") + var objs []fs.Object + for _, fileName := range fileNames[2:] { + contents := "bad file name: " + fileName + file := fstest.NewItem(fileName, contents, t1) + objs = append(objs, PutTestContents(ctx, t, f, &file, contents, true)) + } + + // Check they arrived + // This uses walk.Walk with a max size set to make sure we don't use ListR + check := func(f fs.Fs, dir string, wantFileNames, wantDirNames []string) { + t.Helper() + var gotFileNames, gotDirNames []string + require.NoError(t, walk.Walk(ctx, f, dir, true, 100, func(path string, entries fs.DirEntries, err error) error { + if err != nil { + return err + } + for _, entry := range entries { + if _, isObj := entry.(fs.Object); isObj { + gotFileNames = append(gotFileNames, entry.Remote()) + } else { + gotDirNames = append(gotDirNames, entry.Remote()) + } + } + return nil + })) + sort.Strings(wantDirNames) + sort.Strings(wantFileNames) + sort.Strings(gotDirNames) + sort.Strings(gotFileNames) + assert.Equal(t, wantFileNames, gotFileNames) + assert.Equal(t, wantDirNames, gotDirNames) + } + check(f, "", fileNames, dirNames) + check(f, "/", []string{ + "/.leadingdot", + "///tripleslash", + "//doubleslash", + }, []string{ + "//", + "///", + }) + check(f, "//", []string{ + "///tripleslash", + "//doubleslash", + }, []string{ + "///", + }) + check(f, "dir", []string{ + "dir/.leadingdot", + "dir///tripleslash", + "dir//doubleslash", + }, []string{ + "dir/", + "dir//", + }) + check(f, "dir/", []string{ + "dir///tripleslash", + "dir//doubleslash", + }, []string{ + "dir//", + }) + check(f, "dir//", []string{ + "dir///tripleslash", + }, nil) + + // Now create a backend not at the root of a bucket + f2, err := fs.NewFs(ctx, subRemoteName+"/dir") + require.NoError(t, err) + check(f2, "", []string{ + ".leadingdot", + "//tripleslash", + "/doubleslash", + }, []string{ + "/", + "//", + }) + check(f2, "/", []string{ + "//tripleslash", + "/doubleslash", + }, []string{ + "//", + }) + check(f2, "//", []string{ + "//tripleslash", + }, []string(nil)) + + // Remove the objects + for _, obj := range objs { + assert.NoError(t, obj.Remove(ctx)) + } + + // Check they are gone + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{ + "hello? sausage", + "hello? sausage/êé", + "hello? sausage/êé/Hello, 世界", + "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", + }, fs.GetModifyWindow(ctx, f)) + }) + // State of remote at the moment the internal tests are called InternalTestFiles = []fstest.Item{file1, file2}