From b3217d2cace8b365b1f5f00a7fd93bd228243be1 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 10 Jun 2018 11:54:44 +0100 Subject: [PATCH] serve webdav: make Content-Type without reading the file and add --etag-hash Before this change x/net/webdav would open each file to find out its Content-Type. Now we override the FileInfo and provide that directly from rclone. An --etag-hash has also been implemented to override the ETag with the hash passed in. Fixes #2273 --- cmd/serve/webdav/webdav.go | 128 +++++++++++++++++++++++++++++--- cmd/serve/webdav/webdav_test.go | 8 ++ 2 files changed, 126 insertions(+), 10 deletions(-) diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go index a7d4b6bc4..8a42e1a69 100644 --- a/cmd/serve/webdav/webdav.go +++ b/cmd/serve/webdav/webdav.go @@ -1,8 +1,5 @@ package webdav -// FIXME need to fix directory listings reading each file - make an -// override for getcontenttype property? - import ( "net/http" "os" @@ -11,6 +8,7 @@ import ( "github.com/ncw/rclone/cmd/serve/httplib" "github.com/ncw/rclone/cmd/serve/httplib/httpflags" "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/hash" "github.com/ncw/rclone/fs/log" "github.com/ncw/rclone/vfs" "github.com/ncw/rclone/vfs/vfsflags" @@ -20,9 +18,15 @@ import ( "golang.org/x/net/webdav" ) +var ( + hashName string + hashType = hash.None +) + func init() { httpflags.AddFlags(Command.Flags()) vfsflags.AddFlags(Command.Flags()) + Command.Flags().StringVar(&hashName, "etag-hash", "", "Which hash to use for the ETag, or auto or blank for off") } // Command definition for cobra @@ -35,17 +39,41 @@ remote over HTTP via the webdav protocol. This can be viewed with a webdav client or you can make a remote of type webdav to read and write it. -NB at the moment each directory listing reads the start of each file -which is undesirable: see https://github.com/golang/go/issues/22577 +### Webdav options + +#### --etag-hash + +This controls the ETag header. Without this flag the ETag will be +based on the ModTime and Size of the object. + +If this flag is set to "auto" then rclone will choose the first +supported hash on the backend or you can use a named hash such as +"MD5" or "SHA-1". + +Use "rclone hashsum" to see the full list. + ` + httplib.Help + vfs.Help, - Run: func(command *cobra.Command, args []string) { + RunE: func(command *cobra.Command, args []string) error { cmd.CheckArgs(1, 1, command, args) f := cmd.NewFsSrc(args) + hashType = hash.None + if hashName == "auto" { + hashType = f.Hashes().GetOne() + } else if hashName != "" { + err := hashType.Set(hashName) + if err != nil { + return err + } + } + if hashType != hash.None { + fs.Debugf(f, "Using hash %v for ETag", hashType) + } cmd.Run(false, false, command, func() error { w := newWebDAV(f, &httpflags.Opt) w.serve() return nil }) + return nil }, } @@ -116,7 +144,11 @@ func (w *WebDAV) Mkdir(ctx context.Context, name string, perm os.FileMode) (err // OpenFile opens a file or a directory func (w *WebDAV) OpenFile(ctx context.Context, name string, flags int, perm os.FileMode) (file webdav.File, err error) { defer log.Trace(name, "flags=%v, perm=%v", flags, perm)("err = %v", &err) - return w.vfs.OpenFile(name, flags, perm) + f, err := w.vfs.OpenFile(name, flags, perm) + if err != nil { + return nil, err + } + return Handle{f}, nil } // RemoveAll removes a file or a directory and its contents @@ -142,8 +174,84 @@ func (w *WebDAV) Rename(ctx context.Context, oldName, newName string) (err error // Stat returns info about the file or directory func (w *WebDAV) Stat(ctx context.Context, name string) (fi os.FileInfo, err error) { defer log.Trace(name, "")("fi=%+v, err = %v", &fi, &err) - return w.vfs.Stat(name) + fi, err = w.vfs.Stat(name) + if err != nil { + return nil, err + } + return FileInfo{fi}, nil } -// check interface -var _ os.FileInfo = vfs.Node(nil) +// Handle represents an open file +type Handle struct { + vfs.Handle +} + +// Readdir reads directory entries from the handle +func (h Handle) Readdir(count int) (fis []os.FileInfo, err error) { + fis, err = h.Handle.Readdir(count) + if err != nil { + return nil, err + } + // Wrap each FileInfo + for i := range fis { + fis[i] = FileInfo{fis[i]} + } + return fis, nil +} + +// Stat the handle +func (h Handle) Stat() (fi os.FileInfo, err error) { + fi, err = h.Handle.Stat() + if err != nil { + return nil, err + } + return FileInfo{fi}, nil +} + +// FileInfo represents info about a file satisfying os.FileInfo and +// also some additional interfaces for webdav for ETag and ContentType +type FileInfo struct { + os.FileInfo +} + +// ETag returns an ETag for the FileInfo +func (fi FileInfo) ETag(ctx context.Context) (etag string, err error) { + defer log.Trace(fi, "")("etag=%q, err=%v", &etag, &err) + if hashType == hash.None { + return "", webdav.ErrNotImplemented + } + node, ok := (fi.FileInfo).(vfs.Node) + if !ok { + fs.Errorf(fi, "Expecting vfs.Node, got %T", fi.FileInfo) + return "", webdav.ErrNotImplemented + } + entry := node.DirEntry() + o, ok := entry.(fs.Object) + if !ok { + return "", webdav.ErrNotImplemented + } + hash, err := o.Hash(hashType) + if err != nil || hash == "" { + return "", webdav.ErrNotImplemented + } + return `"` + hash + `"`, nil +} + +// ContentType returns a content type for the FileInfo +func (fi FileInfo) ContentType(ctx context.Context) (contentType string, err error) { + defer log.Trace(fi, "")("etag=%q, err=%v", &contentType, &err) + node, ok := (fi.FileInfo).(vfs.Node) + if !ok { + fs.Errorf(fi, "Expecting vfs.Node, got %T", fi.FileInfo) + return "application/octet-stream", nil + } + entry := node.DirEntry() + switch x := entry.(type) { + case fs.Object: + return fs.MimeType(x), nil + case fs.Directory: + return "inode/directory", nil + } + fs.Errorf(fi, "Expecting fs.Object or fs.Directory, got %T", entry) + return "application/octet-stream", nil +} diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.go index 831310611..aecd2f15d 100644 --- a/cmd/serve/webdav/webdav_test.go +++ b/cmd/serve/webdav/webdav_test.go @@ -16,6 +16,7 @@ import ( "github.com/ncw/rclone/cmd/serve/httplib" "github.com/ncw/rclone/fstest" "github.com/stretchr/testify/assert" + "golang.org/x/net/webdav" ) const ( @@ -23,6 +24,13 @@ const ( testURL = "http://" + testBindAddress + "/" ) +// check interfaces +var ( + _ os.FileInfo = FileInfo{nil} + _ webdav.ETager = FileInfo{nil} + _ webdav.ContentTyper = FileInfo{nil} +) + // TestWebDav runs the webdav server then runs the unit tests for the // webdav remote against it. func TestWebDav(t *testing.T) {