diff --git a/vfs/rc.go b/vfs/rc.go index f34074e56..2fde29d1c 100644 --- a/vfs/rc.go +++ b/vfs/rc.go @@ -2,155 +2,57 @@ package vfs import ( "context" + "fmt" "strconv" "strings" "time" "github.com/pkg/errors" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/rc" ) -// Add remote control for the VFS -func (vfs *VFS) addRC() { +const getVFSHelp = ` +This command takes an "fs" parameter. If this parameter is not +supplied and if there is only one VFS in use then that VFS will be +used. If there is more than one VFS in use then the "fs" parameter +must be supplied.` + +// GetVFS gets a VFS with config name "fs" from the cache or returns an error. +// +// If "fs" is not set and there is one and only one VFS in the active +// cache then it returns it. This is for backwards compatibility. +func getVFS(in rc.Params) (vfs *VFS, err error) { + fsString, err := in.GetString("fs") + if rc.IsErrParamNotFound(err) { + var count int + vfs, count = activeCacheEntries() + if count == 1 { + return vfs, nil + } else if count == 0 { + return nil, errors.New(`no VFS active and "fs" parameter not supplied`) + } + return nil, errors.New(`more than one VFS active - need "fs" parameter`) + } else if err != nil { + return nil, err + } + activeMu.Lock() + defer activeMu.Unlock() + fsString = cache.Canonicalize(fsString) + activeVFS := active[fsString] + if len(activeVFS) == 0 { + return nil, errors.Errorf("no VFS found with name %q", fsString) + } else if len(activeVFS) > 1 { + return nil, errors.Errorf("more than one VFS active with name %q", fsString) + } + return activeVFS[0], nil +} + +func init() { rc.Add(rc.Call{ - Path: "vfs/forget", - Fn: func(ctx context.Context, in rc.Params) (out rc.Params, err error) { - root, err := vfs.Root() - if err != nil { - return nil, err - } - - forgotten := []string{} - if len(in) == 0 { - root.ForgetAll() - } else { - for k, v := range in { - path, ok := v.(string) - if !ok { - return out, errors.Errorf("value must be string %q=%v", k, v) - } - path = strings.Trim(path, "/") - if strings.HasPrefix(k, "file") { - root.ForgetPath(path, fs.EntryObject) - } else if strings.HasPrefix(k, "dir") { - root.ForgetPath(path, fs.EntryDirectory) - } else { - return out, errors.Errorf("unknown key %q", k) - } - forgotten = append(forgotten, path) - } - } - out = rc.Params{ - "forgotten": forgotten, - } - return out, nil - }, - Title: "Forget files or directories in the directory cache.", - Help: ` -This forgets the paths in the directory cache causing them to be -re-read from the remote when needed. - -If no paths are passed in then it will forget all the paths in the -directory cache. - - rclone rc vfs/forget - -Otherwise pass files or dirs in as file=path or dir=path. Any -parameter key starting with file will forget that file and any -starting with dir will forget that dir, eg - - rclone rc vfs/forget file=hello file2=goodbye dir=home/junk - -`, - }) - rc.Add(rc.Call{ - Path: "vfs/refresh", - Fn: func(ctx context.Context, in rc.Params) (out rc.Params, err error) { - root, err := vfs.Root() - if err != nil { - return nil, err - } - getDir := func(path string) (*Dir, error) { - path = strings.Trim(path, "/") - segments := strings.Split(path, "/") - var node Node = root - for _, s := range segments { - if dir, ok := node.(*Dir); ok { - node, err = dir.stat(s) - if err != nil { - return nil, err - } - } - } - if dir, ok := node.(*Dir); ok { - return dir, nil - } - return nil, EINVAL - } - - recursive := false - { - const k = "recursive" - - if v, ok := in[k]; ok { - s, ok := v.(string) - if !ok { - return out, errors.Errorf("value must be string %q=%v", k, v) - } - recursive, err = strconv.ParseBool(s) - if err != nil { - return out, errors.Errorf("invalid value %q=%v", k, v) - } - delete(in, k) - } - } - - result := map[string]string{} - if len(in) == 0 { - if recursive { - err = root.readDirTree() - } else { - err = root.readDir() - } - if err != nil { - result[""] = err.Error() - } else { - result[""] = "OK" - } - } else { - for k, v := range in { - path, ok := v.(string) - if !ok { - return out, errors.Errorf("value must be string %q=%v", k, v) - } - if strings.HasPrefix(k, "dir") { - dir, err := getDir(path) - if err != nil { - result[path] = err.Error() - } else { - if recursive { - err = dir.readDirTree() - } else { - err = dir.readDir() - } - if err != nil { - result[path] = err.Error() - } else { - result[path] = "OK" - } - - } - } else { - return out, errors.Errorf("unknown key %q", k) - } - } - } - out = rc.Params{ - "result": result, - } - return out, nil - }, + Path: "vfs/refresh", + Fn: rcRefresh, Title: "Refresh the directory cache.", Help: ` This reads the directories for the specified paths and freshens the @@ -167,12 +69,223 @@ starting with dir will refresh that directory, eg If the parameter recursive=true is given the whole directory tree will get refreshed. This refresh will use --fast-list if enabled. - -`, +` + getVFSHelp, }) +} + +func rcRefresh(ctx context.Context, in rc.Params) (out rc.Params, err error) { + vfs, err := getVFS(in) + if err != nil { + return nil, err + } + + root, err := vfs.Root() + if err != nil { + return nil, err + } + getDir := func(path string) (*Dir, error) { + path = strings.Trim(path, "/") + segments := strings.Split(path, "/") + var node Node = root + for _, s := range segments { + if dir, ok := node.(*Dir); ok { + node, err = dir.stat(s) + if err != nil { + return nil, err + } + } + } + if dir, ok := node.(*Dir); ok { + return dir, nil + } + return nil, EINVAL + } + + recursive := false + { + const k = "recursive" + + if v, ok := in[k]; ok { + s, ok := v.(string) + if !ok { + return out, errors.Errorf("value must be string %q=%v", k, v) + } + recursive, err = strconv.ParseBool(s) + if err != nil { + return out, errors.Errorf("invalid value %q=%v", k, v) + } + delete(in, k) + } + } + + result := map[string]string{} + if len(in) == 0 { + if recursive { + err = root.readDirTree() + } else { + err = root.readDir() + } + if err != nil { + result[""] = err.Error() + } else { + result[""] = "OK" + } + } else { + for k, v := range in { + path, ok := v.(string) + if !ok { + return out, errors.Errorf("value must be string %q=%v", k, v) + } + if strings.HasPrefix(k, "dir") { + dir, err := getDir(path) + if err != nil { + result[path] = err.Error() + } else { + if recursive { + err = dir.readDirTree() + } else { + err = dir.readDir() + } + if err != nil { + result[path] = err.Error() + } else { + result[path] = "OK" + } + } + } else { + return out, errors.Errorf("unknown key %q", k) + } + } + } + out = rc.Params{ + "result": result, + } + return out, nil +} + +// Add remote control for the VFS +func init() { + rc.Add(rc.Call{ + Path: "vfs/forget", + Fn: rcForget, + Title: "Forget files or directories in the directory cache.", + Help: ` +This forgets the paths in the directory cache causing them to be +re-read from the remote when needed. + +If no paths are passed in then it will forget all the paths in the +directory cache. + + rclone rc vfs/forget + +Otherwise pass files or dirs in as file=path or dir=path. Any +parameter key starting with file will forget that file and any +starting with dir will forget that dir, eg + + rclone rc vfs/forget file=hello file2=goodbye dir=home/junk +` + getVFSHelp, + }) +} + +func rcForget(ctx context.Context, in rc.Params) (out rc.Params, err error) { + vfs, err := getVFS(in) + if err != nil { + return nil, err + } + + root, err := vfs.Root() + if err != nil { + return nil, err + } + + forgotten := []string{} + if len(in) == 0 { + root.ForgetAll() + } else { + for k, v := range in { + path, ok := v.(string) + if !ok { + return out, errors.Errorf("value must be string %q=%v", k, v) + } + path = strings.Trim(path, "/") + if strings.HasPrefix(k, "file") { + root.ForgetPath(path, fs.EntryObject) + } else if strings.HasPrefix(k, "dir") { + root.ForgetPath(path, fs.EntryDirectory) + } else { + return out, errors.Errorf("unknown key %q", k) + } + forgotten = append(forgotten, path) + } + } + out = rc.Params{ + "forgotten": forgotten, + } + return out, nil +} + +func getDuration(k string, v interface{}) (time.Duration, error) { + s, ok := v.(string) + if !ok { + return 0, errors.Errorf("value must be string %q=%v", k, v) + } + interval, err := fs.ParseDuration(s) + if err != nil { + return 0, errors.Wrap(err, "parse duration") + } + return interval, nil +} + +func getInterval(in rc.Params) (time.Duration, bool, error) { + k := "interval" + v, ok := in[k] + if !ok { + return 0, false, nil + } + interval, err := getDuration(k, v) + if err != nil { + return 0, true, err + } + if interval < 0 { + return 0, true, errors.New("interval must be >= 0") + } + delete(in, k) + return interval, true, nil +} + +func getTimeout(in rc.Params) (time.Duration, error) { + k := "timeout" + v, ok := in[k] + if !ok { + return 10 * time.Second, nil + } + timeout, err := getDuration(k, v) + if err != nil { + return 0, err + } + delete(in, k) + return timeout, nil +} + +func getStatus(vfs *VFS, in rc.Params) (out rc.Params, err error) { + for k, v := range in { + return nil, errors.Errorf("invalid parameter: %s=%s", k, v) + } + return rc.Params{ + "enabled": vfs.Opt.PollInterval != 0, + "supported": vfs.pollChan != nil, + "interval": map[string]interface{}{ + "raw": vfs.Opt.PollInterval, + "seconds": vfs.Opt.PollInterval / time.Second, + "string": vfs.Opt.PollInterval.String(), + }, + }, nil +} + +func init() { rc.Add(rc.Call{ Path: "vfs/poll-interval", - Fn: rcPollFunc(vfs), + Fn: rcPollInterval, Title: "Get the status or update the value of the poll-interval option.", Help: ` Without any parameter given this returns the current status of the @@ -194,102 +307,82 @@ not reached. If poll-interval is updated or disabled temporarily, some changes might not get picked up by the polling function, depending on the used remote. -`, +` + getVFSHelp, }) } -func rcPollFunc(vfs *VFS) (rcPollFunc rc.Func) { - getDuration := func(k string, v interface{}) (time.Duration, error) { - s, ok := v.(string) - if !ok { - return 0, errors.Errorf("value must be string %q=%v", k, v) - } - interval, err := fs.ParseDuration(s) - if err != nil { - return 0, errors.Wrap(err, "parse duration") - } - return interval, nil - } - getInterval := func(in rc.Params) (time.Duration, bool, error) { - k := "interval" - v, ok := in[k] - if !ok { - return 0, false, nil - } - interval, err := getDuration(k, v) - if err != nil { - return 0, true, err - } - if interval < 0 { - return 0, true, errors.New("interval must be >= 0") - } - delete(in, k) - return interval, true, nil - } - getTimeout := func(in rc.Params) (time.Duration, error) { - k := "timeout" - v, ok := in[k] - if !ok { - return 10 * time.Second, nil - } - timeout, err := getDuration(k, v) - if err != nil { - return 0, err - } - delete(in, k) - return timeout, nil +func rcPollInterval(ctx context.Context, in rc.Params) (out rc.Params, err error) { + vfs, err := getVFS(in) + if err != nil { + return nil, err } - _status := func(in rc.Params) (out rc.Params, err error) { - for k, v := range in { - return nil, errors.Errorf("invalid parameter: %s=%s", k, v) - } - return rc.Params{ - "enabled": vfs.Opt.PollInterval != 0, - "supported": vfs.pollChan != nil, - "interval": map[string]interface{}{ - "raw": vfs.Opt.PollInterval, - "seconds": vfs.Opt.PollInterval / time.Second, - "string": vfs.Opt.PollInterval.String(), - }, - }, nil + interval, intervalPresent, err := getInterval(in) + if err != nil { + return nil, err + } + timeout, err := getTimeout(in) + if err != nil { + return nil, err + } + for k, v := range in { + return nil, errors.Errorf("invalid parameter: %s=%s", k, v) + } + if vfs.pollChan == nil { + return nil, errors.New("poll-interval is not supported by this remote") } - return func(ctx context.Context, in rc.Params) (out rc.Params, err error) { - interval, intervalPresent, err := getInterval(in) - if err != nil { - return nil, err - } - timeout, err := getTimeout(in) - if err != nil { - return nil, err - } - for k, v := range in { - return nil, errors.Errorf("invalid parameter: %s=%s", k, v) - } - if vfs.pollChan == nil { - return nil, errors.New("poll-interval is not supported by this remote") - } - if !intervalPresent { - return _status(in) - } - var timeoutHit bool - var timeoutChan <-chan time.Time - if timeout > 0 { - timer := time.NewTimer(timeout) - defer timer.Stop() - timeoutChan = timer.C - } - select { - case vfs.pollChan <- interval: - vfs.Opt.PollInterval = interval - case <-timeoutChan: - timeoutHit = true - } - out, err = _status(in) - if out != nil { - out["timeout"] = timeoutHit - } - return + if !intervalPresent { + return getStatus(vfs, in) } + var timeoutHit bool + var timeoutChan <-chan time.Time + if timeout > 0 { + timer := time.NewTimer(timeout) + defer timer.Stop() + timeoutChan = timer.C + } + select { + case vfs.pollChan <- interval: + vfs.Opt.PollInterval = interval + case <-timeoutChan: + timeoutHit = true + } + out, err = getStatus(vfs, in) + if out != nil { + out["timeout"] = timeoutHit + } + return +} + +func init() { + rc.Add(rc.Call{ + Path: "vfs/list", + Title: "List active VFSes.", + Help: ` +This lists the active VFSes. + +It returns a list under the key "vfses" where the values are the VFS +names that could be passed to the other VFS commands in the "fs" +parameter.`, + Fn: rcList, + }) +} + +func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) { + activeMu.Lock() + defer activeMu.Unlock() + var names = []string{} + for name, vfses := range active { + if len(vfses) == 1 { + names = append(names, name) + } else { + for i := range vfses { + names = append(names, fmt.Sprintf("%s[%d]", name, i)) + } + } + } + out = rc.Params{} + out["vfses"] = names + return out, nil } diff --git a/vfs/rc_test.go b/vfs/rc_test.go new file mode 100644 index 000000000..0e31ef0a9 --- /dev/null +++ b/vfs/rc_test.go @@ -0,0 +1,118 @@ +package vfs + +import ( + "context" + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/rc" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/vfs/vfscommon" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func rcNewRun(t *testing.T, method string) (r *fstest.Run, vfs *VFS, cleanup func(), call *rc.Call) { + if *fstest.RemoteName != "" { + t.Skip("Skipping test on non local remote") + } + r, vfs, cleanup = newTestVFS(t) + call = rc.Calls.Get(method) + assert.NotNil(t, call) + return r, vfs, cleanup, call +} + +func TestRcGetVFS(t *testing.T) { + in := rc.Params{} + vfs, err := getVFS(in) + require.Error(t, err) + assert.Contains(t, err.Error(), "no VFS active") + assert.Nil(t, vfs) + + r, vfs2, cleanup := newTestVFS(t) + defer cleanup() + + vfs, err = getVFS(in) + require.NoError(t, err) + assert.True(t, vfs == vfs2) + + inPresent := rc.Params{"fs": fs.ConfigString(r.Fremote)} + vfs, err = getVFS(inPresent) + require.NoError(t, err) + assert.True(t, vfs == vfs2) + + inWrong := rc.Params{"fs": fs.ConfigString(r.Fremote) + "notfound"} + vfs, err = getVFS(inWrong) + require.Error(t, err) + assert.Contains(t, err.Error(), "no VFS found with name") + assert.Nil(t, vfs) + + opt := vfscommon.DefaultOpt + opt.NoModTime = true + vfs3 := New(r.Fremote, &opt) + defer vfs3.Shutdown() + + vfs, err = getVFS(in) + require.Error(t, err) + assert.Contains(t, err.Error(), "more than one VFS active - need") + assert.Nil(t, vfs) + + vfs, err = getVFS(inPresent) + require.Error(t, err) + assert.Contains(t, err.Error(), "more than one VFS active with name") + assert.Nil(t, vfs) +} + +func TestRcForget(t *testing.T) { + r, vfs, cleanup, call := rcNewRun(t, "vfs/forget") + defer cleanup() + _, _ = r, vfs + out, err := call.Fn(context.Background(), nil) + require.NoError(t, err) + assert.Equal(t, rc.Params{ + "forgotten": []string{}, + }, out) + // FIXME needs more tests +} + +func TestRcRefresh(t *testing.T) { + r, vfs, cleanup, call := rcNewRun(t, "vfs/refresh") + defer cleanup() + _, _ = r, vfs + out, err := call.Fn(context.Background(), nil) + require.NoError(t, err) + assert.Equal(t, rc.Params{ + "result": map[string]string{ + "": "OK", + }, + }, out) + // FIXME needs more tests +} + +func TestRcPollInterval(t *testing.T) { + r, vfs, cleanup, call := rcNewRun(t, "vfs/poll-interval") + defer cleanup() + _ = vfs + if r.Fremote.Features().ChangeNotify == nil { + t.Skip("ChangeNotify not supported") + } + out, err := call.Fn(context.Background(), nil) + require.NoError(t, err) + assert.Equal(t, rc.Params{}, out) + // FIXME needs more tests +} + +func TestRcList(t *testing.T) { + r, vfs, cleanup, call := rcNewRun(t, "vfs/list") + defer cleanup() + _ = vfs + + out, err := call.Fn(context.Background(), nil) + require.NoError(t, err) + + assert.Equal(t, rc.Params{ + "vfses": []string{ + fs.ConfigString(r.Fremote), + }, + }, out) +} diff --git a/vfs/vfs.go b/vfs/vfs.go index cc016b927..4fa915d92 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -204,9 +204,6 @@ func New(f fs.Fs, opt *vfscommon.Options) *VFS { vfs.SetCacheMode(vfs.Opt.CacheMode) - // add the remote control - vfs.addRC() - // Pin the Fs into the cache so that when we use cache.NewFs // with the same remote string we get this one. The Pin is // removed by Shutdown