diff --git a/README.md b/README.md index 484dd7110..71e178a54 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Rclone is a command line program to sync files and directories to and from * Yandex Disk * SFTP * The local filesystem + * FTP Features diff --git a/docs/content/about.md b/docs/content/about.md index f38695b06..06acfab77 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -25,6 +25,7 @@ Rclone is a command line program to sync files and directories to and from * Yandex Disk * SFTP * The local filesystem + * FTP Features diff --git a/docs/content/docs.md b/docs/content/docs.md index 35453fdb7..29166f271 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -31,6 +31,7 @@ See the following for detailed instructions for * [Microsoft OneDrive](/onedrive/) * [Yandex Disk](/yandex/) * [SFTP](/sftp/) + * [FTP](/ftp/) * [Crypt](/crypt/) - to encrypt other remotes Usage diff --git a/docs/content/ftp.md b/docs/content/ftp.md new file mode 100644 index 000000000..d8bbaf73a --- /dev/null +++ b/docs/content/ftp.md @@ -0,0 +1,35 @@ +--- +title: "FTP" +description: "Rclone docs for FTP backend" +date: "2017-01-01" +--- + + FTP +------------------------------ + +FTP support is provided via +[github.com/jlaffaye/ftp](https://godoc.org/github.com/jlaffaye/ftp) +package. + +### Configuration ### + +An Ftp backend only needs an Url and and username and password. With +anonymous FTP server you will need to use `anonymous` as username and +your email address as password. + +Example: +``` +[remote] +type = Ftp +username = anonymous +password = john.snow@example.org +url = ftp://ftp.kernel.org/pub +``` + +### Unsupported features ### + +FTP backends does not support: + +* Any hash mechanism +* Modified Time +* remote copy/move diff --git a/docs/content/overview.md b/docs/content/overview.md index 4c16d0c95..9ff8055c1 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -29,6 +29,7 @@ Here is an overview of the major features of each cloud storage system. | Yandex Disk | MD5 | Yes | No | No | R/W | | SFTP | - | Yes | Depends | No | - | | The local filesystem | All | Yes | Depends | No | - | +| FTP | None | No | Yes | No | - | ### Hash ### @@ -117,6 +118,7 @@ operations more efficient. | Yandex Disk | Yes | No | No | No | No [#575](https://github.com/ncw/rclone/issues/575) | | SFTP | No | No | Yes | Yes | No | | The local filesystem | Yes | No | Yes | Yes | No | +| FTP | No | No | No | No | No | ### Purge ### diff --git a/fs/all/all.go b/fs/all/all.go index 986339e3a..c144c75b4 100644 --- a/fs/all/all.go +++ b/fs/all/all.go @@ -15,4 +15,5 @@ import ( _ "github.com/ncw/rclone/sftp" _ "github.com/ncw/rclone/swift" _ "github.com/ncw/rclone/yandex" + _ "github.com/ncw/rclone/ftp" ) diff --git a/fs/test_all.go b/fs/test_all.go index 4e54a50ab..eca634d4a 100644 --- a/fs/test_all.go +++ b/fs/test_all.go @@ -35,6 +35,7 @@ var ( "TestSftp:", "TestSwift:", "TestYandex:", + "TestFTP:", } binary = "fs.test" // Flags diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go index f6caec15b..f1bd7b609 100644 --- a/fstest/fstests/gen_tests.go +++ b/fstest/fstests/gen_tests.go @@ -142,5 +142,6 @@ func main() { generateTestProgram(t, fns, "Crypt", "2") generateTestProgram(t, fns, "Crypt", "3") generateTestProgram(t, fns, "Sftp", "") + generateTestProgram(t, fns, "FTP", "") log.Printf("Done") } diff --git a/ftp/ftp.go b/ftp/ftp.go new file mode 100644 index 000000000..e7e900f58 --- /dev/null +++ b/ftp/ftp.go @@ -0,0 +1,433 @@ +// Package fs is a generic file system interface for rclone object storage systems +package ftp + +import ( + "fmt" + "github.com/jlaffaye/ftp" + "github.com/ncw/rclone/fs" + "io" + "path/filepath" + "regexp" + "strconv" + "time" + "sync" + "strings" +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "Ftp", + Description: "FTP interface", + NewFs: NewFs, + Options: []fs.Option{ + { + Name: "username", + Help: "Username", + }, { + Name: "password", + Help: "Password", + }, { + Name: "url", + Help: "FTP url", + }, + }, + }) +} + +type Url struct { + Scheme string + Host string + Port int + Path string +} + +type Fs struct { + name string // name of this remote + c *ftp.ServerConn // the connection to the FTP server + root string // the path we are working on if any + url Url + mu sync.Mutex +} + +type Object struct { + fs *Fs + remote string + info *FileInfo +} + +type FileInfo struct { + Name string + Size uint64 + ModTime time.Time + IsDir bool +} + + + +// Implements ReadCloser for FTP objects. +type FtpReadCloser struct { + remote string + c *ftp.ServerConn + fd io.ReadCloser +} + +///////////////// +// Url methods // +///////////////// +func (u *Url) ToDial() string { + return fmt.Sprintf("%s:%d", u.Host, u.Port) +} + +func (u *Url) String() string { + return fmt.Sprintf("ftp://%s:%d/%s", u.Host, u.Port, u.Path) +} + +func parseUrl(url string) Url { + // This is *similar* to the RFC 3986 regexp but it matches the + // port independently from the host + r, _ := regexp.Compile("^(([^:/?#]+):)?(//([^/?#:]*))?(:([0-9]+))?([^?#]*)(\\?([^#]*))?(#(.*))?") + + data := r.FindAllStringSubmatch(url, -1) + + if data[0][5] == "" { data[0][5] = "21" } + port, _ := strconv.Atoi(data[0][5]) + return Url{data[0][2], data[0][4], port, data[0][7]} +} + +//////////////// +// Fs Methods // +//////////////// + +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { + fs.Debug(f, "Trying to put file %s", src.Remote()) + o := &Object{ + fs: f, + remote: src.Remote(), + } + err := o.Update(in, src) + return o, err +} + +func (f *Fs) Rmdir(dir string) error { + // This is actually a recursive remove directory + f.mu.Lock() + files, _ := f.c.List(filepath.Join(f.root, dir)) + f.mu.Unlock() + for i:= range files { + if files[i].Type == ftp.EntryTypeFolder { + f.Rmdir(filepath.Join(dir, files[i].Name)) + } + } + f.mu.Lock() + err := f.c.RemoveDir(filepath.Join(f.root, dir)) + f.mu.Unlock() + return err +} + +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +func (f *Fs) String() string { + return fmt.Sprintf("FTP Connection to %s", f.url.String()) +} + +// Hash are not supported +func (f *Fs) Hashes() fs.HashSet { + return 0 +} + +// Modified Time not supported +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +func (f *Fs) mkdir(abspath string) error { + _, err := f.GetInfo(abspath) + if err != nil { + fs.Debug(f, "Trying to create directory %s", abspath) + f.mu.Lock() + err := f.c.MakeDir(abspath) + f.mu.Unlock() + if err != nil { + return err + } + } + return err +} + +func (f *Fs) Mkdir(dir string) error { + // This actually works as mkdir -p + fs.Debug(f, "ENTER function 'Mkdir' on '%s/%s'", f.root, dir) + defer fs.Debug(f, "EXIT function 'Mkdir' on '%s/%s'", f.root, dir) + abspath := filepath.Join(f.root, dir) + tokens := strings.Split(abspath, "/") + curdir := "" + for i:= range tokens { + curdir += "/" + tokens[i] + f.mkdir(curdir) + } + return nil +} + +func (f *Fs) GetInfo(remote string) (*FileInfo, error) { + fs.Debug(f, "ENTER function 'GetInfo' on file %s", remote) + defer fs.Debug(f, "EXIT function 'GetInfo'") + dir := filepath.Dir(remote) + base := filepath.Base(remote) + + f.mu.Lock() + files, _ := f.c.List(dir) + f.mu.Unlock() + for i:= range files { + if files[i].Name == base { + info := &FileInfo{ + Name: remote, + Size: files[i].Size, + ModTime: files[i].Time, + IsDir: files[i].Type == ftp.EntryTypeFolder, + } + return info, nil + } + } + return nil, fs.ErrorObjectNotFound +} + +func (f *Fs) NewObject(remote string) (fs.Object, error) { + fs.Debug(f, "ENTER function 'NewObject' called with remote %s", remote) + defer fs.Debug(f, "EXIT function 'NewObject'") + dir := filepath.Dir(remote) + base := filepath.Base(remote) + + f.mu.Lock() + files, _ := f.c.List(dir) + f.mu.Unlock() + for i:= range files { + if files[i].Name == base { + o := &Object{ + fs: f, + remote: remote, + } + info := &FileInfo{ + Name: remote, + Size: files[i].Size, + ModTime: files[i].Time, + } + o.info = info + + return o, nil + } + } + return nil, fs.ErrorObjectNotFound +} + +func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { + fs.Debug(f, "ENTER function 'list'") + defer fs.Debug(f, "EXIT function 'list'") + f.mu.Lock() + files, _ := f.c.List(filepath.Join(f.root, dir)) + f.mu.Unlock() + for i:= range files { + object := files[i] + newremote := filepath.Join(dir, object.Name) + switch object.Type { + case ftp.EntryTypeFolder: + if out.IncludeDirectory(newremote){ + d := &fs.Dir{ + Name: newremote, + When: object.Time, + Bytes: 0, + Count: -1, + } + if curlevel < out.Level(){ + f.list(out, filepath.Join(dir, object.Name), curlevel +1 ) + } + if out.AddDir(d) { + return + } + } + default: + o := &Object{ + fs: f, + remote: newremote, + } + info := &FileInfo{ + Name: newremote, + Size: object.Size, + ModTime: object.Time, + } + o.info = info + if out.Add(o) { + return + } + } + } +} + +func (f *Fs) List(out fs.ListOpts, dir string) { + fs.Debug(f, "ENTER function 'List' on directory '%s/%s'", f.root, dir) + defer fs.Debug(f, "EXIT function 'List' for directory '%s/%s'", f.root, dir) + f.list(out, dir, 1) + out.Finished() +} + +//////////////////// +// Object methods // +//////////////////// + +func (o *Object) Hash(t fs.HashType) (string, error) { + return "", fs.ErrHashUnsupported +} + +func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { + path := filepath.Join(o.fs.root, o.remote) + fs.Debug(o.fs, "ENTER function 'Open' on file '%s' in root '%s'", o.remote, o.fs.root) + defer fs.Debug(o.fs, "EXIT function 'Open' %s", path) + c, _, err := ftpConnection(o.fs.name, o.fs.root) + if err != nil { + return nil, err + } + fd, err := c.Retr(path) + if err != nil { + return nil, err + } + return FtpReadCloser{path, c, fd}, nil +} + +func (o *Object) Remote() string { + return o.remote +} + +func (o *Object) Remove() error { + path := filepath.Join(o.fs.root, o.remote) + fs.Debug(o, "ENTER function 'Remove' for obejct at %s", path) + defer fs.Debug(o, "EXIT function 'Remove' for obejct at %s", path) + // Check if it's a directory or a file + info, _ := o.fs.GetInfo(path) + var err error + if info.IsDir { + err = o.fs.Rmdir(o.remote) + } else { + o.fs.mu.Lock() + err = o.fs.c.Delete(path) + o.fs.mu.Unlock() + } + return err +} + +func (o *Object) SetModTime(modTime time.Time) error { + return nil +} + +func (o *Object) Fs() fs.Info { + return o.fs +} + +func (o *Object) ModTime() time.Time { + return o.info.ModTime +} + +func (o *Object) Size() int64 { + return int64(o.info.Size) +} + +func (o *Object) Storable() bool { + return true +} + +func (o *Object) String() string { + return fmt.Sprintf("FTP file at %s/%s", o.fs.url.String(), o.remote) +} + +func (o *Object) MakeAllDir() { + tokens := strings.Split(filepath.Dir(o.remote), "/") + dir := "" + for i:= range tokens { + dir += tokens[i]+"/" + o.fs.Mkdir(dir) + } +} +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { + // Create all upper directory first... + o.MakeAllDir() + path := filepath.Join(o.fs.root, o.remote) + c, _, _ := ftpConnection(o.fs.name, o.fs.root) + err := c.Stor(path, in) + o.info, _ = o.fs.GetInfo(path) + return err +} + +/////////////////////////// +// FtpReadCloser methods // +/////////////////////////// + +func (f FtpReadCloser) Read(p []byte) (int, error) { + return f.fd.Read(p) +} + +func (f FtpReadCloser) Close() error { + err := f.fd.Close() + defer f.c.Quit() + if err != nil { + return nil + } + return nil +} + +// This mutex is only used by ftpConnection. We create a new ftp +// connection for each transfer, but we need to serialize it otherwise +// Dial() and Login() might be mixed... +var globalMux = sync.Mutex{} + +func ftpConnection(name, root string) (*ftp.ServerConn, Url, error) { + // Open a new connection to the FTP server. + url := fs.ConfigFileGet(name, "url") + user := fs.ConfigFileGet(name, "username") + pass := fs.ConfigFileGet(name, "password") + u := parseUrl(url) + u.Path = filepath.Join(u.Path, root) + fs.Debug(nil, "New ftp Connection with name %s and url %s (path %s)", name, u.String(), u.Path) + globalMux.Lock() + defer globalMux.Unlock() + c, err := ftp.DialTimeout(u.ToDial(), 30*time.Second) + if err != nil { + fs.ErrorLog(nil, "Error while Dialing %s: %s", u.ToDial(), err) + return nil, u, err + } + err = c.Login(user, pass) + if err != nil { + fs.ErrorLog(nil, "Error while Logging in into %s: %s", u.ToDial(), err) + return nil, u, err + } + return c, u, nil +} + + + +// Register the FS +func NewFs(name, root string) (fs.Fs, error) { + fs.Debug(nil, "ENTER function 'NewFs' with name %s and root %s", name, root) + defer fs.Debug(nil, "EXIT function 'NewFs'") + c, u, err := ftpConnection(name, root) + if err != nil { + return nil, err + } + fs := &Fs{ + name: name, + root: u.Path, + c: c, + url: u, + mu: sync.Mutex{}, + } + return fs, err +} + +var ( + _ fs.Fs = &Fs{} +) diff --git a/ftp/ftp_internal_test.go b/ftp/ftp_internal_test.go new file mode 100644 index 000000000..cea83a48e --- /dev/null +++ b/ftp/ftp_internal_test.go @@ -0,0 +1,36 @@ +package ftp + +import "testing" + +func TestParseUrlToDial(t *testing.T){ + for _, test := range []struct { + in string + want string + }{ + {"ftp://foo.bar", "foo.bar:21"}, + {"http://foo.bar", "foo.bar:21"}, + {"ftp:/foo.bar:123", "foo.bar:123"}, + } { + u := parseUrl(test.in) + got := u.ToDial() + if got != test.want { + t.Logf("%q: want %q got %q", test.in, test.want, got) + } + } +} + +func TestParseUrlPath(t *testing.T){ + for _, test := range []struct { + in string + want string + }{ + {"ftp://foo.bar/", "/"}, + {"ftp://foo.bar/debian", "/debian"}, + {"ftp://foo.bar", "/"}, + } { + u := parseUrl(test.in) + if u.Path != test.want { + t.Logf("%q: want %q got %q", test.in, test.want, u.Path) + } + } +} diff --git a/ftp/ftp_test.go b/ftp/ftp_test.go new file mode 100644 index 000000000..7b223fdcd --- /dev/null +++ b/ftp/ftp_test.go @@ -0,0 +1,62 @@ +// Test FTP filesystem interface +// +// Automatically generated - DO NOT EDIT +// Regenerate with: make gen_tests +package ftp_test + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + "github.com/ncw/rclone/ftp" +) + +func TestSetup(t *testing.T) { + fstests.NilObject = fs.Object((*ftp.Object)(nil)) + fstests.RemoteName = "TestFTP:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } +func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }