From 6669165b6b511ad2f6c988fe90b9da71db32e60b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 6 Dec 2016 18:00:24 +0000 Subject: [PATCH] serve http command to serve a remote over HTTP This implements a basic webserver to serve an rclone remote over HTTP. It also sets up the framework for adding more types of server later. --- cmd/all/all.go | 1 + cmd/serve/http/http.go | 251 ++++++++++++++++++ cmd/serve/http/http_test.go | 170 ++++++++++++ cmd/serve/http/testdata/files/hidden.txt | 1 + cmd/serve/http/testdata/files/hidden/file.txt | 1 + cmd/serve/http/testdata/files/one%.txt | 1 + cmd/serve/http/testdata/files/three/a.txt | 1 + cmd/serve/http/testdata/files/three/b.txt | 1 + cmd/serve/http/testdata/files/two.txt | 1 + cmd/serve/http/testdata/golden/a.txt | 1 + .../http/testdata/golden/dirnotfound.html | 1 + cmd/serve/http/testdata/golden/hidden.txt | 1 + cmd/serve/http/testdata/golden/hiddendir.html | 1 + cmd/serve/http/testdata/golden/index.html | 13 + cmd/serve/http/testdata/golden/indexhead.txt | 0 cmd/serve/http/testdata/golden/indexpost.txt | 1 + cmd/serve/http/testdata/golden/notfound.html | 1 + cmd/serve/http/testdata/golden/one.txt | 1 + cmd/serve/http/testdata/golden/onehead.txt | 0 cmd/serve/http/testdata/golden/onepost.txt | 1 + cmd/serve/http/testdata/golden/three.html | 12 + cmd/serve/serve.go | 33 +++ 22 files changed, 494 insertions(+) create mode 100644 cmd/serve/http/http.go create mode 100644 cmd/serve/http/http_test.go create mode 100644 cmd/serve/http/testdata/files/hidden.txt create mode 100644 cmd/serve/http/testdata/files/hidden/file.txt create mode 100644 cmd/serve/http/testdata/files/one%.txt create mode 100644 cmd/serve/http/testdata/files/three/a.txt create mode 100644 cmd/serve/http/testdata/files/three/b.txt create mode 100644 cmd/serve/http/testdata/files/two.txt create mode 100644 cmd/serve/http/testdata/golden/a.txt create mode 100644 cmd/serve/http/testdata/golden/dirnotfound.html create mode 100644 cmd/serve/http/testdata/golden/hidden.txt create mode 100644 cmd/serve/http/testdata/golden/hiddendir.html create mode 100644 cmd/serve/http/testdata/golden/index.html create mode 100644 cmd/serve/http/testdata/golden/indexhead.txt create mode 100644 cmd/serve/http/testdata/golden/indexpost.txt create mode 100644 cmd/serve/http/testdata/golden/notfound.html create mode 100644 cmd/serve/http/testdata/golden/one.txt create mode 100644 cmd/serve/http/testdata/golden/onehead.txt create mode 100644 cmd/serve/http/testdata/golden/onepost.txt create mode 100644 cmd/serve/http/testdata/golden/three.html create mode 100644 cmd/serve/serve.go diff --git a/cmd/all/all.go b/cmd/all/all.go index 3aa2dee9b..9c8ccc6a4 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -38,6 +38,7 @@ import ( _ "github.com/ncw/rclone/cmd/rcat" _ "github.com/ncw/rclone/cmd/rmdir" _ "github.com/ncw/rclone/cmd/rmdirs" + _ "github.com/ncw/rclone/cmd/serve" _ "github.com/ncw/rclone/cmd/sha1sum" _ "github.com/ncw/rclone/cmd/size" _ "github.com/ncw/rclone/cmd/sync" diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go new file mode 100644 index 000000000..0e7cf9ae6 --- /dev/null +++ b/cmd/serve/http/http.go @@ -0,0 +1,251 @@ +package http + +import ( + "fmt" + "html/template" + "io" + "log" + "net/http" + "path" + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs" + "github.com/spf13/cobra" +) + +// Globals +var ( + bindAddress = "localhost:8080" + readWrite = false +) + +func init() { + Command.Flags().StringVarP(&bindAddress, "addr", "", bindAddress, "IPaddress:Port to bind server to.") + // Command.Flags().BoolVarP(&readWrite, "rw", "", readWrite, "Serve in read/write mode.") +} + +// Command definition for cobra +var Command = &cobra.Command{ + Use: "http remote:path", + Short: `Serve the remote over HTTP.`, + Long: `rclone serve http implements a basic web server to serve the remote +over HTTP. This can be viewed in a web browser or you can make a +remote of type http read from it. + +Use --addr to specify which IP address and port the server should +listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all +IPs. By default it only listens on localhost. + +You can use the filter flags (eg --include, --exclude) to control what +is served. + +The server will log errors. Use -v to see access logs. + +--bwlimit will be respected for file transfers. Use --stats to +control the stats printing. +`, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(1, 1, command, args) + f := cmd.NewFsSrc(args) + cmd.Run(false, true, command, func() error { + s := server{ + f: f, + bindAddress: bindAddress, + readWrite: readWrite, + } + s.serve() + return nil + }) + }, +} + +// server contains everything to run the server +type server struct { + f fs.Fs + bindAddress string + readWrite bool +} + +// serve creates the http server +func (s *server) serve() { + mux := http.NewServeMux() + mux.HandleFunc("/", s.handler) + // FIXME make a transport? + httpServer := &http.Server{ + Addr: s.bindAddress, + Handler: mux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + fs.Logf(s.f, "Serving on http://%s/", bindAddress) + log.Fatal(httpServer.ListenAndServe()) +} + +// handler reads incoming requests and dispatches them +func (s *server) handler(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" && r.Method != "HEAD" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + urlPath := r.URL.Path + isDir := strings.HasSuffix(urlPath, "/") + remote := strings.Trim(urlPath, "/") + if isDir { + s.serveDir(w, r, remote) + } else { + s.serveFile(w, r, remote) + } +} + +// entry is a directory entry +type entry struct { + remote string + URL string + Leaf string +} + +// entries represents a directory +type entries []entry + +// indexPage is a directory listing template +var indexPage = ` + + + +{{ .Title }} + + +

{{ .Title }}

+{{ range $i := .Entries }}{{ $i.Leaf }}
+{{ end }} + +` + +// indexTemplate is the instantiated indexPage +var indexTemplate = template.Must(template.New("index").Parse(indexPage)) + +// indexData is used to fill in the indexTemplate +type indexData struct { + Title string + Entries entries +} + +// error returns an http.StatusInternalServerError and logs the error +func internalError(what interface{}, w http.ResponseWriter, text string, err error) { + fs.Stats.Error() + fs.Errorf(what, "%s: %v", text, err) + http.Error(w, text+".", http.StatusInternalServerError) +} + +// serveDir serves a directory index at dirRemote +func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) { + // Check the directory is included in the filters + if !fs.Config.Filter.IncludeDirectory(dirRemote) { + fs.Infof(dirRemote, "%s: Directory not found (filtered)", r.RemoteAddr) + http.Error(w, "Directory not found", http.StatusNotFound) + return + } + + // List the directory + dirEntries, err := fs.ListDirSorted(s.f, false, dirRemote) + if err == fs.ErrorDirNotFound { + fs.Infof(dirRemote, "%s: Directory not found", r.RemoteAddr) + http.Error(w, "Directory not found", http.StatusNotFound) + return + } else if err != nil { + internalError(dirRemote, w, "Failed to list directory", err) + return + } + + var out entries + for _, o := range dirEntries { + remote := strings.Trim(o.Remote(), "/") + leaf := path.Base(remote) + urlRemote := leaf + if _, ok := o.(*fs.Dir); ok { + leaf += "/" + urlRemote += "/" + } + out = append(out, entry{remote: remote, URL: urlRemote, Leaf: leaf}) + } + + // Account the transfer + fs.Stats.Transferring(dirRemote) + defer fs.Stats.DoneTransferring(dirRemote, true) + + fs.Infof(dirRemote, "%s: Serving directory", r.RemoteAddr) + err = indexTemplate.Execute(w, indexData{ + Entries: out, + Title: fmt.Sprintf("Directory listing of /%s", dirRemote), + }) + if err != nil { + internalError(dirRemote, w, "Failed to render template", err) + return + } +} + +// serveFile serves a file object at remote +func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string) { + // FIXME could cache the directories and objects... + obj, err := s.f.NewObject(remote) + if err == fs.ErrorObjectNotFound { + fs.Infof(remote, "%s: File not found", r.RemoteAddr) + http.Error(w, "File not found", http.StatusNotFound) + return + } else if err != nil { + internalError(remote, w, "Failed to find file", err) + return + } + + // Check the object is included in the filters + if !fs.Config.Filter.IncludeObject(obj) { + fs.Infof(remote, "%s: File not found (filtered)", r.RemoteAddr) + http.Error(w, "File not found", http.StatusNotFound) + return + } + + // Set content length since we know how long the object is + w.Header().Set("Content-Length", strconv.FormatInt(obj.Size(), 10)) + + // Set content type + mimeType := fs.MimeType(obj) + if mimeType == "application/octet-stream" && path.Ext(remote) == "" { + // Leave header blank so http server guesses + } else { + w.Header().Set("Content-Type", mimeType) + } + + // If HEAD no need to read the object since we have set the headers + if r.Method == "HEAD" { + return + } + + // open the object + in, err := obj.Open() + if err != nil { + internalError(remote, w, "Failed to open file", err) + return + } + defer func() { + err := in.Close() + if err != nil { + fs.Errorf(remote, "Failed to close file: %v", err) + } + }() + + // Account the transfer + fs.Stats.Transferring(remote) + defer fs.Stats.DoneTransferring(remote, true) + in = fs.NewAccount(in, obj).WithBuffer() // account the transfer + + // Copy the contents of the object to the output + fs.Infof(remote, "%s: Serving file", r.RemoteAddr) + _, err = io.Copy(w, in) + if err != nil { + fs.Errorf(remote, "Failed to write file: %v", err) + } +} diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go new file mode 100644 index 000000000..f8ee1bc91 --- /dev/null +++ b/cmd/serve/http/http_test.go @@ -0,0 +1,170 @@ +package http + +import ( + "flag" + "io/ioutil" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/ncw/rclone/fs" + _ "github.com/ncw/rclone/local" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var updateGolden = flag.Bool("updategolden", false, "update golden files for regression test") + +const ( + testBindAddress = "localhost:51777" + testURL = "http://" + testBindAddress + "/" +) + +func startServer(t *testing.T, f fs.Fs) { + s := server{ + f: f, + bindAddress: testBindAddress, + readWrite: false, + } + + go s.serve() + + // try to connect to the test server + pause := time.Millisecond + for i := 0; i < 10; i++ { + conn, err := net.Dial("tcp", testBindAddress) + if err == nil { + _ = conn.Close() + return + } + // t.Logf("couldn't connect, sleeping for %v: %v", pause, err) + time.Sleep(pause) + pause *= 2 + } + t.Fatal("couldn't connect to server") + +} + +func TestInit(t *testing.T) { + // Configure the remote + fs.LoadConfig() + // fs.Config.LogLevel = fs.LogLevelDebug + // fs.Config.DumpHeaders = true + // fs.Config.DumpBodies = true + + // exclude files called hidden.txt and directories called hidden + require.NoError(t, fs.Config.Filter.AddRule("- hidden.txt")) + require.NoError(t, fs.Config.Filter.AddRule("- hidden/**")) + + // Create a test Fs + f, err := fs.NewFs("testdata/files") + require.NoError(t, err) + + startServer(t, f) +} + +// check body against the file, or re-write body if -updategolden is +// set. +func checkGolden(t *testing.T, fileName string, got []byte) { + if *updateGolden { + t.Logf("Updating golden file %q", fileName) + err := ioutil.WriteFile(fileName, got, 0666) + require.NoError(t, err) + } else { + want, err := ioutil.ReadFile(fileName) + require.NoError(t, err) + wants := strings.Split(string(want), "\n") + gots := strings.Split(string(got), "\n") + assert.Equal(t, wants, gots, fileName) + } +} + +func TestGets(t *testing.T) { + for _, test := range []struct { + URL string + Status int + Golden string + Method string + }{ + { + URL: "", + Status: http.StatusOK, + Golden: "testdata/golden/index.html", + }, + { + URL: "notfound", + Status: http.StatusNotFound, + Golden: "testdata/golden/notfound.html", + }, + { + URL: "dirnotfound/", + Status: http.StatusNotFound, + Golden: "testdata/golden/dirnotfound.html", + }, + { + URL: "hidden/", + Status: http.StatusNotFound, + Golden: "testdata/golden/hiddendir.html", + }, + { + URL: "one%25.txt", + Status: http.StatusOK, + Golden: "testdata/golden/one.txt", + }, + { + URL: "hidden.txt", + Status: http.StatusNotFound, + Golden: "testdata/golden/hidden.txt", + }, + { + URL: "three/", + Status: http.StatusOK, + Golden: "testdata/golden/three.html", + }, + { + URL: "three/a.txt", + Status: http.StatusOK, + Golden: "testdata/golden/a.txt", + }, + { + URL: "", + Method: "HEAD", + Status: http.StatusOK, + Golden: "testdata/golden/indexhead.txt", + }, + { + URL: "one%25.txt", + Method: "HEAD", + Status: http.StatusOK, + Golden: "testdata/golden/onehead.txt", + }, + { + URL: "", + Method: "POST", + Status: http.StatusMethodNotAllowed, + Golden: "testdata/golden/indexpost.txt", + }, + { + URL: "one%25.txt", + Method: "POST", + Status: http.StatusMethodNotAllowed, + Golden: "testdata/golden/onepost.txt", + }, + } { + method := test.Method + if method == "" { + method = "GET" + } + req, err := http.NewRequest(method, testURL+test.URL, nil) + require.NoError(t, err) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, test.Status, resp.StatusCode, test.Golden) + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + checkGolden(t, test.Golden, body) + } +} diff --git a/cmd/serve/http/testdata/files/hidden.txt b/cmd/serve/http/testdata/files/hidden.txt new file mode 100644 index 000000000..136c05e0d --- /dev/null +++ b/cmd/serve/http/testdata/files/hidden.txt @@ -0,0 +1 @@ +hidden diff --git a/cmd/serve/http/testdata/files/hidden/file.txt b/cmd/serve/http/testdata/files/hidden/file.txt new file mode 100644 index 000000000..f4c22a0c9 --- /dev/null +++ b/cmd/serve/http/testdata/files/hidden/file.txt @@ -0,0 +1 @@ +hiddenfile diff --git a/cmd/serve/http/testdata/files/one%.txt b/cmd/serve/http/testdata/files/one%.txt new file mode 100644 index 000000000..07f46ed0f --- /dev/null +++ b/cmd/serve/http/testdata/files/one%.txt @@ -0,0 +1 @@ +one% diff --git a/cmd/serve/http/testdata/files/three/a.txt b/cmd/serve/http/testdata/files/three/a.txt new file mode 100644 index 000000000..2bdf67abb --- /dev/null +++ b/cmd/serve/http/testdata/files/three/a.txt @@ -0,0 +1 @@ +three diff --git a/cmd/serve/http/testdata/files/three/b.txt b/cmd/serve/http/testdata/files/three/b.txt new file mode 100644 index 000000000..510f0cb61 --- /dev/null +++ b/cmd/serve/http/testdata/files/three/b.txt @@ -0,0 +1 @@ +threeb diff --git a/cmd/serve/http/testdata/files/two.txt b/cmd/serve/http/testdata/files/two.txt new file mode 100644 index 000000000..f719efd43 --- /dev/null +++ b/cmd/serve/http/testdata/files/two.txt @@ -0,0 +1 @@ +two diff --git a/cmd/serve/http/testdata/golden/a.txt b/cmd/serve/http/testdata/golden/a.txt new file mode 100644 index 000000000..2bdf67abb --- /dev/null +++ b/cmd/serve/http/testdata/golden/a.txt @@ -0,0 +1 @@ +three diff --git a/cmd/serve/http/testdata/golden/dirnotfound.html b/cmd/serve/http/testdata/golden/dirnotfound.html new file mode 100644 index 000000000..4c30256ec --- /dev/null +++ b/cmd/serve/http/testdata/golden/dirnotfound.html @@ -0,0 +1 @@ +Directory not found diff --git a/cmd/serve/http/testdata/golden/hidden.txt b/cmd/serve/http/testdata/golden/hidden.txt new file mode 100644 index 000000000..99bc2a2c3 --- /dev/null +++ b/cmd/serve/http/testdata/golden/hidden.txt @@ -0,0 +1 @@ +File not found diff --git a/cmd/serve/http/testdata/golden/hiddendir.html b/cmd/serve/http/testdata/golden/hiddendir.html new file mode 100644 index 000000000..4c30256ec --- /dev/null +++ b/cmd/serve/http/testdata/golden/hiddendir.html @@ -0,0 +1 @@ +Directory not found diff --git a/cmd/serve/http/testdata/golden/index.html b/cmd/serve/http/testdata/golden/index.html new file mode 100644 index 000000000..8afb9b697 --- /dev/null +++ b/cmd/serve/http/testdata/golden/index.html @@ -0,0 +1,13 @@ + + + + +Directory listing of / + + +

Directory listing of /

+one%.txt
+three/
+two.txt
+ + diff --git a/cmd/serve/http/testdata/golden/indexhead.txt b/cmd/serve/http/testdata/golden/indexhead.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/serve/http/testdata/golden/indexpost.txt b/cmd/serve/http/testdata/golden/indexpost.txt new file mode 100644 index 000000000..85a67dcae --- /dev/null +++ b/cmd/serve/http/testdata/golden/indexpost.txt @@ -0,0 +1 @@ +Method not allowed diff --git a/cmd/serve/http/testdata/golden/notfound.html b/cmd/serve/http/testdata/golden/notfound.html new file mode 100644 index 000000000..99bc2a2c3 --- /dev/null +++ b/cmd/serve/http/testdata/golden/notfound.html @@ -0,0 +1 @@ +File not found diff --git a/cmd/serve/http/testdata/golden/one.txt b/cmd/serve/http/testdata/golden/one.txt new file mode 100644 index 000000000..07f46ed0f --- /dev/null +++ b/cmd/serve/http/testdata/golden/one.txt @@ -0,0 +1 @@ +one% diff --git a/cmd/serve/http/testdata/golden/onehead.txt b/cmd/serve/http/testdata/golden/onehead.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/serve/http/testdata/golden/onepost.txt b/cmd/serve/http/testdata/golden/onepost.txt new file mode 100644 index 000000000..85a67dcae --- /dev/null +++ b/cmd/serve/http/testdata/golden/onepost.txt @@ -0,0 +1 @@ +Method not allowed diff --git a/cmd/serve/http/testdata/golden/three.html b/cmd/serve/http/testdata/golden/three.html new file mode 100644 index 000000000..85ea95184 --- /dev/null +++ b/cmd/serve/http/testdata/golden/three.html @@ -0,0 +1,12 @@ + + + + +Directory listing of /three + + +

Directory listing of /three

+a.txt
+b.txt
+ + diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go new file mode 100644 index 000000000..ad99df14d --- /dev/null +++ b/cmd/serve/serve.go @@ -0,0 +1,33 @@ +package serve + +import ( + "errors" + + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/cmd/serve/http" + "github.com/spf13/cobra" +) + +func init() { + Command.AddCommand(http.Command) + cmd.Root.AddCommand(Command) +} + +// Command definition for cobra +var Command = &cobra.Command{ + Use: "serve [opts] ", + Short: `Serve a remote over a protocol.`, + Long: `rclone serve is used to serve a remote over a given protocol. This +command requires the use of a subcommand to specify the protocol, eg + + rclone serve http remote: + +Each subcommand has its own options which you can see in their help. +`, + RunE: func(command *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("serve requires a protocol, eg 'rclone serve http remote:'") + } + return errors.New("unknown protocol") + }, +}