mirror of https://github.com/rclone/rclone.git
serve http command to serve a remote over HTTP
This implements a basic webserver to serve an rclone remote over HTTP. It also sets up the framework for adding more types of server later.
This commit is contained in:
Nick Craig-Wood
parent
a06bcd4c57
commit
6669165b6b
|
|
@ -38,6 +38,7 @@ import (
|
||||||
_ "github.com/ncw/rclone/cmd/rcat"
|
_ "github.com/ncw/rclone/cmd/rcat"
|
||||||
_ "github.com/ncw/rclone/cmd/rmdir"
|
_ "github.com/ncw/rclone/cmd/rmdir"
|
||||||
_ "github.com/ncw/rclone/cmd/rmdirs"
|
_ "github.com/ncw/rclone/cmd/rmdirs"
|
||||||
|
_ "github.com/ncw/rclone/cmd/serve"
|
||||||
_ "github.com/ncw/rclone/cmd/sha1sum"
|
_ "github.com/ncw/rclone/cmd/sha1sum"
|
||||||
_ "github.com/ncw/rclone/cmd/size"
|
_ "github.com/ncw/rclone/cmd/size"
|
||||||
_ "github.com/ncw/rclone/cmd/sync"
|
_ "github.com/ncw/rclone/cmd/sync"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,251 @@
|
||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"html/template"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"path"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/ncw/rclone/cmd"
|
||||||
|
"github.com/ncw/rclone/fs"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Globals
|
||||||
|
var (
|
||||||
|
bindAddress = "localhost:8080"
|
||||||
|
readWrite = false
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
Command.Flags().StringVarP(&bindAddress, "addr", "", bindAddress, "IPaddress:Port to bind server to.")
|
||||||
|
// Command.Flags().BoolVarP(&readWrite, "rw", "", readWrite, "Serve in read/write mode.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Command definition for cobra
|
||||||
|
var Command = &cobra.Command{
|
||||||
|
Use: "http remote:path",
|
||||||
|
Short: `Serve the remote over HTTP.`,
|
||||||
|
Long: `rclone serve http implements a basic web server to serve the remote
|
||||||
|
over HTTP. This can be viewed in a web browser or you can make a
|
||||||
|
remote of type http read from it.
|
||||||
|
|
||||||
|
Use --addr to specify which IP address and port the server should
|
||||||
|
listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
|
||||||
|
IPs. By default it only listens on localhost.
|
||||||
|
|
||||||
|
You can use the filter flags (eg --include, --exclude) to control what
|
||||||
|
is served.
|
||||||
|
|
||||||
|
The server will log errors. Use -v to see access logs.
|
||||||
|
|
||||||
|
--bwlimit will be respected for file transfers. Use --stats to
|
||||||
|
control the stats printing.
|
||||||
|
`,
|
||||||
|
Run: func(command *cobra.Command, args []string) {
|
||||||
|
cmd.CheckArgs(1, 1, command, args)
|
||||||
|
f := cmd.NewFsSrc(args)
|
||||||
|
cmd.Run(false, true, command, func() error {
|
||||||
|
s := server{
|
||||||
|
f: f,
|
||||||
|
bindAddress: bindAddress,
|
||||||
|
readWrite: readWrite,
|
||||||
|
}
|
||||||
|
s.serve()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// server contains everything to run the server
|
||||||
|
type server struct {
|
||||||
|
f fs.Fs
|
||||||
|
bindAddress string
|
||||||
|
readWrite bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve creates the http server
|
||||||
|
func (s *server) serve() {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.HandleFunc("/", s.handler)
|
||||||
|
// FIXME make a transport?
|
||||||
|
httpServer := &http.Server{
|
||||||
|
Addr: s.bindAddress,
|
||||||
|
Handler: mux,
|
||||||
|
ReadTimeout: 10 * time.Second,
|
||||||
|
WriteTimeout: 10 * time.Second,
|
||||||
|
MaxHeaderBytes: 1 << 20,
|
||||||
|
}
|
||||||
|
fs.Logf(s.f, "Serving on http://%s/", bindAddress)
|
||||||
|
log.Fatal(httpServer.ListenAndServe())
|
||||||
|
}
|
||||||
|
|
||||||
|
// handler reads incoming requests and dispatches them
|
||||||
|
func (s *server) handler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != "GET" && r.Method != "HEAD" {
|
||||||
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
urlPath := r.URL.Path
|
||||||
|
isDir := strings.HasSuffix(urlPath, "/")
|
||||||
|
remote := strings.Trim(urlPath, "/")
|
||||||
|
if isDir {
|
||||||
|
s.serveDir(w, r, remote)
|
||||||
|
} else {
|
||||||
|
s.serveFile(w, r, remote)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// entry is a directory entry
|
||||||
|
type entry struct {
|
||||||
|
remote string
|
||||||
|
URL string
|
||||||
|
Leaf string
|
||||||
|
}
|
||||||
|
|
||||||
|
// entries represents a directory
|
||||||
|
type entries []entry
|
||||||
|
|
||||||
|
// indexPage is a directory listing template
|
||||||
|
var indexPage = `<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>{{ .Title }}</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>{{ .Title }}</h1>
|
||||||
|
{{ range $i := .Entries }}<a href="{{ $i.URL }}">{{ $i.Leaf }}</a><br />
|
||||||
|
{{ end }}</body>
|
||||||
|
</html>
|
||||||
|
`
|
||||||
|
|
||||||
|
// indexTemplate is the instantiated indexPage
|
||||||
|
var indexTemplate = template.Must(template.New("index").Parse(indexPage))
|
||||||
|
|
||||||
|
// indexData is used to fill in the indexTemplate
|
||||||
|
type indexData struct {
|
||||||
|
Title string
|
||||||
|
Entries entries
|
||||||
|
}
|
||||||
|
|
||||||
|
// error returns an http.StatusInternalServerError and logs the error
|
||||||
|
func internalError(what interface{}, w http.ResponseWriter, text string, err error) {
|
||||||
|
fs.Stats.Error()
|
||||||
|
fs.Errorf(what, "%s: %v", text, err)
|
||||||
|
http.Error(w, text+".", http.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// serveDir serves a directory index at dirRemote
|
||||||
|
func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
|
||||||
|
// Check the directory is included in the filters
|
||||||
|
if !fs.Config.Filter.IncludeDirectory(dirRemote) {
|
||||||
|
fs.Infof(dirRemote, "%s: Directory not found (filtered)", r.RemoteAddr)
|
||||||
|
http.Error(w, "Directory not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// List the directory
|
||||||
|
dirEntries, err := fs.ListDirSorted(s.f, false, dirRemote)
|
||||||
|
if err == fs.ErrorDirNotFound {
|
||||||
|
fs.Infof(dirRemote, "%s: Directory not found", r.RemoteAddr)
|
||||||
|
http.Error(w, "Directory not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
} else if err != nil {
|
||||||
|
internalError(dirRemote, w, "Failed to list directory", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var out entries
|
||||||
|
for _, o := range dirEntries {
|
||||||
|
remote := strings.Trim(o.Remote(), "/")
|
||||||
|
leaf := path.Base(remote)
|
||||||
|
urlRemote := leaf
|
||||||
|
if _, ok := o.(*fs.Dir); ok {
|
||||||
|
leaf += "/"
|
||||||
|
urlRemote += "/"
|
||||||
|
}
|
||||||
|
out = append(out, entry{remote: remote, URL: urlRemote, Leaf: leaf})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Account the transfer
|
||||||
|
fs.Stats.Transferring(dirRemote)
|
||||||
|
defer fs.Stats.DoneTransferring(dirRemote, true)
|
||||||
|
|
||||||
|
fs.Infof(dirRemote, "%s: Serving directory", r.RemoteAddr)
|
||||||
|
err = indexTemplate.Execute(w, indexData{
|
||||||
|
Entries: out,
|
||||||
|
Title: fmt.Sprintf("Directory listing of /%s", dirRemote),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
internalError(dirRemote, w, "Failed to render template", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// serveFile serves a file object at remote
|
||||||
|
func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
|
||||||
|
// FIXME could cache the directories and objects...
|
||||||
|
obj, err := s.f.NewObject(remote)
|
||||||
|
if err == fs.ErrorObjectNotFound {
|
||||||
|
fs.Infof(remote, "%s: File not found", r.RemoteAddr)
|
||||||
|
http.Error(w, "File not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
} else if err != nil {
|
||||||
|
internalError(remote, w, "Failed to find file", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check the object is included in the filters
|
||||||
|
if !fs.Config.Filter.IncludeObject(obj) {
|
||||||
|
fs.Infof(remote, "%s: File not found (filtered)", r.RemoteAddr)
|
||||||
|
http.Error(w, "File not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set content length since we know how long the object is
|
||||||
|
w.Header().Set("Content-Length", strconv.FormatInt(obj.Size(), 10))
|
||||||
|
|
||||||
|
// Set content type
|
||||||
|
mimeType := fs.MimeType(obj)
|
||||||
|
if mimeType == "application/octet-stream" && path.Ext(remote) == "" {
|
||||||
|
// Leave header blank so http server guesses
|
||||||
|
} else {
|
||||||
|
w.Header().Set("Content-Type", mimeType)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If HEAD no need to read the object since we have set the headers
|
||||||
|
if r.Method == "HEAD" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// open the object
|
||||||
|
in, err := obj.Open()
|
||||||
|
if err != nil {
|
||||||
|
internalError(remote, w, "Failed to open file", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
err := in.Close()
|
||||||
|
if err != nil {
|
||||||
|
fs.Errorf(remote, "Failed to close file: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Account the transfer
|
||||||
|
fs.Stats.Transferring(remote)
|
||||||
|
defer fs.Stats.DoneTransferring(remote, true)
|
||||||
|
in = fs.NewAccount(in, obj).WithBuffer() // account the transfer
|
||||||
|
|
||||||
|
// Copy the contents of the object to the output
|
||||||
|
fs.Infof(remote, "%s: Serving file", r.RemoteAddr)
|
||||||
|
_, err = io.Copy(w, in)
|
||||||
|
if err != nil {
|
||||||
|
fs.Errorf(remote, "Failed to write file: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,170 @@
|
||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"io/ioutil"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/ncw/rclone/fs"
|
||||||
|
_ "github.com/ncw/rclone/local"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
var updateGolden = flag.Bool("updategolden", false, "update golden files for regression test")
|
||||||
|
|
||||||
|
const (
|
||||||
|
testBindAddress = "localhost:51777"
|
||||||
|
testURL = "http://" + testBindAddress + "/"
|
||||||
|
)
|
||||||
|
|
||||||
|
func startServer(t *testing.T, f fs.Fs) {
|
||||||
|
s := server{
|
||||||
|
f: f,
|
||||||
|
bindAddress: testBindAddress,
|
||||||
|
readWrite: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
go s.serve()
|
||||||
|
|
||||||
|
// try to connect to the test server
|
||||||
|
pause := time.Millisecond
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
conn, err := net.Dial("tcp", testBindAddress)
|
||||||
|
if err == nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// t.Logf("couldn't connect, sleeping for %v: %v", pause, err)
|
||||||
|
time.Sleep(pause)
|
||||||
|
pause *= 2
|
||||||
|
}
|
||||||
|
t.Fatal("couldn't connect to server")
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInit(t *testing.T) {
|
||||||
|
// Configure the remote
|
||||||
|
fs.LoadConfig()
|
||||||
|
// fs.Config.LogLevel = fs.LogLevelDebug
|
||||||
|
// fs.Config.DumpHeaders = true
|
||||||
|
// fs.Config.DumpBodies = true
|
||||||
|
|
||||||
|
// exclude files called hidden.txt and directories called hidden
|
||||||
|
require.NoError(t, fs.Config.Filter.AddRule("- hidden.txt"))
|
||||||
|
require.NoError(t, fs.Config.Filter.AddRule("- hidden/**"))
|
||||||
|
|
||||||
|
// Create a test Fs
|
||||||
|
f, err := fs.NewFs("testdata/files")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
startServer(t, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// check body against the file, or re-write body if -updategolden is
|
||||||
|
// set.
|
||||||
|
func checkGolden(t *testing.T, fileName string, got []byte) {
|
||||||
|
if *updateGolden {
|
||||||
|
t.Logf("Updating golden file %q", fileName)
|
||||||
|
err := ioutil.WriteFile(fileName, got, 0666)
|
||||||
|
require.NoError(t, err)
|
||||||
|
} else {
|
||||||
|
want, err := ioutil.ReadFile(fileName)
|
||||||
|
require.NoError(t, err)
|
||||||
|
wants := strings.Split(string(want), "\n")
|
||||||
|
gots := strings.Split(string(got), "\n")
|
||||||
|
assert.Equal(t, wants, gots, fileName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGets(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
URL string
|
||||||
|
Status int
|
||||||
|
Golden string
|
||||||
|
Method string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
URL: "",
|
||||||
|
Status: http.StatusOK,
|
||||||
|
Golden: "testdata/golden/index.html",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "notfound",
|
||||||
|
Status: http.StatusNotFound,
|
||||||
|
Golden: "testdata/golden/notfound.html",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "dirnotfound/",
|
||||||
|
Status: http.StatusNotFound,
|
||||||
|
Golden: "testdata/golden/dirnotfound.html",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "hidden/",
|
||||||
|
Status: http.StatusNotFound,
|
||||||
|
Golden: "testdata/golden/hiddendir.html",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "one%25.txt",
|
||||||
|
Status: http.StatusOK,
|
||||||
|
Golden: "testdata/golden/one.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "hidden.txt",
|
||||||
|
Status: http.StatusNotFound,
|
||||||
|
Golden: "testdata/golden/hidden.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "three/",
|
||||||
|
Status: http.StatusOK,
|
||||||
|
Golden: "testdata/golden/three.html",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "three/a.txt",
|
||||||
|
Status: http.StatusOK,
|
||||||
|
Golden: "testdata/golden/a.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "",
|
||||||
|
Method: "HEAD",
|
||||||
|
Status: http.StatusOK,
|
||||||
|
Golden: "testdata/golden/indexhead.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "one%25.txt",
|
||||||
|
Method: "HEAD",
|
||||||
|
Status: http.StatusOK,
|
||||||
|
Golden: "testdata/golden/onehead.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "",
|
||||||
|
Method: "POST",
|
||||||
|
Status: http.StatusMethodNotAllowed,
|
||||||
|
Golden: "testdata/golden/indexpost.txt",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
URL: "one%25.txt",
|
||||||
|
Method: "POST",
|
||||||
|
Status: http.StatusMethodNotAllowed,
|
||||||
|
Golden: "testdata/golden/onepost.txt",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
method := test.Method
|
||||||
|
if method == "" {
|
||||||
|
method = "GET"
|
||||||
|
}
|
||||||
|
req, err := http.NewRequest(method, testURL+test.URL, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, test.Status, resp.StatusCode, test.Golden)
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
checkGolden(t, test.Golden, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
hidden
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
hiddenfile
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
one%
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
three
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
threeb
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
two
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
three
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
Directory not found
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
File not found
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
Directory not found
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>Directory listing of /</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Directory listing of /</h1>
|
||||||
|
<a href="one%25.txt">one%.txt</a><br />
|
||||||
|
<a href="three/">three/</a><br />
|
||||||
|
<a href="two.txt">two.txt</a><br />
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
Method not allowed
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
File not found
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
one%
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
Method not allowed
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>Directory listing of /three</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Directory listing of /three</h1>
|
||||||
|
<a href="a.txt">a.txt</a><br />
|
||||||
|
<a href="b.txt">b.txt</a><br />
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
package serve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"github.com/ncw/rclone/cmd"
|
||||||
|
"github.com/ncw/rclone/cmd/serve/http"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
Command.AddCommand(http.Command)
|
||||||
|
cmd.Root.AddCommand(Command)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Command definition for cobra
|
||||||
|
var Command = &cobra.Command{
|
||||||
|
Use: "serve <protocol> [opts] <remote>",
|
||||||
|
Short: `Serve a remote over a protocol.`,
|
||||||
|
Long: `rclone serve is used to serve a remote over a given protocol. This
|
||||||
|
command requires the use of a subcommand to specify the protocol, eg
|
||||||
|
|
||||||
|
rclone serve http remote:
|
||||||
|
|
||||||
|
Each subcommand has its own options which you can see in their help.
|
||||||
|
`,
|
||||||
|
RunE: func(command *cobra.Command, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New("serve requires a protocol, eg 'rclone serve http remote:'")
|
||||||
|
}
|
||||||
|
return errors.New("unknown protocol")
|
||||||
|
},
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue