Merge pull request #84 from mutantmonkey/abstract_storage

use abstracted storage for flexibility

backends/backends.go

@@ -0,0 +1,23 @@
+package backends
+
+import (
+	"io"
+	"net/http"
+)
+
+type ReadSeekCloser interface {
+	io.Reader
+	io.Closer
+	io.Seeker
+	io.ReaderAt
+}
+
+type StorageBackend interface {
+	Delete(key string) error
+	Exists(key string) (bool, error)
+	Get(key string) ([]byte, error)
+	Put(key string, r io.Reader) (int64, error)
+	Open(key string) (ReadSeekCloser, error)
+	ServeFile(key string, w http.ResponseWriter, r *http.Request)
+	Size(key string) (int64, error)
+}

backends/localfs/localfs.go

@@ -0,0 +1,70 @@
+package localfs
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+
+	"github.com/andreimarcu/linx-server/backends"
+)
+
+type LocalfsBackend struct {
+	basePath string
+}
+
+func (b LocalfsBackend) Delete(key string) error {
+	return os.Remove(path.Join(b.basePath, key))
+}
+
+func (b LocalfsBackend) Exists(key string) (bool, error) {
+	_, err := os.Stat(path.Join(b.basePath, key))
+	return err == nil, err
+}
+
+func (b LocalfsBackend) Get(key string) ([]byte, error) {
+	return ioutil.ReadFile(path.Join(b.basePath, key))
+}
+
+func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) {
+	dst, err := os.Create(path.Join(b.basePath, key))
+	if err != nil {
+		return 0, err
+	}
+	defer dst.Close()
+
+	bytes, err := io.Copy(dst, r)
+	if bytes == 0 {
+		b.Delete(key)
+		return bytes, errors.New("Empty file")
+	} else if err != nil {
+		b.Delete(key)
+		return bytes, err
+	}
+
+	return bytes, err
+}
+
+func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) {
+	return os.Open(path.Join(b.basePath, key))
+}
+
+func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) {
+	filePath := path.Join(b.basePath, key)
+	http.ServeFile(w, r, filePath)
+}
+
+func (b LocalfsBackend) Size(key string) (int64, error) {
+	fileInfo, err := os.Stat(path.Join(b.basePath, key))
+	if err != nil {
+		return 0, err
+	}
+
+	return fileInfo.Size(), nil
+}
+
+func NewLocalfsBackend(basePath string) LocalfsBackend {
+	return LocalfsBackend{basePath: basePath}
+}

delete.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"path"
 
 	"github.com/zenazn/goji/web"
 )
@@ -13,11 +12,9 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	requestKey := r.Header.Get("Linx-Delete-Key")
 
 	filename := c.URLParams["name"]
-	filePath := path.Join(Config.filesDir, filename)
-	metaPath := path.Join(Config.metaDir, filename)
 
 	// Ensure requested file actually exists
-	if _, readErr := os.Stat(filePath); os.IsNotExist(readErr) {
+	if _, readErr := fileBackend.Exists(filename); os.IsNotExist(readErr) {
 		notFoundHandler(c, w, r) // 404 - file doesn't exist
 		return
 	}
@@ -30,8 +27,8 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	}
 
 	if metadata.DeleteKey == requestKey {
-		fileDelErr := os.Remove(filePath)
+		fileDelErr := fileBackend.Delete(filename)
-		metaDelErr := os.Remove(metaPath)
+		metaDelErr := metaBackend.Delete(filename)
 
 		if (fileDelErr != nil) || (metaDelErr != nil) {
 			oopsHandler(c, w, r, RespPLAIN, "Could not delete")

display.go

@@ -2,10 +2,7 @@ package main
 
 import (
 	"encoding/json"
-	"io/ioutil"
 	"net/http"
-	"os"
-	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -22,7 +19,6 @@ const maxDisplayFileSizeBytes = 1024 * 512
 
 func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	fileName := c.URLParams["name"]
-	filePath := path.Join(Config.filesDir, fileName)
 
 	err := checkFile(fileName)
 	if err == NotFoundErr {
@@ -43,7 +39,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	extra := make(map[string]string)
 	lines := []string{}
 
-	file, _ := os.Open(filePath)
+	file, _ := fileBackend.Open(fileName)
 	defer file.Close()
 
 	header := make([]byte, 512)
@@ -79,7 +75,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 
 	} else if extension == "story" {
 		if metadata.Size < maxDisplayFileSizeBytes {
-			bytes, err := ioutil.ReadFile(filePath)
+			bytes, err := fileBackend.Get(fileName)
 			if err == nil {
 				extra["contents"] = string(bytes)
 				lines = strings.Split(extra["contents"], "\n")
@@ -89,7 +85,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 
 	} else if extension == "md" {
 		if metadata.Size < maxDisplayFileSizeBytes {
-			bytes, err := ioutil.ReadFile(filePath)
+			bytes, err := fileBackend.Get(fileName)
 			if err == nil {
 				unsafe := blackfriday.MarkdownCommon(bytes)
 				html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
@@ -101,7 +97,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 
 	} else if strings.HasPrefix(metadata.Mimetype, "text/") || supportedBinExtension(extension) {
 		if metadata.Size < maxDisplayFileSizeBytes {
-			bytes, err := ioutil.ReadFile(filePath)
+			bytes, err := fileBackend.Get(fileName)
 			if err == nil {
 				extra["extension"] = extension
 				extra["lang_hl"], extra["lang_ace"] = extensionToHlAndAceLangs(extension)

fileserve.go

@@ -3,8 +3,6 @@ package main
 import (
 	"net/http"
 	"net/url"
-	"os"
-	"path"
 	"strings"
 
 	"github.com/zenazn/goji/web"
@@ -12,7 +10,6 @@ import (
 
 func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	fileName := c.URLParams["name"]
-	filePath := path.Join(Config.filesDir, fileName)
 
 	err := checkFile(fileName)
 	if err == NotFoundErr {
@@ -35,7 +32,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy)
 
-	http.ServeFile(w, r, filePath)
+	fileBackend.ServeFile(fileName, w, r)
 }
 
 func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@@ -63,9 +60,7 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 }
 
 func checkFile(filename string) error {
-	filePath := path.Join(Config.filesDir, filename)
+	_, err := fileBackend.Exists(filename)
-
-	_, err := os.Stat(filePath)
 	if err != nil {
 		return NotFoundErr
 	}
@@ -76,8 +71,8 @@ func checkFile(filename string) error {
 	}
 
 	if expired {
-		os.Remove(path.Join(Config.filesDir, filename))
+		fileBackend.Delete(filename)
-		os.Remove(path.Join(Config.metaDir, filename))
+		metaBackend.Delete(filename)
 		return NotFoundErr
 	}
 

meta.go

@@ -3,6 +3,7 @@ package main
 import (
 	"archive/tar"
 	"archive/zip"
+	"bytes"
 	"compress/bzip2"
 	"compress/gzip"
 	"crypto/sha256"
@@ -10,9 +11,6 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
-	"io/ioutil"
-	"os"
-	"path"
 	"sort"
 	"time"
 	"unicode"
@@ -43,14 +41,17 @@ var NotFoundErr = errors.New("File not found.")
 var BadMetadata = errors.New("Corrupted metadata.")
 
 func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
-	file, err := os.Open(path.Join(Config.filesDir, fName))
+	file, err := fileBackend.Open(fName)
-	fileInfo, err := os.Stat(path.Join(Config.filesDir, fName))
 	if err != nil {
 		return
 	}
 	defer file.Close()
 
-	m.Size = fileInfo.Size()
+	m.Size, err = fileBackend.Size(fName)
+	if err != nil {
+		return
+	}
+
 	m.Expiry = exp
 
 	if delKey == "" {
@@ -138,12 +139,6 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e
 }
 
 func metadataWrite(filename string, metadata *Metadata) error {
-	file, err := os.Create(path.Join(Config.metaDir, filename))
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
 	mjson := MetadataJSON{}
 	mjson.DeleteKey = metadata.DeleteKey
 	mjson.Mimetype = metadata.Mimetype
@@ -157,8 +152,7 @@ func metadataWrite(filename string, metadata *Metadata) error {
 		return err
 	}
 
-	_, err = file.Write(byt)
+	if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil {
-	if err != nil {
 		return err
 	}
 
@@ -166,7 +160,7 @@ func metadataWrite(filename string, metadata *Metadata) error {
 }
 
 func metadataRead(filename string) (metadata Metadata, err error) {
-	b, err := ioutil.ReadFile(path.Join(Config.metaDir, filename))
+	b, err := metaBackend.Get(filename)
 	if err != nil {
 		// Metadata does not exist, generate one
 		newMData, err := generateMetadata(filename, neverExpire, "")
@@ -174,7 +168,8 @@ func metadataRead(filename string) (metadata Metadata, err error) {
 			return metadata, err
 		}
 		metadataWrite(filename, &newMData)
-		b, err = ioutil.ReadFile(path.Join(Config.metaDir, filename))
+
+		b, err = metaBackend.Get(filename)
 		if err != nil {
 			return metadata, BadMetadata
 		}

server.go

@@ -14,6 +14,8 @@ import (
 	"time"
 
 	"github.com/GeertJohan/go.rice"
+	"github.com/andreimarcu/linx-server/backends"
+	"github.com/andreimarcu/linx-server/backends/localfs"
 	"github.com/flosch/pongo2"
 	"github.com/vharitonsky/iniflags"
 	"github.com/zenazn/goji/graceful"
@@ -61,6 +63,8 @@ var staticBox *rice.Box
 var timeStarted time.Time
 var timeStartedStr string
 var remoteAuthKeys []string
+var metaBackend backends.StorageBackend
+var fileBackend backends.StorageBackend
 
 func setup() *web.Mux {
 	mux := web.New()
@@ -118,6 +122,9 @@ func setup() *web.Mux {
 		Config.sitePath = "/"
 	}
 
+	metaBackend = localfs.NewLocalfsBackend(Config.metaDir)
+	fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
+
 	// Template setup
 	p2l, err := NewPongo2TemplatesLoader()
 	if err != nil {

torrent.go

@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"os"
-	"path"
 	"time"
 
 	"github.com/zeebo/bencode"
@@ -37,7 +35,7 @@ func hashPiece(piece []byte) []byte {
 	return h.Sum(nil)
 }
 
-func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, error) {
+func createTorrent(fileName string, f io.ReadCloser, r *http.Request) ([]byte, error) {
 	chunk := make([]byte, TORRENT_PIECE_LENGTH)
 
 	torrent := Torrent{
@@ -49,11 +47,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
 		UrlList: []string{fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName)},
 	}
 
-	f, err := os.Open(filePath)
-	if err != nil {
-		return []byte{}, err
-	}
-
 	for {
 		n, err := f.Read(chunk)
 		if err == io.EOF {
@@ -66,8 +59,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
 		torrent.Info.Pieces += string(hashPiece(chunk[:n]))
 	}
 
-	f.Close()
-
 	data, err := bencode.EncodeBytes(&torrent)
 	if err != nil {
 		return []byte{}, err
@@ -78,7 +69,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
 
 func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	fileName := c.URLParams["name"]
-	filePath := path.Join(Config.filesDir, fileName)
 
 	err := checkFile(fileName)
 	if err == NotFoundErr {
@@ -89,7 +79,14 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	encoded, err := createTorrent(fileName, filePath, r)
+	f, err := fileBackend.Open(fileName)
+	if err != nil {
+		oopsHandler(c, w, r, RespHTML, "Could not create torrent.")
+		return
+	}
+	defer f.Close()
+
+	encoded, err := createTorrent(fileName, f, r)
 	if err != nil {
 		oopsHandler(c, w, r, RespHTML, "Could not create torrent.")
 		return

torrent_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"testing"
 
 	"github.com/zeebo/bencode"
@@ -11,7 +12,13 @@ func TestCreateTorrent(t *testing.T) {
 	fileName := "server.go"
 	var decoded Torrent
 
-	encoded, err := createTorrent(fileName, fileName, nil)
+	f, err := os.Open("server.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	encoded, err := createTorrent(fileName, f, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -47,7 +54,13 @@ func TestCreateTorrent(t *testing.T) {
 func TestCreateTorrentWithImage(t *testing.T) {
 	var decoded Torrent
 
-	encoded, err := createTorrent("test.jpg", "static/images/404.jpg", nil)
+	f, err := os.Open("static/images/404.jpg")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	encoded, err := createTorrent("test.jpg", f, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

upload.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -232,9 +231,8 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
 
 	upload.Filename = strings.Join([]string{barename, extension}, ".")
 
-	_, err = os.Stat(path.Join(Config.filesDir, upload.Filename))
+	fileexists, _ := fileBackend.Exists(upload.Filename)
 
-	fileexists := err == nil
 	// Check if the delete key matches, in which case overwrite
 	if fileexists {
 		metad, merr := metadataRead(upload.Filename)
@@ -254,20 +252,13 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
 		}
 		upload.Filename = strings.Join([]string{barename, extension}, ".")
 
-		_, err = os.Stat(path.Join(Config.filesDir, upload.Filename))
+		fileexists, err = fileBackend.Exists(upload.Filename)
-		fileexists = err == nil
 	}
 
 	if fileBlacklist[strings.ToLower(upload.Filename)] {
 		return upload, errors.New("Prohibited filename")
 	}
 
-	dst, err := os.Create(path.Join(Config.filesDir, upload.Filename))
-	if err != nil {
-		return
-	}
-	defer dst.Close()
-
 	// Get the rest of the metadata needed for storage
 	var expiry time.Time
 	if upReq.expiry == 0 {
@@ -276,29 +267,22 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
 		expiry = time.Now().Add(upReq.expiry)
 	}
 
-	bytes, err := io.Copy(dst, io.MultiReader(bytes.NewReader(header), upReq.src))
+	bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
-	if bytes == 0 {
+	if err != nil {
-		os.Remove(path.Join(Config.filesDir, upload.Filename))
+		return upload, err
-		return upload, errors.New("Empty file")
-
-	} else if err != nil {
-		os.Remove(path.Join(Config.filesDir, upload.Filename))
-		return
 	} else if bytes > Config.maxSize {
-		os.Remove(path.Join(Config.filesDir, upload.Filename))
+		fileBackend.Delete(upload.Filename)
 		return upload, errors.New("File too large")
 	}
 
 	upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
 	if err != nil {
-		os.Remove(path.Join(Config.filesDir, upload.Filename))
+		fileBackend.Delete(upload.Filename)
-		os.Remove(path.Join(Config.metaDir, upload.Filename))
 		return
 	}
 	err = metadataWrite(upload.Filename, &upload.Metadata)
 	if err != nil {
-		os.Remove(path.Join(Config.filesDir, upload.Filename))
+		fileBackend.Delete(upload.Filename)
-		os.Remove(path.Join(Config.metaDir, upload.Filename))
 		return
 	}
 	return