Profouzors
/
ProfouzorsLinx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #84 from mutantmonkey/abstract_storage 

use abstracted storage for flexibility
This commit is contained in:
Andrei Marcu 2016-06-14 21:10:46 -07:00 committed by GitHub
parent 61eb32a79b fcd18eceec
commit 9c6088cfe5
10 changed files with 154 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
backends/backends.go Normal file
View File

@ -0,0 +1,23 @@
package backends
import (
"io"
"net/http"
)
type ReadSeekCloser interface {
io.Reader
io.Closer
io.Seeker
io.ReaderAt
}
type StorageBackend interface {
Delete(key string) error
Exists(key string) (bool, error)
Get(key string) ([]byte, error)
Put(key string, r io.Reader) (int64, error)
Open(key string) (ReadSeekCloser, error)
ServeFile(key string, w http.ResponseWriter, r *http.Request)
Size(key string) (int64, error)
}

70
backends/localfs/localfs.go Normal file
View File

@ -0,0 +1,70 @@
package localfs
import (
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"github.com/andreimarcu/linx-server/backends"
)
type LocalfsBackend struct {
basePath string
}
func (b LocalfsBackend) Delete(key string) error {
return os.Remove(path.Join(b.basePath, key))
}
func (b LocalfsBackend) Exists(key string) (bool, error) {
_, err := os.Stat(path.Join(b.basePath, key))
return err == nil, err
}
func (b LocalfsBackend) Get(key string) ([]byte, error) {
return ioutil.ReadFile(path.Join(b.basePath, key))
}
func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) {
dst, err := os.Create(path.Join(b.basePath, key))
if err != nil {
return 0, err
}
defer dst.Close()
bytes, err := io.Copy(dst, r)
if bytes == 0 {
b.Delete(key)
return bytes, errors.New("Empty file")
} else if err != nil {
b.Delete(key)
return bytes, err
}
return bytes, err
}
func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) {
return os.Open(path.Join(b.basePath, key))
}
func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) {
filePath := path.Join(b.basePath, key)
http.ServeFile(w, r, filePath)
}
func (b LocalfsBackend) Size(key string) (int64, error) {
fileInfo, err := os.Stat(path.Join(b.basePath, key))
if err != nil {
return 0, err
}
return fileInfo.Size(), nil
}
func NewLocalfsBackend(basePath string) LocalfsBackend {
return LocalfsBackend{basePath: basePath}
}

9
delete.go
View File

@ -4,7 +4,6 @@ import (
"fmt"
"net/http"
"os"
"path"
"github.com/zenazn/goji/web"
)
@ -13,11 +12,9 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
requestKey := r.Header.Get("Linx-Delete-Key")
filename := c.URLParams["name"]
filePath := path.Join(Config.filesDir, filename)
metaPath := path.Join(Config.metaDir, filename)
// Ensure requested file actually exists
if _, readErr := os.Stat(filePath); os.IsNotExist(readErr) {
if _, readErr := fileBackend.Exists(filename); os.IsNotExist(readErr) {
notFoundHandler(c, w, r) // 404 - file doesn't exist
return
}
@ -30,8 +27,8 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
if metadata.DeleteKey == requestKey {
fileDelErr := os.Remove(filePath)
metaDelErr := os.Remove(metaPath)
fileDelErr := fileBackend.Delete(filename)
metaDelErr := metaBackend.Delete(filename)
if (fileDelErr != nil) || (metaDelErr != nil) {
oopsHandler(c, w, r, RespPLAIN, "Could not delete")

12
display.go
View File

@ -2,10 +2,7 @@ package main
import (
"encoding/json"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
@ -22,7 +19,6 @@ const maxDisplayFileSizeBytes = 1024 * 512
func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
err := checkFile(fileName)
if err == NotFoundErr {
@ -43,7 +39,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
extra := make(map[string]string)
lines := []string{}
file, _ := os.Open(filePath)
file, _ := fileBackend.Open(fileName)
defer file.Close()
header := make([]byte, 512)
@ -79,7 +75,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} else if extension == "story" {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath)
bytes, err := fileBackend.Get(fileName)
if err == nil {
extra["contents"] = string(bytes)
lines = strings.Split(extra["contents"], "\n")
@ -89,7 +85,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} else if extension == "md" {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath)
bytes, err := fileBackend.Get(fileName)
if err == nil {
unsafe := blackfriday.MarkdownCommon(bytes)
html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
@ -101,7 +97,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} else if strings.HasPrefix(metadata.Mimetype, "text/") || supportedBinExtension(extension) {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath)
bytes, err := fileBackend.Get(fileName)
if err == nil {
extra["extension"] = extension
extra["lang_hl"], extra["lang_ace"] = extensionToHlAndAceLangs(extension)

13
fileserve.go
View File

@ -3,8 +3,6 @@ package main
import (
"net/http"
"net/url"
"os"
"path"
"strings"
"github.com/zenazn/goji/web"
@ -12,7 +10,6 @@ import (
func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
err := checkFile(fileName)
if err == NotFoundErr {
@ -35,7 +32,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy)
http.ServeFile(w, r, filePath)
fileBackend.ServeFile(fileName, w, r)
}
func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@ -63,9 +60,7 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
func checkFile(filename string) error {
filePath := path.Join(Config.filesDir, filename)
_, err := os.Stat(filePath)
_, err := fileBackend.Exists(filename)
if err != nil {
return NotFoundErr
}
@ -76,8 +71,8 @@ func checkFile(filename string) error {
}
if expired {
os.Remove(path.Join(Config.filesDir, filename))
os.Remove(path.Join(Config.metaDir, filename))
fileBackend.Delete(filename)
metaBackend.Delete(filename)
return NotFoundErr
}

27
meta.go
View File

@ -3,6 +3,7 @@ package main
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"crypto/sha256"
@ -10,9 +11,6 @@ import (
"encoding/json"
"errors"
"io"
"io/ioutil"
"os"
"path"
"sort"
"time"
"unicode"
@ -43,14 +41,17 @@ var NotFoundErr = errors.New("File not found.")
var BadMetadata = errors.New("Corrupted metadata.")
func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
file, err := os.Open(path.Join(Config.filesDir, fName))
fileInfo, err := os.Stat(path.Join(Config.filesDir, fName))
file, err := fileBackend.Open(fName)
if err != nil {
return
}
defer file.Close()
m.Size = fileInfo.Size()
m.Size, err = fileBackend.Size(fName)
if err != nil {
return
}
m.Expiry = exp
if delKey == "" {
@ -138,12 +139,6 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e
}
func metadataWrite(filename string, metadata *Metadata) error {
file, err := os.Create(path.Join(Config.metaDir, filename))
if err != nil {
return err
}
defer file.Close()
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
@ -157,8 +152,7 @@ func metadataWrite(filename string, metadata *Metadata) error {
return err
}
_, err = file.Write(byt)
if err != nil {
if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil {
return err
}
@ -166,7 +160,7 @@ func metadataWrite(filename string, metadata *Metadata) error {
}
func metadataRead(filename string) (metadata Metadata, err error) {
b, err := ioutil.ReadFile(path.Join(Config.metaDir, filename))
b, err := metaBackend.Get(filename)
if err != nil {
// Metadata does not exist, generate one
newMData, err := generateMetadata(filename, neverExpire, "")
@ -174,7 +168,8 @@ func metadataRead(filename string) (metadata Metadata, err error) {
return metadata, err
}
metadataWrite(filename, &newMData)
b, err = ioutil.ReadFile(path.Join(Config.metaDir, filename))
b, err = metaBackend.Get(filename)
if err != nil {
return metadata, BadMetadata
}

7
server.go
View File

@ -14,6 +14,8 @@ import (
"time"
"github.com/GeertJohan/go.rice"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/flosch/pongo2"
"github.com/vharitonsky/iniflags"
"github.com/zenazn/goji/graceful"
@ -61,6 +63,8 @@ var staticBox *rice.Box
var timeStarted time.Time
var timeStartedStr string
var remoteAuthKeys []string
var metaBackend backends.StorageBackend
var fileBackend backends.StorageBackend
func setup() *web.Mux {
mux := web.New()
@ -118,6 +122,9 @@ func setup() *web.Mux {
Config.sitePath = "/"
}
metaBackend = localfs.NewLocalfsBackend(Config.metaDir)
fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
// Template setup
p2l, err := NewPongo2TemplatesLoader()
if err != nil {

21
torrent.go
View File

@ -6,8 +6,6 @@ import (
"fmt"
"io"
"net/http"
"os"
"path"
"time"
"github.com/zeebo/bencode"
@ -37,7 +35,7 @@ func hashPiece(piece []byte) []byte {
return h.Sum(nil)
}
func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, error) {
func createTorrent(fileName string, f io.ReadCloser, r *http.Request) ([]byte, error) {
chunk := make([]byte, TORRENT_PIECE_LENGTH)
torrent := Torrent{
@ -49,11 +47,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
UrlList: []string{fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName)},
}
f, err := os.Open(filePath)
if err != nil {
return []byte{}, err
}
for {
n, err := f.Read(chunk)
if err == io.EOF {
@ -66,8 +59,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
torrent.Info.Pieces += string(hashPiece(chunk[:n]))
}
f.Close()
data, err := bencode.EncodeBytes(&torrent)
if err != nil {
return []byte{}, err
@ -78,7 +69,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
err := checkFile(fileName)
if err == NotFoundErr {
@ -89,7 +79,14 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
return
}
encoded, err := createTorrent(fileName, filePath, r)
f, err := fileBackend.Open(fileName)
if err != nil {
oopsHandler(c, w, r, RespHTML, "Could not create torrent.")
return
}
defer f.Close()
encoded, err := createTorrent(fileName, f, r)
if err != nil {
oopsHandler(c, w, r, RespHTML, "Could not create torrent.")
return

17
torrent_test.go
View File

@ -2,6 +2,7 @@ package main
import (
"fmt"
"os"
"testing"
"github.com/zeebo/bencode"
@ -11,7 +12,13 @@ func TestCreateTorrent(t *testing.T) {
fileName := "server.go"
var decoded Torrent
encoded, err := createTorrent(fileName, fileName, nil)
f, err := os.Open("server.go")
if err != nil {
t.Fatal(err)
}
defer f.Close()
encoded, err := createTorrent(fileName, f, nil)
if err != nil {
t.Fatal(err)
}
@ -47,7 +54,13 @@ func TestCreateTorrent(t *testing.T) {
func TestCreateTorrentWithImage(t *testing.T) {
var decoded Torrent
encoded, err := createTorrent("test.jpg", "static/images/404.jpg", nil)
f, err := os.Open("static/images/404.jpg")
if err != nil {
t.Fatal(err)
}
defer f.Close()
encoded, err := createTorrent("test.jpg", f, nil)
if err != nil {
t.Fatal(err)
}

32
upload.go
View File

@ -8,7 +8,6 @@ import (
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
@ -232,9 +231,8 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
upload.Filename = strings.Join([]string{barename, extension}, ".")
_, err = os.Stat(path.Join(Config.filesDir, upload.Filename))
fileexists, _ := fileBackend.Exists(upload.Filename)
fileexists := err == nil
// Check if the delete key matches, in which case overwrite
if fileexists {
metad, merr := metadataRead(upload.Filename)
@ -254,20 +252,13 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
}
upload.Filename = strings.Join([]string{barename, extension}, ".")
_, err = os.Stat(path.Join(Config.filesDir, upload.Filename))
fileexists = err == nil
fileexists, err = fileBackend.Exists(upload.Filename)
}
if fileBlacklist[strings.ToLower(upload.Filename)] {
return upload, errors.New("Prohibited filename")
}
dst, err := os.Create(path.Join(Config.filesDir, upload.Filename))
if err != nil {
return
}
defer dst.Close()
// Get the rest of the metadata needed for storage
var expiry time.Time
if upReq.expiry == 0 {
@ -276,29 +267,22 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
expiry = time.Now().Add(upReq.expiry)
}
bytes, err := io.Copy(dst, io.MultiReader(bytes.NewReader(header), upReq.src))
if bytes == 0 {
os.Remove(path.Join(Config.filesDir, upload.Filename))
return upload, errors.New("Empty file")
} else if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
return
bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
if err != nil {
return upload, err
} else if bytes > Config.maxSize {
os.Remove(path.Join(Config.filesDir, upload.Filename))
fileBackend.Delete(upload.Filename)
return upload, errors.New("File too large")
}
upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
fileBackend.Delete(upload.Filename)
return
}
err = metadataWrite(upload.Filename, &upload.Metadata)
if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
fileBackend.Delete(upload.Filename)
return
}
return