From b7fadd96765fdcc45e023c6e4c60280b1b5906d5 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Mon, 1 May 2017 21:25:56 -0700 Subject: [PATCH 1/4] Add linx-cleanup tool This doesn't completely fix #116, but it makes setting up a cron job to do cleanup much more pleasant. --- .gitignore | 1 + backends/localfs/localfs.go | 15 ++++++ backends/meta.go | 23 +++++++++ backends/metajson/metajson.go | 73 +++++++++++++++++++++++++++ backends/{backends.go => storage.go} | 5 ++ delete.go | 2 +- display.go | 3 +- expiry.go | 23 +++------ expiry/expiry.go | 13 +++++ fileserve.go | 5 +- linx-cleanup/cleanup.go | 45 +++++++++++++++++ meta.go | 75 ++++------------------------ server.go | 7 ++- torrent.go | 3 +- upload.go | 20 ++++---- 15 files changed, 216 insertions(+), 97 deletions(-) create mode 100644 backends/meta.go create mode 100644 backends/metajson/metajson.go rename backends/{backends.go => storage.go} (84%) create mode 100644 expiry/expiry.go create mode 100644 linx-cleanup/cleanup.go diff --git a/.gitignore b/.gitignore index 79de414..bd9da44 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ _testmain.go linx-server files/ meta/ +linx-cleanup diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index 148cf2e..b55c986 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -65,6 +65,21 @@ func (b LocalfsBackend) Size(key string) (int64, error) { return fileInfo.Size(), nil } +func (b LocalfsBackend) List() ([]string, error) { + var output []string + + files, err := ioutil.ReadDir(b.basePath) + if err != nil { + return nil, err + } + + for _, file := range files { + output = append(output, file.Name()) + } + + return output, nil +} + func NewLocalfsBackend(basePath string) LocalfsBackend { return LocalfsBackend{basePath: basePath} } diff --git a/backends/meta.go b/backends/meta.go new file mode 100644 index 0000000..eb17d5e --- /dev/null +++ b/backends/meta.go @@ -0,0 +1,23 @@ +package backends + +import ( + "errors" + "time" +) + +type MetaBackend interface { + Get(key string) (Metadata, error) + Put(key string, metadata *Metadata) error +} + +type Metadata struct { + DeleteKey string + Sha256sum string + Mimetype string + Size int64 + Expiry time.Time + ArchiveFiles []string + ShortURL string +} + +var BadMetadata = errors.New("Corrupted metadata.") diff --git a/backends/metajson/metajson.go b/backends/metajson/metajson.go new file mode 100644 index 0000000..6e76dd4 --- /dev/null +++ b/backends/metajson/metajson.go @@ -0,0 +1,73 @@ +package metajson + +import ( + "bytes" + "encoding/json" + "time" + + "github.com/andreimarcu/linx-server/backends" +) + +type MetadataJSON struct { + DeleteKey string `json:"delete_key"` + Sha256sum string `json:"sha256sum"` + Mimetype string `json:"mimetype"` + Size int64 `json:"size"` + Expiry int64 `json:"expiry"` + ArchiveFiles []string `json:"archive_files,omitempty"` + ShortURL string `json:"short_url"` +} + +type MetaJSONBackend struct { + storage backends.MetaStorageBackend +} + +func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error { + mjson := MetadataJSON{} + mjson.DeleteKey = metadata.DeleteKey + mjson.Mimetype = metadata.Mimetype + mjson.ArchiveFiles = metadata.ArchiveFiles + mjson.Sha256sum = metadata.Sha256sum + mjson.Expiry = metadata.Expiry.Unix() + mjson.Size = metadata.Size + mjson.ShortURL = metadata.ShortURL + + byt, err := json.Marshal(mjson) + if err != nil { + return err + } + + if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil { + return err + } + + return nil +} + +func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) { + b, err := m.storage.Get(key) + if err != nil { + return metadata, backends.BadMetadata + } + + mjson := MetadataJSON{} + + err = json.Unmarshal(b, &mjson) + if err != nil { + return metadata, backends.BadMetadata + } + + metadata.DeleteKey = mjson.DeleteKey + metadata.Mimetype = mjson.Mimetype + metadata.ArchiveFiles = mjson.ArchiveFiles + metadata.Sha256sum = mjson.Sha256sum + metadata.Expiry = time.Unix(mjson.Expiry, 0) + metadata.Size = mjson.Size + metadata.ShortURL = mjson.ShortURL + + return +} + +func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend { + return MetaJSONBackend{storage: storage} +} diff --git a/backends/backends.go b/backends/storage.go similarity index 84% rename from backends/backends.go rename to backends/storage.go index 42a33f0..2b51a2c 100644 --- a/backends/backends.go +++ b/backends/storage.go @@ -21,3 +21,8 @@ type StorageBackend interface { ServeFile(key string, w http.ResponseWriter, r *http.Request) Size(key string) (int64, error) } + +type MetaStorageBackend interface { + StorageBackend + List() ([]string, error) +} diff --git a/delete.go b/delete.go index e42e623..61c6fa8 100644 --- a/delete.go +++ b/delete.go @@ -28,7 +28,7 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) { if metadata.DeleteKey == requestKey { fileDelErr := fileBackend.Delete(filename) - metaDelErr := metaBackend.Delete(filename) + metaDelErr := metaStorageBackend.Delete(filename) if (fileDelErr != nil) || (metaDelErr != nil) { oopsHandler(c, w, r, RespPLAIN, "Could not delete") diff --git a/display.go b/display.go index b196477..c6d8470 100644 --- a/display.go +++ b/display.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/andreimarcu/linx-server/expiry" "github.com/dustin/go-humanize" "github.com/flosch/pongo2" "github.com/microcosm-cc/bluemonday" @@ -32,7 +33,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { return } var expiryHuman string - if metadata.Expiry != neverExpire { + if metadata.Expiry != expiry.NeverExpire { expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "") } sizeHuman := humanize.Bytes(uint64(metadata.Size)) diff --git a/expiry.go b/expiry.go index 4172f51..6d8887d 100644 --- a/expiry.go +++ b/expiry.go @@ -3,6 +3,7 @@ package main import ( "time" + "github.com/andreimarcu/linx-server/expiry" "github.com/dustin/go-humanize" ) @@ -21,14 +22,6 @@ type ExpirationTime struct { Human string } -var neverExpire = time.Unix(0, 0) - -// Determine if a file with expiry set to "ts" has expired yet -func isTsExpired(ts time.Time) bool { - now := time.Now() - return ts != neverExpire && now.After(ts) -} - // Determine if the given filename is expired func isFileExpired(filename string) (bool, error) { metadata, err := metadataRead(filename) @@ -36,7 +29,7 @@ func isFileExpired(filename string) (bool, error) { return false, err } - return isTsExpired(metadata.Expiry), nil + return expiry.IsTsExpired(metadata.Expiry), nil } // Return a list of expiration times and their humanized versions @@ -45,16 +38,16 @@ func listExpirationTimes() []ExpirationTime { actualExpiryInList := false var expiryList []ExpirationTime - for _, expiry := range defaultExpiryList { - if Config.maxExpiry == 0 || expiry <= Config.maxExpiry { - if expiry == Config.maxExpiry { + for _, expiryEntry := range defaultExpiryList { + if Config.maxExpiry == 0 || expiryEntry <= Config.maxExpiry { + if expiryEntry == Config.maxExpiry { actualExpiryInList = true } - duration := time.Duration(expiry) * time.Second + duration := time.Duration(expiryEntry) * time.Second expiryList = append(expiryList, ExpirationTime{ - expiry, - humanize.RelTime(epoch, epoch.Add(duration), "", ""), + Seconds: expiryEntry, + Human: humanize.RelTime(epoch, epoch.Add(duration), "", ""), }) } } diff --git a/expiry/expiry.go b/expiry/expiry.go new file mode 100644 index 0000000..a1a4e54 --- /dev/null +++ b/expiry/expiry.go @@ -0,0 +1,13 @@ +package expiry + +import ( + "time" +) + +var NeverExpire = time.Unix(0, 0) + +// Determine if a file with expiry set to "ts" has expired yet +func IsTsExpired(ts time.Time) bool { + now := time.Now() + return ts != NeverExpire && now.After(ts) +} diff --git a/fileserve.go b/fileserve.go index 1a48a74..f7f87a6 100644 --- a/fileserve.go +++ b/fileserve.go @@ -5,6 +5,7 @@ import ( "net/url" "strings" + "github.com/andreimarcu/linx-server/backends" "github.com/zenazn/goji/web" ) @@ -15,7 +16,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { if err == NotFoundErr { notFoundHandler(c, w, r) return - } else if err == BadMetadata { + } else if err == backends.BadMetadata { oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") return } @@ -72,7 +73,7 @@ func checkFile(filename string) error { if expired { fileBackend.Delete(filename) - metaBackend.Delete(filename) + metaStorageBackend.Delete(filename) return NotFoundErr } diff --git a/linx-cleanup/cleanup.go b/linx-cleanup/cleanup.go new file mode 100644 index 0000000..7607fe6 --- /dev/null +++ b/linx-cleanup/cleanup.go @@ -0,0 +1,45 @@ +package main + +import ( + "flag" + "log" + + "github.com/andreimarcu/linx-server/backends/localfs" + "github.com/andreimarcu/linx-server/backends/metajson" + "github.com/andreimarcu/linx-server/expiry" +) + +func main() { + var filesDir string + var metaDir string + var noLogs bool + + flag.StringVar(&filesDir, "filespath", "files/", + "path to files directory") + flag.StringVar(&metaDir, "metapath", "meta/", + "path to metadata directory") + flag.BoolVar(&noLogs, "nologs", false, + "don't log deleted files") + + metaStorageBackend := localfs.NewLocalfsBackend(metaDir) + metaBackend := metajson.NewMetaJSONBackend(metaStorageBackend) + fileBackend := localfs.NewLocalfsBackend(filesDir) + + files, err := metaStorageBackend.List() + if err != nil { + panic(err) + } + + for _, filename := range files { + metadata, err := metaBackend.Get(filename) + if err != nil { + log.Printf("Failed to find metadata for %s", filename) + } + + if expiry.IsTsExpired(metadata.Expiry) { + log.Printf("Delete %s", filename) + fileBackend.Delete(filename) + metaStorageBackend.Delete(filename) + } + } +} diff --git a/meta.go b/meta.go index d2fb637..10cd4e1 100644 --- a/meta.go +++ b/meta.go @@ -3,46 +3,25 @@ package main import ( "archive/tar" "archive/zip" - "bytes" "compress/bzip2" "compress/gzip" "crypto/sha256" "encoding/hex" - "encoding/json" "errors" "io" "sort" "time" "unicode" + "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/expiry" "github.com/dchest/uniuri" "gopkg.in/h2non/filetype.v1" ) -type MetadataJSON struct { - DeleteKey string `json:"delete_key"` - Sha256sum string `json:"sha256sum"` - Mimetype string `json:"mimetype"` - Size int64 `json:"size"` - Expiry int64 `json:"expiry"` - ArchiveFiles []string `json:"archive_files,omitempty"` - ShortURL string `json:"short_url"` -} - -type Metadata struct { - DeleteKey string - Sha256sum string - Mimetype string - Size int64 - Expiry time.Time - ArchiveFiles []string - ShortURL string -} - var NotFoundErr = errors.New("File not found.") -var BadMetadata = errors.New("Corrupted metadata.") -func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) { +func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) { file, err := fileBackend.Open(fName) if err != nil { return @@ -145,59 +124,23 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e return } -func metadataWrite(filename string, metadata *Metadata) error { - mjson := MetadataJSON{} - mjson.DeleteKey = metadata.DeleteKey - mjson.Mimetype = metadata.Mimetype - mjson.ArchiveFiles = metadata.ArchiveFiles - mjson.Sha256sum = metadata.Sha256sum - mjson.Expiry = metadata.Expiry.Unix() - mjson.Size = metadata.Size - mjson.ShortURL = metadata.ShortURL - - byt, err := json.Marshal(mjson) - if err != nil { - return err - } - - if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil { - return err - } - - return nil +func metadataWrite(filename string, metadata *backends.Metadata) error { + return metaBackend.Put(filename, metadata) } -func metadataRead(filename string) (metadata Metadata, err error) { - b, err := metaBackend.Get(filename) +func metadataRead(filename string) (metadata backends.Metadata, err error) { + metadata, err = metaBackend.Get(filename) if err != nil { // Metadata does not exist, generate one - newMData, err := generateMetadata(filename, neverExpire, "") + newMData, err := generateMetadata(filename, expiry.NeverExpire, "") if err != nil { return metadata, err } metadataWrite(filename, &newMData) - b, err = metaBackend.Get(filename) - if err != nil { - return metadata, BadMetadata - } + metadata, err = metaBackend.Get(filename) } - mjson := MetadataJSON{} - - err = json.Unmarshal(b, &mjson) - if err != nil { - return metadata, BadMetadata - } - - metadata.DeleteKey = mjson.DeleteKey - metadata.Mimetype = mjson.Mimetype - metadata.ArchiveFiles = mjson.ArchiveFiles - metadata.Sha256sum = mjson.Sha256sum - metadata.Expiry = time.Unix(mjson.Expiry, 0) - metadata.Size = mjson.Size - metadata.ShortURL = mjson.ShortURL - return } diff --git a/server.go b/server.go index ac229fe..daaf89e 100644 --- a/server.go +++ b/server.go @@ -16,6 +16,7 @@ import ( "github.com/GeertJohan/go.rice" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends/localfs" + "github.com/andreimarcu/linx-server/backends/metajson" "github.com/flosch/pongo2" "github.com/vharitonsky/iniflags" "github.com/zenazn/goji/graceful" @@ -65,7 +66,8 @@ var staticBox *rice.Box var timeStarted time.Time var timeStartedStr string var remoteAuthKeys []string -var metaBackend backends.StorageBackend +var metaStorageBackend backends.MetaStorageBackend +var metaBackend backends.MetaBackend var fileBackend backends.StorageBackend func setup() *web.Mux { @@ -124,7 +126,8 @@ func setup() *web.Mux { Config.sitePath = "/" } - metaBackend = localfs.NewLocalfsBackend(Config.metaDir) + metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir) + metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend) fileBackend = localfs.NewLocalfsBackend(Config.filesDir) // Template setup diff --git a/torrent.go b/torrent.go index f0f4d48..23361b5 100644 --- a/torrent.go +++ b/torrent.go @@ -8,6 +8,7 @@ import ( "net/http" "time" + "github.com/andreimarcu/linx-server/backends" "github.com/zeebo/bencode" "github.com/zenazn/goji/web" ) @@ -74,7 +75,7 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { if err == NotFoundErr { notFoundHandler(c, w, r) return - } else if err == BadMetadata { + } else if err == backends.BadMetadata { oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") return } diff --git a/upload.go b/upload.go index 9cf8713..b7195c3 100644 --- a/upload.go +++ b/upload.go @@ -15,6 +15,8 @@ import ( "strings" "time" + "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/expiry" "github.com/dchest/uniuri" "github.com/zenazn/goji/web" "gopkg.in/h2non/filetype.v1" @@ -41,7 +43,7 @@ type UploadRequest struct { // Metadata associated with a file as it would actually be stored type Upload struct { Filename string // Final filename on disk - Metadata Metadata + Metadata backends.Metadata } func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { @@ -258,11 +260,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { } // Get the rest of the metadata needed for storage - var expiry time.Time + var fileExpiry time.Time if upReq.expiry == 0 { - expiry = neverExpire + fileExpiry = expiry.NeverExpire } else { - expiry = time.Now().Add(upReq.expiry) + fileExpiry = time.Now().Add(upReq.expiry) } bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src)) @@ -273,7 +275,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { return upload, errors.New("File too large") } - upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey) + upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey) if err != nil { fileBackend.Delete(upload.Filename) return @@ -342,14 +344,14 @@ func parseExpiry(expStr string) time.Duration { if expStr == "" { return time.Duration(Config.maxExpiry) * time.Second } else { - expiry, err := strconv.ParseUint(expStr, 10, 64) + fileExpiry, err := strconv.ParseUint(expStr, 10, 64) if err != nil { return time.Duration(Config.maxExpiry) * time.Second } else { - if Config.maxExpiry > 0 && (expiry > Config.maxExpiry || expiry == 0) { - expiry = Config.maxExpiry + if Config.maxExpiry > 0 && (fileExpiry > Config.maxExpiry || fileExpiry == 0) { + fileExpiry = Config.maxExpiry } - return time.Duration(expiry) * time.Second + return time.Duration(fileExpiry) * time.Second } } } From c6f62fccdf8cfa2926e4cd18eb11be225fa1e5f9 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Mon, 1 May 2017 21:49:27 -0700 Subject: [PATCH 2/4] Drop Mercurial from Dockerfile We no longer have any Mercurial dependencies, so we don't need to install it anymore. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6ff81bd..60e189d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:alpine RUN set -ex \ - && apk add --no-cache --virtual .build-deps git mercurial \ + && apk add --no-cache --virtual .build-deps git \ && go get github.com/andreimarcu/linx-server \ && apk del .build-deps From 32b537a057a3e1fe3c8632ba2c32c314fabd3deb Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Wed, 3 May 2017 21:12:41 -0700 Subject: [PATCH 3/4] Add missing `flag.Parse()` call --- linx-cleanup/cleanup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/linx-cleanup/cleanup.go b/linx-cleanup/cleanup.go index 7607fe6..a8dc37f 100644 --- a/linx-cleanup/cleanup.go +++ b/linx-cleanup/cleanup.go @@ -20,6 +20,7 @@ func main() { "path to metadata directory") flag.BoolVar(&noLogs, "nologs", false, "don't log deleted files") + flag.Parse() metaStorageBackend := localfs.NewLocalfsBackend(metaDir) metaBackend := metajson.NewMetaJSONBackend(metaStorageBackend) From a69aa95a87061e575fc5919a678029649fb85bbe Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Thu, 4 May 2017 21:43:52 -0700 Subject: [PATCH 4/4] Add `linx-cleanup` info to readme --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 1921855..7fa6b68 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,23 @@ allowhotlink = true A helper utility ```linx-genkey``` is provided which hashes keys to the format required in the auth files. +Cleaning up expired files +------------------------- +When files expire, access is disabled immediately, but the files and metadata +will persist on disk until someone attempts to access them. If you'd like to +automatically clean up files that have expired, you can use the included +`linx-cleanup` utility. To run it automatically, use a cronjob or similar type +of scheduled task. + +You should be careful to ensure that only one instance of `linx-client` runs at +a time to avoid unexpected behavior. It does not implement any type of locking. + +#### Options +- ```-filespath files/``` -- Path to stored uploads (default is files/) +- ```-metapath meta/``` -- Path to stored information about uploads (default is meta/) +- ```-nologs``` -- (optionally) disable deletion logs in stdout + + Deployment ---------- Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/).