Merge pull request #124 from mutantmonkey/cleanup_tool

Add linx-cleanup tool

.gitignore

@@ -29,3 +29,4 @@ _testmain.go
 linx-server
 files/
 meta/
+linx-cleanup

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:alpine
 
 RUN set -ex \
-        && apk add --no-cache --virtual .build-deps git mercurial \
+        && apk add --no-cache --virtual .build-deps git \
         && go get github.com/andreimarcu/linx-server \
         && apk del .build-deps
 

README.md

@@ -71,6 +71,23 @@ allowhotlink = true
 A helper utility ```linx-genkey``` is provided which hashes keys to the format required in the auth files.
 
 
+Cleaning up expired files
+-------------------------
+When files expire, access is disabled immediately, but the files and metadata
+will persist on disk until someone attempts to access them. If you'd like to
+automatically clean up files that have expired, you can use the included
+`linx-cleanup` utility. To run it automatically, use a cronjob or similar type
+of scheduled task.
+
+You should be careful to ensure that only one instance of `linx-client` runs at
+a time to avoid unexpected behavior. It does not implement any type of locking.
+
+#### Options
+- ```-filespath files/``` -- Path to stored uploads (default is files/)
+- ```-metapath meta/``` -- Path to stored information about uploads (default is meta/)
+- ```-nologs``` -- (optionally) disable deletion logs in stdout
+
+
 Deployment
 ----------
 Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/).

backends/localfs/localfs.go

@@ -65,6 +65,21 @@ func (b LocalfsBackend) Size(key string) (int64, error) {
 	return fileInfo.Size(), nil
 }
 
+func (b LocalfsBackend) List() ([]string, error) {
+	var output []string
+
+	files, err := ioutil.ReadDir(b.basePath)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		output = append(output, file.Name())
+	}
+
+	return output, nil
+}
+
 func NewLocalfsBackend(basePath string) LocalfsBackend {
 	return LocalfsBackend{basePath: basePath}
 }

backends/meta.go

@@ -0,0 +1,23 @@
+package backends
+
+import (
+	"errors"
+	"time"
+)
+
+type MetaBackend interface {
+	Get(key string) (Metadata, error)
+	Put(key string, metadata *Metadata) error
+}
+
+type Metadata struct {
+	DeleteKey    string
+	Sha256sum    string
+	Mimetype     string
+	Size         int64
+	Expiry       time.Time
+	ArchiveFiles []string
+	ShortURL     string
+}
+
+var BadMetadata = errors.New("Corrupted metadata.")

backends/metajson/metajson.go

@@ -0,0 +1,73 @@
+package metajson
+
+import (
+	"bytes"
+	"encoding/json"
+	"time"
+
+	"github.com/andreimarcu/linx-server/backends"
+)
+
+type MetadataJSON struct {
+	DeleteKey    string   `json:"delete_key"`
+	Sha256sum    string   `json:"sha256sum"`
+	Mimetype     string   `json:"mimetype"`
+	Size         int64    `json:"size"`
+	Expiry       int64    `json:"expiry"`
+	ArchiveFiles []string `json:"archive_files,omitempty"`
+	ShortURL     string   `json:"short_url"`
+}
+
+type MetaJSONBackend struct {
+	storage backends.MetaStorageBackend
+}
+
+func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error {
+	mjson := MetadataJSON{}
+	mjson.DeleteKey = metadata.DeleteKey
+	mjson.Mimetype = metadata.Mimetype
+	mjson.ArchiveFiles = metadata.ArchiveFiles
+	mjson.Sha256sum = metadata.Sha256sum
+	mjson.Expiry = metadata.Expiry.Unix()
+	mjson.Size = metadata.Size
+	mjson.ShortURL = metadata.ShortURL
+
+	byt, err := json.Marshal(mjson)
+	if err != nil {
+		return err
+	}
+
+	if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) {
+	b, err := m.storage.Get(key)
+	if err != nil {
+		return metadata, backends.BadMetadata
+	}
+
+	mjson := MetadataJSON{}
+
+	err = json.Unmarshal(b, &mjson)
+	if err != nil {
+		return metadata, backends.BadMetadata
+	}
+
+	metadata.DeleteKey = mjson.DeleteKey
+	metadata.Mimetype = mjson.Mimetype
+	metadata.ArchiveFiles = mjson.ArchiveFiles
+	metadata.Sha256sum = mjson.Sha256sum
+	metadata.Expiry = time.Unix(mjson.Expiry, 0)
+	metadata.Size = mjson.Size
+	metadata.ShortURL = mjson.ShortURL
+
+	return
+}
+
+func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend {
+	return MetaJSONBackend{storage: storage}
+}

backends/storage.go

@@ -21,3 +21,8 @@ type StorageBackend interface {
 	ServeFile(key string, w http.ResponseWriter, r *http.Request)
 	Size(key string) (int64, error)
 }
+
+type MetaStorageBackend interface {
+	StorageBackend
+	List() ([]string, error)
+}

delete.go

@@ -28,7 +28,7 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 
 	if metadata.DeleteKey == requestKey {
 		fileDelErr := fileBackend.Delete(filename)
-		metaDelErr := metaBackend.Delete(filename)
+		metaDelErr := metaStorageBackend.Delete(filename)
 
 		if (fileDelErr != nil) || (metaDelErr != nil) {
 			oopsHandler(c, w, r, RespPLAIN, "Could not delete")

display.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/andreimarcu/linx-server/expiry"
 	"github.com/dustin/go-humanize"
 	"github.com/flosch/pongo2"
 	"github.com/microcosm-cc/bluemonday"
@@ -32,7 +33,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	var expiryHuman string
-	if metadata.Expiry != neverExpire {
+	if metadata.Expiry != expiry.NeverExpire {
 		expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "")
 	}
 	sizeHuman := humanize.Bytes(uint64(metadata.Size))

expiry.go

@@ -3,6 +3,7 @@ package main
 import (
 	"time"
 
+	"github.com/andreimarcu/linx-server/expiry"
 	"github.com/dustin/go-humanize"
 )
 
@@ -21,14 +22,6 @@ type ExpirationTime struct {
 	Human   string
 }
 
-var neverExpire = time.Unix(0, 0)
-
-// Determine if a file with expiry set to "ts" has expired yet
-func isTsExpired(ts time.Time) bool {
-	now := time.Now()
-	return ts != neverExpire && now.After(ts)
-}
-
 // Determine if the given filename is expired
 func isFileExpired(filename string) (bool, error) {
 	metadata, err := metadataRead(filename)
@@ -36,7 +29,7 @@ func isFileExpired(filename string) (bool, error) {
 		return false, err
 	}
 
-	return isTsExpired(metadata.Expiry), nil
+	return expiry.IsTsExpired(metadata.Expiry), nil
 }
 
 // Return a list of expiration times and their humanized versions
@@ -45,16 +38,16 @@ func listExpirationTimes() []ExpirationTime {
 	actualExpiryInList := false
 	var expiryList []ExpirationTime
 
-	for _, expiry := range defaultExpiryList {
-		if Config.maxExpiry == 0 || expiry <= Config.maxExpiry {
-			if expiry == Config.maxExpiry {
+	for _, expiryEntry := range defaultExpiryList {
+		if Config.maxExpiry == 0 || expiryEntry <= Config.maxExpiry {
+			if expiryEntry == Config.maxExpiry {
 				actualExpiryInList = true
 			}
 
-			duration := time.Duration(expiry) * time.Second
+			duration := time.Duration(expiryEntry) * time.Second
 			expiryList = append(expiryList, ExpirationTime{
-				expiry,
-				humanize.RelTime(epoch, epoch.Add(duration), "", ""),
+				Seconds: expiryEntry,
+				Human:   humanize.RelTime(epoch, epoch.Add(duration), "", ""),
 			})
 		}
 	}

expiry/expiry.go

@@ -0,0 +1,13 @@
+package expiry
+
+import (
+	"time"
+)
+
+var NeverExpire = time.Unix(0, 0)
+
+// Determine if a file with expiry set to "ts" has expired yet
+func IsTsExpired(ts time.Time) bool {
+	now := time.Now()
+	return ts != NeverExpire && now.After(ts)
+}

fileserve.go

@@ -5,6 +5,7 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/andreimarcu/linx-server/backends"
 	"github.com/zenazn/goji/web"
 )
 
@@ -15,7 +16,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	if err == NotFoundErr {
 		notFoundHandler(c, w, r)
 		return
-	} else if err == BadMetadata {
+	} else if err == backends.BadMetadata {
 		oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
 		return
 	}
@@ -72,7 +73,7 @@ func checkFile(filename string) error {
 
 	if expired {
 		fileBackend.Delete(filename)
-		metaBackend.Delete(filename)
+		metaStorageBackend.Delete(filename)
 		return NotFoundErr
 	}
 

linx-cleanup/cleanup.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/andreimarcu/linx-server/backends/localfs"
+	"github.com/andreimarcu/linx-server/backends/metajson"
+	"github.com/andreimarcu/linx-server/expiry"
+)
+
+func main() {
+	var filesDir string
+	var metaDir string
+	var noLogs bool
+
+	flag.StringVar(&filesDir, "filespath", "files/",
+		"path to files directory")
+	flag.StringVar(&metaDir, "metapath", "meta/",
+		"path to metadata directory")
+	flag.BoolVar(&noLogs, "nologs", false,
+		"don't log deleted files")
+	flag.Parse()
+
+	metaStorageBackend := localfs.NewLocalfsBackend(metaDir)
+	metaBackend := metajson.NewMetaJSONBackend(metaStorageBackend)
+	fileBackend := localfs.NewLocalfsBackend(filesDir)
+
+	files, err := metaStorageBackend.List()
+	if err != nil {
+		panic(err)
+	}
+
+	for _, filename := range files {
+		metadata, err := metaBackend.Get(filename)
+		if err != nil {
+			log.Printf("Failed to find metadata for %s", filename)
+		}
+
+		if expiry.IsTsExpired(metadata.Expiry) {
+			log.Printf("Delete %s", filename)
+			fileBackend.Delete(filename)
+			metaStorageBackend.Delete(filename)
+		}
+	}
+}

meta.go

@@ -3,46 +3,25 @@ package main
 import (
 	"archive/tar"
 	"archive/zip"
-	"bytes"
 	"compress/bzip2"
 	"compress/gzip"
 	"crypto/sha256"
 	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"io"
 	"sort"
 	"time"
 	"unicode"
 
+	"github.com/andreimarcu/linx-server/backends"
+	"github.com/andreimarcu/linx-server/expiry"
 	"github.com/dchest/uniuri"
 	"gopkg.in/h2non/filetype.v1"
 )
 
-type MetadataJSON struct {
-	DeleteKey    string   `json:"delete_key"`
-	Sha256sum    string   `json:"sha256sum"`
-	Mimetype     string   `json:"mimetype"`
-	Size         int64    `json:"size"`
-	Expiry       int64    `json:"expiry"`
-	ArchiveFiles []string `json:"archive_files,omitempty"`
-	ShortURL     string   `json:"short_url"`
-}
-
-type Metadata struct {
-	DeleteKey    string
-	Sha256sum    string
-	Mimetype     string
-	Size         int64
-	Expiry       time.Time
-	ArchiveFiles []string
-	ShortURL     string
-}
-
 var NotFoundErr = errors.New("File not found.")
-var BadMetadata = errors.New("Corrupted metadata.")
 
-func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
+func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) {
 	file, err := fileBackend.Open(fName)
 	if err != nil {
 		return
@@ -145,58 +124,22 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e
 	return
 }
 
-func metadataWrite(filename string, metadata *Metadata) error {
-	mjson := MetadataJSON{}
-	mjson.DeleteKey = metadata.DeleteKey
-	mjson.Mimetype = metadata.Mimetype
-	mjson.ArchiveFiles = metadata.ArchiveFiles
-	mjson.Sha256sum = metadata.Sha256sum
-	mjson.Expiry = metadata.Expiry.Unix()
-	mjson.Size = metadata.Size
-	mjson.ShortURL = metadata.ShortURL
-
-	byt, err := json.Marshal(mjson)
-	if err != nil {
-		return err
+func metadataWrite(filename string, metadata *backends.Metadata) error {
+	return metaBackend.Put(filename, metadata)
 }
 
-	if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func metadataRead(filename string) (metadata Metadata, err error) {
-	b, err := metaBackend.Get(filename)
+func metadataRead(filename string) (metadata backends.Metadata, err error) {
+	metadata, err = metaBackend.Get(filename)
 	if err != nil {
 		// Metadata does not exist, generate one
-		newMData, err := generateMetadata(filename, neverExpire, "")
+		newMData, err := generateMetadata(filename, expiry.NeverExpire, "")
 		if err != nil {
 			return metadata, err
 		}
 		metadataWrite(filename, &newMData)
 
-		b, err = metaBackend.Get(filename)
-		if err != nil {
-			return metadata, BadMetadata
+		metadata, err = metaBackend.Get(filename)
 	}
-	}
-
-	mjson := MetadataJSON{}
-
-	err = json.Unmarshal(b, &mjson)
-	if err != nil {
-		return metadata, BadMetadata
-	}
-
-	metadata.DeleteKey = mjson.DeleteKey
-	metadata.Mimetype = mjson.Mimetype
-	metadata.ArchiveFiles = mjson.ArchiveFiles
-	metadata.Sha256sum = mjson.Sha256sum
-	metadata.Expiry = time.Unix(mjson.Expiry, 0)
-	metadata.Size = mjson.Size
-	metadata.ShortURL = mjson.ShortURL
 
 	return
 }

server.go

@@ -16,6 +16,7 @@ import (
 	"github.com/GeertJohan/go.rice"
 	"github.com/andreimarcu/linx-server/backends"
 	"github.com/andreimarcu/linx-server/backends/localfs"
+	"github.com/andreimarcu/linx-server/backends/metajson"
 	"github.com/flosch/pongo2"
 	"github.com/vharitonsky/iniflags"
 	"github.com/zenazn/goji/graceful"
@@ -65,7 +66,8 @@ var staticBox *rice.Box
 var timeStarted time.Time
 var timeStartedStr string
 var remoteAuthKeys []string
-var metaBackend backends.StorageBackend
+var metaStorageBackend backends.MetaStorageBackend
+var metaBackend backends.MetaBackend
 var fileBackend backends.StorageBackend
 
 func setup() *web.Mux {
@@ -124,7 +126,8 @@ func setup() *web.Mux {
 		Config.sitePath = "/"
 	}
 
-	metaBackend = localfs.NewLocalfsBackend(Config.metaDir)
+	metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir)
+	metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend)
 	fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
 
 	// Template setup

torrent.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/andreimarcu/linx-server/backends"
 	"github.com/zeebo/bencode"
 	"github.com/zenazn/goji/web"
 )
@@ -74,7 +75,7 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	if err == NotFoundErr {
 		notFoundHandler(c, w, r)
 		return
-	} else if err == BadMetadata {
+	} else if err == backends.BadMetadata {
 		oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
 		return
 	}

upload.go

@@ -15,6 +15,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/andreimarcu/linx-server/backends"
+	"github.com/andreimarcu/linx-server/expiry"
 	"github.com/dchest/uniuri"
 	"github.com/zenazn/goji/web"
 	"gopkg.in/h2non/filetype.v1"
@@ -41,7 +43,7 @@ type UploadRequest struct {
 // Metadata associated with a file as it would actually be stored
 type Upload struct {
 	Filename string // Final filename on disk
-	Metadata Metadata
+	Metadata backends.Metadata
 }
 
 func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@@ -258,11 +260,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
 	}
 
 	// Get the rest of the metadata needed for storage
-	var expiry time.Time
+	var fileExpiry time.Time
 	if upReq.expiry == 0 {
-		expiry = neverExpire
+		fileExpiry = expiry.NeverExpire
 	} else {
-		expiry = time.Now().Add(upReq.expiry)
+		fileExpiry = time.Now().Add(upReq.expiry)
 	}
 
 	bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
@@ -273,7 +275,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
 		return upload, errors.New("File too large")
 	}
 
-	upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
+	upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey)
 	if err != nil {
 		fileBackend.Delete(upload.Filename)
 		return
@@ -342,14 +344,14 @@ func parseExpiry(expStr string) time.Duration {
 	if expStr == "" {
 		return time.Duration(Config.maxExpiry) * time.Second
 	} else {
-		expiry, err := strconv.ParseUint(expStr, 10, 64)
+		fileExpiry, err := strconv.ParseUint(expStr, 10, 64)
 		if err != nil {
 			return time.Duration(Config.maxExpiry) * time.Second
 		} else {
-			if Config.maxExpiry > 0 && (expiry > Config.maxExpiry || expiry == 0) {
-				expiry = Config.maxExpiry
+			if Config.maxExpiry > 0 && (fileExpiry > Config.maxExpiry || fileExpiry == 0) {
+				fileExpiry = Config.maxExpiry
 			}
-			return time.Duration(expiry) * time.Second
+			return time.Duration(fileExpiry) * time.Second
 		}
 	}
 }