From 4a9a4f7be2084cf8019c45136df0520a5b407602 Mon Sep 17 00:00:00 2001 From: BBaoVanC Date: Thu, 23 Dec 2021 18:24:50 -0600 Subject: [PATCH] Add check for locks in cleanup.go --- backends/localfs/localfs.go | 13 +++++++++++++ backends/s3/s3.go | 5 +++++ backends/storage.go | 2 +- cleanup/cleanup.go | 11 +++++++++++ linx-cleanup/linx-cleanup.go | 3 +++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index bd68d20..9a72891 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -2,6 +2,7 @@ package localfs import ( "encoding/json" + "errors" "io" "io/ioutil" "net/http" @@ -151,6 +152,18 @@ func (b LocalfsBackend) Unlock(filename string) (err error) { return } +func (b LocalfsBackend) CheckLock(filename string) (locked bool, err error) { + lockPath := path.Join(b.locksPath, filename) + + if _, err := os.Stat(lockPath); errors.Is(err, os.ErrNotExist) { + return false, nil + } else { + return true, nil + } + + return false, err +} + func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (m backends.Metadata, err error) { filePath := path.Join(b.filesPath, key) diff --git a/backends/s3/s3.go b/backends/s3/s3.go index b48837f..17f6725 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -167,6 +167,11 @@ func (b S3Backend) Unlock(filename string) (err error) { return } +func (b S3Backend) CheckLock(filename string) (locked bool, err error) { + log.Printf("Locking is not supported on S3") + return +} + func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (m backends.Metadata, err error) { tmpDst, err := ioutil.TempFile("", "linx-server-upload") if err != nil { diff --git a/backends/storage.go b/backends/storage.go index 119ddb6..02e2a57 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -14,6 +14,7 @@ type StorageBackend interface { Get(key string) (Metadata, io.ReadCloser, error) Lock(filename string) (error) Unlock(filename string) (error) + CheckLock(filename string) (bool, error) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (Metadata, error) PutMetadata(key string, m Metadata) error ServeFile(key string, w http.ResponseWriter, r *http.Request) error @@ -27,4 +28,3 @@ type MetaStorageBackend interface { var NotFoundErr = errors.New("File not found.") var FileEmptyError = errors.New("Empty file") -var FileLockedError = errors.New("Locked file") diff --git a/cleanup/cleanup.go b/cleanup/cleanup.go index 6230b00..97789d5 100644 --- a/cleanup/cleanup.go +++ b/cleanup/cleanup.go @@ -17,6 +17,15 @@ func Cleanup(filesDir string, metaDir string, locksDir string, noLogs bool) { } for _, filename := range files { + locked, err := fileBackend.CheckLock(filename) + if err != nil { + log.Printf("Error checking if %s is locked: %s", filename, err) + } + if locked { + log.Printf("%s is locked, it will be ignored", filename) + continue + } + metadata, err := fileBackend.Head(filename) if err != nil { if !noLogs { @@ -36,7 +45,9 @@ func Cleanup(filesDir string, metaDir string, locksDir string, noLogs bool) { func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, locksDir string, noLogs bool) { c := time.Tick(minutes) for range c { + log.Printf("Running periodic cleanup") Cleanup(filesDir, metaDir, locksDir, noLogs) + log.Printf("Finished periodic cleanup") } } diff --git a/linx-cleanup/linx-cleanup.go b/linx-cleanup/linx-cleanup.go index 13b3ef1..7953ab4 100644 --- a/linx-cleanup/linx-cleanup.go +++ b/linx-cleanup/linx-cleanup.go @@ -9,12 +9,15 @@ import ( func main() { var filesDir string var metaDir string + var locksDir string var noLogs bool flag.StringVar(&filesDir, "filespath", "files/", "path to files directory") flag.StringVar(&metaDir, "metapath", "meta/", "path to metadata directory") + flag.StringVar(&metaDir, "lockspath", "locks/", + "path to metadata directory") flag.BoolVar(&noLogs, "nologs", false, "don't log deleted files") flag.Parse()