From 151515f516c615c15f418c3753c400db21a4bd12 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Wed, 13 May 2020 17:37:33 -0700 Subject: [PATCH] Cleanup: Integrate expired files periodic cleanup --- .gitignore | 4 +-- README.md | 33 +++++++------------ {linx-cleanup => cleanup}/cleanup.go | 26 +++++++-------- linx-cleanup/README.md | 19 +++++++++++ linx-cleanup/linx-cleanup.go | 23 +++++++++++++ .../linx-cleanup.service | 0 .../linx-cleanup.timer | 0 server.go | 8 +++++ 8 files changed, 75 insertions(+), 38 deletions(-) rename {linx-cleanup => cleanup}/cleanup.go (63%) create mode 100644 linx-cleanup/README.md create mode 100644 linx-cleanup/linx-cleanup.go rename linx-cleanup.service => linx-cleanup/linx-cleanup.service (100%) rename linx-cleanup.timer => linx-cleanup/linx-cleanup.timer (100%) diff --git a/.gitignore b/.gitignore index c6750eb..ec613f1 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,8 @@ _testmain.go *.prof linx-server -linx-cleanup -linx-genkey +linx-cleanup/linx-cleanup +linx-genkey/linx-genkey files/ meta/ binaries/ diff --git a/README.md b/README.md index c7e0adf..567d646 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,17 @@ allowhotlink = true | ```-force-random-filename``` | (optionally) force the use of random filenames | ```-custompagespath "custom_pages"``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation + +#### Cleaning up expired files +When files expire, access is disabled immediately, but the files and metadata +will persist on disk until someone attempts to access them. You can set the following option to run cleanup every few minutes. This can also be done using a separate utility found the linx-cleanup directory. + + +|Option|Description +|------|----------- +| ```-cleanup-every-minutes 5``` | How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed) + + #### Require API Keys for uploads |Option|Description @@ -127,26 +138,6 @@ The following storage backends are available: |------|----------- | ```-fastcgi``` | serve through fastcgi - -Cleaning up expired files -------------------------- -When files expire, access is disabled immediately, but the files and metadata -will persist on disk until someone attempts to access them. If you'd like to -automatically clean up files that have expired, you can use the included -`linx-cleanup` utility. To run it automatically, use a cronjob or similar type -of scheduled task. - -You should be careful to ensure that only one instance of `linx-cleanup` runs at -a time to avoid unexpected behavior. It does not implement any type of locking. - - -|Option|Description -|------|----------- -| ```-filespath files/``` | Path to stored uploads (default is files/) -| ```-nologs``` | (optionally) disable deletion logs in stdout -| ```-metapath meta/``` | Path to stored information about uploads (default is meta/) - - Deployment ---------- Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/). @@ -207,4 +198,4 @@ along with this program. If not, see . Author ------- -Andrei Marcu, http://andreim.net/ +Andrei Marcu, https://andreim.net/ diff --git a/linx-cleanup/cleanup.go b/cleanup/cleanup.go similarity index 63% rename from linx-cleanup/cleanup.go rename to cleanup/cleanup.go index 88c2bce..5920c22 100644 --- a/linx-cleanup/cleanup.go +++ b/cleanup/cleanup.go @@ -1,26 +1,14 @@ -package main +package cleanup import ( - "flag" "log" + "time" "github.com/andreimarcu/linx-server/backends/localfs" "github.com/andreimarcu/linx-server/expiry" ) -func main() { - var filesDir string - var metaDir string - var noLogs bool - - flag.StringVar(&filesDir, "filespath", "files/", - "path to files directory") - flag.StringVar(&metaDir, "metapath", "meta/", - "path to metadata directory") - flag.BoolVar(&noLogs, "nologs", false, - "don't log deleted files") - flag.Parse() - +func Cleanup(filesDir string, metaDir string, noLogs bool) { fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir) files, err := fileBackend.List() @@ -44,3 +32,11 @@ func main() { } } } + +func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, noLogs bool) { + c := time.Tick(minutes) + for range c { + Cleanup(filesDir, metaDir, noLogs) + } + +} diff --git a/linx-cleanup/README.md b/linx-cleanup/README.md new file mode 100644 index 0000000..7d4f4a3 --- /dev/null +++ b/linx-cleanup/README.md @@ -0,0 +1,19 @@ + +linx-cleanup +------------------------- +When files expire, access is disabled immediately, but the files and metadata +will persist on disk until someone attempts to access them. + +If you'd like to automatically clean up files that have expired, you can use the included `linx-cleanup` utility. To run it automatically, use a cronjob or similar type +of scheduled task. + +You should be careful to ensure that only one instance of `linx-cleanup` runs at +a time to avoid unexpected behavior. It does not implement any type of locking. + + +|Option|Description +|------|----------- +| ```-filespath files/``` | Path to stored uploads (default is files/) +| ```-nologs``` | (optionally) disable deletion logs in stdout +| ```-metapath meta/``` | Path to stored information about uploads (default is meta/) + diff --git a/linx-cleanup/linx-cleanup.go b/linx-cleanup/linx-cleanup.go new file mode 100644 index 0000000..13b3ef1 --- /dev/null +++ b/linx-cleanup/linx-cleanup.go @@ -0,0 +1,23 @@ +package main + +import ( + "flag" + + "github.com/andreimarcu/linx-server/cleanup" +) + +func main() { + var filesDir string + var metaDir string + var noLogs bool + + flag.StringVar(&filesDir, "filespath", "files/", + "path to files directory") + flag.StringVar(&metaDir, "metapath", "meta/", + "path to metadata directory") + flag.BoolVar(&noLogs, "nologs", false, + "don't log deleted files") + flag.Parse() + + cleanup.Cleanup(filesDir, metaDir, noLogs) +} diff --git a/linx-cleanup.service b/linx-cleanup/linx-cleanup.service similarity index 100% rename from linx-cleanup.service rename to linx-cleanup/linx-cleanup.service diff --git a/linx-cleanup.timer b/linx-cleanup/linx-cleanup.timer similarity index 100% rename from linx-cleanup.timer rename to linx-cleanup/linx-cleanup.timer diff --git a/server.go b/server.go index e1f0c0a..dae3491 100644 --- a/server.go +++ b/server.go @@ -19,6 +19,7 @@ import ( "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends/localfs" "github.com/andreimarcu/linx-server/backends/s3" + "github.com/andreimarcu/linx-server/cleanup" "github.com/flosch/pongo2" "github.com/vharitonsky/iniflags" "github.com/zenazn/goji/graceful" @@ -71,6 +72,7 @@ var Config struct { forceRandomFilename bool accessKeyCookieExpiry uint64 customPagesDir string + cleanupEveryMinutes uint64 } var Templates = make(map[string]*pongo2.Template) @@ -150,6 +152,10 @@ func setup() *web.Mux { storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint, Config.s3ForcePathStyle) } else { storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir) + if Config.cleanupEveryMinutes > 0 { + go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.noLogs) + } + } // Template setup @@ -311,6 +317,8 @@ func main() { flag.Uint64Var(&Config.accessKeyCookieExpiry, "access-cookie-expiry", 0, "Expiration time for access key cookies in seconds (set 0 to use session cookies)") flag.StringVar(&Config.customPagesDir, "custompagespath", "", "path to directory containing .md files to render as custom pages") + flag.Uint64Var(&Config.cleanupEveryMinutes, "cleanup-every-minutes", 0, + "How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)") iniflags.Parse()