Cleanup: Integrate expired files periodic cleanup

This commit is contained in:
Andrei Marcu 2020-05-13 17:37:33 -07:00
parent bc599ae018
commit 151515f516
8 changed files with 75 additions and 38 deletions

4
.gitignore vendored
View File

@ -29,8 +29,8 @@ _testmain.go
*.prof
linx-server
linx-cleanup
linx-genkey
linx-cleanup/linx-cleanup
linx-genkey/linx-genkey
files/
meta/
binaries/

View File

@ -92,6 +92,17 @@ allowhotlink = true
| ```-force-random-filename``` | (optionally) force the use of random filenames
| ```-custompagespath "custom_pages"``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation
#### Cleaning up expired files
When files expire, access is disabled immediately, but the files and metadata
will persist on disk until someone attempts to access them. You can set the following option to run cleanup every few minutes. This can also be done using a separate utility found the linx-cleanup directory.
|Option|Description
|------|-----------
| ```-cleanup-every-minutes 5``` | How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)
#### Require API Keys for uploads
|Option|Description
@ -127,26 +138,6 @@ The following storage backends are available:
|------|-----------
| ```-fastcgi``` | serve through fastcgi
Cleaning up expired files
-------------------------
When files expire, access is disabled immediately, but the files and metadata
will persist on disk until someone attempts to access them. If you'd like to
automatically clean up files that have expired, you can use the included
`linx-cleanup` utility. To run it automatically, use a cronjob or similar type
of scheduled task.
You should be careful to ensure that only one instance of `linx-cleanup` runs at
a time to avoid unexpected behavior. It does not implement any type of locking.
|Option|Description
|------|-----------
| ```-filespath files/``` | Path to stored uploads (default is files/)
| ```-nologs``` | (optionally) disable deletion logs in stdout
| ```-metapath meta/``` | Path to stored information about uploads (default is meta/)
Deployment
----------
Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/).
@ -207,4 +198,4 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
Author
-------
Andrei Marcu, http://andreim.net/
Andrei Marcu, https://andreim.net/

View File

@ -1,26 +1,14 @@
package main
package cleanup
import (
"flag"
"log"
"time"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/expiry"
)
func main() {
var filesDir string
var metaDir string
var noLogs bool
flag.StringVar(&filesDir, "filespath", "files/",
"path to files directory")
flag.StringVar(&metaDir, "metapath", "meta/",
"path to metadata directory")
flag.BoolVar(&noLogs, "nologs", false,
"don't log deleted files")
flag.Parse()
func Cleanup(filesDir string, metaDir string, noLogs bool) {
fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir)
files, err := fileBackend.List()
@ -44,3 +32,11 @@ func main() {
}
}
}
func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, noLogs bool) {
c := time.Tick(minutes)
for range c {
Cleanup(filesDir, metaDir, noLogs)
}
}

19
linx-cleanup/README.md Normal file
View File

@ -0,0 +1,19 @@
linx-cleanup
-------------------------
When files expire, access is disabled immediately, but the files and metadata
will persist on disk until someone attempts to access them.
If you'd like to automatically clean up files that have expired, you can use the included `linx-cleanup` utility. To run it automatically, use a cronjob or similar type
of scheduled task.
You should be careful to ensure that only one instance of `linx-cleanup` runs at
a time to avoid unexpected behavior. It does not implement any type of locking.
|Option|Description
|------|-----------
| ```-filespath files/``` | Path to stored uploads (default is files/)
| ```-nologs``` | (optionally) disable deletion logs in stdout
| ```-metapath meta/``` | Path to stored information about uploads (default is meta/)

View File

@ -0,0 +1,23 @@
package main
import (
"flag"
"github.com/andreimarcu/linx-server/cleanup"
)
func main() {
var filesDir string
var metaDir string
var noLogs bool
flag.StringVar(&filesDir, "filespath", "files/",
"path to files directory")
flag.StringVar(&metaDir, "metapath", "meta/",
"path to metadata directory")
flag.BoolVar(&noLogs, "nologs", false,
"don't log deleted files")
flag.Parse()
cleanup.Cleanup(filesDir, metaDir, noLogs)
}

View File

@ -19,6 +19,7 @@ import (
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/backends/s3"
"github.com/andreimarcu/linx-server/cleanup"
"github.com/flosch/pongo2"
"github.com/vharitonsky/iniflags"
"github.com/zenazn/goji/graceful"
@ -71,6 +72,7 @@ var Config struct {
forceRandomFilename bool
accessKeyCookieExpiry uint64
customPagesDir string
cleanupEveryMinutes uint64
}
var Templates = make(map[string]*pongo2.Template)
@ -150,6 +152,10 @@ func setup() *web.Mux {
storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint, Config.s3ForcePathStyle)
} else {
storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir)
if Config.cleanupEveryMinutes > 0 {
go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.noLogs)
}
}
// Template setup
@ -311,6 +317,8 @@ func main() {
flag.Uint64Var(&Config.accessKeyCookieExpiry, "access-cookie-expiry", 0, "Expiration time for access key cookies in seconds (set 0 to use session cookies)")
flag.StringVar(&Config.customPagesDir, "custompagespath", "",
"path to directory containing .md files to render as custom pages")
flag.Uint64Var(&Config.cleanupEveryMinutes, "cleanup-every-minutes", 0,
"How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)")
iniflags.Parse()