Cleanup: Integrate expired files periodic cleanup
This commit is contained in:
parent
bc599ae018
commit
151515f516
|
@ -29,8 +29,8 @@ _testmain.go
|
|||
*.prof
|
||||
|
||||
linx-server
|
||||
linx-cleanup
|
||||
linx-genkey
|
||||
linx-cleanup/linx-cleanup
|
||||
linx-genkey/linx-genkey
|
||||
files/
|
||||
meta/
|
||||
binaries/
|
||||
|
|
33
README.md
33
README.md
|
@ -92,6 +92,17 @@ allowhotlink = true
|
|||
| ```-force-random-filename``` | (optionally) force the use of random filenames
|
||||
| ```-custompagespath "custom_pages"``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation
|
||||
|
||||
|
||||
#### Cleaning up expired files
|
||||
When files expire, access is disabled immediately, but the files and metadata
|
||||
will persist on disk until someone attempts to access them. You can set the following option to run cleanup every few minutes. This can also be done using a separate utility found the linx-cleanup directory.
|
||||
|
||||
|
||||
|Option|Description
|
||||
|------|-----------
|
||||
| ```-cleanup-every-minutes 5``` | How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)
|
||||
|
||||
|
||||
#### Require API Keys for uploads
|
||||
|
||||
|Option|Description
|
||||
|
@ -127,26 +138,6 @@ The following storage backends are available:
|
|||
|------|-----------
|
||||
| ```-fastcgi``` | serve through fastcgi
|
||||
|
||||
|
||||
Cleaning up expired files
|
||||
-------------------------
|
||||
When files expire, access is disabled immediately, but the files and metadata
|
||||
will persist on disk until someone attempts to access them. If you'd like to
|
||||
automatically clean up files that have expired, you can use the included
|
||||
`linx-cleanup` utility. To run it automatically, use a cronjob or similar type
|
||||
of scheduled task.
|
||||
|
||||
You should be careful to ensure that only one instance of `linx-cleanup` runs at
|
||||
a time to avoid unexpected behavior. It does not implement any type of locking.
|
||||
|
||||
|
||||
|Option|Description
|
||||
|------|-----------
|
||||
| ```-filespath files/``` | Path to stored uploads (default is files/)
|
||||
| ```-nologs``` | (optionally) disable deletion logs in stdout
|
||||
| ```-metapath meta/``` | Path to stored information about uploads (default is meta/)
|
||||
|
||||
|
||||
Deployment
|
||||
----------
|
||||
Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/).
|
||||
|
@ -207,4 +198,4 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|||
|
||||
Author
|
||||
-------
|
||||
Andrei Marcu, http://andreim.net/
|
||||
Andrei Marcu, https://andreim.net/
|
||||
|
|
|
@ -1,26 +1,14 @@
|
|||
package main
|
||||
package cleanup
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/andreimarcu/linx-server/backends/localfs"
|
||||
"github.com/andreimarcu/linx-server/expiry"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var filesDir string
|
||||
var metaDir string
|
||||
var noLogs bool
|
||||
|
||||
flag.StringVar(&filesDir, "filespath", "files/",
|
||||
"path to files directory")
|
||||
flag.StringVar(&metaDir, "metapath", "meta/",
|
||||
"path to metadata directory")
|
||||
flag.BoolVar(&noLogs, "nologs", false,
|
||||
"don't log deleted files")
|
||||
flag.Parse()
|
||||
|
||||
func Cleanup(filesDir string, metaDir string, noLogs bool) {
|
||||
fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir)
|
||||
|
||||
files, err := fileBackend.List()
|
||||
|
@ -44,3 +32,11 @@ func main() {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, noLogs bool) {
|
||||
c := time.Tick(minutes)
|
||||
for range c {
|
||||
Cleanup(filesDir, metaDir, noLogs)
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,19 @@
|
|||
|
||||
linx-cleanup
|
||||
-------------------------
|
||||
When files expire, access is disabled immediately, but the files and metadata
|
||||
will persist on disk until someone attempts to access them.
|
||||
|
||||
If you'd like to automatically clean up files that have expired, you can use the included `linx-cleanup` utility. To run it automatically, use a cronjob or similar type
|
||||
of scheduled task.
|
||||
|
||||
You should be careful to ensure that only one instance of `linx-cleanup` runs at
|
||||
a time to avoid unexpected behavior. It does not implement any type of locking.
|
||||
|
||||
|
||||
|Option|Description
|
||||
|------|-----------
|
||||
| ```-filespath files/``` | Path to stored uploads (default is files/)
|
||||
| ```-nologs``` | (optionally) disable deletion logs in stdout
|
||||
| ```-metapath meta/``` | Path to stored information about uploads (default is meta/)
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
||||
"github.com/andreimarcu/linx-server/cleanup"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var filesDir string
|
||||
var metaDir string
|
||||
var noLogs bool
|
||||
|
||||
flag.StringVar(&filesDir, "filespath", "files/",
|
||||
"path to files directory")
|
||||
flag.StringVar(&metaDir, "metapath", "meta/",
|
||||
"path to metadata directory")
|
||||
flag.BoolVar(&noLogs, "nologs", false,
|
||||
"don't log deleted files")
|
||||
flag.Parse()
|
||||
|
||||
cleanup.Cleanup(filesDir, metaDir, noLogs)
|
||||
}
|
|
@ -19,6 +19,7 @@ import (
|
|||
"github.com/andreimarcu/linx-server/backends"
|
||||
"github.com/andreimarcu/linx-server/backends/localfs"
|
||||
"github.com/andreimarcu/linx-server/backends/s3"
|
||||
"github.com/andreimarcu/linx-server/cleanup"
|
||||
"github.com/flosch/pongo2"
|
||||
"github.com/vharitonsky/iniflags"
|
||||
"github.com/zenazn/goji/graceful"
|
||||
|
@ -71,6 +72,7 @@ var Config struct {
|
|||
forceRandomFilename bool
|
||||
accessKeyCookieExpiry uint64
|
||||
customPagesDir string
|
||||
cleanupEveryMinutes uint64
|
||||
}
|
||||
|
||||
var Templates = make(map[string]*pongo2.Template)
|
||||
|
@ -150,6 +152,10 @@ func setup() *web.Mux {
|
|||
storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint, Config.s3ForcePathStyle)
|
||||
} else {
|
||||
storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir)
|
||||
if Config.cleanupEveryMinutes > 0 {
|
||||
go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.noLogs)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Template setup
|
||||
|
@ -311,6 +317,8 @@ func main() {
|
|||
flag.Uint64Var(&Config.accessKeyCookieExpiry, "access-cookie-expiry", 0, "Expiration time for access key cookies in seconds (set 0 to use session cookies)")
|
||||
flag.StringVar(&Config.customPagesDir, "custompagespath", "",
|
||||
"path to directory containing .md files to render as custom pages")
|
||||
flag.Uint64Var(&Config.cleanupEveryMinutes, "cleanup-every-minutes", 0,
|
||||
"How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)")
|
||||
|
||||
iniflags.Parse()
|
||||
|
||||
|
|
Loading…
Reference in New Issue