From ffeaaeb2b9e8b261aaf45df4e9f46f4f0d4902c0 Mon Sep 17 00:00:00 2001 From: Marcelo Waisman Date: Thu, 13 Feb 2025 22:26:44 +0200 Subject: [PATCH] sqlite: using ncruces/go-sqlite3 instead of mattn/go-sqlite3 for sqlite --- backend/sqlite/sqlite.go | 15 +++++++++++---- backend/sqlite/sqlite_utils.go | 18 ++++++++++++------ go.mod | 14 ++++++++------ go.sum | 24 ++++++++++++++++-------- 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/backend/sqlite/sqlite.go b/backend/sqlite/sqlite.go index 0565c4525..b4ea37029 100644 --- a/backend/sqlite/sqlite.go +++ b/backend/sqlite/sqlite.go @@ -1,3 +1,4 @@ +// Package sqlite provides an interface to the sqlite backend. package sqlite import ( @@ -8,7 +9,9 @@ import ( "strings" "time" - _ "github.com/mattn/go-sqlite3" // SQLite driver + // SQLite driver + _ "github.com/ncruces/go-sqlite3/driver" + _ "github.com/ncruces/go-sqlite3/embed" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" @@ -32,7 +35,6 @@ func init() { if err != nil { return nil, fmt.Errorf("failed to get connection: %w", err) } - defer db.Close() if err := initSqlite(db); err != nil { return nil, fmt.Errorf("failed to initialize tables: %w", err) @@ -52,6 +54,7 @@ func init() { }) } +// NewFs constructs a new filesystem given a root path and rclone configuration options func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) { fs.Debugf(nil, "[NewFs] name: %q root: %q", name, root) opt := new(Options) @@ -191,7 +194,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { // Return an error if it doesn't exist or isn't empty func (f *Fs) Rmdir(ctx context.Context, dir string) error { fullPath := f.fullPath(dir) - fs.Debugf(nil, "[Rmdir] attemting removing dir: %q", fullPath) + fs.Debugf(nil, "[Rmdir] attempting removing dir: %q", fullPath) return f.rmDir(fullPath) } @@ -255,7 +258,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Remove deletes the remote Object func (o *Object) Remove(ctx context.Context) error { fs.Debugf(nil, "[Remove] removing object: %q", o.info.Filename) - o.fs.remove(o.info.Filename) + err := o.fs.remove(o.info.Filename) + if err != nil { + return err + } return nil } @@ -293,6 +299,7 @@ func (f *Fs) String() string { return fmt.Sprintf("['%s']", f.root) } +// Precision denotes that setting modification times is not supported func (f *Fs) Precision() time.Duration { return time.Millisecond } diff --git a/backend/sqlite/sqlite_utils.go b/backend/sqlite/sqlite_utils.go index f66d82b9d..aae741b79 100644 --- a/backend/sqlite/sqlite_utils.go +++ b/backend/sqlite/sqlite_utils.go @@ -10,10 +10,10 @@ import ( "strings" "time" - _ "github.com/mattn/go-sqlite3" "github.com/rclone/rclone/fs" ) +// An Fs is a representation of a remote SQLite Fs type Fs struct { name string root string @@ -22,6 +22,7 @@ type Fs struct { db *sql.DB } +// An File is a representation of an actual file row in the database type File struct { Filename string Type string @@ -31,6 +32,7 @@ type File struct { SHA1 string } +// An Object on the remote SQLite Fs type Object struct { fs *Fs // what this object is part of info File // info about the object @@ -40,6 +42,7 @@ type Object struct { sha1 string // SHA-1 of the object content } +// Options represent the configuration of the SQLite backend type Options struct { SqliteFile string } @@ -61,7 +64,7 @@ CREATE INDEX IF NOT EXISTS idx_filename_prefix ON files(filename); ` func getConnection(dbPath string) (*sql.DB, error) { - db, err := sql.Open("sqlite3", dbPath) + db, err := sql.Open("sqlite3", "file:"+dbPath) if err != nil { return nil, fmt.Errorf("failed to open database: %w", err) } @@ -89,7 +92,7 @@ func initSqlite(db *sql.DB) error { func (f *Fs) findFile(fullPath string) (*File, error) { fs.Debugf(nil, "[findFile] fullPath: %q", fullPath) - var err error = nil + var err error if f.db == nil { f.db, err = getConnection(f.opt.SqliteFile) if err != nil { @@ -123,7 +126,7 @@ func (f *Fs) fileExists(fullPath string) bool { } func (f *Fs) getFiles(fullPath string) (*[]File, error) { - var err error = nil + var err error if f.db == nil { f.db, err = getConnection(f.opt.SqliteFile) if err != nil { @@ -134,7 +137,6 @@ func (f *Fs) getFiles(fullPath string) (*[]File, error) { if err != nil { return nil, fmt.Errorf("failed to query files: %w", err) } - defer rows.Close() var files []File fileOrDirExists := false @@ -155,6 +157,10 @@ func (f *Fs) getFiles(fullPath string) (*[]File, error) { if err = rows.Err(); err != nil { return nil, fmt.Errorf("error iterating rows: %w", err) } + err = rows.Close() + if err != nil { + return nil, fmt.Errorf("failed to close rows: %w", err) + } if !fileOrDirExists { return nil, fs.ErrorDirNotFound } @@ -193,7 +199,7 @@ func (f *Fs) mkDir(fullPath string) error { func (f *Fs) rmDir(fullPath string) error { fs.Debugf(nil, "[rmdir] fullPath: %q", fullPath) - var err error = nil + var err error // Check if directory is empty result, err := f.hasFiles(fullPath) if err != nil { diff --git a/go.mod b/go.mod index 315fb171b..2a124243d 100644 --- a/go.mod +++ b/go.mod @@ -49,10 +49,10 @@ require ( github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 github.com/mattn/go-colorable v0.1.14 github.com/mattn/go-runewidth v0.0.16 - github.com/mattn/go-sqlite3 v1.14.17 github.com/minio/minio-go/v7 v7.0.83 github.com/mitchellh/go-homedir v1.1.0 github.com/moby/sys/mountinfo v0.7.2 + github.com/ncruces/go-sqlite3 v0.23.0 github.com/ncw/swift/v2 v2.0.3 github.com/oracle/oci-go-sdk/v65 v65.81.1 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -81,12 +81,12 @@ require ( github.com/zeebo/blake3 v0.2.4 go.etcd.io/bbolt v1.3.11 goftp.io/server/v2 v2.0.1 - golang.org/x/crypto v0.32.0 + golang.org/x/crypto v0.33.0 golang.org/x/net v0.34.0 golang.org/x/oauth2 v0.25.0 - golang.org/x/sync v0.10.0 - golang.org/x/sys v0.29.0 - golang.org/x/text v0.21.0 + golang.org/x/sync v0.11.0 + golang.org/x/sys v0.30.0 + golang.org/x/text v0.22.0 golang.org/x/time v0.9.0 google.golang.org/api v0.216.0 gopkg.in/validator.v2 v2.0.1 @@ -188,6 +188,7 @@ require ( github.com/minio/xxml v0.0.3 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/julianday v1.0.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/panjf2000/ants/v2 v2.9.1 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect @@ -207,6 +208,7 @@ require ( github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect + github.com/tetratelabs/wazero v1.8.2 // indirect github.com/tklauser/go-sysconf v0.3.13 // indirect github.com/tklauser/numcpus v0.7.0 // indirect github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 // indirect @@ -239,5 +241,5 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.1 github.com/pkg/xattr v0.4.10 golang.org/x/mobile v0.0.0-20250106192035-c31d5b91ecc3 - golang.org/x/term v0.28.0 + golang.org/x/term v0.29.0 ) diff --git a/go.sum b/go.sum index 7a44765c1..1835e6bb1 100644 --- a/go.sum +++ b/go.sum @@ -460,8 +460,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= @@ -482,6 +480,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-sqlite3 v0.23.0 h1:90j/ar8Ywu2AtsfDl5WhO9sgP/rNk76BcKGIzAHO8AQ= +github.com/ncruces/go-sqlite3 v0.23.0/go.mod h1:gq2nriHSczOs11SqGW5+0X+SgLdkdj4K+j4F/AhQ+8g= +github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= +github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -609,6 +611,8 @@ github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc= github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= +github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4= +github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= @@ -696,8 +700,8 @@ golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -806,8 +810,9 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -861,8 +866,9 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -877,8 +883,9 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -893,8 +900,9 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=