sqlite: using ncruces/go-sqlite3 instead of mattn/go-sqlite3 for sqlite

This commit is contained in:
Marcelo Waisman 2025-02-13 22:26:44 +02:00
parent 343271e0d0
commit ffeaaeb2b9
4 changed files with 47 additions and 24 deletions

View File

@ -1,3 +1,4 @@
// Package sqlite provides an interface to the sqlite backend.
package sqlite
import (
@ -8,7 +9,9 @@ import (
"strings"
"time"
_ "github.com/mattn/go-sqlite3" // SQLite driver
// SQLite driver
_ "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
@ -32,7 +35,6 @@ func init() {
if err != nil {
return nil, fmt.Errorf("failed to get connection: %w", err)
}
defer db.Close()
if err := initSqlite(db); err != nil {
return nil, fmt.Errorf("failed to initialize tables: %w", err)
@ -52,6 +54,7 @@ func init() {
})
}
// NewFs constructs a new filesystem given a root path and rclone configuration options
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
fs.Debugf(nil, "[NewFs] name: %q root: %q", name, root)
opt := new(Options)
@ -191,7 +194,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// Return an error if it doesn't exist or isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
fullPath := f.fullPath(dir)
fs.Debugf(nil, "[Rmdir] attemting removing dir: %q", fullPath)
fs.Debugf(nil, "[Rmdir] attempting removing dir: %q", fullPath)
return f.rmDir(fullPath)
}
@ -255,7 +258,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Remove deletes the remote Object
func (o *Object) Remove(ctx context.Context) error {
fs.Debugf(nil, "[Remove] removing object: %q", o.info.Filename)
o.fs.remove(o.info.Filename)
err := o.fs.remove(o.info.Filename)
if err != nil {
return err
}
return nil
}
@ -293,6 +299,7 @@ func (f *Fs) String() string {
return fmt.Sprintf("['%s']", f.root)
}
// Precision denotes that setting modification times is not supported
func (f *Fs) Precision() time.Duration {
return time.Millisecond
}

View File

@ -10,10 +10,10 @@ import (
"strings"
"time"
_ "github.com/mattn/go-sqlite3"
"github.com/rclone/rclone/fs"
)
// An Fs is a representation of a remote SQLite Fs
type Fs struct {
name string
root string
@ -22,6 +22,7 @@ type Fs struct {
db *sql.DB
}
// An File is a representation of an actual file row in the database
type File struct {
Filename string
Type string
@ -31,6 +32,7 @@ type File struct {
SHA1 string
}
// An Object on the remote SQLite Fs
type Object struct {
fs *Fs // what this object is part of
info File // info about the object
@ -40,6 +42,7 @@ type Object struct {
sha1 string // SHA-1 of the object content
}
// Options represent the configuration of the SQLite backend
type Options struct {
SqliteFile string
}
@ -61,7 +64,7 @@ CREATE INDEX IF NOT EXISTS idx_filename_prefix ON files(filename);
`
func getConnection(dbPath string) (*sql.DB, error) {
db, err := sql.Open("sqlite3", dbPath)
db, err := sql.Open("sqlite3", "file:"+dbPath)
if err != nil {
return nil, fmt.Errorf("failed to open database: %w", err)
}
@ -89,7 +92,7 @@ func initSqlite(db *sql.DB) error {
func (f *Fs) findFile(fullPath string) (*File, error) {
fs.Debugf(nil, "[findFile] fullPath: %q", fullPath)
var err error = nil
var err error
if f.db == nil {
f.db, err = getConnection(f.opt.SqliteFile)
if err != nil {
@ -123,7 +126,7 @@ func (f *Fs) fileExists(fullPath string) bool {
}
func (f *Fs) getFiles(fullPath string) (*[]File, error) {
var err error = nil
var err error
if f.db == nil {
f.db, err = getConnection(f.opt.SqliteFile)
if err != nil {
@ -134,7 +137,6 @@ func (f *Fs) getFiles(fullPath string) (*[]File, error) {
if err != nil {
return nil, fmt.Errorf("failed to query files: %w", err)
}
defer rows.Close()
var files []File
fileOrDirExists := false
@ -155,6 +157,10 @@ func (f *Fs) getFiles(fullPath string) (*[]File, error) {
if err = rows.Err(); err != nil {
return nil, fmt.Errorf("error iterating rows: %w", err)
}
err = rows.Close()
if err != nil {
return nil, fmt.Errorf("failed to close rows: %w", err)
}
if !fileOrDirExists {
return nil, fs.ErrorDirNotFound
}
@ -193,7 +199,7 @@ func (f *Fs) mkDir(fullPath string) error {
func (f *Fs) rmDir(fullPath string) error {
fs.Debugf(nil, "[rmdir] fullPath: %q", fullPath)
var err error = nil
var err error
// Check if directory is empty
result, err := f.hasFiles(fullPath)
if err != nil {

14
go.mod
View File

@ -49,10 +49,10 @@ require (
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
github.com/mattn/go-colorable v0.1.14
github.com/mattn/go-runewidth v0.0.16
github.com/mattn/go-sqlite3 v1.14.17
github.com/minio/minio-go/v7 v7.0.83
github.com/mitchellh/go-homedir v1.1.0
github.com/moby/sys/mountinfo v0.7.2
github.com/ncruces/go-sqlite3 v0.23.0
github.com/ncw/swift/v2 v2.0.3
github.com/oracle/oci-go-sdk/v65 v65.81.1
github.com/patrickmn/go-cache v2.1.0+incompatible
@ -81,12 +81,12 @@ require (
github.com/zeebo/blake3 v0.2.4
go.etcd.io/bbolt v1.3.11
goftp.io/server/v2 v2.0.1
golang.org/x/crypto v0.32.0
golang.org/x/crypto v0.33.0
golang.org/x/net v0.34.0
golang.org/x/oauth2 v0.25.0
golang.org/x/sync v0.10.0
golang.org/x/sys v0.29.0
golang.org/x/text v0.21.0
golang.org/x/sync v0.11.0
golang.org/x/sys v0.30.0
golang.org/x/text v0.22.0
golang.org/x/time v0.9.0
google.golang.org/api v0.216.0
gopkg.in/validator.v2 v2.0.1
@ -188,6 +188,7 @@ require (
github.com/minio/xxml v0.0.3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/panjf2000/ants/v2 v2.9.1 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
@ -207,6 +208,7 @@ require (
github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect
github.com/sony/gobreaker v0.5.0 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
github.com/tetratelabs/wazero v1.8.2 // indirect
github.com/tklauser/go-sysconf v0.3.13 // indirect
github.com/tklauser/numcpus v0.7.0 // indirect
github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 // indirect
@ -239,5 +241,5 @@ require (
github.com/golang-jwt/jwt/v4 v4.5.1
github.com/pkg/xattr v0.4.10
golang.org/x/mobile v0.0.0-20250106192035-c31d5b91ecc3
golang.org/x/term v0.28.0
golang.org/x/term v0.29.0
)

24
go.sum
View File

@ -460,8 +460,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
@ -482,6 +480,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/ncruces/go-sqlite3 v0.23.0 h1:90j/ar8Ywu2AtsfDl5WhO9sgP/rNk76BcKGIzAHO8AQ=
github.com/ncruces/go-sqlite3 v0.23.0/go.mod h1:gq2nriHSczOs11SqGW5+0X+SgLdkdj4K+j4F/AhQ+8g=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg=
github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@ -609,6 +611,8 @@ github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf
github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc=
github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4=
github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0=
github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4=
@ -696,8 +700,8 @@ golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -806,8 +810,9 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -861,8 +866,9 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@ -877,8 +883,9 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -893,8 +900,9 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=