From 7dc45206905d50f29010763d71857c344a979034 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Wed, 8 Jan 2025 16:36:41 +0000 Subject: [PATCH] Fix #207. --- ext/serdes/serdes.go | 151 ++++++++++++++++++++++++++++++++++++++ ext/serdes/serdes_test.go | 68 +++++++++++++++++ vfs/memdb/memdb.go | 4 +- vtab.go | 2 +- 4 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 ext/serdes/serdes.go create mode 100644 ext/serdes/serdes_test.go diff --git a/ext/serdes/serdes.go b/ext/serdes/serdes.go new file mode 100644 index 00000000..2e4aac42 --- /dev/null +++ b/ext/serdes/serdes.go @@ -0,0 +1,151 @@ +// Package serdes provides functions to (de)serialize databases. +package serdes + +import ( + "io" + "sync" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/vfs" +) + +func init() { + vfs.Register(vfsName, sliceVFS{}) +} + +// Serialize backs up a database into a byte slice. +// +// https://sqlite.org/c3ref/serialize.html +func Serialize(db *sqlite3.Conn, schema string) ([]byte, error) { + var file sliceFile + openMtx.Lock() + openFile = &file + err := db.Backup(schema, "file:db?vfs="+vfsName) + return file.data, err +} + +// Deserialize restores a database from a byte slice, +// DESTROYING any contents previously stored in schema. +// +// To non-destructively open a database from a byte slice, +// consider alternatives like the ["reader"] or ["memdb"] VFSes. +// +// This differs from the similarly named SQLite API +// in that it DOES NOT disconnect from schema +// to reopen as an in-memory database. +// +// https://sqlite.org/c3ref/deserialize.html +// +// ["memdb"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb +// ["reader"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs +func Deserialize(db *sqlite3.Conn, schema string, data []byte) error { + openMtx.Lock() + openFile = &sliceFile{data} + return db.Restore(schema, "file:db?vfs="+vfsName) +} + +var ( + openMtx sync.Mutex + openFile *sliceFile +) + +const vfsName = "github.com/ncruces/go-sqlite3/ext/deserialize.sliceVFS" + +type sliceVFS struct{} + +func (sliceVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) { + if flags&vfs.OPEN_MAIN_DB == 0 { + // notest // OPEN_MEMORY + return nil, flags, sqlite3.CANTOPEN + } + + file := openFile + openFile = nil + openMtx.Unlock() + + if file.data != nil { + flags |= vfs.OPEN_READONLY + } + flags |= vfs.OPEN_MEMORY + return file, flags, nil +} + +func (sliceVFS) Delete(name string, dirSync bool) error { + // notest // OPEN_MEMORY + return sqlite3.IOERR_DELETE +} + +func (sliceVFS) Access(name string, flag vfs.AccessFlag) (bool, error) { + return name == "db", nil +} + +func (sliceVFS) FullPathname(name string) (string, error) { + return name, nil +} + +type sliceFile struct{ data []byte } + +func (f *sliceFile) ReadAt(b []byte, off int64) (n int, err error) { + if d := f.data; off < int64(len(d)) { + n = copy(b, d[off:]) + } + if n == 0 { + err = io.EOF + } + return +} + +func (f *sliceFile) WriteAt(b []byte, off int64) (n int, err error) { + if d := f.data; off > int64(len(d)) { + f.data = append(d, make([]byte, off-int64(len(d)))...) + } + d := append(f.data[:off], b...) + if len(d) > len(f.data) { + f.data = d + } + return len(b), nil +} + +func (f *sliceFile) Size() (int64, error) { + return int64(len(f.data)), nil +} + +func (f *sliceFile) Truncate(size int64) error { + if d := f.data; size < int64(len(d)) { + f.data = d[:size] + } + return nil +} + +func (f *sliceFile) SizeHint(size int64) error { + if d := f.data; size > int64(len(d)) { + f.data = append(d, make([]byte, size-int64(len(d)))...) + } + return nil +} + +func (*sliceFile) Close() error { return nil } + +func (*sliceFile) Sync(flag vfs.SyncFlag) error { return nil } + +func (*sliceFile) Lock(lock vfs.LockLevel) error { return nil } + +func (*sliceFile) Unlock(lock vfs.LockLevel) error { return nil } + +func (*sliceFile) CheckReservedLock() (bool, error) { + // notest // OPEN_MEMORY + return false, nil +} + +func (*sliceFile) SectorSize() int { + // notest // IOCAP_POWERSAFE_OVERWRITE + return 0 +} + +func (*sliceFile) DeviceCharacteristics() vfs.DeviceCharacteristic { + return vfs.IOCAP_ATOMIC | + vfs.IOCAP_SAFE_APPEND | + vfs.IOCAP_SEQUENTIAL | + vfs.IOCAP_POWERSAFE_OVERWRITE | + vfs.IOCAP_SUBPAGE_READ +} diff --git a/ext/serdes/serdes_test.go b/ext/serdes/serdes_test.go new file mode 100644 index 00000000..fdf53251 --- /dev/null +++ b/ext/serdes/serdes_test.go @@ -0,0 +1,68 @@ +package serdes_test + +import ( + "io" + "net/http" + "testing" + + "github.com/ncruces/go-sqlite3" + _ "github.com/ncruces/go-sqlite3/embed" + "github.com/ncruces/go-sqlite3/ext/serdes" +) + +func TestDeserialize(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + input, err := httpGet() + if err != nil { + t.Fatal(err) + } + + db, err := sqlite3.Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + err = serdes.Deserialize(db, "temp", input) + if err != nil { + t.Fatal(err) + } + + output, err := serdes.Serialize(db, "temp") + if err != nil { + t.Fatal(err) + } + + if len(input) != len(output) { + t.Fatal("lengths are different") + } + for i := range input { + // These may be different. + switch { + case 24 <= i && i < 28: + // File change counter. + continue + case 40 <= i && i < 44: + // Schema cookie. + continue + case 92 <= i && i < 100: + // SQLite version that wrote the file. + continue + } + if input[i] != output[i] { + t.Errorf("difference at %d: %d %d", i, input[i], output[i]) + } + } +} + +func httpGet() ([]byte, error) { + res, err := http.Get("https://raw.githubusercontent.com/jpwhite3/northwind-SQLite3/refs/heads/main/dist/northwind.db") + if err != nil { + return nil, err + } + defer res.Body.Close() + return io.ReadAll(res.Body) +} diff --git a/vfs/memdb/memdb.go b/vfs/memdb/memdb.go index 686f8e9a..4adb2dde 100644 --- a/vfs/memdb/memdb.go +++ b/vfs/memdb/memdb.go @@ -62,11 +62,11 @@ func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, err } func (memVFS) Delete(name string, dirSync bool) error { - return sqlite3.IOERR_DELETE + return sqlite3.IOERR_DELETE_NOENT // used to delete journals } func (memVFS) Access(name string, flag vfs.AccessFlag) (bool, error) { - return false, nil + return false, nil // used to check for journals } func (memVFS) FullPathname(name string) (string, error) { diff --git a/vtab.go b/vtab.go index 98348623..1998a528 100644 --- a/vtab.go +++ b/vtab.go @@ -242,7 +242,7 @@ type VTabSavepointer interface { // A VTabCursor may optionally implement // [io.Closer] to free resources. // -// http://sqlite.org/c3ref/vtab_cursor.html +// https://sqlite.org/c3ref/vtab_cursor.html type VTabCursor interface { // https://sqlite.org/vtab.html#xfilter Filter(idxNum int, idxStr string, arg ...Value) error