From ca4d16598a87a20db06d45df9759bc763ded8ac7 Mon Sep 17 00:00:00 2001 From: sputn1ck Date: Mon, 25 May 2026 08:25:45 +0200 Subject: [PATCH 1/3] db: Add wasm SQLite wallet storage Add wasm-safe SQLite open, migrate, and error handling through go-wasmsqlite. Browser builds use OPFS while native builds keep the existing database/sql path. Move lwwallet walletdb and seed storage behind native and wasm files so browser wallets can persist state in OPFS. --- btcwbackend/boarding_backend.go | 2 + btcwbackend/chain_backend.go | 2 + btcwbackend/config.go | 2 + btcwbackend/doc.go | 2 + btcwbackend/log.go | 2 + btcwbackend/neutrino.go | 2 + btcwbackend/wallet.go | 2 + btcwbackend/wasm_stub.go | 129 +++++ darepod/fs_native.go | 10 + darepod/fs_wasm.go | 10 + darepod/seed_manager.go | 39 -- darepod/seed_storage_native.go | 45 ++ darepod/seed_storage_wasm.go | 62 +++ darepod/server.go | 2 +- db/migrate/driver_native.go | 37 ++ db/migrate/driver_wasm.go | 25 + db/migrate/migrations.go | 27 - db/migrate/sqlite_wasm_driver.go | 258 +++++++++ db/postgres.go | 60 --- db/postgres_fixture.go | 2 + db/postgres_test_helpers.go | 71 +++ db/sqlerrors.go | 51 +- db/sqlerrors_native.go | 52 ++ db/sqlerrors_wasm.go | 42 ++ db/sqlite.go | 67 +-- db/sqlite_open.go | 50 ++ db/sqlite_open_native.go | 63 +++ db/sqlite_open_wasm.go | 129 +++++ go.mod | 8 +- go.sum | 20 +- internal/sqlbase/LICENSE | 19 + internal/sqlbase/db.go | 317 +++++++++++ internal/sqlbase/db_conn_set.go | 90 ++++ internal/sqlbase/log.go | 14 + internal/sqlbase/readwrite_bucket.go | 491 ++++++++++++++++++ internal/sqlbase/readwrite_cursor.go | 232 +++++++++ internal/sqlbase/readwrite_tx.go | 234 +++++++++ internal/sqlbase/schema.go | 73 +++ lwwallet/wallet.go | 17 +- lwwallet/walletdb_native.go | 18 + lwwallet/walletdb_wasm.go | 115 ++++ .../migrations/000001_swap_sessions.up.sql | 13 +- sdk/swaps/store.go | 67 +-- swapclientserver/fs_native.go | 13 + swapclientserver/fs_wasm.go | 9 + swapclientserver/service.go | 2 +- 46 files changed, 2718 insertions(+), 279 deletions(-) create mode 100644 btcwbackend/wasm_stub.go create mode 100644 darepod/fs_native.go create mode 100644 darepod/fs_wasm.go create mode 100644 darepod/seed_storage_native.go create mode 100644 darepod/seed_storage_wasm.go create mode 100644 db/migrate/driver_native.go create mode 100644 db/migrate/driver_wasm.go create mode 100644 db/migrate/sqlite_wasm_driver.go create mode 100644 db/postgres_test_helpers.go create mode 100644 db/sqlerrors_native.go create mode 100644 db/sqlerrors_wasm.go create mode 100644 db/sqlite_open.go create mode 100644 db/sqlite_open_native.go create mode 100644 db/sqlite_open_wasm.go create mode 100644 internal/sqlbase/LICENSE create mode 100644 internal/sqlbase/db.go create mode 100644 internal/sqlbase/db_conn_set.go create mode 100644 internal/sqlbase/log.go create mode 100644 internal/sqlbase/readwrite_bucket.go create mode 100644 internal/sqlbase/readwrite_cursor.go create mode 100644 internal/sqlbase/readwrite_tx.go create mode 100644 internal/sqlbase/schema.go create mode 100644 lwwallet/walletdb_native.go create mode 100644 lwwallet/walletdb_wasm.go create mode 100644 swapclientserver/fs_native.go create mode 100644 swapclientserver/fs_wasm.go diff --git a/btcwbackend/boarding_backend.go b/btcwbackend/boarding_backend.go index 64d87d1bb..5ab914faa 100644 --- a/btcwbackend/boarding_backend.go +++ b/btcwbackend/boarding_backend.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + package btcwbackend import ( diff --git a/btcwbackend/chain_backend.go b/btcwbackend/chain_backend.go index 679e670f1..ab5416e91 100644 --- a/btcwbackend/chain_backend.go +++ b/btcwbackend/chain_backend.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + package btcwbackend import ( diff --git a/btcwbackend/config.go b/btcwbackend/config.go index aca0ea266..cb4330f85 100644 --- a/btcwbackend/config.go +++ b/btcwbackend/config.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + package btcwbackend import ( diff --git a/btcwbackend/doc.go b/btcwbackend/doc.go index a0a74701d..ab399e459 100644 --- a/btcwbackend/doc.go +++ b/btcwbackend/doc.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + // Package btcwbackend provides a lightweight in-process Bitcoin wallet backed // by LND's btcwallet and a neutrino (BIP 157/158) chain backend. It wraps // lnwallet/btcwallet.BtcWallet with a neutrino-based chain.Interface, diff --git a/btcwbackend/log.go b/btcwbackend/log.go index f7e681e22..dc38069f5 100644 --- a/btcwbackend/log.go +++ b/btcwbackend/log.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + package btcwbackend // Subsystem defines the logging code for this subsystem. diff --git a/btcwbackend/neutrino.go b/btcwbackend/neutrino.go index 59680e166..33f217df5 100644 --- a/btcwbackend/neutrino.go +++ b/btcwbackend/neutrino.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + package btcwbackend import ( diff --git a/btcwbackend/wallet.go b/btcwbackend/wallet.go index 9bfd9e6a4..95a33a036 100644 --- a/btcwbackend/wallet.go +++ b/btcwbackend/wallet.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + package btcwbackend import ( diff --git a/btcwbackend/wasm_stub.go b/btcwbackend/wasm_stub.go new file mode 100644 index 000000000..365892838 --- /dev/null +++ b/btcwbackend/wasm_stub.go @@ -0,0 +1,129 @@ +//go:build js && wasm + +// Package btcwbackend is unavailable in browser WASM builds. +package btcwbackend + +import ( + "context" + "fmt" + "time" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/darepo-client/chainbackends" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/lightninglabs/darepo-client/wallet" + "github.com/lightninglabs/darepo-client/walletcore" + fn "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/keychain" +) + +const ( + // Subsystem defines the logging code for this subsystem. + Subsystem = "BTCW" + + // DefaultFeeMinUpdateTimeout is the default minimum interval between + // fee estimation API queries. + DefaultFeeMinUpdateTimeout = 5 * time.Minute + + // DefaultFeeMaxUpdateTimeout is the default maximum interval between + // fee estimation API queries. + DefaultFeeMaxUpdateTimeout = 20 * time.Minute +) + +// Config holds the configuration for the native neutrino-backed wallet. +type Config struct { + walletcore.Config + + NeutrinoDataDir string + ConnectPeers []string + AddPeers []string + BlockHeadersSource string + FilterHeadersSource string + FeeURL string + FeeMinUpdateTimeout time.Duration + FeeMaxUpdateTimeout time.Duration + PackageSubmitter chainbackends.PackageSubmitter + PersistFilters bool + DisableGlobalLoggers bool +} + +// WithLogger returns a new config with the given logger set. +func (c Config) WithLogger(log btclog.Logger) Config { + c.Log = fn.Some(log) + + return c +} + +// NeutrinoServiceOption configures a NeutrinoService. +type NeutrinoServiceOption func(*NeutrinoService) + +// NeutrinoService is a browser stub for the native neutrino service. +type NeutrinoService struct{} + +// WithoutGlobalDependencyLoggers disables native package-global loggers. +func WithoutGlobalDependencyLoggers() NeutrinoServiceOption { + return func(*NeutrinoService) {} +} + +// NewNeutrinoService reports that neutrino is unavailable in browser builds. +func NewNeutrinoService(string, *chaincfg.Params, []string, []string, bool, + string, string, btclog.Logger, + ...NeutrinoServiceOption) (*NeutrinoService, error) { + + return nil, fmt.Errorf("btcwallet backend is not available in wasm") +} + +// Start reports that neutrino is unavailable in browser builds. +func (n *NeutrinoService) Start(context.Context) error { + return fmt.Errorf("btcwallet backend is not available in wasm") +} + +// Stop is a no-op for the browser stub. +func (n *NeutrinoService) Stop() error { + return nil +} + +// Wallet is a browser stub for the native neutrino-backed wallet. +type Wallet struct { + walletcore.Wallet +} + +// New reports that btcwallet is unavailable in browser builds. +func New(Config) (*Wallet, error) { + return nil, fmt.Errorf("btcwallet backend is not available in wasm") +} + +// NewWithNeutrino reports that btcwallet is unavailable in browser builds. +func NewWithNeutrino(Config, *NeutrinoService) (*Wallet, error) { + return nil, fmt.Errorf("btcwallet backend is not available in wasm") +} + +// Start reports that btcwallet is unavailable in browser builds. +func (w *Wallet) Start() error { + return fmt.Errorf("btcwallet backend is not available in wasm") +} + +// Stop is a no-op for the browser stub. +func (w *Wallet) Stop() {} + +// BoardingBackend reports that no btcwallet boarding backend exists in WASM. +func (w *Wallet) BoardingBackend() wallet.BoardingBackend { + return nil +} + +// ChainBackend reports that no btcwallet chain backend exists in WASM. +func (w *Wallet) ChainBackend() chainsource.ChainBackend { + return nil +} + +// KeyRing returns the embedded walletcore key ring, if one was set by tests. +func (w *Wallet) KeyRing() keychain.SecretKeyRing { + return w.Wallet.KeyRing +} + +// IsSynced reports that the unavailable browser stub is not synced. +func (w *Wallet) IsSynced() (bool, int64, error) { + return false, 0, fmt.Errorf("btcwallet backend is not available in " + + "wasm") +} diff --git a/darepod/fs_native.go b/darepod/fs_native.go new file mode 100644 index 000000000..4a74714b1 --- /dev/null +++ b/darepod/fs_native.go @@ -0,0 +1,10 @@ +//go:build !js || !wasm + +package darepod + +import "os" + +// ensureDataDir creates a host filesystem directory for daemon state. +func ensureDataDir(path string) error { + return os.MkdirAll(path, 0700) +} diff --git a/darepod/fs_wasm.go b/darepod/fs_wasm.go new file mode 100644 index 000000000..562716528 --- /dev/null +++ b/darepod/fs_wasm.go @@ -0,0 +1,10 @@ +//go:build js && wasm + +package darepod + +// ensureDataDir is a no-op in browser builds. Persistent state uses +// OPFS-backed SQLite and browser storage instead of host filesystem +// directories, and os.MkdirAll is not implemented under js/wasm. +func ensureDataDir(string) error { + return nil +} diff --git a/darepod/seed_manager.go b/darepod/seed_manager.go index 488e1e714..650fa957e 100644 --- a/darepod/seed_manager.go +++ b/darepod/seed_manager.go @@ -292,42 +292,3 @@ func LoadPasswordFromFile(path string) ([]byte, error) { func SeedFilePath(networkDir string) string { return filepath.Join(networkDir, seedFileBaseName) } - -// SaveEncryptedSeed writes the encrypted seed ciphertext to disk at -// the given path. The file is created with restrictive permissions -// (0600) to prevent unauthorized access. -func SaveEncryptedSeed(path string, ciphertext []byte) error { - // Ensure the parent directory exists. - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0700); err != nil { - return fmt.Errorf("creating directory %q: %w", dir, err) - } - - if err := os.WriteFile(path, ciphertext, 0600); err != nil { - return fmt.Errorf("writing seed file %q: %w", path, err) - } - - return nil -} - -// LoadEncryptedSeed reads the encrypted seed ciphertext from disk. It -// returns an error if the file does not exist or cannot be read. -func LoadEncryptedSeed(path string) ([]byte, error) { - // Seed file path is an operator-supplied config value. - data, err := os.ReadFile(path) //nolint:gosec // G304 - if err != nil { - return nil, fmt.Errorf("reading seed file %q: %w", path, err) - } - - return data, nil -} - -// SeedFileExists returns true if an encrypted seed file exists at the -// expected path within the network data directory. -func SeedFileExists(networkDir string) bool { - path := SeedFilePath(networkDir) - - _, err := os.Stat(path) - - return err == nil -} diff --git a/darepod/seed_storage_native.go b/darepod/seed_storage_native.go new file mode 100644 index 000000000..86faec566 --- /dev/null +++ b/darepod/seed_storage_native.go @@ -0,0 +1,45 @@ +//go:build !js || !wasm + +package darepod + +import ( + "fmt" + "os" + "path/filepath" +) + +// SaveEncryptedSeed writes the encrypted seed ciphertext to disk at the given +// path. The file is created with restrictive permissions to prevent +// unauthorized access. +func SaveEncryptedSeed(path string, ciphertext []byte) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("creating directory %q: %w", dir, err) + } + + if err := os.WriteFile(path, ciphertext, 0600); err != nil { + return fmt.Errorf("writing seed file %q: %w", path, err) + } + + return nil +} + +// LoadEncryptedSeed reads the encrypted seed ciphertext from disk. +func LoadEncryptedSeed(path string) ([]byte, error) { + data, err := os.ReadFile(path) //nolint:gosec // G304 + if err != nil { + return nil, fmt.Errorf("reading seed file %q: %w", path, err) + } + + return data, nil +} + +// SeedFileExists returns true if an encrypted seed file exists at the expected +// path within the network data directory. +func SeedFileExists(networkDir string) bool { + path := SeedFilePath(networkDir) + + _, err := os.Stat(path) + + return err == nil +} diff --git a/darepod/seed_storage_wasm.go b/darepod/seed_storage_wasm.go new file mode 100644 index 000000000..79250aef9 --- /dev/null +++ b/darepod/seed_storage_wasm.go @@ -0,0 +1,62 @@ +//go:build js && wasm + +package darepod + +import ( + "encoding/base64" + "fmt" + "syscall/js" +) + +const wasmSeedStoragePrefix = "darepod:encrypted-seed:" + +// SaveEncryptedSeed stores the encrypted seed in browser localStorage. The +// payload is already password-encrypted by EncryptSeed before this boundary. +func SaveEncryptedSeed(path string, ciphertext []byte) error { + storage := js.Global().Get("localStorage") + if storage.IsUndefined() || storage.IsNull() { + return fmt.Errorf("browser localStorage is unavailable") + } + + storage.Call( + "setItem", wasmSeedStoragePrefix+path, + base64.StdEncoding.EncodeToString(ciphertext), + ) + + return nil +} + +// LoadEncryptedSeed reads the encrypted seed ciphertext from browser +// localStorage. +func LoadEncryptedSeed(path string) ([]byte, error) { + storage := js.Global().Get("localStorage") + if storage.IsUndefined() || storage.IsNull() { + return nil, fmt.Errorf("browser localStorage is unavailable") + } + + value := storage.Call("getItem", wasmSeedStoragePrefix+path) + if value.IsNull() || value.IsUndefined() { + return nil, fmt.Errorf("reading seed file %q: not found", path) + } + + data, err := base64.StdEncoding.DecodeString(value.String()) + if err != nil { + return nil, fmt.Errorf("decode seed file %q: %w", path, err) + } + + return data, nil +} + +// SeedFileExists returns true if an encrypted seed exists in browser +// localStorage for the network data directory. +func SeedFileExists(networkDir string) bool { + path := SeedFilePath(networkDir) + storage := js.Global().Get("localStorage") + if storage.IsUndefined() || storage.IsNull() { + return false + } + + value := storage.Call("getItem", wasmSeedStoragePrefix+path) + + return !value.IsNull() && !value.IsUndefined() +} diff --git a/darepod/server.go b/darepod/server.go index 01f547aec..6673efcef 100644 --- a/darepod/server.go +++ b/darepod/server.go @@ -3292,7 +3292,7 @@ func (s *Server) handleInboundRPC(ctx context.Context, func (s *Server) initDatabase(ctx context.Context) error { networkDir := s.cfg.NetworkDir() - if err := os.MkdirAll(networkDir, 0700); err != nil { + if err := ensureDataDir(networkDir); err != nil { return fmt.Errorf("unable to create data dir: %w", err) } diff --git a/db/migrate/driver_native.go b/db/migrate/driver_native.go new file mode 100644 index 000000000..a0e53e336 --- /dev/null +++ b/db/migrate/driver_native.go @@ -0,0 +1,37 @@ +//go:build !js || !wasm + +package migrate + +import ( + "database/sql" + "fmt" + + "github.com/golang-migrate/migrate/v4/database" + postgresmigrate "github.com/golang-migrate/migrate/v4/database/postgres" + sqlitemigrate "github.com/golang-migrate/migrate/v4/database/sqlite" + "github.com/lightninglabs/darepo-client/db/sqlc" +) + +// newMigrationDriver creates a native migration driver for the given backend. +func newMigrationDriver(db *sql.DB, backend sqlc.BackendType, + migrationsTable string) (database.Driver, error) { + + switch backend { + case sqlc.BackendTypeSqlite: + cfg := &sqlitemigrate.Config{ + MigrationsTable: migrationsTable, + } + + return sqlitemigrate.WithInstance(db, cfg) + + case sqlc.BackendTypePostgres: + cfg := &postgresmigrate.Config{ + MigrationsTable: migrationsTable, + } + + return postgresmigrate.WithInstance(db, cfg) + + default: + return nil, fmt.Errorf("unsupported backend: %v", backend) + } +} diff --git a/db/migrate/driver_wasm.go b/db/migrate/driver_wasm.go new file mode 100644 index 000000000..b814eef13 --- /dev/null +++ b/db/migrate/driver_wasm.go @@ -0,0 +1,25 @@ +//go:build js && wasm + +package migrate + +import ( + "database/sql" + "fmt" + + "github.com/golang-migrate/migrate/v4/database" + "github.com/lightninglabs/darepo-client/db/sqlc" +) + +// newMigrationDriver creates a browser-compatible migration driver. +func newMigrationDriver(db *sql.DB, backend sqlc.BackendType, + migrationsTable string) (database.Driver, error) { + + switch backend { + case sqlc.BackendTypeSqlite: + return newWASMSQLiteMigrationDriver(db, migrationsTable) + + default: + return nil, fmt.Errorf("unsupported wasm migration backend: %v", + backend) + } +} diff --git a/db/migrate/migrations.go b/db/migrate/migrations.go index 7c2a1bf0d..18ba977ac 100644 --- a/db/migrate/migrations.go +++ b/db/migrate/migrations.go @@ -13,9 +13,6 @@ import ( "github.com/btcsuite/btclog/v2" golangmigrate "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database" - postgresmigrate "github.com/golang-migrate/migrate/v4/database/postgres" - sqlitemigrate "github.com/golang-migrate/migrate/v4/database/sqlite" "github.com/golang-migrate/migrate/v4/source/iofs" "github.com/lightninglabs/darepo-client/db/sqlc" ) @@ -209,30 +206,6 @@ func verifyVersionState(mig *golangmigrate.Migrate, return int(version), nil } -// newMigrationDriver creates a migration driver for the given backend. -func newMigrationDriver(db *sql.DB, backend sqlc.BackendType, - migrationsTable string) (database.Driver, error) { - - switch backend { - case sqlc.BackendTypeSqlite: - cfg := &sqlitemigrate.Config{ - MigrationsTable: migrationsTable, - } - - return sqlitemigrate.WithInstance(db, cfg) - - case sqlc.BackendTypePostgres: - cfg := &postgresmigrate.Config{ - MigrationsTable: migrationsTable, - } - - return postgresmigrate.WithInstance(db, cfg) - - default: - return nil, fmt.Errorf("unsupported backend: %v", backend) - } -} - // migrationLogger wraps a btclog.Logger for golang-migrate logging. type migrationLogger struct { log btclog.Logger diff --git a/db/migrate/sqlite_wasm_driver.go b/db/migrate/sqlite_wasm_driver.go new file mode 100644 index 000000000..dfd88c681 --- /dev/null +++ b/db/migrate/sqlite_wasm_driver.go @@ -0,0 +1,258 @@ +//go:build js && wasm + +package migrate + +import ( + "database/sql" + "errors" + "fmt" + "io" + "strings" + "sync/atomic" + + "github.com/golang-migrate/migrate/v4/database" +) + +// wasmSQLiteDriver implements golang-migrate's database.Driver contract for +// the browser-backed wasmsqlite database/sql driver. +type wasmSQLiteDriver struct { + db *sql.DB + migrationsTable string + isLocked atomic.Bool +} + +// newWASMSQLiteMigrationDriver creates a migration driver that avoids +// importing golang-migrate's modernc-backed sqlite driver in js/wasm builds. +func newWASMSQLiteMigrationDriver(db *sql.DB, + migrationsTable string) (database.Driver, error) { + + if db == nil { + return nil, fmt.Errorf("db is nil") + } + if migrationsTable == "" { + migrationsTable = "schema_migrations" + } + + driver := &wasmSQLiteDriver{ + db: db, + migrationsTable: migrationsTable, + } + if err := driver.ensureVersionTable(); err != nil { + return nil, err + } + + return driver, nil +} + +// Open is unsupported because callers provide an already-open browser DB. +func (d *wasmSQLiteDriver) Open(string) (database.Driver, error) { + return nil, fmt.Errorf("open is unsupported for wasm sqlite migrations") +} + +// Close leaves the caller-owned database handle open. +func (d *wasmSQLiteDriver) Close() error { + return nil +} + +// Lock acquires a process-local migration lock. +func (d *wasmSQLiteDriver) Lock() error { + if !d.isLocked.CompareAndSwap(false, true) { + return database.ErrLocked + } + + return nil +} + +// Unlock releases a process-local migration lock. +func (d *wasmSQLiteDriver) Unlock() error { + if !d.isLocked.CompareAndSwap(true, false) { + return database.ErrNotLocked + } + + return nil +} + +// Run executes one migration inside a transaction. +func (d *wasmSQLiteDriver) Run(migration io.Reader) error { + migrationBytes, err := io.ReadAll(migration) + if err != nil { + return err + } + + query := string(migrationBytes) + tx, err := d.db.Begin() + if err != nil { + return &database.Error{ + OrigErr: err, + Err: "transaction start failed", + } + } + + if _, err := tx.Exec(query); err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + err = errors.Join(err, rollbackErr) + } + + return &database.Error{ + OrigErr: err, + Query: migrationBytes, + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{ + OrigErr: err, + Err: "transaction commit failed", + } + } + + return nil +} + +// SetVersion updates the migration bookkeeping row. +func (d *wasmSQLiteDriver) SetVersion(version int, dirty bool) error { + tx, err := d.db.Begin() + if err != nil { + return &database.Error{ + OrigErr: err, + Err: "transaction start failed", + } + } + + deleteQuery := fmt.Sprintf("DELETE FROM %s", + quoteSQLiteIdentifier(d.migrationsTable)) + if _, err := tx.Exec(deleteQuery); err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + err = errors.Join(err, rollbackErr) + } + + return &database.Error{ + OrigErr: err, + Query: []byte(deleteQuery), + } + } + + if version >= 0 || (version == database.NilVersion && dirty) { + insertQuery := fmt.Sprintf("INSERT INTO %s (version, dirty) "+ + "VALUES (?, ?)", + quoteSQLiteIdentifier(d.migrationsTable)) + if _, err := tx.Exec(insertQuery, version, dirty); err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + err = errors.Join(err, rollbackErr) + } + + return &database.Error{ + OrigErr: err, + Query: []byte(insertQuery), + } + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{ + OrigErr: err, + Err: "transaction commit failed", + } + } + + return nil +} + +// Version returns the current migration version. +func (d *wasmSQLiteDriver) Version() (int, bool, error) { + query := fmt.Sprintf("SELECT version, dirty FROM %s LIMIT 1", + quoteSQLiteIdentifier(d.migrationsTable)) + + var version int + var dirty bool + err := d.db.QueryRow(query).Scan(&version, &dirty) + switch { + case errors.Is(err, sql.ErrNoRows): + return database.NilVersion, false, nil + + case err != nil: + return database.NilVersion, false, &database.Error{ + OrigErr: err, + Query: []byte(query), + } + + default: + return version, dirty, nil + } +} + +// Drop drops user tables and vacuums the database. +func (d *wasmSQLiteDriver) Drop() error { + const query = `SELECT name FROM sqlite_master WHERE type = 'table' ` + + `AND name NOT LIKE 'sqlite_%'` + + rows, err := d.db.Query(query) + if err != nil { + return &database.Error{ + OrigErr: err, + Query: []byte(query), + } + } + defer rows.Close() + + var tables []string + for rows.Next() { + var table string + if err := rows.Scan(&table); err != nil { + return &database.Error{ + OrigErr: err, + Query: []byte(query), + } + } + + tables = append(tables, table) + } + if err := rows.Err(); err != nil { + return &database.Error{ + OrigErr: err, + Query: []byte(query), + } + } + + for _, table := range tables { + dropQuery := "DROP TABLE " + quoteSQLiteIdentifier(table) + if _, err := d.db.Exec(dropQuery); err != nil { + return &database.Error{ + OrigErr: err, + Query: []byte(dropQuery), + } + } + } + + if _, err := d.db.Exec("VACUUM"); err != nil { + return &database.Error{ + OrigErr: err, + Query: []byte("VACUUM"), + } + } + + return nil +} + +func (d *wasmSQLiteDriver) ensureVersionTable() error { + table := quoteSQLiteIdentifier(d.migrationsTable) + index := quoteSQLiteIdentifier(d.migrationsTable + "_version_unique") + query := fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s (version INTEGER, dirty BOOLEAN); +CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (version); +`, + table, index, table) + + if _, err := d.db.Exec(query); err != nil { + return &database.Error{ + OrigErr: err, + Query: []byte(query), + } + } + + return nil +} + +func quoteSQLiteIdentifier(identifier string) string { + return `"` + strings.ReplaceAll(identifier, `"`, `""`) + `"` +} diff --git a/db/postgres.go b/db/postgres.go index 3b204407a..e030ff266 100644 --- a/db/postgres.go +++ b/db/postgres.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "log/slog" - "testing" "time" "github.com/btcsuite/btclog/v2" @@ -15,7 +14,6 @@ import ( dbmigrate "github.com/lightninglabs/darepo-client/db/migrate" "github.com/lightninglabs/darepo-client/db/sqlc" fn "github.com/lightningnetwork/lnd/fn/v2" - "github.com/stretchr/testify/require" ) const ( @@ -30,13 +28,6 @@ const ( ) var ( - // DefaultPostgresFixtureLifetime is the default maximum time a Postgres - // test fixture is being kept alive. After that time the docker - // container will be terminated forcefully, even if the tests aren't - // fully executed yet. So this time needs to be chosen correctly to be - // longer than the longest expected individual test run time. - DefaultPostgresFixtureLifetime = 60 * time.Minute - // postgresSchemaReplacements is a map of schema strings that need to be // replaced for postgres. This is needed because we write the schemas // to work with sqlite primarily, and postgres has some differences. @@ -230,54 +221,3 @@ func (s *PostgresStore) ExecuteMigrations(target MigrationTarget, return nil } - -// NewTestPostgresDB is a helper function that creates a Postgres database for -// testing. -func NewTestPostgresDB(t testing.TB) *PostgresStore { - t.Helper() - - t.Logf("Creating new Postgres DB for testing") - - // For tests, use a simple logger that outputs to the test log. - log := btclog.Disabled - - sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime, true) - store, err := NewPostgresStore(sqlFixture.GetConfig(), log) - require.NoError(t, err) - - t.Cleanup(func() { - sqlFixture.TearDown(t) - }) - - return store -} - -// NewTestPostgresDBWithVersion is a helper function that creates a Postgres -// database for testing and migrates it to the given version. -func NewTestPostgresDBWithVersion(t testing.TB, version uint) *PostgresStore { - t.Helper() - - t.Logf( - "Creating new Postgres DB for testing, migrating to version %d", - version, - ) - - // For tests, use a simple logger that outputs to the test log. - log := btclog.Disabled - - sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime, true) - storeCfg := sqlFixture.GetConfig() - storeCfg.SkipMigrations = true - - store, err := NewPostgresStore(storeCfg, log) - require.NoError(t, err) - - err = store.ExecuteMigrations(TargetVersion(version)) - require.NoError(t, err) - - t.Cleanup(func() { - sqlFixture.TearDown(t) - }) - - return store -} diff --git a/db/postgres_fixture.go b/db/postgres_fixture.go index a1f8e7549..780b1cf73 100644 --- a/db/postgres_fixture.go +++ b/db/postgres_fixture.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + package db import ( diff --git a/db/postgres_test_helpers.go b/db/postgres_test_helpers.go new file mode 100644 index 000000000..415dd28cd --- /dev/null +++ b/db/postgres_test_helpers.go @@ -0,0 +1,71 @@ +//go:build !js || !wasm + +package db + +import ( + "testing" + "time" + + "github.com/btcsuite/btclog/v2" + "github.com/stretchr/testify/require" +) + +var ( + // DefaultPostgresFixtureLifetime is the default maximum time a Postgres + // test fixture is being kept alive. After that time the docker + // container will be terminated forcefully, even if the tests aren't + // fully executed yet. So this time needs to be chosen correctly to be + // longer than the longest expected individual test run time. + DefaultPostgresFixtureLifetime = 60 * time.Minute +) + +// NewTestPostgresDB is a helper function that creates a Postgres database for +// testing. +func NewTestPostgresDB(t testing.TB) *PostgresStore { + t.Helper() + + t.Logf("Creating new Postgres DB for testing") + + // For tests, use a simple logger that outputs to the test log. + log := btclog.Disabled + + sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime, true) + store, err := NewPostgresStore(sqlFixture.GetConfig(), log) + require.NoError(t, err) + + t.Cleanup(func() { + sqlFixture.TearDown(t) + }) + + return store +} + +// NewTestPostgresDBWithVersion is a helper function that creates a Postgres +// database for testing and migrates it to the given version. +func NewTestPostgresDBWithVersion(t testing.TB, version uint) *PostgresStore { + t.Helper() + + t.Logf( + "Creating new Postgres DB for testing, migrating to version %d", + version, + ) + + // For tests, use a simple logger that outputs to the test log. + log := btclog.Disabled + + sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime, true) + storeCfg := sqlFixture.GetConfig() + storeCfg.SkipMigrations = true + + store, err := NewPostgresStore(storeCfg, log) + require.NoError(t, err) + + err = store.ExecuteMigrations(TargetVersion(version)) + require.NoError(t, err) + + t.Cleanup(func() { + sqlFixture.TearDown(t) + }) + + return store +} diff --git a/db/sqlerrors.go b/db/sqlerrors.go index 154859b41..745cbd33d 100644 --- a/db/sqlerrors.go +++ b/db/sqlerrors.go @@ -9,8 +9,6 @@ import ( pgconnv4 "github.com/jackc/pgconn" "github.com/jackc/pgerrcode" pgconnv5 "github.com/jackc/pgx/v5/pgconn" - "modernc.org/sqlite" - sqlite3 "modernc.org/sqlite/lib" ) // isDBClosedError reports whether err indicates the underlying sql handle @@ -52,10 +50,8 @@ var ( // MapSQLError attempts to interpret a given error as a database agnostic SQL // error. func MapSQLError(err error) error { - // Attempt to interpret the error as a sqlite error. - var sqliteErr *sqlite.Error - if errors.As(err, &sqliteErr) { - return parseSqliteError(sqliteErr) + if mapped := mapSQLiteError(err); mapped != nil { + return mapped } // Attempt to interpret the error as a postgres error. The pgx v4 and @@ -80,49 +76,6 @@ func MapSQLError(err error) error { return err } -// parseSqliteError attempts to parse a sqlite error as a database agnostic -// SQL error. -func parseSqliteError(sqliteErr *sqlite.Error) error { - switch sqliteErr.Code() { - // Handle unique constraint violation error. - case sqlite3.SQLITE_CONSTRAINT_UNIQUE, - sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: - return &ErrSQLUniqueConstraintViolation{ - DBError: sqliteErr, - } - - // Database is currently busy, so we'll need to try again. - case sqlite3.SQLITE_BUSY: - return &ErrSerializationError{ - DBError: sqliteErr, - } - - // A write operation could not continue because of a conflict within - // the same database connection. - case sqlite3.SQLITE_LOCKED, sqlite3.SQLITE_BUSY_SNAPSHOT: - return &ErrDeadlockError{ - DBError: sqliteErr, - } - - // Generic error, need to parse the message further. - case sqlite3.SQLITE_ERROR: - errMsg := sqliteErr.Error() - - switch { - case strings.Contains(errMsg, "no such table"): - return &ErrSchemaError{ - DBError: sqliteErr, - } - - default: - return fmt.Errorf("unknown sqlite error: %w", sqliteErr) - } - - default: - return fmt.Errorf("unknown sqlite error: %w", sqliteErr) - } -} - // classifyPostgresError maps a postgres SQLSTATE code to a database agnostic // SQL error. func classifyPostgresError(code string, dbErr error) error { diff --git a/db/sqlerrors_native.go b/db/sqlerrors_native.go new file mode 100644 index 000000000..d2e4782cc --- /dev/null +++ b/db/sqlerrors_native.go @@ -0,0 +1,52 @@ +//go:build !js || !wasm + +package db + +import ( + "errors" + "fmt" + "strings" + + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" +) + +// mapSQLiteError attempts to parse native SQLite errors as database agnostic +// SQL errors. +func mapSQLiteError(err error) error { + var sqliteErr *sqlite.Error + if !errors.As(err, &sqliteErr) { + return nil + } + + switch sqliteErr.Code() { + case sqlite3.SQLITE_CONSTRAINT_UNIQUE, + sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: + return &ErrSQLUniqueConstraintViolation{ + DBError: sqliteErr, + } + + case sqlite3.SQLITE_BUSY: + return &ErrSerializationError{ + DBError: sqliteErr, + } + + case sqlite3.SQLITE_LOCKED, sqlite3.SQLITE_BUSY_SNAPSHOT: + return &ErrDeadlockError{ + DBError: sqliteErr, + } + + case sqlite3.SQLITE_ERROR: + errMsg := sqliteErr.Error() + if strings.Contains(errMsg, "no such table") { + return &ErrSchemaError{ + DBError: sqliteErr, + } + } + + return fmt.Errorf("unknown sqlite error: %w", sqliteErr) + + default: + return fmt.Errorf("unknown sqlite error: %w", sqliteErr) + } +} diff --git a/db/sqlerrors_wasm.go b/db/sqlerrors_wasm.go new file mode 100644 index 000000000..bfa65941a --- /dev/null +++ b/db/sqlerrors_wasm.go @@ -0,0 +1,42 @@ +//go:build js && wasm + +package db + +import "strings" + +// mapSQLiteError classifies browser SQLite errors using stable message +// fragments surfaced through the wasmsqlite bridge. +func mapSQLiteError(err error) error { + if err == nil { + return nil + } + + msg := strings.ToLower(err.Error()) + switch { + case strings.Contains(msg, "unique constraint failed"), + strings.Contains(msg, "constraint failed"), + strings.Contains(msg, "primary key"): + return &ErrSQLUniqueConstraintViolation{ + DBError: err, + } + + case strings.Contains(msg, "database is locked"), + strings.Contains(msg, "database table is locked"): + return &ErrDeadlockError{ + DBError: err, + } + + case strings.Contains(msg, "database is busy"): + return &ErrSerializationError{ + DBError: err, + } + + case strings.Contains(msg, "no such table"): + return &ErrSchemaError{ + DBError: err, + } + + default: + return nil + } +} diff --git a/db/sqlite.go b/db/sqlite.go index c1d44f76b..5e0a7a7b0 100644 --- a/db/sqlite.go +++ b/db/sqlite.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "log/slog" - "net/url" "path/filepath" "strconv" "strings" @@ -19,19 +18,9 @@ import ( "github.com/lightninglabs/darepo-client/db/sqlc" fn "github.com/lightningnetwork/lnd/fn/v2" "github.com/stretchr/testify/require" - _ "modernc.org/sqlite" // Register relevant drivers. ) const ( - // sqliteOptionPrefix is the string prefix sqlite uses to set various - // options. This is used in the following format: - // * sqliteOptionPrefix || option_name = option_value. - sqliteOptionPrefix = "_pragma" - - // sqliteTxLockImmediate is a dsn option used to ensure that write - // transactions are started immediately. - sqliteTxLockImmediate = "_txlock=immediate" - // defaultMaxConns is the number of permitted active and idle // connections. We want to limit this so it isn't unlimited. We use the // same value for the number of idle connections as, this can speed up @@ -142,17 +131,14 @@ func NewSqliteStore(cfg *SqliteConfig, // The set of pragma options are accepted using query options. For now // we only want to ensure that foreign key constraints are properly // enforced. - pragmaOptions := []struct { - name string - value string - }{ + pragmaOptions := []SQLitePragma{ { - name: "foreign_keys", - value: "on", + Name: "foreign_keys", + Value: "on", }, { - name: "journal_mode", - value: "WAL", + Name: "journal_mode", + Value: "WAL", }, { // busy_timeout caps how long SQLite will wait on @@ -167,8 +153,8 @@ func NewSqliteStore(cfg *SqliteConfig, // begin-tx failures, which masquerade as "mailbox // full" or "Failed to lease message" upstream and // confuse production diagnosis. - name: "busy_timeout", - value: "30000", + Name: "busy_timeout", + Value: "30000", }, { // The synchronous pragma governs commit durability. @@ -180,8 +166,8 @@ func NewSqliteStore(cfg *SqliteConfig, // configurable so operators can trade durability for // performance. See resolveSqliteSynchronous for the // accepted values. - name: "synchronous", - value: synchronous, + Name: "synchronous", + Value: synchronous, }, { // fullfsync uses the correct fsync system call on macOS @@ -192,24 +178,10 @@ func NewSqliteStore(cfg *SqliteConfig, // F_FULLFSYNC waits on a full hardware cache flush, so // the config exposes an opt-out for write-heavy // deployments. Enabled by default. - name: "fullfsync", - value: strconv.FormatBool(!cfg.NoFullfsync), + Name: "fullfsync", + Value: strconv.FormatBool(!cfg.NoFullfsync), }, } - sqliteOptions := make(url.Values) - for _, option := range pragmaOptions { - sqliteOptions.Add( - sqliteOptionPrefix, - fmt.Sprintf("%v=%v", option.name, option.value), - ) - } - - // Construct the DSN which is just the database file name, appended - // with the series of pragma options as a query URL string. For more - // details on the formatting here, see the modernc.org/sqlite docs: - // https://pkg.go.dev/modernc.org/sqlite#Driver.Open. - dsn := fmt.Sprintf("%v?%v&%v", cfg.DatabaseFileName, - sqliteOptions.Encode(), sqliteTxLockImmediate) ctx := context.Background() storeLog.InfoS(ctx, "Opening SQLite database", @@ -220,16 +192,23 @@ func NewSqliteStore(cfg *SqliteConfig, slog.Duration("conn_max_lifetime", defaultConnMaxLifetime), ) - db, err := sql.Open("sqlite", dsn) + openResult, err := OpenSQLiteDatabase(SQLiteOpenConfig{ + DatabaseFileName: cfg.DatabaseFileName, + Pragmas: pragmaOptions, + TxLockImmediate: true, + MaxOpenConns: defaultMaxConns, + MaxIdleConns: defaultMaxConns, + ConnMaxLifetime: defaultConnMaxLifetime, + }) if err != nil { return nil, err } - db.SetMaxOpenConns(defaultMaxConns) - db.SetMaxIdleConns(defaultMaxConns) - db.SetConnMaxLifetime(defaultConnMaxLifetime) + db := openResult.DB - storeLog.DebugS(ctx, "SQLite connection pool configured") + storeLog.DebugS(ctx, "SQLite connection pool configured", + slog.String("driver", openResult.DriverName), + ) // Persist the resolved logger into the config option so the // logger(ctx) helper can retrieve it without keeping a separate diff --git a/db/sqlite_open.go b/db/sqlite_open.go new file mode 100644 index 000000000..072e8e98e --- /dev/null +++ b/db/sqlite_open.go @@ -0,0 +1,50 @@ +package db + +import ( + "database/sql" + "time" +) + +// SQLitePragma is one PRAGMA setting applied when a SQLite database opens. +type SQLitePragma struct { + Name string + Value string +} + +// SQLiteOpenConfig contains the driver-neutral SQLite open settings shared by +// the native and browser-backed database handles. +type SQLiteOpenConfig struct { + // DatabaseFileName is the native filename or logical browser OPFS name. + DatabaseFileName string + + // Pragmas are applied by the selected driver at open time when + // possible. + Pragmas []SQLitePragma + + // TxLockImmediate requests immediate write transactions when the driver + // supports that mode. + TxLockImmediate bool + + // MaxOpenConns bounds the database/sql connection pool. + MaxOpenConns int + + // MaxIdleConns bounds idle connections in the database/sql pool. + MaxIdleConns int + + // ConnMaxLifetime limits how long one SQL connection is reused. + ConnMaxLifetime time.Duration +} + +// SQLiteOpenResult returns the opened SQL handle and driver details useful for +// logging and tests. +type SQLiteOpenResult struct { + DB *sql.DB + DriverName string + DSN string +} + +// OpenSQLiteDatabase opens a SQLite handle using the driver selected for the +// current build target. +func OpenSQLiteDatabase(cfg SQLiteOpenConfig) (*SQLiteOpenResult, error) { + return openSQLiteDatabase(cfg) +} diff --git a/db/sqlite_open_native.go b/db/sqlite_open_native.go new file mode 100644 index 000000000..f3ccd0a4f --- /dev/null +++ b/db/sqlite_open_native.go @@ -0,0 +1,63 @@ +//go:build !js || !wasm + +package db + +import ( + "database/sql" + "fmt" + "net/url" + + _ "modernc.org/sqlite" // Register native SQLite driver. +) + +const ( + // sqliteOptionPrefix is the modernc SQLite DSN prefix for pragma + // settings. + sqliteOptionPrefix = "_pragma" + + // sqliteTxLockImmediate starts write transactions immediately. + sqliteTxLockImmediate = "_txlock=immediate" +) + +// openSQLiteDatabase opens SQLite through the native modernc driver. +func openSQLiteDatabase(cfg SQLiteOpenConfig) (*SQLiteOpenResult, error) { + sqliteOptions := make(url.Values) + for _, pragma := range cfg.Pragmas { + sqliteOptions.Add( + sqliteOptionPrefix, + fmt.Sprintf("%s=%s", pragma.Name, pragma.Value), + ) + } + + dsn := fmt.Sprintf("%s?%s", cfg.DatabaseFileName, + sqliteOptions.Encode()) + if cfg.TxLockImmediate { + dsn = fmt.Sprintf("%s&%s", dsn, sqliteTxLockImmediate) + } + + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + + configureSQLitePool(db, cfg) + + return &SQLiteOpenResult{ + DB: db, + DriverName: "sqlite", + DSN: dsn, + }, nil +} + +// configureSQLitePool applies database/sql pool settings when present. +func configureSQLitePool(db *sql.DB, cfg SQLiteOpenConfig) { + if cfg.MaxOpenConns > 0 { + db.SetMaxOpenConns(cfg.MaxOpenConns) + } + if cfg.MaxIdleConns > 0 { + db.SetMaxIdleConns(cfg.MaxIdleConns) + } + if cfg.ConnMaxLifetime > 0 { + db.SetConnMaxLifetime(cfg.ConnMaxLifetime) + } +} diff --git a/db/sqlite_open_wasm.go b/db/sqlite_open_wasm.go new file mode 100644 index 000000000..5a483a0d3 --- /dev/null +++ b/db/sqlite_open_wasm.go @@ -0,0 +1,129 @@ +//go:build js && wasm + +package db + +import ( + "database/sql" + "fmt" + "hash/fnv" + "net/url" + "path/filepath" + "strings" + "time" + + _ "github.com/lightninglabs/go-wasmsqlite" +) + +const ( + wasmSQLiteDriverName = "wasmsqlite" + wasmSQLiteVFS = "opfs" +) + +// openSQLiteDatabase opens SQLite through the browser-backed wasmsqlite +// driver. +func openSQLiteDatabase(cfg SQLiteOpenConfig) (*SQLiteOpenResult, error) { + values := url.Values{} + values.Set("file", browserSQLiteFileName(cfg.DatabaseFileName)) + values.Set("vfs", wasmSQLiteVFS) + values.Set("mode", "rwc") + + pragmas := make([]string, 0, len(cfg.Pragmas)+1) + for _, pragma := range cfg.Pragmas { + switch strings.ToLower(pragma.Name) { + case "busy_timeout": + values.Set("busy_timeout", pragma.Value) + + case "journal_mode": + values.Set("journal_mode", pragma.Value) + + case "fullfsync": + // fullfsync is a native filesystem durability hint and + // is not meaningful for browser OPFS. + + default: + pragmas = append( + pragmas, pragma.Name+"="+pragma.Value, + ) + } + } + + pragmas = append(pragmas, "locking_mode=EXCLUSIVE") + values.Set("pragma", strings.Join(pragmas, ";")) + + dsn := values.Encode() + db, err := openWASMSQLiteWithRetry(dsn) + if err != nil { + return nil, err + } + + return &SQLiteOpenResult{ + DB: db, + DriverName: wasmSQLiteDriverName, + DSN: dsn, + }, nil +} + +// openWASMSQLiteWithRetry smooths over reload-time OPFS release races. Each +// retry uses a fresh database/sql handle because a failed go-wasmsqlite open +// can leave the worker tracking the filename as open. +func openWASMSQLiteWithRetry(dsn string) (*sql.DB, error) { + var lastErr error + + for attempt := 0; attempt < 25; attempt++ { + db, err := sql.Open(wasmSQLiteDriverName, dsn) + if err != nil { + return nil, err + } + + // OPFS SQLite handles must be single-connection handles. + // Multiple SQL connections would race the same browser database + // through one worker. + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + + err = db.Ping() + if err == nil { + return db, nil + } + + _ = db.Close() + if !isWASMCantOpen(err) { + return nil, err + } + + lastErr = err + time.Sleep(200 * time.Millisecond) + } + + return nil, lastErr +} + +// isWASMCantOpen identifies the SQLite error returned while OPFS still holds a +// file lock from a just-unloaded page runtime. +func isWASMCantOpen(err error) bool { + return strings.Contains(err.Error(), "SQLITE_CANTOPEN") || + strings.Contains(err.Error(), "unable to open database file") +} + +// browserSQLiteFileName maps native paths to stable origin-local OPFS names. +// The full path is hashed into the name, not just its basename, so databases +// that share a basename across different data dirs or networks (e.g. the +// regtest and signet client.db, or two swaps.db) map to distinct OPFS files +// within one browser origin instead of silently colliding. This mirrors the +// scheme lwwallet uses for its own OPFS wallet database. +func browserSQLiteFileName(name string) string { + normalized := filepath.ToSlash(filepath.Clean(name)) + base := filepath.Base(normalized) + if base == "." || base == "/" || base == "" { + base = "arkd.db" + normalized = base + } + + hasher := fnv.New64a() + _, _ = hasher.Write([]byte(normalized)) + + ext := filepath.Ext(base) + stem := strings.TrimSuffix(base, ext) + + return fmt.Sprintf("/%s-%016x%s", stem, hasher.Sum64(), ext) +} diff --git a/go.mod b/go.mod index d0ada9e5f..4e422f4b6 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/btcsuite/btcwallet v0.16.17 github.com/btcsuite/btcwallet/walletdb v1.5.1 github.com/btcsuite/btcwallet/wtxmgr v1.5.6 - github.com/golang-migrate/migrate/v4 v4.17.0 + github.com/golang-migrate/migrate/v4 v4.19.1 github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 @@ -57,6 +57,8 @@ require ( pgregory.net/rapid v1.2.0 ) +require github.com/lightninglabs/go-wasmsqlite v0.0.0-20260627090804-0dce68fc5287 + require ( dario.cat/mergo v1.0.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect @@ -138,7 +140,7 @@ require ( github.com/lightningnetwork/lnd/cert v1.2.2 // indirect github.com/lightningnetwork/lnd/healthcheck v1.2.6 // indirect github.com/lightningnetwork/lnd/queue v1.1.2-0.20260417172355-dfb7d57826c7 // indirect - github.com/lightningnetwork/lnd/sqldb v1.0.13-0.20260417172355-dfb7d57826c7 // indirect + github.com/lightningnetwork/lnd/sqldb v1.0.13-0.20260417172355-dfb7d57826c7 github.com/lightningnetwork/lnd/ticker v1.1.1 // indirect github.com/lightningnetwork/lnd/tor v1.1.6 // indirect github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796 // indirect @@ -207,7 +209,7 @@ require ( golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/sys v0.43.0 // indirect golang.org/x/text v0.36.0 // indirect golang.org/x/tools v0.43.0 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect diff --git a/go.sum b/go.sum index 8f422e2d5..ae36733cf 100644 --- a/go.sum +++ b/go.sum @@ -709,6 +709,12 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 h1:UQ4AU+BGti3Sy/aLU8KVseYKNALcX9UXY6DfpwQ6J8E= +github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= +github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= +github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -818,6 +824,8 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -842,6 +850,12 @@ github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqw github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -1112,6 +1126,8 @@ github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lightninglabs/go-wasmsqlite v0.0.0-20260627090804-0dce68fc5287 h1:q/7nFGq1Eln2DrV51SddoGP54Yfgvuxq6JmQHG+D8LA= +github.com/lightninglabs/go-wasmsqlite v0.0.0-20260627090804-0dce68fc5287/go.mod h1:qtLtxJq5KD9oP2I4lj0qRIWCU/fN8eNfTrHie4YrKTQ= github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf h1:HZKvJUHlcXI/f/O0Avg7t8sqkPo78HFzjmeYFl6DPnc= github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf/go.mod h1:vxmQPeIQxPf6Jf9rM8R+B4rKBqLA2AjttNxkFBL2Plk= github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.4-0.20250610182311-2f1d46ef18b7 h1:373o5lNr1udAdhcf5+zq/0dYpRtkvYLl8Lk6wG7I0DY= @@ -1752,8 +1768,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/sqlbase/LICENSE b/internal/sqlbase/LICENSE new file mode 100644 index 000000000..cfab3dac7 --- /dev/null +++ b/internal/sqlbase/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2015-2022 Lightning Labs and The Lightning Network Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/internal/sqlbase/db.go b/internal/sqlbase/db.go new file mode 100644 index 000000000..4c0b38a67 --- /dev/null +++ b/internal/sqlbase/db.go @@ -0,0 +1,317 @@ +//go:build js && wasm + +package sqlbase + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "strings" + "sync" + "time" + + "github.com/btcsuite/btcwallet/walletdb" + "github.com/lightningnetwork/lnd/sqldb" +) + +const ( + // kvTableName is the name of the table that will contain all the kv + // pairs. + kvTableName = "kv" + + // DefaultNumTxRetries is the default number of times we'll retry a + // transaction if it fails with an error that permits transaction + // repetition. + DefaultNumTxRetries = 50 +) + +// Config holds a set of configuration options of a sql database connection. +type Config struct { + // DriverName is the string that defines the registered sql driver that + // is to be used. + DriverName string + + // Dsn is the database connection string that will be used to connect + // to the db. + Dsn string + + // Timeout is the time after which a query to the db will be canceled if + // it has not yet completed. + Timeout time.Duration + + // Schema is the name of the schema under which the sql tables should be + // created. It should be left empty for backends like sqlite that do not + // support having more than one schema. + Schema string + + // TableNamePrefix is the name that should be used as a table name + // prefix when constructing the KV style table. + TableNamePrefix string + + // SQLiteCmdReplacements define a one-to-one string mapping of sql + // keywords to the strings that should replace those keywords in any + // commands. Note that the sqlite keywords to be replaced are + // case-sensitive. + SQLiteCmdReplacements SQLiteCmdReplacements + + // WithTxLevelLock when set will ensure that there is a transaction + // level lock. + // + // NOTE: Temporary, should be removed when all parts of the LND code + // are more resilient against concurrent db access.. + WithTxLevelLock bool +} + +// db holds a reference to the sql db connection. +type db struct { + // cfg is the sql db connection config. + cfg *Config + + // prefix is the table name prefix that is used to simulate namespaces. + // We don't use schemas because at least sqlite does not support that. + prefix string + + // ctx is the overall context for the database driver. + // + // TODO: This is an anti-pattern that is in place until the kvdb + // interface supports a context. + ctx context.Context + + // db is the underlying database connection instance. + db *sql.DB + + // table is the name of the table that contains the data for all + // top-level buckets that have keys that cannot be mapped to a distinct + // sql table. + table string + + // lock is the global write lock that ensures single writer. This is + // only used if cfg.WithTxLevelLock is set. + lock sync.RWMutex +} + +// Enforce db implements the walletdb.DB interface. +var _ walletdb.DB = (*db)(nil) + +var ( + // dbConns is a global set of database connections. + dbConns *dbConnSet + dbConnsMu sync.Mutex +) + +// Init initializes the global set of database connections. +func Init(maxConnections int) { + dbConnsMu.Lock() + defer dbConnsMu.Unlock() + + if dbConns != nil { + return + } + + dbConns = newDbConnSet(maxConnections) +} + +// NewSqlBackend returns a db object initialized with the passed backend +// config. If database connection cannot be established, then returns error. +func NewSqlBackend(ctx context.Context, cfg *Config) (*db, error) { + dbConnsMu.Lock() + defer dbConnsMu.Unlock() + + if dbConns == nil { + return nil, errors.New("db connection set not initialized") + } + + if cfg.TableNamePrefix == "" { + return nil, errors.New("empty table name prefix") + } + + table := fmt.Sprintf("%s_%s", cfg.TableNamePrefix, kvTableName) + + query := newKVSchemaCreationCmd( + table, cfg.Schema, cfg.SQLiteCmdReplacements, + ) + + dbConn, err := dbConns.Open(cfg.DriverName, cfg.Dsn) + if err != nil { + return nil, err + } + + _, err = dbConn.ExecContext(ctx, query) + if err != nil { + _ = dbConns.Close(cfg.Dsn) + + return nil, err + } + + return &db{ + cfg: cfg, + ctx: ctx, + db: dbConn, + table: table, + prefix: cfg.TableNamePrefix, + }, nil +} + +// getTimeoutCtx gets a timeout context for database requests. +func (db *db) getTimeoutCtx() (context.Context, func()) { + if db.cfg.Timeout == time.Duration(0) { + return db.ctx, func() {} + } + + return context.WithTimeout(db.ctx, db.cfg.Timeout) +} + +// getPrefixedTableName returns a table name for this prefix (namespace). +func (db *db) getPrefixedTableName(table string) string { + return fmt.Sprintf("%s_%s", db.prefix, table) +} + +// catchPanic executes the specified function. If a panic occurs, it is returned +// as an error value. +func catchPanic(f func() error) (err error) { + defer func() { + if r := recover(); r != nil { + switch data := r.(type) { + case error: + err = data + + default: + err = errors.New(fmt.Sprintf("%v", data)) + } + + // Before we issue a critical log which'll cause the + // daemon to shut down, we'll first check if this is a + // DB serialization error. If so, then we don't need to + // log as we can retry safely and avoid tearing + // everything down. + if sqldb.IsSerializationError(sqldb.MapSQLError(err)) { + log.Tracef("Detected db serialization error "+ + "via panic: %v", err) + } else { + log.Criticalf("Caught unhandled error: %v", r) + } + } + }() + + err = f() + + return +} + +// View opens a database read transaction and executes the function f with the +// transaction passed as a parameter. After f exits, the transaction is rolled +// back. If f errors, its error is returned, not a rollback error (if any +// occur). The passed reset function is called before the start of the +// transaction and can be used to reset intermediate state. As callers may +// expect retries of the f closure (depending on the database backend used), the +// reset function will be called before each retry respectively. +func (db *db) View(f func(tx walletdb.ReadTx) error, reset func()) error { + return db.executeTransaction( + func(tx walletdb.ReadWriteTx) error { + return f(tx.(walletdb.ReadTx)) + }, + reset, true, + ) +} + +// Update opens a database read/write transaction and executes the function f +// with the transaction passed as a parameter. After f exits, if f did not +// error, the transaction is committed. Otherwise, if f did error, the +// transaction is rolled back. If the rollback fails, the original error +// returned by f is still returned. If the commit fails, the commit error is +// returned. As callers may expect retries of the f closure, the reset function +// will be called before each retry respectively. +func (db *db) Update(f func(tx walletdb.ReadWriteTx) error, + reset func()) error { + + return db.executeTransaction(f, reset, false) +} + +// executeTransaction creates a new read-only or read-write transaction and +// executes the given function within it. +func (db *db) executeTransaction(f func(tx walletdb.ReadWriteTx) error, + reset func(), readOnly bool) error { + + makeTx := func() (sqldb.Tx, error) { + return newReadWriteTx(db, readOnly) + } + + execTxBody := func(tx sqldb.Tx) error { + kvTx, ok := tx.(*readWriteTx) + if !ok { + return fmt.Errorf("expected *readWriteTx, got %T", tx) + } + + reset() + return catchPanic(func() error { return f(kvTx) }) + } + + onBackoff := func(retry int, delay time.Duration) { + log.Tracef("Retrying transaction due to tx serialization "+ + "error, attempt_number=%v, delay=%v", retry, delay) + } + + rollbackTx := func(tx sqldb.Tx) error { + kvTx, ok := tx.(*readWriteTx) + if !ok { + return fmt.Errorf("expected *readWriteTx, got %T", tx) + } + + return attemptRollback(kvTx) + } + + return sqldb.ExecuteSQLTransactionWithRetry( + db.ctx, makeTx, rollbackTx, execTxBody, onBackoff, + DefaultNumTxRetries, + ) +} + +// PrintStats returns all collected stats pretty printed into a string. +func (db *db) PrintStats() string { + return "stats not supported by SQL driver" +} + +// BeginReadWriteTx opens a database read+write transaction. +func (db *db) BeginReadWriteTx() (walletdb.ReadWriteTx, error) { + return newReadWriteTx(db, false) +} + +// BeginReadTx opens a database read transaction. +func (db *db) BeginReadTx() (walletdb.ReadTx, error) { + return newReadWriteTx(db, true) +} + +// Copy writes a copy of the database to the provided writer. This call will +// start a read-only transaction to perform all operations. +// This function is part of the walletdb.Db interface implementation. +func (db *db) Copy(w io.Writer) error { + return errors.New("not implemented") +} + +// Close cleanly shuts down the database and syncs all data. +// This function is part of the walletdb.Db interface implementation. +func (db *db) Close() error { + dbConnsMu.Lock() + defer dbConnsMu.Unlock() + + log.Infof("Closing database %v", db.prefix) + + return dbConns.Close(db.cfg.Dsn) +} + +// attemptRollback attempts to roll back the transaction, and if it fails, it +// will return the error. If the transaction was already closed, it will return +// nil. +func attemptRollback(tx *readWriteTx) error { + rollbackErr := tx.Rollback() + if rollbackErr != nil && + !errors.Is(rollbackErr, walletdb.ErrTxClosed) && + !strings.Contains(rollbackErr.Error(), "conn closed") { + + return fmt.Errorf("error rolling back tx: %w", rollbackErr) + } + + return nil +} diff --git a/internal/sqlbase/db_conn_set.go b/internal/sqlbase/db_conn_set.go new file mode 100644 index 000000000..081b60b0d --- /dev/null +++ b/internal/sqlbase/db_conn_set.go @@ -0,0 +1,90 @@ +//go:build js && wasm + +package sqlbase + +import ( + "database/sql" + "fmt" + "sync" +) + +// dbConn stores the actual connection and a user count. +type dbConn struct { + db *sql.DB + count int +} + +// dbConnSet stores a set of connections. +type dbConnSet struct { + dbConn map[string]*dbConn + maxConnections int + + // mu is used to guard access to the dbConn map. + mu sync.Mutex +} + +// newDbConnSet initializes a new set of connections. +func newDbConnSet(maxConnections int) *dbConnSet { + return &dbConnSet{ + dbConn: make(map[string]*dbConn), + maxConnections: maxConnections, + } +} + +// Open opens a new database connection. If a connection already exists for the +// given dsn, the existing connection is returned. +func (d *dbConnSet) Open(driver, dsn string) (*sql.DB, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if dbConn, ok := d.dbConn[dsn]; ok { + dbConn.count++ + + return dbConn.db, nil + } + + db, err := sql.Open(driver, dsn) + if err != nil { + return nil, err + } + + // Limit maximum number of open connections. This is useful to prevent + // the server from running out of connections and returning an error. + // With this client-side limit in place, lnd will wait for a connection + // to become available. + if d.maxConnections != 0 { + db.SetMaxOpenConns(d.maxConnections) + } + + d.dbConn[dsn] = &dbConn{ + db: db, + count: 1, + } + + return db, nil +} + +// Close closes the connection with the given dsn. If there are still other +// users of the same connection, this function does nothing. +func (d *dbConnSet) Close(dsn string) error { + d.mu.Lock() + defer d.mu.Unlock() + + dbConn, ok := d.dbConn[dsn] + if !ok { + return fmt.Errorf("connection not found: %v", dsn) + } + + // Reduce user count. + dbConn.count-- + + // Do not close if there are other users. + if dbConn.count > 0 { + return nil + } + + // Close connection. + delete(d.dbConn, dsn) + + return dbConn.db.Close() +} diff --git a/internal/sqlbase/log.go b/internal/sqlbase/log.go new file mode 100644 index 000000000..e31810ea8 --- /dev/null +++ b/internal/sqlbase/log.go @@ -0,0 +1,14 @@ +//go:build js && wasm + +package sqlbase + +import "github.com/btcsuite/btclog/v2" + +// log is a logger that is initialized as disabled. This means the package will +// not perform any logging by default until a logger is set. +var log = btclog.Disabled + +// UseLogger uses a specified Logger to output package logging info. +func UseLogger(logger btclog.Logger) { + log = logger +} diff --git a/internal/sqlbase/readwrite_bucket.go b/internal/sqlbase/readwrite_bucket.go new file mode 100644 index 000000000..c069fe321 --- /dev/null +++ b/internal/sqlbase/readwrite_bucket.go @@ -0,0 +1,491 @@ +//go:build js && wasm + +package sqlbase + +import ( + "database/sql" + "errors" + "fmt" + + "github.com/btcsuite/btcwallet/walletdb" +) + +// readWriteBucket stores the bucket id and the buckets transaction. +type readWriteBucket struct { + // id is used to identify the bucket. If id is null, it refers to the + // root bucket. + id *int64 + + // tx holds the parent transaction. + tx *readWriteTx + + table string +} + +// newReadWriteBucket creates a new rw bucket with the passed transaction +// and bucket id. +func newReadWriteBucket(tx *readWriteTx, id *int64) *readWriteBucket { + return &readWriteBucket{ + id: id, + tx: tx, + table: tx.db.table, + } +} + +// NestedReadBucket retrieves a nested read bucket with the given key. +// Returns nil if the bucket does not exist. +func (b *readWriteBucket) NestedReadBucket(key []byte) walletdb.ReadBucket { + return b.NestedReadWriteBucket(key) +} + +func parentSelector(id *int64) string { + if id == nil { + return "parent_id IS NULL" + } + + return fmt.Sprintf("parent_id=%v", *id) +} + +// ForEach invokes the passed function with every key/value pair in +// the bucket. This includes nested buckets, in which case the value +// is nil, but it does not include the key/value pairs within those +// nested buckets. +func (b *readWriteBucket) ForEach(cb func(k, v []byte) error) error { + cursor := b.ReadWriteCursor() + + k, v := cursor.First() + for k != nil { + err := cb(k, v) + if err != nil { + return err + } + + k, v = cursor.Next() + } + + return nil +} + +// Get returns the value for the given key. Returns nil if the key does +// not exist in this bucket. +func (b *readWriteBucket) Get(key []byte) []byte { + // Return nil if the key is empty. + if len(key) == 0 { + return nil + } + + var value *[]byte + row, cancel := b.tx.QueryRow( + "SELECT value FROM "+b.table+" WHERE "+parentSelector(b.id)+ + " AND key=$1", + key, + ) + defer cancel() + err := row.Scan(&value) + + switch { + case err == sql.ErrNoRows: + return nil + + case err != nil: + panic(err) + } + + // A NULL value column marks a sub-bucket row rather than a stored + // value: buckets are created with a NULL value, so a row can exist + // for this key while value scans back as a nil pointer. Per the + // walletdb contract, Get on a key that names a bucket returns nil, so + // guard here before dereferencing value below (otherwise len(*value) + // panics on a nil pointer for nested-bucket keys). + if value == nil { + return nil + } + + // When an empty byte array is stored as the value, Sqlite will decode + // that into nil whereas postgres will decode that as an empty byte + // array. Since returning nil is taken to mean that no value has ever + // been written, we ensure here that we at least return an empty array + // so that nil checks will fail. + if len(*value) == 0 { + return []byte{} + } + + return *value +} + +// ReadCursor returns a new read-only cursor for this bucket. +func (b *readWriteBucket) ReadCursor() walletdb.ReadCursor { + return newReadWriteCursor(b) +} + +// NestedReadWriteBucket retrieves a nested bucket with the given key. +// Returns nil if the bucket does not exist. +func (b *readWriteBucket) NestedReadWriteBucket( + key []byte) walletdb.ReadWriteBucket { + + if len(key) == 0 { + return nil + } + + var id int64 + row, cancel := b.tx.QueryRow( + "SELECT id FROM "+b.table+" WHERE "+parentSelector(b.id)+ + " AND key=$1 AND value IS NULL", + key, + ) + defer cancel() + err := row.Scan(&id) + + switch { + case err == sql.ErrNoRows: + return nil + + case err != nil: + panic(err) + } + + return newReadWriteBucket(b.tx, &id) +} + +// CreateBucket creates and returns a new nested bucket with the given key. +// Returns ErrBucketExists if the bucket already exists, ErrBucketNameRequired +// if the key is empty, or ErrIncompatibleValue if the key value is otherwise +// invalid for the particular database implementation. Other errors are +// possible depending on the implementation. +func (b *readWriteBucket) CreateBucket(key []byte) (walletdb.ReadWriteBucket, + error) { + + if len(key) == 0 { + return nil, walletdb.ErrBucketNameRequired + } + + // Check to see if the bucket already exists. + var ( + value *[]byte + id int64 + ) + row, cancel := b.tx.QueryRow( + "SELECT id,value FROM "+b.table+" WHERE "+parentSelector(b.id)+ + " AND key=$1", + key, + ) + defer cancel() + err := row.Scan(&id, &value) + + switch { + case err == sql.ErrNoRows: + case err == nil && value == nil: + return nil, walletdb.ErrBucketExists + + case err == nil && value != nil: + return nil, walletdb.ErrIncompatibleValue + + case err != nil: + return nil, err + } + + // Bucket does not yet exist, so create it. Postgres will generate a + // bucket id for the new bucket. + row, cancel = b.tx.QueryRow( + "INSERT INTO "+b.table+" (parent_id, key) "+ + "VALUES($1, $2) RETURNING id", + b.id, + key, + ) + defer cancel() + err = row.Scan(&id) + if err != nil { + return nil, err + } + + return newReadWriteBucket(b.tx, &id), nil +} + +// CreateBucketIfNotExists creates and returns a new nested bucket with +// the given key if it does not already exist. Returns +// ErrBucketNameRequired if the key is empty or ErrIncompatibleValue +// if the key value is otherwise invalid for the particular database +// backend. Other errors are possible depending on the implementation. +func (b *readWriteBucket) CreateBucketIfNotExists(key []byte) ( + walletdb.ReadWriteBucket, error) { + + if len(key) == 0 { + return nil, walletdb.ErrBucketNameRequired + } + + // Check to see if the bucket already exists. + var ( + value *[]byte + id int64 + ) + row, cancel := b.tx.QueryRow( + "SELECT id,value FROM "+b.table+" WHERE "+parentSelector(b.id)+ + " AND key=$1", + key, + ) + defer cancel() + err := row.Scan(&id, &value) + + switch { + // Bucket does not yet exist, so create it now. Postgres will generate a + // bucket id for the new bucket. + case err == sql.ErrNoRows: + row, cancel := b.tx.QueryRow( + "INSERT INTO "+b.table+" (parent_id, key) "+ + "VALUES($1, $2) RETURNING id", + b.id, + key, + ) + defer cancel() + err := row.Scan(&id) + if err != nil { + return nil, err + } + + case err == nil && value != nil: + return nil, walletdb.ErrIncompatibleValue + + case err != nil: + return nil, err + } + + return newReadWriteBucket(b.tx, &id), nil +} + +// DeleteNestedBucket deletes the nested bucket and its sub-buckets +// pointed to by the passed key. All values in the bucket and sub-buckets +// will be deleted as well. +func (b *readWriteBucket) DeleteNestedBucket(key []byte) error { + if len(key) == 0 { + return walletdb.ErrIncompatibleValue + } + + result, err := b.tx.Exec( + "DELETE FROM "+b.table+" WHERE "+parentSelector(b.id)+ + " AND key=$1 AND value IS NULL", + key, + ) + if err != nil { + return err + } + + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return walletdb.ErrBucketNotFound + } + + return nil +} + +// Put updates the value for the passed key. +// Returns ErrKeyRequired if te passed key is empty. +func (b *readWriteBucket) Put(key, value []byte) error { + if len(key) == 0 { + return walletdb.ErrKeyRequired + } + + // Prevent NULL being written for an empty value slice. + if value == nil { + value = []byte{} + } + + var ( + result sql.Result + err error + ) + + // We are putting a value in a bucket in this table. Try to insert the + // key first. If the key already exists (ON CONFLICT), update the key. + // Do not update a NULL value, because this indicates that the key + // contains a sub-bucket. This case will be caught via RowsAffected + // below. + if b.id == nil { + // ON CONFLICT requires the WHERE parent_id IS NULL hint to let + // Postgres find the NULL-parent_id unique index (_unp). + result, err = b.tx.Exec( + "INSERT INTO "+b.table+" (key, value) VALUES($1, $2) "+ + "ON CONFLICT (key) WHERE parent_id IS NULL "+ + "DO UPDATE SET value=$2 "+ + "WHERE "+b.table+".value IS NOT NULL", + key, + value, + ) + } else { + // ON CONFLICT requires the WHERE parent_id NOT IS NULL hint to + // let Postgres find the non-NULL-parent_id unique index + // (
_up). + result, err = b.tx.Exec( + "INSERT INTO "+b.table+" (key, value, parent_id) "+ + "VALUES($1, $2, $3) "+ + "ON CONFLICT (key, parent_id) "+ + "WHERE parent_id IS NOT NULL "+ + "DO UPDATE SET value=$2 "+ + "WHERE "+b.table+".value IS NOT NULL", + key, + value, + b.id, + ) + } + if err != nil { + return err + } + + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows != 1 { + return walletdb.ErrIncompatibleValue + } + + return nil +} + +// Delete deletes the key/value pointed to by the passed key. +// Returns ErrKeyRequired if the passed key is empty. +func (b *readWriteBucket) Delete(key []byte) error { + if key == nil { + return nil + } + if len(key) == 0 { + return walletdb.ErrKeyRequired + } + + // Check to see if a bucket with this key exists. + var dummy int + row, cancel := b.tx.QueryRow( + "SELECT 1 FROM "+b.table+" WHERE "+parentSelector(b.id)+ + " AND key=$1 AND value IS NULL", + key, + ) + defer cancel() + err := row.Scan(&dummy) + switch { + // No bucket exists, proceed to deletion of the key. + case err == sql.ErrNoRows: + case err != nil: + return err + + // Bucket exists. + default: + return walletdb.ErrIncompatibleValue + } + + _, err = b.tx.Exec( + "DELETE FROM "+b.table+" WHERE key=$1 AND "+ + parentSelector(b.id)+" AND value IS NOT NULL", + key, + ) + if err != nil { + return err + } + + return nil +} + +// ReadWriteCursor returns a new read-write cursor for this bucket. +func (b *readWriteBucket) ReadWriteCursor() walletdb.ReadWriteCursor { + return newReadWriteCursor(b) +} + +// Tx returns the buckets transaction. +func (b *readWriteBucket) Tx() walletdb.ReadWriteTx { + return b.tx +} + +// NextSequence returns an autoincrementing sequence number for this bucket. +// Note that this is not a thread safe function and as such it must not be used +// for synchronization. +func (b *readWriteBucket) NextSequence() (uint64, error) { + seq := b.Sequence() + 1 + + return seq, b.SetSequence(seq) +} + +// SetSequence updates the sequence number for the bucket. +func (b *readWriteBucket) SetSequence(v uint64) error { + if b.id == nil { + panic("sequence not supported on top level bucket") + } + + result, err := b.tx.Exec( + "UPDATE "+b.table+" SET sequence=$2 WHERE id=$1", b.id, + int64(v), + ) + if err != nil { + return err + } + + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows != 1 { + return errors.New("cannot set sequence") + } + + return nil +} + +// Sequence returns the current sequence number for this bucket without +// incrementing it. +func (b *readWriteBucket) Sequence() uint64 { + if b.id == nil { + panic("sequence not supported on top level bucket") + } + + var seq int64 + row, cancel := b.tx.QueryRow( + "SELECT sequence FROM "+b.table+" WHERE id=$1 "+ + "AND sequence IS NOT NULL", + b.id, + ) + defer cancel() + err := row.Scan(&seq) + + switch { + case err == sql.ErrNoRows: + return 0 + + case err != nil: + panic(err) + } + + return uint64(seq) +} + +// Prefetch will attempt to prefetch all values under a path from the passed +// bucket. +func (b *readWriteBucket) Prefetch(paths ...[]string) {} + +// ForAll is an optimized version of ForEach with the limitation that no +// additional queries can be executed within the callback. +func (b *readWriteBucket) ForAll(cb func(k, v []byte) error) error { + rows, cancel, err := b.tx.Query( + "SELECT key, value FROM " + b.table + " WHERE " + + parentSelector(b.id) + " ORDER BY key", + ) + if err != nil { + return err + } + defer cancel() + + for rows.Next() { + var key, value []byte + + err := rows.Scan(&key, &value) + if err != nil { + return err + } + + err = cb(key, value) + if err != nil { + return err + } + } + + return nil +} diff --git a/internal/sqlbase/readwrite_cursor.go b/internal/sqlbase/readwrite_cursor.go new file mode 100644 index 000000000..02f3fd900 --- /dev/null +++ b/internal/sqlbase/readwrite_cursor.go @@ -0,0 +1,232 @@ +//go:build js && wasm + +package sqlbase + +import ( + "database/sql" + + "github.com/btcsuite/btcwallet/walletdb" +) + +// readWriteCursor holds a reference to the cursors bucket, the value +// prefix and the current key used while iterating. +type readWriteCursor struct { + bucket *readWriteBucket + + // currKey holds the current key of the cursor. + currKey []byte +} + +func newReadWriteCursor(b *readWriteBucket) *readWriteCursor { + return &readWriteCursor{ + bucket: b, + } +} + +// First positions the cursor at the first key/value pair and returns +// the pair. +func (c *readWriteCursor) First() ([]byte, []byte) { + var ( + key []byte + value []byte + ) + row, cancel := c.bucket.tx.QueryRow( + "SELECT key, value FROM " + c.bucket.table + " WHERE " + + parentSelector(c.bucket.id) + + " ORDER BY key LIMIT 1", + ) + defer cancel() + err := row.Scan(&key, &value) + + switch { + case err == sql.ErrNoRows: + return nil, nil + + case err != nil: + panic(err) + } + + // Copy current key to prevent modification by the caller. + c.currKey = make([]byte, len(key)) + copy(c.currKey, key) + + return key, value +} + +// Last positions the cursor at the last key/value pair and returns the +// pair. +func (c *readWriteCursor) Last() ([]byte, []byte) { + var ( + key []byte + value []byte + ) + row, cancel := c.bucket.tx.QueryRow( + "SELECT key, value FROM " + c.bucket.table + " WHERE " + + parentSelector(c.bucket.id) + + " ORDER BY key DESC LIMIT 1", + ) + defer cancel() + err := row.Scan(&key, &value) + + switch { + case err == sql.ErrNoRows: + return nil, nil + + case err != nil: + panic(err) + } + + // Copy current key to prevent modification by the caller. + c.currKey = make([]byte, len(key)) + copy(c.currKey, key) + + return key, value +} + +// Next moves the cursor one key/value pair forward and returns the new +// pair. +func (c *readWriteCursor) Next() ([]byte, []byte) { + var ( + key []byte + value []byte + ) + row, cancel := c.bucket.tx.QueryRow( + "SELECT key, value FROM "+c.bucket.table+" WHERE "+ + parentSelector(c.bucket.id)+ + " AND key>$1 ORDER BY key LIMIT 1", + c.currKey, + ) + defer cancel() + err := row.Scan(&key, &value) + + switch { + case err == sql.ErrNoRows: + return nil, nil + + case err != nil: + panic(err) + } + + // Copy current key to prevent modification by the caller. + c.currKey = make([]byte, len(key)) + copy(c.currKey, key) + + return key, value +} + +// Prev moves the cursor one key/value pair backward and returns the new +// pair. +func (c *readWriteCursor) Prev() ([]byte, []byte) { + var ( + key []byte + value []byte + ) + row, cancel := c.bucket.tx.QueryRow( + "SELECT key, value FROM "+c.bucket.table+" WHERE "+ + parentSelector(c.bucket.id)+ + " AND key<$1 ORDER BY key DESC LIMIT 1", + c.currKey, + ) + defer cancel() + err := row.Scan(&key, &value) + + switch { + case err == sql.ErrNoRows: + return nil, nil + + case err != nil: + panic(err) + } + + // Copy current key to prevent modification by the caller. + c.currKey = make([]byte, len(key)) + copy(c.currKey, key) + + return key, value +} + +// Seek positions the cursor at the passed seek key. If the key does +// not exist, the cursor is moved to the next key after seek. Returns +// the new pair. +func (c *readWriteCursor) Seek(seek []byte) ([]byte, []byte) { + // Convert nil to empty slice, otherwise sql mapping won't be correct + // and no keys are found. + if seek == nil { + seek = []byte{} + } + + var ( + key []byte + value []byte + ) + row, cancel := c.bucket.tx.QueryRow( + "SELECT key, value FROM "+c.bucket.table+" WHERE "+ + parentSelector(c.bucket.id)+ + " AND key>=$1 ORDER BY key LIMIT 1", + seek, + ) + defer cancel() + err := row.Scan(&key, &value) + + switch { + case err == sql.ErrNoRows: + return nil, nil + + case err != nil: + panic(err) + } + + // Copy current key to prevent modification by the caller. + c.currKey = make([]byte, len(key)) + copy(c.currKey, key) + + return key, value +} + +// Delete removes the current key/value pair the cursor is at without +// invalidating the cursor. Returns ErrIncompatibleValue if attempted +// when the cursor points to a nested bucket. +func (c *readWriteCursor) Delete() error { + // Get first record at or after cursor. + var key []byte + row, cancel := c.bucket.tx.QueryRow( + "SELECT key FROM "+c.bucket.table+" WHERE "+ + parentSelector(c.bucket.id)+ + " AND key>=$1 ORDER BY key LIMIT 1", + c.currKey, + ) + defer cancel() + err := row.Scan(&key) + + switch { + case err == sql.ErrNoRows: + return nil + + case err != nil: + panic(err) + } + + // Delete record. + result, err := c.bucket.tx.Exec( + "DELETE FROM "+c.bucket.table+" WHERE "+ + parentSelector(c.bucket.id)+ + " AND key=$1 AND value IS NOT NULL", + key, + ) + if err != nil { + panic(err) + } + + rows, err := result.RowsAffected() + if err != nil { + return err + } + + // The key exists but nothing has been deleted. This means that the key + // must have been a bucket key. + if rows != 1 { + return walletdb.ErrIncompatibleValue + } + + return err +} diff --git a/internal/sqlbase/readwrite_tx.go b/internal/sqlbase/readwrite_tx.go new file mode 100644 index 000000000..d49862752 --- /dev/null +++ b/internal/sqlbase/readwrite_tx.go @@ -0,0 +1,234 @@ +//go:build js && wasm + +package sqlbase + +import ( + "context" + "database/sql" + "sync" + + "github.com/btcsuite/btcwallet/walletdb" +) + +// readWriteTx holds a reference to an open postgres transaction. +type readWriteTx struct { + db *db + tx *sql.Tx + + // onCommit gets called upon commit. + onCommit func() + + // active is true if the transaction hasn't been committed yet. + active bool + + // locker is a pointer to the global db lock. + locker sync.Locker +} + +// newReadWriteTx creates an rw transaction using a connection from the +// specified pool. +func newReadWriteTx(db *db, readOnly bool) (*readWriteTx, error) { + locker := newNoopLocker() + if db.cfg.WithTxLevelLock { + // Obtain the global lock instance. An alternative here is to + // obtain a database lock from Postgres. Unfortunately there is + // no database-level lock in Postgres, meaning that each table + // would need to be locked individually. Perhaps an advisory + // lock could perform this function too. + locker = &db.lock + if readOnly { + locker = db.lock.RLocker() + } + } + locker.Lock() + + // Start the transaction. Don't use the timeout context because it would + // be applied to the transaction as a whole. If possible, mark the + // transaction as read-only to make sure that potential programming + // errors cannot cause changes to the database. + tx, err := db.db.BeginTx( + context.Background(), + &sql.TxOptions{ + ReadOnly: readOnly, + Isolation: sql.LevelSerializable, + }, + ) + if err != nil { + locker.Unlock() + + return nil, err + } + + return &readWriteTx{ + db: db, + tx: tx, + active: true, + locker: locker, + }, nil +} + +// ReadBucket opens the root bucket for read only access. If the bucket +// described by the key does not exist, nil is returned. +func (tx *readWriteTx) ReadBucket(key []byte) walletdb.ReadBucket { + return tx.ReadWriteBucket(key) +} + +// ForEachBucket iterates through all top level buckets. +func (tx *readWriteTx) ForEachBucket(fn func(key []byte) error) error { + // Fetch binary top level buckets. + bucket := newReadWriteBucket(tx, nil) + err := bucket.ForEach(func(k, _ []byte) error { + return fn(k) + }) + + return err +} + +// Rollback closes the transaction, discarding changes (if any) if the +// database was modified by a write transaction. +func (tx *readWriteTx) Rollback() error { + // If the transaction has been closed roolback will fail. + if !tx.active { + return walletdb.ErrTxClosed + } + + err := tx.tx.Rollback() + + // Unlock the transaction regardless of the error result. + tx.active = false + tx.locker.Unlock() + + return err +} + +// ReadWriteBucket opens the root bucket for read/write access. If the +// bucket described by the key does not exist, nil is returned. +func (tx *readWriteTx) ReadWriteBucket(key []byte) walletdb.ReadWriteBucket { + if len(key) == 0 { + return nil + } + + bucket := newReadWriteBucket(tx, nil) + + return bucket.NestedReadWriteBucket(key) +} + +// CreateTopLevelBucket creates the top level bucket for a key if it +// does not exist. The newly-created bucket it returned. +func (tx *readWriteTx) CreateTopLevelBucket(key []byte) ( + walletdb.ReadWriteBucket, error) { + + if len(key) == 0 { + return nil, walletdb.ErrBucketNameRequired + } + + bucket := newReadWriteBucket(tx, nil) + + return bucket.CreateBucketIfNotExists(key) +} + +// DeleteTopLevelBucket deletes the top level bucket for a key. This +// errors if the bucket can not be found or the key keys a single value +// instead of a bucket. +func (tx *readWriteTx) DeleteTopLevelBucket(key []byte) error { + // Execute a cascading delete on the key. + result, err := tx.Exec( + "DELETE FROM "+tx.db.table+" WHERE key=$1 "+ + "AND parent_id IS NULL", + key, + ) + if err != nil { + return err + } + + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return walletdb.ErrBucketNotFound + } + + return nil +} + +// Commit commits the transaction if not already committed. +func (tx *readWriteTx) Commit() error { + // Commit will fail if the transaction is already committed. + if !tx.active { + return walletdb.ErrTxClosed + } + + // Try committing the transaction. + err := tx.tx.Commit() + if err == nil && tx.onCommit != nil { + tx.onCommit() + } + + // Unlock the transaction regardless of the error result. + tx.active = false + tx.locker.Unlock() + + return err +} + +// OnCommit sets the commit callback (overriding if already set). +func (tx *readWriteTx) OnCommit(cb func()) { + tx.onCommit = cb +} + +// QueryRow executes a QueryRow call with a timeout context. +func (tx *readWriteTx) QueryRow(query string, args ...interface{}) (*sql.Row, + func()) { + + ctx, cancel := tx.db.getTimeoutCtx() + + return tx.tx.QueryRowContext(ctx, query, args...), cancel +} + +// Query executes a multi-row query call with a timeout context. +func (tx *readWriteTx) Query(query string, args ...interface{}) (*sql.Rows, + func(), error) { + + ctx, cancel := tx.db.getTimeoutCtx() + rows, err := tx.tx.QueryContext(ctx, query, args...) + if err != nil { + cancel() + + return nil, func() {}, err + } + + return rows, cancel, nil +} + +// Exec executes a Exec call with a timeout context. +func (tx *readWriteTx) Exec(query string, args ...interface{}) (sql.Result, + error) { + + ctx, cancel := tx.db.getTimeoutCtx() + defer cancel() + + return tx.tx.ExecContext(ctx, query, args...) +} + +// noopLocker is an implementation of a no-op sync.Locker. +type noopLocker struct{} + +// newNoopLocker creates a new noopLocker. +func newNoopLocker() sync.Locker { + return &noopLocker{} +} + +// Lock is a noop. +// +// NOTE: this is part of the sync.Locker interface. +func (n *noopLocker) Lock() { +} + +// Unlock is a noop. +// +// NOTE: this is part of the sync.Locker interface. +func (n *noopLocker) Unlock() { +} + +var _ sync.Locker = (*noopLocker)(nil) diff --git a/internal/sqlbase/schema.go b/internal/sqlbase/schema.go new file mode 100644 index 000000000..9578365fe --- /dev/null +++ b/internal/sqlbase/schema.go @@ -0,0 +1,73 @@ +//go:build js && wasm + +package sqlbase + +import ( + "fmt" + "strings" +) + +// SQLiteCmdReplacements is a one to one mapping of sqlite keywords that should +// be replaced by the mapped strings in any command. Note that the sqlite +// keywords to be replaced are case-sensitive. +type SQLiteCmdReplacements map[string]string + +func newKVSchemaCreationCmd(table, schema string, + replacements SQLiteCmdReplacements) string { + + var ( + tableInSchema = table + finalCmd string + ) + if schema != "" { + finalCmd = fmt.Sprintf(`CREATE SCHEMA IF NOT EXISTS ` + schema + + `;`) + + tableInSchema = fmt.Sprintf("%s.%s", schema, table) + } + + // Construct the sql statements to set up a kv table in postgres. Every + // row points to the bucket that it is one via its parent_id field. A + // NULL parent_id means that the key belongs to the uppermost bucket in + // this table. A constraint on parent_id is enforcing referential + // integrity. + // + // Furthermore, there is a
_p index on parent_id that is required + // for the foreign key constraint. + // + // Finally, there are unique indices on (parent_id, key) to prevent the + // same key being present in a bucket more than once (
_up and + //
_unp). In postgres, a single index wouldn't enforce the unique + // constraint on rows with a NULL parent_id. Therefore, two indices are + // defined. + // + // The replacements map can be used to replace any sqlite keywords. + // Callers should note that the sqlite keywords are case-sensitive. + finalCmd += fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS ` + tableInSchema + ` +( + key BLOB NOT NULL, + value BLOB, + parent_id BIGINT, + id INTEGER PRIMARY KEY, + sequence BIGINT, + CONSTRAINT ` + table + `_parent FOREIGN KEY (parent_id) + REFERENCES ` + tableInSchema + ` (id) + ON UPDATE NO ACTION + ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS ` + table + `_p + ON ` + tableInSchema + ` (parent_id); +CREATE UNIQUE INDEX IF NOT EXISTS ` + table + `_up + ON ` + tableInSchema + ` + (parent_id, key) WHERE parent_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS ` + table + `_unp + ON ` + tableInSchema + ` (key) WHERE parent_id IS NULL; +`) + + for from, to := range replacements { + finalCmd = strings.Replace(finalCmd, from, to, -1) + } + + return finalCmd +} diff --git a/lwwallet/wallet.go b/lwwallet/wallet.go index c99e550d2..a3ba43fc1 100644 --- a/lwwallet/wallet.go +++ b/lwwallet/wallet.go @@ -59,9 +59,9 @@ type Wallet struct { } // New creates a new lightweight wallet from the given configuration. -// The caller must provide a DBDir for btcwallet's bbolt database and -// is responsible for managing the directory's lifecycle (creation -// before calling New, cleanup after Stop if desired). +// The caller must provide a DBDir for btcwallet's wallet database. Native +// builds use that path for btcwallet's bbolt database, while browser builds +// derive a stable OPFS SQLite database name from it. func New(cfg Config) (*Wallet, error) { // Constructors run before a contextual logger is guaranteed, // so default to a disabled logger when one was not explicitly @@ -97,6 +97,11 @@ func New(cfg Config) (*Wallet, error) { walletcore.DefaultBlockCacheSize, ) + loaderOptions, err := newWalletLoaderOptions(cfg) + if err != nil { + return nil, fmt.Errorf("create wallet loader options: %w", err) + } + btcw, err := btcwallet.New(btcwallet.Config{ PrivatePass: walletcore.WalletPassphrase, PublicPass: walletcore.WalletPassphrase, @@ -106,11 +111,7 @@ func New(cfg Config) (*Wallet, error) { NetParams: cfg.ChainParams, CoinType: coinType, RecoveryWindow: cfg.RecoveryWindow, - LoaderOptions: []btcwallet.LoaderOption{ - btcwallet.LoaderWithLocalWalletDB( - cfg.DBDir, false, 60*time.Second, - ), - }, + LoaderOptions: loaderOptions, }, blockCache) if err != nil { return nil, fmt.Errorf("create btcwallet: %w", err) diff --git a/lwwallet/walletdb_native.go b/lwwallet/walletdb_native.go new file mode 100644 index 000000000..a9b8548d0 --- /dev/null +++ b/lwwallet/walletdb_native.go @@ -0,0 +1,18 @@ +//go:build !js || !wasm + +package lwwallet + +import ( + "time" + + "github.com/lightningnetwork/lnd/lnwallet/btcwallet" +) + +// newWalletLoaderOptions returns the native btcwallet bbolt loader options. +func newWalletLoaderOptions(cfg Config) ([]btcwallet.LoaderOption, error) { + return []btcwallet.LoaderOption{ + btcwallet.LoaderWithLocalWalletDB( + cfg.DBDir, false, 60*time.Second, + ), + }, nil +} diff --git a/lwwallet/walletdb_wasm.go b/lwwallet/walletdb_wasm.go new file mode 100644 index 000000000..08f0e7619 --- /dev/null +++ b/lwwallet/walletdb_wasm.go @@ -0,0 +1,115 @@ +//go:build js && wasm + +package lwwallet + +import ( + "context" + "fmt" + "hash/fnv" + "net/url" + "path/filepath" + "strings" + "time" + + "github.com/btcsuite/btcwallet/walletdb" + "github.com/lightninglabs/darepo-client/internal/sqlbase" + _ "github.com/lightninglabs/go-wasmsqlite" + "github.com/lightningnetwork/lnd/lnwallet/btcwallet" +) + +const ( + wasmWalletDBDriverName = "wasmsqlite" + wasmWalletDBTablePrefix = "walletdb" + wasmWalletDBTimeout = 30 * time.Second + wasmWalletDBBusyTimeoutMS = "30000" + wasmWalletDBMaxConnections = 1 + wasmWalletDBFileNamePattern = "/wallet-%016x.db" +) + +// newWalletLoaderOptions opens btcwallet through an OPFS-backed SQLite +// walletdb implementation for browser builds. +func newWalletLoaderOptions(cfg Config) ([]btcwallet.LoaderOption, error) { + db, err := openWASMWalletDB(cfg.DBDir) + if err != nil { + return nil, err + } + + return []btcwallet.LoaderOption{ + btcwallet.LoaderWithExternalWalletDB(db), + }, nil +} + +// openWASMWalletDB opens btcwallet's walletdb on top of the same browser +// SQLite/OPFS driver used by the daemon and swap stores. +func openWASMWalletDB(dbDir string) (walletdb.DB, error) { + sqlbase.Init(wasmWalletDBMaxConnections) + + cfg := &sqlbase.Config{ + DriverName: wasmWalletDBDriverName, + Dsn: wasmWalletDBDSN(dbDir), + Timeout: wasmWalletDBTimeout, + TableNamePrefix: wasmWalletDBTablePrefix, + WithTxLevelLock: true, + } + + var lastErr error + for attempt := 0; attempt < 25; attempt++ { + db, err := sqlbase.NewSqlBackend(context.Background(), cfg) + if err == nil { + return db, nil + } + if !isWASMWalletCantOpen(err) { + return nil, fmt.Errorf("open OPFS wallet database: %w", + err) + } + + lastErr = err + time.Sleep(200 * time.Millisecond) + } + + return nil, fmt.Errorf("open OPFS wallet database: %w", lastErr) +} + +// wasmWalletDBDSN returns a go-wasmsqlite DSN for btcwallet's SQL walletdb. +func wasmWalletDBDSN(dbDir string) string { + values := url.Values{} + values.Set("file", wasmWalletDBFileName(dbDir)) + values.Set("vfs", "opfs") + values.Set("mode", "rwc") + values.Set("busy_timeout", wasmWalletDBBusyTimeoutMS) + values.Set("journal_mode", "WAL") + values.Set( + "pragma", + strings.Join( + []string{ + "foreign_keys=on", + "auto_vacuum=incremental", + "locking_mode=EXCLUSIVE", + }, ";", + ), + ) + + return values.Encode() +} + +// wasmWalletDBFileName maps a native wallet DB directory to a stable +// origin-local OPFS database name. +func wasmWalletDBFileName(dbDir string) string { + normalized := filepath.ToSlash(filepath.Clean(dbDir)) + normalized = strings.TrimSpace(normalized) + if normalized == "" || normalized == "." { + normalized = "wallet" + } + + hasher := fnv.New64a() + _, _ = hasher.Write([]byte(normalized)) + + return fmt.Sprintf(wasmWalletDBFileNamePattern, hasher.Sum64()) +} + +// isWASMWalletCantOpen identifies the SQLite error returned while OPFS still +// holds the wallet database from a just-unloaded page runtime. +func isWASMWalletCantOpen(err error) bool { + return strings.Contains(err.Error(), "SQLITE_CANTOPEN") || + strings.Contains(err.Error(), "unable to open database file") +} diff --git a/sdk/swaps/migrations/000001_swap_sessions.up.sql b/sdk/swaps/migrations/000001_swap_sessions.up.sql index 927cee59d..828b5e2b7 100644 --- a/sdk/swaps/migrations/000001_swap_sessions.up.sql +++ b/sdk/swaps/migrations/000001_swap_sessions.up.sql @@ -43,7 +43,8 @@ CREATE TABLE IF NOT EXISTS receive_swaps ( operator_pubkey BLOB NOT NULL, -- swap_server_pubkey is the swap server key used in the expected vHTLC. - swap_server_pubkey BLOB NOT NULL, + -- It is NULL until the server's HTLC event is accepted. + swap_server_pubkey BLOB, -- refund_locktime is the absolute refund locktime negotiated for the -- expected vHTLC. @@ -60,12 +61,14 @@ CREATE TABLE IF NOT EXISTS receive_swaps ( unilateral_refund_without_receiver_delay BIGINT NOT NULL, -- vhtlc_pkscript is the exact vHTLC output script the client expects the - -- swap server to fund. - vhtlc_pkscript BLOB NOT NULL, + -- swap server to fund. It is NULL until the server's HTLC event is + -- accepted. + vhtlc_pkscript BLOB, -- vhtlc_policy_template is the semantic vHTLC policy template sent into - -- the daemon when claiming the receive-side vHTLC. - vhtlc_policy_template BLOB NOT NULL, + -- the daemon when claiming the receive-side vHTLC. It is NULL until the + -- server's HTLC event is accepted. + vhtlc_policy_template BLOB, -- vhtlc_outpoint is the observed funded vHTLC outpoint once indexed. vhtlc_outpoint TEXT NOT NULL DEFAULT '', diff --git a/sdk/swaps/store.go b/sdk/swaps/store.go index 8c8d57a81..915480fb6 100644 --- a/sdk/swaps/store.go +++ b/sdk/swaps/store.go @@ -6,15 +6,14 @@ import ( "embed" "fmt" "log/slog" - "net/url" "path/filepath" "time" "github.com/btcsuite/btclog/v2" + clientdb "github.com/lightninglabs/darepo-client/db" dbmigrate "github.com/lightninglabs/darepo-client/db/migrate" dbsqlc "github.com/lightninglabs/darepo-client/db/sqlc" swapsqlc "github.com/lightninglabs/darepo-client/sdk/swaps/sqlc" - _ "modernc.org/sqlite" ) const ( @@ -30,15 +29,6 @@ const ( // for the swap-client schema. defaultMigrationDatabaseName = "swap_client" - // sqliteOptionPrefix is the modernc SQLite DSN prefix for pragma - // settings. - sqliteOptionPrefix = "_pragma" - - // sqliteTxLockImmediate starts write transactions immediately so state - // persistence fails fast under contention instead of stalling halfway - // through a swap step. - sqliteTxLockImmediate = "_txlock=immediate" - // defaultMaxConns bounds the swap store connection pool just like // the main client database. defaultMaxConns = 25 @@ -95,56 +85,53 @@ func NewSqliteStore(cfg *SqliteStoreConfig, log btclog.Logger) (*Store, error) { log = btclog.Disabled } - pragmaOptions := []struct { - name string - value string - }{ + pragmaOptions := []clientdb.SQLitePragma{ { - name: "foreign_keys", - value: "on", + Name: "foreign_keys", + Value: "on", }, { - name: "journal_mode", - value: "WAL", + Name: "journal_mode", + Value: "WAL", }, { - name: "busy_timeout", - value: "5000", + Name: "busy_timeout", + Value: "5000", }, { - name: "synchronous", - value: "full", + Name: "synchronous", + Value: "full", }, { - name: "fullfsync", - value: "true", + Name: "fullfsync", + Value: "true", }, } - sqliteOptions := make(url.Values) - for _, option := range pragmaOptions { - sqliteOptions.Add( - sqliteOptionPrefix, - fmt.Sprintf("%s=%s", option.name, option.value), - ) - } - - dsn := fmt.Sprintf("%s?%s&%s", cfg.DatabaseFileName, - sqliteOptions.Encode(), sqliteTxLockImmediate) - ctx := context.Background() log.InfoS(ctx, "Opening swap SQLite database", slog.String("db_file", cfg.DatabaseFileName), ) - db, err := sql.Open("sqlite", dsn) + openResult, err := clientdb.OpenSQLiteDatabase( + clientdb.SQLiteOpenConfig{ + DatabaseFileName: cfg.DatabaseFileName, + Pragmas: pragmaOptions, + TxLockImmediate: true, + MaxOpenConns: defaultMaxConns, + MaxIdleConns: defaultMaxConns, + ConnMaxLifetime: defaultConnMaxLifetime, + }, + ) if err != nil { return nil, fmt.Errorf("open swap sqlite db: %w", err) } - db.SetMaxOpenConns(defaultMaxConns) - db.SetMaxIdleConns(defaultMaxConns) - db.SetConnMaxLifetime(defaultConnMaxLifetime) + db := openResult.DB + + log.DebugS(ctx, "Swap SQLite connection pool configured", + slog.String("driver", openResult.DriverName), + ) if !cfg.SkipMigrations { err = RunMigrations(db, log) diff --git a/swapclientserver/fs_native.go b/swapclientserver/fs_native.go new file mode 100644 index 000000000..cdb873ac6 --- /dev/null +++ b/swapclientserver/fs_native.go @@ -0,0 +1,13 @@ +//go:build swapruntime && (!js || !wasm) + +package swapclientserver + +import ( + "os" + "path/filepath" +) + +// ensureSwapDBDir creates the host directory that contains the swap database. +func ensureSwapDBDir(dbPath string) error { + return os.MkdirAll(filepath.Dir(dbPath), 0o700) +} diff --git a/swapclientserver/fs_wasm.go b/swapclientserver/fs_wasm.go new file mode 100644 index 000000000..93a38c601 --- /dev/null +++ b/swapclientserver/fs_wasm.go @@ -0,0 +1,9 @@ +//go:build swapruntime && js && wasm + +package swapclientserver + +// ensureSwapDBDir is a no-op in browser builds. The SQLite driver maps the +// configured database filename to OPFS, where host directories do not exist. +func ensureSwapDBDir(string) error { + return nil +} diff --git a/swapclientserver/service.go b/swapclientserver/service.go index 788fa1bd5..fe2add9e6 100644 --- a/swapclientserver/service.go +++ b/swapclientserver/service.go @@ -354,7 +354,7 @@ func newSwapClientService(ctx context.Context, rpcServer *darepod.RPCServer, if err != nil { return nil, nil, err } - if err := os.MkdirAll(filepath.Dir(dbPath), 0o700); err != nil { + if err := ensureSwapDBDir(dbPath); err != nil { return nil, nil, fmt.Errorf("create swap db dir: %w", err) } From 705264d4aee087a896955b354454e2a72c8d1751 Mon Sep 17 00:00:00 2001 From: sputn1ck Date: Thu, 25 Jun 2026 16:12:04 -0700 Subject: [PATCH 2/3] walletdk: build the embedded runtime for js/wasm The embedded daemon path was walled off from js/wasm builds: embedded.go, embedded_config.go, errmap.go, and the swapruntime / walletdkrpc tag pairs all carried a !js constraint, so walletdk.Start, the wallet RPC subserver, and the error-reconstruct interceptor did not exist in a browser build at all. Drop those constraints so the in-process bufconn runtime compiles under GOOS=js, and disable the public HTTP gateway on the embedded path, since a browser has no TCP listener to bind. sdk/ark splits its embedded boot into a native file (embedded.go, now tagged !js || !wasm) and a browser stub (embedded_wasm.go) whose StartEmbedded reports that the native daemon runtime is unavailable, with the shared grpc readiness helper lifted into transport.go. The browser wallet boots the daemon through walletdk.Start rather than sdk/ark, so the stub keeps the package compiling for js consumers that only need the remote Ark types. --- sdk/ark/embedded.go | 67 +---------------------------- sdk/ark/embedded_wasm.go | 29 +++++++++++++ sdk/ark/transport.go | 73 ++++++++++++++++++++++++++++++++ sdk/walletdk/embedded.go | 9 +++- sdk/walletdk/embedded_config.go | 2 - sdk/walletdk/errmap.go | 2 - sdk/walletdk/swapruntime.go | 2 +- sdk/walletdk/swapruntime_stub.go | 2 +- sdk/walletdk/walletdkrpc.go | 2 +- sdk/walletdk/walletdkrpc_stub.go | 2 +- 10 files changed, 115 insertions(+), 75 deletions(-) create mode 100644 sdk/ark/embedded_wasm.go create mode 100644 sdk/ark/transport.go diff --git a/sdk/ark/embedded.go b/sdk/ark/embedded.go index 9153d5eda..fd8577ffc 100644 --- a/sdk/ark/embedded.go +++ b/sdk/ark/embedded.go @@ -1,3 +1,5 @@ +//go:build !js || !wasm + package ark import ( @@ -9,17 +11,10 @@ import ( "github.com/lightninglabs/darepo-client/daemonrpc" "github.com/lightninglabs/darepo-client/darepod" "google.golang.org/grpc" - "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/test/bufconn" ) -const ( - // defaultBufConnSize is the size of the in-memory listener buffer used - // by embedded clients that do not provide their own transport. - defaultBufConnSize = 1 << 20 -) - // EmbeddedConfig configures an in-process daemon runtime managed by the SDK. type EmbeddedConfig struct { // DaemonConfig is the full darepod configuration snapshot to clone @@ -246,61 +241,3 @@ func waitForRunExit(ctx context.Context, runErrChan <-chan error) error { ctx.Err()) } } - -// waitForReady forces a new client connection to attempt dialing and waits -// until it reaches READY, the embedded daemon exits, or the caller's context -// expires. -func waitForReady(ctx context.Context, conn *grpc.ClientConn, - runDoneChan <-chan error) error { - - conn.Connect() - - for { - state := conn.GetState() - if state == connectivity.Ready { - return nil - } - - if state == connectivity.Shutdown { - return fmt.Errorf("grpc connection shut down before " + - "readiness") - } - - waitCtx, waitCancel := context.WithCancel(ctx) - runExitErr := make(chan error, 1) - go func() { - select { - case runErr := <-runDoneChan: - if runErr != nil { - runExitErr <- fmt.Errorf("embedded "+ - "daemon exited before "+ - "readiness: %w", runErr) - } else { - runExitErr <- fmt.Errorf("embedded " + - "daemon exited before " + - "readiness") - } - - waitCancel() - - case <-waitCtx.Done(): - } - }() - - if !conn.WaitForStateChange(waitCtx, state) { - waitCancel() - - select { - case err := <-runExitErr: - return err - - default: - } - - return fmt.Errorf("wait for grpc readiness: %w", - ctx.Err()) - } - - waitCancel() - } -} diff --git a/sdk/ark/embedded_wasm.go b/sdk/ark/embedded_wasm.go new file mode 100644 index 000000000..c5d7b8dd5 --- /dev/null +++ b/sdk/ark/embedded_wasm.go @@ -0,0 +1,29 @@ +//go:build js && wasm + +package ark + +import ( + "context" + "fmt" + + "google.golang.org/grpc" +) + +// EmbeddedConfig configures an in-process daemon runtime managed by the SDK. +type EmbeddedConfig struct { + // DaemonConfig is intentionally untyped in browser builds so packages + // that only need remote Ark SDK types do not import the native daemon. + DaemonConfig any + + // BufferSize is ignored in browser builds. + BufferSize int + + // DialOptions is ignored in browser builds. + DialOptions []grpc.DialOption +} + +// StartEmbedded is not available in browser builds until the daemon runtime +// has a WASM-safe construction path. +func StartEmbedded(context.Context, EmbeddedConfig) (*Client, error) { + return nil, fmt.Errorf("embedded ark runtime is not available in wasm") +} diff --git a/sdk/ark/transport.go b/sdk/ark/transport.go new file mode 100644 index 000000000..52fafc0af --- /dev/null +++ b/sdk/ark/transport.go @@ -0,0 +1,73 @@ +package ark + +import ( + "context" + "fmt" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" +) + +const ( + // defaultBufConnSize is the size of the in-memory listener buffer used + // by embedded clients that do not provide their own transport. + defaultBufConnSize = 1 << 20 +) + +// waitForReady forces a new client connection to attempt dialing and waits +// until it reaches READY, the embedded daemon exits, or the caller's context +// expires. +func waitForReady(ctx context.Context, conn *grpc.ClientConn, + runDoneChan <-chan error) error { + + conn.Connect() + + for { + state := conn.GetState() + if state == connectivity.Ready { + return nil + } + + if state == connectivity.Shutdown { + return fmt.Errorf("grpc connection shut down before " + + "readiness") + } + + waitCtx, waitCancel := context.WithCancel(ctx) + runExitErr := make(chan error, 1) + go func() { + select { + case runErr := <-runDoneChan: + if runErr != nil { + runExitErr <- fmt.Errorf("embedded "+ + "daemon exited before "+ + "readiness: %w", runErr) + } else { + runExitErr <- fmt.Errorf("embedded " + + "daemon exited before " + + "readiness") + } + + waitCancel() + + case <-waitCtx.Done(): + } + }() + + if !conn.WaitForStateChange(waitCtx, state) { + waitCancel() + + select { + case err := <-runExitErr: + return err + + default: + } + + return fmt.Errorf("wait for grpc readiness: %w", + ctx.Err()) + } + + waitCancel() + } +} diff --git a/sdk/walletdk/embedded.go b/sdk/walletdk/embedded.go index 37a340f6a..a40ef0c39 100644 --- a/sdk/walletdk/embedded.go +++ b/sdk/walletdk/embedded.go @@ -1,5 +1,3 @@ -//go:build !js - package walletdk import ( @@ -129,6 +127,13 @@ func Start(ctx context.Context, cfg Config, opts ...Option) (*Client, error) { daemonCfg.RPC = &darepod.RPCConfig{} } daemonCfg.RPC.Listener = listener + if daemonCfg.RPC.Gateway != nil { + // Embedded walletdk talks to darepod through the private + // bufconn listener above. The public HTTP gateway would need a + // browser-incompatible TCP listener in WASM and is not needed + // by native embedders using this in-process API. + daemonCfg.RPC.Gateway.Enabled = false + } if err := daemonCfg.Validate(); err != nil { _ = listener.Close() diff --git a/sdk/walletdk/embedded_config.go b/sdk/walletdk/embedded_config.go index 7a71a26aa..8b00028e1 100644 --- a/sdk/walletdk/embedded_config.go +++ b/sdk/walletdk/embedded_config.go @@ -1,5 +1,3 @@ -//go:build !js - package walletdk import ( diff --git a/sdk/walletdk/errmap.go b/sdk/walletdk/errmap.go index b5a8a1669..de5c49ef3 100644 --- a/sdk/walletdk/errmap.go +++ b/sdk/walletdk/errmap.go @@ -1,5 +1,3 @@ -//go:build !js - package walletdk import ( diff --git a/sdk/walletdk/swapruntime.go b/sdk/walletdk/swapruntime.go index 97c57cbb8..e0066eee1 100644 --- a/sdk/walletdk/swapruntime.go +++ b/sdk/walletdk/swapruntime.go @@ -1,4 +1,4 @@ -//go:build swapruntime && !js +//go:build swapruntime package walletdk diff --git a/sdk/walletdk/swapruntime_stub.go b/sdk/walletdk/swapruntime_stub.go index 7d6e6cffc..764a883c6 100644 --- a/sdk/walletdk/swapruntime_stub.go +++ b/sdk/walletdk/swapruntime_stub.go @@ -1,4 +1,4 @@ -//go:build !swapruntime && !js +//go:build !swapruntime package walletdk diff --git a/sdk/walletdk/walletdkrpc.go b/sdk/walletdk/walletdkrpc.go index 4b41dcde8..7bcbd974a 100644 --- a/sdk/walletdk/walletdkrpc.go +++ b/sdk/walletdk/walletdkrpc.go @@ -1,4 +1,4 @@ -//go:build walletdkrpc && swapruntime && !js +//go:build walletdkrpc && swapruntime package walletdk diff --git a/sdk/walletdk/walletdkrpc_stub.go b/sdk/walletdk/walletdkrpc_stub.go index 0eb7c46f1..0297a9e7a 100644 --- a/sdk/walletdk/walletdkrpc_stub.go +++ b/sdk/walletdk/walletdkrpc_stub.go @@ -1,4 +1,4 @@ -//go:build (!walletdkrpc || !swapruntime) && !js +//go:build !walletdkrpc || !swapruntime package walletdk From b91300a899ebc75072a9a68838c9929597237fa4 Mon Sep 17 00:00:00 2001 From: sputn1ck Date: Thu, 25 Jun 2026 16:12:04 -0700 Subject: [PATCH 3/3] cmd/walletdk-wasm: add browser bridge over the mobile facade Expose the embedded walletdk runtime to browser JavaScript as a thin syscall/js adapter over the sdk/walletdk/mobile JSON facade. Every verb takes a JS request object and resolves a Promise with the decoded JSON response, so the daemon, swap, and OOR machinery all run in the single browser VM with no separate gateway process. Routing through the shared mobile facade keeps one source of truth with the gomobile bindings, so the bridge cannot drift from the walletdk.Client API the way a bespoke per-method dispatch would. A native stub keeps go build ./... green off-target, where the js/wasm main has no entry point, and a wasm-wallet make target assembles the stripped blob alongside wasm_exec.js and the go-wasmsqlite OPFS runtime assets so the browser bundle is buildable in one step. --- Makefile | 25 ++- cmd/walletdk-wasm/main.go | 318 ++++++++++++++++++++++++++++++++++++++ cmd/walletdk-wasm/stub.go | 21 +++ 3 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 cmd/walletdk-wasm/main.go create mode 100644 cmd/walletdk-wasm/stub.go diff --git a/Makefile b/Makefile index b446f3555..5d76599ab 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ .PHONY: ast-lint ast-grep-fix .PHONY: unit unit-cover unit-race unit-swapruntime check-go-version build install clean release .PHONY: build build-swapruntime build-swapclient build-walletdkrpc rpc install install-swapruntime install-walletdkrpc help clean-networks -.PHONY: mobile mobile-android mobile-ios +.PHONY: mobile mobile-android mobile-ios wasm-wallet .PHONY: systest systest-verbose .PHONY: commitmsg-lint commitmsg-fmt commitmsg-reword @@ -449,6 +449,29 @@ mobile-ios: #? Build the iOS .xcframework for sdk/walletdk @$(call print, "Building iOS .xcframework for walletdk.") ./sdk/walletdk/mobile/gen_bindings.sh ios +WASM_WALLET_OUT := bin/wasm +WASMSQLITE_DIR := $(shell $(GOCC) list -m -f '{{.Dir}}' github.com/lightninglabs/go-wasmsqlite 2>/dev/null) + +wasm-wallet: #? Build the walletdk browser wasm blob + runtime assets into bin/wasm + @$(call print, "Building walletdk browser wasm blob.") + $(RM) -r $(WASM_WALLET_OUT) + mkdir -p $(WASM_WALLET_OUT) + GOOS=js GOARCH=wasm $(GOBUILD) -trimpath -ldflags="-s -w" \ + -tags="mobile walletdkrpc swapruntime" \ + -o $(WASM_WALLET_OUT)/walletdk.wasm ./cmd/walletdk-wasm + gzip -9 -c $(WASM_WALLET_OUT)/walletdk.wasm \ + > $(WASM_WALLET_OUT)/walletdk.wasm.gz + cp "$$($(GOCC) env GOROOT)/lib/wasm/wasm_exec.js" $(WASM_WALLET_OUT)/ + cp $(WASMSQLITE_DIR)/assets/sqlite3.js $(WASM_WALLET_OUT)/ + cp $(WASMSQLITE_DIR)/assets/sqlite3.wasm $(WASM_WALLET_OUT)/ + cp $(WASMSQLITE_DIR)/assets/sqlite3-opfs-async-proxy.js $(WASM_WALLET_OUT)/ + cp $(WASMSQLITE_DIR)/bridge/sqlite-bridge.js $(WASM_WALLET_OUT)/ + cp $(WASMSQLITE_DIR)/bridge/sqlite-worker.js $(WASM_WALLET_OUT)/ + # The go-wasmsqlite assets are read-only in the module cache; make the + # copies writable so re-runs (and callers staging the bundle) aren't + # blocked by a read-only destination. + chmod -R u+w $(WASM_WALLET_OUT) + install: #? Build and install binaries to GOPATH/bin @$(call print, "Installing binaries.") $(GOINSTALL) -trimpath -tags="$(DEV_TAGS)" $(DEV_LDFLAGS) ./cmd/merge-sql-schemas diff --git a/cmd/walletdk-wasm/main.go b/cmd/walletdk-wasm/main.go new file mode 100644 index 000000000..83b0bf170 --- /dev/null +++ b/cmd/walletdk-wasm/main.go @@ -0,0 +1,318 @@ +//go:build js && wasm + +// Command walletdk-wasm exposes the embedded walletdk runtime to browser +// JavaScript. It is a thin syscall/js adapter over the sdk/walletdk/mobile +// JSON facade: every verb takes a JS request object and resolves a JS response +// object, so the daemon, swap, and OOR machinery all run in-process in the one +// browser VM with no separate gateway. The facade is the single source of +// truth shared with the gomobile bindings, so this bridge never reaches into +// the walletdk.Client API directly and cannot drift from it. +// +// Build with: GOOS=js GOARCH=wasm go build \ +// -tags "mobile walletdkrpc swapruntime" ./cmd/walletdk-wasm +package main + +import ( + "errors" + "io" + "syscall/js" + + "github.com/lightninglabs/darepo-client/sdk/walletdk/mobile" +) + +// main installs the browser entry point and then parks the Go runtime so the +// exported callbacks stay live for the lifetime of the page. +func main() { + js.Global().Set("walletdkCall", js.FuncOf(walletCall)) + js.Global().Call("dispatchEvent", js.Global().Get("CustomEvent").New( + "walletdk-ready", + )) + + select {} +} + +// walletCall is the single JS entry point. It takes a method name and an +// optional request object and returns a Promise that resolves with the verb's +// JSON response decoded into a JS value (or rejects with an Error). +func walletCall(_ js.Value, args []js.Value) any { + if len(args) == 0 { + return rejected(errors.New("method is required")) + } + + method := args[0].String() + + var req js.Value + if len(args) > 1 { + req = args[1] + } + + switch method { + // Lifecycle. + case "start": + cfg := startConfig(req) + + return promise(func() (any, error) { + return js.Null(), mobile.Start(cfg) + }) + + case "stop": + return promise(func() (any, error) { + return js.Null(), mobile.Stop() + }) + + // No-argument verbs returning JSON. + case "getInfo": + return promise(jsonNoArg(mobile.GetInfo)) + + case "balance": + return promise(jsonNoArg(mobile.Balance)) + + case "status": + return promise(jsonNoArg(mobile.Status)) + + // Request/response verbs returning JSON. + case "createWallet": + return promise(jsonVerb(req, mobile.CreateWallet)) + + case "unlockWallet": + return promise(jsonVerb(req, mobile.UnlockWallet)) + + case "deposit": + return promise(jsonVerb(req, mobile.Deposit)) + + case "receive": + return promise(jsonVerb(req, mobile.Receive)) + + case "prepareSend": + return promise(jsonVerb(req, mobile.PrepareSend)) + + case "sendPrepared": + return promise(jsonVerb(req, mobile.SendPrepared)) + + case "list": + return promise(jsonVerb(req, mobile.List)) + + case "exit": + return promise(jsonVerb(req, mobile.Exit)) + + case "exitStatus": + return promise(jsonVerb(req, mobile.ExitStatus)) + + case "getExitPlan": + return promise(jsonVerb(req, mobile.GetExitPlan)) + + case "sweepWallet": + return promise(jsonVerb(req, mobile.SweepWallet)) + + // Scalar convenience verbs for the hottest UI paths. + case "confirmedBalanceSat": + return promise(func() (any, error) { + return mobile.ConfirmedBalanceSat() + }) + + case "pendingInboundSat": + return promise(func() (any, error) { + return mobile.PendingInboundSat() + }) + + case "walletReady": + return promise(func() (any, error) { + return mobile.WalletReady() + }) + + case "isRunning": + return promise(func() (any, error) { + return mobile.IsRunning(), nil + }) + + // Streaming verb, exposed as a pull handle. + case "subscribe": + return promise(func() (any, error) { + sub, err := mobile.Subscribe(jsonBytes(req)) + if err != nil { + return nil, err + } + + return subscriptionHandle(sub), nil + }) + + default: + return rejected(errors.New("unknown method: " + method)) + } +} + +// jsonNoArg adapts a facade verb that takes no request and returns a JSON body. +func jsonNoArg(fn func() ([]byte, error)) func() (any, error) { + return func() (any, error) { + out, err := fn() + if err != nil { + return nil, err + } + + return parse(out), nil + } +} + +// jsonVerb adapts a facade verb that takes a JSON request and returns a JSON +// body, marshalling the JS request object on the way in. +func jsonVerb(req js.Value, + fn func([]byte) ([]byte, error)) func() (any, error) { + + return func() (any, error) { + out, err := fn(jsonBytes(req)) + if err != nil { + return nil, err + } + + return parse(out), nil + } +} + +// subscriptionHandle wraps a pull-based mobile.Subscription as a JS object with +// next() (resolving the next entry, or null at end of stream) and close() +// methods. The handle owns its js.Func callbacks and releases them on close. +func subscriptionHandle(sub *mobile.Subscription) js.Value { + handle := js.Global().Get("Object").New() + + var nextFn, closeFn js.Func + + nextFn = js.FuncOf(func(_ js.Value, _ []js.Value) any { + return promise(func() (any, error) { + entry, err := sub.Next() + switch { + case errors.Is(err, io.EOF): + return js.Null(), nil + + case err != nil: + return nil, err + + default: + return parse(entry), nil + } + }) + }) + + closeFn = js.FuncOf(func(_ js.Value, _ []js.Value) any { + err := sub.Close() + nextFn.Release() + closeFn.Release() + if err != nil { + return jsError(err) + } + + return js.Null() + }) + + handle.Set("next", nextFn) + handle.Set("close", closeFn) + + return handle +} + +// promise runs fn on a fresh goroutine and surfaces its result as a JS Promise. +// A panic in fn is recovered and rejected so it never kills the Go runtime. +func promise(fn func() (any, error)) any { + var executor js.Func + executor = js.FuncOf(func(_ js.Value, args []js.Value) any { + resolve, reject := args[0], args[1] + + go func() { + defer func() { + if r := recover(); r != nil { + reject.Invoke( + jsError( + errors.New( + "walletdk panic"), + ), + ) + } + }() + + res, err := fn() + if err != nil { + reject.Invoke(jsError(err)) + + return + } + + resolve.Invoke(res) + }() + + return nil + }) + + // The Promise constructor invokes the executor synchronously, so by the + // time New returns the callback has already run and is never called + // again. Release it now, otherwise every wallet call leaks a Go + // callback handle for the lifetime of the page. + p := js.Global().Get("Promise").New(executor) + executor.Release() + + return p +} + +// rejected returns an immediately-rejected Promise carrying err. +func rejected(err error) any { + return js.Global().Get("Promise").Call("reject", jsError(err)) +} + +// jsError builds a JS Error from a Go error. +func jsError(err error) js.Value { + return js.Global().Get("Error").New(err.Error()) +} + +// browserDataDir is a WASM-safe default data directory. A browser has no $HOME +// for the daemon's `~` expansion to resolve against, and persistent state lives +// in OPFS-backed SQLite keyed by hashed file names rather than host directories +// (see darepod.ensureDataDir, a no-op under js/wasm), so any fixed in-origin +// path is all the embedded daemon needs. +const browserDataDir = "/darepo" + +// startConfig renders the start request as a config JSON string, injecting the +// browser-safe data dir when the caller didn't set one. Without it the embedded +// daemon's config validation expands the default `~/.darepod` via +// os.UserHomeDir, which fails with "$HOME is not defined" under wasm_exec.js +// and aborts start before the wallet boots. +func startConfig(req js.Value) string { + if req.IsUndefined() || req.IsNull() { + req = js.Global().Get("Object").New() + } + + if v := req.Get("data_dir"); v.IsUndefined() || v.IsNull() || + v.String() == "" { + + req.Set("data_dir", browserDataDir) + } + + return stringify(req) +} + +// stringify renders a JS request value as a JSON string, or "" when absent. +func stringify(v js.Value) string { + if v.IsUndefined() || v.IsNull() { + return "" + } + + return js.Global().Get("JSON").Call("stringify", v).String() +} + +// jsonBytes renders a JS request value as JSON request bytes, or nil when +// absent (the facade treats a nil body as the zero request). +func jsonBytes(v js.Value) []byte { + s := stringify(v) + if s == "" { + return nil + } + + return []byte(s) +} + +// parse decodes a JSON response body into a JS value, mapping an empty body to +// null. +func parse(b []byte) js.Value { + if len(b) == 0 { + return js.Null() + } + + return js.Global().Get("JSON").Call("parse", string(b)) +} diff --git a/cmd/walletdk-wasm/stub.go b/cmd/walletdk-wasm/stub.go new file mode 100644 index 000000000..9ba64ed79 --- /dev/null +++ b/cmd/walletdk-wasm/stub.go @@ -0,0 +1,21 @@ +//go:build !js || !wasm + +// Command walletdk-wasm is only meaningful as a js/wasm target. This stub +// keeps the package buildable (and `go build ./...` green) on native +// toolchains by providing a main that explains how to build the real binary. +package main + +import ( + "fmt" + "os" +) + +// main reports that walletdk-wasm must be built for js/wasm. +func main() { + fmt.Fprintln( + os.Stderr, "walletdk-wasm is only supported on js/wasm; "+ + "build with GOOS=js GOARCH=wasm -tags \"mobile "+ + "walletdkrpc swapruntime\"", + ) + os.Exit(1) +}