From 63887f4809fe8ecca3faacd3cba023f2d70e28ed Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Fri, 26 Jan 2024 17:24:07 +0200 Subject: [PATCH 01/12] flush table change updates on trailing edge. Prevents race conditions in watched queries --- .changeset/warm-foxes-act.md | 5 +++++ apps/supabase-todolist | 2 +- .../src/client/AbstractPowerSyncDatabase.ts | 22 +++++++++++-------- yarn.lock | 2 +- 4 files changed, 20 insertions(+), 11 deletions(-) create mode 100644 .changeset/warm-foxes-act.md diff --git a/.changeset/warm-foxes-act.md b/.changeset/warm-foxes-act.md new file mode 100644 index 000000000..dc516f0e4 --- /dev/null +++ b/.changeset/warm-foxes-act.md @@ -0,0 +1,5 @@ +--- +'@journeyapps/powersync-sdk-common': patch +--- + +Fixed table change updates to be throttled and flushed on the trailing edge to avoid race conditions in watched queries. diff --git a/apps/supabase-todolist b/apps/supabase-todolist index ef67b6397..3bb0b34be 160000 --- a/apps/supabase-todolist +++ b/apps/supabase-todolist @@ -1 +1 @@ -Subproject commit ef67b6397cf1cbf70c7512dfeb3eddb0719bc8c8 +Subproject commit 3bb0b34be007be5f5bd056b9585dad763559f3b1 diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts index 3257e9b98..0a635033e 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts @@ -499,15 +499,19 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver((eventOptions) => { - const flushTableUpdates = _.throttle(async () => { - const intersection = _.intersection(watchedTables, throttledTableUpdates); - if (intersection.length) { - eventOptions.push({ - changedTables: intersection - }); - } - throttledTableUpdates = []; - }, throttleMs); + const flushTableUpdates = _.throttle( + async () => { + const intersection = _.intersection(watchedTables, throttledTableUpdates); + if (intersection.length) { + eventOptions.push({ + changedTables: intersection + }); + } + throttledTableUpdates = []; + }, + throttleMs, + { leading: false, trailing: true } + ); const dispose = this.database.registerListener({ tablesUpdated: async (update) => { diff --git a/yarn.lock b/yarn.lock index a63f460fe..ef834527f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2220,7 +2220,7 @@ "@types/yargs" "^17.0.8" chalk "^4.0.0" -"@journeyapps/react-native-quick-sqlite@^1.0.0", "@journeyapps/react-native-quick-sqlite@^1.1.0": +"@journeyapps/react-native-quick-sqlite@^1.1.0": version "1.1.0" resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-1.1.0.tgz#cf4aa6694b7232d0f86e565fdba4e41ef15d80cc" integrity sha512-Pg6VA6ABC7N5FrNB5eqTgNsKdzzmDSp5aBtnQh1BlcZu7ISPZdCcKo+ZJtKyzTAWpc17LIttvJwxez6zBxUdOw== From a0887111b56cafe847353da2e2c3934f5c53029b Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 29 Jan 2024 17:52:02 +0200 Subject: [PATCH 02/12] added viewname for tables --- .../src/client/AbstractPowerSyncDatabase.ts | 44 +++++++++--- .../src/db/schema/Schema.ts | 6 ++ .../src/db/schema/Table.ts | 72 ++++++++++++++++++- .../powersync-sdk-common/src/utils/strings.ts | 4 ++ 4 files changed, 116 insertions(+), 10 deletions(-) diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts index 0a635033e..fa15c81b1 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts @@ -18,6 +18,12 @@ import { CrudEntry } from './sync/bucket/CrudEntry'; import { mutexRunExclusive } from '../utils/mutex'; import { BaseObserver } from '../utils/BaseObserver'; import { EventIterator } from 'event-iterator'; +import { AssertionError } from 'assert'; +import { quoteIdentifier } from 'src/utils/strings'; + +export interface DisconnectAndClearOptions { + clearLocal?: boolean; +} export interface PowerSyncDatabaseOptions { schema: Schema; @@ -57,6 +63,10 @@ export interface PowerSyncDBListener extends StreamingSyncImplementationListener const POWERSYNC_TABLE_MATCH = /(^ps_data__|^ps_data_local__)/; +const DEFAULT_DISCONNECT_CLEAR_OPTIONS: DisconnectAndClearOptions = { + clearLocal: true +}; + export const DEFAULT_WATCH_THROTTLE_MS = 30; export const DEFAULT_POWERSYNC_DB_OPTIONS = { @@ -90,6 +100,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver void; protected _isReadyPromise: Promise; + protected _schema: Schema; constructor(protected options: PowerSyncDatabaseOptions) { super(); @@ -97,6 +108,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver cb.initialized?.()); } + async updateSchema(schema: Schema) { + if (this.abortController) { + throw new AssertionError({ message: 'Cannot update schema while connected' }); + } + + schema.validate(); + this._schema = schema; + await this.database.execute('SELECT powersync_replace_schema(?)', [JSON.stringify(this.schema.toJSON())]); + } + /** * Queues a CRUD upload when internal CRUD tables have been updated */ @@ -208,24 +229,31 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { - await tx.execute(`DELETE FROM ${PSInternalTable.OPLOG} WHERE 1`); - await tx.execute(`DELETE FROM ${PSInternalTable.CRUD} WHERE 1`); - await tx.execute(`DELETE FROM ${PSInternalTable.BUCKETS} WHERE 1`); + await tx.execute(`DELETE FROM ${PSInternalTable.OPLOG}`); + await tx.execute(`DELETE FROM ${PSInternalTable.CRUD}`); + await tx.execute(`DELETE FROM ${PSInternalTable.BUCKETS}`); + + const tableGlob = clearLocal ? 'ps_data_*' : 'ps_data__*'; const existingTableRows = await tx.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'" + ` + SELECT name FROM sqlite_master WHERE type='table' AND name GLOB ? + `, + [tableGlob] ); if (!existingTableRows.rows.length) { return; } for (const row of existingTableRows.rows._array) { - await tx.execute(`DELETE FROM ${row.name} WHERE 1`); + await tx.execute(`DELETE FROM ${quoteIdentifier(row.name)} WHERE 1`); } }); } diff --git a/packages/powersync-sdk-common/src/db/schema/Schema.ts b/packages/powersync-sdk-common/src/db/schema/Schema.ts index a2d290558..56bd93816 100644 --- a/packages/powersync-sdk-common/src/db/schema/Schema.ts +++ b/packages/powersync-sdk-common/src/db/schema/Schema.ts @@ -3,6 +3,12 @@ import type { Table } from './Table'; export class Schema { constructor(public tables: Table[]) {} + validate() { + for (const table of this.tables) { + table.validate(); + } + } + toJSON() { return { tables: this.tables.map((t) => t.toJSON()) diff --git a/packages/powersync-sdk-common/src/db/schema/Table.ts b/packages/powersync-sdk-common/src/db/schema/Table.ts index a31aa9a8a..64836c2a8 100644 --- a/packages/powersync-sdk-common/src/db/schema/Table.ts +++ b/packages/powersync-sdk-common/src/db/schema/Table.ts @@ -1,12 +1,18 @@ +import _ from 'lodash'; import { Column } from '../Column'; import type { Index } from './Index'; +import { AssertionError } from 'assert'; export interface TableOptions { + /** + * The synced table name, matching sync rules + */ name: string; columns: Column[]; indexes?: Index[]; localOnly?: boolean; insertOnly?: boolean; + viewName?: string; } export const DEFAULT_TABLE_OPTIONS: Partial = { @@ -15,6 +21,8 @@ export const DEFAULT_TABLE_OPTIONS: Partial = { localOnly: false }; +export const InvalidSQLCharacters = /[\"\'%,\.#\s\[\]]/; + export class Table { protected options: TableOptions; @@ -34,6 +42,14 @@ export class Table { return this.options.name; } + get viewNameOverride() { + return this.options.viewName; + } + + get viewName() { + return this.viewNameOverride || this.name; + } + get columns() { return this.options.columns; } @@ -59,13 +75,65 @@ export class Table { } get validName() { - // TODO verify - return !/[\"\'%,\.#\s\[\]]/.test(this.name); + return _.chain([this.name, this.viewNameOverride]) + .compact() + .every((name) => !InvalidSQLCharacters.test(name)) + .value(); + } + + validate() { + if (InvalidSQLCharacters.test(this.name)) { + throw new AssertionError({ message: `Invalid characters in table name: ${this.name}` }); + } else if (this.viewNameOverride && InvalidSQLCharacters.test(this.viewNameOverride!)) { + throw new AssertionError({ + message: ` + Invalid characters in view name: ${this.viewNameOverride}` + }); + } + + const columnNames = new Set(); + columnNames.add('id'); + for (const column in this.columns) { + if (column == 'id') { + throw new AssertionError({ + message: `${this.name}: id column is automatically added, custom id columns are not supported` + }); + } else if (columnNames.has(column)) { + throw new AssertionError({ message: `Duplicate column ${column}` }); + } else if (InvalidSQLCharacters.test(column)) { + throw new AssertionError({ message: `Invalid characters in column name: $name.${column}` }); + } + columnNames.add(column); + } + + const indexNames = new Set(); + + for (const index of this.indexes) { + if (indexNames.has(index.name)) { + throw new AssertionError({ message: `Duplicate index $name.${index}` }); + } else if (InvalidSQLCharacters.test(index.name)) { + throw new AssertionError({ + message: ` + Invalid characters in index name: $name.${index}` + }); + } + + for (const column of index.columns) { + if (!columnNames.has(column.name)) { + throw new AssertionError({ + message: `Column $name.${column.name} not found for index ${index.name}` + }); + } + } + + indexNames.add(index.name); + } } toJSON() { return { name: this.name, + viewName: this.viewName, local_only: this.localOnly, insert_only: this.insertOnly, columns: this.columns.map((c) => c.toJSON()), diff --git a/packages/powersync-sdk-common/src/utils/strings.ts b/packages/powersync-sdk-common/src/utils/strings.ts index 66a636439..057a2f6ad 100644 --- a/packages/powersync-sdk-common/src/utils/strings.ts +++ b/packages/powersync-sdk-common/src/utils/strings.ts @@ -5,3 +5,7 @@ export function quoteString(s: string) { export function quoteJsonPath(path: string) { return quoteString(`$.${path}`); } + +export function quoteIdentifier(s: string) { + return `"${s.replaceAll('"', '""')}"`; +} From ad3e93f7585c20d786810b67656b43f61c95d984 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 30 Jan 2024 12:32:46 +0200 Subject: [PATCH 03/12] added changesets --- .changeset/curly-peas-argue.md | 5 +++++ .changeset/warm-foxes-act.md | 2 +- .changeset/yellow-kangaroos-allow.md | 5 +++++ .../src/client/AbstractPowerSyncDatabase.ts | 2 +- packages/powersync-sdk-react-native/package.json | 2 +- yarn.lock | 8 ++++++++ 6 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 .changeset/curly-peas-argue.md create mode 100644 .changeset/yellow-kangaroos-allow.md diff --git a/.changeset/curly-peas-argue.md b/.changeset/curly-peas-argue.md new file mode 100644 index 000000000..d83948449 --- /dev/null +++ b/.changeset/curly-peas-argue.md @@ -0,0 +1,5 @@ +--- +'@journeyapps/powersync-sdk-common': patch +--- + +Added `viewName` option to Schema Table definitions. This allows for overriding a table's view name. diff --git a/.changeset/warm-foxes-act.md b/.changeset/warm-foxes-act.md index dc516f0e4..a9ca64b8e 100644 --- a/.changeset/warm-foxes-act.md +++ b/.changeset/warm-foxes-act.md @@ -2,4 +2,4 @@ '@journeyapps/powersync-sdk-common': patch --- -Fixed table change updates to be throttled and flushed on the trailing edge to avoid race conditions in watched queries. +Improved table change updates to be throttled on the trailing edge. This prevents unnecessary query on both the leading and rising edge. diff --git a/.changeset/yellow-kangaroos-allow.md b/.changeset/yellow-kangaroos-allow.md new file mode 100644 index 000000000..1b0a645c4 --- /dev/null +++ b/.changeset/yellow-kangaroos-allow.md @@ -0,0 +1,5 @@ +--- +'@journeyapps/powersync-sdk-react-native': patch +--- + +Bumped powrsync-sqlite-core to v0.1.5. Depdendent projects should run `pod repo update && pod update` in the `ios` folder for updates to reflect. diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts index fa15c81b1..abe6cb843 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts @@ -19,7 +19,7 @@ import { mutexRunExclusive } from '../utils/mutex'; import { BaseObserver } from '../utils/BaseObserver'; import { EventIterator } from 'event-iterator'; import { AssertionError } from 'assert'; -import { quoteIdentifier } from 'src/utils/strings'; +import { quoteIdentifier } from '../utils/strings'; export interface DisconnectAndClearOptions { clearLocal?: boolean; diff --git a/packages/powersync-sdk-react-native/package.json b/packages/powersync-sdk-react-native/package.json index e37cefe27..103133d20 100644 --- a/packages/powersync-sdk-react-native/package.json +++ b/packages/powersync-sdk-react-native/package.json @@ -44,7 +44,7 @@ "async-lock": "^1.4.0" }, "devDependencies": { - "@journeyapps/react-native-quick-sqlite": "^1.1.0", + "@journeyapps/react-native-quick-sqlite": "0.0.0-dev-20240130102422", "@types/async-lock": "^1.4.0", "react-native": "0.72.4", "react": "18.2.0", diff --git a/yarn.lock b/yarn.lock index ef834527f..fc110e236 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2220,6 +2220,14 @@ "@types/yargs" "^17.0.8" chalk "^4.0.0" +"@journeyapps/react-native-quick-sqlite@0.0.0-dev-20240130102422": + version "0.0.0-dev-20240130102422" + resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-0.0.0-dev-20240130102422.tgz#00f599008f99c12f7c425345819b87bf58b445e6" + integrity sha512-iwkEQdTHdZZw4BcoyRmv6O3iSconCgv/868zNVK1/Z87UbxEew1WKNDkuigUJsl70kI+C8AkrAfjmP6cKDKX5w== + dependencies: + lodash "^4.17.21" + uuid "3.4.0" + "@journeyapps/react-native-quick-sqlite@^1.1.0": version "1.1.0" resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-1.1.0.tgz#cf4aa6694b7232d0f86e565fdba4e41ef15d80cc" From f4cd47afb23572abd4c5ab0cfac85a92e94bd33b Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 30 Jan 2024 13:43:30 +0200 Subject: [PATCH 04/12] use view_name as payload --- packages/powersync-sdk-common/src/db/schema/Table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/powersync-sdk-common/src/db/schema/Table.ts b/packages/powersync-sdk-common/src/db/schema/Table.ts index 64836c2a8..b73cf7e88 100644 --- a/packages/powersync-sdk-common/src/db/schema/Table.ts +++ b/packages/powersync-sdk-common/src/db/schema/Table.ts @@ -133,7 +133,7 @@ export class Table { toJSON() { return { name: this.name, - viewName: this.viewName, + view_name: this.viewName, local_only: this.localOnly, insert_only: this.insertOnly, columns: this.columns.map((c) => c.toJSON()), From b8f7cf6a65dda1bcfff8c02acaa84a1106e5dc89 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 30 Jan 2024 13:58:56 +0200 Subject: [PATCH 05/12] use standard JS errors. Assert not supported in React Native --- .../src/client/AbstractPowerSyncDatabase.ts | 3 +- .../src/db/schema/Table.ts | 28 ++++++------------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts index abe6cb843..60d9fc992 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts @@ -18,7 +18,6 @@ import { CrudEntry } from './sync/bucket/CrudEntry'; import { mutexRunExclusive } from '../utils/mutex'; import { BaseObserver } from '../utils/BaseObserver'; import { EventIterator } from 'event-iterator'; -import { AssertionError } from 'assert'; import { quoteIdentifier } from '../utils/strings'; export interface DisconnectAndClearOptions { @@ -165,7 +164,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver(); columnNames.add('id'); for (const column in this.columns) { if (column == 'id') { - throw new AssertionError({ - message: `${this.name}: id column is automatically added, custom id columns are not supported` - }); + throw new Error(`${this.name}: id column is automatically added, custom id columns are not supported`); } else if (columnNames.has(column)) { - throw new AssertionError({ message: `Duplicate column ${column}` }); + throw new Error(`Duplicate column ${column}`); } else if (InvalidSQLCharacters.test(column)) { - throw new AssertionError({ message: `Invalid characters in column name: $name.${column}` }); + throw new Error(`Invalid characters in column name: $name.${column}`); } columnNames.add(column); } @@ -110,19 +105,14 @@ export class Table { for (const index of this.indexes) { if (indexNames.has(index.name)) { - throw new AssertionError({ message: `Duplicate index $name.${index}` }); + throw new Error(`Duplicate index $name.${index}`); } else if (InvalidSQLCharacters.test(index.name)) { - throw new AssertionError({ - message: ` - Invalid characters in index name: $name.${index}` - }); + throw new Error(`Invalid characters in index name: $name.${index}`); } for (const column of index.columns) { if (!columnNames.has(column.name)) { - throw new AssertionError({ - message: `Column $name.${column.name} not found for index ${index.name}` - }); + throw new Error(`Column $name.${column.name} not found for index ${index.name}`); } } From eba6e878080382f07ae52e7b8d53c8429e5a7117 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 30 Jan 2024 15:07:19 +0200 Subject: [PATCH 06/12] validate schema changes before application. --- apps/supabase-todolist | 2 +- .../src/client/AbstractPowerSyncDatabase.ts | 1 + .../src/db/schema/Schema.ts | 1 + .../src/db/schema/Table.ts | 15 ++++++++------- yarn.lock | 18 +++++++++--------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/apps/supabase-todolist b/apps/supabase-todolist index 3bb0b34be..1496d6c4a 160000 --- a/apps/supabase-todolist +++ b/apps/supabase-todolist @@ -1 +1 @@ -Subproject commit 3bb0b34be007be5f5bd056b9585dad763559f3b1 +Subproject commit 1496d6c4a4b38a2ad4eb895d22fea60aa8ba3e67 diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts index 60d9fc992..465731d9c 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts @@ -158,6 +158,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver cb.initialized?.()); } diff --git a/packages/powersync-sdk-common/src/db/schema/Schema.ts b/packages/powersync-sdk-common/src/db/schema/Schema.ts index 56bd93816..702c7018a 100644 --- a/packages/powersync-sdk-common/src/db/schema/Schema.ts +++ b/packages/powersync-sdk-common/src/db/schema/Schema.ts @@ -10,6 +10,7 @@ export class Schema { } toJSON() { + this.validate(); return { tables: this.tables.map((t) => t.toJSON()) }; diff --git a/packages/powersync-sdk-common/src/db/schema/Table.ts b/packages/powersync-sdk-common/src/db/schema/Table.ts index 56f656259..8d484e07f 100644 --- a/packages/powersync-sdk-common/src/db/schema/Table.ts +++ b/packages/powersync-sdk-common/src/db/schema/Table.ts @@ -90,15 +90,16 @@ export class Table { const columnNames = new Set(); columnNames.add('id'); - for (const column in this.columns) { - if (column == 'id') { + for (const column of this.columns) { + const { name: columnName } = column; + if (column.name == 'id') { throw new Error(`${this.name}: id column is automatically added, custom id columns are not supported`); - } else if (columnNames.has(column)) { - throw new Error(`Duplicate column ${column}`); - } else if (InvalidSQLCharacters.test(column)) { + } else if (columnNames.has(columnName)) { + throw new Error(`Duplicate column ${columnName}`); + } else if (InvalidSQLCharacters.test(columnName)) { throw new Error(`Invalid characters in column name: $name.${column}`); } - columnNames.add(column); + columnNames.add(columnName); } const indexNames = new Set(); @@ -112,7 +113,7 @@ export class Table { for (const column of index.columns) { if (!columnNames.has(column.name)) { - throw new Error(`Column $name.${column.name} not found for index ${index.name}`); + throw new Error(`Column ${column.name} not found for index ${index.name}`); } } diff --git a/yarn.lock b/yarn.lock index fc110e236..c8c703c42 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2228,14 +2228,6 @@ lodash "^4.17.21" uuid "3.4.0" -"@journeyapps/react-native-quick-sqlite@^1.1.0": - version "1.1.0" - resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-1.1.0.tgz#cf4aa6694b7232d0f86e565fdba4e41ef15d80cc" - integrity sha512-Pg6VA6ABC7N5FrNB5eqTgNsKdzzmDSp5aBtnQh1BlcZu7ISPZdCcKo+ZJtKyzTAWpc17LIttvJwxez6zBxUdOw== - dependencies: - lodash "^4.17.21" - uuid "3.4.0" - "@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2": version "0.3.3" resolved "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098" @@ -3611,7 +3603,7 @@ ajv-keywords@^5.1.0: dependencies: fast-deep-equal "^3.1.3" -ajv@^8.0.0, ajv@^8.9.0: +ajv@^8.0.0, ajv@^8.11.0, ajv@^8.9.0: version "8.12.0" resolved "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1" integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== @@ -5374,6 +5366,14 @@ expo-asset@~8.10.1: path-browserify "^1.0.0" url-parse "^1.5.9" +expo-build-properties@~0.8.3: + version "0.8.3" + resolved "https://registry.npmjs.org/expo-build-properties/-/expo-build-properties-0.8.3.tgz#fbfa156e9619bebda71c66af9a26ebc3490b2365" + integrity sha512-kEDDuAadHqJTkvCGK4fXYHVrePiJO1DjyW95AicmwuGwQvGJydYFbuoauf9ybAU+4UH4arhbce8gHI3ZpIj3Jw== + dependencies: + ajv "^8.11.0" + semver "^7.5.3" + expo-camera@~13.4.4: version "13.4.4" resolved "https://registry.npmjs.org/expo-camera/-/expo-camera-13.4.4.tgz#e01ead31a3150398d37e94c307f6937480680690" From 8e67836f98fbb3a6601a448f9082ea96ec217edc Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 30 Jan 2024 15:32:06 +0200 Subject: [PATCH 07/12] minor issue --- .changeset/curly-peas-argue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/curly-peas-argue.md b/.changeset/curly-peas-argue.md index d83948449..6df32a5b9 100644 --- a/.changeset/curly-peas-argue.md +++ b/.changeset/curly-peas-argue.md @@ -1,5 +1,5 @@ --- -'@journeyapps/powersync-sdk-common': patch +'@journeyapps/powersync-sdk-common': minor --- Added `viewName` option to Schema Table definitions. This allows for overriding a table's view name. From 9e61f8329b6cec170b7d65c3ddcfbed4a2e4f3d5 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 30 Jan 2024 15:38:10 +0200 Subject: [PATCH 08/12] warn on schema validation error --- .../src/client/AbstractPowerSyncDatabase.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts index 465731d9c..bad6431b8 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts @@ -168,7 +168,16 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver Date: Wed, 31 Jan 2024 15:20:01 +0200 Subject: [PATCH 09/12] update powersync-sqlite-core v0.1.6 --- .changeset/yellow-kangaroos-allow.md | 2 +- apps/supabase-todolist | 2 +- packages/powersync-sdk-react-native/package.json | 4 ++-- yarn.lock | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.changeset/yellow-kangaroos-allow.md b/.changeset/yellow-kangaroos-allow.md index 1b0a645c4..e3aa870f4 100644 --- a/.changeset/yellow-kangaroos-allow.md +++ b/.changeset/yellow-kangaroos-allow.md @@ -2,4 +2,4 @@ '@journeyapps/powersync-sdk-react-native': patch --- -Bumped powrsync-sqlite-core to v0.1.5. Depdendent projects should run `pod repo update && pod update` in the `ios` folder for updates to reflect. +Bumped powersync-sqlite-core to v0.1.6. Depdendent projects should run `pod repo update && pod update` in the `ios` folder for updates to reflect. diff --git a/apps/supabase-todolist b/apps/supabase-todolist index 1496d6c4a..1fa46abf9 160000 --- a/apps/supabase-todolist +++ b/apps/supabase-todolist @@ -1 +1 @@ -Subproject commit 1496d6c4a4b38a2ad4eb895d22fea60aa8ba3e67 +Subproject commit 1fa46abf9cd4e9af0586e82030f5b4b0a9cb93e7 diff --git a/packages/powersync-sdk-react-native/package.json b/packages/powersync-sdk-react-native/package.json index 103133d20..75bf1e5d4 100644 --- a/packages/powersync-sdk-react-native/package.json +++ b/packages/powersync-sdk-react-native/package.json @@ -27,7 +27,7 @@ }, "homepage": "https://docs.powersync.co/", "peerDependencies": { - "@journeyapps/react-native-quick-sqlite": "^1.1.0", + "@journeyapps/react-native-quick-sqlite": "0.0.0-dev-20240131124845", "base-64": "^1.0.0", "react": "*", "react-native": "*", @@ -44,7 +44,7 @@ "async-lock": "^1.4.0" }, "devDependencies": { - "@journeyapps/react-native-quick-sqlite": "0.0.0-dev-20240130102422", + "@journeyapps/react-native-quick-sqlite": "0.0.0-dev-20240131124845", "@types/async-lock": "^1.4.0", "react-native": "0.72.4", "react": "18.2.0", diff --git a/yarn.lock b/yarn.lock index c8c703c42..badfbea0e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2220,10 +2220,10 @@ "@types/yargs" "^17.0.8" chalk "^4.0.0" -"@journeyapps/react-native-quick-sqlite@0.0.0-dev-20240130102422": - version "0.0.0-dev-20240130102422" - resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-0.0.0-dev-20240130102422.tgz#00f599008f99c12f7c425345819b87bf58b445e6" - integrity sha512-iwkEQdTHdZZw4BcoyRmv6O3iSconCgv/868zNVK1/Z87UbxEew1WKNDkuigUJsl70kI+C8AkrAfjmP6cKDKX5w== +"@journeyapps/react-native-quick-sqlite@0.0.0-dev-20240131124845": + version "0.0.0-dev-20240131124845" + resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-0.0.0-dev-20240131124845.tgz#ebed4dceade9eb2d5009f4ffb32d255014aa9c63" + integrity sha512-0zo6gtR7IveQBBq1hchJtS5+VQ8fYRIdN0BnDrhz9RMzuAA75mO8MocHFXgH3oKNaO2rNLlpOXXS/c9SO8ty6Q== dependencies: lodash "^4.17.21" uuid "3.4.0" From c6138f51287385c5f717a1797bec799851f9c06e Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 31 Jan 2024 16:44:27 +0200 Subject: [PATCH 10/12] don't validate schema on --- .changeset/eight-squids-peel.md | 7 +++++++ .changeset/yellow-kangaroos-allow.md | 5 ----- packages/powersync-sdk-common/src/db/schema/Schema.ts | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 .changeset/eight-squids-peel.md delete mode 100644 .changeset/yellow-kangaroos-allow.md diff --git a/.changeset/eight-squids-peel.md b/.changeset/eight-squids-peel.md new file mode 100644 index 000000000..0e4743795 --- /dev/null +++ b/.changeset/eight-squids-peel.md @@ -0,0 +1,7 @@ +--- +'@journeyapps/powersync-sdk-react-native': minor +--- + +Bumped powersync-sqlite-core to v0.1.6. dependant projects should: +- Upgrade to `@journeyapps/react-native-quick-sqlite@1.1.1` +- run `pod repo update && pod update` in the `ios` folder for updates to reflect. diff --git a/.changeset/yellow-kangaroos-allow.md b/.changeset/yellow-kangaroos-allow.md deleted file mode 100644 index e3aa870f4..000000000 --- a/.changeset/yellow-kangaroos-allow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@journeyapps/powersync-sdk-react-native': patch ---- - -Bumped powersync-sqlite-core to v0.1.6. Depdendent projects should run `pod repo update && pod update` in the `ios` folder for updates to reflect. diff --git a/packages/powersync-sdk-common/src/db/schema/Schema.ts b/packages/powersync-sdk-common/src/db/schema/Schema.ts index 702c7018a..56bd93816 100644 --- a/packages/powersync-sdk-common/src/db/schema/Schema.ts +++ b/packages/powersync-sdk-common/src/db/schema/Schema.ts @@ -10,7 +10,6 @@ export class Schema { } toJSON() { - this.validate(); return { tables: this.tables.map((t) => t.toJSON()) }; From 787668a2963228a43da737975adbad8758aa5f5c Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 31 Jan 2024 17:29:41 +0200 Subject: [PATCH 11/12] add global sync locks --- .changeset/chilled-poets-think.md | 5 ++++ .../client/AbstractPowerSyncOpenFactory.ts | 5 +++- .../AbstractStreamingSyncImplementation.ts | 9 +++++-- .../powersync-sdk-react-native/package.json | 4 +-- .../src/db/PowerSyncDatabase.ts | 3 ++- .../RNQSDBOpenFactory.ts | 13 +++++++++- .../ReactNativeStreamingSyncImplementation.ts | 26 ++++++++++++++++++- yarn.lock | 8 ++++++ 8 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 .changeset/chilled-poets-think.md diff --git a/.changeset/chilled-poets-think.md b/.changeset/chilled-poets-think.md new file mode 100644 index 000000000..55f75263b --- /dev/null +++ b/.changeset/chilled-poets-think.md @@ -0,0 +1,5 @@ +--- +'@journeyapps/powersync-sdk-react-native': patch +--- + +Added global locks for syncing connections. Added warning when creating multiple Powersync instances. diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncOpenFactory.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncOpenFactory.ts index bcc7fe62c..7d7e8ab0f 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncOpenFactory.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncOpenFactory.ts @@ -1,3 +1,4 @@ +import Logger from 'js-logger'; import { DBAdapter } from '../db/DBAdapter'; import { Schema } from '../db/schema/Schema'; import { AbstractPowerSyncDatabase, PowerSyncDatabaseOptions } from './AbstractPowerSyncDatabase'; @@ -15,7 +16,9 @@ export interface PowerSyncOpenFactoryOptions extends Partial { export interface AbstractStreamingSyncImplementationOptions { adapter: BucketStorageAdapter; - remote: AbstractRemote; uploadCrud: () => Promise; + crudUploadThrottleMs?: number; + /** + * An identifier for which PowerSync DB this sync implementation is + * linked to. Most commonly DB name, but not restricted to DB name. + */ + identifier?: string; logger?: ILogger; + remote: AbstractRemote; retryDelayMs?: number; - crudUploadThrottleMs?: number; } export interface StreamingSyncImplementationListener extends BaseListener { diff --git a/packages/powersync-sdk-react-native/package.json b/packages/powersync-sdk-react-native/package.json index 75bf1e5d4..271ff7618 100644 --- a/packages/powersync-sdk-react-native/package.json +++ b/packages/powersync-sdk-react-native/package.json @@ -27,7 +27,7 @@ }, "homepage": "https://docs.powersync.co/", "peerDependencies": { - "@journeyapps/react-native-quick-sqlite": "0.0.0-dev-20240131124845", + "@journeyapps/react-native-quick-sqlite": "^1.1.1", "base-64": "^1.0.0", "react": "*", "react-native": "*", @@ -44,7 +44,7 @@ "async-lock": "^1.4.0" }, "devDependencies": { - "@journeyapps/react-native-quick-sqlite": "0.0.0-dev-20240131124845", + "@journeyapps/react-native-quick-sqlite": "^1.1.1", "@types/async-lock": "^1.4.0", "react-native": "0.72.4", "react": "18.2.0", diff --git a/packages/powersync-sdk-react-native/src/db/PowerSyncDatabase.ts b/packages/powersync-sdk-react-native/src/db/PowerSyncDatabase.ts index 10a918a97..3534382be 100644 --- a/packages/powersync-sdk-react-native/src/db/PowerSyncDatabase.ts +++ b/packages/powersync-sdk-react-native/src/db/PowerSyncDatabase.ts @@ -28,7 +28,8 @@ export class PowerSyncDatabase extends AbstractPowerSyncDatabase { await connector.uploadData(this); }, retryDelayMs: this.options.retryDelay, - crudUploadThrottleMs: this.options.crudUploadThrottleMs + crudUploadThrottleMs: this.options.crudUploadThrottleMs, + identifier: this.options.database.name }); } } diff --git a/packages/powersync-sdk-react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBOpenFactory.ts b/packages/powersync-sdk-react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBOpenFactory.ts index 772dd877d..c8f811c1e 100644 --- a/packages/powersync-sdk-react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBOpenFactory.ts +++ b/packages/powersync-sdk-react-native/src/db/adapters/react-native-quick-sqlite/RNQSDBOpenFactory.ts @@ -4,12 +4,19 @@ import { AbstractPowerSyncDatabase, AbstractPowerSyncDatabaseOpenFactory, DBAdapter, - PowerSyncDatabaseOptions + PowerSyncDatabaseOptions, + PowerSyncOpenFactoryOptions } from '@journeyapps/powersync-sdk-common'; import { PowerSyncDatabase } from '../../../db/PowerSyncDatabase'; import { RNQSDBAdapter } from './RNQSDBAdapter'; export class RNQSPowerSyncDatabaseOpenFactory extends AbstractPowerSyncDatabaseOpenFactory { + protected instanceGenerated: boolean; + + constructor(options: PowerSyncOpenFactoryOptions) { + super(options); + this.instanceGenerated = false; + } protected openDB(): DBAdapter { /** * React Native Quick SQLite opens files relative to the `Documents`dir on iOS and the `Files` @@ -38,6 +45,10 @@ export class RNQSPowerSyncDatabaseOpenFactory extends AbstractPowerSyncDatabaseO } generateInstance(options: PowerSyncDatabaseOptions): AbstractPowerSyncDatabase { + if (this.instanceGenerated) { + this.options.logger?.warn('Generating multiple PowerSync instances can sometimes cause unexpected results.'); + } + this.instanceGenerated = true; return new PowerSyncDatabase(options); } } diff --git a/packages/powersync-sdk-react-native/src/sync/stream/ReactNativeStreamingSyncImplementation.ts b/packages/powersync-sdk-react-native/src/sync/stream/ReactNativeStreamingSyncImplementation.ts index cfa90b2cb..55eafaae7 100644 --- a/packages/powersync-sdk-react-native/src/sync/stream/ReactNativeStreamingSyncImplementation.ts +++ b/packages/powersync-sdk-react-native/src/sync/stream/ReactNativeStreamingSyncImplementation.ts @@ -5,14 +5,38 @@ import { LockType } from '@journeyapps/powersync-sdk-common'; import Lock from 'async-lock'; + +/** + * Global locks which prevent multiple instances from syncing + * concurrently. + */ +const LOCKS = new Map>(); + export class ReactNativeStreamingSyncImplementation extends AbstractStreamingSyncImplementation { locks: Map; constructor(options: AbstractStreamingSyncImplementationOptions) { super(options); - this.locks = new Map(); + this.initLocks(); + } + + /** + * Configures global locks on sync process + */ + initLocks() { + const { identifier } = this.options; + if (identifier && LOCKS.has(identifier)) { + this.locks = LOCKS.get(identifier); + return; + } + + this.locks = new Map(); this.locks.set(LockType.CRUD, new Lock()); this.locks.set(LockType.SYNC, new Lock()); + + if (identifier) { + LOCKS.set(identifier, this.locks); + } } obtainLock(lockOptions: LockOptions): Promise { diff --git a/yarn.lock b/yarn.lock index badfbea0e..3405e26db 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2228,6 +2228,14 @@ lodash "^4.17.21" uuid "3.4.0" +"@journeyapps/react-native-quick-sqlite@^1.1.1": + version "1.1.1" + resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-1.1.1.tgz#2b13d739ca73026453717c887246f8e4a23f5f44" + integrity sha512-5I8zUZoFRgtnagjQygqnSyWG0L39ycFvum+ytsMmFW8LY1GpEZ+a3I6fWggwM82/Cc9YOheCVRIz47g0uBBZug== + dependencies: + lodash "^4.17.21" + uuid "3.4.0" + "@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2": version "0.3.3" resolved "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098" From db5ee0f73d55bcf022271ad1950227414f8b1d34 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 31 Jan 2024 17:33:52 +0200 Subject: [PATCH 12/12] update repo --- apps/supabase-todolist | 2 +- yarn.lock | 20 ++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/apps/supabase-todolist b/apps/supabase-todolist index 1fa46abf9..3bb0b34be 160000 --- a/apps/supabase-todolist +++ b/apps/supabase-todolist @@ -1 +1 @@ -Subproject commit 1fa46abf9cd4e9af0586e82030f5b4b0a9cb93e7 +Subproject commit 3bb0b34be007be5f5bd056b9585dad763559f3b1 diff --git a/yarn.lock b/yarn.lock index 3405e26db..ff2adb18a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2220,15 +2220,7 @@ "@types/yargs" "^17.0.8" chalk "^4.0.0" -"@journeyapps/react-native-quick-sqlite@0.0.0-dev-20240131124845": - version "0.0.0-dev-20240131124845" - resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-0.0.0-dev-20240131124845.tgz#ebed4dceade9eb2d5009f4ffb32d255014aa9c63" - integrity sha512-0zo6gtR7IveQBBq1hchJtS5+VQ8fYRIdN0BnDrhz9RMzuAA75mO8MocHFXgH3oKNaO2rNLlpOXXS/c9SO8ty6Q== - dependencies: - lodash "^4.17.21" - uuid "3.4.0" - -"@journeyapps/react-native-quick-sqlite@^1.1.1": +"@journeyapps/react-native-quick-sqlite@^1.1.0", "@journeyapps/react-native-quick-sqlite@^1.1.1": version "1.1.1" resolved "https://registry.npmjs.org/@journeyapps/react-native-quick-sqlite/-/react-native-quick-sqlite-1.1.1.tgz#2b13d739ca73026453717c887246f8e4a23f5f44" integrity sha512-5I8zUZoFRgtnagjQygqnSyWG0L39ycFvum+ytsMmFW8LY1GpEZ+a3I6fWggwM82/Cc9YOheCVRIz47g0uBBZug== @@ -3611,7 +3603,7 @@ ajv-keywords@^5.1.0: dependencies: fast-deep-equal "^3.1.3" -ajv@^8.0.0, ajv@^8.11.0, ajv@^8.9.0: +ajv@^8.0.0, ajv@^8.9.0: version "8.12.0" resolved "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1" integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== @@ -5374,14 +5366,6 @@ expo-asset@~8.10.1: path-browserify "^1.0.0" url-parse "^1.5.9" -expo-build-properties@~0.8.3: - version "0.8.3" - resolved "https://registry.npmjs.org/expo-build-properties/-/expo-build-properties-0.8.3.tgz#fbfa156e9619bebda71c66af9a26ebc3490b2365" - integrity sha512-kEDDuAadHqJTkvCGK4fXYHVrePiJO1DjyW95AicmwuGwQvGJydYFbuoauf9ybAU+4UH4arhbce8gHI3ZpIj3Jw== - dependencies: - ajv "^8.11.0" - semver "^7.5.3" - expo-camera@~13.4.4: version "13.4.4" resolved "https://registry.npmjs.org/expo-camera/-/expo-camera-13.4.4.tgz#e01ead31a3150398d37e94c307f6937480680690"