fix(policy): run pg/sqlite tests, misc dual db compatibility fixes

.github/workflows/build-test.yml

@@ -34,6 +34,7 @@ jobs:
         strategy:
             matrix:
                 node-version: [20.x]
+                provider: [sqlite, postgresql]
 
         steps:
             - name: Checkout
@@ -76,4 +77,6 @@ jobs:
               run: pnpm run lint
 
             - name: Test
+              env:
+                  TEST_DB_PROVIDER: ${{ matrix.provider }}
               run: pnpm run test

packages/language/res/stdlib.zmodel

@@ -123,8 +123,10 @@ function future(): Any {
 } @@@expressionContext([AccessPolicy])
 
 /**
- * If the field value contains the search string. By default, the search is case-sensitive,
- * but you can override the behavior with the "caseInSensitive" argument.
+ * Checks if the field value contains the search string. By default, the search is case-sensitive, and
+ * "LIKE" operator is used to match. If `caseInSensitive` is true, "ILIKE" operator is used if
+ * supported, otherwise it still falls back to "LIKE" and delivers whatever the database's 
+ * behavior is.
  */
 function contains(field: String, search: String, caseInSensitive: Boolean?): Boolean {
 } @@@expressionContext([AccessPolicy, ValidationRule])
@@ -136,15 +138,21 @@ function search(field: String, search: String): Boolean {
 } @@@expressionContext([AccessPolicy])
 
 /**
- * If the field value starts with the search string
+ * Checks the field value starts with the search string. By default, the search is case-sensitive, and
+ * "LIKE" operator is used to match. If `caseInSensitive` is true, "ILIKE" operator is used if
+ * supported, otherwise it still falls back to "LIKE" and delivers whatever the database's 
+ * behavior is.
  */
-function startsWith(field: String, search: String): Boolean {
+function startsWith(field: String, search: String, caseInSensitive: Boolean?): Boolean {
 } @@@expressionContext([AccessPolicy, ValidationRule])
 
 /**
- * If the field value ends with the search string
+ * Checks if the field value ends with the search string. By default, the search is case-sensitive, and
+ * "LIKE" operator is used to match. If `caseInSensitive` is true, "ILIKE" operator is used if
+ * supported, otherwise it still falls back to "LIKE" and delivers whatever the database's 
+ * behavior is.
  */
-function endsWith(field: String, search: String): Boolean {
+function endsWith(field: String, search: String, caseInSensitive: Boolean?): Boolean {
 } @@@expressionContext([AccessPolicy, ValidationRule])
 
 /**

packages/runtime/package.json

@@ -8,6 +8,8 @@
         "watch": "tsup-node --watch",
         "lint": "eslint src --ext ts",
         "test": "vitest run && pnpm test:typecheck",
+        "test:sqlite": "TEST_DB_PROVIDER=sqlite vitest run",
+        "test:postgresql": "TEST_DB_PROVIDER=postgresql vitest run",
         "test:generate": "tsx test/scripts/generate.ts",
         "test:typecheck": "tsc --project tsconfig.test.json",
         "pack": "pnpm pack"

packages/runtime/src/client/client-impl.ts

@@ -61,7 +61,7 @@ export class ClientImpl<Schema extends SchemaDef> {
         executor?: QueryExecutor,
     ) {
         this.$schema = schema;
-        this.$options = options ?? ({} as ClientOptions<Schema>);
+        this.$options = options;
 
         this.$options.functions = {
             ...BuiltinFunctions,
@@ -326,7 +326,7 @@ export class ClientImpl<Schema extends SchemaDef> {
 
 function createClientProxy<Schema extends SchemaDef>(client: ClientImpl<Schema>): ClientImpl<Schema> {
     const inputValidator = new InputValidator(client.$schema);
-    const resultProcessor = new ResultProcessor(client.$schema);
+    const resultProcessor = new ResultProcessor(client.$schema, client.$options);
 
     return new Proxy(client, {
         get: (target, prop, receiver) => {

packages/runtime/src/client/crud/dialects/base-dialect.ts

@@ -45,6 +45,10 @@ export abstract class BaseCrudDialect<Schema extends SchemaDef> {
         return value;
     }
 
+    transformOutput(value: unknown, _type: BuiltinType) {
+        return value;
+    }
+
     // #region common query builders
 
     buildSelectModel(eb: ExpressionBuilder<any, any>, model: string, modelAlias: string) {
@@ -1255,5 +1259,15 @@ export abstract class BaseCrudDialect<Schema extends SchemaDef> {
      */
     abstract get supportInsertWithDefault(): boolean;
 
+    /**
+     * Gets the SQL column type for the given field definition.
+     */
+    abstract getFieldSqlType(fieldDef: FieldDef): string;
+
+    /*
+     * Gets the string casing behavior for the dialect.
+     */
+    abstract getStringCasingBehavior(): { supportsILike: boolean; likeCaseSensitive: boolean };
+
     // #endregion
 }

packages/runtime/src/client/crud/dialects/postgresql.ts

@@ -1,4 +1,5 @@
 import { invariant } from '@zenstackhq/common-helpers';
+import Decimal from 'decimal.js';
 import {
     sql,
     type Expression,
@@ -11,6 +12,8 @@ import { match } from 'ts-pattern';
 import type { BuiltinType, FieldDef, GetModels, SchemaDef } from '../../../schema';
 import { DELEGATE_JOINED_FIELD_PREFIX } from '../../constants';
 import type { FindArgs } from '../../crud-types';
+import { QueryError } from '../../errors';
+import type { ClientOptions } from '../../options';
 import {
     buildJoinPairs,
     getDelegateDescendantModels,
@@ -23,6 +26,10 @@ import {
 import { BaseCrudDialect } from './base-dialect';
 
 export class PostgresCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect<Schema> {
+    constructor(schema: Schema, options: ClientOptions<Schema>) {
+        super(schema, options);
+    }
+
     override get provider() {
         return 'postgresql' as const;
     }
@@ -44,13 +51,69 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends BaseCrudDiale
         } else {
             return match(type)
                 .with('DateTime', () =>
-                    value instanceof Date ? value : typeof value === 'string' ? new Date(value) : value,
+                    value instanceof Date
+                        ? value.toISOString()
+                        : typeof value === 'string'
+                          ? new Date(value).toISOString()
+                          : value,
                 )
                 .with('Decimal', () => (value !== null ? value.toString() : value))
                 .otherwise(() => value);
         }
     }
 
+    override transformOutput(value: unknown, type: BuiltinType) {
+        if (value === null || value === undefined) {
+            return value;
+        }
+        return match(type)
+            .with('DateTime', () => this.transformOutputDate(value))
+            .with('Bytes', () => this.transformOutputBytes(value))
+            .with('BigInt', () => this.transformOutputBigInt(value))
+            .with('Decimal', () => this.transformDecimal(value))
+            .otherwise(() => super.transformOutput(value, type));
+    }
+
+    private transformOutputBigInt(value: unknown) {
+        if (typeof value === 'bigint') {
+            return value;
+        }
+        invariant(
+            typeof value === 'string' || typeof value === 'number',
+            `Expected string or number, got ${typeof value}`,
+        );
+        return BigInt(value);
+    }
+
+    private transformDecimal(value: unknown) {
+        if (value instanceof Decimal) {
+            return value;
+        }
+        invariant(
+            typeof value === 'string' || typeof value === 'number' || value instanceof Decimal,
+            `Expected string, number or Decimal, got ${typeof value}`,
+        );
+        return new Decimal(value);
+    }
+
+    private transformOutputDate(value: unknown) {
+        if (typeof value === 'string') {
+            return new Date(value);
+        } else if (value instanceof Date && this.options.fixPostgresTimezone !== false) {
+            // SPECIAL NOTES:
+            // node-pg has a terrible quirk that it returns the date value in local timezone
+            // as a `Date` object although for `DateTime` field the data in DB is stored in UTC
+            // see: https://github.com/brianc/node-postgres/issues/429
+            return new Date(value.getTime() - value.getTimezoneOffset() * 60 * 1000);
+        } else {
+            return value;
+        }
+    }
+
+    private transformOutputBytes(value: unknown) {
+        return Buffer.isBuffer(value) ? Uint8Array.from(value) : value;
+    }
+
     override buildRelationSelection(
         query: SelectQueryBuilder<any, any, any>,
         model: string,
@@ -370,4 +433,42 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends BaseCrudDiale
     override get supportInsertWithDefault() {
         return true;
     }
+
+    override getFieldSqlType(fieldDef: FieldDef) {
+        // TODO: respect `@db.x` attributes
+        if (fieldDef.relation) {
+            throw new QueryError('Cannot get SQL type of a relation field');
+        }
+
+        let result: string;
+
+        if (this.schema.enums?.[fieldDef.type]) {
+            // enums are treated as text
+            result = 'text';
+        } else {
+            result = match(fieldDef.type)
+                .with('String', () => 'text')
+                .with('Boolean', () => 'boolean')
+                .with('Int', () => 'integer')
+                .with('BigInt', () => 'bigint')
+                .with('Float', () => 'double precision')
+                .with('Decimal', () => 'decimal')
+                .with('DateTime', () => 'timestamp')
+                .with('Bytes', () => 'bytea')
+                .with('Json', () => 'jsonb')
+                // fallback to text
+                .otherwise(() => 'text');
+        }
+
+        if (fieldDef.array) {
+            result += '[]';
+        }
+
+        return result;
+    }
+
+    override getStringCasingBehavior() {
+        // Postgres `LIKE` is case-sensitive, `ILIKE` is case-insensitive
+        return { supportsILike: true, likeCaseSensitive: true };
+    }
 }

packages/runtime/src/client/crud/dialects/sqlite.ts

@@ -1,5 +1,5 @@
 import { invariant } from '@zenstackhq/common-helpers';
-import type Decimal from 'decimal.js';
+import Decimal from 'decimal.js';
 import {
     ExpressionWrapper,
     sql,
@@ -9,9 +9,10 @@ import {
     type SelectQueryBuilder,
 } from 'kysely';
 import { match } from 'ts-pattern';
-import type { BuiltinType, GetModels, SchemaDef } from '../../../schema';
+import type { BuiltinType, FieldDef, GetModels, SchemaDef } from '../../../schema';
 import { DELEGATE_JOINED_FIELD_PREFIX } from '../../constants';
 import type { FindArgs } from '../../crud-types';
+import { QueryError } from '../../errors';
 import {
     getDelegateDescendantModels,
     getManyToManyRelation,
@@ -41,7 +42,13 @@ export class SqliteCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect
             } else {
                 return match(type)
                     .with('Boolean', () => (value ? 1 : 0))
-                    .with('DateTime', () => (value instanceof Date ? value.toISOString() : value))
+                    .with('DateTime', () =>
+                        value instanceof Date
+                            ? value.toISOString()
+                            : typeof value === 'string'
+                              ? new Date(value).toISOString()
+                              : value,
+                    )
                     .with('Decimal', () => (value as Decimal).toString())
                     .with('Bytes', () => Buffer.from(value as Uint8Array))
                     .with('Json', () => JSON.stringify(value))
@@ -50,6 +57,76 @@ export class SqliteCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect
         }
     }
 
+    override transformOutput(value: unknown, type: BuiltinType) {
+        if (value === null || value === undefined) {
+            return value;
+        } else if (this.schema.typeDefs && type in this.schema.typeDefs) {
+            // typed JSON field
+            return this.transformOutputJson(value);
+        } else {
+            return match(type)
+                .with('Boolean', () => this.transformOutputBoolean(value))
+                .with('DateTime', () => this.transformOutputDate(value))
+                .with('Bytes', () => this.transformOutputBytes(value))
+                .with('Decimal', () => this.transformOutputDecimal(value))
+                .with('BigInt', () => this.transformOutputBigInt(value))
+                .with('Json', () => this.transformOutputJson(value))
+                .otherwise(() => super.transformOutput(value, type));
+        }
+    }
+
+    private transformOutputDecimal(value: unknown) {
+        if (value instanceof Decimal) {
+            return value;
+        }
+        invariant(
+            typeof value === 'string' || typeof value === 'number' || value instanceof Decimal,
+            `Expected string, number or Decimal, got ${typeof value}`,
+        );
+        return new Decimal(value);
+    }
+
+    private transformOutputBigInt(value: unknown) {
+        if (typeof value === 'bigint') {
+            return value;
+        }
+        invariant(
+            typeof value === 'string' || typeof value === 'number',
+            `Expected string or number, got ${typeof value}`,
+        );
+        return BigInt(value);
+    }
+
+    private transformOutputBoolean(value: unknown) {
+        return !!value;
+    }
+
+    private transformOutputDate(value: unknown) {
+        if (typeof value === 'number') {
+            return new Date(value);
+        } else if (typeof value === 'string') {
+            return new Date(value);
+        } else {
+            return value;
+        }
+    }
+
+    private transformOutputBytes(value: unknown) {
+        return Buffer.isBuffer(value) ? Uint8Array.from(value) : value;
+    }
+
+    private transformOutputJson(value: unknown) {
+        // better-sqlite3 typically returns JSON as string; be tolerant
+        if (typeof value === 'string') {
+            try {
+                return JSON.parse(value);
+            } catch (e) {
+                throw new QueryError('Invalid JSON returned', e);
+            }
+        }
+        return value;
+    }
+
     override buildRelationSelection(
         query: SelectQueryBuilder<any, any, any>,
         model: string,
@@ -301,4 +378,39 @@ export class SqliteCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect
     override get supportInsertWithDefault() {
         return false;
     }
+
+    override getFieldSqlType(fieldDef: FieldDef) {
+        // TODO: respect `@db.x` attributes
+        if (fieldDef.relation) {
+            throw new QueryError('Cannot get SQL type of a relation field');
+        }
+        if (fieldDef.array) {
+            throw new QueryError('SQLite does not support scalar list type');
+        }
+
+        if (this.schema.enums?.[fieldDef.type]) {
+            // enums are stored as text
+            return 'text';
+        }
+
+        return (
+            match(fieldDef.type)
+                .with('String', () => 'text')
+                .with('Boolean', () => 'integer')
+                .with('Int', () => 'integer')
+                .with('BigInt', () => 'integer')
+                .with('Float', () => 'real')
+                .with('Decimal', () => 'decimal')
+                .with('DateTime', () => 'numeric')
+                .with('Bytes', () => 'blob')
+                .with('Json', () => 'jsonb')
+                // fallback to text
+                .otherwise(() => 'text')
+        );
+    }
+
+    override getStringCasingBehavior() {
+        // SQLite `LIKE` is case-insensitive, and there is no `ILIKE`
+        return { supportsILike: false, likeCaseSensitive: false };
+    }
 }