From 8a45194e92c65bb827cb487baddb7059e135e880 Mon Sep 17 00:00:00 2001 From: "Marc J. Schmidt" Date: Sun, 1 Oct 2023 18:52:45 +0200 Subject: [PATCH] feature(orm): custom typed database.raw() support this makes it possible to write custom SQL or custom pipelines (for Mongo) and get the correct type deserialised. fix(mongo): handle getMore() correctly when batchSize is too big. Also allow to set batchSize in a query manually using `Query.withBatchSize(32)`. fixes #486 --- packages/mongo/src/adapter.ts | 58 ++++++++++++++++++- .../mongo/src/client/command/aggregate.ts | 39 ++++++++++--- packages/mongo/src/client/command/count.ts | 2 +- .../src/client/command/createCollection.ts | 2 +- .../mongo/src/client/command/createIndexes.ts | 2 +- packages/mongo/src/client/command/delete.ts | 2 +- .../mongo/src/client/command/dropIndexes.ts | 2 +- packages/mongo/src/client/command/find.ts | 31 ++++++++-- .../mongo/src/client/command/findAndModify.ts | 2 +- packages/mongo/src/client/command/getMore.ts | 15 +++++ packages/mongo/src/client/command/insert.ts | 2 +- packages/mongo/src/client/command/update.ts | 2 +- packages/mongo/src/client/config.ts | 2 +- packages/mongo/src/client/connection.ts | 1 + packages/mongo/src/query.resolver.ts | 7 ++- packages/mongo/tests/mongo.spec.ts | 52 +++++++++++++++++ packages/orm-integration/src/various.ts | 20 +++---- packages/orm/src/database-adapter.ts | 2 +- packages/orm/src/database-session.ts | 7 ++- packages/orm/src/database.ts | 3 +- packages/orm/src/query.ts | 20 ++++++- packages/sql/src/sql-adapter.ts | 57 +++++++++++++++--- packages/sqlite/tests/sqlite.spec.ts | 26 +++++++++ 23 files changed, 303 insertions(+), 53 deletions(-) create mode 100644 packages/mongo/src/client/command/getMore.ts diff --git a/packages/mongo/src/adapter.ts b/packages/mongo/src/adapter.ts index d95adbd0a..2f506c077 100644 --- a/packages/mongo/src/adapter.ts +++ b/packages/mongo/src/adapter.ts @@ -8,8 +8,8 @@ * You should have received a copy of the MIT License along with this program. */ -import { DatabaseAdapter, DatabaseAdapterQueryFactory, DatabaseEntityRegistry, DatabaseSession, OrmEntity } from '@deepkit/orm'; -import { AbstractClassType, ClassType } from '@deepkit/core'; +import { DatabaseAdapter, DatabaseAdapterQueryFactory, DatabaseEntityRegistry, DatabaseSession, FindQuery, ItemNotFound, OrmEntity, RawFactory } from '@deepkit/orm'; +import { AbstractClassType, ClassType, isArray } from '@deepkit/core'; import { MongoDatabaseQuery } from './query.js'; import { MongoPersistence } from './persistence.js'; import { MongoClient } from './client/client.js'; @@ -19,7 +19,9 @@ import { MongoDatabaseTransaction } from './client/connection.js'; import { CreateIndex, CreateIndexesCommand } from './client/command/createIndexes.js'; import { DropIndexesCommand } from './client/command/dropIndexes.js'; import { CreateCollectionCommand } from './client/command/createCollection.js'; -import { entity, ReceiveType, ReflectionClass } from '@deepkit/type'; +import { entity, ReceiveType, ReflectionClass, resolveReceiveType } from '@deepkit/type'; +import { Command } from './client/command/command.js'; +import { AggregateCommand } from './client/command/aggregate.js'; export class MongoDatabaseQueryFactory extends DatabaseAdapterQueryFactory { constructor( @@ -35,6 +37,52 @@ export class MongoDatabaseQueryFactory extends DatabaseAdapterQueryFactory { } } +class MongoRawCommandQuery implements FindQuery { + constructor( + protected session: DatabaseSession, + protected client: MongoClient, + protected command: Command, + ) { + } + + async find(): Promise { + const res = await this.client.execute(this.command); + return res as any; + } + + async findOneOrUndefined(): Promise { + const res = await this.client.execute(this.command); + if (isArray(res)) return res[0]; + return res; + } + + async findOne(): Promise { + const item = await this.findOneOrUndefined(); + if (!item) throw new ItemNotFound('Could not find item'); + return item; + } +} + +export class MongoRawFactory implements RawFactory<[Command]> { + constructor( + protected session: DatabaseSession, + protected client: MongoClient, + ) { + } + + create( + commandOrPipeline: Command | any[], + type?: ReceiveType, + resultType?: ReceiveType, + ): MongoRawCommandQuery { + type = resolveReceiveType(type); + const resultSchema = resultType ? resolveReceiveType(resultType) : undefined; + + const command = isArray(commandOrPipeline) ? new AggregateCommand(ReflectionClass.from(type), commandOrPipeline, resultSchema) : commandOrPipeline; + return new MongoRawCommandQuery(this.session, this.client, command); + } +} + export class MongoDatabaseAdapter extends DatabaseAdapter { public readonly client: MongoClient; @@ -55,6 +103,10 @@ export class MongoDatabaseAdapter extends DatabaseAdapter { this.ormSequences = ReflectionClass.from(OrmSequence); } + rawFactory(session: DatabaseSession): MongoRawFactory { + return new MongoRawFactory(session, this.client); + } + getName(): string { return 'mongo'; } diff --git a/packages/mongo/src/client/command/aggregate.ts b/packages/mongo/src/client/command/aggregate.ts index 56b9833fa..13b33077e 100644 --- a/packages/mongo/src/client/command/aggregate.ts +++ b/packages/mongo/src/client/command/aggregate.ts @@ -10,8 +10,9 @@ import { toFastProperties } from '@deepkit/core'; import { BaseResponse, Command } from './command.js'; -import { InlineRuntimeType, ReflectionClass, Type, typeOf, UUID } from '@deepkit/type'; +import { getTypeJitContainer, InlineRuntimeType, isType, ReflectionClass, Type, typeOf, UUID } from '@deepkit/type'; import { MongoError } from '../error.js'; +import { GetMoreMessage } from './getMore.js'; interface AggregateMessage { aggregate: string; @@ -28,29 +29,31 @@ interface AggregateMessage { export class AggregateCommand extends Command { partial: boolean = false; + batchSize: number = 1_000_000; constructor( public schema: ReflectionClass, public pipeline: any[] = [], - public resultSchema?: ReflectionClass, + public resultSchema?: ReflectionClass | Type, ) { super(); } async execute(config, host, transaction): Promise { const cmd = { - aggregate: this.schema.collectionName || this.schema.name || 'unknown', + aggregate: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', pipeline: this.pipeline, cursor: { - batchSize: 1_000_000, //todo make configurable + batchSize: this.batchSize } }; if (transaction) transaction.applyTransaction(cmd); - const resultSchema = this.resultSchema || this.schema; + let resultSchema = this.resultSchema || this.schema; + if (resultSchema && !isType(resultSchema)) resultSchema = resultSchema.type; - const jit = resultSchema.getJitContainer(); + const jit = getTypeJitContainer(resultSchema); let specialisedResponse: Type | undefined = this.partial ? jit.mdbAggregatePartial : jit.mdbAggregate; if (!specialisedResponse) { @@ -82,14 +85,32 @@ export class AggregateCommand extends Command { } interface Response extends BaseResponse { - cursor: { id: BigInt, firstBatch?: any[], nextBatch?: any[] }; + cursor: { id: bigint, firstBatch?: any[], nextBatch?: any[] }; } const res = await this.sendAndWait(cmd, undefined, specialisedResponse); if (!res.cursor.firstBatch) throw new MongoError(`No firstBatch received`); - //todo: implement fetchMore and decrease batchSize - return res.cursor.firstBatch; + const result: R[] = res.cursor.firstBatch; + + let cursorId = res.cursor.id; + while (cursorId) { + const nextCommand = { + getMore: cursorId, + $db: cmd.$db, + collection: cmd.aggregate, + batchSize: cmd.cursor.batchSize, + }; + if (transaction) transaction.applyTransaction(nextCommand); + const next = await this.sendAndWait(nextCommand, undefined, specialisedResponse); + + if (next.cursor.nextBatch) { + result.push(...next.cursor.nextBatch); + } + cursorId = next.cursor.id; + } + + return result; } needsWritableHost(): boolean { diff --git a/packages/mongo/src/client/command/count.ts b/packages/mongo/src/client/command/count.ts index 177036581..c4caa27f0 100644 --- a/packages/mongo/src/client/command/count.ts +++ b/packages/mongo/src/client/command/count.ts @@ -39,7 +39,7 @@ export class CountCommand> extends Command { async execute(config, host, transaction): Promise { const cmd: any = { - count: this.schema.collectionName || this.schema.name || 'unknown', + count: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', query: this.query, limit: this.limit, diff --git a/packages/mongo/src/client/command/createCollection.ts b/packages/mongo/src/client/command/createCollection.ts index 5060fc026..60d1d3135 100644 --- a/packages/mongo/src/client/command/createCollection.ts +++ b/packages/mongo/src/client/command/createCollection.ts @@ -25,7 +25,7 @@ export class CreateCollectionCommand> extends Com async execute(config, host, transaction): Promise { const cmd: any = { - create: this.schema.collectionName || this.schema.name || 'unknown', + create: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', }; diff --git a/packages/mongo/src/client/command/createIndexes.ts b/packages/mongo/src/client/command/createIndexes.ts index 27024910e..f0edcf366 100644 --- a/packages/mongo/src/client/command/createIndexes.ts +++ b/packages/mongo/src/client/command/createIndexes.ts @@ -36,7 +36,7 @@ export class CreateIndexesCommand> extends Comman async execute(config, host, transaction): Promise { const cmd: any = { - createIndexes: this.schema.collectionName || this.schema.name || 'unknown', + createIndexes: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', indexes: this.indexes, }; diff --git a/packages/mongo/src/client/command/delete.ts b/packages/mongo/src/client/command/delete.ts index d3b9cdb36..3b8995e17 100644 --- a/packages/mongo/src/client/command/delete.ts +++ b/packages/mongo/src/client/command/delete.ts @@ -38,7 +38,7 @@ export class DeleteCommand> extends Command { async execute(config, host, transaction): Promise { const cmd = { - delete: this.schema.collectionName || this.schema.name || 'unknown', + delete: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', deletes: [ { diff --git a/packages/mongo/src/client/command/dropIndexes.ts b/packages/mongo/src/client/command/dropIndexes.ts index a25420a60..578f63397 100644 --- a/packages/mongo/src/client/command/dropIndexes.ts +++ b/packages/mongo/src/client/command/dropIndexes.ts @@ -27,7 +27,7 @@ export class DropIndexesCommand> extends Command async execute(config, host, transaction): Promise { const cmd: any = { - dropIndexes: this.schema.collectionName || this.schema.name || 'unknown', + dropIndexes: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', index: this.names }; diff --git a/packages/mongo/src/client/command/find.ts b/packages/mongo/src/client/command/find.ts index 680e50e0b..372eec654 100644 --- a/packages/mongo/src/client/command/find.ts +++ b/packages/mongo/src/client/command/find.ts @@ -13,6 +13,7 @@ import { toFastProperties } from '@deepkit/core'; import { DEEP_SORT } from '../../query.model.js'; import { InlineRuntimeType, ReflectionClass, ReflectionKind, typeOf, TypeUnion, UUID } from '@deepkit/type'; import { MongoError } from '../error.js'; +import { GetMoreMessage } from './getMore.js'; interface FindSchema { find: string; @@ -30,6 +31,8 @@ interface FindSchema { } export class FindCommand extends Command { + batchSize: number = 1_000_000; + constructor( public schema: ReflectionClass, public filter: { [name: string]: any } = {}, @@ -43,12 +46,12 @@ export class FindCommand extends Command { async execute(config, host, transaction): Promise { const cmd: FindSchema = { - find: this.schema.collectionName || this.schema.name || 'unknown', + find: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', filter: this.filter, limit: this.limit, skip: this.skip, - batchSize: 1_000_000, //todo make configurable + batchSize: this.batchSize, }; if (transaction) transaction.applyTransaction(cmd); @@ -119,14 +122,32 @@ export class FindCommand extends Command { } interface Response extends BaseResponse { - cursor: { id: BigInt, firstBatch?: any[], nextBatch?: any[] }; + cursor: { id: bigint, firstBatch?: any[], nextBatch?: any[] }; } const res = await this.sendAndWait(cmd, undefined, specialisedResponse); if (!res.cursor.firstBatch) throw new MongoError(`No firstBatch received`); - //todo: implement fetchMore and decrease batchSize - return res.cursor.firstBatch; + const result: T[] = res.cursor.firstBatch; + + let cursorId = res.cursor.id; + while (cursorId) { + const nextCommand = { + getMore: cursorId, + $db: cmd.$db, + collection: cmd.find, + batchSize: cmd.batchSize, + }; + if (transaction) transaction.applyTransaction(nextCommand); + const next = await this.sendAndWait(nextCommand, undefined, specialisedResponse); + + if (next.cursor.nextBatch) { + result.push(...next.cursor.nextBatch); + } + cursorId = next.cursor.id; + } + + return result; } needsWritableHost(): boolean { diff --git a/packages/mongo/src/client/command/findAndModify.ts b/packages/mongo/src/client/command/findAndModify.ts index 97aa6946a..a241af365 100644 --- a/packages/mongo/src/client/command/findAndModify.ts +++ b/packages/mongo/src/client/command/findAndModify.ts @@ -47,7 +47,7 @@ export class FindAndModifyCommand> extends Comman for (const name of this.fields) fields[name] = 1; const cmd: any = { - findAndModify: this.schema.collectionName || this.schema.name || 'unknown', + findAndModify: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', query: this.query, update: this.update, diff --git a/packages/mongo/src/client/command/getMore.ts b/packages/mongo/src/client/command/getMore.ts new file mode 100644 index 000000000..150345d66 --- /dev/null +++ b/packages/mongo/src/client/command/getMore.ts @@ -0,0 +1,15 @@ +import { UUID } from '@deepkit/type'; + +export interface GetMoreMessage { + getMore: bigint; + $db: string; + collection: string; + batchSize?: number; + maxTimeMS?: number; + comment?: string; + + lsid?: { id: UUID }, + txnNumber?: number, + startTransaction?: boolean, + autocommit?: boolean, +} diff --git a/packages/mongo/src/client/command/insert.ts b/packages/mongo/src/client/command/insert.ts index 73070ba5b..c1004bfc8 100644 --- a/packages/mongo/src/client/command/insert.ts +++ b/packages/mongo/src/client/command/insert.ts @@ -35,7 +35,7 @@ export class InsertCommand extends Command { async execute(config, host, transaction): Promise { const cmd: any = { - insert: this.schema.collectionName || this.schema.name || 'unknown', + insert: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', documents: this.documents, }; diff --git a/packages/mongo/src/client/command/update.ts b/packages/mongo/src/client/command/update.ts index d3174cad1..e714118b6 100644 --- a/packages/mongo/src/client/command/update.ts +++ b/packages/mongo/src/client/command/update.ts @@ -40,7 +40,7 @@ export class UpdateCommand> extends Command { async execute(config, host, transaction): Promise { const cmd = { - update: this.schema.collectionName || this.schema.name || 'unknown', + update: this.schema.getCollectionName() || 'unknown', $db: this.schema.databaseSchemaName || config.defaultDb || 'admin', updates: this.updates }; diff --git a/packages/mongo/src/client/config.ts b/packages/mongo/src/client/config.ts index 992256381..316eafc11 100644 --- a/packages/mongo/src/client/config.ts +++ b/packages/mongo/src/client/config.ts @@ -138,7 +138,7 @@ export class MongoClientConfig { } resolveCollectionName(schema: ReflectionClass): string { - return schema.collectionName || schema.name || 'unknown'; + return schema.getCollectionName() || 'unknown'; } @singleStack() diff --git a/packages/mongo/src/client/connection.ts b/packages/mongo/src/client/connection.ts index a9406c573..b37b02ffd 100644 --- a/packages/mongo/src/client/connection.ts +++ b/packages/mongo/src/client/connection.ts @@ -220,6 +220,7 @@ export class MongoDatabaseTransaction extends DatabaseTransaction { async begin() { if (!this.connection) return; + // see https://github.com/mongodb/specifications/blob/master/source/sessions/driver-sessions.rst this.lsid = { id: uuid() }; this.txnNumber = MongoDatabaseTransaction.txnNumber++; // const res = await this.connection.execute(new StartSessionCommand()); diff --git a/packages/mongo/src/query.resolver.ts b/packages/mongo/src/query.resolver.ts index d290d7a38..91c94a67c 100644 --- a/packages/mongo/src/query.resolver.ts +++ b/packages/mongo/src/query.resolver.ts @@ -310,6 +310,7 @@ export class MongoQueryResolver extends GenericQueryResolve const pipeline = this.buildAggregationPipeline(model); const resultsSchema = model.isAggregate() ? this.getCachedAggregationSchema(model) : this.getSchemaWithJoins(); const command = new AggregateCommand(this.classSchema, pipeline, resultsSchema); + if (model.batchSize) command.batchSize = model.batchSize; command.partial = model.isPartial(); const items = await connection.execute(command); if (model.isAggregate()) { @@ -318,14 +319,16 @@ export class MongoQueryResolver extends GenericQueryResolve return items.map(v => formatter.hydrate(model, v)); } else { - const items = await connection.execute(new FindCommand( + const command = new FindCommand( this.classSchema, getMongoFilter(this.classSchema, model), this.getProjection(this.classSchema, model.select), this.getSortFromModel(model.sort), model.limit, model.skip, - )); + ); + if (model.batchSize) command.batchSize = model.batchSize; + const items = await connection.execute(command); return items.map(v => formatter.hydrate(model, v)); } } finally { diff --git a/packages/mongo/tests/mongo.spec.ts b/packages/mongo/tests/mongo.spec.ts index d239b8e13..f81049aa0 100644 --- a/packages/mongo/tests/mongo.spec.ts +++ b/packages/mongo/tests/mongo.spec.ts @@ -565,3 +565,55 @@ test('aggregation without accumulators', async () => { { downloadsSum: 0, category: 'pdfs' }, ]); }); + +test('raw', async () => { + class Model { + public _id: MongoId & PrimaryKey = ''; + + constructor(public id: number) { + } + } + + const db = await createDatabase('raw'); + + { + const session = db.createSession(); + for (let i = 0; i < 1000; i++) session.add(new Model(i)); + await session.commit(); + } + + const result = await db.raw([{ $match: { id: { $gt: 500 } } }, { $count: 'count' }]).findOne(); + expect(result.count).toBe(499); + + const items = await db.raw([{ $match: { id: { $lt: 500 } } }]).find(); + expect(items.length).toBe(500); + expect(items[0]).toBeInstanceOf(Model); +}); + +test('batch', async () => { + class Model { + public _id: MongoId & PrimaryKey = ''; + + constructor(public id: number) { + } + } + + const db = await createDatabase('batch'); + { + const session = db.createSession(); + for (let i = 0; i < 1000; i++) session.add(new Model(i)); + await session.commit(); + } + + { + const items = await db.query(Model).withBatchSize(10).find(); + expect(items.length).toBe(1000); + } + + { + const session = db.createSession(); + session.useTransaction(); + const items = await session.query(Model).withBatchSize(10).find(); + expect(items.length).toBe(1000); + } +}); diff --git a/packages/orm-integration/src/various.ts b/packages/orm-integration/src/various.ts index 8182733e5..eb0b564a2 100644 --- a/packages/orm-integration/src/various.ts +++ b/packages/orm-integration/src/various.ts @@ -22,29 +22,30 @@ export const variousTests = { await database.persist(cast({ username: 'peter' })); await database.persist(cast({ username: 'marie' })); + type Count = {count: number}; + { - const result = await database.raw(sql`SELECT count(*) as count + const result = await database.raw(sql`SELECT count(*) as count FROM ${user}`).findOne(); expect(result.count).toBe(2); } { - const result = await database.createSession().raw(sql`SELECT count(*) as count + const result = await database.createSession().raw(sql`SELECT count(*) as count FROM ${user}`).findOne(); expect(result.count).toBe(2); } { const id = 1; - const result = await database.createSession().raw(sql`SELECT count(*) as count + const result = await database.createSession().raw(sql`SELECT count(*) as count FROM ${user} WHERE id > ${id}`).findOne(); expect(result.count).toBe(1); } { - const result = await database.raw(sql`SELECT * - FROM ${user}`).find(); + const result = await database.raw(sql`SELECT * FROM ${user}`).find(); expect(result).toEqual([ { id: 1, username: 'peter' }, { id: 2, username: 'marie' }, @@ -52,20 +53,17 @@ export const variousTests = { } { - const result = await database.createSession().raw(sql`SELECT * - FROM ${user}`).find(); + const result = await database.createSession().raw(sql`SELECT * FROM ${user}`).find(); expect(result).toEqual([ { id: 1, username: 'peter' }, { id: 2, username: 'marie' }, ]); } - await database.raw(sql`DELETE - FROM ${user}`).execute(); + await database.raw(sql`DELETE FROM ${user}`).execute(); { - const result = await database.raw(sql`SELECT count(*) as count - FROM ${user}`).findOne(); + const result = await database.raw(sql`SELECT count(*) as count FROM ${user}`).findOne(); expect(result.count).toBe(0); } database.disconnect(); diff --git a/packages/orm/src/database-adapter.ts b/packages/orm/src/database-adapter.ts index e900c6f22..4385af451 100644 --- a/packages/orm/src/database-adapter.ts +++ b/packages/orm/src/database-adapter.ts @@ -39,7 +39,7 @@ export abstract class DatabasePersistence { } export class RawFactory> { - create(...args: A): any { + create(...args: A): any { throw new Error(`Current database adapter does not support raw mode.`); } } diff --git a/packages/orm/src/database-session.ts b/packages/orm/src/database-session.ts index dcc0a9f23..12081d6b9 100644 --- a/packages/orm/src/database-session.ts +++ b/packages/orm/src/database-session.ts @@ -11,7 +11,7 @@ import type { DatabaseAdapter, DatabasePersistence, DatabasePersistenceChangeSet } from './database-adapter.js'; import { DatabaseEntityRegistry } from './database-adapter.js'; import { DatabaseValidationError, OrmEntity } from './type.js'; -import { AbstractClassType, ClassType, CustomError } from '@deepkit/core'; +import { AbstractClassType, ClassType, CustomError, forwardTypeArguments } from '@deepkit/core'; import { getPrimaryKeyExtractor, isReferenceInstance, @@ -322,7 +322,10 @@ export class DatabaseSession { this.query = query as any; const factory = this.adapter.rawFactory(this); - this.raw = factory.create.bind(factory); + this.raw = (...args: any[]) => { + forwardTypeArguments(this.raw, factory.create); + return factory.create(...args); + }; } /** diff --git a/packages/orm/src/database.ts b/packages/orm/src/database.ts index abe7e450c..83c727a57 100644 --- a/packages/orm/src/database.ts +++ b/packages/orm/src/database.ts @@ -8,7 +8,7 @@ * You should have received a copy of the MIT License along with this program. */ -import { AbstractClassType, ClassType, getClassName, getClassTypeFromInstance } from '@deepkit/core'; +import { AbstractClassType, ClassType, forwardTypeArguments, getClassName, getClassTypeFromInstance } from '@deepkit/core'; import { entityAnnotation, EntityOptions, @@ -163,6 +163,7 @@ export class Database { const session = this.createSession(); session.withIdentityMap = false; if (!session.raw) throw new Error('Adapter has no raw mode'); + forwardTypeArguments(this.raw, session.raw); return session.raw(...args); }; diff --git a/packages/orm/src/query.ts b/packages/orm/src/query.ts index f7319b309..2eb61e229 100644 --- a/packages/orm/src/query.ts +++ b/packages/orm/src/query.ts @@ -103,6 +103,7 @@ export class DatabaseQueryModel(); public returning: (keyof T & string)[] = []; + public batchSize?: number; isLazyLoaded(field: string): boolean { return this.lazyLoad.has(field); @@ -139,6 +140,7 @@ export class DatabaseQueryModel { return c as any; } + withBatchSize(batchSize: number): this { + const c = this.clone(); + c.model.batchSize = batchSize; + return c as any; + } + groupBy[]>(...field: K): this { const c = this.clone(); c.model.groupBy = new Set([...field as string[]]); @@ -628,6 +636,14 @@ export abstract class GenericQueryResolver, patchResult: PatchResult): Promise; } +export interface FindQuery { + findOneOrUndefined(): Promise; + + findOne(): Promise; + + find(): Promise; +} + export type Methods = { [K in keyof T]: K extends keyof Query ? never : T[K] extends ((...args: any[]) => any) ? K : never }[keyof T]; /** @@ -660,7 +676,9 @@ export class Query extends BaseQuery { this.model.withIdentityMap = session.withIdentityMap; } - static from & { _: () => T }, T extends ReturnType['_']>, B extends ClassType>>(this: B, query: Q): Replace, Resolve> { + static from & { + _: () => T + }, T extends ReturnType['_']>, B extends ClassType>>(this: B, query: Q): Replace, Resolve> { const result = (new this(query.classSchema, query.session, query.resolver)); result.model = query.model.clone(result); return result as any; diff --git a/packages/sql/src/sql-adapter.ts b/packages/sql/src/sql-adapter.ts index e602e230d..8fdb94081 100644 --- a/packages/sql/src/sql-adapter.ts +++ b/packages/sql/src/sql-adapter.ts @@ -23,7 +23,9 @@ import { DatabaseTransaction, DeleteResult, FilterQuery, + FindQuery, GenericQueryResolver, + ItemNotFound, OrmEntity, PatchResult, Query, @@ -33,11 +35,23 @@ import { SORT_ORDER } from '@deepkit/orm'; import { AbstractClassType, ClassType, isArray, isClass } from '@deepkit/core'; -import { Changes, entity, getPartialSerializeFunction, getSerializeFunction, PrimaryKey, ReceiveType, ReflectionClass } from '@deepkit/type'; +import { + castFunction, + Changes, + entity, + getPartialSerializeFunction, + getSerializeFunction, + PrimaryKey, + ReceiveType, + ReflectionClass, + ReflectionKind, + resolveReceiveType, + Type +} from '@deepkit/type'; import { DefaultPlatform, SqlPlaceholderStrategy } from './platform/default-platform.js'; import { Sql, SqlBuilder } from './sql-builder.js'; import { SqlFormatter } from './sql-formatter.js'; -import { DatabaseComparator, DatabaseModel, Table } from './schema/table.js'; +import { DatabaseComparator, DatabaseModel } from './schema/table.js'; import { Stopwatch } from '@deepkit/stopwatch'; export type SORT_TYPE = SORT_ORDER | { $meta: 'textScore' }; @@ -329,6 +343,8 @@ export function identifier(id: string) { return new SQLQueryIdentifier(id); } +export type SqlStatement = { sql: string, params: any[] }; + export class SqlQuery { constructor(public parts: ReadonlyArray) { } @@ -341,7 +357,7 @@ export class SqlQuery { platform: DefaultPlatform, placeholderStrategy: SqlPlaceholderStrategy, tableName?: string - ): { sql: string, params: any[] } { + ): SqlStatement { let sql = ''; const params: any[] = []; @@ -488,12 +504,13 @@ export class SqlMigrationHandler { } } -export class RawQuery { +export class RawQuery implements FindQuery { constructor( protected session: DatabaseSession, protected connectionPool: SQLConnectionPool, protected platform: DefaultPlatform, protected sql: SqlQuery, + protected type: Type, ) { } @@ -511,23 +528,44 @@ export class RawQuery { } } + /** + * Returns the SQL statement with placeholders replaced with the actual values. + */ + getSql(): SqlStatement { + return this.sql.convertToSQL(this.platform, new this.platform.placeholderStrategy); + } + /** * Returns the raw result of a single row. + * + * Note that this does not resolve/map joins. Use the regular database.query() for that. */ - async findOne(): Promise { + async findOneOrUndefined(): Promise { return (await this.find())[0]; } + /** + * Note that this does not resolve/map joins. Use the regular database.query() for that. + */ + async findOne(): Promise { + const item = await this.findOneOrUndefined(); + if (!item) throw new ItemNotFound('Item not found'); + return item; + } + /** * Returns the full result of a raw query. + * + * Note that this does not resolve/map joins. Use the regular database.query() for that. */ - async find(): Promise { + async find(): Promise { const sql = this.sql.convertToSQL(this.platform, new this.platform.placeholderStrategy); const connection = await this.connectionPool.getConnection(this.session.logger, this.session.assignedTransaction, this.session.stopwatch); try { + const caster = castFunction(undefined, undefined, undefined, this.type); const res = await connection.execAndReturnAll(sql.sql, sql.params); - return isArray(res) ? [...res] : []; + return (isArray(res) ? [...res] : []).map(v => caster(v)) as T[]; } finally { connection.release(); } @@ -542,8 +580,9 @@ export class SqlRawFactory implements RawFactory<[SqlQuery]> { ) { } - create(sql: SqlQuery): RawQuery { - return new RawQuery(this.session, this.connectionPool, this.platform, sql); + create(sql: SqlQuery, type?: ReceiveType): RawQuery { + type = type ? resolveReceiveType(type) : { kind: ReflectionKind.any }; + return new RawQuery(this.session, this.connectionPool, this.platform, sql, type); } } diff --git a/packages/sqlite/tests/sqlite.spec.ts b/packages/sqlite/tests/sqlite.spec.ts index bf8eea80b..789ef2a54 100644 --- a/packages/sqlite/tests/sqlite.spec.ts +++ b/packages/sqlite/tests/sqlite.spec.ts @@ -6,6 +6,7 @@ import { SQLiteDatabaseAdapter, SQLiteDatabaseTransaction } from '../src/sqlite- import { sleep } from '@deepkit/core'; import { AutoIncrement, BackReference, cast, Entity, entity, isReferenceInstance, PrimaryKey, Reference, ReflectionClass, serialize, typeOf, UUID, uuid } from '@deepkit/type'; import { DatabaseEntityRegistry } from '@deepkit/orm'; +import { sql } from '@deepkit/sql'; test('reflection circular reference', () => { const user = ReflectionClass.from(User); @@ -266,6 +267,31 @@ test('connection pool', async () => { } }); +test('raw', async () => { + class User { + id!: number & PrimaryKey; + name!: string; + } + + const database = await databaseFactory([ReflectionClass.from()]); + + await database.persist({ id: 1, name: 'Peter' }); + await database.persist({ id: 2, name: 'Marie' }); + + { + const row = await database.raw<{ count: bigint }>(sql`SELECT count(*) as count FROM user`).findOne(); + expect(row.count).toBe(2n); + } + + { + const rows = await database.raw(sql`SELECT * FROM user`).find(); + expect(rows.length).toBe(2); + expect(rows[0]).toEqual({ id: 1, name: 'Peter' }); + expect(rows[1]).toEqual({ id: 2, name: 'Marie' }); + expect(rows[0]).toBeInstanceOf(User); + } +}); + test(':memory: connection pool', async () => { const sqlite = new SQLiteDatabaseAdapter(':memory:');