chore: organize config, validation, decorators (#8118)

* refactor: validation

* refactor: utilities

* refactor: config

server/src/infra/infra.config.ts → server/src/config.ts

@@ -1,7 +1,42 @@
 import { RegisterQueueOptions } from '@nestjs/bullmq';
+import { ConfigModuleOptions } from '@nestjs/config';
 import { QueueOptions } from 'bullmq';
 import { RedisOptions } from 'ioredis';
+import Joi from 'joi';
 import { QueueName } from 'src/domain/job/job.constants';
+import { LogLevel } from 'src/infra/entities/system-config.entity';
+
+const WHEN_DB_URL_SET = Joi.when('DB_URL', {
+  is: Joi.exist(),
+  then: Joi.string().optional(),
+  otherwise: Joi.string().required(),
+});
+
+export const immichAppConfig: ConfigModuleOptions = {
+  envFilePath: '.env',
+  isGlobal: true,
+  validationSchema: Joi.object({
+    NODE_ENV: Joi.string().optional().valid('development', 'production', 'staging').default('development'),
+    LOG_LEVEL: Joi.string()
+      .optional()
+      .valid(...Object.values(LogLevel)),
+
+    DB_USERNAME: WHEN_DB_URL_SET,
+    DB_PASSWORD: WHEN_DB_URL_SET,
+    DB_DATABASE_NAME: WHEN_DB_URL_SET,
+    DB_URL: Joi.string().optional(),
+    DB_VECTOR_EXTENSION: Joi.string().optional().valid('pgvector', 'pgvecto.rs').default('pgvecto.rs'),
+
+    MACHINE_LEARNING_PORT: Joi.number().optional(),
+    MICROSERVICES_PORT: Joi.number().optional(),
+    IMMICH_METRICS_PORT: Joi.number().optional(),
+
+    IMMICH_METRICS: Joi.boolean().optional().default(false),
+    IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),
+    IMMICH_API_METRICS: Joi.boolean().optional().default(false),
+    IMMICH_IO_METRICS: Joi.boolean().optional().default(false),
+  }),
+};
 
 function parseRedisConfig(): RedisOptions {
   const redisUrl = process.env.REDIS_URL;

server/src/decorators.ts

@@ -0,0 +1,124 @@
+import { SetMetadata } from '@nestjs/common';
+import _ from 'lodash';
+import { setUnion } from 'src/utils';
+
+// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the
+// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching
+// by a list of IDs) requires splitting the query into multiple chunks.
+// We are rounding down this limit, as queries commonly include other filters and parameters.
+export const DATABASE_PARAMETER_CHUNK_SIZE = 65_500;
+
+/**
+ * Chunks an array or set into smaller collections of the same type and specified size.
+ *
+ * @param collection The collection to chunk.
+ * @param size The size of each chunk.
+ */
+function chunks<T>(collection: Array<T>, size: number): Array<Array<T>>;
+function chunks<T>(collection: Set<T>, size: number): Array<Set<T>>;
+function chunks<T>(collection: Array<T> | Set<T>, size: number): Array<Array<T>> | Array<Set<T>> {
+  if (collection instanceof Set) {
+    const result = [];
+    let chunk = new Set<T>();
+    for (const element of collection) {
+      chunk.add(element);
+      if (chunk.size === size) {
+        result.push(chunk);
+        chunk = new Set<T>();
+      }
+    }
+    if (chunk.size > 0) {
+      result.push(chunk);
+    }
+    return result;
+  } else {
+    return _.chunk(collection, size);
+  }
+}
+
+/**
+ * Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
+ * to overcome the maximum number of parameters allowed by the database driver.
+ *
+ * @param options.paramIndex The index of the function parameter to chunk. Defaults to 0.
+ * @param options.flatten Whether to flatten the results. Defaults to false.
+ */
+export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
+  return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
+    const originalMethod = descriptor.value;
+    const parameterIndex = options.paramIndex ?? 0;
+    descriptor.value = async function (...arguments_: any[]) {
+      const argument = arguments_[parameterIndex];
+
+      // Early return if argument length is less than or equal to the chunk size.
+      if (
+        (Array.isArray(argument) && argument.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
+        (argument instanceof Set && argument.size <= DATABASE_PARAMETER_CHUNK_SIZE)
+      ) {
+        return await originalMethod.apply(this, arguments_);
+      }
+
+      return Promise.all(
+        chunks(argument, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
+          await Reflect.apply(originalMethod, this, [
+            ...arguments_.slice(0, parameterIndex),
+            chunk,
+            ...arguments_.slice(parameterIndex + 1),
+          ]);
+        }),
+      ).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
+    };
+  };
+}
+
+export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator {
+  return Chunked({ ...options, mergeFn: _.flatten });
+}
+
+export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
+  return Chunked({ ...options, mergeFn: setUnion });
+}
+
+// https://stackoverflow.com/a/74898678
+export function DecorateAll(
+  decorator: <T>(
+    target: any,
+    propertyKey: string,
+    descriptor: TypedPropertyDescriptor<T>,
+  ) => TypedPropertyDescriptor<T> | void,
+) {
+  return (target: any) => {
+    const descriptors = Object.getOwnPropertyDescriptors(target.prototype);
+    for (const [propName, descriptor] of Object.entries(descriptors)) {
+      const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor';
+      if (!isMethod) {
+        continue;
+      }
+      decorator({ ...target, constructor: { ...target.constructor, name: target.name } as any }, propName, descriptor);
+      Object.defineProperty(target.prototype, propName, descriptor);
+    }
+  };
+}
+
+const UUID = '00000000-0000-4000-a000-000000000000';
+
+export const DummyValue = {
+  UUID,
+  UUID_SET: new Set([UUID]),
+  PAGINATION: { take: 10, skip: 0 },
+  EMAIL: '[email protected]',
+  STRING: 'abcdefghi',
+  BUFFER: Buffer.from('abcdefghi'),
+  DATE: new Date(),
+  TIME_BUCKET: '2024-01-01T00:00:00.000Z',
+};
+
+export const GENERATE_SQL_KEY = 'generate-sql-key';
+
+export interface GenerateSqlQueries {
+  name?: string;
+  params: unknown[];
+}
+
+/** Decorator to enable versioning/tracking of generated Sql */
+export const GenerateSql = (...options: GenerateSqlQueries[]) => SetMetadata(GENERATE_SQL_KEY, options);

server/src/domain/access/access.core.ts

@@ -1,8 +1,8 @@
 import { BadRequestException, UnauthorizedException } from '@nestjs/common';
 import { AuthDto } from 'src/domain/auth/auth.dto';
-import { setDifference, setIsEqual, setUnion } from 'src/domain/domain.util';
 import { IAccessRepository } from 'src/domain/repositories/access.repository';
 import { SharedLinkEntity } from 'src/infra/entities/shared-link.entity';
+import { setDifference, setIsEqual, setUnion } from 'src/utils';
 
 export enum Permission {
   ACTIVITY_CREATE = 'activity.create',

server/src/domain/activity/activity.dto.ts

@@ -1,8 +1,8 @@
 import { ApiProperty } from '@nestjs/swagger';
 import { IsEnum, IsNotEmpty, IsString, ValidateIf } from 'class-validator';
-import { Optional, ValidateUUID } from 'src/domain/domain.util';
 import { UserDto, mapSimpleUser } from 'src/domain/user/response-dto/user-response.dto';
 import { ActivityEntity } from 'src/infra/entities/activity.entity';
+import { Optional, ValidateUUID } from 'src/validation';
 
 export enum ReactionType {
   COMMENT = 'comment',

server/src/domain/album/album-response.dto.ts

@@ -1,9 +1,9 @@
 import { ApiProperty } from '@nestjs/swagger';
 import { AssetResponseDto, mapAsset } from 'src/domain/asset/response-dto/asset-response.dto';
 import { AuthDto } from 'src/domain/auth/auth.dto';
-import { Optional } from 'src/domain/domain.util';
 import { UserResponseDto, mapUser } from 'src/domain/user/response-dto/user-response.dto';
 import { AlbumEntity, AssetOrder } from 'src/infra/entities/album.entity';
+import { Optional } from 'src/validation';
 
 export class AlbumResponseDto {
   id!: string;

server/src/domain/album/album.service.ts

@@ -14,14 +14,14 @@ import { AlbumInfoDto } from 'src/domain/album/dto/album.dto';
 import { GetAlbumsDto } from 'src/domain/album/dto/get-albums.dto';
 import { BulkIdErrorReason, BulkIdResponseDto, BulkIdsDto } from 'src/domain/asset/response-dto/asset-ids-response.dto';
 import { AuthDto } from 'src/domain/auth/auth.dto';
-import { setUnion } from 'src/domain/domain.util';
 import { IAccessRepository } from 'src/domain/repositories/access.repository';
 import { AlbumAssetCount, AlbumInfoOptions, IAlbumRepository } from 'src/domain/repositories/album.repository';
 import { IAssetRepository } from 'src/domain/repositories/asset.repository';
 import { IUserRepository } from 'src/domain/repositories/user.repository';
 import { AlbumEntity } from 'src/infra/entities/album.entity';
 import { AssetEntity } from 'src/infra/entities/asset.entity';
 import { UserEntity } from 'src/infra/entities/user.entity';
+import { setUnion } from 'src/utils';
 
 @Injectable()
 export class AlbumService {

server/src/domain/album/dto/album-add-users.dto.ts

@@ -1,5 +1,5 @@
 import { ArrayNotEmpty } from 'class-validator';
-import { ValidateUUID } from 'src/domain/domain.util';
+import { ValidateUUID } from 'src/validation';
 
 export class AddUsersDto {
   @ValidateUUID({ each: true })

server/src/domain/album/dto/album-create.dto.ts

@@ -1,6 +1,6 @@
 import { ApiProperty } from '@nestjs/swagger';
 import { IsString } from 'class-validator';
-import { Optional, ValidateUUID } from 'src/domain/domain.util';
+import { Optional, ValidateUUID } from 'src/validation';
 
 export class CreateAlbumDto {
   @IsString()

server/src/domain/album/dto/album-update.dto.ts

@@ -1,7 +1,7 @@
 import { ApiProperty } from '@nestjs/swagger';
 import { IsEnum, IsString } from 'class-validator';
-import { Optional, ValidateBoolean, ValidateUUID } from 'src/domain/domain.util';
 import { AssetOrder } from 'src/infra/entities/album.entity';
+import { Optional, ValidateBoolean, ValidateUUID } from 'src/validation';
 
 export class UpdateAlbumDto {
   @Optional()

server/src/domain/album/dto/album.dto.ts

@@ -1,4 +1,4 @@
-import { ValidateBoolean } from 'src/domain/domain.util';
+import { ValidateBoolean } from 'src/validation';
 
 export class AlbumInfoDto {
   @ValidateBoolean({ optional: true })

server/src/domain/album/dto/get-albums.dto.ts

@@ -1,4 +1,4 @@
-import { ValidateBoolean, ValidateUUID } from 'src/domain/domain.util';
+import { ValidateBoolean, ValidateUUID } from 'src/validation';
 
 export class GetAlbumsDto {
   @ValidateBoolean({ optional: true })

server/src/domain/api-key/api-key.dto.ts

@@ -1,5 +1,5 @@
 import { IsNotEmpty, IsString } from 'class-validator';
-import { Optional } from 'src/domain/domain.util';
+import { Optional } from 'src/validation';
 export class APIKeyCreateDto {
   @IsString()
   @IsNotEmpty()

server/src/domain/asset/asset.service.ts

@@ -21,7 +21,6 @@ import { MapMarkerResponseDto } from 'src/domain/asset/response-dto/map-marker-r
 import { TimeBucketResponseDto } from 'src/domain/asset/response-dto/time-bucket-response.dto';
 import { AuthDto } from 'src/domain/auth/auth.dto';
 import { mimeTypes } from 'src/domain/domain.constant';
-import { usePagination } from 'src/domain/domain.util';
 import { JOBS_ASSET_PAGINATION_SIZE, JobName } from 'src/domain/job/job.constants';
 import { IAssetDeletionJob, ISidecarWriteJob } from 'src/domain/job/job.interface';
 import { IAccessRepository } from 'src/domain/repositories/access.repository';
@@ -38,6 +37,7 @@ import { SystemConfigCore } from 'src/domain/system-config/system-config.core';
 import { AssetEntity } from 'src/infra/entities/asset.entity';
 import { LibraryType } from 'src/infra/entities/library.entity';
 import { ImmichLogger } from 'src/infra/logger';
+import { usePagination } from 'src/utils';
 
 export enum UploadFieldName {
   ASSET_DATA = 'assetData',

server/src/domain/asset/dto/asset-ids.dto.ts

@@ -1,6 +1,6 @@
 import { ApiProperty } from '@nestjs/swagger';
 import { IsEnum } from 'class-validator';
-import { ValidateUUID } from 'src/domain/domain.util';
+import { ValidateUUID } from 'src/validation';
 
 export class AssetIdsDto {
   @ValidateUUID({ each: true })

server/src/domain/asset/dto/asset-stack.dto.ts

@@ -1,4 +1,4 @@
-import { ValidateUUID } from 'src/domain/domain.util';
+import { ValidateUUID } from 'src/validation';
 
 export class UpdateStackParentDto {
   @ValidateUUID()

server/src/domain/asset/dto/asset-statistics.dto.ts

@@ -1,7 +1,7 @@
 import { ApiProperty } from '@nestjs/swagger';
-import { ValidateBoolean } from 'src/domain/domain.util';
 import { AssetStats } from 'src/domain/repositories/asset.repository';
 import { AssetType } from 'src/infra/entities/asset.entity';
+import { ValidateBoolean } from 'src/validation';
 
 export class AssetStatsDto {
   @ValidateBoolean({ optional: true })

server/src/domain/asset/dto/asset.dto.ts

@@ -10,7 +10,7 @@ import {
   ValidateIf,
 } from 'class-validator';
 import { BulkIdsDto } from 'src/domain/asset/response-dto/asset-ids-response.dto';
-import { Optional, ValidateBoolean, ValidateUUID } from 'src/domain/domain.util';
+import { Optional, ValidateBoolean, ValidateUUID } from 'src/validation';
 
 export class DeviceIdDto {
   @IsNotEmpty()

server/src/domain/asset/dto/map-marker.dto.ts

@@ -1,4 +1,4 @@
-import { ValidateBoolean, ValidateDate } from 'src/domain/domain.util';
+import { ValidateBoolean, ValidateDate } from 'src/validation';
 
 export class MapMarkerDto {
   @ValidateBoolean({ optional: true })

server/src/domain/asset/dto/time-bucket.dto.ts

@@ -1,8 +1,8 @@
 import { ApiProperty } from '@nestjs/swagger';
 import { IsEnum, IsNotEmpty, IsString } from 'class-validator';
-import { Optional, ValidateBoolean, ValidateUUID } from 'src/domain/domain.util';
 import { TimeBucketSize } from 'src/domain/repositories/asset.repository';
 import { AssetOrder } from 'src/infra/entities/album.entity';
+import { Optional, ValidateBoolean, ValidateUUID } from 'src/validation';
 
 export class TimeBucketDto {
   @IsNotEmpty()

server/src/domain/asset/response-dto/asset-ids-response.dto.ts

@@ -1,4 +1,4 @@
-import { ValidateUUID } from 'src/domain/domain.util';
+import { ValidateUUID } from 'src/validation';
 
 /** @deprecated Use `BulkIdResponseDto` instead */
 export enum AssetIdErrorReason {

server/src/domain/audit/audit.dto.ts

@@ -1,9 +1,9 @@
 import { ApiProperty } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
 import { IsArray, IsEnum, IsString, IsUUID, ValidateNested } from 'class-validator';
-import { Optional, ValidateDate, ValidateUUID } from 'src/domain/domain.util';
 import { EntityType } from 'src/infra/entities/audit.entity';
 import { AssetPathType, PathType, PersonPathType, UserPathType } from 'src/infra/entities/move.entity';
+import { Optional, ValidateDate, ValidateUUID } from 'src/validation';
 
 const PathEnum = Object.values({ ...AssetPathType, ...PersonPathType, ...UserPathType });
 

server/src/domain/audit/audit.service.ts

@@ -12,7 +12,6 @@ import {
 } from 'src/domain/audit/audit.dto';
 import { AuthDto } from 'src/domain/auth/auth.dto';
 import { AUDIT_LOG_MAX_DURATION } from 'src/domain/domain.constant';
-import { usePagination } from 'src/domain/domain.util';
 import { JOBS_ASSET_PAGINATION_SIZE } from 'src/domain/job/job.constants';
 import { IAccessRepository } from 'src/domain/repositories/access.repository';
 import { IAssetRepository } from 'src/domain/repositories/asset.repository';
@@ -26,6 +25,7 @@ import { StorageCore, StorageFolder } from 'src/domain/storage/storage.core';
 import { DatabaseAction } from 'src/infra/entities/audit.entity';
 import { AssetPathType, PersonPathType, UserPathType } from 'src/infra/entities/move.entity';
 import { ImmichLogger } from 'src/infra/logger';
+import { usePagination } from 'src/utils';
 
 @Injectable()
 export class AuditService {

server/src/domain/auth/auth.service.ts

@@ -34,7 +34,6 @@ import {
   mapLoginResponse,
   mapUserToken,
 } from 'src/domain/auth/auth.dto';
-import { HumanReadableSize } from 'src/domain/domain.util';
 import { IAccessRepository } from 'src/domain/repositories/access.repository';
 import { IKeyRepository } from 'src/domain/repositories/api-key.repository';
 import { ICryptoRepository } from 'src/domain/repositories/crypto.repository';
@@ -49,6 +48,7 @@ import { UserCore } from 'src/domain/user/user.core';
 import { SystemConfig } from 'src/infra/entities/system-config.entity';
 import { UserEntity } from 'src/infra/entities/user.entity';
 import { ImmichLogger } from 'src/infra/logger';
+import { HumanReadableSize } from 'src/utils';
 
 export interface LoginDetails {
   isSecure: boolean;