feat: schema-check

docs/src/pages/errors.md

@@ -32,3 +32,7 @@ If you would like to migrate from one media location to another, simply successf
 4. Start up Immich
 
 After version `1.136.0`, Immich can detect when a media location has moved and will automatically update the database paths to keep them in sync.
+
+## Schema drift
+
+Schema drift is when the database schema is out of sync with the code. This could be the result of manual database tinkering, issues during a database restore, or something else. Schema drift can lead to data corruption, application bugs, and other unpredictable behavior. Please reconcile the differences as soon as possible. Specifically, missing `CONSTRAINT`s can result in duplicate assets being uploaded, since the server relies on a checksum `CONSTRAINT` to prevent duplicates.

server/src/commands/index.ts

@@ -9,6 +9,7 @@ import {
 import { DisableOAuthLogin, EnableOAuthLogin } from 'src/commands/oauth-login';
 import { DisablePasswordLoginCommand, EnablePasswordLoginCommand } from 'src/commands/password-login';
 import { PromptPasswordQuestions, ResetAdminPasswordCommand } from 'src/commands/reset-admin-password.command';
+import { SchemaCheck } from 'src/commands/schema-check';
 import { VersionCommand } from 'src/commands/version.command';
 
 export const commandsAndQuestions = [
@@ -28,4 +29,5 @@ export const commandsAndQuestions = [
   ChangeMediaLocationCommand,
   PromptMediaLocationQuestions,
   PromptConfirmMoveQuestions,
+  SchemaCheck,
 ];

server/src/commands/schema-check.ts

@@ -0,0 +1,60 @@
+import { Command, CommandRunner } from 'nest-commander';
+import { ErrorMessages } from 'src/constants';
+import { CliService } from 'src/services/cli.service';
+import { asHuman } from 'src/sql-tools/schema-diff';
+
+@Command({
+  name: 'schema-check',
+  description: 'Verify database migrations and check for schema drift',
+})
+export class SchemaCheck extends CommandRunner {
+  constructor(private service: CliService) {
+    super();
+  }
+
+  async run(): Promise<void> {
+    try {
+      const { migrations, drift } = await this.service.schemaReport();
+
+      if (migrations.every((item) => item.status === 'applied')) {
+        console.log('Migrations are up to date');
+      } else {
+        console.log('Migration issues detected:');
+        for (const migration of migrations) {
+          switch (migration.status) {
+            case 'deleted': {
+              console.log(`  - ${migration.name} was applied, but the file no longer exists on disk`);
+              break;
+            }
+
+            case 'missing': {
+              console.log(`  - ${migration.name} exists, but has not been applied to the database`);
+              break;
+            }
+          }
+        }
+      }
+
+      if (drift.items.length === 0) {
+        console.log('\nNo schema drift detected');
+      } else {
+        console.log(`\n${ErrorMessages.SchemaDrift}`);
+        for (const item of drift.items) {
+          console.log(`  - ${item.type}: ` + asHuman(item));
+        }
+
+        console.log(`
+
+The below SQL is automatically generated and may be helpful for resolving drift. ** Use at your own risk! **
+
+\`\`\`sql
+${drift.asSql().join('\n')}
+\`\`\`
+`);
+      }
+    } catch (error) {
+      console.error(error);
+      console.error('Unable to debug migrations');
+    }
+  }
+}

server/src/constants.ts

@@ -4,6 +4,13 @@ import { dirname, join } from 'node:path';
 import { SemVer } from 'semver';
 import { ApiTag, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum';
 
+export const ErrorMessages = {
+  InconsistentMediaLocation:
+    'Detected an inconsistent media location. For more information, see https://docs.immich.app/errors#inconsistent-media-location',
+  SchemaDrift: `Detected schema drift. For more information, see https://docs.immich.app/errors#schema-drift`,
+  TypeOrmUpgrade: 'Invalid upgrade path. For more information, see https://docs.immich.app/errors/#typeorm-upgrade',
+};
+
 export const POSTGRES_VERSION_RANGE = '>=14.0.0';
 export const VECTORCHORD_VERSION_RANGE = '>=0.3 <2';
 export const VECTORS_VERSION_RANGE = '>=0.2 <0.4';

server/src/repositories/database.repository.ts

@@ -19,7 +19,9 @@ import { GenerateSql } from 'src/decorators';
 import { DatabaseExtension, DatabaseLock, VectorIndex } from 'src/enum';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
+import 'src/schema'; // make sure all schema definitions are imported for schemaFromCode
 import { DB } from 'src/schema';
+import { schemaDiff, schemaFromCode, schemaFromDatabase } from 'src/sql-tools';
 import { ExtensionVersion, VectorExtension, VectorUpdateResult } from 'src/types';
 import { vectorIndexQuery } from 'src/utils/database';
 import { isValidInteger } from 'src/validation';
@@ -281,6 +283,27 @@ export class DatabaseRepository {
     return rows[0].db;
   }
 
+  getMigrations() {
+    return this.db.selectFrom('kysely_migrations').select(['name', 'timestamp']).orderBy('name', 'asc').execute();
+  }
+
+  async getSchemaDrift() {
+    const source = schemaFromCode({ overrides: true, namingStrategy: 'default' });
+    const target = await schemaFromDatabase(this.db, {});
+
+    const drift = schemaDiff(source, target, {
+      tables: { ignoreExtra: true },
+      constraints: { ignoreExtra: false },
+      indexes: { ignoreExtra: true },
+      triggers: { ignoreExtra: true },
+      columns: { ignoreExtra: true },
+      functions: { ignoreExtra: false },
+      parameters: { ignoreExtra: true },
+    });
+
+    return drift;
+  }
+
   async getDimensionSize(table: string, column = 'embedding'): Promise<number> {
     const { rows } = await sql<{ dimsize: number }>`
       SELECT atttypmod as dimsize

server/src/repositories/metadata.repository.ts

@@ -106,7 +106,7 @@ export class MetadataRepository {
 
   readTags(path: string): Promise<ImmichTags> {
     const args = mimeTypes.isVideo(path) ? ['-ee'] : [];
-    return this.exiftool.read(path, args).catch((error) => {
+    return this.exiftool.read(path, { readArgs: args }).catch((error) => {
       this.logger.warn(`Error reading exif data (${path}): ${error}\n${error?.stack}`);
       return {};
     }) as Promise<ImmichTags>;

server/src/schema/index.ts

@@ -168,6 +168,8 @@ export interface Migrations {
 }
 
 export interface DB {
+  kysely_migrations: { timestamp: string; name: string };
+
   activity: ActivityTable;
 
   album: AlbumTable;

server/src/schema/migrations/1744910873969-InitialMigration.ts

@@ -1,4 +1,5 @@
 import { Kysely, sql } from 'kysely';
+import { ErrorMessages } from 'src/constants';
 import { DatabaseExtension } from 'src/enum';
 import { getVectorExtension } from 'src/repositories/database.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
@@ -16,9 +17,7 @@ export async function up(db: Kysely<any>): Promise<void> {
       rows: [lastMigration],
     } = await lastMigrationSql.execute(db);
     if (lastMigration?.name !== 'AddMissingIndex1744910873956') {
-      throw new Error(
-        'Invalid upgrade path. For more information, see https://docs.immich.app/errors/#typeorm-upgrade',
-      );
+      throw new Error(ErrorMessages.TypeOrmUpgrade);
     }
     logger.log('Database has up to date TypeORM migrations, skipping initial Kysely migration');
     return;

server/src/services/cli.service.ts

@@ -1,15 +1,59 @@
 import { Injectable } from '@nestjs/common';
-import { isAbsolute } from 'node:path';
+import { isAbsolute, join } from 'node:path';
 import { SALT_ROUNDS } from 'src/constants';
 import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
 import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto';
 import { MaintenanceAction, SystemMetadataKey } from 'src/enum';
 import { BaseService } from 'src/services/base.service';
+import { schemaDiff } from 'src/sql-tools';
 import { createMaintenanceLoginUrl, generateMaintenanceSecret } from 'src/utils/maintenance';
 import { getExternalDomain } from 'src/utils/misc';
 
+export type SchemaReport = {
+  migrations: MigrationStatus[];
+  drift: ReturnType<typeof schemaDiff>;
+};
+
+type MigrationStatus = {
+  name: string;
+  status: 'applied' | 'missing' | 'deleted';
+};
+
 @Injectable()
 export class CliService extends BaseService {
+  async schemaReport(): Promise<SchemaReport> {
+    // eslint-disable-next-line unicorn/prefer-module
+    const allFiles = await this.storageRepository.readdir(join(__dirname, '../schema/migrations'));
+    const files = allFiles.filter((file) => file.endsWith('.js')).map((file) => file.slice(0, -3));
+    const rows = await this.databaseRepository.getMigrations();
+    const filesSet = new Set(files);
+    const rowsSet = new Set(rows.map((item) => item.name));
+    const combined = [...filesSet, ...rowsSet].toSorted();
+
+    const migrations: MigrationStatus[] = [];
+
+    for (const name of combined) {
+      if (filesSet.has(name) && rowsSet.has(name)) {
+        migrations.push({ name, status: 'applied' });
+        continue;
+      }
+
+      if (filesSet.has(name) && !rowsSet.has(name)) {
+        migrations.push({ name, status: 'missing' });
+        continue;
+      }
+
+      if (!filesSet.has(name) && rowsSet.has(name)) {
+        migrations.push({ name, status: 'deleted' });
+        continue;
+      }
+    }
+
+    const drift = await this.databaseRepository.getSchemaDrift();
+
+    return { migrations, drift };
+  }
+
   async listUsers(): Promise<UserAdminResponseDto[]> {
     const users = await this.userRepository.getList({ withDeleted: true });
     return users.map((user) => mapUserAdmin(user));

server/src/services/database.service.spec.ts

@@ -21,6 +21,11 @@ describe(DatabaseService.name, () => {
     extensionRange = '0.2.x';
     mocks.database.getVectorExtension.mockResolvedValue(DatabaseExtension.VectorChord);
     mocks.database.getExtensionVersionRange.mockReturnValue(extensionRange);
+    mocks.database.getSchemaDrift.mockResolvedValue({
+      items: [],
+      asSql: () => [],
+      asHuman: () => [],
+    });
 
     versionBelowRange = '0.1.0';
     minVersionInRange = '0.2.0';

server/src/services/database.service.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import semver from 'semver';
-import { EXTENSION_NAMES, VECTOR_EXTENSIONS } from 'src/constants';
+import { ErrorMessages, EXTENSION_NAMES, VECTOR_EXTENSIONS } from 'src/constants';
 import { OnEvent } from 'src/decorators';
 import { BootstrapEventPriority, DatabaseExtension, DatabaseLock, VectorIndex } from 'src/enum';
 import { BaseService } from 'src/services/base.service';
@@ -124,6 +124,17 @@ export class DatabaseService extends BaseService {
       const { database } = this.configRepository.getEnv();
       if (!database.skipMigrations) {
         await this.databaseRepository.runMigrations();
+
+        this.logger.log('Checking for schema drift');
+        const drift = await this.databaseRepository.getSchemaDrift();
+        if (drift.items.length === 0) {
+          this.logger.log('No schema drift detected');
+        } else {
+          this.logger.warn(`${ErrorMessages.SchemaDrift} or run \`immich-admin schema-check\``);
+          for (const warning of drift.asHuman()) {
+            this.logger.warn(`  - ${warning}`);
+          }
+        }
       }
       await Promise.all([
         this.databaseRepository.prewarm(VectorIndex.Clip),

server/src/services/storage.service.ts

@@ -1,5 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { join } from 'node:path';
+import { ErrorMessages } from 'src/constants';
 import { StorageCore } from 'src/cores/storage.core';
 import { OnEvent, OnJob } from 'src/decorators';
 import {
@@ -114,9 +115,7 @@ export class StorageService extends BaseService {
           this.logger.log(`Media location changed (from=${previous}, to=${current})`);
 
           if (!path.startsWith(previous)) {
-            throw new Error(
-              'Detected an inconsistent media location. For more information, see https://docs.immich.app/errors#inconsistent-media-location',
-            );
+            throw new Error(ErrorMessages.InconsistentMediaLocation);
           }
 
           this.logger.warn(

server/src/sql-tools/comparers/column.comparer.spec.ts

@@ -15,7 +15,7 @@ const testColumn: DatabaseColumn = {
 describe('compareColumns', () => {
   describe('onExtra', () => {
     it('should work', () => {
-      expect(compareColumns.onExtra(testColumn)).toEqual([
+      expect(compareColumns().onExtra(testColumn)).toEqual([
         {
           tableName: 'table1',
           columnName: 'test',
@@ -28,7 +28,7 @@ describe('compareColumns', () => {
 
   describe('onMissing', () => {
     it('should work', () => {
-      expect(compareColumns.onMissing(testColumn)).toEqual([
+      expect(compareColumns().onMissing(testColumn)).toEqual([
         {
           type: 'ColumnAdd',
           column: testColumn,
@@ -40,14 +40,14 @@ describe('compareColumns', () => {
 
   describe('onCompare', () => {
     it('should work', () => {
-      expect(compareColumns.onCompare(testColumn, testColumn)).toEqual([]);
+      expect(compareColumns().onCompare(testColumn, testColumn)).toEqual([]);
     });
 
     it('should detect a change in type', () => {
       const source: DatabaseColumn = { ...testColumn };
       const target: DatabaseColumn = { ...testColumn, type: 'text' };
       const reason = 'column type is different (character varying vs text)';
-      expect(compareColumns.onCompare(source, target)).toEqual([
+      expect(compareColumns().onCompare(source, target)).toEqual([
         {
           columnName: 'test',
           tableName: 'table1',
@@ -66,7 +66,7 @@ describe('compareColumns', () => {
       const source: DatabaseColumn = { ...testColumn, nullable: true };
       const target: DatabaseColumn = { ...testColumn, nullable: true, default: "''" };
       const reason = `default is different (null vs '')`;
-      expect(compareColumns.onCompare(source, target)).toEqual([
+      expect(compareColumns().onCompare(source, target)).toEqual([
         {
           columnName: 'test',
           tableName: 'table1',
@@ -83,7 +83,7 @@ describe('compareColumns', () => {
       const source: DatabaseColumn = { ...testColumn, comment: 'new comment' };
       const target: DatabaseColumn = { ...testColumn, comment: 'old comment' };
       const reason = 'comment is different (new comment vs old comment)';
-      expect(compareColumns.onCompare(source, target)).toEqual([
+      expect(compareColumns().onCompare(source, target)).toEqual([
         {
           columnName: 'test',
           tableName: 'table1',