V2 migration algorithm: add tests for model versions

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts

@@ -187,7 +187,7 @@ describe('validateTypeMigrations', () => {
       });
 
       expect(() => validate({ type, kibanaVersion: '3.2.3' })).toThrowErrorMatchingInlineSnapshot(
-        `"Type foo: Uusing modelVersions requires to specify switchToModelVersionAt"`
+        `"Type foo: Using modelVersions requires to specify switchToModelVersionAt"`
       );
     });
 
@@ -234,6 +234,15 @@ describe('validateTypeMigrations', () => {
         `"Type foo: gaps between model versions aren't allowed (missing versions: 2,4,5)"`
       );
     });
+
+    it('does not throw passing an empty model version map', () => {
+      const type = createType({
+        name: 'foo',
+        modelVersions: {},
+      });
+
+      expect(() => validate({ type, kibanaVersion: '3.2.3' })).not.toThrow();
+    });
   });
 
   describe('modelVersions mapping additions', () => {

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts

@@ -73,45 +73,47 @@ export function validateTypeMigrations({
     const modelVersionMap =
       typeof type.modelVersions === 'function' ? type.modelVersions() : type.modelVersions ?? {};
 
-    if (Object.keys(modelVersionMap).length > 0 && !type.switchToModelVersionAt) {
-      throw new Error(
-        `Type ${type.name}: Uusing modelVersions requires to specify switchToModelVersionAt`
-      );
-    }
+    if (Object.keys(modelVersionMap).length > 0) {
+      if (!type.switchToModelVersionAt) {
+        throw new Error(
+          `Type ${type.name}: Using modelVersions requires to specify switchToModelVersionAt`
+        );
+      }
 
-    Object.entries(modelVersionMap).forEach(([version, definition]) => {
-      assertValidModelVersion(version);
-    });
+      Object.entries(modelVersionMap).forEach(([version, definition]) => {
+        assertValidModelVersion(version);
+      });
 
-    const { min: minVersion, max: maxVersion } = Object.keys(modelVersionMap).reduce(
-      (minMax, rawVersion) => {
-        const version = Number.parseInt(rawVersion, 10);
-        minMax.min = Math.min(minMax.min, version);
-        minMax.max = Math.max(minMax.max, version);
-        return minMax;
-      },
-      { min: Infinity, max: -Infinity }
-    );
+      const { min: minVersion, max: maxVersion } = Object.keys(modelVersionMap).reduce(
+        (minMax, rawVersion) => {
+          const version = Number.parseInt(rawVersion, 10);
+          minMax.min = Math.min(minMax.min, version);
+          minMax.max = Math.max(minMax.max, version);
+          return minMax;
+        },
+        { min: Infinity, max: -Infinity }
+      );
 
-    if (minVersion > 1) {
-      throw new Error(`Type ${type.name}: model versioning must start with version 1`);
-    }
+      if (minVersion > 1) {
+        throw new Error(`Type ${type.name}: model versioning must start with version 1`);
+      }
 
-    validateAddedMappings(type.name, type.mappings, modelVersionMap);
+      validateAddedMappings(type.name, type.mappings, modelVersionMap);
 
-    const missingVersions = getMissingVersions(
-      minVersion,
-      maxVersion,
-      Object.keys(modelVersionMap).map((v) => Number.parseInt(v, 10))
-    );
-    if (missingVersions.length) {
-      throw new Error(
-        `Type ${
-          type.name
-        }: gaps between model versions aren't allowed (missing versions: ${missingVersions.join(
-          ','
-        )})`
+      const missingVersions = getMissingVersions(
+        minVersion,
+        maxVersion,
+        Object.keys(modelVersionMap).map((v) => Number.parseInt(v, 10))
       );
+      if (missingVersions.length) {
+        throw new Error(
+          `Type ${
+            type.name
+          }: gaps between model versions aren't allowed (missing versions: ${missingVersions.join(
+            ','
+          )})`
+        );
+      }
     }
   }
 

src/core/server/integration_tests/saved_objects/migrations/group4/jest.integration.config.js

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  // TODO replace the line below with
+  // preset: '@kbn/test/jest_integration_node
+  // to do so, we must fix all integration tests first
+  // see https://github.com/elastic/kibana/pull/130255/
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../../../../../../..',
+  roots: ['<rootDir>/src/core/server/integration_tests/saved_objects/migrations/group4'],
+  // must override to match all test given there is no `integration_tests` subfolder
+  testMatch: ['**/*.test.{js,mjs,ts,tsx}'],
+};

src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_same_stack_version.test.ts

@@ -0,0 +1,184 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import fs from 'fs/promises';
+import { range, sortBy } from 'lodash';
+import { type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
+import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server';
+import { modelVersionToVirtualVersion } from '@kbn/core-saved-objects-base-server-internal';
+import '../jest_matchers';
+import { getKibanaMigratorTestKit, startElasticsearch } from '../kibana_migrator_test_kit';
+import { delay, createType, parseLogFile } from '../test_utils';
+import { getBaseMigratorParams } from '../fixtures/zdt_base.fixtures';
+
+const logFilePath = Path.join(__dirname, 'v2_with_mv_same_stack_version.test.log');
+
+const NB_DOCS_PER_TYPE = 25;
+
+describe('V2 algorithm - using model versions - upgrade without stack version increase', () => {
+  let esServer: TestElasticsearchUtils['es'];
+
+  beforeAll(async () => {
+    await fs.unlink(logFilePath).catch(() => {});
+    esServer = await startElasticsearch();
+  });
+
+  afterAll(async () => {
+    await esServer?.stop();
+    await delay(10);
+  });
+
+  const getTestModelVersionType = ({ beforeUpgrade }: { beforeUpgrade: boolean }) => {
+    const type = createType({
+      name: 'test_mv',
+      namespaceType: 'single',
+      migrations: {},
+      switchToModelVersionAt: '8.8.0',
+      modelVersions: {
+        1: {
+          changes: [],
+        },
+      },
+      mappings: {
+        properties: {
+          field1: { type: 'text' },
+          field2: { type: 'text' },
+        },
+      },
+    });
+
+    if (!beforeUpgrade) {
+      Object.assign<typeof type, Partial<typeof type>>(type, {
+        modelVersions: {
+          ...type.modelVersions,
+          2: {
+            changes: [
+              {
+                type: 'mappings_addition',
+                addedMappings: {
+                  field3: { type: 'text' },
+                },
+              },
+              {
+                type: 'data_backfill',
+                transform: (document) => {
+                  document.attributes.field3 = 'test_mv-backfilled';
+                  return { document };
+                },
+              },
+            ],
+          },
+        },
+        mappings: {
+          ...type.mappings,
+          properties: {
+            ...type.mappings.properties,
+            field3: { type: 'text' },
+          },
+        },
+      });
+    }
+
+    return type;
+  };
+
+  const createBaseline = async () => {
+    const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({
+      ...getBaseMigratorParams({
+        migrationAlgorithm: 'v2',
+        kibanaVersion: '8.8.0',
+      }),
+      types: [getTestModelVersionType({ beforeUpgrade: true })],
+    });
+    await runMigrations();
+
+    const mvObjs = range(NB_DOCS_PER_TYPE).map<SavedObjectsBulkCreateObject>((number) => ({
+      id: `mv-${String(number).padStart(3, '0')}`,
+      type: 'test_mv',
+      attributes: {
+        field1: `f1-${number}`,
+        field2: `f2-${number}`,
+      },
+    }));
+
+    await savedObjectsRepository.bulkCreate(mvObjs);
+  };
+
+  it('migrates the documents', async () => {
+    await createBaseline();
+
+    const modelVersionType = getTestModelVersionType({ beforeUpgrade: false });
+
+    const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({
+      ...getBaseMigratorParams({ migrationAlgorithm: 'v2', kibanaVersion: '8.8.0' }),
+      logFilePath,
+      types: [modelVersionType],
+    });
+    await runMigrations();
+
+    const indices = await client.indices.get({ index: '.kibana*' });
+    expect(Object.keys(indices)).toEqual(['.kibana_8.8.0_001']);
+
+    const index = indices['.kibana_8.8.0_001'];
+    const mappings = index.mappings ?? {};
+    const mappingMeta = mappings._meta ?? {};
+
+    expect(mappings.properties).toEqual(
+      expect.objectContaining({
+        test_mv: modelVersionType.mappings,
+      })
+    );
+
+    expect(mappingMeta).toEqual({
+      indexTypesMap: {
+        '.kibana': ['test_mv'],
+      },
+      migrationMappingPropertyHashes: expect.any(Object),
+    });
+
+    const { saved_objects: testMvDocs } = await savedObjectsRepository.find({
+      type: 'test_mv',
+      perPage: 1000,
+    });
+
+    expect(testMvDocs).toHaveLength(NB_DOCS_PER_TYPE);
+
+    const testMvData = sortBy(testMvDocs, 'id').map((object) => ({
+      id: object.id,
+      type: object.type,
+      attributes: object.attributes,
+      version: object.typeMigrationVersion,
+    }));
+
+    expect(testMvData).toEqual(
+      range(NB_DOCS_PER_TYPE).map((number) => ({
+        id: `mv-${String(number).padStart(3, '0')}`,
+        type: 'test_mv',
+        attributes: {
+          field1: `f1-${number}`,
+          field2: `f2-${number}`,
+          field3: 'test_mv-backfilled',
+        },
+        version: modelVersionToVirtualVersion(2),
+      }))
+    );
+
+    const records = await parseLogFile(logFilePath);
+    expect(records).toContainLogEntries(
+      [
+        'INIT -> WAIT_FOR_YELLOW_SOURCE',
+        'CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES',
+        'Migration completed',
+      ],
+      {
+        ordered: true,
+      }
+    );
+  });
+});