diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts index 8fbaf06b56f80..f2a43545319a0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts @@ -187,7 +187,7 @@ describe('validateTypeMigrations', () => { }); expect(() => validate({ type, kibanaVersion: '3.2.3' })).toThrowErrorMatchingInlineSnapshot( - `"Type foo: Uusing modelVersions requires to specify switchToModelVersionAt"` + `"Type foo: Using modelVersions requires to specify switchToModelVersionAt"` ); }); @@ -234,6 +234,15 @@ describe('validateTypeMigrations', () => { `"Type foo: gaps between model versions aren't allowed (missing versions: 2,4,5)"` ); }); + + it('does not throw passing an empty model version map', () => { + const type = createType({ + name: 'foo', + modelVersions: {}, + }); + + expect(() => validate({ type, kibanaVersion: '3.2.3' })).not.toThrow(); + }); }); describe('modelVersions mapping additions', () => { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts index 42ee0ad6ae8e1..be3ff2e0e325b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts @@ -73,45 +73,47 @@ export function validateTypeMigrations({ const modelVersionMap = typeof type.modelVersions === 'function' ? type.modelVersions() : type.modelVersions ?? {}; - if (Object.keys(modelVersionMap).length > 0 && !type.switchToModelVersionAt) { - throw new Error( - `Type ${type.name}: Uusing modelVersions requires to specify switchToModelVersionAt` - ); - } + if (Object.keys(modelVersionMap).length > 0) { + if (!type.switchToModelVersionAt) { + throw new Error( + `Type ${type.name}: Using modelVersions requires to specify switchToModelVersionAt` + ); + } - Object.entries(modelVersionMap).forEach(([version, definition]) => { - assertValidModelVersion(version); - }); + Object.entries(modelVersionMap).forEach(([version, definition]) => { + assertValidModelVersion(version); + }); - const { min: minVersion, max: maxVersion } = Object.keys(modelVersionMap).reduce( - (minMax, rawVersion) => { - const version = Number.parseInt(rawVersion, 10); - minMax.min = Math.min(minMax.min, version); - minMax.max = Math.max(minMax.max, version); - return minMax; - }, - { min: Infinity, max: -Infinity } - ); + const { min: minVersion, max: maxVersion } = Object.keys(modelVersionMap).reduce( + (minMax, rawVersion) => { + const version = Number.parseInt(rawVersion, 10); + minMax.min = Math.min(minMax.min, version); + minMax.max = Math.max(minMax.max, version); + return minMax; + }, + { min: Infinity, max: -Infinity } + ); - if (minVersion > 1) { - throw new Error(`Type ${type.name}: model versioning must start with version 1`); - } + if (minVersion > 1) { + throw new Error(`Type ${type.name}: model versioning must start with version 1`); + } - validateAddedMappings(type.name, type.mappings, modelVersionMap); + validateAddedMappings(type.name, type.mappings, modelVersionMap); - const missingVersions = getMissingVersions( - minVersion, - maxVersion, - Object.keys(modelVersionMap).map((v) => Number.parseInt(v, 10)) - ); - if (missingVersions.length) { - throw new Error( - `Type ${ - type.name - }: gaps between model versions aren't allowed (missing versions: ${missingVersions.join( - ',' - )})` + const missingVersions = getMissingVersions( + minVersion, + maxVersion, + Object.keys(modelVersionMap).map((v) => Number.parseInt(v, 10)) ); + if (missingVersions.length) { + throw new Error( + `Type ${ + type.name + }: gaps between model versions aren't allowed (missing versions: ${missingVersions.join( + ',' + )})` + ); + } } } diff --git a/src/core/server/integration_tests/saved_objects/migrations/group4/jest.integration.config.js b/src/core/server/integration_tests/saved_objects/migrations/group4/jest.integration.config.js new file mode 100644 index 0000000000000..9b2c7be87ac8e --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group4/jest.integration.config.js @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + // TODO replace the line below with + // preset: '@kbn/test/jest_integration_node + // to do so, we must fix all integration tests first + // see https://github.com/elastic/kibana/pull/130255/ + preset: '@kbn/test/jest_integration', + rootDir: '../../../../../../..', + roots: ['/src/core/server/integration_tests/saved_objects/migrations/group4'], + // must override to match all test given there is no `integration_tests` subfolder + testMatch: ['**/*.test.{js,mjs,ts,tsx}'], +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_same_stack_version.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_same_stack_version.test.ts new file mode 100644 index 0000000000000..c20a9d1aa6f75 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_same_stack_version.test.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range, sortBy } from 'lodash'; +import { type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import { modelVersionToVirtualVersion } from '@kbn/core-saved-objects-base-server-internal'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit, startElasticsearch } from '../kibana_migrator_test_kit'; +import { delay, createType, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams } from '../fixtures/zdt_base.fixtures'; + +const logFilePath = Path.join(__dirname, 'v2_with_mv_same_stack_version.test.log'); + +const NB_DOCS_PER_TYPE = 25; + +describe('V2 algorithm - using model versions - upgrade without stack version increase', () => { + let esServer: TestElasticsearchUtils['es']; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const getTestModelVersionType = ({ beforeUpgrade }: { beforeUpgrade: boolean }) => { + const type = createType({ + name: 'test_mv', + namespaceType: 'single', + migrations: {}, + switchToModelVersionAt: '8.8.0', + modelVersions: { + 1: { + changes: [], + }, + }, + mappings: { + properties: { + field1: { type: 'text' }, + field2: { type: 'text' }, + }, + }, + }); + + if (!beforeUpgrade) { + Object.assign>(type, { + modelVersions: { + ...type.modelVersions, + 2: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + field3: { type: 'text' }, + }, + }, + { + type: 'data_backfill', + transform: (document) => { + document.attributes.field3 = 'test_mv-backfilled'; + return { document }; + }, + }, + ], + }, + }, + mappings: { + ...type.mappings, + properties: { + ...type.mappings.properties, + field3: { type: 'text' }, + }, + }, + }); + } + + return type; + }; + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams({ + migrationAlgorithm: 'v2', + kibanaVersion: '8.8.0', + }), + types: [getTestModelVersionType({ beforeUpgrade: true })], + }); + await runMigrations(); + + const mvObjs = range(NB_DOCS_PER_TYPE).map((number) => ({ + id: `mv-${String(number).padStart(3, '0')}`, + type: 'test_mv', + attributes: { + field1: `f1-${number}`, + field2: `f2-${number}`, + }, + })); + + await savedObjectsRepository.bulkCreate(mvObjs); + }; + + it('migrates the documents', async () => { + await createBaseline(); + + const modelVersionType = getTestModelVersionType({ beforeUpgrade: false }); + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams({ migrationAlgorithm: 'v2', kibanaVersion: '8.8.0' }), + logFilePath, + types: [modelVersionType], + }); + await runMigrations(); + + const indices = await client.indices.get({ index: '.kibana*' }); + expect(Object.keys(indices)).toEqual(['.kibana_8.8.0_001']); + + const index = indices['.kibana_8.8.0_001']; + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(mappings.properties).toEqual( + expect.objectContaining({ + test_mv: modelVersionType.mappings, + }) + ); + + expect(mappingMeta).toEqual({ + indexTypesMap: { + '.kibana': ['test_mv'], + }, + migrationMappingPropertyHashes: expect.any(Object), + }); + + const { saved_objects: testMvDocs } = await savedObjectsRepository.find({ + type: 'test_mv', + perPage: 1000, + }); + + expect(testMvDocs).toHaveLength(NB_DOCS_PER_TYPE); + + const testMvData = sortBy(testMvDocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + version: object.typeMigrationVersion, + })); + + expect(testMvData).toEqual( + range(NB_DOCS_PER_TYPE).map((number) => ({ + id: `mv-${String(number).padStart(3, '0')}`, + type: 'test_mv', + attributes: { + field1: `f1-${number}`, + field2: `f2-${number}`, + field3: 'test_mv-backfilled', + }, + version: modelVersionToVirtualVersion(2), + })) + ); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntries( + [ + 'INIT -> WAIT_FOR_YELLOW_SOURCE', + 'CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES', + 'Migration completed', + ], + { + ordered: true, + } + ); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_stack_version_bump.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_stack_version_bump.test.ts new file mode 100644 index 0000000000000..e259e0f12d178 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_stack_version_bump.test.ts @@ -0,0 +1,278 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range, sortBy } from 'lodash'; +import { type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import { modelVersionToVirtualVersion } from '@kbn/core-saved-objects-base-server-internal'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit, startElasticsearch } from '../kibana_migrator_test_kit'; +import { delay, createType, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams } from '../fixtures/zdt_base.fixtures'; + +const logFilePath = Path.join(__dirname, 'v2_with_mv_stack_version_bump.test.log'); + +const NB_DOCS_PER_TYPE = 100; + +describe('V2 algorithm - using model versions - stack version bump scenario', () => { + let esServer: TestElasticsearchUtils['es']; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const getTestSwitchType = ({ beforeUpgrade }: { beforeUpgrade: boolean }) => { + const type = createType({ + name: 'test_switch', + namespaceType: 'single', + migrations: { + '8.7.0': (doc) => { + return doc; + }, + }, + modelVersions: {}, + mappings: { + properties: { + field1: { type: 'text' }, + field2: { type: 'text' }, + }, + }, + }); + + if (!beforeUpgrade) { + Object.assign>(type, { + switchToModelVersionAt: '8.8.0', + modelVersions: { + 1: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + field3: { type: 'text' }, + }, + }, + { + type: 'data_backfill', + transform: (document) => { + document.attributes.field3 = 'test_switch-backfilled'; + return { document }; + }, + }, + ], + }, + }, + mappings: { + ...type.mappings, + properties: { + ...type.mappings.properties, + field3: { type: 'text' }, + }, + }, + }); + } + + return type; + }; + + const getTestModelVersionType = ({ beforeUpgrade }: { beforeUpgrade: boolean }) => { + const type = createType({ + name: 'test_mv', + namespaceType: 'single', + migrations: {}, + switchToModelVersionAt: '8.8.0', + modelVersions: { + 1: { + changes: [], + }, + }, + mappings: { + properties: { + field1: { type: 'text' }, + field2: { type: 'text' }, + }, + }, + }); + + if (!beforeUpgrade) { + Object.assign>(type, { + modelVersions: { + ...type.modelVersions, + 2: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + field3: { type: 'text' }, + }, + }, + { + type: 'data_backfill', + transform: (document) => { + document.attributes.field3 = 'test_mv-backfilled'; + return { document }; + }, + }, + ], + }, + }, + mappings: { + ...type.mappings, + properties: { + ...type.mappings.properties, + field3: { type: 'text' }, + }, + }, + }); + } + + return type; + }; + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams({ + migrationAlgorithm: 'v2', + kibanaVersion: '8.8.0', + }), + types: [ + getTestSwitchType({ beforeUpgrade: true }), + getTestModelVersionType({ beforeUpgrade: true }), + ], + }); + await runMigrations(); + + const switchObjs = range(NB_DOCS_PER_TYPE).map((number) => ({ + id: `switch-${String(number).padStart(3, '0')}`, + type: 'test_switch', + attributes: { + field1: `f1-${number}`, + field2: `f2-${number}`, + }, + })); + + await savedObjectsRepository.bulkCreate(switchObjs); + + const mvObjs = range(NB_DOCS_PER_TYPE).map((number) => ({ + id: `mv-${String(number).padStart(3, '0')}`, + type: 'test_mv', + attributes: { + field1: `f1-${number}`, + field2: `f2-${number}`, + }, + })); + + await savedObjectsRepository.bulkCreate(mvObjs); + }; + + it('migrates the documents', async () => { + await createBaseline(); + + const switchType = getTestSwitchType({ beforeUpgrade: false }); + const modelVersionType = getTestModelVersionType({ beforeUpgrade: false }); + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams({ migrationAlgorithm: 'v2' }), + logFilePath, + types: [switchType, modelVersionType], + }); + await runMigrations(); + + const indices = await client.indices.get({ index: '.kibana*' }); + expect(Object.keys(indices)).toEqual(['.kibana_8.8.0_001']); + + const index = indices['.kibana_8.8.0_001']; + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(mappings.properties).toEqual( + expect.objectContaining({ + test_switch: switchType.mappings, + test_mv: modelVersionType.mappings, + }) + ); + + expect(mappingMeta).toEqual({ + indexTypesMap: { + '.kibana': ['test_mv', 'test_switch'], + }, + migrationMappingPropertyHashes: expect.any(Object), + }); + + const { saved_objects: testSwitchDocs } = await savedObjectsRepository.find({ + type: 'test_switch', + perPage: 1000, + }); + const { saved_objects: testMvDocs } = await savedObjectsRepository.find({ + type: 'test_mv', + perPage: 1000, + }); + + expect(testSwitchDocs).toHaveLength(NB_DOCS_PER_TYPE); + expect(testMvDocs).toHaveLength(NB_DOCS_PER_TYPE); + + const testSwitchDocsData = sortBy(testSwitchDocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + version: object.typeMigrationVersion, + })); + + expect(testSwitchDocsData).toEqual( + range(NB_DOCS_PER_TYPE).map((number) => ({ + id: `switch-${String(number).padStart(3, '0')}`, + type: 'test_switch', + attributes: { + field1: `f1-${number}`, + field2: `f2-${number}`, + field3: 'test_switch-backfilled', + }, + version: modelVersionToVirtualVersion(1), + })) + ); + + const testMvData = sortBy(testMvDocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + version: object.typeMigrationVersion, + })); + + expect(testMvData).toEqual( + range(NB_DOCS_PER_TYPE).map((number) => ({ + id: `mv-${String(number).padStart(3, '0')}`, + type: 'test_mv', + attributes: { + field1: `f1-${number}`, + field2: `f2-${number}`, + field3: 'test_mv-backfilled', + }, + version: modelVersionToVirtualVersion(2), + })) + ); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntries( + [ + 'INIT -> WAIT_FOR_YELLOW_SOURCE', + 'CLEANUP_UNKNOWN_AND_EXCLUDED_WAIT_FOR_TASK -> PREPARE_COMPATIBLE_MIGRATION', + 'Migration completed', + ], + { + ordered: true, + } + ); + }); +});