diff --git a/docs/setup/upgrade/resolving-migration-failures.asciidoc b/docs/setup/upgrade/resolving-migration-failures.asciidoc index 596674237eb7a..81ddac9a94900 100644 --- a/docs/setup/upgrade/resolving-migration-failures.asciidoc +++ b/docs/setup/upgrade/resolving-migration-failures.asciidoc @@ -120,10 +120,7 @@ If you fail to remedy this, your upgrade to 8.0+ will fail with a message like: [source,sh] -------------------------------------------- -Unable to complete saved object migrations for the [.kibana] index: Migration failed because some documents were found which use unknown saved object types: -- "firstDocId" (type "someType") -- "secondtDocId" (type "someType") -- "thirdDocId" (type "someOtherType") +Unable to complete saved object migrations for the [.kibana] index: Migration failed because some documents were found which use unknown saved object types: someType,someOtherType To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration. -------------------------------------------- diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/extract_errors.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/extract_errors.test.ts index 5eb3947388d4f..01ade23e05db1 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/extract_errors.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/extract_errors.test.ts @@ -29,10 +29,7 @@ describe('extractUnknownDocFailureReason', () => { }, ]) ).toMatchInlineSnapshot(` - "Migration failed because some documents were found which use unknown saved object types: - - \\"unknownType:12\\" (type: \\"unknownType\\") - - \\"anotherUnknownType:42\\" (type: \\"anotherUnknownType\\") - + "Migration failed because some documents were found which use unknown saved object types: unknownType,anotherUnknownType To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration. Please refer to some-url.co for more information." `); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/extract_errors.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/extract_errors.ts index edfdc98244985..0ed10e9e8f38a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/extract_errors.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/extract_errors.ts @@ -52,10 +52,12 @@ export function extractUnknownDocFailureReason( resolveMigrationFailuresUrl: string, unknownDocs: DocumentIdAndType[] ): string { + const typesSet = new Set(unknownDocs.map(({ type }) => type)); return ( - `Migration failed because some documents were found which use unknown saved object types:\n` + - unknownDocs.map((doc) => `- "${doc.id}" (type: "${doc.type}")\n`).join('') + - `\nTo proceed with the migration you can configure Kibana to discard unknown saved objects for this migration.\n` + + `Migration failed because some documents were found which use unknown saved object types: ${Array.from( + typesSet.values() + )}\n` + + `To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration.\n` + `Please refer to ${resolveMigrationFailuresUrl} for more information.` ); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index e619daa73cdae..14b171ac097da 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -559,13 +559,21 @@ export const model = (currentState: State, resW: ResponseType): deleteByQueryTaskId: res.right.taskId, }; } else { + const reason = extractUnknownDocFailureReason( + stateP.migrationDocLinks.resolveMigrationFailures, + res.left.unknownDocs + ); return { ...stateP, controlState: 'FATAL', - reason: extractUnknownDocFailureReason( - stateP.migrationDocLinks.resolveMigrationFailures, - res.left.unknownDocs - ), + reason, + logs: [ + ...logs, + { + level: 'error', + message: reason, + }, + ], }; } } else if (stateP.controlState === 'CLEANUP_UNKNOWN_AND_EXCLUDED_WAIT_FOR_TASK') { @@ -700,13 +708,22 @@ export const model = (currentState: State, resW: ResponseType): if (isTypeof(res.right, 'unknown_docs_found')) { if (!stateP.discardUnknownObjects) { + const reason = extractUnknownDocFailureReason( + stateP.migrationDocLinks.resolveMigrationFailures, + res.right.unknownDocs + ); + return { ...stateP, controlState: 'FATAL', - reason: extractUnknownDocFailureReason( - stateP.migrationDocLinks.resolveMigrationFailures, - res.right.unknownDocs - ), + reason, + logs: [ + ...logs, + { + level: 'error', + message: reason, + }, + ], }; } @@ -879,6 +896,13 @@ export const model = (currentState: State, resW: ResponseType): corruptDocumentIds: [], transformErrors: [], progress: createInitialProgress(), + logs: [ + ...logs, + { + level: 'info', + message: `REINDEX_SOURCE_TO_TEMP_OPEN_PIT PitId:${res.right.pitId}`, + }, + ], }; } else { throwBadResponse(stateP, res); diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip index 46cc61cbe7b5f..d079624d95988 100644 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip and b/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_with_unknown_so.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_with_unknown_so.zip deleted file mode 100644 index 3312515024b91..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_with_unknown_so.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7_13_corrupt_and_transform_failures_docs.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7_13_corrupt_and_transform_failures_docs.zip deleted file mode 100644 index b808bdc4f59ce..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7_13_corrupt_and_transform_failures_docs.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_document_migration_failure.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_document_migration_failure.zip deleted file mode 100644 index 9dc4de75c5d98..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_document_migration_failure.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_migrated_with_corrupt_outdated_docs.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_migrated_with_corrupt_outdated_docs.zip deleted file mode 100644 index 726df7782cda3..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_migrated_with_corrupt_outdated_docs.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_migrated_with_outdated_docs.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_migrated_with_outdated_docs.zip deleted file mode 100644 index 1fa9f061e15ff..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/8.0.0_migrated_with_outdated_docs.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_transform_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_transform_failures.test.ts deleted file mode 100644 index c186b4c0613ef..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_transform_failures.test.ts +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import Fs from 'fs'; -import Util from 'util'; -import { Env } from '@kbn/config'; -import { REPO_ROOT } from '@kbn/repo-info'; -import { getEnvOptions } from '@kbn/config-mocks'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Root } from '@kbn/core-root-server-internal'; -import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'; -import { getMigrationDocLink } from '../test_utils'; -import { - createRootWithCorePlugins, - TestElasticsearchUtils, - createTestServers as createkbnServerTestServers, -} from '@kbn/core-test-helpers-kbn-server'; -import { - MAIN_SAVED_OBJECT_INDEX, - TASK_MANAGER_SAVED_OBJECT_INDEX, - ANALYTICS_SAVED_OBJECT_INDEX, -} from '@kbn/core-saved-objects-server'; - -const migrationDocLink = getMigrationDocLink().resolveMigrationFailures; -const logFilePath = Path.join(__dirname, '7_13_corrupt_transform_failures.log'); - -const asyncUnlink = Util.promisify(Fs.unlink); - -async function removeLogFile() { - // ignore errors if it doesn't exist - await asyncUnlink(logFilePath).catch(() => void 0); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - - beforeAll(async () => { - await removeLogFile(); - }); - - afterEach(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - describe('when `migrations.discardCorruptObjects` does not match current kibana version', () => { - it('fails to migrate when corrupt objects and transform errors are encountered', async () => { - const { startES } = createTestServers(); - root = createRoot(); - esServer = await startES(); - await rootPrebootAndSetup(root); - - try { - await root.start(); - } catch (err) { - const errorMessage = err.message; - const errorLines = errorMessage.split('\n'); - const errorMessageWithoutStack = errorLines - .filter((line: string) => !line.includes(' at ')) - .join('\n'); - - expect(errorMessageWithoutStack).toMatchInlineSnapshot(` - "Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 7 corrupt saved object documents were found: P2SQfHkBs3dBRGh--No5, QGSZfHkBs3dBRGh-ANoD, QWSZfHkBs3dBRGh-hNob, QmSZfHkBs3dBRGh-w9qH, one, two, Q2SZfHkBs3dBRGh-9dp2 - 7 transformation errors were encountered: - - space:default: Error: Migration function for version 7.14.0 threw an error - Caused by: - TypeError: Cannot set properties of undefined (setting 'bar') - - space:first: Error: Migration function for version 7.14.0 threw an error - Caused by: - TypeError: Cannot set properties of undefined (setting 'bar') - - space:forth: Error: Migration function for version 7.14.0 threw an error - Caused by: - TypeError: Cannot set properties of undefined (setting 'bar') - - space:second: Error: Migration function for version 7.14.0 threw an error - Caused by: - TypeError: Cannot set properties of undefined (setting 'bar') - - space:fifth: Error: Migration function for version 7.14.0 threw an error - Caused by: - TypeError: Cannot set properties of undefined (setting 'bar') - - space:third: Error: Migration function for version 7.14.0 threw an error - Caused by: - TypeError: Cannot set properties of undefined (setting 'bar') - - space:sixth: Error: Migration function for version 7.14.0 threw an error - Caused by: - TypeError: Cannot set properties of undefined (setting 'bar') - - To allow migrations to proceed, please delete or fix these documents. - Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration. - Please refer to ${migrationDocLink} for more information." - `); - return; - } - // Fail test if above expression doesn't throw anything. - expect('to throw').toBe('but did not'); - }); - }); - - describe('when `migrations.discardCorruptObjects` matches current kibana version', () => { - it('proceeds with the migration, ignoring corrupt objects and transform errors', async () => { - const { startES } = createTestServers(); - const currentVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; - root = createRoot(currentVersion); - esServer = await startES(); - await rootPrebootAndSetup(root); - - await expect(root.start()).resolves.not.toThrowError(); - // TODO check that the destination indices contain data, but NOT the conflicting objects - - const esClient: ElasticsearchClient = esServer.es.getClient(); - const docs = await esClient.search({ - index: [ - MAIN_SAVED_OBJECT_INDEX, - TASK_MANAGER_SAVED_OBJECT_INDEX, - ANALYTICS_SAVED_OBJECT_INDEX, - ], - _source: false, - fields: ['_id'], - size: 50, - }); - - // 34 saved objects (11 tasks + 23 misc) + 14 corrupt (discarded) = 48 total in the old indices - expect((docs.hits.total as SearchTotalHits).value).toEqual(34); - expect(docs.hits.hits.map(({ _id }) => _id).sort()).toEqual([ - 'config:7.13.0', - 'index-pattern:logs-*', - 'index-pattern:metrics-*', - 'task:Actions-actions_telemetry', - 'task:Actions-cleanup_failed_action_executions', - 'task:Alerting-alerting_health_check', - 'task:Alerting-alerting_telemetry', - 'task:Alerts-alerts_invalidate_api_keys', - 'task:Lens-lens_telemetry', - 'task:apm-telemetry-task', - 'task:data_enhanced_search_sessions_monitor', - 'task:endpoint:user-artifact-packager:1.0.0', - 'task:security:endpoint-diagnostics:1.0.0', - 'task:session_cleanup', - 'ui-metric:console:DELETE_delete', - 'ui-metric:console:GET_get', - 'ui-metric:console:GET_search', - 'ui-metric:console:POST_delete_by_query', - 'ui-metric:console:POST_index', - 'ui-metric:console:PUT_indices.put_mapping', - 'usage-counters:uiCounter:21052021:click:global_search_bar:user_navigated_to_application', - 'usage-counters:uiCounter:21052021:click:global_search_bar:user_navigated_to_application_unknown', - 'usage-counters:uiCounter:21052021:count:console:DELETE_delete', - 'usage-counters:uiCounter:21052021:count:console:GET_cat.aliases', - 'usage-counters:uiCounter:21052021:count:console:GET_cat.indices', - 'usage-counters:uiCounter:21052021:count:console:GET_get', - 'usage-counters:uiCounter:21052021:count:console:GET_search', - 'usage-counters:uiCounter:21052021:count:console:POST_delete_by_query', - 'usage-counters:uiCounter:21052021:count:console:POST_index', - 'usage-counters:uiCounter:21052021:count:console:PUT_indices.put_mapping', - 'usage-counters:uiCounter:21052021:count:global_search_bar:search_focus', - 'usage-counters:uiCounter:21052021:count:global_search_bar:search_request', - 'usage-counters:uiCounter:21052021:count:global_search_bar:shortcut_used', - 'usage-counters:uiCounter:21052021:loaded:console:opened_app', - ]); - }); - }); -}); - -function createTestServers() { - return createkbnServerTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - // example of original 'foo' SO with corrupt id: - // _id: one - // { - // foo: { - // name: 'one', - // }, - // type: 'foo', - // references: [], - // migrationVersion: { - // foo: '7.13.0', - // }, - // "coreMigrationVersion": "7.13.0", - // "updated_at": "2021-05-16T18:16:45.450Z" - // }, - - // SO that will fail transformation: - // { - // type: 'space', - // space: {}, - // }, - // - // - dataArchive: Path.join( - __dirname, - '..', - 'archives', - '7_13_corrupt_and_transform_failures_docs.zip' - ), - }, - }, - }); -} - -function createRoot(discardCorruptObjects?: string) { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - batchSize: 5, - discardCorruptObjects, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss: true, - } - ); -} - -async function rootPrebootAndSetup(root: Root) { - await root.preboot(); - const coreSetup = await root.setup(); - - coreSetup.savedObjects.registerType({ - name: 'foo', - hidden: false, - mappings: { - properties: {}, - }, - namespaceType: 'agnostic', - migrations: { - '7.14.0': (doc) => doc, - }, - }); - - // registering the `space` type with a throwing migration fn to avoid the migration failing for unknown types - coreSetup.savedObjects.registerType({ - name: 'space', - hidden: false, - mappings: { - properties: {}, - }, - namespaceType: 'single', - migrations: { - '7.14.0': (doc) => { - doc.attributes.foo.bar = 12; - return doc; - }, - }, - }); -} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_unknown_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_unknown_types.test.ts deleted file mode 100644 index 25698f9b63f68..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_unknown_types.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import fs from 'fs/promises'; -import type { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types'; -import { Env } from '@kbn/config'; -import { REPO_ROOT } from '@kbn/repo-info'; -import { getEnvOptions } from '@kbn/config-mocks'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Root } from '@kbn/core-root-server-internal'; -import { - createRootWithCorePlugins, - createTestServers, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; - -const logFilePath = Path.join(__dirname, '7_13_unknown_types.log'); - -async function removeLogFile() { - // ignore errors if it doesn't exist - await fs.unlink(logFilePath).catch(() => void 0); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - let startES: () => Promise; - - beforeAll(async () => { - await removeLogFile(); - }); - - beforeEach(() => { - ({ startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - // dataset contains 2 type of unknown docs - // `foo` documents - // `space` documents (to mimic a migration with disabled plugins) - dataArchive: Path.join(__dirname, '..', 'archives', '7.13.0_with_unknown_so.zip'), - }, - }, - })); - }); - - afterEach(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - describe('when `discardUnknownObjects` does not match current kibana version', () => { - it('fails the migration if unknown types are found in the source index', async () => { - // Start kibana with foo and space types disabled - root = createRoot('7.13.0'); - esServer = await startES(); - await root.preboot(); - await root.setup(); - - try { - await root.start(); - expect('should have thrown').toEqual('but it did not'); - } catch (err) { - const errorMessage = err.message; - - expect( - errorMessage.startsWith( - 'Unable to complete saved object migrations for the [.kibana] index: Migration failed because some documents ' + - 'were found which use unknown saved object types:' - ) - ).toBeTruthy(); - - const unknownDocs = [ - { type: 'space', id: 'space:default' }, - { type: 'space', id: 'space:first' }, - { type: 'space', id: 'space:second' }, - { type: 'space', id: 'space:third' }, - { type: 'space', id: 'space:forth' }, - { type: 'space', id: 'space:fifth' }, - { type: 'space', id: 'space:sixth' }, - { type: 'foo', id: 'P2SQfHkBs3dBRGh--No5' }, - { type: 'foo', id: 'QGSZfHkBs3dBRGh-ANoD' }, - { type: 'foo', id: 'QWSZfHkBs3dBRGh-hNob' }, - ]; - - unknownDocs.forEach(({ id, type }) => { - expect(errorMessage).toEqual(expect.stringContaining(`- "${id}" (type: "${type}")`)); - }); - - const client = esServer.es.getClient(); - const { body: response } = await client.indices.getSettings( - { index: '.kibana_7.13.0_001' }, - { meta: true } - ); - const settings = response['.kibana_7.13.0_001'].settings as IndicesIndexSettings; - expect(settings.index).not.toBeUndefined(); - expect(settings.index!.blocks?.write).not.toEqual('true'); - } - }); - }); - - describe('when `discardUnknownObjects` matches current kibana version', () => { - const currentVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; - - it('discards the documents with unknown types and finishes the migration successfully', async () => { - // Start kibana with foo and space types disabled - root = createRoot(currentVersion); - esServer = await startES(); - await root.preboot(); - await root.setup(); - - // the migration process should finish successfully - await expect(root.start()).resolves.not.toThrowError(); - - const esClient: ElasticsearchClient = esServer.es.getClient(); - const body = await esClient.count({ q: 'type:foo|space' }); - expect(body.count).toEqual(0); - }); - }); -}); - -function createRoot(discardUnknownObjects?: string) { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - batchSize: 5, - discardUnknownObjects, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss: true, - } - ); -} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/__snapshots__/v2_migration.test.ts.snap b/src/core/server/integration_tests/saved_objects/migrations/group1/__snapshots__/v2_migration.test.ts.snap new file mode 100644 index 0000000000000..1afb9e03cb6a2 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/__snapshots__/v2_migration.test.ts.snap @@ -0,0 +1,504 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`v2 migration to a newer stack version with transform errors collects corrupt saved object documents across batches 1`] = ` +"Error: Cannot convert 'complex' objects with values that are multiple of 100 41e28595-9710-4583-aeb3-41c5899aa2ce +Error: Cannot convert 'complex' objects with values that are multiple of 100 a1f15bce-d9a2-4a4d-a280-c05f2eda99f1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 06d3374e-193b-43d1-9c1a-0c8829cb0c51 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b26f9630-7de4-45ec-a27c-c041b97fbe57 +Error: Cannot convert 'complex' objects with values that are multiple of 100 21c18cf5-825a-4847-a2e2-5c670033a560 +Error: Cannot convert 'complex' objects with values that are multiple of 100 cd75cc1c-e915-4573-b353-f3f2ee01cf47 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8bee00b6-36dd-46e7-b055-f366cc37df48 +Error: Cannot convert 'complex' objects with values that are multiple of 100 2e675eed-4f4e-4151-99fd-83a32b5f8e79 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c3de3e3c-b38c-4264-a6f6-e59eff793980 +Error: Cannot convert 'complex' objects with values that are multiple of 100 da9b73a8-b9f9-4073-a9bb-c853243c2dd8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 6d3de4cc-2c2c-4b28-b8dc-4d2f607ae4f2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 31fe1208-4702-495f-95c9-d3f948576791 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e1e75224-dfb4-465e-9688-093ef4a663e6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d30d55f5-5513-436b-b0c1-2c2fc9dffcc3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8a5cd239-c02c-4370-abb5-f084d7f2d418 +Error: Cannot convert 'complex' objects with values that are multiple of 100 5eeded25-9d6c-45bd-9d7d-33657c67e73d +Error: Cannot convert 'complex' objects with values that are multiple of 100 90849a74-a3cf-4587-8ba8-d0c9480015c6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 94f659fb-73d9-49a7-8a82-50e67c259ddb +Error: Cannot convert 'complex' objects with values that are multiple of 100 6548dc97-0f55-462b-a4d2-dc0bceb7fbdd +Error: Cannot convert 'complex' objects with values that are multiple of 100 e6b6316d-c839-4b75-9abb-18da0d0d3f31 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1dadcae8-a0f7-49eb-bbf0-81fcc81061d7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1d1391ac-7564-4a97-8b65-f3c50bf56615 +Error: Cannot convert 'complex' objects with values that are multiple of 100 cbedd055-0619-46f4-ae2c-68fff60eef45 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9907c207-e47f-4a1e-91a8-173915eefd8b +Error: Cannot convert 'complex' objects with values that are multiple of 100 cd7fdcdc-3d77-4b43-9bb9-41b715ba41a4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c144775d-b0d6-42c1-ad0d-7a4350687c85 +Error: Cannot convert 'complex' objects with values that are multiple of 100 306a4982-da6b-463f-b157-9dc33579cd28 +Error: Cannot convert 'complex' objects with values that are multiple of 100 df6c1595-68ef-41f6-b649-30952d586195 +Error: Cannot convert 'complex' objects with values that are multiple of 100 5d3e825d-3db7-4a04-be91-b1b92e00386a +Error: Cannot convert 'complex' objects with values that are multiple of 100 d257437f-529b-482d-a981-2f26bd79388f +Error: Cannot convert 'complex' objects with values that are multiple of 100 dac6c6c6-d329-4dd5-acd1-b2585917d736 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e415176e-b3cd-41d1-9ba5-06c702a13cb3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 64163b20-da0b-482b-90d2-347e73e8c068 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f195b8b9-c1a4-4724-b861-2a533a58e6b4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d53b617d-94b7-4701-99a5-a194c6b34e0c +Error: Cannot convert 'complex' objects with values that are multiple of 100 e350a554-2086-4d36-a7ea-3898befc1792 +Error: Cannot convert 'complex' objects with values that are multiple of 100 2de91b3b-f69f-4e64-8d79-b432aedd46ec +Error: Cannot convert 'complex' objects with values that are multiple of 100 9cfc58ad-94fa-4b6c-b776-3b93942fdabc +Error: Cannot convert 'complex' objects with values that are multiple of 100 efcbc446-c869-4837-861c-20c787a50872 +Error: Cannot convert 'complex' objects with values that are multiple of 100 70e0b310-e8b6-4fcf-8a1d-fb9e28b0d1ed +Error: Cannot convert 'complex' objects with values that are multiple of 100 df92c6c1-7405-4fe2-849b-e1aec71f44d0 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9b3839aa-5cad-4b30-ae0e-18340b566535 +Error: Cannot convert 'complex' objects with values that are multiple of 100 7cbad964-9f31-4a5c-87d7-734bdf08b42a +Error: Cannot convert 'complex' objects with values that are multiple of 100 5ffc4d92-df45-4151-b6e9-71e18dc67070 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fb3d87d6-ba7b-41d2-ac79-e1e6f402a19c +Error: Cannot convert 'complex' objects with values that are multiple of 100 2e14fca1-ad13-410c-a6dd-969a3793aa78 +Error: Cannot convert 'complex' objects with values that are multiple of 100 691efa7f-9177-4971-996a-c6f6c1218046 +Error: Cannot convert 'complex' objects with values that are multiple of 100 34c47c79-369e-456c-b1af-286e247929cd +Error: Cannot convert 'complex' objects with values that are multiple of 100 add9a30d-8050-4370-bd44-237b5cce5d93 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f0423b5a-56a3-40a3-97ec-f30ad0187e63 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8b25b058-5931-4973-aa86-8dbbd91e4bd1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 097d4e9f-afa6-4db0-a35b-183051116ed5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d7b3edcb-ae7d-4df8-ae38-e627aaeb46be +Error: Cannot convert 'complex' objects with values that are multiple of 100 344a9dee-5de8-4e50-af66-374c6b7675a7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b2c6c79e-6d9f-42a0-b4ab-91fb321b021f +Error: Cannot convert 'complex' objects with values that are multiple of 100 09452338-c116-4d86-b59a-be071b0cd689 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f0ee199a-e921-484f-b551-f4aec202ea27 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3d12c404-4dfc-40d9-9795-44f6292ce5ae +Error: Cannot convert 'complex' objects with values that are multiple of 100 31ca239d-a3cf-43c0-8ef3-a1d37c7e311f +Error: Cannot convert 'complex' objects with values that are multiple of 100 6f45a809-e866-4b7d-812e-21e9180133fe +Error: Cannot convert 'complex' objects with values that are multiple of 100 9ff098a5-0738-4f8c-b595-0dd615e5ce2b +Error: Cannot convert 'complex' objects with values that are multiple of 100 611156d0-d452-4582-8ef1-4b9f68ce40de +Error: Cannot convert 'complex' objects with values that are multiple of 100 6e55ba50-6717-4250-8c5e-0d34e608cf55 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f204be38-103a-40cc-b599-9f9a6dc2b676 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8e54c8e7-80f1-4a27-b96b-f36b5279b154 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d33b9b61-4bb2-4935-a71f-7194918f86ab +Error: Cannot convert 'complex' objects with values that are multiple of 100 46042cbf-2da6-4ff3-8dec-b92f1e53910f +Error: Cannot convert 'complex' objects with values that are multiple of 100 004c3a5d-caa0-474e-a631-a9bee6fc432a +Error: Cannot convert 'complex' objects with values that are multiple of 100 a744e441-7653-4486-b39d-bef59af32c46 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d9e2ebda-2370-4660-aa84-311a9a56ccc3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4e57d594-45ba-4b8e-a2fa-814a0fc2db0b +Error: Cannot convert 'complex' objects with values that are multiple of 100 228fe220-b046-48c3-b0dd-bfe3cc15b3af +Error: Cannot convert 'complex' objects with values that are multiple of 100 66afe76d-ab04-4381-aaf6-5a07b7043300 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f1f48caa-5e33-49b3-8c8e-f9e47cc46553 +Error: Cannot convert 'complex' objects with values that are multiple of 100 6d1de2f4-6995-43e3-86e9-34da77dfb160 +Error: Cannot convert 'complex' objects with values that are multiple of 100 56911ce1-c027-4ef3-81fa-c8e99ee3c6bd +Error: Cannot convert 'complex' objects with values that are multiple of 100 a230e01b-356f-48c8-9aa9-3360c15bce9b +Error: Cannot convert 'complex' objects with values that are multiple of 100 2f0e1193-0f35-48f7-8b28-4bf142984a4e +Error: Cannot convert 'complex' objects with values that are multiple of 100 1168bfdc-0b3d-43df-8fd4-b2f111509710 +Error: Cannot convert 'complex' objects with values that are multiple of 100 752d393a-61e6-4d3d-b3cb-b25c976f5dc9 +Error: Cannot convert 'complex' objects with values that are multiple of 100 2afef4d0-5fb2-4dbc-8edc-cec127ba7566 +Error: Cannot convert 'complex' objects with values that are multiple of 100 bea60590-a57c-495b-be43-d29720cd1995 +Error: Cannot convert 'complex' objects with values that are multiple of 100 511c29d5-c8c6-4cfe-b9c5-f7e3272cfddd +Error: Cannot convert 'complex' objects with values that are multiple of 100 c4e2e677-0165-49d1-9ee9-bb102a4446e8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 eb0e6312-35df-4f4b-8ede-9f4ff94f323d +Error: Cannot convert 'complex' objects with values that are multiple of 100 a86f12f2-a3cd-48e9-b5d8-37a67994149b +Error: Cannot convert 'complex' objects with values that are multiple of 100 26c96126-4450-4096-afe7-126b0a6da7ee +Error: Cannot convert 'complex' objects with values that are multiple of 100 d2f4a4e9-34bd-4ac8-bccf-96ccd7cfec38 +Error: Cannot convert 'complex' objects with values that are multiple of 100 864a2090-6d3a-459d-9c53-fbf30433e5d2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e15dcebb-733f-44d1-97dd-90a176770d4b +Error: Cannot convert 'complex' objects with values that are multiple of 100 cf3dc9c8-a377-41fa-84db-71c7a1669655 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9764f40e-ade5-4623-9b77-f2e6286458c3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 57bec873-0fa6-40f0-bb6d-73479aa12f12 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b49cf0b4-072a-4e01-8099-e95783073d5c +Error: Cannot convert 'complex' objects with values that are multiple of 100 39067bbd-ed2b-4788-be3b-4df0041cc009 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0916d31e-5b51-4acd-9735-75bdad4aaec8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 13660507-d9e4-4dd8-a464-2142a27b4793 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d06b9296-5bcf-41df-a0f3-b556f289f06a +Error: Cannot convert 'complex' objects with values that are multiple of 100 e7cf5995-4bb0-4d3a-8165-66be5d9854fc +Error: Cannot convert 'complex' objects with values that are multiple of 100 b8d7dc9f-0bb2-43e7-872e-a0098080470b +Error: Cannot convert 'complex' objects with values that are multiple of 100 2adf568c-1055-4013-bf2c-faa7e433700f +Error: Cannot convert 'complex' objects with values that are multiple of 100 764d554e-413e-4cd3-aeb5-366bf11ca769 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b4677231-92c0-4f0e-8dad-3e2a6d09fb44 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a2f0f8ea-0ccd-4bf3-bde6-8572018f571a +Error: Cannot convert 'complex' objects with values that are multiple of 100 44e15623-8b56-4907-96dd-64fbc78c9262 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1f14b977-c54b-4eef-a48f-573055169eb1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 5fa91020-c284-4a04-8aa3-b198dde8cf14 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f12ddc96-95ff-45a2-8872-847828ce41d0 +Error: Cannot convert 'complex' objects with values that are multiple of 100 983d0fd8-90c7-4ade-ac03-4532b9ccf4d3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8d44f665-d001-4e45-8019-809c32b4dfd7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 00c94e57-87fb-4004-812a-2f30e6465b28 +Error: Cannot convert 'complex' objects with values that are multiple of 100 59bc74cf-26df-4d25-bc7e-a720eb946667 +Error: Cannot convert 'complex' objects with values that are multiple of 100 efe7b66d-feae-4938-aced-1f0818c91e2c +Error: Cannot convert 'complex' objects with values that are multiple of 100 10e68507-5e58-4a43-a16d-3830394341aa +Error: Cannot convert 'complex' objects with values that are multiple of 100 dbe48539-bb49-41b5-93b9-3d05158fa20b +Error: Cannot convert 'complex' objects with values that are multiple of 100 ca0e995a-4189-42ab-a082-f342705470b5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 966811b4-c0ee-4244-98cc-66b2f31855cd +Error: Cannot convert 'complex' objects with values that are multiple of 100 fa6918f8-b132-4720-8828-ded1777cb7d3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 7ca309ac-fc0a-4f5c-90fe-b57bee2e1cc2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d3d9abce-39b2-42a9-a0b4-bb1b0d7179cb +Error: Cannot convert 'complex' objects with values that are multiple of 100 2a540aa3-f0e1-4d20-a623-c05c595e7cd3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 986eba02-ca59-4f0d-92ff-62fc277aa87c +Error: Cannot convert 'complex' objects with values that are multiple of 100 11142ec5-ff29-4577-8936-56f69c2f4dbd +Error: Cannot convert 'complex' objects with values that are multiple of 100 15cffaae-7c1b-40b5-92fc-9e5e0b42174a +Error: Cannot convert 'complex' objects with values that are multiple of 100 8d2c15ec-5caa-44df-be9a-cd8c3a0039fd +Error: Cannot convert 'complex' objects with values that are multiple of 100 82025688-7cc1-4a1b-ba26-fba279c0a559 +Error: Cannot convert 'complex' objects with values that are multiple of 100 548d80cd-6a87-48aa-8fcb-05af29dde400 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3ebe2353-d7e7-44e5-b5c9-92cbc5ccb239 +Error: Cannot convert 'complex' objects with values that are multiple of 100 189e1010-10e9-4a25-9eef-14760fe80cfa +Error: Cannot convert 'complex' objects with values that are multiple of 100 9b88d281-e9d7-494a-85f6-fa8c0a3f236b +Error: Cannot convert 'complex' objects with values that are multiple of 100 19e7a4de-012f-4d69-be2f-bd09f9c26350 +Error: Cannot convert 'complex' objects with values that are multiple of 100 734c74d5-e70a-4776-87e1-51bdf5577f3c +Error: Cannot convert 'complex' objects with values that are multiple of 100 164ca6f1-2cbe-4440-8652-8515df19075f +Error: Cannot convert 'complex' objects with values that are multiple of 100 cc288178-bfc9-428c-902f-5fbefb4f1958 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ae2b9311-4084-438d-824b-f83ad45b35eb +Error: Cannot convert 'complex' objects with values that are multiple of 100 8e8bfb88-f4c1-4929-8b45-4b9a8b912ae8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8254f416-603b-4e71-8a10-3df84be64e7b +Error: Cannot convert 'complex' objects with values that are multiple of 100 15facbbc-6934-4c7d-8278-68a177bc1d40 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fadf9362-aea6-47e1-94da-39da2f59dc6d +Error: Cannot convert 'complex' objects with values that are multiple of 100 a193a83c-fd64-4197-aae8-91f5fc23844e +Error: Cannot convert 'complex' objects with values that are multiple of 100 6ebf1dbe-de76-4539-8af8-7b12e47f5ce2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 27f5606b-234a-4dc0-9e40-75cd74fe32c6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1b06e644-bec5-4723-b040-490ea6110fb7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c81ec835-94a9-4b84-9c55-98bfd27fc3bc +Error: Cannot convert 'complex' objects with values that are multiple of 100 a5e1812e-dbdb-4400-80b6-e7b70e058887 +Error: Cannot convert 'complex' objects with values that are multiple of 100 99a76670-22fd-4365-bfa8-15ee48761260 +Error: Cannot convert 'complex' objects with values that are multiple of 100 7eeb5f71-ac2f-4fd8-99cd-8b987bcbf70c +Error: Cannot convert 'complex' objects with values that are multiple of 100 aefdb49d-511c-4fa2-aab5-56a89428f260 +Error: Cannot convert 'complex' objects with values that are multiple of 100 cb0e6562-5e52-4ab2-adec-97be213f3dae +Error: Cannot convert 'complex' objects with values that are multiple of 100 c292e57b-59d5-47b2-a52f-753fddb7f444 +Error: Cannot convert 'complex' objects with values that are multiple of 100 658fc86d-0406-433c-a29c-01de07173c9f +Error: Cannot convert 'complex' objects with values that are multiple of 100 4df986b0-01a7-4ca0-84b7-e872168da048 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9097e835-d1ae-4c88-9e29-63ed0160b127 +Error: Cannot convert 'complex' objects with values that are multiple of 100 44a33056-b110-4114-9073-a50498e2564b +Error: Cannot convert 'complex' objects with values that are multiple of 100 b001f6bd-5435-4da0-9632-c269768c243c +Error: Cannot convert 'complex' objects with values that are multiple of 100 7bf727c5-58d7-4860-a496-e05c76d5fca8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b7fbaa4a-b30b-4fe0-b2de-2baddaff4dd5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1233fb9c-bc84-4997-b980-724163cd9c99 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0cc4ea6c-9882-4c38-99a3-f0e61136cf97 +Error: Cannot convert 'complex' objects with values that are multiple of 100 87623762-5557-4d50-874b-e783224d2d68 +Error: Cannot convert 'complex' objects with values that are multiple of 100 5c92c79e-6c4a-4482-8f88-7e5663c194f2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a1191ea3-3e5f-46de-b764-6a81182dab19 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b9159be4-5cb2-4f25-a089-85fcd9b03b33 +Error: Cannot convert 'complex' objects with values that are multiple of 100 eb14b9ab-004d-4e6e-b254-02e8af9a351e +Error: Cannot convert 'complex' objects with values that are multiple of 100 bff8783a-a2d6-4424-8119-a16cf72bb52a +Error: Cannot convert 'complex' objects with values that are multiple of 100 28509c5a-462b-483f-883a-be26b4f462f4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 32cca2bf-0c3a-4d40-baea-44833532aeb9 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ebb778ab-992a-4438-919d-4e8d627c2058 +Error: Cannot convert 'complex' objects with values that are multiple of 100 81bc0a2a-ea4d-4944-b2c9-12e4b3bbee61 +Error: Cannot convert 'complex' objects with values that are multiple of 100 7a257c3e-90af-483c-9ed3-737bb77560eb +Error: Cannot convert 'complex' objects with values that are multiple of 100 f2ae2465-949d-4cc9-bdda-89ff964adc0f +Error: Cannot convert 'complex' objects with values that are multiple of 100 f9394e2f-6652-4161-9bd7-fbcfc218b942 +Error: Cannot convert 'complex' objects with values that are multiple of 100 61b99930-5dc3-4e7f-9a61-6e22fcfa5507 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a54b457e-0164-4773-84d9-2bc49bf335a2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b01dcfa3-b6fc-42e3-8682-a5a3349d22d5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ce4b8596-3ae9-4f2a-9a9e-ceb96fbf0171 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fea29815-35c9-494f-9912-4718d4c65d93 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c4e210ed-5d5e-4329-867b-565de7ec14c2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e49e182c-44c9-442b-92ef-fc83433f7e50 +Error: Cannot convert 'complex' objects with values that are multiple of 100 7faac5b2-c36b-4359-bb24-367b77815592 +Error: Cannot convert 'complex' objects with values that are multiple of 100 27fe1231-fed2-41a1-859d-503134f5cc83 +Error: Cannot convert 'complex' objects with values that are multiple of 100 762d235a-1b99-46ed-ad7f-b9e28eea5822 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b5b9992a-bcaa-4b9a-8754-29a60068ac6a +Error: Cannot convert 'complex' objects with values that are multiple of 100 f48be8be-0f44-4f2f-920d-d75ca5e970ae +Error: Cannot convert 'complex' objects with values that are multiple of 100 1369a5fb-8312-4bba-ab79-ea71a9fcd036 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d11c8331-a724-4d85-90b8-aa084d8b258d +Error: Cannot convert 'complex' objects with values that are multiple of 100 ea9019d5-26d0-4477-8c9f-172dfc107f25 +Error: Cannot convert 'complex' objects with values that are multiple of 100 725ee684-9dbf-4ff8-8b8d-b8e2172500b8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 04001900-1655-4f6e-86b8-bbd88504d50f +Error: Cannot convert 'complex' objects with values that are multiple of 100 37bebf97-9ce2-4589-a804-ff69527bafba +Error: Cannot convert 'complex' objects with values that are multiple of 100 cff8daf0-c177-49a6-961b-2cc32217af7b +Error: Cannot convert 'complex' objects with values that are multiple of 100 bbb15e49-0126-4e2d-9518-1c5c2b227263 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b7664d87-c87d-4e06-94d2-c51d904c92fd +Error: Cannot convert 'complex' objects with values that are multiple of 100 534a357a-1f04-4243-aa89-c70879656c66 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0393aa29-9cee-4168-b802-6519c004f49a +Error: Cannot convert 'complex' objects with values that are multiple of 100 1975467d-7832-47e5-a5be-cb24ea2cd405 +Error: Cannot convert 'complex' objects with values that are multiple of 100 27b07229-9c16-43d3-985e-af00336c1970 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e382b6bd-c9a7-4f00-bf5d-b1a829c1486a +Error: Cannot convert 'complex' objects with values that are multiple of 100 8ce42577-57b7-4c3c-9d12-bf09899ec75a +Error: Cannot convert 'complex' objects with values that are multiple of 100 71ee1840-416e-4a24-901b-ac009814738f +Error: Cannot convert 'complex' objects with values that are multiple of 100 286fc315-4643-48ef-b0bb-f18ad22888a0 +Error: Cannot convert 'complex' objects with values that are multiple of 100 68bba5c0-23d5-41ef-b6f5-c094c152427d +Error: Cannot convert 'complex' objects with values that are multiple of 100 7272613f-ee42-4761-89e5-ed4eb10203e2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8d5aec0b-0dd1-4a82-a77c-d35d4db1daf7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 24532830-c3ec-4d6b-8d98-59c38e6255c6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a203d1f2-71bc-4eb1-b479-f650fff57e91 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e3abbd23-7f6b-4c19-a8c9-1faa4ec84cef +Error: Cannot convert 'complex' objects with values that are multiple of 100 229cd51f-526c-44b0-bb96-6aab38f58a4c +Error: Cannot convert 'complex' objects with values that are multiple of 100 e96389f1-03ff-471f-b507-bd666c5ea5cd +Error: Cannot convert 'complex' objects with values that are multiple of 100 b9a71b4f-37e6-4a73-975f-5b0c0fba5327 +Error: Cannot convert 'complex' objects with values that are multiple of 100 231b2ab5-c0bf-4d62-8d30-b83fb55230f1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d1786358-b15d-4e74-8cb1-131d2344bd37 +Error: Cannot convert 'complex' objects with values that are multiple of 100 af4cecfa-6005-4564-812e-61228f66dd62 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ee1bedce-e0db-4a2f-be29-6301e6d136d1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c5f0a020-e588-4fb1-9b1c-57dca2644408 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3969b1f7-3374-402b-a1cc-13ae0b85227a +Error: Cannot convert 'complex' objects with values that are multiple of 100 1db8da28-cb63-4a24-9010-3010d2ab15d2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fee08b75-21dc-4b18-9ef5-8a327798dc60 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f728f4c9-bdee-42aa-a0bb-d9299d344fa5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 231ffa66-55f8-4534-a1d1-738cd4462dfd +Error: Cannot convert 'complex' objects with values that are multiple of 100 2bdef19b-3373-48d9-a41c-87b98afd9651 +Error: Cannot convert 'complex' objects with values that are multiple of 100 2d90f0ae-4e99-4460-b8a6-c396841ff476 +Error: Cannot convert 'complex' objects with values that are multiple of 100 afd1c4ce-110d-4327-9910-04b1f9b83bc4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b01d65b1-38ec-4fc0-bbd1-416f4b941b17 +Error: Cannot convert 'complex' objects with values that are multiple of 100 341c61e2-e62c-4ff4-b761-2b98ac8188fc +Error: Cannot convert 'complex' objects with values that are multiple of 100 6996cad7-588b-4176-bd81-85d4d75fa1c5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 6e7bd135-84a5-4ba2-805b-5bcb271e2a95 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4564b434-4925-4745-b9a5-0f77c4f191e3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b16e3d8d-5c80-489c-9fd4-f88745d259b6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 865d6ee7-dca7-47db-afc8-94b64a86b48d +Error: Cannot convert 'complex' objects with values that are multiple of 100 608e9069-0b56-45ea-8f0d-1a953d8cdc48 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9e29e291-753d-46e7-af8a-5a75839cd1f6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e0d13e93-7834-42d7-9381-ff2c80c787c3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fe800046-61e8-495c-923e-85ed71aed4fd +Error: Cannot convert 'complex' objects with values that are multiple of 100 2db7c305-6ae1-462d-8c78-e862a54e50d2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a3f5c29b-707e-4ae2-b955-651580661246 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3750d10d-d277-496a-8f9c-49592827a9fe +Error: Cannot convert 'complex' objects with values that are multiple of 100 3cb8f69c-19f8-48ad-9f3e-111f92fa9695 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b52b53c7-4dc9-4c26-8027-af970a4664f6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f23836c4-ce59-4a01-bf58-d4ce2a6b4e8d +Error: Cannot convert 'complex' objects with values that are multiple of 100 fe76654b-fb1f-41cc-96c9-ee566f9754eb +Error: Cannot convert 'complex' objects with values that are multiple of 100 8ce8d356-20f0-4e87-b417-a53be0c55764 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f8d692f9-1403-4aa3-b13b-03438cdc01fb +Error: Cannot convert 'complex' objects with values that are multiple of 100 018b0a78-63db-4898-b7e5-6c8d03f7dbaa +Error: Cannot convert 'complex' objects with values that are multiple of 100 0fd3dbb5-6929-473c-aeae-302fbbf47efd +Error: Cannot convert 'complex' objects with values that are multiple of 100 a1c3c2f6-0693-4925-ad24-f33e73edb24b +Error: Cannot convert 'complex' objects with values that are multiple of 100 e563fdae-26e3-4124-a38e-d2281182729f +Error: Cannot convert 'complex' objects with values that are multiple of 100 d32195f5-3312-4baf-82ee-48f322cf2484 +Error: Cannot convert 'complex' objects with values that are multiple of 100 564d8d76-6500-4bf4-8986-6f29f7f939c3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 98cd0dc0-4e58-46e4-8b01-b732b7f4e6e7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e70a7175-6a62-4e45-bb99-8d203f32bf5e +Error: Cannot convert 'complex' objects with values that are multiple of 100 71826df0-ce85-4797-afae-dd6b85210dbe +Error: Cannot convert 'complex' objects with values that are multiple of 100 e98f260f-8d59-474f-8899-f98401b4e9c6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 bec0c13f-3430-495d-9864-3983adfc733e +Error: Cannot convert 'complex' objects with values that are multiple of 100 12ea60a7-de0d-4062-be1a-c66987a55552 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b75c805b-a614-4c82-a2c3-5bf7f756e617 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4bf2ca7b-f1ad-4f0c-8ad9-858d841ca726 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fdeb53ce-76f2-45c2-8d97-56c019ddc4c2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 99d50e6a-a9fb-4699-a7a3-e4771393faf1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fe83ee65-dfb1-4ca0-877c-82437619b7b6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fda80a13-3f7f-4a0a-853a-a5a0b69aeee3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d59fa1ed-ab65-4428-b2c1-dc7737ba58aa +Error: Cannot convert 'complex' objects with values that are multiple of 100 7e26c6f6-09df-49f3-87bd-8e810abb1475 +Error: Cannot convert 'complex' objects with values that are multiple of 100 54dbb234-2cb4-4c91-bb2f-36f0d1dcbc71 +Error: Cannot convert 'complex' objects with values that are multiple of 100 91997e38-b01d-4197-945e-0a453cfc79f5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ed6c8871-f227-4476-b78d-5f8c30c5167f +Error: Cannot convert 'complex' objects with values that are multiple of 100 a06ab564-8762-4438-884a-425fb643ab4e +Error: Cannot convert 'complex' objects with values that are multiple of 100 cf9dcebb-ad19-4fe8-8f51-67533d20ab32 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f5847615-a940-4fa5-885a-4085aef251c8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9e12ba9f-4d99-490e-8778-deff73ef71e7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 35321be9-5341-434f-adaf-f729ee87e683 +Error: Cannot convert 'complex' objects with values that are multiple of 100 aa0365d7-d1c5-4788-933e-b27650cbfda4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f54adb98-cf22-4511-b78c-1b79344b5fe2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 69060ee5-c9d1-4b78-98b0-c562b3793eae +Error: Cannot convert 'complex' objects with values that are multiple of 100 5f52db20-e3a0-49c6-9808-293dff5d2e0c +Error: Cannot convert 'complex' objects with values that are multiple of 100 755dde68-33dc-479d-a93a-406769187f23 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4b50ec74-48a7-4e00-9f0d-1cda8bcdc4b9 +Error: Cannot convert 'complex' objects with values that are multiple of 100 48ec2e7d-cdd2-45d3-a166-01bd72f04b1b +Error: Cannot convert 'complex' objects with values that are multiple of 100 a7ec7e57-4289-4c25-899a-fe77a7c20ce6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d836e77c-94ae-456d-b03b-77efba0f30dd +Error: Cannot convert 'complex' objects with values that are multiple of 100 5fbed3af-b212-4c4e-a228-21fdf20aeb1d +Error: Cannot convert 'complex' objects with values that are multiple of 100 6a974c64-b794-438e-a866-0f8853ae80f7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 38bc381b-d4de-4916-8962-ceb88130635e +Error: Cannot convert 'complex' objects with values that are multiple of 100 c160e280-9308-4712-adee-689884ebeaf6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 52a193cd-f566-4ff2-878d-d70e1139df2e +Error: Cannot convert 'complex' objects with values that are multiple of 100 2f7c0d32-e3a1-4ff0-a3e4-3b45b7389b1f +Error: Cannot convert 'complex' objects with values that are multiple of 100 04f80d54-03f4-4e79-8c46-e5072d36007e +Error: Cannot convert 'complex' objects with values that are multiple of 100 05c5bbfb-8858-4439-a695-c854c4e28569 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ab3bcc0d-8f88-4b5e-acb1-10cb89be118d +Error: Cannot convert 'complex' objects with values that are multiple of 100 588eebea-7807-424f-b1ae-192600f028f3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1a613324-9ebe-4370-a1be-5b8540b5e7e6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a79e69c6-56ea-46ac-8d19-29616c9f43ff +Error: Cannot convert 'complex' objects with values that are multiple of 100 45a8b609-1690-43dc-8fe9-73ed45a996d4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a0c66af2-c52d-4c74-8cf5-d6bd7aae57e1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3a547379-f469-4920-b1b3-0975bf3532e6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 eb989e38-26ad-4173-bc37-23246af599e6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 cc6e8592-1257-4791-810c-68fc59a67450 +Error: Cannot convert 'complex' objects with values that are multiple of 100 5adf294e-af67-447f-8ae8-1bf236cd61da +Error: Cannot convert 'complex' objects with values that are multiple of 100 c71e7626-1497-4727-a9c6-1fa438b57655 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ed9eaec9-6984-42d2-b617-d417a5d8fd8a +Error: Cannot convert 'complex' objects with values that are multiple of 100 b053ad16-b8d1-47e3-a4e6-7d4e0c41013a +Error: Cannot convert 'complex' objects with values that are multiple of 100 368f4560-1acb-4011-a8c0-31f5b7ddb78e +Error: Cannot convert 'complex' objects with values that are multiple of 100 94651677-44bb-46c3-a041-66e9af680854 +Error: Cannot convert 'complex' objects with values that are multiple of 100 212907b6-8457-41e1-82d7-1f17ff8418f8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 dd79b637-25db-4b9e-97a5-cd42efed3fca +Error: Cannot convert 'complex' objects with values that are multiple of 100 e55a86ff-1abd-4cc0-b3f1-186c94a21ca2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 44f22c9a-f7d7-49f8-b105-ec4793913617 +Error: Cannot convert 'complex' objects with values that are multiple of 100 2d7b720d-0418-41b3-8992-d3a06aeb0d87 +Error: Cannot convert 'complex' objects with values that are multiple of 100 229a2508-2949-44e3-abef-b95a5314141a +Error: Cannot convert 'complex' objects with values that are multiple of 100 374ce0e5-9abb-4d1b-905e-3ad020b4e6ad +Error: Cannot convert 'complex' objects with values that are multiple of 100 9280b201-2c4d-4f1a-a2b3-563cef526ec6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 239fea1a-5ce9-45c0-8c6f-7795b87f5005 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e0b95739-008c-4303-b483-59778837edf7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 23c1442d-5fe8-408e-8c91-6fe0b52c3880 +Error: Cannot convert 'complex' objects with values that are multiple of 100 57ec1c1d-604d-45c1-9505-eeee7c5272d3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9c34c4bb-783d-4527-bb40-62dc77ca9e9e +Error: Cannot convert 'complex' objects with values that are multiple of 100 ca0f3c78-c217-4089-9713-b7960c47e709 +Error: Cannot convert 'complex' objects with values that are multiple of 100 5e301cd0-18fc-413f-b89e-89b39c9b7d61 +Error: Cannot convert 'complex' objects with values that are multiple of 100 54a514cb-6a16-4ed5-bb7b-f1dab5e391ce +Error: Cannot convert 'complex' objects with values that are multiple of 100 c48229f1-b060-44f2-8e84-0094f3c52568 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0adef79b-0a77-4e0c-9d45-ea7ac9efc99e +Error: Cannot convert 'complex' objects with values that are multiple of 100 41ae9d1a-19cc-43e2-bf43-74119c256171 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0c8c5586-9310-448c-b769-57cd8b434f1b +Error: Cannot convert 'complex' objects with values that are multiple of 100 5abeddb1-b16e-4cae-9743-bd8b3967d68d +Error: Cannot convert 'complex' objects with values that are multiple of 100 9a16a8f0-5acb-4be4-b01b-ba70ee703d79 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1ec57983-aa25-43fb-b619-75c05db355cb +Error: Cannot convert 'complex' objects with values that are multiple of 100 8219be58-e54e-4779-a46a-46af2e26dd65 +Error: Cannot convert 'complex' objects with values that are multiple of 100 36d49a42-405f-4489-98ee-15988fb02655 +Error: Cannot convert 'complex' objects with values that are multiple of 100 084945de-d37e-4363-81de-c5d80c6330d5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 7804d5ed-5c7d-4836-9a1f-ad04160cc721 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f00fa376-1e1e-40c1-96b9-32240c1f86e4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3fcb887f-e1a6-4353-8473-b1a9e036458e +Error: Cannot convert 'complex' objects with values that are multiple of 100 d5faec5c-9145-41f7-a0f8-d0e9f712d67c +Error: Cannot convert 'complex' objects with values that are multiple of 100 3cd05855-6007-47c8-a1ed-eeb0a0a3f610 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d0398a7a-c268-4d74-8d78-d86fd8fce735 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ac77927f-7557-4393-bd23-87ff74f91f3d +Error: Cannot convert 'complex' objects with values that are multiple of 100 a070e880-ef24-43fb-a9f0-1497bca8e13f +Error: Cannot convert 'complex' objects with values that are multiple of 100 841a6aa7-6d31-43ab-a40c-9d002f052f40 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c5dc76ec-d278-48be-8bf7-68a7fb8ed76c +Error: Cannot convert 'complex' objects with values that are multiple of 100 aa9519cd-f645-4434-b72a-874c0f527303 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c3865152-f2cb-4dad-8846-1a19346af136 +Error: Cannot convert 'complex' objects with values that are multiple of 100 48dd71da-8dc4-4c71-a4bf-5c9472b09bae +Error: Cannot convert 'complex' objects with values that are multiple of 100 7e91868d-55f3-4b56-965e-2a32d66a770a +Error: Cannot convert 'complex' objects with values that are multiple of 100 f4bf79b4-9370-443b-b17d-867982e36d87 +Error: Cannot convert 'complex' objects with values that are multiple of 100 01e803c0-5489-4fbb-9fc6-4dced641ad9e +Error: Cannot convert 'complex' objects with values that are multiple of 100 bf0eb3cb-db70-410e-8007-7a010e226622 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8663a0e7-0597-43c6-85b0-0c107f06c929 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fd613c6a-0c56-4cfd-80b9-fc437a630e7e +Error: Cannot convert 'complex' objects with values that are multiple of 100 615d25b9-df43-412f-bcd1-3110063dcc4f +Error: Cannot convert 'complex' objects with values that are multiple of 100 c931910e-3fd8-4c33-812f-fd49cea925e4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 87565d23-6d9a-4465-b67b-667221c4df3b +Error: Cannot convert 'complex' objects with values that are multiple of 100 8a7652b0-c400-4b4e-af8b-a8da5e0dbd0b +Error: Cannot convert 'complex' objects with values that are multiple of 100 dc059883-7c96-4edd-9a7c-30b18a57b4b5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d0b6a1ca-0e5f-4648-b419-b6b9ec00e9b2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8817a5ba-a093-490b-a7f1-2e859bc0b345 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4e08f021-3d6e-4786-b7a1-8b6d2d7dfa35 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0a0b9977-b713-4357-a90d-c58bf18ea94e +Error: Cannot convert 'complex' objects with values that are multiple of 100 7c5f6a0c-5d52-4391-8f45-ba337aa7ce0f +Error: Cannot convert 'complex' objects with values that are multiple of 100 6742e115-ede8-4bcd-adf9-bf7b154be106 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c68bbfa4-c190-43eb-bde7-d8567e65fea8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1ce1caee-6441-4372-9d95-406a025043f6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 073ed719-6211-45cf-9c9b-341591230cb8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 fa3a3fa7-5266-439e-82cf-3283a86fcd8a +Error: Cannot convert 'complex' objects with values that are multiple of 100 2acea925-7794-4f02-975d-800426fdc744 +Error: Cannot convert 'complex' objects with values that are multiple of 100 dd850e29-182a-4e75-ac56-f62f6389d8bc +Error: Cannot convert 'complex' objects with values that are multiple of 100 58289e3d-11b8-428a-ab3b-f34bc67820eb +Error: Cannot convert 'complex' objects with values that are multiple of 100 3ca1ac43-75db-4d32-ae67-1ca19baa7ebd +Error: Cannot convert 'complex' objects with values that are multiple of 100 589f4c6e-b82a-4c42-a1dc-232cc7ae30b5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 965526db-d773-4f01-bfa1-f3208298e6ca +Error: Cannot convert 'complex' objects with values that are multiple of 100 9daa6a46-3033-4547-8b05-2da1b21710de +Error: Cannot convert 'complex' objects with values that are multiple of 100 fa03b3b2-feb5-41e9-a7fd-fa14b7a2ae74 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8250e2f4-bcdf-46cc-8963-9d1f64a03bc0 +Error: Cannot convert 'complex' objects with values that are multiple of 100 388dba12-07a7-4d3d-88e5-fb5159499697 +Error: Cannot convert 'complex' objects with values that are multiple of 100 72e51f1d-6962-4670-94a9-dff3e8010067 +Error: Cannot convert 'complex' objects with values that are multiple of 100 01f68652-6627-451f-8293-966d19d1a654 +Error: Cannot convert 'complex' objects with values that are multiple of 100 18fd85dc-2843-41ba-b193-df23ce0cefc2 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8f6161e8-f89c-47e8-9147-e4d90ab675ba +Error: Cannot convert 'complex' objects with values that are multiple of 100 eb2132af-a15a-452d-83df-b6974c48b592 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1ee5c1bf-820f-4012-b0f3-5cd013974aeb +Error: Cannot convert 'complex' objects with values that are multiple of 100 37f6c099-3ae5-48de-8a32-002311f36560 +Error: Cannot convert 'complex' objects with values that are multiple of 100 dfd5562d-a445-4b00-8a03-42c3b29a7f6d +Error: Cannot convert 'complex' objects with values that are multiple of 100 f39d88ab-5d99-4c89-84ce-6ddda83fe359 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ab1e91c2-b37c-4c98-a0c1-50424b19ede3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4172f9b5-54f7-4146-af18-f062e51c6fe8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 67c046e1-87a6-44c6-8224-9d7a017e851a +Error: Cannot convert 'complex' objects with values that are multiple of 100 4ff6d019-a524-446c-9c7a-5eed5695411f +Error: Cannot convert 'complex' objects with values that are multiple of 100 88a75507-1fcb-45aa-bb3d-6e9e3f8cefe5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 18d1f9c2-1ca9-47d5-8539-01c1523be687 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0efec7af-4f1a-4c71-ba74-ad25a9446586 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c30d943d-fafe-42c7-9886-976ee2ada664 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ce6e9245-f2a4-48cc-b65e-0aa16bfa6c65 +Error: Cannot convert 'complex' objects with values that are multiple of 100 91bf39b0-e9d9-43c7-bdf2-67c2702acede +Error: Cannot convert 'complex' objects with values that are multiple of 100 3034c39f-ae49-47c7-aa85-55e9b6aa8bc3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 53185dec-a065-4b83-a04e-4c385c931f3b +Error: Cannot convert 'complex' objects with values that are multiple of 100 23f8479f-4784-41d2-aecc-9bed73c8d13b +Error: Cannot convert 'complex' objects with values that are multiple of 100 5fff295a-72f2-48d7-b598-07bdfd0bda6f +Error: Cannot convert 'complex' objects with values that are multiple of 100 7882e9a7-6827-4c61-a930-29d5fc7acfe4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 91a6b535-fee4-4a32-b4f0-528805ae6aaa +Error: Cannot convert 'complex' objects with values that are multiple of 100 a0a7d21f-ba2c-4887-b24d-89298be19b2c +Error: Cannot convert 'complex' objects with values that are multiple of 100 87a39676-e6c1-41ef-a1c7-058a0cb2baa7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 5989e439-91d4-4f59-8bd9-4cc8d8cd5134 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d024758f-9a32-4c8c-90f9-f7d48487b6a9 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ae1fdfa2-5230-474e-bf4e-346266de3058 +Error: Cannot convert 'complex' objects with values that are multiple of 100 27f992f2-336a-4bb4-8a71-faa05d02b8d4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a69b163a-c0ff-45c3-974a-70b1fe651605 +Error: Cannot convert 'complex' objects with values that are multiple of 100 2fa56428-6c22-4da0-af9d-a0e4cdea372b +Error: Cannot convert 'complex' objects with values that are multiple of 100 cb4df774-3555-465f-9388-77a77527d071 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e53a66db-4188-4e30-b0e0-257517103e6c +Error: Cannot convert 'complex' objects with values that are multiple of 100 44389dbc-d1ba-48a6-b247-317f26c2fcba +Error: Cannot convert 'complex' objects with values that are multiple of 100 0210e5db-a487-4137-88c1-b2be15c767a9 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c07a8297-a53c-4da3-b8fd-e79bbe28eb58 +Error: Cannot convert 'complex' objects with values that are multiple of 100 bf3ce749-2bfa-4016-8990-179b7fa3f4e4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d309f6b1-3439-4ab2-bfde-10c835bf5fbc +Error: Cannot convert 'complex' objects with values that are multiple of 100 59ed9c39-fbbf-465c-ad52-3f7c920a3b22 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3fe9bfd7-a802-4559-8e75-7029537eea62 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3cbf3aef-2b29-4144-b3b1-339b4831bed6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 7ba876c6-d99c-4bbd-9d2f-6b0dd2ebc022 +Error: Cannot convert 'complex' objects with values that are multiple of 100 35ce8d6c-853c-4a23-9858-6cda1c0ae9b4 +Error: Cannot convert 'complex' objects with values that are multiple of 100 682cb31c-19ee-42bd-b9f9-99a6a7d05be1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ebf6c057-88ee-42a4-adcc-564a696c45c5 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ec10133d-b87a-4fdf-be5c-bd5cbed145ad +Error: Cannot convert 'complex' objects with values that are multiple of 100 20e6d960-ae74-4816-8111-bdd885e4ab4f +Error: Cannot convert 'complex' objects with values that are multiple of 100 28be65c9-d273-43ec-a516-42714f0c806f +Error: Cannot convert 'complex' objects with values that are multiple of 100 afaf66e2-5b90-4ef9-aa5e-8f5da044fadb +Error: Cannot convert 'complex' objects with values that are multiple of 100 0073e468-5a37-4b04-9126-e5cd82fb7c22 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4758b312-86a8-49d5-a616-b12e8abea612 +Error: Cannot convert 'complex' objects with values that are multiple of 100 f843d079-1645-434c-b2e2-6744546bf518 +Error: Cannot convert 'complex' objects with values that are multiple of 100 22a1f87e-9e51-4bd0-b3f9-33edf9409dcd +Error: Cannot convert 'complex' objects with values that are multiple of 100 6754ee03-4dbd-453e-a571-a710008b8139 +Error: Cannot convert 'complex' objects with values that are multiple of 100 6134569a-59ac-4da6-b5db-60f14d5ea564 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b465c27a-7fb0-4a86-bfec-f0ba8d68b325 +Error: Cannot convert 'complex' objects with values that are multiple of 100 eac4b609-c6cf-405f-b46a-8b0c3e59cd91 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a749cb3c-25a6-4195-bc0e-a281ebdb1b0f +Error: Cannot convert 'complex' objects with values that are multiple of 100 33acd3cf-6b83-4aaa-8a0d-d429eeda042a +Error: Cannot convert 'complex' objects with values that are multiple of 100 bc0f3e94-7b4a-4330-8124-59e0b9ab10cd +Error: Cannot convert 'complex' objects with values that are multiple of 100 c87c35fb-2f70-473a-af97-5b74ed8891fb +Error: Cannot convert 'complex' objects with values that are multiple of 100 e0935148-92e1-427c-872a-92fe3114c6db +Error: Cannot convert 'complex' objects with values that are multiple of 100 e18cfdc8-dc65-4c70-82fb-b61fa05f2a97 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b2bf8917-003f-4e4e-b4b1-2b95cf5d1e02 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4f7cbdb2-7600-442f-beaf-304e80d7d632 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9fbdbb8b-19c1-4fa3-befe-afad6f66196b +Error: Cannot convert 'complex' objects with values that are multiple of 100 b7328f10-bc80-4f5c-801b-c14ffaa99160 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d9520d35-57c2-4f8a-ad32-28d473c2e3f0 +Error: Cannot convert 'complex' objects with values that are multiple of 100 454a86bd-1993-4cff-be5e-dc98cee9c658 +Error: Cannot convert 'complex' objects with values that are multiple of 100 d06dd080-cd3c-42de-8bcb-aa78fc3fed17 +Error: Cannot convert 'complex' objects with values that are multiple of 100 675c2090-6f88-427f-b357-424fcf2e8777 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0655c1b2-24b9-43e4-a9d7-9b4711a80ff3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 7518e96e-5fad-4286-bc04-b1273d2ef59a +Error: Cannot convert 'complex' objects with values that are multiple of 100 818eb169-1f16-410b-9524-e3422a5b626f +Error: Cannot convert 'complex' objects with values that are multiple of 100 618b9e7e-0d59-44a5-b233-25fa09475bfd +Error: Cannot convert 'complex' objects with values that are multiple of 100 a72700a8-dbcc-410e-9c3e-2b029f2a05d9 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c106c66f-836d-4b3f-aed0-7a33dac4cc76 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e8ea6f1b-ed4d-41b4-b1f6-7075c1dff0cf +Error: Cannot convert 'complex' objects with values that are multiple of 100 42fa8bc4-2ada-4523-a6ad-c1602649c744 +Error: Cannot convert 'complex' objects with values that are multiple of 100 87fc12a1-553b-45e4-bc74-0de6b5782350 +Error: Cannot convert 'complex' objects with values that are multiple of 100 ae5c4f18-74b6-4080-9c2c-1c596d984a23 +Error: Cannot convert 'complex' objects with values that are multiple of 100 63bc0d30-e592-4524-987f-6691970dc654 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b3425416-92a2-427e-8323-c5df12536fb7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 92dd95f3-033f-49ab-b42d-1f9e9b41dc13 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b003a035-9097-42b7-91f9-7fa35309a082 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c3c0c870-3081-411e-b487-8089859267de +Error: Cannot convert 'complex' objects with values that are multiple of 100 bf208759-db3f-4b74-aa94-d7d2bb9db63c +Error: Cannot convert 'complex' objects with values that are multiple of 100 2e478ba5-228d-428c-83b3-d1b18313908f +Error: Cannot convert 'complex' objects with values that are multiple of 100 af51f2f7-a382-4806-bc4e-69512a8d532a +Error: Cannot convert 'complex' objects with values that are multiple of 100 037c332e-110f-42bf-9001-a3caf8574ee6 +Error: Cannot convert 'complex' objects with values that are multiple of 100 a918d9a6-87e9-47e8-8a07-de4ebd94dae3 +Error: Cannot convert 'complex' objects with values that are multiple of 100 92750112-af31-41bd-ac90-7c9c830fccee +Error: Cannot convert 'complex' objects with values that are multiple of 100 8d320909-453b-4540-aa3e-2e50e09f12e1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 76b14a34-6ea2-42ef-b3d6-7d9144325a15 +Error: Cannot convert 'complex' objects with values that are multiple of 100 2f6c93f8-f860-486c-8fbe-ed171504b251 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0767434b-8092-4d94-be6f-34d85929dbda +Error: Cannot convert 'complex' objects with values that are multiple of 100 638b5ee8-209d-4bf1-93a0-33f23db7a8a0 +Error: Cannot convert 'complex' objects with values that are multiple of 100 648143b7-e3b1-4e32-a55c-26b017e03bd7 +Error: Cannot convert 'complex' objects with values that are multiple of 100 8fa6afc0-b651-4043-a94e-47250f8e9653 +Error: Cannot convert 'complex' objects with values that are multiple of 100 08b1cfb0-227b-4724-8f60-891d37c29cd8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1e6699fc-1049-4f57-9b64-e8ef22ec16f8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c0e01e41-9651-4079-9dc4-84e659c8772b +Error: Cannot convert 'complex' objects with values that are multiple of 100 f8ce5a55-3aae-444f-83ef-10d9b5c9b8d0 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0edced15-3190-4466-8eac-25fd18f160c8 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3db5ff36-9fbe-4e7e-8b61-d3302232b877 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e17925ae-2d1e-48e5-b16e-9c353ea2e9ff +Error: Cannot convert 'complex' objects with values that are multiple of 100 2c7532b9-95c2-4f3c-8742-b7526a6cde42 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b4caa59c-3e96-4316-aea8-b55148a1fc8b +Error: Cannot convert 'complex' objects with values that are multiple of 100 125dc8c0-4a09-462a-bdf8-19b944affaeb +Error: Cannot convert 'complex' objects with values that are multiple of 100 05234d6d-e547-4713-9069-0520c8b5d6a1 +Error: Cannot convert 'complex' objects with values that are multiple of 100 e5e5af91-3ebd-4707-b7a7-808c2fc87a33 +Error: Cannot convert 'complex' objects with values that are multiple of 100 b79430be-4be2-4e67-adb8-c7f593b7be12 +Error: Cannot convert 'complex' objects with values that are multiple of 100 6349acc6-f70b-474c-a4e1-d74eb5a31638 +Error: Cannot convert 'complex' objects with values that are multiple of 100 4b134f8b-192a-45c3-a543-bf76cd6716a0 +Error: Cannot convert 'complex' objects with values that are multiple of 100 c6468fe5-7ba4-4b6b-9994-a9648e0904ad +Error: Cannot convert 'complex' objects with values that are multiple of 100 0e10a6e2-08f2-44d6-a22d-8bd3e0800a79 +Error: Cannot convert 'complex' objects with values that are multiple of 100 111971d2-74df-4d2b-9fe8-328641d1ff17 +Error: Cannot convert 'complex' objects with values that are multiple of 100 1444dbd3-18c3-423f-b8ed-a01a41ee4731 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9a3e6eff-82c4-441d-8345-fcd4352721bd +Error: Cannot convert 'complex' objects with values that are multiple of 100 20d7476b-bd4b-47a7-9937-87b0fa57d111 +Error: Cannot convert 'complex' objects with values that are multiple of 100 3d3ff888-81b9-47f7-ae2f-24d2c4d4713f +Error: Cannot convert 'complex' objects with values that are multiple of 100 c7041ac7-46bc-4b9b-bf88-a27a1734b154 +Error: Cannot convert 'complex' objects with values that are multiple of 100 0afaa370-f20a-4ae1-b871-2d2675bcdf23 +Error: Cannot convert 'complex' objects with values that are multiple of 100 2888ea65-718b-448b-a2dd-77c56924f630 +Error: Cannot convert 'complex' objects with values that are multiple of 100 9abefd72-25c0-48e3-9239-13a44dbddee6" +`; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts index a150a449db7fc..5981c2759d5ca 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts @@ -9,11 +9,11 @@ import { join } from 'path'; import { omit } from 'lodash'; +import JSON5 from 'json5'; import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; import { - nextMinor, defaultKibanaIndex, defaultKibanaTaskIndex, startElasticsearch, @@ -21,34 +21,27 @@ import { type KibanaMigratorTestKit, readLog, clearLog, + currentVersion, } from '../kibana_migrator_test_kit'; import { BASELINE_DOCUMENTS_PER_TYPE_500K, BASELINE_TEST_ARCHIVE_500K, } from '../kibana_migrator_archive_utils'; -import { getReindexingMigratorTestKit } from '../kibana_migrator_test_kit.fixtures'; +import { + baselineTypes, + getReindexingBaselineTypes, + getReindexingMigratorTestKit, + getUpToDateMigratorTestKit, +} from '../kibana_migrator_test_kit.fixtures'; import { delay } from '../test_utils'; const logFilePath = join(__dirname, 'v2_migration.log'); describe('v2 migration', () => { let esServer: TestElasticsearchUtils; - let kit: KibanaMigratorTestKit; - let migrationResults: MigrationResult[]; beforeAll(async () => { esServer = await startElasticsearch({ dataArchive: BASELINE_TEST_ARCHIVE_500K }); - await clearLog(logFilePath); - kit = await getReindexingMigratorTestKit({ - logFilePath, - filterDeprecated: true, - settings: { - migrations: { - discardUnknownObjects: nextMinor, - }, - }, - }); - migrationResults = await kit.runMigrations(); }); afterAll(async () => { @@ -58,100 +51,328 @@ describe('v2 migration', () => { } }); - describe('a migrator performing a reindexing migration', () => { - describe('when an index contains SO types with incompatible mappings', () => { - it('executes the reindexing migration steps', async () => { - const logs = await readLog(logFilePath); - expect(logs).toMatch(`[${defaultKibanaIndex}] INIT -> WAIT_FOR_YELLOW_SOURCE.`); - expect(logs).toMatch( - `[${defaultKibanaIndex}] WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES.` - ); - expect(logs).toMatch( - `[${defaultKibanaIndex}] UPDATE_SOURCE_MAPPINGS_PROPERTIES -> CHECK_CLUSTER_ROUTING_ALLOCATION.` - ); - expect(logs).toMatch( - `[${defaultKibanaIndex}] CHECK_CLUSTER_ROUTING_ALLOCATION -> CHECK_UNKNOWN_DOCUMENTS.` - ); - expect(logs).toMatch( - `[${defaultKibanaIndex}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.` - ); + describe('to the current stack version', () => { + let upToDateKit: KibanaMigratorTestKit; + let migrationResults: MigrationResult[]; + + beforeAll(async () => { + await clearLog(logFilePath); + upToDateKit = await getUpToDateMigratorTestKit({ + logFilePath, + kibanaVersion: currentVersion, + }); + migrationResults = await upToDateKit.runMigrations(); + }); + + it('skips UPDATE_TARGET_MAPPINGS_PROPERTIES if there are no changes in the mappings', async () => { + const logs = await readLog(logFilePath); + expect(logs).not.toMatch('CREATE_NEW_TARGET'); + expect(logs).toMatch( + `[${defaultKibanaIndex}] CHECK_TARGET_MAPPINGS -> CHECK_VERSION_INDEX_READY_ACTIONS` + ); + expect(logs).toMatch( + `[${defaultKibanaTaskIndex}] CHECK_TARGET_MAPPINGS -> CHECK_VERSION_INDEX_READY_ACTIONS` + ); + expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS_PROPERTIES'); + expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK'); + expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS_META'); + }); + + it(`returns a 'patched' status for each SO index`, () => { + // omit elapsedMs as it varies in each execution + expect(migrationResults.map((result) => omit(result, 'elapsedMs'))).toMatchInlineSnapshot(` + Array [ + Object { + "destIndex": ".kibana_migrator_9.0.0_001", + "status": "patched", + }, + Object { + "destIndex": ".kibana_migrator_tasks_9.0.0_001", + "status": "patched", + }, + ] + `); + }); + + it('each migrator takes less than 10 seconds', () => { + expect( + (migrationResults as Array<{ elapsedMs?: number }>).every( + ({ elapsedMs }) => !elapsedMs || elapsedMs < 10000 + ) + ).toEqual(true); + }); + }); + + describe('to a newer stack version', () => { + describe('with unknown types', () => { + let unknownTypesKit: KibanaMigratorTestKit; + let logs: string; + + beforeAll(async () => { + await clearLog(logFilePath); + unknownTypesKit = await getReindexingMigratorTestKit({ + logFilePath, + // filter out 'task' objects in order to not spawn that migrator for this test + types: getReindexingBaselineTypes(true).filter(({ name }) => name !== 'task'), + settings: { + migrations: { + discardUnknownObjects: currentVersion, // instead of the actual target, 'nextMinor' + }, + }, + }); + }); + + it('fails if Kibana is not configured to discard unknown objects', async () => { + await expect(unknownTypesKit.runMigrations()).rejects.toThrowErrorMatchingInlineSnapshot(` + "Unable to complete saved object migrations for the [.kibana_migrator] index: Migration failed because some documents were found which use unknown saved object types: deprecated + To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration. + Please refer to https://www.elastic.co/guide/en/kibana/master/resolve-migrations-failures.html for more information." + `); + logs = await readLog(logFilePath); expect(logs).toMatch( - `[${defaultKibanaIndex}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.` + 'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be discarded.' ); expect(logs).toMatch( - `[${defaultKibanaIndex}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.` + `[${defaultKibanaIndex}] Migration failed because some documents were found which use unknown saved object types: deprecated` ); - expect(logs).toMatch(`[${defaultKibanaIndex}] MARK_VERSION_INDEX_READY -> DONE.`); - - expect(logs).not.toMatch(`[${defaultKibanaIndex}] CREATE_NEW_TARGET`); - expect(logs).not.toMatch(`[${defaultKibanaIndex}] CLEANUP_UNKNOWN_AND_EXCLUDED`); - expect(logs).not.toMatch(`[${defaultKibanaIndex}] PREPARE_COMPATIBLE_MIGRATION`); + expect(logs).toMatch(`[${defaultKibanaIndex}] CHECK_UNKNOWN_DOCUMENTS -> FATAL.`); }); }); - describe('copies the right documents over to the target indices', () => { - let primaryIndexCounts: Record; - let taskIndexCounts: Record; + describe('with transform errors', () => { + let transformErrorsKit: KibanaMigratorTestKit; + let logs: string; beforeAll(async () => { - primaryIndexCounts = await getAggregatedTypesCount(kit.client, defaultKibanaIndex); - taskIndexCounts = await getAggregatedTypesCount(kit.client, defaultKibanaTaskIndex); + await clearLog(logFilePath); + transformErrorsKit = await getReindexingMigratorTestKit({ + logFilePath, + // filter out 'task' objects in order to not spawn that migrator for this test + types: getReindexingBaselineTypes(true).filter(({ name }) => name !== 'task'), + settings: { + migrations: { + discardCorruptObjects: currentVersion, // instead of the actual target, 'nextMinor' + }, + }, + }); }); - it('copies documents to the right indices depending on their types', () => { - expect(primaryIndexCounts.basic).toBeDefined(); - expect(primaryIndexCounts.complex).toBeDefined(); - expect(primaryIndexCounts.task).not.toBeDefined(); + it('collects corrupt saved object documents across batches', async () => { + try { + await transformErrorsKit.runMigrations(); + } catch (error) { + const lines = error.message + .split('\n') + .filter((line: string) => line.includes(`'complex'`)) + .join('\n'); + expect(lines).toMatchSnapshot(); + } + }); - expect(taskIndexCounts.basic).not.toBeDefined(); - expect(taskIndexCounts.complex).not.toBeDefined(); - expect(taskIndexCounts.task).toBeDefined(); + it('fails if Kibana is not configured to discard transform errors', async () => { + logs = await readLog(logFilePath); + expect(logs).toMatch( + `Cannot convert 'complex' objects with values that are multiple of 100` + ); + expect(logs).toMatch(`[${defaultKibanaIndex}] REINDEX_SOURCE_TO_TEMP_READ -> FATAL.`); }); - it('discards REMOVED_TYPES', () => { - expect(primaryIndexCounts.server).not.toBeDefined(); - expect(taskIndexCounts.server).not.toBeDefined(); + it('closes reindex PIT upon failure', async () => { + const lineWithPit = logs + .split('\n') + .find((line) => + line.includes(`[${defaultKibanaIndex}] REINDEX_SOURCE_TO_TEMP_OPEN_PIT PitId:`) + ); + + expect(lineWithPit).toBeTruthy(); + + const id = JSON5.parse(lineWithPit!).message.split(':')[1]; + expect(id).toBeTruthy(); + + await expect( + transformErrorsKit.client.search({ + pit: { id }, + }) + // throws an exception that cannot search with closed PIT + ).rejects.toThrow(/search_phase_execution_exception/); }); + }); + + describe('configured to discard transform errors and unknown types', () => { + let kit: KibanaMigratorTestKit; + let migrationResults: MigrationResult[]; + let logs: string; - it('discards unknown types', () => { - expect(primaryIndexCounts.deprecated).not.toBeDefined(); - expect(taskIndexCounts.deprecated).not.toBeDefined(); + beforeAll(async () => { + await clearLog(logFilePath); + kit = await getReindexingMigratorTestKit({ + logFilePath, + filterDeprecated: true, + }); + migrationResults = await kit.runMigrations(); + logs = await readLog(logFilePath); }); - it('copies all of the documents', () => { - expect(primaryIndexCounts.basic).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); - expect(taskIndexCounts.task).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + it('migrates documents to the highest version', async () => { + const typeMigrationVersions: Record = { + basic: '10.1.0', // did not define any model versions + complex: '10.2.0', + task: '10.2.0', + }; + + const resultSets = await Promise.all( + baselineTypes.map(({ name: type }) => + kit.client.search({ + index: [defaultKibanaIndex, defaultKibanaTaskIndex], + query: { + bool: { + should: [ + { + term: { type }, + }, + ], + }, + }, + }) + ) + ); + + expect( + resultSets + .flatMap((result) => result.hits.hits) + .every( + (document) => + document._source.typeMigrationVersion === + typeMigrationVersions[document._source.type] + ) + ).toEqual(true); }); - it('executes the excludeOnUpgrade hook', () => { - expect(primaryIndexCounts.complex).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K / 2); + describe('a migrator performing a compatible upgrade migration', () => { + it('updates target mappings when mappings have changed', () => { + expect(logs).toMatch( + `[${defaultKibanaTaskIndex}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.` + ); + expect(logs).toMatch( + `[${defaultKibanaTaskIndex}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.` + ); + expect(logs).toMatch( + `[${defaultKibanaTaskIndex}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.` + ); + expect(logs).toMatch( + `[${defaultKibanaTaskIndex}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.` + ); + }); + + it('updates the version aliases during the PREPARE_COMPATIBLE_MIGRATION step', () => { + expect(logs).toMatch(`[${defaultKibanaTaskIndex}] PREPARE_COMPATIBLE_MIGRATION`); + expect(logs).not.toMatch(`[${defaultKibanaTaskIndex}] MARK_VERSION_INDEX_READY`); + expect(logs).toMatch( + `[${defaultKibanaTaskIndex}] CHECK_VERSION_INDEX_READY_ACTIONS -> DONE.` + ); + }); }); - }); - it('returns a migrated status for each SO index', () => { - // omit elapsedMs as it varies in each execution - expect(migrationResults.map((result) => omit(result, 'elapsedMs'))).toMatchInlineSnapshot(` - Array [ - Object { - "destIndex": ".kibana_migrator_9.1.0_001", - "sourceIndex": ".kibana_migrator_9.0.0_001", - "status": "migrated", - }, - Object { - "destIndex": ".kibana_migrator_tasks_9.0.0_001", - "sourceIndex": ".kibana_migrator_tasks_9.0.0_001", - "status": "migrated", - }, - ] - `); - }); + describe('a migrator performing a reindexing migration', () => { + describe('when an index contains SO types with incompatible mappings', () => { + it('executes the reindexing migration steps', () => { + expect(logs).toMatch(`[${defaultKibanaIndex}] INIT -> WAIT_FOR_YELLOW_SOURCE.`); + expect(logs).toMatch( + `[${defaultKibanaIndex}] WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] UPDATE_SOURCE_MAPPINGS_PROPERTIES -> CHECK_CLUSTER_ROUTING_ALLOCATION.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] CHECK_CLUSTER_ROUTING_ALLOCATION -> CHECK_UNKNOWN_DOCUMENTS.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.` + ); + expect(logs).toMatch(`[${defaultKibanaIndex}] MARK_VERSION_INDEX_READY -> DONE.`); - it('each migrator takes less than 60 seconds', () => { - expect( - (migrationResults as Array<{ elapsedMs?: number }>).every( - ({ elapsedMs }) => !elapsedMs || elapsedMs < 60000 - ) - ).toEqual(true); + expect(logs).not.toMatch(`[${defaultKibanaIndex}] CREATE_NEW_TARGET`); + expect(logs).not.toMatch(`[${defaultKibanaIndex}] CLEANUP_UNKNOWN_AND_EXCLUDED`); + expect(logs).not.toMatch(`[${defaultKibanaIndex}] PREPARE_COMPATIBLE_MIGRATION`); + }); + }); + + describe('copies the right documents over to the target indices', () => { + let primaryIndexCounts: Record; + let taskIndexCounts: Record; + + beforeAll(async () => { + primaryIndexCounts = await getAggregatedTypesCount(kit.client, defaultKibanaIndex); + taskIndexCounts = await getAggregatedTypesCount(kit.client, defaultKibanaTaskIndex); + }); + + it('copies documents to the right indices depending on their types', () => { + expect(primaryIndexCounts.basic).toBeDefined(); + expect(primaryIndexCounts.complex).toBeDefined(); + expect(primaryIndexCounts.task).not.toBeDefined(); + + expect(taskIndexCounts.basic).not.toBeDefined(); + expect(taskIndexCounts.complex).not.toBeDefined(); + expect(taskIndexCounts.task).toBeDefined(); + }); + + it('discards REMOVED_TYPES', () => { + expect(primaryIndexCounts.server).not.toBeDefined(); + expect(taskIndexCounts.server).not.toBeDefined(); + }); + + it('discards unknown types', () => { + expect(primaryIndexCounts.deprecated).not.toBeDefined(); + expect(taskIndexCounts.deprecated).not.toBeDefined(); + }); + + it('copies all of the documents', () => { + expect(primaryIndexCounts.basic).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + expect(taskIndexCounts.task).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + }); + + it('executes the excludeOnUpgrade hook', () => { + // we discard the second half with exclude on upgrade (firstHalf !== true) + // then we discard half all multiples of 100 (1% of them) + expect(primaryIndexCounts.complex).toEqual( + BASELINE_DOCUMENTS_PER_TYPE_500K / 2 - BASELINE_DOCUMENTS_PER_TYPE_500K / 2 / 100 + ); + }); + }); + + it('returns a migrated status for each SO index', () => { + // omit elapsedMs as it varies in each execution + expect(migrationResults.map((result) => omit(result, 'elapsedMs'))) + .toMatchInlineSnapshot(` + Array [ + Object { + "destIndex": ".kibana_migrator_9.1.0_001", + "sourceIndex": ".kibana_migrator_9.0.0_001", + "status": "migrated", + }, + Object { + "destIndex": ".kibana_migrator_tasks_9.0.0_001", + "sourceIndex": ".kibana_migrator_tasks_9.0.0_001", + "status": "migrated", + }, + ] + `); + }); + + it('each migrator takes less than 60 seconds', () => { + expect( + (migrationResults as Array<{ elapsedMs?: number }>).every( + ({ elapsedMs }) => !elapsedMs || elapsedMs < 60000 + ) + ).toEqual(true); + }); + }); }); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_target_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_target_mappings.test.ts deleted file mode 100644 index 4c0721eb04e6f..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_target_mappings.test.ts +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import fs from 'fs/promises'; -import { Env } from '@kbn/config'; -import { REPO_ROOT } from '@kbn/repo-info'; -import { getEnvOptions } from '@kbn/config-mocks'; -import { Root } from '@kbn/core-root-server-internal'; -import { - createRootWithCorePlugins, - createTestServers, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; - -const logFilePath = Path.join(__dirname, 'check_target_mappings.log'); - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2 - CHECK_TARGET_MAPPINGS', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - let logs: string; - - beforeEach(async () => { - await fs.unlink(logFilePath).catch(() => {}); - }); - - afterEach(async () => { - await root?.shutdown(); - await esServer?.stop(); - }); - - it('is not run for new installations', async () => { - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - }, - }, - }); - - root = createRoot(); - esServer = await startES(); - await root.preboot(); - await root.setup(); - await root.start(); - - // Check for migration steps present in the logs - logs = await fs.readFile(logFilePath, 'utf-8'); - - expect(logs).toMatch('CREATE_NEW_TARGET'); - expect(logs).not.toMatch('CHECK_TARGET_MAPPINGS'); - }); - - describe('when the indices are aligned with the stack version', () => { - it('skips UPDATE_TARGET_MAPPINGS_PROPERTIES if there are no changes in the mappings', async () => { - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - }, - }, - }); - - esServer = await startES(); - - // start Kibana a first time to create the system indices - root = createRoot(); - await root.preboot(); - await root.setup(); - await root.start(); - - // stop Kibana and remove logs - await root.shutdown(); - await fs.unlink(logFilePath).catch(() => {}); - - root = createRoot(); - await root.preboot(); - await root.setup(); - await root.start(); - - // Check for migration steps present in the logs - logs = await fs.readFile(logFilePath, 'utf-8'); - expect(logs).not.toMatch('CREATE_NEW_TARGET'); - expect(logs).toMatch('CHECK_TARGET_MAPPINGS -> CHECK_VERSION_INDEX_READY_ACTIONS'); - expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS_PROPERTIES'); - expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK'); - expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS_META'); - }); - }); - - describe('when upgrading to a newer stack version', () => { - const currentVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; - - it('runs UPDATE_TARGET_MAPPINGS_PROPERTIES when mappings have changed', async () => { - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - dataArchive: Path.join(__dirname, '..', 'archives', '8.4.0_with_sample_data_logs.zip'), - }, - }, - }); - - esServer = await startES(); - - // start Kibana a first time to create the system indices - root = createRoot(currentVersion); // we discard a bunch of SO that have become unknown since 8.4.0 - await root.preboot(); - await root.setup(); - await root.start(); - - // Check for migration steps present in the logs - logs = await fs.readFile(logFilePath, 'utf-8'); - expect(logs).not.toMatch('[.kibana] CREATE_NEW_TARGET'); - expect(logs).toMatch('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES'); - expect(logs).toMatch( - 'UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK' - ); - expect(logs).toMatch( - 'UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META' - ); - expect(logs).toMatch('UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS'); - expect(logs).toMatch('Migration completed'); - }); - }); -}); - -function createRoot(discardUnknownObjects?: string, customKibanaVersion?: string) { - return createRootWithCorePlugins( - { - migrations: { - discardUnknownObjects, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss: true, - }, - customKibanaVersion - ); -} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/cleanup.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/cleanup.test.ts deleted file mode 100644 index 057db91eb7b55..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/cleanup.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import Fs from 'fs'; -import Util from 'util'; -import JSON5 from 'json5'; -import { type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { getMigrationDocLink } from '../test_utils'; -import { - clearLog, - currentVersion, - defaultKibanaIndex, - getKibanaMigratorTestKit, - nextMinor, - startElasticsearch, -} from '../kibana_migrator_test_kit'; - -const migrationDocLink = getMigrationDocLink().resolveMigrationFailures; -const logFilePath = Path.join(__dirname, 'cleanup.log'); - -const asyncReadFile = Util.promisify(Fs.readFile); - -describe('migration v2', () => { - let esServer: TestElasticsearchUtils['es']; - let esClient: ElasticsearchClient; - - beforeAll(async () => { - esServer = await startElasticsearch(); - }); - - beforeEach(async () => { - esClient = await setupBaseline(); - await clearLog(logFilePath); - }); - - it('clean ups if migration fails', async () => { - const { runMigrations } = await setupNextMinor(); - - await expect(runMigrations()).rejects.toThrowErrorMatchingInlineSnapshot(` - "Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: Migrations failed. Reason: 1 corrupt saved object documents were found: corrupt:2baf4de0-a6d4-11ed-ba5a-39196fc76e60 - - To allow migrations to proceed, please delete or fix these documents. - Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration. - Please refer to ${migrationDocLink} for more information." - `); - - const logFileContent = await asyncReadFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)); - - const logRecordWithPit = records.find( - (rec) => rec.message === `[${defaultKibanaIndex}] REINDEX_SOURCE_TO_TEMP_OPEN_PIT RESPONSE` - ); - - expect(logRecordWithPit).toBeTruthy(); - }); - - afterEach(async () => { - await esClient?.indices.delete({ index: `${defaultKibanaIndex}_${currentVersion}_001` }); - }); - - afterAll(async () => { - await esServer?.stop(); - }); -}); - -const setupBaseline = async () => { - const typesCurrent: SavedObjectsType[] = [ - { - name: 'complex', - hidden: false, - namespaceType: 'agnostic', - mappings: { - properties: { - name: { type: 'text' }, - value: { type: 'integer' }, - }, - }, - migrations: {}, - }, - ]; - - const savedObjects = [ - { - id: 'complex:4baf4de0-a6d4-11ed-ba5a-39196fc76e60', - body: { - type: 'complex', - complex: { - name: 'foo', - value: 5, - }, - references: [], - coreMigrationVersion: currentVersion, - updated_at: '2023-02-07T11:04:44.914Z', - created_at: '2023-02-07T11:04:44.914Z', - }, - }, - { - id: 'corrupt:2baf4de0-a6d4-11ed-ba5a-39196fc76e60', // incorrect id => corrupt object - body: { - type: 'complex', - complex: { - name: 'bar', - value: 3, - }, - references: [], - coreMigrationVersion: currentVersion, - updated_at: '2023-02-07T11:04:44.914Z', - created_at: '2023-02-07T11:04:44.914Z', - }, - }, - ]; - - const { runMigrations, client } = await getKibanaMigratorTestKit({ - types: typesCurrent, - logFilePath, - }); - - await runMigrations(); - - // inject corrupt saved objects directly using esClient - await Promise.all( - savedObjects.map((savedObject) => - client.create({ - index: defaultKibanaIndex, - refresh: 'wait_for', - ...savedObject, - }) - ) - ); - - return client; -}; - -const setupNextMinor = async () => { - const typesNextMinor: SavedObjectsType[] = [ - { - name: 'complex', - hidden: false, - namespaceType: 'agnostic', - mappings: { - properties: { - name: { type: 'keyword' }, - value: { type: 'long' }, - }, - }, - migrations: { - [nextMinor]: (doc) => doc, - }, - }, - ]; - - return await getKibanaMigratorTestKit({ - types: typesNextMinor, - kibanaVersion: nextMinor, - logFilePath, - settings: { - migrations: { - skip: false, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - appenders: ['file'], - level: 'debug', // DEBUG logs are required to retrieve the PIT _id from the action response logs - }, - ], - }, - }, - }); -}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/collects_corrupt_docs.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/collects_corrupt_docs.test.ts deleted file mode 100644 index 2e78229709060..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/collects_corrupt_docs.test.ts +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import Fs from 'fs'; -import Util from 'util'; -import { - createTestServers, - createRootWithCorePlugins, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import { Root } from '@kbn/core-root-server-internal'; -import { getMigrationDocLink } from '../test_utils'; - -const migrationDocLink = getMigrationDocLink().resolveMigrationFailures; -const logFilePath = Path.join(__dirname, 'collects_corrupt_docs.log'); - -const asyncUnlink = Util.promisify(Fs.unlink); - -async function removeLogFile() { - // ignore errors if it doesn't exist - await asyncUnlink(logFilePath).catch(() => void 0); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2 with corrupt saved object documents', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - - beforeAll(async () => { - await removeLogFile(); - }); - - afterAll(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - it('collects corrupt saved object documents across batches', async () => { - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - // contains 4 `foo` objects, all with a `migrationVersion` of `7.13.0` - // - foo:1 and foo:2 have correct values for their `number` property (13 and 42 respectively) - // - foo:3 and foo:4 don't have the property, and will fail during the `7.14.0` registered migration - // contains migrated index with 8.0 aliases to skip migration, but run outdated doc search - dataArchive: Path.join( - __dirname, - '..', - 'archives', - '8.0.0_document_migration_failure.zip' - ), - }, - }, - }); - - root = createRoot(); - - esServer = await startES(); - await root.preboot(); - const coreSetup = await root.setup(); - - coreSetup.savedObjects.registerType({ - name: 'foo', - hidden: false, - mappings: { - properties: { - number: { type: 'integer' }, - }, - }, - namespaceType: 'agnostic', - migrations: { - '7.14.0': (doc) => { - if (doc.attributes.number === undefined) { - throw new Error('"number" attribute should be present'); - } - doc.attributes = { - ...doc.attributes, - number: doc.attributes.number + 9000, - }; - return doc; - }, - }, - }); - - try { - await root.start(); - expect(true).toEqual(false); - } catch (err) { - const errorMessage = err.message as string; - const errorLines = errorMessage.split('\n'); - const errorMessageWithoutStack = errorLines - .filter((line: string) => !line.includes(' at ')) - .join('\n'); - - expect(errorMessageWithoutStack).toMatchInlineSnapshot(` - "Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 2 transformation errors were encountered: - - foo:3: Error: Migration function for version 7.14.0 threw an error - Caused by: - Error: \\"number\\" attribute should be present - - foo:4: Error: Migration function for version 7.14.0 threw an error - Caused by: - Error: \\"number\\" attribute should be present - - To allow migrations to proceed, please delete or fix these documents. - Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration. - Please refer to ${migrationDocLink} for more information." - `); - - expectMatchOrder(errorLines, [ - { - mode: 'equal', - value: '- foo:3: Error: Migration function for version 7.14.0 threw an error', - }, - { - mode: 'contain', - value: 'at transform', - }, - { - mode: 'equal', - value: 'Caused by:', - }, - { - mode: 'equal', - value: 'Error: "number" attribute should be present', - }, - { - mode: 'contain', - value: 'at 7.14.0', - }, - { - mode: 'equal', - value: '- foo:4: Error: Migration function for version 7.14.0 threw an error', - }, - { - mode: 'contain', - value: 'at transform', - }, - { - mode: 'equal', - value: 'Caused by:', - }, - { - mode: 'equal', - value: 'Error: "number" attribute should be present', - }, - { - mode: 'contain', - value: 'at 7.14.0', - }, - ]); - } - }); -}); - -function createRoot() { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - batchSize: 5, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - appenders: ['file'], - level: 'info', - }, - ], - }, - }, - { - oss: false, - } - ); -} - -type FindInOrderPattern = { mode: 'equal'; value: string } | { mode: 'contain'; value: string }; - -const expectMatchOrder = (lines: string[], patterns: FindInOrderPattern[]) => { - let lineIdx = 0; - let patternIdx = 0; - - while (lineIdx < lines.length && patternIdx < patterns.length) { - const line = lines[lineIdx]; - const pattern = patterns[patternIdx]; - if (lineMatch(line, pattern)) { - patternIdx++; - } - lineIdx++; - } - - expect(patternIdx).toEqual(patterns.length); -}; - -const lineMatch = (line: string, pattern: FindInOrderPattern) => { - if (pattern.mode === 'contain') { - return line.trim().includes(pattern.value.trim()); - } - return line.trim() === pattern.value.trim(); -}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/corrupt_outdated_docs.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/corrupt_outdated_docs.test.ts deleted file mode 100644 index b342882f468b3..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/corrupt_outdated_docs.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import Fs from 'fs'; -import Util from 'util'; -import { - createTestServers, - createRootWithCorePlugins, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import { Root } from '@kbn/core-root-server-internal'; - -const logFilePath = Path.join(__dirname, 'corrupt_outdated_docs.log'); - -const asyncUnlink = Util.promisify(Fs.unlink); - -async function removeLogFile() { - // ignore errors if it doesn't exist - await asyncUnlink(logFilePath).catch(() => void 0); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2 with corrupt saved object documents', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - - beforeAll(async () => { - await removeLogFile(); - }); - - afterAll(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - it.skip('collects corrupt saved object documents across batches', async () => { - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - // original uncorrupt SO: - // { - // type: 'foo', // 'bar', 'baz' - // foo: {}, // bar: {}, baz: {} - // migrationVersion: { - // foo: '7.13.0', - // }, - // }, - // original corrupt SO example: - // { - // id: 'bar:123' // '123' etc - // type: 'foo', - // foo: {}, - // migrationVersion: { - // foo: '7.13.0', - // }, - // }, - // contains migrated index with 8.0 aliases to skip migration, but run outdated doc search - dataArchive: Path.join( - __dirname, - 'archives', - '8.0.0_migrated_with_corrupt_outdated_docs.zip' - ), - }, - }, - }); - - root = createRoot(); - - esServer = await startES(); - await root.preboot(); - const coreSetup = await root.setup(); - - coreSetup.savedObjects.registerType({ - name: 'foo', - hidden: false, - mappings: { properties: {} }, - namespaceType: 'agnostic', - migrations: { - '7.14.0': (doc) => doc, - }, - }); - coreSetup.savedObjects.registerType({ - name: 'bar', - hidden: false, - mappings: { properties: {} }, - namespaceType: 'agnostic', - migrations: { - '7.14.0': (doc) => doc, - }, - }); - coreSetup.savedObjects.registerType({ - name: 'baz', - hidden: false, - mappings: { properties: {} }, - namespaceType: 'agnostic', - migrations: { - '7.14.0': (doc) => doc, - }, - }); - try { - await root.start(); - } catch (err) { - const errorMessage = err.message; - expect( - errorMessage.startsWith( - 'Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 19 corrupt saved object documents were found: ' - ) - ).toBeTruthy(); - expect( - errorMessage.endsWith( - 'To allow migrations to proceed, please delete or fix these documents.' - ) - ).toBeTruthy(); - const expectedCorruptDocIds = [ - '"foo:my_name"', - '"123"', - '"456"', - '"789"', - '"foo:other_name"', - '"bar:123"', - '"baz:123"', - '"bar:345"', - '"bar:890"', - '"baz:456"', - '"baz:789"', - '"bar:other_name"', - '"baz:other_name"', - '"bar:my_name"', - '"baz:my_name"', - '"foo:123"', - '"foo:456"', - '"foo:789"', - '"foo:other"', - ]; - for (const corruptDocId of expectedCorruptDocIds) { - expect(errorMessage.includes(corruptDocId)).toBeTruthy(); - } - } - }); -}); - -function createRoot() { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - batchSize: 5, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - appenders: ['file'], - level: 'info', - }, - ], - }, - }, - { - oss: true, - } - ); -} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts index 9550e46136a47..9bd876166c246 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts @@ -107,8 +107,7 @@ async function createRoot({ logFileName }: CreateRootConfig) { // suite is very long, the 10mins default can cause timeouts jest.setTimeout(15 * 60 * 1000); -// FLAKY: https://github.com/elastic/kibana/issues/156117 -describe.skip('migration v2', () => { +describe('migration v2', () => { let esServer: TestElasticsearchUtils; let rootA: Root; let rootB: Root; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/outdated_docs.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/outdated_docs.test.ts deleted file mode 100644 index 2c45dab10dff2..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/outdated_docs.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import Fs from 'fs'; -import Util from 'util'; -import { kibanaPackageJson as pkg } from '@kbn/repo-info'; -import { - createRootWithCorePlugins, - createTestServers, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Root } from '@kbn/core-root-server-internal'; - -const logFilePath = Path.join(__dirname, 'outdated_docs.log'); - -const asyncUnlink = Util.promisify(Fs.unlink); -async function removeLogFile() { - // ignore errors if it doesn't exist - await asyncUnlink(logFilePath).catch(() => void 0); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - - beforeAll(async () => { - await removeLogFile(); - }); - - afterAll(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - it('migrates the documents to the highest version', async () => { - const migratedIndexAlias = `.kibana_${pkg.version}`; - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - // original SO: - // { - // type: 'foo', - // foo: {}, - // migrationVersion: { - // foo: '7.13.0', - // }, - // }, - // contains migrated index with 8.0 aliases to skip migration, but run outdated doc search - dataArchive: Path.join( - __dirname, - '..', - 'archives', - '8.0.0_migrated_with_outdated_docs.zip' - ), - }, - }, - }); - - root = createRoot(); - - esServer = await startES(); - await root.preboot(); - const coreSetup = await root.setup(); - - coreSetup.savedObjects.registerType({ - name: 'foo', - hidden: false, - mappings: { properties: {} }, - namespaceType: 'agnostic', - migrations: { - '7.14.0': (doc) => doc, - }, - }); - - const coreStart = await root.start(); - const esClient = coreStart.elasticsearch.client.asInternalUser; - - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(1); - const [doc] = migratedDocs; - expect(doc._source.coreMigrationVersion).toBe('8.8.0'); - expect(doc._source.typeMigrationVersion).toBe('7.14.0'); - }); -}); - -function createRoot() { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss: true, - } - ); -} - -async function fetchDocs(esClient: ElasticsearchClient, index: string) { - const body = await esClient.search({ - index, - body: { - query: { - bool: { - should: [ - { - term: { type: 'foo' }, - }, - ], - }, - }, - }, - }); - - return body.hits.hits; -} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.test.ts index 8cfcb1ea5b745..9c42e966b329e 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.test.ts @@ -137,32 +137,6 @@ describe('when upgrading to a new stack version', () => { await clearLog(); }); - it('fails if unknown documents exist', async () => { - // remove the 'deprecated' type from the mappings, so that it is considered unknown - const { runMigrations } = await getUpToDateMigratorTestKit({ - filterDeprecated: true, - }); - - try { - await runMigrations(); - } catch (err) { - const errorMessage = err.message; - expect(errorMessage).toMatch( - `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: Migration failed because some documents were found which use unknown saved object types:` - ); - expect(errorMessage).toMatch( - 'To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration.' - ); - expect(errorMessage).toMatch(/deprecated:.*\(type: "deprecated"\)/); - } - - const logs = await readLog(); - expect(logs).toMatch('INIT -> WAIT_FOR_YELLOW_SOURCE.'); - expect(logs).toMatch('WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES.'); - expect(logs).toMatch('UPDATE_SOURCE_MAPPINGS_PROPERTIES -> CLEANUP_UNKNOWN_AND_EXCLUDED.'); - expect(logs).toMatch('CLEANUP_UNKNOWN_AND_EXCLUDED -> FATAL.'); - }); - it('proceeds if there are no unknown documents', async () => { const { client, runMigrations } = await getUpToDateMigratorTestKit(); @@ -289,35 +263,6 @@ describe('when upgrading to a new stack version', () => { await clearLog(); }); - it('fails if unknown documents exist', async () => { - const { runMigrations } = await getCompatibleMigratorTestKit({ - filterDeprecated: true, // remove the 'deprecated' type from the mappings, so that it is considered unknown - }); - - try { - await runMigrations(); - } catch (err) { - const errorMessage = err.message; - expect(errorMessage).toMatch( - `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: Migration failed because some documents were found which use unknown saved object types:` - ); - expect(errorMessage).toMatch( - 'To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration.' - ); - expect(errorMessage).toMatch(/deprecated:.*\(type: "deprecated"\)/); - } - - const logs = await readLog(); - expect(logs).toMatch(`[${defaultKibanaIndex}] INIT -> WAIT_FOR_YELLOW_SOURCE.`); - expect(logs).toMatch( - `[${defaultKibanaIndex}] WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES.` - ); // this step is run only if mappings are compatible but NOT equal - expect(logs).toMatch( - `[${defaultKibanaIndex}] UPDATE_SOURCE_MAPPINGS_PROPERTIES -> CLEANUP_UNKNOWN_AND_EXCLUDED.` - ); - expect(logs).toMatch(`[${defaultKibanaIndex}] CLEANUP_UNKNOWN_AND_EXCLUDED -> FATAL.`); - }); - it('proceeds if there are no unknown documents', async () => { const { client, runMigrations } = await getCompatibleMigratorTestKit(); @@ -376,10 +321,11 @@ describe('when upgrading to a new stack version', () => { expect(logs).toMatch('MARK_VERSION_INDEX_READY -> DONE'); const counts = await getAggregatedTypesCount(client); + // for 'complex' objects, we discard second half and also multiples of 100 expect(counts).toMatchInlineSnapshot(` Object { "basic": 10, - "complex": 5, + "complex": 4, "deprecated": 10, "task": 10, } diff --git a/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts index b000f0d775358..358ceea5c006a 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts @@ -26,8 +26,8 @@ import { getReindexingBaselineTypes, } from '../kibana_migrator_test_kit.fixtures'; -export const logFilePathFirstRun = join(__dirname, 'dot_kibana_split_1st_run.test.log'); -export const logFilePathSecondRun = join(__dirname, 'dot_kibana_split_2nd_run.test.log'); +export const logFilePathFirstRun = join(__dirname, 'single_migrator_failures_1st_run.test.log'); +export const logFilePathSecondRun = join(__dirname, 'single_migrator_failures_2nd_run.test.log'); const kibanaSplitIndex = `${defaultKibanaIndex}_split`; const tasksToNewIndex = getReindexingBaselineTypes(true).map((type) => { @@ -74,6 +74,7 @@ describe('split .kibana index into multiple system indices', () => { settings: { migrations: { discardUnknownObjects: nextMinor, + discardCorruptObjects: nextMinor, }, }, }); @@ -249,6 +250,7 @@ describe('split .kibana index into multiple system indices', () => { settings: { migrations: { discardUnknownObjects: nextMinor, + discardCorruptObjects: nextMinor, }, }, }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts index 73437885fc7a7..1ec300c075ff2 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts @@ -141,6 +141,40 @@ export const getReindexingBaselineTypes = (filterDeprecated: boolean) => createdAt: { type: 'date' }, }, }, + { + type: 'unsafe_transform', + transformFn: (doc) => { + if (doc.attributes.value % 100 === 0) { + throw new Error( + `Cannot convert 'complex' objects with values that are multiple of 100 ${doc.id}` + ); + } + return { document: doc }; + }, + }, + ], + }, + }, + }; + } else if (type.name === 'task') { + return { + ...type, + mappings: { + properties: { + ...type.mappings.properties, + lastRun: { type: 'date' }, + }, + }, + modelVersions: { + ...type.modelVersions, + 2: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + lastRun: { type: 'date' }, + }, + }, ], }, }, @@ -268,13 +302,21 @@ export const getCompatibleMigratorTestKit = async ({ export const getReindexingMigratorTestKit = async ({ logFilePath = defaultLogFilePath, filterDeprecated = false, + types = getReindexingBaselineTypes(filterDeprecated), kibanaVersion = nextMinor, settings = {}, }: GetMutatedMigratorParams = {}) => { return await getKibanaMigratorTestKit({ logFilePath, - types: getReindexingBaselineTypes(filterDeprecated), + types, kibanaVersion, - settings, + settings: { + ...settings, + migrations: { + discardUnknownObjects: nextMinor, + discardCorruptObjects: nextMinor, + ...settings.migrations, + }, + }, }); };