diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts index 8072675b96b7e..3c47ea01ecd72 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts @@ -652,6 +652,11 @@ describe('migrations v2 model', () => { expect(newState.controlState).toBe('WAIT_FOR_YELLOW_SOURCE'); expect(newState.sourceIndex.value).toBe('.kibana_7.invalid.0_001'); + expect(newState.aliases).toEqual({ + '.kibana': '.kibana_7.invalid.0_001', + '.kibana_7.11.0': '.kibana_7.11.0_001', + '.kibana_7.12.0': '.kibana_7.invalid.0_001', + }); }); test('INIT -> WAIT_FOR_YELLOW_SOURCE when migrating from a v2 migrations index (>= 7.11.0)', () => { @@ -683,6 +688,10 @@ describe('migrations v2 model', () => { expect(newState.sourceIndex.value).toBe('.kibana_7.11.0_001'); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); + expect(newState.aliases).toEqual({ + '.kibana': '.kibana_7.11.0_001', + '.kibana_7.11.0': '.kibana_7.11.0_001', + }); }); test('INIT -> WAIT_FOR_YELLOW_SOURCE when migrating from a v1 migrations index (>= 6.5 < 7.11.0)', () => { @@ -701,6 +710,9 @@ describe('migrations v2 model', () => { expect(newState.sourceIndex.value).toBe('.kibana_3'); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); + expect(newState.aliases).toEqual({ + '.kibana': '.kibana_3', + }); }); test('INIT -> LEGACY_SET_WRITE_BLOCK when migrating from a legacy index (>= 6.0.0 < 6.5)', () => { const res: ResponseType<'INIT'> = Either.right({ @@ -807,67 +819,6 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); - - describe('when upgrading to a new stack version', () => { - const unchangedMappingsState: State = { - ...baseState, - controlState: 'INIT', - kibanaVersion: '7.12.0', // new version! - currentAlias: '.kibana', - versionAlias: '.kibana_7.12.0', - versionIndex: '.kibana_7.11.0_001', - }; - it('INIT -> PREPARE_COMPATIBLE_MIGRATION when the mappings have not changed', () => { - const res: ResponseType<'INIT'> = Either.right({ - '.kibana_7.11.0_001': { - aliases: { - '.kibana': {}, - '.kibana_7.11.0': {}, - }, - mappings: indexMapping, - settings: {}, - }, - }); - const newState = model(unchangedMappingsState, res) as PrepareCompatibleMigration; - - expect(newState.controlState).toEqual('PREPARE_COMPATIBLE_MIGRATION'); - expect(newState.targetIndexRawMappings).toEqual({ - _meta: { - migrationMappingPropertyHashes: { - new_saved_object_type: '4a11183eee21e6fbad864f7a30b39ad0', - }, - }, - properties: { - new_saved_object_type: { - properties: { - value: { - type: 'text', - }, - }, - }, - }, - }); - expect(newState.versionAlias).toEqual('.kibana_7.12.0'); - expect(newState.currentAlias).toEqual('.kibana'); - // will point to - expect(newState.targetIndex).toEqual('.kibana_7.11.0_001'); - expect(newState.preTransformDocsActions).toEqual([ - { - add: { - alias: '.kibana_7.12.0', - index: '.kibana_7.11.0_001', - }, - }, - { - remove: { - alias: '.kibana_7.11.0', - index: '.kibana_7.11.0_001', - must_exist: true, - }, - }, - ]); - }); - }); }); }); @@ -1259,25 +1210,129 @@ describe('migrations v2 model', () => { }); describe('WAIT_FOR_YELLOW_SOURCE', () => { - const someMappings = { - properties: {}, - } as const; - const waitForYellowSourceState: WaitForYellowSourceState = { ...baseState, controlState: 'WAIT_FOR_YELLOW_SOURCE', - sourceIndex: Option.some('.kibana_3') as Option.Some, - sourceIndexMappings: someMappings, + sourceIndex: Option.some('.kibana_7.11.0_001') as Option.Some, + sourceIndexMappings: baseState.targetIndexMappings, + aliases: { + '.kibana': '.kibana_7.11.0_001', + '.kibana_7.11.0': '.kibana_7.11.0_001', + }, }; - test('WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS if action succeeds', () => { - const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({}); - const newState = model(waitForYellowSourceState, res); - expect(newState.controlState).toEqual('CHECK_UNKNOWN_DOCUMENTS'); + describe('if action succeeds', () => { + test('it resets retry count and delay', () => { + const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({}); + const testState = { + ...waitForYellowSourceState, + retryCount: 1, + retryDelay: 2000, + }; + const newState = model(testState, res); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); - expect(newState).toMatchObject({ - controlState: 'CHECK_UNKNOWN_DOCUMENTS', - sourceIndex: Option.some('.kibana_3'), + describe('and mappings match (diffMappings == false)', () => { + const unchangedMappingsState: State = { + ...waitForYellowSourceState, + controlState: 'WAIT_FOR_YELLOW_SOURCE', + kibanaVersion: '7.12.0', // new version! + currentAlias: '.kibana', + versionAlias: '.kibana_7.12.0', + versionIndex: '.kibana_7.11.0_001', + }; + + test('WAIT_FOR_YELLOW_SOURCE -> PREPARE_COMPATIBLE_MIGRATION', () => { + const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({ + '.kibana_7.11.0_001': { + aliases: { + '.kibana': {}, + '.kibana_7.11.0': {}, + }, + mappings: indexMapping, + settings: {}, + }, + }); + const newState = model(unchangedMappingsState, res) as PrepareCompatibleMigration; + + expect(newState.controlState).toEqual('PREPARE_COMPATIBLE_MIGRATION'); + expect(newState.targetIndexRawMappings).toEqual({ + _meta: { + migrationMappingPropertyHashes: { + new_saved_object_type: '4a11183eee21e6fbad864f7a30b39ad0', + }, + }, + properties: { + new_saved_object_type: { + properties: { + value: { + type: 'text', + }, + }, + }, + }, + }); + expect(newState.versionAlias).toEqual('.kibana_7.12.0'); + expect(newState.currentAlias).toEqual('.kibana'); + // will point to + expect(newState.targetIndex).toEqual('.kibana_7.11.0_001'); + expect(newState.preTransformDocsActions).toEqual([ + { + add: { + alias: '.kibana_7.12.0', + index: '.kibana_7.11.0_001', + }, + }, + { + remove: { + alias: '.kibana_7.11.0', + index: '.kibana_7.11.0_001', + must_exist: true, + }, + }, + ]); + }); + }); + + describe('and mappings DO NOT match (diffMappings == true)', () => { + const actualMappings: IndexMapping = { + properties: { + new_saved_object_type: { + properties: { + value: { type: 'integer' }, + }, + }, + }, + _meta: { + migrationMappingPropertyHashes: { + new_saved_object_type: '5b11183eee21e6fbad864f7a30b39be1', + }, + }, + }; + + const changedMappingsState: State = { + ...waitForYellowSourceState, + controlState: 'WAIT_FOR_YELLOW_SOURCE', + kibanaVersion: '7.12.0', // new version! + currentAlias: '.kibana', + versionAlias: '.kibana_7.12.0', + versionIndex: '.kibana_7.11.0_001', + sourceIndexMappings: actualMappings, + }; + + test('WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS', () => { + const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({}); + const newState = model(changedMappingsState, res); + expect(newState.controlState).toEqual('CHECK_UNKNOWN_DOCUMENTS'); + + expect(newState).toMatchObject({ + controlState: 'CHECK_UNKNOWN_DOCUMENTS', + sourceIndex: Option.some('.kibana_7.11.0_001'), + sourceIndexMappings: actualMappings, + }); + }); }); }); @@ -1297,22 +1352,6 @@ describe('migrations v2 model', () => { } `); }); - - test('WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS resets retry count and delay if action succeeds', () => { - const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({}); - const testState = { - ...waitForYellowSourceState, - retryCount: 1, - retryDelay: 2000, - }; - const newState = model(testState, res); - expect(newState.controlState).toEqual('CHECK_UNKNOWN_DOCUMENTS'); - - expect(newState).toMatchObject({ - controlState: 'CHECK_UNKNOWN_DOCUMENTS', - sourceIndex: Option.some('.kibana_3'), - }); - }); }); describe('CHECK_UNKNOWN_DOCUMENTS', () => { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index 44de571c13f5d..167a5d3771b78 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -158,48 +158,14 @@ export const model = (currentState: State, resW: ResponseType): }, ], }; - } else if ( - // source exists - Boolean(indices[source!]?.mappings?._meta?.migrationMappingPropertyHashes) && - // ...and mappings are unchanged - !diffMappings( - /* actual */ - indices[source!].mappings, - /* expected */ - stateP.targetIndexMappings - ) - ) { - const targetIndex = source!; - const sourceMappings = indices[source!].mappings; - - return { - ...stateP, - controlState: 'PREPARE_COMPATIBLE_MIGRATION', - sourceIndex: Option.none, - targetIndex, - targetIndexRawMappings: sourceMappings, - targetIndexMappings: mergeMigrationMappingPropertyHashes( - stateP.targetIndexMappings, - sourceMappings - ), - preTransformDocsActions: [ - // Point the version alias to the source index. This let's other Kibana - // instances know that a migration for the current version is "done" - // even though we may be waiting for document transformations to finish. - { add: { index: source!, alias: stateP.versionAlias } }, - ...buildRemoveAliasActions(source!, Object.keys(aliases), [ - stateP.currentAlias, - stateP.versionAlias, - ]), - ], - versionIndexReadyActions: Option.none, - }; } else if ( // If the `.kibana` alias exists source != null ) { + // CHECKPOINT here we decide to go for yellow source return { ...stateP, + aliases, controlState: 'WAIT_FOR_YELLOW_SOURCE', sourceIndex: Option.some(source!) as Option.Some, sourceIndexMappings: indices[source!].mappings, @@ -481,10 +447,50 @@ export const model = (currentState: State, resW: ResponseType): } else if (stateP.controlState === 'WAIT_FOR_YELLOW_SOURCE') { const res = resW as ExcludeRetryableEsError>; if (Either.isRight(res)) { - return { - ...stateP, - controlState: 'CHECK_UNKNOWN_DOCUMENTS', - }; + // check the existing mappings to see if we can avoid reindexing + if ( + // source exists + Boolean(stateP.sourceIndexMappings._meta?.migrationMappingPropertyHashes) && + // ...and mappings are unchanged + !diffMappings( + /* actual */ + stateP.sourceIndexMappings, + /* expected */ + stateP.targetIndexMappings + ) + ) { + // The source index .kibana is pointing to. E.g: ".xx8.7.0_001" + const source = stateP.sourceIndex.value; + + return { + ...stateP, + controlState: 'PREPARE_COMPATIBLE_MIGRATION', + sourceIndex: Option.none, + targetIndex: source!, + targetIndexRawMappings: stateP.sourceIndexMappings, + targetIndexMappings: mergeMigrationMappingPropertyHashes( + stateP.targetIndexMappings, + stateP.sourceIndexMappings + ), + preTransformDocsActions: [ + // Point the version alias to the source index. This let's other Kibana + // instances know that a migration for the current version is "done" + // even though we may be waiting for document transformations to finish. + { add: { index: source!, alias: stateP.versionAlias } }, + ...buildRemoveAliasActions(source!, Object.keys(stateP.aliases), [ + stateP.currentAlias, + stateP.versionAlias, + ]), + ], + versionIndexReadyActions: Option.none, + }; + } else { + // the mappings have changed, but changes might still be compatible + return { + ...stateP, + controlState: 'CHECK_UNKNOWN_DOCUMENTS', + }; + } } else if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'index_not_yellow_timeout')) { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts index e3873122329fd..f3a59fadf2dd4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts @@ -215,6 +215,7 @@ export interface WaitForYellowSourceState extends BaseState { readonly controlState: 'WAIT_FOR_YELLOW_SOURCE'; readonly sourceIndex: Option.Some; readonly sourceIndexMappings: IndexMapping; + readonly aliases: Record; } export interface CheckUnknownDocumentsState extends BaseState { diff --git a/src/core/server/integration_tests/saved_objects/migrations/skip_reindex.test.ts b/src/core/server/integration_tests/saved_objects/migrations/skip_reindex.test.ts index df64ccb853753..1699dbae46613 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/skip_reindex.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/skip_reindex.test.ts @@ -63,7 +63,8 @@ describe('skip reindexing', () => { logs = await fs.readFile(logFilePath, 'utf-8'); - expect(logs).toMatch('INIT -> PREPARE_COMPATIBLE_MIGRATION'); + expect(logs).toMatch('INIT -> WAIT_FOR_YELLOW_SOURCE'); + expect(logs).toMatch('WAIT_FOR_YELLOW_SOURCE -> PREPARE_COMPATIBLE_MIGRATION'); expect(logs).toMatch('PREPARE_COMPATIBLE_MIGRATION -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'); expect(logs).toMatch('CHECK_TARGET_MAPPINGS -> CHECK_VERSION_INDEX_READY_ACTIONS'); expect(logs).toMatch('CHECK_VERSION_INDEX_READY_ACTIONS -> DONE');