diff --git a/src/core_plugins/elasticsearch/lib/__tests__/health_check.js b/src/core_plugins/elasticsearch/lib/__tests__/health_check.js index d5102f1f640f2..da6a6b6fcb428 100644 --- a/src/core_plugins/elasticsearch/lib/__tests__/health_check.js +++ b/src/core_plugins/elasticsearch/lib/__tests__/health_check.js @@ -39,7 +39,7 @@ describe('plugins/elasticsearch', () => { cluster = { callWithInternalUser: sinon.stub() }; cluster.callWithInternalUser.withArgs('index', sinon.match.any).returns(Promise.resolve()); - cluster.callWithInternalUser.withArgs('create', sinon.match.any).returns(Promise.resolve({ _id: 1, _version: 1 })); + cluster.callWithInternalUser.withArgs('create', sinon.match.any).returns(Promise.resolve({ _id: '1', _version: 1 })); cluster.callWithInternalUser.withArgs('mget', sinon.match.any).returns(Promise.resolve({ ok: true })); cluster.callWithInternalUser.withArgs('get', sinon.match.any).returns(Promise.resolve({ found: false })); cluster.callWithInternalUser.withArgs('search', sinon.match.any).returns(Promise.resolve({ hits: { hits: [] } })); diff --git a/src/server/saved_objects/client/__tests__/saved_objects_client.js b/src/server/saved_objects/client/__tests__/saved_objects_client.js index 95d25f89925d3..24dec0e7848c0 100644 --- a/src/server/saved_objects/client/__tests__/saved_objects_client.js +++ b/src/server/saved_objects/client/__tests__/saved_objects_client.js @@ -1,6 +1,7 @@ import expect from 'expect.js'; import sinon from 'sinon'; import { SavedObjectsClient } from '../saved_objects_client'; +import { createIdQuery } from '../lib/create_id_query'; describe('SavedObjectsClient', () => { let callAdminCluster; @@ -167,7 +168,7 @@ describe('SavedObjectsClient', () => { errors: false, items: [{ create: { - _type: 'foo', + _type: 'config', _id: 'one', error: { reason: 'type[config] missing' @@ -175,8 +176,8 @@ describe('SavedObjectsClient', () => { } }, { create: { - _type: 'config', - _id: 'one', + _type: 'index-pattern', + _id: 'two', _version: 2 } }] @@ -190,13 +191,13 @@ describe('SavedObjectsClient', () => { expect(response).to.eql([ { id: 'one', - type: 'foo', + type: 'config', version: undefined, attributes: { title: 'Test One' }, error: { message: 'type[config] missing' } }, { - id: 'one', - type: 'config', + id: 'two', + type: 'index-pattern', version: 2, attributes: { title: 'Test Two' }, error: undefined @@ -215,8 +216,8 @@ describe('SavedObjectsClient', () => { } }, { create: { - _type: 'config', - _id: 'one', + _type: 'index-pattern', + _id: 'two', _version: 2 } }] @@ -235,8 +236,8 @@ describe('SavedObjectsClient', () => { attributes: { title: 'Test One' }, error: undefined }, { - id: 'one', - type: 'config', + id: 'two', + type: 'index-pattern', version: 2, attributes: { title: 'Test Two' }, error: undefined @@ -247,7 +248,9 @@ describe('SavedObjectsClient', () => { describe('#delete', () => { it('throws notFound when ES is unable to find the document', (done) => { - callAdminCluster.returns(Promise.resolve({ found: false })); + callAdminCluster.returns(Promise.resolve({ + deleted: 0 + })); savedObjectsClient.delete('index-pattern', 'logstash-*').then(() => { done('failed'); @@ -263,10 +266,9 @@ describe('SavedObjectsClient', () => { expect(callAdminCluster.calledOnce).to.be(true); const args = callAdminCluster.getCall(0).args; - expect(args[0]).to.be('delete'); + expect(args[0]).to.be('deleteByQuery'); expect(args[1]).to.eql({ - type: 'index-pattern', - id: 'logstash-*', + body: createIdQuery({ type: 'index-pattern', id: 'logstash-*' }), refresh: 'wait_for', index: '.kibana-test' }); @@ -311,7 +313,23 @@ describe('SavedObjectsClient', () => { const expectedQuery = { bool: { must: [{ match_all: {} }], - filter: [{ term: { _type: 'index-pattern' } }] + filter: [ + { + bool: { + should: [ + { + term: { + _type: 'index-pattern' + } + }, { + term: { + type: 'index-pattern' + } + } + ] + } + } + ] } }; @@ -364,18 +382,26 @@ describe('SavedObjectsClient', () => { expect(callAdminCluster.calledOnce).to.be(true); const options = callAdminCluster.getCall(0).args[1]; - expect(options._source).to.eql('title'); + expect(options._source).to.eql([ + '*.title', 'type', 'title' + ]); }); }); describe('#get', () => { it('formats Elasticsearch response', async () => { callAdminCluster.returns(Promise.resolve({ - _id: 'logstash-*', - _type: 'index-pattern', - _version: 2, - _source: { - title: 'Testing' + hits: { + hits: [ + { + _id: 'logstash-*', + _type: 'index-pattern', + _version: 2, + _source: { + title: 'Testing' + } + } + ] } })); @@ -392,20 +418,20 @@ describe('SavedObjectsClient', () => { }); describe('#bulkGet', () => { - it('accepts a array of mixed type and ids', async () => { + it('accepts an array of mixed type and ids', async () => { await savedObjectsClient.bulkGet([ { id: 'one', type: 'config' }, - { id: 'two', type: 'index-pattern' }, - { id: 'three' } + { id: 'two', type: 'index-pattern' } ]); expect(callAdminCluster.calledOnce).to.be(true); const options = callAdminCluster.getCall(0).args[1]; - expect(options.body.docs).to.eql([ - { _type: 'config', _id: 'one' }, - { _type: 'index-pattern', _id: 'two' }, - { _type: undefined, _id: 'three' } + expect(options.body).to.eql([ + {}, + createIdQuery({ type: 'config', id: 'one' }), + {}, + createIdQuery({ type: 'index-pattern', id: 'two' }) ]); }); @@ -416,23 +442,31 @@ describe('SavedObjectsClient', () => { expect(callAdminCluster.notCalled).to.be(true); }); - it('omits missed objects', async () => { + it('reports error on missed objects', async () => { callAdminCluster.returns(Promise.resolve({ - docs:[{ - _type: 'config', - _id: 'bad', - found: false - }, { - _type: 'config', - _id: 'good', - found: true, - _version: 2, - _source: { title: 'Test' } - }] + responses: [ + { + hits: { + hits: [ + { + _id: 'good', + _type: 'doc', + _version: 2, + _source: { + type: 'config', + config: { + title: 'Test' + } + } + } + ] + } + } + ] })); const { saved_objects: savedObjects } = await savedObjectsClient.bulkGet( - ['good', 'bad', 'config'] + [{ id: 'good', type: 'config' }, { id: 'bad', type: 'config' }] ); expect(savedObjects).to.have.length(1); diff --git a/src/server/saved_objects/client/__tests__/saved_objects_client_mappings.js b/src/server/saved_objects/client/__tests__/saved_objects_client_mappings.js new file mode 100644 index 0000000000000..19ef0d9f84aa9 --- /dev/null +++ b/src/server/saved_objects/client/__tests__/saved_objects_client_mappings.js @@ -0,0 +1,155 @@ +import elasticsearch from 'elasticsearch'; +import expect from 'expect.js'; +import sinon from 'sinon'; + +import { SavedObjectsClient } from '../saved_objects_client'; +const { BadRequest } = elasticsearch.errors; + +describe('SavedObjectsClient', () => { + let callAdminCluster; + let savedObjectsClient; + const illegalArgumentException = { + type: 'illegal_argument_exception', + reason: 'Rejecting mapping update to [.kibana-v6] as the final mapping would have more than 1 type: [doc, foo]' + }; + + describe('mapping', () => { + beforeEach(() => { + callAdminCluster = sinon.stub(); + savedObjectsClient = new SavedObjectsClient('.kibana-test', {}, callAdminCluster); + }); + + afterEach(() => { + callAdminCluster.reset(); + }); + + + describe('#create', () => { + it('falls back to v6 mapping', async () => { + const error = new BadRequest('[illegal_argument_exception] Rejecting mapping update to [.kibana-v6]', { + body: { + error: illegalArgumentException + } + }); + + callAdminCluster + .onFirstCall().throws(error) + .onSecondCall().returns(Promise.resolve({ _type: 'index-pattern', _id: 'logstash-*', _version: 2 })); + + const response = await savedObjectsClient.create('index-pattern', { + title: 'Logstash' + }); + + expect(response).to.eql({ + type: 'index-pattern', + id: 'logstash-*', + version: 2, + attributes: { + title: 'Logstash', + } + }); + }); + }); + + describe('#bulkCreate', () => { + it('falls back to v6 mappings', async () => { + const firstResponse = { + errors: true, + items: [{ + create: { + _type: 'config', + _id: 'one', + _version: 2, + status: 400, + error: illegalArgumentException + } + }, { + create: { + _type: 'index-pattern', + _id: 'two', + _version: 2, + status: 400, + error: illegalArgumentException + } + }] + }; + + const secondResponse = { + errors: false, + items: [{ + create: { + _type: 'config', + _id: 'one', + _version: 2 + } + }, { + create: { + _type: 'index-pattern', + _id: 'two', + _version: 2 + } + }] + }; + + callAdminCluster + .onFirstCall().returns(Promise.resolve(firstResponse)) + .onSecondCall().returns(Promise.resolve(secondResponse)); + + const response = await savedObjectsClient.bulkCreate([ + { type: 'config', id: 'one', attributes: { title: 'Test One' } }, + { type: 'index-pattern', id: 'two', attributes: { title: 'Test Two' } } + ]); + + expect(response).to.eql([ + { + id: 'one', + type: 'config', + version: 2, + attributes: { title: 'Test One' }, + error: undefined + }, { + id: 'two', + type: 'index-pattern', + version: 2, + attributes: { title: 'Test Two' }, + error: undefined + } + ]); + }); + }); + + describe('update', () => { + it('falls back to v6 mappings', async () => { + const id = 'logstash-*'; + const type = 'index-pattern'; + const version = 2; + const attributes = { title: 'Testing' }; + + const error = new BadRequest('[document_missing_exception] [config][logstash-*]: document missing', { + body: { + error: { + type: 'document_missing_exception' + } + } + }); + + callAdminCluster + .onFirstCall().throws(error) + .onSecondCall().returns(Promise.resolve({ + _id: id, + _type: type, + _version: version, + result: 'updated' + })); + + const response = await savedObjectsClient.update('index-pattern', 'logstash-*', attributes); + expect(response).to.eql({ + id, + type, + version, + attributes + }); + }); + }); + }); +}); diff --git a/src/server/saved_objects/client/lib/__tests__/compatibility.js b/src/server/saved_objects/client/lib/__tests__/compatibility.js new file mode 100644 index 0000000000000..33322e5320460 --- /dev/null +++ b/src/server/saved_objects/client/lib/__tests__/compatibility.js @@ -0,0 +1,53 @@ +import expect from 'expect.js'; +import { v5BulkCreate, v6BulkCreate } from '../compatibility'; + +describe('compatibility', () => { + const testObjects = [ + { type: 'index-pattern', id: 'one', attributes: { title: 'Test Index Pattern' } }, + { type: 'config', id: 'two', attributes: { title: 'Test Config Value' } } + ]; + + describe('v5BulkCreate', () => { + it('handles default options', () => { + const objects = v5BulkCreate(testObjects); + expect(objects).to.eql([ + { create: { _type: 'index-pattern', _id: 'one' } }, + { title: 'Test Index Pattern' }, + { create: { _type: 'config', _id: 'two' } }, + { title: 'Test Config Value' } + ]); + }); + + it('uses index action for options.overwrite=true', () => { + const objects = v5BulkCreate(testObjects, { overwrite: true }); + expect(objects).to.eql([ + { index: { _type: 'index-pattern', _id: 'one' } }, + { title: 'Test Index Pattern' }, + { index: { _type: 'config', _id: 'two' } }, + { title: 'Test Config Value' } + ]); + }); + }); + + describe('v6BulkCreate', () => { + it('handles default options', () => { + const objects = v6BulkCreate(testObjects); + expect(objects).to.eql([ + { create: { _type: 'doc', _id: 'index-pattern:one' } }, + { type: 'index-pattern', 'index-pattern': { title: 'Test Index Pattern' } }, + { create: { _type: 'doc', _id: 'config:two' } }, + { type: 'config', config: { title: 'Test Config Value' } } + ]); + }); + + it('uses index action for options.overwrite=true', () => { + const objects = v6BulkCreate(testObjects, { overwrite: true }); + expect(objects).to.eql([ + { index: { _type: 'doc', _id: 'index-pattern:one' } }, + { type: 'index-pattern', 'index-pattern': { title: 'Test Index Pattern' } }, + { index: { _type: 'doc', _id: 'config:two' } }, + { type: 'config', config: { title: 'Test Config Value' } } + ]); + }); + }); +}); diff --git a/src/server/saved_objects/client/lib/__tests__/create_find_query.js b/src/server/saved_objects/client/lib/__tests__/create_find_query.js index b97b6b3c3c8a7..60c78a59b446a 100644 --- a/src/server/saved_objects/client/lib/__tests__/create_find_query.js +++ b/src/server/saved_objects/client/lib/__tests__/create_find_query.js @@ -15,8 +15,20 @@ describe('createFindQuery', () => { query: { bool: { filter: [{ - term: { - _type: 'index-pattern' + bool: { + should: [ + { + term: { + _type: 'index-pattern' + } + }, + { + term: { + type: 'index-pattern' + } + } + ] + } }], must: [{ diff --git a/src/server/saved_objects/client/lib/__tests__/create_id_query.js b/src/server/saved_objects/client/lib/__tests__/create_id_query.js new file mode 100644 index 0000000000000..735e17743baeb --- /dev/null +++ b/src/server/saved_objects/client/lib/__tests__/create_id_query.js @@ -0,0 +1,48 @@ +import expect from 'expect.js'; +import { createIdQuery } from '../create_id_query'; + +describe('createIdQuery', () => { + it('takes an id and type', () => { + const query = createIdQuery({ id: 'foo', type: 'bar' }); + + const expectedQuery = { + version: true, + size: 1, + query: { + bool: { + should: [ + // v5 document + { + bool: { + must: [ + { term: { _id: 'foo' } }, + { term: { _type: 'bar' } } + ] + } + }, + // migrated v5 document + { + bool: { + must: [ + { term: { _id: 'bar:foo' } }, + { term: { type: 'bar' } } + ] + } + }, + // v6 document + { + bool: { + must: [ + { term: { _id: 'foo' } }, + { term: { type: 'bar' } } + ] + } + }, + ] + } + } + }; + + expect(query).to.eql(expectedQuery); + }); +}); diff --git a/src/server/saved_objects/client/lib/__tests__/included_fields.js b/src/server/saved_objects/client/lib/__tests__/included_fields.js new file mode 100644 index 0000000000000..f506cbfd48c4b --- /dev/null +++ b/src/server/saved_objects/client/lib/__tests__/included_fields.js @@ -0,0 +1,44 @@ +import expect from 'expect.js'; +import { includedFields } from '../included_fields'; + +describe('includedFields', () => { + it('returns undefined if fields are not provided', () => { + expect(includedFields()).to.be(undefined); + }); + + it('includes type', () => { + const fields = includedFields('config', 'foo'); + expect(fields).to.have.length(3); + expect(fields).to.contain('type'); + }); + + it('accepts field as string', () => { + const fields = includedFields('config', 'foo'); + expect(fields).to.have.length(3); + expect(fields).to.contain('config.foo'); + }); + + it('accepts fields as an array', () => { + const fields = includedFields('config', ['foo', 'bar']); + + expect(fields).to.have.length(5); + expect(fields).to.contain('config.foo'); + expect(fields).to.contain('config.bar'); + }); + + it('uses wildcard when type is not provided', () => { + const fields = includedFields(undefined, 'foo'); + expect(fields).to.have.length(3); + expect(fields).to.contain('*.foo'); + }); + + describe('v5 compatibility', () => { + it('includes legacy field path', () => { + const fields = includedFields('config', ['foo', 'bar']); + + expect(fields).to.have.length(5); + expect(fields).to.contain('foo'); + expect(fields).to.contain('bar'); + }); + }); +}); diff --git a/src/server/saved_objects/client/lib/__tests__/normalize_es_doc.js b/src/server/saved_objects/client/lib/__tests__/normalize_es_doc.js new file mode 100644 index 0000000000000..51b828d2fd862 --- /dev/null +++ b/src/server/saved_objects/client/lib/__tests__/normalize_es_doc.js @@ -0,0 +1,105 @@ +import expect from 'expect.js'; +import { normalizeEsDoc } from '../normalize_es_doc'; + +describe('normalizeEsDoc', () => { + it('handle legacy doc types', () => { + const doc = { + _id: 'foo', + _type: 'test', + _version: 2, + _source: { title: 'test' } + }; + + expect(normalizeEsDoc(doc)).to.eql({ + id: 'foo', + type: 'test', + version: 2, + attributes: { title: 'test' } + }); + }); + + it('handle migrated single doc type', () => { + const doc = { + _id: 'test:foo', + _type: 'doc', + _version: 2, + _source: { type: 'test', test: { title: 'test' } } + }; + + expect(normalizeEsDoc(doc)).to.eql({ + id: 'foo', + type: 'test', + version: 2, + attributes: { title: 'test' } + }); + }); + + it('handles an overwritten type', () => { + const doc = { + _type: 'doc', + _id: 'test:foo', + _version: 2, + _source: { type: 'test', test: { title: 'test' } } + }; + const overrides = { type: 'test' }; + + expect(normalizeEsDoc(doc, overrides)).to.eql({ + id: 'foo', + type: 'test', + version: 2, + attributes: { title: 'test' } + }); + }); + + it('can add additional keys', () => { + const doc = { + _type: 'doc', + _id: 'test:foo', + _version: 2, + _source: { type: 'test', test: { title: 'test' } } + }; + const overrides = { error: 'An error!' }; + + expect(normalizeEsDoc(doc, overrides)).to.eql({ + id: 'foo', + type: 'test', + version: 2, + attributes: { title: 'test' }, + error: 'An error!' + }); + }); + + it('handles already prefixed ids with the type', () => { + const doc = { + _type: 'doc', + _id: 'test:test:foo', + _version: 2, + _source: { type: 'test', test: { title: 'test' } } + }; + const overrides = { error: 'An error!' }; + + expect(normalizeEsDoc(doc, overrides)).to.eql({ + id: 'test:foo', + type: 'test', + version: 2, + attributes: { title: 'test' }, + error: 'An error!' + }); + }); + + it('handles legacy doc having an attribute the same as type', () => { + const doc = { + _id: 'foo', + _type: 'test', + _version: 2, + _source: { test: 'test' } + }; + + expect(normalizeEsDoc(doc)).to.eql({ + id: 'foo', + type: 'test', + version: 2, + attributes: { test: 'test' } + }); + }); +}); diff --git a/src/server/saved_objects/client/lib/compatibility.js b/src/server/saved_objects/client/lib/compatibility.js new file mode 100644 index 0000000000000..3d78bf19c2105 --- /dev/null +++ b/src/server/saved_objects/client/lib/compatibility.js @@ -0,0 +1,42 @@ +import { V6_TYPE } from '../saved_objects_client'; + +/** + * @param {array} objects - [{ type, id, attributes }] + * @param {object} [options={}] + * @property {boolean} [options.overwrite=false] - overrides existing documents + * @returns {array} + */ +export function v5BulkCreate(objects, options = {}) { + return objects.reduce((acc, object) => { + const method = object.id && !options.overwrite ? 'create' : 'index'; + + acc.push({ [method]: { _type: object.type, _id: object.id } }); + acc.push(object.attributes); + + return acc; + }, []); +} + +/** + * @param {array} objects - [{ type, id, attributes }] + * @param {object} [options={}] + * @property {boolean} [options.overwrite=false] - overrides existing documents + * @returns {array} + */ +export function v6BulkCreate(objects, options = {}) { + return objects.reduce((acc, object) => { + const method = object.id && !options.overwrite ? 'create' : 'index'; + + acc.push({ [method]: { + _type: V6_TYPE, + _id: object.id ? `${object.type}:${object.id}` : undefined + } }); + + acc.push(Object.assign({}, + { type: object.type }, + { [object.type]: object.attributes } + )); + + return acc; + }, []); +} diff --git a/src/server/saved_objects/client/lib/create_find_query.js b/src/server/saved_objects/client/lib/create_find_query.js index c088323c6a0f7..8cb6db72216ec 100644 --- a/src/server/saved_objects/client/lib/create_find_query.js +++ b/src/server/saved_objects/client/lib/create_find_query.js @@ -14,8 +14,19 @@ export function createFindQuery(mappings, options = {}) { if (type) { bool.filter.push({ - term: { - _type: type + bool: { + should: [ + { + term: { + _type: type + } + }, + { + term: { + type + } + } + ] } }); } diff --git a/src/server/saved_objects/client/lib/create_id_query.js b/src/server/saved_objects/client/lib/create_id_query.js new file mode 100644 index 0000000000000..359adb39ccb64 --- /dev/null +++ b/src/server/saved_objects/client/lib/create_id_query.js @@ -0,0 +1,45 @@ +/** + * Finds a document by either its v5 or v6 format + * + * @param type The documents type + * @param id The documents id or legacy id +**/ +export function createIdQuery({ type, id }) { + return { + version: true, + size: 1, + query: { + bool: { + should: [ + // v5 document + { + bool: { + must: [ + { term: { _id: id } }, + { term: { _type: type } } + ] + } + }, + // migrated v5 document + { + bool: { + must: [ + { term: { _id: `${type}:${id}` } }, + { term: { type: type } } + ] + } + }, + // v6 document + { + bool: { + must: [ + { term: { _id: id } }, + { term: { type: type } } + ] + } + }, + ] + } + } + }; +} diff --git a/src/server/saved_objects/client/lib/handle_es_error.js b/src/server/saved_objects/client/lib/handle_es_error.js index a6a1b695a79db..725d954a1ac61 100644 --- a/src/server/saved_objects/client/lib/handle_es_error.js +++ b/src/server/saved_objects/client/lib/handle_es_error.js @@ -13,12 +13,20 @@ const { BadRequest } = elasticsearch.errors; +export function isSingleTypeError(error) { + if (!error) return; + + return error.type === 'illegal_argument_exception' && + error.reason.match(/the final mapping would have more than 1 type/); +} + export function handleEsError(error) { if (!(error instanceof Error)) { throw new Error('Expected an instance of Error'); } - const reason = get(error, 'body.error.reason'); + const { reason, type } = get(error, 'body.error', {}); + const details = { type }; if ( error instanceof ConnectionFault || @@ -30,19 +38,23 @@ export function handleEsError(error) { } if (error instanceof Conflict) { - throw Boom.conflict(reason); + throw Boom.conflict(reason, details); } if (error instanceof Forbidden) { - throw Boom.forbidden(reason); + throw Boom.forbidden(reason, details); } if (error instanceof NotFound) { - throw Boom.notFound(reason); + throw Boom.notFound(reason, details); } if (error instanceof BadRequest) { - throw Boom.badRequest(reason); + if (isSingleTypeError(get(error, 'body.error'))) { + details.type = 'is_single_type'; + } + + throw Boom.badRequest(reason, details); } throw error; diff --git a/src/server/saved_objects/client/lib/included_fields.js b/src/server/saved_objects/client/lib/included_fields.js new file mode 100644 index 0000000000000..fe0f591b7b94b --- /dev/null +++ b/src/server/saved_objects/client/lib/included_fields.js @@ -0,0 +1,18 @@ +/** + * Provides an array of paths for ES source filtering + * + * @param {string} type + * @param {string|array} fields + * @returns {array} + */ +export function includedFields(type, fields) { + if (!fields || fields.length === 0) return; + + // convert to an array + const sourceFields = typeof fields === 'string' ? [fields] : fields; + const sourceType = type || '*'; + + return sourceFields.map(f => `${sourceType}.${f}`) + .concat('type') + .concat(fields); // v5 compatibility +} diff --git a/src/server/saved_objects/client/lib/index.js b/src/server/saved_objects/client/lib/index.js index b0e3483b16b31..f5973cdc01428 100644 --- a/src/server/saved_objects/client/lib/index.js +++ b/src/server/saved_objects/client/lib/index.js @@ -1,2 +1,6 @@ export { createFindQuery } from './create_find_query'; -export { handleEsError } from './handle_es_error'; +export { createIdQuery } from './create_id_query'; +export { handleEsError, isSingleTypeError } from './handle_es_error'; +export { v5BulkCreate, v6BulkCreate } from './compatibility'; +export { normalizeEsDoc } from './normalize_es_doc'; +export { includedFields } from './included_fields'; diff --git a/src/server/saved_objects/client/lib/normalize_es_doc.js b/src/server/saved_objects/client/lib/normalize_es_doc.js new file mode 100644 index 0000000000000..061efece25158 --- /dev/null +++ b/src/server/saved_objects/client/lib/normalize_es_doc.js @@ -0,0 +1,29 @@ +import { get } from 'lodash'; +import { V6_TYPE } from '../saved_objects_client'; + +export function normalizeEsDoc(doc, overrides = {}) { + if (!doc) return {}; + + let type; + let id = doc._id; + let attributes; + + if (doc._type === V6_TYPE) { + type = overrides.type || get(doc, '_source.type'); + attributes = get(doc, `_source.${type}`); + + // migrated v5 indices and objects created with a specified ID + // have the type prefixed to the id. + id = doc._id.replace(`${type}:`, ''); + } else { + type = overrides.type || doc._type; + attributes = doc._source; + } + + return Object.assign({}, { + id, + type, + version: doc._version, + attributes + }, overrides); +} diff --git a/src/server/saved_objects/client/saved_objects_client.js b/src/server/saved_objects/client/saved_objects_client.js index dd80381ef7437..578c3281dbdc0 100644 --- a/src/server/saved_objects/client/saved_objects_client.js +++ b/src/server/saved_objects/client/saved_objects_client.js @@ -3,9 +3,17 @@ import { get } from 'lodash'; import { createFindQuery, + createIdQuery, handleEsError, + isSingleTypeError, + v5BulkCreate, + v6BulkCreate, + normalizeEsDoc, + includedFields } from './lib'; +export const V6_TYPE = 'doc'; + export class SavedObjectsClient { constructor(kibanaIndex, mappings, callAdminCluster) { this._kibanaIndex = kibanaIndex; @@ -25,19 +33,21 @@ export class SavedObjectsClient { */ async create(type, attributes = {}, options = {}) { const method = options.id && !options.overwrite ? 'create' : 'index'; - const response = await this._withKibanaIndex(method, { + const response = await this._withKibanaIndexAndMappingFallback(method, { type, id: options.id, body: attributes, refresh: 'wait_for' + }, { + type: V6_TYPE, + id: options.id ? `${type}:${options.id}` : undefined, + body: { + type, + [type]: attributes + } }); - return { - id: response._id, - type: response._type, - version: response._version, - attributes - }; + return normalizeEsDoc(response, { type, attributes }); } /** @@ -45,31 +55,42 @@ export class SavedObjectsClient { * * @param {array} objects - [{ type, id, attributes }] * @param {object} [options={}] - * @property {boolean} [options.overwrite=false] - overrides existing documents + * @property {boolean} [options.force=false] - overrides existing documents + * @property {string} [options.format=v5] * @returns {promise} - [{ id, type, version, attributes, error: { message } }] */ async bulkCreate(objects, options = {}) { - const body = objects.reduce((acc, object) => { - const method = get(options, 'overwrite', false) === false && object.id ? 'create' : 'index'; + const { format = 'v5' } = options; - acc.push({ [method]: { _type: object.type, _id: object.id } }); - acc.push(object.attributes); + const bulkCreate = format === 'v5' ? v5BulkCreate : v6BulkCreate; + const response = await this._withKibanaIndex('bulk', { + body: bulkCreate(objects, options), + refresh: 'wait_for' + }); - return acc; - }, []); + const items = get(response, 'items', []); + const missingTypesCount = items.filter(item => { + const method = Object.keys(item)[0]; + return isSingleTypeError(get(item, `${method}.error`)); + }).length; + + const formatFallback = format === 'v5' && items.length > 0 && items.length === missingTypesCount; - return await this._withKibanaIndex('bulk', { body, refresh: 'wait_for' }) - .then(resp => get(resp, 'items', []).map((resp, i) => { - const method = Object.keys(resp)[0]; - - return { - id: resp[method]._id, - type: resp[method]._type, - version: resp[method]._version, - attributes: objects[i].attributes, - error: resp[method].error ? { message: get(resp[method], 'error.reason') } : undefined - }; - })); + if (formatFallback) { + return this.bulkCreate(objects, Object.assign({}, options, { format: 'v6' })); + } + + return get(response, 'items', []).map((resp, i) => { + const method = Object.keys(resp)[0]; + const { id, type, attributes } = objects[i]; + + return normalizeEsDoc(resp[method], { + id, + type, + attributes, + error: resp[method].error ? { message: get(resp[method], 'error.reason') } : undefined + }); + }); } /** @@ -80,13 +101,12 @@ export class SavedObjectsClient { * @returns {promise} */ async delete(type, id) { - const response = await this._withKibanaIndex('delete', { - type, - id, + const response = await this._withKibanaIndex('deleteByQuery', { + body: createIdQuery({ type, id }), refresh: 'wait_for' }); - if (get(response, 'found') === false) { + if (get(response, 'deleted') === 0) { throw Boom.notFound(); } } @@ -99,8 +119,9 @@ export class SavedObjectsClient { * Query field argument for more information * @property {integer} [options.page=1] * @property {integer} [options.perPage=20] - * @property {array} options.sort - * @property {array} options.fields + * @property {string} options.sortField + * @property {string} options.sortOrder + * @property {array|string} options.fields * @returns {promise} - { saved_objects: [{ id, type, version, attributes }], total, per_page, page } */ async find(options = {}) { @@ -116,8 +137,7 @@ export class SavedObjectsClient { } = options; const esOptions = { - type, - _source: fields, + _source: includedFields(type, fields), size: perPage, from: perPage * (page - 1), body: createFindQuery(this._mappings, { search, searchFields, type, sortField, sortOrder }) @@ -126,13 +146,8 @@ export class SavedObjectsClient { const response = await this._withKibanaIndex('search', esOptions); return { - saved_objects: get(response, 'hits.hits', []).map(r => { - return { - id: r._id, - type: r._type, - version: r._version, - attributes: r._source - }; + saved_objects: get(response, 'hits.hits', []).map(hit => { + return normalizeEsDoc(hit); }), total: get(response, 'hits.total', 0), per_page: perPage, @@ -141,27 +156,6 @@ export class SavedObjectsClient { }; } - /** - * Gets a single object - * - * @param {string} type - * @param {string} id - * @returns {promise} - { id, type, version, attributes } - */ - async get(type, id) { - const response = await this._withKibanaIndex('get', { - type, - id, - }); - - return { - id: response._id, - type: response._type, - version: response._version, - attributes: response._source - }; - } - /** * Returns an array of objects by id * @@ -179,25 +173,46 @@ export class SavedObjectsClient { return { saved_objects: [] }; } - const docs = objects.map(doc => { - return { _type: get(doc, 'type'), _id: get(doc, 'id') }; - }); + const docs = objects.reduce((acc, { type, id }) => { + return [...acc, {}, createIdQuery({ type, id })]; + }, []); - const response = await this._withKibanaIndex('mget', { body: { docs } }) - .then(resp => get(resp, 'docs', []).filter(resp => resp.found)); + const response = await this._withKibanaIndex('msearch', { body: docs }); + const responses = get(response, 'responses', []); return { - saved_objects: response.map(r => { - return { - id: r._id, - type: r._type, - version: r._version, - attributes: r._source - }; + saved_objects: responses.map((r, i) => { + const [hit] = get(r, 'hits.hits', []); + + if (!hit) { + return Object.assign({}, objects[i], { + error: { statusCode: 404, message: 'Not found' } + }); + } + + return normalizeEsDoc(hit, objects[i]); }) }; } + /** + * Gets a single object + * + * @param {string} type + * @param {string} id + * @returns {promise} - { id, type, version, attributes } + */ + async get(type, id) { + const response = await this._withKibanaIndex('search', { body: createIdQuery({ type, id }) }); + const [hit] = get(response, 'hits.hits', []); + + if (!hit) { + throw Boom.notFound(); + } + + return normalizeEsDoc(hit); + } + /** * Updates an object * @@ -208,22 +223,40 @@ export class SavedObjectsClient { * @returns {promise} */ async update(type, id, attributes, options = {}) { - const response = await this._withKibanaIndex('update', { - type, + const response = await this._withKibanaIndexAndMappingFallback('update', { id, + type, version: options.version, + refresh: 'wait_for', body: { doc: attributes - }, - refresh: 'wait_for' + } + }, { + type: V6_TYPE, + body: { + doc: { + [type]: attributes + } + } }); - return { - id: id, - type: type, - version: get(response, '_version'), - attributes: attributes + return normalizeEsDoc(response, { id, type, attributes }); + } + + _withKibanaIndexAndMappingFallback(method, params, fallbackParams) { + const fallbacks = { + 'create': ['is_single_type'], + 'index': ['is_single_type'], + 'update': ['document_missing_exception'] }; + + return this._withKibanaIndex(method, params).catch(err => { + if (get(fallbacks, method, []).includes(get(err, 'data.type'))) { + return this._withKibanaIndex(method, Object.assign({}, params, fallbackParams)); + } + + throw err; + }); } async _withKibanaIndex(method, params) {