From c466be55b8a8b58a0c1b32bfdb2e35cd865006c9 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 22 Jun 2023 13:42:26 -0700 Subject: [PATCH 1/4] Merge in kibana backend from osints/dev Signed-off-by: Simeon Widdis --- .../integrations/__test__/builder.test.ts | 326 +++++++++++++++ .../__test__/kibana_backend.test.ts | 384 ++++++++++++++++++ .../integrations/integrations_builder.ts | 102 +++++ .../integrations_kibana_backend.ts | 186 +++++++++ server/adaptors/integrations/types.ts | 6 +- server/adaptors/integrations/validators.ts | 11 +- .../__tests__/integrations_router.test.ts | 27 +- .../integrations/integrations_router.ts | 4 +- 8 files changed, 1012 insertions(+), 34 deletions(-) create mode 100644 server/adaptors/integrations/__test__/builder.test.ts create mode 100644 server/adaptors/integrations/__test__/kibana_backend.test.ts create mode 100644 server/adaptors/integrations/integrations_builder.ts create mode 100644 server/adaptors/integrations/integrations_kibana_backend.ts diff --git a/server/adaptors/integrations/__test__/builder.test.ts b/server/adaptors/integrations/__test__/builder.test.ts new file mode 100644 index 0000000000..81af1d6f69 --- /dev/null +++ b/server/adaptors/integrations/__test__/builder.test.ts @@ -0,0 +1,326 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from '../../../../../../src/core/server'; +import { IntegrationInstanceBuilder } from '../integrations_builder'; +import { Integration } from '../repository/integration'; + +const mockSavedObjectsClient: SavedObjectsClientContract = ({ + bulkCreate: jest.fn(), + create: jest.fn(), + delete: jest.fn(), + find: jest.fn(), + get: jest.fn(), + update: jest.fn(), +} as unknown) as SavedObjectsClientContract; + +const sampleIntegration: Integration = ({ + deepCheck: jest.fn().mockResolvedValue(true), + getAssets: jest.fn().mockResolvedValue({ + savedObjects: [ + { + id: 'asset1', + references: [{ id: 'ref1' }], + }, + { + id: 'asset2', + references: [{ id: 'ref2' }], + }, + ], + }), + getConfig: jest.fn().mockResolvedValue({ + name: 'integration-template', + type: 'integration-type', + }), +} as unknown) as Integration; + +describe('IntegrationInstanceBuilder', () => { + let builder: IntegrationInstanceBuilder; + + beforeEach(() => { + builder = new IntegrationInstanceBuilder(mockSavedObjectsClient); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('build', () => { + it('should build an integration instance', async () => { + const options = { + dataSource: 'instance-datasource', + name: 'instance-name', + }; + + const remappedAssets = [ + { + id: 'remapped-asset1', + references: [{ id: 'remapped-ref1' }], + }, + { + id: 'remapped-asset2', + references: [{ id: 'remapped-ref2' }], + }, + ]; + const postAssetsResponse = { + saved_objects: [ + { id: 'created-asset1', type: 'dashboard', attributes: { title: 'Dashboard 1' } }, + { id: 'created-asset2', type: 'visualization', attributes: { title: 'Visualization 1' } }, + ], + }; + const expectedInstance = { + name: 'instance-name', + templateName: 'integration-template', + dataSource: 'instance-datasource', + creationDate: expect.any(String), + assets: [ + { + assetType: 'dashboard', + assetId: 'created-asset1', + status: 'available', + isDefaultAsset: true, + description: 'Dashboard 1', + }, + { + assetType: 'visualization', + assetId: 'created-asset2', + status: 'available', + isDefaultAsset: false, + description: 'Visualization 1', + }, + ], + }; + + // Mock the implementation of the methods in the Integration class + sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); + sampleIntegration.getAssets = jest.fn().mockResolvedValue({ savedObjects: remappedAssets }); + sampleIntegration.getConfig = jest.fn().mockResolvedValue({ + name: 'integration-template', + type: 'integration-type', + }); + + // Mock builder sub-methods + const remapIDsSpy = jest.spyOn(builder, 'remapIDs'); + const postAssetsSpy = jest.spyOn(builder, 'postAssets'); + + (mockSavedObjectsClient.bulkCreate as jest.Mock).mockResolvedValue(postAssetsResponse); + + const instance = await builder.build(sampleIntegration, options); + + expect(sampleIntegration.deepCheck).toHaveBeenCalled(); + expect(sampleIntegration.getAssets).toHaveBeenCalled(); + expect(remapIDsSpy).toHaveBeenCalledWith(remappedAssets); + expect(postAssetsSpy).toHaveBeenCalledWith(remappedAssets); + expect(instance).toEqual(expectedInstance); + }); + + it('should reject with an error if integration is not valid', async () => { + const options = { + dataSource: 'instance-datasource', + name: 'instance-name', + }; + sampleIntegration.deepCheck = jest.fn().mockResolvedValue(false); + + await expect(builder.build(sampleIntegration, options)).rejects.toThrowError( + 'Integration is not valid' + ); + }); + + it('should reject with an error if getAssets throws an error', async () => { + const options = { + dataSource: 'instance-datasource', + name: 'instance-name', + }; + + const errorMessage = 'Failed to get assets'; + sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); + sampleIntegration.getAssets = jest.fn().mockRejectedValue(new Error(errorMessage)); + + await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(errorMessage); + }); + + it('should reject with an error if postAssets throws an error', async () => { + const options = { + dataSource: 'instance-datasource', + name: 'instance-name', + }; + const remappedAssets = [ + { + id: 'remapped-asset1', + references: [{ id: 'remapped-ref1' }], + }, + ]; + const errorMessage = 'Failed to post assets'; + sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); + sampleIntegration.getAssets = jest.fn().mockResolvedValue({ savedObjects: remappedAssets }); + builder.postAssets = jest.fn().mockRejectedValue(new Error(errorMessage)); + + await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(errorMessage); + }); + + it('should reject with an error if getConfig returns null', async () => { + const options = { + dataSource: 'instance-datasource', + name: 'instance-name', + }; + sampleIntegration.getConfig = jest.fn().mockResolvedValue(null); + + await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(); + }); + }); + + describe('remapIDs', () => { + it('should remap IDs and references in assets', () => { + const assets = [ + { + id: 'asset1', + references: [{ id: 'ref1' }, { id: 'ref2' }], + }, + { + id: 'asset2', + references: [{ id: 'ref1' }, { id: 'ref3' }], + }, + ]; + const expectedRemappedAssets = [ + { + id: expect.any(String), + references: [{ id: expect.any(String) }, { id: expect.any(String) }], + }, + { + id: expect.any(String), + references: [{ id: expect.any(String) }, { id: expect.any(String) }], + }, + ]; + + const remappedAssets = builder.remapIDs(assets); + + expect(remappedAssets).toEqual(expectedRemappedAssets); + }); + }); + + describe('postAssets', () => { + it('should post assets and return asset references', async () => { + const assets = [ + { + id: 'asset1', + type: 'dashboard', + attributes: { title: 'Dashboard 1' }, + }, + { + id: 'asset2', + type: 'visualization', + attributes: { title: 'Visualization 1' }, + }, + ]; + const expectedRefs = [ + { + assetType: 'dashboard', + assetId: 'created-asset1', + status: 'available', + isDefaultAsset: true, + description: 'Dashboard 1', + }, + { + assetType: 'visualization', + assetId: 'created-asset2', + status: 'available', + isDefaultAsset: false, + description: 'Visualization 1', + }, + ]; + const bulkCreateResponse = { + saved_objects: [ + { id: 'created-asset1', type: 'dashboard', attributes: { title: 'Dashboard 1' } }, + { id: 'created-asset2', type: 'visualization', attributes: { title: 'Visualization 1' } }, + ], + }; + + (mockSavedObjectsClient.bulkCreate as jest.Mock).mockResolvedValue(bulkCreateResponse); + + const refs = await builder.postAssets(assets); + + expect(mockSavedObjectsClient.bulkCreate).toHaveBeenCalledWith(assets); + expect(refs).toEqual(expectedRefs); + }); + + it('should reject with an error if bulkCreate throws an error', async () => { + const assets = [ + { + id: 'asset1', + type: 'dashboard', + attributes: { title: 'Dashboard 1' }, + }, + ]; + const errorMessage = 'Failed to create assets'; + (mockSavedObjectsClient.bulkCreate as jest.Mock).mockRejectedValue(new Error(errorMessage)); + + await expect(builder.postAssets(assets)).rejects.toThrowError(errorMessage); + }); + }); + + describe('buildInstance', () => { + it('should build an integration instance', async () => { + const integration = { + getConfig: jest.fn().mockResolvedValue({ + name: 'integration-template', + type: 'integration-type', + }), + }; + const refs = [ + { + assetType: 'dashboard', + assetId: 'created-asset1', + status: 'available', + isDefaultAsset: true, + description: 'Dashboard 1', + }, + ]; + const options = { + dataSource: 'instance-datasource', + name: 'instance-name', + }; + const expectedInstance = { + name: 'instance-name', + templateName: 'integration-template', + dataSource: 'instance-datasource', + tags: undefined, + creationDate: expect.any(String), + assets: refs, + }; + + const instance = await builder.buildInstance( + (integration as unknown) as Integration, + refs, + options + ); + + expect(integration.getConfig).toHaveBeenCalled(); + expect(instance).toEqual(expectedInstance); + }); + + it('should reject with an error if getConfig returns null', async () => { + const integration = { + getConfig: jest.fn().mockResolvedValue(null), + }; + const refs = [ + { + assetType: 'dashboard', + assetId: 'created-asset1', + status: 'available', + isDefaultAsset: true, + description: 'Dashboard 1', + }, + ]; + const options = { + dataSource: 'instance-datasource', + name: 'instance-name', + }; + + await expect( + builder.buildInstance((integration as unknown) as Integration, refs, options) + ).rejects.toThrowError(); + }); + }); +}); diff --git a/server/adaptors/integrations/__test__/kibana_backend.test.ts b/server/adaptors/integrations/__test__/kibana_backend.test.ts new file mode 100644 index 0000000000..63d62764ce --- /dev/null +++ b/server/adaptors/integrations/__test__/kibana_backend.test.ts @@ -0,0 +1,384 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IntegrationsKibanaBackend } from '../integrations_kibana_backend'; +import { SavedObject, SavedObjectsClientContract } from '../../../../../../src/core/server/types'; +import { Repository } from '../repository/repository'; +import { IntegrationInstanceBuilder } from '../integrations_builder'; +import { Integration } from '../repository/integration'; +import { SavedObjectsFindResponse } from '../../../../../../src/core/server'; + +describe('IntegrationsKibanaBackend', () => { + let mockSavedObjectsClient: jest.Mocked; + let mockRepository: jest.Mocked; + let backend: IntegrationsKibanaBackend; + + beforeEach(() => { + mockSavedObjectsClient = { + get: jest.fn(), + find: jest.fn(), + create: jest.fn(), + delete: jest.fn(), + } as any; + mockRepository = { + getIntegration: jest.fn(), + getIntegrationList: jest.fn(), + } as any; + backend = new IntegrationsKibanaBackend(mockSavedObjectsClient, mockRepository); + }); + + describe('deleteIntegrationInstance', () => { + it('should delete the integration instance and associated assets', async () => { + const instanceId = 'instance-id'; + const asset1Id = 'asset1-id'; + const asset2Id = 'asset2-id'; + + const instanceData = { + attributes: { + assets: [ + { assetId: asset1Id, assetType: 'asset-type-1' }, + { assetId: asset2Id, assetType: 'asset-type-2' }, + ], + }, + }; + + mockSavedObjectsClient.get.mockResolvedValue(instanceData as SavedObject); + mockSavedObjectsClient.delete.mockResolvedValueOnce({}); + mockSavedObjectsClient.delete.mockResolvedValueOnce({}); + mockSavedObjectsClient.delete.mockResolvedValueOnce({}); + + const result = await backend.deleteIntegrationInstance(instanceId); + + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('integration-instance', instanceId); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith('asset-type-1', asset1Id); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith('asset-type-2', asset2Id); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith( + 'integration-instance', + instanceId + ); + expect(result).toEqual([asset1Id, asset2Id, instanceId]); + }); + + it('should handle a 404 error when getting the integration instance', async () => { + const instanceId = 'instance-id'; + + mockSavedObjectsClient.get.mockRejectedValue({ output: { statusCode: 404 } }); + + const result = await backend.deleteIntegrationInstance(instanceId); + + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('integration-instance', instanceId); + expect(result).toEqual([instanceId]); + }); + + it('should handle a non-404 error when getting the integration instance', async () => { + const instanceId = 'instance-id'; + const error = new Error('Internal Server Error'); + + mockSavedObjectsClient.get.mockRejectedValue(error); + + await expect(backend.deleteIntegrationInstance(instanceId)).rejects.toThrow(error); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('integration-instance', instanceId); + }); + + it('should handle a 404 error when deleting assets', async () => { + const instanceId = 'instance-id'; + const asset1Id = 'asset1-id'; + const asset2Id = 'asset2-id'; + + const instanceData = { + attributes: { + assets: [ + { assetId: asset1Id, assetType: 'asset-type-1' }, + { assetId: asset2Id, assetType: 'asset-type-2' }, + ], + }, + }; + + mockSavedObjectsClient.get.mockResolvedValue(instanceData as SavedObject); + mockSavedObjectsClient.delete.mockRejectedValueOnce({ output: { statusCode: 404 } }); + mockSavedObjectsClient.delete.mockRejectedValueOnce({ output: { statusCode: 404 } }); + mockSavedObjectsClient.delete.mockRejectedValueOnce({ output: { statusCode: 404 } }); + + const result = await backend.deleteIntegrationInstance(instanceId); + + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('integration-instance', instanceId); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith('asset-type-1', asset1Id); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith('asset-type-2', asset2Id); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith( + 'integration-instance', + instanceId + ); + expect(result).toEqual([asset1Id, asset2Id, instanceId]); + }); + + it('should handle a non-404 error when deleting assets', async () => { + const instanceId = 'instance-id'; + const asset1Id = 'asset1-id'; + const asset2Id = 'asset2-id'; + + const instanceData = { + attributes: { + assets: [ + { assetId: asset1Id, assetType: 'asset-type-1' }, + { assetId: asset2Id, assetType: 'asset-type-2' }, + ], + }, + }; + + const error = new Error('Internal Server Error'); + + mockSavedObjectsClient.get.mockResolvedValue(instanceData as SavedObject); + mockSavedObjectsClient.delete.mockRejectedValueOnce({ output: { statusCode: 404 } }); + mockSavedObjectsClient.delete.mockRejectedValueOnce(error); + + await expect(backend.deleteIntegrationInstance(instanceId)).rejects.toThrow(error); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('integration-instance', instanceId); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith('asset-type-1', asset1Id); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith('asset-type-2', asset2Id); + expect(mockSavedObjectsClient.delete).toHaveBeenCalledWith( + 'integration-instance', + instanceId + ); + }); + }); + + describe('getIntegrationTemplates', () => { + it('should get integration templates by name', async () => { + const query = { name: 'template1' }; + const integration = { getConfig: jest.fn().mockResolvedValue({ name: 'template1' }) }; + mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + + const result = await backend.getIntegrationTemplates(query); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(query.name); + expect(integration.getConfig).toHaveBeenCalled(); + expect(result).toEqual({ hits: [await integration.getConfig()] }); + }); + + it('should get all integration templates', async () => { + const integrationList = [ + { getConfig: jest.fn().mockResolvedValue({ name: 'template1' }) }, + { getConfig: jest.fn().mockResolvedValue(null) }, + { getConfig: jest.fn().mockResolvedValue({ name: 'template2' }) }, + ]; + mockRepository.getIntegrationList.mockResolvedValue( + (integrationList as unknown) as Integration[] + ); + + const result = await backend.getIntegrationTemplates(); + + expect(mockRepository.getIntegrationList).toHaveBeenCalled(); + expect(integrationList[0].getConfig).toHaveBeenCalled(); + expect(integrationList[1].getConfig).toHaveBeenCalled(); + expect(integrationList[2].getConfig).toHaveBeenCalled(); + expect(result).toEqual({ + hits: [await integrationList[0].getConfig(), await integrationList[2].getConfig()], + }); + }); + }); + + describe('getIntegrationInstances', () => { + it('should get all integration instances', async () => { + const savedObjects = [ + { id: 'instance1', attributes: { name: 'instance1' } }, + { id: 'instance2', attributes: { name: 'instance2' } }, + ]; + const findResult = { total: savedObjects.length, saved_objects: savedObjects }; + mockSavedObjectsClient.find.mockResolvedValue( + (findResult as unknown) as SavedObjectsFindResponse + ); + + const result = await backend.getIntegrationInstances(); + + expect(mockSavedObjectsClient.find).toHaveBeenCalledWith({ type: 'integration-instance' }); + expect(result).toEqual({ + total: findResult.total, + hits: savedObjects.map((obj) => ({ id: obj.id, ...obj.attributes })), + }); + }); + }); + + describe('getIntegrationInstance', () => { + it('should get integration instance by ID', async () => { + const instanceId = 'instance1'; + const integrationInstance = { id: instanceId, attributes: { name: 'instance1' } }; + mockSavedObjectsClient.get.mockResolvedValue(integrationInstance as SavedObject); + + const result = await backend.getIntegrationInstance({ id: instanceId }); + + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('integration-instance', instanceId); + expect(result).toEqual({ id: instanceId, status: 'available', name: 'instance1' }); + }); + }); + + describe('loadIntegrationInstance', () => { + it('should load and create an integration instance', async () => { + const templateName = 'template1'; + const name = 'instance1'; + const template = { + getConfig: jest.fn().mockResolvedValue({ name: templateName }), + }; + const instanceBuilder = { + build: jest.fn().mockResolvedValue({ name, dataset: 'nginx', namespace: 'prod' }), + }; + const createdInstance = { name, dataset: 'nginx', namespace: 'prod' }; + mockRepository.getIntegration.mockResolvedValue((template as unknown) as Integration); + mockSavedObjectsClient.create.mockResolvedValue(({ + result: 'created', + } as unknown) as SavedObject); + backend.instanceBuilder = (instanceBuilder as unknown) as IntegrationInstanceBuilder; + + const result = await backend.loadIntegrationInstance(templateName, name, 'datasource'); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(templateName); + expect(instanceBuilder.build).toHaveBeenCalledWith(template, { + name, + dataSource: 'datasource', + }); + expect(mockSavedObjectsClient.create).toHaveBeenCalledWith( + 'integration-instance', + createdInstance + ); + expect(result).toEqual(createdInstance); + }); + + it('should reject with a 404 if template is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue(null); + + await expect( + backend.loadIntegrationInstance(templateName, 'instance1', 'datasource') + ).rejects.toHaveProperty('statusCode', 404); + }); + + it('should reject with an error status if building fails', async () => { + const templateName = 'template1'; + const name = 'instance1'; + const template = { + getConfig: jest.fn().mockResolvedValue({ name: templateName }), + }; + const instanceBuilder = { + build: jest.fn().mockRejectedValue(new Error('Failed to build instance')), + }; + backend.instanceBuilder = (instanceBuilder as unknown) as IntegrationInstanceBuilder; + mockRepository.getIntegration.mockResolvedValue((template as unknown) as Integration); + + await expect( + backend.loadIntegrationInstance(templateName, name, 'datasource') + ).rejects.toHaveProperty('statusCode'); + }); + }); + + describe('getStatic', () => { + it('should get static asset data', async () => { + const templateName = 'template1'; + const staticPath = 'path/to/static'; + const assetData = Buffer.from('asset data'); + const integration = { + getStatic: jest.fn().mockResolvedValue(assetData), + }; + mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + + const result = await backend.getStatic(templateName, staticPath); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(templateName); + expect(integration.getStatic).toHaveBeenCalledWith(staticPath); + expect(result).toEqual(assetData); + }); + + it('should reject with a 404 if asset is not found', async () => { + const templateName = 'template1'; + const staticPath = 'path/to/static'; + mockRepository.getIntegration.mockResolvedValue(null); + + await expect(backend.getStatic(templateName, staticPath)).rejects.toHaveProperty( + 'statusCode', + 404 + ); + }); + }); + + describe('getAssetStatus', () => { + it('should return "available" if all assets are available', async () => { + const assets = [ + { assetId: 'asset1', assetType: 'type1' }, + { assetId: 'asset2', assetType: 'type2' }, + ]; + + const result = await backend.getAssetStatus(assets as AssetReference[]); + + expect(result).toBe('available'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledTimes(2); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type1', 'asset1'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type2', 'asset2'); + }); + + it('should return "unavailable" if every asset is unavailable', async () => { + mockSavedObjectsClient.get = jest + .fn() + .mockRejectedValueOnce({ output: { statusCode: 404 } }) + .mockRejectedValueOnce({ output: { statusCode: 404 } }) + .mockRejectedValueOnce({ output: { statusCode: 404 } }); + + const assets = [ + { assetId: 'asset1', assetType: 'type1' }, + { assetId: 'asset2', assetType: 'type2' }, + { assetId: 'asset3', assetType: 'type3' }, + ]; + + const result = await backend.getAssetStatus(assets as AssetReference[]); + + expect(result).toBe('unavailable'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledTimes(3); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type1', 'asset1'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type2', 'asset2'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type3', 'asset3'); + }); + + it('should return "partially-available" if some assets are available and some are unavailable', async () => { + mockSavedObjectsClient.get = jest + .fn() + .mockResolvedValueOnce({}) // Available + .mockRejectedValueOnce({ output: { statusCode: 404 } }) // Unavailable + .mockResolvedValueOnce({}); // Available + + const assets = [ + { assetId: 'asset1', assetType: 'type1' }, + { assetId: 'asset2', assetType: 'type2' }, + { assetId: 'asset3', assetType: 'type3' }, + ]; + + const result = await backend.getAssetStatus(assets as AssetReference[]); + + expect(result).toBe('partially-available'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledTimes(3); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type1', 'asset1'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type2', 'asset2'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type3', 'asset3'); + }); + + it('should return "unknown" if at least one asset has an unknown status', async () => { + mockSavedObjectsClient.get = jest + .fn() + .mockResolvedValueOnce({}) // Available + .mockRejectedValueOnce({}) // Unknown + .mockResolvedValueOnce({}); // Available + + const assets = [ + { assetId: 'asset1', assetType: 'type1' }, + { assetId: 'asset2', assetType: 'type2' }, + { assetId: 'asset3', assetType: 'type3' }, + ]; + + const result = await backend.getAssetStatus(assets as AssetReference[]); + + expect(result).toBe('unknown'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledTimes(3); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type1', 'asset1'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type2', 'asset2'); + expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('type3', 'asset3'); + }); + }); +}); diff --git a/server/adaptors/integrations/integrations_builder.ts b/server/adaptors/integrations/integrations_builder.ts new file mode 100644 index 0000000000..b12e1a1321 --- /dev/null +++ b/server/adaptors/integrations/integrations_builder.ts @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { v4 as uuidv4 } from 'uuid'; +import { uuidRx } from 'public/components/custom_panels/redux/panel_slice'; +import { SavedObjectsClientContract } from '../../../../../src/core/server'; +import { Integration } from './repository/integration'; +import { SavedObjectsBulkCreateObject } from '../../../../../src/core/public'; + +interface BuilderOptions { + name: string; + dataSource: string; +} + +export class IntegrationInstanceBuilder { + client: SavedObjectsClientContract; + + constructor(client: SavedObjectsClientContract) { + this.client = client; + } + + async build(integration: Integration, options: BuilderOptions): Promise { + const instance = integration + .deepCheck() + .then((result) => { + if (!result) { + return Promise.reject(new Error('Integration is not valid')); + } + }) + .then(() => integration.getAssets()) + .then((assets) => this.remapIDs(assets.savedObjects!)) + .then((assets) => this.remapDataSource(assets, options.dataSource)) + .then((assets) => this.postAssets(assets)) + .then((refs) => this.buildInstance(integration, refs, options)); + return instance; + } + + remapDataSource(assets: any[], dataSource: string | undefined): any[] { + if (!dataSource) return assets; + assets = assets.map((asset) => { + if (asset.type === 'index-pattern') { + asset.attributes.title = dataSource; + } + return asset; + }); + return assets; + } + + remapIDs(assets: any[]): any[] { + const toRemap = assets.filter((asset) => asset.id); + const idMap = new Map(); + return toRemap.map((item) => { + if (!idMap.has(item.id)) { + idMap.set(item.id, uuidv4()); + } + item.id = idMap.get(item.id)!; + for (let ref = 0; ref < item.references.length; ref++) { + const refId = item.references[ref].id; + if (!idMap.has(refId)) { + idMap.set(refId, uuidv4()); + } + item.references[ref].id = idMap.get(refId)!; + } + return item; + }); + } + + async postAssets(assets: any[]): Promise { + try { + const response = await this.client.bulkCreate(assets as SavedObjectsBulkCreateObject[]); + const refs: AssetReference[] = response.saved_objects.map((obj: any) => { + return { + assetType: obj.type, + assetId: obj.id, + status: 'available', // Assuming a successfully created object is available + isDefaultAsset: obj.type === 'dashboard', // Assuming for now that dashboards are default + description: obj.attributes?.title, + }; + }); + return Promise.resolve(refs); + } catch (err: any) { + return Promise.reject(err); + } + } + + async buildInstance( + integration: Integration, + refs: AssetReference[], + options: BuilderOptions + ): Promise { + const config: IntegrationTemplate = (await integration.getConfig())!; + return Promise.resolve({ + name: options.name, + templateName: config.name, + dataSource: options.dataSource, + creationDate: new Date().toISOString(), + assets: refs, + }); + } +} diff --git a/server/adaptors/integrations/integrations_kibana_backend.ts b/server/adaptors/integrations/integrations_kibana_backend.ts new file mode 100644 index 0000000000..c7da2c49fb --- /dev/null +++ b/server/adaptors/integrations/integrations_kibana_backend.ts @@ -0,0 +1,186 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'path'; +import { IntegrationsAdaptor } from './integrations_adaptor'; +import { SavedObject, SavedObjectsClientContract } from '../../../../../src/core/server/types'; +import { IntegrationInstanceBuilder } from './integrations_builder'; +import { Repository } from './repository/repository'; + +export class IntegrationsKibanaBackend implements IntegrationsAdaptor { + client: SavedObjectsClientContract; + instanceBuilder: IntegrationInstanceBuilder; + repository: Repository; + + constructor(client: SavedObjectsClientContract, repository?: Repository) { + this.client = client; + this.repository = repository ?? new Repository(path.join(__dirname, '__data__/repository')); + this.instanceBuilder = new IntegrationInstanceBuilder(this.client); + } + + deleteIntegrationInstance = async (id: string): Promise => { + let children: any; + try { + children = await this.client.get('integration-instance', id); + } catch (err: any) { + return err.output?.statusCode === 404 ? Promise.resolve([id]) : Promise.reject(err); + } + + const toDelete = children.attributes.assets + .filter((i: any) => i.assetId) + .map((i: any) => { + return { id: i.assetId, type: i.assetType }; + }); + toDelete.push({ id, type: 'integration-instance' }); + + const result = Promise.all( + toDelete.map( + async (asset: { type: string; id: string }): Promise => { + try { + await this.client.delete(asset.type, asset.id); + return Promise.resolve(asset.id); + } catch (err: any) { + return err.output?.statusCode === 404 ? Promise.resolve(asset.id) : Promise.reject(err); + } + } + ) + ); + return result; + }; + + getIntegrationTemplates = async ( + query?: IntegrationTemplateQuery + ): Promise => { + if (query?.name) { + const integration = await this.repository.getIntegration(query.name); + const config = await integration?.getConfig(); + return Promise.resolve({ hits: config ? [config] : [] }); + } + const integrationList = await this.repository.getIntegrationList(); + const configList = await Promise.all(integrationList.map((x) => x.getConfig())); + return Promise.resolve({ hits: configList.filter((x) => x !== null) as IntegrationTemplate[] }); + }; + + getIntegrationInstances = async ( + _query?: IntegrationInstanceQuery + ): Promise => { + const result = await this.client.find({ type: 'integration-instance' }); + return Promise.resolve({ + total: result.total, + hits: result.saved_objects?.map((x) => ({ + ...x.attributes!, + id: x.id, + })) as IntegrationInstanceResult[], + }); + }; + + getIntegrationInstance = async ( + query?: IntegrationInstanceQuery + ): Promise => { + const result = await this.client.get('integration-instance', `${query!.id}`); + return Promise.resolve(this.buildInstanceResponse(result)); + }; + + buildInstanceResponse = async ( + savedObj: SavedObject + ): Promise => { + const assets: AssetReference[] | undefined = (savedObj.attributes as any)?.assets; + const status: string = assets ? await this.getAssetStatus(assets) : 'available'; + + return { + id: savedObj.id, + status, + ...(savedObj.attributes as any), + }; + }; + + getAssetStatus = async (assets: AssetReference[]): Promise => { + const statuses: Array<{ id: string; status: string }> = await Promise.all( + assets.map(async (asset) => { + try { + await this.client.get(asset.assetType, asset.assetId); + return { id: asset.assetId, status: 'available' }; + } catch (err: any) { + const statusCode = err.output?.statusCode; + if (statusCode && 400 <= statusCode && statusCode < 500) { + return { id: asset.assetId, status: 'unavailable' }; + } + console.error('Failed to get asset status', err); + return { id: asset.assetId, status: 'unknown' }; + } + }) + ); + + const [available, unavailable, unknown] = [ + statuses.filter((x) => x.status === 'available').length, + statuses.filter((x) => x.status === 'unavailable').length, + statuses.filter((x) => x.status === 'unknown').length, + ]; + if (unknown > 0) return 'unknown'; + if (unavailable > 0 && available > 0) return 'partially-available'; + if (unavailable > 0) return 'unavailable'; + return 'available'; + }; + + loadIntegrationInstance = async ( + templateName: string, + name: string, + dataSource: string + ): Promise => { + const template = await this.repository.getIntegration(templateName); + if (template === null) { + return Promise.reject({ + message: `Template ${templateName} not found`, + statusCode: 404, + }); + } + try { + const result = await this.instanceBuilder.build(template, { + name, + dataSource, + }); + await this.client.create('integration-instance', result); + return Promise.resolve(result); + } catch (err: any) { + return Promise.reject({ + message: err.message, + statusCode: 500, + }); + } + }; + + getStatic = async (templateName: string, staticPath: string): Promise => { + const data = await (await this.repository.getIntegration(templateName))?.getStatic(staticPath); + if (!data) { + return Promise.reject({ + message: `Asset ${staticPath} not found`, + statusCode: 404, + }); + } + return Promise.resolve(data); + }; + + getSchemas = async (templateName: string): Promise => { + const integration = await this.repository.getIntegration(templateName); + if (integration === null) { + return Promise.reject({ + message: `Template ${templateName} not found`, + statusCode: 404, + }); + } + return Promise.resolve(integration.getSchemas()); + }; + + getAssets = async (templateName: string): Promise<{ savedObjects?: any }> => { + const integration = await this.repository.getIntegration(templateName); + if (integration === null) { + return Promise.reject({ + message: `Template ${templateName} not found`, + statusCode: 404, + }); + } + return Promise.resolve(integration.getAssets()); + }; +} diff --git a/server/adaptors/integrations/types.ts b/server/adaptors/integrations/types.ts index 58293580fb..8e8d51ca89 100644 --- a/server/adaptors/integrations/types.ts +++ b/server/adaptors/integrations/types.ts @@ -55,11 +55,7 @@ interface IntegrationTemplateQuery { interface IntegrationInstance { name: string; templateName: string; - dataSource: { - sourceType: string; - dataset: string; - namespace: string; - }; + dataSource: string; creationDate: string; assets: AssetReference[]; } diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index 0bc7029b0d..7c1964587e 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -87,16 +87,7 @@ const instanceSchema: JSONSchemaType = { properties: { name: { type: 'string' }, templateName: { type: 'string' }, - dataSource: { - type: 'object', - properties: { - sourceType: { type: 'string' }, - dataset: { type: 'string' }, - namespace: { type: 'string' }, - }, - required: ['sourceType', 'dataset', 'namespace'], - additionalProperties: false, - }, + dataSource: { type: 'string' }, creationDate: { type: 'string' }, assets: { type: 'array', diff --git a/server/routes/integrations/__tests__/integrations_router.test.ts b/server/routes/integrations/__tests__/integrations_router.test.ts index c74a2e8db0..75b1f44c29 100644 --- a/server/routes/integrations/__tests__/integrations_router.test.ts +++ b/server/routes/integrations/__tests__/integrations_router.test.ts @@ -1,29 +1,23 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - +import { DeepPartial } from 'redux'; import { OpenSearchDashboardsResponseFactory } from '../../../../../../src/core/server/http/router'; import { handleWithCallback } from '../integrations_router'; import { IntegrationsAdaptor } from 'server/adaptors/integrations/integrations_adaptor'; -describe('handleWithCallback', () => { - let adaptorMock: jest.Mocked; - let responseMock: jest.Mocked; +jest + .mock('../../../../../../src/core/server', () => jest.fn()) + .mock('../../../../../../src/core/server/http/router', () => jest.fn()); - beforeEach(() => { - adaptorMock = {} as any; - responseMock = { - custom: jest.fn((data) => data), - ok: jest.fn((data) => data), - } as any; - }); +describe('Data wrapper', () => { + const adaptorMock: Partial = {}; + const responseMock: DeepPartial = { + custom: jest.fn((data) => data), + ok: jest.fn((data) => data), + }; it('retrieves data from the callback method', async () => { const callback = jest.fn((_) => { return { test: 'data' }; }); - const result = await handleWithCallback( adaptorMock as IntegrationsAdaptor, responseMock as OpenSearchDashboardsResponseFactory, @@ -39,7 +33,6 @@ describe('handleWithCallback', () => { const callback = jest.fn((_) => { throw new Error('test error'); }); - const result = await handleWithCallback( adaptorMock as IntegrationsAdaptor, responseMock as OpenSearchDashboardsResponseFactory, diff --git a/server/routes/integrations/integrations_router.ts b/server/routes/integrations/integrations_router.ts index 2faf078623..d4bfa4760b 100644 --- a/server/routes/integrations/integrations_router.ts +++ b/server/routes/integrations/integrations_router.ts @@ -12,6 +12,7 @@ import { OpenSearchDashboardsRequest, OpenSearchDashboardsResponseFactory, } from '../../../../../src/core/server/http/router'; +import { IntegrationsKibanaBackend } from '../../adaptors/integrations/integrations_kibana_backend'; /** * Handle an `OpenSearchDashboardsRequest` using the provided `callback` function. @@ -52,8 +53,7 @@ const getAdaptor = ( context: RequestHandlerContext, _request: OpenSearchDashboardsRequest ): IntegrationsAdaptor => { - // Stub - return {} as IntegrationsAdaptor; + return new IntegrationsKibanaBackend(context.core.savedObjects.client); }; export function registerIntegrationsRoute(router: IRouter) { From 7aaa2a5a6f617b0c8d0dfe4973c7bfb5f65e2d6a Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 22 Jun 2023 13:44:51 -0700 Subject: [PATCH 2/4] Add integration type to .kibana from osints/dev Signed-off-by: Simeon Widdis --- server/plugin.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/server/plugin.ts b/server/plugin.ts index f315f809b4..5dab615726 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -78,7 +78,37 @@ export class ObservabilityPlugin }, }; + const integrationInstanceType: SavedObjectsType = { + name: 'integration-instance', + hidden: false, + namespaceType: 'single', + mappings: { + dynamic: false, + properties: { + name: { + type: 'text', + }, + templateName: { + type: 'text', + }, + dataSource: { + type: 'nested', + }, + creationDate: { + type: 'date', + }, + addedBy: { + type: 'text', + }, + assets: { + type: 'nested', + }, + }, + }, + }; + core.savedObjects.registerType(obsPanelType); + core.savedObjects.registerType(integrationInstanceType); // Register server side APIs setupRoutes({ router, client: openSearchObservabilityClient }); From ef310a81879d68868cbcb170ba76c985c0838697 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 26 Jun 2023 09:30:30 -0700 Subject: [PATCH 3/4] Re-add license header Signed-off-by: Simeon Widdis --- .../integrations/__tests__/integrations_router.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/routes/integrations/__tests__/integrations_router.test.ts b/server/routes/integrations/__tests__/integrations_router.test.ts index 75b1f44c29..15d2bac28b 100644 --- a/server/routes/integrations/__tests__/integrations_router.test.ts +++ b/server/routes/integrations/__tests__/integrations_router.test.ts @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { DeepPartial } from 'redux'; import { OpenSearchDashboardsResponseFactory } from '../../../../../../src/core/server/http/router'; import { handleWithCallback } from '../integrations_router'; From 43a21567cd825c4773e15545fa7357a44b3b2e09 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 28 Jun 2023 09:41:56 -0700 Subject: [PATCH 4/4] Fix integrations type Signed-off-by: Simeon Widdis --- server/plugin.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/plugin.ts b/server/plugin.ts index 5dab615726..31db6b3efc 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -92,14 +92,11 @@ export class ObservabilityPlugin type: 'text', }, dataSource: { - type: 'nested', + type: 'text', }, creationDate: { type: 'date', }, - addedBy: { - type: 'text', - }, assets: { type: 'nested', },