diff --git a/x-pack/legacy/plugins/integrations_manager/common/routes.ts b/x-pack/legacy/plugins/integrations_manager/common/routes.ts index 6b594c26e0869..82f41066784bf 100644 --- a/x-pack/legacy/plugins/integrations_manager/common/routes.ts +++ b/x-pack/legacy/plugins/integrations_manager/common/routes.ts @@ -8,8 +8,8 @@ import { PLUGIN_ID } from './constants'; export const API_ROOT = `/api/${PLUGIN_ID}`; export const API_LIST_PATTERN = `${API_ROOT}/list`; export const API_INFO_PATTERN = `${API_ROOT}/package/{pkgkey}`; -export const API_INSTALL_PATTERN = `${API_ROOT}/install/{pkgkey}/{feature?}`; -export const API_DELETE_PATTERN = `${API_ROOT}/delete/{pkgkey}/{feature?}`; +export const API_INSTALL_PATTERN = `${API_ROOT}/install/{pkgkey}/{asset?}`; +export const API_DELETE_PATTERN = `${API_ROOT}/delete/{pkgkey}/{asset?}`; export function getListPath() { return API_LIST_PATTERN; diff --git a/x-pack/legacy/plugins/integrations_manager/common/types.ts b/x-pack/legacy/plugins/integrations_manager/common/types.ts index a8dde6e80ed45..388e08148a282 100644 --- a/x-pack/legacy/plugins/integrations_manager/common/types.ts +++ b/x-pack/legacy/plugins/integrations_manager/common/types.ts @@ -12,11 +12,11 @@ import { } from 'src/core/server/saved_objects'; type AssetReference = Pick; -export interface Installation extends SavedObjectAttributes { +export interface InstallationAttributes extends SavedObjectAttributes { installed: AssetReference[]; } -export type InstallationSavedObject = SavedObject; +export type Installation = SavedObject; // the contract with the registry export type RegistryList = RegistryListItem[]; @@ -49,15 +49,17 @@ export interface RegistryPackage { // the public HTTP response types export type IntegrationList = IntegrationListItem[]; -export type IntegrationListItem = Installed | NotInstalled; +export type IntegrationListItem = Installable; -export type IntegrationInfo = Installed | NotInstalled; +export type IntegrationInfo = Installable; -type Installed = T & { +export type Installable = Installed | NotInstalled; + +export type Installed = T & { status: 'installed'; - savedObject: InstallationSavedObject; + savedObject: Installation; }; -type NotInstalled = T & { +export type NotInstalled = T & { status: 'not_installed'; }; diff --git a/x-pack/legacy/plugins/integrations_manager/public/screens/detail.tsx b/x-pack/legacy/plugins/integrations_manager/public/screens/detail.tsx index 783ddfeacc50d..cdc6198186da6 100644 --- a/x-pack/legacy/plugins/integrations_manager/public/screens/detail.tsx +++ b/x-pack/legacy/plugins/integrations_manager/public/screens/detail.tsx @@ -10,12 +10,9 @@ import { IntegrationInfo } from '../../common/types'; export function Detail(props: { package: string }) { const [info, setInfo] = useState(null); - useEffect( - () => { - getIntegrationInfoByKey(props.package).then(setInfo); - }, - [props.package] - ); + useEffect(() => { + getIntegrationInfoByKey(props.package).then(setInfo); + }, [props.package]); // don't have designs for loading/empty states if (!info) return null; diff --git a/x-pack/legacy/plugins/integrations_manager/server/integrations.ts b/x-pack/legacy/plugins/integrations_manager/server/integrations.ts deleted file mode 100644 index aeb9dda27d461..0000000000000 --- a/x-pack/legacy/plugins/integrations_manager/server/integrations.ts +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - SavedObject, - SavedObjectReference, - SavedObjectsBulkGetObject, - SavedObjectsClientContract, -} from 'src/core/server/saved_objects'; -import { SAVED_OBJECT_TYPE } from '../common/constants'; -import { - Request, - InstallationSavedObject, - IntegrationInfo, - IntegrationListItem, - RegistryList, - RegistryListItem, - RegistryPackage, -} from '../common/types'; -import { cacheGet } from './cache'; -import * as Registry from './registry'; -import { getClient } from './saved_objects'; - -interface PackageRequest extends Request { - params: { - pkgkey: string; - }; -} - -interface InstallFeatureRequest extends PackageRequest { - params: { - pkgkey: string; - feature: string; - }; -} - -export async function handleGetList(req: Request) { - const registryItems = await Registry.fetchList(); - const searchObjects: SavedObjectsBulkGetObject[] = registryItems.map(({ name, version }) => ({ - type: SAVED_OBJECT_TYPE, - id: `${name}-${version}`, - })); - const client = getClient(req); - const results = await client.bulkGet(searchObjects); - const savedObjects: InstallationSavedObject[] = results.saved_objects.filter(o => !o.error); // ignore errors for now - const integrationList = createIntegrationList(registryItems, savedObjects); - - return integrationList; -} - -export async function handleGetInfo(req: PackageRequest) { - const { pkgkey } = req.params; - const item = await Registry.fetchInfo(pkgkey); - const savedObject = await getInstallationObject(getClient(req), pkgkey); - const installation = createInstallationObject(item, savedObject); - - return installation; -} - -export async function handleRequestInstall(req: InstallFeatureRequest) { - const { pkgkey, feature } = req.params; - - if (feature === 'dashboard') { - const toBeSavedObjects = await getObjects(pkgkey, feature); - const client = getClient(req); - const createResults = await client.bulkCreate(toBeSavedObjects, { overwrite: true }); - const installed = createResults.saved_objects.map(({ id, type }) => ({ id, type })); - const mgrResults = await client.create( - SAVED_OBJECT_TYPE, - { installed }, - { id: pkgkey, overwrite: true } - ); - - return mgrResults; - } - - return { - pkgkey, - feature, - created: [], - }; -} - -export async function handleRequestDelete(req: InstallFeatureRequest) { - const { pkgkey, feature } = req.params; - const client = getClient(req); - - const installation = await getInstallationObject(client, pkgkey); - const installedObjects = (installation && installation.attributes.installed) || []; - - // Delete the manager saved object with references to the asset objects - // could also update with [] or some other state - await client.delete(SAVED_OBJECT_TYPE, pkgkey); - - // ASK: should the manager uninstall the assets it installed - // or just the references in SAVED_OBJECT_TYPE? - if (feature === 'dashboard') { - // Delete the installed assets - const deletePromises = installedObjects.map(async ({ id, type }) => client.delete(type, id)); - await Promise.all(deletePromises); - } - - return { - pkgkey, - feature, - deleted: installedObjects, - }; -} - -export async function getObjects(pkgkey: string, type: string): Promise { - const paths = await Registry.getArchiveInfo(`${pkgkey}.tar.gz`); - const toBeSavedObjects = paths.reduce((map, path) => { - collectReferences(map, { path, desiredType: 'dashboard' }); - return map; - }, new Map()); - - return Array.from(toBeSavedObjects.values(), ensureJsonValues); -} - -function getAsset(key: string) { - const value = cacheGet(key); - if (value !== undefined) { - const json = value.toString('utf8'); - return JSON.parse(json); - } -} - -interface CollectReferencesOptions { - path: string; - desiredType: string; // TODO: from enum or similar of acceptable asset types -} -function collectReferences( - toBeSavedObjects: Map = new Map(), - { path, desiredType = 'dashboard' }: CollectReferencesOptions -) { - const [pkgkey, service, type, file] = path.split('/'); - if (type !== desiredType) return; - if (toBeSavedObjects.has(path)) return; - if (!/\.json$/.test(path)) return; - - const asset = getAsset(path); - if (!asset.type) asset.type = type; - if (!asset.id) asset.id = file.replace('.json', ''); - toBeSavedObjects.set(path, asset); - - const references: SavedObjectReference[] = asset.references; - return references.reduce((map, reference) => { - collectReferences(toBeSavedObjects, { - path: `${pkgkey}/${service}/${reference.type}/${reference.id}.json`, - desiredType: reference.type, - }); - return map; - }, toBeSavedObjects); -} - -// the assets from the registry are malformed -// https://github.com/elastic/integrations-registry/issues/42 -function ensureJsonValues(obj: SavedObject) { - const { attributes } = obj; - if ( - attributes.kibanaSavedObjectMeta && - typeof attributes.kibanaSavedObjectMeta.searchSourceJSON !== 'string' - ) { - attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.stringify( - attributes.kibanaSavedObjectMeta.searchSourceJSON - ); - } - ['optionsJSON', 'panelsJSON', 'uiStateJSON', 'visState'] - .filter(key => typeof attributes[key] !== 'string') - .forEach(key => (attributes[key] = JSON.stringify(attributes[key]))); - return obj; -} - -function createIntegrationList( - registryItems: RegistryList, - integrationObjects: InstallationSavedObject[] -) { - const integrationList = registryItems.map(item => - createInstallationObject( - item, - integrationObjects.find(({ id }) => id === `${item.name}-${item.version}`) - ) - ); - - return integrationList.sort(sortByName); -} - -function createInstallationObject( - item: RegistryPackage, - savedObject?: InstallationSavedObject -): IntegrationInfo; -function createInstallationObject( - item: RegistryListItem, - savedObject?: InstallationSavedObject -): IntegrationListItem; -function createInstallationObject( - obj: RegistryPackage | RegistryListItem, - savedObject?: InstallationSavedObject -) { - return savedObject - ? { - ...obj, - status: 'installed', - savedObject, - } - : { - ...obj, - status: 'not_installed', - }; -} - -function sortByName(a: { name: string }, b: { name: string }) { - if (a.name > b.name) { - return 1; - } else if (a.name < b.name) { - return -1; - } else { - return 0; - } -} - -async function getInstallationObject( - client: SavedObjectsClientContract, - pkgkey: string -): Promise { - return client.get(SAVED_OBJECT_TYPE, pkgkey).catch(e => undefined); -} diff --git a/x-pack/legacy/plugins/integrations_manager/server/integrations/data.ts b/x-pack/legacy/plugins/integrations_manager/server/integrations/data.ts new file mode 100644 index 0000000000000..35cce5703d9ca --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/server/integrations/data.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + SavedObject, + SavedObjectsBulkGetObject, + SavedObjectsClientContract, +} from 'src/core/server/saved_objects'; +import { Installation, InstallationAttributes, Installable } from '../../common/types'; +import { SAVED_OBJECT_TYPE } from '../../common/constants'; +import * as Registry from '../registry'; + +export async function getIntegrations(client: SavedObjectsClientContract) { + const registryItems = await Registry.fetchList(); + const searchObjects: SavedObjectsBulkGetObject[] = registryItems.map(({ name, version }) => ({ + type: SAVED_OBJECT_TYPE, + id: `${name}-${version}`, + })); + + const results = await client.bulkGet(searchObjects); + const savedObjects = results.saved_objects.filter(o => !o.error); // ignore errors for now + const integrationList = registryItems + .map(item => + createInstallableFrom( + item, + savedObjects.find(({ id }) => id === `${item.name}-${item.version}`) + ) + ) + .sort(sortByName); + + return integrationList; +} + +export async function getIntegrationInfo(client: SavedObjectsClientContract, pkgkey: string) { + const [item, savedObject] = await Promise.all([ + Registry.fetchInfo(pkgkey), + getInstallationObject(client, pkgkey), + ]); + const installation = createInstallableFrom(item, savedObject); + + return installation; +} + +export async function getInstallationObject(client: SavedObjectsClientContract, pkgkey: string) { + return client.get(SAVED_OBJECT_TYPE, pkgkey).catch(e => undefined); +} + +export async function installAssets( + client: SavedObjectsClientContract, + pkgkey: string, + filter = (entry: Registry.ArchiveEntry): boolean => true +) { + const toBeSavedObjects = await Registry.getObjects(pkgkey, filter); + const createResults = await client.bulkCreate(toBeSavedObjects, { + overwrite: true, + }); + const createdObjects: SavedObject[] = createResults.saved_objects; + const installed = createdObjects.map(({ id, type }) => ({ id, type })); + const results = await client.create( + SAVED_OBJECT_TYPE, + { installed }, + { id: pkgkey, overwrite: true } + ); + + return results; +} + +export async function removeInstallation(client: SavedObjectsClientContract, pkgkey: string) { + const installation = await getInstallationObject(client, pkgkey); + const installedObjects = (installation && installation.attributes.installed) || []; + + // Delete the manager saved object with references to the asset objects + // could also update with [] or some other state + await client.delete(SAVED_OBJECT_TYPE, pkgkey); + + // Delete the installed assets + const deletePromises = installedObjects.map(async ({ id, type }) => client.delete(type, id)); + await Promise.all(deletePromises); + + // successful delete's in SO client return {}. return something more useful + return installedObjects; +} + +function sortByName(a: { name: string }, b: { name: string }) { + if (a.name > b.name) { + return 1; + } else if (a.name < b.name) { + return -1; + } else { + return 0; + } +} + +function createInstallableFrom(from: T, savedObject?: Installation): Installable { + return savedObject + ? { + ...from, + status: 'installed', + savedObject, + } + : { + ...from, + status: 'not_installed', + }; +} diff --git a/x-pack/legacy/plugins/integrations_manager/server/integrations/handlers.ts b/x-pack/legacy/plugins/integrations_manager/server/integrations/handlers.ts new file mode 100644 index 0000000000000..3f5f4e9452057 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/server/integrations/handlers.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Request } from '../../common/types'; +import { ArchiveEntry, pathParts } from '../registry'; +import { getClient } from '../saved_objects'; +import { getIntegrations, getIntegrationInfo, installAssets, removeInstallation } from './data'; + +interface PackageRequest extends Request { + params: { + pkgkey: string; + }; +} + +interface InstallAssetRequest extends Request { + params: AssetRequestParams; +} + +interface DeleteAssetRequest extends Request { + params: AssetRequestParams; +} + +type AssetRequestParams = PackageRequest['params'] & { + asset: string; +}; + +export async function handleGetList(req: Request) { + const client = getClient(req); + const integrationList = await getIntegrations(client); + + return integrationList; +} + +export async function handleGetInfo(req: PackageRequest) { + const { pkgkey } = req.params; + const client = getClient(req); + const installation = await getIntegrationInfo(client, pkgkey); + + return installation; +} + +export async function handleRequestInstall(req: InstallAssetRequest) { + const { pkgkey, asset } = req.params; + const created = []; + + if (asset === 'dashboard') { + const client = getClient(req); + const object = await installAssets(client, pkgkey, (entry: ArchiveEntry) => { + const { type } = pathParts(entry.path); + return type === asset; + }); + + created.push(object); + } + + return { + pkgkey, + asset, + created, + }; +} + +export async function handleRequestDelete(req: DeleteAssetRequest) { + const { pkgkey, asset } = req.params; + const client = getClient(req); + const deleted = await removeInstallation(client, pkgkey); + + return { + pkgkey, + asset, + deleted, + }; +} diff --git a/x-pack/legacy/plugins/integrations_manager/server/integrations/index.ts b/x-pack/legacy/plugins/integrations_manager/server/integrations/index.ts new file mode 100644 index 0000000000000..9e640ddb30dc6 --- /dev/null +++ b/x-pack/legacy/plugins/integrations_manager/server/integrations/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './data'; +export * from './handlers'; diff --git a/x-pack/legacy/plugins/integrations_manager/server/registry.ts b/x-pack/legacy/plugins/integrations_manager/server/registry.ts index c6adb81c15090..2497b53415b1a 100644 --- a/x-pack/legacy/plugins/integrations_manager/server/registry.ts +++ b/x-pack/legacy/plugins/integrations_manager/server/registry.ts @@ -4,12 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ +import { SavedObject } from 'src/core/server/saved_objects'; import { RegistryList, RegistryPackage } from '../common/types'; import { cacheGet, cacheSet, cacheHas } from './cache'; import { ArchiveEntry, untarBuffer, unzipBuffer } from './extract'; import { fetchUrl, getResponseStream } from './requests'; import { streamToBuffer } from './streams'; +export { ArchiveEntry } from './extract'; + const REGISTRY = process.env.REGISTRY || 'http://integrations-registry.app.elstc.co'; export async function fetchList(): Promise { @@ -27,9 +30,13 @@ export async function getArchiveInfo( const paths: string[] = []; const onEntry = (entry: ArchiveEntry) => { const { path, buffer } = entry; - paths.push(path); + const { file } = pathParts(path); + if (!file) return; if (cacheHas(path)) return; - if (buffer) cacheSet(path, buffer); + if (buffer) { + cacheSet(path, buffer); + paths.push(path); + } }; await extract(key, filter, onEntry); @@ -37,6 +44,51 @@ export async function getArchiveInfo( return paths; } +export async function getObjects( + pkgkey: string, + filter = (entry: ArchiveEntry): boolean => true +): Promise { + // Create a Map b/c some values, especially index-patterns, are referenced multiple times + const objects: Map = new Map(); + + // Get paths which match the given filter + const paths = await getArchiveInfo(`${pkgkey}.tar.gz`, filter); + + // Get all objects which matched filter. Add them to the Map + const rootObjects = paths.map(getObject); + rootObjects.forEach(obj => objects.set(obj.id, obj)); + + // Each of those objects might have `references` property like [{id, type, name}] + for (const object of rootObjects) { + // For each of those objects + for (const reference of object.references) { + // Get the objects they reference. Call same function with a new filter + const referencedObjects = await getObjects(pkgkey, (entry: ArchiveEntry) => { + // Skip anything we've already stored + if (objects.has(reference.id)) return false; + + // Is the archive entry the reference we want? + const { type, file } = pathParts(entry.path); + const isType = type === reference.type; + const isJson = file === `${reference.id}.json`; + return isType && isJson; + }); + + // Add referenced objects to the Map + referencedObjects.forEach(ro => objects.set(ro.id, ro)); + } + } + + // return the array of unique objects + return Array.from(objects.values()); +} + +export function pathParts(path: string) { + const [pkgkey, service, type, file] = path.split('/'); + + return { pkgkey, service, type, file }; +} + async function extract( key: string, filter = (entry: ArchiveEntry): boolean => true, @@ -65,3 +117,37 @@ async function getOrFetchArchiveBuffer(key: string): Promise { async function fetchArchiveBuffer(key: string): Promise { return getResponseStream(`${REGISTRY}/package/${key}`).then(streamToBuffer); } + +// the assets from the registry are malformed +// https://github.com/elastic/integrations-registry/issues/42 +function ensureJsonValues(obj: SavedObject) { + const { attributes } = obj; + if ( + attributes.kibanaSavedObjectMeta && + typeof attributes.kibanaSavedObjectMeta.searchSourceJSON !== 'string' + ) { + attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.stringify( + attributes.kibanaSavedObjectMeta.searchSourceJSON + ); + } + ['optionsJSON', 'panelsJSON', 'uiStateJSON', 'visState'] + .filter(key => typeof attributes[key] !== 'string') + .forEach(key => (attributes[key] = JSON.stringify(attributes[key]))); + return obj; +} + +function getObject(key: string) { + const buffer = cacheGet(key); + if (buffer === undefined) throw new Error(`Cannot find asset ${key}`); + + // cache values are buffers. convert to string / JSON + const json = buffer.toString('utf8'); + // convert that to an object & address issues with the formatting of some parts + const asset = ensureJsonValues(JSON.parse(json)); + + const { type, file } = pathParts(key); + if (!asset.type) asset.type = type; + if (!asset.id) asset.id = file.replace('.json', ''); + + return asset; +}