[Backport 2.x] JSON Catalog Reader for Integrations

server/adaptors/integrations/__test__/builder.test.ts

@@ -5,7 +5,13 @@
 
 import { SavedObjectsClientContract } from '../../../../../../src/core/server';
 import { IntegrationInstanceBuilder } from '../integrations_builder';
-import { IntegrationReader } from '../repository/integration';
+import { IntegrationReader } from '../repository/integration_reader';
+import * as mockUtils from '../repository/utils';
+
+jest.mock('../repository/utils', () => ({
+  ...jest.requireActual('../repository/utils'),
+  deepCheck: jest.fn(),
+}));
 
 const mockSavedObjectsClient: SavedObjectsClientContract = ({
   bulkCreate: jest.fn(),
@@ -17,7 +23,6 @@ const mockSavedObjectsClient: SavedObjectsClientContract = ({
 } as unknown) as SavedObjectsClientContract;
 
 const sampleIntegration: IntegrationReader = ({
-  deepCheck: jest.fn().mockResolvedValue(true),
   getAssets: jest.fn().mockResolvedValue({
     savedObjects: [
       {
@@ -104,8 +109,12 @@ describe('IntegrationInstanceBuilder', () => {
         },
       };
 
+      jest
+        .spyOn(mockUtils, 'deepCheck')
+        .mockResolvedValue({ ok: true, value: mockTemplate as IntegrationConfig });
+
       // Mock the implementation of the methods in the Integration class
-      sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: mockTemplate });
+      // sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: mockTemplate });
       sampleIntegration.getAssets = jest
         .fn()
         .mockResolvedValue({ ok: true, value: { savedObjects: remappedAssets } });
@@ -119,7 +128,6 @@ describe('IntegrationInstanceBuilder', () => {
 
       const instance = await builder.build(sampleIntegration, options);
 
-      expect(sampleIntegration.deepCheck).toHaveBeenCalled();
       expect(sampleIntegration.getAssets).toHaveBeenCalled();
       expect(remapIDsSpy).toHaveBeenCalledWith(remappedAssets);
       expect(postAssetsSpy).toHaveBeenCalledWith(remappedAssets);
@@ -131,8 +139,8 @@ describe('IntegrationInstanceBuilder', () => {
         dataSource: 'instance-datasource',
         name: 'instance-name',
       };
-      sampleIntegration.deepCheck = jest
-        .fn()
+      jest
+        .spyOn(mockUtils, 'deepCheck')
         .mockResolvedValue({ ok: false, error: new Error('Mock error') });
 
       await expect(builder.build(sampleIntegration, options)).rejects.toThrowError('Mock error');
@@ -145,7 +153,9 @@ describe('IntegrationInstanceBuilder', () => {
       };
 
       const errorMessage = 'Failed to get assets';
-      sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: {} });
+      jest
+        .spyOn(mockUtils, 'deepCheck')
+        .mockResolvedValue({ ok: true, value: ({} as unknown) as IntegrationConfig });
       sampleIntegration.getAssets = jest
         .fn()
         .mockResolvedValue({ ok: false, error: new Error(errorMessage) });
@@ -165,7 +175,9 @@ describe('IntegrationInstanceBuilder', () => {
         },
       ];
       const errorMessage = 'Failed to post assets';
-      sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: {} });
+      jest
+        .spyOn(mockUtils, 'deepCheck')
+        .mockResolvedValue({ ok: true, value: ({} as unknown) as IntegrationConfig });
       sampleIntegration.getAssets = jest
         .fn()
         .mockResolvedValue({ ok: true, value: { savedObjects: remappedAssets } });
@@ -180,10 +192,14 @@ describe('IntegrationInstanceBuilder', () => {
       const assets = [
         {
           id: 'asset1',
+          type: 'unknown',
+          attributes: { title: 'asset1' },
           references: [{ id: 'ref1' }, { id: 'ref2' }],
         },
         {
           id: 'asset2',
+          type: 'unknown',
+          attributes: { title: 'asset1' },
           references: [{ id: 'ref1' }, { id: 'ref3' }],
         },
       ];
@@ -200,7 +216,7 @@ describe('IntegrationInstanceBuilder', () => {
 
       const remappedAssets = builder.remapIDs(assets);
 
-      expect(remappedAssets).toEqual(expectedRemappedAssets);
+      expect(remappedAssets).toMatchObject(expectedRemappedAssets);
     });
   });
 

server/adaptors/integrations/__test__/json_repository.test.ts

@@ -0,0 +1,225 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Serialization tests for integrations in the local repository.
+ */
+
+import { TemplateManager } from '../repository/repository';
+import { IntegrationReader } from '../repository/integration_reader';
+import path from 'path';
+import * as fs from 'fs/promises';
+import { JsonCatalogDataAdaptor } from '../repository/json_data_adaptor';
+import { deepCheck, foldResults } from '../repository/utils';
+
+const fetchSerializedIntegrations = async (): Promise<Result<SerializedIntegration[], Error>> => {
+  const directory = path.join(__dirname, '../__data__/repository');
+  const folders = await fs.readdir(directory);
+  const readers = await Promise.all(
+    folders.map(async (folder) => {
+      const integPath = path.join(directory, folder);
+      if (!(await fs.lstat(integPath)).isDirectory()) {
+        // If it's not a directory (e.g. a README), skip it
+        return Promise.resolve(null);
+      }
+      // Otherwise, all directories must be integrations
+      return new IntegrationReader(integPath);
+    })
+  );
+  const serializedIntegrationResults = await Promise.all(
+    (readers.filter((x) => x !== null) as IntegrationReader[]).map((r) => r.serialize())
+  );
+  return foldResults(serializedIntegrationResults);
+};
+
+describe('The Local Serialized Catalog', () => {
+  it('Should serialize without errors', async () => {
+    const serialized = await fetchSerializedIntegrations();
+    expect(serialized.ok).toBe(true);
+  });
+
+  it('Should pass deep validation for all serialized integrations', async () => {
+    const serialized = await fetchSerializedIntegrations();
+    const repository = new TemplateManager(
+      '.',
+      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[])
+    );
+
+    for (const integ of await repository.getIntegrationList()) {
+      const validationResult = await deepCheck(integ);
+      await expect(validationResult).toHaveProperty('ok', true);
+    }
+  });
+
+  it('Should correctly retrieve a logo', async () => {
+    const serialized = await fetchSerializedIntegrations();
+    const repository = new TemplateManager(
+      '.',
+      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[])
+    );
+    const integration = (await repository.getIntegration('nginx')) as IntegrationReader;
+    const logoStatic = await integration.getStatic('logo.svg');
+
+    expect(logoStatic).toHaveProperty('ok', true);
+    expect((logoStatic.value as Buffer).length).toBeGreaterThan(1000);
+  });
+
+  it('Should correctly retrieve a gallery image', async () => {
+    const serialized = await fetchSerializedIntegrations();
+    const repository = new TemplateManager(
+      '.',
+      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[])
+    );
+    const integration = (await repository.getIntegration('nginx')) as IntegrationReader;
+    const logoStatic = await integration.getStatic('dashboard1.png');
+
+    expect(logoStatic).toHaveProperty('ok', true);
+    expect((logoStatic.value as Buffer).length).toBeGreaterThan(1000);
+  });
+
+  it('Should correctly retrieve a dark mode logo', async () => {
+    const TEST_INTEGRATION = 'nginx';
+    const serialized = await fetchSerializedIntegrations();
+    const config = (serialized.value as SerializedIntegration[]).filter(
+      (integ: { name: string; components: unknown[] }) => integ.name === TEST_INTEGRATION
+    )[0];
+
+    if (!config.statics) {
+      throw new Error('NginX integration missing statics (invalid test)');
+    }
+    config.statics.darkModeGallery = config.statics.gallery;
+    config.statics.darkModeLogo = {
+      ...(config.statics.logo as SerializedStaticAsset),
+      path: 'dark_logo.svg',
+    };
+
+    const reader = new IntegrationReader('nginx', new JsonCatalogDataAdaptor([config]));
+
+    await expect(reader.getStatic('dark_logo.svg')).resolves.toHaveProperty('ok', true);
+  });
+
+  it('Should correctly re-serialize', async () => {
+    const TEST_INTEGRATION = 'nginx';
+    const serialized = await fetchSerializedIntegrations();
+    const config = (serialized.value as SerializedIntegration[]).filter(
+      (integ: { name: string }) => integ.name === TEST_INTEGRATION
+    )[0];
+
+    const reader = new IntegrationReader('nginx', new JsonCatalogDataAdaptor([config]));
+    const reserialized = await reader.serialize();
+
+    expect(reserialized.value).toEqual(config);
+  });
+
+  it('Should correctly re-serialize with dark mode values', async () => {
+    const TEST_INTEGRATION = 'nginx';
+    const serialized = await fetchSerializedIntegrations();
+    const config = (serialized.value as SerializedIntegration[]).filter(
+      (integ: { name: string }) => integ.name === TEST_INTEGRATION
+    )[0];
+
+    if (!config.statics) {
+      throw new Error('NginX integration missing statics (invalid test)');
+    }
+    config.statics.darkModeGallery = config.statics.gallery;
+    config.statics.darkModeLogo = {
+      ...(config.statics.logo as SerializedStaticAsset),
+      path: 'dark_logo.svg',
+    };
+
+    const reader = new IntegrationReader('nginx', new JsonCatalogDataAdaptor([config]));
+    const reserialized = await reader.serialize();
+
+    expect(reserialized.value).toEqual(config);
+  });
+});
+
+describe('Integration validation', () => {
+  it('Should correctly fail an integration without schemas', async () => {
+    const TEST_INTEGRATION = 'nginx';
+    const serialized = await fetchSerializedIntegrations();
+    const transformedSerialized = (serialized.value as SerializedIntegration[])
+      .filter((integ: { name: string; components: unknown[] }) => integ.name === TEST_INTEGRATION)
+      .map((integ) => {
+        return {
+          ...integ,
+          components: [] as SerializedIntegrationComponent[],
+        };
+      });
+    const integration = new IntegrationReader(
+      TEST_INTEGRATION,
+      new JsonCatalogDataAdaptor(transformedSerialized)
+    );
+
+    await expect(deepCheck(integration)).resolves.toHaveProperty('ok', false);
+  });
+
+  it('Should correctly fail an integration without assets', async () => {
+    const TEST_INTEGRATION = 'nginx';
+    const serialized = await fetchSerializedIntegrations();
+    const transformedSerialized = (serialized.value as SerializedIntegration[])
+      .filter((integ: { name: string; components: unknown[] }) => integ.name === TEST_INTEGRATION)
+      .map((integ) => {
+        return {
+          ...integ,
+          assets: {} as SerializedIntegrationAssets,
+        };
+      });
+    const integration = new IntegrationReader(
+      TEST_INTEGRATION,
+      new JsonCatalogDataAdaptor(transformedSerialized)
+    );
+
+    await expect(deepCheck(integration)).resolves.toHaveProperty('ok', false);
+  });
+});
+
+describe('JSON Catalog with invalid data', () => {
+  it('Should report an error if images are missing data', async () => {
+    const TEST_INTEGRATION = 'nginx';
+    const serialized = await fetchSerializedIntegrations();
+    const baseConfig = (serialized.value as SerializedIntegration[]).filter(
+      (integ: { name: string; components: unknown[] }) => integ.name === TEST_INTEGRATION
+    )[0];
+
+    if (!baseConfig.statics) {
+      throw new Error('NginX integration missing statics (invalid test)');
+    }
+
+    baseConfig.statics = {
+      logo: { path: 'logo.svg' } as SerializedStaticAsset,
+      darkModeLogo: { path: 'dm_logo.svg' } as SerializedStaticAsset,
+      gallery: [{ path: '1.png' }] as SerializedStaticAsset[],
+      darkModeGallery: [{ path: 'dm_1.png' }] as SerializedStaticAsset[],
+    };
+    const reader = new IntegrationReader(
+      TEST_INTEGRATION,
+      new JsonCatalogDataAdaptor([baseConfig])
+    );
+
+    await expect(reader.getStatic('logo.svg')).resolves.toHaveProperty('ok', false);
+    await expect(reader.getStatic('dm_logo.svg')).resolves.toHaveProperty('ok', false);
+    await expect(reader.getStatic('1.png')).resolves.toHaveProperty('ok', false);
+    await expect(reader.getStatic('dm_1.png')).resolves.toHaveProperty('ok', false);
+  });
+
+  it('Should report an error on read if a schema has invalid JSON', async () => {
+    const TEST_INTEGRATION = 'nginx';
+    const serialized = await fetchSerializedIntegrations();
+    const baseConfig = (serialized.value as SerializedIntegration[]).filter(
+      (integ: { name: string; components: unknown[] }) => integ.name === TEST_INTEGRATION
+    )[0];
+
+    expect(baseConfig.components.length).toBeGreaterThanOrEqual(2);
+    baseConfig.components[1].data = '{"invalid_json": true';
+
+    const reader = new IntegrationReader(
+      TEST_INTEGRATION,
+      new JsonCatalogDataAdaptor([baseConfig])
+    );
+
+    await expect(reader.getSchemas()).resolves.toHaveProperty('ok', false);
+  });
+});

server/adaptors/integrations/__test__/local_repository.test.ts → server/adaptors/integrations/__test__/local_fs_repository.test.ts

@@ -3,10 +3,15 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+/**
+ * This file is used as integration tests for Integrations Repository functionality.
+ */
+
 import { TemplateManager } from '../repository/repository';
-import { IntegrationReader } from '../repository/integration';
+import { IntegrationReader } from '../repository/integration_reader';
 import path from 'path';
 import * as fs from 'fs/promises';
+import { deepCheck } from '../repository/utils';
 
 describe('The local repository', () => {
   it('Should only contain valid integration directories or files.', async () => {
@@ -21,7 +26,7 @@ describe('The local repository', () => {
         }
         // Otherwise, all directories must be integrations
         const integ = new IntegrationReader(integPath);
-        expect(integ.getConfig()).resolves.toHaveProperty('ok', true);
+        await expect(integ.getConfig()).resolves.toHaveProperty('ok', true);
       })
     );
   });
@@ -33,7 +38,7 @@ describe('The local repository', () => {
     const integrations: IntegrationReader[] = await repository.getIntegrationList();
     await Promise.all(
       integrations.map(async (i) => {
-        const result = await i.deepCheck();
+        const result = await deepCheck(i);
         if (!result.ok) {
           console.error(result.error);
         }
@@ -42,3 +47,28 @@ describe('The local repository', () => {
     );
   });
 });
+
+describe('Local Nginx Integration', () => {
+  it('Should serialize without errors', async () => {
+    const repository: TemplateManager = new TemplateManager(
+      path.join(__dirname, '../__data__/repository')
+    );
+    const integration = await repository.getIntegration('nginx');
+
+    await expect(integration?.serialize()).resolves.toHaveProperty('ok', true);
+  });
+
+  it('Should serialize to include the config', async () => {
+    const repository: TemplateManager = new TemplateManager(
+      path.join(__dirname, '../__data__/repository')
+    );
+    const integration = await repository.getIntegration('nginx');
+    const config = await integration!.getConfig();
+    const serialized = await integration!.serialize();
+
+    expect(serialized).toHaveProperty('ok', true);
+    expect((serialized as { value: object }).value).toMatchObject(
+      (config as { value: object }).value
+    );
+  });
+});