feat: allow extensions to read/write data

Composer/packages/electron-server/src/main.ts

@@ -62,7 +62,8 @@ async function createAppDataDir() {
   const localPublishPath: string = join(composerAppDataPath, 'hostedBots');
   const azurePublishPath: string = join(composerAppDataPath, 'publishBots');
   process.env.COMPOSER_APP_DATA = join(composerAppDataPath, 'data.json'); // path to the actual data file
-  process.env.COMPOSER_EXTENSION_DATA = join(composerAppDataPath, 'extensions.json');
+  process.env.COMPOSER_EXTENSION_MANIFEST = join(composerAppDataPath, 'extensions.json');
+  process.env.COMPOSER_EXTENSION_DATA_DIR = join(composerAppDataPath, 'extension-data');
   process.env.COMPOSER_REMOTE_EXTENSIONS_DIR = join(composerAppDataPath, 'extensions');
 
   log('creating composer app data path at: ', composerAppDataPath);

Composer/packages/extension/package.json

@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "@botframework-composer/types": "*",
+    "@types/debug": "^4.1.5",
     "@types/passport": "^1.0.3",
     "debug": "^4.1.1",
     "fs-extra": "^9.0.1",

Composer/packages/extension/src/__tests__/setupEnv.ts

@@ -3,6 +3,7 @@
 
 import path from 'path';
 
-process.env.COMPOSER_EXTENSION_DATA = path.resolve(__dirname, '__manifest__.json');
+process.env.COMPOSER_EXTENSION_MANIFEST = path.resolve(__dirname, '__manifest__.json');
+process.env.COMPOSER_EXTENSION_DATA_DIR = path.resolve(__dirname, '__extension-data__');
 process.env.COMPOSER_BUILTIN_EXTENSIONS_DIR = path.resolve(__dirname, '__builtin__');
 process.env.COMPOSER_REMOTE_EXTENSIONS_DIR = path.resolve(__dirname, '__remote__');

Composer/packages/extension/src/extensionContext.ts

@@ -1,20 +1,12 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import path from 'path';
-import fs from 'fs';
-
 import passport from 'passport';
 import { Express } from 'express';
 import { pathToRegexp } from 'path-to-regexp';
-import glob from 'globby';
 import formatMessage from 'format-message';
 import { UserIdentity, ExtensionCollection, RuntimeTemplate } from '@botframework-composer/types';
 
-import logger from './logger';
-import { ExtensionRegistration } from './extensionRegistration';
-
-const log = logger.extend('extension-context');
 export const DEFAULT_RUNTIME = 'csharp-azurewebapp';
 
 class ExtensionContext {
@@ -68,56 +60,6 @@ class ExtensionContext {
     });
   }
 
-  public async loadPlugin(name: string, description: string, thisPlugin: any) {
-    log('Loading extension: %s', name);
-
-    try {
-      const pluginRegistration = new ExtensionRegistration(this, name, description);
-      if (typeof thisPlugin.default === 'function') {
-        // the module exported just an init function
-        await thisPlugin.default.call(null, pluginRegistration);
-      } else if (thisPlugin.default && thisPlugin.default.initialize) {
-        // the module exported an object with an initialize method
-        await thisPlugin.default.initialize.call(null, pluginRegistration);
-      } else if (thisPlugin.initialize && typeof thisPlugin.initialize === 'function') {
-        // the module exported an object with an initialize method
-        await thisPlugin.initialize.call(null, pluginRegistration);
-      } else {
-        throw new Error(formatMessage('Could not init plugin'));
-      }
-    } catch (err) {
-      log('Error loading extension: %s', name);
-      // eslint-disable-next-line no-console
-      console.error(err);
-    }
-  }
-
-  public async loadPluginFromFile(packageJsonPath: string) {
-    const packageJSON = fs.readFileSync(packageJsonPath, 'utf8');
-    const json = JSON.parse(packageJSON);
-
-    if (json.composer?.enabled !== false) {
-      const modulePath = path.dirname(packageJsonPath);
-      try {
-        // eslint-disable-next-line security/detect-non-literal-require, @typescript-eslint/no-var-requires
-        const thisPlugin = require(modulePath);
-        this.loadPlugin(json.name, json.description, thisPlugin);
-      } catch (err) {
-        log('Error:', err?.message);
-      }
-    } else {
-      // noop - this is not a composer plugin
-    }
-  }
-
-  /** @deprecated */
-  public async loadPluginsFromFolder(dir: string) {
-    const plugins = await glob('*/package.json', { cwd: dir, dot: true });
-    for (const p in plugins) {
-      await this.loadPluginFromFile(path.join(dir, plugins[p]));
-    }
-  }
-
   // get the runtime template currently used from project
   public getRuntimeByProject(project): RuntimeTemplate {
     const type = project.settings.runtime?.key || DEFAULT_RUNTIME;

Composer/packages/extension/src/extensionRegistration.ts

@@ -1,20 +1,23 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import path from 'path';
+
 import { RequestHandler } from 'express-serve-static-core';
 import { Debugger } from 'debug';
 import { PublishPlugin, RuntimeTemplate, BotTemplate } from '@botframework-composer/types';
 
-import logger from './logger';
+import log from './logger';
 import { ExtensionContext } from './extensionContext';
-
-const log = logger.extend('extension-registration');
+import { Store } from './storage/store';
 
 export class ExtensionRegistration {
   public context: typeof ExtensionContext;
   private _name: string;
   private _description: string;
   private _log: Debugger;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private _store: Store<any> | null = null;
 
   constructor(context: typeof ExtensionContext, name: string, description: string) {
     this.context = context;
@@ -43,9 +46,19 @@ export class ExtensionRegistration {
     return this._log;
   }
 
+  public get store() {
+    if (this._store === null) {
+      const storePath = path.join(this.dataDir, `${this.name}.json`);
+      this._store = new Store(storePath, {}, this.log);
+    }
+
+    return this._store;
+  }
+
   /**************************************************************************************
    * Storage related features
    *************************************************************************************/
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   public async useStorage(customStorageClass: any) {
     if (!this.context.extensions.storage.customStorageClass) {
       this.context.extensions.storage.customStorageClass = customStorageClass;
@@ -208,4 +221,13 @@ export class ExtensionRegistration {
       this.context.extensions.authentication.allowedUrls.push(url);
     }
   }
+
+  private get dataDir() {
+    /* istanbul ignore next */
+    if (!process.env.COMPOSER_EXTENSION_DATA_DIR) {
+      throw new Error('COMPOSER_EXTENSION_DATA_DIR must be set.');
+    }
+
+    return process.env.COMPOSER_EXTENSION_DATA_DIR;
+  }
 }

Composer/packages/extension/src/manager/manager.ts

@@ -12,6 +12,7 @@ import logger from '../logger';
 import { ExtensionManifestStore } from '../storage/extensionManifestStore';
 import { search, downloadPackage } from '../utils/npm';
 import { isSubdirectory } from '../utils/isSubdirectory';
+import { ExtensionRegistration } from '../extensionRegistration';
 
 const log = logger.extend('manager');
 
@@ -59,6 +60,7 @@ export class ExtensionManagerImp {
    */
   public async loadAll() {
     await ensureDir(this.remoteDir);
+    await ensureDir(this.dataDir);
 
     await this.loadFromDir(this.builtinDir, true);
     await this.loadFromDir(this.remoteDir);
@@ -131,11 +133,23 @@ export class ExtensionManagerImp {
       // eslint-disable-next-line @typescript-eslint/no-var-requires, security/detect-non-literal-require
       const extension = metadata?.path && require(metadata.path);
 
-      if (!extension) {
+      if (!extension || !metadata) {
         throw new Error(`Extension not found: ${id}`);
       }
 
-      await ExtensionContext.loadPlugin(id, '', extension);
+      const registration = new ExtensionRegistration(ExtensionContext, metadata.id, metadata.description);
+      if (typeof extension.default === 'function') {
+        // the module exported just an init function
+        await extension.default.call(null, registration);
+      } else if (extension.default && extension.default.initialize) {
+        // the module exported an object with an initialize method
+        await extension.default.initialize.call(null, registration);
+      } else if (extension.initialize && typeof extension.initialize === 'function') {
+        // the module exported an object with an initialize method
+        await extension.initialize.call(null, registration);
+      } else {
+        throw new Error('Could not init extension');
+      }
     } catch (err) {
       log('Unable to load extension `%s`', id);
       log('%O', err);
@@ -245,7 +259,7 @@ export class ExtensionManagerImp {
   private get manifest() {
     /* istanbul ignore next */
     if (!this._manifest) {
-      this._manifest = new ExtensionManifestStore(process.env.COMPOSER_EXTENSION_DATA as string);
+      this._manifest = new ExtensionManifestStore(process.env.COMPOSER_EXTENSION_MANIFEST as string);
     }
 
     return this._manifest;
@@ -268,6 +282,15 @@ export class ExtensionManagerImp {
 
     return process.env.COMPOSER_REMOTE_EXTENSIONS_DIR;
   }
+
+  private get dataDir() {
+    /* istanbul ignore next */
+    if (!process.env.COMPOSER_EXTENSION_DATA_DIR) {
+      throw new Error('COMPOSER_EXTENSION_DATA_DIR must be set.');
+    }
+
+    return process.env.COMPOSER_EXTENSION_DATA_DIR;
+  }
 }
 
 const ExtensionManager = new ExtensionManagerImp();

Composer/packages/extension/src/storage/__tests__/extensionManifestStore.test.ts

@@ -1,113 +1,57 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import fs from 'fs';
-import path from 'path';
+import { ExtensionManifestStore, ExtensionManifest } from '../extensionManifestStore';
 
-import { existsSync, writeJsonSync, readJsonSync } from 'fs-extra';
+const manifestPath = '../../../__manifest__.json';
 
-import { ExtensionManifestStore } from '../extensionManifestStore';
+const currentManifest = ({
+  extension1: {
+    id: 'extension1',
+  },
+} as unknown) as ExtensionManifest;
 
-jest.mock('fs-extra', () => ({
-  existsSync: jest.fn(),
-  readJsonSync: jest.fn(),
-  writeJsonSync: jest.fn(),
-}));
+let store = new ExtensionManifestStore(manifestPath);
 
-const manifestPath = path.resolve(__dirname, '../../../__manifest__.json');
-
-afterAll(() => {
-  if (fs.existsSync(manifestPath)) {
-    fs.unlinkSync(manifestPath);
-  }
-});
-
-describe('when the manifest does not exist', () => {
-  it('creates one with the default data', () => {
-    (existsSync as jest.Mock).mockReturnValue(false);
-    new ExtensionManifestStore(manifestPath);
-    expect(writeJsonSync).toHaveBeenCalledWith(manifestPath, {}, { spaces: 2 });
+beforeEach(() => {
+  store = new ExtensionManifestStore(manifestPath);
+  Object.entries(currentManifest).forEach(([id, data]) => {
+    store.updateExtensionConfig(id, data ?? {});
   });
 });
 
-describe('when the manifest already exists', () => {
-  const currentManifest = {
-    extension1: {
-      id: 'extension1',
-    },
-  };
-
-  beforeAll(() => {
-    if (fs.existsSync(manifestPath)) {
-      fs.unlinkSync(manifestPath);
-    }
-
-    fs.writeFileSync(manifestPath, JSON.stringify({}));
-  });
-
-  beforeEach(() => {
-    (existsSync as jest.Mock).mockReturnValue(true);
-    (readJsonSync as jest.Mock).mockImplementation((path) => {
-      if (path === manifestPath) {
-        return { ...currentManifest };
-      }
-    });
+describe('#getExtensionConfig', () => {
+  it('returns the extension metadata', () => {
+    expect(store.getExtensionConfig('extension1')).toEqual(currentManifest.extension1);
   });
+});
 
-  it('reads from the manifest', () => {
-    new ExtensionManifestStore(manifestPath);
-    expect(readJsonSync).toHaveBeenCalledWith(manifestPath);
-  });
-
-  describe('#getExtensionConfig', () => {
-    it('returns the extension metadata', () => {
-      const store = new ExtensionManifestStore(manifestPath);
-
-      expect(store.getExtensionConfig('extension1')).toEqual(currentManifest.extension1);
-    });
+describe('#getExtensions', () => {
+  it('returns all extension metadata', () => {
+    expect(store.getExtensions()).toEqual(currentManifest);
   });
+});
 
-  describe('#getExtensions', () => {
-    it('returns all extension metadata', () => {
-      const store = new ExtensionManifestStore(manifestPath);
+describe('#updateExtensionConfig', () => {
+  it('creates a new entry if config does not exist', () => {
+    const newExtension = { id: 'newExtension' };
+    store.updateExtensionConfig('newExtension', newExtension);
 
-      expect(store.getExtensions()).toEqual(currentManifest);
-    });
+    expect(store.getExtensionConfig('newExtension')).toEqual(newExtension);
   });
 
-  describe('#updateExtensionConfig', () => {
-    it('creates a new entry if config does not exist', () => {
-      const newExtension = { id: 'newExtension' };
-      const store = new ExtensionManifestStore(manifestPath);
-      store.updateExtensionConfig('newExtension', newExtension);
+  it('updates the entry if config exist', () => {
+    store.updateExtensionConfig('extension1', { name: 'new name' });
 
-      expect(writeJsonSync).toHaveBeenCalledWith(
-        manifestPath,
-        { ...currentManifest, newExtension },
-        expect.any(Object)
-      );
-    });
-
-    it('updates the entry if config exist', () => {
-      const store = new ExtensionManifestStore(manifestPath);
-      store.updateExtensionConfig('extension1', { name: 'new name' });
-
-      expect(writeJsonSync).toHaveBeenCalledWith(
-        manifestPath,
-        { extension1: { id: 'extension1', name: 'new name' } },
-        expect.any(Object)
-      );
-    });
+    expect(store.getExtensionConfig('extension1')).toEqual({ id: 'extension1', name: 'new name' });
   });
+});
 
-  describe('#removeExtension', () => {
-    it('removes the extension from the manifest', () => {
-      const store = new ExtensionManifestStore(manifestPath);
-      store.removeExtension('extension1');
+describe('#removeExtension', () => {
+  it('removes the extension from the manifest', () => {
+    store.removeExtension('extension1');
 
-      expect(writeJsonSync).toHaveBeenCalledWith(manifestPath, {}, expect.any(Object));
-      expect(store.getExtensionConfig('extension1')).toBeUndefined();
-      expect(store.getExtensions()).toEqual({});
-    });
+    expect(store.getExtensionConfig('extension1')).toBeUndefined();
+    expect(store.getExtensions()).toEqual({});
   });
 });