feature: Supporting core+package templates stored on NuGet

Composer/packages/client/__tests__/components/home.test.tsx

@@ -3,7 +3,7 @@
 
 import * as React from 'react';
 import { fireEvent, render } from '@botframework-composer/test-utils';
-import { ProjectTemplate } from '@bfc/shared';
+import { BotTemplate } from '@bfc/shared';
 
 import { RecentBotList } from '../../src/pages/home/RecentBotList';
 import { ExampleList } from '../../src/pages/home/ExampleList';
@@ -32,7 +32,7 @@ describe('<Home/>', () => {
     const templates = [
       { description: 'echo bot', id: 'EchoBot', name: 'Echo Bot' },
       { description: 'empty bot', id: 'EmptyBot', name: 'Empty Bot' },
-    ] as ProjectTemplate[];
+    ] as BotTemplate[];
     const onClickTemplate = jest.fn((item) => item);
     const { container, getByText } = render(<ExampleList examples={templates} onClick={onClickTemplate} />);
     expect(container).toHaveTextContent('Echo Bot');

Composer/packages/client/src/components/CreationFlow/CreateOptions.tsx

@@ -20,7 +20,7 @@ import {
   DetailsRow,
 } from 'office-ui-fabric-react/lib/DetailsList';
 import { Sticky, StickyPositionType } from 'office-ui-fabric-react/lib/Sticky';
-import { ProjectTemplate } from '@bfc/shared';
+import { BotTemplate } from '@bfc/shared';
 import { DialogWrapper, DialogTypes } from '@bfc/ui-shared';
 import { NeutralColors } from '@uifabric/fluent-theme';
 import { RouteComponentProps } from '@reach/router';
@@ -105,7 +105,7 @@ const optionKeys = {
 
 // -------------------- CreateOptions -------------------- //
 type CreateOptionsProps = {
-  templates: ProjectTemplate[];
+  templates: BotTemplate[];
   onDismiss: () => void;
   onNext: (data: string) => void;
 } & RouteComponentProps<{}>;
@@ -119,7 +119,7 @@ export function CreateOptions(props: CreateOptionsProps) {
   const selection = useMemo(() => {
     return new Selection({
       onSelectionChanged: () => {
-        const t = selection.getSelection()[0] as ProjectTemplate;
+        const t = selection.getSelection()[0] as BotTemplate;
         if (t) {
           setCurrentTemplate(t.id);
         }

Composer/packages/client/src/pages/home/ExampleList.tsx

@@ -6,7 +6,7 @@ import { jsx } from '@emotion/core';
 import React from 'react';
 import { ScrollablePane, ScrollbarVisibility } from 'office-ui-fabric-react/lib/ScrollablePane';
 import { List } from 'office-ui-fabric-react/lib/List';
-import { ProjectTemplate } from '@bfc/shared';
+import { BotTemplate } from '@bfc/shared';
 
 import * as exampleIcons from '../../images/samples';
 
@@ -20,7 +20,7 @@ import {
 } from './styles';
 
 interface ExampleListProps {
-  examples: ProjectTemplate[];
+  examples: BotTemplate[];
   onClick: (templateId: string) => void;
 }
 
@@ -35,7 +35,7 @@ const resolveIcon = (exampleId: string): string => {
 export const ExampleList: React.FC<ExampleListProps> = (props) => {
   const { onClick, examples } = props;
 
-  function onRenderCell(item?: ProjectTemplate): React.ReactNode {
+  function onRenderCell(item?: BotTemplate): React.ReactNode {
     if (!item) {
       return;
     }

Composer/packages/client/src/recoilModel/atoms/appState.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import { atom, atomFamily } from 'recoil';
-import { FormDialogSchemaTemplate, FeatureFlagMap, ProjectTemplate, UserSettings } from '@bfc/shared';
+import { FormDialogSchemaTemplate, FeatureFlagMap, BotTemplate, UserSettings } from '@bfc/shared';
 import { ExtensionMetadata } from '@bfc/extension-client';
 
 import {
@@ -52,7 +52,7 @@ export const recentProjectsState = atom<any[]>({
   default: [],
 });
 
-export const templateProjectsState = atom<ProjectTemplate[]>({
+export const templateProjectsState = atom<BotTemplate[]>({
   key: getFullyQualifiedKey('templateProjects'),
   default: [],
 });

Composer/packages/client/src/recoilModel/selectors/projectTemplates.ts

@@ -18,6 +18,16 @@ export const filteredTemplatesSelector = selector({
         filteredTemplates.splice(vaTemplateIndex, 1);
       }
     }
+    if (!featureFlags?.REMOTE_TEMPLATE_CREATION_EXPERIENCE?.enabled) {
+      let index = filteredTemplates.length - 1;
+
+      while (index >= 0) {
+        if (!filteredTemplates[index].path) {
+          filteredTemplates.splice(index, 1);
+        }
+        index -= 1;
+      }
+    }
     return filteredTemplates;
   },
 });

Composer/packages/lib/shared/src/featureFlagUtils/index.ts

@@ -14,7 +14,7 @@ export type FeatureFlag = {
   enabled: boolean;
 };
 
-export type FeatureFlagKey = 'VA_CREATION' | 'FORM_DIALOG';
+export type FeatureFlagKey = 'VA_CREATION' | 'FORM_DIALOG' | 'REMOTE_TEMPLATE_CREATION_EXPERIENCE';
 
 export type FeatureFlagMap = Record<FeatureFlagKey, FeatureFlag>;
 
@@ -31,4 +31,10 @@ export const getDefaultFeatureFlags = (): FeatureFlagMap => ({
     isHidden: false,
     enabled: false,
   },
+  REMOTE_TEMPLATE_CREATION_EXPERIENCE: {
+    displayName: formatMessage('Remote template creation experience'),
+    description: formatMessage('Changes the new bot flow to use templates stored as packages on Nuget/npm'),
+    isHidden: false,
+    enabled: false,
+  },
 });

Composer/packages/server/package.json

@@ -62,6 +62,7 @@
     "@bfc/lg-languageserver": "*",
     "@bfc/lu-languageserver": "*",
     "@bfc/shared": "*",
+    "@microsoft/bf-dialog": "4.11.0-dev.20201025.69cf2b9",
     "@microsoft/bf-dispatcher": "^4.11.0-beta.20201016.393c6b2",
     "@microsoft/bf-generate-library": "^4.10.0-daily.20201026.178799",
     "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128",

Composer/packages/server/src/controllers/project.ts

@@ -6,6 +6,7 @@ import * as fs from 'fs';
 import { Request, Response } from 'express';
 import { Archiver } from 'archiver';
 import { ExtensionContext } from '@bfc/extension';
+import { SchemaMerger } from '@microsoft/bf-dialog/lib/library/schemaMerger';
 
 import log from '../logger';
 import { BotProjectService } from '../services/project';
@@ -54,6 +55,33 @@ async function createProject(req: Request, res: Response) {
     await AssetService.manager.copyBoilerplate(currentProject.dataDir, currentProject.fileStorage);
 
     if (currentProject !== undefined) {
+      if (currentProject?.settings?.runtime.customRuntime === true) {
+        const runtime = ExtensionContext.getRuntimeByProject(currentProject);
+        const runtimePath = currentProject.settings.runtime.path;
+
+        if (!fs.existsSync(runtimePath)) {
+          await runtime.eject(currentProject, currentProject.fileStorage);
+        }
+
+        // install all dependencies and build the app
+        await runtime.build(runtimePath, currentProject);
+
+        const manifestFile = runtime.identifyManifest(runtimePath);
+
+        // run the merge command to merge all package dependencies from the template to the bot project
+        const realMerge = new SchemaMerger(
+          [manifestFile],
+          Path.join(currentProject.dataDir, 'schemas/sdk'),
+          Path.join(currentProject.dataDir, 'dialogs/imported'),
+          false,
+          false,
+          console.log,
+          console.warn,
+          console.error
+        );
+
+        await realMerge.merge();
+      }
       await currentProject.updateBotInfo(name, description, preserveRoot);
       if (schemaUrl) {
         await currentProject.saveSchemaToProject(schemaUrl, locationRef.path);

Composer/packages/server/src/locales/en-US.json

@@ -464,6 +464,9 @@
   "change_recognizer_3145b93d": {
     "message": "Change Recognizer"
   },
+  "changes_the_new_bot_flow_to_use_templates_stored_a_bfe5178c": {
+    "message": "Changes the new bot flow to use templates stored as packages on Nuget/npm"
+  },
   "check_for_updates_and_install_them_automatically_50337340": {
     "message": "Check for updates and install them automatically."
   },
@@ -2075,6 +2078,9 @@
   "reloading_49d2f661": {
     "message": "Reloading"
   },
+  "remote_template_creation_experience_2f9ea9e1": {
+    "message": "Remote template creation experience"
+  },
   "remove_f47dc62a": {
     "message": "Remove"
   },

Composer/packages/server/src/models/asset/assetManager.ts

@@ -3,9 +3,12 @@
 
 import fs from 'fs';
 import path from 'path';
+import { exec } from 'child_process';
+import { promisify } from 'util';
 
+import del from 'del';
 import find from 'lodash/find';
-import { UserIdentity, ExtensionContext } from '@bfc/extension';
+import { UserIdentity, ExtensionContext, BotTemplate } from '@bfc/extension';
 
 import log from '../../logger';
 import { LocalDiskStorage } from '../storage/localDiskStorage';
@@ -16,6 +19,8 @@ import StorageService from '../../services/storage';
 import { IFileStorage } from '../storage/interface';
 import { BotProject } from '../bot/botProject';
 
+const execAsync = promisify(exec);
+
 export class AssetManager {
   public templateStorage: LocalDiskStorage;
   private _botProjectFileTemplate;
@@ -52,13 +57,52 @@ export class AssetManager {
     return ref;
   }
 
+  private async getRemoteTemplate(template: BotTemplate, destinationPath: string) {
+    // install package
+    const { stderr: initErr } = await execAsync(`dotnet new -i ${template.packageName}`);
+    if (initErr) {
+      throw new Error(initErr);
+    }
+    const { stderr: initErr2 } = await execAsync(`dotnet new ${template.id}`, {
+      cwd: destinationPath,
+    });
+    if (initErr2) {
+      throw new Error(initErr2);
+    }
+  }
+
   private async copyDataFilesTo(templateId: string, dstDir: string, dstStorage: IFileStorage, locale?: string) {
     const template = find(ExtensionContext.extensions.botTemplates, { id: templateId });
-    if (template === undefined || template.path === undefined) {
+    if (template === undefined || (template.path === undefined && template.packageName === undefined)) {
       throw new Error(`no such template with id ${templateId}`);
     }
-    // copy Composer data files
-    await copyDir(template.path, this.templateStorage, dstDir, dstStorage);
+
+    let templateSrcPath = template.path;
+    const isHostedTemplate = !templateSrcPath;
+    if (isHostedTemplate) {
+      // create empty temp directory on server for holding externally hosted template src
+      templateSrcPath = path.resolve(__dirname, '../../../temp');
+      fs.mkdir(templateSrcPath, (err) => {
+        if (err) {
+          throw new Error('Error creating temp directory for external template storage');
+        }
+      });
+      await this.getRemoteTemplate(template, templateSrcPath);
+    }
+
+    if (templateSrcPath) {
+      // copy Composer data files
+      await copyDir(templateSrcPath, this.templateStorage, dstDir, dstStorage);
+
+      if (isHostedTemplate) {
+        try {
+          await del(templateSrcPath);
+        } catch (err) {
+          throw new Error('Issue deleting temp generated file for external template assets');
+        }
+      }
+    }
+
     // if we have a locale override, copy those files over too
     if (locale != null) {
       const localePath = path.join(__dirname, '..', '..', '..', 'schemas', `sdk.${locale}.schema`);
@@ -76,7 +120,7 @@ export class AssetManager {
   public async copyBoilerplate(dstDir: string, dstStorage: IFileStorage) {
     for (const boilerplate of ExtensionContext.extensions.baseTemplates) {
       const boilerplatePath = boilerplate.path;
-      if (await this.templateStorage.exists(boilerplatePath)) {
+      if (boilerplatePath && (await this.templateStorage.exists(boilerplatePath))) {
         await copyDir(boilerplatePath, this.templateStorage, dstDir, dstStorage);
       }
     }
@@ -109,19 +153,21 @@ export class AssetManager {
       return undefined;
     }
     const boilerplate = ExtensionContext.extensions.baseTemplates[0];
-    const location = Path.join(boilerplate.path, 'scripts', 'package.json');
-    try {
-      if (fs.existsSync(location)) {
-        const raw = fs.readFileSync(location, 'utf8');
-        const json = JSON.parse(raw);
-        if (json && json.version) {
-          return json.version;
-        } else {
-          return undefined;
+    if (boilerplate.path) {
+      const location = Path.join(boilerplate.path, 'scripts', 'package.json');
+      try {
+        if (fs.existsSync(location)) {
+          const raw = fs.readFileSync(location, 'utf8');
+          const json = JSON.parse(raw);
+          if (json && json.version) {
+            return json.version;
+          } else {
+            return undefined;
+          }
         }
+      } catch (err) {
+        return undefined;
       }
-    } catch (err) {
-      return undefined;
     }
   }
 
@@ -131,15 +177,17 @@ export class AssetManager {
     }
     const boilerplate = ExtensionContext.extensions.botTemplates[0];
 
-    const location = Path.join(boilerplate.path, `${boilerplate.id}.botproj`);
-    try {
-      if (fs.existsSync(location)) {
-        const raw = fs.readFileSync(location, 'utf8');
-        const json = JSON.parse(raw);
-        return json;
+    if (boilerplate.path) {
+      const location = Path.join(boilerplate.path, `${boilerplate.id}.botproj`);
+      try {
+        if (fs.existsSync(location)) {
+          const raw = fs.readFileSync(location, 'utf8');
+          const json = JSON.parse(raw);
+          return json;
+        }
+      } catch (err) {
+        return '';
       }
-    } catch (err) {
-      return '';
     }
   }
 }

Composer/packages/server/src/models/bot/botProject.ts

@@ -199,6 +199,15 @@ export class BotProject implements IBotProject {
   public getEnvSettings = async (obfuscate: boolean) => {
     const settings = await this.settingManager.get(obfuscate);
 
+    // Resolve relative path for custom runtime if the path is relative
+    if (settings?.runtime?.customRuntime && settings.runtime.path && !Path.isAbsolute(settings.runtime.path)) {
+      const absolutePath = Path.resolve(this.dir, 'settings', settings.runtime.path);
+      if (fs.existsSync(absolutePath)) {
+        settings.runtime.path = absolutePath;
+        await this.updateEnvSettings(settings);
+      }
+    }
+
     // fix old bot have no language settings
     if (!settings?.defaultLanguage) {
       settings.defaultLanguage = defaultLanguage;

Composer/packages/types/src/runtime.ts

@@ -37,11 +37,15 @@ export type BotTemplate = {
   name: string;
   description: string;
   /* absolute path */
-  path: string;
+  path?: string;
   /* tags for further grouping and search secenario */
   tags?: string[];
   /* list of supported runtime versions */
   support?: string[];
+  packageName?: string;
+  packageSource?: string;
+  packageVersion?: string;
+  index?: number;
 };
 
 export type RuntimeTemplate = {
@@ -53,6 +57,8 @@ export type RuntimeTemplate = {
 
   run: (project: IBotProject, localDisk?: any) => Promise<void>;
 
+  identifyManifest: (runtimePath: string) => string;
+
   /** build for deploy method */
   buildDeploy: (
     runtimePath: string,