diff --git a/extensions/azurePublish/src/node/deploy.ts b/extensions/azurePublish/src/node/deploy.ts index e2eb58739c..d76bd46cba 100644 --- a/extensions/azurePublish/src/node/deploy.ts +++ b/extensions/azurePublish/src/node/deploy.ts @@ -9,10 +9,12 @@ import * as rp from 'request-promise'; import archiver from 'archiver'; import { AzureBotService } from '@azure/arm-botservice'; import { TokenCredentials } from '@azure/ms-rest-js'; +import { composeRenderFunction } from '@uifabric/utilities'; import { BotProjectDeployConfig, BotProjectDeployLoggerType } from './types'; import { build, publishLuisToPrediction } from './luisAndQnA'; import { AzurePublishErrors, createCustomizeError, stringifyError } from './utils/errorHandler'; +import { copyDir } from './utils/copyDir'; import { KeyVaultApi } from './keyvaultHelper/keyvaultApi'; import { KeyVaultApiConfig } from './keyvaultHelper/keyvaultApiConfig'; @@ -114,13 +116,24 @@ export class BotProjectDeploy { // this returns a pathToArtifacts where the deployable version lives. const pathToArtifacts = await this.runtime.buildDeploy(this.projPath, project, settings, profileName); + // COPY MANIFESTS TO wwwroot/manifests + // eslint-disable-next-line security/detect-non-literal-fs-filename + if (await project.fileStorage.exists(path.join(pathToArtifacts, 'manifests'))) { + await copyDir( + path.join(pathToArtifacts, 'manifests'), + project.fileStorage, + path.join(pathToArtifacts, 'wwwroot', 'manifests'), + project.fileStorage + ); + } + // STEP 4: ZIP THE ASSETS // Build a zip file of the project this.logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: 'Creating build artifact...', }); - await this.zipDirectory(pathToArtifacts, this.zipPath); + await this.zipDirectory(pathToArtifacts, settings, this.zipPath); this.logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: 'Build artifact ready!', @@ -146,7 +159,7 @@ export class BotProjectDeploy { } } - private async zipDirectory(source: string, out: string) { + private async zipDirectory(source: string, settings: any, out: string) { console.log(`Zip the files in ${source} into a zip file ${out}`); try { const archive = archiver('zip', { zlib: { level: 9 } }); @@ -157,12 +170,15 @@ export class BotProjectDeploy { .glob('**/*', { cwd: source, dot: true, - ignore: ['**/code.zip'], // , 'node_modules/**/*' + ignore: ['**/code.zip', '**/settings/appsettings.json'], }) .on('error', (err) => reject(err)) .pipe(stream); stream.on('close', () => resolve()); + + // write the merged settings to the deploy artifact + archive.append(JSON.stringify(settings, null, 2), { name: 'settings/appsettings.json' }); archive.finalize(); }); } catch (error) { diff --git a/extensions/azurePublish/src/node/index.ts b/extensions/azurePublish/src/node/index.ts index 87543cee36..d67a56e3c6 100644 --- a/extensions/azurePublish/src/node/index.ts +++ b/extensions/azurePublish/src/node/index.ts @@ -5,7 +5,7 @@ import path from 'path'; import formatMessage from 'format-message'; import md5 from 'md5'; -import { copy, rmdir, emptyDir, readJson, pathExists, writeJson, mkdirSync, writeFileSync } from 'fs-extra'; +import { readJson, pathExists, writeJson } from 'fs-extra'; import { Debugger } from 'debug'; import { IBotProject, @@ -26,7 +26,7 @@ import { BotProjectProvision } from './provision'; import { BackgroundProcessManager } from './backgroundProcessManager'; import { ProvisionConfig } from './provision'; import schema from './schema'; -import { stringifyError, AzurePublishErrors, createCustomizeError } from './utils/errorHandler'; +import { stringifyError } from './utils/errorHandler'; import { ProcessStatus } from './types'; // This option controls whether the history is serialized to a file between sessions with Composer @@ -52,12 +52,6 @@ interface PublishConfig { [key: string]: any; } -interface ResourceType { - key: string; - // other keys TBD - [key: string]: any; -} - interface ProvisionHistoryItem { profileName: string; jobId: string; // use for unique each provision @@ -111,32 +105,6 @@ export default async (composer: IExtensionRegistration): Promise => { this.bundleId = bundleId; } - private baseRuntimeFolder = process.env.AZURE_PUBLISH_PATH || path.resolve(__dirname, `../../publishBots`); - - /*******************************************************************************************************************************/ - /* These methods generate all the necessary paths to various files */ - /*******************************************************************************************************************************/ - - private getRuntimeTemplateMode = (runtimeKey?: string): string => { - // The "mode" is kept the same for backward compatibility with original folder names - const { runtimeType } = parseRuntimeKey(runtimeKey); - return runtimeType === 'functions' ? 'azurefunctions' : 'azurewebapp'; - }; - - // path to working folder containing all the assets - private getRuntimeFolder = (key: string) => { - return path.resolve(this.baseRuntimeFolder, `${key}`); - }; - - // path to the runtime code inside the working folder - private getProjectFolder = (key: string, template: string) => { - return path.resolve(this.baseRuntimeFolder, `${key}/${template}`); - }; - - // path to the declarative assets - private getBotFolder = (key: string, template: string) => - path.resolve(this.getProjectFolder(key, template), 'ComposerDialogs'); - /*******************************************************************************************************************************/ /* These methods deal with the publishing history displayed in the Composer UI */ /*******************************************************************************************************************************/ @@ -182,84 +150,6 @@ export default async (composer: IExtensionRegistration): Promise => { /* These methods implement the publish actions */ /*******************************************************************************************************************************/ /** - * Prepare a bot to be built and deployed by copying the runtime and declarative assets into a temporary folder - * @param project - * @param settings - * @param srcTemplate - * @param resourcekey - */ - private init = async (project: any, srcTemplate: string, resourcekey: string, runtime: any) => { - try { - // point to the declarative assets (possibly in remote storage) - const botFiles = project.getProject().files; - - const mode = this.getRuntimeTemplateMode(runtime?.key); - - // include both pre-release and release identifiers here - // TODO: eventually we can clean this up when the "old" runtime is deprecated - // (old runtime support is the else block below) - if (isUsingAdaptiveRuntime(runtime)) { - const buildFolder = this.getProjectFolder(resourcekey, mode); - - // clean up from any previous deploys - await this.cleanup(resourcekey); - - // copy bot and runtime into projFolder - await copy(srcTemplate, buildFolder); - } else { - const botFolder = this.getBotFolder(resourcekey, mode); - const runtimeFolder = this.getRuntimeFolder(resourcekey); - - // clean up from any previous deploys - await this.cleanup(resourcekey); - - // create the temporary folder to contain this project - mkdirSync(runtimeFolder, { recursive: true }); - - // create the ComposerDialogs/ folder - mkdirSync(botFolder, { recursive: true }); - - let manifestPath; - for (const file of botFiles) { - const pattern = /manifests\/[0-9A-z-]*.json/; - if (file.relativePath.match(pattern)) { - manifestPath = path.dirname(file.path); - } - // save bot files - const filePath = path.resolve(botFolder, file.relativePath); - if (!(await pathExists(path.dirname(filePath)))) { - mkdirSync(path.dirname(filePath), { recursive: true }); - } - writeFileSync(filePath, file.content); - } - - // save manifest - runtime.setSkillManifest(runtimeFolder, project.fileStorage, manifestPath, project.fileStorage, mode); - - // copy bot and runtime into projFolder - await copy(srcTemplate, runtimeFolder); - } - } catch (error) { - throw createCustomizeError( - AzurePublishErrors.INITIALIZE_ERROR, - `Error during init publish folder, ${error.message}` - ); - } - }; - - /** - * Remove any previous version of a project's working files - * @param resourcekey - */ - private async cleanup(resourcekey: string) { - try { - const projFolder = this.getRuntimeFolder(resourcekey); - await emptyDir(projFolder); - await rmdir(projFolder); - } catch (error) { - this.logger('$O', error); - } - } /** * Take the project from a given folder, build it, and push it to Azure. @@ -294,7 +184,7 @@ export default async (composer: IExtensionRegistration): Promise => { } }, accessToken: accessToken, - projPath: this.getProjectFolder(resourcekey, mode), + projPath: project.getRuntimePath(), runtime: runtime, }); @@ -311,8 +201,6 @@ export default async (composer: IExtensionRegistration): Promise => { await this.updateHistory(botId, profileName, publishResultFromStatus(status).result); // clean up the background process BackgroundProcessManager.removeProcess(jobId); - // clean up post-deploy - await this.cleanup(resourcekey); }; /*******************************************************************************************************************************/ @@ -341,20 +229,6 @@ export default async (composer: IExtensionRegistration): Promise => { // get the appropriate runtime template which contains methods to build and configure the runtime const runtime = composer.getRuntimeByProject(project); // set runtime code path as runtime template folder path - let runtimeCodePath = runtime.path; - - // If the project is using an "ejected" runtime, use that version of the code instead of the built-in template - if ( - project.settings && - project.settings.runtime && - project.settings.runtime.customRuntime === true && - project.settings.runtime.path - ) - runtimeCodePath = project.getRuntimePath(); // get computed absolute path - - // Prepare the temporary project - // this writes all the settings to the root settings/appsettings.json file - await this.init(project, runtimeCodePath, resourcekey, runtime); // Merge all the settings // this combines the bot-wide settings, the environment specific settings, and 2 new fields needed for deployed bots @@ -390,7 +264,6 @@ export default async (composer: IExtensionRegistration): Promise => { publishResultFromStatus(BackgroundProcessManager.getStatus(jobId)).result ); BackgroundProcessManager.removeProcess(jobId); - this.cleanup(resourcekey); } }; @@ -567,8 +440,6 @@ export default async (composer: IExtensionRegistration): Promise => { const status = publishResultFromStatus(BackgroundProcessManager.getStatus(jobId)); await this.updateHistory(botId, profileName, status.result); BackgroundProcessManager.removeProcess(jobId); - this.cleanup(resourcekey as string); - return status; } }; diff --git a/extensions/azurePublish/src/node/utils/copyDir.ts b/extensions/azurePublish/src/node/utils/copyDir.ts new file mode 100644 index 0000000000..d5b321adef --- /dev/null +++ b/extensions/azurePublish/src/node/utils/copyDir.ts @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +/* eslint-disable security/detect-non-literal-fs-filename */ + +import Path from 'path'; + +import { IFileStorage } from './interface'; + +export async function copyDir( + srcDir: string, + srcStorage: IFileStorage, + dstDir: string, + dstStorage: IFileStorage, + pathsToExclude?: Set +) { + if (!(await srcStorage.exists(srcDir)) || !(await srcStorage.stat(srcDir)).isDir) { + throw new Error(`No such dir ${srcDir}}`); + } + + if (!(await dstStorage.exists(dstDir))) { + await dstStorage.mkDir(dstDir, { recursive: true }); + } + + const paths = await srcStorage.readDir(srcDir); + + for (const path of paths) { + const srcPath = Path.join(srcDir, path); + if (pathsToExclude?.has(srcPath)) { + continue; + } + const dstPath = Path.join(dstDir, path); + + if ((await srcStorage.stat(srcPath)).isFile) { + // copy files + const content = await srcStorage.readFile(srcPath); + await dstStorage.writeFile(dstPath, content); + } else { + // recursively copy dirs + await copyDir(srcPath, srcStorage, dstPath, dstStorage, pathsToExclude); + } + } +} diff --git a/extensions/runtimes/src/index.ts b/extensions/runtimes/src/index.ts index 2b91b0b996..abe25ce9ac 100644 --- a/extensions/runtimes/src/index.ts +++ b/extensions/runtimes/src/index.ts @@ -316,13 +316,6 @@ export default async (composer: any): Promise => { } throw new Error(`Runtime already exists at ${destPath}`); }, - setSkillManifest: async ( - dstRuntimePath: string, - dstStorage: IFileStorage, - srcManifestDir: string, - srcStorage: IFileStorage, - mode = 'azurewebapp' - ) => {}, }); /** @@ -431,36 +424,9 @@ export default async (composer: any): Promise => { return; } - // write settings to disk in the appropriate location - const settingsPath = path.join(publishFolder, 'settings', 'appsettings.json'); - if (!(await fs.pathExists(path.dirname(settingsPath)))) { - await fs.mkdirp(path.dirname(settingsPath)); - } - await fs.writeFile(settingsPath, JSON.stringify(settings, null, 2)); - // return the location of the build artifiacts return publishFolder; }, - setSkillManifest: async ( - dstRuntimePath: string, - dstStorage: IFileStorage, - srcManifestDir: string, - srcStorage: IFileStorage, - mode = 'azurewebapp' // set default as azurewebapp - ) => { - // update manifst into runtime wwwroot - if (mode === 'azurewebapp') { - const manifestDstDir = path.resolve(dstRuntimePath, 'azurewebapp', 'wwwroot', 'manifests'); - - if (await fs.pathExists(manifestDstDir)) { - await removeDirAndFiles(manifestDstDir); - } - - if (await fs.pathExists(srcManifestDir)) { - await copyDir(srcManifestDir, srcStorage, manifestDstDir, dstStorage); - } - } - }, }); composer.addRuntimeTemplate({ @@ -569,37 +535,9 @@ export default async (composer: any): Promise => { throw err; return; } - - // write settings to disk in the appropriate location - const settingsPath = path.join(publishFolder, 'settings', 'appsettings.json'); - if (!(await fs.pathExists(path.dirname(settingsPath)))) { - await fs.mkdirp(path.dirname(settingsPath)); - } - await fs.writeFile(settingsPath, JSON.stringify(settings, null, 2)); - // return the location of the build artifiacts return publishFolder; }, - setSkillManifest: async ( - dstRuntimePath: string, - dstStorage: IFileStorage, - srcManifestDir: string, - srcStorage: IFileStorage, - mode = 'azurewebapp' // set default as azurewebapp - ) => { - // update manifst into runtime wwwroot - if (mode === 'azurewebapp') { - const manifestDstDir = path.resolve(dstRuntimePath, 'azurewebapp', 'wwwroot', 'manifests'); - - if (await fs.pathExists(manifestDstDir)) { - await removeDirAndFiles(manifestDstDir); - } - - if (await fs.pathExists(srcManifestDir)) { - await copyDir(srcManifestDir, srcStorage, manifestDstDir, dstStorage); - } - } - }, }); /** @@ -669,23 +607,9 @@ export default async (composer: any): Promise => { if (installErr) { composer.log(installErr); } - // write settings to disk in the appropriate location - const settingsPath = path.join(runtimePath, 'settings', 'appsettings.json'); - if (!(await fs.pathExists(path.dirname(settingsPath)))) { - await fs.mkdirp(path.dirname(settingsPath)); - } - await fs.writeFile(settingsPath, JSON.stringify(settings, null, 2)); - composer.log('BUILD COMPLETE'); return path.resolve(runtimePath, '.'); }, - setSkillManifest: async ( - dstRuntimePath: string, - dstStorage: IFileStorage, - srcManifestDir: string, - srcStorage: IFileStorage, - mode = 'azurewebapp' - ) => {}, }); composer.addRuntimeTemplate({ @@ -750,22 +674,8 @@ export default async (composer: any): Promise => { if (installErr) { composer.log(installErr); } - // write settings to disk in the appropriate location - const settingsPath = path.join(runtimePath, 'settings', 'appsettings.json'); - if (!(await fs.pathExists(path.dirname(settingsPath)))) { - await fs.mkdirp(path.dirname(settingsPath)); - } - await fs.writeFile(settingsPath, JSON.stringify(settings, null, 2)); - composer.log('BUILD COMPLETE'); return path.resolve(runtimePath, '.'); }, - setSkillManifest: async ( - dstRuntimePath: string, - dstStorage: IFileStorage, - srcManifestDir: string, - srcStorage: IFileStorage, - mode = 'azurewebapp' - ) => {}, }); };