Zz/codegen pipeline

tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts

@@ -1,16 +1,29 @@
 #!/usr/bin/env node
 import {
+    CompletedEvent,
     GenerateAndBuildOutput,
+    generateTotalResult,
     getGenerateAndBuildOutput,
+    getTaskResults,
+    InProgressEvent,
     logger,
+    PipelineRunEvent,
     requireJsonc,
     SDK,
+    SDKPipelineStatus,
+    TaskResult,
     TaskResultStatus,
+    Trigger,
 } from '@azure-tools/sdk-generation-lib';
 import * as fs from 'fs';
 import * as path from 'path';
 
-import { PrepareArtifactFilesInput, prepareArtifactFilesInput } from '../../cliSchema/prepareArtifactFilesCliConfig';
+import {
+    PrepareArtifactFilesInput,
+    prepareArtifactFilesInput,
+    prepareResultArtifactInput,
+    PrepareResultArtifactInput,
+} from '../../cliSchema/prepareArtifactFilesCliConfig';
 import { getFileListInPackageFolder } from '../../utils/git';
 
 function copyFile(filePath: string, targetDir: string) {
@@ -29,11 +42,6 @@ async function prepareSourceCode(
     artifactDir: string
 ) {
     for (const p of generateAndBuildOutput.packages) {
-        const result = p.result;
-        if (result === TaskResultStatus.Failure) {
-            logger.warn(`Build ${p.packageName} failed, skipped it`);
-            continue;
-        }
         const packageName = p.packageName;
         const packagePaths = p.path;
 
@@ -59,11 +67,6 @@ async function prepareSourceCode(
 
 async function prepareArtifacts(generateAndBuildOutput: GenerateAndBuildOutput, language: string, artifactDir: string) {
     for (const p of generateAndBuildOutput.packages) {
-        const result = p.result;
-        if (result === TaskResultStatus.Failure) {
-            logger.warn(`Build ${p.packageName} failed, skipped it`);
-            continue;
-        }
         const artifacts = p.artifacts;
         if (!artifacts) {
             // artifacts is optional
@@ -80,7 +83,8 @@ async function prepareArtifacts(generateAndBuildOutput: GenerateAndBuildOutput,
 
 function validateInput(config: PrepareArtifactFilesInput) {
     if (!fs.existsSync(config.generateAndBuildOutputFile)) {
-        throw new Error(`generateAndBuildOutputFile:${config.generateAndBuildOutputFile} isn's exist!`);
+        logger.error(`generateAndBuildOutputFile:${config.generateAndBuildOutputFile} isn's exist!`);
+        process.exit(0);
     }
     if (!fs.existsSync(config.artifactDir)) {
         throw new Error(`Invalid artifactDir:${config.artifactDir}!`);
@@ -90,7 +94,7 @@ function validateInput(config: PrepareArtifactFilesInput) {
     }
 }
 
-async function main() {
+async function prepareSourceCodeAndArtifacts() {
     prepareArtifactFilesInput.validate();
     const config: PrepareArtifactFilesInput = prepareArtifactFilesInput.getProperties();
 
@@ -103,6 +107,96 @@ async function main() {
     await prepareArtifacts(generateAndBuildOutput, config.language, config.artifactDir);
 }
 
+function validateResultInput(config: PrepareResultArtifactInput) {
+    if (!fs.existsSync(config.artifactDir)) {
+        throw new Error(`Invalid artifactDir:${config.artifactDir}!`);
+    }
+}
+
+function getTrigger(config: PrepareResultArtifactInput): Trigger {
+    let trigger: Trigger;
+    try {
+        trigger = JSON.parse(config.trigger);
+    } catch (error) {
+        logger.error(`Wrong json format:` + config.trigger);
+        throw new Error(error);
+    }
+
+    return trigger;
+}
+
+function prepareResult(pipelineStatus: SDKPipelineStatus) {
+    prepareResultArtifactInput.validate();
+    const config: PrepareResultArtifactInput = prepareResultArtifactInput.getProperties();
+
+    validateResultInput(config);
+    const trigger: Trigger = getTrigger(config);
+    let event: PipelineRunEvent = undefined;
+
+    switch (pipelineStatus) {
+        case 'in_progress':
+            event = {
+                status: 'in_progress',
+                trigger: trigger,
+                pipelineBuildId: config.pipelineBuildId,
+            } as InProgressEvent;
+            break;
+        case 'completed':
+            if (!config.resultsPath || !config.logPath) {
+                throw new Error(`Invalid completed event parameter!`);
+            }
+
+            const taskResults: TaskResult[] = getTaskResults(config.resultsPath);
+            const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId);
+            event = {
+                status: 'completed',
+                trigger: trigger,
+                pipelineBuildId: config.pipelineBuildId,
+                logPath: config.logPath,
+                result: taskTotalResult,
+            } as CompletedEvent;
+            break;
+        default:
+            throw new Error(`Unsupported status: ` + (pipelineStatus as string));
+    }
+
+    fs.writeFileSync(config.artifactDir + `/` + pipelineStatus + `/result.json`, JSON.stringify(event, null, 2), {
+        encoding: 'utf-8',
+    });
+}
+
+async function main() {
+    const args = parseArgs(process.argv);
+    const pipelineStatus = args['pipelineStatus'];
+
+    if (pipelineStatus === 'completed') {
+        prepareResult(pipelineStatus);
+        await prepareSourceCodeAndArtifacts();
+    } else if (pipelineStatus === 'in_progress') {
+        prepareResult(pipelineStatus);
+    } else {
+        throw new Error(`Unknown pipelineStatus:${pipelineStatus}!`);
+    }
+}
+
+/**
+ * Parse a list of command line arguments.
+ * @param argv List of cli args(process.argv)
+ */
+const flagRegex = /^--([^=:]+)([=:](.+))?$/;
+export function parseArgs(argv: string[]) {
+    const result: any = {};
+    for (const arg of argv) {
+        const match = flagRegex.exec(arg);
+        if (match) {
+            const key = match[1];
+            const rawValue = match[3];
+            result[key] = rawValue;
+        }
+    }
+    return result;
+}
+
 main().catch((e) => {
     logger.error(`${e.message}
      ${e.stack}`);

tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts

@@ -5,6 +5,7 @@ import {
     CodeGeneration,
     CompletedEvent,
     generateTotalResult,
+    getTaskResults,
     InProgressEvent,
     logger,
     MongoConnectContext,
@@ -17,7 +18,7 @@ import {
     SDKPipelineStatus,
     StorageType,
     TaskResult,
-    Trigger
+    Trigger,
 } from '@azure-tools/sdk-generation-lib';
 import * as fs from 'fs';
 
@@ -29,7 +30,7 @@ import {
     ResultPublisherDBResultInput,
     resultPublisherDBResultInput,
     ResultPublisherEventHubInput,
-    resultPublisherEventHubInput
+    resultPublisherEventHubInput,
 } from '../../cliSchema/publishResultConfig';
 
 async function publishBlob() {
@@ -39,7 +40,7 @@ async function publishBlob() {
         pipelineBuildId: config.pipelineBuildId,
         sdkGenerationName: config.sdkGenerationName,
         azureStorageBlobSasUrl: config.azureStorageBlobSasUrl,
-        azureBlobContainerName: config.azureBlobContainerName
+        azureBlobContainerName: config.azureBlobContainerName,
     };
     const resultBlobPublisher: ResultBlobPublisher = new ResultBlobPublisher(context);
     await resultBlobPublisher.uploadLogsAndResult(config.logsAndResultPath, config.taskName as AzureSDKTaskName);
@@ -75,7 +76,7 @@ function initMongoConnectContext(config: ResultPublisherDBCodeGenerationInput):
         database: config.mongodb.database,
         ssl: config.mongodb.ssl,
         synchronize: true,
-        logging: true
+        logging: true,
     };
 
     return mongoConnectContext;
@@ -123,19 +124,6 @@ function getTrigger(config: ResultPublisherEventHubInput): Trigger {
     return trigger;
 }
 
-function getTaskResults(taskResultsPath: string): TaskResult[] {
-    const taskResultsPathArray = JSON.parse(taskResultsPath);
-    const taskResults: TaskResult[] = [];
-    for (const taskResultPath of taskResultsPathArray) {
-        if (fs.existsSync(taskResultPath)) {
-            taskResults.push(requireJsonc(taskResultPath));
-        } else {
-            logger.warn(`${taskResultPath} isn't exist, skip.`);
-        }
-    }
-    return taskResults;
-}
-
 async function publishEventhub(pipelineStatus: SDKPipelineStatus) {
     resultPublisherEventHubInput.validate();
     const config: ResultPublisherEventHubInput = resultPublisherEventHubInput.getProperties();
@@ -144,37 +132,37 @@ async function publishEventhub(pipelineStatus: SDKPipelineStatus) {
     const publisher: ResultEventhubPublisher = new ResultEventhubPublisher(config.eventHubConnectionString);
 
     switch (pipelineStatus) {
-    case 'queued':
-        event = {
-            status: 'queued',
-            trigger: trigger,
-            pipelineBuildId: config.pipelineBuildId
-        } as QueuedEvent;
-        break;
-    case 'in_progress':
-        event = {
-            status: 'in_progress',
-            trigger: trigger,
-            pipelineBuildId: config.pipelineBuildId
-        } as InProgressEvent;
-        break;
-    case 'completed':
-        if (!config.resultsPath || !config.logPath) {
-            throw new Error(`Invalid completed event parameter!`);
-        }
+        case 'queued':
+            event = {
+                status: 'queued',
+                trigger: trigger,
+                pipelineBuildId: config.pipelineBuildId,
+            } as QueuedEvent;
+            break;
+        case 'in_progress':
+            event = {
+                status: 'in_progress',
+                trigger: trigger,
+                pipelineBuildId: config.pipelineBuildId,
+            } as InProgressEvent;
+            break;
+        case 'completed':
+            if (!config.resultsPath || !config.logPath) {
+                throw new Error(`Invalid completed event parameter!`);
+            }
 
-        const taskResults: TaskResult[] = getTaskResults(config.resultsPath);
-        const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId);
-        event = {
-            status: 'completed',
-            trigger: trigger,
-            pipelineBuildId: config.pipelineBuildId,
-            logPath: config.logPath,
-            result: taskTotalResult
-        } as CompletedEvent;
-        break;
-    default:
-        throw new Error(`Unsupported status: ` + (pipelineStatus as string));
+            const taskResults: TaskResult[] = getTaskResults(config.resultsPath);
+            const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId);
+            event = {
+                status: 'completed',
+                trigger: trigger,
+                pipelineBuildId: config.pipelineBuildId,
+                logPath: config.logPath,
+                result: taskTotalResult,
+            } as CompletedEvent;
+            break;
+        default:
+            throw new Error(`Unsupported status: ` + (pipelineStatus as string));
     }
     await publisher.publishEvent(event);
     await publisher.close();
@@ -186,17 +174,17 @@ async function main() {
     const pipelineStatus = args['pipelineStatus'];
 
     switch (storageType as StorageType) {
-    case StorageType.Blob:
-        await publishBlob();
-        break;
-    case StorageType.Db:
-        await publishDB(pipelineStatus);
-        break;
-    case StorageType.EventHub:
-        await publishEventhub(pipelineStatus);
-        break;
-    default:
-        throw new Error(`Unknown storageType:${storageType}!`);
+        case StorageType.Blob:
+            await publishBlob();
+            break;
+        case StorageType.Db:
+            await publishDB(pipelineStatus);
+            break;
+        case StorageType.EventHub:
+            await publishEventhub(pipelineStatus);
+            break;
+        default:
+            throw new Error(`Unknown storageType:${storageType}!`);
     }
 }
 

tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts

@@ -17,45 +17,45 @@ export const generateResultCliInput = convict<GenerateResultCliInput>({
     pipelineBuildId: {
         default: null,
         format: assertNullOrEmpty,
-        arg: 'buildId'
+        arg: 'buildId',
     },
     logfile: {
         default: null,
         format: assertNullOrEmpty,
-        arg: 'logfile'
+        arg: 'logfile',
     },
     logFilterStr: {
         default: null,
         nullable: true,
         format: String,
-        arg: 'logFilterStr'
+        arg: 'logFilterStr',
     },
     taskName: {
         default: null,
         format: ['init', 'generateAndBuild', 'mockTest', 'liveTest'],
-        arg: 'taskName'
+        arg: 'taskName',
     },
     exeResult: {
         default: null,
         nullable: true,
-        format: ['success', 'failure'],
-        arg: 'exeResult'
+        format: ['succeeded', 'failed'],
+        arg: 'exeResult',
     },
     taskOutputPath: {
         default: null,
         nullable: true,
         format: String,
-        arg: 'taskOutputPath'
+        arg: 'taskOutputPath',
     },
     resultOutputPath: {
         default: null,
         format: assertNullOrEmpty,
-        arg: 'resultOutputPath'
+        arg: 'resultOutputPath',
     },
     dockerResultFile: {
         default: null,
         nullable: true,
         format: String,
-        arg: 'dockerResultFile'
-    }
+        arg: 'dockerResultFile',
+    },
 });