diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts index 56ef1b40227..91c55d790b7 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts @@ -1,16 +1,29 @@ #!/usr/bin/env node import { + CompletedEvent, GenerateAndBuildOutput, + generateTotalResult, getGenerateAndBuildOutput, + getTaskResults, + InProgressEvent, logger, + PipelineRunEvent, requireJsonc, SDK, + SDKPipelineStatus, + TaskResult, TaskResultStatus, + Trigger, } from '@azure-tools/sdk-generation-lib'; import * as fs from 'fs'; import * as path from 'path'; -import { PrepareArtifactFilesInput, prepareArtifactFilesInput } from '../../cliSchema/prepareArtifactFilesCliConfig'; +import { + PrepareArtifactFilesInput, + prepareArtifactFilesInput, + prepareResultArtifactInput, + PrepareResultArtifactInput, +} from '../../cliSchema/prepareArtifactFilesCliConfig'; import { getFileListInPackageFolder } from '../../utils/git'; function copyFile(filePath: string, targetDir: string) { @@ -29,11 +42,6 @@ async function prepareSourceCode( artifactDir: string ) { for (const p of generateAndBuildOutput.packages) { - const result = p.result; - if (result === TaskResultStatus.Failure) { - logger.warn(`Build ${p.packageName} failed, skipped it`); - continue; - } const packageName = p.packageName; const packagePaths = p.path; @@ -59,11 +67,6 @@ async function prepareSourceCode( async function prepareArtifacts(generateAndBuildOutput: GenerateAndBuildOutput, language: string, artifactDir: string) { for (const p of generateAndBuildOutput.packages) { - const result = p.result; - if (result === TaskResultStatus.Failure) { - logger.warn(`Build ${p.packageName} failed, skipped it`); - continue; - } const artifacts = p.artifacts; if (!artifacts) { // artifacts is optional @@ -80,7 +83,8 @@ async function prepareArtifacts(generateAndBuildOutput: GenerateAndBuildOutput, function validateInput(config: PrepareArtifactFilesInput) { if (!fs.existsSync(config.generateAndBuildOutputFile)) { - throw new Error(`generateAndBuildOutputFile:${config.generateAndBuildOutputFile} isn's exist!`); + logger.error(`generateAndBuildOutputFile:${config.generateAndBuildOutputFile} isn's exist!`); + process.exit(0); } if (!fs.existsSync(config.artifactDir)) { throw new Error(`Invalid artifactDir:${config.artifactDir}!`); @@ -90,7 +94,7 @@ function validateInput(config: PrepareArtifactFilesInput) { } } -async function main() { +async function prepareSourceCodeAndArtifacts() { prepareArtifactFilesInput.validate(); const config: PrepareArtifactFilesInput = prepareArtifactFilesInput.getProperties(); @@ -103,6 +107,96 @@ async function main() { await prepareArtifacts(generateAndBuildOutput, config.language, config.artifactDir); } +function validateResultInput(config: PrepareResultArtifactInput) { + if (!fs.existsSync(config.artifactDir)) { + throw new Error(`Invalid artifactDir:${config.artifactDir}!`); + } +} + +function getTrigger(config: PrepareResultArtifactInput): Trigger { + let trigger: Trigger; + try { + trigger = JSON.parse(config.trigger); + } catch (error) { + logger.error(`Wrong json format:` + config.trigger); + throw new Error(error); + } + + return trigger; +} + +function prepareResult(pipelineStatus: SDKPipelineStatus) { + prepareResultArtifactInput.validate(); + const config: PrepareResultArtifactInput = prepareResultArtifactInput.getProperties(); + + validateResultInput(config); + const trigger: Trigger = getTrigger(config); + let event: PipelineRunEvent = undefined; + + switch (pipelineStatus) { + case 'in_progress': + event = { + status: 'in_progress', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + } as InProgressEvent; + break; + case 'completed': + if (!config.resultsPath || !config.logPath) { + throw new Error(`Invalid completed event parameter!`); + } + + const taskResults: TaskResult[] = getTaskResults(config.resultsPath); + const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); + event = { + status: 'completed', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + logPath: config.logPath, + result: taskTotalResult, + } as CompletedEvent; + break; + default: + throw new Error(`Unsupported status: ` + (pipelineStatus as string)); + } + + fs.writeFileSync(config.artifactDir + `/` + pipelineStatus + `/result.json`, JSON.stringify(event, null, 2), { + encoding: 'utf-8', + }); +} + +async function main() { + const args = parseArgs(process.argv); + const pipelineStatus = args['pipelineStatus']; + + if (pipelineStatus === 'completed') { + prepareResult(pipelineStatus); + await prepareSourceCodeAndArtifacts(); + } else if (pipelineStatus === 'in_progress') { + prepareResult(pipelineStatus); + } else { + throw new Error(`Unknown pipelineStatus:${pipelineStatus}!`); + } +} + +/** + * Parse a list of command line arguments. + * @param argv List of cli args(process.argv) + */ +const flagRegex = /^--([^=:]+)([=:](.+))?$/; +export function parseArgs(argv: string[]) { + const result: any = {}; + for (const arg of argv) { + const match = flagRegex.exec(arg); + if (match) { + const key = match[1]; + const rawValue = match[3]; + result[key] = rawValue; + } + } + return result; +} + main().catch((e) => { logger.error(`${e.message} ${e.stack}`); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts index 1e395fe778c..4d84f28f21b 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts @@ -5,6 +5,7 @@ import { CodeGeneration, CompletedEvent, generateTotalResult, + getTaskResults, InProgressEvent, logger, MongoConnectContext, @@ -17,7 +18,7 @@ import { SDKPipelineStatus, StorageType, TaskResult, - Trigger + Trigger, } from '@azure-tools/sdk-generation-lib'; import * as fs from 'fs'; @@ -29,7 +30,7 @@ import { ResultPublisherDBResultInput, resultPublisherDBResultInput, ResultPublisherEventHubInput, - resultPublisherEventHubInput + resultPublisherEventHubInput, } from '../../cliSchema/publishResultConfig'; async function publishBlob() { @@ -39,7 +40,7 @@ async function publishBlob() { pipelineBuildId: config.pipelineBuildId, sdkGenerationName: config.sdkGenerationName, azureStorageBlobSasUrl: config.azureStorageBlobSasUrl, - azureBlobContainerName: config.azureBlobContainerName + azureBlobContainerName: config.azureBlobContainerName, }; const resultBlobPublisher: ResultBlobPublisher = new ResultBlobPublisher(context); await resultBlobPublisher.uploadLogsAndResult(config.logsAndResultPath, config.taskName as AzureSDKTaskName); @@ -75,7 +76,7 @@ function initMongoConnectContext(config: ResultPublisherDBCodeGenerationInput): database: config.mongodb.database, ssl: config.mongodb.ssl, synchronize: true, - logging: true + logging: true, }; return mongoConnectContext; @@ -123,19 +124,6 @@ function getTrigger(config: ResultPublisherEventHubInput): Trigger { return trigger; } -function getTaskResults(taskResultsPath: string): TaskResult[] { - const taskResultsPathArray = JSON.parse(taskResultsPath); - const taskResults: TaskResult[] = []; - for (const taskResultPath of taskResultsPathArray) { - if (fs.existsSync(taskResultPath)) { - taskResults.push(requireJsonc(taskResultPath)); - } else { - logger.warn(`${taskResultPath} isn't exist, skip.`); - } - } - return taskResults; -} - async function publishEventhub(pipelineStatus: SDKPipelineStatus) { resultPublisherEventHubInput.validate(); const config: ResultPublisherEventHubInput = resultPublisherEventHubInput.getProperties(); @@ -144,37 +132,37 @@ async function publishEventhub(pipelineStatus: SDKPipelineStatus) { const publisher: ResultEventhubPublisher = new ResultEventhubPublisher(config.eventHubConnectionString); switch (pipelineStatus) { - case 'queued': - event = { - status: 'queued', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId - } as QueuedEvent; - break; - case 'in_progress': - event = { - status: 'in_progress', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId - } as InProgressEvent; - break; - case 'completed': - if (!config.resultsPath || !config.logPath) { - throw new Error(`Invalid completed event parameter!`); - } + case 'queued': + event = { + status: 'queued', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + } as QueuedEvent; + break; + case 'in_progress': + event = { + status: 'in_progress', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + } as InProgressEvent; + break; + case 'completed': + if (!config.resultsPath || !config.logPath) { + throw new Error(`Invalid completed event parameter!`); + } - const taskResults: TaskResult[] = getTaskResults(config.resultsPath); - const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); - event = { - status: 'completed', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId, - logPath: config.logPath, - result: taskTotalResult - } as CompletedEvent; - break; - default: - throw new Error(`Unsupported status: ` + (pipelineStatus as string)); + const taskResults: TaskResult[] = getTaskResults(config.resultsPath); + const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); + event = { + status: 'completed', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + logPath: config.logPath, + result: taskTotalResult, + } as CompletedEvent; + break; + default: + throw new Error(`Unsupported status: ` + (pipelineStatus as string)); } await publisher.publishEvent(event); await publisher.close(); @@ -186,17 +174,17 @@ async function main() { const pipelineStatus = args['pipelineStatus']; switch (storageType as StorageType) { - case StorageType.Blob: - await publishBlob(); - break; - case StorageType.Db: - await publishDB(pipelineStatus); - break; - case StorageType.EventHub: - await publishEventhub(pipelineStatus); - break; - default: - throw new Error(`Unknown storageType:${storageType}!`); + case StorageType.Blob: + await publishBlob(); + break; + case StorageType.Db: + await publishDB(pipelineStatus); + break; + case StorageType.EventHub: + await publishEventhub(pipelineStatus); + break; + default: + throw new Error(`Unknown storageType:${storageType}!`); } } diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts index adc0303e6b8..336721672d5 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts @@ -17,45 +17,45 @@ export const generateResultCliInput = convict({ pipelineBuildId: { default: null, format: assertNullOrEmpty, - arg: 'buildId' + arg: 'buildId', }, logfile: { default: null, format: assertNullOrEmpty, - arg: 'logfile' + arg: 'logfile', }, logFilterStr: { default: null, nullable: true, format: String, - arg: 'logFilterStr' + arg: 'logFilterStr', }, taskName: { default: null, format: ['init', 'generateAndBuild', 'mockTest', 'liveTest'], - arg: 'taskName' + arg: 'taskName', }, exeResult: { default: null, nullable: true, - format: ['success', 'failure'], - arg: 'exeResult' + format: ['succeeded', 'failed'], + arg: 'exeResult', }, taskOutputPath: { default: null, nullable: true, format: String, - arg: 'taskOutputPath' + arg: 'taskOutputPath', }, resultOutputPath: { default: null, format: assertNullOrEmpty, - arg: 'resultOutputPath' + arg: 'resultOutputPath', }, dockerResultFile: { default: null, nullable: true, format: String, - arg: 'dockerResultFile' - } + arg: 'dockerResultFile', + }, }); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts index 8c59b602315..2ad5bff4353 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts @@ -12,17 +12,57 @@ export const prepareArtifactFilesInput = convict({ generateAndBuildOutputFile: { default: null, format: assertNullOrEmpty, - arg: 'generateAndBuildOutputFile' + arg: 'generateAndBuildOutputFile', }, artifactDir: { doc: 'The dir to publish artifact', default: null, format: assertNullOrEmpty, - arg: 'artifactDir' + arg: 'artifactDir', }, language: { default: null, format: ['js', 'python', 'go', 'net', 'java'], - arg: 'language' - } + arg: 'language', + }, +}); + +export class PrepareResultArtifactInput { + pipelineBuildId: string; + trigger: string; + artifactDir: string; + logPath?: string; + resultsPath?: string; +} + +export const prepareResultArtifactInput = convict({ + pipelineBuildId: { + default: null, + format: assertNullOrEmpty, + arg: 'buildId', + }, + trigger: { + default: null, + format: assertNullOrEmpty, + arg: 'trigger', + }, + artifactDir: { + doc: 'The dir to publish artifact', + default: null, + format: assertNullOrEmpty, + arg: 'artifactDir', + }, + logPath: { + default: null, + nullable: true, + format: String, + arg: 'logPath', + }, + resultsPath: { + doc: 'task result files array', + default: null, + nullable: true, + format: String, + arg: 'resultsPath', + }, }); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts index c4403cb0f0e..99a9d0c8cff 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts @@ -50,7 +50,7 @@ export class ResultDBPublisher { ssl: this.context.ssl, synchronize: this.context.synchronize, logging: this.context.logging, - entities: [TaskResultEntity, CodeGeneration] + entities: [TaskResultEntity, CodeGeneration], }); } @@ -101,9 +101,9 @@ export class ResultBlobPublisher { const logsAndResultPathArray = JSON.parse(logsAndResultPath); for (const file of logsAndResultPathArray) { if (fs.existsSync(file)) { - const blobName: string = file.includes('.json') ? - `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}-result.json` : - `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}.log`; + const blobName: string = file.includes('.json') + ? `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}-result.json` + : `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}.log`; await this.azureBlobClient.publishBlob(file, blobName); logger.info(`Publish ${file} Success !!!`); } else { diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts index 5bb84f5612e..fd5f1f28917 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts @@ -4,18 +4,20 @@ import { Column, Entity, ObjectIdColumn } from 'typeorm'; import { getTaskBasicConfig, TaskBasicConfig } from './taskBasicConfig'; import { GenerateAndBuildOutput } from './taskInputAndOuputSchemaTypes/GenerateAndBuildOutput'; import { InitOutput } from './taskInputAndOuputSchemaTypes/InitOutput'; +import { logger } from '../utils/logger'; +import { requireJsonc } from '../utils/requireJsonc'; import { TestOutput } from './taskInputAndOuputSchemaTypes/TestOutput'; @Entity('sdkGenerationResults') export class TaskResultEntity { @ObjectIdColumn() - id: string; + id: string; @Column() - key: string; + key: string; @Column() - pipelineBuildId: string; + pipelineBuildId: string; @Column() - taskResult: TaskResult; + taskResult: TaskResult; } export enum TaskResultStatus { @@ -104,7 +106,7 @@ export function setTaskResult(config: TaskBasicConfig, taskName: string) { pipelineBuildId: '', result: TaskResultStatus.Success, errorCount: 0, - warningCount: 0 + warningCount: 0, }; } @@ -122,7 +124,7 @@ export function generateTotalResult(taskResults: TaskResult[], pipelineBuildId: pipelineBuildId: pipelineBuildId, result: TaskResultStatus.Success, errorCount: 0, - messages: [] + messages: [], }; if (taskResults.length === 0) { @@ -144,3 +146,16 @@ export function generateTotalResult(taskResults: TaskResult[], pipelineBuildId: return totalResult; } + +export function getTaskResults(taskResultsPath: string): TaskResult[] { + const taskResultsPathArray = JSON.parse(taskResultsPath); + const taskResults: TaskResult[] = []; + for (const taskResultPath of taskResultsPathArray) { + if (fs.existsSync(taskResultPath)) { + taskResults.push(requireJsonc(taskResultPath)); + } else { + logger.warn(`${taskResultPath} isn't exist, skip.`); + } + } + return taskResults; +} diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml index 1f5ca0117c3..a68efcb5c0d 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -5,6 +5,18 @@ parameters: - name: trigger type: string + - name: triggerType + type: string + + - name: serviceType + type: string + + - name: sdk + type: string + + - name: sdkConfig + type: string + steps: - bash: | publishResult \ @@ -18,15 +30,74 @@ steps: displayName: Publish status env: EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) - + condition: ne('${{ parameters.triggerType }}', 'release') + - bash: | - docker run \ - -v $(SPEC_REPO):/spec-repo \ - -v $(SDK_REPO):/sdk-repo \ - -v /tmp/output:/tmp/output \ - $(DOCKER_IMAGE) \ - --readme=${{ parameters.readmeFile }} - if [ $? -ne 0 ]; then - exit 1 + if [ "${{ parameters.serviceType }}" = "resource-manager" ];then + docker run --privileged \ + -v $(SPEC_REPO):/spec-repo \ + -v $(SDK_REPO):/sdk-repo \ + -v /tmp/output:/tmp/output \ + $(DOCKER_IMAGE) \ + --readme=${{ parameters.readmeFile }} $(DOCKER_EX_CMD) + else + echo '${{ parameters.sdkConfig }}' > /tmp/output/autorest.md + configLen=`cat /tmp/output/autorest.md | wc -L` + echo "configLen is :${configLen}" + #check if config is default '' + if [ 2 -ge ${configLen} ];then + docker run --privileged \ + -v $(SPEC_REPO):/spec-repo \ + -v $(SDK_REPO):/sdk-repo \ + -v /tmp/output:/tmp/output \ + $(DOCKER_IMAGE) \ + --readme=${{ parameters.readmeFile }} $(DOCKER_EX_CMD) + else + docker run --privileged \ + -v $(SPEC_REPO):/spec-repo \ + -v $(SDK_REPO):/sdk-repo \ + -v /tmp/output/autorest.md:/autorest.md \ + -v /tmp/output:/tmp/output \ + $(DOCKER_IMAGE) \ + --readme=${{ parameters.readmeFile }} $(DOCKER_EX_CMD) + fi fi displayName: Generate sdk + + - bash: | + #TODO: delete this bash + echo "check output dir" + ls /tmp/output + echo "show docker.log" + head -n 30 /tmp/output/docker.log + echo "*****" + tail -n 30 /tmp/output/docker.log + echo "show generateAndBuild-task.log" + head -n 30 /tmp/output/generateAndBuild-task.log + echo "*****" + tail -n 30 /tmp/output/generateAndBuild-task.log + echo "show generateAndBuildInput.json" + cat /tmp/output/generateAndBuildInput.json + echo "show generateAndBuildOutputJson.json" + cat /tmp/output/generateAndBuildOutputJson.json + echo "show init-task.log" + head -n 30 /tmp/output/init-task.log + echo "*****" + tail -n 30 /tmp/output/init-task.log + echo "show taskResults.json" + cat /tmp/output/taskResults.json + + echo "show mock-host.log" + head -n 30 /tmp/output/mock-host.log | echo "don't quite even failed" + echo "*****" + tail -n 30 /tmp/output/mock-host.log | echo "don't quite even failed" + + echo "show mockTest-task.log" + head -n 30 /tmp/output/mockTest-task.log | echo "don't quite even failed" + echo "*****" + tail -n 30 /tmp/output/mockTest-task.log | echo "don't quite even failed" + + echo "show mockTestOutput.json" + cat /tmp/output/mockTestOutput.json | echo "don't quite even failed" + displayName: Check output dir + condition: always() \ No newline at end of file diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 961afa8e6fa..53ea20ae8bc 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -13,6 +13,10 @@ parameters: - name: sdkGenerationName type: string + + - name: triggerType + type: string + steps: - bash: | echo "will parse task result" @@ -41,6 +45,79 @@ steps: displayName: Parse task result condition: always() + - bash: | + #TODO: delete this bash + ls -al /tmp/output/ + echo "show generateAndBuild-task-result.json" + cat /tmp/output/generateAndBuild-task-result.json | echo "don't quite even failed" + displayName: Check output dir + condition: always() + + - bash: | + #TODO: delete this bash + echo "will show git info" + git status + displayName: Show git info + workingDirectory: $(SDK_REPO) + + - bash: | + echo "will create PR" + git config --global user.email "$(GIT_USER_EMAIL)" + git config --global user.name "$(GIT_USER_NAME)" + + #packages=`cat /tmp/output/generateAndBuildOutputJson.json| jq -c '.packages|.[]'` + package=`cat /tmp/output/generateAndBuildOutputJson.json| jq -c '.packages|.[]'` + #for package in ${packages} + #do + packageName=`echo ${package} | jq -r '.packageName'` + paths=`echo ${package} | jq -r '.path|.[]'` + for path in ${paths} + do + git add ${path} + done + git commit -m '[auto]create by sdk generation' + if [ "${{ parameters.serviceType }}" = "resource-manager" ];then + branchName=sdkGen/${{ parameters.sdkGenerationName }}/arm-${{ parameters.service }}-${packageName} + else + branchName=sdkGen/${{ parameters.sdkGenerationName }}/dpg-${{ parameters.service }}-${packageName} + fi + git checkout -b ${branchName} + pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(azuresdk-github-pat)@#") + #TODO: push PR to Azure if is release to Azure no Azure SDK + pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#$(SDK_OWNER)#$(SDK_PUSH_OWNER)#") + git push ${pushSdkRepoUrl} --force + echo "git push finished:$?" + sdkRepoPullsApi=$(echo $(sdkRepoUrl) | sed "s#https://#https://api.#") + sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#$(SDK_OWNER)#repos/$(SDK_PUSH_OWNER)#") + sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#.git\$#/pulls#") + echo "will use api:${sdkRepoPullsApi}" + RESULT=$(curl \ + -u "$(REPO_OWNER)":"$(azuresdk-github-pat)" \ + -H "Accept: application/vnd.github.v3+json" \ + -X POST \ + -d '{"title":"[AutoPR-${{ parameters.service }}]autogenerated","body":"","base":"'$(sdkMainBranch)'", "head":"$(SDK_PUSH_OWNER):'${branchName}'"}' \ + ${sdkRepoPullsApi}) + echo "creat pr finished:$?" + + prNumber=`echo ${RESULT} | jq -r '.number'` + echo "##vso[task.setvariable variable=prNumber;]${prNumber}" + prLink=`echo ${RESULT} | jq -r '.html_url'` + echo "##vso[task.setvariable variable=prLink;]${prLink}" + echo "##vso[task.setvariable variable=branchName;]${branchName}" + echo "##vso[task.setvariable variable=packageName;]${packageName}" + packageFolder=`echo ${package} | jq -r '.packageFolder'` + serviceDir=`echo $packageFolder | awk '{split($0,array,"/");print array[2]}' ` + if [ "${{ parameters.sdk }}" = "js" ] && [ "${{ parameters.serviceType }}" = "resource-manager" ];then + echo "##vso[task.setvariable variable=releasePipeline;]${{ parameters.sdk }} - ${serviceDir} - mgmt" + else + echo "##vso[task.setvariable variable=releasePipeline;]${{ parameters.sdk }} - ${serviceDir}" + fi + + git reset HEAD~ + #done + workingDirectory: $(SDK_REPO) + displayName: Submit Pull Request + - bash: | cp /tmp/output/docker.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-full.log cp /tmp/output/init-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-init.log @@ -50,31 +127,53 @@ steps: cp /tmp/output/mockTest-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest.log | echo "wait all language support mock test" cp /tmp/output/mockTest-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest-result.json | echo "wait all language support mock test" prepareArtifactFiles \ + --pipelineStatus=completed \ --artifactDir="$(System.ArtifactsDirectory)" \ + --buildId="$(Build.BuildId)" \ --generateAndBuildOutputFile="/tmp/output/generateAndBuildOutputJson.json" \ - --language="${{ parameters.sdk }}" - if [ $? -ne 0 ]; then - exit 1 + --trigger='${{ parameters.trigger }}' \ + --logPath="tmplogpath.com" \ + --language="${{ parameters.sdk }}" \ + --resultsPath='["/tmp/output/init-task-result.json", + "/tmp/output/generateAndBuild-task-result.json", + "/tmp/output/mockTest-task-result.json"]' + + #TODO: delete next line + cat /tmp/output/generateAndBuild-task-result.json + cat $(System.ArtifactsDirectory)/completed/result.json + + if [ -n "$(prLink)" ];then + newtaskresult=$(cat $(System.ArtifactsDirectory)/completed/result.json | \ + jq --arg prNumber "$(prNumber)" --arg prLink "$(prLink)" --arg branchName "$(branchName)" \ + --arg packageName "$(packageName)" --arg serviceType "${{ parameters.serviceType }}" \ + --arg releasePipeline "$(releasePipeline)" \ + '.result += {prNumber:$prNumber, prLink:$prLink, branchName:$branchName, serviceType:$serviceType, releasePipeline:$releasePipeline} | + .package += [{packageName:$packageName}]' ) + echo ${newtaskresult} > $(System.ArtifactsDirectory)/completed/result.json fi + + #TODO: delete next line + cat $(System.ArtifactsDirectory)/completed/result.json displayName: Prepare artifact workingDirectory: $(SDK_REPO) condition: always() - task: PublishPipelineArtifact@1 - displayName: Publish SDK Generation pipeline artifact + displayName: Publish artifact inputs: targetPath: $(System.ArtifactsDirectory)/${{ parameters.sdk }}/artifact/ artifactName: sdk-generation-artifact continueOnError: true - task: PublishPipelineArtifact@1 - displayName: Publish SDK Generation pipeline sourceCode + displayName: Publish sourceCode inputs: targetPath: $(System.ArtifactsDirectory)/${{ parameters.sdk }}/sourceCode/ artifactName: sdk-generation-sourceCode + continueOnError: true - task: PublishPipelineArtifact@1 - displayName: Publish SDK Generation pipeline logs + displayName: Publish logs inputs: targetPath: $(System.ArtifactsDirectory)/logs/ artifactName: sdk-generation-logs @@ -96,43 +195,11 @@ steps: displayName: Publish status env: EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) - condition: always() - - - bash: | - echo "will create PR" - git config --global user.email "$(GIT_USER_EMAIL)" - git config --global user.name "$(GIT_USER_NAME)" + condition: and(always(), ne('${{ parameters.triggerType }}', 'release')) - packages=`cat /tmp/output/generateAndBuildOutputJson.json | jq -c '.packages | .[]'` - for package in ${packages} - do - packageName=`echo ${package} | jq -r '.packageName'` - paths=`echo ${package} | jq -r '.path |.[]'` - for path in ${paths} - do - git add ${path} - done - git commit -m '[auto]create by sdk generation' - if [ "${{ parameters.serviceType }}" = "resource-manager" ];then - branchName=sdkGen/${{ parameters.sdkGenerationName }}/arm-${{ parameters.service }}-${packageName} - else - branchName=sdkGen/${{ parameters.sdkGenerationName }}/dpg-${{ parameters.service }}-${packageName} - fi - git checkout -b ${branchName} - pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") - pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#Azure#$(REPO_OWNER)#") - git push ${pushSdkRepoUrl} - sdkRepoPullsApi=$(echo $(sdkRepoUrl) | sed "s#https://#https://api.#") - sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#Azure#repos/$(REPO_OWNER)#") - sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#.git\$#/pulls#") - curl \ - -u "$(REPO_OWNER)":"$(REPO_OWNER_ACCESSKEY)" \ - -H "Accept: application/vnd.github.v3+json" \ - -X POST \ - -d '{"title":"[AutoPR-${{ parameters.service }}]autogenerated","body":"","base":"main", "head":"'$(REPO_OWNER)':'${branchName}'"}' \ - ${sdkRepoPullsApi} - - git reset HEAD~ - done - workingDirectory: $(SDK_REPO) - displayName: Submit Pull Request \ No newline at end of file + - task: PublishPipelineArtifact@1 + displayName: Publish status artifact + inputs: + targetPath: $(System.ArtifactsDirectory)/completed/ + artifactName: sdk-generation-completed-result + condition: always() \ No newline at end of file diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml index 1aa1b6aaab2..34f5aab3655 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -2,15 +2,15 @@ parameters: - name: sdk type: string - - name: sdkRepoBaseBranch - type: string - - name: trigger type: string - name: swagger type: object + - name: triggerType + type: string + steps: - task: NodeTool@0 displayName: Specify Node Version @@ -18,6 +18,7 @@ steps: versionSpec: 14.x - bash: | + #install tools cd tools/sdk-generation-pipeline npm install -g @microsoft/rush rush update @@ -27,6 +28,8 @@ steps: PACKAGE_TGZ=`ls azure-tools-sdk-generation-cli*.tgz` npm install -g ${PACKAGE_TGZ} npm install -g autorest + + #mkdir for artifacts cd $(System.ArtifactsDirectory) echo "mkdir for docker output" mkdir -p /tmp/output @@ -35,7 +38,12 @@ steps: cd $(Agent.BuildDirectory) getRepoName SPEC_REPO ${{ parameters.swagger.path }} getRepoName SDK_REPO $(sdkRepoUrl) + #mkdir for result artifact + mkdir -p $(System.ArtifactsDirectory)/in_progress/ + mkdir -p $(System.ArtifactsDirectory)/completed/ + displayName: Setup Pipeline Runtime Environment + - bash: | publishResult \ --storageType=eventhub \ --pipelineStatus=queued \ @@ -44,16 +52,33 @@ steps: if [ $? -ne 0 ]; then exit 1 fi - displayName: Setup Pipeline Runtime Environment + displayName: Publish status env: EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) + condition: ne('${{ parameters.triggerType }}', 'release') + + - bash: | + prepareArtifactFiles \ + --pipelineStatus=in_progress \ + --artifactDir="$(System.ArtifactsDirectory)" \ + --buildId="$(Build.BuildId)" \ + --trigger='${{ parameters.trigger }}' + displayName: Prepare in progress status + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: Publish status artifact + inputs: + targetPath: $(System.ArtifactsDirectory)/in_progress/ + artifactName: sdk-generation-in_progress-result + continueOnError: true - bash: | git clone ${{ parameters.swagger.path }} - downloadSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") - git clone --branch $(sdkRepoBaseBranch) ${downloadSdkRepoUrl} + downloadSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(azuresdk-github-pat)@#") + git clone --branch $(sdkMainBranch) ${downloadSdkRepoUrl} cd $(SPEC_REPO) - export GITHUB_TOKEN=$(REPO_OWNER_ACCESSKEY) + export GITHUB_TOKEN=$(azuresdk-github-pat) if [ -n "${{ parameters.swagger.branch }}" ];then git checkout ${{ parameters.swagger.branch }} elif [ -n "${{ parameters.swagger.pullRequest }}" ];then diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index 9197cedb000..12908cd75ea 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -13,6 +13,9 @@ parameters: - name: service type: string + + - name: sdkRepoBaseBranch + type: string steps: - bash: | if [ -z ${{ parameters.swagger.path }} ];then @@ -22,5 +25,16 @@ steps: displayName: Check parameters - bash: | - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-${{parameters.sdk}}.git" + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" + if [ ${{parameters.sdk}} = "js" ]; then + echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" + elif [ ${{parameters.sdk}} = "net" ]; then + echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" + elif [ ${{parameters.sdk}} = "python" ]; then + echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" + elif [ ${{parameters.sdk}} = "java" ]; then + echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-codegen" + else + echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" + fi displayName: Set sdk repo diff --git a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml index 140b9efbb51..67d1ffa8842 100644 --- a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml +++ b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml @@ -54,6 +54,17 @@ parameters: - name: sdkGenerationName type: string + - name: triggerType + type: string + values: + - release + - ad-hoc + - ci + + - name: sdkConfig + type: string + default: "''" + stages: - stage: SdkGeneration jobs: @@ -65,18 +76,23 @@ stages: swagger: ${{ parameters.swagger }} sdk: ${{ parameters.sdkToGernerate }} service: ${{ parameters.service }} + sdkRepoBaseBranch: ${{ variables.sdkRepoBaseBranch }} - template: ./pipelines/steps/pipeline-setup.yml parameters: sdk: ${{ parameters.sdkToGernerate }} - sdkRepoBaseBranch: ${{ variables.sdkRepoBaseBranch }} trigger: ${{ parameters.clientAgent }} swagger: ${{ parameters.swagger }} + triggerType: ${{ parameters.triggerType }} - template: ./pipelines/steps/generate-sdk.yml parameters: readmeFile: ${{ parameters.relativeReadmeFilePath }} trigger: ${{ parameters.clientAgent }} + triggerType: ${{ parameters.triggerType }} + serviceType: ${{ parameters.serviceType }} + sdk: ${{ parameters.sdkToGernerate }} + sdkConfig: ${{ parameters.sdkConfig }} - template: ./pipelines/steps/parse-and-publish-result.yml parameters: @@ -85,3 +101,4 @@ stages: serviceType: ${{ parameters.serviceType }} trigger: ${{ parameters.clientAgent }} sdkGenerationName: ${{ parameters.sdkGenerationName }} + triggerType: ${{ parameters.triggerType }}