diff --git a/.travis.yml b/.travis.yml index 0f192aeac3bf..f092364e6a12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,32 +1,32 @@ language: node_js node_js: - - "8" + - '8' services: - docker env: matrix: - - MODE=syntax - - MODE=python + - MODE=syntax + - MODE=python # - MODE=node - - MODE=ruby - - MODE=semantic PR_ONLY=true - # - MODE=linter PR_ONLY=false Disabling to save travis-ci resource for now - - MODE=semantic PR_ONLY=false - - MODE=model PR_ONLY=false - - MODE=linter PR_ONLY=true - - MODE=model PR_ONLY=true - - MODE=BreakingChange PR_ONLY=true - - MODE=azurebot PR_ONLY=true + - MODE=ruby + - MODE=semantic PR_ONLY=true + - MODE=semantic PR_ONLY=false + - MODE=model PR_ONLY=false + - MODE=linter PR_ONLY=true + - MODE=model PR_ONLY=true + - MODE=BreakingChange PR_ONLY=true + - MODE=azurebot PR_ONLY=true + - MODE=liveValidation PR_ONLY=true matrix: fast_finish: true allow_failures: - - env: MODE=linter PR_ONLY=false - env: MODE=semantic PR_ONLY=false - env: MODE=model PR_ONLY=false - env: MODE=linter PR_ONLY=true - env: MODE=model PR_ONLY=true - env: MODE=BreakingChange PR_ONLY=true - env: MODE=azurebot PR_ONLY=true + - env: MODE=liveValidation PR_ONLY=true before_install: - docker pull lmazuel/swagger-to-sdk - python -c "import os; print('\n'.join(v for v in os.environ.keys() if v.startswith('TRAVIS')))" > /tmp/env_file @@ -41,12 +41,43 @@ install: - npm install script: - DOCKER_CMD="docker run --rm --env-file /tmp/env_file -e GH_TOKEN -v $PWD:/git-restapi/ lmazuel/swagger-to-sdk" - - if [[ $MODE == 'python' ]]; then $DOCKER_CMD AutorestCI/azure-sdk-for-python --pr-repo-id Azure/azure-sdk-for-python -o master -v; fi - - if [[ $MODE == 'node' ]]; then $DOCKER_CMD AutorestCI/azure-sdk-for-node --pr-repo-id Azure/azure-sdk-for-node -o master -v; fi - - if [[ $MODE == 'ruby' ]]; then $DOCKER_CMD AutorestCI/azure-sdk-for-ruby --pr-repo-id Azure/azure-sdk-for-ruby -o master -v; fi - - if [[ $MODE == 'syntax' ]]; then npm test -- test/syntax.js; fi - - if [[ $MODE == 'linter' ]]; then npm test -- test/linter.js; fi - - if [[ $MODE == 'semantic' ]]; then npm test -- test/semantic.js; fi - - if [[ $MODE == 'model' ]]; then npm test -- test/model.js; fi - - if [[ $MODE == 'BreakingChange' ]]; then node -- scripts/breaking-change.js; fi - - if [[ $MODE == 'azurebot' ]]; then node scripts/momentOfTruth.js; fi + - >- + if [[ $MODE == 'python' ]]; then + $DOCKER_CMD AutorestCI/azure-sdk-for-python --pr-repo-id Azure/azure-sdk-for-python -o master -v + fi + - >- + if [[ $MODE == 'node' ]]; then + $DOCKER_CMD AutorestCI/azure-sdk-for-node --pr-repo-id Azure/azure-sdk-for-node -o master -v + fi + - >- + if [[ $MODE == 'ruby' ]]; then + $DOCKER_CMD AutorestCI/azure-sdk-for-ruby --pr-repo-id Azure/azure-sdk-for-ruby -o master -v + fi + - >- + if [[ $MODE == 'syntax' ]]; then + npm test -- test/syntax.js + fi + - >- + if [[ $MODE == 'linter' ]]; then + npm test -- test/linter.js + fi + - >- + if [[ $MODE == 'semantic' ]]; then + npm test -- test/semantic.js + fi + - >- + if [[ $MODE == 'model' ]]; then + npm test -- test/model.js + fi + - >- + if [[ $MODE == 'BreakingChange' ]]; then + node -- scripts/breaking-change.js + fi + - >- + if [[ $MODE == 'azurebot' ]]; then + node scripts/momentOfTruth.js + fi + - >- + if [[ $MODE == 'liveValidation' ]]; then + node -- scripts/liveValidation.js; + fi diff --git a/package.json b/package.json index d9293640fd08..f6827855e58e 100644 --- a/package.json +++ b/package.json @@ -10,19 +10,19 @@ "description": "Tests for Azure REST API Specifications", "license": "MIT", "devDependencies": { + "@microsoft.azure/async-io": "^1.0.21", + "@microsoft.azure/literate": "^1.0.21", + "@microsoft.azure/polyfill": "^1.0.17", "fs-extra": "^3.0.1", - "mocha": "*", "glob": "^5.0.14", + "js-yaml": "^3.8.2", "json-schema-ref-parser": "^3.1.2", - "request": "^2.61.0", - "z-schema": "^3.16.1", + "mocha": "*", + "oad": "^0.1.9", "oav": "^0.4.1", - "js-yaml": "^3.8.2", - "azure-storage": "^2.1.0", - "@microsoft.azure/literate": "^1.0.21", - "@microsoft.azure/async-io": "^1.0.21", - "@microsoft.azure/polyfill": "^1.0.17", - "oad": "^0.1.9" + "request": "^2.61.0", + "request-promise-native": "^1.0.5", + "z-schema": "^3.16.1" }, "homepage": "https://github.com/azure/azure-rest-api-specs", "repository": { @@ -35,4 +35,4 @@ "scripts": { "test": "mocha -t 500000" } -} +} \ No newline at end of file diff --git a/scripts/liveValidation.js b/scripts/liveValidation.js new file mode 100644 index 000000000000..dc5d28e13e94 --- /dev/null +++ b/scripts/liveValidation.js @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License in the project root for license information. + +'use strict'; + +const utils = require('../test/util/utils'), + request = require('request-promise-native'), + zlib = require('zlib'); + +const repoUrl = utils.getRepoUrl(), + validationService = "https://app.azure-devex-tools.com/api/validations", + branch = utils.getSourceBranch(), + processingDelay = 20, + isRunningInTravisCI = process.env.MODE === 'liveValidation' && process.env.PR_ONLY === 'true', + specsPaths = utils.getFilesChangedInPR(), + regex = /resource-manager[\\|\/](.*?)[\\|\/].*?[\\|\/](.*?)[\\|\/]/, + successThreshold = 90, + validationModels = new Map(); + +let durationInSeconds = parseInt(process.env.LIVE_VALIDATION_DURATION_IN_MINUTES) * 60; +if (isNaN(durationInSeconds)) { + durationInSeconds = 180; +} + +async function runScript() { + // See whether script is in Travis CI context + console.log(`isRunningInTraviSCI: ${isRunningInTravisCI}`); + for (const specPath of specsPaths) { + let matchResult = specPath.match(regex); + + if (matchResult === null) { + continue; + } + + let resourceProvider = matchResult[1]; + let apiVersion = matchResult[2]; + + if (!validationModels.has(resourceProvider)) { + validationModels.set(resourceProvider, new Set()); + } + + validationModels.get(resourceProvider).add(apiVersion); + } + + if (validationModels.size === 0) { + console.log("Change didn't affect any swagger specs. No validation to be done."); + return; + } else if (validationModels.size > 1) { + console.log("WARNING: Multiple resource provider have changes, only the first one will be validated."); + } + + let resourceProvider = validationModels.keys().next().value; + + if (validationModels.get(resourceProvider).size > 1) { + console.log("WARNING: Multiple api versions have changes, only the first one will be validated."); + } + + let apiVersion = validationModels.get(resourceProvider).values().next().value; + + console.log(`Changes detected in a swagger spec.`); + console.log(`RP is: ${resourceProvider}`); + console.log(`ApiVersion is: ${apiVersion}`); + console.log(`Source repo is: ${repoUrl}`); + console.log(`Branch is: ${branch}`); + + console.log(`Making the request to the validation service...`); + + let response = await request.post(validationService).form({ + repoUrl: repoUrl, + branch: branch, + resourceProvider: resourceProvider, + apiVersion: apiVersion, + duration: durationInSeconds + }); + let validationId = JSON.parse(response).validationId; + + let validationResultUrl = `${validationService}/${validationId}`; + console.log(`Request done, results will in ${durationInSeconds} seconds...`); + + await timeout((durationInSeconds + processingDelay) * 1000); + let validationResult = JSON.parse(await request(validationResultUrl)); + + console.log(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"); + console.log(`Results of validation ${validationId}:`); + + let analyticsUrl = await createAnalyticsLink(validationId); + + let failingOperations = []; + let noTrafficOperations = []; + for (const [operationId, operationResult] of Object.entries(validationResult.operationResults)) { + + if (operationResult.operationCount === 0) { + noTrafficOperations.push(operationResult.operationId) + } else if (operationResult.successRate < successThreshold) { + failingOperations.push(operationResult.operationId); + } + + console.log(JSON.stringify(operationResult)); + } + + console.log(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"); + if (failingOperations.length > 0 || noTrafficOperations.length > 0) { + console.log(`The changes in the specs introduced by this PR potentially do not reflect the Service API.`); + + console.log(`Active traffic and success rate > ${successThreshold}% FOR EACH OPERATION is required. Please review the following operations before moving forward.`); + console.log(`SUCCESS RATE < ${successThreshold}%: + ${JSON.stringify(failingOperations)}`); + + if (noTrafficOperations.length > 0) { + console.log(`NO TRAFFIC: + ${JSON.stringify(noTrafficOperations)} + `); + } + console.log(`To inspect the individual failures go to the url (add '| where customDimensions.operationId == ""' to filter for individual operations.): + ${analyticsUrl} + `); + process.exitCode = 1; + } else { + console.log(`SUCCESS RATE: ${validationResult.SuccessRate} > ${successThreshold}. You can move forward:`); + } +} + +function timeout(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +function createAnalyticsLink(validationId) { + return new Promise(resolve => { + const query = ` +traces +| where customDimensions.validationId == "${validationId}" +| where customDimensions.logType == "data" +| where customDimensions.isSuccess == "false" +| project timestamp, message, customDimensions +`; + + zlib.deflate(query, (err, buffer) => { + if (!err) { + let queryParams = buffer.toString('base64'); + let analyticsLink = `https://analytics.applicationinsights.io/subscriptions/6b085460-5f21-477e-ba44-1035046e9101/resourcegroups/openapi-platform-logs/components/openapiAI?q=${queryParams}&apptype=Node.JS×pan=P1D`; + resolve(analyticsLink); + } + }); + }); +} + +runScript().then(success => { + console.log(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"); + console.log(`Thanks for using live validation.`); + console.log(`If you encounter any issue(s), please open issue(s) at https://github.com/Azure/openapi-platform/issues .`); +}).catch(err => { + console.log(err); + process.exitCode = 1; +}); diff --git a/scripts/momentOfTruth.js b/scripts/momentOfTruth.js index 2fecae05cbbf..9fe2ef4aa00a 100644 --- a/scripts/momentOfTruth.js +++ b/scripts/momentOfTruth.js @@ -23,14 +23,9 @@ var filename = `${pullRequestNumber}_${utils.getTimeStamp()}.json`; var logFilepath = path.join(getLogDir(), filename); var finalResult = {}; finalResult["pullRequest"] = pullRequestNumber; -finalResult["repositoryUrl"] = getRepository(); +finalResult["repositoryUrl"] = utils.getRepoUrl(); finalResult["files"] = {}; -// Retrieves Git Repository Url -function getRepository() { - return "https://github.com/" + utils.getRepoName(); -} - // Creates and returns path to the logging directory function getLogDir() { let logDir = path.join(__dirname, '../', 'output'); @@ -68,7 +63,7 @@ async function getLinterResult(swaggerPath) { } let cmd = linterCmd + swaggerPath; console.log(`Executing: ${cmd}`); - const {err, stdout, stderr } = await new Promise(res => exec(cmd, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 64 }, + const { err, stdout, stderr } = await new Promise(res => exec(cmd, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 64 }, (err, stdout, stderr) => res({ err: err, stdout: stdout, stderr: stderr }))); let resultString = stderr; @@ -94,7 +89,7 @@ async function getLinterResult(swaggerPath) { async function uploadToAzureStorage(json) { console.log(`Uploading data...`); - const {error, response, body } = await new Promise(res => request({ + const { error, response, body } = await new Promise(res => request({ url: "http://az-bot.azurewebsites.net/process", method: "POST", json: true, @@ -125,7 +120,7 @@ async function updateResult(spec, errors, beforeOrAfter) { //main function async function runScript() { - // Useful when debugging a test for a particular swagger. + // Useful when debugging a test for a particular swagger. // Just update the regex. That will return an array of filtered items. // configsToProcess = ['/Users/vishrut/git-repos/azure-rest-api-specs/specification/storage/resource-manager/readme.md', // '/Users/vishrut/git-repos/azure-rest-api-specs/specification/web/resource-manager/readme.md']; diff --git a/test/util/utils.js b/test/util/utils.js index ef58686e0c89..facb611e51f7 100644 --- a/test/util/utils.js +++ b/test/util/utils.js @@ -30,7 +30,7 @@ exports.globPath = path.join(__dirname, '../', '../', '/specification/**/*.json' exports.swaggers = glob.sync(exports.globPath, { ignore: ['**/examples/**/*.json', '**/quickstart-templates/*.json', '**/schema/*.json'] }); exports.exampleGlobPath = path.join(__dirname, '../', '../', '/specification/**/examples/**/*.json'); exports.examples = glob.sync(exports.exampleGlobPath); -exports.readmes = glob.sync(path.join(__dirname, '../', '../', '/specification/**/readme.md')); +exports.readmes = glob.sync(path.join(__dirname, '../', '../', '/specification/**/readme.md')); // Remove byte order marker. This catches EF BB BF (the UTF-8 BOM) // because the buffer-to-string conversion in `fs.readFile()` @@ -79,18 +79,17 @@ exports.getTargetBranch = function getTargetBranch() { exports.checkoutTargetBranch = function checkoutTargetBranch() { let targetBranch = exports.getTargetBranch(); let cmds = [`git remote -vv`, `git branch --all`, - `git remote set-branches origin --add ${targetBranch}`, - `git fetch origin ${targetBranch}`, - `git diff`, - `git stash`, - `git checkout ${targetBranch}`, - `git log -3`]; + `git remote set-branches origin --add ${targetBranch}`, + `git fetch origin ${targetBranch}`, + `git diff`, + `git stash`, + `git checkout ${targetBranch}`, + `git log -3`]; console.log(`Changing the branch to ${targetBranch}...`); - for(let cmd of cmds) - { + for (let cmd of cmds) { console.log(cmd); - execSync(cmd, { encoding: 'utf8', stdio: 'inherit' }); + execSync(cmd, { encoding: 'utf8', stdio: 'inherit' }); } } @@ -135,7 +134,7 @@ exports.getPullRequestNumber = function getPullRequestNumber() { * Gets the Repo name. We are using the environment * variable provided by travis-ci. It is called TRAVIS_REPO_SLUG. More info can be found here: * https://docs.travis-ci.com/user/environment-variables/#Convenience-Variables - * @returns {string} PR number or 'undefined'. + * @returns {string} repo name or 'undefined'. */ exports.getRepoName = function getRepoName() { let result = process.env['TRAVIS_REPO_SLUG']; @@ -144,6 +143,16 @@ exports.getRepoName = function getRepoName() { return result; }; +// Retrieves Git Repository Url +/** + * Gets the repo URL + * @returns {string} repo URL or 'undefined' + */ +exports.getRepoUrl = function getRepoUrl() { + let repoName = exports.getRepoName(); + return `https://github.com/${repoName}`; +} + exports.getTimeStamp = function getTimeStamp() { // We pad each value so that sorted directory listings show the files in chronological order function pad(number) { @@ -238,13 +247,13 @@ exports.getFilesChangedInPR = function getFilesChangedInPR() { }); console.log(`>>>> Number of swaggers found in this PR: ${swaggerFilesInPR.length}`); - var deletedFiles = swaggerFilesInPR.filter(function(swaggerFile){ + var deletedFiles = swaggerFilesInPR.filter(function (swaggerFile) { return !fs.existsSync(swaggerFile); }); console.log('>>>>> Files deleted in this PR are as follows:') console.log(deletedFiles); // Remove files that have been deleted in the PR - swaggerFilesInPR = swaggerFilesInPR.filter(function(x) { return deletedFiles.indexOf(x) < 0 }); + swaggerFilesInPR = swaggerFilesInPR.filter(function (x) { return deletedFiles.indexOf(x) < 0 }); result = swaggerFilesInPR; } catch (err) {