diff --git a/src/api/deploy.ts b/src/api/deploy.ts index 373135cde..27cb77780 100644 --- a/src/api/deploy.ts +++ b/src/api/deploy.ts @@ -2,8 +2,11 @@ import { Observable, Observer } from 'rxjs'; import { s3Deploy } from './deploy/aws-s3'; import { codeDeploy } from './deploy/aws-code-deploy'; import { elasticDeploy } from './deploy/aws-elastic'; +import * as envVars from './env-variables'; -export function deploy(preferences: any, container: string, variables: string[]): Observable { +export function deploy( + preferences: any, container: string, variables: envVars.EnvVariables +): Observable { return new Observable((observer: Observer) => { if (preferences) { const provider = preferences.provider; @@ -18,7 +21,9 @@ export function deploy(preferences: any, container: string, variables: string[]) }); } -function deployProvider(provider, preferences, container, variables): Observable { +function deployProvider( + provider: string, preferences: any, container: string, variables: envVars.EnvVariables +): Observable { switch (provider) { case 's3': return s3Deploy(preferences, container, variables); @@ -37,14 +42,10 @@ function deployProvider(provider, preferences, container, variables): Observable } } -export function findFromEnvVariables(variables, property) { - let value = variables.find(v => v.startsWith(property)); - - if (value) { - const tmp = value.split('='); - if (tmp.length > 1) { - return tmp[1]; - } +export function findFromEnvVariables(variables: envVars.EnvVariables, property: string) { + let value = variables[property]; + if (typeof value !== 'undefined') { + return value.value; } return null; diff --git a/src/api/deploy/aws-code-deploy.ts b/src/api/deploy/aws-code-deploy.ts index 63a3424e4..ba752ed3d 100644 --- a/src/api/deploy/aws-code-deploy.ts +++ b/src/api/deploy/aws-code-deploy.ts @@ -5,9 +5,10 @@ import { findFromEnvVariables } from '../deploy'; import * as style from 'ansi-styles'; import { error } from 'util'; import chalk from 'chalk'; +import * as envVars from '../env-variables'; export function codeDeploy( - preferences: any, container: string, variables: string[] + preferences: any, container: string, variables: envVars.EnvVariables ): Observable { return new Observable((observer: Observer) => { @@ -89,7 +90,7 @@ export function codeDeploy( let command = { type: CommandType.deploy, command: `aws configure set aws_access_key_id ${accessKeyId}` }; - dockerExec(container, command) + dockerExec(container, command, variables) .toPromise() .then(result => { if (!(result && result.data === 0)) { @@ -103,7 +104,7 @@ export function codeDeploy( command: `aws configure set aws_secret_access_key ${secretAccessKey}` }; - return dockerExec(container, command).toPromise(); + return dockerExec(container, command, variables).toPromise(); }) .then(result => { if (!(result && result.data === 0)) { @@ -116,7 +117,7 @@ export function codeDeploy( type: CommandType.deploy, command: `aws configure set region ${region}` }; - return dockerExec(container, command).toPromise(); + return dockerExec(container, command, variables).toPromise(); }) .then(result => { if (!(result && result.data === 0)) { @@ -137,7 +138,7 @@ export function codeDeploy( + ` --deployment-group-name ${deployGroup} --service-role-arn ${arn}` }; - return dockerExec(container, command) + return dockerExec(container, command, variables) .toPromise() .then(result => { if (!(result && result.data === 0)) { @@ -182,7 +183,7 @@ export function codeDeploy( return Promise.reject(1); } - return dockerExec(container, command) + return dockerExec(container, command, variables) .toPromise() .then(result => { if (!(result && result.data === 0)) { @@ -211,7 +212,7 @@ export function codeDeploy( }); } -function depGroupExists(container, application, group): Promise { +function depGroupExists(container: string, application: string, group: string): Promise { return new Promise((resolve, reject) => { const command = `aws deploy get-deployment-group --application-name ${application}` + ` --deployment-group ${group}`; diff --git a/src/api/deploy/aws-elastic.ts b/src/api/deploy/aws-elastic.ts index 5b81ade77..75d38e4da 100644 --- a/src/api/deploy/aws-elastic.ts +++ b/src/api/deploy/aws-elastic.ts @@ -4,9 +4,10 @@ import { CommandType } from '../config'; import { findFromEnvVariables } from '../deploy'; import * as style from 'ansi-styles'; import chalk from 'chalk'; +import * as envVars from '../env-variables'; export function elasticDeploy( - preferences: any, container: string, variables: string[] + preferences: any, container: string, variables: envVars.EnvVariables ): Observable { return new Observable((observer: Observer) => { // 1. check preferences @@ -106,7 +107,7 @@ export function elasticDeploy( let command = { type: CommandType.deploy, command: `aws configure set aws_access_key_id ${accessKeyId}` }; - dockerExec(container, command) + dockerExec(container, command, variables) .toPromise() .then(result => { if (!(result && result.data === 0)) { @@ -120,7 +121,7 @@ export function elasticDeploy( command: `aws configure set aws_secret_access_key ${secretAccessKey}` }; - return dockerExec(container, command).toPromise(); + return dockerExec(container, command, variables).toPromise(); }) .then(result => { if (!(result && result.data === 0)) { @@ -133,7 +134,7 @@ export function elasticDeploy( type: CommandType.deploy, command: `aws configure set region ${region}` }; - return dockerExec(container, command).toPromise(); + return dockerExec(container, command, variables).toPromise(); }) .then(result => { if (!(result && result.data === 0)) { @@ -160,7 +161,7 @@ export function elasticDeploy( }; } - return dockerExec(container, command).toPromise(); + return dockerExec(container, command, variables).toPromise(); }) .then(() => { // 3. check if environment exists @@ -176,7 +177,7 @@ export function elasticDeploy( + ` --template-name "${environmentTemplate}"` }; - return dockerExec(container, command) + return dockerExec(container, command, variables) .toPromise() .then(result => { if (!(result && result.data === 0)) { @@ -194,7 +195,7 @@ export function elasticDeploy( + ` --solution-stack-name "${solutionStackName}"` }; - return dockerExec(container, command) + return dockerExec(container, command, variables) .toPromise() .then(result => { if (!(result && result.data === 0)) { @@ -233,7 +234,7 @@ export function elasticDeploy( }); } -function environmentExists(container, environment): Promise { +function environmentExists(container: string, environment: string): Promise { return new Promise((resolve, reject) => { const getEnvCommand = `aws elasticbeanstalk describe-environments --environment-names` + ` "${environment}"`; diff --git a/src/api/deploy/aws-s3.ts b/src/api/deploy/aws-s3.ts index 034cde054..a2b6d60df 100644 --- a/src/api/deploy/aws-s3.ts +++ b/src/api/deploy/aws-s3.ts @@ -4,9 +4,10 @@ import { CommandType } from '../config'; import { findFromEnvVariables } from '../deploy'; import * as style from 'ansi-styles'; import chalk from 'chalk'; +import * as envVars from '../env-variables'; export function s3Deploy( - preferences: any, container: string, variables: string[] + preferences: any, container: string, variables: envVars.EnvVariables ): Observable { return new Observable((observer: Observer) => { @@ -75,7 +76,7 @@ export function s3Deploy( } return Observable - .concat(...commands.map(command => dockerExec(container, command))) + .concat(...commands.map(command => dockerExec(container, command, variables))) .toPromise(); }) .then(result => { @@ -90,7 +91,7 @@ export function s3Deploy( type: CommandType.deploy, command: `aws configure set aws_access_key_id ${accessKeyId}` }; - return dockerExec(container, command).toPromise(); + return dockerExec(container, command, variables).toPromise(); }) .then(result => { if (!(result && result.data === 0)) { @@ -104,7 +105,7 @@ export function s3Deploy( command: `aws configure set aws_secret_access_key ${secretAccessKey}` }; - return dockerExec(container, command).toPromise(); + return dockerExec(container, command, variables).toPromise(); }) .then(result => { if (!(result && result.data === 0)) { @@ -117,7 +118,7 @@ export function s3Deploy( type: CommandType.deploy, command: `aws configure set region ${region}` }; - return dockerExec(container, command).toPromise(); + return dockerExec(container, command, variables).toPromise(); }) .then(result => { if (!(result && result.data === 0)) { @@ -140,7 +141,7 @@ export function s3Deploy( } return Observable - .concat(...application.map(command => dockerExec(container, command))) + .concat(...application.map(command => dockerExec(container, command, variables))) .toPromise(); }) .then(result => { @@ -158,7 +159,7 @@ export function s3Deploy( + ` --s3-location s3://${preferences.bucket}/${zipName}.zip` }; - return dockerExec(container, deploy).toPromise(); + return dockerExec(container, deploy, variables).toPromise(); }) .then(result => { if (!(result && result.data === 0)) { @@ -183,7 +184,7 @@ export function s3Deploy( }); } -function appSpecExists(container): Promise { +function appSpecExists(container: string): Promise { return new Promise((resolve, reject) => { let appSpec = false; dockerExec(container, { type: CommandType.deploy, command: 'ls'}) @@ -199,7 +200,7 @@ function appSpecExists(container): Promise { }); } -function applicationExists(container, application): Promise { +function applicationExists(container: string, application: string): Promise { return new Promise((resolve, reject) => { const getApplicationCommand = 'aws deploy list-applications'; let appExists = false; diff --git a/src/api/docker.ts b/src/api/docker.ts index 46789c5ba..dfb884fa7 100644 --- a/src/api/docker.ts +++ b/src/api/docker.ts @@ -14,7 +14,7 @@ export const docker = new dockerode(); export function createContainer( name: string, image: string, - envs: string[] + envs: envVars.EnvVariables ): Observable { return new Observable(observer => { docker.createContainer({ @@ -23,7 +23,7 @@ export function createContainer( Tty: true, OpenStdin: true, StdinOnce: false, - Env: envs || [], + Env: envVars.serialize(envs) || [], Binds: ['/var/run/docker.sock:/var/run/docker.sock'], Privileged: true, ExposedPorts: { @@ -60,7 +60,9 @@ export function startContainer(id: string): Promise { return docker.getContainer(id).start(); } -export function dockerExec(id: string, cmd: any, env: envVars.EnvVariables = {}): Observable { +export function dockerExec( + id: string, cmd: any, env: envVars.EnvVariables = {} +): Observable { return new Observable(observer => { let exitCode = 255; let command; @@ -125,6 +127,13 @@ export function dockerExec(id: string, cmd: any, env: envVars.EnvVariables = {}) if (str.includes('//') && str.includes('@')) { str = str.replace(/\/\/(.*)@/, '//'); } + + const variable = + Object.keys(env).find(k => env[k].secure && str.indexOf(env[k].value) >= 0); + if (typeof variable !== 'undefined') { + str = str.replace(env[variable].value, '******'); + } + observer.next({ type: 'data', data: str }); } @@ -138,6 +147,22 @@ export function dockerExec(id: string, cmd: any, env: envVars.EnvVariables = {}) }); } +export function dockerPwd(id: string, env: envVars.EnvVariables): Observable { + return new Observable(observer => { + dockerExec(id, { type: CommandType.before_install, command: 'pwd'}, env) + .subscribe(event => { + if (event && event.data && event.type === 'data') { + envVars.set(env, 'ABSTRUSE_BUILD_DIR', event.data.replace('\r\n', '')); + } + }, + err => observer.error(err), + () => { + observer.next({ type: 'env', data: env }); + observer.complete(); + }); + }); +} + export function listContainers(): Promise { return docker.listContainers(); } diff --git a/src/api/env-variables.ts b/src/api/env-variables.ts index f55a1c86e..24c09c079 100644 --- a/src/api/env-variables.ts +++ b/src/api/env-variables.ts @@ -1,23 +1,28 @@ export interface EnvVariables { - [key: string]: string | number | boolean; + [key: string]: { + value: string | number | boolean, + secure: boolean + }; } -export function set(envs: EnvVariables, key: string, value: string | number | boolean): void { - envs[key] = value; +export function set( + envs: EnvVariables, key: string, value: string | number | boolean, secure = false +): void { + envs[key] = { value: value, secure: secure }; } export function unset(envs: EnvVariables, key: string): void { - envs[key] = null; + delete envs[key]; } export function serialize(envs: EnvVariables): string[] { - return Object.keys(envs).map(key => `${key}=${envs[key]}`); + return Object.keys(envs).map(key => `${key}=${envs[key].value}`); } export function unserialize(envs: string[]): EnvVariables { return envs.reduce((acc, curr) => { const splitted = curr.split('='); - acc = Object.assign({}, acc, { [splitted[0]]: splitted[1] }); + acc = Object.assign({}, acc, { [splitted[0]]: { value: splitted[1], secure: false }}); return acc; }, {}); } @@ -36,6 +41,13 @@ export function generate(data: any): EnvVariables { request.data.pullrequest.source.commit && request.data.pullrequest.source.commit.hash || request.data.commit || ''; + const prBranch = request.pr ? request.data.pull_request && request.data.pull_request.head && + request.data.pull_request.head.ref || + request.data.pullrequest && request.data.pullrequest.source && + request.data.pullrequest.source.branch && + request.data.pullrequest.source.branch.name || + request.data.object_attributes && request.data.object_attributes.source_branch || + request.data.pull_request && request.data.pull_request.head_branch : ''; const tag = request.ref && request.ref.startsWith('refs/tags/') ? request.ref.replace('refs/tags/', '') : null; @@ -45,7 +57,7 @@ export function generate(data: any): EnvVariables { set(envs, 'ABSTRUSE_COMMIT', commit); set(envs, 'ABSTRUSE_EVENT_TYPE', request.pr ? 'pull_request' : 'push'); set(envs, 'ABSTRUSE_PULL_REQUEST', request.pr ? request.pr : false); - set(envs, 'ABSTRUSE_PULL_REQUEST_BRANCH', request.pr ? request.branch : ''); + set(envs, 'ABSTRUSE_PULL_REQUEST_BRANCH', prBranch); set(envs, 'ABSTRUSE_TAG', tag); const prSha = request.pr ? commit : ''; @@ -54,15 +66,15 @@ export function generate(data: any): EnvVariables { return envs; } -function init(): EnvVariables { +export function init(): EnvVariables { return [ 'ABSTRUSE_BRANCH', 'ABSTRUSE_BUILD_DIR', 'ABSTRUSE_BUILD_ID', 'ABSTRUSE_JOB_ID', 'ABSTRUSE_COMMIT', 'ABSTRUSE_EVENT_TYPE', 'ABSTRUSE_PULL_REQUEST', 'ABSTRUSE_PULL_REQUEST_BRANCH', - 'ABSTRUSE_TAG', 'ABSTRUSE_PULL_REQEUST_SHA', 'ABSTRUSE_SECURE_ENV_VARS', + 'ABSTRUSE_TAG', 'ABSTRUSE_PULL_REQUEST_SHA', 'ABSTRUSE_SECURE_ENV_VARS', 'ABSTRUSE_TEST_RESULT' ].reduce((acc, curr) => { - acc = Object.assign(acc, { [curr]: null }); + acc = Object.assign(acc, { [curr]: { value: null, secure: false } }); return acc; }, {}); } diff --git a/src/api/process-manager.ts b/src/api/process-manager.ts index d409ad7cf..7db3873a4 100644 --- a/src/api/process-manager.ts +++ b/src/api/process-manager.ts @@ -20,6 +20,7 @@ import { getHttpJsonResponse, getBitBucketAccessToken } from './utils'; import { getConfig } from './setup'; import { sendFailureStatus, sendPendingStatus, sendSuccessStatus } from './commit-status'; import { decrypt } from './security'; +import * as envVars from './env-variables'; export interface BuildMessage { type: string; @@ -118,18 +119,24 @@ export function startJobProcess(proc: JobProcess): Observable<{}> { return new Observable(observer => { getRepositoryByBuildId(proc.build_id) .then(repository => { - const envVariables: string[] = repository.variables.map(v => { + let envs = envVars.generate(proc); + let secureVarirables = false; + repository.variables.forEach(v => { if (!!v.encrypted) { - return `${v.name}=${decrypt(v.value)}`; + secureVarirables = true; + envVars.set(envs, v.name, decrypt(v.value), true); } else { - return `${v.name}=${v.value}`; + envVars.set(envs, v.name, v.value); } }); + envVars.set(envs, 'ABSTRUSE_SECURE_ENV_VARS', secureVarirables); + const jobTimeout = config.jobTimeout ? config.jobTimeout * 1000 : 3600000; const idleTimeout = config.idleTimeout ? config.idleTimeout * 1000 : 3600000; - buildSub[proc.job_id] = startBuildProcess(proc, envVariables, jobTimeout, idleTimeout) + buildSub[proc.job_id] = + startBuildProcess(proc, envs, jobTimeout, idleTimeout) .subscribe(event => { const msg: JobProcessEvent = { build_id: proc.build_id, diff --git a/src/api/process.ts b/src/api/process.ts index 7ea64144d..dfdaffc44 100644 --- a/src/api/process.ts +++ b/src/api/process.ts @@ -33,7 +33,7 @@ export interface ProcessOutput { export function startBuildProcess( proc: JobProcess, - variables: string[], + envs: envVars.EnvVariables, jobTimeout: number, idleTimeout: number ): Observable { @@ -41,15 +41,17 @@ export function startBuildProcess( const image = proc.image_name; const name = 'abstruse_' + proc.build_id + '_' + proc.job_id; - let envs = envVars.generate(proc); - const initEnvs = proc.commands.filter(cmd => { - return typeof cmd.command === 'string' && cmd.command.startsWith('export'); - }) + proc.commands + .filter(cmd => typeof cmd.command === 'string' && cmd.command.startsWith('export')) .map(cmd => cmd.command.replace('export', '')) .reduce((acc, curr) => acc.concat(curr.split(' ')), []) .concat(proc.env.reduce((acc, curr) => acc.concat(curr.split(' ')), [])) - .concat(variables) - .filter(Boolean); + .forEach(env => { + const splitted = env.split('='); + if (splitted.length > 1) { + envVars.set(envs, splitted[0], splitted[1]); + } + }); const gitTypes = [CommandType.git]; const installTypes = [CommandType.before_install, CommandType.install]; @@ -88,7 +90,8 @@ export function startBuildProcess( restoreCache = Observable.concat(...[ executeOutsideContainer(copyRestoreCmd), - docker.dockerExec(name, { command: restoreCmd, type: CommandType.restore_cache, env: envs }) + docker.dockerExec( + name, { command: restoreCmd, type: CommandType.restore_cache, env: envs }) ]); let cacheFolders = proc.cache.map(folder => { @@ -109,12 +112,14 @@ export function startBuildProcess( ].join(''); saveCache = Observable.concat(...[ - docker.dockerExec(name, { command: tarCmd, type: CommandType.store_cache, env: envs }), + docker.dockerExec( + name, { command: tarCmd, type: CommandType.store_cache, env: envs }), executeOutsideContainer(saveTarCmd) ]); } - const sub = docker.createContainer(name, image, initEnvs) + const sub = docker.createContainer(name, image, envs) + .concat(docker.dockerPwd(name, envs)) .concat(...gitCommands.map(cmd => docker.dockerExec(name, cmd, envs))) .concat(restoreCache) .concat(...installCommands.map(cmd => docker.dockerExec(name, cmd, envs))) @@ -122,7 +127,7 @@ export function startBuildProcess( .concat(...scriptCommands.map(cmd => docker.dockerExec(name, cmd, envs))) .concat(...beforeDeployCommands.map(cmd => docker.dockerExec(name, cmd, envs))) .concat(...deployCommands.map(cmd => docker.dockerExec(name, cmd, envs))) - .concat(deploy(deployPreferences, name, initEnvs)) + .concat(deploy(deployPreferences, name, envs)) .concat(...afterDeployCommands.map(cmd => docker.dockerExec(name, cmd, envs))) .timeoutWith(idleTimeout, Observable.throw(new Error('command timeout'))) .takeUntil(Observable.timer(jobTimeout).timeInterval().mergeMap(() => {