From fbf297543f1cc26b7b8e86343081cf9b029101d4 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 7 Aug 2018 12:28:19 +0200 Subject: [PATCH 1/3] Toolkit: support profiles This adds support for AWS profiles to the CDK toolkit. At the same time, it overhauls how the AWS SDK is configured. The configuration via environment variables set at just the right time is removed, and we reimplement some parts of the SDK in an AWS CLI-compatible way to get a consistent view on the account ID and region based on the provided configuration. Fixes a bug in the AWS STS call where it would do two default credential lookups (down to one now). Fixes #480. --- packages/aws-cdk/bin/cdk.ts | 8 +- packages/aws-cdk/lib/api/deploy-stack.ts | 4 +- .../lib/api/util/sdk-load-aws-config.ts | 45 ------- packages/aws-cdk/lib/api/util/sdk.ts | 113 +++++++++++++++--- packages/aws-cdk/lib/api/util/sdk_ini_file.ts | 52 ++++++++ 5 files changed, 154 insertions(+), 68 deletions(-) delete mode 100644 packages/aws-cdk/lib/api/util/sdk-load-aws-config.ts create mode 100644 packages/aws-cdk/lib/api/util/sdk_ini_file.ts diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index 65bd8be5ef00d..9226e8b700db7 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -1,9 +1,6 @@ #!/usr/bin/env node import 'source-map-support/register'; -// Ensure the AWS SDK is properly initialized before anything else. -import '../lib/api/util/sdk-load-aws-config'; - import cxapi = require('@aws-cdk/cx-api'); import cdkUtil = require('@aws-cdk/util'); import childProcess = require('child_process'); @@ -49,6 +46,7 @@ async function parseCommandLineArguments() { .option('ignore-errors', { type: 'boolean', default: false, desc: 'Ignores synthesis errors, which will likely produce an invalid output' }) .option('json', { type: 'boolean', alias: 'j', desc: 'Use JSON output instead of YAML' }) .option('verbose', { type: 'boolean', alias: 'v', desc: 'Show debug logs' }) + .option('profile', { type: 'string', desc: 'Use the indicated AWS profile' }) // tslint:disable-next-line:max-line-length .option('version-reporting', { type: 'boolean', desc: 'Disable insersion of the CDKMetadata resource in synthesized templates', default: undefined }) .command([ 'list', 'ls' ], 'Lists all stacks in the app', yargs => yargs @@ -110,7 +108,9 @@ async function initCommandLine() { debug('Command line arguments:', argv); - const aws = new SDK(); + const aws = new SDK(argv.profile); + // tslint:disable-next-line:no-console + console.log("Account: ", await aws.defaultAccount(), " region: ", aws.defaultRegion()); const availableContextProviders: contextplugins.ProviderMap = { 'availability-zones': new contextplugins.AZContextProviderPlugin(aws), diff --git a/packages/aws-cdk/lib/api/deploy-stack.ts b/packages/aws-cdk/lib/api/deploy-stack.ts index 0de865cedad5b..f744ac05d8384 100644 --- a/packages/aws-cdk/lib/api/deploy-stack.ts +++ b/packages/aws-cdk/lib/api/deploy-stack.ts @@ -23,7 +23,7 @@ export interface DeployStackResult { } export async function deployStack(stack: cxapi.SynthesizedStack, - sdk: SDK = new SDK(), + sdk: SDK, toolkitInfo?: ToolkitInfo, deployName?: string, quiet: boolean = false): Promise { @@ -134,7 +134,7 @@ async function makeBodyParameter(stack: cxapi.SynthesizedStack, toolkitInfo?: To } } -export async function destroyStack(stack: cxapi.StackInfo, sdk: SDK = new SDK(), deployName?: string, quiet: boolean = false) { +export async function destroyStack(stack: cxapi.StackInfo, sdk: SDK, deployName?: string, quiet: boolean = false) { if (!stack.environment) { throw new Error(`The stack ${stack.name} does not have an environment`); } diff --git a/packages/aws-cdk/lib/api/util/sdk-load-aws-config.ts b/packages/aws-cdk/lib/api/util/sdk-load-aws-config.ts deleted file mode 100644 index 366f3d7ee5de6..0000000000000 --- a/packages/aws-cdk/lib/api/util/sdk-load-aws-config.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * IMPORTANT: This **must** be required _before_ 'aws-sdk' is. - * - * This ensures the correct environment is set-up so the AWS SDK properly - * loads up configruation stored in the shared credentials file (usually - * found at ~/.aws/credentials) and the aws config file (usually found at - * ~/.aws/config), if either is present. - * - * @see https://github.com/awslabs/aws-cdk/pull/128 - */ - -import fs = require('fs'); -import os = require('os'); -import path = require('path'); - -const sharedCredentialsFile = - process.env.AWS_SHARED_CREDENTIALS_FILE ? process.env.AWS_SHARED_CREDENTIALS_FILE - : path.join(os.homedir(), '.aws', 'credentials'); -const awsConfigFile = - process.env.AWS_CONFIG_FILE ? process.env.AWS_CONFIG_FILE - : path.join(os.homedir(), '.aws', 'config'); - -if (fs.existsSync(awsConfigFile) && !fs.existsSync(sharedCredentialsFile)) { - /* - * Write an empty credentials file if there's a config file, otherwise the SDK will simply bail out, - * since the credentials file is loaded before the config file is. - */ - fs.writeFileSync(sharedCredentialsFile, ''); -} -if (fs.existsSync(sharedCredentialsFile)) { - // Ensures that region is loaded from ~/.aws/config (https://github.com/aws/aws-sdk-js/pull/1391) - process.env.AWS_SDK_LOAD_CONFIG = '1'; -} - -/* - * Set environment variables so JS AWS SDK behaves as close as possible to AWS CLI. - * @see https://github.com/aws/aws-sdk-js/issues/373 - * @see https://github.com/awslabs/aws-cdk/issues/131 - */ -if (process.env.AWS_DEFAULT_PROFILE && !process.env.AWS_PROFILE) { - process.env.AWS_PROFILE = process.env.AWS_DEFAULT_PROFILE; -} -if (process.env.AWS_DEFAULT_REGION && !process.env.AWS_REGION) { - process.env.AWS_REGION = process.env.AWS_DEFAULT_REGION; -} diff --git a/packages/aws-cdk/lib/api/util/sdk.ts b/packages/aws-cdk/lib/api/util/sdk.ts index 2f12c5db467dc..2979443d3b17b 100644 --- a/packages/aws-cdk/lib/api/util/sdk.ts +++ b/packages/aws-cdk/lib/api/util/sdk.ts @@ -1,9 +1,12 @@ import { Environment} from '@aws-cdk/cx-api'; -import { CloudFormation, config, CredentialProviderChain, EC2, S3, SSM, STS } from 'aws-sdk'; +import AWS = require('aws-sdk'); +import os = require('os'); +import path = require('path'); import { debug } from '../../logging'; import { PluginHost } from '../../plugin'; import { CredentialProviderSource, Mode } from '../aws-auth/credentials'; import { AccountAccessKeyCache } from './account-cache'; +import { SharedIniFile } from './sdk_ini_file'; /** * Source for SDK client objects @@ -19,50 +22,59 @@ export class SDK { private defaultAccountId?: string = undefined; private readonly userAgent: string; private readonly accountCache = new AccountAccessKeyCache(); + private readonly defaultCredentialProvider: AWS.CredentialProviderChain; - constructor() { + constructor(private readonly profile: string | undefined) { // Find the package.json from the main toolkit const pkg = (require.main as any).require('../package.json'); this.userAgent = `${pkg.name}/${pkg.version}`; + + // tslint:disable-next-line:no-console + console.log(new Error().stack); + + // tslint:disable-next-line:no-console + console.log('Profile', profile); + + this.defaultCredentialProvider = makeCLICompatibleCredentialProvider(profile); } - public async cloudFormation(environment: Environment, mode: Mode): Promise { - return new CloudFormation({ + public async cloudFormation(environment: Environment, mode: Mode): Promise { + return new AWS.CloudFormation({ region: environment.region, credentialProvider: await this.getCredentialProvider(environment.account, mode), customUserAgent: this.userAgent }); } - public async ec2(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise { - return new EC2({ + public async ec2(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise { + return new AWS.EC2({ region, credentialProvider: await this.getCredentialProvider(awsAccountId, mode), customUserAgent: this.userAgent }); } - public async ssm(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise { - return new SSM({ + public async ssm(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise { + return new AWS.SSM({ region, credentialProvider: await this.getCredentialProvider(awsAccountId, mode), customUserAgent: this.userAgent }); } - public async s3(environment: Environment, mode: Mode): Promise { - return new S3({ + public async s3(environment: Environment, mode: Mode): Promise { + return new AWS.S3({ region: environment.region, credentialProvider: await this.getCredentialProvider(environment.account, mode), customUserAgent: this.userAgent }); } - public defaultRegion() { - return config.region; + public defaultRegion(): string | undefined { + return getCLICompatibleDefaultRegion(this.profile); } - public async defaultAccount() { + public async defaultAccount(): Promise { if (!this.defaultAccountFetched) { this.defaultAccountId = await this.lookupDefaultAccount(); this.defaultAccountFetched = true; @@ -73,8 +85,7 @@ export class SDK { private async lookupDefaultAccount() { try { debug('Resolving default credentials'); - const chain = new CredentialProviderChain(); - const creds = await chain.resolvePromise(); + const creds = await this.defaultCredentialProvider.resolvePromise(); const accessKeyId = creds.accessKeyId; if (!accessKeyId) { throw new Error('Unable to resolve AWS credentials (setup with "aws configure")'); @@ -83,7 +94,7 @@ export class SDK { const accountId = await this.accountCache.fetch(creds.accessKeyId, async () => { // if we don't have one, resolve from STS and store in cache. debug('Looking up default account ID from STS'); - const result = await new STS().getCallerIdentity().promise(); + const result = await new AWS.STS({ credentials: creds }).getCallerIdentity().promise(); const aid = result.Account; if (!aid) { debug('STS didn\'t return an account ID'); @@ -100,7 +111,7 @@ export class SDK { } } - private async getCredentialProvider(awsAccountId: string | undefined, mode: Mode): Promise { + private async getCredentialProvider(awsAccountId: string | undefined, mode: Mode): Promise { // If requested account is undefined or equal to default account, use default credentials provider. const defaultAccount = await this.defaultAccount(); if (!awsAccountId || awsAccountId === defaultAccount) { @@ -129,3 +140,71 @@ export class SDK { throw new Error(`Need to perform AWS calls for account ${awsAccountId}, but no credentials found. Tried: ${sourceNames}.`); } } + +/** + * Build an AWS CLI-compatible credential chain provider + * + * This is similar to the default credential provider chain created by the SDK + * except it also accepts the profile argument in the constructor (not just from + * the environment). + * + * To mimic the AWS CLI behavior: + * + * - we default to ~/.aws/credentials if environment variable for credentials + * file location is not given (SDK expects explicit environment variable with name). + * - AWS_DEFAULT_PROFILE is also inspected for profile name (not just AWS_PROFILE). + */ +function makeCLICompatibleCredentialProvider(profile: string | undefined) { + profile = profile || process.env.AWS_PROFILE || process.env.AWS_DEFAULT_PROFILE || 'default'; + + // Need to construct filename ourselves, without appropriate environment variables + // no defaults used by JS SDK. + const filename = process.env.AWS_SHARED_CREDENTIALS_FILE || path.join(os.homedir(), '.aws', 'credentials'); + + return new AWS.CredentialProviderChain([ + () => new AWS.EnvironmentCredentials('AWS'), + () => new AWS.EnvironmentCredentials('AMAZON'), + () => new AWS.SharedIniFileCredentials({ profile, filename }), + () => { + // Calling private API + if ((AWS.ECSCredentials.prototype as any).isConfiguredForEcsCredentials()) { + return new AWS.ECSCredentials(); + } + return new AWS.EC2MetadataCredentials(); + } + ]); +} + +/** + * Return the default region in a CLI-compatible way + * + * Mostly copied from node_loader.js, but with the following differences: + * + * - Takes a runtime profile name to load the region from, not just based on environment + * variables at process start. + * - We have needed to create a local copy of the SharedIniFile class because the + * implementation in 'aws-sdk' is private (and the default use of it in the + * SDK does not allow us to specify a profile at runtime). + * - AWS_DEFAULT_PROFILE and AWS_DEFAULT_REGION are also used as environment + * variables to be used to determine the region. + */ +function getCLICompatibleDefaultRegion(profile: string | undefined): string | undefined { + let region = process.env.AWS_REGION || process.env.AMAZON_REGION || + process.env.AWS_DEFAULT_REGION || process.env.AMAZON_DEFAULT_REGION; + + profile = profile || process.env.AWS_PROFILE || process.env.AWS_DEFAULT_PROFILE || 'default'; + + // Defaults inside constructor + const toCheck = [ + {filename: process.env.AWS_SHARED_CREDENTIALS_FILE }, + {isConfig: true, filename: process.env.AWS_CONFIG_FILE}, + ]; + + while (!region && toCheck.length > 0) { + const configFile = new SharedIniFile(toCheck.shift()); + const section = configFile.getProfile(profile); + region = section && section.region; + } + + return region; +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/util/sdk_ini_file.ts b/packages/aws-cdk/lib/api/util/sdk_ini_file.ts new file mode 100644 index 0000000000000..fc21bac2b0e49 --- /dev/null +++ b/packages/aws-cdk/lib/api/util/sdk_ini_file.ts @@ -0,0 +1,52 @@ +/** + * A reimplementation of JS AWS SDK's SharedIniFile class + * + * We need that class to parse the ~/.aws/config file to determine the correct + * region at runtime, but unfortunately it is private upstream. + */ + +import AWS = require('aws-sdk'); +import os = require('os'); +import path = require('path'); + +export interface SharedIniFileOptions { + isConfig?: boolean; + filename?: string; +} + +export class SharedIniFile { + private readonly isConfig: boolean; + private readonly filename: string; + private parsedContents?: {[key: string]: {[key: string]: string}}; + + constructor(options?: SharedIniFileOptions) { + options = options || {}; + this.isConfig = options.isConfig === true; + this.filename = options.filename || this.getDefaultFilepath(); + } + + public getProfile(profile: string) { + this.ensureFileLoaded(); + + const profileIndex = profile !== (AWS as any).util.defaultProfile && this.isConfig ? + 'profile ' + profile : profile; + + return this.parsedContents![profileIndex]; + } + + private getDefaultFilepath(): string { + return path.join( + os.homedir(), + '.aws', + this.isConfig ? 'config' : 'credentials' + ); + } + + private ensureFileLoaded() { + if (!this.parsedContents) { + this.parsedContents = (AWS as any).util.ini.parse( + (AWS as any).util.readFileSync(this.filename) + ); + } + } +} \ No newline at end of file From d2bbfe9aa3696586d9bfc1ae3f87209733bf879a Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 7 Aug 2018 12:46:09 +0200 Subject: [PATCH 2/3] Remove trailing debugging statements --- packages/aws-cdk/bin/cdk.ts | 2 -- packages/aws-cdk/lib/api/util/sdk.ts | 7 ------- 2 files changed, 9 deletions(-) diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index 9226e8b700db7..a299d37ff1270 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -109,8 +109,6 @@ async function initCommandLine() { debug('Command line arguments:', argv); const aws = new SDK(argv.profile); - // tslint:disable-next-line:no-console - console.log("Account: ", await aws.defaultAccount(), " region: ", aws.defaultRegion()); const availableContextProviders: contextplugins.ProviderMap = { 'availability-zones': new contextplugins.AZContextProviderPlugin(aws), diff --git a/packages/aws-cdk/lib/api/util/sdk.ts b/packages/aws-cdk/lib/api/util/sdk.ts index 2979443d3b17b..4901d81c028f9 100644 --- a/packages/aws-cdk/lib/api/util/sdk.ts +++ b/packages/aws-cdk/lib/api/util/sdk.ts @@ -28,13 +28,6 @@ export class SDK { // Find the package.json from the main toolkit const pkg = (require.main as any).require('../package.json'); this.userAgent = `${pkg.name}/${pkg.version}`; - - // tslint:disable-next-line:no-console - console.log(new Error().stack); - - // tslint:disable-next-line:no-console - console.log('Profile', profile); - this.defaultCredentialProvider = makeCLICompatibleCredentialProvider(profile); } From f7a4d7e2b433605c5a9ccd787db960ca608cd690 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 7 Aug 2018 15:07:27 +0200 Subject: [PATCH 3/3] Address review comments --- packages/aws-cdk/bin/cdk.ts | 2 +- packages/aws-cdk/lib/api/util/sdk.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index a299d37ff1270..e73e37f8ff994 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -46,7 +46,7 @@ async function parseCommandLineArguments() { .option('ignore-errors', { type: 'boolean', default: false, desc: 'Ignores synthesis errors, which will likely produce an invalid output' }) .option('json', { type: 'boolean', alias: 'j', desc: 'Use JSON output instead of YAML' }) .option('verbose', { type: 'boolean', alias: 'v', desc: 'Show debug logs' }) - .option('profile', { type: 'string', desc: 'Use the indicated AWS profile' }) + .option('profile', { type: 'string', desc: 'Use the indicated AWS profile as the default environment' }) // tslint:disable-next-line:max-line-length .option('version-reporting', { type: 'boolean', desc: 'Disable insersion of the CDKMetadata resource in synthesized templates', default: undefined }) .command([ 'list', 'ls' ], 'Lists all stacks in the app', yargs => yargs diff --git a/packages/aws-cdk/lib/api/util/sdk.ts b/packages/aws-cdk/lib/api/util/sdk.ts index 4901d81c028f9..cfcde34f1cd14 100644 --- a/packages/aws-cdk/lib/api/util/sdk.ts +++ b/packages/aws-cdk/lib/api/util/sdk.ts @@ -182,9 +182,6 @@ function makeCLICompatibleCredentialProvider(profile: string | undefined) { * variables to be used to determine the region. */ function getCLICompatibleDefaultRegion(profile: string | undefined): string | undefined { - let region = process.env.AWS_REGION || process.env.AMAZON_REGION || - process.env.AWS_DEFAULT_REGION || process.env.AMAZON_DEFAULT_REGION; - profile = profile || process.env.AWS_PROFILE || process.env.AWS_DEFAULT_PROFILE || 'default'; // Defaults inside constructor @@ -193,11 +190,14 @@ function getCLICompatibleDefaultRegion(profile: string | undefined): string | un {isConfig: true, filename: process.env.AWS_CONFIG_FILE}, ]; + let region = process.env.AWS_REGION || process.env.AMAZON_REGION || + process.env.AWS_DEFAULT_REGION || process.env.AMAZON_DEFAULT_REGION; + while (!region && toCheck.length > 0) { const configFile = new SharedIniFile(toCheck.shift()); const section = configFile.getProfile(profile); region = section && section.region; - } + } return region; } \ No newline at end of file