From 3118aed90dddc9dfe8c6c983db94b7e58ccff467 Mon Sep 17 00:00:00 2001 From: Kelvin Lo Date: Thu, 3 Dec 2020 13:33:14 -0800 Subject: [PATCH] Feat: Added Amazon EKS Resource Detector (#1669) --- .../src/detectors/AwsEksDetector.ts | 238 +++++++++++++ .../src/detectors/index.ts | 1 + .../test/detectors/AwsEksDetector.test.ts | 324 ++++++++++++++++++ 3 files changed, 563 insertions(+) create mode 100644 packages/opentelemetry-resource-detector-aws/src/detectors/AwsEksDetector.ts create mode 100644 packages/opentelemetry-resource-detector-aws/test/detectors/AwsEksDetector.test.ts diff --git a/packages/opentelemetry-resource-detector-aws/src/detectors/AwsEksDetector.ts b/packages/opentelemetry-resource-detector-aws/src/detectors/AwsEksDetector.ts new file mode 100644 index 0000000000..5060b7d524 --- /dev/null +++ b/packages/opentelemetry-resource-detector-aws/src/detectors/AwsEksDetector.ts @@ -0,0 +1,238 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Detector, + Resource, + CONTAINER_RESOURCE, + K8S_RESOURCE, + ResourceDetectionConfigWithLogger, +} from '@opentelemetry/resources'; +import * as https from 'https'; +import * as fs from 'fs'; +import * as util from 'util'; + +/** + * The AwsEksDetector can be used to detect if a process is running in AWS Elastic + * Kubernetes and return a {@link Resource} populated with data about the Kubernetes + * plugins of AWS X-Ray. Returns an empty Resource if detection fails. + * + * See https://docs.amazonaws.cn/en_us/xray/latest/devguide/xray-guide.pdf + * for more details about detecting information for Elastic Kubernetes plugins + */ + +export class AwsEksDetector implements Detector { + readonly K8S_SVC_URL = 'kubernetes.default.svc'; + readonly K8S_TOKEN_PATH = + '/var/run/secrets/kubernetes.io/serviceaccount/token'; + readonly K8S_CERT_PATH = + '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'; + readonly AUTH_CONFIGMAP_PATH = + '/api/v1/namespaces/kube-system/configmaps/aws-auth'; + readonly CW_CONFIGMAP_PATH = + '/api/v1/namespaces/amazon-cloudwatch/configmaps/cluster-info'; + readonly CONTAINER_ID_LENGTH = 64; + readonly DEFAULT_CGROUP_PATH = '/proc/self/cgroup'; + readonly TIMEOUT_MS = 2000; + readonly UTF8_UNICODE = 'utf8'; + + private static readFileAsync = util.promisify(fs.readFile); + private static fileAccessAsync = util.promisify(fs.access); + + /** + * The AwsEksDetector can be used to detect if a process is running on Amazon + * Elastic Kubernetes and returns a promise containing a {@link Resource} + * populated with instance metadata. Returns a promise containing an + * empty {@link Resource} if the connection to kubernetes process + * or aws config maps fails + * @param config The resource detection config with a required logger + */ + async detect(config: ResourceDetectionConfigWithLogger): Promise { + try { + await AwsEksDetector.fileAccessAsync(this.K8S_TOKEN_PATH); + const k8scert = await AwsEksDetector.readFileAsync(this.K8S_CERT_PATH); + + if (!this._isEks(config, k8scert)) { + return Resource.empty(); + } + + const containerId = await this._getContainerId(config); + const clusterName = await this._getClusterName(config, k8scert); + + return !containerId && !clusterName + ? Resource.empty() + : new Resource({ + [K8S_RESOURCE.CLUSTER_NAME]: clusterName || '', + [CONTAINER_RESOURCE.ID]: containerId || '', + }); + } catch (e) { + config.logger.warn('Process is not running on K8S', e); + return Resource.empty(); + } + } + + /** + * Attempts to make a connection to AWS Config map which will + * determine whether the process is running on an EKS + * process if the config map is empty or not + * @param config The resource detection config with a required logger + */ + private async _isEks( + config: ResourceDetectionConfigWithLogger, + cert: Buffer + ): Promise { + const options = { + ca: cert, + headers: { + Authorization: await this._getK8sCredHeader(config), + }, + hostname: this.K8S_SVC_URL, + method: 'GET', + path: this.AUTH_CONFIGMAP_PATH, + timeout: this.TIMEOUT_MS, + }; + return !!(await this._fetchString(options)); + } + + /** + * Attempts to make a connection to Amazon Cloudwatch + * Config Maps to grab cluster name + * @param config The resource detection config with a required logger + */ + private async _getClusterName( + config: ResourceDetectionConfigWithLogger, + cert: Buffer + ): Promise { + const options = { + ca: cert, + headers: { + Authorization: await this._getK8sCredHeader(config), + }, + host: this.K8S_SVC_URL, + method: 'GET', + path: this.CW_CONFIGMAP_PATH, + timeout: this.TIMEOUT_MS, + }; + const response = await this._fetchString(options); + try { + return JSON.parse(response).data['cluster.name']; + } catch (e) { + config.logger.warn('Cannot get cluster name on EKS', e); + } + return ''; + } + /** + * Reads the Kubernetes token path and returns kubernetes + * credential header + * @param config The resource detection config with a required logger + */ + private async _getK8sCredHeader( + config: ResourceDetectionConfigWithLogger + ): Promise { + try { + const content = await AwsEksDetector.readFileAsync( + this.K8S_TOKEN_PATH, + this.UTF8_UNICODE + ); + return 'Bearer ' + content; + } catch (e) { + config.logger.warn('Unable to read Kubernetes client token.', e); + } + return ''; + } + + /** + * Read container ID from cgroup file generated from docker which lists the full + * untruncated docker container ID at the end of each line. + * + * The predefined structure of calling /proc/self/cgroup when in a docker container has the structure: + * + * #:xxxxxx:/ + * + * or + * + * #:xxxxxx:/docker/64characterID + * + * This function takes advantage of that fact by just reading the 64-character ID from the end of the + * first line. In EKS, even if we fail to find target file or target file does + * not contain container ID we do not throw an error but throw warning message + * and then return null string + */ + private async _getContainerId( + config: ResourceDetectionConfigWithLogger + ): Promise { + try { + const rawData = await AwsEksDetector.readFileAsync( + this.DEFAULT_CGROUP_PATH, + this.UTF8_UNICODE + ); + const splitData = rawData.trim().split('\n'); + for (const str of splitData) { + if (str.length > this.CONTAINER_ID_LENGTH) { + return str.substring(str.length - this.CONTAINER_ID_LENGTH); + } + } + } catch (e) { + config.logger.warn( + `AwsEksDetector failed to read container ID: ${e.message}` + ); + } + return undefined; + } + + /** + * Establishes an HTTP connection to AWS instance document url. + * If the application is running on an EKS instance, we should be able + * to get back a valid JSON document. Parses that document and stores + * the identity properties in a local map. + */ + private async _fetchString(options: https.RequestOptions): Promise { + return await new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + req.abort(); + reject(new Error('EKS metadata api request timed out.')); + }, 2000); + + const req = https.request(options, res => { + clearTimeout(timeoutId); + const { statusCode } = res; + res.setEncoding(this.UTF8_UNICODE); + let rawData = ''; + res.on('data', chunk => (rawData += chunk)); + res.on('end', () => { + if (statusCode && statusCode >= 200 && statusCode < 300) { + try { + resolve(rawData); + } catch (e) { + reject(e); + } + } else { + reject( + new Error('Failed to load page, status code: ' + statusCode) + ); + } + }); + }); + req.on('error', err => { + clearTimeout(timeoutId); + reject(err); + }); + req.end(); + }); + } +} + +export const awsEksDetector = new AwsEksDetector(); diff --git a/packages/opentelemetry-resource-detector-aws/src/detectors/index.ts b/packages/opentelemetry-resource-detector-aws/src/detectors/index.ts index 01986175c3..e873f14e89 100644 --- a/packages/opentelemetry-resource-detector-aws/src/detectors/index.ts +++ b/packages/opentelemetry-resource-detector-aws/src/detectors/index.ts @@ -17,3 +17,4 @@ export * from './AwsEc2Detector'; export * from './AwsBeanstalkDetector'; export * from './AwsEcsDetector'; +export * from './AwsEksDetector'; diff --git a/packages/opentelemetry-resource-detector-aws/test/detectors/AwsEksDetector.test.ts b/packages/opentelemetry-resource-detector-aws/test/detectors/AwsEksDetector.test.ts new file mode 100644 index 0000000000..ce350e2084 --- /dev/null +++ b/packages/opentelemetry-resource-detector-aws/test/detectors/AwsEksDetector.test.ts @@ -0,0 +1,324 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as nock from 'nock'; +import * as sinon from 'sinon'; +import * as assert from 'assert'; +import { Resource } from '@opentelemetry/resources'; +import { awsEksDetector, AwsEksDetector } from '../../src'; +import { + assertK8sResource, + assertContainerResource, + assertEmptyResource, +} from '@opentelemetry/resources/test/util/resource-assertions'; +import { NoopLogger } from '@opentelemetry/core'; + +const K8S_SVC_URL = awsEksDetector.K8S_SVC_URL; +const AUTH_CONFIGMAP_PATH = awsEksDetector.AUTH_CONFIGMAP_PATH; +const CW_CONFIGMAP_PATH = awsEksDetector.CW_CONFIGMAP_PATH; + +describe('awsEksDetector', () => { + const errorMsg = { + fileNotFoundError: new Error('cannot find cgroup file'), + }; + + const correctCgroupData = + 'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm'; + const mockedClusterResponse = '{"data":{"cluster.name":"my-cluster"}}'; + const mockedAwsAuth = 'my-auth'; + const k8s_token = 'Bearer 31ada4fd-adec-460c-809a-9e56ceb75269'; + let sandbox: sinon.SinonSandbox; + let readStub, fileStub, getCredStub; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + nock.disableNetConnect(); + nock.cleanAll(); + }); + + afterEach(() => { + sandbox.restore(); + nock.enableNetConnect(); + }); + + describe('on successful request', () => { + it('should return an aws_eks_instance_resource', async () => { + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .resolves(); + readStub = sandbox + .stub(AwsEksDetector, 'readFileAsync' as any) + .resolves(correctCgroupData); + getCredStub = sandbox + .stub(awsEksDetector, '_getK8sCredHeader' as any) + .resolves(k8s_token); + const scope = nock('https://' + K8S_SVC_URL) + .persist() + .get(AUTH_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => mockedAwsAuth) + .get(CW_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => mockedClusterResponse); + + const resource: Resource = await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + + scope.done(); + + sandbox.assert.calledOnce(fileStub); + sandbox.assert.calledTwice(readStub); + sandbox.assert.calledTwice(getCredStub); + + assert.ok(resource); + assertK8sResource(resource, { + clusterName: 'my-cluster', + }); + assertContainerResource(resource, { + id: 'bcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm', + }); + }); + + it('should return a resource with clusterName attribute without cgroup file', async () => { + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .resolves(); + readStub = sandbox + .stub(AwsEksDetector, 'readFileAsync' as any) + .onSecondCall() + .rejects(errorMsg.fileNotFoundError); + getCredStub = sandbox + .stub(awsEksDetector, '_getK8sCredHeader' as any) + .resolves(k8s_token); + const scope = nock('https://' + K8S_SVC_URL) + .persist() + .get(AUTH_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => mockedAwsAuth) + .get(CW_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => mockedClusterResponse); + + const resource: Resource = await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + + scope.done(); + + assert.ok(resource); + assertK8sResource(resource, { + clusterName: 'my-cluster', + }); + }); + + it('should return a resource with container ID attribute without a clusterName', async () => { + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .resolves(); + readStub = sandbox + .stub(AwsEksDetector, 'readFileAsync' as any) + .resolves(correctCgroupData); + getCredStub = sandbox + .stub(awsEksDetector, '_getK8sCredHeader' as any) + .resolves(k8s_token); + const scope = nock('https://' + K8S_SVC_URL) + .persist() + .get(AUTH_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => mockedAwsAuth) + .get(CW_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => ''); + + const resource: Resource = await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + + scope.done(); + + assert.ok(resource); + assertContainerResource(resource, { + id: 'bcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm', + }); + }); + + it('should return a resource with clusterName attribute when cgroup file does not contain valid Container ID', async () => { + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .resolves(); + readStub = sandbox + .stub(AwsEksDetector, 'readFileAsync' as any) + .onSecondCall() + .resolves(''); + getCredStub = sandbox + .stub(awsEksDetector, '_getK8sCredHeader' as any) + .resolves(k8s_token); + const scope = nock('https://' + K8S_SVC_URL) + .persist() + .get(AUTH_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => mockedAwsAuth) + .get(CW_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => mockedClusterResponse); + + const resource: Resource = await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + + scope.done(); + + assert.ok(resource); + assert.ok(resource); + assertK8sResource(resource, { + clusterName: 'my-cluster', + }); + }); + + it('should return an empty resource when not running on Eks', async () => { + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .resolves(''); + readStub = sandbox + .stub(AwsEksDetector, 'readFileAsync' as any) + .resolves(correctCgroupData); + getCredStub = sandbox + .stub(awsEksDetector, '_getK8sCredHeader' as any) + .resolves(k8s_token); + const scope = nock('https://' + K8S_SVC_URL) + .persist() + .get(AUTH_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => ''); + + const resource: Resource = await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + + scope.done(); + + assert.ok(resource); + assertEmptyResource(resource); + }); + + it('should return an empty resource when k8s token file does not exist', async () => { + const errorMsg = { + fileNotFoundError: new Error('cannot file k8s token file'), + }; + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .rejects(errorMsg.fileNotFoundError); + + const resource: Resource = await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + + assert.ok(resource); + assertEmptyResource(resource); + }); + + it('should return an empty resource when containerId and clusterName are invalid', async () => { + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .resolves(''); + readStub = sandbox + .stub(AwsEksDetector, 'readFileAsync' as any) + .onSecondCall() + .rejects(errorMsg.fileNotFoundError); + + getCredStub = sandbox + .stub(awsEksDetector, '_getK8sCredHeader' as any) + .resolves(k8s_token); + const scope = nock('https://' + K8S_SVC_URL) + .persist() + .get(AUTH_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => mockedAwsAuth) + .get(CW_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(200, () => ''); + + const resource: Resource = await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + + scope.isDone(); + + assert.ok(resource); + assertEmptyResource(resource); + }); + }); + + describe('on unsuccesful request', () => { + it('should throw when receiving error response code', async () => { + const expectedError = new Error('EKS metadata api request timed out.'); + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .resolves(); + readStub = sandbox + .stub(AwsEksDetector, 'readFileAsync' as any) + .resolves(correctCgroupData); + getCredStub = sandbox + .stub(awsEksDetector, '_getK8sCredHeader' as any) + .resolves(k8s_token); + const scope = nock('https://' + K8S_SVC_URL) + .persist() + .get(AUTH_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .delayConnection(2500) + .reply(200, () => mockedAwsAuth); + + try { + await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + } catch (err) { + assert.deepStrictEqual(err, expectedError); + } + + scope.done(); + }); + + it('should return an empty resource when timed out', async () => { + const expectedError = new Error('Failed to load page, status code: 404'); + fileStub = sandbox + .stub(AwsEksDetector, 'fileAccessAsync' as any) + .resolves(); + readStub = sandbox + .stub(AwsEksDetector, 'readFileAsync' as any) + .resolves(correctCgroupData); + getCredStub = sandbox + .stub(awsEksDetector, '_getK8sCredHeader' as any) + .resolves(k8s_token); + const scope = nock('https://' + K8S_SVC_URL) + .persist() + .get(AUTH_CONFIGMAP_PATH) + .matchHeader('Authorization', k8s_token) + .reply(404, () => new Error()); + + try { + await awsEksDetector.detect({ + logger: new NoopLogger(), + }); + } catch (err) { + assert.deepStrictEqual(err, expectedError); + } + + scope.done(); + }); + }); +});