Introduce middleware directory

package.json

@@ -50,7 +50,7 @@
     "greenkeeper[bot] <greenkeeper[bot]@users.noreply.github.com>"
   ],
   "scripts": {
-    "cover": "nyc --reporter=lcov mocha build/test/*.js && nyc report",
+    "cover": "nyc --reporter=lcov mocha build/test/*.js build/test/**/*.js build/test/**/**/*.js && nyc report",
     "test-no-cover": "mocha build/test/*.js",
     "test": "npm run cover",
     "docs": "jsdoc -c .jsdoc.js",
@@ -72,6 +72,7 @@
     "@google-cloud/paginator": "^0.1.0",
     "@google-cloud/projectify": "^0.3.0",
     "@google-cloud/promisify": "^0.3.0",
+    "@opencensus/propagation-stackdriver": "0.0.4",
     "arrify": "^1.0.1",
     "eventid": "^0.1.2",
     "extend": "^3.0.2",
@@ -81,6 +82,7 @@
     "google-proto-files": "^0.17.0",
     "is": "^3.2.1",
     "lodash.merge": "^4.6.1",
+    "on-finished": "^2.3.0",
     "protobufjs": "^6.8.8",
     "pumpify": "^1.5.1",
     "snakecase-keys": "^2.0.0",
@@ -93,10 +95,12 @@
     "@google-cloud/pubsub": "^0.20.1",
     "@google-cloud/storage": "^2.0.2",
     "@types/arrify": "^1.0.4",
+    "@types/express": "^4.16.0",
     "@types/extend": "^3.0.0",
     "@types/is": "0.0.20",
     "@types/mocha": "^5.2.5",
     "@types/nock": "^9.3.0",
+    "@types/on-finished": "^2.3.1",
     "@types/pumpify": "^1.4.1",
     "@types/through2": "^2.0.34",
     "@types/uuid": "^3.4.4",

src/http-request.ts

@@ -0,0 +1,33 @@
+/*!
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface StackdriverHttpRequest {
+  requestMethod?: string;
+  requestUrl?: string;
+  requestSize?: number;
+  status?: number;
+  responseSize?: number;
+  userAgent?: string;
+  remoteIp?: string;
+  serverIp?: string;
+  referer?: string;
+  latency?: {seconds: number; nanos: number;};
+  cacheLookup?: boolean;
+  cacheHit?: boolean;
+  cacheValidatedWithOriginServer?: boolean;
+  cacheFillBytes?: number;
+  protocol?: string;
+}

src/middleware/context.ts

@@ -0,0 +1,45 @@
+/*!
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as context from '@opencensus/propagation-stackdriver';
+import * as http from 'http';
+
+export type HeaderWrapper = context.HeaderGetter&context.HeaderSetter;
+
+export function makeHeaderWrapper(req: http.IncomingMessage): HeaderWrapper {
+  const wrapper = {
+    setHeader(name: string, value: string) {
+      req.headers[name] = value;
+    },
+    getHeader(name: string) {
+      return req.headers[name];
+    }
+  };
+  return wrapper;
+}
+
+export function getOrInjectContext(headerWrapper: HeaderWrapper):
+    context.SpanContext {
+  let spanContext = context.extract(headerWrapper);
+  if (spanContext) {
+    return spanContext;
+  }
+
+  // We were the first actor to detect lack of context. Establish context.
+  spanContext = context.generate();
+  context.inject(headerWrapper, spanContext);
+  return spanContext;
+}

src/middleware/express/make-http-request.ts

@@ -0,0 +1,37 @@
+/*!
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Types-only import.
+import {Request, Response} from 'express';
+
+import {StackdriverHttpRequest} from '../../http-request';
+
+export function makeHttpRequestData(
+    req: Request, res: Response,
+    latencyMilliseconds: number): StackdriverHttpRequest {
+  return {
+    status: res.statusCode,
+    requestUrl: req.url,
+    requestMethod: req.method,
+    userAgent: req.headers['user-agent'],
+    responseSize:
+        (res.getHeader && Number(res.getHeader('Content-Length'))) || 0,
+    latency: {
+      seconds: Math.floor(latencyMilliseconds / 1e3),
+      nanos: Math.floor((latencyMilliseconds % 1e3) * 1e6)
+    }
+  };
+}

src/middleware/express/make-middleware.ts

@@ -0,0 +1,55 @@
+/*!
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import onFinished = require('on-finished');
+import {getOrInjectContext, makeHeaderWrapper} from '../context';
+// Types-only import.
+import {Request, Response, NextFunction} from 'express';
+import {makeHttpRequestData} from './make-http-request';
+
+const CHILD_LOG_NAME_SUFFIX = 'applog';
+
+export interface AnnotatedRequestType<LoggerType> extends Request {
+  log: LoggerType;
+}
+
+export function makeMiddleware<LoggerType>(
+    projectId: string,
+    emitRequestLog: (httpRequest: HttpRequest, trace: string) => void,
+    makeChildLogger: (childSuffix: string, trace: string) => LoggerType) {
+  return (req: Request, res: Response, next: NextFunction) => {
+    // TODO(ofrobots): use high-resolution timer.
+    const requestStartMs = Date.now();
+
+    const wrapper = makeHeaderWrapper(req);
+
+    const spanContext = getOrInjectContext(wrapper);
+    const trace = `projects/${projectId}/traces/${spanContext.traceId}`;
+
+    // Install a child logger on the request object.
+    (req as AnnotatedRequestType<LoggerType>).log =
+        makeChildLogger(CHILD_LOG_NAME_SUFFIX, trace);
+
+    // Emit a 'Request Log' on the parent logger.
+    onFinished(res, () => {
+      const latencyMs = Date.now() - requestStartMs;
+      const httpRequest = makeHttpRequestData(req, res, latencyMs);
+      emitRequestLog(httpRequest, trace);
+    });
+
+    next();
+  };
+}

test/middleware/express/test-make-http-request.ts

@@ -0,0 +1,40 @@
+/*!
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as assert from 'assert';
+// Types-only import.
+import {Request, Response} from 'express';
+import {makeHttpRequestData} from '../../../src/middleware/express/make-http-request';
+
+describe('middleware/express/make-http-request', () => {
+  it('should convert latency to proto Duration', () => {
+    const fakeRequest = {headers: {}};
+    const fakeResponse = {};
+
+    const h1 = makeHttpRequestData(
+        fakeRequest as Request, fakeResponse as Response, 1003);
+    assert.deepStrictEqual(h1.latency, {seconds: 1, nanos: 3e6});
+
+    const h2 = makeHttpRequestData(
+        fakeRequest as Request, fakeResponse as Response, 9003.1);
+    assert.deepStrictEqual(h2.latency, {seconds: 9, nanos: 3.1e6});
+
+    // Make sure we nanos is uint32.
+    const h3 = makeHttpRequestData(
+        fakeRequest as Request, fakeResponse as Response, 1.0000000001);
+    assert.deepStrictEqual(h3.latency, {seconds: 0, nanos: 1e6});
+  });
+});

test/middleware/express/test-make-middleware.ts

@@ -0,0 +1,126 @@
+/*!
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as assert from 'assert';
+import * as EventEmitter from 'events';
+import * as extend from 'extend';
+import * as proxyquire from 'proxyquire';
+
+const FAKE_PROJECT_ID = 'project-🦄';
+
+function makeFakeRequest() {
+  return {headers: {'content-type': 'application/🍰'}};
+}
+
+function makeFakeResponse() {
+  const ee = new EventEmitter();
+  // tslint:disable-next-line:no-any
+  (ee as any).getHeader = (name: string) => {};
+  return ee;
+}
+
+let getOrInjectContextValue;
+const FAKE_CONTEXT = {
+  getOrInjectContext: () => {
+    return getOrInjectContextValue;
+  }
+};
+
+
+describe('middleware/express/make-middleware', () => {
+  describe('makeMiddleware', () => {
+    const {makeMiddleware} = proxyquire(
+        '../../../src/middleware/express/make-middleware',
+        {'../context': FAKE_CONTEXT});
+
+    it('should return a function accepting 3 arguments', () => {
+      const middleware = makeMiddleware(FAKE_PROJECT_ID, () => {}, () => {});
+      assert.ok(typeof middleware === 'function');
+      assert.ok(middleware.length === 3);
+    });
+
+    describe('middleware', () => {
+      const FAKE_SPAN_CONTEXT = {traceId: 'traceId-🥑'};
+
+      beforeEach(() => {
+        getOrInjectContextValue = undefined;
+      });
+
+      it('should call the next middleware synchronously', () => {
+        getOrInjectContextValue = FAKE_SPAN_CONTEXT;
+        const fakeRequest = makeFakeRequest();
+        const fakeResponse = makeFakeResponse();
+        let called = false;
+
+        const middleware = makeMiddleware(FAKE_PROJECT_ID, () => {}, () => {});
+
+        middleware(fakeRequest, fakeResponse, () => {
+          called = true;
+        });
+        assert.ok(called);
+      });
+
+      it('should call makeChildLogger with correct trace context', () => {
+        const FAKE_CHILD_LOGGER = {log: '🍌'};
+        getOrInjectContextValue = FAKE_SPAN_CONTEXT;
+        const fakeRequest = makeFakeRequest();
+        const fakeResponse = makeFakeResponse();
+
+        function makeChild(childSuffix, trace) {
+          assert.strictEqual(
+              trace,
+              `projects/${FAKE_PROJECT_ID}/traces/${
+                  FAKE_SPAN_CONTEXT.traceId}`);
+          assert.strictEqual(typeof childSuffix, 'string');
+          return FAKE_CHILD_LOGGER;
+        }
+
+        const middleware = makeMiddleware(FAKE_PROJECT_ID, () => {}, makeChild);
+        middleware(fakeRequest, fakeResponse, () => {});
+
+        // Should annotate the request with the child logger.
+        // tslint:disable-next-line:no-any
+        assert.strictEqual((fakeRequest as any).log, FAKE_CHILD_LOGGER);
+      });
+
+      it('should emit a request log when response is finished', (done) => {
+        getOrInjectContextValue = FAKE_SPAN_CONTEXT;
+        const fakeRequest = makeFakeRequest();
+        const fakeResponse = makeFakeResponse();
+        let emitRequestLogCalled = false;
+
+        function emitRequestLog(httpRequest, trace) {
+          assert.strictEqual(
+              trace,
+              `projects/${FAKE_PROJECT_ID}/traces/${
+                  FAKE_SPAN_CONTEXT.traceId}`);
+          // TODO: check httpRequest properties.
+          emitRequestLogCalled = true;
+        }
+
+        const middleware =
+            makeMiddleware(FAKE_PROJECT_ID, emitRequestLog, () => {});
+        middleware(fakeRequest, fakeResponse, () => {});
+
+        setTimeout(() => {
+          fakeResponse.emit('finished');
+          assert.strictEqual(emitRequestLogCalled, true);
+          done();
+        }, 10);
+      });
+    });
+  });
+});