v4.3.0 proposal

.github/workflows/appsec.yml

@@ -130,3 +130,18 @@ jobs:
       - uses: ./.github/actions/node/latest
       - run: yarn test:appsec:plugins:ci
       - uses: codecov/codecov-action@v2
+
+  sourcing:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: cookie
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/node/setup
+      - run: yarn install
+      - uses: ./.github/actions/node/16
+      - run: yarn test:appsec:plugins:ci
+      - uses: ./.github/actions/node/18
+      - run: yarn test:appsec:plugins:ci
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:appsec:plugins:ci

.github/workflows/plugins.yml

@@ -330,6 +330,23 @@ jobs:
         uses: ./.github/actions/testagent/logs
       - uses: codecov/codecov-action@v2
 
+  fetch:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: fetch
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/testagent/start
+      - uses: ./.github/actions/node/setup
+      - run: yarn install
+      - uses: ./.github/actions/node/oldest
+      - run: yarn test:plugins:ci
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:plugins:ci
+      - if: always()
+        uses: ./.github/actions/testagent/logs
+      - uses: codecov/codecov-action@v2
+
   generic-pool:
     runs-on: ubuntu-latest
     env:
@@ -802,7 +819,7 @@ jobs:
           - 5500:5500
       testagent:
         image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:latest
-        env: 
+        env:
           LOG_LEVEL: DEBUG
           TRACE_LANGUAGE: javascript
           DISABLED_CHECKS: trace_content_length
@@ -816,7 +833,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/node/setup
-      - run: yarn install --ignore-engine
+      - run: yarn install --ignore-engines
       - run: yarn services
       - run: yarn test:plugins
       - uses: codecov/codecov-action@v2

.github/workflows/project.yml

@@ -5,7 +5,7 @@ on:
   push:
     branches: [master]
   schedule:
-    - cron: '0 4 * * *'
+    - cron: "0 4 * * *"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
@@ -22,20 +22,34 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: ./.github/actions/node/setup
-      - run: yarn install
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.version }}
       # Disable core dumps since some integration tests intentionally abort and core dump generation takes around 5-10s
+      - run: yarn install
       - run: sudo sysctl -w kernel.core_pattern='|/bin/false'
       - run: yarn test:integration
 
   integration-ci:
     strategy:
       matrix:
         version: [16, latest]
-        framework: [cucumber, cypress, playwright]
+        framework: [cucumber, playwright]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.version }}
+      - run: yarn install
+      - run: yarn test:integration:${{ matrix.framework }}
+
+  integration-cypress:
+    strategy:
+      matrix:
+        version: [16, latest]
+        # 6.7.0 is the minimum version we support
+        cypress-version: [6.7.0, latest]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -44,7 +58,9 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.version }}
-      - run: yarn test:integration:${{ matrix.framework }}
+      - run: yarn test:integration:cypress
+        env:
+          CYPRESS_VERSION: ${{ matrix.cypress-version }}
 
   lint:
     runs-on: ubuntu-latest

.github/workflows/release-dev.yml

@@ -23,3 +23,26 @@ jobs:
           echo "::set-output name=json::$content"
       - run: npm version --no-git-tag-version ${{ fromJson(steps.pkg.outputs.json).version }}-$(git rev-parse --short HEAD)+${{ github.run_id }}.${{ github.run_attempt }}
       - run: npm publish --tag dev
+      - run: |
+          git tag --force dev
+          git push origin :refs/tags/dev
+          git push origin --tags
+
+  injection-image-publish:
+    runs-on: ubuntu-latest
+    needs: ['publish']
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+      - name: Log in to the Container registry
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: npm pack for injection image
+        run: |
+          npm pack dd-trace@dev
+      - uses: ./.github/actions/injection
+        with:
+          init-image-version: dev

.gitlab-ci.yml

@@ -48,7 +48,7 @@ deploy_to_reliability_env:
 deploy_to_docker_registries:
   stage: deploy
   rules:
-    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+    - if: '$CI_COMMIT_TAG =~ /^v.*/ || $CI_COMMIT_TAG == "dev"'
       when: on_success
     - when: manual
       allow_failure: true
@@ -78,3 +78,5 @@ deploy_latest_to_docker_registries:
     IMG_SOURCES: ghcr.io/datadog/dd-trace-js/dd-lib-js-init:$CI_COMMIT_TAG
     IMG_DESTINATIONS: dd-lib-js-init:latest
     IMG_SIGNING: "false"
+    RETRY_COUNT: 5
+    RETRY_DELAY: 300

benchmark/sirun/appsec-iast/README.md

@@ -0,0 +1,9 @@
+This creates 150 HTTP requests from client to server.
+
+The variants are:
+- control tracer with non vulnerable endpoint without iast 
+- tracer with non vulnerable endpoint with iast active and default configuration
+- tracer with non vulnerable endpoint with iast active and sampling 100
+- control tracer with vulnerable endpoint without iast
+- tracer with vulnerable endpoint with iast active and default configuration
+- tracer with vulnerable endpoint with iast active and sampling 100

benchmark/sirun/appsec-iast/client.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { port, reqs } = require('./common')
+const http = require('http')
+
+let connectionsMade = 0
+function request (opts) {
+  http.get(opts, (res) => {
+    res.on('data', () => {})
+    res.on('end', () => {
+      if (++connectionsMade !== reqs) {
+        request(opts)
+      }
+    })
+  }).on('error', (e) => {
+    setTimeout(() => {
+      request(opts)
+    }, 10)
+  })
+}
+
+const path = '/?param=value'
+const opts = {
+  headers: {
+    accept: 'text/html'
+  },
+  port,
+  path
+}
+request(opts)

benchmark/sirun/appsec-iast/common.js

@@ -0,0 +1,6 @@
+'use strict'
+
+module.exports = {
+  port: 3331 + parseInt(process.env.CPU_AFFINITY || '0'),
+  reqs: 200
+}

benchmark/sirun/appsec-iast/meta.json

@@ -0,0 +1,66 @@
+{
+  "name": "appsec-iast",
+  "cachegrind": false,
+  "instructions": true,
+  "iterations": 30,
+  "variants": {
+    "no-vulnerability-control": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-without-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"",
+      "env": {
+        "DD_IAST_ENABLED": "0"
+      }
+    },
+    "no-vulnerability-iast-enabled-default-config": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-without-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"",
+      "baseline": "no-vulnerability-control",
+      "env": {
+        "DD_IAST_ENABLED": "1"
+      }
+    },
+    "no-vulnerability-iast-enabled-always-active": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-without-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"",
+      "baseline": "no-vulnerability-control",
+      "env": {
+        "DD_IAST_ENABLED": "1",
+        "DD_IAST_REQUEST_SAMPLING": "100",
+        "DD_IAST_MAX_CONCURRENT_REQUESTS": "1000",
+        "DD_IAST_MAX_CONTEXT_OPERATIONS": "100"
+      }
+    },
+    "with-vulnerability-control": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-with-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"",
+      "env": {
+        "DD_IAST_ENABLED": "0"
+      }
+    },
+    "with-vulnerability-iast-enabled-default-config": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-with-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"",
+      "baseline": "with-vulnerability-control",
+      "env": {
+        "DD_IAST_ENABLED": "1"
+      }
+    },
+    "with-vulnerability-iast-enabled-always-active": {
+      "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
+      "run": "node --require ../../../init.js server-with-vulnerability.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"",
+      "baseline": "with-vulnerability-control",
+      "env": {
+        "DD_IAST_ENABLED": "1",
+        "DD_IAST_REQUEST_SAMPLING": "100",
+        "DD_IAST_MAX_CONCURRENT_REQUESTS": "1000",
+        "DD_IAST_MAX_CONTEXT_OPERATIONS": "100"
+      }
+    }
+  }
+}

benchmark/sirun/appsec-iast/server-with-vulnerability.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const { port, reqs } = require('./common')
+const express = require('../../../versions/express').get()
+const cookieParser = require('../../../versions/cookie-parser').get()
+const childProcess = require('child_process')
+
+const app = express()
+app.use(cookieParser())
+
+let connectionsMade = 0
+
+function noop () {}
+
+app.get('/', (req, res) => {
+  childProcess.exec('echo #' + req.query.param, noop)
+  res.writeHead(200)
+  res.end('Hello, World!')
+
+  if (++connectionsMade === reqs) {
+    server.close()
+  }
+})
+
+const server = app.listen(port)

benchmark/sirun/appsec-iast/server-without-vulnerability.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const { port, reqs } = require('./common')
+const express = require('../../../versions/express').get()
+const cookieParser = require('../../../versions/cookie-parser').get()
+
+const app = express()
+app.use(cookieParser())
+
+let connectionsMade = 0
+
+app.get('/', (req, res) => {
+  res.writeHead(200)
+  res.end('Hello, World!')
+
+  if (++connectionsMade === reqs) {
+    server.close()
+  }
+})
+
+const server = app.listen(port)

benchmark/sirun/runall.sh

@@ -19,7 +19,7 @@ nvm use 18
   cd ../../ &&
   npm install --global yarn \
     && yarn install --ignore-engines \
-    && PLUGINS="bluebird|q|graphql" yarn services
+    && PLUGINS="bluebird|q|graphql|express" yarn services
 )
 
 # run each test in parallel for a given version of Node.js

docker-compose.yml

@@ -12,7 +12,10 @@ services:
     ports:
       - "127.0.0.1:5432:5432"
   mssql:
-    image: mcr.microsoft.com/mssql/server:2017-latest-ubuntu
+  # A working MSSQL server is not available on ARM.
+  # This image provides _most_ of sqlserver functionalities, but
+  # does not support stored procedures (corresponding tests will fail)
+    image: mcr.microsoft.com/mssql/azure-sql-edge
     environment:
       - "ACCEPT_EULA=Y"
       - "SA_PASSWORD=DD_HUNTER2"

docs/API.md

@@ -40,6 +40,7 @@ tracer.use('pg', {
 <h5 id="generic-pool"></h5>
 <h5 id="google-cloud-pubsub"></h5>
 <h5 id="fastify"></h5>
+<h5 id="fetch"></h5>
 <h5 id="graphql"></h5>
 <h5 id="graphql-tags"></h5>
 <h5 id="graphql-config"></h5>
@@ -110,6 +111,7 @@ tracer.use('pg', {
 * [elasticsearch](./interfaces/plugins.elasticsearch.html)
 * [express](./interfaces/plugins.express.html)
 * [fastify](./interfaces/plugins.fastify.html)
+* [fetch](./interfaces/plugins.fetch.html)
 * [generic-pool](./interfaces/plugins.generic_pool.html)
 * [google-cloud-pubsub](./interfaces/plugins.google_cloud_pubsub.html)
 * [graphql](./interfaces/plugins.graphql.html)

docs/test.ts

@@ -248,6 +248,8 @@ tracer.use('express');
 tracer.use('express', httpServerOptions);
 tracer.use('fastify');
 tracer.use('fastify', httpServerOptions);
+tracer.use('fetch');
+tracer.use('fetch', httpClientOptions);
 tracer.use('generic-pool');
 tracer.use('google-cloud-pubsub');
 tracer.use('graphql');

index.d.ts

@@ -749,6 +749,7 @@ interface Plugins {
   "elasticsearch": plugins.elasticsearch;
   "express": plugins.express;
   "fastify": plugins.fastify;
+  "fetch": plugins.fetch;
   "generic-pool": plugins.generic_pool;
   "google-cloud-pubsub": plugins.google_cloud_pubsub;
   "graphql": plugins.graphql;
@@ -1092,6 +1093,12 @@ declare namespace plugins {
    */
   interface fastify extends HttpServer {}
 
+  /**
+   * This plugin automatically instruments the
+   * [fetch](https://nodejs.org/api/globals.html#fetch) global.
+   */
+  interface fetch extends HttpClient {}
+
   /**
    * This plugin patches the [generic-pool](https://github.com/coopernurse/node-pool)
    * module to bind the callbacks the the caller context.