feat(redis): added endpoint filtering (#1448)

ref INSTA-16323

Author: aryamohanan

packages/collector/src/announceCycle/unannounced.js

@@ -43,6 +43,7 @@ const maxRetryDelay = 60 * 1000; // one minute
  * @typedef {Object} TracingConfig
  * @property {Array.<string>} [extra-http-headers]
  * @property {KafkaTracingConfig} [kafka]
+ * @property {Object.<string, (string | string[])>} [ignore-endpoints]
  * @property {boolean} [span-batching-enabled]
  */
 
@@ -126,6 +127,7 @@ function applyAgentConfiguration(agentResponse) {
   applyExtraHttpHeaderConfiguration(agentResponse);
   applyKafkaTracingConfiguration(agentResponse);
   applySpanBatchingConfiguration(agentResponse);
+  applyIgnoreEndpointsConfiguration(agentResponse);
 }
 
 /**
@@ -220,3 +222,34 @@ function applySpanBatchingConfiguration(agentResponse) {
     agentOpts.config.tracing.spanBatchingEnabled = true;
   }
 }
+
+/**
+ * - The agent configuration currently uses a pipe ('|') as a separator for endpoints.
+ * - This function supports both ('|') and comma (',') to ensure future compatibility.
+ * - Additionally, it supports the `string[]` format for backward compatibility,
+ *   as this was the previously used standard. The final design decision is not yet completed.
+ *   https://github.ibm.com/instana/requests-for-discussion/pull/84
+ *
+ * @param {AgentAnnounceResponse} agentResponse
+ */
+function applyIgnoreEndpointsConfiguration(agentResponse) {
+  if (agentResponse?.tracing?.['ignore-endpoints']) {
+    const endpointTracingConfigFromAgent = agentResponse.tracing['ignore-endpoints'];
+
+    const endpointTracingConfig = Object.fromEntries(
+      Object.entries(endpointTracingConfigFromAgent).map(([service, endpoints]) => {
+        let normalizedEndpoints = null;
+        if (typeof endpoints === 'string') {
+          normalizedEndpoints = endpoints.split(/[|,]/).map(endpoint => endpoint?.trim()?.toLowerCase());
+        } else if (Array.isArray(endpoints)) {
+          normalizedEndpoints = endpoints.map(endpoint => endpoint?.toLowerCase());
+        }
+
+        return [service.toLowerCase(), normalizedEndpoints];
+      })
+    );
+
+    ensureNestedObjectExists(agentOpts.config, ['tracing', 'ignoreEndpoints']);
+    agentOpts.config.tracing.ignoreEndpoints = endpointTracingConfig;
+  }
+}

packages/collector/test/announceCycle/unannounced_test.js

@@ -213,6 +213,97 @@ describe('unannounced state', () => {
         }
       });
     });
+    it('should  apply the configuration to ignore a single endpoint for a package', done => {
+      prepareAnnounceResponse({
+        tracing: {
+          'ignore-endpoints': {
+            redis: 'get'
+          }
+        }
+      });
+      unannouncedState.enter({
+        transitionTo: () => {
+          expect(agentOptsStub.config).to.deep.equal({
+            tracing: {
+              ignoreEndpoints: {
+                redis: ['get']
+              }
+            }
+          });
+          done();
+        }
+      });
+    });
+
+    it('should apply the configuration to ignore multiple endpoints for a package', done => {
+      prepareAnnounceResponse({
+        tracing: {
+          'ignore-endpoints': {
+            redis: 'SET|GET'
+          }
+        }
+      });
+      unannouncedState.enter({
+        transitionTo: () => {
+          expect(agentOptsStub.config).to.deep.equal({
+            tracing: {
+              ignoreEndpoints: {
+                redis: ['set', 'get']
+              }
+            }
+          });
+          done();
+        }
+      });
+    });
+
+    it('should apply tracing configuration to ignore specified endpoints across different packages', done => {
+      prepareAnnounceResponse({
+        tracing: {
+          'ignore-endpoints': {
+            REDIS: 'get|set',
+            dynamodb: 'query'
+          }
+        }
+      });
+      unannouncedState.enter({
+        transitionTo: () => {
+          expect(agentOptsStub.config).to.deep.equal({
+            tracing: {
+              ignoreEndpoints: {
+                redis: ['get', 'set'],
+                dynamodb: ['query']
+              }
+            }
+          });
+          done();
+        }
+      });
+    });
+
+    it('should apply tracing configuration to ignore endpoints when specified using array format', done => {
+      prepareAnnounceResponse({
+        tracing: {
+          'ignore-endpoints': {
+            REDIS: ['get', 'type'],
+            dynamodb: 'query'
+          }
+        }
+      });
+      unannouncedState.enter({
+        transitionTo: () => {
+          expect(agentOptsStub.config).to.deep.equal({
+            tracing: {
+              ignoreEndpoints: {
+                redis: ['get', 'type'],
+                dynamodb: ['query']
+              }
+            }
+          });
+          done();
+        }
+      });
+    });
 
     function prepareAnnounceResponse(announceResponse) {
       agentConnectionStub.announceNodeCollector.callsArgWithAsync(0, null, JSON.stringify(announceResponse));

packages/collector/test/apps/agentStub.js

@@ -38,6 +38,7 @@ const enableSpanBatching = process.env.ENABLE_SPANBATCHING === 'true';
 const kafkaTraceCorrelation = process.env.KAFKA_TRACE_CORRELATION
   ? process.env.KAFKA_TRACE_CORRELATION === 'true'
   : null;
+const ignoreEndpoints = process.env.IGNORE_ENDPOINTS && JSON.parse(process.env.IGNORE_ENDPOINTS);
 
 let discoveries = {};
 let rejectAnnounceAttempts = 0;
@@ -86,7 +87,7 @@ app.put('/com.instana.plugin.nodejs.discovery', (req, res) => {
     }
   };
 
-  if (kafkaTraceCorrelation != null || extraHeaders.length > 0 || enableSpanBatching) {
+  if (kafkaTraceCorrelation != null || extraHeaders.length > 0 || enableSpanBatching || ignoreEndpoints) {
     response.tracing = {};
 
     if (extraHeaders.length > 0) {
@@ -103,8 +104,10 @@ app.put('/com.instana.plugin.nodejs.discovery', (req, res) => {
     if (enableSpanBatching) {
       response.tracing['span-batching-enabled'] = true;
     }
+    if (ignoreEndpoints) {
+      response.tracing['ignore-endpoints'] = ignoreEndpoints;
+    }
   }
-
   res.send(response);
 });
 

packages/collector/test/apps/agentStubControls.js

@@ -43,6 +43,11 @@ class AgentStubControls {
         env.KAFKA_TRACE_CORRELATION = opts.kafkaConfig.traceCorrelation.toString();
       }
     }
+    // This is not the INSTANA_IGNORE_ENDPOINTS env. We  use this "IGNORE_ENDPOINTS" env for the fake agent to
+    // serve the ignore endpoints config to our tracer.
+    if (opts.ignoreEndpoints) {
+      env.IGNORE_ENDPOINTS = JSON.stringify(opts.ignoreEndpoints);
+    }
 
     this.agentStub = spawn('node', [path.join(__dirname, 'agentStub.js')], {
       stdio: config.getAppStdio(),

packages/collector/test/tracing/database/ioredis/test.js

@@ -1281,4 +1281,142 @@ function checkConnection(span, setupType) {
       });
     }
   });
+  mochaSuiteFn('ignore-endpoints:', function () {
+    describe('when ignore-endpoints is enabled via agent configuration', () => {
+      const { AgentStubControls } = require('../../../apps/agentStubControls');
+      const customAgentControls = new AgentStubControls();
+      let controls;
+
+      before(async () => {
+        await customAgentControls.startAgent({
+          ignoreEndpoints: { redis: 'get|set' }
+        });
+
+        controls = new ProcessControls({
+          agentControls: customAgentControls,
+          dirname: __dirname
+        });
+        await controls.startAndWaitForAgentConnection();
+      });
+
+      beforeEach(async () => {
+        await customAgentControls.clearReceivedTraceData();
+      });
+
+      after(async () => {
+        await customAgentControls.stopAgent();
+        await controls.stop();
+      });
+
+      it('should ignore redis spans for ignored endpoints (get, set)', async () => {
+        await controls
+          .sendRequest({
+            method: 'POST',
+            path: '/values',
+            qs: {
+              key: 'discount',
+              value: 50
+            }
+          })
+          .then(async () => {
+            return retry(async () => {
+              const spans = await customAgentControls.getSpans();
+              // 1 x http entry span
+              expect(spans.length).to.equal(1);
+              spans.forEach(span => {
+                expect(span.n).not.to.equal('redis');
+              });
+              expectAtLeastOneMatching(spans, [
+                span => expect(span.n).to.equal('node.http.server'),
+                span => expect(span.data.http.method).to.equal('POST')
+              ]);
+            });
+          });
+      });
+    });
+    describe('when ignore-endpoints is enabled via tracing configuration', async () => {
+      globalAgent.setUpCleanUpHooks();
+      const agentControls = globalAgent.instance;
+      let controls;
+
+      before(async () => {
+        controls = new ProcessControls({
+          useGlobalAgent: true,
+          dirname: __dirname,
+          env: {
+            INSTANA_IGNORE_ENDPOINTS: '{"redis": ["get"]}'
+          }
+        });
+        await controls.start();
+      });
+
+      beforeEach(async () => {
+        await agentControls.clearReceivedTraceData();
+      });
+
+      after(async () => {
+        await controls.stop();
+      });
+
+      afterEach(async () => {
+        await controls.clearIpcMessages();
+      });
+      it('should ignore spans for ignored endpoint (get)', async function () {
+        await controls
+          .sendRequest({
+            method: 'GET',
+            path: '/values',
+            qs: {
+              key: 'discount',
+              value: 50
+            }
+          })
+
+          .then(async () => {
+            return retry(async () => {
+              const spans = await agentControls.getSpans();
+              // 1 x http entry span
+              expect(spans.length).to.equal(1);
+              spans.forEach(span => {
+                expect(span.n).not.to.equal('redis');
+              });
+
+              expectAtLeastOneMatching(spans, [
+                span => expect(span.n).to.equal('node.http.server'),
+                span => expect(span.data.http.method).to.equal('GET')
+              ]);
+            });
+          });
+      });
+      it('should not ignore spans for endpoints that are not in the ignore list', async () => {
+        await controls
+          .sendRequest({
+            method: 'POST',
+            path: '/values',
+            qs: {
+              key: 'discount',
+              value: 50
+            }
+          })
+          .then(async () => {
+            return retry(async () => {
+              const spans = await agentControls.getSpans();
+              expect(spans.length).to.equal(2);
+
+              const entrySpan = expectAtLeastOneMatching(spans, [
+                span => expect(span.n).to.equal('node.http.server'),
+                span => expect(span.data.http.method).to.equal('POST')
+              ]);
+
+              expectExactlyOneMatching(spans, [
+                span => expect(span.t).to.equal(entrySpan.t),
+                span => expect(span.p).to.equal(entrySpan.s),
+                span => expect(span.n).to.equal('redis'),
+                span => expect(span.data.redis.command).to.equal('set')
+              ]);
+            });
+          });
+      });
+    });
+  });
 });