feat: bumped sdk and data pump dependencies and added support for sensitive data event streams

Author: flowcore-platform

bun.lock

@@ -4,8 +4,8 @@
     "": {
       "name": "@flowcore/local-read-model-mcp-server",
       "dependencies": {
-        "@flowcore/data-pump": "^0.3.5",
-        "@flowcore/sdk": "^1.19.1",
+        "@flowcore/data-pump": "^0.13.0",
+        "@flowcore/sdk": "^1.47.1",
         "@flowcore/sdk-oidc-client": "^1.3.1",
         "@flowcore/time-uuid": "^0.2.0",
         "@modelcontextprotocol/sdk": "^1.7.0",
@@ -54,9 +54,9 @@
 
     "@deno/shim-deno-test": ["@deno/[email protected]", "", {}, ""],
 
-    "@flowcore/data-pump": ["@flowcore/data-pump@0.3.5", "", { "dependencies": { "@date-fns/utc": "^2.1.0", "@deno/shim-deno": "~0.18.0", "@sinclair/typebox": "0.32.15", "date-fns": "^4.1.0", "long": "^5.3.1", "nats": "^2.29.1", "prom-client": "^15.1.3", "rxjs": "^7.8.1", "ws": "^8.13.0" } }, "sha512-FNIjhydEWibG8fr1bSdP6JIpTBmPqVZaNfH9UB7rXnBAkjQBurB0nnMo3uvyjXjUKeVfZ/k7I2P5a8l84gKr1A=="],
+    "@flowcore/data-pump": ["@flowcore/data-pump@0.13.0", "", { "dependencies": { "@date-fns/utc": "^2.1.0", "@deno/shim-deno": "~0.18.0", "@sinclair/typebox": "0.32.15", "date-fns": "^4.1.0", "long": "^5.3.1", "nats": "^2.29.1", "prom-client": "^15.1.3", "rxjs": "^7.8.1", "ws": "^8.13.0" } }, "sha512-/k+bHfCFSeKtVm/JfgQ610Ww5bIJOjg4yGKpsbupTLaSUNFrnuUuIc9gTWSnsV+n/YQHwN0Ag157WP8K2Jikag=="],
 
-    "@flowcore/sdk": ["@flowcore/sdk@1.19.1", "", { "dependencies": { "@deno/shim-deno": "~0.18.0", "@sinclair/typebox": "0.32.15", "rxjs": "^7.8.1", "ws": "^8.13.0" } }, ""],
+    "@flowcore/sdk": ["@flowcore/sdk@1.47.1", "", { "dependencies": { "@deno/shim-deno": "~0.18.0", "@sinclair/typebox": "0.32.15", "rxjs": "^7.8.1", "ws": "^8.13.0" } }, "sha512-Z92MoJUVuM3PKqI8jCFdv93DE16dcUG9eYJ/jOJt7bmufabTzHR0aSlLFfr7leUmJqIDTMaxqwh3a8LVWhDpvQ=="],
 
     "@flowcore/sdk-oidc-client": ["@flowcore/[email protected]", "", {}, ""],
 

package.json

@@ -41,8 +41,8 @@
     "typescript": "^5"
   },
   "dependencies": {
-    "@flowcore/data-pump": "^0.3.5",
-    "@flowcore/sdk": "^1.19.1",
+    "@flowcore/data-pump": "^0.13.0",
+    "@flowcore/sdk": "^1.47.1",
     "@flowcore/sdk-oidc-client": "^1.3.1",
     "@flowcore/time-uuid": "^0.2.0",
     "@modelcontextprotocol/sdk": "^1.7.0",

src/duckdb/stream.ts

@@ -30,6 +30,7 @@ interface StreamInfo {
   maxParallelism?: number
   processedTimeBuckets: number
   totalTimeBuckets: number
+  includeSensitiveData?: boolean
 }
 
 /**
@@ -57,6 +58,7 @@ export async function startEventStreamProjection(
   projectorName: string,
   targetTable: string,
   maxParallelism = 5, // Default to 5 parallel time bucket processes
+  includeSensitiveData = false,
 ) {
   const streamId = `stream-${Date.now()}`
 
@@ -75,6 +77,7 @@ export async function startEventStreamProjection(
     maxParallelism,
     processedTimeBuckets: 0,
     totalTimeBuckets: 0,
+    includeSensitiveData,
   }
 
   activeStreams[streamId] = streamInfo
@@ -110,6 +113,7 @@ async function processEventStream(getBearerToken: () => Promise<string>, streamI
     const endState = getState(streamInfo.endDate)
 
     activeDataPumps[streamId] = FlowcoreDataPump.create({
+      includeSensitiveData: streamInfo.includeSensitiveData ?? false,
       auth: {
         getBearerToken: async () => await getBearerToken(),
       },
@@ -238,6 +242,7 @@ export function getStreamInfo(streamId: string) {
       processedTimeBuckets: streamInfo.processedTimeBuckets,
       totalTimeBuckets: streamInfo.totalTimeBuckets,
       maxParallelism: streamInfo.maxParallelism,
+      includeSensitiveData: streamInfo.includeSensitiveData ?? false,
     },
   }
 }
@@ -258,5 +263,6 @@ export function getAllStreams() {
     targetTable: stream.targetTable,
     processedTimeBuckets: stream.processedTimeBuckets,
     totalTimeBuckets: stream.totalTimeBuckets,
+    includeSensitiveData: stream.includeSensitiveData ?? false,
   }))
 }

src/index.ts

@@ -180,7 +180,7 @@ module.exports = projectEvent;
 
 server.tool(
   "start_event_stream_projection",
-  "Start streaming events from Flowcore and projecting them to the database",
+  "Start streaming events from Flowcore and projecting them to the database, Sensitive data should not be needed, but if you need it ask the user if they are sure they want to include sensitive data, is false by default",
   {
     tenant: z.string().describe("Tenant name"),
     dataCore: z.string().describe("Name of the data core"),
@@ -196,6 +196,12 @@ server.tool(
       .describe(
         "Maximum number of events to process in parallel (default: 100), this can be used to speed up the projection, but don't increase it too much as it will use more local resources",
       ),
+    includeSensitiveData: z
+      .boolean()
+      .optional()
+      .describe(
+        "Whether to include sensitive data in the response, CAUTION: This will return sensitive data from the event type, so use with caution, ask the user if they are sure they want to include sensitive data, is false by default",
+      ),
   },
   startEventStreamProjectionHandler(async () => exchangePat(username, pat)),
 )

src/tools/duckdb-tools.ts

@@ -126,6 +126,7 @@ export const startEventStreamProjectionHandler =
     projectorName,
     targetTable,
     maxParallelism,
+    includeSensitiveData,
   }: {
     tenant: string
     dataCore: string
@@ -136,6 +137,7 @@ export const startEventStreamProjectionHandler =
     projectorName: string
     targetTable: string
     maxParallelism?: number
+    includeSensitiveData?: boolean
   }) => {
     try {
       const result = await startEventStreamProjection(
@@ -149,6 +151,7 @@ export const startEventStreamProjectionHandler =
         projectorName,
         targetTable,
         maxParallelism,
+        includeSensitiveData,
       )
 
       return {