[WIP] Adds a cli command to classify usntructured data

.pre-commit-config.yaml

@@ -24,6 +24,7 @@ repos:
       - id: check-json
         exclude: (.vscode/|tsconfig.json|.devcontainer/)
       - id: detect-private-key
+        exclude: (examples/chats/)
       - id: detect-aws-credentials
         args: [--allow-missing-credentials]
       - id: check-yaml

.vscode/settings.json

@@ -61,6 +61,7 @@
     "pipeable",
     "preact",
     "pubspec",
+    "redactions",
     "Requirize",
     "respawned",
     "respawning",

examples/chats/generate_chats.ts

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+// generate-support-chats.mjs
+//
+// Generates fake support chat transcripts with personal data categories.
+// Creates N .txt files in the specified directory.
+//
+// Run with:
+//   node generate-support-chats.mjs --out ./support_chats --count 1000
+
+import { mkdirSync, writeFileSync, existsSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+/**
+ * Generates a random integer between min and max (inclusive).
+ *
+ * @param min - Minimum value (inclusive)
+ * @param max - Maximum value (inclusive)
+ * @returns - Random integer between min and max
+ */
+function randomInt(min: number, max: number): number {
+  return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+/**
+ * Picks a random element from an array.
+ *
+ * @param arr - Array to pick from
+ * @returns - Random element from the array
+ */
+function pick<T>(arr: T[]): T {
+  return arr[randomInt(0, arr.length - 1)];
+}
+
+// ----- Data pools -----
+const agentNames = ['Sofia', 'Liam', 'Ava', 'Noah', 'Maya', 'Ethan'];
+const customerNames = ['Jordan', 'Taylor', 'Sam', 'Alex', 'Casey', 'Riley'];
+
+const openings = [
+  'Hi, I need help with',
+  "Hello—I'm having trouble with",
+  'Can you assist me with',
+  'Good morning, question about',
+];
+
+const issues = [
+  'a double charge on my card',
+  'logging into my account',
+  'a late delivery',
+  'resetting my password',
+  'updating my subscription',
+  'firmware update errors',
+];
+
+// Example personal data categories to sprinkle into chats
+const personalData = [
+  'Social Security Number: 123-45-6789',
+  'User ID: U123456',
+  'Username: jordan_92',
+  'Declared Interests: hiking, cooking',
+  'Profile Picture URL: https://example.com/img/profile123.jpg',
+  'User Preferences: marketing_emails=false',
+  'Profile URL: https://social.example.com/jordan',
+  'IP Address: 192.168.1.42',
+  'MAC Address: 00:1B:44:11:3A:B7',
+  'User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)',
+  'Cookies: sessionid=abcd1234',
+  'Serial Number: SN-1234567890',
+  'Survey Data: Q1=Yes, Q2=No',
+  'Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----',
+  'Email: [email protected]',
+  'Phone: +1-202-555-0172',
+  'Approximate Geolocation: San Francisco, CA',
+  'Date of Birth: 1990-05-12',
+  'Gender: Non-binary',
+  'Political Affiliation: Independent',
+  'Race/Ethnicity: Hispanic',
+  'Religion: None',
+  'Sexual Orientation: Heterosexual',
+  'Union Membership: None',
+  'Biometric Identifier: FaceID registered',
+];
+
+/**
+ * Generate a single chat
+ *
+ * @param chatId - Unique identifier for the chat
+ * @returns - Formatted chat transcript as a string
+ */
+function generateChat(chatId: number): string {
+  const customer = pick(customerNames);
+  const agent = pick(agentNames);
+  const turns = randomInt(8, 18);
+
+  const lines = [];
+  lines.push(`Support Chat Transcript #${chatId}`);
+  lines.push('==================================');
+
+  for (let i = 0; i < turns; i += 1) {
+    if (i % 2 === 0) {
+      // customer
+      const msg =
+        i === 0
+          ? `${pick(openings)} ${pick(issues)}.`
+          : chance(0.2)
+          ? `Here is my ${pick(personalData)}`
+          : 'Can you check on that?';
+      lines.push(`${customer}: ${msg}`);
+    } else {
+      // agent
+      const msg = chance(0.3)
+        ? `Thanks, can you also provide your ${pick(personalData)}?`
+        : 'Let me look that up for you.';
+      lines.push(`${agent}: ${msg}`);
+    }
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Chance function to simulate probability.
+ *
+ * @param p - Probability of true (0-1)
+ * @returns - True with probability p
+ */
+function chance(p: number): boolean {
+  return Math.random() < p;
+}
+
+const filename = fileURLToPath(import.meta.url);
+const dir = dirname(filename);
+
+const OUT_DIR = join(dir, './support_chats');
+const COUNT = Number(process.env.COUNT || 1000);
+
+if (!existsSync(OUT_DIR)) mkdirSync(OUT_DIR, { recursive: true });
+
+for (let i = 1; i <= COUNT; i += 1) {
+  const chat = generateChat(i);
+  const filename = join(OUT_DIR, `chat_${i.toString().padStart(4, '0')}.txt`);
+  writeFileSync(filename, chat, 'utf8');
+}
+
+const logger = console;
+logger.log(`✅ Generated ${COUNT} support chat transcripts in ${OUT_DIR}`);

examples/chats/support_chats/chat_0001.txt

@@ -0,0 +1,16 @@
+Support Chat Transcript #1
+==================================
+Taylor: Hi, I need help with logging into my account.
+Sofia: Let me look that up for you.
+Taylor: Can you check on that?
+Sofia: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?
+Taylor: Can you check on that?
+Sofia: Let me look that up for you.
+Taylor: Can you check on that?
+Sofia: Let me look that up for you.
+Taylor: Can you check on that?
+Sofia: Let me look that up for you.
+Taylor: Can you check on that?
+Sofia: Thanks, can you also provide your Union Membership: None?
+Taylor: Can you check on that?
+Sofia: Let me look that up for you.

examples/chats/support_chats/chat_0002.txt

@@ -0,0 +1,17 @@
+Support Chat Transcript #2
+==================================
+Casey: Good morning, question about a late delivery.
+Sofia: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?
+Casey: Can you check on that?
+Sofia: Let me look that up for you.
+Casey: Here is my Profile Picture URL: https://example.com/img/profile123.jpg
+Sofia: Let me look that up for you.
+Casey: Can you check on that?
+Sofia: Let me look that up for you.
+Casey: Can you check on that?
+Sofia: Let me look that up for you.
+Casey: Can you check on that?
+Sofia: Let me look that up for you.
+Casey: Can you check on that?
+Sofia: Let me look that up for you.
+Casey: Here is my Profile URL: https://social.example.com/jordan

examples/chats/support_chats/chat_0003.txt

@@ -0,0 +1,13 @@
+Support Chat Transcript #3
+==================================
+Sam: Good morning, question about logging into my account.
+Liam: Let me look that up for you.
+Sam: Can you check on that?
+Liam: Let me look that up for you.
+Sam: Can you check on that?
+Liam: Let me look that up for you.
+Sam: Can you check on that?
+Liam: Let me look that up for you.
+Sam: Can you check on that?
+Liam: Let me look that up for you.
+Sam: Can you check on that?

examples/chats/support_chats/chat_0004.txt

@@ -0,0 +1,10 @@
+Support Chat Transcript #4
+==================================
+Sam: Good morning, question about updating my subscription.
+Ava: Let me look that up for you.
+Sam: Can you check on that?
+Ava: Let me look that up for you.
+Sam: Here is my IP Address: 192.168.1.42
+Ava: Let me look that up for you.
+Sam: Can you check on that?
+Ava: Let me look that up for you.

examples/chats/support_chats/chat_0005.txt

@@ -0,0 +1,13 @@
+Support Chat Transcript #5
+==================================
+Taylor: Hi, I need help with a late delivery.
+Liam: Let me look that up for you.
+Taylor: Here is my Survey Data: Q1=Yes, Q2=No
+Liam: Let me look that up for you.
+Taylor: Can you check on that?
+Liam: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?
+Taylor: Can you check on that?
+Liam: Let me look that up for you.
+Taylor: Can you check on that?
+Liam: Thanks, can you also provide your Race/Ethnicity: Hispanic?
+Taylor: Can you check on that?

examples/chats/support_chats/chat_0006.txt

@@ -0,0 +1,11 @@
+Support Chat Transcript #6
+==================================
+Riley: Hello—I'm having trouble with a double charge on my card.
+Liam: Thanks, can you also provide your User ID: U123456?
+Riley: Can you check on that?
+Liam: Thanks, can you also provide your Sexual Orientation: Heterosexual?
+Riley: Can you check on that?
+Liam: Let me look that up for you.
+Riley: Here is my Race/Ethnicity: Hispanic
+Liam: Let me look that up for you.
+Riley: Can you check on that?

examples/chats/support_chats/chat_0007.txt

@@ -0,0 +1,15 @@
+Support Chat Transcript #7
+==================================
+Riley: Hi, I need help with a double charge on my card.
+Sofia: Let me look that up for you.
+Riley: Can you check on that?
+Sofia: Thanks, can you also provide your Cookies: sessionid=abcd1234?
+Riley: Here is my Approximate Geolocation: San Francisco, CA
+Sofia: Let me look that up for you.
+Riley: Here is my Race/Ethnicity: Hispanic
+Sofia: Let me look that up for you.
+Riley: Can you check on that?
+Sofia: Let me look that up for you.
+Riley: Can you check on that?
+Sofia: Thanks, can you also provide your Serial Number: SN-1234567890?
+Riley: Here is my User Preferences: marketing_emails=false

examples/chats/support_chats/chat_0008.txt

@@ -0,0 +1,10 @@
+Support Chat Transcript #8
+==================================
+Casey: Hello—I'm having trouble with a late delivery.
+Ethan: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?
+Casey: Can you check on that?
+Ethan: Let me look that up for you.
+Casey: Can you check on that?
+Ethan: Thanks, can you also provide your Phone: +1-202-555-0172?
+Casey: Can you check on that?
+Ethan: Let me look that up for you.

examples/chats/support_chats/chat_0009.txt

@@ -0,0 +1,17 @@
+Support Chat Transcript #9
+==================================
+Taylor: Good morning, question about firmware update errors.
+Sofia: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----?
+Taylor: Can you check on that?
+Sofia: Let me look that up for you.
+Taylor: Can you check on that?
+Sofia: Let me look that up for you.
+Taylor: Can you check on that?
+Sofia: Thanks, can you also provide your Serial Number: SN-1234567890?
+Taylor: Can you check on that?
+Sofia: Let me look that up for you.
+Taylor: Can you check on that?
+Sofia: Let me look that up for you.
+Taylor: Here is my MAC Address: 00:1B:44:11:3A:B7
+Sofia: Let me look that up for you.
+Taylor: Can you check on that?

examples/chats/support_chats/chat_0010.txt

@@ -0,0 +1,12 @@
+Support Chat Transcript #10
+==================================
+Sam: Can you assist me with updating my subscription.
+Ethan: Let me look that up for you.
+Sam: Can you check on that?
+Ethan: Let me look that up for you.
+Sam: Can you check on that?
+Ethan: Thanks, can you also provide your Survey Data: Q1=Yes, Q2=No?
+Sam: Can you check on that?
+Ethan: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA?
+Sam: Can you check on that?
+Ethan: Thanks, can you also provide your Declared Interests: hiking, cooking?

examples/chats/support_chats/chat_0011.txt

@@ -0,0 +1,20 @@
+Support Chat Transcript #11
+==================================
+Sam: Hi, I need help with firmware update errors.
+Maya: Thanks, can you also provide your Religion: None?
+Sam: Can you check on that?
+Maya: Let me look that up for you.
+Sam: Can you check on that?
+Maya: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?
+Sam: Here is my Approximate Geolocation: San Francisco, CA
+Maya: Let me look that up for you.
+Sam: Can you check on that?
+Maya: Let me look that up for you.
+Sam: Here is my Date of Birth: 1990-05-12
+Maya: Thanks, can you also provide your User ID: U123456?
+Sam: Can you check on that?
+Maya: Let me look that up for you.
+Sam: Here is my Email: [email protected]
+Maya: Let me look that up for you.
+Sam: Can you check on that?
+Maya: Thanks, can you also provide your Email: [email protected]?

examples/chats/support_chats/chat_0012.txt

@@ -0,0 +1,12 @@
+Support Chat Transcript #12
+==================================
+Sam: Good morning, question about a double charge on my card.
+Ava: Thanks, can you also provide your Email: [email protected]?
+Sam: Can you check on that?
+Ava: Thanks, can you also provide your Political Affiliation: Independent?
+Sam: Can you check on that?
+Ava: Let me look that up for you.
+Sam: Can you check on that?
+Ava: Let me look that up for you.
+Sam: Can you check on that?
+Ava: Let me look that up for you.

examples/chats/support_chats/chat_0013.txt

@@ -0,0 +1,19 @@
+Support Chat Transcript #13
+==================================
+Sam: Good morning, question about updating my subscription.
+Noah: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?
+Sam: Can you check on that?
+Noah: Thanks, can you also provide your Union Membership: None?
+Sam: Can you check on that?
+Noah: Let me look that up for you.
+Sam: Can you check on that?
+Noah: Let me look that up for you.
+Sam: Here is my Serial Number: SN-1234567890
+Noah: Thanks, can you also provide your Union Membership: None?
+Sam: Can you check on that?
+Noah: Let me look that up for you.
+Sam: Here is my Social Security Number: 123-45-6789
+Noah: Thanks, can you also provide your IP Address: 192.168.1.42?
+Sam: Can you check on that?
+Noah: Let me look that up for you.
+Sam: Here is my MAC Address: 00:1B:44:11:3A:B7

examples/chats/support_chats/chat_0014.txt

@@ -0,0 +1,17 @@
+Support Chat Transcript #14
+==================================
+Casey: Can you assist me with logging into my account.
+Ava: Thanks, can you also provide your Race/Ethnicity: Hispanic?
+Casey: Can you check on that?
+Ava: Let me look that up for you.
+Casey: Here is my Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----
+Ava: Let me look that up for you.
+Casey: Can you check on that?
+Ava: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?
+Casey: Here is my Profile Picture URL: https://example.com/img/profile123.jpg
+Ava: Let me look that up for you.
+Casey: Here is my Race/Ethnicity: Hispanic
+Ava: Let me look that up for you.
+Casey: Can you check on that?
+Ava: Let me look that up for you.
+Casey: Can you check on that?