Merge pull request #337 from ninoseki/add-only

feat: add only option

README.md

@@ -27,11 +27,12 @@ $ ioc-extractor --help
 Usage: ioc-extractor [options]
 
 Options:
-  --no-strict     Disable strict option
-  --no-refang     Disable refang option
-  --no-sort       Disable sort option
-  -p, --punycode  Enable punycode option
-  -h, --help      display help for command
+  --no-strict            Disable strict option
+  --no-refang            Disable refang option
+  --no-sort              Disable sort option
+  -p, --punycode         Enable punycode option
+  -o, --only <types...>  Show only specific IoC types
+  -h, --help             display help for command
 ```
 
 ```bash
@@ -61,6 +62,14 @@ $ echo "1.1.1.1 8.8.8.8 example.com" | ioc-extractor | jq
   "urls": [],
   "xmrs": []
 }
+
+$ echo "1.1.1.1 8.8.8.8" | ioc-extractor --only ipv4s | jq
+{
+  "ipv4s": [
+    "1.1.1.1",
+    "8.8.8.8"
+  ]
+}
 ```
 
 ### As a library
@@ -85,7 +94,7 @@ console.log(ioc.domains);
 - [sort](#sort)
 - [strict](#strict)
 
-If you want to extract a specific type of IoC, you can use extract function.
+If you want to extract a specific type of IoC, you can use an extract function by IoC type.
 
 ```ts
 import {
@@ -116,6 +125,17 @@ Network related extract functions (e.g. `extractDomains`) can take the following
 
 See [docs](https://ninoseki.github.io/ioc-extractor/) for more details.
 
+Alternatively, if you want to extract a list of specific IoC types at once, you can use `partialExtractIOC`.
+
+```ts
+import { partialExtractIOC } from "ioc-extractor";
+
+const input = "1.1.1[.]1 google(.)com f6f8179ac71eaabff12b8c024342109b";
+const ioc = partialExtractIOC(input, ["ipv4s", "domains"]);
+console.log(ioc);
+// => {"ipv4s":["1.1.1.1"],"domains":["google.com"]}
+```
+
 ## IoC Types
 
 This package supports the following IoCs:

spec/index.spec.ts

@@ -1,9 +1,9 @@
 import * as fs from "fs";
 import * as path from "path";
 
-import { extractIOC } from "@/index";
+import { extractIOC, partialExtractIOC } from "@/index";
 
-describe("IOCExtractor", () => {
+describe("extractIOC", () => {
   describe("simple input", () => {
     it("should extract IOCs from the input", () => {
       const input =
@@ -113,3 +113,20 @@ describe("IOCExtractor", () => {
     });
   });
 });
+
+describe("partialExtractIOC", () => {
+  describe("simple input", () => {
+    it("should only extract specific IoCs from the input", () => {
+      const input =
+        "1.1.1[.]1 2.2.2 . 2 google(.)com テスト.example.com https://www.google[.]com http://テスト.example.com f6f8179ac71eaabff12b8c024342109b 275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f UA-26296840-4 test@テスト.example.com example.nope";
+      const ioc = partialExtractIOC(input, ["domains"]);
+
+      expect(ioc.domains).toEqual([
+        "example.com",
+        "google.com",
+        "www.google.com",
+      ]);
+      expect(ioc.ipv4s).toBeUndefined();
+    });
+  });
+});

src/bin/ioc-extractor.ts

@@ -2,7 +2,11 @@
 import { program } from "commander";
 import getStdin from "get-stdin";
 
-import { extractIOC, Options } from "../index";
+import { IOCKey, Options } from "@/types";
+
+import { extractIOC, partialExtractIOC } from "../index";
+
+type OnlyOptions = Options & Partial<{ only: IOCKey[] }>;
 
 (async (): Promise<void> => {
   const input = (await getStdin()).trim();
@@ -11,11 +15,15 @@ import { extractIOC, Options } from "../index";
     .option("--no-strict", "Disable strict option")
     .option("--no-refang", "Disable refang option")
     .option("--no-sort", "Disable sort option")
-    .option("-p, --punycode", "Enable punycode option");
+    .option("-p, --punycode", "Enable punycode option")
+    .option("-o, --only <types...>", "Show only specific IoC types");
   program.parse();
 
-  const options = <Options>program.opts();
-  const ioc = extractIOC(input, options);
+  const options = <OnlyOptions>program.opts();
+
+  const ioc = options.only
+    ? partialExtractIOC(input, options.only, options)
+    : extractIOC(input, options);
 
   // eslint-disable-next-line no-console
   console.log(JSON.stringify(ioc));

src/index.ts

@@ -57,7 +57,7 @@ import {
   isURL,
   isXMR,
 } from "./aux/validators";
-import type { IOC, Options } from "./types";
+import type { IOC, IOCKey, Options } from "./types";
 
 export {
   extractASN,
@@ -121,36 +121,14 @@ export {
 export type { IOC, Options };
 
 export class IOCExtractor {
-  /**
-   * Returns an IoC in data
-   *
-   * @static
-   * @param {string} data A string
-   * @param {Options} options
-   * @returns {IOC}
-   * @memberof IOCExtractor
-   */
-  public static extractIOC(
-    data: string,
-    options: Options = {
-      strict: true,
-      refang: true,
-      punycode: false,
-      sort: true,
-    },
-  ): IOC {
-    const extractor = new IOCExtractor(data);
-    return extractor.extractIOC(options);
-  }
+  private s: string;
 
-  private data: string;
-
-  public constructor(data: string) {
-    this.data = data;
+  public constructor(s: string) {
+    this.s = s;
   }
 
   /**
-   * Returns an IOC of the data
+   * Extract IoCs from a string
    *
    * @returns {IOC}
    * @param {Options} options
@@ -165,7 +143,7 @@ export class IOCExtractor {
     },
   ): IOC {
     // Apply refang
-    let normalized = options.refang ? refang(this.data) : this.data;
+    let normalized = options.refang ? refang(this.s) : this.s;
     // Apply punycode conversion
     normalized = options.punycode
       ? unicodeToASCII(normalized, {
@@ -196,24 +174,98 @@ export class IOCExtractor {
     };
     return ioc;
   }
+
+  /**
+   * Partially extract IoCs a string
+   *
+   * @returns {IOC}
+   * @param {IOCKey[]} only
+   * @param {Options} options
+   * @memberof IOCExtractor
+   */
+  public partialExtractIOC(
+    only: IOCKey[],
+    options: Options = {
+      strict: true,
+      refang: true,
+      punycode: false,
+      sort: true,
+    },
+  ): Partial<IOC> {
+    // Apply refang
+    let normalized = options.refang ? refang(this.s) : this.s;
+    // Apply punycode conversion
+    normalized = options.punycode
+      ? unicodeToASCII(normalized, {
+          ignoreInvalidPunycode: true,
+          transitionalProcessing: true,
+        })
+      : normalized;
+
+    const funcByType = {
+      asns: extractASNs,
+      btcs: extractBTCs,
+      cves: extractCVEs,
+      domains: extractDomains,
+      emails: extractEmails,
+      eths: extractETHs,
+      gaPubIDs: extractGAPubIDs,
+      gaTrackIDs: extractGATrackIDs,
+      ipv4s: extractIPv4s,
+      ipv6s: extractIPv6s,
+      macAddresses: extractMacAddresses,
+      md5s: extractMD5s,
+      sha1s: extractSHA1s,
+      sha256s: extractSHA256s,
+      sha512s: extractSHA512s,
+      ssdeeps: extractSSDEEPs,
+      urls: extractURLs,
+      xmrs: extractXMRs,
+    };
+
+    return Object.fromEntries(
+      only.map((key) => [key, funcByType[key](normalized, options)]),
+    ) as Partial<IOC>;
+  }
 }
 
 /**
- * Returns an IoC of data
+ * Extract IoCs from a string
  *
  * @export
- * @param {string} data A string
+ * @param {string} s A string
  * @param {Options} options
  * @returns {IOC}
  */
 export function extractIOC(
-  data: string,
+  s: string,
   options: Options = {
     strict: true,
     refang: true,
     punycode: false,
     sort: true,
   },
 ): IOC {
-  return IOCExtractor.extractIOC(data, options);
+  return new IOCExtractor(s).extractIOC(options);
+}
+
+/**
+ * Partially extract IoCs from a string
+ *
+ * @export
+ * @param {string} s A string
+ * @param {Options} options
+ * @returns {IOC}
+ */
+export function partialExtractIOC(
+  s: string,
+  only: IOCKey[],
+  options: Options = {
+    strict: true,
+    refang: true,
+    punycode: false,
+    sort: true,
+  },
+): Partial<IOC> {
+  return new IOCExtractor(s).partialExtractIOC(only, options);
 }

src/types.ts

@@ -35,3 +35,23 @@ export interface IOC {
   urls: string[];
   xmrs: string[];
 }
+
+export type IOCKey =
+  | "asns"
+  | "btcs"
+  | "cves"
+  | "domains"
+  | "emails"
+  | "eths"
+  | "gaPubIDs"
+  | "gaTrackIDs"
+  | "ipv4s"
+  | "ipv6s"
+  | "macAddresses"
+  | "md5s"
+  | "sha1s"
+  | "sha256s"
+  | "sha512s"
+  | "ssdeeps"
+  | "urls"
+  | "xmrs";