[ti_anomali] Add Benchmark and Policy Test

packages/ti_anomali/_dev/benchmark/rally/intelligence-benchmark.yml

@@ -0,0 +1,14 @@
+---
+description: Benchmark 100000 anomali.intelligence events ingested.
+data_stream:
+   name: intelligence
+corpora:
+   generator:
+      total_events: 100000
+      template:
+         type: gotext
+         path: ./intelligence-benchmark/template.ndjson
+      config:
+         path: ./intelligence-benchmark/config.yml
+      fields:
+         path: ./intelligence-benchmark/fields.yml

packages/ti_anomali/_dev/benchmark/rally/intelligence-benchmark/config.yml

@@ -0,0 +1,98 @@
+fields:
+  - name: confidence
+    cardinility: 100
+  - name: created_by
+    period: -24h
+  - name: created_ts
+    period: -24h
+  - name: expiration_ts
+    period: -24h
+  - name: feed_id
+    cardinility: 1000000
+  - name: id
+    cardinility: 1000000
+  - name: itype
+    enum:
+      - mal_ip
+      - mal_domain
+      - mal_url
+      - parked_ip
+      - parked_email
+      - parked_url
+  - name: meta_detail2
+    enum:
+      - imported by user 142
+      - imported by user 143
+  - name: meta_severity
+    enum:
+      - low
+      - medium
+      - high
+      - very-high
+  - name: modified_ts
+    period: -24h
+  - name: org
+    enum:
+      - Domains by Proxy, LLC
+      - Alicloud-hk
+  - name: owner_organization_id
+    cardinility: 1000000
+  - name: resource_uri
+    enum:
+      - "/api/v2/intelligence/232020126/"
+      - "/api/v2/intelligence/235548914/"
+      - "/api/v2/intelligence/184982668/"
+  - name: retina_confidence
+    cardinility: 100
+  - name: sort
+    cardinility: 1000000
+  - name: source
+    enum:
+      - Analyst
+      - Default Organization
+  - name: source_reported_confidence
+    cardinility: 100
+  - name: status
+    enum:
+      - active
+      - inactive
+  - name: tags_id
+    cardinility: 1000000
+  - name: tags_name
+    enum:
+      - Domains-contacted-by-samples-which-do-public-IP-checks.
+      - public-ip-check-dns
+      - md5-3d4bf45cc1648d76f9770c7c27afc4b8
+  - name: threat_type
+    enum:
+      - bot
+      - apt
+      - c2
+      - i2p
+      - malware
+  - name: threatscore
+    cardinility: 100
+  - name: timestamp
+    period: -24h
+  - name: tlp
+    enum:
+      - WHITE
+      - AMBER
+  - name: trusted_circle_ids
+    cardinility: 1000000
+  - name: type
+    enum:
+      - domain
+      - ip
+      - url
+  - name: update_id
+    cardinility: 1000000
+  - name: uuid
+    cardinility: 1000000
+  - name: value
+    enum:
+      - "test_mail_remote@test.com"
+      - "test_mail_remote2@test.com"
+      - "test_mail_remote3@test.com"
+      - "test_mail_remote4@test.com"
+      - "test_mail_remote5@test.com"

packages/ti_anomali/_dev/benchmark/rally/intelligence-benchmark/fields.yml

@@ -0,0 +1,70 @@
+- name: asn
+  type: keyword
+- name: can_add_public_tags
+  type: boolean
+- name: confidence
+  type: long
+- name: created_by
+  type: keyword
+- name: created_ts
+  type: date
+- name: description
+  type: keyword
+- name: expiration_ts
+  type: date
+- name: feed_id
+  type: long
+- name: id
+  type: long
+- name: is_anonymous
+  type: boolean
+- name: is_public
+  type: boolean
+- name: itype
+  type: keyword
+- name: meta_detail2
+  type: keyword
+- name: meta_severity
+  type: keyword
+- name: modified_ts
+  type: keyword
+- name: org
+  type: keyword
+- name: owner_organization_id
+  type: long
+- name: resource_uri
+  type: keyword
+- name: retina_confidence
+  type: long
+- name: sort
+  type: long
+- name: source
+  type: keyword
+- name: source_reported_confidence
+  type: long
+- name: status
+  type: keyword
+- name: subtype
+  type: keyword
+- name: tags_id
+  type: keyword
+- name: tags_name
+  type: keyword
+- name: threat_type
+  type: keyword
+- name: threatscore
+  type: long
+- name: timestamp
+  type: date
+- name: tlp
+  type: keyword
+- name: trusted_circle_ids
+  type: keyword
+- name: type
+  type: keyword
+- name: update_id
+  type: long
+- name: uuid
+  type: keyword
+- name: value
+  type: keyword

packages/ti_anomali/_dev/benchmark/rally/intelligence-benchmark/template.ndjson

@@ -0,0 +1,79 @@
+{{- $timestamp := generate "timestamp" -}}
+{{- $source := generate "source" -}}
+{{- $threatscore := generate "threatscore" -}}
+{{- $threat_type := generate "threat_type" -}}
+{{- $trusted_circle_ids := generate "trusted_circle_ids" -}}
+{{- $description := generate "description" -}}
+{{- $sort := generate "sort" -}}
+{{- $resource_uri := generate "resource_uri" -}}
+{{- $modified_ts := generate "modified_ts" -}}
+{{- $update_id := generate "update_id" -}}
+{{- $source_reported_confidence := generate "source_reported_confidence" -}}
+{{- $type := generate "type" -}}
+{{- $uuid := generate "uuid" -}}
+{{- $feed_id := generate "feed_id" -}}
+{{- $retina_confidence := generate "retina_confidence" -}}
+{{- $created_ts := generate "created_ts" -}}
+{{- $id := generate "id" -}}
+{{- $value := generate "value" -}}
+{{- $itype := generate "itype" -}}
+{{- $org := generate "org" -}}
+{{- $confidence := generate "confidence" -}}
+{{- $expiration_ts := generate "expiration_ts" -}}
+{{- $owner_organization_id := generate "owner_organization_id" -}}
+{{- $meta_severity := generate "meta_severity" -}}
+{{- $meta_detail2 := generate "meta_detail2" -}}
+{{- $is_anonymous := generate "is_anonymous" -}}
+{{- $is_public := generate "is_public" -}}
+{{- $asn := generate "asn" -}}
+{{- $status := generate "status" -}}
+{{- $tags_id := generate "tags_id" -}}
+{{- $tags_name := generate "tags_name" -}}
+{{- $can_add_public_tags := generate "can_add_public_tags" -}}
+{{- $subtype := generate "subtype" -}}
+{{- $tlp := generate "tlp" -}}
+{{- $created_by := generate "created_by" -}}
+{
+  "json": {
+    "source": "{{ $source }}",
+    "threatscore": {{ $threatscore }},
+    "threat_type": "{{ $threat_type }}",
+    "trusted_circle_ids": "{{ $trusted_circle_ids }}",
+    "description": "{{ $description }}",
+    "sort": [{{ $sort }}],
+    "resource_uri": "{{ $resource_uri }}",
+    "modified_ts": "{{ $modified_ts }}",
+    "update_id": {{ $update_id }},
+    "source_reported_confidence": {{ $source_reported_confidence }},
+    "type": "{{ $type }}",
+    "uuid": "{{ $uuid }}",
+    "feed_id": {{ $feed_id }},
+    "retina_confidence": {{ $retina_confidence }},
+    "created_ts": "{{ $created_ts }}",
+    "id": {{ $id }},
+    "value": "{{ $value }}",
+    "itype": "{{ $itype }}",
+    "org": "{{ $org }}",
+    "confidence": {{ $confidence }},
+    "expiration_ts": "{{ $expiration_ts }}",
+    "owner_organization_id": {{ $owner_organization_id }},
+    "meta": {
+      "severity": "{{ $meta_severity }}",
+      "detail2": "{{ $meta_detail2 }}"
+    },
+    "is_anonymous": {{ $is_anonymous }},
+    "is_public": {{ $is_public }},
+    "asn": "{{ $asn }}",
+    "status": "{{ $status }}",
+    "tags": [
+      {
+        "id": "{{ $tags_id }}",
+        "name": "{{ $tags_name }}"
+      }
+    ],
+    "can_add_public_tags": {{ $can_add_public_tags }},
+    "subtype": "{{ $subtype }}",
+    "tlp": "{{ $tlp }}",
+    "created_by": "{{ $created_by }}"
+  }
+}

packages/ti_anomali/_dev/benchmark/rally/threatstream-benchmark.yml

@@ -0,0 +1,14 @@
+---
+description: Benchmark 100000 anomali.threatstream events ingested.
+data_stream:
+   name: threatstream
+corpora:
+   generator:
+      total_events: 100000
+      template:
+         type: gotext
+         path: ./threatstream-benchmark/template.ndjson
+      config:
+         path: ./threatstream-benchmark/config.yml
+      fields:
+         path: ./threatstream-benchmark/fields.yml

packages/ti_anomali/_dev/benchmark/rally/threatstream-benchmark/config.yml

@@ -0,0 +1,83 @@
+fields:
+  - name: added_at
+    period: -24h
+  - name: classification
+    enum:
+      - private
+      - public
+  - name: confidence
+    cardinality: 100
+  - name: country
+    enum:
+      - US
+      - DE
+      - IN
+      - VS
+      - RU
+  - name: date_first
+    period: -24h
+  - name: date_last
+    period: -24h
+  - name: detail
+    enum:
+      - phish-kit-sig-id-43111996,Microsoft
+      - first_seen=2020-04-13T09:30:20,IP=192.168.113.221,ciib,o2d,mask=192.168.2.110,popularity=high
+      - 32-bit,date_added=2020-10-09T15:44:05,elf,mips
+      - gnh7,Botnet-DRZ8-,popularity=low,type=2,first_seen=2020-01-07T01:38:35,Botnet-WSPDZDY,mask=192.168.113.180,popularity=low,threat=gu3wn7
+  - name: detail2
+    enum:
+      - imported by user 1
+      - imported by user 710
+  - name: id
+    cardinality: 1000000
+  - name: itype
+    enum:
+      - mal_md5
+      - phish_url
+      - scan_ip
+      - mal_url
+      - mal_domain
+  - name: lat
+    cardinality: 100
+  - name: maltype
+    enum:
+      - phish-kit-sig-id-43111996
+      - 32-bit
+      - malware:mi5n
+  - name: resource_uri
+    enum:
+      - /api/v1/intelligence/P29675942316/
+      - /api/v1/intelligence/22222/
+      - /api/v1/intelligence/111111/
+  - name: severity
+    enum:
+      - very-high
+      - high
+      - medium
+      - low
+  - name: source
+    enum:
+      - Default Organization
+      - Phony generated indicator
+  - name: source_feed_id
+    cardinality: 1000000
+  - name: state
+    value: active
+  - name: timestamp
+    period: -24h
+  - name: trusted_circle_ids
+    cardinality: 1000000
+  - name: update_id
+    cardinality: 1000000
+  - name: url
+    enum:
+      - http://onv7s.example.org/29j3q7kc/4l0za3s?viyrr-vd=hde
+      - http://ureumt8.example.org/ffey/ugwd?770694=x4r5wc-k
+      - https://5wcz6kck.example.net/mankgvtpl/1suq?vx-gvh=00tc4
+      - https://v6cw8.example.org/yw7fom/x6xp?3ck=i3ko4
+      - http://tccdg3.example.net/2vgsz9a/9tzk9?xsy9af-=jz3ibf
+  - name: value_type
+    enum:
+      - url
+      - ip
+      - domain