From 5758105f07dc35f29046a53a18611e2e4690ffc9 Mon Sep 17 00:00:00 2001
From: Dominik Rosiek <drosiek@sumologic.com>
Date: Mon, 8 Mar 2021 10:15:30 +0100
Subject: [PATCH 1/4] example: add example for parsing kubernetes logs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Dominik Rosiek <drosiek@sumologic.com>
Co-authored-by: Patryk Małek <69143962+pmalek-sumo@users.noreply.github.com>
---
 examples/kubernetes/README.md                 | 22 ++++++
 examples/kubernetes/docker-compose.yml        |  8 ++
 examples/kubernetes/otel-collector-config.yml | 79 +++++++++++++++++++
 .../logs/0.log                                |  7 ++
 .../logs/0.log                                |  6 ++
 .../logs/0.log                                |  7 ++
 6 files changed, 129 insertions(+)
 create mode 100644 examples/kubernetes/README.md
 create mode 100644 examples/kubernetes/docker-compose.yml
 create mode 100644 examples/kubernetes/otel-collector-config.yml
 create mode 100644 examples/kubernetes/varlogpods/containerd_logs-0_000011112222333344445555666677778888/logs/0.log
 create mode 100644 examples/kubernetes/varlogpods/crio_logs-0_111122223333444455556666777788889999/logs/0.log
 create mode 100644 examples/kubernetes/varlogpods/docker_logs-0_222233334444555566667777888899990000/logs/0.log

diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md
new file mode 100644
index 000000000000..cadc166d72f5
--- /dev/null
+++ b/examples/kubernetes/README.md
@@ -0,0 +1,22 @@
+# OpenTelemetry Collector Demo
+
+This demo is a sample app to build the collector and exercise its kubernetes logs scrapping functionality.
+
+## Build and Run
+
+Two steps are required to build and run the demo:
+
+1. Build latest docker image in main repository directory `make docker-otelcontribcol`
+1. Switch to this directory and run `docker-compose up`
+
+## Description
+
+`varlogpods` contains example log files placed in kubernetes-like directory structure.
+Each of the directory has different formatted logs in one of three formats (either `CRI-O`, `CRI-Containerd` or `Docker`).
+This directory is mounted to standard location (`/var/log/pods`).
+
+`otel-collector-config` is a configuration to autodetect and parse logs for all of three mentioned formats
+
+## ToDo
+
+To cover kubernetes system logs, logs from journald should be supported as well.
diff --git a/examples/kubernetes/docker-compose.yml b/examples/kubernetes/docker-compose.yml
new file mode 100644
index 000000000000..6bb749ad801d
--- /dev/null
+++ b/examples/kubernetes/docker-compose.yml
@@ -0,0 +1,8 @@
+version: "3"
+services:
+  opentelemetry-collector-contrib:
+    image: otelcontribcol
+    command: ["--config=/etc/otel-collector-config.yml"]
+    volumes:
+      - ./otel-collector-config.yml:/etc/otel-collector-config.yml
+      - ./varlogpods:/var/log/pods
diff --git a/examples/kubernetes/otel-collector-config.yml b/examples/kubernetes/otel-collector-config.yml
new file mode 100644
index 000000000000..3c10d55b8a84
--- /dev/null
+++ b/examples/kubernetes/otel-collector-config.yml
@@ -0,0 +1,79 @@
+receivers:
+  filelog:
+    include:
+      # - /var/log/pods/namespace_pod-name_pod-uid/container-name/*.log
+      - /var/log/pods/*/*/*.log
+    exclude:
+      # - /var/log/pods/namespace_pod-name_pod-uid/container-name/*.log
+      - /var/log/pods/*/otel-collector/*.log
+    start_at: beginning
+    include_file_path: true
+    include_file_name: false
+    operators:
+      # Find out which format is used by kubernetes
+      - type: router
+        id: get-format
+        routes:
+          - output: parser-docker
+            expr: '$$record matches "^\\{"'
+          - output: parser-crio
+            expr: '$$record matches "^[^ Z]+ "'
+          - output: parser-containerd
+            expr: '$$record matches "^[^ Z]+Z"'
+      # Parse CRI-O format
+      - type: regex_parser
+        id: parser-crio
+        regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout_type: gotime
+          layout: '2006-01-02T15:04:05.000000000-07:00'
+      # Parse CRI-Containerd format
+      - type: regex_parser
+        id: parser-containerd
+        regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # Parse Docker format
+      - type: json_parser
+        id: parser-docker
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # Extract metadata from file path
+      - type: regex_parser
+        id: extract_metadata_from_filepath
+        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
+        parse_from: $$attributes.file_path
+      # Move out attributes to Attributes
+      - type: metadata
+        attributes:
+          stream: 'EXPR($.stream)'
+          k8s.container.name: 'EXPR($.container_name)'
+          k8s.namespace.name: 'EXPR($.namespace)'
+          k8s.pod.name: 'EXPR($.pod_name)'
+          run_id: 'EXPR($.run_id)'
+          k8s.pod.uid: 'EXPR($.uid)'
+      # Clean up log record
+      - type: restructure
+        id: clean-up-log-record
+        ops:
+          - remove: logtag
+          - remove: stream
+          - remove: container_name
+          - remove: namespace
+          - remove: pod_name
+          - remove: run_id
+          - remove: uid
+exporters:
+  logging:
+    loglevel: debug
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [logging]
diff --git a/examples/kubernetes/varlogpods/containerd_logs-0_000011112222333344445555666677778888/logs/0.log b/examples/kubernetes/varlogpods/containerd_logs-0_000011112222333344445555666677778888/logs/0.log
new file mode 100644
index 000000000000..4b108cd74770
--- /dev/null
+++ b/examples/kubernetes/varlogpods/containerd_logs-0_000011112222333344445555666677778888/logs/0.log
@@ -0,0 +1,7 @@
+2021-02-16T09:21:15.518430714Z stdout F example: 10 Tue Feb 16 09:21:15 UTC 2021
+2021-02-16T09:21:16.519721603Z stdout F example: 11 Tue Feb 16 09:21:16 UTC 2021
+2021-02-16T09:21:17.521307236Z stdout F example: 12 Tue Feb 16 09:21:17 UTC 2021
+2021-02-16T09:21:18.522445945Z stdout F example: 13 (part of log) Tue Feb 16 09:21:18 UTC 2021
+2021-02-16T09:21:18.522445945Z stdout F example: 13 (end of log) Tue Feb 16 09:21:18 UTC 2021
+2021-02-16T09:21:19.523759881Z stdout F example: 14 Tue Feb 16 09:21:19 UTC 2021
+2021-02-16T09:21:20.545525544Z stdout F example: 15 Tue Feb 16 09:21:20 UTC 2021
diff --git a/examples/kubernetes/varlogpods/crio_logs-0_111122223333444455556666777788889999/logs/0.log b/examples/kubernetes/varlogpods/crio_logs-0_111122223333444455556666777788889999/logs/0.log
new file mode 100644
index 000000000000..e7dc8de5f061
--- /dev/null
+++ b/examples/kubernetes/varlogpods/crio_logs-0_111122223333444455556666777788889999/logs/0.log
@@ -0,0 +1,6 @@
+2021-02-16T08:59:31.252009327+00:00 stdout F example: 11 Tue Feb 16 08:59:31 UTC 2021
+2021-02-16T08:59:32.253038336+00:00 stdout F example: 12 Tue Feb 16 08:59:32 UTC 2021
+2021-02-16T08:59:33.254364766+00:00 stdout P example: 13 (part of log) Tue Feb 16 08:59:33 UTC 2021
+2021-02-16T08:59:33.254364766+00:00 stdout F example: 13 (end of log) Tue Feb 16 08:59:33 UTC 2021
+2021-02-16T08:59:34.255644946+00:00 stdout F example: 14 Tue Feb 16 08:59:34 UTC 2021
+2021-02-16T08:59:35.261801788+00:00 stdout F example: 15 Tue Feb 16 08:59:35 UTC 2021
diff --git a/examples/kubernetes/varlogpods/docker_logs-0_222233334444555566667777888899990000/logs/0.log b/examples/kubernetes/varlogpods/docker_logs-0_222233334444555566667777888899990000/logs/0.log
new file mode 100644
index 000000000000..4cffc5e881fc
--- /dev/null
+++ b/examples/kubernetes/varlogpods/docker_logs-0_222233334444555566667777888899990000/logs/0.log
@@ -0,0 +1,7 @@
+{"log":"example: 12 Tue Feb 16 09:15:12 UTC 2021\n","stream":"stdout","time":"2021-02-16T09:15:12.50286486Z"}
+{"log":"example: 13 (part of log) Tue Feb 16 09:15:13 UTC 2021","stream":"stdout","time":"2021-02-16T09:15:13.503624173Z"}
+{"log":"example: 13 (end of log) Tue Feb 16 09:15:13 UTC 2021\n","stream":"stdout","time":"2021-02-16T09:15:13.503624173Z"}
+{"log":"example: 14 Tue Feb 16 09:15:14 UTC 2021\n","stream":"stdout","time":"2021-02-16T09:15:14.504446654Z"}
+{"log":"example: 15 Tue Feb 16 09:15:15 UTC 2021\n","stream":"stdout","time":"2021-02-16T09:15:15.505407523Z"}
+{"log":"example: 16 Tue Feb 16 09:15:16 UTC 2021\n","stream":"stdout","time":"2021-02-16T09:15:16.51048724Z"}
+{"log":"example: 17 Tue Feb 16 09:15:17 UTC 2021\n","stream":"stdout","time":"2021-02-16T09:15:17.511829776Z"}

From 90c9a13f2b0de4f1e61c425194f225dc6914cdf7 Mon Sep 17 00:00:00 2001
From: Dominik Rosiek <drosiek@sumologic.com>
Date: Mon, 8 Mar 2021 14:00:48 +0100
Subject: [PATCH 2/4] example: kubernetes: add result of performance test to
 README

Signed-off-by: Dominik Rosiek <drosiek@sumologic.com>
---
 examples/kubernetes/README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md
index cadc166d72f5..d7e8c87ff39d 100644
--- a/examples/kubernetes/README.md
+++ b/examples/kubernetes/README.md
@@ -17,6 +17,20 @@ This directory is mounted to standard location (`/var/log/pods`).
 
 `otel-collector-config` is a configuration to autodetect and parse logs for all of three mentioned formats
 
+## Performance Tests
+
+There are multiple tests for various configurations in [`testbed`](../../testbed/tests/log_test.go).
+Following table shows result of example run:
+
+Test                                    |Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Items|Received Items|
+----------------------------------------|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:|
+Log10kDPS/OTLP                          |PASS  |     15s|    15.2|    15.7|         69|         73|    149900|        149900|
+Log10kDPS/filelog                       |PASS  |     15s|    16.5|    18.0|         61|         74|    150000|        150000|
+Log10kDPS/kubernetes_containers         |PASS  |     15s|    42.3|    44.0|         66|         80|    150000|        150000|
+Log10kDPS/k8s_CRI-Containerd            |PASS  |     15s|    36.7|    38.0|         64|         78|    150000|        150000|
+Log10kDPS/k8s_CRI-Containerd_no_attr_ops|PASS  |     15s|    28.9|    29.7|         64|         77|    150000|        150000|
+Log10kDPS/CRI-Containerd                |PASS  |     15s|    19.0|    21.0|         63|         77|    150000|        150000|
+
 ## ToDo
 
 To cover kubernetes system logs, logs from journald should be supported as well.

From 21c5fb86d85667bf4b5b15f336d67d82a6ba28f8 Mon Sep 17 00:00:00 2001
From: Dominik Rosiek <drosiek@sumologic.com>
Date: Wed, 10 Mar 2021 08:58:11 +0100
Subject: [PATCH 3/4] example: kubernetes: add example daemonset

Signed-off-by: Dominik Rosiek <drosiek@sumologic.com>
---
 examples/kubernetes/README.md           |   8 +-
 examples/kubernetes/otel-collector.yaml | 153 ++++++++++++++++++++++++
 2 files changed, 160 insertions(+), 1 deletion(-)
 create mode 100644 examples/kubernetes/otel-collector.yaml

diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md
index d7e8c87ff39d..7b8e3f3a6804 100644
--- a/examples/kubernetes/README.md
+++ b/examples/kubernetes/README.md
@@ -4,12 +4,18 @@ This demo is a sample app to build the collector and exercise its kubernetes log
 
 ## Build and Run
 
+### Kubernetes
+
+Switch to this directory and run following command: `kubectl apply -n <namespace> -f otel-collector.yaml`
+
+### Docker Compose
+
 Two steps are required to build and run the demo:
 
 1. Build latest docker image in main repository directory `make docker-otelcontribcol`
 1. Switch to this directory and run `docker-compose up`
 
-## Description
+#### Description
 
 `varlogpods` contains example log files placed in kubernetes-like directory structure.
 Each of the directory has different formatted logs in one of three formats (either `CRI-O`, `CRI-Containerd` or `Docker`).
diff --git a/examples/kubernetes/otel-collector.yaml b/examples/kubernetes/otel-collector.yaml
new file mode 100644
index 000000000000..eb18509d0870
--- /dev/null
+++ b/examples/kubernetes/otel-collector.yaml
@@ -0,0 +1,153 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-config
+data:
+  config.yaml: |-
+    receivers:
+      filelog:
+        include:
+          # - /var/log/pods/namespace_pod-name_pod-uid/container-name/*.log
+          - /var/log/pods/*/*/*.log
+        exclude:
+          # - /var/log/pods/namespace_pod-name_pod-uid/container-name/*.log
+          - /var/log/pods/*/otel-collector/*.log
+        start_at: beginning
+        include_file_path: true
+        include_file_name: false
+        operators:
+          # Find out which format is used by kubernetes
+          - type: router
+            id: get-format
+            routes:
+              - output: parser-docker
+                expr: '$$record matches "^\\{"'
+              - output: parser-crio
+                expr: '$$record matches "^[^ Z]+ "'
+              - output: parser-containerd
+                expr: '$$record matches "^[^ Z]+Z"'
+          # Parse CRI-O format
+          - type: regex_parser
+            id: parser-crio
+            regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+            output: extract_metadata_from_filepath
+            timestamp:
+              parse_from: time
+              layout_type: gotime
+              layout: '2006-01-02T15:04:05.000000000-07:00'
+          # Parse CRI-Containerd format
+          - type: regex_parser
+            id: parser-containerd
+            regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+            output: extract_metadata_from_filepath
+            timestamp:
+              parse_from: time
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+          # Parse Docker format
+          - type: json_parser
+            id: parser-docker
+            output: extract_metadata_from_filepath
+            timestamp:
+              parse_from: time
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+          # Extract metadata from file path
+          - type: regex_parser
+            id: extract_metadata_from_filepath
+            regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
+            parse_from: $$attributes.file_path
+          # Move out attributes to Attributes
+          - type: metadata
+            attributes:
+              stream: 'EXPR($.stream)'
+              k8s.container.name: 'EXPR($.container_name)'
+              k8s.namespace.name: 'EXPR($.namespace)'
+              k8s.pod.name: 'EXPR($.pod_name)'
+              run_id: 'EXPR($.run_id)'
+              k8s.pod.uid: 'EXPR($.uid)'
+          # Clean up log record
+          - type: restructure
+            id: clean-up-log-record
+            ops:
+              - remove: logtag
+              - remove: stream
+              - remove: container_name
+              - remove: namespace
+              - remove: pod_name
+              - remove: run_id
+              - remove: uid
+    exporters:
+      logging:
+        loglevel: debug
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog]
+          exporters: [logging]
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: otel-collector
+  labels:
+    app: opentelemetry
+    component: otel-collector
+spec:
+  selector:
+    matchLabels:
+      app: opentelemetry
+      component: otel-collector
+  template:
+    metadata:
+      labels:
+        app: opentelemetry
+        component: otel-collector
+    spec:
+      containers:
+      - name: otel-collector
+        image: otel/opentelemetry-collector-contrib:0.22.0
+        resources:
+          limits:
+            cpu: 100m
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - mountPath: /var/log
+          name: varlog
+          readOnly: true
+        - mountPath: /var/lib/docker/containers
+          name: varlibdockercontainers
+          readOnly: true
+        - mountPath: /etc/otel/config.yaml
+          name: data
+          subPath: config.yaml
+          readOnly: true
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: data
+        configMap:
+          name: otel-collector-config
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  labels:
+    app: opentelemetry
+    component: otel-collector
+spec:
+  ports:
+  - name: metrics # Default endpoint for querying metrics.
+    port: 8888
+  selector:
+    component: otel-collector

From b85326ccb0c763b8cfdb1efbedbea333daf7cebd Mon Sep 17 00:00:00 2001
From: Dominik Rosiek <drosiek@sumologic.com>
Date: Mon, 15 Mar 2021 07:43:23 +0100
Subject: [PATCH 4/4] examples: kubernetes: add more explanation about
 inclusion/exclusion logs

Signed-off-by: Dominik Rosiek <drosiek@sumologic.com>
---
 examples/kubernetes/README.md                   | 17 +++++++++++++++++
 examples/kubernetes/otel-collector-config.yml   |  3 +--
 examples/kubernetes/otel-collector.yaml         |  3 +--
 .../otel-collector/0.log                        |  7 +++++++
 4 files changed, 26 insertions(+), 4 deletions(-)
 create mode 100644 examples/kubernetes/varlogpods/otel_otel_888877776666555544443333222211110000/otel-collector/0.log

diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md
index 7b8e3f3a6804..3aba4248c4f1 100644
--- a/examples/kubernetes/README.md
+++ b/examples/kubernetes/README.md
@@ -8,6 +8,23 @@ This demo is a sample app to build the collector and exercise its kubernetes log
 
 Switch to this directory and run following command: `kubectl apply -n <namespace> -f otel-collector.yaml`
 
+#### Include/Exclude Specific Logs
+
+Kubernetes logs are being stored in `/var/log/pods`.
+Path to container logs is constructed using following pattern:
+
+`/var/log/pods/<namespace>_<pod_name>_<pod_uid>/<container>/<run_id>.log`
+
+You can use it to manage from which containers do you want to include and exclude logs.
+
+For example, to include all logs from `default` namespace, you can use following configuration:
+
+```yaml
+include:
+    - /var/log/pods/default_*/*/*.log
+exclude: []
+```
+
 ### Docker Compose
 
 Two steps are required to build and run the demo:
diff --git a/examples/kubernetes/otel-collector-config.yml b/examples/kubernetes/otel-collector-config.yml
index 3c10d55b8a84..e40b27ff973a 100644
--- a/examples/kubernetes/otel-collector-config.yml
+++ b/examples/kubernetes/otel-collector-config.yml
@@ -1,10 +1,9 @@
 receivers:
   filelog:
     include:
-      # - /var/log/pods/namespace_pod-name_pod-uid/container-name/*.log
       - /var/log/pods/*/*/*.log
     exclude:
-      # - /var/log/pods/namespace_pod-name_pod-uid/container-name/*.log
+      # Exclude logs from all containers named otel-collector
       - /var/log/pods/*/otel-collector/*.log
     start_at: beginning
     include_file_path: true
diff --git a/examples/kubernetes/otel-collector.yaml b/examples/kubernetes/otel-collector.yaml
index eb18509d0870..696935832c54 100644
--- a/examples/kubernetes/otel-collector.yaml
+++ b/examples/kubernetes/otel-collector.yaml
@@ -8,10 +8,9 @@ data:
     receivers:
       filelog:
         include:
-          # - /var/log/pods/namespace_pod-name_pod-uid/container-name/*.log
           - /var/log/pods/*/*/*.log
         exclude:
-          # - /var/log/pods/namespace_pod-name_pod-uid/container-name/*.log
+          # Exclude logs from all containers named otel-collector
           - /var/log/pods/*/otel-collector/*.log
         start_at: beginning
         include_file_path: true
diff --git a/examples/kubernetes/varlogpods/otel_otel_888877776666555544443333222211110000/otel-collector/0.log b/examples/kubernetes/varlogpods/otel_otel_888877776666555544443333222211110000/otel-collector/0.log
new file mode 100644
index 000000000000..df3129cf0a7a
--- /dev/null
+++ b/examples/kubernetes/varlogpods/otel_otel_888877776666555544443333222211110000/otel-collector/0.log
@@ -0,0 +1,7 @@
+2021-02-16T09:21:15.518430714Z stdout F otel-collector: 10 Tue Feb 16 09:21:15 UTC 2021
+2021-02-16T09:21:16.519721603Z stdout F otel-collector: 11 Tue Feb 16 09:21:16 UTC 2021
+2021-02-16T09:21:17.521307236Z stdout F otel-collector: 12 Tue Feb 16 09:21:17 UTC 2021
+2021-02-16T09:21:18.522445945Z stdout F otel-collector: 13 (part of log) Tue Feb 16 09:21:18 UTC 2021
+2021-02-16T09:21:18.522445945Z stdout F otel-collector: 13 (end of log) Tue Feb 16 09:21:18 UTC 2021
+2021-02-16T09:21:19.523759881Z stdout F otel-collector: 14 Tue Feb 16 09:21:19 UTC 2021
+2021-02-16T09:21:20.545525544Z stdout F otel-collector: 15 Tue Feb 16 09:21:20 UTC 2021