Merge pull request #99 from openinfradev/loki_apply_tomsa

efk: remove efk on aws-msa-reference

aws-msa-reference/lma/site-values.yaml

@@ -25,8 +25,6 @@ charts:
   override:
     prometheusOperator.nodeSelector: $(nodeSelector)
 
-- name: eck-operator
-
 - name: prometheus
   override:
     prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName: $(storageClassName)
@@ -59,31 +57,6 @@ charts:
     conf.processes: dockerd,kubelet,kube-proxy,ntpd,node
     pod.hostNetwork: false
 
-- name: eck-resource
-  override:
-    kibana.nodeSelector: $(nodeSelector)
-    kibana.server.basePath: /kibana
-    kibana.readinessPath: /kibana/login
-
-    elasticsearch.nodeSets.master.nodeSelector: $(nodeSelector)
-    elasticsearch.nodeSets.master.count: 3
-    elasticsearch.nodeSets.master.javaOpts: "-Xms2g -Xmx2g"
-    elasticsearch.nodeSets.master.limitCpu: 2
-    elasticsearch.nodeSets.master.limitMem: 4Gi
-    elasticsearch.nodeSets.master.pvc.storageClassName: $(storageClassName)
-    elasticsearch.nodeSets.master.pvc.size: 10Gi
-
-    elasticsearch.nodeSets.hotdata.nodeSelector: $(nodeSelector)
-    elasticsearch.nodeSets.hotdata.count: 3
-    elasticsearch.nodeSets.hotdata.javaOpts: "-Xms2g -Xmx2g"
-    elasticsearch.nodeSets.hotdata.limitCpu: 2
-    elasticsearch.nodeSets.hotdata.limitMem: 4Gi
-    elasticsearch.nodeSets.hotdata.pvc.storageClassName: $(storageClassName)
-    elasticsearch.nodeSets.hotdata.pvc.size: 500Gi
-
-    elasticsearch.nodeSets.client.enabled: false
-
-
 - name: grafana
   override:
     adminPassword: password
@@ -124,115 +97,32 @@ charts:
     fluentbit:
       clusterName: $(clusterName)
       outputs:
-        es:
-        - name: taco-es
-          host: eck-elasticsearch-es-http
-          port: 9200
-          dedicatedUser: 
-            username: taco-fluentbit
-            password: password
-            elasticPasswordSecret: eck-elasticsearch-es-elastic-user
-          template:
-            enabled: true
-            ilms:
-            - name: hot-delete-14days
-              json:
-                policy:
-                  phases:
-                    hot:
-                      actions:
-                        rollover:
-                          max_size: 30gb
-                          max_age: 1d
-                          max_docs: 5000000000
-                        set_priority:
-                          priority: 100
-                    delete:
-                      min_age: 14d
-                      actions:
-                        delete: {}
-            - name: hot-delete-7days
-              json:
-                policy:
-                  phases:
-                    hot:
-                      actions:
-                        rollover:
-                          max_size: 30gb
-                          max_age: 1d
-                          max_docs: 5000000000
-                        set_priority:
-                          priority: 100
-                    delete:
-                      min_age: 7d
-                      actions:
-                        delete: {}
-            - name: hot-delete-3hour
-              json:
-                policy:
-                  phases:
-                    hot:
-                      actions:
-                        rollover:
-                          max_size: 30gb
-                          max_age: 1h
-                          max_docs: 5000000000
-                        set_priority:
-                          priority: 100
-                    delete:
-                      min_age: 3h
-                      actions:
-                        delete: {}
-            templates:
-            - name: platform
-              json:
-                index_patterns: "platform*"
-                settings:
-                  refresh_interval: 30s
-                  number_of_shards: 3
-                  number_of_replicas: 1
-                  index.lifecycle.name: hot-delete-14days
-                  index.lifecycle.rollover_alias: platform
-            - name: application
-              json:
-                index_patterns: "container*"
-                settings:
-                  refresh_interval: 30s
-                  number_of_shards: 3
-                  number_of_replicas: 1
-                  index.lifecycle.name: hot-delete-3hour
-                  index.lifecycle.rollover_alias: container
-            - name: syslog
-              json:
-                index_patterns: "syslog*"
-                settings:
-                  refresh_interval: 30s
-                  number_of_shards: 2
-                  number_of_replicas: 1
-                  index.lifecycle.name: hot-delete-14days
-                  index.lifecycle.rollover_alias: syslog
+        loki: 
+        - name: taco-loki
+          host: loki
+          port: 3100
       targetLogs: 
       - tag: kube.*
         bufferChunkSize: 2M
         bufferMaxSize: 5M
         do_not_store_as_default: false
-        es_name: taco-es
         index: container
+        loki_name: taco-loki
         memBufLimit: 20MB
         multi_index:
         - index: platform
-          es_name: taco-es
+          loki_name: taco-loki
           key: $kubernetes['namespace_name']
           value: kube-system|taco-system|lma|argo
         parser: docker
         path: /var/log/containers/*.log
-        type: fluent
+        type: kubernates
         extraArgs:
           multilineParser: docker, cri
       - tag: syslog.*
-        es_name: taco-es
+        loki_name: taco-loki
         index: syslog
-        parser: syslog-rfc5424
+        parser: taco-syslog-parser-for-ubuntu
         path: /var/log/syslog
         type: syslog
 
@@ -244,8 +134,6 @@ charts:
       interval: $(serviceScrapeInterval)
     serviceMonitor.kubelet.interval: 30s
     serviceMonitor.additionalScrapeConfigs:
-    metricbeat.enabled: false
-    kibanaInit.url: http://eck-kibana-dashboard-kb-http.lma.svc.$(clusterName):5601
     grafanaDashboard.istio.enabled: false
     grafanaDashboard.jaeger.enabled: false
     serviceMonitor.istio.enabled: false
@@ -260,16 +148,19 @@ charts:
 - name: kubernetes-event-exporter
   override:
     conf.recievers:
-    - name: default
-      type: elasticsearch
-      config:
-        hosts:
-        - "https://eck-elasticsearch-es-http.lma.svc.siim-dev:9200"
-        index: kube-events
-        username: elastic
-        password: tacoword
-        tls: # optional, advanced options for tls
-          insecureSkipVerify: true
+      - name: loki
+        type: file
+        config:
+          path: "/tmp/kubernetes-event.log"
+    addons:
+      loki: 
+        enabled: true
+        host: loki
+        port: 3100      
+        target_file: "/tmp/kubernetes-event.log"
+
+    conf.default.hosts:
+    - "https://eck-elasticsearch-es-http.lma.svc.$(clusterName):9200"
 
 - name: thanos
   override:

aws-reference/lma/site-values.yaml

@@ -158,8 +158,7 @@ charts:
         host: loki
         port: 3100      
         target_file: "/tmp/kubernetes-event.log"
-
-
+
     conf.default.hosts:
     - "https://eck-elasticsearch-es-http.lma.svc.$(clusterName):9200"