Add thanos to hub quickstart cluster & remote write from all clusters

config/observability/grafana_datasources.yaml

@@ -4,10 +4,10 @@
         {
             "access": "proxy",
             "editable": false,
-            "name": "prometheus",
+            "name": "Kuadrant-Thanos-Hub",
             "orgId": 1,
             "type": "prometheus",
-            "url": "http://prometheus-k8s.monitoring.svc:9090",
+            "url": "http://thanos-query.monitoring.svc:9090",
             "version": 1
         }
     ]

config/thanos/kustomization.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: monitoring
+
+resources:
+  - ./namespace.yaml
+  - ./manifests/thanos-query-deployment.yaml
+  - ./manifests/thanos-query-service.yaml
+  - ./manifests/thanos-query-serviceAccount.yaml
+  - ./manifests/thanos-receive-ingestor-default-service.yaml
+  - ./manifests/thanos-receive-ingestor-default-statefulSet.yaml
+  - ./manifests/thanos-receive-ingestor-serviceAccount.yaml
+  - ./manifests/thanos-receive-router-configmap.yaml
+  - ./manifests/thanos-receive-router-deployment.yaml
+  - ./manifests/thanos-receive-router-service.yaml
+  - ./manifests/thanos-receive-router-serviceAccount.yaml
+  - ./thanos-receive-router-service-loadbalancer.yaml

config/thanos/manifests/thanos-query-deployment.yaml

@@ -0,0 +1,86 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-query
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: v0.29.0
+  name: thanos-query
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: query-layer
+      app.kubernetes.io/instance: thanos-query
+      app.kubernetes.io/name: thanos-query
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: query-layer
+        app.kubernetes.io/instance: thanos-query
+        app.kubernetes.io/name: thanos-query
+        app.kubernetes.io/version: v0.29.0
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-query
+              namespaces:
+              - monitoring
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      containers:
+      - args:
+        - query
+        - --grpc-address=0.0.0.0:10901
+        - --http-address=0.0.0.0:9090
+        - --log.level=info
+        - --log.format=logfmt
+        - --query.replica-label=prometheus_replica
+        - --query.replica-label=rule_replica
+        - --endpoint=dnssrv+_grpc._tcp.thanos-receive-ingestor-default.monitoring.svc.cluster.local:10901
+        - --query.auto-downsampling
+        env:
+        - name: HOST_IP_ADDRESS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        image: quay.io/thanos/thanos:v0.29.0
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 9090
+            scheme: HTTP
+          periodSeconds: 30
+        name: thanos-query
+        ports:
+        - containerPort: 10901
+          name: grpc
+        - containerPort: 9090
+          name: http
+        readinessProbe:
+          failureThreshold: 20
+          httpGet:
+            path: /-/ready
+            port: 9090
+            scheme: HTTP
+          periodSeconds: 5
+        resources: {}
+        terminationMessagePolicy: FallbackToLogsOnError
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        fsGroup: 65534
+        runAsUser: 65534
+      serviceAccountName: thanos-query
+      terminationGracePeriodSeconds: 120

config/thanos/manifests/thanos-query-service.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-query
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: v0.29.0
+  name: thanos-query
+  namespace: monitoring
+spec:
+  ports:
+  - name: grpc
+    port: 10901
+    targetPort: 10901
+  - name: http
+    port: 9090
+    targetPort: 9090
+  selector:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-query
+    app.kubernetes.io/name: thanos-query

config/thanos/manifests/thanos-query-serviceAccount.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-query
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: v0.29.0
+  name: thanos-query
+  namespace: monitoring

config/thanos/manifests/thanos-query-serviceMonitor.yaml

@@ -0,0 +1,25 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-query
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: v0.29.0
+  name: thanos-query
+  namespace: monitoring
+spec:
+  endpoints:
+  - port: http
+    relabelings:
+    - action: replace
+      separator: /
+      sourceLabels:
+      - namespace
+      - pod
+      targetLabel: instance
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: query-layer
+      app.kubernetes.io/instance: thanos-query
+      app.kubernetes.io/name: thanos-query

config/thanos/manifests/thanos-receive-ingestor-default-service.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: database-write-hashring
+    app.kubernetes.io/instance: thanos-receive-ingestor-default
+    app.kubernetes.io/name: thanos-receive
+    app.kubernetes.io/version: v0.29.0
+    controller.receive.thanos.io/hashring: default
+  name: thanos-receive-ingestor-default
+  namespace: monitoring
+spec:
+  clusterIP: None
+  ports:
+  - name: grpc
+    port: 10901
+    targetPort: 10901
+  - name: http
+    port: 10902
+    targetPort: 10902
+  - name: remote-write
+    port: 19291
+    targetPort: 19291
+  selector:
+    app.kubernetes.io/component: database-write-hashring
+    app.kubernetes.io/instance: thanos-receive-ingestor-default
+    app.kubernetes.io/name: thanos-receive
+    controller.receive.thanos.io/hashring: default

config/thanos/manifests/thanos-receive-ingestor-default-statefulSet.yaml

@@ -0,0 +1,149 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app.kubernetes.io/component: database-write-hashring
+    app.kubernetes.io/instance: thanos-receive-ingestor-default
+    app.kubernetes.io/name: thanos-receive
+    app.kubernetes.io/version: v0.29.0
+    controller.receive.thanos.io: thanos-receive-controller
+    controller.receive.thanos.io/hashring: default
+  name: thanos-receive-ingestor-default
+  namespace: monitoring
+spec:
+  minReadySeconds: 0
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: database-write-hashring
+      app.kubernetes.io/instance: thanos-receive-ingestor-default
+      app.kubernetes.io/name: thanos-receive
+      controller.receive.thanos.io/hashring: default
+  serviceName: thanos-receive-ingestor-default
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: database-write-hashring
+        app.kubernetes.io/instance: thanos-receive-ingestor-default
+        app.kubernetes.io/name: thanos-receive
+        app.kubernetes.io/version: v0.29.0
+        controller.receive.thanos.io/hashring: default
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-receive
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - thanos-receive-ingestor-default
+              namespaces:
+              - monitoring
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-receive
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - thanos-receive-ingestor-default
+              namespaces:
+              - monitoring
+              topologyKey: topology.kubernetes.io/zone
+            weight: 100
+      containers:
+      - args:
+        - receive
+        - --log.level=info
+        - --log.format=logfmt
+        - --grpc-address=0.0.0.0:10901
+        - --http-address=0.0.0.0:10902
+        - --remote-write.address=0.0.0.0:19291
+        - --receive.replication-factor=1
+        - --tsdb.path=/var/thanos/receive
+        - --tsdb.retention=15d
+        - --label=replica="$(NAME)"
+        - --label=receive="true"
+        - --receive.local-endpoint=$(NAME).thanos-receive-ingestor-default.$(NAMESPACE).svc.cluster.local:10901
+        - --receive.hashrings-file=/var/lib/thanos-receive/hashrings.json
+        env:
+        - name: NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: HOST_IP_ADDRESS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        image: quay.io/thanos/thanos:v0.29.0
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 8
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
+        name: thanos-receive
+        ports:
+        - containerPort: 10901
+          name: grpc
+        - containerPort: 10902
+          name: http
+        - containerPort: 19291
+          name: remote-write
+        readinessProbe:
+          failureThreshold: 20
+          httpGet:
+            path: /-/ready
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 5
+        resources: {}
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /var/thanos/receive
+          name: data
+          readOnly: false
+        - mountPath: /var/lib/thanos-receive
+          name: hashring-config
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        fsGroup: 65534
+        runAsUser: 65534
+      serviceAccountName: thanos-receive-ingestor
+      terminationGracePeriodSeconds: 900
+      volumes:
+      - configMap:
+          name: hashring-config
+        name: hashring-config
+  volumeClaimTemplates:
+  - metadata:
+      labels:
+        app.kubernetes.io/component: database-write-hashring
+        app.kubernetes.io/instance: thanos-receive-ingestor-default
+        app.kubernetes.io/name: thanos-receive
+        controller.receive.thanos.io/hashring: default
+      name: data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Gi

config/thanos/manifests/thanos-receive-ingestor-serviceAccount.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/component: database-write-hashring
+    app.kubernetes.io/instance: thanos-receive-ingestor
+    app.kubernetes.io/name: thanos-receive
+    app.kubernetes.io/version: v0.29.0
+  name: thanos-receive-ingestor
+  namespace: monitoring

config/thanos/manifests/thanos-receive-ingestor-serviceMonitor.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: database-write-hashring
+    app.kubernetes.io/instance: thanos-receive-ingestor
+    app.kubernetes.io/name: thanos-receive
+    app.kubernetes.io/version: v0.29.0
+  name: thanos-receive-ingestor
+  namespace: monitoring
+spec:
+  endpoints:
+  - port: http
+    relabelings:
+    - action: replace
+      separator: /
+      sourceLabels:
+      - namespace
+      - pod
+      targetLabel: instance
+    - action: replace
+      sourceLabels:
+      - __meta_kubernetes_service_label_controller_receive_thanos_io_shard
+      targetLabel: hashring
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: database-write-hashring
+      app.kubernetes.io/name: thanos-receive