From e5c94eaceb698ee47a516691f17584e165a035e8 Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 22:19:59 +0000 Subject: [PATCH 01/10] feat add headlamp --- .../headlamp/app/helmrelease.yaml | 67 +++++++++++++++++++ .../headlamp/app/kustomization.yaml | 8 +++ .../main/apps/observability/headlamp/ks.yaml | 20 ++++++ kubernetes/shared/repos/helm/headlamp.yaml | 9 +++ .../shared/repos/helm/kustomization.yaml | 1 + 5 files changed, 105 insertions(+) create mode 100644 kubernetes/main/apps/observability/headlamp/app/helmrelease.yaml create mode 100644 kubernetes/main/apps/observability/headlamp/app/kustomization.yaml create mode 100644 kubernetes/main/apps/observability/headlamp/ks.yaml create mode 100644 kubernetes/shared/repos/helm/headlamp.yaml diff --git a/kubernetes/main/apps/observability/headlamp/app/helmrelease.yaml b/kubernetes/main/apps/observability/headlamp/app/helmrelease.yaml new file mode 100644 index 0000000000..4aea501943 --- /dev/null +++ b/kubernetes/main/apps/observability/headlamp/app/helmrelease.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: headlamp +spec: + interval: 30m + chart: + spec: + chart: headlamp + version: 0.26.0 + sourceRef: + kind: HelmRepository + name: headlamp + namespace: flux-system + driftDetection: + mode: enabled + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + strategy: rollback + retries: 3 + values: + fullnameOverride: headlamp + initContainers: + - image: ghcr.io/headlamp-k8s/headlamp-plugin-flux:v0.1.0-beta-1@sha256:5274c581d69b36af6899dfccfc73adeda414e0ae6046f009db8ed3c4271c6add + command: + - /bin/sh + - -c + - mkdir -p /build/plugins && cp -r /plugins/* /build/plugins/ + name: headlamp-plugins + volumeMounts: + - mountPath: /build/plugins + name: headlamp-plugins + config: + oidc: + secret: + create: false + externalSecret: + enabled: true + name: headlamp-oidc + pluginsDir: /build/plugins + podAnnotations: + reloader.stakater.com/auto: "true" + volumeMounts: + - mountPath: /build/plugins + name: headlamp-plugins + volumes: + - name: headlamp-plugins + persistentVolumeClaim: + claimName: headlamp + ingress: + enabled: true + ingressClassName: internal + hosts: + - host: headlamp.rafaribe.com + paths: + - path: / + type: Prefix + backend: + service: + name: headlamp + port: + number: 80 diff --git a/kubernetes/main/apps/observability/headlamp/app/kustomization.yaml b/kubernetes/main/apps/observability/headlamp/app/kustomization.yaml new file mode 100644 index 0000000000..80c88e043a --- /dev/null +++ b/kubernetes/main/apps/observability/headlamp/app/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability +resources: + - ./helmrelease.yaml + - ../../../../../shared/templates/volsync + - ../../../../../shared/templates/gatus/guarded diff --git a/kubernetes/main/apps/observability/headlamp/ks.yaml b/kubernetes/main/apps/observability/headlamp/ks.yaml new file mode 100644 index 0000000000..a1d5147536 --- /dev/null +++ b/kubernetes/main/apps/observability/headlamp/ks.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app headlamp + namespace: flux-system +spec: + targetNamespace: observability + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/main/apps/observability/headlamp/app + prune: true + sourceRef: + kind: GitRepository + name: home-ops + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/shared/repos/helm/headlamp.yaml b/kubernetes/shared/repos/helm/headlamp.yaml new file mode 100644 index 0000000000..1e29a44c4c --- /dev/null +++ b/kubernetes/shared/repos/helm/headlamp.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: headlamp + namespace: flux-system +spec: + interval: 2h + url: https://headlamp-k8s.github.io/headlamp/ diff --git a/kubernetes/shared/repos/helm/kustomization.yaml b/kubernetes/shared/repos/helm/kustomization.yaml index a86b3ba7c6..7fc4bc7894 100644 --- a/kubernetes/shared/repos/helm/kustomization.yaml +++ b/kubernetes/shared/repos/helm/kustomization.yaml @@ -27,6 +27,7 @@ resources: - ./falco.yaml - ./grafana.yaml - ./hajimari.yaml + - ./headlamp.yaml - ./infracloudio.yaml - ./ingress-nginx.yaml - ./intel.yaml From 837d406543de294c0100da19fcfceb92f6ee1ff9 Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 22:21:08 +0000 Subject: [PATCH 02/10] feat: add pihole exporter --- .../apps/observability/exporters/pihole/helmrelease.yaml | 2 +- .../apps/observability/exporters/pihole/kustomization.yaml | 6 ++++++ kubernetes/main/apps/observability/kustomization.yaml | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kubernetes/main/apps/observability/exporters/pihole/helmrelease.yaml b/kubernetes/main/apps/observability/exporters/pihole/helmrelease.yaml index 96503d03a8..815a9214a5 100644 --- a/kubernetes/main/apps/observability/exporters/pihole/helmrelease.yaml +++ b/kubernetes/main/apps/observability/exporters/pihole/helmrelease.yaml @@ -9,7 +9,7 @@ spec: chart: spec: chart: app-template - version: 3.4.0 + version: 3.5.1 sourceRef: kind: HelmRepository name: bjw-s diff --git a/kubernetes/main/apps/observability/exporters/pihole/kustomization.yaml b/kubernetes/main/apps/observability/exporters/pihole/kustomization.yaml index e69de29bb2..2708f09eed 100644 --- a/kubernetes/main/apps/observability/exporters/pihole/kustomization.yaml +++ b/kubernetes/main/apps/observability/exporters/pihole/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./externalsecret.yaml + - ./helmrelease.yaml diff --git a/kubernetes/main/apps/observability/kustomization.yaml b/kubernetes/main/apps/observability/kustomization.yaml index 6deea963df..5e5ba82894 100644 --- a/kubernetes/main/apps/observability/kustomization.yaml +++ b/kubernetes/main/apps/observability/kustomization.yaml @@ -7,8 +7,10 @@ resources: # Flux-Kustomizations - ./alertmanager-discord/ks.yaml - ./alertmanager/ks.yaml + - ./exporters/ks.yaml - ./gatus/ks.yaml - ./grafana/ks.yaml + #- ./headlamp/ks.yaml - ./karma/ks.yaml - ./kepler/ks.yaml - ./kromgo/ks.yaml From c4faa2dc7264bc817844d3f3ebfdf1912b0ac368 Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 22:25:53 +0000 Subject: [PATCH 03/10] feat. add actual budget --- .../apps/services/actual/app/helmrelease.yaml | 86 +++++++++++++++++++ .../services/actual/app/kustomization.yaml | 8 ++ kubernetes/main/apps/services/actual/ks.yaml | 28 ++++++ .../main/apps/services/kustomization.yaml | 1 + 4 files changed, 123 insertions(+) create mode 100644 kubernetes/main/apps/services/actual/app/helmrelease.yaml create mode 100644 kubernetes/main/apps/services/actual/app/kustomization.yaml create mode 100644 kubernetes/main/apps/services/actual/ks.yaml diff --git a/kubernetes/main/apps/services/actual/app/helmrelease.yaml b/kubernetes/main/apps/services/actual/app/helmrelease.yaml new file mode 100644 index 0000000000..e76ea0dd2c --- /dev/null +++ b/kubernetes/main/apps/services/actual/app/helmrelease.yaml @@ -0,0 +1,86 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: actual +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 3.5.1 + interval: 30m + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + + values: + controllers: + actual: + annotations: + reloader.stakater.com/auto: "true" + + pod: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" + + containers: + app: + image: + repository: ghcr.io/actualbudget/actual-server + tag: 24.11.0 + env: + ACTUAL_PORT: &httpPort 5006 + probes: + liveness: + enabled: true + readiness: + enabled: true + custom: true + spec: + httpGet: + path: / + port: *httpPort + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + resources: + requests: + cpu: 12m + memory: 128M + limits: + memory: 512M + + service: + app: + controller: actual + ports: + http: + port: *httpPort + + ingress: + app: + className: "internal" + annotations: + external-dns.alpha.kubernetes.io/target: internal.rafaribe.com + hosts: + - host: actual.rafaribe.com + paths: + - path: / + service: + identifier: app + port: http + + persistence: + data: + existingClaim: actual-data + advancedMounts: + actual: + app: + - path: /data diff --git a/kubernetes/main/apps/services/actual/app/kustomization.yaml b/kubernetes/main/apps/services/actual/app/kustomization.yaml new file mode 100644 index 0000000000..eff822ae3e --- /dev/null +++ b/kubernetes/main/apps/services/actual/app/kustomization.yaml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml + - ../../../../../shared/templates/gatus/guarded + - ../../../../../shared/templates/volsync diff --git a/kubernetes/main/apps/services/actual/ks.yaml b/kubernetes/main/apps/services/actual/ks.yaml new file mode 100644 index 0000000000..432c8213db --- /dev/null +++ b/kubernetes/main/apps/services/actual/ks.yaml @@ -0,0 +1,28 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &appname actual + namespace: flux-system +spec: + targetNamespace: services + commonMetadata: + labels: + app.kubernetes.io/name: *appname + interval: 30m + timeout: 5m + path: "./kubernetes/main/apps/services/actual/app" + prune: true + sourceRef: + kind: GitRepository + name: home-ops + wait: false + dependsOn: + - name: rook-ceph-cluster + - name: volsync + - name: external-secrets-stores + postBuild: + substitute: + APP: *appname + VOLSYNC_CLAIM: actual-data diff --git a/kubernetes/main/apps/services/kustomization.yaml b/kubernetes/main/apps/services/kustomization.yaml index 4bbdfda6e9..be902863f4 100644 --- a/kubernetes/main/apps/services/kustomization.yaml +++ b/kubernetes/main/apps/services/kustomization.yaml @@ -6,6 +6,7 @@ resources: # Pre Flux-Kustomizations - ./namespace.yaml # Flux-Kustomizations + - ./actual/ks.yaml - ./atuin/ks.yaml - ./cyberchef/ks.yaml - ./ferdium/ks.yaml From a3819fd8ab7e592a31d5bbd0cb48eb9db0512aff Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 22:34:57 +0000 Subject: [PATCH 04/10] feat: add searxng --- .../main/apps/services/kustomization.yaml | 1 + .../services/searxng/app/externalsecret.yaml | 21 ++++ .../services/searxng/app/helmrelease.yaml | 110 ++++++++++++++++++ .../services/searxng/app/kustomization.yaml | 14 +++ kubernetes/main/apps/services/searxng/ks.yaml | 0 5 files changed, 146 insertions(+) create mode 100644 kubernetes/main/apps/services/searxng/app/externalsecret.yaml create mode 100644 kubernetes/main/apps/services/searxng/app/helmrelease.yaml create mode 100644 kubernetes/main/apps/services/searxng/app/kustomization.yaml create mode 100644 kubernetes/main/apps/services/searxng/ks.yaml diff --git a/kubernetes/main/apps/services/kustomization.yaml b/kubernetes/main/apps/services/kustomization.yaml index be902863f4..5229acb4d3 100644 --- a/kubernetes/main/apps/services/kustomization.yaml +++ b/kubernetes/main/apps/services/kustomization.yaml @@ -19,6 +19,7 @@ resources: - ./mealie/ks.yaml - ./netboot/ks.yaml - ./paperless/ks.yaml + - ./searxng/ks.yaml - ./smtp-relay/ks.yaml - ./tandoor/ks.yaml - ./thelounge/ks.yaml diff --git a/kubernetes/main/apps/services/searxng/app/externalsecret.yaml b/kubernetes/main/apps/services/searxng/app/externalsecret.yaml new file mode 100644 index 0000000000..2417c47c62 --- /dev/null +++ b/kubernetes/main/apps/services/searxng/app/externalsecret.yaml @@ -0,0 +1,21 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: &app searxng +spec: + refreshInterval: 5m + secretStoreRef: + kind: ClusterSecretStore + name: akeyless-secret-store + target: + name: *app + creationPolicy: Owner + template: + engineVersion: v2 + data: + SEARXNG_SECRET: "{{ .SEARXNG_SECRET_KEY }}" + dataFrom: + - extract: + key: /searxng diff --git a/kubernetes/main/apps/services/searxng/app/helmrelease.yaml b/kubernetes/main/apps/services/searxng/app/helmrelease.yaml new file mode 100644 index 0000000000..7236463aaf --- /dev/null +++ b/kubernetes/main/apps/services/searxng/app/helmrelease.yaml @@ -0,0 +1,110 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: searxng +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 3.5.1 + interval: 30m + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + + values: + controllers: + searxng: + strategy: RollingUpdate + annotations: + reloader.stakater.com/auto: "true" + + containers: + main: + image: + repository: docker.io/searxng/searxng + tag: 2024.6.30-39aaac40d + env: + SEARXNG_BASE_URL: https://search.rafaribe.com + SEARXNG_URL: https://search.rafaribe.com + SEARXNG_PORT: &httpPort 8080 + SEARXNG_REDIS_URL: redis://dragonfly.storage.svc.cluster.local:6379 + envFrom: + - secretRef: + name: searxng-secret + probes: + liveness: &probes + enabled: true + custom: true + spec: + httpGet: + path: /stats + port: 8080 + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + readiness: *probes + resources: + requests: + cpu: 10m + memory: 256Mi + limits: + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + add: + - CHOWN + - SETGID + - SETUID + - DAC_OVERRIDE + + service: + app: + controller: searxng + ports: + http: + port: *httpPort + redis: + controller: redis + ports: + http: + port: 6379 + + ingress: + app: + className: "internal" + annotations: + external-dns.alpha.kubernetes.io/target: internal.rafaribe.com + hosts: + - host: search.rafaribe.com + paths: + - path: / + service: + identifier: app + port: http + + persistence: + config: + type: configMap + name: searxng-configmap + globalMounts: + - path: /etc/searxng/settings.yml + subPath: settings.yml + readOnly: true + - path: /etc/searxng/limiter.toml + subPath: limiter.toml + readOnly: true + tmpfs: + enabled: true + type: emptyDir + globalMounts: + - path: /etc/searxng diff --git a/kubernetes/main/apps/services/searxng/app/kustomization.yaml b/kubernetes/main/apps/services/searxng/app/kustomization.yaml new file mode 100644 index 0000000000..b16819cc58 --- /dev/null +++ b/kubernetes/main/apps/services/searxng/app/kustomization.yaml @@ -0,0 +1,14 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./externalsecret.yaml + - ./helmrelease.yaml +configMapGenerator: + - name: searxng-configmap + files: + - ./resources/limiter.toml + - ./resources/settings.yml +generatorOptions: + disableNameSuffixHash: true diff --git a/kubernetes/main/apps/services/searxng/ks.yaml b/kubernetes/main/apps/services/searxng/ks.yaml new file mode 100644 index 0000000000..e69de29bb2 From 4e8f296f25356c10c567223fffe22d54fb73ca2a Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 22:38:03 +0000 Subject: [PATCH 05/10] feat: add vikunja --- .../main/apps/services/kustomization.yaml | 1 + .../services/vikunja/app/helmrelease.yaml | 92 +++++++++++++++++++ .../services/vikunja/app/kustomization.yaml | 8 ++ kubernetes/main/apps/services/vikunja/ks.yaml | 28 ++++++ 4 files changed, 129 insertions(+) create mode 100644 kubernetes/main/apps/services/vikunja/app/helmrelease.yaml create mode 100644 kubernetes/main/apps/services/vikunja/app/kustomization.yaml create mode 100644 kubernetes/main/apps/services/vikunja/ks.yaml diff --git a/kubernetes/main/apps/services/kustomization.yaml b/kubernetes/main/apps/services/kustomization.yaml index 5229acb4d3..b63de307e8 100644 --- a/kubernetes/main/apps/services/kustomization.yaml +++ b/kubernetes/main/apps/services/kustomization.yaml @@ -24,3 +24,4 @@ resources: - ./tandoor/ks.yaml - ./thelounge/ks.yaml - ./radicale/ks.yaml + - ./vikunja/ks.yaml diff --git a/kubernetes/main/apps/services/vikunja/app/helmrelease.yaml b/kubernetes/main/apps/services/vikunja/app/helmrelease.yaml new file mode 100644 index 0000000000..46c7c7757a --- /dev/null +++ b/kubernetes/main/apps/services/vikunja/app/helmrelease.yaml @@ -0,0 +1,92 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: vikunja +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 3.5.1 + interval: 30m + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + + values: + controllers: + vikunja: + annotations: + reloader.stakater.com/auto: "true" + + pod: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" + + containers: + app: + image: + repository: docker.io/vikunja/vikunja + tag: 0.24.5 + env: + VIKUNJA_DATABASE_TYPE: sqlite + VIKUNJA_DATABASE_PATH: /db/vikunja.db + VIKUNJA_FILES_BASEPATH: /files + VIKUNJA_SERVICE_ENABLECALDAV: false + VIKUNJA_SERVICE_ENABLELINKSHARING: false + VIKUNJA_SERVICE_ENABLEREGISTRATION: false + VIKUNJA_SERVICE_PUBLICURL: https://vikunja.rafaribe.com/ + probes: + liveness: + enabled: true + readiness: + enabled: true + resources: + requests: + cpu: 5m + memory: 192Mi + limits: + memory: 1024Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + service: + app: + controller: vikunja + ports: + http: + port: 3456 + + ingress: + app: + className: "external-nginx" + annotations: + external-dns.alpha.kubernetes.io/target: external.rafaribe.com + hosts: + - host: vikunja.rafaribe.com + paths: + - path: / + service: + identifier: app + port: http + + persistence: + data: + existingClaim: vikunja-data + advancedMounts: + vikunja: + app: + - path: /db + subPath: db + - path: /files + subPath: files diff --git a/kubernetes/main/apps/services/vikunja/app/kustomization.yaml b/kubernetes/main/apps/services/vikunja/app/kustomization.yaml new file mode 100644 index 0000000000..d25bdce87a --- /dev/null +++ b/kubernetes/main/apps/services/vikunja/app/kustomization.yaml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml + - ../../../../../shared/templates/volsync + - ../../../../../shared/templates/gatus/guarded diff --git a/kubernetes/main/apps/services/vikunja/ks.yaml b/kubernetes/main/apps/services/vikunja/ks.yaml new file mode 100644 index 0000000000..d35e28847f --- /dev/null +++ b/kubernetes/main/apps/services/vikunja/ks.yaml @@ -0,0 +1,28 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &appname vikunja + namespace: flux-system +spec: + targetNamespace: services + commonMetadata: + labels: + app.kubernetes.io/name: *appname + interval: 30m + timeout: 5m + path: "./kubernetes/main/apps/services/vikunja/app" + prune: true + sourceRef: + kind: GitRepository + name: home-ops + wait: false + dependsOn: + - name: rook-ceph-cluster + - name: volsync + - name: external-secrets-stores + postBuild: + substitute: + APP: *appname + VOLSYNC_CLAIM: vikunja-data From 62bca90ed5d54f5f38e2f573194cb732c1f2c8b3 Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 22:44:55 +0000 Subject: [PATCH 06/10] fix: emqx --- .../apps/storage/emqx/cluster/cluster.yaml | 11 +- .../storage/emqx/cluster/kustomization.yaml | 6 - .../emqx/cluster/resources/init-mqtt.py | 110 ------------------ kubernetes/main/apps/storage/emqx/ks.yaml | 44 +++---- .../storage/emqx/operator/externalsecret.yaml | 38 ++++-- 5 files changed, 62 insertions(+), 147 deletions(-) delete mode 100644 kubernetes/main/apps/storage/emqx/cluster/resources/init-mqtt.py diff --git a/kubernetes/main/apps/storage/emqx/cluster/cluster.yaml b/kubernetes/main/apps/storage/emqx/cluster/cluster.yaml index e89a516c06..7dca6ed142 100644 --- a/kubernetes/main/apps/storage/emqx/cluster/cluster.yaml +++ b/kubernetes/main/apps/storage/emqx/cluster/cluster.yaml @@ -1,5 +1,5 @@ --- -# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/apps.emqx.io/emqx_v2beta1.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/refs/heads/main/apps.emqx.io/emqx_v2beta1.json apiVersion: apps.emqx.io/v2beta1 kind: EMQX metadata: @@ -24,6 +24,10 @@ spec: type = built_in_database enable = true } + { + type = file + path = "/opt/init-acl" + } ] no_match: "deny" } @@ -41,6 +45,10 @@ spec: mountPath: /opt/init-user.json subPath: init-user.json readOnly: true + - name: init-user + mountPath: /opt/init-acl + subPath: init-acl + readOnly: true extraVolumes: - name: init-user secret: @@ -48,6 +56,7 @@ spec: listenersServiceTemplate: metadata: annotations: + external-dns.alpha.kubernetes.io/hostname: "mqtt.rafaribe.com." lbipam.cilium.io/ips: ${LB_EMQX} spec: type: LoadBalancer diff --git a/kubernetes/main/apps/storage/emqx/cluster/kustomization.yaml b/kubernetes/main/apps/storage/emqx/cluster/kustomization.yaml index d4de36fd16..83d325ddbe 100644 --- a/kubernetes/main/apps/storage/emqx/cluster/kustomization.yaml +++ b/kubernetes/main/apps/storage/emqx/cluster/kustomization.yaml @@ -6,9 +6,3 @@ resources: - ./cluster.yaml - ./ingress.yaml - ./podmonitor.yaml -configMapGenerator: - - name: emqx-init-mqtt-configmap - files: - - init-mqtt.py=./resources/init-mqtt.py -generatorOptions: - disableNameSuffixHash: true diff --git a/kubernetes/main/apps/storage/emqx/cluster/resources/init-mqtt.py b/kubernetes/main/apps/storage/emqx/cluster/resources/init-mqtt.py deleted file mode 100644 index ea9c0a1f3f..0000000000 --- a/kubernetes/main/apps/storage/emqx/cluster/resources/init-mqtt.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import json -import time -from typing import Optional -from urllib.request import Request, urlopen -from urllib.error import URLError - - -class EMQXManager: - def __init__( - self, - emqx_address: str, - admin_username: str, - admin_password: str, - mqtt_username: str, - mqtt_password: str, - ) -> None: - self.emqx_address = emqx_address - self.admin_username = admin_username - self.admin_password = admin_password - self.mqtt_username = mqtt_username - self.mqtt_password = mqtt_password - - def wait_for_emqx(self) -> None: - while True: - try: - response = urlopen(f"http://{self.emqx_address}/api/v5/status") - if response.getcode() == 200: - print("EMQX started, ready to initialize..") - break - except URLError: - print("Waiting for EMQX to start..") - time.sleep(5) - - def get_api_token(self) -> Optional[str]: - data = json.dumps( - {"username": self.admin_username, "password": self.admin_password} - ).encode("utf-8") - req = Request( - f"http://{self.emqx_address}/api/v5/login", - data=data, - headers={"Content-Type": "application/json"}, - ) - try: - with urlopen(req) as response: - response_data = json.loads(response.read().decode("utf-8")) - return response_data.get("token", None) - except URLError as e: - print(f"Error: {e}") - return None - - def create_mqtt_user(self, api_token: str) -> bool: - data = json.dumps( - { - "user_id": self.mqtt_username, - "password": self.mqtt_password, - "is_superuser": True, - } - ).encode("utf-8") - headers = { - "Authorization": f"Bearer {api_token}", - "Content-Type": "application/json", - } - req = Request( - f"http://{self.emqx_address}/api/v5/authentication/password_based:built_in_database/users", - data=data, - headers=headers, - ) - try: - with urlopen(req) as response: - return response.getcode() == 200 - except URLError as e: - print(f"Error: {e}") - return False - - -def main() -> None: - emqx_address = os.environ.get("X_EMQX_ADDRESS") - admin_username = os.environ.get("EMQX_DASHBOARD__DEFAULT_USERNAME") - admin_password = os.environ.get("EMQX_DASHBOARD__DEFAULT_PASSWORD") - mqtt_username = os.environ.get("X_EMQX_MQTT_USERNAME") - mqtt_password = os.environ.get("X_EMQX_MQTT_PASSWORD") - - if not all( - [emqx_address, admin_username, admin_password, mqtt_username, mqtt_password] - ): - print("Missing environment variables.") - return - - emqx_manager = EMQXManager( - emqx_address, admin_username, admin_password, mqtt_username, mqtt_password - ) - emqx_manager.wait_for_emqx() - - api_token = emqx_manager.get_api_token() - if api_token: - success = emqx_manager.create_mqtt_user(api_token) - if success: - print(f"User {mqtt_username} created successfully.") - else: - print(f"Error creating user {mqtt_username} or user already exists.") - else: - print("Login failed.") - - while True: - time.sleep(1) - - -if __name__ == "__main__": - main() diff --git a/kubernetes/main/apps/storage/emqx/ks.yaml b/kubernetes/main/apps/storage/emqx/ks.yaml index 9182996394..ad97277183 100644 --- a/kubernetes/main/apps/storage/emqx/ks.yaml +++ b/kubernetes/main/apps/storage/emqx/ks.yaml @@ -20,28 +20,28 @@ spec: retryInterval: 1m timeout: 5m --- -# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json -# apiVersion: kustomize.toolkit.fluxcd.io/v1 -# kind: Kustomization -# metadata: -# name: &app emqx-cluster -# namespace: flux-system -# spec: -# targetNamespace: storage -# commonMetadata: -# labels: -# app.kubernetes.io/name: *app -# # dependsOn: -# # - name: emqx -# path: ./kubernetes/main/apps/storage/emqx/cluster -# prune: true -# sourceRef: -# kind: GitRepository -# name: home-ops -# wait: true -# interval: 30m -# retryInterval: 1m -# timeout: 5m +yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app emqx-cluster + namespace: flux-system +spec: + targetNamespace: storage + commonMetadata: + labels: + app.kubernetes.io/name: *app + # dependsOn: + # - name: emqx + path: ./kubernetes/main/apps/storage/emqx/cluster + prune: true + sourceRef: + kind: GitRepository + name: home-ops + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m # # --- # # yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json # apiVersion: kustomize.toolkit.fluxcd.io/v1 diff --git a/kubernetes/main/apps/storage/emqx/operator/externalsecret.yaml b/kubernetes/main/apps/storage/emqx/operator/externalsecret.yaml index 327fe1967d..0fc9dc0f8d 100644 --- a/kubernetes/main/apps/storage/emqx/operator/externalsecret.yaml +++ b/kubernetes/main/apps/storage/emqx/operator/externalsecret.yaml @@ -3,22 +3,44 @@ apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: &app mqx + name: &app emqx-secret spec: secretStoreRef: kind: ClusterSecretStore name: akeyless-secret-store target: - name: *app + name: emqx-secret template: engineVersion: v2 data: - EMQX_DASHBOARD__DEFAULT_USERNAME: "{{ .EMQX_DASHBOARD__DEFAULT_USERNAME }}" - EMQX_DASHBOARD__DEFAULT_PASSWORD: "{{ .EMQX_DASHBOARD__DEFAULT_PASSWORD }}" - X_EMQX_MQTT_USERNAME: "{{ .X_EMQX_MQTT_USERNAME }}" - X_EMQX_MQTT_PASSWORD: "{{ .X_EMQX_MQTT_PASSWORD }}" - X_EMQX_APIKEY_KEY: "{{ .X_EMQX_APIKEY_KEY }}" - X_EMQX_APIKEY_SECRET: "{{ .X_EMQX_APIKEY_SECRET }}" + EMQX_DASHBOARD__DEFAULT_USERNAME: '{{ index . "USERNAME" }}' + EMQX_DASHBOARD__DEFAULT_PASSWORD: '{{ index . "password" }}' + dataFrom: + - extract: + key: /emqx +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: emqx-init-user +spec: + secretStoreRef: + kind: ClusterSecretStore + name: akeyless-secret-store + target: + name: emqx-init-user-secret + creationPolicy: Owner + template: + engineVersion: v2 + data: + init-user.json: | + [ + {"user_id": "{{ index . "USERNAME" }}", "password": "{{ index . "password" }}", "is_superuser": true}, + {"user_id": "{{ index . "X_EMQX_MQTT_USERNAME" }}", "password": "{{ index . "X_EMQX_MQTT_PASSWORD" }}", "is_superuser": false} + ] + init-acl: | + {allow, {user, "{{ index . "X_EMQX_MQTT_USERNAME" }}"}, all, ["#"]}. dataFrom: - extract: key: /emqx From bfb2e9a09868c335b0355ba04ee664931c9fa322 Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 23:18:24 +0000 Subject: [PATCH 07/10] chore: storage emqx --- kubernetes/main/apps/storage/emqx/ks.yaml | 4 ++-- kubernetes/main/apps/storage/kustomization.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/main/apps/storage/emqx/ks.yaml b/kubernetes/main/apps/storage/emqx/ks.yaml index ad97277183..eefee8914c 100644 --- a/kubernetes/main/apps/storage/emqx/ks.yaml +++ b/kubernetes/main/apps/storage/emqx/ks.yaml @@ -31,8 +31,8 @@ spec: commonMetadata: labels: app.kubernetes.io/name: *app - # dependsOn: - # - name: emqx + dependsOn: + - name: emqx-operator path: ./kubernetes/main/apps/storage/emqx/cluster prune: true sourceRef: diff --git a/kubernetes/main/apps/storage/kustomization.yaml b/kubernetes/main/apps/storage/kustomization.yaml index 7fffba0e93..48b4563a51 100644 --- a/kubernetes/main/apps/storage/kustomization.yaml +++ b/kubernetes/main/apps/storage/kustomization.yaml @@ -7,9 +7,9 @@ resources: - ./namespace.yaml # Flux-Kustomizations - ./cloudnative-pg/ks.yaml - - ./minio/ks.yaml - ./dragonfly/ks.yaml - - ./obsidian-couchdb/ks.yaml - ./emqx/ks.yaml + - ./minio/ks.yaml + - ./obsidian-couchdb/ks.yaml - ./snapshot-controller/ks.yaml #- ./longhorn/ks.yaml From 45f839d8641f531ffd69097679e0f86e4d6e212e Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 23:20:54 +0000 Subject: [PATCH 08/10] chore: remove trivy operator for now --- kubernetes/main/apps/security/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/main/apps/security/kustomization.yaml b/kubernetes/main/apps/security/kustomization.yaml index 77112fa2a2..586fc7c1e5 100644 --- a/kubernetes/main/apps/security/kustomization.yaml +++ b/kubernetes/main/apps/security/kustomization.yaml @@ -8,5 +8,5 @@ resources: # Flux-Kustomizations - ./authelia/ks.yaml - ./glauth/ks.yaml - - ./trivy-operator/ks.yaml + # - ./trivy-operator/ks.yaml - ./tetragon/ks.yaml From e6e35c9032205da239da7ac641f25ed8847c43d8 Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 23:26:15 +0000 Subject: [PATCH 09/10] chore: stuff --- kubernetes/main/apps/storage/cloudnative-pg/ks.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kubernetes/main/apps/storage/cloudnative-pg/ks.yaml b/kubernetes/main/apps/storage/cloudnative-pg/ks.yaml index 36a4887186..2ba04c9e8f 100644 --- a/kubernetes/main/apps/storage/cloudnative-pg/ks.yaml +++ b/kubernetes/main/apps/storage/cloudnative-pg/ks.yaml @@ -101,8 +101,6 @@ spec: VOLSYNC_CAPACITY: 2Gi GATUS_SUBDOMAIN: pg --- - ---- # yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization From 989760bc97cbf1225dde1d041cb9a55dd1681add Mon Sep 17 00:00:00 2001 From: rafaribe Date: Tue, 26 Nov 2024 23:30:02 +0000 Subject: [PATCH 10/10] chore: stuff --- .github/workflows/flux-diff.yaml | 1 + .github/workflows/flux-image-test.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/flux-diff.yaml b/.github/workflows/flux-diff.yaml index 5e6819df0a..8dfa2bf101 100644 --- a/.github/workflows/flux-diff.yaml +++ b/.github/workflows/flux-diff.yaml @@ -36,6 +36,7 @@ jobs: uses: tj-actions/changed-files@v45 with: files: kubernetes/** + files_ignore: kubernetes/shared/** dir_names: true dir_names_max_depth: 2 matrix: true diff --git a/.github/workflows/flux-image-test.yaml b/.github/workflows/flux-image-test.yaml index f607c4ed5e..c7d0a4ce1e 100644 --- a/.github/workflows/flux-image-test.yaml +++ b/.github/workflows/flux-image-test.yaml @@ -35,6 +35,7 @@ jobs: uses: tj-actions/changed-files@v45 with: files: kubernetes/** + files_ignore: kubernetes/shared/** dir_names: true dir_names_max_depth: 2 matrix: true