feat(gatus): add isolated external probe - #4086
Merged
Merged
Conversation
@@ data @@
# v1/ConfigMap/actions-runner-system/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/ai/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/cert-manager/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/database/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/default/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/flux-system/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/kopiur-system/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/kube-system/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/media/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/network/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/observability/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/openebs-system/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/rook-ceph/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/security/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/system-upgrade/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ data @@
# v1/ConfigMap/web3/cluster-settings
! + one map entry added:
+ ENVOY_GATUS_PROBE_IP: "192.168.0.9"
@@ spec.endpoints @@
# externaldns.k8s.io/v1alpha1/DNSEndpoint/network/cloudflared
! + one list entry added:
+ - dnsName: gatus-probe...PLACEHOLDER_SECRET_DOMAIN..
+ recordType: CNAME
+ targets:
+ - 2a03cc15-eff3-4c5c-a7c5-97877e5b8231.cfargotunnel.com
@@ data.config.yaml @@
# v1/ConfigMap/network/cloudflared-configmap
! ± value change in multiline text (one insert, no deletions)
---
ingress:
- hostname: ..PLACEHOLDER_SECRET_DOMAIN..
originRequest: &originRequest
[one line unchanged)]
keepAliveTimeout: 3600s
originServerName: external...PLACEHOLDER_SECRET_DOMAIN..
tcpKeepAlive: 7200s
service: &svc https://envoy-external.network.svc.cluster.local:443
+ - hostname: gatus-probe...PLACEHOLDER_SECRET_DOMAIN..
+ originRequest:
+ originServerName: gatus-probe...PLACEHOLDER_SECRET_DOMAIN..
+ http2Origin: true
+ service: https://envoy-external-probe.network.svc.cluster.local:443
- hostname: "*...PLACEHOLDER_SECRET_DOMAIN.."
originRequest: *originRequest
service: *svc
- service: http_status:404
@@ spec.values.sidecar.gatewayNames @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/observability/gatus
! - one list entry removed:
- - envoy-external
@@ data.config.yaml @@
# v1/ConfigMap/observability/gatus-configmap
! ± value change in multiline text (one insert, no deletions)
---
web:
port: ${GATUS_WEB_PORT}
storage:
[17 lines unchanged)]
target: 1.1.1.1:53
interval: 1m
endpoints:
+ - name: Gatus external probe
+ group: external
+ url: https://gatus-probe...PLACEHOLDER_SECRET_DOMAIN../health
+ interval: 1m
+ client:
+ dns-resolver: tcp://1.1.1.1:53
+ conditions:
+ - "[STATUS] == 200"
+ alerts:
+ - type: discord
+ enabled: true
+ send-on-resolved: true
+
- name: Cloudflare
group: connectivity
url: icmp://1.1.1.1
interval: 1m
[83 lines unchanged)]
timeout: 30s
conditions:
- "[STATUS] == 200"
- "[RESPONSE_TIME] < 30000"
@@ (root level) @@
# gateway.networking.k8s.io/v1/Gateway/network/envoy-external-probe
! + one document added:
+ apiVersion: gateway.networking.k8s.io/v1
+ kind: Gateway
+ metadata:
+ name: envoy-external-probe
+ namespace: network
+ annotations:
+ external-dns.alpha.kubernetes.io/target: gatus-probe...PLACEHOLDER_SECRET_DOMAIN..
+ labels:
+ kustomize.toolkit.fluxcd.io/name: envoy-gateway
+ kustomize.toolkit.fluxcd.io/namespace: network
+ spec:
+ gatewayClassName: envoy
+ infrastructure:
+ annotations:
+ external-dns.alpha.kubernetes.io/hostname: gatus-probe...PLACEHOLDER_SECRET_DOMAIN..
+ lbipam.cilium.io/ips: "192.168.0.9"
+ listeners:
+ - name: https
+ hostname: gatus-probe...PLACEHOLDER_SECRET_DOMAIN..
+ port: 443
+ protocol: HTTPS
+ tls:
+ certificateRefs:
+ - name: "-.PLACEHOLDER_SECRET_DOMAIN..-production-tls"
+ kind: Secret
+ allowedRoutes:
+ namespaces:
+ from: Selector
+ selector:
+ matchLabels:
+ kubernetes.io/metadata.name: observability
@@ (root level) @@
# gateway.networking.k8s.io/v1/HTTPRoute/observability/gatus-probe
! + one document added:
+ apiVersion: gateway.networking.k8s.io/v1
+ kind: HTTPRoute
+ metadata:
+ name: gatus-probe
+ namespace: observability
+ labels:
+ kustomize.toolkit.fluxcd.io/name: gatus
+ kustomize.toolkit.fluxcd.io/namespace: observability
+ spec:
+ hostnames:
+ - gatus-probe...PLACEHOLDER_SECRET_DOMAIN..
+ parentRefs:
+ - name: envoy-external-probe
+ namespace: network
+ sectionName: https
+ rules:
+ - backendRefs:
+ - name: gatus
+ port: 80
+ matches:
+ - method: GET
+ path:
+ type: Exact
+ value: /health
|
@@ spec.template.spec.initContainers.gatus-sidecar.args @@
# apps/v1/Deployment/observability/gatus
! - one list entry removed:
- - "--gateway-name=envoy-external"
|
AI Automated ReviewAnalysis engine: qwen-3.6-fast@http://litellm.ai.svc.cluster.local/v1 (openai) Recommendation: Approve with minor notes. The PR correctly implements an isolated, Cloudflare-tunneled health check route for Gatus, separating it from the CrowdSec-protected main gateway. The manifests are internally consistent and follow repository conventions. Change-by-Change Findings
Standards Compliance
Required Checks
Unknowns or Needs Verification
|
Tanguille
added a commit
that referenced
this pull request
Jul 27, 2026
…d xmrig guard #4086 added the isolated external probe and dropped envoy-external from the sidecar's gatewayNames in the same change. Since 2026-07-20 22:00 that has left all eight public hostnames undiscovered, and GatusEndpointDown filters on group=~"external|services", so the only externally-alerting endpoint was the probe itself. The probe is a genuine round trip through Cloudflare's edge and back down the tunnel, but it terminates on the dedicated probe gateway, not the envoy-external one that serves users. The xmrig guard recovers only at or below 60C, which neither mining node reaches, so a node that trips once stays gated off. Thresholds are left alone on purpose: on every sustained-mining day control-3 peaked at 89-95C against 66-75C when idle, which is the range that previously forced a cold power cycle. The gate being shut is the correct outcome. What was missing is a signal for the case where it is shut without cause, so the new rule pairs the gate state with the temperature actually sitting in the recovery band.
Tanguille
added a commit
that referenced
this pull request
Jul 27, 2026
…d xmrig guard #4086 added the isolated external probe and dropped envoy-external from the sidecar's gatewayNames in the same change. Since 2026-07-20 22:00 that has left all eight public hostnames undiscovered, and GatusEndpointDown filters on group=~"external|services", so the only externally-alerting endpoint was the probe itself. The probe is a genuine round trip through Cloudflare's edge and back down the tunnel, but it terminates on the dedicated probe gateway, not the envoy-external one that serves users. The xmrig guard recovers only at or below 60C, which neither mining node reaches, so a node that trips once stays gated off. Thresholds are left alone on purpose: on every sustained-mining day control-3 peaked at 89-95C against 66-75C when idle, which is the range that previously forced a cold power cycle. The gate being shut is the correct outcome. What was missing is a signal for the case where it is shut without cause, so the new rule pairs the gate state with the temperature actually sitting in the recovery band.
Tanguille
added a commit
that referenced
this pull request
Jul 27, 2026
* fix(homeassistant): raise mcp memory limit to 512Mi Both ha-mcp containers peak at ~255Mi against a 256Mi limit; the primary server was OOMKilled 13 times in the last 30 days. * fix(observability): defrag etcd daily, right-size VM stack, drop noisy rule etcd-defrag reclaimed 302MB to 130MB per member on its last run but only runs weekly; fragmentation crosses the 50% alert threshold within ~4 days, so etcdDatabaseHighFragmentationRatio fires for roughly half of every week. A full three-member pass costs ~95s. vmsingle, vmagent, vlogs and the log collector all requested less memory than their observed peaks, leaving the entire metrics and logging path burstable and first in line for eviction on a node that is already at 97% of allocatable. vmsingle's 4Gi limit was within 80Mi of its 14d peak. RecordingRulesNoData flags recording rules that are empty by design: count:up0 is count(up == 0), which yields no samples precisely when every scrape target is healthy. * fix(toolhive): raise memory limits on OOMKilled mcp workloads The ha VirtualMCPServer was OOMKilled twice at its 1Gi limit and the observability one at the operator's 512Mi default; both swing by an order of magnitude with session load. karakeep's mcp container peaked at 84% of its 100Mi limit. * fix(observability): unclamp cpu limits on bursty grafana and speedtest Both sit at 100m and burst well past it: grafana throttles on 13% of CFS periods rendering dashboards, and speedtest-exporter on 74%, which caps the throughput it exists to measure. CPU is compressible, so the request still does the isolation work. * fix(observability): restore external gatus discovery, alert on latched xmrig guard #4086 added the isolated external probe and dropped envoy-external from the sidecar's gatewayNames in the same change. Since 2026-07-20 22:00 that has left all eight public hostnames undiscovered, and GatusEndpointDown filters on group=~"external|services", so the only externally-alerting endpoint was the probe itself. The probe is a genuine round trip through Cloudflare's edge and back down the tunnel, but it terminates on the dedicated probe gateway, not the envoy-external one that serves users. The xmrig guard recovers only at or below 60C, which neither mining node reaches, so a node that trips once stays gated off. Thresholds are left alone on purpose: on every sustained-mining day control-3 peaked at 89-95C against 66-75C when idle, which is the range that previously forced a cold power cycle. The gate being shut is the correct outcome. What was missing is a signal for the case where it is shut without cause, so the new rule pairs the gate state with the temperature actually sitting in the recovery band. * fix(xmrig-guard): gate on the Composite sensor and narrow the thermal band The guard read max() across every NVMe hwmon sensor, but only temp1 is Composite, which is the reading the drives' 70C rating specifies and the one smartctl reports. temp2-temp4 are internal die sensors running ~9C hotter with no comparable rating, so a Composite-spec threshold was being compared against the wrong number and every measurement was ~9C pessimistic. On Composite the drives are well inside their rating at idle: median 57.9C and 58.9C, and 0.0% of the last 7d above 70C on either node. Mining is what breaks the rating, peaking at 79.8C and 82.8C, so the gate still has to shut during mining. Trip moves 70C -> 65C and recovery stays at 60C. Idle Composite never reached 65C over 7d, so the trip does not false-fire, and it sits at or below 60C for 100% (control-2) and 90% (control-3) of the time, so recovery is reachable and the permanent latch is gone. Mining raises Composite at up to 1.1C/min, giving ~4.5 minutes across the 65-70C band; the 180s scale-down stabilization did not fit inside that with the 120s dwell and 60s poll, so it drops to 0. Restricting to Composite also shrinks the audited set from 7 and 8 series to 2, which is what failed today when one missing sample tripped the identity check and latched control-2 closed. * fix(xmrig-guard): shed all miners at once and drop the trip to 64C The scale-down policy capped removal at one pod per 60s, so draining three replicas took a further 180s that the previous timing note did not count. Full worst case was 60s to sample the crossing + 120s trip dwell + 60s KEDA polling + 180s drain = 7 minutes, which at the observed 1.1C/min peak put the drives at 72.7C, past their 70C rating. Removing the per-pod cap brings the response to ~4.25 minutes. Pairing that with a 64C trip gives a 6C band worth ~5.5 minutes and lands the peak near 68.7C. Idle Composite never exceeded 62C over 7d on either node, so the lower trip does not false-fire and still leaves 4C of hysteresis above the 60C recovery point. Gradual drain only ever made sense for tracking a falling solar curve, never for a thermal trip, and mining is free to interrupt.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Summary
Validation