fix(proxies): classify refusal cause and hang, add bounded opt-in recovery pass

.env.example

@@ -2332,6 +2332,11 @@ APP_LOG_TO_FILE=true
 # PROXY_HEALTH_ENABLED=true
 # Sweep interval in ms (minimum 60000). Default: 600000 (10min).
 # PROXY_HEALTH_INTERVAL_MS=600000
+
+# Background recovery-pass interval in ms: how often the scheduler re-probes proxies it
+# previously marked unhealthy, so a proxy that comes back is picked up without a restart.
+# Values below 60000 fall back to the default.
+# PROXY_HEALTH_RECOVERY_INTERVAL_MS=600000
 # Reachability probe target for the scheduler and the auto-test endpoint.
 # Point it at an internal/self-hosted URL to avoid the public default.
 # PROXY_HEALTH_TEST_URL=https://httpbin.org/ip

changelog.d/fixes/14233-proxy-health-refusal-cause-recovery.md

@@ -0,0 +1 @@
+- **fix(proxies):** the health sweep now tells a refused destination apart from a dead proxy (a stalled handshake is observed separately from a frank connection failure, and a 451 or proven 403 carries its cause alongside the verdict without touching any proxy status), and an opt-in bounded pass re-tests a proxy the sweep saw refused before its set-aside expires — `PROXY_HEALTH_RECOVERY_ENABLED=true` (default off: no extra probe runs) ([#14233](https://github.com/diegosouzapw/OmniRoute/pull/14233)) — thanks @maxmad64bis

docs/reference/ENVIRONMENT.md

@@ -1139,6 +1139,7 @@ Anthropic-compatible provider instead.
 | `PROXY_HEALTH_UNHEALTHY_CACHE_TTL_MS`           | `2000`                  | `src/lib/proxyHealth.ts`                                                                | Cache TTL for failed proxy health probes. Keep this shorter than `PROXY_HEALTH_CACHE_TTL_MS` so transient proxy timeouts under high concurrency retry quickly without disabling fast-fail for truly dead proxies.                                                                                                                                                                                                                                                                                                                                                          |
 | `PROXY_HEALTH_ENABLED`                          | `true`                  | `src/lib/proxyHealth/scheduler.ts`                                                      | Set `false` to disable the background proxy health scheduler that periodically probes registered proxies.                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
 | `PROXY_HEALTH_INTERVAL_MS`                      | `600000`                | `src/lib/proxyHealth/scheduler.ts`                                                      | Background health-scheduler sweep interval in ms (minimum `60000`).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| `PROXY_HEALTH_RECOVERY_INTERVAL_MS`              | `600000`                | `src/lib/proxyHealth/scheduler.ts`                                                      | Background recovery-pass interval in ms: how often previously-unhealthy proxies are re-probed so a recovered proxy returns to rotation without a restart. Values below `60000` fall back to the default.                                                                                                                                                                                                                                  |
 | `PROXY_HEALTH_TEST_URL`                         | `https://httpbin.org/ip` | `src/lib/proxyHealth/probeTarget.ts`                                                   | Reachability probe target used by the scheduler and the `/api/settings/proxies/auto-test` endpoint. Point it at an internal/self-hosted URL to avoid the public default.                                                                                                                                                                                                                                                                                                                                                                                                   |
 | `PROXY_HEALTH_TEST_CONCURRENCY`                 | `10`                     | `src/lib/proxyHealth/probeTarget.ts`                                                   | Probes started at once per batch, shared by the scheduler and the `/api/settings/proxies/auto-test` endpoint. Floored at 1 and capped at 50.                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `PROXY_HEALTH_TEST_STAGGER_MS`                  | `100`                    | `src/lib/proxyHealth/probeTarget.ts`                                                   | Delay in ms between two probe departures inside a batch. Without it the whole batch leaves at the same moment and a shared egress IP can trip a rate-limited target. Set to `0` to disable the spacing; capped at 5000.                                                                                                                                                                                                                                                                                                                                                    |

src/lib/proxyHealth/decision.ts

@@ -40,7 +40,52 @@
  *       relayed 5xx stays `inconclusive` (policy B) and keeps the streak.
  */
 
-export type ProxyProbeOutcome = "ok" | "fail" | "inconclusive" | "blocked";
+export type ProxyProbeOutcome = "ok" | "fail" | "hang" | "inconclusive" | "blocked";
+
+/**
+ * Sidecar cause for a refused relay. Carried ALONGSIDE the verdict, never
+ * inside it: `decideProxyHealthAction` never branches on a cause, so a cause
+ * can neither write a status nor filter traffic. The only proven 403 is one
+ * the caller proves via `signals.geoProven`; the sweep never proves (HEAD
+ * probe, no body), so sweep 403s always land on `unproven`.
+ */
+export type ProbeCause = "unclassified" | "target_refused" | "unproven";
+
+export interface RefusalCauseSignals {
+  /** The caller proved a geographic motive for this 403. Defaults to false. */
+  geoProven?: boolean;
+}
+
+/**
+ * PURE: classify the cause of a refused relay. 451 is a proven refusal by
+ * status alone; 403 needs an explicit proof; 401/429 claim no cause.
+ */
+export function classifyRefusalCause(status: number, signals: RefusalCauseSignals = {}): ProbeCause {
+  if (status === 451) return "target_refused";
+  if (status === 403) return signals.geoProven === true ? "target_refused" : "unproven";
+  return "unclassified";
+}
+
+export interface ProbeErrorSignals {
+  /** Our own deadline fired (controller aborted). */
+  aborted: boolean;
+  /** The proxy host resolved (a hang implies we got past DNS). */
+  resolved: boolean;
+  /** A TLS handshake completed. */
+  tlsNegotiated: boolean;
+}
+
+/**
+ * PURE: distinguish a stalled handshake from a frank failure. A hang means we
+ * reached the host but the handshake never completed before our own deadline;
+ * anything aborted without those conditions keeps today's `inconclusive`.
+ */
+export function classifyProbeError(signals: ProbeErrorSignals): "hang" | "fail" | "inconclusive" {
+  if (signals.aborted) {
+    return signals.resolved && !signals.tlsNegotiated ? "hang" : "inconclusive";
+  }
+  return "fail";
+}
 
 /** Statuses that mean the TARGET refused this egress IP rather than served it. */
 const TARGET_BLOCK_STATUSES: ReadonlySet<number> = new Set([401, 403, 429]);
@@ -128,7 +173,9 @@ export function decideProxyHealthAction(input: ProxyHealthDecisionInput): ProxyH
     };
   }
 
-  // Conclusive failure.
+  // Conclusive failure. `hang` (a stalled handshake) falls through with `fail`:
+  // same count, same policy C (no status write by default). The verdicts stay
+  // distinct upstream so the sweep can observe hangs separately.
   const failures = priorFailures + 1;
 
   // C: default mode only counts/logs — never downgrades.