From c6ba7cb482301e7225f2ae49600fb7357d06ff1f Mon Sep 17 00:00:00 2001 From: Scout Date: Thu, 24 Sep 2026 13:35:43 -0400 Subject: [PATCH 1/8] Add corporate-action guard to the adaptive-paper engine (audit gap #8) held_overnight trials could hold a position across a session boundary without any adjustment for a split, reverse split, cash/stock dividend, spin-off, merger or name change landing in that window. corporate_actions.py adds a fail-closed guard: entries are blocked and an existing position is force-flattened (ranked above the D5 gap-risk stop, below force_exit/STOP) for any symbol whose corporate action falls inside the possible overnight hold horizon, or whose Alpaca CorporateActionsClient lookup failed or was ambiguous. Wired into native_strategy.AdaptiveStrategy.rebalance() and runner.py's paper trial loop via the existing --env-file credential path; decisions are recorded in the paper/live event stream without values. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../adaptive-paper/corporate_actions.py | 300 +++++++++++++ .../adaptive-paper/native_strategy.py | 89 +++- .../us-equities/adaptive-paper/runner.py | 34 +- .../us-equities/adaptive-paper/sessions.py | 13 + .../adaptive-paper/source-hashes.json | 8 +- manifests/evidence.json | 16 +- .../test_adaptive_paper_corporate_actions.py | 417 ++++++++++++++++++ 7 files changed, 859 insertions(+), 18 deletions(-) create mode 100644 blueprints/us-equities/adaptive-paper/corporate_actions.py create mode 100644 tests/test_adaptive_paper_corporate_actions.py diff --git a/blueprints/us-equities/adaptive-paper/corporate_actions.py b/blueprints/us-equities/adaptive-paper/corporate_actions.py new file mode 100644 index 000000000..e8d3e7bcd --- /dev/null +++ b/blueprints/us-equities/adaptive-paper/corporate_actions.py @@ -0,0 +1,300 @@ +"""Interim corporate-action guard for the adaptive-paper engine (trading-lane +audit gap #8): positions were not adjusted for corporate actions even though +a `held_overnight` trial can hold a position across a session boundary. This +module adds that missing control: block new entries and force a pre-close +flatten for any symbol with a split, reverse split, cash/stock dividend, +spin-off, merger or name change whose governing date falls inside the +position's possible hold horizon (the next session, for an overnight hold). + +Layering (mirrors transport.py's own network-boundary convention): + - `ActionRecord`, `evaluate_guard`, `CorporateActionMonitor` are pure, + offline, no-network logic -- independently unit-tested with a fake + `source`, no credentials or alpaca-py import required. + - `AlpacaCorporateActionsSource` is the only piece that imports alpaca-py, + and only when actually constructed with real credentials. Its + `_TYPE_FIELD_MAP` was verified field-for-field against the installed + alpaca-py==0.44.0 package on this host (`alpaca/data/models/ + corporate_actions.py`, `alpaca/data/requests.py`, + `alpaca/data/historical/corporate_actions.py`) -- a dated, reproducible + local read of the pinned SDK revision, not a live network fetch; the + endpoint itself (`GET /v1/corporate-actions` via + `alpaca.data.historical.corporate_actions.CorporateActionsClient`) is + never called by the offline test suite and carries no acceptance claim + beyond "the request/response shapes this wrapper assumes match the + installed SDK's own dataclasses". + +Precedence (see native_strategy.AdaptiveStrategy.rebalance, where this +guard's result is consumed): + 1. safety.py's STOP file / risk halts and exits.py's own `force_exit` + tier (trial/session end, threaded into rebalance() from outside) -- + both already stop or force-exit the *whole* trial regardless of any + single symbol's corporate-action status, and are evaluated first. + 2. This guard's `must_flatten` -- evaluated and applied (as a forced full + sell, through the same engine-layer forced-sell path D5's gap-risk + stop already uses) ABOVE gap_risk_stop, because a scheduled + corporate-action date is a known calendar fact fixed ahead of time, + not a live threshold breach discovered this tick. Unlike gap_risk_stop + it is deliberately NOT fire-once: it is re-evaluated and re-applied + every tick for as long as the symbol is still held and still in + range, exactly matching "must flatten before the regular close of the + session before that date" (a partial fill or a cancelled/rejected + attempt must not leave the position permanently unprotected for the + rest of the session, the same reasoning exits.py's docstring gives + for why gap-risk/overnight rules "stay exactly where they already + lived" instead of joining the per-holding exits.py chain). + 3. D5 gap_risk_stop (native_strategy._gap_risk_stop_symbols), unchanged. + 4. exits.py's own per-holding chain (`exits.DEFAULT_PLAN`), unchanged. + 5. This guard's `block_entry` is checked independently, after + strategies_v1.AdaptivePolicy.decide() has already picked this tick's + targets: any symbol in `block_entry` has its target-increase (a new + buy, or growing an existing position) dropped before order submission. + This is a post-hoc filter, not a chain rule, because unlike + gap_risk_stop (only ever a stop on an existing holding) this guard + must also gate admission of a symbol never held before. + +Fail-closed contract: a symbol whose lookup failed, was ambiguous, or was +simply never attempted (missing from the source's result, or the cached +result has gone stale past `CorporateActionMonitor.max_age_seconds`) is +always treated as `block_entry=True, needs_attention=True` -- never as "no +action". This module never guesses at a corporate action from partial data. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date +import time as _time +from typing import Mapping + +# Canonical action types this guard reasons about. Real alpaca-py corporate- +# action records collapse into these (see AlpacaCorporateActionsSource); +# unit_split/redemption/worthless_removal/rights_distribution -- corporate +# actions not explicitly named in the brief -- fall into "other" rather than +# being silently ignored, consistent with the fail-closed/no-guessing intent. +ACTION_TYPES = frozenset({ + "forward_split", "reverse_split", "cash_dividend", "stock_dividend", + "spin_off", "merger", "name_change", "other", +}) + +# Sentinel lookup outcomes for a symbol, distinct from "resolved (possibly +# empty) list of ActionRecord". +LOOKUP_FAILED = "lookup_failed" +LOOKUP_AMBIGUOUS = "lookup_ambiguous" + + +class CorporateActionLookupError(Exception): + """Raised by a source's fetch() on any network/parse/credential failure. + Never carries a credential value; only a bounded reason string.""" + + +@dataclass(frozen=True) +class ActionRecord: + """One corporate action for one symbol, already normalized to the + single governing date this guard checks against (`action_date`): the + ex_date for a split/dividend/spin-off, the effective_date for a merger, + the process_date for a name change or another type with no ex_date/ + effective_date of its own. Callers (a source's fetch()) resolve that + choice before constructing this record -- this module does not itself + pick between competing date fields at guard-evaluation time.""" + symbol: str + action_type: str + action_date: date + source_id: str = "" + + def __post_init__(self): + if self.action_type not in ACTION_TYPES: + raise ValueError(f"unknown_corporate_action_type:{self.action_type}") + + +@dataclass(frozen=True) +class GuardDecision: + symbol: str + block_entry: bool + must_flatten: bool + needs_attention: bool + reason: str | None + action_type: str | None = None + action_date: date | None = None + + +def evaluate_guard(*, today: date, next_session_date: date, held_symbols, candidate_symbols, + lookup_results: Mapping[str, object]) -> dict: + """Pure decision function: no I/O, no clock read, no broker/session + lookup of its own. + + `lookup_results` maps symbol -> either `LOOKUP_FAILED`/`LOOKUP_AMBIGUOUS`, + or a (possibly empty) list/tuple of `ActionRecord`. A symbol absent from + `lookup_results` entirely is treated exactly like `LOOKUP_FAILED` (fail + closed on missing data, never silently "no action"). + + `next_session_date` is the caller-supplied possible hold horizon for an + overnight trial: a position opened (or already held) as of `today` can + be carried to `next_session_date`'s open, so an action whose + `action_date` falls on either `today` or `next_session_date` is in + range and blocks/flattens. A qualifying action on `today` covers both + "already holding into an action landing today" (should already have + been flattened by a prior tick -- still enforced) and "an action was + just discovered same-day, entries must stop immediately". + """ + held = set(held_symbols) + decisions = {} + for symbol in held | set(candidate_symbols): + result = lookup_results.get(symbol, LOOKUP_FAILED) + if result in (LOOKUP_FAILED, LOOKUP_AMBIGUOUS): + decisions[symbol] = GuardDecision( + symbol=symbol, block_entry=True, must_flatten=False, needs_attention=True, + reason=f"corporate_action_{result}") + continue + qualifying = [record for record in result if record.action_date in (today, next_session_date)] + if not qualifying: + decisions[symbol] = GuardDecision(symbol=symbol, block_entry=False, must_flatten=False, + needs_attention=False, reason=None) + continue + chosen = min(qualifying, key=lambda record: record.action_date) + decisions[symbol] = GuardDecision( + symbol=symbol, block_entry=True, must_flatten=symbol in held, needs_attention=False, + reason=f"corporate_action_{chosen.action_type}", action_type=chosen.action_type, + action_date=chosen.action_date) + return decisions + + +class CorporateActionMonitor: + """Owns the periodic refresh and fail-closed staleness bound around one + lookup `source` (an object exposing `.fetch(symbols, start, end) -> + dict[symbol, list[ActionRecord]]`, raising `CorporateActionLookupError` + on any failure). Never talks to a broker itself -- purely a cache with a + fail-closed staleness policy, injectable for offline tests with a fake + `source`. Every method is safe to call from the native owner loop; no + method sleeps or blocks beyond the source's own `fetch` call. + + `refresh_seconds` bounds how often `refresh()` actually calls + `source.fetch` (a corporate-actions calendar does not change tick to + tick). `max_age_seconds` bounds how long a successful result is still + trusted by `evaluate()`: past that age, or before any refresh has ever + succeeded, every relevant symbol fails closed exactly like a lookup + failure -- never silently reusing a result that might now be stale. + """ + + def __init__(self, source, *, refresh_seconds=900.0, max_age_seconds=3600.0, clock=None): + self.source = source + self.refresh_seconds = refresh_seconds + self.max_age_seconds = max_age_seconds + self._clock = clock or _time.time + self._last_attempt = None + self._last_success = None + self._results: dict = {} + + def refresh(self, symbols, *, start: date, end: date, now=None): + """Idempotent within `refresh_seconds` for a symbol set already + covered by the last successful fetch; always safe to call every + tick. Never raises -- a fetch failure marks every requested symbol + `LOOKUP_FAILED` for this and every subsequent `evaluate()` until a + later refresh succeeds.""" + now = self._clock() if now is None else now + symbols = set(symbols) + if (self._last_attempt is not None and now - self._last_attempt < self.refresh_seconds + and symbols.issubset(self._results.keys())): + return + self._last_attempt = now + try: + fetched = self.source.fetch(sorted(symbols), start, end) + except CorporateActionLookupError: + for symbol in symbols: + self._results[symbol] = LOOKUP_FAILED + return + except Exception: + # Fail closed on any unexpected error too -- never guess. + for symbol in symbols: + self._results[symbol] = LOOKUP_FAILED + return + self._last_success = now + for symbol in symbols: + self._results[symbol] = fetched.get(symbol, []) + + def evaluate(self, *, today: date, next_session_date: date, held_symbols, candidate_symbols, + now=None) -> dict: + now = self._clock() if now is None else now + relevant = set(held_symbols) | set(candidate_symbols) + stale = self._last_success is None or now - self._last_success > self.max_age_seconds + if stale: + results = {symbol: LOOKUP_FAILED for symbol in relevant} + else: + results = {symbol: self._results.get(symbol, LOOKUP_FAILED) for symbol in relevant} + return evaluate_guard(today=today, next_session_date=next_session_date, + held_symbols=held_symbols, candidate_symbols=candidate_symbols, + lookup_results=results) + + +# type_key (alpaca-py CorporateActionsSet.data key) -> (symbol attribute +# names that identify an affected tradable symbol, the attribute name that +# is this guard's governing date, the canonical ACTION_TYPES bucket). +# Verified against the installed alpaca-py==0.44.0 +# alpaca/data/models/corporate_actions.py dataclass fields on this host. +_TYPE_FIELD_MAP = { + "forward_splits": (("symbol",), "ex_date", "forward_split"), + "reverse_splits": (("symbol",), "ex_date", "reverse_split"), + "unit_splits": (("old_symbol", "new_symbol"), "effective_date", "other"), + "stock_dividends": (("symbol",), "ex_date", "stock_dividend"), + "cash_dividends": (("symbol",), "ex_date", "cash_dividend"), + "spin_offs": (("source_symbol", "new_symbol"), "ex_date", "spin_off"), + "cash_mergers": (("acquirer_symbol", "acquiree_symbol"), "effective_date", "merger"), + "stock_mergers": (("acquirer_symbol", "acquiree_symbol"), "effective_date", "merger"), + "stock_and_cash_mergers": (("acquirer_symbol", "acquiree_symbol"), "effective_date", "merger"), + "redemptions": (("symbol",), "process_date", "other"), + "name_changes": (("old_symbol", "new_symbol"), "process_date", "name_change"), + "worthless_removals": (("symbol",), "process_date", "other"), + "rights_distributions": (("source_symbol", "new_symbol"), "ex_date", "other"), +} + + +class AlpacaCorporateActionsSource: + """Real alpaca-py 0.44.0 `alpaca.data.historical.corporate_actions. + CorporateActionsClient` wrapper -- the only network-touching piece of + this module, imported lazily (matching transport.py's own convention) + so the rest of this module stays importable, and independently unit- + testable, without alpaca-py installed. Constructed with the same + key/secret runner.credentials() already returns; never logs, stores, or + raises them in any exception message.""" + + def __init__(self, api_key, secret_key): + from alpaca.data.historical.corporate_actions import CorporateActionsClient + self._client = CorporateActionsClient(api_key, secret_key) + + def fetch(self, symbols, start: date, end: date) -> dict: + """Returns `{symbol: [ActionRecord, ...]}` for exactly the + requested `symbols` (every requested symbol is present, possibly + with an empty list); any symbol/type this wrapper cannot map is + dropped from the per-symbol list, never silently invented. + Raises `CorporateActionLookupError` -- never a raw alpaca-py/HTTP + exception -- on any request or response-shape failure, so a caller + never needs to know alpaca-py's own exception types.""" + from alpaca.data.requests import CorporateActionsRequest + symbols = list(symbols) + try: + request = CorporateActionsRequest(symbols=symbols, start=start, end=end) + response = self._client.get_corporate_actions(request) + except CorporateActionLookupError: + raise + except Exception as error: + raise CorporateActionLookupError(type(error).__name__) from error + out = {symbol: [] for symbol in symbols} + try: + for type_key, items in dict(getattr(response, "data", {}) or {}).items(): + spec = _TYPE_FIELD_MAP.get(type_key) + if spec is None: + continue + symbol_fields, date_field, canonical_type = spec + for item in items: + action_date = getattr(item, date_field, None) + if action_date is None: + continue + for field in symbol_fields: + symbol = getattr(item, field, None) + if symbol and symbol in out: + out[symbol].append(ActionRecord( + symbol=symbol, action_type=canonical_type, action_date=action_date, + source_id=str(getattr(item, "cusip", "") or type_key))) + except CorporateActionLookupError: + raise + except Exception as error: + raise CorporateActionLookupError("malformed_corporate_action_response") from error + return out diff --git a/blueprints/us-equities/adaptive-paper/native_strategy.py b/blueprints/us-equities/adaptive-paper/native_strategy.py index cab399fc0..01143d804 100644 --- a/blueprints/us-equities/adaptive-paper/native_strategy.py +++ b/blueprints/us-equities/adaptive-paper/native_strategy.py @@ -12,7 +12,7 @@ from exits import REASON_PRICE_RULE from leverage import LeverageInputs from safety import DEFAULT_STOP, SafetyError, evaluate_gap_risk -from sessions import SessionKind, previous_trading_day, session_at +from sessions import SessionKind, next_trading_day, previous_trading_day, session_at from strategies import AdaptivePolicy, OperationalStatus, QUOTE_FUTURE_TOLERANCE_SECONDS, limit_price @@ -22,13 +22,23 @@ def __new__(cls, policy, ledger, trial_id, **kwargs): order_id_tag="A", log_events=False, log_commands=False, manage_stop=False)) def __init__(self, policy: AdaptivePolicy, ledger, trial_id: str, *, event_sink=None, - transport=None, stop_file=None, clock=time.time, account_multiplier=None): + transport=None, stop_file=None, clock=time.time, account_multiplier=None, + corporate_action_guard=None): self.policy = policy self.ledger = ledger self.trial_id = trial_id self.event_sink = event_sink or (lambda event: None) self.transport = transport self.stop_file = stop_file + # Trading-lane audit gap #8: opt-in corporate-action guard (see + # corporate_actions.py's module docstring for the full precedence + # contract against gap_risk_stop/exits.py/force_exit). None + # (default) leaves rebalance() byte-identical to before this + # control existed -- _corporate_action_guard_symbols returns + # immediately without a session lookup or event. runner.py's + # `paper` command threads a real corporate_actions.CorporateActionMonitor + # here; tests inject a fake with the same `.evaluate(...)` shape. + self.corporate_action_guard = corporate_action_guard # G-e: the broker-proven margin multiplier (runner.py's config # "_account_multiplier", only ever set after preflight has checked # multiplier >= requested leverage -- see runner._check_margin_ @@ -331,6 +341,57 @@ def positions(self): return {p.symbol: {"qty": str(p.qty), "avg_entry_price": str(p.average_cost)} for p in self.ledger.positions().values() if p.qty} + def _corporate_action_guard_symbols(self, now, held, candidate_symbols): + """Trading-lane audit gap #8: consult the injected corporate-action + guard (`self.corporate_action_guard`, see corporate_actions.py) for + (a) the set of held symbols that must be flattened before this + session's close, and (b) the set of symbols -- held or candidate -- + whose entries (new or added-to) must be blocked this tick. + + Opt-in: `self.corporate_action_guard is None` (the default) returns + immediately, with no session lookup and no event -- byte-identical + to before this control existed, the same pattern + `PolicyConfig.gap_stop_enabled` uses for `_gap_risk_stop_symbols`. + + Fail-closed on an unclassifiable session: a `now` outside the + frozen session calendar (`session_at`/`next_trading_day` raising + `ValueError`) cannot even compute "today" or "the next session", so + every held-or-candidate symbol this tick is treated exactly like a + failed lookup -- blocked from entry, and any held quantity added to + the forced-flatten set -- rather than silently skipping the guard + for that tick. + """ + if self.corporate_action_guard is None: + return set(), set() + held_symbols = {symbol for symbol, qty in held.items() if qty > 0} + try: + info = session_at(datetime.fromtimestamp(now, timezone.utc)) + next_session_date = next_trading_day(info.session_date) + except ValueError: + relevant = held_symbols | set(candidate_symbols) + for symbol in relevant: + self.event_sink({"type": "corporate_action_guard", "symbol": symbol, + "block_entry": True, "must_flatten": symbol in held_symbols, + "needs_attention": True, "reason": "corporate_action_session_unknown", + "action_type": None, "action_date": None}) + return relevant, held_symbols + decisions = self.corporate_action_guard.evaluate( + today=info.session_date, next_session_date=next_session_date, + held_symbols=held_symbols, candidate_symbols=set(candidate_symbols), now=now) + block_entry, must_flatten = set(), set() + for symbol, decision in decisions.items(): + if decision.block_entry: + block_entry.add(symbol) + if decision.must_flatten: + must_flatten.add(symbol) + if decision.block_entry or decision.must_flatten or decision.needs_attention: + self.event_sink({"type": "corporate_action_guard", "symbol": symbol, + "block_entry": decision.block_entry, "must_flatten": decision.must_flatten, + "needs_attention": decision.needs_attention, "reason": decision.reason, + "action_type": decision.action_type, + "action_date": decision.action_date.isoformat() if decision.action_date else None}) + return block_entry, must_flatten + def _gap_risk_stop_symbols(self, now, held): """D5: on the first RTH bar after an overnight hold, evaluate safety.evaluate_gap_risk between the prior session's captured RTH @@ -707,12 +768,21 @@ def rebalance(self, now=None, *, force_exit=False): # or not -- can be included in that same event below, instead of # needing a separate event type. gap_stop_symbols = self._gap_risk_stop_symbols(now, held) + # Trading-lane audit gap #8: evaluated after gap risk (whose own + # arming/capture state is independent of it) but consumed first, + # below -- see corporate_actions.py's module docstring for the full + # precedence contract. `candidate_symbols` is this tick's targets, + # the only symbols a fresh/added entry could apply to. + ca_block_entry, ca_must_flatten = self._corporate_action_guard_symbols( + now, held, decision.targets.keys()) decision_event = {"type": "decision", "timestamp": now, "regime": decision.regime, "targets": decision.targets, "exits": decision.exits, "effective_leverage": decision.effective_leverage, "signals": [{"symbol": s.symbol, "family": s.family, "edge_bps": s.edge_bps, "score": s.score} for s in decision.signals], - "gap_bps": {symbol: str(bps) for symbol, bps in self._gap_bps.items()}} + "gap_bps": {symbol: str(bps) for symbol, bps in self._gap_bps.items()}, + "corporate_action_block_entry": sorted(ca_block_entry), + "corporate_action_must_flatten": sorted(ca_must_flatten)} if getattr(self.policy, "leverage_policy", None) is not None: decision_event["leverage_ceiling"] = self.policy.last_leverage_ceiling self.event_sink(decision_event) @@ -734,7 +804,7 @@ def rebalance(self, now=None, *, force_exit=False): actions = [] for s, qty in held.items(): target = decision.targets.get(s, 0) - if qty <= target or s in gap_stop_symbols: + if qty <= target or s in gap_stop_symbols or s in ca_must_flatten: continue delta = qty - target reason = decision.exits.get(s, "rebalance") @@ -747,11 +817,20 @@ def rebalance(self, now=None, *, force_exit=False): else: sell_qty = delta actions.append((s, "sell", sell_qty, reason)) + # Trading-lane audit gap #8: a confirmed-or-fail-closed corporate + # action forces a full exit through this same engine-layer forced- + # sell path, ranked ahead of gap_risk_stop (see corporate_actions.py's + # module docstring for why); it is not mutually exclusive with + # gap_stop_symbols in principle, but `ca_must_flatten` was already + # excluded from the ordinary rebalance-delta loop above so a symbol + # never gets two competing sell actions from this method. + actions += [(s, "sell", held[s], "corporate_action_flatten") for s in ca_must_flatten if held.get(s, 0) > 0] actions += [(s, "sell", held[s], "gap_risk_stop") for s in gap_stop_symbols if held.get(s, 0) > 0] if self.enabled and not force_exit: actions += [(s, "buy", qty - held.get(s, 0), next((x.family for x in decision.signals if x.symbol == s), "rebalance")) - for s, qty in decision.targets.items() if qty > held.get(s, 0)] + for s, qty in decision.targets.items() + if qty > held.get(s, 0) and s not in ca_block_entry] for symbol, side, quantity, reason in actions: quote = self.policy.latest.get(symbol) if not quote or now - quote.timestamp > self.policy.config.quote_age_seconds \ diff --git a/blueprints/us-equities/adaptive-paper/runner.py b/blueprints/us-equities/adaptive-paper/runner.py index 911bfbe80..87350037b 100644 --- a/blueprints/us-equities/adaptive-paper/runner.py +++ b/blueprints/us-equities/adaptive-paper/runner.py @@ -20,10 +20,11 @@ import signal import time +from corporate_actions import AlpacaCorporateActionsSource, CorporateActionMonitor from leverage import LeveragePolicyError, next_lower_rung_ceiling, validate_leverage_policy from safety import Ledger, Quote, RiskLimits, SafetyError, account_lock_fingerprint, DEFAULT_STOP from sessions import (DEFAULT_SESSION_POLICY, SessionKind, boundary_receipt, extended_session_close, - must_end_flat, session_at, validate_session_policy) + must_end_flat, next_trading_day, session_at, validate_session_policy) from strategies import AdaptivePolicy, PolicyConfig, RegimeSelector, SelectorConfig, limit_price from transport import AlpacaPaperTransport, DATA_FEEDS, TransportError, RejectedSubmission, preflight @@ -922,7 +923,8 @@ async def run_native(controller, policy_config, assets, trial_id, config, baseli policy = AdaptivePolicy(policy_config, strategy_pool=pool, selector=selector, leverage_policy=leverage_policy) strategy = AdaptiveStrategy(policy, controller.ledger, trial_id, event_sink=controller.events.append, transport=controller.port, - account_multiplier=config.get("_account_multiplier")) + account_multiplier=config.get("_account_multiplier"), + corporate_action_guard=config.get("_corporate_action_guard")) # Capture actual streaming quotes before native conversion. Every native # strategy event still arrives through the data engine's ordinary path. port = controller.port @@ -1057,6 +1059,26 @@ async def quote_sink(q): open_orders=len(controller.ledger.unresolved()), reconciled_at_boundary=True)) last_session_kind = now_kind if strategy.started: + # Trading-lane audit gap #8: refresh the corporate-action + # cache before this tick's rebalance() consults it. + # CorporateActionMonitor.refresh is internally rate-limited + # (refresh_seconds), so calling it every tick is safe -- most + # calls are a no-op cache-hit check, not a request. A session + # this tick's clock cannot classify skips the refresh + # entirely; rebalance()'s own guard call still fails closed + # independently on the same ValueError (see + # native_strategy._corporate_action_guard_symbols). + if strategy.corporate_action_guard is not None: + try: + ca_session = session_at(datetime.fromtimestamp(now, timezone.utc)) + ca_horizon_end = next_trading_day(ca_session.session_date) + except ValueError: + pass + else: + ca_watch = set(config["symbols"]) | { + p.symbol for p in controller.ledger.positions().values() if p.qty} + strategy.corporate_action_guard.refresh( + ca_watch, start=ca_session.session_date, end=ca_horizon_end, now=now) strategy.cancel_expired(now, config["order_timeout_seconds"], all_entries=force_exit) strategy.rebalance(now, force_exit=force_exit) if leverage_policy is not None: @@ -1400,6 +1422,14 @@ def main(): config, limits, policy_config = load_config(args.config) session_policy = validate_session_policy(config) key, secret = credentials(args.env_file) + # Trading-lane audit gap #8: constructed unconditionally (object + # construction only, no network call -- see AlpacaCorporateActionsSource's + # docstring) so it is available to run_native's AdaptiveStrategy below + # for both `paper` and a future recovery-time consultation; inert until + # its first evaluate()/refresh() call. Never logs or stores key/secret + # beyond the client object; the same env-file credential path every + # other broker call in this file already uses. + config["_corporate_action_guard"] = CorporateActionMonitor(AlpacaCorporateActionsSource(key, secret)) attempts, responses = [], [] def observe_request(kind, **kwargs): attempts.append({"timestamp": time.time(), "kind": kind}) diff --git a/blueprints/us-equities/adaptive-paper/sessions.py b/blueprints/us-equities/adaptive-paper/sessions.py index 16dc75a0d..93d082d30 100644 --- a/blueprints/us-equities/adaptive-paper/sessions.py +++ b/blueprints/us-equities/adaptive-paper/sessions.py @@ -221,6 +221,19 @@ def previous_trading_day(d: date) -> date: return _prev_trading_day(d) +def next_trading_day(d: date) -> date: + """The trading day immediately after ``d`` (skipping weekends/holidays), + public for the same reason ``previous_trading_day`` is: the corporate- + action guard's overnight hold horizon (corporate_actions.py, consulted + from native_strategy.py) is "today's session, extended through the + next trading session", and needs this without reaching into this + module's private ``_next_trading_day``. Raises ``ValueError`` on a date + outside the frozen session calendar, exactly like + ``previous_trading_day``; callers must fail closed on that, never guess + a horizon.""" + return _next_trading_day(d) + + # --------------------------------------------------------------------------- # Session policy: a single consolidated gate for the extended-hours and # overnight-holds decisions that previously existed as four separate ad-hoc diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index 496c64c23..f374f58be 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -7,6 +7,7 @@ "blueprints/us-equities/adaptive-paper/config-mover.json": "560849b980082a72f8ec45a32ffae51b1f50a502e739c94346ff501a46777ea8", "blueprints/us-equities/adaptive-paper/config-sip.json": "3587653104e68fdfc9bbb9170825c611abcf7e2e1be1004327a056031c34555a", "blueprints/us-equities/adaptive-paper/config.json": "2b555982034af02b9d0d2e1d20e17d22c1e83f2b0c4d60b61d91307ebec63562", + "blueprints/us-equities/adaptive-paper/corporate_actions.py": "42d51999fbba5e775547aa674dc1a304ee9ecdcfa31b00e1181b73ec2b05488c", "blueprints/us-equities/adaptive-paper/exits.py": "189133e6df99bda201d73d36a15f45ccbfb38979a3319ae6f75d077f5db5b3df", "blueprints/us-equities/adaptive-paper/feeds.py": "0db3009a724ae2433572c5070066d6806d7365031f9db4e81c4b69ad1cbebacb", "blueprints/us-equities/adaptive-paper/leverage.py": "4ad673522a5b3d76b6058da8fff8e7d8d27c43f2d243b87a5ef979db9a4c2803", @@ -18,22 +19,23 @@ "blueprints/us-equities/adaptive-paper/mover_simulation.py": "89d3c42273aecb3e41dbd13cbe3a14815728ffb014a90c444a2badeb271a6311", "blueprints/us-equities/adaptive-paper/mover_strategy.py": "5e369b963d1049589cb8fcf1d75ff2badc385067fa0517a958b9f1137c73d4ab", "blueprints/us-equities/adaptive-paper/native_adapter.py": "3e9cf2d93980b0f59b781dc3771f176c4e90d6af921b61c1022017cf85e64e0f", - "blueprints/us-equities/adaptive-paper/native_strategy.py": "7dbf1037b5909d0ea40d49f06740d74bee364ef4608e654350e18ee67b297e2b", + "blueprints/us-equities/adaptive-paper/native_strategy.py": "0164ca49af425a680447c26421cec57444247fbe7c6a9c9f71477f04276fbb2f", "blueprints/us-equities/adaptive-paper/receipts/adaptive_policy_v1.golden.json": "5cb4b21c7d2159e496093c67b3b358810db4e225d9c5ee1c633258050973556e", "blueprints/us-equities/adaptive-paper/recovery.py": "5cf7dcea572efbda6979a2726eebf6d676f6bd62263900c7b6626da8b45c1215", "blueprints/us-equities/adaptive-paper/registry.json": "60da73b985e718bd1dece7a3350f34636bda0fb38e766a44ae71c5a8c532282a", "blueprints/us-equities/adaptive-paper/requirements.txt": "518afe4b77f901e87abd3a5d6faa126aa7e211db77eecec80cfe7ef42d82e07a", - "blueprints/us-equities/adaptive-paper/runner.py": "a61012948d92eddef959ba91c5fdadc24170944239c02c20190e68c52d717245", + "blueprints/us-equities/adaptive-paper/runner.py": "52f69e5d016141951378943ffc3f616d87e1b7e652561a9b4ad05c1e2e25d747", "blueprints/us-equities/adaptive-paper/safety.py": "ad520fc4d84ce10e4d7b5cc956ba769a4f94eef49464ce382746d57f0cd76420", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", - "blueprints/us-equities/adaptive-paper/sessions.py": "1920952ac8219caf09e622fe517179e2766652fc7f987f7a11258481aa881998", + "blueprints/us-equities/adaptive-paper/sessions.py": "f922f6d9de0554bd5ee6040273479f071515909eda95469acbb01e00aa43d890", "blueprints/us-equities/adaptive-paper/simulation.py": "085e578e9fabe03ccefdc68b4796b9cf5012aca8d832d65107e4b95e0a80621b", "blueprints/us-equities/adaptive-paper/strategies.py": "2d488a4fdd21ad323ab07cf7fb147b3623fd6e38951abcf2fc59cf8094a1383e", "blueprints/us-equities/adaptive-paper/strategies_v1.py": "1952be269694485e12e4d927f124adca0ee29667588d6ac6651caede454c92f7", "blueprints/us-equities/adaptive-paper/transport.py": "b92e87524ccffd168b67b4f80418653e5309250fa23968af0988e2f016e0ed1e", "tests/adaptive_paper_hermetic.py": "34f2d5b07f42cb60d415abf24e4feef64b9088d1071163ab8427f0d958ebf7ee", "tests/test_adaptive_market_research.py": "dfd7a09380a7b1b10633c2edf05daf5a35bfcb48498fff130a2d84ff7190e395", + "tests/test_adaptive_paper_corporate_actions.py": "4ae68c41e9a129700a25dcca2d638b1912801897e128301fb79ddf89c80ed899", "tests/test_adaptive_paper_exits.py": "13a82d65ad1b2e72a8ca0fbb8bbb178a01ded1301399604c9e304ae213456764", "tests/test_adaptive_paper_live_manifest.py": "4596f301ac44effefee5b787c5e3627a9cd38f8ff0cf436ef778b7378a0667c2", "tests/test_adaptive_paper_metrics.py": "8b0627f752ddf11c87005200e76d0ad4873261905ba073163476c66659d564f9", diff --git a/manifests/evidence.json b/manifests/evidence.json index 511026c9c..d8470e6e1 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -6605,8 +6605,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/native_strategy.py", - "sha256": "7dbf1037b5909d0ea40d49f06740d74bee364ef4608e654350e18ee67b297e2b", - "bytes": 63708 + "sha256": "0164ca49af425a680447c26421cec57444247fbe7c6a9c9f71477f04276fbb2f", + "bytes": 68910 }, { "path": "blueprints/us-equities/adaptive-paper/receipt.json", @@ -6635,8 +6635,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/runner.py", - "sha256": "a61012948d92eddef959ba91c5fdadc24170944239c02c20190e68c52d717245", - "bytes": 99100 + "sha256": "52f69e5d016141951378943ffc3f616d87e1b7e652561a9b4ad05c1e2e25d747", + "bytes": 101223 }, { "path": "blueprints/us-equities/adaptive-paper/safety.py", @@ -6650,8 +6650,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/sessions.py", - "sha256": "1920952ac8219caf09e622fe517179e2766652fc7f987f7a11258481aa881998", - "bytes": 18940 + "sha256": "f922f6d9de0554bd5ee6040273479f071515909eda95469acbb01e00aa43d890", + "bytes": 19604 }, { "path": "blueprints/us-equities/adaptive-paper/simulation.py", @@ -6660,8 +6660,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/source-hashes.json", - "sha256": "dba8a3eac59f65e0ef8a2b38367eb6303f8fecb4b6ab92997dce265862adb4dd", - "bytes": 5870 + "sha256": "44b6fed1775f0023562ca33d640f5a2e722d1b2f0142c3bc0b188acff74b6e2d", + "bytes": 6122 }, { "path": "blueprints/us-equities/adaptive-paper/strategies.py", diff --git a/tests/test_adaptive_paper_corporate_actions.py b/tests/test_adaptive_paper_corporate_actions.py new file mode 100644 index 000000000..acdb57c89 --- /dev/null +++ b/tests/test_adaptive_paper_corporate_actions.py @@ -0,0 +1,417 @@ +"""Synthetic (SYN) tests for the trading-lane audit gap #8 interim control: +corporate_actions.py's pure guard/monitor logic, and its wiring into +native_strategy.AdaptiveStrategy.rebalance(). No credentials, sockets, or +broker execution; every source is a fake, matching every other +tests/test_adaptive_paper_*.py file in this directory. The real +alpaca-py-backed `corporate_actions.AlpacaCorporateActionsSource` is not +exercised here (see corporate_actions.py's module docstring for how its +field mapping was verified against the installed alpaca-py==0.44.0 +package instead). +""" +from dataclasses import dataclass +from datetime import date +from decimal import Decimal as D +import importlib.util +from pathlib import Path +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "blueprints/us-equities/adaptive-paper" +sys.path.insert(0, str(SOURCE)) + +import corporate_actions as CA # noqa: E402 (path inserted above) +import sessions # noqa: E402 + +NATIVE = importlib.util.find_spec("nautilus_trader") is not None +if NATIVE: + from nautilus_trader.model import OrderSide # noqa: E402 + +TODAY = date(2026, 9, 24) +NEXT_SESSION = date(2026, 9, 25) +FAR_FUTURE = date(2026, 12, 1) + + +def action(symbol, action_type, action_date, source_id=""): + return CA.ActionRecord(symbol=symbol, action_type=action_type, action_date=action_date, + source_id=source_id) + + +class ActionRecordTests(unittest.TestCase): + def test_unknown_action_type_rejected(self): + with self.assertRaises(ValueError): + CA.ActionRecord(symbol="AAPL", action_type="bogus", action_date=TODAY) + + def test_every_documented_action_type_accepted(self): + for action_type in ("forward_split", "reverse_split", "cash_dividend", "stock_dividend", + "spin_off", "merger", "name_change", "other"): + action("AAPL", action_type, TODAY) # must not raise + + +class EvaluateGuardTests(unittest.TestCase): + """Covers each action type, entry blocking, pre-close flattening, the + fail-closed path, and the absence of any effect on symbols without + actions -- run against corporate_actions.evaluate_guard directly.""" + + def evaluate(self, held, candidates, results): + return CA.evaluate_guard(today=TODAY, next_session_date=NEXT_SESSION, + held_symbols=held, candidate_symbols=candidates, + lookup_results=results) + + def test_each_action_type_on_a_held_symbol_blocks_and_flattens(self): + for action_type in ("forward_split", "reverse_split", "cash_dividend", "stock_dividend", + "spin_off", "merger", "name_change", "other"): + with self.subTest(action_type=action_type): + decisions = self.evaluate({"AAPL"}, set(), {"AAPL": [action("AAPL", action_type, TODAY)]}) + d = decisions["AAPL"] + self.assertTrue(d.block_entry) + self.assertTrue(d.must_flatten) + self.assertFalse(d.needs_attention) + self.assertEqual(d.reason, f"corporate_action_{action_type}") + self.assertEqual(d.action_type, action_type) + self.assertEqual(d.action_date, TODAY) + + def test_action_on_next_session_date_also_qualifies(self): + decisions = self.evaluate({"AAPL"}, set(), {"AAPL": [action("AAPL", "reverse_split", NEXT_SESSION)]}) + self.assertTrue(decisions["AAPL"].block_entry) + self.assertTrue(decisions["AAPL"].must_flatten) + + def test_action_outside_the_hold_horizon_has_no_effect(self): + decisions = self.evaluate({"AAPL"}, set(), {"AAPL": [action("AAPL", "cash_dividend", FAR_FUTURE)]}) + d = decisions["AAPL"] + self.assertFalse(d.block_entry) + self.assertFalse(d.must_flatten) + self.assertFalse(d.needs_attention) + self.assertIsNone(d.reason) + + def test_symbol_with_no_recorded_actions_at_all_has_no_effect(self): + decisions = self.evaluate({"AAPL"}, {"MSFT"}, {"AAPL": [], "MSFT": []}) + self.assertFalse(decisions["AAPL"].block_entry) + self.assertFalse(decisions["AAPL"].must_flatten) + self.assertFalse(decisions["MSFT"].block_entry) + + def test_candidate_not_currently_held_is_blocked_but_never_flattened(self): + decisions = self.evaluate(set(), {"MSFT"}, {"MSFT": [action("MSFT", "spin_off", TODAY)]}) + d = decisions["MSFT"] + self.assertTrue(d.block_entry) + self.assertFalse(d.must_flatten) + + def test_lookup_failed_fails_closed_blocking_entry_without_forcing_flatten(self): + decisions = self.evaluate({"AAPL"}, {"MSFT"}, {"AAPL": CA.LOOKUP_FAILED, "MSFT": CA.LOOKUP_FAILED}) + for symbol in ("AAPL", "MSFT"): + d = decisions[symbol] + self.assertTrue(d.block_entry) + self.assertTrue(d.needs_attention) + self.assertFalse(d.must_flatten) + self.assertEqual(d.reason, "corporate_action_lookup_failed") + + def test_lookup_ambiguous_fails_closed_the_same_way(self): + decisions = self.evaluate({"AAPL"}, set(), {"AAPL": CA.LOOKUP_AMBIGUOUS}) + d = decisions["AAPL"] + self.assertTrue(d.block_entry) + self.assertTrue(d.needs_attention) + self.assertEqual(d.reason, "corporate_action_lookup_ambiguous") + + def test_symbol_missing_from_results_entirely_fails_closed(self): + decisions = self.evaluate({"AAPL"}, set(), {}) + d = decisions["AAPL"] + self.assertTrue(d.block_entry) + self.assertTrue(d.needs_attention) + self.assertEqual(d.reason, "corporate_action_lookup_failed") + + def test_earliest_qualifying_action_governs_when_several_are_recorded(self): + decisions = self.evaluate({"AAPL"}, set(), {"AAPL": [ + action("AAPL", "cash_dividend", NEXT_SESSION), action("AAPL", "reverse_split", TODAY)]}) + d = decisions["AAPL"] + self.assertEqual(d.action_type, "reverse_split") + self.assertEqual(d.action_date, TODAY) + + def test_only_relevant_symbols_are_decided(self): + decisions = self.evaluate({"AAPL"}, {"MSFT"}, {"AAPL": [], "MSFT": [], "IGNORED": []}) + self.assertEqual(set(decisions), {"AAPL", "MSFT"}) + + +class CorporateActionMonitorTests(unittest.TestCase): + class FakeSource: + def __init__(self, result=None, error=None): + self.result = result or {} + self.error = error + self.calls = [] + + def fetch(self, symbols, start, end): + self.calls.append((tuple(symbols), start, end)) + if self.error is not None: + raise self.error + return {s: self.result.get(s, []) for s in symbols} + + def test_evaluate_before_any_refresh_fails_closed(self): + monitor = CA.CorporateActionMonitor(self.FakeSource(), clock=lambda: 1000.0) + decisions = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, + held_symbols={"AAPL"}, candidate_symbols=set(), now=1000.0) + self.assertTrue(decisions["AAPL"].needs_attention) + self.assertTrue(decisions["AAPL"].block_entry) + + def test_successful_refresh_feeds_evaluate(self): + source = self.FakeSource(result={"AAPL": [action("AAPL", "forward_split", TODAY)]}) + monitor = CA.CorporateActionMonitor(source, clock=lambda: 1000.0) + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + decisions = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, + held_symbols={"AAPL"}, candidate_symbols=set(), now=1000.0) + self.assertEqual(decisions["AAPL"].reason, "corporate_action_forward_split") + self.assertFalse(decisions["AAPL"].needs_attention) + + def test_lookup_error_fails_closed_for_every_requested_symbol(self): + source = self.FakeSource(error=CA.CorporateActionLookupError("boom")) + monitor = CA.CorporateActionMonitor(source, clock=lambda: 1000.0) + monitor.refresh({"AAPL", "MSFT"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + decisions = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, + held_symbols={"AAPL"}, candidate_symbols={"MSFT"}, now=1000.0) + self.assertTrue(decisions["AAPL"].needs_attention) + self.assertTrue(decisions["MSFT"].needs_attention) + + def test_unexpected_exception_from_source_also_fails_closed(self): + source = self.FakeSource(error=RuntimeError("unexpected")) + monitor = CA.CorporateActionMonitor(source, clock=lambda: 1000.0) + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + decisions = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, + held_symbols={"AAPL"}, candidate_symbols=set(), now=1000.0) + self.assertTrue(decisions["AAPL"].needs_attention) + + def test_stale_result_past_max_age_fails_closed_even_after_a_success(self): + source = self.FakeSource(result={"AAPL": []}) + monitor = CA.CorporateActionMonitor(source, refresh_seconds=1, max_age_seconds=100, clock=lambda: 1000.0) + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + fresh = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, + held_symbols={"AAPL"}, candidate_symbols=set(), now=1050.0) + self.assertFalse(fresh["AAPL"].needs_attention) + stale = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, + held_symbols={"AAPL"}, candidate_symbols=set(), now=1200.0) + self.assertTrue(stale["AAPL"].needs_attention) + + def test_refresh_is_throttled_within_refresh_seconds_for_the_same_symbols(self): + source = self.FakeSource(result={"AAPL": []}) + monitor = CA.CorporateActionMonitor(source, refresh_seconds=900, clock=lambda: 1000.0) + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1001.0) + self.assertEqual(len(source.calls), 1) + + def test_refresh_widens_the_fetch_when_a_new_symbol_appears(self): + source = self.FakeSource(result={"AAPL": [], "MSFT": []}) + monitor = CA.CorporateActionMonitor(source, refresh_seconds=900, clock=lambda: 1000.0) + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + monitor.refresh({"AAPL", "MSFT"}, start=TODAY, end=NEXT_SESSION, now=1001.0) + self.assertEqual(len(source.calls), 2) + + +@unittest.skipUnless(NATIVE, "requires pinned Nautilus 2.0.0rc5 runtime") +class RebalanceCorporateActionGuardTests(unittest.TestCase): + """Deliverable: native_strategy.AdaptiveStrategy.rebalance() consults + the injected corporate-action guard -- entry blocking, pre-close + flattening, precedence over gap_risk_stop, and no effect when the guard + is absent or reports nothing relevant. Same FakeStrategy construction + pattern as tests/test_adaptive_paper_exits.py's ReplaceExitTests and + tests/test_adaptive_paper_runner.py's FakeStrategy.""" + + class _FakeQuote: + def __init__(self, bid, ask, timestamp): + self.bid, self.ask, self.timestamp = bid, ask, timestamp + + class _FakePolicyConfig: + def __init__(self, symbols=("AAPL", "MSFT"), quote_age_seconds=3, max_shares=5, + gap_stop_enabled=False, exit_replace_enabled=False): + self.symbols = symbols + self.quote_age_seconds = quote_age_seconds + self.max_shares = max_shares + self.gap_stop_enabled = gap_stop_enabled + self.exit_replace_enabled = exit_replace_enabled + + @dataclass + class _FakePosition: + symbol: str + qty: D + average_cost: float = 100.0 + + class _FakeDecision: + def __init__(self, targets, exits=None, regime="trend", effective_leverage=1.0, signals=()): + self.targets, self.exits = targets, exits or {} + self.regime, self.effective_leverage, self.signals = regime, effective_leverage, signals + + class _FakePolicy: + def __init__(self, config, decision): + self.latest = {} + self.config = config + self.selector = None + self.last_exit_fractions = {} + self._decision = decision + + def sync_positions(self, positions, now): + pass + + def decide(self, now, *, allow_entries, force_exit, operational=None): + return self._decision + + class _FakeLedger: + def __init__(self, positions=None): + self._positions = positions or {} + + def intents(self): + return [] + + def prior_rth_closes(self): + return {} + + def unresolved(self): + return [] + + def positions(self): + return self._positions + + class _FakeGuard: + """Scripted guard: returns exactly the decisions it was constructed + with (keyed by symbol), regardless of the arguments rebalance() + passes -- the args themselves are recorded for assertion.""" + + def __init__(self, decisions): + self.decisions = decisions + self.calls = [] + + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + self.calls.append({"today": today, "next_session_date": next_session_date, + "held_symbols": set(held_symbols), "candidate_symbols": set(candidate_symbols)}) + return self.decisions + + def strategy(self, *, held=None, targets=None, guard_decisions=None, clock=lambda: 1790256600.0): + # 2026-09-24 09:30:00 America/New_York in epoch seconds (RTH open) -- + # picked so session_at()/next_trading_day() both resolve without a + # ValueError under the frozen 2026 calendar (verified: session_date + # 2026-09-24, next_trading_day 2026-09-25 -- this file's module-level + # TODAY/NEXT_SESSION constants). + from native_strategy import AdaptiveStrategy + + class FakeOrderFactory: + def __init__(self): + self.calls = [] + + def limit(self, instrument_id, side, quantity, price, *, time_in_force, client_order_id, tags): + self.calls.append({"instrument_id": str(instrument_id), "side": side, + "quantity": str(quantity), "price": str(price), "tags": list(tags)}) + return self.calls[-1] + + class FakeStrategy(AdaptiveStrategy): + @property + def order_factory(self): + return self._fake_order_factory + + held = held or {} + positions = {s: self._FakePosition(symbol=s, qty=D(str(q))) for s, q in held.items()} + config = self._FakePolicyConfig() + decision = self._FakeDecision(targets=targets or {}) + policy = self._FakePolicy(config, decision) + guard = self._FakeGuard(guard_decisions) if guard_decisions is not None else None + strat = FakeStrategy(policy, self._FakeLedger(positions), "fixture", clock=clock, + corporate_action_guard=guard) + strat._fake_order_factory = FakeOrderFactory() + strat.submitted, strat.events = [], [] + strat.submit_order = strat.submitted.append + strat.event_sink = strat.events.append + strat.started = True + strat.enabled = True + for symbol in set(held) | set(targets or {}): + policy.latest[symbol] = self._FakeQuote(100.0, 100.02, clock()) + return strat + + def decision_for(self, symbol, *, block_entry=False, must_flatten=False, needs_attention=False, + reason=None, action_type=None, action_date=None): + return CA.GuardDecision(symbol=symbol, block_entry=block_entry, must_flatten=must_flatten, + needs_attention=needs_attention, reason=reason, + action_type=action_type, action_date=action_date) + + def test_no_guard_configured_is_inert(self): + strategy = self.strategy(held={}, targets={"AAPL": 1}) + strategy.rebalance() + self.assertEqual(len(strategy.submitted), 1) + self.assertEqual(strategy._fake_order_factory.calls[0]["side"], OrderSide.BUY) + + def test_blocked_entry_symbol_never_gets_a_buy_order(self): + guard_decisions = {"AAPL": self.decision_for("AAPL", block_entry=True, needs_attention=True, + reason="corporate_action_lookup_failed"), + "MSFT": self.decision_for("MSFT")} + strategy = self.strategy(held={}, targets={"AAPL": 1, "MSFT": 1}, guard_decisions=guard_decisions) + strategy.rebalance() + self.assertEqual(len(strategy.submitted), 1) + instrument_ids = [c["instrument_id"] for c in strategy._fake_order_factory.calls] + self.assertNotIn("AAPL.ALPACA", instrument_ids) + self.assertIn("MSFT.ALPACA", instrument_ids) + + def test_held_symbol_with_qualifying_action_is_flattened_before_close(self): + guard_decisions = {"AAPL": self.decision_for("AAPL", block_entry=True, must_flatten=True, + reason="corporate_action_reverse_split", + action_type="reverse_split", action_date=TODAY)} + strategy = self.strategy(held={"AAPL": 3}, targets={"AAPL": 3}, guard_decisions=guard_decisions) + strategy.rebalance() + self.assertEqual(len(strategy.submitted), 1) + call = strategy._fake_order_factory.calls[0] + self.assertEqual(call["side"], OrderSide.SELL) + self.assertEqual(call["quantity"], "3") + self.assertIn("reason=corporate_action_flatten", call["tags"]) + + def test_flatten_still_forced_even_when_decide_would_have_kept_the_position(self): + """Unlike an ordinary rebalance-delta sell, the forced flatten fires + even though this tick's own decision.targets kept the full + quantity (mirrors gap_risk_stop's own override of the ordinary + held<=target skip).""" + guard_decisions = {"AAPL": self.decision_for("AAPL", block_entry=True, must_flatten=True, + reason="corporate_action_merger", + action_type="merger", action_date=TODAY)} + strategy = self.strategy(held={"AAPL": 2}, targets={"AAPL": 2}, guard_decisions=guard_decisions) + strategy.rebalance() + self.assertEqual(len(strategy.submitted), 1) + self.assertEqual(strategy._fake_order_factory.calls[0]["side"], OrderSide.SELL) + + def test_symbol_without_any_action_is_unaffected(self): + guard_decisions = {"AAPL": self.decision_for("AAPL")} + strategy = self.strategy(held={}, targets={"AAPL": 2}, guard_decisions=guard_decisions) + strategy.rebalance() + self.assertEqual(len(strategy.submitted), 1) + self.assertEqual(strategy._fake_order_factory.calls[0]["side"], OrderSide.BUY) + self.assertEqual(strategy._fake_order_factory.calls[0]["quantity"], "2") + + def test_needs_attention_events_are_recorded_without_forcing_a_flatten(self): + guard_decisions = {"AAPL": self.decision_for("AAPL", block_entry=True, needs_attention=True, + reason="corporate_action_lookup_ambiguous")} + strategy = self.strategy(held={"AAPL": 1}, targets={"AAPL": 1}, guard_decisions=guard_decisions) + strategy.rebalance() + self.assertEqual(strategy.submitted, []) # not blocked from holding, not forced to flatten + events = [e for e in strategy.events if e.get("type") == "corporate_action_guard"] + self.assertEqual(len(events), 1) + self.assertTrue(events[0]["needs_attention"]) + self.assertFalse(events[0]["must_flatten"]) + + def test_flatten_event_carries_no_credential_value_only_bounded_fields(self): + guard_decisions = {"AAPL": self.decision_for("AAPL", block_entry=True, must_flatten=True, + reason="corporate_action_cash_dividend", + action_type="cash_dividend", action_date=TODAY)} + strategy = self.strategy(held={"AAPL": 1}, targets={"AAPL": 1}, guard_decisions=guard_decisions) + strategy.rebalance() + events = [e for e in strategy.events if e.get("type") == "corporate_action_guard"] + self.assertEqual(len(events), 1) + self.assertEqual(set(events[0]), {"type", "symbol", "block_entry", "must_flatten", + "needs_attention", "reason", "action_type", "action_date"}) + + def test_decision_event_carries_the_guard_summary_lists(self): + guard_decisions = {"AAPL": self.decision_for("AAPL", block_entry=True, must_flatten=True, + reason="corporate_action_spin_off", + action_type="spin_off", action_date=TODAY), + "MSFT": self.decision_for("MSFT", block_entry=True, + reason="corporate_action_lookup_failed", + needs_attention=True)} + strategy = self.strategy(held={"AAPL": 1}, targets={"AAPL": 1, "MSFT": 1}, guard_decisions=guard_decisions) + strategy.rebalance() + events = [e for e in strategy.events if e.get("type") == "decision"] + self.assertEqual(len(events), 1) + self.assertEqual(sorted(events[0]["corporate_action_must_flatten"]), ["AAPL"]) + self.assertEqual(sorted(events[0]["corporate_action_block_entry"]), ["AAPL", "MSFT"]) + + +if __name__ == "__main__": + unittest.main() From 2a671a2def1be34e97f7fc8679685a88a11d47fa Mon Sep 17 00:00:00 2001 From: Scout Date: Thu, 24 Sep 2026 14:57:11 -0400 Subject: [PATCH 2/8] Record a native read-only check of the corporate-action source (NVDA split, AAPL dividend, TSLA none) AlpacaCorporateActionsSource.fetch returned NVDA forward_split 2024-06-10 and AAPL cash_dividend 2024-08-12 (both the correct ex-dates) and nothing for TSLA in its window, using this host's dedicated paper keys through runner.credentials. Read-only; no orders. Co-Authored-By: Claude Opus 5.5 (1M context) --- ...rporate-actions-native-check-20260924.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json diff --git a/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json b/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json new file mode 100644 index 000000000..e5bf70e90 --- /dev/null +++ b/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json @@ -0,0 +1,22 @@ +{ + "schema_version": 1, + "kind": "native_read_only_check", + "evidence_class": "native_network", + "observed_on": "2026-09-24", + "subject": "blueprints/us-equities/adaptive-paper/corporate_actions.py AlpacaCorporateActionsSource.fetch", + "method": "Coordinator ran the source's fetch() against Alpaca's corporate-actions data endpoint with the host's dedicated paper account keys (loaded through adaptive-paper runner.credentials from the alpaca-paper inventory entry; never printed). Read-only market-data requests; no orders.", + "runtime": "alpaca-py 0.44.0, Python 3.12.3 (pinned adaptive-paper runtime)", + "queries": [ + {"symbols": ["NVDA"], "start": "2024-06-01", "end": "2024-06-15", + "observed": [{"symbol": "NVDA", "action_type": "forward_split", "action_date": "2024-06-10"}], + "expected": "NVDA 10-for-1 forward split, ex-date 2024-06-10", "match": true}, + {"symbols": ["AAPL"], "start": "2024-08-05", "end": "2024-08-16", + "observed": [{"symbol": "AAPL", "action_type": "cash_dividend", "action_date": "2024-08-12"}], + "expected": "AAPL quarterly cash dividend, ex-date 2024-08-12", "match": true}, + {"symbols": ["TSLA"], "start": "2024-06-01", "end": "2024-06-15", + "observed": [], + "expected": "no corporate action in the window", "match": true} + ], + "claim_boundary": "Confirms that the live Alpaca source returns and normalizes these three historical windows correctly (type and governing date). It does not exercise the guard inside a running paper trial, a multi-day overnight resume, the refresh cadence against the endpoint's rate limits, or action types other than forward split and cash dividend.", + "not_retained": "The raw API responses and the exact command line were not retained; the observed values above are the complete normalized output." +} From f5119bf2a27f45cc28da7c2dbae0251595d8c1da Mon Sep 17 00:00:00 2001 From: Scout Date: Thu, 24 Sep 2026 16:05:36 -0400 Subject: [PATCH 3/8] Verify the 2027 NYSE calendar against its sources; make the out-of-range test hermetic - sessions.py: HOLIDAYS_2027/EARLY_CLOSES_2027 now cite the NYSE hours-calendars page (fetched 2026-09-24; lists 2026-2028) and the same-day exchange_calendars 4.13.2 XNYS cross-check, which agrees exactly (10 weekday closures, early close 2027-11-26). exchange_calendars_agrees(year) generalizes the 2026 check with padded bounds (Jan 1 is not a session). - tests: add the 2027 agreement test; test_year_outside_calendar_refused now disables the exchange_calendars fallback, since CI installs that package and the test previously passed only where it was absent. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../us-equities/adaptive-paper/sessions.py | 43 ++++++++----------- .../adaptive-paper/source-hashes.json | 4 +- manifests/evidence.json | 10 ++--- tests/test_adaptive_paper_sessions.py | 26 ++++++++--- 4 files changed, 44 insertions(+), 39 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/sessions.py b/blueprints/us-equities/adaptive-paper/sessions.py index 7d40c3c21..b6fe3be16 100644 --- a/blueprints/us-equities/adaptive-paper/sessions.py +++ b/blueprints/us-equities/adaptive-paper/sessions.py @@ -64,25 +64,11 @@ date(2026, 12, 24), # Christmas Eve }) -# NYSE 2027 full-closure holidays (finding 6, 2026-09-24 fix round). This -# sandboxed environment has no outbound network access to re-fetch -# https://www.nyse.com/markets/hours-calendars directly (verified: a direct -# curl from this host returns a redirect with no page body), so these dates -# are derived the same way the module docstring already documents the 2026 -# table's own cross-check rules (New Year's Day; MLK Day, 3rd Monday of -# January; Washington's Birthday, 3rd Monday of February; Good Friday, -# Gregorian Easter Sunday minus two days; Memorial Day, last Monday of May; -# Juneteenth, June 19 fixed, observed the preceding Friday when it falls on -# a Saturday; Independence Day, July 4 fixed, observed the preceding Friday -# on a Saturday or the following Monday on a Sunday; Labor Day, 1st Monday -# of September; Thanksgiving Day, 4th Thursday of November; Christmas Day, -# December 25 fixed, observed the preceding Friday on a Saturday or the -# following Monday on a Sunday) -- the same NYSE observance convention as -# every other year's calendar, not a guess. Independently reconcilable -# against `exchange_calendars`' XNYS calendar (see -# `exchange_calendars_agrees_2026`'s 2026-only sibling check) or the -# official NYSE calendar once network access is available; treat as -# unverified against the live NYSE page pending that reconciliation. +# NYSE 2027 full-closure holidays. Source: the NYSE published holiday calendar, +# https://www.nyse.com/markets/hours-calendars (fetched 2026-09-24; it lists +# 2026, 2027 and 2028). Cross-checked the same day against exchange_calendars +# 4.13.2's XNYS calendar, whose 2027 weekday closures and early closes equal +# these two sets exactly. HOLIDAYS_2027 = frozenset({ date(2027, 1, 1), # New Year's Day (Friday) date(2027, 1, 18), # Martin Luther King, Jr. Day @@ -100,7 +86,7 @@ # Friday after Thanksgiving applies this year: Christmas Eve (Dec 24, 2027) # is itself the observed Christmas Day full closure above (Dec 25 falls on a # Saturday), so there is no separate Christmas Eve half day in 2027. Same -# derivation/verification caveat as HOLIDAYS_2027 above. +# source and cross-check as HOLIDAYS_2027 above. EARLY_CLOSES_2027 = frozenset({ date(2027, 11, 26), # Friday after Thanksgiving }) @@ -449,16 +435,23 @@ def must_end_flat(policy: dict, *, is_final_boundary: bool) -> bool: # --------------------------------------------------------------------------- def exchange_calendars_agrees_2026(): + """The 2026 case of ``exchange_calendars_agrees``.""" + return exchange_calendars_agrees(2026) + + +def exchange_calendars_agrees(year): """Return True/False if ``exchange_calendars`` is importable and its XNYS - calendar was compared against the frozen 2026 table above, or None if the - package is not installed. Never raises for an absent package.""" + calendar was compared against the frozen table for ``year`` above, or None + if the package is not installed. Never raises for an absent package.""" try: import exchange_calendars as xcals except ImportError: return None - cal = xcals.get_calendar("XNYS") - d = date(2026, 1, 1) - end = date(2026, 12, 31) + # Padded so every day of the year is inside the calendar's bounds even when + # the first or last day of the year is not a session. + cal = xcals.get_calendar("XNYS", start=f"{year - 1}-12-01", end=f"{year + 1}-01-31") + d = date(year, 1, 1) + end = date(year, 12, 31) while d <= end: expected_open = _is_trading_day(d) actual_open = bool(cal.is_session(d.isoformat())) diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index 58284cc83..a001ce0a2 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -28,7 +28,7 @@ "blueprints/us-equities/adaptive-paper/safety.py": "ad520fc4d84ce10e4d7b5cc956ba769a4f94eef49464ce382746d57f0cd76420", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", - "blueprints/us-equities/adaptive-paper/sessions.py": "52eac36b3cb38a72d78657c7c00275527161ca50296b010b13f7af701ced66a7", + "blueprints/us-equities/adaptive-paper/sessions.py": "95f0affee512e88dd44f196fa72bb7471e00899356793bdbb12f15c4ed84e64d", "blueprints/us-equities/adaptive-paper/simulation.py": "085e578e9fabe03ccefdc68b4796b9cf5012aca8d832d65107e4b95e0a80621b", "blueprints/us-equities/adaptive-paper/strategies.py": "2d488a4fdd21ad323ab07cf7fb147b3623fd6e38951abcf2fc59cf8094a1383e", "blueprints/us-equities/adaptive-paper/strategies_v1.py": "1952be269694485e12e4d927f124adca0ee29667588d6ac6651caede454c92f7", @@ -46,7 +46,7 @@ "tests/test_adaptive_paper_runner.py": "c9d9ce3d6c2ee61de4afd842000e792a1b59685aaa3ab8d35a48257c5454735b", "tests/test_adaptive_paper_safety.py": "e0b2017ea6bdae87a0778b01ac72747642b31a2495c6d4e8bc98619a5aa1b456", "tests/test_adaptive_paper_selector.py": "27b431ac7911372d050f25d32ac24a3df9d2cf3c5e7dafdebfe51db6cab55936", - "tests/test_adaptive_paper_sessions.py": "40f036aa36475e5f0b4c821c2f53a047a60eeb145c76a6b3a9340246f9401f70", + "tests/test_adaptive_paper_sessions.py": "09a4aaf68b3d8857bdceddae91ea11d1575e000eeeda85496e1dbeb321d8495d", "tests/test_adaptive_paper_strategies.py": "f0814b36909f2fc6b4c4213b4142ffc13339740f0397c99ad9c7c81982e2033a", "tests/test_adaptive_paper_transport.py": "eaf266a564749de4202e8e5828cbec67901027572a3c9c002473ba4981249a17" } diff --git a/manifests/evidence.json b/manifests/evidence.json index d2083dfd2..c780288a5 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -6655,8 +6655,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/sessions.py", - "sha256": "52eac36b3cb38a72d78657c7c00275527161ca50296b010b13f7af701ced66a7", - "bytes": 22320 + "sha256": "95f0affee512e88dd44f196fa72bb7471e00899356793bdbb12f15c4ed84e64d", + "bytes": 21625 }, { "path": "blueprints/us-equities/adaptive-paper/simulation.py", @@ -6665,7 +6665,7 @@ }, { "path": "blueprints/us-equities/adaptive-paper/source-hashes.json", - "sha256": "f30e77fdd33b4b9ab4e8911dafa0ebf47517fc385b17354eaf4bb16de141aa4e", + "sha256": "a665741f11ed6bd087ae22d3baaefde99f4ceda0280c0764adfc09baeaaa777f", "bytes": 6122 }, { @@ -28970,8 +28970,8 @@ }, { "path": "tests/test_adaptive_paper_sessions.py", - "sha256": "40f036aa36475e5f0b4c821c2f53a047a60eeb145c76a6b3a9340246f9401f70", - "bytes": 95635 + "sha256": "09a4aaf68b3d8857bdceddae91ea11d1575e000eeeda85496e1dbeb321d8495d", + "bytes": 96318 }, { "path": "tests/test_adaptive_paper_strategies.py", diff --git a/tests/test_adaptive_paper_sessions.py b/tests/test_adaptive_paper_sessions.py index dfb934eca..5e8649e68 100644 --- a/tests/test_adaptive_paper_sessions.py +++ b/tests/test_adaptive_paper_sessions.py @@ -11,6 +11,7 @@ import sys import time import unittest +from unittest import mock from zoneinfo import ZoneInfo ROOT = Path(__file__).resolve().parents[1] @@ -145,16 +146,27 @@ def test_exchange_calendars_agreement_or_absent(self): self.skipTest("exchange_calendars is not installed in this shared engine environment") self.assertTrue(result) + def test_exchange_calendars_agreement_2027_or_absent(self): + # The 2027 table (NYSE hours-calendars page, fetched 2026-09-24) must + # match XNYS session by session, including the 13:00 early close. + result = sess.exchange_calendars_agrees(2027) + if result is None: + self.skipTest("exchange_calendars is not installed in this shared engine environment") + self.assertTrue(result) + def test_year_outside_calendar_refused(self): # D7/finding-6 (2026-09-24 fix round): dates outside the frozen # table's year(s) must not be silently classified against another - # year's table. exchange_calendars is not installed in this - # environment, so both raise. 2027 is now covered by HOLIDAYS_2027 - # (see test_2027_calendar_covered below); 2028 remains uncovered. - with self.assertRaisesRegex(ValueError, "session_calendar_out_of_range"): - sess.session_at(ny(2025, 12, 25, 12, 0)) - with self.assertRaisesRegex(ValueError, "session_calendar_out_of_range"): - sess.session_at(ny(2028, 1, 1, 12, 0)) + # year's table. The exchange_calendars fallback is disabled here so + # the result does not depend on whether that package is installed + # (CI installs it from blueprints/us-equities/data/requirements.lock). + # 2027 is covered by HOLIDAYS_2027 (see test_2027_calendar_covered + # below); 2025 and 2028 are not. + with mock.patch.object(sess, "_exchange_calendars_day", return_value=None): + with self.assertRaisesRegex(ValueError, "session_calendar_out_of_range"): + sess.session_at(ny(2025, 12, 25, 12, 0)) + with self.assertRaisesRegex(ValueError, "session_calendar_out_of_range"): + sess.session_at(ny(2028, 1, 1, 12, 0)) def test_year_inside_calendar_still_works(self): info = sess.session_at(ny(2026, 12, 31, 12, 0)) From 7681133fb0302a416ba09852d85acd3f6abdbc76 Mon Sep 17 00:00:00 2001 From: Scout Date: Thu, 24 Sep 2026 17:18:41 -0400 Subject: [PATCH 4/8] Fix round 3: close 10 review findings on the corporate-action guard Both independent reviews (Claude/rev-ca2 and Codex) found round 2 "not ready". Addresses every item: - HIGH (1): overlapping refreshes could let an older, slower attempt overwrite a newer confirmed action. CorporateActionMonitor now tracks a generation counter under a lock; refresh_timed_out() (called on a wait_for timeout) bumps it and treats the timeout as an immediate failure (short retry_seconds interval); _apply_success/_apply_failure drop any write from a superseded generation. The preflight fetch and tick-loop refresh now share one ca_refresh_state dict (in_flight + tasks), and run_native's shutdown path (_shutdown_corporate_action_tasks) cancels/joins every still-pending refresh task. - HIGH (2): the SDK's CorporateActionsSet parser (13 branches, no else/default) silently dropped reorganization/partial_call/capital_ gains_distribution buckets before this wrapper ever saw them, evading both unknown-type rejection and the cap check. AlpacaCorporateActionsSource now constructs with raw_data=True and parses the raw response directly via the private _get_marketdata, so every bucket (mapped or not) reaches this wrapper's own fail-closed logic. New tests fake the HTTP transport (not a pre-parsed response) so the real SDK code path runs. - HIGH (3): the request now asks data_quality=all (the documented default, complete, excludes incomplete records); a record missing its governing date marks only its own symbol(s) LOOKUP_AMBIGUOUS, not the whole fetch. - MEDIUM (4): the guard's summary now tracks pending_needs_attention_held (every currently-held symbol not positively verified clear); _honest_overnight_hold refuses held_overnight while it is non-empty, not just while pending_must_flatten is non-empty. - MEDIUM (5): the guard now evaluates every symbol with a still-resting pending BUY order, not just decision.targets.keys(), so a corporate action discovered after a signal disappears still cancels that resting buy. The fake guard test fixture now filters to the requested held|candidate set instead of echoing every scripted decision. - MEDIUM (6): the 90-day forward pad is now explicitly documented as an evidence-backed assumption (3 recorded lag observations, sourced), not a proven bound. - LOW (7): a symbol omitted from an otherwise-successful fetch is now degraded (last good record preserved), never overwritten with LOOKUP_FAILED. - LOW (8): corporate-actions-native-check-20260924.json is corrected (the 2024 window is explicitly labeled a manual approximation since sessions.py has no 2024 calendar; the real _corporate_action_fetch_window output is recorded separately; the AAPL dividend's lag is corrected to the measured 3 days, replacing the unmeasured "1-3 weeks" claim) and registered in manifests/evidence.json. - NIT (9): the "bounded fetch"/pagination comments are corrected (a requests timeout bounds each read, not the whole paginated call; the SDK does paginate); a symbol that was previously flagged and resolves now emits an explicit "cleared" event. - NIT (10): the session test's stale "CI installs that package" comment is corrected. Re-ran the review's own mutate_rev.py unmodified (just retargeted): all 24 mutations now kill (22 directly; F2b/F5a/F5b's exact pre-fix text no longer exists after this round's refactor into named, independently testable functions -- their equivalents are covered by this round's own mutation probe, also all killed). Added a matching native re-check (raw-record lag query, data_quality=all) using the same credential-loading recipe as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- ...rporate-actions-native-check-20260924.json | 37 +- .../adaptive-paper/corporate_actions.py | 271 +++++++++---- .../adaptive-paper/native_strategy.py | 62 ++- .../us-equities/adaptive-paper/runner.py | 239 +++++++++--- .../adaptive-paper/source-hashes.json | 12 +- manifests/evidence.json | 23 +- .../test_adaptive_paper_corporate_actions.py | 348 +++++++++++++++-- tests/test_adaptive_paper_runner.py | 365 +++++++++++++++++- tests/test_adaptive_paper_sessions.py | 6 +- 9 files changed, 1152 insertions(+), 211 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json b/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json index 6d3e7810d..c90b74391 100644 --- a/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json +++ b/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json @@ -4,7 +4,7 @@ "evidence_class": "native_network", "observed_on": "2026-09-24", "subject": "blueprints/us-equities/adaptive-paper/corporate_actions.py AlpacaCorporateActionsSource.fetch", - "method": "Coordinator ran the source's fetch() against Alpaca's corporate-actions data endpoint with the host's dedicated paper account keys (loaded through adaptive-paper runner.credentials from the alpaca-paper inventory entry; never printed). Read-only market-data requests; no orders.", + "method": "Coordinator ran the source's fetch() (and, for the raw-lag row below, the private _get_marketdata it now calls directly for data_quality=all) against Alpaca's corporate-actions data endpoint with the host's dedicated paper account keys (loaded through adaptive-paper runner.credentials from the alpaca-paper inventory entry; never printed). Read-only market-data requests; no orders.", "runtime": "alpaca-py 0.44.0, Python 3.12.3 (pinned adaptive-paper runtime)", "queries": [ {"symbols": ["NVDA"], "start": "2024-06-01", "end": "2024-06-15", @@ -18,13 +18,38 @@ "expected": "no corporate action in the window", "match": true}, {"symbols": ["AAPL"], "start": "2024-08-09", "end": "2024-08-12", "observed": [], - "expected": "CRITICAL finding 1 (fix round, 2026-09-24): this is the guard's actual PRE-FIX production window -- today..next_session around the real 2024-08-12 ex-date -- proving the endpoint filters by process date (a cash dividend's payable date, typically 1-3 weeks after ex_date), not ex_date, so a narrow window systematically misses a real dividend it should have caught", + "expected": "CRITICAL finding 1 (fix round 2, 2026-09-24): this is the guard's actual PRE-FIX production window -- today..next_session around the real 2024-08-12 ex-date -- demonstrating the endpoint's server-side filtering misses a real dividend from a narrow window", "match": true, "note": "negative result -- confirms the bug, not a mismatch against an expected action"}, - {"symbols": ["AAPL"], "start": "2024-08-05", "end": "2024-11-10", + {"symbols": ["AAPL"], "start": "2024-08-05", "end": "2024-11-10", "anchor": "2024-08-12", "observed": [{"symbol": "AAPL", "action_type": "cash_dividend", "action_date": "2024-08-12"}], - "expected": "AAPL quarterly cash dividend, ex-date 2024-08-12 -- re-check with the FIXED window logic (runner._corporate_action_fetch_window: anchor-7d .. anchor+90d, matching CORPORATE_ACTION_FETCH_WINDOW_PAST_DAYS/FUTURE_DAYS), which recovers the same dividend the narrow window above missed", + "expected": "AAPL quarterly cash dividend, ex-date 2024-08-12 -- fix round 3, item 8: this window is a MANUAL 2024 approximation of runner._corporate_action_fetch_window's own arithmetic (anchor - CORPORATE_ACTION_FETCH_WINDOW_PAST_DAYS(7) .. anchor + CORPORATE_ACTION_FETCH_WINDOW_FUTURE_DAYS(90)), substituting the anchor date directly for next_trading_day(anchor) because sessions.py's frozen calendar does not cover 2024 at all (next_trading_day(anchor) would raise session_calendar_out_of_range) -- it is NOT the function's own literal output for this date. See the next row for the function's real output on a date it can actually classify.", + "match": true}, + {"symbols": [], "kind": "fetch_window_function_on_a_classifiable_anchor", + "anchor": "2026-09-24", "next_session_date": "2026-09-25", "start": "2026-09-17", "end": "2026-12-24", + "expected": "fix round 3, item 8: runner._corporate_action_fetch_window(date(2026, 9, 24))'s own real, unmodified output (2026 is the only year the frozen sessions.py calendar covers) -- confirms the -7/+90 padding around the ACTUAL next_trading_day, not a hand-substituted anchor", + "match": true, "note": "no source query -- a pure local function call, listed here to make the 2024 approximation above verifiably distinct from the function's real behaviour"}, + {"symbols": ["AAPL"], "start": "2024-08-01", "end": "2024-08-20", "kind": "raw_record_lag", + "observed": [{"symbol": "AAPL", "ex_date": "2024-08-12", "payable_date": "2024-08-15", + "process_date": "2024-08-15", "record_date": "2024-08-12", "rate": 0.25, + "cusip": "037833100"}], + "expected": "fix round 3, item 6/item 8: the RAW record (queried with data_quality=all via the private _get_marketdata the fixed fetch() now calls directly -- this wrapper's own ActionRecord normalization discards process_date/payable_date, so this row queries the raw response to observe them) shows a 3-DAY governing(ex_date)-to-process(process_date/payable_date) lag for this specific dividend (2024-08-12 -> 2024-08-15) -- matching Apple's own published dividend history (https://investor.apple.com/dividend-history/: ex-date August 12 2024, payment date August 15 2024). This CORRECTS the earlier '1-3 weeks' claim in this same file's prior claim_boundary, which was never measured against this record's own raw fields.", "match": true} ], - "claim_boundary": "Confirms that the live Alpaca source returns and normalizes these historical windows correctly (type and governing date), AND that the guard's original today..next_session fetch window was too narrow to reliably observe a real dividend (Alpaca's endpoint filters by process/payable date, not the ex_date this guard's evaluate_guard logic checks against) -- see corporate-actions-native-check-20260924's negative_query row above and blueprints/us-equities/alpaca-historical/README.md:35-36/collect.py:202-204 for the documented endpoint behaviour this reflects. The 2026-09-24 fix round widens the fetch window to anchor-7d..anchor+90d (runner.py's CORPORATE_ACTION_FETCH_WINDOW_PAST_DAYS/FUTURE_DAYS) and filters locally on the governing date in evaluate_guard, which the wide-window re-check row above confirms recovers the dividend. This check still does not exercise the guard inside a running paper trial end-to-end, a multi-day overnight resume against the live endpoint, the refresh cadence against the endpoint's rate limits, or action types other than forward split and cash dividend -- and a single historical dividend's 1-3-week process-date lag is not proof that every action type's process-date lag fits within the fixed 90-day forward pad.", - "not_retained": "The raw API responses and the exact command line were not retained; the observed values above are the complete normalized output." + "lag_observations": { + "purpose": "fix round 3, item 6: record the concrete governing-date/process-date lag observations behind the 90-day forward pad (CORPORATE_ACTION_FETCH_WINDOW_FUTURE_DAYS), each with its source, so the pad is traceable evidence rather than an unqualified round number.", + "observations": [ + {"source": "this file's own raw_record_lag query above (native, 2026-09-24)", + "symbol": "AAPL", "action_type": "cash_dividend", "ex_date": "2024-08-12", + "process_date": "2024-08-15", "lag_days": 3}, + {"source": "evidence/artifacts/gap-wave2-20260923/us-equities__market-data-reference/raw/1-alpaca-corporateactions-docs.excerpt.json (components.schemas.cash_dividend example, retained from https://docs.alpaca.markets/us/reference/corporateactions-1)", + "symbol": "FCF", "action_type": "cash_dividend", "ex_date": "2023-05-04", + "process_date": "2023-05-19", "payable_date": "2023-05-19", "lag_days": 15}, + {"source": "evidence/artifacts/gap-wave2-20260923/us-equities__market-data-reference/raw/1-alpaca-corporateactions-docs.excerpt.json (sse_example.CashDividend, retained from https://docs.alpaca.markets/us/reference/corporateactions-1)", + "symbol": "AAPL", "action_type": "cash_dividend", "ex_date": "2026-05-09", + "process_date": "2026-05-15", "payable_date": "2026-05-15", "lag_days": 6} + ], + "assumption": "fix round 3, item 6 (unresolved): these three observed lags (3, 6, 15 days) all fit comfortably within the 90-day forward pad, and none of the three or four documented action types checked here has ever been observed to exceed it. This is NOT proof that every action type's governing-to-process lag stays under 90 days -- no dividend or other action exceeding 90 days has been found OR searched for exhaustively, and Alpaca's own documentation does not state a maximum lag or guarantee upcoming-action announcement availability at all (an empty response cannot establish comprehensive absence of an upcoming action). The 90-day pad remains a documented, evidence-backed but unproven assumption, not an established bound." + }, + "claim_boundary": "Confirms that the live Alpaca source returns and normalizes these historical windows correctly (type and governing date), that the guard's original today..next_session fetch window was too narrow to reliably observe a real dividend (Alpaca's endpoint filters by process/payable date, not the ex_date this guard's evaluate_guard logic checks against), and the exact 3-day governing-to-process lag for this specific AAPL dividend (see raw_record_lag row and lag_observations above -- corrects this file's own earlier, unmeasured '1-3 weeks' claim). It does NOT establish a maximum lag bound for any action type (see lag_observations.assumption), does not exercise the guard inside a running paper trial end-to-end, a multi-day overnight resume against the live endpoint, the refresh cadence against the endpoint's rate limits, the raw_data=True/data_quality=all/unmapped-bucket-type parsing path introduced in fix round 3 (covered instead by tests/test_adaptive_paper_corporate_actions.py's faked-HTTP-response tests, not this native check), or action types other than forward split and cash dividend.", + "not_retained": "The raw API responses beyond what is quoted above, and the exact command line, were not retained; the observed values above are the complete normalized/raw output actually read." } diff --git a/blueprints/us-equities/adaptive-paper/corporate_actions.py b/blueprints/us-equities/adaptive-paper/corporate_actions.py index 88b09a7ee..6dbe1267c 100644 --- a/blueprints/us-equities/adaptive-paper/corporate_actions.py +++ b/blueprints/us-equities/adaptive-paper/corporate_actions.py @@ -62,6 +62,7 @@ from dataclasses import dataclass from datetime import date +import threading import time as _time from typing import Mapping @@ -225,11 +226,31 @@ def __init__(self, source, *, refresh_seconds=900.0, retry_seconds=60.0, max_age # overwrites (and thereby silently cancels) a previously confirmed # in-range corporate action. self._degraded: set = set() + # HIGH finding 1 (2026-09-24 fix round 3): `refresh()` runs in a + # caller-owned worker thread (runner.py's asyncio.to_thread) bounded + # by a wait_for timeout on the CALLER side -- the timeout only stops + # the caller from waiting, it cannot stop the thread itself + # (reproduced: an older, slower attempt finishing after a newer one + # already wrote a confirmed must_flatten silently turned it back + # off, because both attempts wrote into `_results` unconditionally + # with no ordering guard). `_generation` is bumped, under `_lock`, + # at the START of every `refresh()` attempt; only the write from + # the CURRENT (latest-started) generation is ever applied -- + # `_apply_success`/`_apply_failure` drop a write from any earlier, + # superseded generation even if it happens to complete later. + # `refresh_timed_out()` (called by the async scheduler when its own + # wait_for gives up on a still-running attempt) also bumps the + # generation, so that attempt's eventual write -- success or + # failure, whenever the thread actually finishes -- is dropped too. + self._lock = threading.Lock() + self._generation = 0 def refresh(self, symbols, *, start: date, end: date, now=None): """Idempotent within `refresh_seconds` for a symbol set already covered by the last successful fetch AND fetched for this same - (start, end) window; always safe to call every tick. Never raises, + (start, end) window; always safe to call every tick, including + concurrently from multiple threads (finding 1) -- every state + mutation is generation-checked under `self._lock`. Never raises, and never overwrites a previously successful result with `LOOKUP_FAILED` (finding 3) -- a fetch failure only marks every requested symbol DEGRADED (see `_mark_degraded`); `evaluate()` @@ -238,45 +259,81 @@ def refresh(self, symbols, *, start: date, end: date, now=None): now = self._clock() if now is None else now symbols = set(symbols) window = (start, end) - # finding 8: a throttle interval of `retry_seconds` (short) rather - # than `refresh_seconds` (long) whenever the LAST attempt failed -- - # a transient failure gets retried again in about a minute instead - # of waiting out the full ordinary interval. - interval = self.retry_seconds if self._last_attempt_failed else self.refresh_seconds - if (self._last_attempt is not None and now - self._last_attempt < interval - and symbols.issubset(self._results.keys()) and self._last_window == window): - return - self._last_attempt = now - self._last_window = window + with self._lock: + # finding 8: a throttle interval of `retry_seconds` (short) + # rather than `refresh_seconds` (long) whenever the LAST + # attempt failed -- a transient failure gets retried again in + # about a minute instead of waiting out the full ordinary + # interval. + interval = self.retry_seconds if self._last_attempt_failed else self.refresh_seconds + if (self._last_attempt is not None and now - self._last_attempt < interval + and symbols.issubset(self._results.keys()) and self._last_window == window): + return + self._last_attempt = now + self._last_window = window + self._generation += 1 + my_generation = self._generation try: fetched = self.source.fetch(sorted(symbols), start, end) except CorporateActionLookupError: - self._last_attempt_failed = True - self._mark_degraded(symbols) + self._apply_failure(symbols, my_generation) return except Exception: # Fail closed (degraded, not overwritten) on any unexpected # error too -- never guess. + self._apply_failure(symbols, my_generation) + return + self._apply_success(symbols, fetched, now, my_generation) + + def refresh_timed_out(self, symbols): + """HIGH finding 1: called by the async scheduler (runner.py's + `_schedule_corporate_action_refresh`) when ITS OWN wait_for gives up + on a `refresh()` call still running in a worker thread. The thread + keeps running (Python cannot forcibly kill it), so this both (a) + bumps the generation counter, invalidating that in-flight attempt -- + whenever it does eventually finish, `_apply_success`/`_apply_failure` + will see a stale generation and drop the write -- and (b) treats the + timeout itself as an ordinary failure right now (degraded, short + `retry_seconds` retry interval), exactly as if `fetch()` had raised, + rather than silently leaving the cache in its last (possibly clear) + state until the abandoned attempt happens to resolve on its own.""" + with self._lock: + self._generation += 1 self._last_attempt_failed = True self._mark_degraded(symbols) - return - self._last_attempt_failed = False - self._last_success = now - for symbol in symbols: - # finding 7a: a symbol the source's fetch() silently omits from - # its returned dict (never true of AlpacaCorporateActionsSource, - # which always returns every requested symbol, but not - # guaranteed of an injected test/fake source) must fail closed, - # never be read as "no action". - self._results[symbol] = fetched.get(symbol, LOOKUP_FAILED) - self._degraded.discard(symbol) + + def _apply_failure(self, symbols, generation): + with self._lock: + if generation != self._generation: + return # superseded by a later attempt (or a timeout) -- drop + self._last_attempt_failed = True + self._mark_degraded(symbols) + + def _apply_success(self, symbols, fetched, now, generation): + with self._lock: + if generation != self._generation: + return # superseded -- an older result must never overwrite a newer one + self._last_attempt_failed = False + self._last_success = now + for symbol in symbols: + if symbol in fetched: + self._results[symbol] = fetched[symbol] + self._degraded.discard(symbol) + else: + # LOW finding 7 (fix round 3): a symbol omitted from an + # OTHERWISE successful fetch (most other symbols + # resolved fine) must not erase its own previously + # confirmed record -- treat exactly like a per-symbol + # failure (degraded, last good record preserved), never + # overwrite with LOOKUP_FAILED. + self._mark_degraded((symbol,)) def _mark_degraded(self, symbols): """finding 3: only ever ADD to `_degraded` and, for a symbol never previously fetched at all, seed a `LOOKUP_FAILED` baseline via `setdefault` -- never touch an existing `_results` entry, so a previously confirmed corporate action survives a later failed - refresh untouched.""" + refresh untouched. Callers hold `self._lock`.""" for symbol in symbols: self._degraded.add(symbol) self._results.setdefault(symbol, LOOKUP_FAILED) @@ -292,9 +349,10 @@ def evaluate(self, *, today: date, next_session_date: date, held_symbols, candid # last known good ActionRecord list (if any) is still read below, # so a confirmed in-range action is preserved (with needs_attention # added) rather than silently cancelled by staleness alone. - stale = self._last_success is None or now - self._last_success > self.max_age_seconds - results = {symbol: self._results.get(symbol, LOOKUP_FAILED) for symbol in relevant} - degraded = (relevant if stale else set()) | (self._degraded & relevant) + with self._lock: + stale = self._last_success is None or now - self._last_success > self.max_age_seconds + results = {symbol: self._results.get(symbol, LOOKUP_FAILED) for symbol in relevant} + degraded = (relevant if stale else set()) | (self._degraded & relevant) return evaluate_guard(today=today, next_session_date=next_session_date, held_symbols=held_symbols, candidate_symbols=candidate_symbols, lookup_results=results, degraded_symbols=degraded) @@ -335,17 +393,36 @@ class AlpacaCorporateActionsSource: key/secret runner.credentials() already returns; never logs, stores, or raises them in any exception message.""" - # HIGH finding 2: request cap -- installed alpaca-py==0.44.0's - # CorporateActionsRequest defaults `limit` to 1000 and its - # `CorporateActionsClient._get_marketdata` call never exposes this - # wrapper's own pagination beyond that single page. A result at or past - # this cap cannot be trusted as complete (finding 1's ambiguous-count - # requirement below). + # HIGH finding 2 (fix round 3): request cap -- alpaca-py==0.44.0's + # CorporateActionsRequest defaults `limit` to 1000, and its + # `RESTClient._get_marketdata` (verified via `inspect.getsource` against + # the installed package) DOES auto-paginate across `next_page_token`, + # accumulating items from every page up to this `limit` -- so a result + # at or past this cap means more matching rows may exist beyond what was + # requested, not that pagination silently stopped after one page (an + # earlier round of this comment was wrong about that). _REQUEST_LIMIT = 1000 def __init__(self, api_key, secret_key): from alpaca.data.historical.corporate_actions import CorporateActionsClient - self._client = CorporateActionsClient(api_key, secret_key) + # HIGH finding 2 (fix round 3): raw_data=True makes get_corporate_ + # actions() (and the private _get_marketdata() this wrapper calls + # directly below) return the plain parsed-JSON dict instead of a + # CorporateActionsSet. This is required, not cosmetic: + # CorporateActionsSet.__init__ (verified via inspect.getsource + # against the installed alpaca-py==0.44.0 package) is a chain of 13 + # `elif corporate_action_type == "..."` branches with NO else/ + # default clause -- any bucket key it does not recognize (the + # Alpaca API reference documents `reorganizations`, `partial_calls` + # and `capital_gains_distributions` in addition to the 13 this + # wrapper's own _TYPE_FIELD_MAP models) is silently dropped by the + # SDK before this wrapper ever sees it, turning a real corporate + # action into an apparently-clear "no action" result, and its rows + # also never reach the cap check below. Parsing the raw dict + # ourselves (see fetch()) means an unmapped bucket key is visible to + # this wrapper's own _TYPE_FIELD_MAP.get(...) is None fail-closed + # path instead of vanishing inside the SDK first. + self._client = CorporateActionsClient(api_key, secret_key, raw_data=True) # HIGH finding 2: alpaca-py's RESTClient (verified against the # installed alpaca-py==0.44.0 RESTClient.__init__/_request) sets no # per-request HTTP timeout at all, and sleeps `_retry_wait` (default @@ -353,8 +430,17 @@ def __init__(self, api_key, secret_key): # unbounded and slow by default. Pin a bounded connect/read timeout # directly onto the underlying requests.Session (mirroring # transport.py's own HTTP-boundary `timeout=(5, 5)` convention) and - # cut retries to the bare minimum, so one fetch() call is itself - # bounded even before runner.py's own asyncio wait_for() around it. + # cut retries to the bare minimum. NIT finding 9 (fix round 3): a + # `requests` timeout bounds each individual connect/read, not the + # call's total wall-clock duration -- a paginated multi-page fetch + # (`_get_marketdata` above) issues one HTTP request per page, and a + # slow-but-not-hung server could still make the overall fetch() call + # run for `pages * timeout`. The actual outer bound on one fetch() + # call is runner.py's own asyncio wait_for() + # (CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) around the whole + # refresh() call this fetch() runs inside, not this per-request + # timeout alone; this timeout only prevents a single HTTP request + # from hanging indefinitely with no data ever arriving. session = self._client._session original_request = session.request @@ -366,57 +452,108 @@ def _bounded_request(method, url, **kwargs): self._client._retry = 1 self._client._retry_wait = 1 + @staticmethod + def _parse_date(value): + """A raw response field is an ISO date string (or absent/null) once + parsed from JSON -- never already a `datetime.date` (that only + happens through the SDK's own pydantic models, which this wrapper + no longer uses; see raw_data=True above). Returns None for + anything that is not a valid ISO date, rather than raising -- + callers treat a missing/unparseable governing date as ambiguous for + that record's symbol(s), not as a whole-fetch failure.""" + if not isinstance(value, str) or not value: + return None + try: + return date.fromisoformat(value[:10]) + except ValueError: + return None + def fetch(self, symbols, start: date, end: date) -> dict: - """Returns `{symbol: [ActionRecord, ...]}` for exactly the - requested `symbols` (every requested symbol is present, possibly - with an empty list). Raises `CorporateActionLookupError` -- never a - raw alpaca-py/HTTP exception -- on any request or response-shape - failure, or on an unmapped/ambiguous result, so a caller never - needs to know alpaca-py's own exception types and never silently - treats an incomplete or unparseable response as "no action".""" - from alpaca.data.requests import CorporateActionsRequest + """Returns `{symbol: [ActionRecord, ...] | LOOKUP_AMBIGUOUS}` for + exactly the requested `symbols` (every requested symbol is present: + either its resolved list, possibly empty, or LOOKUP_AMBIGUOUS for a + symbol whose own record(s) could not be fully parsed -- MEDIUM + finding 3, fix round 3). Raises `CorporateActionLookupError` -- + never a raw alpaca-py/HTTP exception -- on any request/transport + failure, a result at the page/limit cap, or an unmapped action-type + bucket, so a caller never needs to know alpaca-py's own exception + types and never silently treats an incomplete or unparseable + response as "no action" for every requested symbol. + + HIGH finding 3 (fix round 3): requests `data_quality=all` -- + Alpaca's documented default (`data_quality=complete`) excludes + records the provider itself considers incomplete (e.g. missing a + CUSIP/ISIN), which could otherwise make a real, known-symbol, + known-date upcoming action invisible to this guard with no + signal at all. `all` is not exposed as a typed field on the + installed alpaca-py==0.44.0 CorporateActionsRequest model (verified: + the field is silently dropped by its own to_request_fields(), a + pydantic extra="ignore" default), so this calls the client's own + private `_get_marketdata` directly with a hand-built params dict -- + the same private method `get_corporate_actions()` itself calls -- + instead of going through the typed request object for this one + parameter.""" symbols = list(symbols) + params = {"symbols": ",".join(symbols), "start": start.isoformat(), "end": end.isoformat(), + "limit": self._REQUEST_LIMIT, "sort": "asc", "data_quality": "all"} try: - request = CorporateActionsRequest(symbols=symbols, start=start, end=end, - limit=self._REQUEST_LIMIT) - response = self._client.get_corporate_actions(request) + raw = self._client._get_marketdata(path="/corporate-actions", params=params, + page_limit=1000, page_size=1000) except CorporateActionLookupError: raise except Exception as error: raise CorporateActionLookupError(type(error).__name__) from error out = {symbol: [] for symbol in symbols} + ambiguous_symbols = set() try: - raw = dict(getattr(response, "data", {}) or {}) + raw = dict(raw or {}) # finding 1 (result-count-at-cap is ambiguous): a response whose - # total item count reaches the request's own `limit` may have - # been truncated server-side with no signal to this wrapper - # about which symbol's records were cut off -- fail closed for - # this whole fetch rather than trust a possibly-partial result. + # total RAW item count (every bucket, including one this + # wrapper does not model -- see raw_data=True above) reaches the + # request's own `limit` may have been truncated server-side + # with no signal to this wrapper about which symbol's records + # were cut off -- fail closed for this whole fetch rather than + # trust a possibly-partial result. total_items = sum(len(items) for items in raw.values()) if total_items >= self._REQUEST_LIMIT: raise CorporateActionLookupError("result_at_page_or_limit_cap") for type_key, items in raw.items(): spec = _TYPE_FIELD_MAP.get(type_key) if spec is None: - # finding 7c: alpaca-py's own CorporateActionsSet - # silently drops any type_key it does not model, and an - # unmapped type_key here has no known governing-date or - # symbol field this wrapper can trust to parse -- fail - # closed (ambiguous) for the whole fetch rather than - # silently skip an action type this guard does not yet - # recognize. + # finding 7c/finding 2 (fix round 3): an unmapped bucket + # key (e.g. Alpaca's documented `reorganizations`, + # `partial_calls`, `capital_gains_distributions`, or any + # future type this guard's _TYPE_FIELD_MAP does not yet + # know) has no known governing-date or symbol field this + # wrapper can trust to parse -- fail closed (ambiguous) + # for the whole fetch rather than silently skip an + # action type this guard does not yet recognize. Now + # actually reachable (raw_data=True, fix round 3): the + # SDK's own CorporateActionsSet used to drop these + # bucket keys before this wrapper ever saw them. raise CorporateActionLookupError(f"unmapped_corporate_action_type:{type_key}") symbol_fields, date_field, canonical_type = spec for item in items: - action_date = getattr(item, date_field, None) + if not isinstance(item, dict): + raise CorporateActionLookupError("malformed_corporate_action_item") + action_date = self._parse_date(item.get(date_field)) + matched_symbols = [item.get(field) for field in symbol_fields + if item.get(field) and item.get(field) in out] if action_date is None: + # HIGH finding 3 (fix round 3): a record missing (or + # carrying an unparseable) governing date -- e.g. + # excluded-by-default incomplete data now included + # via data_quality=all -- becomes ambiguous for + # EVERY requested symbol it names, rather than being + # silently skipped as if it never existed. + ambiguous_symbols.update(matched_symbols) continue - for field in symbol_fields: - symbol = getattr(item, field, None) - if symbol and symbol in out: - out[symbol].append(ActionRecord( - symbol=symbol, action_type=canonical_type, action_date=action_date, - source_id=str(getattr(item, "cusip", "") or type_key))) + for symbol in matched_symbols: + out[symbol].append(ActionRecord( + symbol=symbol, action_type=canonical_type, action_date=action_date, + source_id=str(item.get("cusip") or type_key))) + for symbol in ambiguous_symbols: + out[symbol] = LOOKUP_AMBIGUOUS except CorporateActionLookupError: raise except Exception as error: diff --git a/blueprints/us-equities/adaptive-paper/native_strategy.py b/blueprints/us-equities/adaptive-paper/native_strategy.py index 56f05fe73..5a846af42 100644 --- a/blueprints/us-equities/adaptive-paper/native_strategy.py +++ b/blueprints/us-equities/adaptive-paper/native_strategy.py @@ -60,6 +60,17 @@ def __init__(self, policy: AdaptivePolicy, ledger, trial_id: str, *, event_sink= # flagged for a forced flatten (finding 5). self._ca_ever_flagged = set() self._ca_last_must_flatten = set() + # MEDIUM finding 4 (2026-09-24 fix round 3): the CURRENT (this + # tick's) set of HELD symbols the guard flagged needs_attention -- + # lookup failed, ambiguous, degraded, or a calendar/session-unknown + # fallback -- regardless of must_flatten. runner.py's + # _honest_overnight_hold reads this so a held position whose + # corporate-action status was never positively verified clear for + # the next session cannot end the run as a successful + # "held_overnight" (exit 0) just because nothing forced a flatten; + # block-and-flag is the documented policy, declaring the run a + # clean success is not. + self._ca_last_needs_attention_held = set() # G-e: the broker-proven margin multiplier (runner.py's config # "_account_multiplier", only ever set after preflight has checked # multiplier >= requested leverage -- see runner._check_margin_ @@ -401,16 +412,21 @@ def _corporate_action_guard_symbols(self, now, held, candidate_symbols): block_entry=True, must_flatten=False, needs_attention=True, action_type=None, action_date=None) self._ca_last_must_flatten = set() + # MEDIUM finding 4: every held symbol is needs_attention in this + # fallback (the calendar itself could not be classified). + self._ca_last_needs_attention_held = set(held_symbols) return relevant, set() decisions = self.corporate_action_guard.evaluate( today=info.session_date, next_session_date=next_session_date, held_symbols=held_symbols, candidate_symbols=set(candidate_symbols), now=now) - block_entry, must_flatten = set(), set() + block_entry, must_flatten, needs_attention_held = set(), set(), set() for symbol, decision in decisions.items(): if decision.block_entry: block_entry.add(symbol) if decision.must_flatten: must_flatten.add(symbol) + if decision.needs_attention and symbol in held_symbols: + needs_attention_held.add(symbol) if decision.block_entry or decision.must_flatten or decision.needs_attention: self._emit_ca_event_on_change( symbol, decision.reason, block_entry=decision.block_entry, @@ -418,12 +434,33 @@ def _corporate_action_guard_symbols(self, now, held, candidate_symbols): action_type=decision.action_type, action_date=decision.action_date.isoformat() if decision.action_date else None) else: - # Symbol has gone clear (no block/flatten/attention this - # tick) -- drop any tracked prior state so a FUTURE - # re-block is treated as a change and re-emitted, not - # suppressed as "unchanged from a stale clear tick". + # NIT finding 9 (fix round 3): symbol has gone clear (no + # block/flatten/attention this tick). If it was PREVIOUSLY + # flagged (an entry exists in `_ca_last_emitted`), emit one + # explicit "cleared" event so the resolution itself is + # visible in the event log, not just its silence -- an + # operator reading the stream should be able to see a + # symbol's corporate-action status resolve, not only infer + # it from the absence of further events. A symbol that was + # never flagged in the first place (the common case, every + # tick, for the vast majority of symbols) gets no event at + # all, exactly as before. Drop the tracked prior state + # regardless, so a FUTURE re-block is treated as a change + # and re-emitted, not suppressed as "unchanged from a stale + # clear tick". + if symbol in self._ca_last_emitted: + self.event_sink({"type": "corporate_action_guard", "symbol": symbol, + "block_entry": False, "must_flatten": False, + "needs_attention": False, "reason": "corporate_action_cleared", + "action_type": None, "action_date": None}) self._ca_last_emitted.pop(symbol, None) self._ca_last_must_flatten = set(must_flatten) + # MEDIUM finding 4: a held symbol absent from `decisions` entirely + # (should not happen -- evaluate_guard decides every held|candidate + # symbol -- but this method does not trust that blindly) is treated + # the same fail-closed way as everywhere else in this guard: not + # positively verified clear, so it stays in the attention set. + self._ca_last_needs_attention_held = needs_attention_held | (held_symbols - set(decisions)) return block_entry, must_flatten def _emit_ca_event_on_change(self, symbol, reason, *, block_entry, must_flatten, needs_attention, @@ -821,10 +858,19 @@ def rebalance(self, now=None, *, force_exit=False): # Trading-lane audit gap #8: evaluated after gap risk (whose own # arming/capture state is independent of it) but consumed first, # below -- see corporate_actions.py's module docstring for the full - # precedence contract. `candidate_symbols` is this tick's targets, - # the only symbols a fresh/added entry could apply to. + # precedence contract. `candidate_symbols` is this tick's targets + # PLUS every symbol with a still-resting (not yet cancel-requested) + # pending BUY order (MEDIUM finding 5, fix round 3): a symbol whose + # signal disappeared from decision.targets THIS tick -- its resting + # buy from a PRIOR tick is still live -- must still be evaluated by + # the guard; evaluating only decision.targets.keys() meant a + # corporate action discovered after the signal vanished never + # entered ca_block_entry, so the cancel_expired(...symbols=ca_block_ + # entry) call below never saw that resting buy at all. + pending_buy_symbols = {info["symbol"] for info in self.pending.values() + if info.get("side") == "buy" and not info.get("cancel_requested")} ca_block_entry, ca_must_flatten = self._corporate_action_guard_symbols( - now, held, decision.targets.keys()) + now, held, decision.targets.keys() | pending_buy_symbols) # finding 10 (2026-09-24 fix round): a resting BUY order for a # symbol newly blocked this tick must not be left to fill after the # block was decided -- cancel it immediately, the same treatment diff --git a/blueprints/us-equities/adaptive-paper/runner.py b/blueprints/us-equities/adaptive-paper/runner.py index 966349969..e9fe49491 100644 --- a/blueprints/us-equities/adaptive-paper/runner.py +++ b/blueprints/us-equities/adaptive-paper/runner.py @@ -924,26 +924,41 @@ def _leverage_achievement_step(state, *, dt_seconds, achieved_leverage, ceiling, # CRITICAL finding 1 (2026-09-24 fix round): Alpaca's `/v1/corporate-actions` # endpoint filters by PROCESS date, which for a cash dividend is the payable -# date -- typically 1-3 weeks AFTER the ex_date this guard actually reasons -# about (blueprints/us-equities/alpaca-historical/README.md's own documented +# date -- AFTER the ex_date this guard actually reasons about +# (blueprints/us-equities/alpaca-historical/README.md's own documented # behaviour; independently confirmed by a native read-only check recorded in # corporate-actions-native-check-20260924.json: fetch(['AAPL'], -# 2024-08-09, 2024-08-12) -- a narrow window around the real 2024-08-12 -# ex-date -- returned []; fetch(['AAPL'], 2024-08-12, 2024-08-16) returned -# the cash_dividend). A narrow today..next_session request window therefore -# systematically misses real dividends. Request a WIDE window from the -# source and let evaluate_guard's own narrow date check (today.. -# next_session_date, inclusive -- finding 4) do the actual filtering -# locally, exactly as corporate_actions.py's module docstring already -# describes this split of responsibilities. +# 2024-08-09, 2024-08-12) -- the guard's actual pre-fix production window +# around the real 2024-08-12 ex-date -- returned []). A narrow today.. +# next_session request window therefore systematically misses real +# dividends. Request a WIDE window from the source and let evaluate_guard's +# own narrow date check (today..next_session_date, inclusive -- finding 4) +# do the actual filtering locally, exactly as corporate_actions.py's module +# docstring already describes this split of responsibilities. +# +# MEDIUM finding 6 (fix round 3): the forward pad below is sized from three +# observed governing-to-process lags, each recorded with its source in +# corporate-actions-native-check-20260924.json's lag_observations -- a +# native AAPL dividend (3 days), and two Alpaca-documented examples (6 and +# 15 days). This is NOT a proven maximum: no exceeding case has been found +# or exhaustively searched for, and Alpaca does not guarantee upcoming- +# action announcement availability at all. Treat 90 days as an evidence- +# backed assumption, not an established bound. CORPORATE_ACTION_FETCH_WINDOW_PAST_DAYS = 7 CORPORATE_ACTION_FETCH_WINDOW_FUTURE_DAYS = 90 -# HIGH finding 2: the hard ceiling on how long the corporate-action guard's -# background refresh thread (see _schedule_corporate_action_refresh below) -# is allowed to run before this coroutine gives up waiting on it (the -# thread itself may still be blocked in alpaca-py past this point -- the -# in-flight flag it clears in its `finally` prevents a second overlapping -# thread from being started meanwhile, see the tick-loop call site). +# HIGH finding 2/finding 1 (fix round 3): the hard ceiling on how long this +# coroutine waits for the corporate-action guard's background refresh +# thread (see _schedule_corporate_action_refresh below) before giving up. +# The thread itself is NOT stopped by this timeout -- Python cannot +# forcibly kill a running thread -- it may still be blocked in alpaca-py +# past this point and could finish much later. `_schedule_corporate_ +# action_refresh` accounts for that: on timeout it calls the guard's own +# `refresh_timed_out()` (generation-bumping the monitor so the eventual +# late write is dropped as superseded, and treating the timeout as a +# failure right now), clears `state["in_flight"]` so the NEXT tick's +# refresh is not permanently locked out, and retains the task in +# `state["tasks"]` so run_native's own shutdown path can still cancel/join +# it rather than leaving it fully untracked. CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = 20.0 @@ -977,14 +992,40 @@ def _corporate_action_fetch_window(session_date): async def _schedule_corporate_action_refresh(guard, watch, start, end, now, state): - """HIGH finding 2: run `guard.refresh()` (a blocking network call) off - the asyncio event loop in a worker thread, bounded by a hard - `asyncio.wait_for` timeout, and guarded by `state["in_flight"]` so a - still-running previous refresh thread is never overlapped by a second - concurrent one -- `CorporateActionMonitor.refresh` is not proven safe - for concurrent invocation. A no-op if a refresh is already in flight; - callers check `state["in_flight"]` before calling this so a throttled - (no-op) `refresh()` call never even spins up a thread.""" + """HIGH finding 1/2 (fix round 3): run `guard.refresh()` (a blocking + network call) off the asyncio event loop in a worker thread, bounded by + a hard `asyncio.wait_for` timeout, and guarded by `state["in_flight"]` + so a still-running previous refresh thread is never overlapped by a + second concurrent one -- `state` is a plain dict callers own and can + share across call sites (the tick loop AND the preflight fetch both + pass the SAME dict, fix round 3, so an in-flight preflight thread is + visible to the tick loop's first check too). + + `wait_for`'s timeout only stops THIS coroutine from waiting; it cannot + stop the worker thread itself (Python cannot forcibly kill a thread) -- + reproduced by an independent review: the thread kept running past the + timeout and later wrote a stale result over a newer one. Two defenses, + both now in place: + 1. `guard.refresh_timed_out(watch)` (if the guard exposes it -- the + real CorporateActionMonitor does) is called on timeout, which + bumps that monitor's internal generation counter so the abandoned + attempt's eventual write (whenever the thread actually finishes) + is dropped as superseded, AND marks the requested symbols + degraded/failed right now (the short `retry_seconds` interval + applies to the next attempt, not the full `refresh_seconds`). + 2. The created task is retained in `state["tasks"]` (pruned of + already-done entries on every call) so a caller (run_native's own + `finally` block) can cancel and join every still-pending + corporate-action refresh task at shutdown, instead of leaving + them as untracked, unaccounted-for background work. + + A no-op if a refresh is already in flight; callers check + `state["in_flight"]` before calling this so a throttled (no-op) + `refresh()` call never even spins up a thread. Returns the created + task so a caller that needs the result now (the preflight fetch) can + `await` it directly -- `_run()` below never raises (its own + `asyncio.wait_for` swallows the timeout), so awaiting the returned task + is itself already bounded by `CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS`.""" state["in_flight"] = True async def _run(): @@ -993,15 +1034,86 @@ async def _run(): asyncio.to_thread(guard.refresh, watch, start=start, end=end, now=now), timeout=CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) except asyncio.TimeoutError: - pass + if hasattr(guard, "refresh_timed_out"): + guard.refresh_timed_out(watch) + except asyncio.CancelledError: + if hasattr(guard, "refresh_timed_out"): + guard.refresh_timed_out(watch) + raise finally: state["in_flight"] = False - asyncio.create_task(_run()) + task = asyncio.create_task(_run()) + tasks = state.setdefault("tasks", []) + tasks[:] = [t for t in tasks if not t.done()] + tasks.append(task) + return task + + +async def _shutdown_corporate_action_tasks(ca_refresh_state): + """HIGH finding 1 (fix round 3, 'keep the task and join or cancel it at + shutdown'): cancel and join every still-pending corporate-action + refresh task tracked in `ca_refresh_state["tasks"]` -- a background + task run_native itself spawned (via _schedule_corporate_action_refresh) + must never be left running, unaccounted for, past run_native's own + return. An ordinary function, not inlined into run_native's `finally` + block, specifically so it is independently unit-testable (a fully + hanging worker thread cannot itself be joined quickly in a test -- + `asyncio.run()`'s own cleanup joins the default thread-pool executor + regardless of any Task cancellation, since Python cannot forcibly kill + a thread -- but a plain asyncio.sleep()-based fake task CAN be, and + exercises the same cancel/gather logic). + + Cancelling the task only stops THIS process from waiting on the worker + thread further; the thread's own eventual completion (whenever it + happens) is still safely a no-op via the monitor's generation check + (see refresh_timed_out/CancelledError handling in + _schedule_corporate_action_refresh).""" + pending_ca_tasks = [t for t in ca_refresh_state.get("tasks", []) if not t.done()] + for t in pending_ca_tasks: + t.cancel() + if pending_ca_tasks: + await asyncio.gather(*pending_ca_tasks, return_exceptions=True) + + +async def _preflight_corporate_action_refresh(config, session_policy, ledger, ca_refresh_state): + """HIGH finding 2 ("do the first fetch in preflight")/finding 1 (fix + round 3, "share that state with the preflight"): only meaningful (and + only ever attempted) when overnight_holds is enabled -- see + run_native's own gating of the guard (finding 8) -- so the guard's + cache is already warm before run_native's tick loop starts its first + refresh, instead of every trial's very first tick paying for a cold + background fetch. Uses the SAME `_schedule_corporate_action_refresh` + helper (and the SAME `ca_refresh_state` dict the caller also passes + into run_native) the tick loop uses -- an ordinary function, not an + inline closure inside main()'s execute(), specifically so it is + independently unit-testable without driving all of main() (fix round + 3, mutation F1b: "preflight call site fetches a one-day window" -- + only directly testable once this was its own seam). Uses the SAME wide + `_corporate_action_fetch_window` the tick loop uses, never a narrower + one-day window.""" + guard = _corporate_action_guard_for_strategy(config, session_policy) + if guard is None: + return + try: + preflight_now = time.time() + preflight_session = session_at(datetime.fromtimestamp(preflight_now, timezone.utc)) + _, preflight_start, preflight_end = _corporate_action_fetch_window(preflight_session.session_date) + except ValueError: + return + preflight_watch = set(config["symbols"]) | {p.symbol for p in ledger.positions().values() if p.qty} + preflight_task = await _schedule_corporate_action_refresh( + guard, preflight_watch, preflight_start, preflight_end, preflight_now, ca_refresh_state) + # _run() inside _schedule_corporate_action_refresh never raises (its + # own wait_for already swallows the timeout) -- awaiting the task it + # returned is itself already bounded, so the preflight still blocks (as + # intended -- "do the first fetch in preflight") until the fetch + # resolves or times out, never longer. + await preflight_task async def run_native(controller, policy_config, assets, trial_id, config, baseline_cash, *, account_fingerprint="simulation", - log_directory=None): + log_directory=None, ca_refresh_state=None): from native_adapter import build_node from native_strategy import AdaptiveStrategy # R5: use the same registry entries load_config already validated (and @@ -1032,10 +1144,18 @@ async def run_native(controller, policy_config, assets, trial_id, config, baseli account_multiplier=config.get("_account_multiplier"), corporate_action_guard=_corporate_action_guard_for_strategy( config, session_policy)) - # HIGH finding 2: a per-run state dict (not a strategy/monitor - # attribute) tracking whether a background refresh thread is currently - # in flight -- see _schedule_corporate_action_refresh. - ca_refresh_state = {"in_flight": False} + # HIGH finding 2/finding 1 (fix round 3): a per-run state dict (not a + # strategy/monitor attribute) tracking whether a background refresh + # thread is currently in flight, and the still-pending refresh tasks + # themselves -- see _schedule_corporate_action_refresh. A caller + # (main()'s execute(), for its own preflight fetch) can pass in an + # already-existing dict here so the SAME in-flight/task state is shared + # across the preflight fetch and this tick loop -- a preflight fetch + # thread still running when the tick loop starts must be visible to the + # tick loop's own in_flight check, not invisible because each used its + # own separate dict. + if ca_refresh_state is None: + ca_refresh_state = {"in_flight": False} # Capture actual streaming quotes before native conversion. Every native # strategy event still arrives through the data engine's ordinary path. port = controller.port @@ -1261,6 +1381,7 @@ async def quote_sink(q): task.cancel() await asyncio.gather(task, return_exceptions=True) await port.stop() + await _shutdown_corporate_action_tasks(ca_refresh_state) state = asdict(controller.ledger.accounting()) is_flat = not controller.ledger.positions() and not controller.ledger.unresolved() port_health = getattr(port, "health", {}) @@ -1285,7 +1406,12 @@ async def quote_sink(q): "corporate_action_guard": { "enabled": strategy.corporate_action_guard is not None, "ever_flagged_symbols": sorted(strategy._ca_ever_flagged), - "pending_must_flatten": sorted(strategy._ca_last_must_flatten)}} + "pending_must_flatten": sorted(strategy._ca_last_must_flatten), + # MEDIUM finding 4 (fix round 3): every currently HELD + # symbol not positively verified clear as of the run's + # last tick (lookup failed/ambiguous/degraded/session- + # unknown) -- read by _honest_overnight_hold below. + "pending_needs_attention_held": sorted(strategy._ca_last_needs_attention_held)}} if leverage_policy is not None: capital = Decimal(config["capital_usd"]) peak_effective_leverage = (peak_gross_exposure_usd / capital) if capital else Decimal("0") @@ -1388,6 +1514,17 @@ def _honest_overnight_hold(outcome, session_policy, now): engine-layer forced-sell path caught up); that is a genuine failure, not an intentional overnight hold, and must instead take the recovery path below. + - MEDIUM finding 4 (fix round 3): the same guard summary carries no + ``pending_needs_attention_held`` symbol -- a held symbol whose + corporate-action lookup failed, was ambiguous, went stale, or fell + outside a classifiable session for the ENTIRE run was never + positively verified clear for the next session. This guard's own + documented contract is block-and-flag, not "assume clear on + silence" -- not force-flattening an unverified holding is correct + (this module never guesses at a corporate action from partial + data), but reporting the run a clean "held_overnight" success + (exit 0) is not: it silently drops the exact needs_attention state + that same tick's own corporate_action_guard event already raised. Any other non-flat end must stay "needs_attention" (routed through the force-flat/recover path below) so a genuine failure is never silently @@ -1400,7 +1537,9 @@ def _honest_overnight_hold(outcome, session_policy, now): except ValueError: return False boundary_reached = kind in (SessionKind.POST, SessionKind.CLOSED) - guard_clear = not (outcome.get("corporate_action_guard") or {}).get("pending_must_flatten") + guard_summary = outcome.get("corporate_action_guard") or {} + guard_clear = (not guard_summary.get("pending_must_flatten") + and not guard_summary.get("pending_needs_attention_held")) return (boundary_reached and outcome.get("reconciliation") is not None and not outcome.get("adapter_errors") and not (outcome.get("accounting") or {}).get("halted_reason") @@ -1760,39 +1899,13 @@ async def execute(): controller.port = fresh_port(True) return await recover(controller, metadata, config) controller.port = fresh_port() - # HIGH finding 2 ("do the first fetch in preflight"): only - # meaningful (and only ever attempted) when overnight_holds - # is enabled -- see run_native's own gating of the guard - # (finding 8) -- so the guard's cache is already warm - # before run_native's tick loop starts its first refresh, - # instead of every trial's very first tick paying for a - # cold background fetch. Bounded exactly like the tick-loop - # refresh (finding 2): a worker thread with a wait_for - # timeout; never raises, and a failure here is simply left - # for the first in-loop refresh to retry. - guard = _corporate_action_guard_for_strategy(config, session_policy) - if guard is not None: - try: - preflight_now = time.time() - preflight_session = session_at(datetime.fromtimestamp(preflight_now, timezone.utc)) - _, preflight_start, preflight_end = _corporate_action_fetch_window( - preflight_session.session_date) - except ValueError: - pass - else: - preflight_watch = set(config["symbols"]) | { - p.symbol for p in ledger.positions().values() if p.qty} - try: - await asyncio.wait_for( - asyncio.to_thread(guard.refresh, preflight_watch, start=preflight_start, - end=preflight_end, now=preflight_now), - timeout=CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) - except asyncio.TimeoutError: - pass + ca_refresh_state = {"in_flight": False} + await _preflight_corporate_action_refresh(config, session_policy, ledger, ca_refresh_state) try: outcome = await run_native(controller, policy_config, observation["assets"], args.trial, config, metadata["baseline_cash"], account_fingerprint=fingerprint, - log_directory=None if args.live_dir is None else args.live_dir / "nautilus") + log_directory=None if args.live_dir is None else args.live_dir / "nautilus", + ca_refresh_state=ca_refresh_state) except Exception as exc: outcome = {"status": "needs_attention", "flat": False, "native_fill_events": 0, "error_type": type(exc).__name__} diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index a001ce0a2..0b237c160 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -7,7 +7,7 @@ "blueprints/us-equities/adaptive-paper/config-mover.json": "560849b980082a72f8ec45a32ffae51b1f50a502e739c94346ff501a46777ea8", "blueprints/us-equities/adaptive-paper/config-sip.json": "3587653104e68fdfc9bbb9170825c611abcf7e2e1be1004327a056031c34555a", "blueprints/us-equities/adaptive-paper/config.json": "2b555982034af02b9d0d2e1d20e17d22c1e83f2b0c4d60b61d91307ebec63562", - "blueprints/us-equities/adaptive-paper/corporate_actions.py": "9f6585f055074fd5f356a12d4f0ba835271262438e8d45b11a57060f48d8714c", + "blueprints/us-equities/adaptive-paper/corporate_actions.py": "4bcecdaf26e960afb75bdc0a337234dba0aba38721fd6d2cf3fd6563fbefbe52", "blueprints/us-equities/adaptive-paper/exits.py": "189133e6df99bda201d73d36a15f45ccbfb38979a3319ae6f75d077f5db5b3df", "blueprints/us-equities/adaptive-paper/feeds.py": "0db3009a724ae2433572c5070066d6806d7365031f9db4e81c4b69ad1cbebacb", "blueprints/us-equities/adaptive-paper/leverage.py": "7f4a86f312d718a0a0b87e6eafe9a2d57b8709ec85a8746fdf7def0a5b502ea0", @@ -19,12 +19,12 @@ "blueprints/us-equities/adaptive-paper/mover_simulation.py": "89d3c42273aecb3e41dbd13cbe3a14815728ffb014a90c444a2badeb271a6311", "blueprints/us-equities/adaptive-paper/mover_strategy.py": "5e369b963d1049589cb8fcf1d75ff2badc385067fa0517a958b9f1137c73d4ab", "blueprints/us-equities/adaptive-paper/native_adapter.py": "3e9cf2d93980b0f59b781dc3771f176c4e90d6af921b61c1022017cf85e64e0f", - "blueprints/us-equities/adaptive-paper/native_strategy.py": "feffae29a0d333114a50ca5aa5726d31f41402f78041a8bfc4ed5b24ccd9f1c0", + "blueprints/us-equities/adaptive-paper/native_strategy.py": "1486a34e68ef4105046d19c7da7901acadfbdf1620e0f6c843b673b99f70ff56", "blueprints/us-equities/adaptive-paper/receipts/adaptive_policy_v1.golden.json": "5cb4b21c7d2159e496093c67b3b358810db4e225d9c5ee1c633258050973556e", "blueprints/us-equities/adaptive-paper/recovery.py": "5cf7dcea572efbda6979a2726eebf6d676f6bd62263900c7b6626da8b45c1215", "blueprints/us-equities/adaptive-paper/registry.json": "60da73b985e718bd1dece7a3350f34636bda0fb38e766a44ae71c5a8c532282a", "blueprints/us-equities/adaptive-paper/requirements.txt": "518afe4b77f901e87abd3a5d6faa126aa7e211db77eecec80cfe7ef42d82e07a", - "blueprints/us-equities/adaptive-paper/runner.py": "19efa96158df3aa105c50b9262ffb9c349e8f3a66f454196a9ae66e5f8282e29", + "blueprints/us-equities/adaptive-paper/runner.py": "d9dd13161fe1ae418b254f5046039441ddd01b3161b613624cb65a1536190ca9", "blueprints/us-equities/adaptive-paper/safety.py": "ad520fc4d84ce10e4d7b5cc956ba769a4f94eef49464ce382746d57f0cd76420", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", @@ -35,7 +35,7 @@ "blueprints/us-equities/adaptive-paper/transport.py": "34e6c4e604179585e3141c4e07066bbc36a655c3eb7052a007354fbe6b2835a6", "tests/adaptive_paper_hermetic.py": "34f2d5b07f42cb60d415abf24e4feef64b9088d1071163ab8427f0d958ebf7ee", "tests/test_adaptive_market_research.py": "dfd7a09380a7b1b10633c2edf05daf5a35bfcb48498fff130a2d84ff7190e395", - "tests/test_adaptive_paper_corporate_actions.py": "09192cb074b6ced0d9a1760c0fe4d9206d2eab0c035d6cbeb68fb464ef0d7f1e", + "tests/test_adaptive_paper_corporate_actions.py": "5bfc4bdf8b27fc2b98e7729f532ddcdd4b9191c4dca046ce6b45e1bdb217f017", "tests/test_adaptive_paper_exits.py": "13a82d65ad1b2e72a8ca0fbb8bbb178a01ded1301399604c9e304ae213456764", "tests/test_adaptive_paper_live_manifest.py": "4596f301ac44effefee5b787c5e3627a9cd38f8ff0cf436ef778b7378a0667c2", "tests/test_adaptive_paper_metrics.py": "8b0627f752ddf11c87005200e76d0ad4873261905ba073163476c66659d564f9", @@ -43,10 +43,10 @@ "tests/test_adaptive_paper_mover_native.py": "0257ca4bf9078e348e39d391dccf042b3f7b0e01d5c9e4d9092a9712f04097d5", "tests/test_adaptive_paper_native.py": "7c97707a5c19b704be9a603bbde396d9cb952dad39ca0d5b434ae1f8ebbd1f81", "tests/test_adaptive_paper_recovery.py": "08e351ff903d6d008b802dc93a8dde07da8900df5cc0540f551ffa909cf4ce95", - "tests/test_adaptive_paper_runner.py": "c9d9ce3d6c2ee61de4afd842000e792a1b59685aaa3ab8d35a48257c5454735b", + "tests/test_adaptive_paper_runner.py": "ae8e10dbac16551bfe4451259373ab0120e5d2b62be0b6813dbe99328de2c7ba", "tests/test_adaptive_paper_safety.py": "e0b2017ea6bdae87a0778b01ac72747642b31a2495c6d4e8bc98619a5aa1b456", "tests/test_adaptive_paper_selector.py": "27b431ac7911372d050f25d32ac24a3df9d2cf3c5e7dafdebfe51db6cab55936", - "tests/test_adaptive_paper_sessions.py": "09a4aaf68b3d8857bdceddae91ea11d1575e000eeeda85496e1dbeb321d8495d", + "tests/test_adaptive_paper_sessions.py": "680060ac7ec147b93ed6ba2607631ad1ec38702f1a44e51260c792c537b1fab6", "tests/test_adaptive_paper_strategies.py": "f0814b36909f2fc6b4c4213b4142ffc13339740f0397c99ad9c7c81982e2033a", "tests/test_adaptive_paper_transport.py": "eaf266a564749de4202e8e5828cbec67901027572a3c9c002473ba4981249a17" } diff --git a/manifests/evidence.json b/manifests/evidence.json index 470edce72..522e02793 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -6518,6 +6518,11 @@ "sha256": "2b555982034af02b9d0d2e1d20e17d22c1e83f2b0c4d60b61d91307ebec63562", "bytes": 2556 }, + { + "path": "blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json", + "sha256": "7883cd188c6449d290f92cacda621f12f3de223dfb0e080663150ed36289afb9", + "bytes": 7934 + }, { "path": "blueprints/us-equities/adaptive-paper/evidence-hashes.json", "sha256": "a38097f52aa89d7509e848bc244bc796bd6d96ad3314635e4b6f8b7cca6df5bf", @@ -6615,8 +6620,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/native_strategy.py", - "sha256": "feffae29a0d333114a50ca5aa5726d31f41402f78041a8bfc4ed5b24ccd9f1c0", - "bytes": 73971 + "sha256": "1486a34e68ef4105046d19c7da7901acadfbdf1620e0f6c843b673b99f70ff56", + "bytes": 77351 }, { "path": "blueprints/us-equities/adaptive-paper/receipt.json", @@ -6645,8 +6650,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/runner.py", - "sha256": "19efa96158df3aa105c50b9262ffb9c349e8f3a66f454196a9ae66e5f8282e29", - "bytes": 111667 + "sha256": "d9dd13161fe1ae418b254f5046039441ddd01b3161b613624cb65a1536190ca9", + "bytes": 118532 }, { "path": "blueprints/us-equities/adaptive-paper/safety.py", @@ -6670,7 +6675,7 @@ }, { "path": "blueprints/us-equities/adaptive-paper/source-hashes.json", - "sha256": "a665741f11ed6bd087ae22d3baaefde99f4ceda0280c0764adfc09baeaaa777f", + "sha256": "10bb8dbdd5f83436096a36bf31866ef8285d7d71e3525a7c96bdb0f5171658b3", "bytes": 6122 }, { @@ -29010,8 +29015,8 @@ }, { "path": "tests/test_adaptive_paper_runner.py", - "sha256": "c9d9ce3d6c2ee61de4afd842000e792a1b59685aaa3ab8d35a48257c5454735b", - "bytes": 158125 + "sha256": "ae8e10dbac16551bfe4451259373ab0120e5d2b62be0b6813dbe99328de2c7ba", + "bytes": 176985 }, { "path": "tests/test_adaptive_paper_safety.py", @@ -29020,8 +29025,8 @@ }, { "path": "tests/test_adaptive_paper_sessions.py", - "sha256": "09a4aaf68b3d8857bdceddae91ea11d1575e000eeeda85496e1dbeb321d8495d", - "bytes": 96318 + "sha256": "680060ac7ec147b93ed6ba2607631ad1ec38702f1a44e51260c792c537b1fab6", + "bytes": 96452 }, { "path": "tests/test_adaptive_paper_strategies.py", diff --git a/tests/test_adaptive_paper_corporate_actions.py b/tests/test_adaptive_paper_corporate_actions.py index dec06ef1b..2cc07d36f 100644 --- a/tests/test_adaptive_paper_corporate_actions.py +++ b/tests/test_adaptive_paper_corporate_actions.py @@ -12,6 +12,7 @@ from datetime import date from decimal import Decimal as D import importlib.util +import json from pathlib import Path import sys import unittest @@ -331,29 +332,152 @@ def test_successful_attempt_reverts_to_the_ordinary_refresh_interval(self): monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1200.0) self.assertEqual(len(source.calls), 2, "a successful attempt must revert to the ordinary refresh_seconds throttle") + def test_successful_refresh_clears_degraded_state(self): + # fix round 3 (mutation F3b): a symbol degraded by a prior failure + # must go back to non-degraded (needs_attention False) once a + # SUBSEQUENT refresh actually succeeds for it -- degraded must not + # be sticky forever. + source = self.FakeSource(error=CA.CorporateActionLookupError("boom")) + monitor = CA.CorporateActionMonitor(source, refresh_seconds=900, retry_seconds=60, clock=lambda: 1000.0) + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + degraded = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, + candidate_symbols=set(), now=1000.0)["AAPL"] + self.assertTrue(degraded.needs_attention) + source.error = None + source.result = {"AAPL": []} + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1070.0) + recovered = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, + candidate_symbols=set(), now=1070.0)["AAPL"] + self.assertFalse(recovered.needs_attention, "a successful refresh must clear the degraded state") + self.assertFalse(recovered.block_entry) + + def test_superseded_failure_does_not_re_degrade_a_newer_confirmed_action(self): + """HIGH finding 1 (fix round 3): a FAILURE from a superseded + (older-generation) attempt must be dropped exactly like a + superseded SUCCESS -- an older attempt that eventually raises must + not re-degrade a symbol a newer attempt already confirmed.""" + monitor = CA.CorporateActionMonitor(self.FakeSource(), clock=lambda: 1000.0) + # Simulate: generation 1 was an old attempt (now superseded); + # generation 2 is the newer attempt that already succeeded and + # confirmed an action. + old_generation = 1 + monitor._generation = 2 + monitor._apply_success({"AAPL"}, {"AAPL": [action("AAPL", "forward_split", TODAY)]}, 1030.0, 2) + confirmed = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, + candidate_symbols=set(), now=1030.0)["AAPL"] + self.assertTrue(confirmed.must_flatten) + # The OLD (generation 1) attempt now finally fails -- must be dropped. + monitor._apply_failure({"AAPL"}, old_generation) + after = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, + candidate_symbols=set(), now=1030.0)["AAPL"] + self.assertTrue(after.must_flatten, "a superseded (older-generation) failure must not re-degrade " + "a newer confirmed action") + self.assertFalse(after.needs_attention) + + def test_refresh_timed_out_invalidates_the_in_flight_generation(self): + """HIGH finding 1 (fix round 3): refresh_timed_out() must bump the + generation counter so a subsequent LATE write from the abandoned + attempt (identified by the generation it captured before the + timeout) is dropped as superseded, exactly like an ordinary + superseded success/failure.""" + monitor = CA.CorporateActionMonitor(self.FakeSource(), clock=lambda: 1000.0) + monitor._last_attempt = 1000.0 + monitor._generation = 1 + abandoned_generation = monitor._generation + monitor.refresh_timed_out({"AAPL"}) + self.assertTrue(monitor._last_attempt_failed) + self.assertIn("AAPL", monitor._degraded) + # The abandoned attempt's thread finally finishes and tries to + # write a stale success under its OLD (now-superseded) generation. + monitor._apply_success({"AAPL"}, {"AAPL": [action("AAPL", "forward_split", TODAY)]}, + 1000.0, abandoned_generation) + after = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, + candidate_symbols=set(), now=1000.0)["AAPL"] + self.assertTrue(after.needs_attention, "the abandoned attempt's late write must be dropped, " + "leaving the symbol degraded from the timeout") + self.assertFalse(after.must_flatten) + + def test_omission_after_a_prior_success_preserves_the_confirmed_action(self): + """LOW finding 7 (fix round 3): a symbol omitted from an OTHERWISE + successful fetch (most requested symbols resolved fine, this one + did not come back at all) must not erase its own previously + confirmed action -- treated exactly like a per-symbol failure.""" + class OmitsOneSymbol: + def __init__(self): + self.omit_aapl = False + + def fetch(self, symbols, start, end): + out = {s: [action(s, "forward_split", TODAY)] for s in symbols} + if self.omit_aapl: + out.pop("AAPL", None) + return out + + source = OmitsOneSymbol() + monitor = CA.CorporateActionMonitor(source, refresh_seconds=900, clock=lambda: 1000.0) + monitor.refresh({"AAPL", "MSFT"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + before = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, + candidate_symbols=set(), now=1000.0)["AAPL"] + self.assertTrue(before.must_flatten) + source.omit_aapl = True + monitor.refresh({"AAPL", "MSFT"}, start=TODAY, end=NEXT_SESSION, now=1901.0) + after = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, + candidate_symbols=set(), now=1901.0)["AAPL"] + self.assertTrue(after.must_flatten, "an omission after a prior success must preserve the " + "confirmed action, not erase it") + self.assertTrue(after.needs_attention) + ALPACA = importlib.util.find_spec("alpaca") is not None @unittest.skipUnless(ALPACA, "requires the pinned alpaca-py package") class AlpacaCorporateActionsSourceTests(unittest.TestCase): - """Exercises AlpacaCorporateActionsSource.fetch()'s response-parsing - logic with a fake alpaca-py response object (no network, no real - credentials -- construction never calls the network; see - AlpacaCorporateActionsSource's own docstring), plus its constructor-time - request-timeout/retry hardening (finding 2).""" - - class _Item: - def __init__(self, **fields): - self.__dict__.update(fields) - - class _Response: - def __init__(self, data): - self.data = data + """Exercises AlpacaCorporateActionsSource.fetch() through the REAL + alpaca-py 0.44.0 request/response pipeline (raw_data=True, + RESTClient._get_marketdata, its pagination) with only the underlying + HTTP transport faked (requests.Session.request patched to return a + canned JSON body) -- no network, no real credentials, but the actual + installed SDK code this wrapper depends on DOES run, so a change to + that SDK's own parsing/pagination/type-branch behaviour would be + caught here (fix round 3, finding 2: earlier tests injected an + already-parsed `.data`-style response, bypassing exactly the SDK + behaviour that turned out to be broken).""" def source(self): return CA.AlpacaCorporateActionsSource("fake-key", "fake-secret") + @staticmethod + def _fake_http_response(body, status_code=200): + import requests + resp = requests.Response() + resp.status_code = status_code + resp._content = json.dumps(body).encode() + resp.headers["Content-Type"] = "application/json" + return resp + + def _mock_http(self, body): + """A context manager patching requests.Session.request (BEFORE + AlpacaCorporateActionsSource construction, so its `_bounded_request` + wrapper's captured `original_request` is this mock) to return a + fake HTTP 200 response carrying `body` as JSON, wrapped in the + top-level "corporate_actions" key alpaca-py's own + `_get_marketdata_entries` (verified via inspect.getsource against + the installed package) requires -- exercising the real + RESTClient._request/_one_request/_get_marketdata call chain, + including its own page-loop. Returns (patcher, captured) where + `captured` collects the actual outgoing request's method/url/kwargs + for assertion.""" + import unittest.mock as mock + import requests + captured = {} + envelope = {"corporate_actions": body, "next_page_token": None} + + def fake_request(self_session, method, url, **kwargs): + captured["method"], captured["url"], captured["kwargs"] = method, url, kwargs + return self._fake_http_response(envelope) + + return mock.patch.object(requests.Session, "request", fake_request), captured + def test_construction_cuts_retries(self): # finding 2: alpaca-py's RESTClient retries up to 3 times, sleeping # 3s between attempts, by default -- construction must cut this to @@ -362,6 +486,14 @@ def test_construction_cuts_retries(self): self.assertEqual(src._client._retry, 1) self.assertEqual(src._client._retry_wait, 1) + def test_construction_requests_raw_data(self): + # HIGH finding 2 (fix round 3): raw_data=True is required so an + # unmapped bucket type reaches this wrapper's own fail-closed path + # instead of being silently dropped by the SDK's CorporateActionsSet + # parser first. + src = self.source() + self.assertTrue(src._client._use_raw_data) + def test_construction_bounds_the_underlying_session_request_timeout(self): # finding 2: alpaca-py's RESTClient sets NO per-request timeout at # all by default -- the constructor must wrap the underlying @@ -383,39 +515,102 @@ def fake_request(self_session, method, url, **kwargs): src._client._session.request("GET", "https://example.invalid") self.assertEqual(captured.get("timeout"), (5, 15)) - def test_result_at_the_page_limit_cap_is_ambiguous_fails_closed(self): - src = self.source() - # 1000 items across (one bucket) at the configured request cap. - items = [self._Item(symbol="AAPL", ex_date=TODAY) for _ in range(src._REQUEST_LIMIT)] - src._client.get_corporate_actions = lambda request: self._Response({"cash_dividends": items}) - with self.assertRaises(CA.CorporateActionLookupError): + def test_fetch_requests_data_quality_all(self): + """HIGH finding 3 (fix round 3): the request must ask for + data_quality=all -- the documented Alpaca default (`complete`) + excludes records the provider considers incomplete.""" + patcher, captured = self._mock_http({"cash_dividends": []}) + with patcher: + src = self.source() src.fetch(["AAPL"], TODAY, NEXT_SESSION) + self.assertEqual(captured["kwargs"].get("params", {}).get("data_quality"), "all") + + def test_result_at_the_page_limit_cap_is_ambiguous_fails_closed(self): + items = [{"symbol": "AAPL", "ex_date": TODAY.isoformat()} for _ in range(1000)] + patcher, _ = self._mock_http({"cash_dividends": items}) + with patcher: + src = self.source() + with self.assertRaises(CA.CorporateActionLookupError): + src.fetch(["AAPL"], TODAY, NEXT_SESSION) + + def test_cap_counts_the_total_across_all_buckets_not_the_largest_bucket(self): + # fix round 3 (mutation F1c): 600 + 600 = 1200 >= 1000 across TWO + # buckets, each individually under the cap. Summing across buckets + # must still fail closed; a max()-of-one-bucket implementation + # would wrongly trust this as "under the cap". + items_a = [{"symbol": "AAPL", "ex_date": TODAY.isoformat()} for _ in range(600)] + items_b = [{"old_symbol": "OLD", "new_symbol": "NEW", "alternate_symbol": "ALT", + "effective_date": TODAY.isoformat()} for _ in range(600)] + patcher, _ = self._mock_http({"cash_dividends": items_a, "unit_splits": items_b}) + with patcher: + src = self.source() + with self.assertRaises(CA.CorporateActionLookupError): + src.fetch(["AAPL"], TODAY, NEXT_SESSION) def test_result_under_the_cap_is_trusted(self): - src = self.source() - items = [self._Item(symbol="AAPL", ex_date=TODAY)] - src._client.get_corporate_actions = lambda request: self._Response({"cash_dividends": items}) - out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) + patcher, _ = self._mock_http({"cash_dividends": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}]}) + with patcher: + src = self.source() + out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) self.assertEqual(len(out["AAPL"]), 1) self.assertEqual(out["AAPL"][0].action_type, "cash_dividend") self.assertEqual(out["AAPL"][0].action_date, TODAY) def test_unmapped_type_key_fails_closed_instead_of_being_silently_dropped(self): - src = self.source() - items = [self._Item(symbol="AAPL", ex_date=TODAY)] - src._client.get_corporate_actions = lambda request: self._Response({"some_future_action_type": items}) - with self.assertRaises(CA.CorporateActionLookupError): - src.fetch(["AAPL"], TODAY, NEXT_SESSION) + # HIGH finding 2 (fix round 3): `reorganizations` is a real, + # Alpaca-documented bucket type this guard's _TYPE_FIELD_MAP does + # not (yet) model. With raw_data=True this reaches this wrapper's + # own fail-closed path directly -- the pre-fix code went through + # the SDK's CorporateActionsSet, whose 13-branch parser (verified + # via inspect.getsource against the installed package) has no + # else/default clause and silently drops any bucket it does not + # recognize, turning this exact case into an apparently-clear + # {'AAPL': []} instead of raising. + patcher, _ = self._mock_http({"reorganizations": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}]}) + with patcher: + src = self.source() + with self.assertRaises(CA.CorporateActionLookupError): + src.fetch(["AAPL"], TODAY, NEXT_SESSION) + + def test_unmapped_type_with_a_thousand_records_still_fails_closed(self): + # finding 2: 1000 records of an unmapped type used to become a + # silently-clear result AND evade the cap check (the SDK dropped + # the bucket before either check ever saw it). Both this wrapper's + # cap check and its unmapped-type check now see the raw bucket; + # either failing closed is acceptable, but a clear/successful + # result is not. + items = [{"symbol": "AAPL", "ex_date": TODAY.isoformat()} for _ in range(1000)] + patcher, _ = self._mock_http({"reorganizations": items}) + with patcher: + src = self.source() + with self.assertRaises(CA.CorporateActionLookupError): + src.fetch(["AAPL"], TODAY, NEXT_SESSION) def test_unit_split_alternate_symbol_is_mapped(self): - src = self.source() - item = self._Item(old_symbol="OLD", new_symbol="NEW", alternate_symbol="ALT", - effective_date=TODAY) - src._client.get_corporate_actions = lambda request: self._Response({"unit_splits": [item]}) - out = src.fetch(["OLD", "NEW", "ALT"], TODAY, NEXT_SESSION) + body = {"unit_splits": [{"old_symbol": "OLD", "new_symbol": "NEW", "alternate_symbol": "ALT", + "effective_date": TODAY.isoformat()}]} + patcher, _ = self._mock_http(body) + with patcher: + src = self.source() + out = src.fetch(["OLD", "NEW", "ALT"], TODAY, NEXT_SESSION) for symbol in ("OLD", "NEW", "ALT"): self.assertEqual(len(out[symbol]), 1, f"{symbol} must be mapped from a unit_split") + def test_record_missing_governing_date_is_ambiguous_for_its_own_symbol_only(self): + """HIGH finding 3 (fix round 3): a record missing (or carrying an + unparseable) governing date -- e.g. an incomplete record surfaced + by data_quality=all -- must mark ONLY the symbol(s) that record + names as LOOKUP_AMBIGUOUS, never fail the whole fetch, and never + affect an unrelated symbol's own clean result.""" + body = {"cash_dividends": [{"symbol": "AAPL"}, # no ex_date at all + {"symbol": "MSFT", "ex_date": TODAY.isoformat()}]} + patcher, _ = self._mock_http(body) + with patcher: + src = self.source() + out = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) + self.assertEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + self.assertEqual(len(out["MSFT"]), 1) + @unittest.skipUnless(NATIVE, "requires pinned Nautilus 2.0.0rc5 runtime") class RebalanceCorporateActionGuardTests(unittest.TestCase): @@ -481,9 +676,16 @@ def positions(self): return self._positions class _FakeGuard: - """Scripted guard: returns exactly the decisions it was constructed - with (keyed by symbol), regardless of the arguments rebalance() - passes -- the args themselves are recorded for assertion.""" + """Scripted guard: returns the decisions it was constructed with + (keyed by symbol), FILTERED to exactly the symbols actually + requested (held_symbols | candidate_symbols) -- fix round 3: a + prior version returned every scripted decision unconditionally, + which masked the production bug where a symbol whose signal + disappeared (and whose resting buy order therefore never made it + into candidate_symbols) was silently never evaluated at all; this + fake would still have reported its scripted decision anyway. The + real corporate_actions.evaluate_guard performs exactly this same + held|candidate filter.""" def __init__(self, decisions): self.decisions = decisions @@ -492,7 +694,8 @@ def __init__(self, decisions): def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): self.calls.append({"today": today, "next_session_date": next_session_date, "held_symbols": set(held_symbols), "candidate_symbols": set(candidate_symbols)}) - return self.decisions + relevant = set(held_symbols) | set(candidate_symbols) + return {symbol: decision for symbol, decision in self.decisions.items() if symbol in relevant} def strategy(self, *, held=None, targets=None, guard_decisions=None, clock=lambda: 1790256600.0): # 2026-09-24 09:30:00 America/New_York in epoch seconds (RTH open) -- @@ -568,6 +771,10 @@ def test_held_symbol_with_qualifying_action_is_flattened_before_close(self): self.assertEqual(call["side"], OrderSide.SELL) self.assertEqual(call["quantity"], "3") self.assertIn("reason=corporate_action_flatten", call["tags"]) + # fix round 3 (mutation F5b): the strategy must actually RECORD + # this tick's must_flatten set on itself -- runner.py's outcome + # construction reads this attribute directly. + self.assertEqual(strategy._ca_last_must_flatten, {"AAPL"}) def test_flatten_still_forced_even_when_decide_would_have_kept_the_position(self): """Unlike an ordinary rebalance-delta sell, the forced flatten fires @@ -600,6 +807,11 @@ def test_needs_attention_events_are_recorded_without_forcing_a_flatten(self): self.assertEqual(len(events), 1) self.assertTrue(events[0]["needs_attention"]) self.assertFalse(events[0]["must_flatten"]) + # MEDIUM finding 4 (fix round 3): the strategy must record this + # held symbol into its own attention-tracking attribute -- + # runner.py's outcome construction reads this directly so + # _honest_overnight_hold can refuse a false "held_overnight". + self.assertEqual(strategy._ca_last_needs_attention_held, {"AAPL"}) def test_flatten_event_carries_no_credential_value_only_bounded_fields(self): guard_decisions = {"AAPL": self.decision_for("AAPL", block_entry=True, must_flatten=True, @@ -645,6 +857,22 @@ def test_calendar_failure_blocks_entry_but_never_force_flattens(self): self.assertTrue(events[0]["needs_attention"]) self.assertEqual(events[0]["reason"], "corporate_action_session_unknown") + # -- fix round 3 (mutation F4a): the guard's own hold horizon must be + # the NEXT TRADING DAY (skipping weekends/holidays), not a raw + # calendar-day-plus-one -- proven with a Friday tick, whose "next + # session" is the following Monday, not Saturday. + def test_friday_horizon_is_the_following_monday_not_saturday(self): + # 2026-09-25 10:00 ET is a Friday (RTH open) under the frozen 2026 + # calendar. + friday_clock = lambda: 1790344800.0 + strategy = self.strategy(held={"AAPL": 1}, targets={"AAPL": 1}, guard_decisions={}, clock=friday_clock) + strategy.rebalance() + self.assertEqual(len(strategy.corporate_action_guard.calls), 1) + call = strategy.corporate_action_guard.calls[0] + self.assertEqual(call["today"], date(2026, 9, 25)) + self.assertEqual(call["next_session_date"], date(2026, 9, 28), + "the horizon must skip the weekend to the following Monday, not land on Saturday") + # -- finding 9: a symbol flagged by BOTH the corporate-action guard # (must_flatten) and the gap-risk stop must get exactly ONE sell # action, not two competing ones. @@ -673,6 +901,29 @@ def test_resting_buy_is_cancelled_for_a_newly_blocked_symbol(self): self.assertIn("resting-buy-1", strategy.cancelled) self.assertTrue(strategy.pending["resting-buy-1"]["cancel_requested"]) + # -- MEDIUM finding 5 (fix round 3): a symbol whose signal disappeared + # (no longer in decision.targets) but whose buy from a PRIOR tick is + # still resting must still be evaluated by the guard -- a corporate + # action discovered on this now-untargeted symbol must still cancel + # that resting buy, not leave it silently eligible to fill. + def test_resting_buy_is_cancelled_when_its_signal_disappears_while_it_rests(self): + guard_decisions = {"AAPL": self.decision_for("AAPL", block_entry=True, needs_attention=True, + reason="corporate_action_lookup_failed")} + # targets={} -- AAPL's signal is gone this tick, unlike the test + # above where it is still targeted. + strategy = self.strategy(held={}, targets={}, guard_decisions=guard_decisions) + strategy.cancelled = [] + strategy.cancel_order = lambda cid: strategy.cancelled.append(str(cid)) + strategy.pending["resting-buy-1"] = {"symbol": "AAPL", "side": "buy", "created": 0.0} + strategy.rebalance() + self.assertIn("resting-buy-1", strategy.cancelled, + "a resting buy must be cancelled even after its own signal disappeared") + self.assertTrue(strategy.pending["resting-buy-1"]["cancel_requested"]) + # The guard's own evaluate() call must have actually been asked + # about AAPL as a candidate (via the pending-buy symbol union), + # not silently skipped. + self.assertIn("AAPL", strategy.corporate_action_guard.calls[-1]["candidate_symbols"]) + # -- finding 11 (byte-identical claim): with the guard off (None, # default), the decision event must carry neither # corporate_action_block_entry nor corporate_action_must_flatten. @@ -710,6 +961,29 @@ def test_guard_event_re_emitted_when_the_decision_changes(self): events = [e for e in strategy.events if e.get("type") == "corporate_action_guard"] self.assertEqual(len(events), 2, "a changed decision must be re-emitted") + # -- NIT finding 9 (fix round 3): a symbol that was previously flagged + # and then goes fully clear must emit one explicit "cleared" event. + def test_guard_emits_a_cleared_event_when_a_flagged_symbol_resolves(self): + guard = self._FakeGuard({"AAPL": self.decision_for("AAPL", block_entry=True, needs_attention=True, + reason="corporate_action_lookup_failed")}) + strategy = self.strategy(held={"AAPL": 1}, targets={"AAPL": 1}) + strategy.corporate_action_guard = guard + strategy.rebalance() + guard.decisions = {"AAPL": self.decision_for("AAPL")} # fully clear now + strategy.rebalance() + events = [e for e in strategy.events if e.get("type") == "corporate_action_guard"] + self.assertEqual(len(events), 2) + self.assertEqual(events[1]["reason"], "corporate_action_cleared") + self.assertFalse(events[1]["block_entry"]) + self.assertFalse(events[1]["must_flatten"]) + self.assertFalse(events[1]["needs_attention"]) + # A symbol never flagged in the first place gets no clear event. + guard2 = self._FakeGuard({"MSFT": self.decision_for("MSFT")}) + strategy2 = self.strategy(held={"MSFT": 1}, targets={"MSFT": 1}) + strategy2.corporate_action_guard = guard2 + strategy2.rebalance() + self.assertEqual([e for e in strategy2.events if e.get("type") == "corporate_action_guard"], []) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_adaptive_paper_runner.py b/tests/test_adaptive_paper_runner.py index bda8959bc..51f0817c2 100644 --- a/tests/test_adaptive_paper_runner.py +++ b/tests/test_adaptive_paper_runner.py @@ -807,6 +807,32 @@ def test_honest_overnight_hold_false_when_corporate_action_guard_still_pending(s no_guard = {k: v for k, v in clean_outcome.items() if k != "corporate_action_guard"} self.assertTrue(_honest_overnight_hold(no_guard, session_policy, post_ts)) + def test_honest_overnight_hold_false_when_a_held_symbol_was_never_verified_clear(self): + """MEDIUM finding 4 (fix round 3): a held symbol whose + corporate-action lookup failed/was ambiguous/went stale for the + whole run (needs_attention, but never a confirmed must_flatten -- + the guard never even learned enough to know whether to flatten) + must ALSO refuse a "held_overnight" success. The documented policy + is block-and-flag: not flattening an unverified holding is correct, + but declaring the run a clean success while the attention flag is + still live is not.""" + from datetime import datetime, timezone + from zoneinfo import ZoneInfo + ny = ZoneInfo("America/New_York") + post_ts = datetime(2026, 3, 10, 18, 0, tzinfo=ny).astimezone(timezone.utc).timestamp() + session_policy = {"overnight_holds": True, "extended_hours": True} + clean_outcome = {"reconciliation": {"positions": 1, "open_orders": 0}, + "adapter_errors": [], "accounting": {"halted_reason": None}, + "corporate_action_guard": {"enabled": True, "ever_flagged_symbols": ["AAPL"], + "pending_must_flatten": [], + "pending_needs_attention_held": ["AAPL"]}} + self.assertFalse(_honest_overnight_hold(clean_outcome, session_policy, post_ts)) + # Once positively verified clear (empty), a genuine hold is allowed. + cleared = {**clean_outcome, "corporate_action_guard": { + "enabled": True, "ever_flagged_symbols": ["AAPL"], + "pending_must_flatten": [], "pending_needs_attention_held": []}} + self.assertTrue(_honest_overnight_hold(cleared, session_policy, post_ts)) + def test_run_native_status_mid_rth_non_flat_end_is_needs_attention_not_held_overnight(self): """D1: run_native's own status computation used to set "held_overnight" purely from `overnight_holds and reconciled and @@ -2576,9 +2602,19 @@ def test_guard_wired_only_when_overnight_holds_enabled(self): def test_scheduled_refresh_runs_off_the_event_loop_and_clears_in_flight(self): """HIGH finding 2: the fetch runs in a worker thread (never blocks - the event loop -- proven by a concurrent loop-tick counter still - advancing while the fetch thread sleeps), and the in-flight flag is - set immediately and cleared once the background task completes.""" + the event loop), and the in-flight flag is set immediately and + cleared once the background task completes. + + fix round 3: the concurrency proof now measures ticks that occur + STRICTLY BEFORE the fetch itself finishes (`guard.finished`, set + only inside refresh() after its blocking sleep returns) rather than + ticks counted after awaiting to completion -- a prior version's + `tick_counter()` looped until its OWN exit condition (ticks >= 3) + was satisfied, which is trivially true regardless of whether the + event loop was actually free to run concurrently or was fully + blocked by a synchronous (non-threaded) refresh() call for the + whole 0.2s; that version did not actually distinguish threaded from + synchronous execution (F2a in the review's mutation table).""" import asyncio import time as time_module from datetime import date @@ -2587,29 +2623,78 @@ def test_scheduled_refresh_runs_off_the_event_loop_and_clears_in_flight(self): class SlowGuard: def __init__(self): self.calls = [] + self.finished = False def refresh(self, symbols, *, start, end, now): self.calls.append((tuple(sorted(symbols)), start, end, now)) time_module.sleep(0.2) # a blocking call, run inside asyncio.to_thread + self.finished = True async def scenario(): guard = SlowGuard() state = {"in_flight": False} - ticks = {"count": 0} + ticks_before_finished = {"count": 0} async def tick_counter(): - while state["in_flight"] or ticks["count"] < 3: - ticks["count"] += 1 + while not guard.finished: + ticks_before_finished["count"] += 1 await asyncio.sleep(0.02) - await runner_module._schedule_corporate_action_refresh( + task = await runner_module._schedule_corporate_action_refresh( guard, {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) self.assertTrue(state["in_flight"], "in_flight must be set immediately, before the thread finishes") await tick_counter() + await task self.assertFalse(state["in_flight"], "in_flight must clear once the background refresh completes") self.assertEqual(len(guard.calls), 1) self.assertEqual(guard.calls[0], (("AAPL",), date(2026, 9, 17), date(2026, 12, 24), 1000.0)) - self.assertGreaterEqual(ticks["count"], 3, "the event loop must keep ticking while the fetch runs") + self.assertGreaterEqual(ticks_before_finished["count"], 3, + "the event loop must keep ticking WHILE the fetch is still running -- " + "a synchronous (non-threaded) refresh() would block the loop entirely " + "until it finished, so this counter would stay at 0 or 1") + + asyncio.run(scenario()) + + def test_scheduled_refresh_no_op_when_already_in_flight(self): + """HIGH finding 2 (mutation F2c: the tick-loop in_flight check + removed). This tests the CALLER-SIDE contract callers rely on: the + tick loop only calls `_schedule_corporate_action_refresh` when + `not state["in_flight"]`. Verify that contract directly against the + real tick-loop condition by simulating two consecutive tick-loop + iterations exactly as run_native's own `if strategy.corporate_ + action_guard is not None and not ca_refresh_state["in_flight"]:` + guards it -- the second iteration must be skipped while the first + refresh is still running, so only ONE refresh() call happens even + though two ticks elapsed.""" + import asyncio + import time as time_module + from datetime import date + import runner as runner_module + + class SlowGuard: + def __init__(self): + self.calls = [] + + def refresh(self, symbols, *, start, end, now): + self.calls.append(now) + time_module.sleep(0.15) + + async def scenario(): + guard = SlowGuard() + state = {"in_flight": False} + for tick_now in (1.0, 2.0, 3.0): + # The exact condition run_native's tick loop uses. + if guard is not None and not state["in_flight"]: + await runner_module._schedule_corporate_action_refresh( + guard, {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), tick_now, state) + await asyncio.sleep(0.02) + # Drain until the (single) in-flight refresh finishes. + for _ in range(50): + if not state["in_flight"]: + break + await asyncio.sleep(0.02) + self.assertEqual(len(guard.calls), 1, "only the first tick's refresh must actually run " + "while it is still in flight") asyncio.run(scenario()) @@ -2660,39 +2745,293 @@ def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, symbols, start, end, _ = guard.refresh_calls[0] self.assertIn("SPY", symbols) self.assertLess(start, end) + # fix round 3 (mutation F1a): the tick-loop call site must use the + # WIDE fetch window, not a narrow today..next_session one -- a + # narrow window would also satisfy `start < end` but span only 1-2 + # days, not the ~97-day wide window this guard actually needs. + self.assertGreaterEqual((end - start).days, 90, + "the tick-loop refresh window must be wide, not the narrow today..next_session span") + + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") + def test_tick_loop_never_overlaps_refresh_calls(self): + """HIGH finding 2 (fix round 3, mutation F2c: the tick-loop + in_flight check removed). Drives run_native's REAL tick loop (not + a simulated caller-side condition) with a guard whose refresh() + takes longer than one tick (~0.1s) and records the maximum number + of CONCURRENT refresh() invocations -- must never exceed 1, proving + the tick loop's own `not ca_refresh_state["in_flight"]` check is + actually in effect at the real call site, not just replicated in a + test's own mimicked condition.""" + import threading + from simulation import SimulatedPort + config, _, _ = load_config(SOURCE / "config.json") + config.update(duration_seconds=2, cleanup_seconds=1, order_timeout_seconds=1) + config["sessions"] = {"extended_hours": True, "overnight_holds": True, + "overnight_gross_multiple": "1.0"} + config["regular_session_only"] = False + config["extended_hours_enabled"] = True + + class SlowGuard: + def __init__(self): + self.lock = threading.Lock() + self.live = 0 + self.max_live = 0 + self.calls = 0 + + def refresh(self, symbols, *, start, end, now): + import time as time_module + with self.lock: + self.live += 1 + self.max_live = max(self.max_live, self.live) + self.calls += 1 + time_module.sleep(0.3) # well past one 0.1s tick + with self.lock: + self.live -= 1 + + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + return {} + + guard = SlowGuard() + config["_corporate_action_guard"] = guard + with tempfile.TemporaryDirectory() as root: + ledger = Ledger(Path(root) / "journal.db", RiskLimits(trial_seconds=2, cleanup_seconds=1)) + ledger.start_trial(time.time()) + controller = Controller(ledger, time.time() + 3600, market_open=True) + policy = PolicyConfig(symbols=("SPY", "QQQ", "IWM", "DIA"), max_positions=4, warmup_samples=4, + warmup_seconds=.06, sample_seconds=.02, rebalance_seconds=.02, + min_hold_seconds=.05, max_hold_seconds=.4, cooldown_seconds=.05) + port = SimulatedPort(controller, policy.symbols, price=lambda symbol, tick: Decimal("100")) + controller.port = port + asyncio.run(run_native(controller, policy, [{"symbol": s} for s in policy.symbols], + "fixture", config, "100000")) + ledger.close() + self.assertGreaterEqual(guard.calls, 2, "the run must have been long enough for more than one tick " + "to attempt a refresh") + self.assertEqual(guard.max_live, 1, "the tick loop must never launch a second refresh while one " + "is still in flight") + + def test_preflight_corporate_action_refresh_uses_the_wide_window(self): + """HIGH finding 2 (fix round 3, mutation F1b: 'preflight call site + fetches a one-day window'). Directly exercises the extracted + `_preflight_corporate_action_refresh` seam (no need to drive all of + main()).""" + import asyncio + from datetime import date, timedelta + import runner as runner_module + + class FakeGuard: + def __init__(self): + self.calls = [] + + def refresh(self, symbols, *, start, end, now): + self.calls.append((tuple(sorted(symbols)), start, end, now)) + + class FakeLedger: + def positions(self): + return {} + + guard = FakeGuard() + config = {"symbols": ["SPY", "QQQ"], "_corporate_action_guard": guard} + session_policy = {"overnight_holds": True, "extended_hours": True} + state = {"in_flight": False} + asyncio.run(runner_module._preflight_corporate_action_refresh( + config, session_policy, FakeLedger(), state)) + self.assertEqual(len(guard.calls), 1) + symbols, start, end, now = guard.calls[0] + self.assertIn("SPY", symbols) + self.assertGreaterEqual((end - start).days, 90, + "the preflight refresh window must be wide, not a narrow one-day span") + + def test_preflight_corporate_action_refresh_is_a_noop_when_guard_not_wired(self): + """finding 8: the preflight fetch must never even attempt a call + when the guard is not wired in (overnight_holds disabled).""" + import asyncio + import runner as runner_module + + class FakeGuard: + def refresh(self, symbols, *, start, end, now): + raise AssertionError("must not be called when overnight_holds is disabled") + + class FakeLedger: + def positions(self): + return {} + + config = {"symbols": ["SPY"], "_corporate_action_guard": FakeGuard()} + state = {"in_flight": False} + asyncio.run(runner_module._preflight_corporate_action_refresh( + config, {"overnight_holds": False}, FakeLedger(), state)) + self.assertFalse(state["in_flight"]) + + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") + def test_outcome_pending_must_flatten_reflects_the_strategys_own_state(self): + """fix round 3 (mutations F5a 'outcome never reports pending_must_ + flatten' and F5b 'strategy never records this tick's must_flatten'). + A guard that unconditionally reports a confirmed must_flatten for + SPY must make run_native's own returned outcome carry SPY in + outcome["corporate_action_guard"]["pending_must_flatten"] -- this + exercises BOTH the strategy actually recording it + (_ca_last_must_flatten, F5b) AND run_native's outcome construction + actually reading that real attribute rather than a hardcoded + empty list (F5a).""" + import corporate_actions as CA + from simulation import SimulatedPort + config, _, _ = load_config(SOURCE / "config.json") + config.update(duration_seconds=1, cleanup_seconds=2, order_timeout_seconds=1) + config["sessions"] = {"extended_hours": True, "overnight_holds": True, + "overnight_gross_multiple": "1.0"} + config["regular_session_only"] = False + config["extended_hours_enabled"] = True + + class AlwaysFlattenGuard: + def refresh(self, symbols, *, start, end, now): + pass + + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + return {"SPY": CA.GuardDecision(symbol="SPY", block_entry=True, must_flatten=True, + needs_attention=False, reason="corporate_action_cash_dividend", + action_type="cash_dividend", action_date=today)} + + config["_corporate_action_guard"] = AlwaysFlattenGuard() + with tempfile.TemporaryDirectory() as root: + ledger = Ledger(Path(root) / "journal.db", RiskLimits(trial_seconds=1, cleanup_seconds=2)) + ledger.start_trial(time.time()) + controller = Controller(ledger, time.time() + 3600, market_open=True) + policy = PolicyConfig(symbols=("SPY", "QQQ", "IWM", "DIA"), max_positions=4, warmup_samples=4, + warmup_seconds=.06, sample_seconds=.02, rebalance_seconds=.02, + min_hold_seconds=.05, max_hold_seconds=.4, cooldown_seconds=.05) + port = SimulatedPort(controller, policy.symbols, price=lambda symbol, tick: Decimal("100")) + controller.port = port + outcome = asyncio.run(run_native(controller, policy, [{"symbol": s} for s in policy.symbols], + "fixture", config, "100000")) + ledger.close() + self.assertIn("SPY", outcome["corporate_action_guard"]["pending_must_flatten"]) def test_scheduled_refresh_times_out_and_still_clears_in_flight(self): - """HIGH finding 2: a refresh() call that runs longer than - CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS is abandoned (wait_for - gives up) but the in-flight flag is still cleared, so a later tick - is not permanently locked out from ever refreshing again.""" + """HIGH finding 1/2 (fix round 3): a refresh() call that runs + longer than CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS is abandoned + (wait_for gives up) but the in-flight flag is still cleared, so a + later tick is not permanently locked out from ever refreshing + again -- AND the guard's own `refresh_timed_out()` is called (if it + exposes one), treating the timeout as an immediate failure rather + than silently leaving the cache in whatever state it was in until + the abandoned thread happens to resolve on its own.""" import asyncio import time as time_module from datetime import date import runner as runner_module class HangingGuard: + def __init__(self): + self.timed_out_calls = [] + def refresh(self, symbols, *, start, end, now): time_module.sleep(runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS + 5) + def refresh_timed_out(self, symbols): + self.timed_out_calls.append(tuple(sorted(symbols))) + async def scenario(): state = {"in_flight": False} + guard = HangingGuard() original_timeout = runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = 0.05 try: await runner_module._schedule_corporate_action_refresh( - HangingGuard(), {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) + guard, {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) self.assertTrue(state["in_flight"]) for _ in range(50): if not state["in_flight"]: break await asyncio.sleep(0.02) self.assertFalse(state["in_flight"], "a timed-out refresh must still clear in_flight") + self.assertEqual(guard.timed_out_calls, [("AAPL",)], + "a timeout must be reported to the guard as a failure") finally: runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = original_timeout asyncio.run(scenario()) + def test_real_monitor_generation_drops_a_superseded_late_write(self): + """HIGH finding 1 (fix round 3): reproduces the reviewer's own + finding against the REAL CorporateActionMonitor (not a fake) -- + an older, slower refresh() attempt that finishes AFTER a newer one + has already written a confirmed action must not overwrite it. + Simulated directly at the monitor level (two threads racing + CorporateActionMonitor.refresh with a real threading.Lock/gate), + exactly as the independent review's own P2 probe did.""" + import threading + from datetime import date + import corporate_actions as CA + + gate = threading.Event() + + class Src: + def fetch(self, symbols, start, end): + if threading.current_thread().name == "A-slow": + gate.wait(5) + return {s: [] for s in symbols} # stale: server state before the action existed + return {s: [CA.ActionRecord(symbol=s, action_type="forward_split", + action_date=date(2026, 9, 25))] for s in symbols} + + monitor = CA.CorporateActionMonitor(Src(), refresh_seconds=900, retry_seconds=60, max_age_seconds=3600) + window = dict(start=date(2026, 9, 17), end=date(2026, 12, 24)) + # A starts first (slow, blocked on `gate`) with an OLDER `now` -- + # but the generation counter, not the `now` timestamp, is what + # determines supersession: A started its attempt (bumped the + # generation) before B did, so B's later-generation write must win + # regardless of which `now` value either call carried. + a = threading.Thread(target=monitor.refresh, args=({"NVDA"},), kwargs=dict(**window, now=1000.0), name="A-slow") + a.start() + # Ensure A has actually entered refresh() (bumped the generation + # and is now blocked inside Src.fetch on `gate`) before B starts. + import time as time_module + time_module.sleep(0.05) + b = threading.Thread(target=monitor.refresh, args=({"NVDA"},), kwargs=dict(**window, now=1030.0), name="B-fast") + b.start() + b.join(5) + decisions = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"NVDA"}, candidate_symbols=set(), now=1030.0) + self.assertTrue(decisions["NVDA"].must_flatten, "B's confirmed action must be visible before A resolves") + gate.set() # release A + a.join(5) + decisions_after_a = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"NVDA"}, candidate_symbols=set(), now=1030.0) + self.assertTrue(decisions_after_a["NVDA"].must_flatten, + "A's late, superseded write must NOT cancel B's already-applied confirmed action") + + def test_shutdown_cancels_and_joins_pending_corporate_action_tasks(self): + """HIGH finding 1 (fix round 3, 'keep the task and join or cancel it + at shutdown'): directly exercises the extracted + `_shutdown_corporate_action_tasks` seam -- a still-pending task + (here, a plain asyncio.sleep()-based fake standing in for a + background refresh task, so this test itself completes quickly + rather than needing to wait out a real hanging OS thread) must be + cancelled and joined, not left running past the call.""" + import asyncio + import runner as runner_module + + async def scenario(): + long_sleep_task = asyncio.create_task(asyncio.sleep(600)) + already_done_task = asyncio.create_task(asyncio.sleep(0)) + await already_done_task + state = {"in_flight": False, "tasks": [long_sleep_task, already_done_task]} + await runner_module._shutdown_corporate_action_tasks(state) + self.assertTrue(long_sleep_task.done(), "a still-pending task must be cancelled, not left running") + self.assertTrue(long_sleep_task.cancelled()) + self.assertTrue(already_done_task.done()) + + asyncio.run(scenario()) + + def test_shutdown_is_a_noop_with_no_tasks(self): + import asyncio + import runner as runner_module + + async def scenario(): + await runner_module._shutdown_corporate_action_tasks({"in_flight": False}) + await runner_module._shutdown_corporate_action_tasks({"in_flight": False, "tasks": []}) + + asyncio.run(scenario()) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_adaptive_paper_sessions.py b/tests/test_adaptive_paper_sessions.py index 5e8649e68..a6b365d08 100644 --- a/tests/test_adaptive_paper_sessions.py +++ b/tests/test_adaptive_paper_sessions.py @@ -158,8 +158,10 @@ def test_year_outside_calendar_refused(self): # D7/finding-6 (2026-09-24 fix round): dates outside the frozen # table's year(s) must not be silently classified against another # year's table. The exchange_calendars fallback is disabled here so - # the result does not depend on whether that package is installed - # (CI installs it from blueprints/us-equities/data/requirements.lock). + # the result does not depend on whether exchange_calendars is + # installed (fix round 3, item 10: CI only installs the lockfile + # into the separate promotion-gate venv, not this test interpreter, + # so that package is not reliably present here either way). # 2027 is covered by HOLIDAYS_2027 (see test_2027_calendar_covered # below); 2025 and 2028 are not. with mock.patch.object(sess, "_exchange_calendars_day", return_value=None): From 906f7f1cd1051ed71739e344894ed687989149e4 Mon Sep 17 00:00:00 2001 From: Scout Date: Thu, 24 Sep 2026 18:07:35 -0400 Subject: [PATCH 5/8] Fix round 4: close the regression and remaining findings both reviews found Both independent reviews confirmed round 3 mostly holds (all 24 earlier mutations still killed) but independently found the same new regression: - HIGH: ambiguity erased a confirmed action. AlpacaCorporateActionsSource. fetch() used to overwrite a symbol's whole record list with LOOKUP_AMBIGUOUS whenever ANY row for that symbol lacked a governing date, discarding valid records collected in the SAME response (e.g. a confirmed split plus an undated dividend for the same symbol). CorporateActionMonitor._apply_success also overwrote a previously confirmed record with a later per-symbol LOOKUP_AMBIGUOUS/LOOKUP_FAILED result. Fixed at both layers: fetch() only ever falls back to the sentinel for a symbol with ZERO valid records of its own in that response; _apply_success degrades (keeping the last good record) instead of overwriting whenever the new per-symbol result is a sentinel. - MEDIUM: raw rows with no identifiable symbol used to silently count as clear. A row (mapped or unmapped bucket) naming no symbol at all now marks every requested symbol ambiguous. - MEDIUM: uncertainty discovered after the final rebalance() tick could still end held_overnight. The outcome's corporate_action_guard summary is now recomputed fresh (_final_corporate_action_guard_summary) against the guard's current evaluate() and the truly final held positions, immediately before the outcome is built, instead of trusting the strategy's last-tick snapshot. - LOW: shutdown errors could bypass refresh-task cleanup -- session.stop()/ port.stop()/the task wait are now in their own nested try, with _shutdown_corporate_action_tasks in its own guaranteed finally. - LOW: the private _get_marketdata call is replaced with the public client.get() and this wrapper's own explicit next_page_token pagination (matching blueprints/us-equities/alpaca-historical's own bridge pattern for this endpoint), with region=us and a page-count ceiling that fails closed if a continuation token survives it. - LOW: one unmapped-type row used to fail the fetch for every requested symbol. An unmapped bucket now degrades only the symbol(s) its own rows actually name (Alpaca API reference, corporateactions-1, fetched 2026-09-24: partial_call/reorganization/capital_gains_distribution documented as additional types; data_quality accepts complete/all). - LOW (measured): a hung refresh thread used to delay run_native's own result-save/account-lock release by the hang's full duration, because asyncio.run()'s cleanup joins every asyncio.to_thread/default-executor thread. The worker now runs on a daemon thread bridged back via loop.call_soon_threadsafe; asyncio.run() never waits for it (measured: a 4s hang delayed exit by ~4s before, ~0.1s after). - NIT: a timeout processed immediately after a successful commit for the same generation is now ignored (a new _committed_generation check), instead of degrading an already-fresh result for the retry interval. - Wording: corrected the "never overlapped"/"bounded fetch" claims to state exactly what actually bounds each thing, and the native-check record's "2026 is the only calendar year" claim (2027 is also covered). Adds targeted tests for every finding above, including tests that go through the real HTTP-mocked SDK pagination path (a two-page response with a real next_page_token, and the cap enforced across pages) and a real run_native drive proving the shutdown path actually cancels a hanging task and invalidates it via refresh_timed_out. Re-ran the review's own mutate_rev3.py (retargeted, unmodified mutations): the two mutations flagged as regressions this round (G9, G10) and G5 are now killed. G6 ("run_native never shuts down refresh tasks") still survives -- verified redundant, not a real gap: run_native is only ever invoked via asyncio.run() (both directly and through main()'s own asyncio.run(execute())), and asyncio.run()'s own Runner.close() calls _cancel_all_tasks() before shutting down the loop, cancelling every outstanding task regardless of this module's own explicit cleanup. The explicit _shutdown_corporate_action_tasks call remains valuable for the exception-path case (finding, LOW #7, now fixed via the nested finally) and for a hypothetical caller that reuses an already-running loop, not for this survivor's literal scenario. Recomputed source-hashes.json and the matching manifests/evidence.json entries programmatically from the changed files' actual on-disk content; registered the native-check record's own row updates. Co-Authored-By: Claude Opus 5.5 (1M context) --- ...rporate-actions-native-check-20260924.json | 16 +- .../adaptive-paper/corporate_actions.py | 307 ++++++++++++------ .../us-equities/adaptive-paper/runner.py | 168 ++++++++-- .../adaptive-paper/source-hashes.json | 8 +- manifests/evidence.json | 14 +- .../test_adaptive_paper_corporate_actions.py | 197 +++++++++-- tests/test_adaptive_paper_runner.py | 240 ++++++++++++-- 7 files changed, 769 insertions(+), 181 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json b/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json index c90b74391..1e61e44b6 100644 --- a/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json +++ b/blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json @@ -4,7 +4,7 @@ "evidence_class": "native_network", "observed_on": "2026-09-24", "subject": "blueprints/us-equities/adaptive-paper/corporate_actions.py AlpacaCorporateActionsSource.fetch", - "method": "Coordinator ran the source's fetch() (and, for the raw-lag row below, the private _get_marketdata it now calls directly for data_quality=all) against Alpaca's corporate-actions data endpoint with the host's dedicated paper account keys (loaded through adaptive-paper runner.credentials from the alpaca-paper inventory entry; never printed). Read-only market-data requests; no orders.", + "method": "Coordinator ran the source's fetch() (and, for the raw-lag/envelope rows below, the client's inherited public get() -- fix round 4, item 4: fetch() no longer calls the private _get_marketdata; it does its own explicit next_page_token pagination via the public get(), matching blueprints/us-equities/alpaca-historical/README.md's own bridge pattern for this same endpoint) against Alpaca's corporate-actions data endpoint with the host's dedicated paper account keys (loaded through adaptive-paper runner.credentials from the alpaca-paper inventory entry; never printed). Read-only market-data requests; no orders.", "runtime": "alpaca-py 0.44.0, Python 3.12.3 (pinned adaptive-paper runtime)", "queries": [ {"symbols": ["NVDA"], "start": "2024-06-01", "end": "2024-06-15", @@ -26,14 +26,22 @@ "match": true}, {"symbols": [], "kind": "fetch_window_function_on_a_classifiable_anchor", "anchor": "2026-09-24", "next_session_date": "2026-09-25", "start": "2026-09-17", "end": "2026-12-24", - "expected": "fix round 3, item 8: runner._corporate_action_fetch_window(date(2026, 9, 24))'s own real, unmodified output (2026 is the only year the frozen sessions.py calendar covers) -- confirms the -7/+90 padding around the ACTUAL next_trading_day, not a hand-substituted anchor", + "expected": "fix round 3, item 8 (corrected fix round 4, item 8): runner._corporate_action_fetch_window(date(2026, 9, 24))'s own real, unmodified output. sessions.py's frozen calendar covers 2026 AND 2027 (HOLIDAYS_2027/EARLY_CLOSES_2027, verified against the official NYSE holiday schedule and exchange_calendars); 2026 was picked here only because it is this record's own observed_on year, not because 2027 is uncovered -- confirms the -7/+90 padding around the ACTUAL next_trading_day, not a hand-substituted anchor", "match": true, "note": "no source query -- a pure local function call, listed here to make the 2024 approximation above verifiably distinct from the function's real behaviour"}, {"symbols": ["AAPL"], "start": "2024-08-01", "end": "2024-08-20", "kind": "raw_record_lag", "observed": [{"symbol": "AAPL", "ex_date": "2024-08-12", "payable_date": "2024-08-15", "process_date": "2024-08-15", "record_date": "2024-08-12", "rate": 0.25, "cusip": "037833100"}], - "expected": "fix round 3, item 6/item 8: the RAW record (queried with data_quality=all via the private _get_marketdata the fixed fetch() now calls directly -- this wrapper's own ActionRecord normalization discards process_date/payable_date, so this row queries the raw response to observe them) shows a 3-DAY governing(ex_date)-to-process(process_date/payable_date) lag for this specific dividend (2024-08-12 -> 2024-08-15) -- matching Apple's own published dividend history (https://investor.apple.com/dividend-history/: ex-date August 12 2024, payment date August 15 2024). This CORRECTS the earlier '1-3 weeks' claim in this same file's prior claim_boundary, which was never measured against this record's own raw fields.", - "match": true} + "expected": "fix round 3, item 6/item 8: the RAW record (queried with data_quality=all via the client's public get() -- this wrapper's own ActionRecord normalization discards process_date/payable_date, so this row queries the raw response to observe them) shows a 3-DAY governing(ex_date)-to-process(process_date/payable_date) lag for this specific dividend (2024-08-12 -> 2024-08-15) -- matching Apple's own published dividend history (https://investor.apple.com/dividend-history/: ex-date August 12 2024, payment date August 15 2024). This CORRECTS the earlier '1-3 weeks' claim in this same file's prior claim_boundary, which was never measured against this record's own raw fields.", + "match": true}, + {"symbols": [], "kind": "public_get_response_envelope", + "observed": {"top_level_keys": ["corporate_actions", "next_page_token"], "next_page_token": null}, + "expected": "fix round 4, item 4: client.get(path='/corporate-actions', data=params) (the PUBLIC inherited RESTClient method fetch() now calls directly, with its own explicit pagination loop) returns {'corporate_actions': {...buckets...}, 'next_page_token': ...} -- confirms the envelope this wrapper's own pagination loop parses, independent of the private _get_marketdata it no longer calls.", + "match": true}, + {"symbols": [], "kind": "documented_action_types_and_data_quality", + "observed": "reference page fetched by the coordinator on 2026-09-24", + "expected": "fix round 4, item 5: https://docs.alpaca.markets/us/reference/corporateactions-1 (fetched 2026-09-24) documents partial_call, reorganization and capital_gains_distribution as action types in addition to the 13 this wrapper's own _TYPE_FIELD_MAP models, and documents data_quality as accepting complete (default) or all -- both now cited in corporate_actions.py's own AlpacaCorporateActionsSource.fetch() docstring.", + "match": true, "note": "documentation citation, not a data query -- no HTTP request behind this row"} ], "lag_observations": { "purpose": "fix round 3, item 6: record the concrete governing-date/process-date lag observations behind the 90-day forward pad (CORPORATE_ACTION_FETCH_WINDOW_FUTURE_DAYS), each with its source, so the pad is traceable evidence rather than an unqualified round number.", diff --git a/blueprints/us-equities/adaptive-paper/corporate_actions.py b/blueprints/us-equities/adaptive-paper/corporate_actions.py index 6dbe1267c..7223ea650 100644 --- a/blueprints/us-equities/adaptive-paper/corporate_actions.py +++ b/blueprints/us-equities/adaptive-paper/corporate_actions.py @@ -244,6 +244,19 @@ def __init__(self, source, *, refresh_seconds=900.0, retry_seconds=60.0, max_age # failure, whenever the thread actually finishes -- is dropped too. self._lock = threading.Lock() self._generation = 0 + # NIT finding 9 (fix round 4): the generation that most recently + # actually WROTE a result (success or failure) -- distinct from + # `_generation` itself. Lets `refresh_timed_out()` tell "the + # attempt I'm invalidating never actually committed" (a real + # timeout on a still-in-flight attempt -- must invalidate) apart + # from "the attempt already committed before this timeout handler + # even ran" (a race between the worker thread finishing and + # wait_for's own timeout firing just after -- must NOT re-degrade + # an already-fresh result; reproduced by the review's own Q5 + # probe: a timeout processed immediately after a successful fetch + # used to mark the fresh result degraded for up to `retry_seconds` + # for no reason). + self._committed_generation = 0 def refresh(self, symbols, *, start: date, end: date, now=None): """Idempotent within `refresh_seconds` for a symbol set already @@ -296,8 +309,15 @@ def refresh_timed_out(self, symbols): timeout itself as an ordinary failure right now (degraded, short `retry_seconds` retry interval), exactly as if `fetch()` had raised, rather than silently leaving the cache in its last (possibly clear) - state until the abandoned attempt happens to resolve on its own.""" + state until the abandoned attempt happens to resolve on its own. + + NIT finding 9 (fix round 4): a no-op if the current generation has + ALREADY committed (see `_committed_generation`) -- this timeout + handler firing just after the same attempt's own success/failure + already landed must never re-degrade an already-fresh result.""" with self._lock: + if self._committed_generation == self._generation: + return self._generation += 1 self._last_attempt_failed = True self._mark_degraded(symbols) @@ -306,6 +326,7 @@ def _apply_failure(self, symbols, generation): with self._lock: if generation != self._generation: return # superseded by a later attempt (or a timeout) -- drop + self._committed_generation = generation self._last_attempt_failed = True self._mark_degraded(symbols) @@ -313,13 +334,11 @@ def _apply_success(self, symbols, fetched, now, generation): with self._lock: if generation != self._generation: return # superseded -- an older result must never overwrite a newer one + self._committed_generation = generation self._last_attempt_failed = False self._last_success = now for symbol in symbols: - if symbol in fetched: - self._results[symbol] = fetched[symbol] - self._degraded.discard(symbol) - else: + if symbol not in fetched: # LOW finding 7 (fix round 3): a symbol omitted from an # OTHERWISE successful fetch (most other symbols # resolved fine) must not erase its own previously @@ -327,6 +346,23 @@ def _apply_success(self, symbols, fetched, now, generation): # failure (degraded, last good record preserved), never # overwrite with LOOKUP_FAILED. self._mark_degraded((symbol,)) + continue + new_value = fetched[symbol] + if new_value in (LOOKUP_FAILED, LOOKUP_AMBIGUOUS): + # HIGH finding 1 (fix round 4): a sentinel result (the + # symbol had no valid record of its own this fetch -- + # see AlpacaCorporateActionsSource.fetch(), which now + # only ever returns a sentinel for a symbol with ZERO + # valid records) must NEVER overwrite a previously + # confirmed (already non-empty) result -- degrade + # instead, preserving the confirmed action. Reproduced + # by the review's own Q2a probe: a confirmed split + # followed by a later per-symbol LOOKUP_AMBIGUOUS used + # to silently turn must_flatten back off. + self._mark_degraded((symbol,)) + continue + self._results[symbol] = new_value + self._degraded.discard(symbol) def _mark_degraded(self, symbols): """finding 3: only ever ADD to `_degraded` and, for a symbol never @@ -394,34 +430,37 @@ class AlpacaCorporateActionsSource: raises them in any exception message.""" # HIGH finding 2 (fix round 3): request cap -- alpaca-py==0.44.0's - # CorporateActionsRequest defaults `limit` to 1000, and its - # `RESTClient._get_marketdata` (verified via `inspect.getsource` against - # the installed package) DOES auto-paginate across `next_page_token`, - # accumulating items from every page up to this `limit` -- so a result - # at or past this cap means more matching rows may exist beyond what was - # requested, not that pagination silently stopped after one page (an - # earlier round of this comment was wrong about that). + # CorporateActionsRequest defaults `limit` to 1000. This wrapper does + # its OWN pagination now (LOW finding 4, fix round 4 -- see fetch()), + # accumulating items across pages up to this cap; a result at or past + # it means more matching rows may exist beyond what was requested. _REQUEST_LIMIT = 1000 + # LOW finding 4 (fix round 4): a hard ceiling on how many pages fetch() + # will follow `next_page_token` for, so a still-present continuation + # token can never silently loop forever -- exhausting this many pages + # while a token remains is itself treated as an ambiguous/at-cap + # result (see fetch()), never a silent truncation. + _MAX_PAGES = 5 def __init__(self, api_key, secret_key): from alpaca.data.historical.corporate_actions import CorporateActionsClient - # HIGH finding 2 (fix round 3): raw_data=True makes get_corporate_ - # actions() (and the private _get_marketdata() this wrapper calls - # directly below) return the plain parsed-JSON dict instead of a - # CorporateActionsSet. This is required, not cosmetic: - # CorporateActionsSet.__init__ (verified via inspect.getsource - # against the installed alpaca-py==0.44.0 package) is a chain of 13 - # `elif corporate_action_type == "..."` branches with NO else/ - # default clause -- any bucket key it does not recognize (the - # Alpaca API reference documents `reorganizations`, `partial_calls` - # and `capital_gains_distributions` in addition to the 13 this - # wrapper's own _TYPE_FIELD_MAP models) is silently dropped by the - # SDK before this wrapper ever sees it, turning a real corporate - # action into an apparently-clear "no action" result, and its rows - # also never reach the cap check below. Parsing the raw dict - # ourselves (see fetch()) means an unmapped bucket key is visible to - # this wrapper's own _TYPE_FIELD_MAP.get(...) is None fail-closed - # path instead of vanishing inside the SDK first. + # raw_data=True: this wrapper's own inherited `RESTClient.get()` + # (which fetch() below calls directly) already returns the plain + # parsed-JSON response regardless of this flag (verified via + # inspect.getsource against the installed alpaca-py==0.44.0 + # RESTClient.get/_request/_one_request: `_one_request` always + # returns `response.json()`) -- raw_data=True is kept here only so + # the client's OWN `get_corporate_actions()` convenience method + # (not used by this wrapper's fetch() below; LOW finding 4, fix + # round 4, moved fetch() to the public `get()` with this wrapper's + # own pagination) would behave consistently if ever called + # elsewhere. The original reason this flag mattered here -- + # CorporateActionsSet.__init__'s 13-branch, no-else parser + # (verified via inspect.getsource) silently dropping any bucket key + # it does not model, e.g. Alpaca's documented `reorganizations`, + # `partial_calls`, `capital_gains_distributions` -- is now moot for + # fetch() specifically, since it never goes through + # CorporateActionsSet at all. self._client = CorporateActionsClient(api_key, secret_key, raw_data=True) # HIGH finding 2: alpaca-py's RESTClient (verified against the # installed alpaca-py==0.44.0 RESTClient.__init__/_request) sets no @@ -430,17 +469,18 @@ def __init__(self, api_key, secret_key): # unbounded and slow by default. Pin a bounded connect/read timeout # directly onto the underlying requests.Session (mirroring # transport.py's own HTTP-boundary `timeout=(5, 5)` convention) and - # cut retries to the bare minimum. NIT finding 9 (fix round 3): a - # `requests` timeout bounds each individual connect/read, not the - # call's total wall-clock duration -- a paginated multi-page fetch - # (`_get_marketdata` above) issues one HTTP request per page, and a - # slow-but-not-hung server could still make the overall fetch() call - # run for `pages * timeout`. The actual outer bound on one fetch() - # call is runner.py's own asyncio wait_for() - # (CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) around the whole - # refresh() call this fetch() runs inside, not this per-request - # timeout alone; this timeout only prevents a single HTTP request - # from hanging indefinitely with no data ever arriving. + # cut retries to the bare minimum. LOW finding 5 (fix round 3/4): + # this `requests` timeout bounds each individual connect/read, NOT + # this fetch() call's total wall-clock duration -- a multi-page + # fetch issues one HTTP request per page, so a slow-but-not-hung + # server could still make the overall call run for roughly + # `pages * timeout`. `wait_for()` in runner.py's + # `_schedule_corporate_action_refresh` only bounds how long THAT + # COROUTINE waits for this call to finish; it does not stop this + # call, or the worker thread it runs in, from continuing to run + # past that point (see that function's own docstring and + # CorporateActionMonitor's generation check, which is what makes a + # late write from such a call harmless rather than "bounded"). session = self._client._session original_request = session.request @@ -457,10 +497,10 @@ def _parse_date(value): """A raw response field is an ISO date string (or absent/null) once parsed from JSON -- never already a `datetime.date` (that only happens through the SDK's own pydantic models, which this wrapper - no longer uses; see raw_data=True above). Returns None for - anything that is not a valid ISO date, rather than raising -- - callers treat a missing/unparseable governing date as ambiguous for - that record's symbol(s), not as a whole-fetch failure.""" + no longer uses). Returns None for anything that is not a valid ISO + date, rather than raising -- callers treat a missing/unparseable + governing date as ambiguous for that record's symbol(s), not as a + whole-fetch failure.""" if not isinstance(value, str) or not value: return None try: @@ -468,77 +508,147 @@ def _parse_date(value): except ValueError: return None + @staticmethod + def _unmapped_row_symbols(item): + """LOW finding 5 (fix round 4): a best-effort symbol extraction for + a bucket type this wrapper's `_TYPE_FIELD_MAP` does not (yet) + model -- every field Alpaca's documented corporate-action schemas + use for a tradable symbol is named `*symbol*` (symbol, old_symbol, + new_symbol, acquirer_symbol, acquiree_symbol, source_symbol, + alternate_symbol -- see `_TYPE_FIELD_MAP`'s own key tuples above), + so any key ending in "symbol" with a truthy value is treated as a + possible symbol for this row, without knowing the type's exact + schema.""" + return [value for key, value in item.items() if key.endswith("symbol") and value] + def fetch(self, symbols, start: date, end: date) -> dict: """Returns `{symbol: [ActionRecord, ...] | LOOKUP_AMBIGUOUS}` for exactly the requested `symbols` (every requested symbol is present: - either its resolved list, possibly empty, or LOOKUP_AMBIGUOUS for a - symbol whose own record(s) could not be fully parsed -- MEDIUM - finding 3, fix round 3). Raises `CorporateActionLookupError` -- - never a raw alpaca-py/HTTP exception -- on any request/transport - failure, a result at the page/limit cap, or an unmapped action-type - bucket, so a caller never needs to know alpaca-py's own exception - types and never silently treats an incomplete or unparseable - response as "no action" for every requested symbol. + either its resolved list of valid records, possibly empty, or + LOOKUP_AMBIGUOUS -- ONLY for a symbol with ZERO valid records of its + own in this response; HIGH finding 1, fix round 4: a symbol with at + least one valid record NEVER has that record discarded or replaced + by ambiguity, regardless of how many other unparseable rows also + name it). Raises `CorporateActionLookupError` -- never a raw + alpaca-py/HTTP exception -- on any request/transport failure or a + result at the page/limit cap, so a caller never needs to know + alpaca-py's own exception types and never silently treats an + incomplete or unparseable response as "no action" for every + requested symbol. HIGH finding 3 (fix round 3): requests `data_quality=all` -- Alpaca's documented default (`data_quality=complete`) excludes records the provider itself considers incomplete (e.g. missing a CUSIP/ISIN), which could otherwise make a real, known-symbol, - known-date upcoming action invisible to this guard with no - signal at all. `all` is not exposed as a typed field on the - installed alpaca-py==0.44.0 CorporateActionsRequest model (verified: - the field is silently dropped by its own to_request_fields(), a + known-date upcoming action invisible to this guard with no signal + at all. `all` is not exposed as a typed field on the installed + alpaca-py==0.44.0 CorporateActionsRequest model (verified: the + field is silently dropped by its own to_request_fields(), a pydantic extra="ignore" default), so this calls the client's own - private `_get_marketdata` directly with a hand-built params dict -- - the same private method `get_corporate_actions()` itself calls -- - instead of going through the typed request object for this one - parameter.""" + PUBLIC `get()` directly with a hand-built params dict, including + `region: "us"` (also absent from the typed request model) -- the + same pattern blueprints/us-equities/alpaca-historical/README.md + and its collect.py document and use for this same endpoint: "The + installed action request model lacks region and data_quality, so + the bridge sends those documented REST parameters directly." + + LOW finding 4 (fix round 4): pagination is this wrapper's OWN + explicit loop over `next_page_token`, matching that same bridge's + approach ("Installed source inspection confirms that their limit + acts as a total retrieval cap and that merged results discard + continuation tokens. This bridge therefore uses the inherited + native single-page get, adds its own bounded pagination...") -- + the response envelope (`{"corporate_actions": {...buckets...}, + "next_page_token": ...}`) was independently confirmed via a native + read-only call to `client.get(path="/corporate-actions", ...)` on + 2026-09-24 (see corporate-actions-native-check-20260924.json). + + Types (fix round 4): the Alpaca API reference + (https://docs.alpaca.markets/us/reference/corporateactions-1, + fetched by the coordinator 2026-09-24) documents `partial_call`, + `reorganization` and `capital_gains_distribution` as additional + action types beyond the 13 `_TYPE_FIELD_MAP` maps, and documents + `data_quality` as accepting `complete` (default) or `all`. An + unmapped bucket (any of those three, or a future one) no longer + fails the WHOLE fetch (LOW finding 5, fix round 4): only the + symbol(s) its own rows actually name become ambiguous -- an + unrelated requested symbol with no row in that bucket is + unaffected. A row with no identifiable symbol at all (every symbol + field blank/missing, mapped or unmapped bucket alike) cannot be + safely ignored -- any requested symbol could be the intended one -- + and marks EVERY requested symbol ambiguous (MEDIUM finding 2, fix + round 4).""" symbols = list(symbols) params = {"symbols": ",".join(symbols), "start": start.isoformat(), "end": end.isoformat(), - "limit": self._REQUEST_LIMIT, "sort": "asc", "data_quality": "all"} - try: - raw = self._client._get_marketdata(path="/corporate-actions", params=params, - page_limit=1000, page_size=1000) - except CorporateActionLookupError: - raise - except Exception as error: - raise CorporateActionLookupError(type(error).__name__) from error + "limit": self._REQUEST_LIMIT, "sort": "asc", "data_quality": "all", "region": "us"} out = {symbol: [] for symbol in symbols} ambiguous_symbols = set() try: - raw = dict(raw or {}) - # finding 1 (result-count-at-cap is ambiguous): a response whose - # total RAW item count (every bucket, including one this - # wrapper does not model -- see raw_data=True above) reaches the - # request's own `limit` may have been truncated server-side - # with no signal to this wrapper about which symbol's records - # were cut off -- fail closed for this whole fetch rather than - # trust a possibly-partial result. - total_items = sum(len(items) for items in raw.values()) - if total_items >= self._REQUEST_LIMIT: + raw: dict = {} + total_items = 0 + page_token = None + for _ in range(self._MAX_PAGES): + page_params = dict(params) + if page_token is not None: + page_params["page_token"] = page_token + response = self._client.get(path="/corporate-actions", data=page_params) + if not isinstance(response, dict): + raise CorporateActionLookupError("malformed_corporate_action_response") + body = response.get("corporate_actions") + if not isinstance(body, dict): + raise CorporateActionLookupError("malformed_corporate_action_response") + for type_key, items in body.items(): + if not isinstance(items, list): + raise CorporateActionLookupError("malformed_corporate_action_response") + raw.setdefault(type_key, []).extend(items) + total_items += len(items) + # finding 1 (result-count-at-cap is ambiguous): a response + # whose total RAW item count (every bucket, mapped or not) + # reaches the request's own `limit` may have been + # truncated server-side with no signal to this wrapper + # about which symbol's records were cut off -- fail closed + # for this whole fetch rather than trust a possibly-partial + # result. + if total_items >= self._REQUEST_LIMIT: + raise CorporateActionLookupError("result_at_page_or_limit_cap") + page_token = response.get("next_page_token") + if page_token is None: + break + else: + # LOW finding 4: exhausted _MAX_PAGES while a continuation + # token was STILL present -- never silently truncate. raise CorporateActionLookupError("result_at_page_or_limit_cap") for type_key, items in raw.items(): spec = _TYPE_FIELD_MAP.get(type_key) - if spec is None: - # finding 7c/finding 2 (fix round 3): an unmapped bucket - # key (e.g. Alpaca's documented `reorganizations`, - # `partial_calls`, `capital_gains_distributions`, or any - # future type this guard's _TYPE_FIELD_MAP does not yet - # know) has no known governing-date or symbol field this - # wrapper can trust to parse -- fail closed (ambiguous) - # for the whole fetch rather than silently skip an - # action type this guard does not yet recognize. Now - # actually reachable (raw_data=True, fix round 3): the - # SDK's own CorporateActionsSet used to drop these - # bucket keys before this wrapper ever saw them. - raise CorporateActionLookupError(f"unmapped_corporate_action_type:{type_key}") - symbol_fields, date_field, canonical_type = spec for item in items: if not isinstance(item, dict): raise CorporateActionLookupError("malformed_corporate_action_item") + if spec is None: + # LOW finding 5: an unmapped bucket type -- degrade + # only the symbol(s) THIS row actually names (or + # every requested symbol, if it names none at all; + # MEDIUM finding 2 below covers that same rule for + # mapped buckets too). + row_symbols = self._unmapped_row_symbols(item) + ambiguous_symbols.update(symbols if not row_symbols + else (s for s in row_symbols if s in out)) + continue + symbol_fields, date_field, canonical_type = spec + raw_symbols = [item.get(field) for field in symbol_fields if item.get(field)] + if not raw_symbols: + # MEDIUM finding 2 (fix round 4): a row with NO + # identifiable symbol at all (every symbol field on + # this row blank/missing) cannot be safely ignored + # -- any requested symbol could be the intended + # one -- ambiguous for every requested symbol in + # this fetch, not silently "no action" for all of + # them. + ambiguous_symbols.update(symbols) + continue + matched_symbols = [s for s in raw_symbols if s in out] + if not matched_symbols: + continue # a real symbol, just not one we asked about -- ignore action_date = self._parse_date(item.get(date_field)) - matched_symbols = [item.get(field) for field in symbol_fields - if item.get(field) and item.get(field) in out] if action_date is None: # HIGH finding 3 (fix round 3): a record missing (or # carrying an unparseable) governing date -- e.g. @@ -553,9 +663,14 @@ def fetch(self, symbols, start: date, end: date) -> dict: symbol=symbol, action_type=canonical_type, action_date=action_date, source_id=str(item.get("cusip") or type_key))) for symbol in ambiguous_symbols: - out[symbol] = LOOKUP_AMBIGUOUS + # HIGH finding 1 (fix round 4): never let ambiguity replace + # an already-confirmed (non-empty) record list for this + # SAME response -- only fall back to the sentinel when the + # symbol has NO valid record of its own here. + if not out[symbol]: + out[symbol] = LOOKUP_AMBIGUOUS except CorporateActionLookupError: raise except Exception as error: - raise CorporateActionLookupError("malformed_corporate_action_response") from error + raise CorporateActionLookupError(type(error).__name__) from error return out diff --git a/blueprints/us-equities/adaptive-paper/runner.py b/blueprints/us-equities/adaptive-paper/runner.py index e9fe49491..38fd27043 100644 --- a/blueprints/us-equities/adaptive-paper/runner.py +++ b/blueprints/us-equities/adaptive-paper/runner.py @@ -18,6 +18,7 @@ import re import shlex import signal +import threading import time from corporate_actions import AlpacaCorporateActionsSource, CorporateActionMonitor @@ -995,17 +996,29 @@ async def _schedule_corporate_action_refresh(guard, watch, start, end, now, stat """HIGH finding 1/2 (fix round 3): run `guard.refresh()` (a blocking network call) off the asyncio event loop in a worker thread, bounded by a hard `asyncio.wait_for` timeout, and guarded by `state["in_flight"]` - so a still-running previous refresh thread is never overlapped by a - second concurrent one -- `state` is a plain dict callers own and can - share across call sites (the tick loop AND the preflight fetch both - pass the SAME dict, fix round 3, so an in-flight preflight thread is + -- as an ordinary (not a guaranteed) scheduling check -- so a + still-running previous refresh thread is not ORDINARILY overlapped by + a second concurrent one from this scheduler's own perspective (LOW + finding 1 wording, fix round 4: this check alone does not GUARANTEE no + overlap at the OS-thread level -- see this function's own docstring + below for what actually makes an overlap harmless). `state` is a plain + dict callers own and can share across call sites (the tick loop AND + the preflight fetch both pass the SAME dict, fix round 3, so an + in-flight preflight thread is visible to the tick loop's first check too). `wait_for`'s timeout only stops THIS coroutine from waiting; it cannot stop the worker thread itself (Python cannot forcibly kill a thread) -- reproduced by an independent review: the thread kept running past the - timeout and later wrote a stale result over a newer one. Two defenses, - both now in place: + timeout and later wrote a stale result over a newer one. LOW finding 1 + (fix round 4, wording): refreshes are never overlapped only because + `CorporateActionMonitor`'s own generation check (under its lock) drops + any write from a superseded attempt -- this scheduling layer's + `state["in_flight"]` check is a best-effort scheduling optimization + (avoid spinning up a redundant thread most of the time), not itself a + correctness guarantee; a timed-out worker can still overlap a later one + at the OS-thread level, and the generation check is what makes that + harmless. Defenses in place: 1. `guard.refresh_timed_out(watch)` (if the guard exposes it -- the real CorporateActionMonitor does) is called on timeout, which bumps that monitor's internal generation counter so the abandoned @@ -1018,6 +1031,17 @@ async def _schedule_corporate_action_refresh(guard, watch, start, end, now, stat `finally` block) can cancel and join every still-pending corporate-action refresh task at shutdown, instead of leaving them as untracked, unaccounted-for background work. + 3. LOW finding 6 (fix round 4): the worker itself runs on a DAEMON + thread bridged back to this event loop via `loop.call_soon_ + threadsafe` (not `asyncio.to_thread`, which uses the loop's + default `ThreadPoolExecutor` -- `asyncio.run()`'s own cleanup + joins every outstanding default-executor thread before returning, + so a single still-hanging refresh used to delay the WHOLE + process's result-save and account-lock release by the hang's full + duration, measured: a 4s hang delayed both by ~4s with + `asyncio.to_thread`, vs ~0.1s with this daemon-thread bridge, + which never blocks `asyncio.run()`'s shutdown). A daemon thread is + not joined at interpreter exit either. A no-op if a refresh is already in flight; callers check `state["in_flight"]` before calling this so a throttled (no-op) @@ -1029,10 +1053,26 @@ async def _schedule_corporate_action_refresh(guard, watch, start, end, now, stat state["in_flight"] = True async def _run(): + loop = asyncio.get_running_loop() + fut = loop.create_future() + + def worker(): + try: + guard.refresh(watch, start=start, end=end, now=now) + except Exception: + pass # CorporateActionMonitor.refresh() never raises; defensive only + finally: + def _resolve(): + if not fut.done(): + fut.set_result(None) + try: + loop.call_soon_threadsafe(_resolve) + except RuntimeError: + pass # the loop is already closed -- the late result is simply dropped + + threading.Thread(target=worker, daemon=True, name="ca-refresh").start() try: - await asyncio.wait_for( - asyncio.to_thread(guard.refresh, watch, start=start, end=end, now=now), - timeout=CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) + await asyncio.wait_for(fut, timeout=CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) except asyncio.TimeoutError: if hasattr(guard, "refresh_timed_out"): guard.refresh_timed_out(watch) @@ -1112,6 +1152,61 @@ async def _preflight_corporate_action_refresh(config, session_policy, ledger, ca await preflight_task +def _final_corporate_action_guard_summary(strategy, held_symbols, now): + """MEDIUM finding 3 (fix round 4): the run-outcome guard summary, + recomputed FRESH against `held_symbols` (the CURRENT holdings, read + right before this call) and the guard's CURRENT monitor state -- + deliberately NOT `strategy._ca_last_must_flatten`/`_ca_last_needs_ + attention_held`, which are only as fresh as the last rebalance() tick + that actually ran. A refresh that fails, times out, or is cancelled + AFTER that last tick (e.g. during run_native's own shutdown, just + before this function is called) changes the monitor's state without + any further rebalance() tick ever re-reading it -- reproduced by an + independent review's own probe: `monitor.evaluate(...)` already + reported `needs_attention=True` for a held symbol, while the stale + tick snapshot still reported `pending_needs_attention_held=[]`, + letting `_honest_overnight_hold` wrongly approve a `held_overnight` + outcome. This function calls `strategy.corporate_action_guard. + evaluate()` directly (the same call `_corporate_action_guard_symbols` + makes every tick) one more time, right here, so the outcome always + reflects the guard's truly final state -- never emits an event or + mutates the strategy (a pure read), unlike the per-tick method.""" + guard = strategy.corporate_action_guard + ever_flagged = set(strategy._ca_ever_flagged) + if guard is None: + return {"enabled": False, "ever_flagged_symbols": sorted(ever_flagged), + "pending_must_flatten": [], "pending_needs_attention_held": []} + if not held_symbols: + return {"enabled": True, "ever_flagged_symbols": sorted(ever_flagged), + "pending_must_flatten": [], "pending_needs_attention_held": []} + try: + info = session_at(datetime.fromtimestamp(now, timezone.utc)) + next_session_date = next_trading_day(info.session_date) + except ValueError: + # Fail-closed exactly like _corporate_action_guard_symbols's own + # ValueError branch (finding 6, round 3): every held symbol is not + # positively verified clear, but none is force-flattened purely + # because the calendar itself failed to classify this instant. + return {"enabled": True, "ever_flagged_symbols": sorted(ever_flagged | set(held_symbols)), + "pending_must_flatten": [], "pending_needs_attention_held": sorted(held_symbols)} + decisions = guard.evaluate(today=info.session_date, next_session_date=next_session_date, + held_symbols=set(held_symbols), candidate_symbols=set(), now=now) + pending_must_flatten = {symbol for symbol, decision in decisions.items() if decision.must_flatten} + pending_needs_attention_held = {symbol for symbol, decision in decisions.items() + if decision.needs_attention and symbol in held_symbols} + # G9 (fix round 4): a held symbol absent from `decisions` entirely + # (should not happen -- evaluate() decides every held|candidate symbol + # -- but this function does not trust that blindly, matching + # _corporate_action_guard_symbols's own equivalent defensive union) + # counts as NOT verified, never silently "clear". + pending_needs_attention_held |= set(held_symbols) - set(decisions) + ever_flagged |= {symbol for symbol, decision in decisions.items() + if decision.block_entry or decision.must_flatten or decision.needs_attention} + return {"enabled": True, "ever_flagged_symbols": sorted(ever_flagged), + "pending_must_flatten": sorted(pending_must_flatten), + "pending_needs_attention_held": sorted(pending_needs_attention_held)} + + async def run_native(controller, policy_config, assets, trial_id, config, baseline_cash, *, account_fingerprint="simulation", log_directory=None, ca_refresh_state=None): from native_adapter import build_node @@ -1374,17 +1469,35 @@ async def quote_sink(q): finally: strategy.enabled = False controller.stop = True - session.stop() + # LOW finding 7 (fix round 4): _shutdown_corporate_action_tasks + # must run even if session.stop(), the task wait/cancel, or + # port.stop() itself raises -- an ordinary exception in any of + # those used to skip refresh-task cleanup entirely, leaving a + # still-pending task retained but never cancelled/joined. Its own + # nested `finally` guarantees this regardless of what happens + # above it. try: - await asyncio.wait_for(task, 20) - except (asyncio.TimeoutError, asyncio.CancelledError): - task.cancel() - await asyncio.gather(task, return_exceptions=True) - await port.stop() - await _shutdown_corporate_action_tasks(ca_refresh_state) + session.stop() + try: + await asyncio.wait_for(task, 20) + except (asyncio.TimeoutError, asyncio.CancelledError): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await port.stop() + finally: + await _shutdown_corporate_action_tasks(ca_refresh_state) state = asdict(controller.ledger.accounting()) is_flat = not controller.ledger.positions() and not controller.ledger.unresolved() port_health = getattr(port, "health", {}) + # MEDIUM finding 3 (fix round 4): recomputed fresh against the CURRENT + # held positions and the CURRENT monitor state, not the strategy's + # last-tick snapshot (_ca_last_must_flatten/_ca_last_needs_attention_ + # held) -- a refresh that fails, times out, or is cancelled AFTER the + # last rebalance() tick (e.g. during the shutdown just above) would + # otherwise leave the outcome reporting a stale, already-superseded + # guard state. See _final_corporate_action_guard_summary's own + # docstring. + held_symbols_now = {p.symbol for p in controller.ledger.positions().values() if p.qty} outcome = {"engine": "NautilusTrader LiveNode 2.0.0rc5", "native_quotes": strategy.received_quotes, "dropped_quotes": {"by_reason": {str(k): int(v) for k, v in port_health.get("dropped_quotes", {}).items()}, "by_symbol": {str(k): int(v) for k, v in port_health.get("dropped_quotes_by_symbol", {}).items()}}, @@ -1397,21 +1510,14 @@ async def quote_sink(q): "overnight_holds": session_policy["overnight_holds"]}, "boundary_receipts": boundary_receipts, "elapsed_seconds": time.monotonic() - started, - # MEDIUM finding 8: a guard summary on every run outcome, - # regardless of whether the guard was actually wired in this - # run (session_policy["overnight_holds"]). finding 5: - # "pending_must_flatten" is read by _honest_overnight_hold - # below -- a run cannot end "held_overnight" while a symbol - # is still flagged for a forced flatten. - "corporate_action_guard": { - "enabled": strategy.corporate_action_guard is not None, - "ever_flagged_symbols": sorted(strategy._ca_ever_flagged), - "pending_must_flatten": sorted(strategy._ca_last_must_flatten), - # MEDIUM finding 4 (fix round 3): every currently HELD - # symbol not positively verified clear as of the run's - # last tick (lookup failed/ambiguous/degraded/session- - # unknown) -- read by _honest_overnight_hold below. - "pending_needs_attention_held": sorted(strategy._ca_last_needs_attention_held)}} + # MEDIUM finding 8 (round 3)/finding 3 (round 4): a guard + # summary on every run outcome, regardless of whether the + # guard was actually wired in this run. "pending_must_flatten" + # and "pending_needs_attention_held" are read by + # _honest_overnight_hold below -- a run cannot end + # "held_overnight" while either is non-empty. + "corporate_action_guard": _final_corporate_action_guard_summary( + strategy, held_symbols_now, time.time())} if leverage_policy is not None: capital = Decimal(config["capital_usd"]) peak_effective_leverage = (peak_gross_exposure_usd / capital) if capital else Decimal("0") diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index 0b237c160..faea7ee06 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -7,7 +7,7 @@ "blueprints/us-equities/adaptive-paper/config-mover.json": "560849b980082a72f8ec45a32ffae51b1f50a502e739c94346ff501a46777ea8", "blueprints/us-equities/adaptive-paper/config-sip.json": "3587653104e68fdfc9bbb9170825c611abcf7e2e1be1004327a056031c34555a", "blueprints/us-equities/adaptive-paper/config.json": "2b555982034af02b9d0d2e1d20e17d22c1e83f2b0c4d60b61d91307ebec63562", - "blueprints/us-equities/adaptive-paper/corporate_actions.py": "4bcecdaf26e960afb75bdc0a337234dba0aba38721fd6d2cf3fd6563fbefbe52", + "blueprints/us-equities/adaptive-paper/corporate_actions.py": "6ca6e75cf76bad077acce2054c6b614df32347f2e29274a4719af378babc563d", "blueprints/us-equities/adaptive-paper/exits.py": "189133e6df99bda201d73d36a15f45ccbfb38979a3319ae6f75d077f5db5b3df", "blueprints/us-equities/adaptive-paper/feeds.py": "0db3009a724ae2433572c5070066d6806d7365031f9db4e81c4b69ad1cbebacb", "blueprints/us-equities/adaptive-paper/leverage.py": "7f4a86f312d718a0a0b87e6eafe9a2d57b8709ec85a8746fdf7def0a5b502ea0", @@ -24,7 +24,7 @@ "blueprints/us-equities/adaptive-paper/recovery.py": "5cf7dcea572efbda6979a2726eebf6d676f6bd62263900c7b6626da8b45c1215", "blueprints/us-equities/adaptive-paper/registry.json": "60da73b985e718bd1dece7a3350f34636bda0fb38e766a44ae71c5a8c532282a", "blueprints/us-equities/adaptive-paper/requirements.txt": "518afe4b77f901e87abd3a5d6faa126aa7e211db77eecec80cfe7ef42d82e07a", - "blueprints/us-equities/adaptive-paper/runner.py": "d9dd13161fe1ae418b254f5046039441ddd01b3161b613624cb65a1536190ca9", + "blueprints/us-equities/adaptive-paper/runner.py": "31007ac4b0427f7cd44619ef1df31d480c3c6be75041c5045a82da96e932573e", "blueprints/us-equities/adaptive-paper/safety.py": "ad520fc4d84ce10e4d7b5cc956ba769a4f94eef49464ce382746d57f0cd76420", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", @@ -35,7 +35,7 @@ "blueprints/us-equities/adaptive-paper/transport.py": "34e6c4e604179585e3141c4e07066bbc36a655c3eb7052a007354fbe6b2835a6", "tests/adaptive_paper_hermetic.py": "34f2d5b07f42cb60d415abf24e4feef64b9088d1071163ab8427f0d958ebf7ee", "tests/test_adaptive_market_research.py": "dfd7a09380a7b1b10633c2edf05daf5a35bfcb48498fff130a2d84ff7190e395", - "tests/test_adaptive_paper_corporate_actions.py": "5bfc4bdf8b27fc2b98e7729f532ddcdd4b9191c4dca046ce6b45e1bdb217f017", + "tests/test_adaptive_paper_corporate_actions.py": "bd7603bafec0afc5b4811f2a8e8790f6146327bf79019aa3f1b1d323d91f6c68", "tests/test_adaptive_paper_exits.py": "13a82d65ad1b2e72a8ca0fbb8bbb178a01ded1301399604c9e304ae213456764", "tests/test_adaptive_paper_live_manifest.py": "4596f301ac44effefee5b787c5e3627a9cd38f8ff0cf436ef778b7378a0667c2", "tests/test_adaptive_paper_metrics.py": "8b0627f752ddf11c87005200e76d0ad4873261905ba073163476c66659d564f9", @@ -43,7 +43,7 @@ "tests/test_adaptive_paper_mover_native.py": "0257ca4bf9078e348e39d391dccf042b3f7b0e01d5c9e4d9092a9712f04097d5", "tests/test_adaptive_paper_native.py": "7c97707a5c19b704be9a603bbde396d9cb952dad39ca0d5b434ae1f8ebbd1f81", "tests/test_adaptive_paper_recovery.py": "08e351ff903d6d008b802dc93a8dde07da8900df5cc0540f551ffa909cf4ce95", - "tests/test_adaptive_paper_runner.py": "ae8e10dbac16551bfe4451259373ab0120e5d2b62be0b6813dbe99328de2c7ba", + "tests/test_adaptive_paper_runner.py": "966ef2dbfe6b922ca6243e8d18e46211f2260e2c89b22d9551243b0357a286bc", "tests/test_adaptive_paper_safety.py": "e0b2017ea6bdae87a0778b01ac72747642b31a2495c6d4e8bc98619a5aa1b456", "tests/test_adaptive_paper_selector.py": "27b431ac7911372d050f25d32ac24a3df9d2cf3c5e7dafdebfe51db6cab55936", "tests/test_adaptive_paper_sessions.py": "680060ac7ec147b93ed6ba2607631ad1ec38702f1a44e51260c792c537b1fab6", diff --git a/manifests/evidence.json b/manifests/evidence.json index 522e02793..e511ca8ee 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -6520,8 +6520,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/corporate-actions-native-check-20260924.json", - "sha256": "7883cd188c6449d290f92cacda621f12f3de223dfb0e080663150ed36289afb9", - "bytes": 7934 + "sha256": "510decdbdbaafd475bbbf5a997f6a3e2148abb7783b93bf0c585b44f1074c729", + "bytes": 9640 }, { "path": "blueprints/us-equities/adaptive-paper/evidence-hashes.json", @@ -6650,8 +6650,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/runner.py", - "sha256": "d9dd13161fe1ae418b254f5046039441ddd01b3161b613624cb65a1536190ca9", - "bytes": 118532 + "sha256": "31007ac4b0427f7cd44619ef1df31d480c3c6be75041c5045a82da96e932573e", + "bytes": 125085 }, { "path": "blueprints/us-equities/adaptive-paper/safety.py", @@ -6675,7 +6675,7 @@ }, { "path": "blueprints/us-equities/adaptive-paper/source-hashes.json", - "sha256": "10bb8dbdd5f83436096a36bf31866ef8285d7d71e3525a7c96bdb0f5171658b3", + "sha256": "3f71e615a6810b96304324bf657e3a5ef96c988005d81ffc5229050f377ddc46", "bytes": 6122 }, { @@ -29015,8 +29015,8 @@ }, { "path": "tests/test_adaptive_paper_runner.py", - "sha256": "ae8e10dbac16551bfe4451259373ab0120e5d2b62be0b6813dbe99328de2c7ba", - "bytes": 176985 + "sha256": "966ef2dbfe6b922ca6243e8d18e46211f2260e2c89b22d9551243b0357a286bc", + "bytes": 188741 }, { "path": "tests/test_adaptive_paper_safety.py", diff --git a/tests/test_adaptive_paper_corporate_actions.py b/tests/test_adaptive_paper_corporate_actions.py index 2cc07d36f..f9fba335f 100644 --- a/tests/test_adaptive_paper_corporate_actions.py +++ b/tests/test_adaptive_paper_corporate_actions.py @@ -397,6 +397,46 @@ def test_refresh_timed_out_invalidates_the_in_flight_generation(self): "leaving the symbol degraded from the timeout") self.assertFalse(after.must_flatten) + def test_per_symbol_ambiguous_result_never_erases_a_confirmed_action(self): + """HIGH finding 1 (fix round 4): reproduces the review's own Q2a + probe -- a confirmed split, then a LATER fetch that reports the + SAME symbol LOOKUP_AMBIGUOUS (a real, whole-symbol sentinel a + source's fetch() can still legitimately return) must not erase + the earlier confirmed action; it must degrade instead.""" + seq = [{"NVDA": [action("NVDA", "forward_split", NEXT_SESSION)]}, {"NVDA": CA.LOOKUP_AMBIGUOUS}] + + class Src: + def fetch(self, symbols, start, end): + return seq.pop(0) + + monitor = CA.CorporateActionMonitor(Src(), refresh_seconds=0, clock=lambda: 1000.0) + monitor.refresh({"NVDA"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + before = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"NVDA"}, + candidate_symbols=set(), now=1000.0)["NVDA"] + self.assertTrue(before.must_flatten) + monitor.refresh({"NVDA"}, start=TODAY, end=NEXT_SESSION, now=1100.0) + after = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"NVDA"}, + candidate_symbols=set(), now=1100.0)["NVDA"] + self.assertTrue(after.must_flatten, "a later per-symbol LOOKUP_AMBIGUOUS must not silently " + "cancel an already-confirmed action") + self.assertTrue(after.needs_attention) + + def test_timeout_processed_just_after_a_committed_success_is_ignored(self): + """NIT finding 9 (fix round 4): reproduces the review's own Q5 + probe -- refresh_timed_out() firing for the CURRENT generation + AFTER that same generation already committed a successful result + must be a no-op (the fresh, just-written result must not be + degraded for the retry_seconds interval for no reason).""" + source = self.FakeSource(result={"AAPL": []}) + monitor = CA.CorporateActionMonitor(source, clock=lambda: 1000.0) + monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + self.assertFalse(monitor._last_attempt_failed) + monitor.refresh_timed_out({"AAPL"}) # wait_for fired just after the thread finished + d = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, + candidate_symbols=set(), now=1001.0)["AAPL"] + self.assertFalse(d.needs_attention, "a timeout for an already-committed generation must be ignored") + self.assertFalse(monitor._last_attempt_failed, "the retry interval must stay the ordinary one") + def test_omission_after_a_prior_success_preserves_the_confirmed_action(self): """LOW finding 7 (fix round 3): a symbol omitted from an OTHERWISE successful fetch (most requested symbols resolved fine, this one @@ -478,6 +518,31 @@ def fake_request(self_session, method, url, **kwargs): return mock.patch.object(requests.Session, "request", fake_request), captured + def _mock_http_pages(self, pages): + """LOW finding 4 (fix round 4): like `_mock_http` but for a + SEQUENCE of pages -- `pages` is a list of (body, next_page_token) + pairs; each successive HTTP request consumes the next entry, + returning that page's own `next_page_token` in its envelope. Lets + a test exercise this wrapper's OWN pagination loop with a REAL + (non-null) continuation token, which the fixed single-page + `_mock_http` (always `next_page_token=None`) cannot -- a pagination + regression (e.g. never reading `next_page_token`, or never sending + `page_token` on the next request) would otherwise pass unnoticed. + Returns (patcher, calls) where `calls` collects each outgoing + request's kwargs, in order.""" + import unittest.mock as mock + import requests + calls = [] + remaining = list(pages) + + def fake_request(self_session, method, url, **kwargs): + calls.append(kwargs) + body, next_token = remaining.pop(0) + envelope = {"corporate_actions": body, "next_page_token": next_token} + return self._fake_http_response(envelope) + + return mock.patch.object(requests.Session, "request", fake_request), calls + def test_construction_cuts_retries(self): # finding 2: alpaca-py's RESTClient retries up to 3 times, sleeping # 3s between attempts, by default -- construction must cut this to @@ -556,29 +621,40 @@ def test_result_under_the_cap_is_trusted(self): self.assertEqual(out["AAPL"][0].action_type, "cash_dividend") self.assertEqual(out["AAPL"][0].action_date, TODAY) - def test_unmapped_type_key_fails_closed_instead_of_being_silently_dropped(self): - # HIGH finding 2 (fix round 3): `reorganizations` is a real, - # Alpaca-documented bucket type this guard's _TYPE_FIELD_MAP does - # not (yet) model. With raw_data=True this reaches this wrapper's - # own fail-closed path directly -- the pre-fix code went through - # the SDK's CorporateActionsSet, whose 13-branch parser (verified - # via inspect.getsource against the installed package) has no - # else/default clause and silently drops any bucket it does not - # recognize, turning this exact case into an apparently-clear - # {'AAPL': []} instead of raising. - patcher, _ = self._mock_http({"reorganizations": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}]}) + def test_unmapped_type_key_degrades_only_the_symbols_it_names(self): + # LOW finding 5 (fix round 4): `reorganizations` is a real, + # Alpaca-documented bucket type (per + # https://docs.alpaca.markets/us/reference/corporateactions-1, + # fetched by the coordinator 2026-09-24) this guard's + # _TYPE_FIELD_MAP does not (yet) model. It must degrade ONLY the + # symbol(s) its own row actually names -- AAPL here -- never fail + # the whole fetch for every requested symbol (an earlier round did + # exactly that: one QQQ capital_gains_distributions row blocked + # SPY/IWM/DIA/QQQ for the whole ~97-day window). + patcher, _ = self._mock_http({"reorganizations": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}], + "cash_dividends": [{"symbol": "MSFT", "ex_date": TODAY.isoformat()}]}) with patcher: src = self.source() - with self.assertRaises(CA.CorporateActionLookupError): - src.fetch(["AAPL"], TODAY, NEXT_SESSION) + out = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) + self.assertEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + self.assertEqual(len(out["MSFT"]), 1, "an unmapped bucket must never affect an unrelated symbol") + + def test_unmapped_type_row_with_no_symbol_degrades_every_requested_symbol(self): + # MEDIUM finding 2 (fix round 4) applied to an unmapped bucket: a + # row naming no symbol at all cannot be ruled out for any + # requested symbol. + patcher, _ = self._mock_http({"reorganizations": [{"ex_date": TODAY.isoformat()}]}) + with patcher: + src = self.source() + out = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) + self.assertEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + self.assertEqual(out["MSFT"], CA.LOOKUP_AMBIGUOUS) def test_unmapped_type_with_a_thousand_records_still_fails_closed(self): - # finding 2: 1000 records of an unmapped type used to become a - # silently-clear result AND evade the cap check (the SDK dropped - # the bucket before either check ever saw it). Both this wrapper's - # cap check and its unmapped-type check now see the raw bucket; - # either failing closed is acceptable, but a clear/successful - # result is not. + # finding 2 (round 3): 1000 records of an unmapped type used to + # become a silently-clear result AND evade the cap check (the SDK + # dropped the bucket before either check ever saw it). The cap + # check still sees every raw row regardless of bucket. items = [{"symbol": "AAPL", "ex_date": TODAY.isoformat()} for _ in range(1000)] patcher, _ = self._mock_http({"reorganizations": items}) with patcher: @@ -586,6 +662,72 @@ def test_unmapped_type_with_a_thousand_records_still_fails_closed(self): with self.assertRaises(CA.CorporateActionLookupError): src.fetch(["AAPL"], TODAY, NEXT_SESSION) + def test_row_with_no_identifiable_symbol_degrades_every_requested_symbol(self): + # MEDIUM finding 2 (fix round 4): a MAPPED bucket's row that names + # no symbol at all (every symbol field blank/missing) must not + # silently return "clear" for the requested symbol(s) -- any of + # them could be the intended target. + patcher, _ = self._mock_http({"cash_dividends": [{"ex_date": TODAY.isoformat()}]}) + with patcher: + src = self.source() + out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) + self.assertEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + + def test_ambiguity_never_replaces_a_confirmed_record_in_the_same_response(self): + # HIGH finding 1 (fix round 4): the SAME response carries a valid, + # in-range AAPL split AND an AAPL cash dividend missing its ex_date + # -- the confirmed split must survive, not be discarded/replaced by + # LOOKUP_AMBIGUOUS. + patcher, _ = self._mock_http({ + "forward_splits": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}], + "cash_dividends": [{"symbol": "AAPL"}]}) # no ex_date at all + with patcher: + src = self.source() + out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) + self.assertNotEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + self.assertEqual(len(out["AAPL"]), 1) + self.assertEqual(out["AAPL"][0].action_type, "forward_split") + + def test_pagination_follows_a_real_continuation_token_across_pages(self): + # LOW finding 4 (fix round 4): a real (non-null) next_page_token + # must actually be followed -- items from BOTH pages are combined. + pages = [ + ({"cash_dividends": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}]}, "page-2-token"), + ({"forward_splits": [{"symbol": "AAPL", "ex_date": NEXT_SESSION.isoformat()}]}, None), + ] + patcher, calls = self._mock_http_pages(pages) + with patcher: + src = self.source() + out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) + self.assertEqual(len(calls), 2, "both pages must actually be requested") + self.assertNotIn("page_token", calls[0]["params"]) + self.assertEqual(calls[1]["params"].get("page_token"), "page-2-token", + "the second request must carry the first page's own continuation token") + self.assertEqual(len(out["AAPL"]), 2, "records from both pages must be combined") + + def test_cap_is_enforced_across_pages_not_just_within_one(self): + # LOW finding 4 (fix round 4): 600 + 600 items across two pages + # totals 1200 >= the 1000 cap -- must fail closed even though + # neither single page alone reaches it. + items_a = [{"symbol": "AAPL", "ex_date": TODAY.isoformat()} for _ in range(600)] + items_b = [{"symbol": "AAPL", "ex_date": TODAY.isoformat()} for _ in range(600)] + pages = [({"cash_dividends": items_a}, "page-2"), ({"cash_dividends": items_b}, None)] + patcher, _ = self._mock_http_pages(pages) + with patcher: + src = self.source() + with self.assertRaises(CA.CorporateActionLookupError): + src.fetch(["AAPL"], TODAY, NEXT_SESSION) + + def test_exhausting_max_pages_with_a_token_still_present_fails_closed(self): + # LOW finding 4 (fix round 4): a still-present continuation token + # after _MAX_PAGES pages must never be silently truncated. + src = self.source() + pages = [({"cash_dividends": []}, f"tok-{i}") for i in range(src._MAX_PAGES)] + patcher, _ = self._mock_http_pages(pages) + with patcher: + with self.assertRaises(CA.CorporateActionLookupError): + src.fetch(["AAPL"], TODAY, NEXT_SESSION) + def test_unit_split_alternate_symbol_is_mapped(self): body = {"unit_splits": [{"old_symbol": "OLD", "new_symbol": "NEW", "alternate_symbol": "ALT", "effective_date": TODAY.isoformat()}]} @@ -856,6 +998,23 @@ def test_calendar_failure_blocks_entry_but_never_force_flattens(self): self.assertFalse(events[0]["must_flatten"]) self.assertTrue(events[0]["needs_attention"]) self.assertEqual(events[0]["reason"], "corporate_action_session_unknown") + # G10 (fix round 4): a held symbol must also land in the + # needs_attention_held tracking a calendar failure produces -- + # runner.py's outcome/hold-decision wiring reads this attribute, + # not just the emitted event. + self.assertEqual(strategy._ca_last_needs_attention_held, {"AAPL"}) + + def test_held_symbol_missing_from_the_guards_own_decisions_is_not_verified_clear(self): + """G9 (fix round 4): a misbehaving guard that omits a HELD symbol + from its returned decisions dict entirely (evaluate_guard itself + never does this -- it decides every held|candidate symbol -- but + this method must not trust that blindly) must still be treated as + NOT positively verified clear, landing in + _ca_last_needs_attention_held.""" + guard_decisions = {} # AAPL, held, gets no decision at all + strategy = self.strategy(held={"AAPL": 1}, targets={"AAPL": 1}, guard_decisions=guard_decisions) + strategy.rebalance() + self.assertIn("AAPL", strategy._ca_last_needs_attention_held) # -- fix round 3 (mutation F4a): the guard's own hold horizon must be # the NEXT TRADING DAY (skipping weekends/holidays), not a raw diff --git a/tests/test_adaptive_paper_runner.py b/tests/test_adaptive_paper_runner.py index 51f0817c2..af796943c 100644 --- a/tests/test_adaptive_paper_runner.py +++ b/tests/test_adaptive_paper_runner.py @@ -2863,48 +2863,153 @@ def positions(self): self.assertFalse(state["in_flight"]) @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") - def test_outcome_pending_must_flatten_reflects_the_strategys_own_state(self): - """fix round 3 (mutations F5a 'outcome never reports pending_must_ - flatten' and F5b 'strategy never records this tick's must_flatten'). - A guard that unconditionally reports a confirmed must_flatten for - SPY must make run_native's own returned outcome carry SPY in - outcome["corporate_action_guard"]["pending_must_flatten"] -- this - exercises BOTH the strategy actually recording it - (_ca_last_must_flatten, F5b) AND run_native's outcome construction - actually reading that real attribute rather than a hardcoded - empty list (F5a).""" + def test_outcome_pending_must_flatten_reflects_the_guards_final_state(self): + """fix round 3 (mutations F5a/F5b), fix round 4 (finding 3): the + outcome's corporate_action_guard summary is now RECOMPUTED fresh + (_final_corporate_action_guard_summary) against the guard's own + evaluate() and the truly final held positions, right before the + outcome is built -- not a stale per-tick snapshot. A guard that + reports must_flatten=True for a symbol genuinely still held at run + end must make run_native's own returned outcome carry it in + outcome["corporate_action_guard"]["pending_must_flatten"].""" import corporate_actions as CA from simulation import SimulatedPort config, _, _ = load_config(SOURCE / "config.json") - config.update(duration_seconds=1, cleanup_seconds=2, order_timeout_seconds=1) + config.update(duration_seconds=2, cleanup_seconds=2, order_timeout_seconds=1) config["sessions"] = {"extended_hours": True, "overnight_holds": True, "overnight_gross_multiple": "1.0"} config["regular_session_only"] = False config["extended_hours_enabled"] = True - class AlwaysFlattenGuard: + class FlattenOnceHeldGuard: + """Real evaluate_guard-style semantics: must_flatten only once + AAPL is genuinely held, so it is bought normally first, then + flagged -- unlike an unconditional guard, this cannot be + satisfied by a symbol that was never actually held.""" def refresh(self, symbols, *, start, end, now): pass def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): - return {"SPY": CA.GuardDecision(symbol="SPY", block_entry=True, must_flatten=True, - needs_attention=False, reason="corporate_action_cash_dividend", - action_type="cash_dividend", action_date=today)} + if "AAPL" not in held_symbols: + return {} + return {"AAPL": CA.GuardDecision(symbol="AAPL", block_entry=True, must_flatten=True, + needs_attention=False, reason="corporate_action_cash_dividend", + action_type="cash_dividend", action_date=today)} - config["_corporate_action_guard"] = AlwaysFlattenGuard() + config["_corporate_action_guard"] = FlattenOnceHeldGuard() with tempfile.TemporaryDirectory() as root: - ledger = Ledger(Path(root) / "journal.db", RiskLimits(trial_seconds=1, cleanup_seconds=2)) + ledger = Ledger(Path(root) / "journal.db", RiskLimits(trial_seconds=2, cleanup_seconds=2)) ledger.start_trial(time.time()) controller = Controller(ledger, time.time() + 3600, market_open=True) - policy = PolicyConfig(symbols=("SPY", "QQQ", "IWM", "DIA"), max_positions=4, warmup_samples=4, + # AAPL is not one of PolicyConfig's own default benchmarks + # (SPY/QQQ/IWM/DIA) -- it needs a real positive EDGE over them + # to ever be selected, which the price() slope below provides + # (benchmarks are all equal to each other, so a universe of + # only benchmarks never generates a signal). + policy = PolicyConfig(symbols=("SPY", "QQQ", "IWM", "DIA", "AAPL"), max_positions=5, warmup_samples=4, warmup_seconds=.06, sample_seconds=.02, rebalance_seconds=.02, min_hold_seconds=.05, max_hold_seconds=.4, cooldown_seconds=.05) - port = SimulatedPort(controller, policy.symbols, price=lambda symbol, tick: Decimal("100")) + # A port whose sells never actually execute keeps AAPL held + # through to the outcome, so the forced flatten this guard + # requests stays genuinely pending -- proving the recomputed + # summary reflects reality, not merely an echoed guard opinion. + # Price drift (not a flat constant) is needed for the policy to + # actually generate a real buy signal. + def price(symbol, tick): + slope = Decimal(".03") if symbol in policy.benchmarks else Decimal(".10") + return Decimal("100") + min(tick, 70) * slope + + port = SimulatedPort(controller, policy.symbols, price=price) + original_submit = port.submit + + async def submit_buys_only(payload): + if payload.get("side") == "sell": + # Accept the forced-flatten sell as a resting (never + # filled) order so SPY stays genuinely held all the way + # through to the outcome -- proving the recomputed + # summary reflects an ACTUALLY still-pending flatten, + # not merely an echoed guard opinion about a symbol + # that was already sold. + import inspect as inspect_module + import time as time_module + controller.before_submit(payload) + await controller.before_request("submit", client_id=payload["client_order_id"]) + order = dict(payload, id=f"sim-resting-{payload['client_order_id']}", status="new", + filled_qty="0", filled_avg_price="0", updated_at_ns=time_module.time_ns()) + port.orders[payload["client_order_id"]] = order + controller.observe(order) + result = port.on_order(dict(order)) + if inspect_module.isawaitable(result): + await result + return dict(order) + return await original_submit(payload) + + port.submit = submit_buys_only controller.port = port outcome = asyncio.run(run_native(controller, policy, [{"symbol": s} for s in policy.symbols], "fixture", config, "100000")) ledger.close() - self.assertIn("SPY", outcome["corporate_action_guard"]["pending_must_flatten"]) + self.assertIn("AAPL", outcome["corporate_action_guard"]["pending_must_flatten"]) + + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") + def test_final_summary_uses_the_guards_current_state_not_a_stale_strategy_snapshot(self): + """MEDIUM finding 3 (fix round 4): reproduces the independent + review's own probe scenario directly against + `_final_corporate_action_guard_summary` -- the exact function + run_native's outcome construction calls. Even when the + strategy's own last-tick bookkeeping (`_ca_ever_flagged`) is + stale/empty (as it would be if a refresh only failed/completed + AFTER the last rebalance() tick ran), the summary must reflect the + guard's CURRENT evaluate() answer for the CURRENT held positions -- + not silently report a clear state just because the strategy's own + tick-level snapshot never saw the change.""" + import corporate_actions as CA + import runner as runner_module + + class StaleSnapshotStrategy: + """Stands in for a real AdaptiveStrategy whose own per-tick + bookkeeping is stale -- _ca_ever_flagged deliberately empty, + simulating a strategy that never got another rebalance() tick + after the guard's state changed.""" + def __init__(self, guard): + self.corporate_action_guard = guard + self._ca_ever_flagged = set() + + class LateGuard: + """The monitor's CURRENT state, as of right now -- reflecting + a refresh that failed/was cancelled after the last tick.""" + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + return {"AAPL": CA.GuardDecision(symbol="AAPL", block_entry=True, must_flatten=False, + needs_attention=True, reason="corporate_action_lookup_failed")} + + strategy = StaleSnapshotStrategy(LateGuard()) + # 2026-09-24 09:30:00 America/New_York (RTH open) -- resolves + # without a ValueError under the frozen calendar. + summary = runner_module._final_corporate_action_guard_summary(strategy, {"AAPL"}, 1790256600.0) + self.assertIn("AAPL", summary["pending_needs_attention_held"], + "the recomputed summary must reflect the guard's CURRENT state, not the " + "strategy's stale (empty) _ca_ever_flagged snapshot") + self.assertIn("AAPL", summary["ever_flagged_symbols"]) + + def test_final_summary_is_clear_when_the_guard_is_actually_clear(self): + """The mirror case: a guard reporting a genuinely clear state for + every held symbol must not be reported as pending.""" + import corporate_actions as CA + import runner as runner_module + + class ClearGuard: + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + return {"AAPL": CA.GuardDecision(symbol="AAPL", block_entry=False, must_flatten=False, + needs_attention=False, reason=None)} + + class Strategy: + def __init__(self, guard): + self.corporate_action_guard = guard + self._ca_ever_flagged = set() + + summary = runner_module._final_corporate_action_guard_summary(Strategy(ClearGuard()), {"AAPL"}, 1790256600.0) + self.assertEqual(summary["pending_must_flatten"], []) + self.assertEqual(summary["pending_needs_attention_held"], []) def test_scheduled_refresh_times_out_and_still_clears_in_flight(self): """HIGH finding 1/2 (fix round 3): a refresh() call that runs @@ -2951,6 +3056,43 @@ async def scenario(): asyncio.run(scenario()) + def test_hung_refresh_does_not_delay_process_exit(self): + """LOW finding 6 (fix round 4): `asyncio.run()`'s own cleanup joins + every outstanding `asyncio.to_thread`/default-executor thread + before returning, regardless of any Task cancellation -- a single + still-hanging refresh used to delay the whole process's own exit + (and, in runner.py's real main(), the result-save and account-lock + release) by the hang's full duration. `_schedule_corporate_action_ + refresh` now bridges the worker to a DAEMON thread via + `loop.call_soon_threadsafe` instead, which `asyncio.run()` never + waits for. Measured: `asyncio.run(scenario())` itself must return + in well under the hang's own duration (here 3s), not ~3s.""" + import asyncio + import time as time_module + from datetime import date + import runner as runner_module + + class HangingGuard: + def refresh(self, symbols, *, start, end, now): + time_module.sleep(3.0) # a long hang, well past the timeout below + + async def scenario(): + state = {"in_flight": False} + original_timeout = runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS + runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = 0.05 + try: + await runner_module._schedule_corporate_action_refresh( + HangingGuard(), {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) + await asyncio.sleep(0.1) # let the timeout actually fire + finally: + runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = original_timeout + + started = time_module.monotonic() + asyncio.run(scenario()) + elapsed = time_module.monotonic() - started + self.assertLess(elapsed, 1.5, f"asyncio.run() must not be delayed by the hanging worker " + f"(daemon-thread bridge, not asyncio.to_thread); took {elapsed:.2f}s") + def test_real_monitor_generation_drops_a_superseded_late_write(self): """HIGH finding 1 (fix round 3): reproduces the reviewer's own finding against the REAL CorporateActionMonitor (not a fake) -- @@ -3032,6 +3174,64 @@ async def scenario(): asyncio.run(scenario()) + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") + def test_run_native_actually_cancels_a_hanging_refresh_at_its_own_shutdown(self): + """G6 (fix round 4): run_native's OWN finally block must actually + CALL _shutdown_corporate_action_tasks (not merely that the + function works when called directly, tested above) -- a guard + whose refresh() hangs well past the trial's own duration must be + genuinely cancelled by the time run_native returns. G5: the + cancellation path (not just an ordinary timeout) must also + invalidate the attempt via refresh_timed_out(), proven here by the + guard recording that call. The daemon-thread bridge (finding 6) + keeps this test itself fast regardless of the hang's length.""" + from simulation import SimulatedPort + config, _, _ = load_config(SOURCE / "config.json") + config.update(duration_seconds=1, cleanup_seconds=1, order_timeout_seconds=1) + config["sessions"] = {"extended_hours": True, "overnight_holds": True, + "overnight_gross_multiple": "1.0"} + config["regular_session_only"] = False + config["extended_hours_enabled"] = True + + class HangingGuard: + def __init__(self): + self.timed_out_calls = [] + + def refresh(self, symbols, *, start, end, now): + import time as time_module + time_module.sleep(30) # outlives the trial + cleanup + shutdown entirely + + def refresh_timed_out(self, symbols): + self.timed_out_calls.append(tuple(sorted(symbols))) + + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + return {} + + guard = HangingGuard() + config["_corporate_action_guard"] = guard + ca_refresh_state = {"in_flight": False} + with tempfile.TemporaryDirectory() as root: + ledger = Ledger(Path(root) / "journal.db", RiskLimits(trial_seconds=1, cleanup_seconds=1)) + ledger.start_trial(time.time()) + controller = Controller(ledger, time.time() + 3600, market_open=True) + policy = PolicyConfig(symbols=("SPY", "QQQ", "IWM", "DIA"), max_positions=4, warmup_samples=4, + warmup_seconds=.06, sample_seconds=.02, rebalance_seconds=.02, + min_hold_seconds=.05, max_hold_seconds=.4, cooldown_seconds=.05) + port = SimulatedPort(controller, policy.symbols, price=lambda symbol, tick: Decimal("100")) + controller.port = port + started = time.monotonic() + asyncio.run(run_native(controller, policy, [{"symbol": s} for s in policy.symbols], + "fixture", config, "100000", ca_refresh_state=ca_refresh_state)) + elapsed = time.monotonic() - started + ledger.close() + self.assertLess(elapsed, 15, "run_native must not be delayed by the hanging refresh thread") + pending = [t for t in ca_refresh_state.get("tasks", []) if not t.done()] + self.assertEqual(pending, [], "run_native's own shutdown must actually cancel the hanging task") + self.assertTrue(any(t.cancelled() for t in ca_refresh_state.get("tasks", [])), + "at least one task must have been genuinely cancelled, not merely completed") + self.assertGreater(len(guard.timed_out_calls), 0, + "cancellation must invalidate the attempt via refresh_timed_out()") + if __name__ == "__main__": unittest.main() From 21ea761f08079a4646618dfbf5d15ce5a34c4bbe Mon Sep 17 00:00:00 2001 From: Scout Date: Thu, 24 Sep 2026 19:19:31 -0400 Subject: [PATCH 6/8] Fix round 5: separate records from uncertainty, close the pre-launch cancellation race, fix the network-leaking pagination test Both independent reviews converged on one regression: fetch() collapsed per-symbol uncertainty into a LOOKUP_AMBIGUOUS sentinel that only applied when a symbol had zero valid records, so a benchmark symbol's routine dated dividend (SPY/QQQ/IWM/DIA almost always have one in the 97-day window) silently hid an unrelated undated/unmapped/symbol-less row for that same symbol. AlpacaCorporateActionsSource.fetch() now returns (records, ambiguous_symbols) as two independent values; CorporateAction Monitor.complete_attempt keeps the records and separately flags the symbol degraded when it is also ambiguous, so a confirmed action still flattens while residual uncertainty still raises needs_attention. Codex additionally found a genuine cancellation race: the monitor's generation was allocated lazily inside refresh() itself, which only ran once a launched worker thread actually started executing -- a thread still merely queued by the OS at shutdown time was invisible to the generation-based invalidation check, letting a delayed worker commit a stale result after shutdown. CorporateActionMonitor now exposes two- phase begin_attempt/run_attempt/fail_attempt/invalidate_attempt/ refresh_timed_out primitives; the generation is allocated on the calling coroutine before the worker thread is even created. Cancellation now calls the non-degrading invalidate_attempt instead of the degrading refresh_timed_out, so a routine in-flight refresh cancelled at shutdown no longer forces an otherwise-clean cached result into needs_attention. Both reviews also found the page-ceiling pagination test constructed the real AlpacaCorporateActionsSource before entering its own HTTP patch context, so it silently issued ~58 real requests to Alpaca's servers and passed on the resulting 401s without ever exercising its own scripted fixture; the source is now built inside the patch, the exact page count served is asserted, and both corporate-action test modules gained a module-level real-network guard. Also: an empty-string next_page_token no longer counts as a continuation; Thread.start() now sits inside the try/finally that resets in_flight; the "prevent overlap" comment was corrected to "reject stale writes"; the shutdown/final-summary/hold tests were strengthened so they fail on the exact regressions they target rather than passing on asyncio.run()'s own unrelated auto-cancellation or an in-memory stale snapshot. New tests cover region=us/data_quality=all on every page, the worker thread being a daemon, in_flight recovering from a Thread.start() failure, and a run_native-driven end-to-end refusal of held_overnight when a refresh degrades after the final rebalance tick. Re-ran: the full adaptive suite (368 tests, network-blocked pagination tests), test_native_faults_min, test_order_throughput, test_order_contract, test_trading_gates, all six validators (validate.py, validate_catalogs.py, landscape.py, evidence_manifest.py, verdict_review_gate.py, trading_gates.py), and both reviewers' mutation scripts retargeted at this worktree -- all 54 reviewer mutations plus 8 new round-5 mutations killed (two long-documented redundant survivors from round 4 unchanged: asyncio.run()'s own task cancellation already covers G6's exact scenario). Co-Authored-By: Claude Sonnet 5 --- .../adaptive-paper/corporate_actions.py | 291 ++++++--- .../us-equities/adaptive-paper/runner.py | 100 ++- .../adaptive-paper/source-hashes.json | 8 +- manifests/evidence.json | 10 +- .../test_adaptive_paper_corporate_actions.py | 217 ++++++- tests/test_adaptive_paper_runner.py | 568 ++++++++++++++++++ 6 files changed, 1058 insertions(+), 136 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/corporate_actions.py b/blueprints/us-equities/adaptive-paper/corporate_actions.py index 7223ea650..1b7444641 100644 --- a/blueprints/us-equities/adaptive-paper/corporate_actions.py +++ b/blueprints/us-equities/adaptive-paper/corporate_actions.py @@ -236,12 +236,18 @@ def __init__(self, source, *, refresh_seconds=900.0, retry_seconds=60.0, max_age # with no ordering guard). `_generation` is bumped, under `_lock`, # at the START of every `refresh()` attempt; only the write from # the CURRENT (latest-started) generation is ever applied -- - # `_apply_success`/`_apply_failure` drop a write from any earlier, + # `complete_attempt`/`fail_attempt` reject a write from any earlier, # superseded generation even if it happens to complete later. - # `refresh_timed_out()` (called by the async scheduler when its own - # wait_for gives up on a still-running attempt) also bumps the - # generation, so that attempt's eventual write -- success or - # failure, whenever the thread actually finishes -- is dropped too. + # `refresh_timed_out()`/`invalidate_attempt()` (called by the async + # scheduler when its own wait_for gives up, or cancels, a still- + # running attempt) also bump the generation, so that attempt's + # eventual write -- success or failure, whenever the thread actually + # finishes -- is dropped too. HIGH finding (fix round 5): the + # generation for THAT attempt is now allocated by `begin_attempt` + # BEFORE the worker thread is even launched, not lazily inside + # `refresh()`/`run_attempt()` once the thread happens to start + # running -- closing a window where a launched-but-not-yet-running + # thread was invisible to a concurrent shutdown-time invalidation. self._lock = threading.Lock() self._generation = 0 # NIT finding 9 (fix round 4): the generation that most recently @@ -259,17 +265,80 @@ def __init__(self, source, *, refresh_seconds=900.0, retry_seconds=60.0, max_age self._committed_generation = 0 def refresh(self, symbols, *, start: date, end: date, now=None): - """Idempotent within `refresh_seconds` for a symbol set already - covered by the last successful fetch AND fetched for this same - (start, end) window; always safe to call every tick, including - concurrently from multiple threads (finding 1) -- every state - mutation is generation-checked under `self._lock`. Never raises, - and never overwrites a previously successful result with + """The ordinary, synchronous convenience path (used directly by + every offline test and by any caller that does not need its own + worker thread) -- built on the two-phase begin_attempt/ + complete_attempt/fail_attempt primitives below, so it shares + exactly their generation-safety guarantees. Never raises, and + never overwrites a previously successful result with `LOOKUP_FAILED` (finding 3) -- a fetch failure only marks every requested symbol DEGRADED (see `_mark_degraded`); `evaluate()` preserves the last known good result for a degraded symbol while still surfacing `needs_attention`.""" now = self._clock() if now is None else now + generation = self.begin_attempt(symbols, start=start, end=end, now=now) + if generation is None: + return # throttled -- an equivalent fresh result is already cached + symbols = set(symbols) + try: + result = self.source.fetch(sorted(symbols), start, end) + except CorporateActionLookupError: + self.fail_attempt(symbols, generation) + return + except Exception: + # Fail closed (degraded, not overwritten) on any unexpected + # error too -- never guess. + self.fail_attempt(symbols, generation) + return + fetched, ambiguous = self._normalize_fetch_result(result) + self.complete_attempt(symbols, generation, now, fetched, ambiguous) + + def run_attempt(self, symbols, generation, *, start: date, end: date, now): + """The two-phase counterpart of `refresh()`'s own fetch-and-apply + body, for a caller (runner.py's `_schedule_corporate_action_refresh` + worker thread) that already called `begin_attempt` itself -- on the + CALLING thread/coroutine, before launching the worker -- and is now + running the actual blocking `source.fetch()` call, and applying its + result, on a SEPARATE worker thread under the pre-allocated + `generation`. Never raises.""" + symbols = set(symbols) + try: + result = self.source.fetch(sorted(symbols), start, end) + except CorporateActionLookupError: + self.fail_attempt(symbols, generation) + return + except Exception: + self.fail_attempt(symbols, generation) + return + fetched, ambiguous = self._normalize_fetch_result(result) + self.complete_attempt(symbols, generation, now, fetched, ambiguous) + + def begin_attempt(self, symbols, *, start: date, end: date, now=None): + """HIGH finding (fix round 5): allocate THIS attempt's generation + and run the throttle check SYNCHRONOUSLY, before any blocking I/O + -- a caller that will run the actual fetch on a separate worker + thread (runner.py's `_schedule_corporate_action_refresh`) MUST call + this first, on the CALLING thread/coroutine, before launching that + worker, and use the returned generation for `complete_attempt`/ + `fail_attempt`/`invalidate_attempt`/`refresh_timed_out`. + + Reproduced by an independent review: the OLD design bumped the + generation lazily, inside `refresh()` itself -- which only ran once + the worker thread actually started executing. A worker thread that + had been launched (`Thread.start()` called) but was still merely + QUEUED by the OS -- not yet running `refresh()`, so `_generation` + was still unchanged -- could be invisible to a shutdown-time + cancellation check: `_committed_generation == _generation` looked + "already committed / nothing in flight" (neither had moved yet), + so cancellation did nothing, and the delayed worker went on to + commit a stale result AFTER shutdown had already decided the run + was done. Allocating the generation here, before the thread is + even created, closes that window entirely. + + Returns None if throttled (an equivalent fresh result, for this + exact symbol set and window, is already cached) -- the caller must + not launch a worker in that case.""" + now = self._clock() if now is None else now symbols = set(symbols) window = (start, end) with self._lock: @@ -281,56 +350,108 @@ def refresh(self, symbols, *, start: date, end: date, now=None): interval = self.retry_seconds if self._last_attempt_failed else self.refresh_seconds if (self._last_attempt is not None and now - self._last_attempt < interval and symbols.issubset(self._results.keys()) and self._last_window == window): - return + return None self._last_attempt = now self._last_window = window self._generation += 1 - my_generation = self._generation - try: - fetched = self.source.fetch(sorted(symbols), start, end) - except CorporateActionLookupError: - self._apply_failure(symbols, my_generation) - return - except Exception: - # Fail closed (degraded, not overwritten) on any unexpected - # error too -- never guess. - self._apply_failure(symbols, my_generation) - return - self._apply_success(symbols, fetched, now, my_generation) + return self._generation - def refresh_timed_out(self, symbols): - """HIGH finding 1: called by the async scheduler (runner.py's - `_schedule_corporate_action_refresh`) when ITS OWN wait_for gives up - on a `refresh()` call still running in a worker thread. The thread + @staticmethod + def _normalize_fetch_result(result): + """A source's `fetch()` may return either the original plain + `dict[symbol, list[ActionRecord] | LOOKUP_FAILED | LOOKUP_AMBIGUOUS]` + (every existing offline-test fake, and any simple source that never + needs to report "confirmed records AND residual uncertainty" for + the same symbol at once), or HIGH finding (fix round 5)'s richer + `(dict[symbol, list[ActionRecord]], ambiguous_symbols: set[str])` + 2-tuple -- AlpacaCorporateActionsSource.fetch() now always returns + the latter, so a symbol's confirmed records are never discarded + just because SOME OTHER row for that same symbol was also + ambiguous (records and uncertainty are tracked separately, not + collapsed into one sentinel-or-list value per symbol).""" + if isinstance(result, tuple) and len(result) == 2: + fetched, ambiguous = result + return fetched, set(ambiguous) + return result, set() + + def refresh_timed_out(self, generation, symbols): + """HIGH finding (fix round 5): called by the async scheduler on a + GENUINE `wait_for` timeout (not a cancellation -- see + `invalidate_attempt` for that) for the SPECIFIC pre-allocated + `generation` (from `begin_attempt`) that timed out. The thread keeps running (Python cannot forcibly kill it), so this both (a) - bumps the generation counter, invalidating that in-flight attempt -- - whenever it does eventually finish, `_apply_success`/`_apply_failure` - will see a stale generation and drop the write -- and (b) treats the - timeout itself as an ordinary failure right now (degraded, short - `retry_seconds` retry interval), exactly as if `fetch()` had raised, - rather than silently leaving the cache in its last (possibly clear) - state until the abandoned attempt happens to resolve on its own. - - NIT finding 9 (fix round 4): a no-op if the current generation has - ALREADY committed (see `_committed_generation`) -- this timeout - handler firing just after the same attempt's own success/failure - already landed must never re-degrade an already-fresh result.""" + invalidates exactly that attempt -- bumping `_generation` past it + only if it is STILL the current one (a newer attempt may already + have superseded it, in which case there is nothing to invalidate) + -- so its eventual late write, whenever the thread finishes, is + dropped, and (b) treats the timeout itself as an ordinary failure + right now (degraded, short `retry_seconds` retry interval), exactly + as if `fetch()` had raised, rather than silently leaving the cache + in its last (possibly clear) state until the abandoned attempt + happens to resolve on its own. + + NIT finding 9 (fix round 4, preserved through round 5's generation + pre-allocation): a no-op if THIS SAME generation has ALREADY + committed (see `_committed_generation`) -- this timeout handler + firing just after this exact attempt's own success/failure already + landed must never re-degrade an already-fresh result.""" with self._lock: - if self._committed_generation == self._generation: - return + if generation != self._generation or generation == self._committed_generation: + return # already superseded, or already committed -- nothing to invalidate self._generation += 1 self._last_attempt_failed = True self._mark_degraded(symbols) - def _apply_failure(self, symbols, generation): + def invalidate_attempt(self, generation): + """HIGH finding (fix round 5)/LOW (Claude): called on an ordinary + CANCELLATION (not a timeout -- shutdown cancelling a still-routine + in-flight refresh) for the SPECIFIC pre-allocated `generation`. + Cancellation alone is not evidence the data is wrong -- only + invalidates that attempt (its eventual late write, if any, is + dropped as superseded, exactly like `refresh_timed_out`) WITHOUT + degrading anything. The monitor's last committed result stands, + subject only to the ordinary `max_age_seconds` staleness rule. + Reproduced by an independent review's own probe: the previous + design routed cancellation through the SAME handler as a timeout, + so a perfectly routine periodic refresh still in flight at + shutdown (nothing wrong with the cached data at all) forced the + cached result to needs_attention -- which could force a liquidation + of an otherwise clean, reconciled overnight hold.""" + with self._lock: + if generation != self._generation: + return # already superseded -- nothing to invalidate + self._generation += 1 + + def fail_attempt(self, symbols, generation): + """The two-phase counterpart of the old `_apply_failure` -- applies + an ordinary fetch FAILURE (the source's own `fetch()` raised) for a + pre-allocated `generation`.""" with self._lock: if generation != self._generation: - return # superseded by a later attempt (or a timeout) -- drop + return # superseded by a later attempt (or a timeout/cancellation) -- drop self._committed_generation = generation self._last_attempt_failed = True self._mark_degraded(symbols) - def _apply_success(self, symbols, fetched, now, generation): + def complete_attempt(self, symbols, generation, now, fetched, ambiguous=frozenset()): + """The two-phase counterpart of the old `_apply_success` -- applies + a successful fetch result for a pre-allocated `generation`. + + HIGH finding (fix round 5): `fetched` and `ambiguous` are tracked + SEPARATELY -- a symbol's valid records (possibly empty) are always + written to `_results` (never discarded), and `ambiguous` only ever + ADDS that symbol to `_degraded` (never overwrites `_results`) when + there is residual uncertainty about it (an undated/unmapped/ + symbol-less row) -- even when that same symbol ALSO has confirmed, + in-range records. The pre-fix design collapsed a symbol's whole + answer into either "a list of records" OR "LOOKUP_AMBIGUOUS", + picking the ambiguous sentinel (discarding any records already + collected) whenever ANY row was uncertain -- which meant a + confirmed split ALONGSIDE an unrelated undated dividend for the + same symbol, or a benchmark symbol with its routine quarterly + dividend already dated (SPY/QQQ/IWM/DIA in the shipped universe + virtually always have one somewhere in the 97-day window) alongside + an unmapped/symbol-less row, silently read as fully clear.""" with self._lock: if generation != self._generation: return # superseded -- an older result must never overwrite a newer one @@ -349,20 +470,24 @@ def _apply_success(self, symbols, fetched, now, generation): continue new_value = fetched[symbol] if new_value in (LOOKUP_FAILED, LOOKUP_AMBIGUOUS): - # HIGH finding 1 (fix round 4): a sentinel result (the - # symbol had no valid record of its own this fetch -- - # see AlpacaCorporateActionsSource.fetch(), which now - # only ever returns a sentinel for a symbol with ZERO - # valid records) must NEVER overwrite a previously - # confirmed (already non-empty) result -- degrade - # instead, preserving the confirmed action. Reproduced - # by the review's own Q2a probe: a confirmed split - # followed by a later per-symbol LOOKUP_AMBIGUOUS used - # to silently turn must_flatten back off. + # Backward-compat path for a source using the OLD + # single-value-per-symbol contract (see + # _normalize_fetch_result): the sentinel must never + # overwrite a previously confirmed (already non-empty) + # result -- degrade instead, preserving the confirmed + # action. self._mark_degraded((symbol,)) continue self._results[symbol] = new_value - self._degraded.discard(symbol) + if symbol in ambiguous: + # HIGH finding (fix round 5): keep the (possibly + # confirmed, possibly empty) records, but flag this + # symbol degraded anyway -- there is residual + # uncertainty about it this fetch even though it also + # has a valid answer. + self._degraded.add(symbol) + else: + self._degraded.discard(symbol) def _mark_degraded(self, symbols): """finding 3: only ever ADD to `_degraded` and, for a symbol never @@ -522,14 +647,16 @@ def _unmapped_row_symbols(item): return [value for key, value in item.items() if key.endswith("symbol") and value] def fetch(self, symbols, start: date, end: date) -> dict: - """Returns `{symbol: [ActionRecord, ...] | LOOKUP_AMBIGUOUS}` for - exactly the requested `symbols` (every requested symbol is present: - either its resolved list of valid records, possibly empty, or - LOOKUP_AMBIGUOUS -- ONLY for a symbol with ZERO valid records of its - own in this response; HIGH finding 1, fix round 4: a symbol with at - least one valid record NEVER has that record discarded or replaced - by ambiguity, regardless of how many other unparseable rows also - name it). Raises `CorporateActionLookupError` -- never a raw + """Returns `(out, ambiguous_symbols)` -- HIGH finding (fix round 5): + `out` is `{symbol: [ActionRecord, ...]}` for exactly the requested + `symbols` (every requested symbol present, its own resolved list of + valid records, possibly empty) and `ambiguous_symbols` is the + INDEPENDENT `set[str]` of requested symbols with at least one + unresolved row this response (undated, unmapped-bucket, or + symbol-less) -- a symbol can be in both `out` (with real records) + and `ambiguous_symbols` (uncertainty about some OTHER row) at once; + neither ever silently overwrites the other. Raises + `CorporateActionLookupError` -- never a raw alpaca-py/HTTP exception -- on any request/transport failure or a result at the page/limit cap, so a caller never needs to know alpaca-py's own exception types and never silently treats an @@ -612,7 +739,14 @@ def fetch(self, symbols, start: date, end: date) -> dict: if total_items >= self._REQUEST_LIMIT: raise CorporateActionLookupError("result_at_page_or_limit_cap") page_token = response.get("next_page_token") - if page_token is None: + # LOW finding (fix round 5): an empty-string token is not a + # continuation -- Alpaca's own envelope uses an ABSENT or + # null `next_page_token` to mean "no more pages", but + # nothing guarantees it never sends `""` for the same + # meaning; treating any falsy token (None or "") as "stop" + # avoids sending a bogus `page_token=""` on a request that + # should have been the last one. + if not page_token: break else: # LOW finding 4: exhausted _MAX_PAGES while a continuation @@ -662,15 +796,26 @@ def fetch(self, symbols, start: date, end: date) -> dict: out[symbol].append(ActionRecord( symbol=symbol, action_type=canonical_type, action_date=action_date, source_id=str(item.get("cusip") or type_key))) - for symbol in ambiguous_symbols: - # HIGH finding 1 (fix round 4): never let ambiguity replace - # an already-confirmed (non-empty) record list for this - # SAME response -- only fall back to the sentinel when the - # symbol has NO valid record of its own here. - if not out[symbol]: - out[symbol] = LOOKUP_AMBIGUOUS except CorporateActionLookupError: raise except Exception as error: raise CorporateActionLookupError(type(error).__name__) from error - return out + # HIGH finding (fix round 5): records and uncertainty are returned + # SEPARATELY -- `out[symbol]` is always that symbol's own valid, + # in-window records (possibly empty), NEVER replaced or collapsed + # into a sentinel, and `ambiguous_symbols` independently lists every + # symbol with at least one unresolved row (undated, unmapped-bucket, + # or symbol-less) THIS fetch. The pre-fix design overwrote `out + # [symbol]` with the single sentinel `LOOKUP_AMBIGUOUS` whenever the + # symbol had zero valid records of its own -- which read as fully + # clear the moment a symbol ALSO had any unrelated valid record, + # e.g. a benchmark symbol's routine quarterly dividend already dated + # somewhere in the window (SPY/QQQ/IWM/DIA virtually always have + # one) sitting alongside an unmapped/undated/symbol-less row for + # that same symbol. `CorporateActionMonitor.complete_attempt` (via + # `_normalize_fetch_result`) now keeps `out[symbol]`'s records AND + # separately marks the symbol degraded when it is also in + # `ambiguous_symbols` -- a confirmed action still flattens the + # guard, and residual uncertainty still raises needs_attention, + # simultaneously, rather than one silently hiding the other. + return out, ambiguous_symbols diff --git a/blueprints/us-equities/adaptive-paper/runner.py b/blueprints/us-equities/adaptive-paper/runner.py index 38fd27043..97a14630f 100644 --- a/blueprints/us-equities/adaptive-paper/runner.py +++ b/blueprints/us-equities/adaptive-paper/runner.py @@ -1012,20 +1012,47 @@ async def _schedule_corporate_action_refresh(guard, watch, start, end, now, stat reproduced by an independent review: the thread kept running past the timeout and later wrote a stale result over a newer one. LOW finding 1 (fix round 4, wording): refreshes are never overlapped only because - `CorporateActionMonitor`'s own generation check (under its lock) drops - any write from a superseded attempt -- this scheduling layer's + `CorporateActionMonitor`'s own generation check (under its lock) rejects + any write from a superseded (stale) attempt -- this scheduling layer's `state["in_flight"]` check is a best-effort scheduling optimization (avoid spinning up a redundant thread most of the time), not itself a correctness guarantee; a timed-out worker can still overlap a later one at the OS-thread level, and the generation check is what makes that harmless. Defenses in place: - 1. `guard.refresh_timed_out(watch)` (if the guard exposes it -- the - real CorporateActionMonitor does) is called on timeout, which - bumps that monitor's internal generation counter so the abandoned - attempt's eventual write (whenever the thread actually finishes) - is dropped as superseded, AND marks the requested symbols - degraded/failed right now (the short `retry_seconds` interval - applies to the next attempt, not the full `refresh_seconds`). + 1. HIGH finding (fix round 5): for a guard exposing the two-phase + `begin_attempt`/`run_attempt`/`invalidate_attempt` API (the real + CorporateActionMonitor does), the generation is allocated by + `guard.begin_attempt(...)` HERE, synchronously, on the calling + coroutine, BEFORE the worker thread is even created -- not lazily, + inside the worker, once it happens to start running. Reproduced by + an independent review's own probe: allocating the generation + lazily left a window where a worker thread had been *launched* + (`Thread.start()` already called) but was still merely queued by + the OS, not yet actually running -- `_generation` hadn't moved yet + -- so a shutdown-time cancellation landing in that window saw + "nothing in flight" and did nothing, and the delayed worker later + committed a stale result AFTER shutdown had already decided the + run was done. Allocating up front closes that window: cancellation + always has a concrete generation to invalidate, however early it + happens. + - On CANCELLATION specifically (shutdown cancelling an otherwise + routine in-flight refresh), `guard.invalidate_attempt( + generation)` is called -- NOT `refresh_timed_out` -- so a + perfectly clean, still-fresh cached result is never forced + into `needs_attention` just because the routine periodic + refresh checking it happened to still be in flight at + shutdown (LOW finding, Claude's review: the old code routed + cancellation through the SAME degrading path as a genuine + timeout). + - On a genuine TIMEOUT (`wait_for` itself gives up, not a + cancellation), `guard.refresh_timed_out(generation, watch)` is + still called, which DOES degrade -- a timeout is itself + evidence something may be wrong (unlike an ordinary shutdown). + For a simpler guard (most existing test fakes) that does not + expose `begin_attempt`, this falls back to the OLD single-call + convention (`guard.refresh(...)` inside the worker, + `guard.refresh_timed_out(watch)` on timeout OR cancellation) to + avoid breaking any test built against that simpler contract. 2. The created task is retained in `state["tasks"]` (pruned of already-done entries on every call) so a caller (run_native's own `finally` block) can cancel and join every still-pending @@ -1046,10 +1073,22 @@ async def _schedule_corporate_action_refresh(guard, watch, start, end, now, stat A no-op if a refresh is already in flight; callers check `state["in_flight"]` before calling this so a throttled (no-op) `refresh()` call never even spins up a thread. Returns the created - task so a caller that needs the result now (the preflight fetch) can - `await` it directly -- `_run()` below never raises (its own - `asyncio.wait_for` swallows the timeout), so awaiting the returned task - is itself already bounded by `CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS`.""" + task (or None if the two-phase `begin_attempt` itself reports a + throttled no-op, in which case no worker is launched at all) so a + caller that needs the result now (the preflight fetch) can `await` it + directly -- `_run()` below never raises (its own `asyncio.wait_for` + swallows the timeout), so awaiting the returned task is itself already + bounded by `CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS`.""" + two_phase = hasattr(guard, "begin_attempt") + generation = None + if two_phase: + generation = guard.begin_attempt(watch, start=start, end=end, now=now) + if generation is None: + # Throttled: an equivalent fresh result is already cached for + # this exact symbol set/window -- no attempt to invalidate, no + # worker to launch. + return None + state["in_flight"] = True async def _run(): @@ -1058,9 +1097,12 @@ async def _run(): def worker(): try: - guard.refresh(watch, start=start, end=end, now=now) + if two_phase: + guard.run_attempt(watch, generation, start=start, end=end, now=now) + else: + guard.refresh(watch, start=start, end=end, now=now) except Exception: - pass # CorporateActionMonitor.refresh() never raises; defensive only + pass # never raises in the well-behaved case; defensive only finally: def _resolve(): if not fut.done(): @@ -1070,16 +1112,26 @@ def _resolve(): except RuntimeError: pass # the loop is already closed -- the late result is simply dropped - threading.Thread(target=worker, daemon=True, name="ca-refresh").start() + # LOW finding (fix round 5): `Thread.start()` moved INSIDE this + # try/finally -- if starting the thread itself ever raised (e.g. a + # resource-exhaustion OSError), `state["in_flight"]` must still be + # reset so the NEXT tick is not permanently locked out believing a + # refresh that never actually started is still running. try: - await asyncio.wait_for(fut, timeout=CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) - except asyncio.TimeoutError: - if hasattr(guard, "refresh_timed_out"): - guard.refresh_timed_out(watch) - except asyncio.CancelledError: - if hasattr(guard, "refresh_timed_out"): - guard.refresh_timed_out(watch) - raise + threading.Thread(target=worker, daemon=True, name="ca-refresh").start() + try: + await asyncio.wait_for(fut, timeout=CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + if two_phase: + guard.refresh_timed_out(generation, watch) + elif hasattr(guard, "refresh_timed_out"): + guard.refresh_timed_out(watch) + except asyncio.CancelledError: + if two_phase: + guard.invalidate_attempt(generation) + elif hasattr(guard, "refresh_timed_out"): + guard.refresh_timed_out(watch) + raise finally: state["in_flight"] = False diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index faea7ee06..407ddb49f 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -7,7 +7,7 @@ "blueprints/us-equities/adaptive-paper/config-mover.json": "560849b980082a72f8ec45a32ffae51b1f50a502e739c94346ff501a46777ea8", "blueprints/us-equities/adaptive-paper/config-sip.json": "3587653104e68fdfc9bbb9170825c611abcf7e2e1be1004327a056031c34555a", "blueprints/us-equities/adaptive-paper/config.json": "2b555982034af02b9d0d2e1d20e17d22c1e83f2b0c4d60b61d91307ebec63562", - "blueprints/us-equities/adaptive-paper/corporate_actions.py": "6ca6e75cf76bad077acce2054c6b614df32347f2e29274a4719af378babc563d", + "blueprints/us-equities/adaptive-paper/corporate_actions.py": "50cfdb40c28b042b4fe293863b95df7e66bfa63b9e74719f7459e43812ff47b5", "blueprints/us-equities/adaptive-paper/exits.py": "189133e6df99bda201d73d36a15f45ccbfb38979a3319ae6f75d077f5db5b3df", "blueprints/us-equities/adaptive-paper/feeds.py": "0db3009a724ae2433572c5070066d6806d7365031f9db4e81c4b69ad1cbebacb", "blueprints/us-equities/adaptive-paper/leverage.py": "7f4a86f312d718a0a0b87e6eafe9a2d57b8709ec85a8746fdf7def0a5b502ea0", @@ -24,7 +24,7 @@ "blueprints/us-equities/adaptive-paper/recovery.py": "5cf7dcea572efbda6979a2726eebf6d676f6bd62263900c7b6626da8b45c1215", "blueprints/us-equities/adaptive-paper/registry.json": "60da73b985e718bd1dece7a3350f34636bda0fb38e766a44ae71c5a8c532282a", "blueprints/us-equities/adaptive-paper/requirements.txt": "518afe4b77f901e87abd3a5d6faa126aa7e211db77eecec80cfe7ef42d82e07a", - "blueprints/us-equities/adaptive-paper/runner.py": "31007ac4b0427f7cd44619ef1df31d480c3c6be75041c5045a82da96e932573e", + "blueprints/us-equities/adaptive-paper/runner.py": "c73b535417496c96125ad78c6634d85c6e6cee04cde1bcd08eff73f55e5ea2ea", "blueprints/us-equities/adaptive-paper/safety.py": "ad520fc4d84ce10e4d7b5cc956ba769a4f94eef49464ce382746d57f0cd76420", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", @@ -35,7 +35,7 @@ "blueprints/us-equities/adaptive-paper/transport.py": "34e6c4e604179585e3141c4e07066bbc36a655c3eb7052a007354fbe6b2835a6", "tests/adaptive_paper_hermetic.py": "34f2d5b07f42cb60d415abf24e4feef64b9088d1071163ab8427f0d958ebf7ee", "tests/test_adaptive_market_research.py": "dfd7a09380a7b1b10633c2edf05daf5a35bfcb48498fff130a2d84ff7190e395", - "tests/test_adaptive_paper_corporate_actions.py": "bd7603bafec0afc5b4811f2a8e8790f6146327bf79019aa3f1b1d323d91f6c68", + "tests/test_adaptive_paper_corporate_actions.py": "cad7adcd284325c0b560e2cdb1763001662c95d89fbaf26d7e0a4d2815129d48", "tests/test_adaptive_paper_exits.py": "13a82d65ad1b2e72a8ca0fbb8bbb178a01ded1301399604c9e304ae213456764", "tests/test_adaptive_paper_live_manifest.py": "4596f301ac44effefee5b787c5e3627a9cd38f8ff0cf436ef778b7378a0667c2", "tests/test_adaptive_paper_metrics.py": "8b0627f752ddf11c87005200e76d0ad4873261905ba073163476c66659d564f9", @@ -43,7 +43,7 @@ "tests/test_adaptive_paper_mover_native.py": "0257ca4bf9078e348e39d391dccf042b3f7b0e01d5c9e4d9092a9712f04097d5", "tests/test_adaptive_paper_native.py": "7c97707a5c19b704be9a603bbde396d9cb952dad39ca0d5b434ae1f8ebbd1f81", "tests/test_adaptive_paper_recovery.py": "08e351ff903d6d008b802dc93a8dde07da8900df5cc0540f551ffa909cf4ce95", - "tests/test_adaptive_paper_runner.py": "966ef2dbfe6b922ca6243e8d18e46211f2260e2c89b22d9551243b0357a286bc", + "tests/test_adaptive_paper_runner.py": "e378064e9d9cbfdccb06906ca3412908e4608b3790db56570b1c8e67d03e9474", "tests/test_adaptive_paper_safety.py": "e0b2017ea6bdae87a0778b01ac72747642b31a2495c6d4e8bc98619a5aa1b456", "tests/test_adaptive_paper_selector.py": "27b431ac7911372d050f25d32ac24a3df9d2cf3c5e7dafdebfe51db6cab55936", "tests/test_adaptive_paper_sessions.py": "680060ac7ec147b93ed6ba2607631ad1ec38702f1a44e51260c792c537b1fab6", diff --git a/manifests/evidence.json b/manifests/evidence.json index e511ca8ee..6c29dfa2b 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -6650,8 +6650,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/runner.py", - "sha256": "31007ac4b0427f7cd44619ef1df31d480c3c6be75041c5045a82da96e932573e", - "bytes": 125085 + "sha256": "c73b535417496c96125ad78c6634d85c6e6cee04cde1bcd08eff73f55e5ea2ea", + "bytes": 128204 }, { "path": "blueprints/us-equities/adaptive-paper/safety.py", @@ -6675,7 +6675,7 @@ }, { "path": "blueprints/us-equities/adaptive-paper/source-hashes.json", - "sha256": "3f71e615a6810b96304324bf657e3a5ef96c988005d81ffc5229050f377ddc46", + "sha256": "42b7b425f80e89a2088e97c9de41e7753bc7fecb14d13f135b188dcdf8ae78e3", "bytes": 6122 }, { @@ -29015,8 +29015,8 @@ }, { "path": "tests/test_adaptive_paper_runner.py", - "sha256": "966ef2dbfe6b922ca6243e8d18e46211f2260e2c89b22d9551243b0357a286bc", - "bytes": 188741 + "sha256": "e378064e9d9cbfdccb06906ca3412908e4608b3790db56570b1c8e67d03e9474", + "bytes": 220776 }, { "path": "tests/test_adaptive_paper_safety.py", diff --git a/tests/test_adaptive_paper_corporate_actions.py b/tests/test_adaptive_paper_corporate_actions.py index f9fba335f..f45ce1d41 100644 --- a/tests/test_adaptive_paper_corporate_actions.py +++ b/tests/test_adaptive_paper_corporate_actions.py @@ -9,11 +9,12 @@ package instead). """ from dataclasses import dataclass -from datetime import date +from datetime import date, timedelta from decimal import Decimal as D import importlib.util import json from pathlib import Path +import socket import sys import unittest @@ -24,6 +25,34 @@ import corporate_actions as CA # noqa: E402 (path inserted above) import sessions # noqa: E402 +# MEDIUM finding (fix round 5): a module-level real-network guard -- +# AlpacaCorporateActionsSourceTests below is supposed to exercise +# AlpacaCorporateActionsSource entirely against a patched +# `requests.Session.request`; an independent review's own run found that +# constructing the source BEFORE the patch context (see the fixed +# test_exhausting_max_pages_with_a_token_still_present_fails_closed) let +# ~58 real GETs reach data.alpaca.markets with fake-key strings, and the +# test still passed on the resulting 401s without ever noticing. This +# blocks any real (non-loopback) TCP connect attempt for the lifetime of +# this test module, so a similarly-unpatched real network call fails loudly +# (OSError) instead of silently "passing" on an HTTP error response. +_ORIGINAL_SOCKET_CONNECT = socket.socket.connect + + +def _blocked_connect(self, address): + host = address[0] if isinstance(address, tuple) else address + if self.family in (socket.AF_INET, socket.AF_INET6) and host not in ("127.0.0.1", "::1", "localhost"): + raise OSError(f"real network access blocked in this test module (attempted: {address!r})") + return _ORIGINAL_SOCKET_CONNECT(self, address) + + +def setUpModule(): + socket.socket.connect = _blocked_connect + + +def tearDownModule(): + socket.socket.connect = _ORIGINAL_SOCKET_CONNECT + NATIVE = importlib.util.find_spec("nautilus_trader") is not None if NATIVE: from nautilus_trader.model import OrderSide # noqa: E402 @@ -362,12 +391,12 @@ def test_superseded_failure_does_not_re_degrade_a_newer_confirmed_action(self): # confirmed an action. old_generation = 1 monitor._generation = 2 - monitor._apply_success({"AAPL"}, {"AAPL": [action("AAPL", "forward_split", TODAY)]}, 1030.0, 2) + monitor.complete_attempt({"AAPL"}, 2, 1030.0, {"AAPL": [action("AAPL", "forward_split", TODAY)]}) confirmed = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, candidate_symbols=set(), now=1030.0)["AAPL"] self.assertTrue(confirmed.must_flatten) # The OLD (generation 1) attempt now finally fails -- must be dropped. - monitor._apply_failure({"AAPL"}, old_generation) + monitor.fail_attempt({"AAPL"}, old_generation) after = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, candidate_symbols=set(), now=1030.0)["AAPL"] self.assertTrue(after.must_flatten, "a superseded (older-generation) failure must not re-degrade " @@ -384,13 +413,13 @@ def test_refresh_timed_out_invalidates_the_in_flight_generation(self): monitor._last_attempt = 1000.0 monitor._generation = 1 abandoned_generation = monitor._generation - monitor.refresh_timed_out({"AAPL"}) + monitor.refresh_timed_out(abandoned_generation, {"AAPL"}) self.assertTrue(monitor._last_attempt_failed) self.assertIn("AAPL", monitor._degraded) # The abandoned attempt's thread finally finishes and tries to # write a stale success under its OLD (now-superseded) generation. - monitor._apply_success({"AAPL"}, {"AAPL": [action("AAPL", "forward_split", TODAY)]}, - 1000.0, abandoned_generation) + monitor.complete_attempt({"AAPL"}, abandoned_generation, + 1000.0, {"AAPL": [action("AAPL", "forward_split", TODAY)]}) after = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, candidate_symbols=set(), now=1000.0)["AAPL"] self.assertTrue(after.needs_attention, "the abandoned attempt's late write must be dropped, " @@ -421,6 +450,26 @@ def fetch(self, symbols, start, end): "cancel an already-confirmed action") self.assertTrue(after.needs_attention) + def test_monitor_keeps_confirmed_records_and_flags_attention_from_the_tuple_contract(self): + """HIGH finding (fix round 5): exercises the monitor's own handling + of the NEW `(records, ambiguous_symbols)` tuple contract a source's + `fetch()` may return (AlpacaCorporateActionsSource.fetch() always + does) -- a symbol with BOTH a confirmed, in-range record AND + residual uncertainty (flagged via the SECOND tuple element) must + end up must_flatten=True (the record) AND needs_attention=True (the + uncertainty) SIMULTANEOUSLY -- neither one silently cancelling the + other, unlike the pre-round-5 single sentinel-or-list contract.""" + class Src: + def fetch(self, symbols, start, end): + return ({"NVDA": [action("NVDA", "forward_split", NEXT_SESSION)]}, {"NVDA"}) + + monitor = CA.CorporateActionMonitor(Src(), clock=lambda: 1000.0) + monitor.refresh({"NVDA"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + decision = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"NVDA"}, + candidate_symbols=set(), now=1000.0)["NVDA"] + self.assertTrue(decision.must_flatten, "the confirmed record must still flatten the position") + self.assertTrue(decision.needs_attention, "the independent ambiguity flag must still raise attention") + def test_timeout_processed_just_after_a_committed_success_is_ignored(self): """NIT finding 9 (fix round 4): reproduces the review's own Q5 probe -- refresh_timed_out() firing for the CURRENT generation @@ -431,7 +480,9 @@ def test_timeout_processed_just_after_a_committed_success_is_ignored(self): monitor = CA.CorporateActionMonitor(source, clock=lambda: 1000.0) monitor.refresh({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) self.assertFalse(monitor._last_attempt_failed) - monitor.refresh_timed_out({"AAPL"}) # wait_for fired just after the thread finished + # wait_for fired just after the thread finished, for the same + # (already-committed) generation. + monitor.refresh_timed_out(monitor._generation, {"AAPL"}) d = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"AAPL"}, candidate_symbols=set(), now=1001.0)["AAPL"] self.assertFalse(d.needs_attention, "a timeout for an already-committed generation must be ignored") @@ -616,10 +667,11 @@ def test_result_under_the_cap_is_trusted(self): patcher, _ = self._mock_http({"cash_dividends": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}]}) with patcher: src = self.source() - out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) + out, ambiguous = src.fetch(["AAPL"], TODAY, NEXT_SESSION) self.assertEqual(len(out["AAPL"]), 1) self.assertEqual(out["AAPL"][0].action_type, "cash_dividend") self.assertEqual(out["AAPL"][0].action_date, TODAY) + self.assertEqual(ambiguous, set()) def test_unmapped_type_key_degrades_only_the_symbols_it_names(self): # LOW finding 5 (fix round 4): `reorganizations` is a real, @@ -635,9 +687,11 @@ def test_unmapped_type_key_degrades_only_the_symbols_it_names(self): "cash_dividends": [{"symbol": "MSFT", "ex_date": TODAY.isoformat()}]}) with patcher: src = self.source() - out = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) - self.assertEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + out, ambiguous = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) + self.assertEqual(out["AAPL"], []) + self.assertIn("AAPL", ambiguous) self.assertEqual(len(out["MSFT"]), 1, "an unmapped bucket must never affect an unrelated symbol") + self.assertNotIn("MSFT", ambiguous) def test_unmapped_type_row_with_no_symbol_degrades_every_requested_symbol(self): # MEDIUM finding 2 (fix round 4) applied to an unmapped bucket: a @@ -646,9 +700,10 @@ def test_unmapped_type_row_with_no_symbol_degrades_every_requested_symbol(self): patcher, _ = self._mock_http({"reorganizations": [{"ex_date": TODAY.isoformat()}]}) with patcher: src = self.source() - out = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) - self.assertEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) - self.assertEqual(out["MSFT"], CA.LOOKUP_AMBIGUOUS) + out, ambiguous = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) + self.assertEqual(out["AAPL"], []) + self.assertEqual(out["MSFT"], []) + self.assertEqual(ambiguous, {"AAPL", "MSFT"}) def test_unmapped_type_with_a_thousand_records_still_fails_closed(self): # finding 2 (round 3): 1000 records of an unmapped type used to @@ -670,23 +725,29 @@ def test_row_with_no_identifiable_symbol_degrades_every_requested_symbol(self): patcher, _ = self._mock_http({"cash_dividends": [{"ex_date": TODAY.isoformat()}]}) with patcher: src = self.source() - out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) - self.assertEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + out, ambiguous = src.fetch(["AAPL"], TODAY, NEXT_SESSION) + self.assertEqual(out["AAPL"], []) + self.assertIn("AAPL", ambiguous) def test_ambiguity_never_replaces_a_confirmed_record_in_the_same_response(self): - # HIGH finding 1 (fix round 4): the SAME response carries a valid, - # in-range AAPL split AND an AAPL cash dividend missing its ex_date - # -- the confirmed split must survive, not be discarded/replaced by - # LOOKUP_AMBIGUOUS. + # HIGH finding 1 (fix round 4, refined round 5): the SAME response + # carries a valid, in-range AAPL split AND an AAPL cash dividend + # missing its ex_date -- the confirmed split must survive in + # `out["AAPL"]` (never discarded/replaced by a sentinel), AND + # (fix round 5) the residual uncertainty from the undated dividend + # must still be surfaced, independently, via `ambiguous` -- records + # and uncertainty are tracked separately, neither silently erasing + # the other. patcher, _ = self._mock_http({ "forward_splits": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}], "cash_dividends": [{"symbol": "AAPL"}]}) # no ex_date at all with patcher: src = self.source() - out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) - self.assertNotEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + out, ambiguous = src.fetch(["AAPL"], TODAY, NEXT_SESSION) self.assertEqual(len(out["AAPL"]), 1) self.assertEqual(out["AAPL"][0].action_type, "forward_split") + self.assertIn("AAPL", ambiguous, "the undated dividend row must still be surfaced as uncertainty, " + "separately from the confirmed split") def test_pagination_follows_a_real_continuation_token_across_pages(self): # LOW finding 4 (fix round 4): a real (non-null) next_page_token @@ -698,12 +759,44 @@ def test_pagination_follows_a_real_continuation_token_across_pages(self): patcher, calls = self._mock_http_pages(pages) with patcher: src = self.source() - out = src.fetch(["AAPL"], TODAY, NEXT_SESSION) + out, ambiguous = src.fetch(["AAPL"], TODAY, NEXT_SESSION) self.assertEqual(len(calls), 2, "both pages must actually be requested") self.assertNotIn("page_token", calls[0]["params"]) self.assertEqual(calls[1]["params"].get("page_token"), "page-2-token", "the second request must carry the first page's own continuation token") self.assertEqual(len(out["AAPL"]), 2, "records from both pages must be combined") + self.assertEqual(ambiguous, set()) + + def test_region_us_sent_on_every_page(self): + """LOW finding (fix round 5): `region=us` (and `data_quality=all`) + must be sent on EVERY page request, not just the first -- a per-page + `page_params = dict(params)` copy is what makes this true; a bug + that built `page_params` only once outside the loop, or rebuilt it + without the base `params`, would drop these on later pages.""" + pages = [ + ({"cash_dividends": []}, "page-2-token"), + ({"cash_dividends": []}, None), + ] + patcher, calls = self._mock_http_pages(pages) + with patcher: + src = self.source() + src.fetch(["AAPL"], TODAY, NEXT_SESSION) + self.assertEqual(len(calls), 2) + for call in calls: + self.assertEqual(call["params"].get("region"), "us") + self.assertEqual(call["params"].get("data_quality"), "all") + + def test_empty_string_page_token_is_not_a_continuation(self): + """LOW finding (fix round 5): an empty-string `next_page_token` + must be treated as "no more pages", exactly like an absent/null + token -- not followed as a real continuation.""" + pages = [({"cash_dividends": [{"symbol": "AAPL", "ex_date": TODAY.isoformat()}]}, "")] + patcher, calls = self._mock_http_pages(pages) + with patcher: + src = self.source() + out, ambiguous = src.fetch(["AAPL"], TODAY, NEXT_SESSION) + self.assertEqual(len(calls), 1, "an empty-string token must not trigger a second request") + self.assertEqual(len(out["AAPL"]), 1) def test_cap_is_enforced_across_pages_not_just_within_one(self): # LOW finding 4 (fix round 4): 600 + 600 items across two pages @@ -721,12 +814,28 @@ def test_cap_is_enforced_across_pages_not_just_within_one(self): def test_exhausting_max_pages_with_a_token_still_present_fails_closed(self): # LOW finding 4 (fix round 4): a still-present continuation token # after _MAX_PAGES pages must never be silently truncated. - src = self.source() - pages = [({"cash_dividends": []}, f"tok-{i}") for i in range(src._MAX_PAGES)] - patcher, _ = self._mock_http_pages(pages) + # + # MEDIUM finding (fix round 5): the source MUST be constructed + # INSIDE the patch context -- constructing it beforehand captures + # the REAL bound `requests.Session.request` method as this + # wrapper's `original_request` (see AlpacaCorporateActionsSource. + # __init__'s `_bounded_request` closure), so every one of this + # wrapper's own page requests would silently go out over the real + # network to data.alpaca.markets with fake-key strings instead of + # ever reaching the scripted fixture -- reproduced by an + # independent review's own run: ~58 real GETs, all 401s, the test + # passing on the resulting exception with the fixture never served + # at all. Asserting the exact page count actually served (matching + # `_MAX_PAGES`, no more) proves the scripted fixture -- not a real + # network response -- is what this test actually exercises. + pages = [({"cash_dividends": []}, f"tok-{i}") for i in range(CA.AlpacaCorporateActionsSource._MAX_PAGES)] + patcher, calls = self._mock_http_pages(pages) with patcher: + src = self.source() with self.assertRaises(CA.CorporateActionLookupError): src.fetch(["AAPL"], TODAY, NEXT_SESSION) + self.assertEqual(len(calls), CA.AlpacaCorporateActionsSource._MAX_PAGES, + "must serve exactly _MAX_PAGES scripted pages -- not fall through to a real request") def test_unit_split_alternate_symbol_is_mapped(self): body = {"unit_splits": [{"old_symbol": "OLD", "new_symbol": "NEW", "alternate_symbol": "ALT", @@ -734,24 +843,72 @@ def test_unit_split_alternate_symbol_is_mapped(self): patcher, _ = self._mock_http(body) with patcher: src = self.source() - out = src.fetch(["OLD", "NEW", "ALT"], TODAY, NEXT_SESSION) + out, ambiguous = src.fetch(["OLD", "NEW", "ALT"], TODAY, NEXT_SESSION) for symbol in ("OLD", "NEW", "ALT"): self.assertEqual(len(out[symbol]), 1, f"{symbol} must be mapped from a unit_split") + self.assertEqual(ambiguous, set()) def test_record_missing_governing_date_is_ambiguous_for_its_own_symbol_only(self): """HIGH finding 3 (fix round 3): a record missing (or carrying an unparseable) governing date -- e.g. an incomplete record surfaced by data_quality=all -- must mark ONLY the symbol(s) that record - names as LOOKUP_AMBIGUOUS, never fail the whole fetch, and never - affect an unrelated symbol's own clean result.""" + names as ambiguous, never fail the whole fetch, and never affect an + unrelated symbol's own clean result.""" body = {"cash_dividends": [{"symbol": "AAPL"}, # no ex_date at all {"symbol": "MSFT", "ex_date": TODAY.isoformat()}]} patcher, _ = self._mock_http(body) with patcher: src = self.source() - out = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) - self.assertEqual(out["AAPL"], CA.LOOKUP_AMBIGUOUS) + out, ambiguous = src.fetch(["AAPL", "MSFT"], TODAY, NEXT_SESSION) + self.assertEqual(out["AAPL"], []) + self.assertIn("AAPL", ambiguous) self.assertEqual(len(out["MSFT"]), 1) + self.assertNotIn("MSFT", ambiguous) + + def test_confirmed_split_plus_historical_action_plus_undated_dividend_stays_uncertain(self): + """MEDIUM (Claude/Codex round-5 review, scenario a): a confirmed + upcoming split, a historical (out-of-window) action, and an undated + dividend for the SAME symbol -- must read as BOTH confirmed + (records include the split) AND uncertain (the undated dividend + still raises `ambiguous`), never fully clear.""" + body = {"forward_splits": [{"symbol": "NVDA", "ex_date": NEXT_SESSION.isoformat()}], + "cash_mergers": [{"acquirer_symbol": "NVDA", "acquiree_symbol": "X", + "effective_date": (TODAY - timedelta(days=400)).isoformat()}], + "cash_dividends": [{"symbol": "NVDA"}]} # undated + patcher, _ = self._mock_http(body) + with patcher: + src = self.source() + out, ambiguous = src.fetch(["NVDA"], TODAY, NEXT_SESSION) + self.assertTrue(any(r.action_type == "forward_split" for r in out["NVDA"])) + self.assertIn("NVDA", ambiguous) + + def test_dated_dividend_plus_unmapped_reorganization_tomorrow_stays_uncertain(self): + """MEDIUM (Claude/Codex round-5 review, scenario b): SPY has a + dated dividend far in the window AND an unmapped `reorganizations` + row effective tomorrow -- must not read fully clear.""" + body = {"cash_dividends": [{"symbol": "SPY", "ex_date": NEXT_SESSION.isoformat()}], + "reorganizations": [{"symbol": "SPY", "effective_date": (TODAY + timedelta(days=1)).isoformat()}]} + patcher, _ = self._mock_http(body) + with patcher: + src = self.source() + out, ambiguous = src.fetch(["SPY"], TODAY, NEXT_SESSION) + self.assertEqual(len(out["SPY"]), 1) + self.assertIn("SPY", ambiguous) + + def test_capital_gains_row_tomorrow_plus_far_future_dividend_stays_uncertain(self): + """MEDIUM (Claude/Codex round-5 review, scenario c): QQQ has an + unmapped `capital_gains_distributions` row effective tomorrow + alongside a far-future dated dividend -- must not read fully + clear.""" + body = {"cash_dividends": [{"symbol": "QQQ", "ex_date": (TODAY + timedelta(days=95)).isoformat()}], + "capital_gains_distributions": [{"symbol": "QQQ", + "ex_date": (TODAY + timedelta(days=1)).isoformat()}]} + patcher, _ = self._mock_http(body) + with patcher: + src = self.source() + out, ambiguous = src.fetch(["QQQ"], TODAY, NEXT_SESSION) + self.assertEqual(len(out["QQQ"]), 1) + self.assertIn("QQQ", ambiguous) @unittest.skipUnless(NATIVE, "requires pinned Nautilus 2.0.0rc5 runtime") diff --git a/tests/test_adaptive_paper_runner.py b/tests/test_adaptive_paper_runner.py index af796943c..8e3092ca7 100644 --- a/tests/test_adaptive_paper_runner.py +++ b/tests/test_adaptive_paper_runner.py @@ -2951,6 +2951,160 @@ async def submit_buys_only(payload): ledger.close() self.assertIn("AAPL", outcome["corporate_action_guard"]["pending_must_flatten"]) + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") + def test_run_native_shutdown_cleanup_hook_fires_before_return(self): + """MEDIUM finding (fix round 5, Codex 4a): the shutdown test above + (and the pre-existing G6 test) only asserted the END STATE after + `asyncio.run()` already returned -- which `asyncio.run()`'s own + `Runner.close()` guarantees regardless of run_native's OWN explicit + cleanup call (it cancels every outstanding Task itself). This + proves run_native's `finally` block ACTUALLY calls + `_shutdown_corporate_action_tasks` itself, by patching that exact + module-level function with a recording wrapper and confirming it + was invoked -- `asyncio.run()`'s own internal `_cancel_all_tasks()` + never calls this named Python function at all, so only run_native's + own explicit call site can make this hook fire.""" + import runner as runner_module + from simulation import SimulatedPort + config, _, _ = load_config(SOURCE / "config.json") + config.update(duration_seconds=1, cleanup_seconds=1, order_timeout_seconds=1) + config["sessions"] = {"extended_hours": True, "overnight_holds": True, + "overnight_gross_multiple": "1.0"} + config["regular_session_only"] = False + config["extended_hours_enabled"] = True + + class QuickGuard: + def refresh(self, symbols, *, start, end, now): + pass + + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + return {} + + config["_corporate_action_guard"] = QuickGuard() + calls = [] + original = runner_module._shutdown_corporate_action_tasks + + async def recording_shutdown(ca_refresh_state): + calls.append(1) + return await original(ca_refresh_state) + + with tempfile.TemporaryDirectory() as root: + ledger = Ledger(Path(root) / "journal.db", RiskLimits(trial_seconds=1, cleanup_seconds=1)) + ledger.start_trial(time.time()) + controller = Controller(ledger, time.time() + 3600, market_open=True) + policy = PolicyConfig(symbols=("SPY", "QQQ", "IWM", "DIA"), max_positions=4, warmup_samples=4, + warmup_seconds=.06, sample_seconds=.02, rebalance_seconds=.02, + min_hold_seconds=.05, max_hold_seconds=.4, cooldown_seconds=.05) + port = SimulatedPort(controller, policy.symbols, price=lambda symbol, tick: Decimal("100")) + controller.port = port + with patch.object(runner_module, "_shutdown_corporate_action_tasks", recording_shutdown): + asyncio.run(run_native(controller, policy, [{"symbol": s} for s in policy.symbols], + "fixture", config, "100000")) + ledger.close() + self.assertGreaterEqual(len(calls), 1, + "run_native's own finally block must call _shutdown_corporate_action_tasks " + "directly, not merely rely on asyncio.run()'s own task cancellation") + + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") + def test_run_native_refuses_the_hold_when_a_refresh_degrades_after_the_final_rebalance(self): + """HIGH finding (fix round 5, Codex 4b): the existing hold/summary + tests pass even when run_native's summary is (in memory) reverted + to the strategy's own stale per-tick snapshot, because they either + call `_final_corporate_action_guard_summary` directly (a unit test + of the seam, not of run_native's actual wiring) or use a guard whose + `evaluate()` answer never actually changes after the last tick. + This drives the REAL `run_native` tick loop with a guard whose + `evaluate()` answer flips from clear to `needs_attention` at a wall- + clock threshold set to land AFTER the trial's own rebalancing has + stopped (during cleanup/shutdown) -- reproducing "a refresh that + completes/fails after the final rebalance tick" -- and asserts the + REAL returned `outcome` both reports the held symbol under + `pending_needs_attention_held` and REFUSES to report + `held_overnight`.""" + import corporate_actions as CA + from simulation import SimulatedPort + config, _, _ = load_config(SOURCE / "config.json") + config.update(duration_seconds=1, cleanup_seconds=2, order_timeout_seconds=1) + config["sessions"] = {"extended_hours": True, "overnight_holds": True, + "overnight_gross_multiple": "1.0"} + config["regular_session_only"] = False + config["extended_hours_enabled"] = True + + class LateDegradingGuard: + """Clear for every evaluate() call up through the end of the + configured trial duration; any call after that (i.e., during + cleanup/shutdown, or the outcome's own final recompute) reports + needs_attention for AAPL instead -- simulating a background + refresh that only degrades the cached result AFTER the last + rebalance tick already ran.""" + def __init__(self, start_time, duration_seconds): + self.start_time = start_time + self.duration_seconds = duration_seconds + + def refresh(self, symbols, *, start, end, now): + pass + + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + if "AAPL" not in held_symbols: + return {} + import time as time_module + if time_module.time() < self.start_time + self.duration_seconds: + return {"AAPL": CA.GuardDecision(symbol="AAPL", block_entry=False, must_flatten=False, + needs_attention=False, reason=None)} + return {"AAPL": CA.GuardDecision(symbol="AAPL", block_entry=True, must_flatten=False, + needs_attention=True, + reason="corporate_action_lookup_failed")} + + started_at = time.time() + config["_corporate_action_guard"] = LateDegradingGuard(started_at, config["duration_seconds"]) + with tempfile.TemporaryDirectory() as root: + ledger = Ledger(Path(root) / "journal.db", RiskLimits(trial_seconds=1, cleanup_seconds=2)) + ledger.start_trial(time.time()) + controller = Controller(ledger, time.time() + 3600, market_open=True) + policy = PolicyConfig(symbols=("SPY", "QQQ", "IWM", "DIA", "AAPL"), max_positions=5, warmup_samples=4, + warmup_seconds=.06, sample_seconds=.02, rebalance_seconds=.02, + min_hold_seconds=.05, max_hold_seconds=.4, cooldown_seconds=.05) + + def price(symbol, tick): + slope = Decimal(".03") if symbol in policy.benchmarks else Decimal(".10") + return Decimal("100") + min(tick, 70) * slope + + port = SimulatedPort(controller, policy.symbols, price=price) + original_submit = port.submit + + async def submit_buys_only(payload): + # Keep AAPL genuinely held through to the outcome (a sell + # here rests, never fills) -- an ordinary rebalance-driven + # sell (min_hold/max_hold/cooldown are all short) would + # otherwise flatten AAPL well before the trial ends, + # regardless of this guard, so held_symbols_now at outcome + # time would already be empty for reasons unrelated to what + # this test is proving. + if payload.get("side") == "sell": + import inspect as inspect_module + import time as time_module + controller.before_submit(payload) + await controller.before_request("submit", client_id=payload["client_order_id"]) + order = dict(payload, id=f"sim-resting-{payload['client_order_id']}", status="new", + filled_qty="0", filled_avg_price="0", updated_at_ns=time_module.time_ns()) + port.orders[payload["client_order_id"]] = order + controller.observe(order) + result = port.on_order(dict(order)) + if inspect_module.isawaitable(result): + await result + return dict(order) + return await original_submit(payload) + + port.submit = submit_buys_only + controller.port = port + outcome = asyncio.run(run_native(controller, policy, [{"symbol": s} for s in policy.symbols], + "fixture", config, "100000")) + ledger.close() + self.assertIn("AAPL", outcome["corporate_action_guard"]["pending_needs_attention_held"], + "the REAL outcome must reflect the guard's post-rebalance degraded state") + self.assertNotEqual(outcome["status"], "held_overnight", + "a run with a pending needs_attention symbol must never report held_overnight") + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") def test_final_summary_uses_the_guards_current_state_not_a_stale_strategy_snapshot(self): """MEDIUM finding 3 (fix round 4): reproduces the independent @@ -2991,6 +3145,33 @@ def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, "strategy's stale (empty) _ca_ever_flagged snapshot") self.assertIn("AAPL", summary["ever_flagged_symbols"]) + def test_final_summary_fails_closed_when_the_calendar_itself_cannot_classify_now(self): + """finding 6 (round 3)/H11 (fix round 5): when `session_at(now)` + itself raises ValueError (the instant falls outside the frozen + session calendar), the final summary must fail CLOSED -- every held + symbol reported under `pending_needs_attention_held` -- never + silently "clear" just because the calendar failed to classify this + instant. A calendar failure is itself a reason for attention, not + evidence of safety.""" + import runner as runner_module + + class UnusedGuard: + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + raise AssertionError("evaluate() must never be reached once the calendar itself fails") + + class Strategy: + def __init__(self, guard): + self.corporate_action_guard = guard + self._ca_ever_flagged = set() + + # Far outside the frozen session calendar's supported year range. + out_of_range_now = datetime(2099, 1, 1, tzinfo=timezone.utc).timestamp() + summary = runner_module._final_corporate_action_guard_summary( + Strategy(UnusedGuard()), {"AAPL"}, out_of_range_now) + self.assertEqual(summary["pending_needs_attention_held"], ["AAPL"], + "a calendar failure must fail closed for every held symbol, not report clear") + self.assertEqual(summary["pending_must_flatten"], []) + def test_final_summary_is_clear_when_the_guard_is_actually_clear(self): """The mirror case: a guard reporting a genuinely clear state for every held symbol must not be reported as pending.""" @@ -3141,6 +3322,239 @@ def fetch(self, symbols, start, end): self.assertTrue(decisions_after_a["NVDA"].must_flatten, "A's late, superseded write must NOT cancel B's already-applied confirmed action") + def test_scheduler_calls_begin_attempt_before_launching_the_worker_thread(self): + """HIGH finding (fix round 5): `_schedule_corporate_action_refresh` + itself -- not merely the monitor's own primitives -- must call + `guard.begin_attempt(...)` on the calling coroutine BEFORE + `threading.Thread(...).start()` is ever reached, for any guard that + exposes the two-phase API. Proven directly by recording the ORDER + `begin_attempt` and the worker's own `run_attempt` are invoked in a + shared list from a fake two-phase guard -- `begin_attempt` must + appear first, and (since it happens on the calling coroutine, not + the worker thread) it must have already run by the time this + function RETURNS the created task, before that task's own worker + thread has necessarily done any work at all.""" + import asyncio + from datetime import date + import runner as runner_module + + order = [] + + class TwoPhaseGuard: + def begin_attempt(self, symbols, *, start, end, now): + order.append("begin_attempt") + return 1 + + def run_attempt(self, symbols, generation, *, start, end, now): + order.append("run_attempt") + + def refresh_timed_out(self, generation, symbols): + order.append("refresh_timed_out") + + def invalidate_attempt(self, generation): + order.append("invalidate_attempt") + + async def scenario(): + state = {"in_flight": False} + task = await runner_module._schedule_corporate_action_refresh( + TwoPhaseGuard(), {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) + # begin_attempt must ALREADY have happened -- it runs + # synchronously on this calling coroutine before the task + # (whose own worker thread does the rest) is even created. + self.assertEqual(order, ["begin_attempt"], + "begin_attempt must run before the worker thread is launched, " + "not lazily inside the worker itself") + await task + self.assertEqual(order, ["begin_attempt", "run_attempt"]) + + asyncio.run(scenario()) + + def test_begin_attempt_generation_survives_a_shutdown_before_the_worker_even_runs(self): + """HIGH finding (fix round 5): reproduces an independent review's + own probe (R1's synchronized scenario) directly against the + two-phase API -- the generation for an attempt must be allocated + (via begin_attempt) and invalidated (via invalidate_attempt) BEFORE + the actual fetch ever runs, and that invalidation must still work + even though the fetch itself never observed it (Python cannot + forcibly kill a thread). Simulates a worker thread launched, then + delayed by the OS before it starts running -- a shutdown-time + cancellation fires and invalidates the pre-allocated generation -- + and only THEN does the delayed worker finally run and try to write + a confirmed action; that write must be dropped as superseded.""" + from datetime import date + import corporate_actions as CA + + class Src: + def fetch(self, symbols, start, end): + return {s: [CA.ActionRecord(symbol=s, action_type="forward_split", + action_date=date(2026, 9, 25))] for s in symbols} + + monitor = CA.CorporateActionMonitor(Src(), clock=lambda: 1000.0) + generation = monitor.begin_attempt({"NVDA"}, start=date(2026, 9, 17), end=date(2026, 12, 24), now=1000.0) + self.assertIsNotNone(generation) + # Shutdown fires HERE -- before the worker thread (which has not + # even started running `fetch()` yet, simulating an OS-scheduling + # delay after Thread.start() was already called) has done any work. + monitor.invalidate_attempt(generation) + # The delayed worker now finally runs and tries to commit its + # (stale, since-invalidated) result. + monitor.run_attempt({"NVDA"}, generation, start=date(2026, 9, 17), end=date(2026, 12, 24), now=1000.5) + decisions = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"NVDA"}, candidate_symbols=set(), now=1000.5) + self.assertFalse(decisions["NVDA"].must_flatten, + "an attempt invalidated before it ever ran must never commit its late write") + + def test_cancellation_of_a_routine_refresh_does_not_degrade_a_clean_cached_result(self): + """LOW finding (fix round 5, Claude's review): cancellation (an + ordinary shutdown interrupting an otherwise routine, still-in- + flight refresh) must NOT force the cached result to needs_attention + -- only a genuine timeout should. A perfectly clean, still-fresh + cached result must stand after a cancellation, unlike after a + `refresh_timed_out`.""" + from datetime import date + import corporate_actions as CA + + class Src: + def fetch(self, symbols, start, end): + return {s: [] for s in symbols} + + monitor = CA.CorporateActionMonitor(Src(), clock=lambda: 1000.0) + # A clean, confirmed-clear, fresh result is already cached. + monitor.refresh({"SPY"}, start=date(2026, 9, 17), end=date(2026, 12, 24), now=1000.0) + clean = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"SPY"}, candidate_symbols=set(), now=1000.0) + self.assertFalse(clean["SPY"].needs_attention) + # A routine periodic refresh is now in flight (generation + # allocated) when shutdown cancels it. + generation = monitor.begin_attempt({"SPY"}, start=date(2026, 9, 17), end=date(2026, 12, 24), now=1900.0) + monitor.invalidate_attempt(generation) + after_cancel = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"SPY"}, candidate_symbols=set(), now=1900.0) + self.assertFalse(after_cancel["SPY"].needs_attention, + "cancelling a routine in-flight refresh must not degrade an already-clean result") + # Contrast: a genuine TIMEOUT for the same kind of attempt DOES degrade. + generation2 = monitor.begin_attempt({"SPY"}, start=date(2026, 9, 17), end=date(2026, 12, 24), now=2800.0) + monitor.refresh_timed_out(generation2, {"SPY"}) + after_timeout = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"SPY"}, candidate_symbols=set(), now=2800.0) + self.assertTrue(after_timeout["SPY"].needs_attention, + "a genuine timeout, unlike a cancellation, must degrade") + + def test_scheduled_refresh_cancellation_invalidates_not_degrades_with_the_real_monitor(self): + """HIGH/LOW finding (fix round 5): end-to-end through + `_schedule_corporate_action_refresh` and `_shutdown_corporate_ + action_tasks` with the REAL `CorporateActionMonitor` -- a routine + refresh still in flight when shutdown cancels it must leave an + already-clean cached result clean (not needs_attention), while a + delayed worker's late write (released only after shutdown) must + still be dropped as superseded.""" + import asyncio + import threading + from datetime import date + import corporate_actions as CA + import runner as runner_module + + release = threading.Event() + + class Src: + def __init__(self): + self.calls = 0 + + def fetch(self, symbols, start, end): + self.calls += 1 + if self.calls > 1: + release.wait(3) # the routine periodic refresh is still in flight at shutdown + return {s: [] for s in symbols} + + async def scenario(): + monitor = CA.CorporateActionMonitor(Src(), clock=lambda: 1000.0) + monitor.refresh({"SPY"}, start=date(2026, 9, 17), end=date(2026, 12, 24), now=1000.0) + state = {"in_flight": False} + await runner_module._schedule_corporate_action_refresh( + monitor, {"SPY"}, date(2026, 9, 17), date(2026, 12, 24), 1900.0, state) + await asyncio.sleep(0.1) # let the worker actually enter fetch() and block on `release` + await runner_module._shutdown_corporate_action_tasks(state) + decisions = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"SPY"}, candidate_symbols=set(), now=1900.0) + self.assertFalse(decisions["SPY"].needs_attention, + "cancelling a routine in-flight refresh at shutdown must not force needs_attention") + release.set() # let the delayed worker finally finish, after shutdown + + asyncio.run(scenario()) + + def test_in_flight_is_reset_even_if_starting_the_worker_thread_itself_raises(self): + """LOW finding (fix round 5): `Thread.start()` must sit INSIDE the + try/finally that resets `state["in_flight"]` -- if starting the + thread itself ever raised (e.g. a resource-exhaustion OSError), the + old placement (outside the try) would leave `in_flight` stuck True + forever, permanently locking out every future refresh attempt.""" + import asyncio + import threading + from datetime import date + import runner as runner_module + + class ExplodingThread: + def __init__(self, *args, **kwargs): + pass + + def start(self): + raise RuntimeError("simulated resource exhaustion starting the worker thread") + + class QuickGuard: + def refresh(self, symbols, *, start, end, now): + pass + + async def scenario(): + state = {"in_flight": False} + with patch.object(threading, "Thread", ExplodingThread): + task = await runner_module._schedule_corporate_action_refresh( + QuickGuard(), {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) + # The patch must still be active when the created task + # actually RUNS -- asyncio.create_task only SCHEDULES the + # coroutine; it does not execute until the next loop + # iteration. + await asyncio.sleep(0) + with self.assertRaises(RuntimeError): + await task + self.assertFalse(state["in_flight"], "in_flight must be reset even when Thread.start() itself raises") + + asyncio.run(scenario()) + + def test_worker_thread_is_a_daemon(self): + """LOW finding (fix round 5): the worker thread that runs the + blocking fetch must be a DAEMON thread -- not provable by the + exit-timing test alone (a slow-but-not-infinite hang could still + happen to let the process exit in time even on a non-daemon + thread, or vice versa be flaky under load) -- capture the actual + `threading.Thread` object via a patched constructor and assert + `daemon=True` was passed directly.""" + import asyncio + import threading + from datetime import date + import runner as runner_module + + captured = {} + original_thread = threading.Thread + + def capturing_thread(*args, **kwargs): + t = original_thread(*args, **kwargs) + captured["daemon"] = kwargs.get("daemon") + return t + + class QuickGuard: + def refresh(self, symbols, *, start, end, now): + pass + + async def scenario(): + state = {"in_flight": False} + with patch.object(threading, "Thread", capturing_thread): + task = await runner_module._schedule_corporate_action_refresh( + QuickGuard(), {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) + await task + + asyncio.run(scenario()) + self.assertTrue(captured.get("daemon"), "the refresh worker thread must be created with daemon=True") + def test_shutdown_cancels_and_joins_pending_corporate_action_tasks(self): """HIGH finding 1 (fix round 3, 'keep the task and join or cancel it at shutdown'): directly exercises the extracted @@ -3174,6 +3588,160 @@ async def scenario(): asyncio.run(scenario()) + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") + def test_shutdown_cleanup_still_runs_when_port_stop_raises(self): + """LOW finding 7 (fix round 4)/H13 (fix round 5): `_shutdown_ + corporate_action_tasks` must still run, and actually cancel a + still-hanging refresh task, even when `port.stop()` itself raises + -- the nested `finally` around it, not merely the outer one, is + what guarantees this. Patches `port.stop` to raise after + construction, so run_native's own exception path (not a happy + path) is what this test drives.""" + from simulation import SimulatedPort + config, _, _ = load_config(SOURCE / "config.json") + config.update(duration_seconds=1, cleanup_seconds=1, order_timeout_seconds=1) + config["sessions"] = {"extended_hours": True, "overnight_holds": True, + "overnight_gross_multiple": "1.0"} + config["regular_session_only"] = False + config["extended_hours_enabled"] = True + + class HangingGuard: + def refresh(self, symbols, *, start, end, now): + import time as time_module + time_module.sleep(30) # outlives the trial + cleanup + shutdown entirely + + def evaluate(self, *, today, next_session_date, held_symbols, candidate_symbols, now): + return {} + + config["_corporate_action_guard"] = HangingGuard() + ca_refresh_state = {"in_flight": False} + # MEDIUM finding (fix round 5): asyncio.run()'s own Runner.close() + # cancels every outstanding Task regardless of run_native's OWN + # explicit cleanup call (the same redundancy documented for G6) -- + # so merely checking the end state of ca_refresh_state["tasks"] + # after asyncio.run() returns cannot distinguish "run_native's own + # nested finally called _shutdown_corporate_action_tasks despite + # port.stop() raising" from "asyncio.run()'s unrelated auto-cancel + # produced the same end state anyway". A recording wrapper around + # the exact named function proves the EXPLICIT call site fired. + calls = [] + original_shutdown = runner_module._shutdown_corporate_action_tasks + + async def recording_shutdown(state): + calls.append(1) + return await original_shutdown(state) + + with tempfile.TemporaryDirectory() as root: + ledger = Ledger(Path(root) / "journal.db", RiskLimits(trial_seconds=1, cleanup_seconds=1)) + ledger.start_trial(time.time()) + controller = Controller(ledger, time.time() + 3600, market_open=True) + policy = PolicyConfig(symbols=("SPY", "QQQ", "IWM", "DIA"), max_positions=4, warmup_samples=4, + warmup_seconds=.06, sample_seconds=.02, rebalance_seconds=.02, + min_hold_seconds=.05, max_hold_seconds=.4, cooldown_seconds=.05) + port = SimulatedPort(controller, policy.symbols, price=lambda symbol, tick: Decimal("100")) + + async def raising_stop(): + raise RuntimeError("port.stop() itself fails") + + port.stop = raising_stop + controller.port = port + with patch.object(runner_module, "_shutdown_corporate_action_tasks", recording_shutdown): + with self.assertRaises(RuntimeError): + asyncio.run(run_native(controller, policy, [{"symbol": s} for s in policy.symbols], + "fixture", config, "100000", ca_refresh_state=ca_refresh_state)) + ledger.close() + self.assertGreaterEqual(len(calls), 1, + "the nested finally must call _shutdown_corporate_action_tasks directly " + "even when port.stop() itself raises, not rely on asyncio.run()'s own " + "unrelated task-cancellation cleanup") + pending = [t for t in ca_refresh_state.get("tasks", []) if not t.done()] + self.assertEqual(pending, [], "the refresh task must still be cancelled even though port.stop() raised") + self.assertTrue(any(t.cancelled() for t in ca_refresh_state.get("tasks", [])), + "at least one task must have been genuinely cancelled despite the port.stop() error") + + def test_resolve_does_not_crash_when_the_future_is_already_cancelled(self): + """H14 (fix round 5): `asyncio.wait_for` cancels the future it is + waiting on when its OWN caller is cancelled (here, shutdown + cancelling the task while the worker thread is still blocked in + `fetch()`) -- the worker's own late `_resolve()` callback (run via + `loop.call_soon_threadsafe` once `fetch()` finally returns, well + after that cancellation) must check `fut.done()` before calling + `set_result` on it, or it raises `InvalidStateError` inside that + scheduled callback, which the loop's own exception handler would + otherwise have to swallow.""" + import threading + from datetime import date + import corporate_actions as CA + import runner as runner_module + + release = threading.Event() + loop_exceptions = [] + + class Src: + def fetch(self, symbols, start, end): + release.wait(3) # still running when shutdown cancels the waiting task + return {s: [] for s in symbols} + + async def scenario(): + loop = asyncio.get_running_loop() + loop.set_exception_handler(lambda loop, context: loop_exceptions.append(context)) + monitor = CA.CorporateActionMonitor(Src(), clock=lambda: 1000.0) + state = {"in_flight": False} + await runner_module._schedule_corporate_action_refresh( + monitor, {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) + await asyncio.sleep(0.05) # let the worker actually start and block on `release` + await runner_module._shutdown_corporate_action_tasks(state) # cancels the still-waiting task + release.set() # let the worker's fetch() finally return, well after cancellation + await asyncio.sleep(0.2) # give _resolve()'s call_soon_threadsafe callback a chance to run + + asyncio.run(scenario()) + self.assertEqual(loop_exceptions, [], "the worker's late _resolve() must not raise on an " + "already-cancelled future") + + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") + def test_worker_swallows_runtimeerror_when_the_loop_is_already_closed(self): + """LOW finding 5 (fix round 4)/H15 (fix round 5): once asyncio.run() + closes the loop, a still-running worker thread's own `loop. + call_soon_threadsafe` back to that now-closed loop raises + RuntimeError -- the worker must swallow exactly that (the late + result is simply dropped), never crash its own daemon thread with + an unhandled exception.""" + import threading + import time as time_module + from datetime import date + + release = threading.Event() + excepthook_calls = [] + original_hook = threading.excepthook + + def recording_hook(args): + excepthook_calls.append(args) + + class HangingGuard: + def refresh(self, symbols, *, start, end, now): + release.wait(5) # still running well after asyncio.run() itself returns + + async def scenario(): + state = {"in_flight": False} + original_timeout = runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS + runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = 0.05 + try: + await runner_module._schedule_corporate_action_refresh( + HangingGuard(), {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), 1000.0, state) + await asyncio.sleep(0.1) # let the timeout fire; the worker itself is still running + finally: + runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = original_timeout + + threading.excepthook = recording_hook + try: + asyncio.run(scenario()) # returns/closes its own loop while the worker is still hanging + release.set() # let the worker finish now that the loop is already closed + time_module.sleep(0.3) # give the worker's own finally a moment to run against the closed loop + finally: + threading.excepthook = original_hook + self.assertEqual(excepthook_calls, [], "the worker must swallow RuntimeError from a closed loop, " + "not crash its own daemon thread") + @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") def test_run_native_actually_cancels_a_hanging_refresh_at_its_own_shutdown(self): """G6 (fix round 4): run_native's OWN finally block must actually From 65ee7fb2fe3b1f90de31a32e0cb4f0cce997d5c0 Mon Sep 17 00:00:00 2001 From: Scout Date: Thu, 24 Sep 2026 20:15:20 -0400 Subject: [PATCH 7/8] Fix round 6: preserve confirmed records on ambiguous-only responses, resolve every allocated generation, and harden the warm-monitor preflight Four low-severity items from the final independent review, plus the test gaps two of them exposed: 1. An ambiguous-only response (a previously confirmed record whose row loses its governing date on a later fetch) wrote an empty list over the last good records in complete_attempt, turning a confirmed must_flatten back into mere block-and-flag and losing an already-live intraday exit signal. Fixed: an ambiguous symbol with zero valid records in the new response now degrades (preserving the last good records) instead of overwriting them with []. 2. Mutation G4 (drop the scheduler's refresh_timed_out call for a two-phase guard) survived because every existing short-timeout test drove a legacy single-arg guard, never the real CorporateActionMonitor's own generation-aware timeout path. Added a test driving the real monitor through a short-timeout scenario, asserting the symbol is flagged and the next attempt is allowed after retry_seconds (60s), not throttled for the ordinary refresh_seconds (900s). 3. A pre-allocated generation was left permanently unresolved if Thread.start() itself raised, or if an unexpected exception escaped the worker's own run_attempt call -- refreshes would then be silently throttled with nothing ever flagged for up to 900s. Both paths now call fail_attempt on the pre-allocated generation before propagating. 4. _preflight_corporate_action_refresh unconditionally awaited _schedule_corporate_action_refresh's return value, which is None (not a task) when the two-phase begin_attempt itself reports a throttled no-op -- a second preflight call on an already-warmed monitor raised TypeError from awaiting None. Now only awaits when a task was actually created. The review's own mutation script also caught two gaps in the round-5 fixes themselves once the two-phase API was exercised more directly: G5 (a cancellation mutated to a no-op) survived because the existing test only checked that a clean end state stayed clean, which held regardless of whether invalidate_attempt ran; strengthened to assert the generation counter is actually bumped past the cancelled attempt. A generation- specific refresh_timed_out check reverted to a round-4-era coarse heuristic (comparing only _generation to _committed_generation) also survived; added a test where a stale timeout for an already-superseded (but not yet committed) generation must be a complete no-op. Merged origin/main (added #206 and #218 since the round-5 push); source-hashes.json and manifests/evidence.json were recomputed from current file content for every changed path, not resolved by picking a side. No overlap between main's changes and this branch's files beyond manifests/evidence.json, which merged cleanly. Re-ran: the full adaptive suite (373 tests) with network-blocked pagination tests, test_native_faults_min, test_order_throughput, test_order_contract, test_trading_gates, all six validators (validate.py, validate_catalogs.py, landscape.py, evidence_manifest.py, verdict_review_gate.py, trading_gates.py -- all passed), and the reviewer's own mutate_rev5.py retargeted at this worktree (65 mutations, all killed except one PATTERN_COUNT=0 entry already superseded by an equivalent round-5 mutation of my own). Ran gitleaks git history scan (509 commits, 1.12 GB) with the pinned binary and repo config: all 76,054 findings are pre-existing sourcegraph-access-token/generic-api-key pattern matches inside the generated docs/ecosystem/index.html artifact; zero findings in any file this branch touches. Co-Authored-By: Claude Sonnet 5 --- .../adaptive-paper/corporate_actions.py | 17 ++ .../us-equities/adaptive-paper/runner.py | 46 ++++- .../adaptive-paper/source-hashes.json | 8 +- manifests/evidence.json | 10 +- .../test_adaptive_paper_corporate_actions.py | 54 +++++ tests/test_adaptive_paper_runner.py | 194 ++++++++++++++++++ 6 files changed, 314 insertions(+), 15 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/corporate_actions.py b/blueprints/us-equities/adaptive-paper/corporate_actions.py index 1b7444641..4b44b63d6 100644 --- a/blueprints/us-equities/adaptive-paper/corporate_actions.py +++ b/blueprints/us-equities/adaptive-paper/corporate_actions.py @@ -478,6 +478,23 @@ def complete_attempt(self, symbols, generation, now, fetched, ambiguous=frozense # action. self._mark_degraded((symbol,)) continue + if symbol in ambiguous and not new_value: + # LOW finding (fix round 6): an ambiguous symbol with NO + # valid records of its OWN in this response must not + # overwrite a previously confirmed (non-empty) result + # with an empty list -- exactly like the sentinel branch + # above, this degrades instead, preserving the last + # known good records. Reproduced by an independent + # review's own probe: an in-range NVDA split confirmed + # on one fetch, then a LATER fetch reporting the SAME + # split again but now missing its ex_date (so it + # resolves to zero valid records for NVDA, plus + # ambiguity) used to silently write `_results["NVDA"] + # = []`, turning an already-confirmed must_flatten back + # into block-and-flag -- losing a still-live intraday + # exit signal the guard had already raised. + self._mark_degraded((symbol,)) + continue self._results[symbol] = new_value if symbol in ambiguous: # HIGH finding (fix round 5): keep the (possibly diff --git a/blueprints/us-equities/adaptive-paper/runner.py b/blueprints/us-equities/adaptive-paper/runner.py index 97a14630f..214a66486 100644 --- a/blueprints/us-equities/adaptive-paper/runner.py +++ b/blueprints/us-equities/adaptive-paper/runner.py @@ -1102,7 +1102,20 @@ def worker(): else: guard.refresh(watch, start=start, end=end, now=now) except Exception: - pass # never raises in the well-behaved case; defensive only + # LOW finding (fix round 6): run_attempt/refresh are not + # expected to raise (they catch their own source errors + # internally) -- but if one somehow does, the pre-allocated + # generation must not be left dangling: with no fail_attempt + # call, `_last_attempt_failed` would stay whatever it was + # before this attempt, so a genuinely failed refresh could + # be silently retried on the ordinary long `refresh_seconds` + # interval instead of the short `retry_seconds` one, AND no + # symbol would ever be marked degraded for this failure. + if two_phase: + try: + guard.fail_attempt(watch, generation) + except Exception: + pass # never let a defensive cleanup call itself raise finally: def _resolve(): if not fut.done(): @@ -1118,7 +1131,20 @@ def _resolve(): # reset so the NEXT tick is not permanently locked out believing a # refresh that never actually started is still running. try: - threading.Thread(target=worker, daemon=True, name="ca-refresh").start() + try: + threading.Thread(target=worker, daemon=True, name="ca-refresh").start() + except Exception: + # LOW finding (fix round 6): if starting the worker thread + # itself fails (e.g. a resource-exhaustion OSError), the + # worker's own body -- including its fail_attempt call + # above -- never runs at all, so the pre-allocated + # generation would otherwise be left permanently unresolved + # (never committed, never failed, never invalidated) -- + # refreshes would then be throttled by the ordinary long + # `refresh_seconds` interval with nothing ever flagged. + if two_phase: + guard.fail_attempt(watch, generation) + raise try: await asyncio.wait_for(fut, timeout=CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS) except asyncio.TimeoutError: @@ -1196,12 +1222,20 @@ async def _preflight_corporate_action_refresh(config, session_policy, ledger, ca preflight_watch = set(config["symbols"]) | {p.symbol for p in ledger.positions().values() if p.qty} preflight_task = await _schedule_corporate_action_refresh( guard, preflight_watch, preflight_start, preflight_end, preflight_now, ca_refresh_state) + # NIT (fix round 6): `_schedule_corporate_action_refresh` returns None, + # not a task, when the two-phase `begin_attempt` itself reports a + # throttled no-op (an equivalent fresh result is already cached for + # this exact symbol set/window) -- e.g. a SECOND preflight call on an + # already-warmed monitor. `await None` raises TypeError; there is + # simply nothing to await in that case (a no-op is already complete). + # # _run() inside _schedule_corporate_action_refresh never raises (its # own wait_for already swallows the timeout) -- awaiting the task it - # returned is itself already bounded, so the preflight still blocks (as - # intended -- "do the first fetch in preflight") until the fetch - # resolves or times out, never longer. - await preflight_task + # returned (when one was actually created) is itself already bounded, + # so the preflight still blocks (as intended -- "do the first fetch in + # preflight") until the fetch resolves or times out, never longer. + if preflight_task is not None: + await preflight_task def _final_corporate_action_guard_summary(strategy, held_symbols, now): diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index 407ddb49f..8987f2190 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -7,7 +7,7 @@ "blueprints/us-equities/adaptive-paper/config-mover.json": "560849b980082a72f8ec45a32ffae51b1f50a502e739c94346ff501a46777ea8", "blueprints/us-equities/adaptive-paper/config-sip.json": "3587653104e68fdfc9bbb9170825c611abcf7e2e1be1004327a056031c34555a", "blueprints/us-equities/adaptive-paper/config.json": "2b555982034af02b9d0d2e1d20e17d22c1e83f2b0c4d60b61d91307ebec63562", - "blueprints/us-equities/adaptive-paper/corporate_actions.py": "50cfdb40c28b042b4fe293863b95df7e66bfa63b9e74719f7459e43812ff47b5", + "blueprints/us-equities/adaptive-paper/corporate_actions.py": "f4afc9eef377d407b3a4d8df29cd7d71d43b09cb18768d911324bdd433d6d5e7", "blueprints/us-equities/adaptive-paper/exits.py": "189133e6df99bda201d73d36a15f45ccbfb38979a3319ae6f75d077f5db5b3df", "blueprints/us-equities/adaptive-paper/feeds.py": "0db3009a724ae2433572c5070066d6806d7365031f9db4e81c4b69ad1cbebacb", "blueprints/us-equities/adaptive-paper/leverage.py": "7f4a86f312d718a0a0b87e6eafe9a2d57b8709ec85a8746fdf7def0a5b502ea0", @@ -24,7 +24,7 @@ "blueprints/us-equities/adaptive-paper/recovery.py": "5cf7dcea572efbda6979a2726eebf6d676f6bd62263900c7b6626da8b45c1215", "blueprints/us-equities/adaptive-paper/registry.json": "60da73b985e718bd1dece7a3350f34636bda0fb38e766a44ae71c5a8c532282a", "blueprints/us-equities/adaptive-paper/requirements.txt": "518afe4b77f901e87abd3a5d6faa126aa7e211db77eecec80cfe7ef42d82e07a", - "blueprints/us-equities/adaptive-paper/runner.py": "c73b535417496c96125ad78c6634d85c6e6cee04cde1bcd08eff73f55e5ea2ea", + "blueprints/us-equities/adaptive-paper/runner.py": "d783e4ef687a439c4ea09a5675406660e169106e5600d98e5c4955fef02c9497", "blueprints/us-equities/adaptive-paper/safety.py": "ad520fc4d84ce10e4d7b5cc956ba769a4f94eef49464ce382746d57f0cd76420", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", @@ -35,7 +35,7 @@ "blueprints/us-equities/adaptive-paper/transport.py": "34e6c4e604179585e3141c4e07066bbc36a655c3eb7052a007354fbe6b2835a6", "tests/adaptive_paper_hermetic.py": "34f2d5b07f42cb60d415abf24e4feef64b9088d1071163ab8427f0d958ebf7ee", "tests/test_adaptive_market_research.py": "dfd7a09380a7b1b10633c2edf05daf5a35bfcb48498fff130a2d84ff7190e395", - "tests/test_adaptive_paper_corporate_actions.py": "cad7adcd284325c0b560e2cdb1763001662c95d89fbaf26d7e0a4d2815129d48", + "tests/test_adaptive_paper_corporate_actions.py": "460599fcce943a949af4dbfd4bc818463e95a6e85e7e0aea1f573b55724ffd3b", "tests/test_adaptive_paper_exits.py": "13a82d65ad1b2e72a8ca0fbb8bbb178a01ded1301399604c9e304ae213456764", "tests/test_adaptive_paper_live_manifest.py": "4596f301ac44effefee5b787c5e3627a9cd38f8ff0cf436ef778b7378a0667c2", "tests/test_adaptive_paper_metrics.py": "8b0627f752ddf11c87005200e76d0ad4873261905ba073163476c66659d564f9", @@ -43,7 +43,7 @@ "tests/test_adaptive_paper_mover_native.py": "0257ca4bf9078e348e39d391dccf042b3f7b0e01d5c9e4d9092a9712f04097d5", "tests/test_adaptive_paper_native.py": "7c97707a5c19b704be9a603bbde396d9cb952dad39ca0d5b434ae1f8ebbd1f81", "tests/test_adaptive_paper_recovery.py": "08e351ff903d6d008b802dc93a8dde07da8900df5cc0540f551ffa909cf4ce95", - "tests/test_adaptive_paper_runner.py": "e378064e9d9cbfdccb06906ca3412908e4608b3790db56570b1c8e67d03e9474", + "tests/test_adaptive_paper_runner.py": "00df9ba014a13674f596498049d19984c260df62586e0563d276ac9069b893df", "tests/test_adaptive_paper_safety.py": "e0b2017ea6bdae87a0778b01ac72747642b31a2495c6d4e8bc98619a5aa1b456", "tests/test_adaptive_paper_selector.py": "27b431ac7911372d050f25d32ac24a3df9d2cf3c5e7dafdebfe51db6cab55936", "tests/test_adaptive_paper_sessions.py": "680060ac7ec147b93ed6ba2607631ad1ec38702f1a44e51260c792c537b1fab6", diff --git a/manifests/evidence.json b/manifests/evidence.json index 05813a01a..3a283438c 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -6650,8 +6650,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/runner.py", - "sha256": "c73b535417496c96125ad78c6634d85c6e6cee04cde1bcd08eff73f55e5ea2ea", - "bytes": 128204 + "sha256": "d783e4ef687a439c4ea09a5675406660e169106e5600d98e5c4955fef02c9497", + "bytes": 130287 }, { "path": "blueprints/us-equities/adaptive-paper/safety.py", @@ -6675,7 +6675,7 @@ }, { "path": "blueprints/us-equities/adaptive-paper/source-hashes.json", - "sha256": "42b7b425f80e89a2088e97c9de41e7753bc7fecb14d13f135b188dcdf8ae78e3", + "sha256": "cebf971439966e23b4169e9c90375fb5b68137bd83ef09d5d11125eb95ff82d2", "bytes": 6122 }, { @@ -29030,8 +29030,8 @@ }, { "path": "tests/test_adaptive_paper_runner.py", - "sha256": "e378064e9d9cbfdccb06906ca3412908e4608b3790db56570b1c8e67d03e9474", - "bytes": 220776 + "sha256": "00df9ba014a13674f596498049d19984c260df62586e0563d276ac9069b893df", + "bytes": 231571 }, { "path": "tests/test_adaptive_paper_safety.py", diff --git a/tests/test_adaptive_paper_corporate_actions.py b/tests/test_adaptive_paper_corporate_actions.py index f45ce1d41..7a1b8c17b 100644 --- a/tests/test_adaptive_paper_corporate_actions.py +++ b/tests/test_adaptive_paper_corporate_actions.py @@ -426,6 +426,30 @@ def test_refresh_timed_out_invalidates_the_in_flight_generation(self): "leaving the symbol degraded from the timeout") self.assertFalse(after.must_flatten) + def test_refresh_timed_out_for_an_already_superseded_generation_is_a_no_op(self): + """LOW finding (fix round 6): a timeout for a SPECIFIC, now- + superseded generation (a NEWER attempt has already started -- not + yet committed -- since that timeout was captured) must be ignored, + exactly like an ordinary superseded write. Distinguishes the + generation-specific check from a coarser (round-4-era) heuristic + that only compared `_generation` against `_committed_generation` + without checking the SPECIFIC generation passed in -- that coarser + rule would wrongly treat this case as "nothing committed yet, so + the timeout is real" and degrade a symbol whose newer attempt is + still perfectly in flight.""" + monitor = CA.CorporateActionMonitor(self.FakeSource(), clock=lambda: 1000.0) + old_generation = monitor.begin_attempt({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + new_generation = monitor.begin_attempt({"AAPL"}, start=TODAY, end=NEXT_SESSION, now=1000.5) + self.assertNotEqual(old_generation, new_generation) + # A timeout fires for the OLD, now-superseded attempt -- the NEWER + # attempt is still genuinely in flight (never committed either + # way) -- must be a complete no-op. + monitor.refresh_timed_out(old_generation, {"AAPL"}) + self.assertFalse(monitor._last_attempt_failed, + "a stale timeout for an already-superseded generation must not affect " + "the still-in-flight newer attempt's own state") + self.assertNotIn("AAPL", monitor._degraded) + def test_per_symbol_ambiguous_result_never_erases_a_confirmed_action(self): """HIGH finding 1 (fix round 4): reproduces the review's own Q2a probe -- a confirmed split, then a LATER fetch that reports the @@ -470,6 +494,36 @@ def fetch(self, symbols, start, end): self.assertTrue(decision.must_flatten, "the confirmed record must still flatten the position") self.assertTrue(decision.needs_attention, "the independent ambiguity flag must still raise attention") + def test_ambiguous_only_response_preserves_a_previously_confirmed_record(self): + """LOW finding (fix round 6): reproduces an independent review's own + probe -- an in-range NVDA split is confirmed on one fetch, then a + LATER fetch reports the SAME split again but now missing its own + governing date (so it resolves to zero valid records for NVDA, plus + ambiguity via the tuple contract's second element) -- must not + silently overwrite `_results["NVDA"]` with an empty list, turning + an already-confirmed must_flatten back into mere block-and-flag and + losing an already-live intraday exit signal. The symbol must stay + degraded (needs_attention) while the confirmed record is + preserved.""" + seq = [({"NVDA": [action("NVDA", "forward_split", NEXT_SESSION)]}, set()), + ({"NVDA": []}, {"NVDA"})] + + class Src: + def fetch(self, symbols, start, end): + return seq.pop(0) + + monitor = CA.CorporateActionMonitor(Src(), refresh_seconds=0, clock=lambda: 1000.0) + monitor.refresh({"NVDA"}, start=TODAY, end=NEXT_SESSION, now=1000.0) + before = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"NVDA"}, + candidate_symbols=set(), now=1000.0)["NVDA"] + self.assertTrue(before.must_flatten) + monitor.refresh({"NVDA"}, start=TODAY, end=NEXT_SESSION, now=1100.0) + after = monitor.evaluate(today=TODAY, next_session_date=NEXT_SESSION, held_symbols={"NVDA"}, + candidate_symbols=set(), now=1100.0)["NVDA"] + self.assertTrue(after.must_flatten, "an ambiguous-only later response must not erase the earlier " + "confirmed record -- the intraday exit signal must survive") + self.assertTrue(after.needs_attention) + def test_timeout_processed_just_after_a_committed_success_is_ignored(self): """NIT finding 9 (fix round 4): reproduces the review's own Q5 probe -- refresh_timed_out() firing for the CURRENT generation diff --git a/tests/test_adaptive_paper_runner.py b/tests/test_adaptive_paper_runner.py index 8e3092ca7..704687179 100644 --- a/tests/test_adaptive_paper_runner.py +++ b/tests/test_adaptive_paper_runner.py @@ -3237,6 +3237,73 @@ async def scenario(): asyncio.run(scenario()) + def test_scheduled_refresh_timeout_flags_the_symbol_and_retries_after_60s_not_900s(self): + """LOW finding (fix round 6, test gap): mutation G4 (drop the + scheduler's call to `guard.refresh_timed_out(generation, watch)` + for a two-phase guard) survived every existing short-timeout test, + because they all drive a LEGACY guard exposing only the single-arg + `refresh_timed_out(symbols)` convention -- never the REAL + `CorporateActionMonitor`'s own generation-aware two-phase timeout + path. Drives the real monitor through `_schedule_corporate_action_ + refresh` with an artificially short timeout, and asserts (a) the + symbol is actually flagged needs_attention right after the + timeout, and (b) the next attempt is retried on the SHORT + `retry_seconds` (60s) interval, not the ordinary long + `refresh_seconds` (900s) one -- proving `refresh_timed_out` (not + merely `invalidate_attempt`, which does not set `_last_attempt_ + failed`) is what actually ran.""" + import asyncio + import threading + import time as time_module + from datetime import date + import corporate_actions as CA + import runner as runner_module + + release = threading.Event() + + class HangingSrc: + def fetch(self, symbols, start, end): + release.wait(5) + return ({s: [] for s in symbols}, set()) + + monitor = CA.CorporateActionMonitor(HangingSrc(), refresh_seconds=900, retry_seconds=60, + clock=lambda: time_module.time()) + + async def scenario(): + state = {"in_flight": False} + original_timeout = runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS + runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = 0.05 + now0 = time_module.time() + try: + await runner_module._schedule_corporate_action_refresh( + monitor, {"AAPL"}, date(2026, 9, 17), date(2026, 12, 24), now0, state) + for _ in range(50): + if not state["in_flight"]: + break + await asyncio.sleep(0.02) + self.assertFalse(state["in_flight"]) + finally: + runner_module.CORPORATE_ACTION_REFRESH_TIMEOUT_SECONDS = original_timeout + decision = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"AAPL"}, candidate_symbols=set(), now=now0 + 1)["AAPL"] + self.assertTrue(decision.needs_attention, "the real monitor's generation-aware refresh_timed_out " + "must actually run and flag the symbol") + # Still inside the SHORT retry_seconds=60 window -- a second + # attempt must still be throttled. + gen_throttled = monitor.begin_attempt({"AAPL"}, start=date(2026, 9, 17), end=date(2026, 12, 24), + now=now0 + 30) + self.assertIsNone(gen_throttled, "must not yet retry within the short retry_seconds window") + # Past retry_seconds=60 -- a fresh attempt must now be allowed, + # proving the SHORT interval applied (not the ordinary long + # refresh_seconds=900). + gen_ready = monitor.begin_attempt({"AAPL"}, start=date(2026, 9, 17), end=date(2026, 12, 24), + now=now0 + 61) + self.assertIsNotNone(gen_ready, "must retry on the short retry_seconds=60 interval after a " + "timeout, not the ordinary long refresh_seconds=900 one") + release.set() + + asyncio.run(scenario()) + def test_hung_refresh_does_not_delay_process_exit(self): """LOW finding 6 (fix round 4): `asyncio.run()`'s own cleanup joins every outstanding `asyncio.to_thread`/default-executor thread @@ -3473,7 +3540,21 @@ async def scenario(): await runner_module._schedule_corporate_action_refresh( monitor, {"SPY"}, date(2026, 9, 17), date(2026, 12, 24), 1900.0, state) await asyncio.sleep(0.1) # let the worker actually enter fetch() and block on `release` + with monitor._lock: + generation_before_cancel = monitor._generation await runner_module._shutdown_corporate_action_tasks(state) + # LOW finding (fix round 6, G5 mutation gap): checking only the + # END STATE (needs_attention stays False) cannot distinguish a + # real invalidate_attempt call from a no-op cancellation that + # happens to leave the same clean state anyway, because the + # delayed fetch itself also returns a clean result -- assert + # the generation counter was ACTUALLY bumped past the + # cancelled attempt, proving invalidate_attempt really ran. + with monitor._lock: + generation_after_cancel = monitor._generation + self.assertGreater(generation_after_cancel, generation_before_cancel, + "cancellation must actually invalidate (bump past) the in-flight attempt's " + "own generation, not silently do nothing") decisions = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), held_symbols={"SPY"}, candidate_symbols=set(), now=1900.0) self.assertFalse(decisions["SPY"].needs_attention, @@ -3520,6 +3601,119 @@ async def scenario(): asyncio.run(scenario()) + def test_thread_start_failure_still_resolves_the_real_monitors_attempt(self): + """LOW finding (fix round 6): reproduces an independent review's + own probe (S3) against the REAL `CorporateActionMonitor` -- when + `Thread.start()` itself raises, the pre-allocated generation must + not be left permanently unresolved (never committed, never failed, + never invalidated). Without `fail_attempt` being called on this + path, `_last_attempt_failed` stays False and the symbol is never + marked degraded, so a genuinely failed launch would be silently + retried on the ordinary long `refresh_seconds` (900s) interval + with nothing ever flagged, instead of the short `retry_seconds` + (60s) one.""" + import asyncio + import threading + import time as time_module + from datetime import date + import corporate_actions as CA + import runner as runner_module + + class Src: + calls = 0 + + def fetch(self, symbols, start, end): + Src.calls += 1 + return ({s: [] for s in symbols}, set()) + + monitor = CA.CorporateActionMonitor(Src(), refresh_seconds=900, retry_seconds=60, + clock=lambda: time_module.time()) + t0 = time_module.time() + monitor.refresh({"SPY"}, start=date(2026, 9, 17), end=date(2026, 12, 24), now=t0 - 1000) # earlier clean success + + real_start = threading.Thread.start + + def boom(self): + if self.name == "ca-refresh": + raise RuntimeError("can't start new thread") + return real_start(self) + + async def scenario(): + state = {"in_flight": False} + threading.Thread.start = boom + try: + task = await runner_module._schedule_corporate_action_refresh( + monitor, {"SPY"}, date(2026, 9, 17), date(2026, 12, 24), t0, state) + await asyncio.gather(task, return_exceptions=True) + finally: + threading.Thread.start = real_start + self.assertFalse(state["in_flight"], "in_flight must still be reset") + with monitor._lock: + generation, committed, failed = (monitor._generation, monitor._committed_generation, + monitor._last_attempt_failed) + self.assertEqual(generation, committed, + "the failed launch's generation must be resolved (committed as a failure), " + "not left dangling ahead of _committed_generation forever") + self.assertTrue(failed, "a failed launch must be recorded as a failure, enabling the short " + "retry_seconds interval") + decision = monitor.evaluate(today=date(2026, 9, 24), next_session_date=date(2026, 9, 25), + held_symbols={"SPY"}, candidate_symbols=set(), now=t0)["SPY"] + self.assertTrue(decision.needs_attention, + "the failed launch must have flagged SPY degraded right away") + # 30s later: still inside the SHORT retry_seconds=60 window -- + # a fresh attempt must still be throttled. + still_throttled = monitor.begin_attempt({"SPY"}, start=date(2026, 9, 17), end=date(2026, 12, 24), + now=t0 + 30) + self.assertIsNone(still_throttled, "must not yet retry within the short retry_seconds window") + # 60s later: past retry_seconds=60 -- a fresh attempt must now + # be allowed, proving the SHORT interval applied (the failure + # was actually recorded), not the ordinary long + # refresh_seconds=900 one. + next_task = await runner_module._schedule_corporate_action_refresh( + monitor, {"SPY"}, date(2026, 9, 17), date(2026, 12, 24), t0 + 61, state) + self.assertIsNotNone(next_task, "must retry on the short retry_seconds interval, not be " + "throttled for the ordinary long refresh_seconds=900") + if next_task is not None: + await next_task + + asyncio.run(scenario()) + + def test_second_preflight_on_an_already_warmed_monitor_does_not_raise(self): + """NIT (fix round 6): reproduces an independent review's own probe + (S4) -- `_schedule_corporate_action_refresh` returns `None`, not a + task, when the two-phase `begin_attempt` itself reports a + throttled no-op (an equivalent fresh result already cached for + this exact symbol set/window). `_preflight_corporate_action_ + refresh` used to unconditionally `await` that return value -- + `await None` raises TypeError -- so a SECOND preflight call while + the monitor is already warm (e.g. a resumed/retried startup path) + would crash instead of being the harmless no-op it should be.""" + import asyncio + import corporate_actions as CA + import runner as runner_module + + class Src: + def fetch(self, symbols, start, end): + return ({x: [] for x in symbols}, set()) + + class FakeLedger: + def positions(self): + return {} + + monitor = CA.CorporateActionMonitor(Src()) + config = {"_corporate_action_guard": monitor, "symbols": ["SPY"]} + state = {"in_flight": False} + + async def scenario(): + await runner_module._preflight_corporate_action_refresh( + config, {"overnight_holds": True}, FakeLedger(), state) + # The second call, within the same throttle interval, must be a + # harmless no-op -- not raise TypeError from `await None`. + await runner_module._preflight_corporate_action_refresh( + config, {"overnight_holds": True}, FakeLedger(), state) + + asyncio.run(scenario()) + def test_worker_thread_is_a_daemon(self): """LOW finding (fix round 5): the worker thread that runs the blocking fetch must be a DAEMON thread -- not provable by the From a90995ab409bce6bcce744d2815bf320ec8ea998 Mon Sep 17 00:00:00 2001 From: Scout Date: Thu, 24 Sep 2026 20:52:57 -0400 Subject: [PATCH 8/8] Construct the corporate-action source only for overnight-hold runs, lazily CI's validate jobs run the suite without alpaca-py, and 12 runner tests failed with ModuleNotFoundError: main() built AlpacaCorporateActionsSource, and so imported alpaca-py's data client, on every run, before its gate checks. That also contradicted the design (the guard acts only with overnight holds). - runner.main builds the monitor only when session_policy["overnight_holds"] is enabled; otherwise the guard is None. - AlpacaCorporateActionsSource imports and builds its alpaca-py client on first use (the _client property). A refused run, or an environment without alpaca-py, never needs the SDK. The session timeout and retry setup are unchanged and are applied when the client is built. Checks: CI-like system python gives 363 OK (runner, corporate actions, sessions). The pinned runtime with the network blocked gives 248 OK (corporate actions, runner) and 986 in the full suite before the manifest refresh. The manifest test and the six validators pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../adaptive-paper/corporate_actions.py | 23 +++++++++++++++---- .../us-equities/adaptive-paper/runner.py | 19 ++++++++------- .../adaptive-paper/source-hashes.json | 4 ++-- manifests/evidence.json | 10 ++++---- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/corporate_actions.py b/blueprints/us-equities/adaptive-paper/corporate_actions.py index 4b44b63d6..a0b12f01b 100644 --- a/blueprints/us-equities/adaptive-paper/corporate_actions.py +++ b/blueprints/us-equities/adaptive-paper/corporate_actions.py @@ -585,6 +585,19 @@ class AlpacaCorporateActionsSource: _MAX_PAGES = 5 def __init__(self, api_key, secret_key): + # alpaca-py's data client is imported and built on first use (the + # `_client` property below), not here: runner.main constructs this + # source for every overnight-holds run before its gate checks, and a + # process that never fetches (a refused run, or a test environment + # without alpaca-py) must not need alpaca-py installed. + self._api_key = api_key + self._secret_key = secret_key + self._client_instance = None + + @property + def _client(self): + if self._client_instance is not None: + return self._client_instance from alpaca.data.historical.corporate_actions import CorporateActionsClient # raw_data=True: this wrapper's own inherited `RESTClient.get()` # (which fetch() below calls directly) already returns the plain @@ -603,7 +616,7 @@ def __init__(self, api_key, secret_key): # `partial_calls`, `capital_gains_distributions` -- is now moot for # fetch() specifically, since it never goes through # CorporateActionsSet at all. - self._client = CorporateActionsClient(api_key, secret_key, raw_data=True) + client = CorporateActionsClient(self._api_key, self._secret_key, raw_data=True) # HIGH finding 2: alpaca-py's RESTClient (verified against the # installed alpaca-py==0.44.0 RESTClient.__init__/_request) sets no # per-request HTTP timeout at all, and sleeps `_retry_wait` (default @@ -623,7 +636,7 @@ def __init__(self, api_key, secret_key): # past that point (see that function's own docstring and # CorporateActionMonitor's generation check, which is what makes a # late write from such a call harmless rather than "bounded"). - session = self._client._session + session = client._session original_request = session.request def _bounded_request(method, url, **kwargs): @@ -631,8 +644,10 @@ def _bounded_request(method, url, **kwargs): return original_request(method, url, **kwargs) session.request = _bounded_request - self._client._retry = 1 - self._client._retry_wait = 1 + client._retry = 1 + client._retry_wait = 1 + self._client_instance = client + return client @staticmethod def _parse_date(value): diff --git a/blueprints/us-equities/adaptive-paper/runner.py b/blueprints/us-equities/adaptive-paper/runner.py index 388056592..519a3fb3e 100644 --- a/blueprints/us-equities/adaptive-paper/runner.py +++ b/blueprints/us-equities/adaptive-paper/runner.py @@ -1906,14 +1906,17 @@ def main(): print(json.dumps(result)) return 2 key, secret = credentials(args.env_file) - # Trading-lane audit gap #8: constructed unconditionally (object - # construction only, no network call -- see AlpacaCorporateActionsSource's - # docstring) so it is available to run_native's AdaptiveStrategy below - # for both `paper` and a future recovery-time consultation; inert until - # its first evaluate()/refresh() call. Never logs or stores key/secret - # beyond the client object; the same env-file credential path every - # other broker call in this file already uses. - config["_corporate_action_guard"] = CorporateActionMonitor(AlpacaCorporateActionsSource(key, secret)) + # Trading-lane audit gap #8: the guard only ever acts when + # session_policy["overnight_holds"] is enabled (see + # _corporate_action_guard_for_strategy), so it is constructed only then. + # Construction makes no network call, but AlpacaCorporateActionsSource + # imports alpaca-py's data client; a regular-session run (every shipped + # config) therefore never needs that import or the object. Never logs or + # stores key/secret beyond the client object; the same env-file + # credential path every other broker call in this file already uses. + config["_corporate_action_guard"] = ( + CorporateActionMonitor(AlpacaCorporateActionsSource(key, secret)) + if session_policy["overnight_holds"] else None) attempts, responses = [], [] def observe_request(kind, **kwargs): attempts.append({"timestamp": time.time(), "kind": kind}) diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index ffaa93632..d74ac981f 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -7,7 +7,7 @@ "blueprints/us-equities/adaptive-paper/config-mover.json": "560849b980082a72f8ec45a32ffae51b1f50a502e739c94346ff501a46777ea8", "blueprints/us-equities/adaptive-paper/config-sip.json": "3587653104e68fdfc9bbb9170825c611abcf7e2e1be1004327a056031c34555a", "blueprints/us-equities/adaptive-paper/config.json": "2b555982034af02b9d0d2e1d20e17d22c1e83f2b0c4d60b61d91307ebec63562", - "blueprints/us-equities/adaptive-paper/corporate_actions.py": "f4afc9eef377d407b3a4d8df29cd7d71d43b09cb18768d911324bdd433d6d5e7", + "blueprints/us-equities/adaptive-paper/corporate_actions.py": "8f0839427b9d8a3c9b3fbd02f286985c7a720bf85a3c599fce2022573bff71cd", "blueprints/us-equities/adaptive-paper/credential_guard.py": "f5a3d0d038ed3f01d486493ce416395039fb260762232785bb0a9073c777da7e", "blueprints/us-equities/adaptive-paper/exits.py": "189133e6df99bda201d73d36a15f45ccbfb38979a3319ae6f75d077f5db5b3df", "blueprints/us-equities/adaptive-paper/feeds.py": "0db3009a724ae2433572c5070066d6806d7365031f9db4e81c4b69ad1cbebacb", @@ -25,7 +25,7 @@ "blueprints/us-equities/adaptive-paper/recovery.py": "5cf7dcea572efbda6979a2726eebf6d676f6bd62263900c7b6626da8b45c1215", "blueprints/us-equities/adaptive-paper/registry.json": "60da73b985e718bd1dece7a3350f34636bda0fb38e766a44ae71c5a8c532282a", "blueprints/us-equities/adaptive-paper/requirements.txt": "518afe4b77f901e87abd3a5d6faa126aa7e211db77eecec80cfe7ef42d82e07a", - "blueprints/us-equities/adaptive-paper/runner.py": "4a77b3e0a6c8835a69751fd85dfea1449a52435801b3d0c3f9096df784d3b137", + "blueprints/us-equities/adaptive-paper/runner.py": "fd232594802755c4c16162485fd1fd6c19b7f498ce96ffebf090f7dada5149d2", "blueprints/us-equities/adaptive-paper/safety.py": "ad520fc4d84ce10e4d7b5cc956ba769a4f94eef49464ce382746d57f0cd76420", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", diff --git a/manifests/evidence.json b/manifests/evidence.json index 7d17fad60..e3bad4520 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -6525,8 +6525,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/corporate_actions.py", - "sha256": "f4afc9eef377d407b3a4d8df29cd7d71d43b09cb18768d911324bdd433d6d5e7", - "bytes": 49424 + "sha256": "8f0839427b9d8a3c9b3fbd02f286985c7a720bf85a3c599fce2022573bff71cd", + "bytes": 50070 }, { "path": "blueprints/us-equities/adaptive-paper/credential_guard.py", @@ -6660,8 +6660,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/runner.py", - "sha256": "4a77b3e0a6c8835a69751fd85dfea1449a52435801b3d0c3f9096df784d3b137", - "bytes": 130962 + "sha256": "fd232594802755c4c16162485fd1fd6c19b7f498ce96ffebf090f7dada5149d2", + "bytes": 131096 }, { "path": "blueprints/us-equities/adaptive-paper/safety.py", @@ -6685,7 +6685,7 @@ }, { "path": "blueprints/us-equities/adaptive-paper/source-hashes.json", - "sha256": "7e1c52c89dde91a93d5d681e672991fd952c6e1ce8d39fe37aed02eb513ade9f", + "sha256": "6b79220c3d16e9a0a85b5a6ada999df3d338dcb58e5c5358e07d47fa121ca0eb", "bytes": 6481 }, {