From 95e481b05f52037b0b47137d350d70462f62acba Mon Sep 17 00:00:00 2001 From: Scout Date: Wed, 30 Sep 2026 11:06:23 -0400 Subject: [PATCH 1/7] Adaptive paper: failing-first tests for broker FEE activities Account 2's recover attempts on 2026-09-30 ended cash_mismatch_or_unmodeled_fees: three paper FEE activities for 2026-09-29 sells (REG -0.19, TAF -0.27, CAT -0.01) that the engine does not model. tests/test_adaptive_paper_fees.py covers the allow-listed GET /v2/account/activities/FEE read, normalize_fee_activity, the snapshot's fee list, the schema-3 ledger fee record, reconcile, runner.main's next-trial check, each lane's fee window and the mover receipts. At this commit 35 of its 38 tests fail; three regression guards pass. Co-Authored-By: Claude Opus 5.5 --- tests/test_adaptive_paper_fees.py | 942 ++++++++++++++++++++++++++++++ 1 file changed, 942 insertions(+) create mode 100644 tests/test_adaptive_paper_fees.py diff --git a/tests/test_adaptive_paper_fees.py b/tests/test_adaptive_paper_fees.py new file mode 100644 index 000000000..61664b2fd --- /dev/null +++ b/tests/test_adaptive_paper_fees.py @@ -0,0 +1,942 @@ +"""Broker FEE activities in the adaptive-paper engine (trading-lane brief, 2026-09-30). + +Every account 2 recover attempt ended cash_mismatch_or_unmodeled_fees: three paper FEE +activities for 2026-09-29 sells (REG -0.19, TAF -0.27, CAT -0.01 USD) the engine did not +model. These tests cover the allow-listed transport read and its normalization, the +snapshot's fee list, the ledger's schema-3 fee record, reconciliation, runner.main's +next-trial check, each lane's fee window and the mover receipts. + +Local synthetic fixtures only: no credentials, broker requests or network. Activity ids +are built at runtime, as in test_adaptive_paper_transport.py (no literal UUID in +published files). +""" +from __future__ import annotations + +import asyncio +from contextlib import nullcontext +from datetime import datetime, timezone +from decimal import Decimal as D +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import sys +import tempfile +import time +import unittest +from unittest.mock import Mock, patch +import uuid + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "blueprints/us-equities/adaptive-paper" +sys.path.insert(0, str(SOURCE)) + +import mover_runner # noqa: E402 +import runner as runner_module # noqa: E402 +import safety # noqa: E402 +import transport as t # noqa: E402 +from safety import Ledger, RiskLimits, SafetyError # noqa: E402 + +try: + import alpaca # noqa: F401 + import requests + HAS_SDK = True +except ImportError: + HAS_SDK = False + +try: + import nautilus_trader # noqa: F401 + import mover + import native_adapter + from mover_simulation import MoverSimulatedPort, piecewise_path + NATIVE = HAS_SDK +except ImportError: + NATIVE = False + +try: # package mode (python -m unittest tests.x) or discover -s tests (top-level modules) + from .adaptive_paper_hermetic import patch_default_stop, restore_default_stop +except ImportError: + from adaptive_paper_hermetic import patch_default_stop, restore_default_stop # noqa: E402 + +_HERMETIC_TOKEN = None + + +def setUpModule(): + global _HERMETIC_TOKEN + _HERMETIC_TOKEN = patch_default_stop() + + +def tearDownModule(): + restore_default_stop(_HERMETIC_TOKEN) + + +# The description of a FEE activity carries the account number; this stands in for it. +ACCOUNT_TEXT = "fixture-description-with-the-account-number" +# The three FEE activities measured on account 2 for 2026-09-29 sells (brief, 2026-09-30). +MEASURED = (("REG", "-0.19"), ("TAF", "-0.27"), ("CAT", "-0.01")) +# An instant of the adaptive lane's first trial (trial.json started_at). +T0 = 1_790_000_000.0 + + +def activity_id(index, stamp="20260930040000"): + """A documented "::" activity id, built at runtime.""" + return "%s%03d::%s" % (stamp, index, uuid.UUID(int=0xFEE0000 + index)) + + +def fee_row(index=0, sub_type="REG", amount="-0.19", **changes): + """One raw NonTradeActivity of activity_type FEE, shaped like the by-type reference's example.""" + row = {"activity_type": "FEE", "activity_sub_type": sub_type, "id": activity_id(index), + "date": "2026-09-29", "net_amount": amount, "currency": "USD", "status": "executed", + "created_at": "2026-09-30T04:00:00.123456Z", "description": ACCOUNT_TEXT} + row.update(changes) + return row + + +def measured_rows(): + return [fee_row(i, sub_type, amount) for i, (sub_type, amount) in enumerate(MEASURED)] + + +def measured_fees(): + """What transport.normalize_fee_activity returns for measured_rows().""" + return [{"id": activity_id(i), "date": "2026-09-29", "net_amount": amount, "sub_type": sub_type} + for i, (sub_type, amount) in enumerate(MEASURED)] + + +def response(payload=None, status=200): + value = requests.Response() + value.status_code = status + value._content = json.dumps(payload).encode() if payload is not None else b"" + return value + + +def utc(epoch): + return datetime.fromtimestamp(epoch, timezone.utc) + + +# --------------------------------------------------------------------------- +# Item 2: normalization +# --------------------------------------------------------------------------- + +class FeeNormalization(unittest.TestCase): + def test_documented_fee_row_keeps_four_fields_and_never_the_description(self): + for row, expected in zip(measured_rows(), measured_fees()): + with self.subTest(sub_type=expected["sub_type"]): + result = t.normalize_fee_activity(row) + self.assertEqual(result, expected) + self.assertNotIn(ACCOUNT_TEXT, json.dumps(result)) + + def test_every_documented_fee_sub_type_and_an_absent_or_null_one(self): + for sub_type in ("REG", "TAF", "LCT", "ORF", "OCC", "NRC", "NRV", "COM", "CAT"): + with self.subTest(sub_type=sub_type): + self.assertEqual(t.normalize_fee_activity(fee_row(sub_type=sub_type))["sub_type"], sub_type) + absent = fee_row() + del absent["activity_sub_type"] + self.assertEqual(t.normalize_fee_activity(absent)["sub_type"], "UNSPECIFIED") + self.assertEqual(t.normalize_fee_activity(fee_row(sub_type=None))["sub_type"], "UNSPECIFIED") + + def test_currency_and_status_may_be_absent(self): + row = fee_row() + del row["currency"], row["status"], row["description"], row["created_at"] + self.assertEqual(t.normalize_fee_activity(row), measured_fees()[0]) + + def test_a_credit_and_trailing_zeros(self): + self.assertEqual(t.normalize_fee_activity(fee_row(amount="0.05"))["net_amount"], "0.05") + self.assertEqual(t.normalize_fee_activity(fee_row(amount="-0.270"))["net_amount"], "-0.27") + + def test_anything_else_fails_closed_without_provider_text(self): + def without(key): + row = fee_row() + del row[key] + return row + cases = { + "div": fee_row(activity_type="DIV"), "csw": fee_row(activity_type="CSW"), + "fill": fee_row(activity_type="FILL"), "lowercase_type": fee_row(activity_type="fee"), + "no_type": without("activity_type"), + "bad_id": fee_row(id="20260930::not-a-uuid"), "uuid_only_id": fee_row(id=str(uuid.UUID(int=7))), + "int_id": fee_row(id=7), "no_id": without("id"), + "text_amount": fee_row(amount="abc"), "nan": fee_row(amount="NaN"), "infinite": fee_row(amount="-Infinity"), + "empty_amount": fee_row(amount=""), "null_amount": fee_row(amount=None), "float_amount": fee_row(amount=-0.19), + "bool_amount": fee_row(amount=True), "no_amount": without("net_amount"), + "timestamp_date": fee_row(date="2026-09-29T00:00:00Z"), "slash_date": fee_row(date="2026/09/29"), + "impossible_date": fee_row(date="2026-02-30"), "null_date": fee_row(date=None), "no_date": without("date"), + "unknown_sub_type": fee_row(sub_type="XYZ"), "lowercase_sub_type": fee_row(sub_type="reg"), + "dividend_sub_type": fee_row(sub_type="CDIV"), "numeric_sub_type": fee_row(sub_type=5), + "empty_sub_type": fee_row(sub_type=""), + "euro": fee_row(currency="EUR"), "null_currency": fee_row(currency=None), + "correction": fee_row(status="correct"), "canceled": fee_row(status="canceled"), + "null_status": fee_row(status=None), + "not_a_dict": ["FEE"], "none": None} + for label, row in cases.items(): + with self.subTest(label=label): + with self.assertRaises(t.TransportError) as caught: + t.normalize_fee_activity(row) + self.assertNotIn(ACCOUNT_TEXT, str(caught.exception)) + + +# --------------------------------------------------------------------------- +# Item 1: the one allow-listed read +# --------------------------------------------------------------------------- + +AFTER = "2026-09-29T08:00:00Z" + + +@unittest.skipUnless(HAS_SDK, "requires isolated reviewed alpaca-py runtime") +class FeeHTTPBoundary(unittest.TestCase): + def setUp(self): + self.budget = Mock(return_value=None) + self.session = t.GuardedSession(origin=t.PAPER_URL, before_request=self.budget) + self.addCleanup(self.session.close) + + def test_the_fee_read_is_allowed_only_after_one_utc_instant_and_is_a_budgeted_read(self): + url = t.PAPER_URL + t.ACTIVITY_FEE_PATH + self.assertEqual(t.ACTIVITY_FEE_PATH, "/v2/account/activities/FEE") + allowed = [{"after": AFTER, "direction": "asc"}, + {"after": AFTER, "direction": "asc", "page_size": 100}, + {"after": AFTER, "direction": "asc", "page_size": 1, "page_token": activity_id(1)}] + with patch.object(self.session._session, "request", return_value=response([])) as request: + for params in allowed: + self.session.request("GET", url, params=params) + self.assertEqual(request.call_count, 3) + self.assertEqual([c.args[0] for c in self.budget.call_args_list], ["read"] * 3) + + def test_every_other_path_parameter_or_method_is_refused_before_the_budget(self): + url = t.PAPER_URL + t.ACTIVITY_FEE_PATH + good = {"after": AFTER, "direction": "asc"} + refused = [ + ("GET", t.PAPER_URL + "/v2/account/activities", good), + ("GET", t.PAPER_URL + "/v2/account/activities/DIV", good), + ("GET", t.PAPER_URL + "/v2/account/activities/CSW", good), + ("GET", t.PAPER_URL + "/v2/account/activities/fee", good), + ("GET", url + "/", good), + ("GET", t.PAPER_URL + t.ACTIVITY_FILL_PATH, good), + ("GET", url + "?after=" + AFTER + "&direction=asc", None), + ("GET", url, None), ("GET", url, {}), + ("GET", url, {"direction": "asc"}), + ("GET", url, {"after": AFTER}), + ("GET", url, {**good, "after": "2026-09-29"}), + ("GET", url, {**good, "after": "2026-09-29T08:00:00.5Z"}), + ("GET", url, {**good, "after": "2026-09-29T08:00:00+00:00"}), + ("GET", url, {**good, "after": "2026-09-29t08:00:00z"}), + ("GET", url, {**good, "after": "2026-13-40T25:61:61Z"}), + ("GET", url, {**good, "after": 1790000000}), + ("GET", url, {**good, "direction": "desc"}), + ("GET", url, {**good, "page_size": 0}), ("GET", url, {**good, "page_size": 101}), + ("GET", url, {**good, "page_size": "100"}), ("GET", url, {**good, "page_size": True}), + ("GET", url, {**good, "page_token": "not::a-token"}), ("GET", url, {**good, "page_token": 5}), + ("GET", url, {**good, "until": AFTER}), ("GET", url, {**good, "date": "2026-09-30"}), + ("GET", url, {**good, "order_id": str(uuid.UUID(int=1))}), + ("GET", url, {**good, "activity_types": "FEE"}), + ("GET", url, {**good, "category": "non_trade_activity"}), + ("POST", url, good), ("DELETE", url, good)] + with patch.object(self.session._session, "request") as request: + for method, target, params in refused: + with self.subTest(method=method, url=target, params=params), self.assertRaises(t.TransportError): + self.session.request(method, target, params=params) + request.assert_not_called() + self.budget.assert_not_called() + + +# --------------------------------------------------------------------------- +# Item 3: the snapshot's fee list (and the standalone read item 7 uses) +# --------------------------------------------------------------------------- + +HISTORY = datetime(2026, 9, 28, 12, 0, 0, 250000, tzinfo=timezone.utc) # the lane's first trial +BASELINE = datetime(2026, 9, 29, 8, 0, 0, 750000, tzinfo=timezone.utc) # this trial's cash baseline +FEE_PATH = "/v2/account/activities/FEE" + + +@unittest.skipUnless(HAS_SDK, "requires isolated reviewed alpaca-py runtime") +class FeeSnapshot(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.budgets = [] + self.port = self.make_port(fee_history_start=BASELINE) + + async def asyncTearDown(self): + await self.port.stop() + + def make_port(self, **kwargs): + def budget(kind, client_id=None): + self.budgets.append(kind) + port = t.AlpacaPaperTransport("fixture-key", "fixture-secret", ["SPY"], before_request=budget, + before_submit=lambda intent: None, sink_observation=lambda order: None, + history_start=HISTORY, **kwargs) + port._loop = asyncio.get_running_loop() + return port + + async def replace_port(self, **kwargs): + await self.port.stop() + self.port = self.make_port(**kwargs) + + def serve(self, fee_pages): + pages, calls = list(fee_pages), [] + + def request(method, url, **kwargs): + path = t.urlsplit(url).path + calls.append((method, path, kwargs.get("params"))) + if path == "/v2/account": + return response({"cash": "1000", "equity": "1000", "buying_power": "1000"}) + if path in ("/v2/positions", "/v2/orders"): + return response([]) + if path == FEE_PATH: + page = pages.pop(0) + return page if isinstance(page, requests.Response) else response(page) + raise AssertionError("unexpected request " + path) + return request, calls + + async def snapshot(self, fee_pages): + request, calls = self.serve(fee_pages) + with patch.object(self.port._client._session._session, "request", side_effect=request): + return await self.port.snapshot(), calls + + async def test_snapshot_lists_the_fees_created_after_the_fee_window_oldest_first(self): + snapshot, calls = await self.snapshot([measured_rows()]) + self.assertEqual(snapshot["fees"], measured_fees()) + self.assertEqual([c for c in calls if c[1] == FEE_PATH], + [("GET", FEE_PATH, {"after": "2026-09-29T08:00:00Z", "direction": "asc", "page_size": 100})]) + # The order history keeps its own window; the fees are read last, after the account's cash. + self.assertEqual([c[2]["after"] for c in calls if c[1] == "/v2/orders" and c[2].get("status") == "all"], + [HISTORY.isoformat()]) + self.assertEqual(calls[-1][1], FEE_PATH) + self.assertEqual(set(self.budgets), {"read"}) + self.assertEqual(snapshot["fee_history_start"], BASELINE.isoformat()) + self.assertNotIn(ACCOUNT_TEXT, json.dumps(snapshot, default=str)) + + async def test_the_fee_window_defaults_to_the_history_start(self): + await self.replace_port() + snapshot, calls = await self.snapshot([[]]) + self.assertEqual(snapshot["fees"], []) + self.assertEqual([c[2]["after"] for c in calls if c[1] == FEE_PATH], ["2026-09-28T12:00:00Z"]) + + async def test_fee_pages_continue_from_the_last_activity_id(self): + first = [fee_row(i, "TAF", "-0.01") for i in range(100)] + second = [fee_row(100, "CAT", "-0.01")] + snapshot, calls = await self.snapshot([first, second]) + self.assertEqual(len(snapshot["fees"]), 101) + self.assertEqual([f["id"] for f in snapshot["fees"]], [row["id"] for row in first + second]) + self.assertEqual([c[2] for c in calls if c[1] == FEE_PATH][1], + {"after": "2026-09-29T08:00:00Z", "direction": "asc", "page_size": 100, + "page_token": first[-1]["id"]}) + + async def test_a_full_page_at_the_bound_is_incomplete_not_an_empty_list(self): + self.port.max_snapshot_pages = 1 + request, _ = self.serve([[fee_row(i, "TAF", "-0.01") for i in range(100)]]) + with patch.object(self.port._client._session._session, "request", side_effect=request): + with self.assertRaisesRegex(t.TransportError, "snapshot incomplete"): + await self.port.snapshot() + self.assertIn("snapshot_incomplete", self.port.health["reasons"]) + + async def test_a_failed_or_malformed_fee_read_freezes_instead_of_listing_no_fees(self): + cases = {"http_500": [response({"code": 50010000, "message": "fixture"}, 500)], + "not_a_list": [{"activities": []}], + "dividend_row": [[fee_row(activity_type="DIV")]], + "withdrawal_row": [[fee_row(activity_type="CSW")]], + "duplicate_id": [[fee_row(0), fee_row(0)]], + "repeated_across_pages": [[fee_row(i, "TAF", "-0.01") for i in range(100)], [fee_row(0)]], + "oversized_page": [[fee_row(i, "TAF", "-0.01") for i in range(101)]]} + for label, pages in cases.items(): + with self.subTest(label=label): + await self.replace_port(fee_history_start=BASELINE) + request, _ = self.serve(pages) + with patch.object(self.port._client._session._session, "request", side_effect=request): + with self.assertRaisesRegex(t.TransportError, "snapshot incomplete"): + await self.port.snapshot() + self.assertIn("snapshot_incomplete", self.port.health["reasons"]) + + async def test_a_naive_fee_history_start_is_refused(self): + with self.assertRaises(t.TransportError): + self.make_port(fee_history_start=datetime(2026, 9, 29, 8, 0)) + + async def test_fee_activities_reads_through_a_fresh_allow_listed_session(self): + budgets = [] + request, calls = self.serve([measured_rows()]) + with patch("requests.Session.request", side_effect=request): + fees = t.fee_activities("fixture-key", "fixture-secret", after=BASELINE, + before_request=lambda kind, **kwargs: budgets.append(kind)) + self.assertEqual(fees, measured_fees()) + self.assertEqual(budgets, ["read"]) + self.assertEqual(calls, [("GET", FEE_PATH, {"after": "2026-09-29T08:00:00Z", "direction": "asc", + "page_size": 100})]) + with patch("requests.Session.request") as sent: + with self.assertRaises(t.TransportError): + t.fee_activities("fixture-key", "fixture-secret", after=datetime(2026, 9, 29), + before_request=budgets.append) + sent.assert_not_called() + + +# --------------------------------------------------------------------------- +# Item 5: the ledger (schema 3) +# --------------------------------------------------------------------------- + +class FeeLedger(unittest.TestCase): + NOW = 1_790_000_000.0 + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) / "ledger.sqlite3" + self.ledger = Ledger(self.path) + self.addCleanup(lambda: self.ledger.close()) + + def reopen(self): + self.ledger.close() + self.ledger = Ledger(self.path) + + def dump(self): + return tuple(self.ledger.db.iterdump()) + + def schema_version(self): + return self.ledger.db.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone()[0] + + def has_fees_table(self, db=None): + return (db or self.ledger.db).execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='fees'").fetchone() is not None + + def test_a_new_ledger_is_schema_3_with_the_fees_table(self): + self.assertEqual(self.schema_version(), "3") + columns = [(r[1], r[2], r[3], r[5]) for r in self.ledger.db.execute("PRAGMA table_info(fees)")] + self.assertEqual(columns, [("activity_id", "TEXT", 0, 1), ("date", "TEXT", 1, 0), + ("net_amount", "TEXT", 1, 0), ("sub_type", "TEXT", 1, 0), + ("recorded_at", "REAL", 1, 0)]) + + def test_schema_2_1_and_unversioned_ledgers_migrate_to_3(self): + for version in ("2", "1", None): + with self.subTest(version=version): + self.ledger.db.execute("DROP TABLE fees") + if version is None: + self.ledger.db.execute("DELETE FROM meta WHERE key='schema_version'") + else: + self.ledger.db.execute("UPDATE meta SET value=? WHERE key='schema_version'", (version,)) + if version in ("1", None): + self.ledger.db.execute("ALTER TABLE executions DROP COLUMN execution_time_ns") + self.reopen() + self.assertEqual(self.schema_version(), "3") + self.assertTrue(self.has_fees_table()) + self.reopen() # a migrated ledger opens unchanged + self.assertEqual(self.schema_version(), "3") + self.assertEqual(self.ledger.record_fees(measured_fees(), self.NOW), measured_fees()) + + def test_any_other_schema_version_is_refused_before_any_change(self): + for version in ("4", "0", "3.0", ""): + with self.subTest(version=version): + self.ledger.db.execute("DROP TABLE IF EXISTS fees") + self.ledger.db.execute("UPDATE meta SET value=? WHERE key='schema_version'", (version,)) + self.ledger.close() + with self.assertRaisesRegex(SafetyError, "^unsupported_ledger_schema$"): + Ledger(self.path) + with sqlite3.connect(self.path) as raw: + self.assertFalse(self.has_fees_table(raw)) + self.assertEqual(raw.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone()[0], + version) + raw.execute("UPDATE meta SET value='2' WHERE key='schema_version'") + raw.close() + self.ledger = Ledger(self.path) + + def test_the_previous_engine_refuses_a_migrated_ledger(self): + # The exact open check of Ledger.__init__ at 11227bfd (this change's base), before any insert. + version = self.schema_version() + with self.assertRaisesRegex(SafetyError, "^unsupported_ledger_schema$"): + if version not in (None, "1", "2"): + raise SafetyError("unsupported_ledger_schema") + + def test_record_fees_books_cash_realized_and_loss_once_with_one_event_each(self): + self.assertEqual(self.ledger.record_fees(measured_fees(), self.NOW), measured_fees()) + state = self.ledger.accounting() + self.assertEqual((state.cash_delta_usd, state.realized_pnl_usd, state.cumulative_realized_loss_usd, + state.gross_loss_usd, state.peak_pnl_usd, state.drawdown_usd), + (D("-0.47"), D("-0.47"), D("0.47"), D("0.47"), D("0"), D("0.47"))) + self.assertEqual([(r["activity_id"], r["date"], r["net_amount"], r["sub_type"], r["recorded_at"]) + for r in self.ledger.fees()], + [(f["id"], f["date"], D(f["net_amount"]), f["sub_type"], self.NOW) for f in measured_fees()]) + events = [json.loads(r[0]) for r in self.ledger.db.execute( + "SELECT payload FROM events WHERE kind='fee_recorded' ORDER BY id")] + self.assertEqual(events, [{"activity_id": f["id"], "date": f["date"], "net_amount": f["net_amount"], + "sub_type": f["sub_type"]} for f in measured_fees()]) + + def test_re_recording_known_fees_changes_nothing(self): + self.ledger.record_fees(measured_fees(), self.NOW) + before = self.dump() + self.assertEqual(self.ledger.record_fees(measured_fees(), self.NOW + 60), []) + self.assertEqual(self.ledger.record_fees(list(reversed(measured_fees()))[:2], self.NOW + 120), []) + same_value = dict(measured_fees()[0], net_amount="-0.190") + self.assertEqual(self.ledger.record_fees([same_value], self.NOW + 150), []) + self.assertEqual(self.ledger.record_fees([], self.NOW + 180), []) + self.assertEqual(self.dump(), before) + + def test_a_changed_fee_activity_is_refused_and_the_whole_batch_rolls_back(self): + known, fresh = measured_fees()[:2] + self.ledger.record_fees([known], self.NOW) + before = self.dump() + for change in ({"net_amount": "-0.20"}, {"date": "2026-09-30"}, {"sub_type": "TAF"}, + {"sub_type": "UNSPECIFIED"}): + with self.subTest(change=change): + with self.assertRaisesRegex(SafetyError, "^fee_activity_changed$"): + self.ledger.record_fees([fresh, dict(known, **change)], self.NOW + 60) + self.assertEqual(self.dump(), before) + + def test_a_fee_credit_raises_realized_and_the_peak_but_never_lowers_the_loss(self): + credit = {"id": activity_id(9), "date": "2026-09-29", "net_amount": "0.10", "sub_type": "UNSPECIFIED"} + self.ledger.record_fees([credit], self.NOW) + state = self.ledger.accounting() + self.assertEqual((state.realized_pnl_usd, state.cumulative_realized_loss_usd, state.peak_pnl_usd, + state.cash_delta_usd), (D("0.10"), D("0"), D("0.10"), D("0.10"))) + self.ledger.record_fees(measured_fees(), self.NOW + 1) + state = self.ledger.accounting() + self.assertEqual((state.realized_pnl_usd, state.cumulative_realized_loss_usd, state.peak_pnl_usd, + state.drawdown_usd), (D("-0.37"), D("0.47"), D("0.10"), D("0.47"))) + + def test_fees_count_against_the_gross_loss_budget(self): + ledger = Ledger(Path(self.tmp.name) / "tight.sqlite3", RiskLimits(max_gross_loss_usd=D("0.40"))) + self.addCleanup(ledger.close) + self.assertIsNone(ledger.halted_reason()) + ledger.record_fees(measured_fees(), self.NOW) + self.assertEqual(ledger.halted_reason(), "gross_loss_cap_reached") + + def test_malformed_fee_rows_are_refused_before_any_write(self): + before = self.dump() + good = measured_fees()[0] + cases = {"not_a_list": good, "none": None, "text": "fees", "row_not_a_dict": [["x"]], + "missing_key": [{k: v for k, v in good.items() if k != "sub_type"}], + "description_key": [dict(good, description=ACCOUNT_TEXT)], + "bad_id": [dict(good, id="7")], "bad_date": [dict(good, date="2026-9-29")], + "impossible_date": [dict(good, date="2026-02-30")], "text_amount": [dict(good, net_amount="abc")], + "exponent_amount": [dict(good, net_amount="-1E-2")], "plus_amount": [dict(good, net_amount="+0.19")], + "ten_decimals": [dict(good, net_amount="-0.0000000001")], + "eleven_digits": [dict(good, net_amount="-10000000000")], + "float_amount": [dict(good, net_amount=-0.19)], "unknown_sub_type": [dict(good, sub_type="XYZ")], + "null_sub_type": [dict(good, sub_type=None)], + "one_bad_row_in_a_batch": [measured_fees()[1], dict(good, sub_type="CDIV")]} + for label, fees in cases.items(): + with self.subTest(label=label): + with self.assertRaises(SafetyError): + self.ledger.record_fees(fees, self.NOW) + self.assertEqual(self.dump(), before) + with self.assertRaises(SafetyError): + self.ledger.record_fees(measured_fees(), float("nan")) + self.assertEqual(self.dump(), before) + + def test_rederived_accounting_keeps_recorded_fees_next_to_symbol_money(self): + self.ledger.start_trial(self.NOW) + quote = safety.Quote("SPY", "100", "100.01", self.NOW) + self.ledger.reserve_intent("buy-1", "SPY", "buy", "1", "100.02", quote=quote, now=self.NOW, market_open=True, + session_close=self.NOW + 3600, stop_file=Path(self.tmp.name) / "STOP") + self.ledger.record_order("buy-1", "broker-buy-1", "filled", "1", "100", timestamp=self.NOW) + self.ledger.record_fees(measured_fees(), self.NOW + 1) + expected = (D("-100.47"), D("-0.47"), D("0.47")) + state = self.ledger.accounting() + self.assertEqual((state.cash_delta_usd, state.realized_pnl_usd, state.cumulative_realized_loss_usd), expected) + self.ledger.db.execute("DELETE FROM meta WHERE key='execution_accounting:rule'") + self.reopen() # the open-time re-derivation runs + state = self.ledger.accounting() + self.assertEqual((state.cash_delta_usd, state.realized_pnl_usd, state.cumulative_realized_loss_usd), expected) + + +# --------------------------------------------------------------------------- +# Items 6 and 8: reconciliation +# --------------------------------------------------------------------------- + +class FeeReconciliation(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.ledger = self.new_ledger("ledger") + + def new_ledger(self, name): + ledger = Ledger(Path(self.tmp.name) / (name + ".sqlite3")) + self.addCleanup(ledger.close) + return ledger + + @staticmethod + def snapshot(cash, fees=None): + value = {"complete": True, "orders": [], "positions": [], "account": {"cash": cash}} + if fees is not None: + value["fees"] = fees + return value + + def test_the_measured_gap_fails_without_fees_and_reconciles_with_the_three_rows(self): + for fees in (None, []): + with self.subTest(fees=fees), self.assertRaisesRegex(SafetyError, "^cash_mismatch_or_unmodeled_fees$"): + runner_module.reconcile(self.ledger, self.snapshot("99999.53", fees), "100000") + proof = runner_module.reconcile(self.ledger, self.snapshot("99999.53", measured_fees()), "100000") + self.assertEqual((proof["cash_match"], proof["cash_delta_usd"]), (True, "-0.47")) + self.assertEqual([f["activity_id"] for f in self.ledger.fees()], [f["id"] for f in measured_fees()]) + self.assertEqual(runner_module.reconcile(self.ledger, self.snapshot("99999.53", measured_fees()), "100000"), + proof) + self.assertEqual(len(self.ledger.fees()), 3) + self.assertEqual(self.ledger.accounting().cash_delta_usd, D("-0.47")) + + def test_a_snapshot_without_the_fees_key_behaves_as_before(self): + before = tuple(self.ledger.db.iterdump()) + proof = runner_module.reconcile(self.ledger, self.snapshot("100000"), "100000") + self.assertEqual(proof, {"positions_match": True, "cash_match": True, "cash_delta_usd": "0", + "open_orders": 0, "positions": 0}) + self.assertEqual(tuple(self.ledger.db.iterdump()), before) + + def test_other_account_activity_still_fails_closed(self): + # A dividend credit, a cash withdrawal or a cash journal beside the recorded fees stays unexplained. + for label, cash in (("DIV", "100000.55"), ("CSW", "99899.53"), ("JNLC", "99999.00")): + with self.subTest(activity=label), self.assertRaisesRegex(SafetyError, "^cash_mismatch_or_unmodeled_fees$"): + runner_module.reconcile(self.ledger, self.snapshot(cash, measured_fees()), "100000") + self.assertEqual(self.ledger.accounting().cash_delta_usd, D("-0.47")) # the fees themselves are recorded + # A non-FEE activity can never be passed off as a fee: the ledger refuses its sub-type. + dividend = {"id": activity_id(5), "date": "2026-09-29", "net_amount": "1.02", "sub_type": "CDIV"} + with self.assertRaises(SafetyError): + runner_module.reconcile(self.new_ledger("dividend"), self.snapshot("100001.02", [dividend]), "100000") + + def test_a_fee_the_per_trial_baseline_already_holds_must_not_reach_reconcile(self): + # Why the mover's fee window is its own trial's start, not the lane's started_at: trial 1 ended flat + # at broker cash 100000 (ledger cash_delta 0); overnight the day's three fees posted (cash 99999.53); + # command_paper then recomputed trial 2's baseline as preflight cash minus the ledger's cash_delta + # (mover_runner.py), which already holds them. Handing trial 2 those fees again double counts. + baseline = format(D("99999.53") - self.ledger.accounting().cash_delta_usd, "f") + with self.assertRaisesRegex(SafetyError, "^cash_mismatch_or_unmodeled_fees$"): + runner_module.reconcile(self.ledger, self.snapshot("99999.53", measured_fees()), baseline) + fresh = self.new_ledger("windowed") + baseline = format(D("99999.53") - fresh.accounting().cash_delta_usd, "f") + self.assertTrue(runner_module.reconcile(fresh, self.snapshot("99999.53", []), baseline)["cash_match"]) + + def test_mover_reconcile_records_fees_through_runner_reconcile(self): + proof = mover_runner.mover_reconcile(self.ledger, self.snapshot("99999.53", measured_fees()), "100000") + self.assertTrue(proof["cash_match"]) + self.assertEqual(len(self.ledger.fees()), 3) + + +# --------------------------------------------------------------------------- +# Items 4 and 7: runner.main (adaptive lane): fee window and next-trial check +# --------------------------------------------------------------------------- + +def paper_ready_observation(now_ns, symbols=("SPY",)): + """The shape test_adaptive_paper_runner.py's _paper_ready_observation uses.""" + return {"account": {"status": "ACTIVE", "currency": "USD", "cash": "10000", "equity": "30000", + "trading_blocked": False, "account_blocked": False, "trade_suspended_by_user": False}, + "clock": {"timestamp_ns": now_ns, "received_at_ns": now_ns, "next_close_ns": now_ns + 3600_000_000_000, + "is_open": True}, + "positions": [], "orders": [], + "assets": [{"symbol": s, "status": "active", "tradable": True} for s in symbols], + "quotes": [{"symbol": s, "ts_ns": now_ns} for s in symbols], + "account_identity_sha256": "fixture-account"} + + +def paper_ready_config(symbols=("SPY",)): + return {"capital_usd": "10000", "duration_seconds": 60, "cleanup_seconds": 10, + "symbols": list(symbols), "benchmarks": list(symbols), "quote_max_age_seconds": 5, + "feed": "iex", "regular_session_only": True, "extended_hours_enabled": False, + "sessions": {"extended_hours": False, "overnight_holds": False, "overnight_gross_multiple": "1.0"}} + + +def full_gate_result(input_sha256): + return {"status": "pass", "input_sha256": input_sha256, "row_count": 1, + "checks": [{"name": name, "status": "pass", "detail": "ok"} + for name in sorted(runner_module._GATE_CHECK_NAMES)], + "versions": {"pandera": "0.33.1"}, "checked_at": "2026-09-22T00:00:00+00:00"} + + +class AdaptiveLaneFees(unittest.TestCase): + """runner.main keeps its first trial's baseline_cash (trial.json started_at), so its fee window is + started_at, and its next-trial cash check records the fees posted since then before comparing.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.env_file = self.root / "paper.env" + self.env_file.write_text("APCA_API_KEY_ID=fixture-key\nAPCA_API_SECRET_KEY=fixture-secret\n") + os.chmod(self.env_file, 0o600) + self.config_file = self.root / "config.json" + self.config_file.write_text("{}") + self.output = self.root / "output.json" + self.universe = self.root / "universe.csv" + self.universe.write_text("symbol\nSPY\n") + self.gate = self.root / "gate-result.json" + self.gate.write_text(json.dumps(full_gate_result(hashlib.sha256(self.universe.read_bytes()).hexdigest()))) + self.observation = paper_ready_observation(time.time_ns()) + self.state = self.root / "state" / "fixture-account" / "adaptive" + self.state.mkdir(parents=True) + (self.state / "trial.json").write_text(json.dumps({ + "trial_id": "fee-1", "started_at": T0, "current_trial_started_at": T0 + 86400, + "config_sha256": hashlib.sha256(self.config_file.read_bytes()).hexdigest(), + "baseline_cash": "10000.47", "phase": "finished", "status": "passed"})) + + def run_main(self, command, fees, cash): + self.observation["account"]["cash"] = cash + seen = {} + sentinel = RuntimeError("transport_constructed") + + def fake_fees(key, secret, *, after, before_request, request_observer=None, **kwargs): + seen["fee_after"] = after + before_request("read") + return [dict(fee) for fee in fees] + + def fake_transport(*args, **kwargs): + seen["transport"] = kwargs + raise sentinel + + argv = ["runner.py", command, "--env-file", str(self.env_file), "--config", str(self.config_file), + "--output", str(self.output), "--state-root", str(self.root / "state"), "--trial", "fee-2"] + if command == "paper": + argv += ["--gate-result", str(self.gate), "--snapshot", str(self.universe)] + raised = None + with patch.object(sys, "argv", argv), \ + patch.object(runner_module, "load_config", return_value=(paper_ready_config(), RiskLimits(), None)), \ + patch.object(runner_module, "credentials", return_value=("fixture-key", "fixture-secret")), \ + patch.object(runner_module, "preflight", return_value=self.observation), \ + patch.object(runner_module, "account_lock_fingerprint", lambda _fingerprint: nullcontext()), \ + patch.object(runner_module, "fee_activities", fake_fees, create=True), \ + patch.object(runner_module, "AlpacaPaperTransport", fake_transport): + try: + runner_module.main() + except Exception as exc: # noqa: BLE001 -- the sentinel says the transport was reached + raised = exc + return seen, raised, sentinel + + def ledger(self): + ledger = Ledger(self.state / "ledger.sqlite3", RiskLimits()) + self.addCleanup(ledger.close) + return ledger + + def test_fees_posted_since_the_baseline_are_recorded_before_the_next_trial_check(self): + seen, raised, sentinel = self.run_main("paper", measured_fees(), "10000.00") + self.assertIs(raised, sentinel, raised) # the next-trial cash check passed + self.assertEqual(seen["fee_after"], utc(T0)) + self.assertEqual((seen["transport"]["fee_history_start"], seen["transport"]["history_start"]), + (utc(T0), utc(T0))) + ledger = self.ledger() + self.assertEqual([f["activity_id"] for f in ledger.fees()], [f["id"] for f in measured_fees()]) + self.assertEqual(ledger.accounting().cash_delta_usd, D("-0.47")) + self.assertEqual([r[0] for r in ledger.db.execute("SELECT kind FROM requests")], ["read"]) + + def test_an_unexplained_gap_still_refuses_the_next_trial(self): + for fees, cash in (([], "10000.00"), (measured_fees(), "10001.02")): + with self.subTest(fees=len(fees), cash=cash): + seen, raised, sentinel = self.run_main("paper", fees, cash) + self.assertIsInstance(raised, SafetyError) + self.assertEqual(str(raised), "next_trial_cash_mismatch") + self.assertNotIn("transport", seen) + + def test_the_adaptive_recover_transport_reads_fees_from_started_at(self): + self.observation["positions"] = [{"symbol": "SPY", "qty": "1"}] + seen, raised, sentinel = self.run_main("recover", [], "10000.00") + self.assertIs(raised, sentinel, raised) + self.assertEqual((seen["transport"]["fee_history_start"], seen["transport"]["history_start"]), + (utc(T0), utc(T0))) + self.assertNotIn("fee_after", seen) # recover records fees through its own snapshots + + +# --------------------------------------------------------------------------- +# Items 4 and 9: mover helpers, fee window and receipts +# --------------------------------------------------------------------------- + +class MoverFeeHelpers(unittest.TestCase): + def test_fee_receipt_counts_totals_and_sub_types_without_ids(self): + rows = [{"activity_id": f["id"], "date": f["date"], "net_amount": D(f["net_amount"]), + "sub_type": f["sub_type"], "recorded_at": T0} for f in measured_fees()] + summary = mover_runner.fee_receipt(rows) + self.assertEqual(summary, {"count": 3, "total_usd": "-0.47", + "sub_types": {"CAT": {"count": 1, "total_usd": "-0.01"}, + "REG": {"count": 1, "total_usd": "-0.19"}, + "TAF": {"count": 1, "total_usd": "-0.27"}}}) + text = json.dumps(summary) + for fee in measured_fees(): + self.assertNotIn(fee["id"].split("::")[1], text) + self.assertEqual(mover_runner.fee_receipt([]), {"count": 0, "total_usd": "0", "sub_types": {}}) + + def test_the_mover_fee_window_is_the_trials_own_baseline_time(self): + self.assertEqual(mover_runner.fee_window_start({"started_at": T0 - 86400, "current_trial_started_at": T0}), + utc(T0)) + for metadata in ({}, {"started_at": T0}, {"current_trial_started_at": None}, + {"current_trial_started_at": "1790000000"}, {"current_trial_started_at": True}, + {"current_trial_started_at": float("nan")}, {"current_trial_started_at": -1.0}, None, []): + with self.subTest(metadata=metadata): + with self.assertRaisesRegex(SafetyError, "^trial_metadata_lacks_baseline_time$"): + mover_runner.fee_window_start(metadata) + + +CONFIG = SOURCE / "config-mover.json" +SCAN_TIME = datetime(2026, 9, 24, 12, 0, 5, tzinfo=timezone.utc).timestamp() # 08:00:05 ET +PRE_TS = datetime(2026, 9, 24, 12, 30, tzinfo=timezone.utc) # 08:30 ET + + +def scan_row(symbol, rank, price): + return {"symbol": symbol, "rank": rank, "price_at_t": price, "dollar_volume_at_t": "50000000", + "entry_bar_dollar_volume": "2000000"} + + +def scan_raw(rows): + return json.dumps({"schema_version": 1, "kind": "mover_scan", "protocol": "mover-early-entry-v1-20260924", + "rule": "08:00|G20|V1000000|any", "scan_time": "2026-09-24T12:00:05Z", + "regime": {"factor": "1"}, "symbols": rows}).encode() + + +@unittest.skipUnless(NATIVE, "requires pinned combined native runtime") +class MoverLaneFees(unittest.TestCase): + """``mover_runner.py paper``/``recover`` end to end with a fake preflight and the synthetic port in + place of AlpacaPaperTransport (as test_adaptive_paper_mover_native.MoverPaperCommandWiring), plus a + broker-side FEE list the port filters by the transport's fee window. A SYN wiring fixture.""" + + SERVER = datetime(2026, 9, 24, 12, 0, 30, tzinfo=timezone.utc).timestamp() # 08:00:30 ET, PRE + FINGERPRINT = "f" * 64 + + def setUp(self): + self.broker = None + self.sell_blocked_ports = 0 + self.windows = [] # (fee_history_start, history_start) per transport built + self.posted = [] # (created_at epoch, raw FEE row) the broker lists + self.post_on_port = None # post the measured fees as this transport (1-based) is built + + def post_fees(self): + """The broker posts the measured FEE activities now: cash falls by 0.47 and the rows are listed.""" + created = time.time() + self.posted.extend((created, row) for row in measured_rows()) + self.broker.cash -= D("0.47") + + def observation(self, symbols): + server_ns, now_ns = int(self.SERVER * 1e9), time.time_ns() + cash = format(self.broker.cash, "f") if self.broker is not None else "100000" + positions = [] if self.broker is None else [ + {"symbol": s, "qty": format(q, "f"), "avg_entry_price": "10"} for s, q in self.broker.positions.items() if q] + orders = [] if self.broker is None else [ + self.broker._public(o) for o in self.broker.orders.values() if o["status"] in ("new", "partially_filled")] + return {"account": {"status": "ACTIVE", "currency": "USD", "cash": cash, "equity": cash, + "buying_power": cash, "trading_blocked": False, "account_blocked": False, + "trade_suspended_by_user": False}, + "account_identity_sha256": self.FINGERPRINT, + "clock": {"is_open": False, "received_at_ns": server_ns, "timestamp_ns": server_ns, + "next_close_ns": server_ns + 8 * 3600 * 10 ** 9, "next_open_ns": server_ns + 5400 * 10 ** 9}, + "positions": positions, "orders": orders, "open_orders_complete": True, + "assets": [{"symbol": s, "status": "active", "tradable": True} for s in symbols], + "quotes": [{"symbol": s, "bid": "100", "ask": "100.01", "ts_ns": now_ns} for s in symbols], + "quote_errors": {}} + + def fast_config(self, root): + data = json.loads(CONFIG.read_text()) + data["order_timeout_seconds"] = 1 + path = Path(root) / "config-fast.json" + path.write_text(json.dumps(data)) + return path + + def trial_json(self, root): + return json.loads((Path(root) / "state" / self.FINGERPRINT / "mover" / "trial.json").read_text()) + + def run_command(self, root, trial, *, command="paper", config=None): + import signal + import transport + out = Path(root) / f"{trial or command}.json" + real_load_scan, real_build_plan, real_flag = (mover_runner.load_scan, mover_runner.build_plan, + native_adapter.order_extended_hours_flag) + + def compressed_plan(settings, limits, scan, session, *, t0, **kwargs): + timing = mover.Timing(t0, t0 + 2.0, 1.0, 1.0, t0 + 4.0, t0 + 8.0, None, 1.5) + return real_build_plan(settings, limits, scan, session, t0=t0, timing=timing, **kwargs) + + def fake_transport(key, secret, symbols, **kwargs): + controller = kwargs["before_request"].__self__ + port = MoverSimulatedPort(controller, symbols, path=piecewise_path({s: [(0, D("10.00"), D("0.02"))] + for s in symbols}), + extended_hours_allowed=kwargs.get("extended_hours_allowed", False)) + if self.broker is not None: + port.orders, port.positions, port.cash = self.broker.orders, self.broker.positions, self.broker.cash + port.t0 = self.broker.t0 + port.fill_sells = self.sell_blocked_ports <= 0 + self.sell_blocked_ports -= 1 + window = kwargs.get("fee_history_start") + self.windows.append((window, kwargs.get("history_start"))) + listed = port.snapshot + + async def snapshot(): + result = await listed() + if window is not None: + # The broker's `after` filter on created_at; the rows pass the real normalization. + result["fees"] = [t.normalize_fee_activity(row) for created, row in self.posted + if created > window.timestamp()] + return result + port.snapshot = snapshot + self.broker = port + if self.post_on_port == len(self.windows): + self.post_fees() + return port + + args = [command, "--env-file", str(Path(root) / "unused.env"), "--output", str(out), + "--state-root", str(Path(root) / "state")] + if command == "paper": + args += ["--scan", str(Path(root) / "scan.json"), "--trial", trial] + if config is not None: + args += ["--config", str(config)] + handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + try: + with patch.object(mover_runner, "credentials", return_value=("key", "secret")), \ + patch.object(transport, "preflight", lambda key, secret, symbols, **kw: self.observation(symbols)), \ + patch.object(transport, "AlpacaPaperTransport", fake_transport), \ + patch.object(mover_runner, "load_scan", + lambda raw, settings, *, now: real_load_scan(raw, settings, now=SCAN_TIME + 20)), \ + patch.object(mover_runner, "build_plan", compressed_plan), \ + patch.object(mover_runner, "_controller_session", lambda close, now, policy: (now + 36000, True)), \ + patch.object(native_adapter, "order_extended_hours_flag", lambda ts, p: real_flag(PRE_TS, p)), \ + patch("builtins.print"): + code = mover_runner.main(args) + finally: + for sig, handler in handlers.items(): + signal.signal(sig, handler) + return code, json.loads(out.read_text()) + + def test_each_mover_transport_reads_fees_from_its_trials_own_baseline_time(self): + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + self.assertEqual(self.run_command(root, "lane-1", config=config)[0], 0) + first = self.trial_json(root) + self.assertEqual(self.run_command(root, "lane-2", config=config)[0], 0) + second = self.trial_json(root) + self.assertEqual(second["started_at"], first["started_at"]) + self.assertLess(first["current_trial_started_at"], second["current_trial_started_at"]) + self.assertEqual(self.windows, [(utc(first["current_trial_started_at"]), utc(first["started_at"])), + (utc(second["current_trial_started_at"]), utc(second["started_at"]))]) + code, result = self.run_command(root, None, command="recover", config=config) + self.assertEqual((code, result["status"]), (0, "passed")) + self.assertEqual(self.windows[-1], (utc(second["current_trial_started_at"]), utc(second["started_at"]))) + + def test_recover_records_fees_posted_after_the_trial_and_reports_them(self): + # The account 2 case: the trial ends needs_attention holding a position, the day's REG/TAF/CAT + # then post, and every recover ended cash_mismatch_or_unmodeled_fees. + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + self.sell_blocked_ports = 2 # the trial's port and the forced recovery's leave sells resting + code, receipt = self.run_command(root, "ext-1", config=config) + self.assertEqual((code, receipt["status"], receipt["flat"]), (3, "needs_attention", False)) + self.assertEqual(receipt["fees_recorded"], {"count": 0, "total_usd": "0", "sub_types": {}}) + self.post_fees() + code, result = self.run_command(root, None, command="recover", config=config) + self.assertEqual((code, result["kind"], result["status"], result["flat"], result["errors"]), + (0, "mover_recovery_receipt", "passed", True, [])) + self.assertEqual(result["fees_recorded"], {"count": 3, "total_usd": "-0.47", + "sub_types": {"CAT": {"count": 1, "total_usd": "-0.01"}, + "REG": {"count": 1, "total_usd": "-0.19"}, + "TAF": {"count": 1, "total_usd": "-0.27"}}}) + text = json.dumps(result) + for fee in measured_fees(): + self.assertNotIn(fee["id"].split("::")[1], text) + self.assertNotIn(ACCOUNT_TEXT, text) + self.assertEqual(self.trial_json(root)["phase"], "finished") + # The next trial's own baseline already holds the posted fees: nothing re-recorded, no mismatch. + code, receipt = self.run_command(root, "ext-2", config=config) + self.assertEqual((code, receipt["status"], receipt["flat"]), (0, "passed", True), receipt.get("error_reason")) + self.assertEqual(receipt["fees_recorded"]["count"], 0) + self.assertTrue(receipt["totals"]["pnl_consistent"], receipt["totals"]) + + def test_fees_posted_during_a_trial_are_recorded_and_the_receipt_stays_consistent(self): + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + self.post_on_port = 1 # the broker posts fees just after the trial's transport exists + code, receipt = self.run_command(root, "ext-1", config=config) + self.assertEqual((code, receipt["status"], receipt["flat"]), (0, "passed", True), receipt.get("error_reason")) + self.assertEqual((receipt["fees_recorded"]["count"], receipt["fees_recorded"]["total_usd"]), (3, "-0.47")) + totals = receipt["totals"] + self.assertEqual(totals["fees_recorded_usd"], "-0.47") + self.assertEqual(D(totals["ledger_realized_pnl_delta_usd"]), D(totals["realized_pnl_usd"]) + D("-0.47")) + self.assertTrue(totals["pnl_consistent"], totals) + self.assertNotIn(ACCOUNT_TEXT, json.dumps(receipt)) + + +if __name__ == "__main__": + unittest.main() From f65cfbeeda751dc5efb613a5726e0a8a02306d14 Mon Sep 17 00:00:00 2001 From: Scout Date: Wed, 30 Sep 2026 13:24:31 -0400 Subject: [PATCH 2/7] Adaptive paper: record broker FEE activities in ledger cash accounting Alpaca paper posts REG/TAF/CAT FEE activities the day after a sell (measured 2026-09-30 on account 2: -0.47 USD for 2026-09-29), and runner.reconcile failed closed on the unexplained gap, so no residual held across a fee posting could be recovered. - transport: one more allow-listed read, GET /v2/account/activities/FEE with after (UTC YYYY-MM-DDTHH:MM:SSZ) and direction asc required, optional page_size 1-100 and activity-id page_token; normalize_fee_activity keeps {id, date, net_amount, sub_type} only (the description is never read); snapshot() lists "fees" after fee_history_start, read last, page-bounded; fee_activities() is the same read on a fresh read-only client. - safety: ledger schema 3 (one way; older engines refuse it) with a fees table; record_fees books each new id once into cash_delta, realized and realized_loss (a credit never lowers the loss), refuses a changed id, and the accounting re-derivation adds recorded fees back. - runner.reconcile records snapshot["fees"] before the unchanged 0.01 USD cash comparison; runner.main's next-trial check reads and records fees since started_at (each GET charged as a read) before comparing. - mover_runner: the fee window is trial.json current_trial_started_at, since command_paper recomputes baseline_cash per trial (mover_runner.py:757); trial and recovery receipts carry fees_recorded (no ids), and pnl_consistent adds the recorded fees. - tests/test_adaptive_paper_transport.py: the flat-snapshot test now expects the fifth request (the FEE read) and asserts snapshot["fees"] == [] (stricter, not loosened). Offline gate (9 modules, pinned venv): 440 tests OK. Co-Authored-By: Claude Opus 5.5 --- .../adaptive-paper/mover_runner.py | 58 +++++++- .../us-equities/adaptive-paper/runner.py | 26 +++- .../us-equities/adaptive-paper/safety.py | 85 ++++++++++- .../us-equities/adaptive-paper/transport.py | 140 +++++++++++++++++- tests/test_adaptive_paper_transport.py | 6 +- 5 files changed, 298 insertions(+), 17 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/mover_runner.py b/blueprints/us-equities/adaptive-paper/mover_runner.py index ab904d277..c4213879a 100644 --- a/blueprints/us-equities/adaptive-paper/mover_runner.py +++ b/blueprints/us-equities/adaptive-paper/mover_runner.py @@ -23,6 +23,7 @@ from decimal import Decimal import hashlib import json +import math from pathlib import Path import re import signal @@ -145,13 +146,40 @@ def observe(self, order): def mover_reconcile(ledger, snapshot, baseline_cash): """runner.reconcile over the ledger's own orders plus every foreign order that is not terminal (which still fails as external_order_detected). Positions and cash are - compared unchanged; the mover's baseline is the trial's own starting cash.""" + compared unchanged, after runner.reconcile records the snapshot's FEE activities; the + mover's baseline is the trial's own starting cash.""" owned = {i.client_id for i in ledger.intents()} orders = [o for o in snapshot.get("orders", []) if o.get("client_order_id") in owned or o.get("status") not in FOREIGN_TERMINAL_STATUSES] return reconcile(ledger, {**snapshot, "orders": orders}, baseline_cash) +def fee_window_start(metadata): + """The instant a mover transport lists FEE activities from: this trial's own cash + baseline. command_paper recomputes baseline_cash at every trial start (preflight cash + minus the ledger's cash_delta), so a fee posted before the trial started is already in + that baseline and recording it again would count it twice; trial.json started_at (the + lane's first trial) would do exactly that for the fees posted after each trading day. + runner.main keeps its first trial's baseline instead and reads fees from started_at.""" + value = metadata.get("current_trial_started_at") if isinstance(metadata, dict) else None + if type(value) not in (int, float) or not math.isfinite(value) or value <= 0: + raise SafetyError("trial_metadata_lacks_baseline_time") + return datetime.fromtimestamp(value, timezone.utc) + + +def fee_receipt(rows): + """The FEE activities one command recorded (Ledger.fees rows): count, total and per + sub-type count and total. Never an activity id.""" + by_type = {} + for row in rows: + entry = by_type.setdefault(row["sub_type"], [0, Decimal(0)]) + entry[0] += 1 + entry[1] += row["net_amount"] + return {"count": len(rows), "total_usd": text(sum((row["net_amount"] for row in rows), Decimal(0))), + "sub_types": {sub_type: {"count": count, "total_usd": text(total)} + for sub_type, (count, total) in sorted(by_type.items())}} + + def scope_recovery_adoption(port, ledger): """Wrap a fresh recovery port so recovery.recover adopts only intents it can own. @@ -459,11 +487,15 @@ def plan_receipt(plan): def build_receipt(*, plan, outcome, config_sha256, scan, ledger, ledger_before, prefixes, preflight=None, - extra=None): + extra=None, fees_recorded=()): """The trial receipt. Per symbol: scan fields, intended notional and caps, orders (client ids, hashed broker refs, submit/accept/fill times and prices), exit reason - and realized P&L; totals; start/end reconciliation. No account id or balance.""" + and realized P&L; totals; start/end reconciliation; the FEE activities recorded in the + run (``fees_recorded``, Ledger.fees rows; count, total and sub-types only). The ledger's + realized delta is the per-symbol P&L plus those fees. No account id or balance.""" after = ledger.accounting() + fees_recorded = list(fees_recorded) + fee_total = sum((row["net_amount"] for row in fees_recorded), Decimal(0)) intents = {intent.client_id: intent for intent in ledger.intents()} pnl = pnl_by_symbol(intents.values(), prefixes) recovery_prefix = f"rec-{plan.trial_id}-" @@ -492,7 +524,9 @@ def build_receipt(*, plan, outcome, config_sha256, scan, ledger, ledger_before, "realized_pnl_usd": text(sum(per_symbol, Decimal(0))) if all_flat else None, "ledger_realized_pnl_delta_usd": text(realized_delta), "ledger_cash_delta_usd": text(after.cash_delta_usd - ledger_before.cash_delta_usd), - "pnl_consistent": bool(all_flat and abs(sum(per_symbol, Decimal(0)) - realized_delta) <= Decimal("0.01")), + "fees_recorded_usd": text(fee_total), + "pnl_consistent": bool(all_flat and abs(sum(per_symbol, Decimal(0)) + fee_total - realized_delta) + <= Decimal("0.01")), "native_fill_events": outcome.get("native_fill_events", 0), "native_rejections": outcome.get("native_rejections", 0)} receipt = { @@ -508,6 +542,7 @@ def build_receipt(*, plan, outcome, config_sha256, scan, ledger, ledger_before, "reconciliation": {"start": {"flat_account_and_no_open_orders": True, "checked_by": "runner.validate_preflight and the native adapter's startup snapshot"}, "end": outcome.get("reconciliation"), "recovery": outcome.get("recovery")}, + "fees_recorded": fee_receipt(fees_recorded), "ledger_risk": {"lifetime_realized_pnl_usd": text(after.realized_pnl_usd), "lifetime_gross_loss_usd": text(after.gross_loss_usd), "drawdown_usd": text(after.drawdown_usd), "halted_reason": after.halted_reason}, @@ -733,6 +768,8 @@ def command_paper(args): "inter_trial_cash_change_usd": None if inter_trial is None else format(inter_trial, "f"), "lane_state": session_state_after(session_plan, equity_end=equity), "history": history} save(metadata_path, metadata) + fee_window = fee_window_start(metadata) + known_fees = {row["activity_id"] for row in ledger.fees()} controller_close, market_open = _controller_session(close, now, session_policy) controller = MoverController(ledger, controller_close, market_open=market_open, max_entry_notional_usd=settings.max_entry_notional_usd) @@ -757,6 +794,8 @@ def fresh_port(): required_quote_symbols=list(settings.benchmarks), # As runner.main: from the lane's first trial, so reconcile sees every owned intent. history_start=datetime.fromtimestamp(metadata["started_at"], timezone.utc), + # FEE activities from this trial's own cash baseline (fee_window_start). + fee_history_start=fee_window, extended_hours_allowed=session_policy["extended_hours"], include_margin=limits.leverage is not None)) @@ -792,7 +831,8 @@ async def execute(): ledger_before=before, prefixes=prefixes, preflight=summary, extra={"shared_account_override": bool(args.allow_shared_account), "inter_trial_cash_changed": (None if inter_trial is None else - abs(inter_trial) > Decimal("0.01"))}) + abs(inter_trial) > Decimal("0.01"))}, + fees_recorded=[row for row in ledger.fees() if row["activity_id"] not in known_fees]) phase, exit_code = trial_phase_and_exit_code(receipt) after = ledger.accounting() equity_end = limits.capital_usd + after.realized_pnl_usd @@ -850,6 +890,9 @@ def command_recover(args): controller_close, market_open = _controller_session(close, time.time(), session_policy) controller = MoverController(ledger, controller_close, market_open=market_open, max_entry_notional_usd=settings.max_entry_notional_usd) + # The trial's own cash baseline (trial.json baseline_cash) opens the FEE window. + fee_window = fee_window_start(metadata) + known_fees = {row["activity_id"] for row in ledger.fees()} # #215: only the benchmark quotes gate the recovery transport; each owned # symbol's freshness gates its own exit (recovery.recover). controller.port = controller.bind(AlpacaPaperTransport( @@ -859,10 +902,13 @@ def command_recover(args): request_observer=responses.append, quote_timeout=settings.stream_quote_timeout_seconds, feed=config["feed"], required_quote_symbols=list(settings.benchmarks), history_start=datetime.fromtimestamp(metadata["started_at"], timezone.utc), + fee_history_start=fee_window, extended_hours_allowed=session_policy["extended_hours"], include_margin=include_margin)) result = asyncio.run(recover_mover(controller, metadata, trial_config)) result.update(kind="mover_recovery_receipt", protocol=PROTOCOL_ID, evidence_class="PAPER", - trial_id=metadata["trial_id"]) + trial_id=metadata["trial_id"], + fees_recorded=fee_receipt([row for row in ledger.fees() + if row["activity_id"] not in known_fees])) phase, exit_code = trial_phase_and_exit_code(result) metadata.update(phase=phase, status=result["status"], recovered_at=time.time()) save(metadata_path, metadata) diff --git a/blueprints/us-equities/adaptive-paper/runner.py b/blueprints/us-equities/adaptive-paper/runner.py index f86cb0e2d..80958fcef 100644 --- a/blueprints/us-equities/adaptive-paper/runner.py +++ b/blueprints/us-equities/adaptive-paper/runner.py @@ -36,7 +36,7 @@ must_end_flat, next_trading_day, session_at, validate_session_policy) from strategies import AdaptivePolicy, PolicyConfig, RegimeSelector, SelectorConfig, limit_price from transport import (AlpacaPaperTransport, DATA_FEEDS, TransportError, RejectedSubmission, order_contract_status, - preflight, halt_statuses_supported, nasdaq_halt_seed) + preflight, halt_statuses_supported, nasdaq_halt_seed, fee_activities) SOURCE = Path(__file__).resolve().parent LAST_OUTPUT = None @@ -758,6 +758,12 @@ def reconcile(ledger, snapshot, baseline_cash): expected = {p.symbol: p.qty for p in ledger.positions().values() if p.qty} if actual != expected: raise SafetyError("position_mismatch") + if "fees" in snapshot: + # Broker FEE activities (REG, TAF, CAT, ...) created since the ledger's cash baseline, + # from the transport's allow-listed read, are recorded durably before the comparison. + # The comparison and its 0.01 USD tolerance are unchanged: deposits, withdrawals, + # journals, dividends, interest and every other activity stay unexplained and fail it. + ledger.record_fees(snapshot["fees"], time.time()) cash_delta = Decimal(snapshot["account"]["cash"]) - Decimal(baseline_cash) if abs(cash_delta - ledger.accounting().cash_delta_usd) > Decimal("0.01"): raise SafetyError("cash_mismatch_or_unmodeled_fees") @@ -2380,6 +2386,22 @@ def observe_request(kind, **kwargs): if (ledger.positions() or ledger.unresolved()) and not resumable_hold: raise SafetyError("next_trial_requires_recovery") if not resumable_hold: + # baseline_cash is the first trial's cash (started_at), so FEE activities + # posted since then, such as the day's REG/TAF/CAT after the last trial + # ended, are recorded durably first, through the transport's allow-listed + # read and normalization. Each GET is charged to the durable budget as a + # read, and `now` follows those reads, so the next trial never starts + # before a recorded request. Any other unexplained change still refuses. + fee_reads = [] + fees = fee_activities(key, secret, + after=datetime.fromtimestamp(previous_metadata["started_at"], timezone.utc), + before_request=lambda kind, **_: fee_reads.append(time.time()), + request_observer=responses.append) + for timestamp in fee_reads: + if ledger.request_budget(timestamp, "read"): + raise SafetyError("preflight_budget_inconsistent") + now = time.time() + ledger.record_fees(fees, now) expected_cash = Decimal(previous_metadata["baseline_cash"]) + ledger.accounting().cash_delta_usd if abs(Decimal(observation["account"]["cash"]) - expected_cash) > Decimal("0.01"): raise SafetyError("next_trial_cash_mismatch") @@ -2410,6 +2432,8 @@ def fresh_port(recovering=False): quote_timeout=config["quote_max_age_seconds"], feed=config["feed"], required_quote_symbols=needed if recovering and needed else config["benchmarks"], history_start=datetime.fromtimestamp(metadata["started_at"], timezone.utc), + # This lane keeps its first trial's baseline_cash, read at started_at. + fee_history_start=datetime.fromtimestamp(metadata["started_at"], timezone.utc), extended_hours_allowed=session_policy["extended_hours"], include_margin=leverage_policy is not None)) for sig in (signal.SIGINT, signal.SIGTERM): diff --git a/blueprints/us-equities/adaptive-paper/safety.py b/blueprints/us-equities/adaptive-paper/safety.py index 76398a6be..48fe476ce 100644 --- a/blueprints/us-equities/adaptive-paper/safety.py +++ b/blueprints/us-equities/adaptive-paper/safety.py @@ -53,6 +53,13 @@ AVERAGE_ROUNDING = D("0.0000005") EXECUTION_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9:_.\-]{0,127}") EXECUTION_ACCOUNTING_RULE = "v4-symbol-execution-time-fallback" +# Broker FEE activities (schema 3), as transport.normalize_fee_activity returns them: an +# activity id "::" (transport.ACTIVITY_ID; this module imports no transport +# code), a YYYY-MM-DD date, a signed decimal net_amount (negative for a charge) and a +# sub-type from the by-type reference's FEE list, or UNSPECIFIED when the broker gave none. +FEE_ACTIVITY_ID = re.compile(r"[0-9]{1,32}::[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") +FEE_AMOUNT = re.compile(r"-?[0-9]{1,10}(?:\.[0-9]{1,9})?") +FEE_SUB_TYPES = frozenset({"REG", "TAF", "LCT", "ORF", "OCC", "NRC", "NRV", "COM", "CAT", "UNSPECIFIED"}) def execution_from_observation(row): @@ -121,6 +128,26 @@ def instant(value): return float(value) +def _fee_row(fee): + """(activity_id, date, net_amount, sub_type) of one normalized FEE activity, or SafetyError.""" + if type(fee) is not dict or set(fee) != {"id", "date", "net_amount", "sub_type"}: + raise SafetyError("invalid_fee_activity") + activity_id, day, amount, sub_type = fee["id"], fee["date"], fee["net_amount"], fee["sub_type"] + if type(activity_id) is not str or not FEE_ACTIVITY_ID.fullmatch(activity_id): + raise SafetyError("invalid_fee_activity") + if type(day) is not str or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", day): + raise SafetyError("invalid_fee_date") + try: + day = date.fromisoformat(day).isoformat() + except ValueError: + raise SafetyError("invalid_fee_date") from None + if type(amount) is not str or not FEE_AMOUNT.fullmatch(amount): + raise SafetyError("invalid_fee_amount") + if type(sub_type) is not str or sub_type not in FEE_SUB_TYPES: + raise SafetyError("invalid_fee_sub_type") + return activity_id, day, D(amount), sub_type + + def symbol_name(value): if type(value) is not str or not re.fullmatch(r"[A-Z]{1,5}(?:[.-][A-Z])?", value): raise SafetyError("invalid_symbol") @@ -440,13 +467,20 @@ def __init__(self, db_path, limits=None): raise SafetyError("persisted_risk_limits_differ") self._set("limits", frozen) version = self._get("schema_version") - if version not in (None, "1", "2"): + if version not in (None, "1", "2", "3"): raise SafetyError("unsupported_ledger_schema") if version in (None, "1"): # Old `at` is the observation's floating-point order.updated_at, # never evidence of an execution time. Keep those fills untimed. self.db.execute("ALTER TABLE executions ADD COLUMN execution_time_ns INTEGER") - self._set("schema_version", "2") + # Schema 3 (2026-09-30): broker FEE activities (record_fees), created inside + # this transaction and only after the version check, so a refused ledger is + # never changed. The migration is one way: engines before it accept only + # None/1/2 and so refuse a migrated ledger (fail closed). + self.db.execute("CREATE TABLE IF NOT EXISTS fees(activity_id TEXT PRIMARY KEY, " + "date TEXT NOT NULL, net_amount TEXT NOT NULL, sub_type TEXT NOT NULL, " + "recorded_at REAL NOT NULL)") + self._set("schema_version", "3") self.db.execute("CREATE INDEX IF NOT EXISTS intents_symbol ON intents(symbol,client_id)") self.db.execute("CREATE INDEX IF NOT EXISTS events_client_kind ON events(client_id,kind,id)") self.db.execute("CREATE INDEX IF NOT EXISTS events_adopted_symbol ON " @@ -1121,6 +1155,12 @@ def _rederive_execution_accounting(self): changed = True self._set_symbol_money(symbol, money) totals = [a + b for a, b in zip(totals, money)] + # Recorded FEE activities belong to no symbol's journal: add their money back, as + # record_fees booked it, so a re-derivation never drops a fee from cash, realized P&L + # or the realized loss. + for (amount,) in self.db.execute("SELECT net_amount FROM fees"): + amount = D(amount) + totals = [totals[0] + amount, totals[1] + amount, totals[2] + max(ZERO, -amount)] for name, amount in zip(("cash_delta", "realized", "realized_loss"), totals): changed |= amount != D(self._get(name)) self._set(name, amount) @@ -1278,6 +1318,40 @@ def record_order(self, client_id, broker_id, status, cumulative_qty, average_pri self._refresh_risk(at) return True + def record_fees(self, fees, now): + """Durably record broker FEE activities (transport.normalize_fee_activity rows) and + return the newly recorded ones. Every row is validated before the transaction; each + new activity id is inserted once, all in one transaction. A new fee adds net_amount + (negative for a charge) to cash_delta and to realized, and max(0, -net_amount) to + realized_loss: a fee is a realized cost that counts against the gross-loss budget, + and a credit never lowers the loss. peak_pnl and the risk halts then follow through + _refresh_risk, as after a fill. A known id with the same date, amount and sub-type is + a no-op; with any other value the whole batch is refused (fee_activity_changed).""" + now = instant(now) + if type(fees) not in (list, tuple): + raise SafetyError("invalid_fee_activities") + rows = [_fee_row(fee) for fee in fees] + with self._transaction(): + recorded = [] + for activity_id, day, amount, sub_type in rows: + known = self.db.execute("SELECT date, net_amount, sub_type FROM fees WHERE activity_id=?", + (activity_id,)).fetchone() + if known is not None: + if (known["date"], D(known["net_amount"]), known["sub_type"]) != (day, amount, sub_type): + raise SafetyError("fee_activity_changed") + continue + text = _canonical(amount) + self.db.execute("INSERT INTO fees(activity_id,date,net_amount,sub_type,recorded_at) " + "VALUES (?,?,?,?,?)", (activity_id, day, text, sub_type, now)) + for name, delta in (("cash_delta", amount), ("realized", amount), + ("realized_loss", max(ZERO, -amount))): + self._set(name, D(self._get(name)) + delta) + self._event("fee_recorded", activity_id=activity_id, date=day, net_amount=text, sub_type=sub_type) + recorded.append({"id": activity_id, "date": day, "net_amount": text, "sub_type": sub_type}) + if recorded: + self._refresh_risk(now) + return recorded + def request_budget(self, now, kind, client_id=None): """Return 0 after durable reservation, or delay <=60s WITHOUT reservation. @@ -1452,6 +1526,13 @@ def accounting(self): context.prec = 40 return self._state() + def fees(self): + """Recorded broker FEE activities in recording order (read-only).""" + with self._lock: + return [{"activity_id": r["activity_id"], "date": r["date"], "net_amount": D(r["net_amount"]), + "sub_type": r["sub_type"], "recorded_at": r["recorded_at"]} + for r in self.db.execute("SELECT * FROM fees ORDER BY recorded_at, rowid")] + def close(self): with self._lock: self.db.close() diff --git a/blueprints/us-equities/adaptive-paper/transport.py b/blueprints/us-equities/adaptive-paper/transport.py index f97bf4dd2..ff53f3397 100644 --- a/blueprints/us-equities/adaptive-paper/transport.py +++ b/blueprints/us-equities/adaptive-paper/transport.py @@ -41,6 +41,19 @@ ACTIVITY_FILL_PATH = "/v2/account/activities/FILL" ACTIVITY_ID = re.compile(r"[0-9]{1,32}::([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})\Z") ACTIVITY_PAGE_SIZE = 100 +# The same endpoint with activity_type FEE ("Fee denominated in USD"), read after one UTC +# instant: the ledger's cash baseline. Reference (updatedAt 2026-05-27, fetched 2026-09-30): +# https://docs.alpaca.markets/us/reference/getaccountactivitiesbyactivitytype-1.md. after: "Get +# activities created after this date. Both formats YYYY-MM-DD and YYYY-MM-DDTHH:MM:SSZ are +# supported."; direction: "The chronological order of response based on the activity datetime." +# (default desc, so asc is always sent); page_size: minimum 1, maximum 100; page_token: "Provide +# the ID of the last activity from the last page to retrieve the next set of results." Its +# ActivitySubType lists for FEE: REG (Regulatory Fee), TAF (Trading Activity Fee), LCT, ORF, OCC, +# NRC, NRV, COM (Commission) and CAT (Consolidated Audit Trail Fee). +ACTIVITY_FEE_PATH = "/v2/account/activities/FEE" +FEE_SUB_TYPES = frozenset({"REG", "TAF", "LCT", "ORF", "OCC", "NRC", "NRV", "COM", "CAT"}) +FEE_AFTER = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z\Z") +FEE_DATE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}\Z") # trade_updates events that carry one execution's own qty, price and execution_id. EXECUTION_EVENTS = frozenset({"fill", "partial_fill"}) EXECUTION_FIELDS = ("event", "execution_id", "event_qty", "event_price", "execution_time_ns") @@ -417,6 +430,93 @@ def _activity_params(params): and (token is None or (isinstance(token, str) and bool(ACTIVITY_ID.fullmatch(token))))) +def fee_after_text(moment): + """The documented UTC ``after`` value (YYYY-MM-DDTHH:MM:SSZ) for a timezone-aware instant, + truncated to the second: the window can open up to one second early, never late. A fee in + that second that a cash baseline already holds is then counted twice and fails the cash + check once (fail closed); it is never silently dropped.""" + if not isinstance(moment, datetime) or moment.tzinfo is None: + raise TransportError("timezone-aware fee window required") + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _fee_after(value): + if not isinstance(value, str) or not FEE_AFTER.fullmatch(value): + return False + try: + datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return False + return True + + +def _fee_activity_params(params): + """Only the documented FEE read after one UTC instant: ``after`` (YYYY-MM-DDTHH:MM:SSZ) and + direction asc are required, page_size 1-100 and an activity-id page_token are optional. No + other filter (order_id, date, until, ...) is ever sent.""" + if not isinstance(params, dict) or not set(params) <= {"after", "direction", "page_size", "page_token"}: + return False + size, token = params.get("page_size", ACTIVITY_PAGE_SIZE), params.get("page_token") + return (_fee_after(params.get("after")) and params.get("direction") == "asc" + and type(size) is int and 1 <= size <= ACTIVITY_PAGE_SIZE + and (token is None or (isinstance(token, str) and bool(ACTIVITY_ID.fullmatch(token))))) + + +def normalize_fee_activity(raw): + """One documented NonTradeActivity of activity_type FEE: ``{id, date, net_amount, sub_type}`` + and nothing else. Account Activities (https://docs.alpaca.markets/us/docs/account-activities.md, + updatedAt 2026-05-25): id "Can be sent as `page_token`"; date "The date on which the activity + occurred or on which the transaction associated with the activity settled."; net_amount "The + net amount of money (positive or negative) associated with the activity." (a documented + string). An absent or null activity_sub_type is UNSPECIFIED; currency must be absent or USD + and status absent or executed (the by-type reference also lists correct and canceled, which + this engine does not model). The description field, which carries the account number, is + never read. Raises TransportError for any other shape; provider text is never retained.""" + if not isinstance(raw, dict) or raw.get("activity_type") != "FEE": + raise TransportError("invalid fee activity") + activity_id, day, amount = raw.get("id"), raw.get("date"), raw.get("net_amount") + sub_type = raw.get("activity_sub_type") + if (not isinstance(activity_id, str) or not ACTIVITY_ID.fullmatch(activity_id) + or not isinstance(day, str) or not FEE_DATE.fullmatch(day) or not isinstance(amount, str) + or (sub_type is not None and (not isinstance(sub_type, str) or sub_type not in FEE_SUB_TYPES)) + or ("currency" in raw and raw["currency"] != "USD") + or ("status" in raw and raw["status"] != "executed")): + raise TransportError("invalid fee activity") + try: + datetime.strptime(day, "%Y-%m-%d") + except ValueError: + raise TransportError("invalid fee activity") from None + return {"id": activity_id, "date": day, "net_amount": decimal_string(amount), + "sub_type": sub_type or "UNSPECIFIED"} + + +def _collect_fee_pages(client, fee_after, max_pages): + """Every FEE activity created after the timezone-aware instant ``fee_after``, oldest first, + normalized. Each page repeats the filters and continues from the last activity id, as + alpaca-py 0.44.0 BrokerClient._get_account_activities_iterator keeps its request fields and + sets page_token to the last result's id. A page bound reached is "completeness unproven", + never a partial list; a repeated id means pagination did not advance.""" + after = fee_after_text(fee_after) + fees, seen, token = [], set(), None + for _ in range(max_pages): + params = {"after": after, "direction": "asc", "page_size": ACTIVITY_PAGE_SIZE} + if token is not None: + params["page_token"] = token + page = client.get("/account/activities/FEE", params) + if not isinstance(page, list) or len(page) > ACTIVITY_PAGE_SIZE: + raise TransportError("invalid fee activity page") + rows = [normalize_fee_activity(row) for row in page] + ids = [row["id"] for row in rows] + if len(set(ids)) != len(ids) or seen.intersection(ids): + raise TransportError("fee activity pagination did not advance") + fees.extend(rows) + seen.update(ids) + if len(page) < ACTIVITY_PAGE_SIZE: + return fees + token = ids[-1] + raise TransportError("fee activity page bound reached; completeness unproven") + + # Best-effort CTA/UTP quote-condition code that can mark an individual NBBO # quote update as halted ("H"). Alpaca's primary halt signal is the separate # trading status stream message parsed by normalize_trading_status below, not @@ -793,8 +893,10 @@ def request(self, method, url, **kwargs): "/v2/orders:by_client_order_id"} asset = path.startswith("/v2/assets/") and SYMBOL.fullmatch(path[len("/v2/assets/"):]) order_id = path.startswith("/v2/orders/") and UUID.fullmatch(path[len("/v2/orders/"):]) - # FILL activities only, and only filtered by one broker order (fill gap-fill). - activities = path == ACTIVITY_FILL_PATH and _activity_params(kwargs.get("params")) + # FILL activities only filtered by one broker order (fill gap-fill), and FEE + # activities only after one UTC instant (the ledger's cash baseline). Both are reads. + activities = ((path == ACTIVITY_FILL_PATH and _activity_params(kwargs.get("params"))) + or (path == ACTIVITY_FEE_PATH and _fee_activity_params(kwargs.get("params")))) allowed = ((method == "GET" and (path in reads or asset or order_id or activities)) or (not self.read_only and method == "POST" and path == "/v2/orders") or (not self.read_only and method == "DELETE" and order_id)) @@ -908,6 +1010,21 @@ def preflight(api_key, secret_key, symbols, *, feed="iex", before_request, reque data._session.close() +def fee_activities(api_key, secret_key, *, after, before_request, request_observer=None, max_pages=20): + """Read-only FEE activities created after the timezone-aware instant ``after``, oldest first, + for a caller that records them durably before a cash comparison it makes outside a transport + snapshot (runner.main's next-trial check). The snapshot's own allow-listed read, normalization + and page bound, on a fresh read-only client; ``before_request`` sees each GET as "read".""" + fee_after_text(after) # a naive instant is refused before any client or request + if type(max_pages) is not int or not 1 <= max_pages <= 100: + raise TransportError("bounded fee activity pages required") + trading = _sdk_client(api_key, secret_key, before_request, request_observer, read_only=True) + try: + return _collect_fee_pages(trading, after, max_pages) + finally: + trading._session.close() + + def _symbols(symbols): result = tuple(sorted(set(symbols))) if not result or len(result) > 30 or any(not isinstance(s, str) or not SYMBOL.fullmatch(s) for s in result): @@ -999,7 +1116,8 @@ def __init__(self, api_key, secret_key, symbols, *, before_request, before_submi sink_observation, queue_size=1024, quote_timeout=5.0, start_timeout=15.0, order_update_timeout=10.0, request_observer=None, history_start=None, max_snapshot_pages=20, required_quote_symbols=None, feed="iex", - extended_hours_allowed=False, include_margin=False, sink_status=None): + extended_hours_allowed=False, include_margin=False, sink_status=None, + fee_history_start=None): # sink_status receives every normalized trading status message of a subscribed # symbol (normalize_trading_status) on the owning loop, like sink_observation. self.sink_status = sink_status @@ -1027,6 +1145,12 @@ def __init__(self, api_key, secret_key, symbols, *, before_request, before_submi self.history_start = history_start or datetime.now(timezone.utc) if self.history_start.tzinfo is None or not 1 <= max_snapshot_pages <= 100: raise TransportError("bounded snapshot with timezone-aware history start required") + # snapshot() lists the FEE activities created after this instant: the ledger's cash + # baseline, which the caller knows (runner.main: the lane's first trial, whose cash is + # baseline_cash; mover_runner: each trial's own start, since it recomputes baseline_cash + # per trial). Defaults to history_start. + self.fee_history_start = fee_history_start or self.history_start + fee_after_text(self.fee_history_start) self.max_snapshot_pages = max_snapshot_pages self._events = queue.Queue(maxsize=queue_size) self._state_lock = threading.RLock() @@ -1562,9 +1686,12 @@ def collect(): "avg_entry_price": decimal_string(p["avg_entry_price"])} for p in self._client.get_all_positions()] orders = self._pages("open") + self._pages("all", after=self.history_start) - return account, positions, orders + # Last, after the account's cash: every listed fee is a posted activity. A fee + # posted between the two reads fails one cash comparison (fail closed). + fees = _collect_fee_pages(self._client, self.fee_history_start, self.max_snapshot_pages) + return account, positions, orders, fees try: - account, positions, raw_orders = await asyncio.to_thread(collect) + account, positions, raw_orders, fees = await asyncio.to_thread(collect) orders = {} for raw in raw_orders: order = await self._observe(normalize_order(raw)) @@ -1577,7 +1704,8 @@ def collect(): raise TransportError("owned intent absent from complete snapshot") orders[key] = found return {"account": account, "orders": list(orders.values()), "positions": positions, - "complete": True, "history_start": self.history_start.isoformat(), + "fees": fees, "complete": True, "history_start": self.history_start.isoformat(), + "fee_history_start": self.fee_history_start.isoformat(), "scope": "all_open_and_recent_plus_owned", "health": self.health} except Exception: self.freeze_health("snapshot_incomplete") diff --git a/tests/test_adaptive_paper_transport.py b/tests/test_adaptive_paper_transport.py index f95ebc177..cb6e29f3f 100644 --- a/tests/test_adaptive_paper_transport.py +++ b/tests/test_adaptive_paper_transport.py @@ -838,13 +838,15 @@ def request(method, url, **kwargs): async def test_proven_not_sent_intent_does_not_break_flat_snapshot(self): self.port.adopt_intents([intent()]) self.port._not_sent.add("trial-1") + # account, positions, open orders, all orders, then the FEE activities (read last). values = [response({"cash": "1000", "equity": "1000", "buying_power": "1000"}), - response([]), response([]), response([])] + response([]), response([]), response([]), response([])] with patch.object(self.port._client._session._session, "request", side_effect=values) as request: snapshot = await self.port.snapshot() self.assertTrue(snapshot["complete"]) self.assertEqual(snapshot["positions"], []) - self.assertEqual(request.call_count, 4) + self.assertEqual(snapshot["fees"], []) + self.assertEqual(request.call_count, 5) async def test_late_event_cannot_undo_cumulative_fill(self): final = t.normalize_order(order(filled_qty="1", filled_avg_price="100", status="filled")) From 941ec30004990d2d54ed855eabe717cc8ac8e573 Mon Sep 17 00:00:00 2001 From: Scout Date: Wed, 30 Sep 2026 13:24:36 -0400 Subject: [PATCH 3/7] Adaptive paper: document FEE reconciliation and refresh source hashes README-safety.md gains "Broker FEE activities" (sources quoted, fetched 2026-09-30) and a dated correction of the 2026-09-25 statement that paper does not simulate regulatory fees; README-mover.md and README-transport.md describe the fee window and the new read. source-hashes.json: the five changed files' digests and the new tests/test_adaptive_paper_fees.py. Co-Authored-By: Claude Opus 5.5 --- .../adaptive-paper/README-mover.md | 14 +- .../adaptive-paper/README-safety.md | 130 +++++++++++++++++- .../adaptive-paper/README-transport.md | 16 ++- .../adaptive-paper/source-hashes.json | 11 +- 4 files changed, 150 insertions(+), 21 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/README-mover.md b/blueprints/us-equities/adaptive-paper/README-mover.md index f64da0ff7..311f41205 100644 --- a/blueprints/us-equities/adaptive-paper/README-mover.md +++ b/blueprints/us-equities/adaptive-paper/README-mover.md @@ -155,11 +155,15 @@ exits, `floor(max_order_notional_usd / bid)` whole shares. ledger cap, the largest position exits (`gross_cap_guard`). The ledger halts permanently above its cap, and a rising mover would otherwise trip it. - **Reconciliation.** It runs at start (flat, no open orders), every 30 s while nothing - is in flight, and at the end: flat, and cash delta equal to this trial's fills. The - baseline is this trial's own starting cash. The engine's between-trial cash check is - kept as an observation rather than a refusal: the receipt carries - `inter_trial_cash_changed`, and the private `trial.json` holds the delta. Quote-driven - orders are suspended while a snapshot is in flight. + is in flight, and at the end: flat, and cash delta equal to this trial's fills plus + the broker FEE activities recorded from the snapshot. The baseline is this trial's + own starting cash, so the paper and recover transports read FEE activities created + after this trial's start (`current_trial_started_at`; a fee posted earlier is already + in the baseline). The receipt and the recovery receipt report `fees_recorded` (count, + total and sub-types, no ids); see README-safety.md, "Broker FEE activities". The + engine's between-trial cash check is kept as an observation rather than a refusal: + the receipt carries `inter_trial_cash_changed`, and the private `trial.json` holds + the delta. Quote-driven orders are suspended while a snapshot is in flight. ## Recovery diff --git a/blueprints/us-equities/adaptive-paper/README-safety.md b/blueprints/us-equities/adaptive-paper/README-safety.md index 6b5d7cb61..558dbfd8c 100644 --- a/blueprints/us-equities/adaptive-paper/README-safety.md +++ b/blueprints/us-equities/adaptive-paper/README-safety.md @@ -104,6 +104,10 @@ limit-order mechanism guarantees a flat finish. overlapping execution, or an execution price beyond the order's limit fails (`execution_conflict_requires_reconciliation`, `execution_overlap_requires_reconciliation`, `incremental_fill_violates_limit`). +- `record_fees(fees, now)` durably records broker FEE activities (the + `{id, date, net_amount, sub_type}` rows `transport.normalize_fee_activity` + returns) and returns the newly recorded ones; `fees()` lists what is recorded. + See "Broker FEE activities" below. - `request_budget(now, kind, client_id=None)` supports `submit`, `read`, `cancel` and `data_read`. Zero means an attempt was durably reserved. A positive delay means nothing was reserved: wait within the caller's remaining deadline, then @@ -149,10 +153,12 @@ fill delta; that derived notional is checked against the limit only beyond Alpac average at the limit no longer freezes, and the booked notional then differs from the executions' by at most that rounding. Cost basis uses weighted average inventory accounting. `cash_delta_usd` is the exact sum of observed gross -buy/sell cash flows; broker fees and other account activity require independent -reconciliation. `cumulative_realized_loss_usd` accumulates losing realized deltas; -wins do not erase it. `gross_loss_usd` adds current negative marked position P&L. -Drawdown measures the fall from the persisted peak total marked P&L. +buy/sell cash flows plus the broker FEE activities the ledger records (see "Broker +FEE activities" below); every other account activity requires independent +reconciliation. `cumulative_realized_loss_usd` accumulates losing realized deltas, +a recorded fee included; wins do not erase it. `gross_loss_usd` adds current +negative marked position P&L. Drawdown measures the fall from the persisted peak +total marked P&L. The transport/coordinator must validate event ownership, reconcile stream gaps, compare broker positions and cash, and stop on unexplained differences. Per-order @@ -171,6 +177,110 @@ a fresh trial or reset request history to hide unresolved state. Identify the actual recovery adapter: a direct SDK fractional exit does not establish that a whole-share native engine adapter supports fractional execution. +## Broker FEE activities (recorded, 2026-09-30) + +Every account 2 `mover_runner.py recover` attempt on 2026-09-30 (06:00 to 06:51 ET) +ended `needs_attention` with `cash_mismatch_or_unmodeled_fees` and no order: broker +cash minus the trial's baseline cash minus the ledger's execution cash flow was +-0.47 USD, three FEE activities for the account's 2026-09-29 sells (REG -0.19, +TAF -0.27, CAT -0.01). Paper posts these fees (see the correction under "Financing +costs"), so the engine records them. Sources (fetched 2026-09-30): + +* Alpaca, "Account Activities" (https://docs.alpaca.markets/us/docs/account-activities.md, + updatedAt 2026-05-25): a NonTradeActivity `id` is "An ID for the activity. Always + in `::` format. Can be sent as `page_token` in requests to facilitate the paging + of results." (the page renders the `::` form as `::`); + `net_amount` is "The net amount of money (positive or negative) associated with + the activity."; `date` is "The date on which the activity occurred or on which + the transaction associated with the activity settled."; `FEE` is "Fee + denominated in USD". +* Alpaca, "Retrieve Account Activities of Specific Type" + (https://docs.alpaca.markets/us/reference/getaccountactivitiesbyactivitytype-1.md, + updatedAt 2026-05-27): `GET /v2/account/activities/{activity_type}`; `after`: "Get + activities created after this date. Both formats YYYY-MM-DD and + YYYY-MM-DDTHH:MM:SSZ are supported."; its `date` filter adds "For non-trade + activities such as fees, the creation date is typically the day after the trade + date (in UTC)."; `page_size` 1 to 100; `page_token`: "Provide the ID of the last + activity from the last page to retrieve the next set of results."; the + NonTradeActivities example carries `activity_sub_type`, `activity_type`, + `created_at`, `currency`, `date`, `id`, `net_amount` and `status` (`executed`, + `correct` or `canceled`); ActivitySubType for FEE: REG (Regulatory Fee), TAF + (Trading Activity Fee), LCT, ORF, OCC, NRC, NRV, COM (Commission), CAT + (Consolidated Audit Trail Fee). +* Alpaca, "Regulatory Fees" (https://docs.alpaca.markets/docs/regulatory-fees, + updatedAt 2025-10-03): equities pay the Trading Activity Fee (TAF) on "Sells + only" and the Consolidated Audit Trail (CAT) fee on "Buys and sells"; "Alpaca's + trading system keeps track of the accrued FEE amounts intraday and deducts the + pending amounts from account balances." and "At EOD, we charge each account the + fees for that trading day". + +**Transport read.** `transport.py` allows one more read, a budgeted `read`: `GET +/v2/account/activities/FEE` with `after` (a UTC `YYYY-MM-DDTHH:MM:SSZ` string) and +`direction` `asc` required and an optional `page_size` 1-100 and activity-id +`page_token`. Every other path, parameter or method stays refused before the +budget hook. `normalize_fee_activity` accepts only `activity_type` FEE, an activity +id, a decimal string `net_amount`, a `YYYY-MM-DD` `date`, a documented FEE +sub-type (absent or null is recorded as `UNSPECIFIED`), `currency` absent or USD +and `status` absent or `executed`; anything else fails closed. It returns `{id, +date, net_amount, sub_type}` only: the `description` field, which carries the +account number, is never read, stored or logged. + +**Snapshot.** `AlpacaPaperTransport(fee_history_start=...)` (default +`history_start`) makes `snapshot()` list `"fees"`: every FEE activity created after +that instant, oldest first, read last (after the account's cash), 100 per page +within `max_snapshot_pages`. A page bound reached is "completeness unproven" and +any fee-read failure makes the snapshot incomplete (frozen), never an empty list. +`transport.fee_activities(...)` is the same read on a fresh read-only client. + +**Ledger schema 3.** Table `fees(activity_id TEXT PRIMARY KEY, date TEXT NOT +NULL, net_amount TEXT NOT NULL, sub_type TEXT NOT NULL, recorded_at REAL NOT +NULL)`. Opening a schema None/1/2 ledger creates it and sets `schema_version` "3"; +the migration is one way (engines before it accept only None/1/2 and refuse a +migrated ledger), and any other version raises `unsupported_ledger_schema` before +any change. `record_fees` validates every row, then inserts each new activity id +once, in one transaction: `net_amount` is added to `cash_delta` and to realized +P&L and `max(0, -net_amount)` to the realized loss, so a fee is a realized cost +that counts against the gross-loss budget and a credit never lowers it; peak P&L +and the risk halts follow as after a fill, and one `fee_recorded` event carries +the id, date, amount and sub-type. A known id is a no-op; the same id with another +date, amount or sub-type raises `fee_activity_changed` and records nothing from +that batch. The open-time accounting re-derivation adds recorded fees back beside +the per-symbol money. + +**Reconciliation.** `runner.reconcile`, and `mover_reconcile` through it, records +`snapshot["fees"]`, when the key is present, before its cash comparison; the +comparison and its 0.01 USD tolerance are unchanged. Deposits, withdrawals, +journals, dividends, interest and every other activity stay unexplained and still +fail it. + +**Fee window: the instant of the ledger's cash baseline.** `runner.main` (adaptive +lane) keeps its first trial's `baseline_cash`, read at trial.json `started_at`: its +transport reads fees from `started_at`, and its next-trial check +(`next_trial_cash_mismatch`) first reads the FEE activities posted since +`started_at` (`fee_activities`, each GET charged to the durable budget as a read) +and records them. `mover_runner.py` recomputes `baseline_cash` at every trial start +(preflight cash minus the ledger's `cash_delta`), so a fee posted before that start +is already inside the baseline: its paper and recover transports read fees from +trial.json `current_trial_started_at` (`mover_runner.fee_window_start`). A +`started_at` window there would hand each trial the fees posted overnight after +the previous one and fail its first reconciliation. + +**Receipts.** The mover trial receipt and the `mover_recovery_receipt` carry +`fees_recorded` (count, total and per sub-type count and total; no activity ids). +The trial receipt's `totals.fees_recorded_usd` joins the per-symbol P&L in +`pnl_consistent`. + +**Limits.** Fees posted between mover trials are absorbed into the next trial's +baseline and never enter the mover ledger's realized loss (as before this change). +A fee posted between one snapshot's account read and its fee read, or between a +preflight and the trial start, fails one cash comparison (fail closed). If paper +cash ever reflects the intraday pending accrual the Regulatory Fees page describes +before the FEE activity posts, reconciliation fails closed until it posts; the +2026-09-30 measurement, a gap equal to the three posted activities, did not show +that. The adaptive lane's fee window grows with the lane's life (more pages per +snapshot, bounded by `max_snapshot_pages`). `tests/test_adaptive_paper_fees.py` +covers this section with local synthetic fixtures (no broker request). + ## Leverage schedule (opt-in, `leverage-schedule-v1-20260922`) The default lane stays 1x: `runner.load_config` refuses `max_leverage > 1` @@ -334,9 +444,15 @@ margin-interest cost of an overnight hold. Sources (fetched 2026-09-25): month end. A settlement-date debit balance at the end of day Friday incurs 3 days of interest (Fri, Sat, Sun), since no trade settles over the weekend. * Alpaca, "Paper Trading" (https://docs.alpaca.markets/docs/paper-trading): - paper does not simulate regulatory fees or dividends; its "Paper vs Live" - table marks Borrow Fees "Coming Soon"; it does not say whether paper posts - margin interest at all. + its "Paper vs Live" table marks Borrow Fees "Coming Soon"; it does not say + whether paper posts margin interest at all. **Correction (2026-09-30):** this + bullet also said that paper "does not simulate regulatory fees or dividends". + The page, re-fetched 2026-09-30 (updatedAt 2026-07-07), still lists + "Regulatory fees" and "Dividends" under "paper trading does not account for", + while its "Rules and Assumptions" say only "Paper trading account **does NOT** + simulate dividends." A paper account showed three FEE activities (REG, TAF, + CAT) for its 2026-09-29 sells on 2026-09-30, so paper does post regulatory + fees; the engine records them (see "Broker FEE activities"). **Settlement convention:** `financing.settlement_date` is T+1, the next NYSE *trading* day (`sessions.next_trading_day`) -- not the separate SIFMA diff --git a/blueprints/us-equities/adaptive-paper/README-transport.md b/blueprints/us-equities/adaptive-paper/README-transport.md index e651d2687..8a7c58582 100644 --- a/blueprints/us-equities/adaptive-paper/README-transport.md +++ b/blueprints/us-equities/adaptive-paper/README-transport.md @@ -78,10 +78,18 @@ cumulative state never moves back. `fill_activities(order_id)` returns one order executions from `GET /v2/account/activities/FILL` filtered by the documented `order_id` parameter (ascending, 100 per page, the last activity id as `page_token`), each with its own `qty` and `price` and the order's `cum_qty` after it, and requires -them to tile the filled quantity from zero. It is the only activities read the -guarded session allows: another activity type, filter, page size or method is refused -before any request. An activity id is `::`; its 36-character UUID is -the native trade id. +them to tile the filled quantity from zero. The guarded session allows exactly one +other activities read (2026-09-30): `GET /v2/account/activities/FEE` with `after` (a +UTC `YYYY-MM-DDTHH:MM:SSZ` instant) and `direction` `asc`, plus an optional +`page_size` 1-100 and activity-id `page_token`. Another activity type, filter, page +size or method is refused before any request. `snapshot()` lists `"fees"`, the FEE +activities created after the transport's `fee_history_start` (default +`history_start`; the caller passes its ledger's cash-baseline instant), normalized by +`normalize_fee_activity` to `{id, date, net_amount, sub_type}` without the +`description` field, read after the account, bounded by `max_snapshot_pages`; a +fee-read failure makes the snapshot incomplete. `fee_activities(...)` is the same +read on a fresh read-only client (see README-safety.md, "Broker FEE activities"). An +activity id is `::`; its 36-character UUID is the native trade id. ## Pre-submission order-contract boundary diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index e7816849a..bc763bae6 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -18,7 +18,7 @@ "blueprints/us-equities/adaptive-paper/market_research.py": "bba9cf77c3c3c6dff32dec1cb01fa8c1957c6b012b7d5bd9693fcc268d557158", "blueprints/us-equities/adaptive-paper/metrics.py": "437fce8aaca350b9d8fe5d5b68212879d82e118f01d93fdcd333233709b80397", "blueprints/us-equities/adaptive-paper/mover.py": "ac34fd84485a054b27fb857b5ef3b0dfb7f7dee9c407eb42d4579e9e80e5ee88", - "blueprints/us-equities/adaptive-paper/mover_runner.py": "03f9107e40feaaefc6e896db3e20d246139f8ef8b02e1a7d7da79199e62bb7f5", + "blueprints/us-equities/adaptive-paper/mover_runner.py": "d1e79e5578d40caf50e01c4cb1e6905042e24e081df489dce2790388fbcec940", "blueprints/us-equities/adaptive-paper/mover_simulation.py": "8ac5955de1e814bb26fbfbca068a5719b9033b2bae0d8e50e78d7a78a546761e", "blueprints/us-equities/adaptive-paper/mover_strategy.py": "4730c5e535aa480030d6484669a9d8565ca25e4e1ea50ea12a45bd1892536cba", "blueprints/us-equities/adaptive-paper/native_adapter.py": "fbf530884c2ad3b9e0c2b1f989eda399531647cc1db3c56dc9bcac10174fea47", @@ -27,15 +27,15 @@ "blueprints/us-equities/adaptive-paper/recovery.py": "38b6cf4f9247577f72337ca3dbd242505a43ad5cbc5cbe21349b0f1beb2f3e51", "blueprints/us-equities/adaptive-paper/registry.json": "60da73b985e718bd1dece7a3350f34636bda0fb38e766a44ae71c5a8c532282a", "blueprints/us-equities/adaptive-paper/requirements.txt": "518afe4b77f901e87abd3a5d6faa126aa7e211db77eecec80cfe7ef42d82e07a", - "blueprints/us-equities/adaptive-paper/runner.py": "27bc3fdc80472c3ee218d3723e506288ebc6eef536dafc8365b30474213a6ae4", - "blueprints/us-equities/adaptive-paper/safety.py": "c2d44faa1e4d369b8882dec9782ef32174c404b723edd2419833146b24eb2357", + "blueprints/us-equities/adaptive-paper/runner.py": "1e0f796f97bbe38d2e516bb8148954ddf7112edd6f1fd14b1d2d1c4ab8d6fb44", + "blueprints/us-equities/adaptive-paper/safety.py": "3a0d8f991969d471e5d67dd3265a02c14cdf411cb3761c96e69e0d5efbc3c802", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", "blueprints/us-equities/adaptive-paper/sessions.py": "95f0affee512e88dd44f196fa72bb7471e00899356793bdbb12f15c4ed84e64d", "blueprints/us-equities/adaptive-paper/simulation.py": "0119e0efe3b498be3c415eee9c1a0be922c9b5b17e3b774aa7528668c4eb0b30", "blueprints/us-equities/adaptive-paper/strategies.py": "2d488a4fdd21ad323ab07cf7fb147b3623fd6e38951abcf2fc59cf8094a1383e", "blueprints/us-equities/adaptive-paper/strategies_v1.py": "1952be269694485e12e4d927f124adca0ee29667588d6ac6651caede454c92f7", - "blueprints/us-equities/adaptive-paper/transport.py": "e3038bd5562fecc7cea52df47c4ff6b94ddae7e85ce545d6684e651b856b89d9", + "blueprints/us-equities/adaptive-paper/transport.py": "e0ec1d3f4b7a5e3fb8e6d1af0ccd0f033e6a790199baae3f9b8d14758d1f565b", "tests/_credential_mutation_driver.py": "bfe35e69702016718452ed702aab20a0a33da5723e28a61de513b3c268849301", "tests/adaptive_paper_hermetic.py": "0f41601cd10abf10ccba4f17d9c35716bb766a5e9951a4390649d6c9387fc1da", "tests/test_adaptive_market_research.py": "772b4fd467d4405bd098a440c54de4bfd61f91bbb80693c052077b867f9e2fd6", @@ -43,6 +43,7 @@ "tests/test_adaptive_paper_credential_race.py": "1b0b6521045858fc9c974fc9e50e390ac888c7091f5f92606feb1bf49f7ef022", "tests/test_adaptive_paper_credential_source.py": "32c3aa93e40392a2af60078dd4ba6c222e195a9a47b22a0bfc0f52a92814382a", "tests/test_adaptive_paper_exits.py": "13a82d65ad1b2e72a8ca0fbb8bbb178a01ded1301399604c9e304ae213456764", + "tests/test_adaptive_paper_fees.py": "ff162965f2e04bbd17c39b6c83062ef42c54f2a2fc82de73a5ee78223d70fc97", "tests/test_adaptive_paper_financing.py": "8d9f1f21f9d53c9ed120281409bdcb97b8c9ddacc615d0cdfa392cfcf111fa90", "tests/test_adaptive_paper_live_manifest.py": "4596f301ac44effefee5b787c5e3627a9cd38f8ff0cf436ef778b7378a0667c2", "tests/test_adaptive_paper_metrics.py": "1d6eaf940021fbaba3ac687a3d3a2d9f6e5492e85a5a372621299d4363ec3ee5", @@ -55,5 +56,5 @@ "tests/test_adaptive_paper_selector.py": "27b431ac7911372d050f25d32ac24a3df9d2cf3c5e7dafdebfe51db6cab55936", "tests/test_adaptive_paper_sessions.py": "1e8b33ec8b496807468067112f5b7f0a2a6d6ce897ac7412605678b94984fc6a", "tests/test_adaptive_paper_strategies.py": "f0814b36909f2fc6b4c4213b4142ffc13339740f0397c99ad9c7c81982e2033a", - "tests/test_adaptive_paper_transport.py": "e4914d1755fb70c23688cb118dc482ea95c15e57ebb3a081813f8083a385b396" + "tests/test_adaptive_paper_transport.py": "7d75e7ae89dc048cd62628fcfd666ff18eaf3daedbed836b0a451ddaf7a45f87" } From 4fd85451a3495486916f0e703b19b3aaa07a1153 Mon Sep 17 00:00:00 2001 From: Scout Date: Wed, 30 Sep 2026 15:15:45 -0400 Subject: [PATCH 4/7] Adaptive paper: fee review round 1 repairs (checkpoint, per-fee risk, admitted reads) Two independent reviews (gpt-6.1-sol and Claude Opus) of 7f9e12f2 found: - the fee window and the cash baseline were taken at different moments; - risk was refreshed only once per fee batch; - the adaptive next-trial fee read was charged to the budget after sending; - there were untestable privacy and tolerance guarantees. - transport.fee_checkpoint: one read-only client reads F1 (FEE list), the account, then F2, and refuses a start when F1 != F2. - mover_runner.command_paper and runner.main's first trial book F2 before the baseline (checkpoint cash - cash_delta) and save fee_window_start. - Every later snapshot, recover and the adaptive next-trial read use that same window. - Legacy metadata keeps current_trial_started_at (mover) or started_at (adaptive), so the account-2 residual's recovery reads after 2026-09-29T20:25:19Z. - Ledger.record_fees refreshes risk after each new fee; a risk halt now replaces recovery_only (stricter; sells stay allowed). - Checkpoint and next-trial GETs are admitted through the durable read budget before sending, and failed pages keep their reservation. - Tests: batch-versus-separate risk, the checkpoint timelines, legacy window, budget refusal before HTTP, description access trap with stdout/stderr/log capture, exact 0.01 tolerance boundaries. The copied accept-list test is renamed. - Docs: the checkpoint design, its assumption and limits; the "fails once" claims removed. Builder: gpt-6.1-sol (ultra) test-first in the owned worktree (fee-fix/REPAIR-BRIEF.md). Coordinator gate on this tree: 9 modules, 462 tests OK. Co-Authored-By: Claude Opus 5.5 --- .../adaptive-paper/README-mover.md | 12 +- .../adaptive-paper/README-safety.md | 66 ++- .../adaptive-paper/README-transport.md | 13 +- .../adaptive-paper/mover_runner.py | 46 +- .../us-equities/adaptive-paper/runner.py | 54 +- .../us-equities/adaptive-paper/safety.py | 5 +- .../us-equities/adaptive-paper/transport.py | 45 +- tests/test_adaptive_paper_fees.py | 469 ++++++++++++++++-- tests/test_adaptive_paper_mover_native.py | 6 + 9 files changed, 609 insertions(+), 107 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/README-mover.md b/blueprints/us-equities/adaptive-paper/README-mover.md index 311f41205..acce62a92 100644 --- a/blueprints/us-equities/adaptive-paper/README-mover.md +++ b/blueprints/us-equities/adaptive-paper/README-mover.md @@ -156,10 +156,14 @@ exits, `floor(max_order_notional_usd / bid)` whole shares. permanently above its cap, and a rising mover would otherwise trip it. - **Reconciliation.** It runs at start (flat, no open orders), every 30 s while nothing is in flight, and at the end: flat, and cash delta equal to this trial's fills plus - the broker FEE activities recorded from the snapshot. The baseline is this trial's - own starting cash, so the paper and recover transports read FEE activities created - after this trial's start (`current_trial_started_at`; a fee posted earlier is already - in the baseline). The receipt and the recovery receipt report `fees_recorded` (count, + the broker FEE activities recorded from the snapshot. Before every trial, a budgeted + F1/account/F2 checkpoint refuses differing fee maps, books stable fees, then computes + baseline as checkpoint cash minus ledger cash delta before beginning the trial. The + lineage's `fee_window_start` is saved and reused by every paper/recover snapshot; + legacy recovery without that key retains `current_trial_started_at`. This assumes + fee activity visibility coincides with its inclusion in cash. A checkpoint refusal + leaves the trial unentered for a later retry. The receipt and the recovery receipt + report `fees_recorded` (count, total and sub-types, no ids); see README-safety.md, "Broker FEE activities". The engine's between-trial cash check is kept as an observation rather than a refusal: the receipt carries `inter_trial_cash_changed`, and the private `trial.json` holds diff --git a/blueprints/us-equities/adaptive-paper/README-safety.md b/blueprints/us-equities/adaptive-paper/README-safety.md index 558dbfd8c..b703865e7 100644 --- a/blueprints/us-equities/adaptive-paper/README-safety.md +++ b/blueprints/us-equities/adaptive-paper/README-safety.md @@ -241,7 +241,7 @@ any change. `record_fees` validates every row, then inserts each new activity id once, in one transaction: `net_amount` is added to `cash_delta` and to realized P&L and `max(0, -net_amount)` to the realized loss, so a fee is a realized cost that counts against the gross-loss budget and a credit never lowers it; peak P&L -and the risk halts follow as after a fill, and one `fee_recorded` event carries +and the risk halts refresh after each newly inserted fee, as after each fill, and one `fee_recorded` event carries the id, date, amount and sub-type. A known id is a no-op; the same id with another date, amount or sub-type raises `fee_activity_changed` and records nothing from that batch. The open-time accounting re-derivation adds recorded fees back beside @@ -253,33 +253,55 @@ comparison and its 0.01 USD tolerance are unchanged. Deposits, withdrawals, journals, dividends, interest and every other activity stay unexplained and still fail it. -**Fee window: the instant of the ledger's cash baseline.** `runner.main` (adaptive -lane) keeps its first trial's `baseline_cash`, read at trial.json `started_at`: its -transport reads fees from `started_at`, and its next-trial check -(`next_trial_cash_mismatch`) first reads the FEE activities posted since -`started_at` (`fee_activities`, each GET charged to the durable budget as a read) -and records them. `mover_runner.py` recomputes `baseline_cash` at every trial start -(preflight cash minus the ledger's `cash_delta`), so a fee posted before that start -is already inside the baseline: its paper and recover transports read fees from -trial.json `current_trial_started_at` (`mover_runner.fee_window_start`). A -`started_at` window there would hand each trial the fees posted overnight after -the previous one and fail its first reconciliation. +**Fee window: a consistent checkpoint (repair brief, 2026-09-30).** Under the +account lock, `transport.fee_checkpoint` uses one fresh read-only client to read +F1 (the FEE list after L), then the account, then F2 with the same filter. It +compares id-to-normalized-row maps and refuses a start if they differ. Every GET +is admitted synchronously through the ledger's durable read budget before sending. +The mover uses the saved `fee_window_start` of a continuing lineage (legacy: +`started_at`); a new lineage takes L immediately before the checkpoint. It books +F2 before `begin_next_trial` and before taking the accounting snapshot, then +computes `baseline_cash = checkpoint account cash - ledger cash_delta`. The +between-trial cash observation also uses checkpoint cash. `trial.json` saves L +as epoch seconds; every paper/recover snapshot uses its same formatted `after` +string. The adaptive lane does this at its first trial, keeps that baseline, and +uses L for snapshots and its synchronously admitted next-trial fee read (legacy: +`started_at`). The old preflight account read precedes `started_at`. + +Assumption: a FEE activity is visible in the activity list exactly when its +amount is in account cash. Under that assumption, every fee in F2 is in both +checkpoint cash and ledger `cash_delta` before computing the baseline. Later +fees after the fixed cutoff are listed and booked once by id. Fees before the +cutoff are in cash and never listed, so the per-trial baseline neither double +counts nor misses a fee. Alpaca's documented `after` has whole-second precision: +formatting fractional L can include a fee from earlier within that second, which +is also booked before baseline and deduplicated thereafter. Fees earlier engines +absorbed into a baseline are booked at the next checkpoint; recomputing baseline +after booking preserves cash reconciliation. Fee losses still consume the risk +budget during recovery: a cap halt replaces `recovery_only`, while the sell-only +restriction remains in force. **Receipts.** The mover trial receipt and the `mover_recovery_receipt` carry `fees_recorded` (count, total and per sub-type count and total; no activity ids). The trial receipt's `totals.fees_recorded_usd` joins the per-symbol P&L in `pnl_consistent`. -**Limits.** Fees posted between mover trials are absorbed into the next trial's -baseline and never enter the mover ledger's realized loss (as before this change). -A fee posted between one snapshot's account read and its fee read, or between a -preflight and the trial start, fails one cash comparison (fail closed). If paper -cash ever reflects the intraday pending accrual the Regulatory Fees page describes -before the FEE activity posts, reconciliation fails closed until it posts; the -2026-09-30 measurement, a gap equal to the three posted activities, did not show -that. The adaptive lane's fee window grows with the lane's life (more pages per -snapshot, bounded by `max_snapshot_pages`). `tests/test_adaptive_paper_fees.py` -covers this section with local synthetic fixtures (no broker request). +**Limits.** A fee posted during the checkpoint refuses that start; the next +start retries. Fees of a lineage's last day that post after its last trial are +never booked into that lineage: cash remains consistent because the next lineage +absorbs them into its baseline (except any included by the whole-second cutoff, +which are booked before baseline). A fee posted between a later snapshot's +account and fee reads can still cause a cash mismatch; reconciliation fails +closed. Under the Regulatory Fees page's intraday-accrual hypothesis, a mover +trial starting after an earlier trial's accrual stays blocked once the FEE posts +inside its window, until an operator re-baselines. The 2026-09-30 measured gap +equaled the three posted activities; that observation did not establish pending +accrual behavior. Legacy mover metadata without `fee_window_start` keeps +`current_trial_started_at` for recovery, including the account-2 residual's +2026-09-29T20:25:19Z window. Legacy adaptive metadata keeps `started_at`. Both +lineages' fee windows grow with their life (more pages per snapshot, bounded by +`max_snapshot_pages`). `tests/test_adaptive_paper_fees.py` covers this section +with local synthetic fixtures (no broker request). ## Leverage schedule (opt-in, `leverage-schedule-v1-20260922`) diff --git a/blueprints/us-equities/adaptive-paper/README-transport.md b/blueprints/us-equities/adaptive-paper/README-transport.md index 8a7c58582..a63c09c33 100644 --- a/blueprints/us-equities/adaptive-paper/README-transport.md +++ b/blueprints/us-equities/adaptive-paper/README-transport.md @@ -84,11 +84,20 @@ UTC `YYYY-MM-DDTHH:MM:SSZ` instant) and `direction` `asc`, plus an optional `page_size` 1-100 and activity-id `page_token`. Another activity type, filter, page size or method is refused before any request. `snapshot()` lists `"fees"`, the FEE activities created after the transport's `fee_history_start` (default -`history_start`; the caller passes its ledger's cash-baseline instant), normalized by +`history_start`; new callers reuse their saved lineage checkpoint window), normalized by `normalize_fee_activity` to `{id, date, net_amount, sub_type}` without the `description` field, read after the account, bounded by `max_snapshot_pages`; a fee-read failure makes the snapshot incomplete. `fee_activities(...)` is the same -read on a fresh read-only client (see README-safety.md, "Broker FEE activities"). An +read on a fresh read-only client. `fee_checkpoint(...)` uses one fresh read-only +client for F1/account/F2 with the same `after`, and refuses changed id-to-normalized-row +maps as `fee_activity_posted_during_checkpoint`. Each GET is admitted synchronously +as a durable `read` before sending. Assuming fee visibility exactly coincides with +its inclusion in cash, callers book F2 before calculating baseline as checkpoint +cash minus ledger cash delta, and reuse that formatted cutoff on later reads. +Whole-second formatting can include earlier same-second fees; booking before baseline +and deduplicating by id keeps cash consistent. A refused checkpoint leaves a start +unentered for retry. Legacy mover recovery uses `current_trial_started_at`; legacy +adaptive metadata uses `started_at` (see README-safety.md, "Broker FEE activities"). An activity id is `::`; its 36-character UUID is the native trade id. ## Pre-submission order-contract boundary diff --git a/blueprints/us-equities/adaptive-paper/mover_runner.py b/blueprints/us-equities/adaptive-paper/mover_runner.py index c4213879a..f55e776cb 100644 --- a/blueprints/us-equities/adaptive-paper/mover_runner.py +++ b/blueprints/us-equities/adaptive-paper/mover_runner.py @@ -147,7 +147,7 @@ def mover_reconcile(ledger, snapshot, baseline_cash): """runner.reconcile over the ledger's own orders plus every foreign order that is not terminal (which still fails as external_order_detected). Positions and cash are compared unchanged, after runner.reconcile records the snapshot's FEE activities; the - mover's baseline is the trial's own starting cash.""" + mover's baseline is recomputed from checkpoint cash after booking its fees.""" owned = {i.client_id for i in ledger.intents()} orders = [o for o in snapshot.get("orders", []) if o.get("client_order_id") in owned or o.get("status") not in FOREIGN_TERMINAL_STATUSES] @@ -155,13 +155,13 @@ def mover_reconcile(ledger, snapshot, baseline_cash): def fee_window_start(metadata): - """The instant a mover transport lists FEE activities from: this trial's own cash - baseline. command_paper recomputes baseline_cash at every trial start (preflight cash - minus the ledger's cash_delta), so a fee posted before the trial started is already in - that baseline and recording it again would count it twice; trial.json started_at (the - lane's first trial) would do exactly that for the fees posted after each trading day. - runner.main keeps its first trial's baseline instead and reads fees from started_at.""" - value = metadata.get("current_trial_started_at") if isinstance(metadata, dict) else None + """The mover lineage's saved checkpoint window, reused by paper and recover. + Checkpoint fees are booked before baseline is recomputed; the fixed formatted cutoff + then lists later fees once by id. Legacy trials without fee_window_start retain + current_trial_started_at, including the account-2 residual's 2026-09-29T20:25:19Z. + A present invalid checkpoint window is refused rather than falling back.""" + value = (metadata.get("fee_window_start", metadata.get("current_trial_started_at")) + if isinstance(metadata, dict) else None) if type(value) not in (int, float) or not math.isfinite(value) or value <= 0: raise SafetyError("trial_metadata_lacks_baseline_time") return datetime.fromtimestamp(value, timezone.utc) @@ -655,7 +655,8 @@ def command_paper(args): LAST_EVIDENCE_CLASS = "PAPER" if not TRIAL_ID.fullmatch(args.trial): raise ValueError("invalid_trial_id") - from transport import AlpacaPaperTransport, TransportError, halt_statuses_supported, nasdaq_halt_seed, preflight + from transport import (AlpacaPaperTransport, TransportError, halt_statuses_supported, nasdaq_halt_seed, + preflight, fee_checkpoint) config, limits, settings = load_mover_config(args.config) config_sha = _sha256(args.config.read_bytes()) try: @@ -736,6 +737,19 @@ def command_paper(args): raise SafetyError("preflight_budget_inconsistent") if ledger.positions() or ledger.unresolved(): raise SafetyError("next_trial_requires_recovery") + # Keep the receipt's accounting baseline before every fee recorded in + # this invocation; sizing and cash baseline use accounting AFTER booking. + receipt_before = ledger.accounting() + known_fees = {row["activity_id"] for row in ledger.fees()} + def admit_checkpoint_read(kind, **_): + if ledger.request_budget(time.time(), "read"): + raise SafetyError("fee_checkpoint_budget_exhausted") + window_start = (previous.get("fee_window_start", previous["started_at"]) + if previous is not None else time.time()) + checkpoint = fee_checkpoint(key, secret, after=fee_window_start({"fee_window_start": window_start}), + before_request=admit_checkpoint_read, request_observer=responses.append) + ledger.record_fees(checkpoint["fees"], time.time()) + now = time.time() # trial must start after its checkpoint's reserved GETs scan = load_scan(raw, settings, now=now) # freshness again, at trial start before = ledger.accounting() equity = limits.capital_usd + before.realized_pnl_usd @@ -749,27 +763,27 @@ def command_paper(args): plan = build_plan(settings, limits, scan, session_plan, trial_id=args.trial, evidence_class="PAPER", t0=now, equity=equity, engine_leverage_multiple=multiple) ledger.begin_next_trial(now, args.trial) - except SafetyError as exc: + except (SafetyError, TransportError) as exc: return not_started(args.output, evidence_class="PAPER", reason=str(exc), stage="trial_start", config_sha256=config_sha, scan_sha256=scan_sha, preflight=summary) - baseline = Decimal(observation["account"]["cash"]) - before.cash_delta_usd + baseline = Decimal(checkpoint["account"]["cash"]) - before.cash_delta_usd # The engine's next_trial_cash_mismatch quantity, kept as an observation: the # baseline is per trial, so activity between mover trials (e.g. another lane on # a shared account) is recorded here instead of refusing the trial. inter_trial = (None if previous is None or previous.get("baseline_cash") is None else - Decimal(observation["account"]["cash"]) + Decimal(checkpoint["account"]["cash"]) - (Decimal(previous["baseline_cash"]) + before.cash_delta_usd)) history = list((previous or {}).get("history", [])) metadata = {"lane": "mover", "protocol": PROTOCOL_ID, "trial_id": args.trial, "config_sha256": config_sha, "scan_sha256": scan_sha, "symbols": list(plan.symbol_names()), "shared_account_override": bool(args.allow_shared_account), "started_at": (previous or {}).get("started_at", now), "current_trial_started_at": now, + "fee_window_start": window_start, "baseline_cash": format(baseline, "f"), "phase": "starting", "inter_trial_cash_change_usd": None if inter_trial is None else format(inter_trial, "f"), "lane_state": session_state_after(session_plan, equity_end=equity), "history": history} save(metadata_path, metadata) fee_window = fee_window_start(metadata) - known_fees = {row["activity_id"] for row in ledger.fees()} controller_close, market_open = _controller_session(close, now, session_policy) controller = MoverController(ledger, controller_close, market_open=market_open, max_entry_notional_usd=settings.max_entry_notional_usd) @@ -794,7 +808,7 @@ def fresh_port(): required_quote_symbols=list(settings.benchmarks), # As runner.main: from the lane's first trial, so reconcile sees every owned intent. history_start=datetime.fromtimestamp(metadata["started_at"], timezone.utc), - # FEE activities from this trial's own cash baseline (fee_window_start). + # The fixed checkpoint window, shared by every trial of this lineage. fee_history_start=fee_window, extended_hours_allowed=session_policy["extended_hours"], include_margin=limits.leverage is not None)) @@ -828,7 +842,7 @@ async def execute(): outcome = asyncio.run(execute()) receipt = build_receipt(plan=plan, outcome=outcome, config_sha256=config_sha, scan=scan, ledger=ledger, - ledger_before=before, prefixes=prefixes, preflight=summary, + ledger_before=receipt_before, prefixes=prefixes, preflight=summary, extra={"shared_account_override": bool(args.allow_shared_account), "inter_trial_cash_changed": (None if inter_trial is None else abs(inter_trial) > Decimal("0.01"))}, @@ -890,7 +904,7 @@ def command_recover(args): controller_close, market_open = _controller_session(close, time.time(), session_policy) controller = MoverController(ledger, controller_close, market_open=market_open, max_entry_notional_usd=settings.max_entry_notional_usd) - # The trial's own cash baseline (trial.json baseline_cash) opens the FEE window. + # Reuse the checkpoint window; legacy residuals keep their original trial window. fee_window = fee_window_start(metadata) known_fees = {row["activity_id"] for row in ledger.fees()} # #215: only the benchmark quotes gate the recovery transport; each owned diff --git a/blueprints/us-equities/adaptive-paper/runner.py b/blueprints/us-equities/adaptive-paper/runner.py index 80958fcef..e73d23b3a 100644 --- a/blueprints/us-equities/adaptive-paper/runner.py +++ b/blueprints/us-equities/adaptive-paper/runner.py @@ -36,7 +36,7 @@ must_end_flat, next_trading_day, session_at, validate_session_policy) from strategies import AdaptivePolicy, PolicyConfig, RegimeSelector, SelectorConfig, limit_price from transport import (AlpacaPaperTransport, DATA_FEEDS, TransportError, RejectedSubmission, order_contract_status, - preflight, halt_statuses_supported, nasdaq_halt_seed, fee_activities) + preflight, halt_statuses_supported, nasdaq_halt_seed, fee_activities, fee_checkpoint) SOURCE = Path(__file__).resolve().parent LAST_OUTPUT = None @@ -2380,26 +2380,45 @@ def observe_request(kind, **kwargs): if metadata["config_sha256"] != summary["config_sha256"]: raise SafetyError("recovery_config_differs_from_frozen_trial") else: + fee_start = (previous_metadata.get("fee_window_start", previous_metadata["started_at"]) + if previous_metadata else None) + if previous_metadata is None: + def admit_checkpoint_read(kind, **_): + if ledger.request_budget(time.time(), "read"): + raise SafetyError("fee_checkpoint_budget_exhausted") + fee_start = time.time() + try: + checkpoint = fee_checkpoint(key, secret, after=datetime.fromtimestamp(fee_start, timezone.utc), + before_request=admit_checkpoint_read, + request_observer=responses.append) + except (TransportError, SafetyError) as exc: + if str(exc) not in ("fee_activity_posted_during_checkpoint", "fee_checkpoint_budget_exhausted"): + raise + summary.update(status="not_started", stage="trial_start", reason=str(exc)) + save(args.output, summary) + print(json.dumps({"status": "not_started", "stage": "trial_start", "reason": str(exc), + "orders_submitted": 0})) + return 2 + ledger.record_fees(checkpoint["fees"], time.time()) + baseline_cash = format(Decimal(checkpoint["account"]["cash"]) + - ledger.accounting().cash_delta_usd, "f") + now = time.time() if previous_metadata: if previous_metadata["config_sha256"] != summary["config_sha256"]: raise SafetyError("next_trial_config_differs_from_frozen_limits") if (ledger.positions() or ledger.unresolved()) and not resumable_hold: raise SafetyError("next_trial_requires_recovery") if not resumable_hold: - # baseline_cash is the first trial's cash (started_at), so FEE activities - # posted since then, such as the day's REG/TAF/CAT after the last trial - # ended, are recorded durably first, through the transport's allow-listed - # read and normalization. Each GET is charged to the durable budget as a - # read, and `now` follows those reads, so the next trial never starts - # before a recorded request. Any other unexplained change still refuses. - fee_reads = [] + # Keep the first checkpoint's cash baseline and fee window (legacy: + # started_at). Reserve every read durably before sending, including + # failed pages, and book the fees before comparing unchanged cash tolerance. + def admit_next_fee_read(kind, **_): + if ledger.request_budget(time.time(), "read"): + raise SafetyError("preflight_budget_inconsistent") fees = fee_activities(key, secret, - after=datetime.fromtimestamp(previous_metadata["started_at"], timezone.utc), - before_request=lambda kind, **_: fee_reads.append(time.time()), + after=datetime.fromtimestamp(fee_start, timezone.utc), + before_request=admit_next_fee_read, request_observer=responses.append) - for timestamp in fee_reads: - if ledger.request_budget(timestamp, "read"): - raise SafetyError("preflight_budget_inconsistent") now = time.time() ledger.record_fees(fees, now) expected_cash = Decimal(previous_metadata["baseline_cash"]) + ledger.accounting().cash_delta_usd @@ -2420,7 +2439,8 @@ def observe_request(kind, **kwargs): metadata = {"trial_id": args.trial, "started_at": previous_metadata["started_at"] if previous_metadata else now, "current_trial_started_at": now, "config_sha256": summary["config_sha256"], - "baseline_cash": previous_metadata["baseline_cash"] if previous_metadata else observation["account"]["cash"], + "fee_window_start": fee_start, + "baseline_cash": previous_metadata["baseline_cash"] if previous_metadata else baseline_cash, "phase": "starting"} save(metadata_path, metadata) def fresh_port(recovering=False): @@ -2432,8 +2452,10 @@ def fresh_port(recovering=False): quote_timeout=config["quote_max_age_seconds"], feed=config["feed"], required_quote_symbols=needed if recovering and needed else config["benchmarks"], history_start=datetime.fromtimestamp(metadata["started_at"], timezone.utc), - # This lane keeps its first trial's baseline_cash, read at started_at. - fee_history_start=datetime.fromtimestamp(metadata["started_at"], timezone.utc), + # This lane keeps its first checkpoint baseline and fee window; legacy + # metadata keeps started_at, which follows the original account read. + fee_history_start=datetime.fromtimestamp(metadata.get("fee_window_start", metadata["started_at"]), + timezone.utc), extended_hours_allowed=session_policy["extended_hours"], include_margin=leverage_policy is not None)) for sig in (signal.SIGINT, signal.SIGTERM): diff --git a/blueprints/us-equities/adaptive-paper/safety.py b/blueprints/us-equities/adaptive-paper/safety.py index 48fe476ce..b34b14d16 100644 --- a/blueprints/us-equities/adaptive-paper/safety.py +++ b/blueprints/us-equities/adaptive-paper/safety.py @@ -802,7 +802,7 @@ def _refresh_risk(self, now=None): reason = "drawdown_cap_reached" elif state.gross_exposure_usd > cap: reason = "gross_exposure_cap_exceeded" - if reason and not state.halted_reason: + if reason and state.halted_reason in (None, "recovery_only"): self._set("halted_reason", reason) self._event("risk_halt", reason=reason) return self._state() @@ -1346,10 +1346,9 @@ def record_fees(self, fees, now): for name, delta in (("cash_delta", amount), ("realized", amount), ("realized_loss", max(ZERO, -amount))): self._set(name, D(self._get(name)) + delta) + self._refresh_risk(now) self._event("fee_recorded", activity_id=activity_id, date=day, net_amount=text, sub_type=sub_type) recorded.append({"id": activity_id, "date": day, "net_amount": text, "sub_type": sub_type}) - if recorded: - self._refresh_risk(now) return recorded def request_budget(self, now, kind, client_id=None): diff --git a/blueprints/us-equities/adaptive-paper/transport.py b/blueprints/us-equities/adaptive-paper/transport.py index ff53f3397..f58a60b30 100644 --- a/blueprints/us-equities/adaptive-paper/transport.py +++ b/blueprints/us-equities/adaptive-paper/transport.py @@ -432,9 +432,9 @@ def _activity_params(params): def fee_after_text(moment): """The documented UTC ``after`` value (YYYY-MM-DDTHH:MM:SSZ) for a timezone-aware instant, - truncated to the second: the window can open up to one second early, never late. A fee in - that second that a cash baseline already holds is then counted twice and fails the cash - check once (fail closed); it is never silently dropped.""" + truncated to the second. The checkpoint and every subsequent fee read use this same + formatted cutoff. Fees from earlier within that second are booked by the checkpoint + before computing the cash baseline, and subsequent reads deduplicate them by id.""" if not isinstance(moment, datetime) or moment.tzinfo is None: raise TransportError("timezone-aware fee window required") return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -1025,6 +1025,35 @@ def fee_activities(api_key, secret_key, *, after, before_request, request_observ trading._session.close() +def fee_checkpoint(api_key, secret_key, *, after, before_request, request_observer=None, max_pages=20): + """Read F1, account cash, then F2 on one fresh read-only client; refuse a changed + id -> normalized row map. Reuse the Alpaca account-activities endpoint and pagination + documented above; before_request synchronously admits each GET as a read. + + Assuming a FEE activity is visible exactly when its amount is in account cash, + every fee in F2 is in checkpoint cash and must be booked into ledger cash_delta + before computing baseline = cash - cash_delta. Later fees after the fixed formatted + L cutoff are booked once by id; fees before that cutoff are in cash and never listed. + Thus the per-trial baseline neither double counts nor misses a fee. Truncation can + include an earlier fee in L's fractional second; it too is booked before baseline. + Fees earlier engines absorbed into baseline are booked at the next checkpoint and + baseline is recomputed afterwards, preserving cash reconciliation. + """ + fee_after_text(after) + if type(max_pages) is not int or not 1 <= max_pages <= 100: + raise TransportError("bounded fee activity pages required") + trading = _sdk_client(api_key, secret_key, before_request, request_observer, read_only=True) + try: + first = _collect_fee_pages(trading, after, max_pages) + account = normalize_account(trading.get_account()) + second = _collect_fee_pages(trading, after, max_pages) + if {row["id"]: row for row in first} != {row["id"]: row for row in second}: + raise TransportError("fee_activity_posted_during_checkpoint") + return {"account": account, "fees": second} + finally: + trading._session.close() + + def _symbols(symbols): result = tuple(sorted(set(symbols))) if not result or len(result) > 30 or any(not isinstance(s, str) or not SYMBOL.fullmatch(s) for s in result): @@ -1145,10 +1174,8 @@ def __init__(self, api_key, secret_key, symbols, *, before_request, before_submi self.history_start = history_start or datetime.now(timezone.utc) if self.history_start.tzinfo is None or not 1 <= max_snapshot_pages <= 100: raise TransportError("bounded snapshot with timezone-aware history start required") - # snapshot() lists the FEE activities created after this instant: the ledger's cash - # baseline, which the caller knows (runner.main: the lane's first trial, whose cash is - # baseline_cash; mover_runner: each trial's own start, since it recomputes baseline_cash - # per trial). Defaults to history_start. + # The caller reuses its checkpoint's fixed lineage window for every snapshot. + # Legacy callers keep their recorded baseline window. Defaults to history_start. self.fee_history_start = fee_history_start or self.history_start fee_after_text(self.fee_history_start) self.max_snapshot_pages = max_snapshot_pages @@ -1686,8 +1713,8 @@ def collect(): "avg_entry_price": decimal_string(p["avg_entry_price"])} for p in self._client.get_all_positions()] orders = self._pages("open") + self._pages("all", after=self.history_start) - # Last, after the account's cash: every listed fee is a posted activity. A fee - # posted between the two reads fails one cash comparison (fail closed). + # Last, after account cash, through the caller's fixed fee window. A fee + # posted between these reads can cause a cash mismatch; fail closed. fees = _collect_fee_pages(self._client, self.fee_history_start, self.max_snapshot_pages) return account, positions, orders, fees try: diff --git a/tests/test_adaptive_paper_fees.py b/tests/test_adaptive_paper_fees.py index 61664b2fd..492715180 100644 --- a/tests/test_adaptive_paper_fees.py +++ b/tests/test_adaptive_paper_fees.py @@ -13,11 +13,13 @@ from __future__ import annotations import asyncio -from contextlib import nullcontext +from contextlib import nullcontext, redirect_stdout, redirect_stderr from datetime import datetime, timezone from decimal import Decimal as D import hashlib import json +import io +import logging import os from pathlib import Path import sqlite3 @@ -114,11 +116,53 @@ def utc(epoch): return datetime.fromtimestamp(epoch, timezone.utc) +class FeePrivacyCapture: + """Capture all output around every fee fixture, including rejected rows.""" + def setUp(self): + super().setUp() + self.fee_output = io.StringIO() + self.enterContext(redirect_stdout(self.fee_output)) + self.enterContext(redirect_stderr(self.fee_output)) + logger = logging.getLogger() + handler = logging.StreamHandler(self.fee_output) + logger.addHandler(handler) + self.addCleanup(logger.removeHandler, handler) + previous_level = logger.level + logger.setLevel(logging.NOTSET) + self.addCleanup(logger.setLevel, previous_level) + original_handle = logging.Logger.handle + def capture_log(emitter, record): + self.fee_output.write(record.getMessage() + "\n") + return original_handle(emitter, record) + self.enterContext(patch.object(logging.Logger, "handle", capture_log)) + self.addCleanup(lambda: self.assertNotIn(ACCOUNT_TEXT, self.fee_output.getvalue())) + + +class DescriptionUnreadable(dict): + """Reading description, even by enumerating items, is a privacy failure.""" + def __getitem__(self, key): + if key == "description": + raise AssertionError("description was read") + return super().__getitem__(key) + + def get(self, key, default=None): + if key == "description": + raise AssertionError("description was read") + return super().get(key, default) + + def items(self): + for key in self: + yield key, self[key] + + # --------------------------------------------------------------------------- # Item 2: normalization # --------------------------------------------------------------------------- -class FeeNormalization(unittest.TestCase): +class FeeNormalization(FeePrivacyCapture, unittest.TestCase): + def test_description_is_never_accessed(self): + self.assertEqual(t.normalize_fee_activity(DescriptionUnreadable(fee_row())), measured_fees()[0]) + def test_documented_fee_row_keeps_four_fields_and_never_the_description(self): for row, expected in zip(measured_rows(), measured_fees()): with self.subTest(sub_type=expected["sub_type"]): @@ -182,8 +226,9 @@ def without(key): @unittest.skipUnless(HAS_SDK, "requires isolated reviewed alpaca-py runtime") -class FeeHTTPBoundary(unittest.TestCase): +class FeeHTTPBoundary(FeePrivacyCapture, unittest.TestCase): def setUp(self): + super().setUp() self.budget = Mock(return_value=None) self.session = t.GuardedSession(origin=t.PAPER_URL, before_request=self.budget) self.addCleanup(self.session.close) @@ -247,7 +292,7 @@ def test_every_other_path_parameter_or_method_is_refused_before_the_budget(self) @unittest.skipUnless(HAS_SDK, "requires isolated reviewed alpaca-py runtime") -class FeeSnapshot(unittest.IsolatedAsyncioTestCase): +class FeeSnapshot(FeePrivacyCapture, unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): self.budgets = [] self.port = self.make_port(fee_history_start=BASELINE) @@ -364,14 +409,49 @@ async def test_fee_activities_reads_through_a_fresh_allow_listed_session(self): sent.assert_not_called() +@unittest.skipUnless(HAS_SDK, "requires isolated reviewed alpaca-py runtime") +class FeeCheckpoint(FeePrivacyCapture, unittest.TestCase): + serve = FeeSnapshot.serve + + def test_checkpoint_returns_f2_and_normalized_cash_on_one_read_only_client(self): + budgets = [] + request, calls = self.serve([measured_rows(), list(reversed(measured_rows()))]) + with patch("requests.Session.request", side_effect=request): + checkpoint = t.fee_checkpoint("fixture-key", "fixture-secret", after=BASELINE, + before_request=lambda kind, **_: budgets.append(kind)) + self.assertEqual(checkpoint, {"account": {"cash": "1000", "equity": "1000", "buying_power": "1000"}, + "fees": list(reversed(measured_fees()))}) + self.assertEqual([c[1] for c in calls], [FEE_PATH, "/v2/account", FEE_PATH]) + self.assertEqual(budgets, ["read"] * 3) + self.assertEqual(calls[0][2], calls[2][2]) + + def test_checkpoint_refuses_a_fee_posted_between_its_reads(self): + request, calls = self.serve([[], measured_rows()]) + with patch("requests.Session.request", side_effect=request): + with self.assertRaisesRegex(t.TransportError, "^fee_activity_posted_during_checkpoint$"): + t.fee_checkpoint("fixture-key", "fixture-secret", after=BASELINE, + before_request=lambda kind, **_: None) + self.assertEqual([c[1] for c in calls], [FEE_PATH, "/v2/account", FEE_PATH]) + + def test_checkpoint_refuses_a_changed_normalized_row(self): + for changes in ({"net_amount": "-0.20"}, {"date": "2026-09-28"}, {"activity_sub_type": "TAF"}): + with self.subTest(changes=changes): + request, _ = self.serve([[fee_row()], [fee_row(**changes)]]) + with patch("requests.Session.request", side_effect=request): + with self.assertRaisesRegex(t.TransportError, "^fee_activity_posted_during_checkpoint$"): + t.fee_checkpoint("fixture-key", "fixture-secret", after=BASELINE, + before_request=lambda kind, **_: None) + + # --------------------------------------------------------------------------- # Item 5: the ledger (schema 3) # --------------------------------------------------------------------------- -class FeeLedger(unittest.TestCase): +class FeeLedger(FeePrivacyCapture, unittest.TestCase): NOW = 1_790_000_000.0 def setUp(self): + super().setUp() self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.path = Path(self.tmp.name) / "ledger.sqlite3" @@ -432,12 +512,68 @@ def test_any_other_schema_version_is_refused_before_any_change(self): raw.close() self.ledger = Ledger(self.path) - def test_the_previous_engine_refuses_a_migrated_ledger(self): - # The exact open check of Ledger.__init__ at 11227bfd (this change's base), before any insert. - version = self.schema_version() - with self.assertRaisesRegex(SafetyError, "^unsupported_ledger_schema$"): - if version not in (None, "1", "2"): - raise SafetyError("unsupported_ledger_schema") + def test_migrated_schema_is_outside_the_previous_accept_list(self): + """Compare with None/1/2 copied from b528bb55 safety.py:443; no previous engine runs.""" + self.assertNotIn(self.schema_version(), (None, "1", "2")) + + def check_batch_risk(self, amounts, expected): + limits = RiskLimits(max_drawdown_usd=D("0.5"), max_gross_loss_usd=D("10")) + ledgers = [Ledger(Path(self.tmp.name) / (name + ".sqlite3"), limits) for name in ("batch", "single")] + for ledger in ledgers: + self.addCleanup(ledger.close) + ledger.start_trial(self.NOW) + fees = [dict(measured_fees()[i], net_amount=amount) for i, amount in enumerate(amounts)] + ledgers[0].record_fees(fees, self.NOW) + for fee in fees: + ledgers[1].record_fees([fee], self.NOW) + for ledger in ledgers: + state = ledger.accounting() + self.assertEqual((state.peak_pnl_usd, state.drawdown_usd, state.halted_reason), expected) + with self.assertRaisesRegex(SafetyError, "^drawdown_cap_reached$"): + ledger.reserve_intent("after-fees", "SPY", "buy", "1", "100.02", + quote=safety.Quote("SPY", "100", "100.01", self.NOW + 1), + now=self.NOW + 1, market_open=True, session_close=self.NOW + 3600, + stop_file=Path(self.tmp.name) / "STOP") + + def test_credit_then_debit_batch_preserves_the_intermediate_peak_and_halt(self): + self.check_batch_risk(("1", "-1"), (D("1"), D("1"), "drawdown_cap_reached")) + + def test_debit_then_credit_batch_preserves_the_intermediate_halt(self): + self.check_batch_risk(("-1", "1"), (D("0"), D("0"), "drawdown_cap_reached")) + + def recovery_position(self, limits): + ledger = Ledger(Path(self.tmp.name) / "recovery.sqlite3", limits) + self.addCleanup(ledger.close) + ledger.start_trial(self.NOW) + ledger.reserve_intent("buy", "SPY", "buy", "1", "100.02", + quote=safety.Quote("SPY", "100", "100.01", self.NOW), now=self.NOW, + market_open=True, session_close=self.NOW + 3600, stop_file=Path(self.tmp.name) / "STOP") + ledger.record_order("buy", "broker-buy", "filled", "1", "100", timestamp=self.NOW) + ledger.begin_recovery(self.NOW + 1) + return ledger + + def finish_recovery(self, ledger): + ledger.reserve_intent("sell", "SPY", "sell", "1", "99.98", + quote=safety.Quote("SPY", "100", "100.01", self.NOW + 3), now=self.NOW + 3, + market_open=True, session_close=self.NOW + 3600, stop_file=Path(self.tmp.name) / "STOP") + ledger.record_order("sell", "broker-sell", "filled", "1", "100", timestamp=self.NOW + 3) + self.assertFalse(ledger.positions()) + + def test_measured_recovery_fees_preserve_recovery_only_and_allow_the_sell(self): + ledger = self.recovery_position(RiskLimits()) + ledger.record_fees(measured_fees(), self.NOW + 2) + self.assertEqual(ledger.halted_reason(), "recovery_only") + self.assertEqual(ledger._get("recovery_only"), "1") + self.finish_recovery(ledger) + + def test_recovery_fee_crossing_the_loss_cap_cannot_be_cleared_by_the_next_trial(self): + ledger = self.recovery_position(RiskLimits(max_gross_loss_usd=D("0.40"))) + ledger.record_fees(measured_fees(), self.NOW + 2) + self.assertEqual(ledger.halted_reason(), "gross_loss_cap_reached") + self.assertEqual(ledger._get("recovery_only"), "1") + self.finish_recovery(ledger) + with self.assertRaisesRegex(SafetyError, "^next_trial_cannot_clear_risk_halt$"): + ledger.begin_next_trial(self.NOW + 4, "after-fees") def test_record_fees_books_cash_realized_and_loss_once_with_one_event_each(self): self.assertEqual(self.ledger.record_fees(measured_fees(), self.NOW), measured_fees()) @@ -535,8 +671,9 @@ def test_rederived_accounting_keeps_recorded_fees_next_to_symbol_money(self): # Items 6 and 8: reconciliation # --------------------------------------------------------------------------- -class FeeReconciliation(unittest.TestCase): +class FeeReconciliation(FeePrivacyCapture, unittest.TestCase): def setUp(self): + super().setUp() self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.ledger = self.new_ledger("ledger") @@ -572,6 +709,15 @@ def test_a_snapshot_without_the_fees_key_behaves_as_before(self): "open_orders": 0, "positions": 0}) self.assertEqual(tuple(self.ledger.db.iterdump()), before) + def test_cash_tolerance_is_exactly_one_cent_with_and_without_fees(self): + for fees, cash_at_one_cent, cash_at_two_cents in ((None, "100000.01", "100000.02"), + (measured_fees(), "99999.54", "99999.55")): + with self.subTest(fees=fees): + ledger = self.new_ledger("boundary-" + str(fees is not None)) + self.assertTrue(runner_module.reconcile(ledger, self.snapshot(cash_at_one_cent, fees), "100000")["cash_match"]) + with self.assertRaisesRegex(SafetyError, "^cash_mismatch_or_unmodeled_fees$"): + runner_module.reconcile(ledger, self.snapshot(cash_at_two_cents, fees), "100000") + def test_other_account_activity_still_fails_closed(self): # A dividend credit, a cash withdrawal or a cash journal beside the recorded fees stays unexplained. for label, cash in (("DIV", "100000.55"), ("CSW", "99899.53"), ("JNLC", "99999.00")): @@ -583,11 +729,9 @@ def test_other_account_activity_still_fails_closed(self): with self.assertRaises(SafetyError): runner_module.reconcile(self.new_ledger("dividend"), self.snapshot("100001.02", [dividend]), "100000") - def test_a_fee_the_per_trial_baseline_already_holds_must_not_reach_reconcile(self): - # Why the mover's fee window is its own trial's start, not the lane's started_at: trial 1 ended flat - # at broker cash 100000 (ledger cash_delta 0); overnight the day's three fees posted (cash 99999.53); - # command_paper then recomputed trial 2's baseline as preflight cash minus the ledger's cash_delta - # (mover_runner.py), which already holds them. Handing trial 2 those fees again double counts. + def test_an_unbooked_fee_absorbed_into_an_arbitrary_baseline_counts_twice(self): + # A caller that fixes a baseline before booking a fee already included in cash + # double counts it. The checkpoint avoids this by booking before computing baseline. baseline = format(D("99999.53") - self.ledger.accounting().cash_delta_usd, "f") with self.assertRaisesRegex(SafetyError, "^cash_mismatch_or_unmodeled_fees$"): runner_module.reconcile(self.ledger, self.snapshot("99999.53", measured_fees()), baseline) @@ -631,11 +775,12 @@ def full_gate_result(input_sha256): "versions": {"pandera": "0.33.1"}, "checked_at": "2026-09-22T00:00:00+00:00"} -class AdaptiveLaneFees(unittest.TestCase): - """runner.main keeps its first trial's baseline_cash (trial.json started_at), so its fee window is - started_at, and its next-trial cash check records the fees posted since then before comparing.""" +class AdaptiveLaneFees(FeePrivacyCapture, unittest.TestCase): + """runner.main keeps its first checkpoint's cash baseline and fee window; legacy + metadata retains started_at. The next-trial check books fees before comparing.""" def setUp(self): + super().setUp() self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.root = Path(self.tmp.name) @@ -657,7 +802,7 @@ def setUp(self): "config_sha256": hashlib.sha256(self.config_file.read_bytes()).hexdigest(), "baseline_cash": "10000.47", "phase": "finished", "status": "passed"})) - def run_main(self, command, fees, cash): + def run_main(self, command, fees, cash, *, fee_reader=None, checkpoint=None): self.observation["account"]["cash"] = cash seen = {} sentinel = RuntimeError("transport_constructed") @@ -671,6 +816,12 @@ def fake_transport(*args, **kwargs): seen["transport"] = kwargs raise sentinel + def fake_checkpoint(key, secret, *, after, before_request, **kwargs): + seen["checkpoint_after"] = after + for _ in range(3): + before_request("read") + return {"account": {"cash": cash}, "fees": fees} + argv = ["runner.py", command, "--env-file", str(self.env_file), "--config", str(self.config_file), "--output", str(self.output), "--state-root", str(self.root / "state"), "--trial", "fee-2"] if command == "paper": @@ -681,10 +832,11 @@ def fake_transport(*args, **kwargs): patch.object(runner_module, "credentials", return_value=("fixture-key", "fixture-secret")), \ patch.object(runner_module, "preflight", return_value=self.observation), \ patch.object(runner_module, "account_lock_fingerprint", lambda _fingerprint: nullcontext()), \ - patch.object(runner_module, "fee_activities", fake_fees, create=True), \ + patch.object(runner_module, "fee_activities", fee_reader or fake_fees), \ + patch.object(runner_module, "fee_checkpoint", checkpoint or fake_checkpoint, create=True), \ patch.object(runner_module, "AlpacaPaperTransport", fake_transport): try: - runner_module.main() + seen["return_code"] = runner_module.main() except Exception as exc: # noqa: BLE001 -- the sentinel says the transport was reached raised = exc return seen, raised, sentinel @@ -721,12 +873,101 @@ def test_the_adaptive_recover_transport_reads_fees_from_started_at(self): (utc(T0), utc(T0))) self.assertNotIn("fee_after", seen) # recover records fees through its own snapshots + def test_first_adaptive_trial_books_checkpoint_fees_before_fixing_baseline(self): + (self.state / "trial.json").unlink() + def checkpoint(key, secret, *, after, before_request, **kwargs): + for _ in range(3): + before_request("read") + return {"account": {"cash": "9999.53"}, "fees": measured_fees()} + seen, raised, sentinel = self.run_main("paper", measured_fees(), "10000", checkpoint=checkpoint) + self.assertIs(raised, sentinel, raised) + metadata = json.loads((self.state / "trial.json").read_text()) + self.assertEqual(D(metadata["baseline_cash"]), D("10000")) + self.assertEqual(seen["transport"]["fee_history_start"], utc(metadata["fee_window_start"])) + ledger = self.ledger() + self.assertEqual(ledger.accounting().cash_delta_usd, D("-0.47")) + self.assertEqual(len(ledger.fees()), 3) + self.assertTrue(runner_module.reconcile(ledger, FeeReconciliation.snapshot("9999.53", measured_fees()), + metadata["baseline_cash"])["cash_match"]) + + def test_adaptive_lineage_reuses_the_checkpoint_window_for_reads_and_transport(self): + path = self.state / "trial.json" + metadata = json.loads(path.read_text()) + metadata["fee_window_start"] = T0 - 3600 + path.write_text(json.dumps(metadata)) + seen, raised, sentinel = self.run_main("paper", measured_fees(), "10000") + self.assertIs(raised, sentinel, raised) + self.assertEqual(seen["fee_after"], utc(T0 - 3600)) + self.assertEqual(seen["transport"]["fee_history_start"], utc(T0 - 3600)) + self.assertEqual(seen["transport"]["history_start"], utc(T0)) + self.assertEqual(json.loads(path.read_text())["fee_window_start"], T0 - 3600) + + def test_first_adaptive_checkpoint_refusals_leave_the_trial_unentered(self): + path = self.state / "trial.json" + path.unlink() + for error in (t.TransportError("fee_activity_posted_during_checkpoint"), + SafetyError("fee_checkpoint_budget_exhausted")): + with self.subTest(reason=str(error)): + seen, raised, _ = self.run_main("paper", [], "10000", checkpoint=Mock(side_effect=error)) + self.assertIsNone(raised) + self.assertEqual(seen["return_code"], 2) + result = json.loads(self.output.read_text()) + self.assertEqual((result["status"], result["stage"], result["reason"]), + ("not_started", "trial_start", str(error))) + self.assertNotIn("transport", seen) + self.assertFalse(path.exists()) + ledger = self.ledger() + self.assertFalse(ledger.intents()) + self.assertEqual(ledger.db.execute("SELECT COUNT(*) FROM trials").fetchone()[0], 0) + + @unittest.skipUnless(HAS_SDK, "requires isolated reviewed alpaca-py runtime") + def test_full_next_trial_budget_prevents_the_http_read(self): + now = time.time() + ledger = self.ledger() + for _ in range(ledger.limits.max_rest_per_minute): + self.assertEqual(ledger.request_budget(now, "read"), 0) + with patch("requests.Session.request") as sent: + seen, raised, _ = self.run_main("paper", [], "10000.47", fee_reader=t.fee_activities) + self.assertIsInstance(raised, SafetyError) + sent.assert_not_called() + self.assertNotIn("transport", seen) + + @unittest.skipUnless(HAS_SDK, "requires isolated reviewed alpaca-py runtime") + def test_failed_next_trial_pagination_charges_every_sent_read(self): + page = [fee_row(i, "CAT", "-0.01") for i in range(100)] + with patch("requests.Session.request", side_effect=[response(page), requests.RequestException("fixture")]) as sent: + _, raised, _ = self.run_main("paper", [], "10000.47", fee_reader=t.fee_activities) + self.assertIsNotNone(raised) + self.assertEqual(sent.call_count, 2) + self.assertEqual([r[0] for r in self.ledger().db.execute("SELECT kind FROM requests")], ["read", "read"]) + + def test_next_trial_cash_tolerance_is_exactly_one_cent_with_and_without_fees(self): + path = self.state / "trial.json" + original = path.read_text() + for fees, one, two in (([], "10000.48", "10000.49"), (measured_fees(), "10000.01", "10000.02")): + with self.subTest(fees=len(fees)): + (self.state / "ledger.sqlite3").unlink(missing_ok=True) + path.write_text(original) + _, raised, sentinel = self.run_main("paper", fees, one) + self.assertIs(raised, sentinel, raised) + path.write_text(original) + _, raised, _ = self.run_main("paper", fees, two) + self.assertIsInstance(raised, SafetyError) + self.assertEqual(str(raised), "next_trial_cash_mismatch") + # --------------------------------------------------------------------------- # Items 4 and 9: mover helpers, fee window and receipts # --------------------------------------------------------------------------- -class MoverFeeHelpers(unittest.TestCase): +class MoverFeeHelpers(FeePrivacyCapture, unittest.TestCase): + def test_checkpoint_window_is_preferred_and_invalid_values_are_refused(self): + self.assertEqual(mover_runner.fee_window_start({"fee_window_start": T0 - 3600, + "current_trial_started_at": T0}), utc(T0 - 3600)) + for value in (None, True, "1790000000", float("nan"), float("inf"), 0, -1): + with self.subTest(value=value), self.assertRaises(SafetyError): + mover_runner.fee_window_start({"fee_window_start": value, "current_trial_started_at": T0}) + def test_fee_receipt_counts_totals_and_sub_types_without_ids(self): rows = [{"activity_id": f["id"], "date": f["date"], "net_amount": D(f["net_amount"]), "sub_type": f["sub_type"], "recorded_at": T0} for f in measured_fees()] @@ -768,7 +1009,7 @@ def scan_raw(rows): @unittest.skipUnless(NATIVE, "requires pinned combined native runtime") -class MoverLaneFees(unittest.TestCase): +class MoverLaneFees(FeePrivacyCapture, unittest.TestCase): """``mover_runner.py paper``/``recover`` end to end with a fake preflight and the synthetic port in place of AlpacaPaperTransport (as test_adaptive_paper_mover_native.MoverPaperCommandWiring), plus a broker-side FEE list the port filters by the transport's fee window. A SYN wiring fixture.""" @@ -777,21 +1018,30 @@ class MoverLaneFees(unittest.TestCase): FINGERPRINT = "f" * 64 def setUp(self): + super().setUp() self.broker = None self.sell_blocked_ports = 0 self.windows = [] # (fee_history_start, history_start) per transport built self.posted = [] # (created_at epoch, raw FEE row) the broker lists self.post_on_port = None # post the measured fees as this transport (1-based) is built + self.checkpoint_hook = None + self.checkpoint_override = None + self.after_checkpoint = None + self.start_cash = D("100000") + self.time_offset = 0 def post_fees(self): """The broker posts the measured FEE activities now: cash falls by 0.47 and the rows are listed.""" created = time.time() self.posted.extend((created, row) for row in measured_rows()) - self.broker.cash -= D("0.47") + if self.broker is None: + self.start_cash -= D("0.47") + else: + self.broker.cash -= D("0.47") def observation(self, symbols): server_ns, now_ns = int(self.SERVER * 1e9), time.time_ns() - cash = format(self.broker.cash, "f") if self.broker is not None else "100000" + cash = format(self.broker.cash if self.broker is not None else self.start_cash, "f") positions = [] if self.broker is None else [ {"symbol": s, "qty": format(q, "f"), "avg_entry_price": "10"} for s, q in self.broker.positions.items() if q] orders = [] if self.broker is None else [ @@ -817,12 +1067,13 @@ def fast_config(self, root): def trial_json(self, root): return json.loads((Path(root) / "state" / self.FINGERPRINT / "mover" / "trial.json").read_text()) - def run_command(self, root, trial, *, command="paper", config=None): + def run_command(self, root, trial, *, command="paper", config=None, checkpoint_seam=False): import signal import transport out = Path(root) / f"{trial or command}.json" real_load_scan, real_build_plan, real_flag = (mover_runner.load_scan, mover_runner.build_plan, native_adapter.order_extended_hours_flag) + real_time = time.time def compressed_plan(settings, limits, scan, session, *, t0, **kwargs): timing = mover.Timing(t0, t0 + 2.0, 1.0, 1.0, t0 + 4.0, t0 + 8.0, None, 1.5) @@ -836,6 +1087,8 @@ def fake_transport(key, secret, symbols, **kwargs): if self.broker is not None: port.orders, port.positions, port.cash = self.broker.orders, self.broker.positions, self.broker.cash port.t0 = self.broker.t0 + else: + port.cash = self.start_cash port.fill_sells = self.sell_blocked_ports <= 0 self.sell_blocked_ports -= 1 window = kwargs.get("fee_history_start") @@ -846,8 +1099,7 @@ async def snapshot(): result = await listed() if window is not None: # The broker's `after` filter on created_at; the rows pass the real normalization. - result["fees"] = [t.normalize_fee_activity(row) for created, row in self.posted - if created > window.timestamp()] + result["fees"] = self.list_fees(window) return result port.snapshot = snapshot self.broker = port @@ -855,6 +1107,25 @@ async def snapshot(): self.post_fees() return port + def fake_checkpoint(key, secret, *, after, before_request, **kwargs): + if self.checkpoint_hook: + self.checkpoint_hook(after) + for _ in range(3): + before_request("read") + result = {"account": self.observation([])["account"], "fees": self.list_fees(after)} + if self.after_checkpoint: + self.after_checkpoint(after) + return result + + async def reconciled_trial(controller, plan, config, baseline_cash, **kwargs): + """Start/checkpoint seam: actual snapshot reconciliation without a native node.""" + proof = mover_runner.mover_reconcile(controller.ledger, await controller.port.snapshot(), baseline_cash) + return {"status": "passed", "flat": True, "native_fill_events": 0, "legs": [], "reconciliation": proof} + + async def reconciled_recovery(controller, metadata, config): + proof = mover_runner.mover_reconcile(controller.ledger, await controller.port.snapshot(), metadata["baseline_cash"]) + return {"status": "passed", "flat": True, "errors": [], "reconciliation": proof} + args = [command, "--env-file", str(Path(root) / "unused.env"), "--output", str(out), "--state-root", str(Path(root) / "state")] if command == "paper": @@ -865,20 +1136,138 @@ async def snapshot(): try: with patch.object(mover_runner, "credentials", return_value=("key", "secret")), \ patch.object(transport, "preflight", lambda key, secret, symbols, **kw: self.observation(symbols)), \ + patch.object(transport, "fee_checkpoint", self.checkpoint_override or fake_checkpoint, create=True), \ patch.object(transport, "AlpacaPaperTransport", fake_transport), \ patch.object(mover_runner, "load_scan", lambda raw, settings, *, now: real_load_scan(raw, settings, now=SCAN_TIME + 20)), \ patch.object(mover_runner, "build_plan", compressed_plan), \ patch.object(mover_runner, "_controller_session", lambda close, now, policy: (now + 36000, True)), \ patch.object(native_adapter, "order_extended_hours_flag", lambda ts, p: real_flag(PRE_TS, p)), \ - patch("builtins.print"): + (patch("time.time", lambda: real_time() + self.time_offset) if checkpoint_seam else nullcontext()), \ + (patch.object(mover_runner, "run_mover", reconciled_trial) if checkpoint_seam else nullcontext()), \ + (patch.object(mover_runner, "recover_mover", reconciled_recovery) if checkpoint_seam else nullcontext()): code = mover_runner.main(args) finally: for sig, handler in handlers.items(): signal.signal(sig, handler) return code, json.loads(out.read_text()) - def test_each_mover_transport_reads_fees_from_its_trials_own_baseline_time(self): + def list_fees(self, window): + cutoff = datetime.strptime(t.fee_after_text(window), "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() + return [t.normalize_fee_activity(row) for created, row in self.posted if created > cutoff] + + def test_mover_checkpoint_books_preflight_to_start_fees_before_the_baseline(self): + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + self.checkpoint_hook = lambda after: self.post_fees() + code, receipt = self.run_command(root, "checkpoint-1", config=config, checkpoint_seam=True) + self.assertEqual((code, receipt["status"], receipt["flat"]), (0, "passed", True), receipt) + self.assertEqual(receipt["fees_recorded"]["total_usd"], "-0.47") + self.assertTrue(receipt["totals"]["pnl_consistent"], receipt["totals"]) + metadata = self.trial_json(root) + self.assertEqual(D(metadata["baseline_cash"]), D("100000")) + self.checkpoint_hook = None + code, result = self.run_command(root, None, command="recover", config=config, checkpoint_seam=True) + self.assertEqual((code, result["status"], result["flat"]), (0, "passed", True), result) + ledger = Ledger(Path(root) / "state" / self.FINGERPRINT / "mover" / "ledger.sqlite3", + mover_runner.load_mover_config(config)[1]) + try: + self.assertEqual(len(ledger.fees()), 3) + finally: + ledger.close() + + def test_mover_same_second_fee_before_account_read_is_booked_once(self): + def post_before_fractional_start(after): + start = after.timestamp() + created = int(start) + (start - int(start)) / 2 + self.assertLess(created, start) + self.posted.extend((created, row) for row in measured_rows()) + self.start_cash -= D("0.47") + self.checkpoint_hook = post_before_fractional_start + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + code, receipt = self.run_command(root, "same-second", config=config, checkpoint_seam=True) + self.assertEqual((code, receipt["status"]), (0, "passed"), receipt) + self.assertEqual(receipt["fees_recorded"]["count"], 3) + self.assertEqual(D(self.trial_json(root)["baseline_cash"]), D("100000")) + code, result = self.run_command(root, None, command="recover", config=config, checkpoint_seam=True) + self.assertEqual((code, result["status"]), (0, "passed"), result) + self.assertEqual(result["fees_recorded"]["count"], 0) + + def test_fee_posted_after_checkpoint_before_trial_start_remains_in_the_window(self): + def post_after_f2(after): + self.post_fees() + # Move the trial clock across a second so the trial-start cutoff mutant + # excludes these fees even after the API's whole-second formatting. + self.time_offset += 2 + self.after_checkpoint = post_after_f2 + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + code, receipt = self.run_command(root, "after-f2", config=config, checkpoint_seam=True) + self.assertEqual((code, receipt["status"]), (0, "passed"), receipt) + self.assertEqual(receipt["fees_recorded"]["count"], 3) + metadata = self.trial_json(root) + self.assertLess(metadata["fee_window_start"], metadata["current_trial_started_at"] - 1) + + def test_checkpoint_instability_refuses_start_and_the_next_start_retries(self): + self.checkpoint_override = Mock(side_effect=t.TransportError("fee_activity_posted_during_checkpoint")) + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + code, receipt = self.run_command(root, "retry-1", config=config, checkpoint_seam=True) + self.assertEqual((code, receipt["status"], receipt["stage"], receipt["reason"]), + (2, "not_started", "trial_start", "fee_activity_posted_during_checkpoint")) + self.assertFalse((Path(root) / "state" / self.FINGERPRINT / "mover" / "trial.json").exists()) + self.checkpoint_override = None + self.assertEqual(self.run_command(root, "retry-1", config=config, checkpoint_seam=True)[0], 0) + + def test_second_mover_start_reuses_the_first_checkpoint_window(self): + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + self.assertEqual(self.run_command(root, "window-1", config=config, checkpoint_seam=True)[0], 0) + first = self.trial_json(root) + self.post_fees() # fees earlier engines absorbed into the next trial baseline + code, receipt = self.run_command(root, "window-2", config=config, checkpoint_seam=True) + self.assertEqual((code, receipt["status"]), (0, "passed"), receipt) + second = self.trial_json(root) + self.assertEqual(second["fee_window_start"], first["fee_window_start"]) + self.assertEqual(D(second["baseline_cash"]), D(first["baseline_cash"])) + self.assertEqual(receipt["fees_recorded"]["count"], 3) + self.assertTrue(receipt["totals"]["pnl_consistent"]) + self.assertEqual(self.windows[-1][0], utc(first["fee_window_start"])) + + def test_full_mover_budget_refuses_checkpoint_before_any_http_call(self): + with tempfile.TemporaryDirectory() as root: + config = self.fast_config(root) + (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) + state = Path(root) / "state" / self.FINGERPRINT / "mover" + _, limits, _ = mover_runner.load_mover_config(config) + ledger = Ledger(state / "ledger.sqlite3", limits) + try: + now = time.time() + for _ in range(limits.max_rest_per_minute): + self.assertEqual(ledger.request_budget(now, "read"), 0) + finally: + ledger.close() + self.checkpoint_override = getattr(t, "fee_checkpoint", None) + with patch("requests.Session.request") as sent: + code, receipt = self.run_command(root, "budget", config=config) + self.assertEqual((code, receipt["status"], receipt["stage"], receipt["reason"]), + (2, "not_started", "trial_start", "fee_checkpoint_budget_exhausted")) + sent.assert_not_called() + self.assertFalse((state / "trial.json").exists()) + ledger = Ledger(state / "ledger.sqlite3", limits) + try: + self.assertFalse(ledger.intents()) + self.assertEqual(ledger.db.execute("SELECT COUNT(*) FROM trials").fetchone()[0], 0) + finally: + ledger.close() + + def test_each_mover_trial_and_recover_reuses_the_lineage_checkpoint_window(self): with tempfile.TemporaryDirectory() as root: config = self.fast_config(root) (Path(root) / "scan.json").write_bytes(scan_raw([scan_row("AAA", 1, "10.00")])) @@ -888,11 +1277,12 @@ def test_each_mover_transport_reads_fees_from_its_trials_own_baseline_time(self) second = self.trial_json(root) self.assertEqual(second["started_at"], first["started_at"]) self.assertLess(first["current_trial_started_at"], second["current_trial_started_at"]) - self.assertEqual(self.windows, [(utc(first["current_trial_started_at"]), utc(first["started_at"])), - (utc(second["current_trial_started_at"]), utc(second["started_at"]))]) + self.assertEqual(second["fee_window_start"], first["fee_window_start"]) + self.assertEqual(self.windows, [(utc(first["fee_window_start"]), utc(first["started_at"])), + (utc(first["fee_window_start"]), utc(second["started_at"]))]) code, result = self.run_command(root, None, command="recover", config=config) self.assertEqual((code, result["status"]), (0, "passed")) - self.assertEqual(self.windows[-1], (utc(second["current_trial_started_at"]), utc(second["started_at"]))) + self.assertEqual(self.windows[-1], (utc(first["fee_window_start"]), utc(second["started_at"]))) def test_recover_records_fees_posted_after_the_trial_and_reports_them(self): # The account 2 case: the trial ends needs_attention holding a position, the day's REG/TAF/CAT @@ -904,10 +1294,19 @@ def test_recover_records_fees_posted_after_the_trial_and_reports_them(self): code, receipt = self.run_command(root, "ext-1", config=config) self.assertEqual((code, receipt["status"], receipt["flat"]), (3, "needs_attention", False)) self.assertEqual(receipt["fees_recorded"], {"count": 0, "total_usd": "0", "sub_types": {}}) + # The real residual's legacy metadata has no checkpoint window. + path = Path(root) / "state" / self.FINGERPRINT / "mover" / "trial.json" + metadata = json.loads(path.read_text()) + metadata.pop("fee_window_start", None) + legacy = datetime(2026, 9, 29, 20, 25, 19, tzinfo=timezone.utc) + metadata["current_trial_started_at"] = legacy.timestamp() + path.write_text(json.dumps(metadata)) self.post_fees() code, result = self.run_command(root, None, command="recover", config=config) self.assertEqual((code, result["kind"], result["status"], result["flat"], result["errors"]), (0, "mover_recovery_receipt", "passed", True, [])) + self.assertEqual(self.windows[-1][0], legacy) + self.assertEqual(t.fee_after_text(self.windows[-1][0]), "2026-09-29T20:25:19Z") self.assertEqual(result["fees_recorded"], {"count": 3, "total_usd": "-0.47", "sub_types": {"CAT": {"count": 1, "total_usd": "-0.01"}, "REG": {"count": 1, "total_usd": "-0.19"}, diff --git a/tests/test_adaptive_paper_mover_native.py b/tests/test_adaptive_paper_mover_native.py index 75a253831..4c9301f1e 100644 --- a/tests/test_adaptive_paper_mover_native.py +++ b/tests/test_adaptive_paper_mover_native.py @@ -611,6 +611,11 @@ def fake_transport(key, secret, symbols, **kwargs): self.broker = port return port + def fake_checkpoint(key, secret, *, after, before_request, **kwargs): + for _ in range(3): + before_request("read") + return {"account": self.observation([])["account"], "fees": []} + args = [command, "--env-file", str(Path(root) / "unused.env"), "--output", str(out), "--state-root", str(Path(root) / "state")] if command == "paper": @@ -621,6 +626,7 @@ def fake_transport(key, secret, symbols, **kwargs): try: with patch.object(mover_runner, "credentials", return_value=("key", "secret")), \ patch.object(transport, "preflight", lambda key, secret, symbols, **kw: self.observation(symbols)), \ + patch.object(transport, "fee_checkpoint", fake_checkpoint), \ patch.object(transport, "AlpacaPaperTransport", fake_transport), \ patch.object(mover_runner, "load_scan", lambda raw, settings, *, now: real_load_scan(raw, settings, now=SCAN_TIME + 20)), \ From 1ff3166efe35f41a72712e5516c5a1fc2b15c175 Mon Sep 17 00:00:00 2001 From: Scout Date: Wed, 30 Sep 2026 16:04:18 -0400 Subject: [PATCH 5/7] Adaptive paper: re-check residuals (recovery halt semantics, adaptive and clock limits) After the repair's re-check (gpt-6.1-sol ultra: one medium doc finding; Claude Opus: pass with five lows): - README-safety.md: fees posting after a lineage's final snapshot are booked if a later paper or recover runs on it; any risk cap that trips during recovery now halts for good (not fee-specific), with the sell-only exit unaffected; adaptive legacy-baseline and next-trial read-order limits; the refused-start receipt gap; the local-clock bound (validate_preflight's 0.25 s clock_drift guard). - tests: test_any_cap_trip_during_recovery_is_sticky_and_the_sell_still_completes (a mark lifting the residual above the gross-exposure cap during recovery); it fails under the pre-repair `if reason and not state.halted_reason`. Co-Authored-By: Claude Opus 5.5 --- .../adaptive-paper/README-safety.md | 41 +++++++++++++++---- tests/test_adaptive_paper_fees.py | 12 ++++++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/README-safety.md b/blueprints/us-equities/adaptive-paper/README-safety.md index b703865e7..2872dd0f5 100644 --- a/blueprints/us-equities/adaptive-paper/README-safety.md +++ b/blueprints/us-equities/adaptive-paper/README-safety.md @@ -278,8 +278,14 @@ formatting fractional L can include a fee from earlier within that second, which is also booked before baseline and deduplicated thereafter. Fees earlier engines absorbed into a baseline are booked at the next checkpoint; recomputing baseline after booking preserves cash reconciliation. Fee losses still consume the risk -budget during recovery: a cap halt replaces `recovery_only`, while the sell-only -restriction remains in force. +budget during recovery. More generally, any risk cap that trips during a recovery +(a fee, a fill, or a mark that lifts the residual above the gross-exposure cap) +now replaces `recovery_only` with that halt reason, exactly as it would halt a +trial. The halt is permanent for the ledger (`next_trial_cannot_clear_risk_halt`), +the sell-only exit is not blocked, and a flat recovery still ends `finished`. +Before this change a cap trip during recovery stayed masked as `recovery_only`: +loss-budget crossings were still refused at the next trial +(`next_trial_risk_budget_exhausted`), but an exposure-cap trip was cleared. **Receipts.** The mover trial receipt and the `mover_recovery_receipt` carry `fees_recorded` (count, total and per sub-type count and total; no activity ids). @@ -287,10 +293,13 @@ The trial receipt's `totals.fees_recorded_usd` joins the per-symbol P&L in `pnl_consistent`. **Limits.** A fee posted during the checkpoint refuses that start; the next -start retries. Fees of a lineage's last day that post after its last trial are -never booked into that lineage: cash remains consistent because the next lineage -absorbs them into its baseline (except any included by the whole-second cutoff, -which are booked before baseline). A fee posted between a later snapshot's +start retries. Fees that post after a lineage's final snapshot are booked into +that lineage only if a later `paper` or `recover` runs on it: `recover` accepts a +finished trial, keeps its fee window and books any new fee there (they then count +against that lineage's budgets). A lineage that is retired and never invoked again +never books them; cash stays consistent because the next lineage absorbs them into +its baseline (except any included by the whole-second cutoff, which are booked +before baseline). A fee posted between a later snapshot's account and fee reads can still cause a cash mismatch; reconciliation fails closed. Under the Regulatory Fees page's intraday-accrual hypothesis, a mover trial starting after an earlier trial's accrual stays blocked once the FEE posts @@ -298,8 +307,24 @@ inside its window, until an operator re-baselines. The 2026-09-30 measured gap equaled the three posted activities; that observation did not establish pending accrual behavior. Legacy mover metadata without `fee_window_start` keeps `current_trial_started_at` for recovery, including the account-2 residual's -2026-09-29T20:25:19Z window. Legacy adaptive metadata keeps `started_at`. Both -lineages' fee windows grow with their life (more pages per snapshot, bounded by +2026-09-29T20:25:19Z window. Legacy adaptive metadata keeps `started_at`, but its +baseline came from a preflight account read taken before `started_at`. A fee +created between that read and `started_at` is never listed, and one created in the +truncated second before the read is counted twice. Either leaves that adaptive lane +mismatched until an operator re-baselines. New adaptive lanes use the checkpoint. +The adaptive lane's next-trial check compares the preflight's cash, which is read +before its fee read, so a fee posted between those two reads is booked but refuses +that start (`next_trial_cash_mismatch`); a retry then reconciles. A fee a +checkpoint books at a start that `begin_next_trial` then refuses (for example, one +that trips a cap) is in the ledger and its `fee_recorded` event, but in no +receipt's `fees_recorded`. The checkpoint's window L is the local clock, while the +broker filters on its own `created_at`. If the local clock runs ahead of the broker's +by more than the gap between the truncated L and the account read, a fee created +inside that gap is neither listed nor in the baseline, and it stays mismatched. The +gap is bounded: `validate_preflight` refuses a clock drift above 0.25 s +(`clock_drift`) just before the checkpoint. It affects only a new lineage's first +checkpoint. Both lineages' fee +windows grow with their life (more pages per snapshot, bounded by `max_snapshot_pages`). `tests/test_adaptive_paper_fees.py` covers this section with local synthetic fixtures (no broker request). diff --git a/tests/test_adaptive_paper_fees.py b/tests/test_adaptive_paper_fees.py index 492715180..e27b61fa6 100644 --- a/tests/test_adaptive_paper_fees.py +++ b/tests/test_adaptive_paper_fees.py @@ -575,6 +575,18 @@ def test_recovery_fee_crossing_the_loss_cap_cannot_be_cleared_by_the_next_trial( with self.assertRaisesRegex(SafetyError, "^next_trial_cannot_clear_risk_halt$"): ledger.begin_next_trial(self.NOW + 4, "after-fees") + def test_any_cap_trip_during_recovery_is_sticky_and_the_sell_still_completes(self): + # The recovery_only replacement in _refresh_risk is not fee-specific: a mark that + # lifts the residual above the gross-exposure cap during recovery also halts for + # good, exactly as it would inside a trial. The sell-only exit is not blocked. + ledger = self.recovery_position(RiskLimits(max_gross_exposure_usd=D("150"), max_order_notional_usd=D("150"))) + ledger.mark_to_market([safety.Quote("SPY", "200", "200.01", self.NOW + 2)], self.NOW + 2) + self.assertEqual(ledger.halted_reason(), "gross_exposure_cap_exceeded") + self.assertEqual(ledger._get("recovery_only"), "1") + self.finish_recovery(ledger) + with self.assertRaisesRegex(SafetyError, "^next_trial_cannot_clear_risk_halt$"): + ledger.begin_next_trial(self.NOW + 4, "after-spike") + def test_record_fees_books_cash_realized_and_loss_once_with_one_event_each(self): self.assertEqual(self.ledger.record_fees(measured_fees(), self.NOW), measured_fees()) state = self.ledger.accounting() From 53a5f3cb3f6d8b7bc457e5117f477cb10df1a06b Mon Sep 17 00:00:00 2001 From: Scout Date: Wed, 30 Sep 2026 16:04:36 -0400 Subject: [PATCH 6/7] Adaptive paper: refresh source hashes after the fee review rounds Co-Authored-By: Claude Opus 5.5 --- .../us-equities/adaptive-paper/source-hashes.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/blueprints/us-equities/adaptive-paper/source-hashes.json b/blueprints/us-equities/adaptive-paper/source-hashes.json index bc763bae6..868e97655 100644 --- a/blueprints/us-equities/adaptive-paper/source-hashes.json +++ b/blueprints/us-equities/adaptive-paper/source-hashes.json @@ -18,7 +18,7 @@ "blueprints/us-equities/adaptive-paper/market_research.py": "bba9cf77c3c3c6dff32dec1cb01fa8c1957c6b012b7d5bd9693fcc268d557158", "blueprints/us-equities/adaptive-paper/metrics.py": "437fce8aaca350b9d8fe5d5b68212879d82e118f01d93fdcd333233709b80397", "blueprints/us-equities/adaptive-paper/mover.py": "ac34fd84485a054b27fb857b5ef3b0dfb7f7dee9c407eb42d4579e9e80e5ee88", - "blueprints/us-equities/adaptive-paper/mover_runner.py": "d1e79e5578d40caf50e01c4cb1e6905042e24e081df489dce2790388fbcec940", + "blueprints/us-equities/adaptive-paper/mover_runner.py": "aaa2e70202e011993c6827cda808155ef7737d752073245ceded55fd8b1eb5d7", "blueprints/us-equities/adaptive-paper/mover_simulation.py": "8ac5955de1e814bb26fbfbca068a5719b9033b2bae0d8e50e78d7a78a546761e", "blueprints/us-equities/adaptive-paper/mover_strategy.py": "4730c5e535aa480030d6484669a9d8565ca25e4e1ea50ea12a45bd1892536cba", "blueprints/us-equities/adaptive-paper/native_adapter.py": "fbf530884c2ad3b9e0c2b1f989eda399531647cc1db3c56dc9bcac10174fea47", @@ -27,15 +27,15 @@ "blueprints/us-equities/adaptive-paper/recovery.py": "38b6cf4f9247577f72337ca3dbd242505a43ad5cbc5cbe21349b0f1beb2f3e51", "blueprints/us-equities/adaptive-paper/registry.json": "60da73b985e718bd1dece7a3350f34636bda0fb38e766a44ae71c5a8c532282a", "blueprints/us-equities/adaptive-paper/requirements.txt": "518afe4b77f901e87abd3a5d6faa126aa7e211db77eecec80cfe7ef42d82e07a", - "blueprints/us-equities/adaptive-paper/runner.py": "1e0f796f97bbe38d2e516bb8148954ddf7112edd6f1fd14b1d2d1c4ab8d6fb44", - "blueprints/us-equities/adaptive-paper/safety.py": "3a0d8f991969d471e5d67dd3265a02c14cdf411cb3761c96e69e0d5efbc3c802", + "blueprints/us-equities/adaptive-paper/runner.py": "34ab492f450431e79a3d5d5c6de32185b22590e1a109b3c5fd976db0aa0aa29e", + "blueprints/us-equities/adaptive-paper/safety.py": "f43def730273be28dbb2b4a791dff31ca7e3f661735eb45e35ebccb75c4d0285", "blueprints/us-equities/adaptive-paper/scheduled_trial.sh": "58a5b656875bda629443768cd0c69a4be28cbf90dfc9f5ae8782d7dcf3ac7d76", "blueprints/us-equities/adaptive-paper/selector.py": "28de8ebf66b6e7135979554f73692ede772167682007d8c6ba41da6fb6fb39b3", "blueprints/us-equities/adaptive-paper/sessions.py": "95f0affee512e88dd44f196fa72bb7471e00899356793bdbb12f15c4ed84e64d", "blueprints/us-equities/adaptive-paper/simulation.py": "0119e0efe3b498be3c415eee9c1a0be922c9b5b17e3b774aa7528668c4eb0b30", "blueprints/us-equities/adaptive-paper/strategies.py": "2d488a4fdd21ad323ab07cf7fb147b3623fd6e38951abcf2fc59cf8094a1383e", "blueprints/us-equities/adaptive-paper/strategies_v1.py": "1952be269694485e12e4d927f124adca0ee29667588d6ac6651caede454c92f7", - "blueprints/us-equities/adaptive-paper/transport.py": "e0ec1d3f4b7a5e3fb8e6d1af0ccd0f033e6a790199baae3f9b8d14758d1f565b", + "blueprints/us-equities/adaptive-paper/transport.py": "8d1222f7b8e90cf2af0cd6efb88eb70704cf900c480a097e86576a8382435b16", "tests/_credential_mutation_driver.py": "bfe35e69702016718452ed702aab20a0a33da5723e28a61de513b3c268849301", "tests/adaptive_paper_hermetic.py": "0f41601cd10abf10ccba4f17d9c35716bb766a5e9951a4390649d6c9387fc1da", "tests/test_adaptive_market_research.py": "772b4fd467d4405bd098a440c54de4bfd61f91bbb80693c052077b867f9e2fd6", @@ -43,12 +43,12 @@ "tests/test_adaptive_paper_credential_race.py": "1b0b6521045858fc9c974fc9e50e390ac888c7091f5f92606feb1bf49f7ef022", "tests/test_adaptive_paper_credential_source.py": "32c3aa93e40392a2af60078dd4ba6c222e195a9a47b22a0bfc0f52a92814382a", "tests/test_adaptive_paper_exits.py": "13a82d65ad1b2e72a8ca0fbb8bbb178a01ded1301399604c9e304ae213456764", - "tests/test_adaptive_paper_fees.py": "ff162965f2e04bbd17c39b6c83062ef42c54f2a2fc82de73a5ee78223d70fc97", + "tests/test_adaptive_paper_fees.py": "9a181eb3d2e20ff34ac6551022bf252d51688ab3014162ca696a37b45bdc5221", "tests/test_adaptive_paper_financing.py": "8d9f1f21f9d53c9ed120281409bdcb97b8c9ddacc615d0cdfa392cfcf111fa90", "tests/test_adaptive_paper_live_manifest.py": "4596f301ac44effefee5b787c5e3627a9cd38f8ff0cf436ef778b7378a0667c2", "tests/test_adaptive_paper_metrics.py": "1d6eaf940021fbaba3ac687a3d3a2d9f6e5492e85a5a372621299d4363ec3ee5", "tests/test_adaptive_paper_mover.py": "51878f9e0121b856e2157db1626c61a2653ff0ac181fdb89b092dc8e763f0062", - "tests/test_adaptive_paper_mover_native.py": "2c176fb4e25827fe6b5c0a7ef1277769cd402bf33b28a31ed5c2c7a6da8c3220", + "tests/test_adaptive_paper_mover_native.py": "2cf17ed5dd956d18b973fcdcf0ea1412dee8eec7040d29fa1d2e28781b7c16fa", "tests/test_adaptive_paper_native.py": "728458624788243abcaf653f6a20a5a6201cfa9ec3328e8d7eb3fd58fdfa94b0", "tests/test_adaptive_paper_recovery.py": "6da67e1b83d887cbaaeaf289434a595dacc79517a6b74b61e2adf622f84462cb", "tests/test_adaptive_paper_runner.py": "29c4c48a8103cadb7906719d7145a9b14192b7d35486bb9eaa934faad5eac0b7", From 92e3963b3063a21c095b3472301e8f299da80d24 Mon Sep 17 00:00:00 2001 From: Scout Date: Wed, 30 Sep 2026 20:58:51 -0400 Subject: [PATCH 7/7] Register the fee-reconciliation engine files in the evidence manifest Re-registers the eleven files this branch adds or changes (docs/lanes.md hot-file protocol, last commit, taken from main's manifest). scripts/validate.py and scripts/evidence_manifest.py --check: passed. Co-Authored-By: Claude Opus 5.5 --- manifests/evidence.json | 45 +++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/manifests/evidence.json b/manifests/evidence.json index 890029a93..980343730 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -10845,8 +10845,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/README-mover.md", - "sha256": "bdea9283cabf8a3202f01359109d463a78cb2735a199e0f93baf05486152d31a", - "bytes": 21164 + "sha256": "a74574de8726cae6aef8caed2f041c13acbd13832a916e1d7c4a8527b7944e03", + "bytes": 21829 }, { "path": "blueprints/us-equities/adaptive-paper/README-native.md", @@ -10865,13 +10865,13 @@ }, { "path": "blueprints/us-equities/adaptive-paper/README-safety.md", - "sha256": "cea19e941f514591c410b28d71909b8f240678074f7c5627b1609ccc74d03e38", - "bytes": 25821 + "sha256": "c3966d55a867612e50fe3cd0d841060b3d8944ae77ad39fb4a050bc0def0bba2", + "bytes": 37314 }, { "path": "blueprints/us-equities/adaptive-paper/README-transport.md", - "sha256": "7b7837cd6866ef36271887edf261c0ab1862e04b49ff2dcbb2e3e74262fa6e23", - "bytes": 17640 + "sha256": "5a4e4a1aa7bd58e6b906604e2ab1c635cbf0417613b5c94546e6e2af968bb86e", + "bytes": 19095 }, { "path": "blueprints/us-equities/adaptive-paper/README.md", @@ -10990,8 +10990,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/mover_runner.py", - "sha256": "03f9107e40feaaefc6e896db3e20d246139f8ef8b02e1a7d7da79199e62bb7f5", - "bytes": 61848 + "sha256": "aaa2e70202e011993c6827cda808155ef7737d752073245ceded55fd8b1eb5d7", + "bytes": 65768 }, { "path": "blueprints/us-equities/adaptive-paper/mover_simulation.py", @@ -11160,13 +11160,13 @@ }, { "path": "blueprints/us-equities/adaptive-paper/runner.py", - "sha256": "27bc3fdc80472c3ee218d3723e506288ebc6eef536dafc8365b30474213a6ae4", - "bytes": 150357 + "sha256": "34ab492f450431e79a3d5d5c6de32185b22590e1a109b3c5fd976db0aa0aa29e", + "bytes": 153832 }, { "path": "blueprints/us-equities/adaptive-paper/safety.py", - "sha256": "c2d44faa1e4d369b8882dec9782ef32174c404b723edd2419833146b24eb2357", - "bytes": 83006 + "sha256": "f43def730273be28dbb2b4a791dff31ca7e3f661735eb45e35ebccb75c4d0285", + "bytes": 88369 }, { "path": "blueprints/us-equities/adaptive-paper/selector.py", @@ -11185,8 +11185,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/source-hashes.json", - "sha256": "f0fb64b0a3700f9fa7e2b14910c035dec66addf17e4bd12467b10d4f07e1e180", - "bytes": 6969 + "sha256": "c9b6f5f8f39fe6595fa13c054420e1e96633038e895495f9a4a5d6b5da0ab93a", + "bytes": 7076 }, { "path": "blueprints/us-equities/adaptive-paper/strategies.py", @@ -11200,8 +11200,8 @@ }, { "path": "blueprints/us-equities/adaptive-paper/transport.py", - "sha256": "e3038bd5562fecc7cea52df47c4ff6b94ddae7e85ce545d6684e651b856b89d9", - "bytes": 83300 + "sha256": "8d1222f7b8e90cf2af0cd6efb88eb70704cf900c480a097e86576a8382435b16", + "bytes": 92936 }, { "path": "blueprints/us-equities/adaptive-paper/trials/20260923-interrupted/README.md", @@ -45488,6 +45488,11 @@ "sha256": "32c3aa93e40392a2af60078dd4ba6c222e195a9a47b22a0bfc0f52a92814382a", "bytes": 25425 }, + { + "path": "tests/test_adaptive_paper_fees.py", + "sha256": "9a181eb3d2e20ff34ac6551022bf252d51688ab3014162ca696a37b45bdc5221", + "bytes": 78129 + }, { "path": "tests/test_adaptive_paper_financing.py", "sha256": "8d9f1f21f9d53c9ed120281409bdcb97b8c9ddacc615d0cdfa392cfcf111fa90", @@ -45515,8 +45520,8 @@ }, { "path": "tests/test_adaptive_paper_mover_native.py", - "sha256": "2c176fb4e25827fe6b5c0a7ef1277769cd402bf33b28a31ed5c2c7a6da8c3220", - "bytes": 54865 + "sha256": "2cf17ed5dd956d18b973fcdcf0ea1412dee8eec7040d29fa1d2e28781b7c16fa", + "bytes": 55169 }, { "path": "tests/test_adaptive_paper_native.py", @@ -45550,8 +45555,8 @@ }, { "path": "tests/test_adaptive_paper_transport.py", - "sha256": "e4914d1755fb70c23688cb118dc482ea95c15e57ebb3a081813f8083a385b396", - "bytes": 106024 + "sha256": "7d75e7ae89dc048cd62628fcfd666ff18eaf3daedbed836b0a451ddaf7a45f87", + "bytes": 106177 }, { "path": "tests/test_adjudicate.py",