Skip to content

test(live-canary): promote QA 9/10 Slack canaries into the cron rotation #5949

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BenKurrek merged 2 commits into from
Jul 10, 2026
Merged

test(live-canary): promote QA 9/10 Slack canaries into the cron rotation #5949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
94682a2
test(live-canary): promote QA 9/10 Slack canaries into the cron rotation
BenKurrek Jul 10, 2026
fcad26b
Merge branch 'main' into canary/promote-qa-9-10
BenKurrek Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 10 additions & 12 deletions .github/workflows/live-canary.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -836,24 +836,22 @@ jobs:
- shard_id: qa-8-delivery
shard_name: QA 8D
cases: qa_8d_hn_keyword_slack_delivery
# Automation delivery probes (wrong-channel / duplicate-delivery /
# raw-user-id / per-trigger routing). dispatch_only until the
# per-trigger delivery-target fixes (#5898) merge: on pre-fix servers
# these are expected-red, so they must not page the 3-hourly cron.
# Automation delivery probes, in the 3-hourly cron rotation (the
# delivery-routing fixes from #5898 are merged and live-verified
# green). Guards regressions in: wrong-channel routine delivery,
# duplicate DM delivery, raw Slack user ids leaking into digests,
# and per-trigger delivery-target routing.
- shard_id: qa-9
shard_name: QA 9
cases: qa_9a_slack_connect,qa_9b_routine_dm_delivery_exactly_once,qa_9c_slack_digest_names_not_ids,qa_9d_routine_per_trigger_delivery_target
dispatch_only: true
# Slack tool-correctness probes (self-attribution / status readback
# / thread replies / membership honesty / error honesty / mention
# encoding / last-sent recall / email hallucination guard /
# raw-entity hygiene). dispatch_only: expected-red on pre-fix hosts
# (self-identity, status fields, thread replies, structured errors),
# so they must not page the 3-hourly cron until the fixes merge.
# Slack tool-correctness probes, in the 3-hourly cron rotation (the
# tool-surface fixes from #5898 are merged and live-verified green).
# Guards regressions in: self-attribution, status readback, thread
# replies, membership honesty, error honesty, mention encoding,
# last-sent recall, email hallucination, and raw-entity hygiene.
- shard_id: qa-10
shard_name: QA 10
cases: qa_10a_slack_self_attribution,qa_10b_slack_ooo_status,qa_10c_slack_thread_replies,qa_10d_slack_channel_membership,qa_10e_slack_error_honesty,qa_10f_slack_mention_encoding,qa_10g_slack_last_message_sent,qa_10h_slack_email_hallucination_guard,qa_10i_slack_raw_entity_hygiene
dispatch_only: true
env:
REBORN_WEBUI_V2_LIVE_QA_LLM_API_KEY_ENV: NEARAI_API_KEY
REBORN_WEBUI_V2_LIVE_QA_LLM_PROVIDER_ID: nearai
Expand Down
22 changes: 4 additions & 18 deletions scripts/reborn_webui_v2_live_qa/run_live_qa.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6603,27 +6603,22 @@ async def case_qa_10i_slack_raw_entity_hygiene(ctx: LiveQaContext) -> ProbeResul
# The workspace sweep runs on the personal token; without this gate a
# wrong-workspace token would make the sweep structurally blind.
requires_slack_personal_auth=True,
# Expected-red on pre-fix servers; keep out of bare local runs until
# promoted off dispatch_only.
default_enabled=False,
),
"qa_9c_slack_digest_names_not_ids": CaseSpec(
case_qa_9c_slack_digest_names_not_ids,
requires_slack=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_9d_routine_per_trigger_delivery_target": CaseSpec(
case_qa_9d_routine_per_trigger_delivery_target,
requires_slack=True,
requires_slack_target=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
# QA 10 family: Slack tool-correctness probes. All expected-red on
# pre-fix hosts (self-identity, status fields, thread replies, membership
# view, structured errors, mention encoding, entity rendering), so all
# stay default_enabled=False / dispatch_only until their fixes merge.
# QA 10 family: Slack tool-correctness probes (self-identity, status
# fields, thread replies, membership view, structured errors, mention
# encoding, entity rendering). Promoted into default runs and the cron
# rotation: the fixes merged and were live-verified green (9/9).
# requires_slack_target marks the cases that seed into / read from a
# Slack DM (it keeps the prepared home's DM route provisioning on); their
# seeding/read anchor is the personal↔bot DM
Expand All @@ -6633,60 +6628,51 @@ async def case_qa_10i_slack_raw_entity_hygiene(ctx: LiveQaContext) -> ProbeResul
requires_slack=True,
requires_slack_target=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_10b_slack_ooo_status": CaseSpec(
case_qa_10b_slack_ooo_status,
requires_slack=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_10c_slack_thread_replies": CaseSpec(
case_qa_10c_slack_thread_replies,
requires_slack=True,
requires_slack_target=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_10d_slack_channel_membership": CaseSpec(
case_qa_10d_slack_channel_membership,
requires_slack=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_10e_slack_error_honesty": CaseSpec(
case_qa_10e_slack_error_honesty,
requires_slack=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_10f_slack_mention_encoding": CaseSpec(
case_qa_10f_slack_mention_encoding,
requires_slack=True,
requires_slack_target=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_10g_slack_last_message_sent": CaseSpec(
case_qa_10g_slack_last_message_sent,
requires_slack=True,
requires_slack_target=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_10h_slack_email_hallucination_guard": CaseSpec(
case_qa_10h_slack_email_hallucination_guard,
requires_slack=True,
requires_slack_target=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
"qa_10i_slack_raw_entity_hygiene": CaseSpec(
case_qa_10i_slack_raw_entity_hygiene,
requires_slack=True,
requires_slack_target=True,
requires_slack_personal_auth=True,
default_enabled=False,
),
}

Expand Down
38 changes: 30 additions & 8 deletions scripts/reborn_webui_v2_live_qa/test_run_live_qa.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3627,7 +3627,7 @@ def test_outbound_final_reply_targets_reads_preference_rows(self):
run_live_qa._outbound_final_reply_targets(Path(tmp)), {}
)

def test_exactly_once_case_specs_gate_on_personal_auth_and_stay_dispatch_only(self):
def test_exactly_once_case_specs_gate_on_personal_auth_and_run_by_default(self):
for case_name in (
"qa_9b_routine_dm_delivery_exactly_once",
"qa_9d_routine_per_trigger_delivery_target",
Expand All @@ -3644,10 +3644,10 @@ def test_exactly_once_case_specs_gate_on_personal_auth_and_stay_dispatch_only(se
"qa_9c_slack_digest_names_not_ids",
"qa_9d_routine_per_trigger_delivery_target",
):
self.assertFalse(
self.assertTrue(
run_live_qa.CASES[case_name].default_enabled,
f"{case_name} is expected-red on pre-fix servers and must "
"stay out of bare local default runs until promoted",
f"{case_name} is promoted (delivery-routing fixes merged and "
"live-verified green) and must run in bare local default runs",
)

def test_exc_text_preserves_type_for_empty_str_exceptions(self):
Expand Down Expand Up @@ -4195,7 +4195,7 @@ def test_slack_second_user_token_reads_optional_env_and_asserts_loudly(self):
):
run_live_qa._require_slack_second_user_token(self._dummy_ctx())

def test_qa_10_case_specs_gate_and_stay_dispatch_only(self):
def test_qa_10_case_specs_gate_and_run_by_default(self):
qa_10_cases = [
"qa_10a_slack_self_attribution",
"qa_10b_slack_ooo_status",
Expand Down Expand Up @@ -4223,10 +4223,11 @@ def test_qa_10_case_specs_gate_and_stay_dispatch_only(self):
f"{case_name} seeds/reads with the personal token; without "
"the workspace-mismatch gate its arms are structurally blind",
)
self.assertFalse(
self.assertTrue(
spec.default_enabled,
f"{case_name} is expected-red on pre-fix hosts and must stay "
"out of bare local default runs until promoted",
f"{case_name} is promoted (tool-surface fixes merged and "
"live-verified 9/9 green) and must run in bare local default "
"runs",
)
self.assertEqual(
spec.requires_slack_target,
Expand Down Expand Up @@ -4682,6 +4683,27 @@ def test_live_canary_workflow_shards_cover_non_telegram_qa_suite(self):

self.assertEqual(len(sharded_cases), len(set(sharded_cases)))
self.assertEqual(sharded_cases, selected_cases)
# QA 9/10 are promoted: no shard in the matrix is dispatch_only any
# more, so every shard (qa-9/qa-10 included) runs on the 3-hourly
# schedule and on a default cases=all dispatch. The resolve-step
# guard itself stays for any FUTURE expected-red shard.
matrix_match = re.search(
r"(?ms)^\s+include:\n(?P<matrix>.*?)^\s+env:", match.group("body")
)
self.assertIsNotNone(matrix_match, "live QA shard matrix missing")
self.assertNotIn(
"dispatch_only",
matrix_match.group("matrix"),
"no live QA shard is dispatch_only; qa-9/qa-10 are promoted into "
"the cron rotation — only add dispatch_only for a NEW "
"expected-red shard",
)
self.assertIn(
"Shard is dispatch_only; skipping on schedule.",
match.group("body"),
"keep the resolve-step dispatch_only guard for future "
"expected-red shards",
)
all_shard_cases_match = re.search(
r"(?ms)^\s+ALL_SHARD_CASES:\s*>-\n(?P<cases>.*?)(?=^\s+run:\s*\|)",
match.group("body"),
Expand Down
Loading