diff --git a/bin/fm-backlog-transition-lib.sh b/bin/fm-backlog-transition-lib.sh index 6a27558a2b3..87fedd947cf 100644 --- a/bin/fm-backlog-transition-lib.sh +++ b/bin/fm-backlog-transition-lib.sh @@ -53,16 +53,16 @@ # the preserved task. The writer and replay share one complete-record validator, # so teardown never publishes or acts on a close replay would reject. # The validator pins the data path to this home's configured root before any -# recovery mutation, then re-runs exactly that close. +# recovery mutation; replay then selects the transition from the current row. # `tasks-axi done` on an already-closed task backfills links # without moving the close date, so replay is idempotent. Ordinary dispatch needs # no marker: it publishes the meta first, so a crash leaves the meta itself as # the evidence that the row is owed a start. -# A captain-held row uses the same record with a `mode=retain` line: replay then -# records the deliverable and reopens the row instead of closing it, and never -# closes a row that reads as an open captain call. An answer that closes the row -# first applies any supported retained artifact from the validated record, then -# replay simply retires the record. +# A captain-held row uses the same record with a `mode=retain` line. Replay +# records the deliverable and reopens the row only while it remains an open +# captain call, never closing it. If the hold was released before replay, the +# finished row closes instead of reopening unheld work. If an answer already +# closed a retained row, replay simply retires the satisfied record. # Set by fm_backlog_transition_applies for a return-1 exemption. # shellcheck disable=SC2034 # Output global, read by the sourcing caller. @@ -1396,6 +1396,12 @@ fm_backlog_close_marker_replay() { # printf '%s' "$rest" } +recorded_resolved_hold_set() { + local body + body=$(decode_shown_value "$1") || return 1 + printf '%s\n' "$body" | awk ' + /^Resolution mode: / { + if (getline > 0 && /^Resolves hold set: /) { + sub(/^Resolves hold set: /, "") + print + } + exit + } + ' +} + closed_answer_replay_mode_compatible() { # case "$1" in answered|repaired|routed) return 0 ;; @@ -525,11 +547,13 @@ closed_answer_replay_mode_compatible() { # # The record's label is what keeps an evidence-backed reconciliation from # reading as the captain's own words. `reconciled` closes a call that went moot # and carries verified evidence; every other mode carries what the captain said. -resolution_block() { # +resolution_block() { local label='Captain decision:' [ "$1" != reconciled ] || label='Reconciliation evidence:' - printf 'Resolution recorded by fm-captain-hold.\nDecision digest: %s\nResolution mode: %s\n\n%s\n%s\n' \ - "$DECISION_DIGEST" "$1" "$label" "$DECISION_TEXT" + printf 'Resolution recorded by fm-captain-hold.\nDecision digest: %s\nResolution mode: %s\n' \ + "$DECISION_DIGEST" "$1" + [ -z "$2" ] || printf 'Resolves hold set: %s\n' "$2" + printf '\n%s\n%s\n' "$label" "$DECISION_TEXT" } # Durable state of one captain call: an active captain hold (annotations @@ -925,7 +949,7 @@ verify_entry_durable() { # ; prints " &2; exit 2; } shift while [ "$#" -gt 0 ]; do @@ -952,12 +976,12 @@ command_hold() { *) fail "--until must be a YYYY-MM-DD date: $until" ;; esac fi + acquire_task_control_lock "$id" hold_set=${FM_CAPTAIN_HOLD_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)} case "$hold_set" in [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]Z) : ;; *) fail "FM_CAPTAIN_HOLD_NOW must be a UTC YYYY-MM-DDTHH:MM:SSZ timestamp" ;; esac - acquire_task_control_lock "$id" require_tasks_axi if task_show "$id"; then show=$TASK_SHOW_OUTPUT @@ -999,6 +1023,16 @@ command_hold() { # snapshot may see the harmless stamp by itself, but can never see a newly # held task without the timestamp that defines this hold lifecycle's age. task_show_or_fail "$id" "task $id disappeared before recording its hold-set stamp" + if [ "$preserve_hold_set" = 0 ] \ + || [ -z "$(body_hold_set_timestamp "$(show_field_value "$show" body)")" ]; then + resolved_hold_set=$(recorded_resolved_hold_set "$(show_field "$show" body)") + while [ "$hold_set" = "$resolved_hold_set" ]; do + [ -z "${FM_CAPTAIN_HOLD_NOW:-}" ] \ + || fail "FM_CAPTAIN_HOLD_NOW collides with the newest resolved hold for task $id; choose a later timestamp" + sleep 1 + hold_set=$(date -u +%Y-%m-%dT%H:%M:%SZ) + done + fi write_hold_set_stamp "$id" "$(show_field "$show" body)" "$hold_set" "$preserve_hold_set" task_show_or_fail "$id" "task $id disappeared while recording its hold-set stamp" [ -n "$(body_hold_set_timestamp "$(show_field_value "$show" body)")" ] \ @@ -1039,10 +1073,10 @@ command_hold() { # Successful closure removes the stamp to restore resolution-first ordering. write_resolution_record() { # local id=$1 mode=$2 body=$3 new_body tmp hold_set - new_body=$(resolution_block "$mode") body=$(decode_shown_value "$body") \ || fail "could not decode the existing body for $id" hold_set=$(body_hold_set_timestamp "$body") + new_body=$(resolution_block "$mode" "$hold_set") if [ -n "$hold_set" ]; then body=${body#"Captain hold set: $hold_set"} case "$body" in @@ -1095,6 +1129,13 @@ apply_pending_retained_artifact() { # esac } +# True while a worker still owns the work item: its runtime record exists or the +# backlog still lists it In flight. Completing such an item would record a +# landing that has not happened, so a keyed answer only ever releases it. +live_work_item() { # + [ "$2" = in_flight ] || [ -f "$STATE/$1.meta" ] +} + close_answered() { # if [ "$2" = 1 ]; then tasks_axi unhold "$1" >/dev/null @@ -1127,13 +1168,15 @@ remove_interrupted_answer_stamp() { # } command_answer() { - local id=${1:-} decision_file='' release=0 show state hold_kind body outcome recorded_mode occurrence + local id=${1:-} decision_file='' release=0 auto_release=0 show state hold_kind body outcome recorded_mode occurrence + local cur resolved_hold_set [ "$#" -ge 1 ] || { usage >&2; exit 2; } shift while [ "$#" -gt 0 ]; do case "$1" in --decision-file) shift; decision_file=${1:-} ;; --release) release=1 ;; + --auto-release) auto_release=1 ;; *) usage >&2; exit 2 ;; esac shift @@ -1147,6 +1190,13 @@ command_answer() { state=$(show_field "$show" state) hold_kind=$(show_field_value "$show" hold_kind) body=$(show_field "$show" body) + if [ "$auto_release" = 1 ] && [ "$state" != "done" ] \ + && { live_work_item "$id" "$state" \ + || { [ "$hold_kind" != captain ] \ + && [ "$(recorded_decision_digest "$body" || true)" = "$DECISION_DIGEST" ] \ + && [ "$(recorded_resolution_mode "$body" || true)" = released ]; }; }; then + release=1 + fi if [ "$release" = 1 ]; then outcome=released; else outcome=answered; fi # The occurrence the parent line names: the record about to be written is # one past those already in the body, and a retry names the newest one. @@ -1158,16 +1208,17 @@ command_answer() { [ "$(recorded_decision_digest "$body" || true)" = "$DECISION_DIGEST" ] \ || fail "captain-held task $id records a different captain decision" recorded_mode=$(recorded_resolution_mode "$body" || true) - closed_answer_replay_mode_compatible "$recorded_mode" "$body" \ + { closed_answer_replay_mode_compatible "$recorded_mode" "$body" \ + || { [ "$auto_release" = 1 ] && [ "$recorded_mode" = released ]; }; } \ || fail "task $id records this resolution with mode ${recorded_mode:-unknown}; it is not a captain-answer replay" [ "$release" = 0 ] \ || fail "task $id records this answer with mode ${recorded_mode:-unknown}; --release cannot reopen a closed task" remove_interrupted_answer_stamp "$id" - if [ "$recorded_mode" = repaired ]; then - publish_parent_resolution_then_retire "$id" $((occurrence - 1)) "answered (repaired)" - else - publish_parent_resolution_then_retire "$id" $((occurrence - 1)) answered - fi + case "$recorded_mode" in + repaired) publish_parent_resolution_then_retire "$id" $((occurrence - 1)) "answered (repaired)" ;; + released) publish_parent_resolution_then_retire "$id" $((occurrence - 1)) released ;; + *) publish_parent_resolution_then_retire "$id" $((occurrence - 1)) answered ;; + esac printf 'answered: %s\n' "$id" return 0 fi @@ -1190,17 +1241,22 @@ command_answer() { fi if [ "$hold_kind" = captain ]; then - # Actively the captain's item (a date-expired hold keeps its annotations - # and stays answerable). A matching record means an interrupted close to - # finish; a different digest is a NEW answer on a re-held task and gets - # its own record on top. Either way the close mode is the caller's flag, - # checked against an interrupted close's recorded mode so a retry cannot - # silently flip a release into a close. + cur=$(body_hold_set_timestamp "$(decode_shown_value "$body")") + resolved_hold_set=$(recorded_resolved_hold_set "$body") if body_has_resolution_record "$body" \ - && [ "$(recorded_decision_digest "$body" || true)" = "$DECISION_DIGEST" ]; then + && [ "$(recorded_decision_digest "$body" || true)" = "$DECISION_DIGEST" ] \ + && { [ -z "$resolved_hold_set" ] \ + || { [ -n "$cur" ] && [ "$resolved_hold_set" = "$cur" ]; }; }; then recorded_mode=$(recorded_resolution_mode "$body" || true) case "$recorded_mode" in - released) [ "$release" = 1 ] || fail "task $id records this answer as a release; retry with --release" ;; + released) + if [ "$release" = 0 ]; then + [ "$auto_release" = 1 ] || fail "task $id records this answer as a release; retry with --release" + if [ -z "$resolved_hold_set" ]; then + fail "task $id records this answer as a release; close it with reconcile or a direct answer" + fi + fi + ;; answered|routed) [ "$release" = 0 ] || fail "task $id records this answer as a close; retry without --release" ;; *) fail "task $id records this resolution with mode ${recorded_mode:-unknown}; it is not a captain-answer replay" ;; esac @@ -1208,7 +1264,10 @@ command_answer() { fail "could not close answered captain-held task $id" fi remove_interrupted_answer_stamp "$id" - publish_parent_resolution_then_retire "$id" $((occurrence - 1)) "$outcome" + case "$recorded_mode" in + released) publish_parent_resolution_then_retire "$id" $((occurrence - 1)) released ;; + *) publish_parent_resolution_then_retire "$id" $((occurrence - 1)) "$outcome" ;; + esac printf '%s: %s\n' "$outcome" "$id" return 0 fi @@ -1397,7 +1456,7 @@ command_answers() { skipped=$((skipped + 1)) continue fi - release_flag='' + release_flag=--auto-release case "${mode:-}" in ''|done) : ;; release) release_flag=--release ;; @@ -1449,16 +1508,39 @@ command_answers() { && { [ "$recorded_digest" = "$digest" ] \ || { case "$body" in *"Resolution recorded by fm-decision-hold."*) true ;; *) false ;; esac \ && [ -n "$legacy_digest" ] && [ "$recorded_digest" = "$legacy_digest" ]; }; }; then - if { [ -z "$release_flag" ] && [ "$state" = "done" ] \ - && closed_answer_replay_mode_compatible "$recorded_mode" "$body"; } \ - || { [ "$release_flag" = --release ] && [ "$state" != "done" ] \ - && [ "$hold_kind" != captain ] && [ "$recorded_mode" = released ]; }; then + if { [ "$release_flag" = --auto-release ] && [ "$state" = "done" ] \ + && { closed_answer_replay_mode_compatible "$recorded_mode" "$body" \ + || [ "$recorded_mode" = released ]; }; } \ + || { [ "$state" != "done" ] && [ "$hold_kind" != captain ] \ + && [ "$recorded_mode" = released ]; }; then + acquire_task_control_lock "$id" + if ! task_show "$id"; then + release_task_control_lock || fail "cannot release task control for $id" + printf 'skipped: %s (absent)\n' "$id" + skipped=$((skipped + 1)) + continue + fi + show=$TASK_SHOW_OUTPUT + state=$(show_field "$show" state) + hold_kind=$(show_field_value "$show" hold_kind) + if [ "$(show_field "$show" body)" != "$body" ] \ + || ! { { [ "$release_flag" = --auto-release ] && [ "$state" = "done" ] \ + && { closed_answer_replay_mode_compatible "$recorded_mode" "$body" \ + || [ "$recorded_mode" = released ]; }; } \ + || { [ "$state" != "done" ] && [ "$hold_kind" != captain ] \ + && [ "$recorded_mode" = released ]; }; }; then + release_task_control_lock || fail "cannot release task control for $id" + printf 'skipped: %s (task changed before answer replay)\n' "$id" + skipped=$((skipped + 1)) + continue + fi occurrence=$(resolution_record_count "$body") case "$recorded_mode" in repaired) publish_parent_resolution_then_retire "$id" "$occurrence" "answered (repaired)" ;; released) publish_parent_resolution_then_retire "$id" "$occurrence" released ;; *) publish_parent_resolution_then_retire "$id" "$occurrence" answered ;; esac + release_task_control_lock || fail "cannot release task control for $id" printf 'closed: %s\n' "$id" closed=$((closed + 1)) continue @@ -1474,8 +1556,7 @@ command_answers() { skipped=$((skipped + 1)) continue fi - # shellcheck disable=SC2086 # release_flag is empty or a single literal flag. - if "$0" answer "$id" --decision-file "$tmp" $release_flag /dev/null 2>"$err"; then + if "$0" answer "$id" --decision-file "$tmp" "$release_flag" /dev/null 2>"$err"; then # A parent-channel delivery problem is reported on stderr by the answer # path even when the close succeeded; keep it visible. [ ! -s "$err" ] || cat "$err" >&2 diff --git a/docs/captain-hold-lifecycle.md b/docs/captain-hold-lifecycle.md index 1770201ef16..023d672aef7 100644 --- a/docs/captain-hold-lifecycle.md +++ b/docs/captain-hold-lifecycle.md @@ -90,9 +90,7 @@ A matching retry also completes any resolution-first normalization left unfinish ### Answer retries and tasks closed elsewhere -- An exact retry is idempotent only when the requested close mode matches the newest record. -- A drifted answer or a mode mismatch is rejected. -- A re-held task accepts a new answer as a new record on top. +[`bin/fm-captain-hold.sh --help`](../bin/fm-captain-hold.sh) owns direct-answer retry compatibility; [answer-time resolution](#answer-time-resolution) explains automatic recovery. On a task closed outside the script, `answer` records the missing block only when the captain-hold annotations tasks-axi preserves through a close prove the captain owned it. It also verifies the task stays closed. @@ -160,7 +158,7 @@ It remains on the appropriate Captain's Call or Charted Next decision surface in The interrupted close/retention marker's publication and replay safety gates are owned by [`bin/fm-backlog-transition-lib.sh`](../bin/fm-backlog-transition-lib.sh)'s CRASH RECOVERY header. -If the captain answers before replay, `answer` validates that record and copies any supported retained pull request or report into the row before closing it. +If a direct answer closes the row before replay, `answer` validates that record and copies any supported retained pull request or report into the row before closing it. A retained Gerrit change URL is instead recorded as a `Gerrit change ` note on that close. Replay then retires the record. @@ -174,7 +172,7 @@ They are recorded for separate upstream work rather than representing defects in - A relocated retained report cannot reach the row, because tasks-axi accepts only `data//report.md`. `done` reports `Task report link must be a data//report.md path`, and `update` reports `--report must be a data//report.md path`. -When an interrupted retention leaves such a relocated report in the validated pending-close record, `answer` skips only that known-unsupported row artifact and closes normally. +When an interrupted retention leaves such a relocated report in the validated pending-close record, `answer`, or cleanup replay converting retention to completion after a release, skips only that known-unsupported row artifact and closes normally. The delivery then remains absent from Recently Landed instead of wedging the captain's answer. A pending-close record that fails validation outright is a different case, and it still refuses the answer. @@ -190,26 +188,24 @@ Only `answer` with the captain's words or evidence-backed `reconcile close` reso "A keyed answer resolves its matching captain-held task" is one capability with one owner. `answers` is its channel-agnostic entry point. -It reads `\t\t