diff --git a/adoption/bootstrap-macos.sh b/adoption/bootstrap-macos.sh index ff79e6f6c..a6c5db5d1 100755 --- a/adoption/bootstrap-macos.sh +++ b/adoption/bootstrap-macos.sh @@ -8,6 +8,11 @@ # mapfile, no associative arrays, no ${var,,}. set -Eeuo pipefail +# Round 3k: see the identical export in adoption/launchd/launchd-agents.sh +# for why (offline fault-injection tests only; no production path reads +# this). +export ADOPTION_SCRIPT_PID="$$" + usage() { printf '%s\n' \ 'Usage: bash bootstrap-macos.sh --profile [--skip-system-packages]' \ diff --git a/adoption/launchd/launchd-agents.sh b/adoption/launchd/launchd-agents.sh index 680a48bb7..a0dde1655 100755 --- a/adoption/launchd/launchd-agents.sh +++ b/adoption/launchd/launchd-agents.sh @@ -8,6 +8,17 @@ # variable is used instead). set -Eeuo pipefail +# Round 3k (coordinator, WSL-host CPU-load flakiness fix): exported purely +# for offline fault-injection tests. A shim that must signal THIS script's +# own top-level process reads this instead of $PPID -- $PPID is correct +# today for every foreign command this script calls directly (none of +# them run inside a $(...) or a pipeline, so $PPID already equals $$ at +# each call site), but that is not an invariant future edits are +# guaranteed to preserve; bootout_and_wait's own subshell-vs-plain-call +# comment below records exactly how that assumption broke once already. +# No production code path reads this variable. +export ADOPTION_SCRIPT_PID="$$" + usage() { printf '%s\n' \ 'Usage: bash launchd-agents.sh render|lint|install|status|remove [options]' \ diff --git a/manifests/evidence.json b/manifests/evidence.json index 414952063..e4f902d84 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -3144,8 +3144,8 @@ }, { "path": "adoption/bootstrap-macos.sh", - "sha256": "4de962dbcd13ea7878c17c24562aad582980ebf47f687e7ed8049e3ab7d8e511", - "bytes": 57933 + "sha256": "c12ec11732ec79de29808155af3ae6029850b9d008f3b9add73ac14db41ef9f6", + "bytes": 58122 }, { "path": "adoption/bootstrap.md", @@ -3189,8 +3189,8 @@ }, { "path": "adoption/launchd/launchd-agents.sh", - "sha256": "1a7508626e59509ee97bfc2517463ba9ed771f5279ca5cfa7abc119403deba3e", - "bytes": 37066 + "sha256": "8d607da363de3a92552e76c88043fc0576168a015e5f30898d07ee3cf4839e6e", + "bytes": 37719 }, { "path": "adoption/lifecycle.md", @@ -27745,8 +27745,8 @@ }, { "path": "tests/test_adoption_bootstrap_macos.py", - "sha256": "747420333a546becb4504a993bcda06bc514a82dae0183f78c6029bcee7c6f2b", - "bytes": 161914 + "sha256": "adb957701308abd587555a3161791e4c53ab56747aa768b594f03d5d15949e36", + "bytes": 165300 }, { "path": "tests/test_adoption_contract.py", @@ -27760,8 +27760,8 @@ }, { "path": "tests/test_adoption_launchd.py", - "sha256": "78f2190610770ff7af561886dbb63ea0e70c2afae2e100c9405109e4107f4b0f", - "bytes": 97478 + "sha256": "f50a8f8e3106e0ec34151dff4ac549c60e19f1109e3555fab7fad689af5f34c2", + "bytes": 111506 }, { "path": "tests/test_adoption_status.py", diff --git a/tests/test_adoption_bootstrap_macos.py b/tests/test_adoption_bootstrap_macos.py index 23255e287..7669560ab 100644 --- a/tests/test_adoption_bootstrap_macos.py +++ b/tests/test_adoption_bootstrap_macos.py @@ -721,6 +721,53 @@ def _shell_functions(text: str, *names: str) -> str: return "".join(blocks) +def _signal_and_wait_for_exit(signal_name: str, indent: str = " ", target_var: str = "ADOPTION_SCRIPT_PID") -> str: + """Round 3k: a fault-injection shim's real-command-then-signal tail. + + Signals `target_var` -- the harness's own top-level PID, exported as + `ADOPTION_SCRIPT_PID` right after its `set -Eeuo pipefail` (see + _run_install_npm), never `$PPID` (that half is genuinely + load-independent: see tests/test_adoption_launchd.py's + SignalShimMechanismProofTests.test_ppid_inside_a_command_substitution_ + subshell_is_not_the_script for a deterministic, no-load reproduction). + + Round 3k's FIRST draft of this helper also polled `kill -0 "$target"` + in a loop, waiting for the target to have fully exited before this + shim itself returns. That design was WRONG and is not used: this shim + is always the target's own SYNCHRONOUS foreground child (exactly how + real npm/mv/ln are invoked), so the target cannot possibly exit -- + its own `wait()` cannot return -- while still blocked waiting for THIS + shim to finish. A shim that waits for the target to disappear before + it disappears itself is circular, not a handshake; it does not fail + loudly either, since its own 30s bound always fires (proven directly: + every affected test took ~30-63s per run in this project's first + commit of this helper, passing only because the shim's own timeout + path, not a real interruption, produced a nonzero exit the assertions + happened to accept as "not 0"). + + What actually determines correctness, confirmed by direct, repeated + (bash 5.2, this host, 10/10 both with and without an added sleep) + testing: bash defers running a trapped signal's handler until whatever + is CURRENTLY in the foreground finishes -- here, this shim itself -- + and then services it immediately afterward, before the script's own + next line, regardless of how long that takes. Ordering is therefore + already guaranteed by bash's synchronous foreground-child execution + model alone, once the signal reaches the right PID (the fix above). + This tail still sleeps briefly (0.5s, comfortably above the 0.2-0.3s + this project's earlier rounds used) purely as defensive margin for the + real `kill()` syscall's effect to be recorded before this process + image goes away -- not to "win a race" against the trap, which this + shim structurally cannot observe completing. Same helper (and the + same design) as tests/test_adoption_launchd.py's own copy. + """ + p = indent + return ( + f'{p}target="${target_var}"\n' + f'{p}kill -{signal_name} "$target"\n' + f'{p}sleep 0.5\n' + ) + + class LlamaWrapperQuotingTests(unittest.TestCase): """Local integration check of the generated llama-server wrapper. @@ -1231,6 +1278,11 @@ def _run_install_npm(self, tmp_path: Path, pins_path: Path, id_: str, version: s harness = tmp_path / "install-npm-e2e-harness.sh" harness.write_text( "set -Eeuo pipefail\n" + # Round 3k: exported for this harness's own fault-injection + # shims (npm/mv/ln) to signal deterministically -- see + # _signal_and_wait_for_exit's own comment for why, never + # $PPID. + + 'export ADOPTION_SCRIPT_PID="$$"\n' + _shell_functions(SCRIPT_PATH.read_text(), "canonical_path", "prune_old_version", "npm_package_name", "install_platform_dependency", "install_npm", "cleanup") # The script-level pending_migration_prefix/_dest globals and the @@ -1261,7 +1313,13 @@ def _run_install_npm(self, tmp_path: Path, pins_path: Path, id_: str, version: s + "}\n" + f'install_npm {id_} {version} {wrapper_url} {wrapper_sha256} {ignore_scripts}\n' ) - result = subprocess.run(["bash", str(harness)], capture_output=True, text=True, timeout=60, + # Round 3k: 90s, not 60s -- a fault-injection shim's own bounded + # wait for the script to react to a signal (_signal_and_wait_for_ + # exit) can itself take up to 30s under real CPU load before it + # gives up; this must stay comfortably above npm install's own + # time plus that 30s so a genuine slow-but-eventual pass is never + # mistaken for a hang. + result = subprocess.run(["bash", str(harness)], capture_output=True, text=True, timeout=90, env={**os.environ, **(extra_env or {})}) return result, eco_root @@ -1906,21 +1964,22 @@ def test_signal_between_the_one_time_migration_and_the_flip_restores_the_real_di 'case "$dest" in\n' " *.migrating.*)\n" ' /bin/mv "$@" || exit $?\n' - f' kill -{signal_name} "$PPID"\n' - # Stay alive briefly after sending the signal: the - # parent bash is blocked in wait() for THIS process, - # and a signal delivered while it is still blocked - # there reliably interrupts that wait immediately; - # if this shim exited instantly instead, bash's - # wait() could already be returning (child exited - # normally) around the same moment the kernel is - # still only queuing the signal, letting bash race - # ahead through the rest of the flip -- successfully - # -- before ever servicing it, which would make this - # test flaky rather than a reliable reproduction of - # the coordinator's exact "signal mid-swap" scenario. - " sleep 0.2\n" - " exit 0\n" + # Round 3k: was a fixed `kill -SIG "$PPID"; sleep + # 0.2; exit 0`, wrong on two counts fixed since -- + # see _signal_and_wait_for_exit's own docstring for + # both. What this line now does: signal + # ADOPTION_SCRIPT_PID (never $PPID), then pause + # briefly (0.5s, a fixed defensive margin, not a + # wait for anything specific) before this shim + # itself returns. It does NOT block until the + # parent has exited -- this shim is the parent's + # own synchronous foreground child, so the parent + # cannot exit while still waiting on it; ordering + # is guaranteed instead by bash servicing the + # pending trap immediately once this shim (its + # current foreground child) naturally returns. + + _signal_and_wait_for_exit(signal_name, indent=" ") + + " exit 0\n" " ;;\n" "esac\n" 'exec /bin/mv "$@"\n' @@ -2084,9 +2143,8 @@ def test_a_signal_during_npm_install_leaves_final_prefix_untouched(self): "#!/bin/sh\n" 'if [ "$1" = "install" ]; then\n' f' {real_npm} "$@" || exit $?\n' - f' kill -{signal_name} "$PPID"\n' - " sleep 0.3\n" - " exit 0\n" + + _signal_and_wait_for_exit(signal_name, indent=" ") + + " exit 0\n" "fi\n" f'exec {real_npm} "$@"\n' ) @@ -2222,9 +2280,8 @@ def test_a_signal_after_creating_tmp_link_leaves_final_prefix_untouched(self): f'case "$dest" in\n' f" {stage_dir}/*-link.*)\n" ' /bin/ln "$@" || exit $?\n' - f' kill -{signal_name} "$PPID"\n' - " sleep 0.3\n" - " exit 0\n" + + _signal_and_wait_for_exit(signal_name, indent=" ") + + " exit 0\n" " ;;\n" "esac\n" 'exec /bin/ln "$@"\n' diff --git a/tests/test_adoption_launchd.py b/tests/test_adoption_launchd.py index 9f961e459..082b047c3 100644 --- a/tests/test_adoption_launchd.py +++ b/tests/test_adoption_launchd.py @@ -66,6 +66,245 @@ def _shell_functions(text: str, *names: str) -> str: return "".join(blocks) +def _signal_and_wait_for_exit(signal_name: str, indent: str = " ", target_var: str = "ADOPTION_SCRIPT_PID") -> str: + """Round 3k: a fault-injection shim's real-command-then-signal tail. + + Signals `target_var` -- launchd-agents.sh's own top-level PID, which it + exports itself (see the export next to its `set -Eeuo pipefail`), never + `$PPID` (that half is genuinely load-independent: see + SignalShimMechanismProofTests.test_ppid_inside_a_command_substitution_ + subshell_is_not_the_script for a deterministic, no-load reproduction of + exactly how `$PPID` fails once ANY step is ever wrapped in `$(...)`, + matching the pre-round-3i bootout_and_wait bug this project already + fixed once). + + Round 3k's FIRST draft of this helper also polled `kill -0 "$target"` + in a loop, waiting for the target to have fully exited before this + shim itself returns. That design was WRONG and is not used: this shim + is always the target's own SYNCHRONOUS foreground child (exactly how + real npm/mv/ln/launchctl are invoked), so the target cannot possibly + exit -- its own `wait()` cannot return -- while still blocked waiting + for THIS shim to finish. A shim that waits for the target to disappear + before it disappears itself is circular, not a handshake; it does not + fail loudly either, since its own 30s bound always fires (proven + directly: every affected test took ~30-63s per run in this project's + first commit of this helper, passing only because the shim's own + timeout path, not a real interruption, produced a nonzero exit the + assertions happened to accept as "not 0"). + + What actually determines correctness, confirmed by direct, repeated + (bash 5.2, this host, 10/10 both with and without an added sleep) + testing: bash defers running a trapped signal's handler until whatever + is CURRENTLY in the foreground finishes -- here, this shim itself -- + and then services it immediately afterward, before the script's own + next line, regardless of how long that takes. Ordering is therefore + already guaranteed by bash's synchronous foreground-child execution + model alone, once the signal reaches the right PID (the fix above). + This tail still sleeps briefly (0.5s, comfortably above the 0.2-0.3s + this project's earlier rounds used) purely as defensive margin for the + real `kill()` syscall's effect to be recorded before this process + image goes away -- not to "win a race" against the trap, which this + shim structurally cannot observe completing. + """ + p = indent + return ( + f'{p}target="${target_var}"\n' + f'{p}kill -{signal_name} "$target"\n' + f'{p}sleep 0.5\n' + ) + + +class SignalShimMechanismProofTests(unittest.TestCase): + """Round 3k, coordinator's deterministic (load-INDEPENDENT) proof of the + mechanism the fix relies on, alongside the CPU-load reproduction + attempts recorded in the round's own handoff. + + 1. TARGET (a real bug, fixed): a shim invoked from inside a command- + substitution subshell -- `x="$(...)"`, the exact construct this + codebase already uses elsewhere (launchctl_label_state's own + `print_output="$(launchctl print ...)"`, and pre-round-3i's + `result="$(bootout_and_wait ...)"`) -- forks a NEW process for that + substitution. `$PPID` read from inside it is that subshell's own + pid, never the top-level script's, so `kill -TERM "$PPID"` there + signals a process with no trap of its own (subshells reset trap + dispositions unless explicitly re-armed inside them); the subshell + just dies, and the top-level script's own TERM trap never runs at + all -- not a matter of odds, every single time. An explicitly + exported target (`export ADOPTION_SCRIPT_PID="$$"` at the script's + own top level) is unaffected: exported variables are inherited into + a subshell unchanged, so the same shim signals the right process + regardless of how many subshells sit between it and the caller. + + 2. ORDERING (this project's own first draft of the fix was WRONG, and + this test proves why, in place of the "poll kill -0 " design + originally suggested): a shim that is the target's own SYNCHRONOUS + foreground child -- true of every real shim in this suite (npm, + mv, ln, launchctl are all invoked directly, never via $(...)) -- + cannot observe the target exit, because the target's own `wait()` + cannot return while still blocked waiting for this shim. Polling + `kill -0` on the target from inside the shim is therefore circular, + not a handshake: it hangs until ITS OWN bound fires, every run, and + does so SILENTLY (a timeout there still yields the nonzero exit + these tests check for, so it does not "fail loud" the way it looks + like it should -- it makes every affected test take ~30s longer + instead). What this project's own repeated, direct testing shows + instead: bash defers a trapped signal's handler until whatever is + CURRENTLY in the foreground finishes -- here, the shim itself -- + then services it immediately afterward, before the script's own + next line, regardless of how long the shim's own post-kill sleep + is (0s and 0.5s were both tested, 10/10 each). Ordering is already + guaranteed by bash's synchronous foreground-child execution model, + once the signal reaches the right PID (property 1). + """ + + def test_ppid_inside_a_command_substitution_subshell_is_not_the_script(self): + for target_var, expect_trap_fires in (("PPID", False), ("ADOPTION_SCRIPT_PID", True)): + with self.subTest(target_var=target_var): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + marker = tmp_path / "trap-ran" + shim_dir = tmp_path / "shim" + shim_dir.mkdir() + (shim_dir / "fake-tool").write_text( + "#!/bin/sh\n" + f'kill -TERM "${target_var}"\n' + "sleep 0.3\n" + ) + (shim_dir / "fake-tool").chmod(0o755) + script = tmp_path / "target.sh" + script.write_text( + "set -Eeuo pipefail\n" + 'export ADOPTION_SCRIPT_PID="$$"\n' + f'trap \'touch {str(marker)!r}; exit 143\' TERM\n' + # The exact shape of the pre-round-3i bootout_and_wait + # bug: the script wraps its OWN call to a foreign- + # command-invoking step in $(...), as a small + # function that does something with the exit status + # AFTER calling fake-tool (captures $?, returns it) -- + # not a bare `run_it() { fake-tool; }`, which bash + # can (and does; confirmed directly) exec-replace the + # subshell with, keeping fake-tool's own $PPID equal + # to the top-level script's and hiding the bug + # entirely. bootout_and_wait's real body has exactly + # this shape: a call, then more statements using its + # exit status, which is what forces bash to keep the + # subshell alive as a genuine separate process rather + # than exec-optimizing it away. + 'run_it() { fake-tool; local rc=$?; return "$rc"; }\n' + 'x="$(run_it)"\n' + "sleep 5\n" # only ever reached if the trap never fired + ) + script.chmod(0o755) + result = subprocess.run( + ["bash", str(script)], + capture_output=True, text=True, timeout=15, + env={**os.environ, "PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}"}, + ) + self.assertEqual( + marker.exists(), expect_trap_fires, + f"target_var={target_var}: rc={result.returncode} " + f"stdout={result.stdout!r} stderr={result.stderr!r}") + + def test_the_trap_runs_before_the_scripts_next_line_once_the_shim_exits(self): + # Proves the ORDERING property _signal_and_wait_for_exit actually + # relies on (see property 2 in the class docstring): regardless of + # the shim's own post-kill sleep -- none at all, or the 0.5s this + # fix uses -- the target's trap always preempts the script's next + # line. `reached` existing would mean the trap was skipped, which + # this project's own repeated testing (10/10 each case) never + # observed. + for extra_sleep in ("0", "0.5"): + with self.subTest(shim_post_kill_sleep=extra_sleep): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + marker = tmp_path / "trap-ran" + reached = tmp_path / "reached-after-shim" + shim_dir = tmp_path / "shim" + shim_dir.mkdir() + (shim_dir / "fake-tool").write_text( + "#!/bin/sh\n" + 'kill -TERM "$ADOPTION_SCRIPT_PID"\n' + f'sleep {extra_sleep}\n' + "exit 0\n" + ) + (shim_dir / "fake-tool").chmod(0o755) + script = tmp_path / "target.sh" + script.write_text( + "set -Eeuo pipefail\n" + 'export ADOPTION_SCRIPT_PID="$$"\n' + f'trap \'touch {str(marker)!r}; exit 143\' TERM\n' + # Direct invocation -- fake-tool is this script's own + # synchronous foreground child, exactly like real + # npm/mv/ln/launchctl calls (never $(...) -- see the + # test above for that half). + 'fake-tool\n' + f'touch {str(reached)!r}\n' + ) + script.chmod(0o755) + result = subprocess.run( + ["bash", str(script)], + capture_output=True, text=True, timeout=10, + env={**os.environ, "PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}"}, + ) + self.assertTrue( + marker.exists(), + f"trap never ran: rc={result.returncode} " + f"stdout={result.stdout!r} stderr={result.stderr!r}") + self.assertFalse( + reached.exists(), + "the trap must preempt the script's next line, not run after it") + + def test_the_actual_helper_through_a_subshell_uses_the_fixed_target_not_ppid(self): + # Codex P2 (#146): the two tests above each build their OWN shim + # script by hand, so neither would notice a regression in + # _signal_and_wait_for_exit ITSELF -- the class would still pass + # even if the real fix were reverted. This one runs the ACTUAL + # helper's generated text (not a hand-written stand-in) as the + # shim, through the same non-optimizable command-substitution + # subshell as test_ppid_inside_a_command_substitution_subshell_is_ + # not_the_script above, against a script that exports + # ADOPTION_SCRIPT_PID exactly like the real product scripts do. + # Manually confirmed while writing this commit: temporarily + # changing _signal_and_wait_for_exit's own `target_var` default + # back to "PPID" makes this exact test fail (marker never + # created); the default was restored to "ADOPTION_SCRIPT_PID" + # before committing. + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + marker = tmp_path / "trap-ran" + shim_dir = tmp_path / "shim" + shim_dir.mkdir() + (shim_dir / "fake-tool").write_text( + "#!/bin/sh\n" + _signal_and_wait_for_exit("TERM", indent="") + ) + (shim_dir / "fake-tool").chmod(0o755) + script = tmp_path / "target.sh" + script.write_text( + "set -Eeuo pipefail\n" + 'export ADOPTION_SCRIPT_PID="$$"\n' + f'trap \'touch {str(marker)!r}; exit 143\' TERM\n' + # Identical non-optimizable shape to the subshell test + # above -- a function that inspects $? after calling + # fake-tool, so bash cannot exec-replace the subshell with + # fake-tool directly (which would hide a $PPID regression + # by leaving fake-tool's own $PPID equal to the script's). + 'run_it() { fake-tool; local rc=$?; return "$rc"; }\n' + 'x="$(run_it)"\n' + "sleep 5\n" # only ever reached if the trap never fired + ) + script.chmod(0o755) + result = subprocess.run( + ["bash", str(script)], + capture_output=True, text=True, timeout=15, + env={**os.environ, "PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}"}, + ) + self.assertTrue( + marker.exists(), + "the real helper's default target must reach the script " + f"through a subshell: rc={result.returncode} " + f"stdout={result.stdout!r} stderr={result.stderr!r}") + + class TemplateRenderAndSchemaTests(unittest.TestCase): def setUp(self): self.templates = templates() @@ -538,9 +777,14 @@ def _launchctl_shim(self, directory: Path, log: Path) -> Path: return shim def _run(self, args, env): + # Round 3k: 60s, not 30s -- a fault-injection shim's own bounded + # wait for the script to react to a signal (_signal_and_wait_for_ + # exit) can itself take up to 30s under real CPU load before it + # gives up; this must stay comfortably above that so a genuine + # slow-but-eventual pass is never mistaken for a hang. return subprocess.run( ["bash", str(SCRIPT_PATH), *args], - capture_output=True, text=True, timeout=30, env=env, + capture_output=True, text=True, timeout=60, env=env, ) def test_help_exits_zero(self): @@ -1366,8 +1610,9 @@ def _matrix_binary_shim(self, tmp_path, marker_dir, log, binary, target, mode, s "*.plist.new.*" temp-file path pattern (this script's ONLY staging target) for binary in ("cp", "mv"). `mode` is "failure" (never performs the real action) or a signal name (performs the real - action, then signals $PPID -- the same real-command-then-signal - design test_adoption_bootstrap_macos.py's own signal shims use). + action, then runs _signal_and_wait_for_exit -- the same + deterministic-target, poll-to-completion design + test_adoption_bootstrap_macos.py's own signal shims use, round 3k). For launchctl, every OTHER subcommand keeps working exactly like _launchctl_shim (marker-based, reading/writing marker_dir), so the matrix isolates exactly one step at a time. @@ -1389,13 +1634,9 @@ def _matrix_binary_shim(self, tmp_path, marker_dir, log, binary, target, mode, s f' rm -f "{marker_dir!s}/$label.loaded.marker"') else: real = ":" - lines += [ - f" {subcmd})", - f" {real}", - f' kill -{signal_name} "$PPID"', - " sleep 0.3", - " exit 0 ;;", - ] + lines += [f" {subcmd})", f" {real}"] + lines += _signal_and_wait_for_exit(signal_name, indent=" ").rstrip("\n").split("\n") + lines += [" exit 0 ;;"] elif subcmd == "bootstrap": lines += [ " bootstrap)", @@ -1443,9 +1684,8 @@ def _matrix_binary_shim(self, tmp_path, marker_dir, log, binary, target, mode, s body = ('case "$check" in\n' " *.plist.new.*)\n" f' /bin/{binary} "$@" || exit $?\n' - f' kill -{signal_name} "$PPID"\n' - " sleep 0.3\n" - " exit 0\n" + + _signal_and_wait_for_exit(signal_name, indent=" ") + + " exit 0\n" " ;;\n" "esac\n" f'exec /bin/{binary} "$@"\n') @@ -1735,9 +1975,8 @@ def test_retry_converges_while_an_earlier_runs_teardown_is_still_asynchronously_ " exit 36\n" " fi\n" f' touch "{async_shim!s}/teardown-started"\n' - f' kill -{signal_name} "$PPID"\n' - " sleep 0.3\n" - " exit 0 ;;\n" + + _signal_and_wait_for_exit(signal_name, indent=" ") + + " exit 0 ;;\n" ' enable) exit 0 ;;\n' ' *) exit 0 ;;\n' "esac\n"