diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index b1b9e7ecc98..6ee36de485d 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -9,15 +9,26 @@ steps: steps: - label: "Simple · Model Executor Test · Shard %N/%t" agent_pool: mi300_1 - parallelism: 2 + parallelism: 3 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking + timeout_in_minutes: 90 commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" + - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" + + - label: "Simple · Omni Processor Test" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + timeout_in_minutes: 60 + commands: + - export VLLM_ROCM_USE_AITER=0 + - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'" - label: "Simple · Diffusion Test · Shard %N/%t" agent_pool: mi300_1 @@ -25,6 +36,7 @@ steps: depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking + timeout_in_minutes: 45 commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 585f089a271..e46b559f318 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -9,26 +9,39 @@ steps: steps: - label: "Simple · Model Executor Test · Shard %N/%t" agent_pool: mi300_1 - parallelism: 2 + parallelism: 3 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking + timeout_in_minutes: 90 commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" + - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - - label: "Simple · Diffusion Test" + - label: "Simple · Omni Processor Test" agent_pool: mi300_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking + timeout_in_minutes: 60 + commands: + - export VLLM_ROCM_USE_AITER=0 + - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'" + + - label: "Simple · Diffusion Test · Shard %N/%t" + agent_pool: mi300_1 + parallelism: 4 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + timeout_in_minutes: 45 commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py" + - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - label: "Simple · Engine&Entrypoints Test" agent_pool: mi300_1 @@ -69,6 +82,27 @@ steps: - group: ":card_index_dividers: Diffusion Test" steps: + # Keep real FlashAttention execution out of the CPU-marked Simple lane. + # AITER may need to build several kernels on a cold worker, so run one + # pytest worker with job-local build directories and a bounded timeout. + - label: "ROCm · FlashAttention/AITER GPU Coverage" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: NonBlocking + timeout_in_minutes: 60 + commands: + - export VLLM_ROCM_USE_AITER=1 + - export AITER_JIT_DIR="/tmp/vllm-omni-aiter-$$BUILDKITE_JOB_ID" + - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-torch-extensions-$$BUILDKITE_JOB_ID" + - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR" + - >- + timeout --signal=TERM --kill-after=2m 55m + pytest -s -v -n 1 + tests/diffusion/attention/ + -m 'core_model and rocm and MI325 and cards_1' + --run-level "core_model" + - label: "Diffusion · Batch Test" agent_pool: mi300_1 depends_on: amd-build @@ -160,6 +194,39 @@ steps: # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model +- label: "ROCm · Engine & Model Executor GPU Coverage (R2-01)" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: NonBlocking + timeout_in_minutes: 20 + artifact_paths: + - "artifacts/rocm-r2-01/**/*" + commands: + # Native AMD tests run from /app/vllm-omni, while Buildkite uploads + # artifacts relative to its checkout directory. + - export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01" + - mkdir -p "$$R2_01_ARTIFACT_DIR" + # The CUDA ready job also covers tests/worker/test_batched_omni_output.py. + # Keep the initial ROCm burn-in scoped to the requested directories. + # Verbose pytest output and JUnit retain node IDs without a second + # collection-only pass over the same GPU-backed test tree. + # The CUDA/L4-only CosyVoice3 ragged-flow parity test cold-JITs multiple + # AITER kernels on ROCm and is covered by the CUDA ready lane. + - >- + pytest -s -v -ra + tests/engine/ tests/model_executor/ + -m 'core_model and cuda and not + (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' + -k 'not test_code2wav_streaming_batch_matches_ragged_flow_numerics' + --run-level "core_model" + --durations=0 + --junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml" + 2>&1 | tee "$$R2_01_ARTIFACT_DIR/pytest.log" + - >- + tail -n 100 "$$R2_01_ARTIFACT_DIR/pytest.log" > + "$$R2_01_ARTIFACT_DIR/pytest-summary.txt" + - label: "Engine Test" agent_pool: mi300_1 depends_on: amd-build @@ -172,6 +239,71 @@ steps: pytest -s -v tests/engine/test_async_omni_engine_abort.py ' +- label: "ROCm · Entrypoints GPU Coverage (R2-02)" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: NonBlocking + timeout_in_minutes: 30 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + artifact_paths: + - "artifacts/rocm-r2-02/**/*" + commands: + # Native AMD tests run from /app/vllm-omni, while Buildkite uploads + # artifacts relative to its checkout directory. + - export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02" + - mkdir -p "$$R2_02_ARTIFACT_DIR" + - >- + ps -eo pid=,ppid=,lstart=,stat=,comm= > + "$$R2_02_ARTIFACT_DIR/processes-before.txt" + # Verbose pytest output and JUnit retain selected node IDs without a + # second collection-only pass over the same GPU-backed test tree. + # The inner timeout leaves five minutes for teardown verification and + # artifact generation before the Buildkite step timeout. + - >- + timeout --signal=TERM --kill-after=1m 25m + pytest -s -v -ra + tests/entrypoints/ + -m 'core_model and cuda and not + (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' + --run-level "core_model" + --durations=0 + --junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml" + 2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log" + - >- + tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" > + "$$R2_02_ARTIFACT_DIR/pytest-summary.txt" + - | + sleep 5 + ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt" + # Compare PID plus process start time so PID reuse cannot hide a leak. + # Double dollars preserve awk field references through Buildkite interpolation. + awk ' + NR == FNR { + if ($$9 != "ps") { + baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1 + } + next + } + $$9 != "ps" { + identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7 + if (!(identity in baseline)) { + print + leaked = 1 + } + } + END { + if (leaked) { + print "status=FAIL leaked_processes_detected=1" + exit 1 + } + print "status=PASS leaked_processes=0" + } + ' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \ + "$$R2_02_ARTIFACT_DIR/processes-after.txt" \ + 2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt" + # - label: "Omni Model Test Qwen3-Omni" # agent_pool: mi300_2 # depends_on: amd-build @@ -201,6 +333,12 @@ steps: depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking + # A ROCm hardware hang aborts the interpreter with 134. Let Buildkite + # create one fresh pod instead of retrying on the poisoned GPU process. + retry: + automatic: + - exit_status: 134 + limit: 1 commands: - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model" diff --git a/.buildkite/amd/test-template-amd-omni.j2 b/.buildkite/amd/test-template-amd-omni.j2 index e2042b96ab3..11a4086b6a7 100644 --- a/.buildkite/amd/test-template-amd-omni.j2 +++ b/.buildkite/amd/test-template-amd-omni.j2 @@ -156,6 +156,14 @@ steps: {% for path in step.artifact_paths %} - "{{ path }}" {% endfor %} +{% endif %} +{% if step.retry %} + retry: + automatic: +{% for retry_rule in step.retry.automatic %} + - exit_status: {{ retry_rule.exit_status }} + limit: {{ retry_rule.limit }} +{% endfor %} {% endif %} {% if step.timeout_in_minutes %} timeout_in_minutes: {{ step.timeout_in_minutes }} @@ -224,6 +232,14 @@ steps: {% for path in step.artifact_paths %} - "{{ path }}" {% endfor %} +{% endif %} +{% if step.retry %} + retry: + automatic: +{% for retry_rule in step.retry.automatic %} + - exit_status: {{ retry_rule.exit_status }} + limit: {{ retry_rule.limit }} +{% endfor %} {% endif %} {% if step.timeout_in_minutes %} timeout_in_minutes: {{ step.timeout_in_minutes }} diff --git a/.buildkite/common/ci_source_file_dependencies.yml b/.buildkite/common/ci_source_file_dependencies.yml index 33def4de450..067af9fa689 100644 --- a/.buildkite/common/ci_source_file_dependencies.yml +++ b/.buildkite/common/ci_source_file_dependencies.yml @@ -38,9 +38,14 @@ x-minicpmo-4-5: &minicpmo_4_5 x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex - vllm_omni/deploy/minicpmo_4_5_duplex.yaml - vllm_omni/engine/duplex/ + - vllm_omni/engine/duplex_omni_engine.py + - vllm_omni/engine/duplex_orchestrator.py + - vllm_omni/engine/omni_engine_base.py - vllm_omni/entrypoints/duplex/ + - vllm_omni/entrypoints/duplex_omni.py + - vllm_omni/entrypoints/async_omni_base.py + - vllm_omni/protocol/ - vllm_omni/clients/ - - vllm_omni/entrypoints/duplex_request_client.py - vllm_omni/outputs/duplex.py - vllm_omni/model_executor/duplex_sampling.py - vllm_omni/experimental/fullduplex/ @@ -95,6 +100,7 @@ x-hunyuan-image3: &hunyuan_image3 x-minimax-h3: &minimax_h3 - vllm_omni/diffusion/models/minimax_h3/ + - vllm_omni/diffusion/layers/indexed_modulation.py x-hunyuan-video: &hunyuan_video - vllm_omni/diffusion/models/hunyuan_video/ @@ -249,8 +255,13 @@ source_file_dependencies: - tests/assets/minicpmo_4_5/ - tests/clients/ - tests/engine/duplex/ + - tests/engine/test_duplex_omni_engine.py + - tests/engine/test_duplex_orchestrator.py - tests/entrypoints/duplex/ + - tests/entrypoints/test_duplex_omni.py + - tests/entrypoints/openai_api/test_duplex_api_server.py - tests/model_executor/models/minicpmo_4_5/duplex/ + - tests/protocol/ omni_minicpmo_4_5_duplex_perf: - *minicpmo_4_5 diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index d600e8fbef8..87d4c536231 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -66,7 +66,9 @@ steps: # 0.26 startup regression is understood and fixed. timeout_in_minutes: 90 commands: - - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" + # Turn-based server VAD over /v1/realtime is deferred to RFC #7181 PR 5 + # (see the P0 thread on #7413); the test stays in the tree and runs again there. + - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" --deselect tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_server_vad_multi_turn_without_client_commit mirror_hardwares: h100_2 - label: "Entrypoint Test with L4" diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 1fdb1ef4a3f..adc94f9e9d6 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -223,6 +223,17 @@ steps: -m "full_model and diffusion and H100 and B200 and cards_1" --run-level "full_model" + - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU" + timeout_in_minutes: 120 + commands: + - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client" + - >- + pytest -sv + tests/e2e/online_serving/test_pi05_expansion.py + -m "full_model and diffusion and H100 and cards_1" + --run-level "full_model" + mirror_hardwares: h100_1 + - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU" source_file_dependencies: diffusion_qwen_image_function timeout_in_minutes: 120 diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 67f03dbb680..53b9bc4d019 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -38,6 +38,12 @@ steps: depends_on: upload-ready-pipeline steps: + - label: "Diffusion · FlashAttention Test" + timeout_in_minutes: 20 + commands: + - timeout 15m pytest -s -v tests/diffusion/attention/ -m 'core_model and cuda and L4 and cards_1' --run-level "core_model" + mirror_hardwares: l4_1 + - label: "Diffusion · Offloader Test" commands: - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model" diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 7809fcbc7c4..70ab5935673 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -191,13 +191,14 @@ steps: # from downgrading transformers; see # examples/offline_inference/text_to_speech/README.md. - label: "TTS · Gepard-1.0 · L4" - timeout_in_minutes: 60 + timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt - >- pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py + tests/e2e/online_serving/test_gepard_tts.py -m "tts and cuda" --run-level "full_model" mirror_hardwares: l4_1 diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index f64fa711f64..0c39a3f93a1 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -235,6 +235,37 @@ steps: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" steps: + - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test" + source_file_dependencies: diffusion_hunyuan_video_function + timeout_in_minutes: 180 + mirror_hardwares: a2b3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + HF_HUB_OFFLINE: "1" + commands: + - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "npu and A2" --run-level "full_model" + - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Perf Test" + source_file_dependencies: diffusion_hunyuan_video_perf + key: nightly-diffusion-x2v-performance-hunyuanvideo15 + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + HF_HUB_OFFLINE: "1" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT - label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test" source_file_dependencies: diffusion_wan22_function mirror_hardwares: a2b3_npu_4 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c92c7b22f8c..e77f7091448 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -67,6 +67,11 @@ # File-level overrides within entrypoints/ /vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao /vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter +# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex +# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the +# engine depends on it — so it is owned by the serving-boundary document rather +# than by any one consumer. — docs/design/module/entrypoints.md +/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao # Error contracts — docs/design/module/error_contracts.md /vllm_omni/errors.py @alex-jw-brooks @NickCao diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index 0d607b30a4c..f9ee3b0c107 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -52,6 +52,7 @@ This extension offers the following nodes based on the output modalities (at **C - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks - **TTS** and **TTS Voice Clone** for TTS tasks +- **Generate Music** for lyrics-and-description-to-music generation This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**) @@ -139,6 +140,93 @@ You can configure per-stage sampling parameters for multi-stage models. > > Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect. +For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`), +3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most +12 connected inputs in total. Any mixture containing at least one image or video +is supported; empty and audio-only references are rejected. For example, you can +combine 6 images, 3 videos, and 3 audio clips in one request. + +Within each media type, references follow slot-number order, skipping unconnected +slots. For example, connecting `image_2` and `image_9` sends `image_2` as the first +image and `image_9` as the second. Each image slot uses the first image in its batch. +Existing connections to `image_1`, `image_2`, `audio_1`, `audio_2`, `video_1`, and +`video_2` remain valid in saved workflows. + +#### MiniMax-H3 Reference to Video + +Load [MiniMax-H3 Reference to Video](example_workflows/vLLM-Omni%20MiniMax-H3%20Reference%20to%20Video.json) +from **Templates → ComfyUI-vLLM-Omni**, or drag the JSON onto the canvas. +It adapts the [official ComfyUI R2V workflow](https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_r2v.json) +to remote vLLM-Omni execution. Model loading, sampling, and VAE decoding run on +the server; ComfyUI loads references and saves the returned video. + +Use an extension version containing the audio-preserving output fix from +[#7456](https://github.com/vllm-project/vllm-omni/pull/7456). Older versions can +drop the server's audio while decoding the response. + +1. Start a Ref2VA-capable service using the [MiniMax-H3 recipe](../../recipes/MiniMaxAI/MiniMax-H3.md). + Set **Generate Video** to its `/v1` URL and served model name. +2. Select your own image in **Load Image**. The template's filenames are placeholders + relative to ComfyUI's input directory; no sample assets or model weights are bundled. +3. Connect **Load Video** and **Load Audio** when needed. Duplicate the loaders to + fill more reference slots, or disconnect the image when using video-only input. + Keep the references connected to **Generate Video** and leave `frame` disconnected. +4. Match prompt tags such as ``, `