diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 09019619f8..319dbc55aa 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -67,7 +67,6 @@ def __init__(self, config: Config, input_address: str, output_address: str): ) self.input_thread.start() - self.profile_enbaled = config.torch_profiler_dir is not None self.mark_trace = getattr(config, "mark_trace", False) init_exit_handler(self) self._init_data_parallel(config) @@ -106,11 +105,6 @@ def __init__(self, config: Config, input_address: str, output_address: str): config.num_kvcache_blocks = num_blocks if not config.enforce_eager: - # Start profiler before cudagraph capture only if mark-trace is enabled. - if self.profile_enbaled and self.mark_trace: - self.runner_mgr.call_func( - "start_profiler", "capture_graph", wait_out=True - ) cap_cost, bs, pool_bytes = self.runner_mgr.call_func( "capture_cudagraph", wait_out=True ) @@ -118,9 +112,6 @@ def __init__(self, config: Config, input_address: str, output_address: str): f"{self.label}: cudagraph capture{bs} cost: {cap_cost:.2f} " f"seconds, pool: {pool_bytes / (1 << 30):.2f}GB" ) - if self.profile_enbaled and self.mark_trace: - # Persist a dedicated capture-graph trace immediately. - self.runner_mgr.call_func("stop_profiler", wait_out=True) good = True finally: logger.info( @@ -287,6 +278,7 @@ def _process_engine_step_inner(self): # Run the model forward pass if there are actual sequences has_seqs = len(scheduled_batch.req_ids) > 0 if has_seqs: + self.scheduler.compute_detailed_aggregates(scheduled_batch, seqs) fwd_out = self.runner_mgr.call_func( "forward", scheduled_batch, wait_out=True ) diff --git a/atom/model_engine/engine_utility.py b/atom/model_engine/engine_utility.py index 2a60062e14..3195c74ec9 100644 --- a/atom/model_engine/engine_utility.py +++ b/atom/model_engine/engine_utility.py @@ -247,6 +247,10 @@ def _handle_configure_hidden_states(self, args: dict): def _handle_start_profile(self, args: dict): result = self.runner_mgr.call_func("start_profiler", wait_out=True) + # Flip the scheduler flag so per-iteration detailed aggregates + # (compute_detailed_aggregates) are emitted while profiling is active. + if self.scheduler is not None: + self.scheduler.profile_active = True logger.info(f"{self.label}: profiler started") self.output_queue.put_nowait( ("UTILITY_RESPONSE", {"cmd": "start_profile", "result": result}) @@ -255,6 +259,8 @@ def _handle_start_profile(self, args: dict): def _handle_stop_profile(self, args: dict): logger.info(f"{self.label}: stopping profiler...") result = self.runner_mgr.call_func("stop_profiler", wait_out=True) + if self.scheduler is not None: + self.scheduler.profile_active = False logger.info(f"{self.label}: profiler stopped, result={result}") self.output_queue.put_nowait( ("UTILITY_RESPONSE", {"cmd": "stop_profile", "result": result}) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 457cd363c1..4f8b989cf5 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -2714,6 +2714,26 @@ def prepare_model(self, batch: ScheduledBatch): needs_independent_noise, ) + @staticmethod + def _detailed_label_suffix(batch: Optional[ScheduledBatch]) -> str: + """Detailed attention aggregates for the trace label, or ``""``. + + These fields are only populated by + `Scheduler.compute_detailed_aggregates` when profiling is active + and ``ATOM_ENABLE_DETAILED_ANNOTATION`` is set, so on the normal + (unprofiled) path this returns an empty string without any extra work. + Appending here keeps the annotation on the ``prefill[]``/``decode[]`` + ``record_function`` (a GPU-recognized layer) instead of nesting an + extra span above ``run_model``. + """ + if batch is None or batch.detailed_sqsq is None: + return "" + return ( + f" sqsq={batch.detailed_sqsq}" + f" sqsk={batch.detailed_sqsk}" + f" sk={batch.detailed_sk}" + ) + def _build_pcp_balanced_slices( self, batch: ScheduledBatch, @@ -2878,6 +2898,7 @@ def run_model( # the label shows bs=/ when they differ. graph_bs=context.graph_bs if forward_mode.use_cudagraph else None, batch=batch, + detailed_suffix=self._detailed_label_suffix(batch), ) # PCP+TBO prefill: per-group round-robin stripe before UBatchWrapper (see @@ -3200,6 +3221,7 @@ def forward(self, batch: ScheduledBatch) -> ScheduledBatchOutput: needs_independent_noise=needs_independent_noise, ) reset_forward_context() + return fwd_output @torch.inference_mode() @@ -3299,6 +3321,60 @@ def propose_draft_token_ids( verify_scheduler.record_ell(batch.req_ids[: batch.total_seqs_num]) return self.tokenID_processor.prepare_draft_ids(batch, draft_token) + def start_capture_profiler(self): + """Set up the per-bs CUDA graph capture profiler (profiles in place). + + Profiles the capture phase as graphs are captured and writes one trace + per batch size, per rank (``bs__rank.json.gz``). Enabled on + every rank when a torch profiler dir is set and mark-trace is on. + """ + self._capture_profile_enabled = ( + self.profiler_dir is not None and self.mark_trace + ) + if self._capture_profile_enabled: + self._profile_bs_idx = 0 + self.capture_traces_dir = os.path.join(self.profiler_dir, "capture_traces") + os.makedirs(self.capture_traces_dir, exist_ok=True) + logger.info(f"{self.label}: Starting CUDA graph capture profiler...") + + def on_trace_ready(prof): + # Invariant: exactly two prof.step() calls happen per captured + # batch size (schedule wait=1 + active=1, repeat=0), so + # on_trace_ready fires once per bs, in self.graph_bs order. + # This is a profiling-only diagnostic; log-and-skip rather than + # assert so a cadence mismatch can never abort CUDA-graph + # capture at server startup (and isn't stripped under python -O). + if self._profile_bs_idx >= len(self.graph_bs): + logger.warning( + "capture profiler fired %d times but only %d batch " + "sizes were captured; skipping extra trace. Check the " + "prof.step() cadence in capture_cudagraph.", + self._profile_bs_idx + 1, + len(self.graph_bs), + ) + return + bs = self.graph_bs[self._profile_bs_idx] + trace_file = os.path.join( + self.capture_traces_dir, f"bs_{bs}_rank{self.rank}.json.gz" + ) + prof.export_chrome_trace(trace_file) + logger.info(f"Saved trace for bs={bs} to {trace_file}") + self._profile_bs_idx += 1 + + self.capture_profiler = torch_profiler.profile( + activities=[ + torch_profiler.ProfilerActivity.CUDA, + torch_profiler.ProfilerActivity.CPU, + ], + schedule=torch_profiler.schedule(wait=1, warmup=0, active=1, repeat=0), + record_shapes=True, + with_stack=True, + profile_memory=False, + on_trace_ready=on_trace_ready, + ) + else: + self.capture_profiler = nullcontext() + @torch.inference_mode() def _piecewise_cg_active(self) -> bool: """True when the compiled model's dense pieces self-capture PIECEWISE @@ -3577,6 +3653,9 @@ def capture_cudagraph(self): # TBO graphs don't capture compute_logits, so disable logits_in_graph. self.logits_in_graph = self.world_size == 1 and not is_tbo + # start capture profiler + self.start_capture_profiler() + @contextmanager def pause_gc(): # No GC during capture: a finalizer's hipModuleUnload aborts it (HIP 900). @@ -3609,7 +3688,7 @@ def pause_gc(): "max_q_len" in inspect.signature(build_capture).parameters ) - with pause_gc(), graph_capture() as capture_ctx: + with pause_gc(), graph_capture() as capture_ctx, self.capture_profiler as prof: for max_q_len in q_buckets: capture_range = ( tqdm.tqdm(self.graph_bs) if self.rank == 0 else self.graph_bs @@ -3685,6 +3764,8 @@ def pause_gc(): outputs[:num_tokens] = model_output if self.logits_in_graph: self.model.compute_logits(outputs[:num_tokens]) + if prof is not None: + prof.step() if _piecewise: # PIECEWISE: no manual whole-forward graph; the compiled @@ -3742,6 +3823,8 @@ def pause_gc(): self.graphs[(bs, max_q_len)] = graph if self.logits_in_graph and ubatch_slices is None: self.graph_logits[(bs, max_q_len)] = graph_logits + if prof is not None: + prof.step() if graph_aux is not None: self.graph_aux_hidden[(bs, max_q_len)] = graph_aux torch.cuda.synchronize() diff --git a/atom/model_engine/run_labels.py b/atom/model_engine/run_labels.py index 7a54381605..d2bb70b290 100644 --- a/atom/model_engine/run_labels.py +++ b/atom/model_engine/run_labels.py @@ -43,12 +43,17 @@ def build_run_label( tbo_on: bool, bs: int, batch: Optional["ScheduledBatch"], + detailed_suffix: str = "", graph_bs: Optional[int] = None, ) -> str: """Build the ``record_function`` label for one forward pass. Pure function (no runtime state) — see module docstring for the taxonomy. + ``detailed_suffix`` is an already-formatted ``" key=value ..."`` fragment + (empty on the normal path) appended inside the brackets; see + ``ModelRunner._detailed_label_suffix``. + ``graph_bs`` is the padded batch size the CUDAGraph actually replays. When it is given and exceeds the real ``bs`` (CUDAGraph path only), the label shows ``bs=/`` so the padding is visible in the trace. @@ -81,6 +86,7 @@ def build_run_label( else: ctx_str = f"{ctx[:3].tolist()}...+{len(ctx) - 3}" label += f" tok={batch.total_tokens_num} ctx={ctx_str}" + label += detailed_suffix if tbo_on: label += " tbo=1" label += "]" diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 4573042657..978777a1fc 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -29,6 +29,7 @@ from atom.model_engine.block_manager import BlockManager from atom.model_engine.request import RequestOutput from atom.model_engine.sequence import Sequence, SequenceStatus, SequenceType +from atom.utils import envs logger = logging.getLogger("atom") @@ -354,6 +355,14 @@ def __init__( self.dspark_dp_bs = None self.dspark_dp_total_tokens = None + # Detailed attention aggregates (set by Scheduler.compute_detailed_aggregates + # when profiling is active and ATOM_ENABLE_DETAILED_ANNOTATION is set). + # None on the normal path; consumed by ModelRunner.run_model to extend + # the prefill[]/decode[] trace labels. + self.detailed_sqsq: int | None = None # sum N_Q^2 + self.detailed_sqsk: int | None = None # sum N_Q*N_KV + self.detailed_sk: int | None = None # sum N_KV + # Collect multimodal data from prefill sequences self.multimodal_data = {} for seq in seqs.values(): @@ -476,6 +485,11 @@ def __init__(self, config: Config): self.cache_stats: Optional[CacheStats] = ( CacheStats() if config.enable_prefix_caching else None ) + self.profile_active = False + # Cache the env flag once (env vars are fixed at process start) so the + # per-iteration compute_detailed_aggregates never pays an os.getenv. + self._detailed_annotation_enabled = envs.ATOM_ENABLE_DETAILED_ANNOTATION + self.enable_chunked_prefill = config.enable_chunked_prefill # V4 SWA correctness on a prefix-cache hit is now handled entirely in # BlockManager: `_swa_bounded_hit` bounds the hit so the boundary's @@ -1729,6 +1743,61 @@ def postprocess( self.running.remove(seq) return finished_seqs + def compute_detailed_aggregates( + self, + scheduled_batch: ScheduledBatch, + seqs: dict[int, Sequence], + ) -> None: + """Attach detailed attention aggregates to *scheduled_batch* in place. + + Only the quadratic terms genuinely needed for a downstream attention-FLOP + estimate are computed here. The request counts and total query tokens are + already emitted by the ``prefill[]``/``decode[]`` labels in + :meth:`ModelRunner.run_model`, so this avoids duplicating them. + + The following batch-level sums are stored on the batch and appended to + those labels by the runner: + + sqsq — sum of N_Q^2 (per request) + sqsk — sum of N_Q*N_KV (per request) + sk — sum of N_KV (per request) + + where ``N_Q`` is the number of query tokens scheduled for a request and + ``N_KV`` is its KV length (cached + new tokens for prefill, full + sequence length for decode). Aggregating over every request in the + batch gives the total for that single forward, which is exactly the + quantity a per-iteration attention-FLOP estimate needs. For MTP/spec-decode a + decode step schedules ``mtp_k + 1`` query tokens, so the scheduled + token count is used as ``N_Q`` for both branches (rather than a + hardcoded 1) to avoid undercounting. + + This is a no-op (leaves the fields ``None``) unless profiling is active + and ``ATOM_ENABLE_DETAILED_ANNOTATION`` is set. + """ + if not self.profile_active or not self._detailed_annotation_enabled: + return + + sqsq = 0 # sum N_Q^2 + sqsk = 0 # sum N_Q*N_KV + sk = 0 # sum N_KV + for seq, num_tokens in zip(seqs.values(), scheduled_batch.num_scheduled_tokens): + # Cast to Python int: num_scheduled_tokens is np.int32, so nq*nq / + # nq*nkv would overflow once a prefill/chunk exceeds ~46341 tokens + # (e.g. np.int32(65536)**2 == 0), silently corrupting the estimate. + nq = int(num_tokens) # query tokens scheduled this forward + if seq.type == SequenceType.DECODE: + nkv = int(seq.num_tokens) # full sequence length + else: + # PREFILL: KV length = cached + new tokens. + nkv = int(seq.num_cached_tokens) + nq + sqsq += nq * nq + sqsk += nq * nkv + sk += nkv + + scheduled_batch.detailed_sqsq = sqsq + scheduled_batch.detailed_sqsk = sqsk + scheduled_batch.detailed_sk = sk + def _connector_flag(self, name: str) -> bool: return bool(getattr(self.kv_connector, name, False)) diff --git a/atom/utils/envs.py b/atom/utils/envs.py index e31524ea52..a19b1a9b41 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -137,6 +137,11 @@ # --- Profiling & Logging --- "ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR", None), "ATOM_PROFILER_MORE": lambda: os.getenv("ATOM_PROFILER_MORE", "0") == "1", + # When profiling is active, append detailed attention aggregates (sqsq, sqsk, sk) + # to the prefill[]/decode[] trace labels emitted by ModelRunner.run_model. + "ATOM_ENABLE_DETAILED_ANNOTATION": lambda: ( + os.getenv("ATOM_ENABLE_DETAILED_ANNOTATION", "0") == "1" + ), "ATOM_PROFILER_TIMEOUT": lambda: float(os.getenv("ATOM_PROFILER_TIMEOUT", "300")), "ATOM_LOG_MORE": lambda: int(os.getenv("ATOM_LOG_MORE", "0")) != 0, # RTL (rocm-trace-lite) GPU kernel tracing — set to output directory to enable. diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 5caf305e26..76ff14dead 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -110,6 +110,7 @@ land. See `atom/model_ops/v4_backend_gate.py` for the selector. |----------|------|---------|-------------| | **ATOM_TORCH_PROFILER_DIR** | str | — | When set, enables PyTorch profiler and writes traces to this directory. Create subdirectories per rank (e.g., `rank_0`, `dp0_tp0`). | | **ATOM_PROFILER_MORE** | bool | 0 (false) | When `ATOM_TORCH_PROFILER_DIR` is set and this is `1`, enables detailed profiling: `record_shapes`, `with_stack`, and `profile_memory`. | +| **ATOM_ENABLE_DETAILED_ANNOTATION** | bool | 0 (false) | When profiling is active, appends detailed attention aggregates to the `prefill[]`/`decode[]` trace labels: `sqsq` (Σ N_Q²), `sqsk` (Σ N_Q·N_KV), and `sk` (Σ N_KV), where N_Q is the scheduled query tokens and N_KV the KV length per request. Used to estimate attention FLOPs for downstream roofline analysis. | | **ATOM_LOG_MORE** | bool | 0 (false) | If set to `1`, use verbose logging format (includes process name, PID, path, line number, function name). | ### Debug dump (`atom.utils.debug_helper`) diff --git a/docs/serving_benchmarking_guide.md b/docs/serving_benchmarking_guide.md index 790f561443..f66c320074 100644 --- a/docs/serving_benchmarking_guide.md +++ b/docs/serving_benchmarking_guide.md @@ -350,6 +350,7 @@ the programmatic API. | `ATOM_TORCH_PROFILER_DIR` env var | Sets the default `torch_profiler_dir` in `Config` | | `ATOM_PROFILER_MORE=1` env var | Enables detailed profiling: `record_shapes`, `with_stack`, `profile_memory` | | `ATOM_PROFILER_TIMEOUT=` env var | Overrides the `stop_profile` timeout; default is 300 seconds | +| `ATOM_ENABLE_DETAILED_ANNOTATION=1` env var | Appends attention FLOP aggregates (`sqsq`, `sqsk`, `sk`) to the `prefill[]`/`decode[]` trace labels while profiling is active (see [CUDA-graph capture traces](#cuda-graph-capture-traces)) | When a profiler directory is configured, each worker saves traces to a rank-specific subdirectory: @@ -431,6 +432,32 @@ python -m atom.benchmarks.benchmark_serving \ This sends `POST /start_profile` before the benchmark and `POST /stop_profile` after completion. +### CUDA-graph capture traces + +During CUDA-graph capture (server bring-up), ATOM can emit one trace file per +captured batch size instead of a single combined blob. This makes each graph's +capture cost easy to inspect in isolation and keeps individual trace files +small. Capture-trace profiling is gated on `--mark-trace` (with +`--torch-profiler-dir`/`ATOM_TORCH_PROFILER_DIR` set); the warmup phase is +omitted so only meaningful capture work is retained. + +The per-batch-size traces are written to: + +``` +{profiler_dir}/capture_traces/bs__rank.json.gz +``` + +where `` is the captured batch size and `` the worker rank. Each file +is a gzip-compressed Chrome trace viewable with `chrome://tracing` or +TensorBoard. + +To additionally annotate the run-phase `prefill[]`/`decode[]` labels with the +attention FLOP aggregates used for roofline analysis, set +`ATOM_ENABLE_DETAILED_ANNOTATION=1` (see [Configuration](#configuration)). The added +fields are `sqsq` (Σ N_Q²), `sqsk` (Σ N_Q·N_KV), and `sk` (Σ N_KV), summed over +every request in the forward. These are attention-quadratic terms only — a full +roofline still requires GEMM FLOPs and bytes moved. + ## Speculative decoding (MTP) ATOM supports Multi-Token Prediction (MTP) for DeepSeek models using the diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 723c6f339e..cbdf1d54d0 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -6,6 +6,8 @@ from types import SimpleNamespace from unittest import mock +import numpy as np + from atom.model_engine.scheduler import ( ScheduledBatch, Scheduler, @@ -824,3 +826,120 @@ def test_normal_decode_window_unchanged(self): ) assert list(batch.scheduled_tokens) == toks[-(mtp_k + 1) :] + + +# ── detailed annotation aggregates ────────────────────────────────────────── + + +class TestComputeDetailedAggregates: + """Unit tests for Scheduler.compute_detailed_aggregates (pure Python). + + The method only touches ``self.profile_active`` and the cached + ``self._detailed_annotation_enabled`` flag, so a lightweight + SimpleNamespace stands in for both the scheduler and the sequences — no + GPU or full Scheduler construction required. + """ + + @staticmethod + def _make_batch(num_scheduled_tokens): + return SimpleNamespace( + num_scheduled_tokens=num_scheduled_tokens, + detailed_sqsq=None, + detailed_sqsk=None, + detailed_sk=None, + ) + + @staticmethod + def _make_seqs(): + # Two prefill requests + one decode request. + # prefill A: N_Q=4, cached=2 -> N_KV=6 -> sqsq 16, sqsk 24, sk 6 + # prefill B: N_Q=3, cached=0 -> N_KV=3 -> sqsq 9, sqsk 9, sk 3 + # decode C: N_Q=1, -> N_KV=10 -> sqsq 1, sqsk 10, sk 10 + return { + 0: SimpleNamespace( + type=SequenceType.PREFILL, num_tokens=6, num_cached_tokens=2 + ), + 1: SimpleNamespace( + type=SequenceType.PREFILL, num_tokens=3, num_cached_tokens=0 + ), + 2: SimpleNamespace( + type=SequenceType.DECODE, num_tokens=10, num_cached_tokens=9 + ), + } + + def test_aggregates_when_enabled(self): + fake_self = SimpleNamespace( + profile_active=True, _detailed_annotation_enabled=True + ) + batch = self._make_batch([4, 3, 1]) + + Scheduler.compute_detailed_aggregates(fake_self, batch, self._make_seqs()) + + assert batch.detailed_sqsq == 16 + 9 + 1 + assert batch.detailed_sqsk == 24 + 9 + 10 + assert batch.detailed_sk == 6 + 3 + 10 + + def test_noop_when_flag_disabled(self): + fake_self = SimpleNamespace( + profile_active=True, _detailed_annotation_enabled=False + ) + batch = self._make_batch([4, 3, 1]) + + Scheduler.compute_detailed_aggregates(fake_self, batch, self._make_seqs()) + + assert batch.detailed_sqsq is None + assert batch.detailed_sqsk is None + assert batch.detailed_sk is None + + def test_noop_when_profiling_inactive(self): + fake_self = SimpleNamespace( + profile_active=False, _detailed_annotation_enabled=True + ) + batch = self._make_batch([4, 3, 1]) + + Scheduler.compute_detailed_aggregates(fake_self, batch, self._make_seqs()) + + assert batch.detailed_sqsq is None + assert batch.detailed_sqsk is None + assert batch.detailed_sk is None + + def test_no_int32_overflow_large_prefill(self): + # Regression: num_scheduled_tokens is np.int32, so nq*nq must not + # overflow for long prefills. np.int32(65536)**2 wraps to 0, which + # would silently corrupt the estimate the feature exists to produce. + fake_self = SimpleNamespace( + profile_active=True, _detailed_annotation_enabled=True + ) + nq = 65536 + batch = self._make_batch(np.asarray([nq], dtype=np.int32)) + seqs = { + 0: SimpleNamespace( + type=SequenceType.PREFILL, num_tokens=nq, num_cached_tokens=0 + ) + } + + Scheduler.compute_detailed_aggregates(fake_self, batch, seqs) + + assert batch.detailed_sqsq == nq * nq # 4294967296, not 0 + assert batch.detailed_sqsk == nq * nq + assert batch.detailed_sk == nq + assert isinstance(batch.detailed_sqsq, int) + + def test_decode_counts_scheduled_query_tokens(self): + # MTP/spec-decode schedules mtp_k+1 query tokens; nq must reflect the + # scheduled count rather than a hardcoded 1 (otherwise undercounted). + fake_self = SimpleNamespace( + profile_active=True, _detailed_annotation_enabled=True + ) + batch = self._make_batch(np.asarray([3], dtype=np.int32)) + seqs = { + 0: SimpleNamespace( + type=SequenceType.DECODE, num_tokens=100, num_cached_tokens=97 + ) + } + + Scheduler.compute_detailed_aggregates(fake_self, batch, seqs) + + assert batch.detailed_sqsq == 9 # 3^2 + assert batch.detailed_sqsk == 300 # 3 * 100 + assert batch.detailed_sk == 100 diff --git a/unit-report.xml b/unit-report.xml new file mode 100644 index 0000000000..3177eda8b1 --- /dev/null +++ b/unit-report.xml @@ -0,0 +1 @@ +('/home/mohbasit@amd.com/ATOM/tests/test_decode_indices_paged.py', 10, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_dspark_swa_fp8_2buff.py', 17, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_eplb_metadata.py', 14, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_eplb_module_a.py', 15, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_eplb_module_b.py', 13, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_eplb_module_c.py', 16, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_eplb_module_d.py', 15, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_eplb_module_e.py', 14, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_kv_connector_scheduler.py', 16, 'Skipped: kv_transfer_engine was split into the moriio subpackage (#690); test imports need path updates by the disaggregation owner')('/home/mohbasit@amd.com/ATOM/tests/test_prefill_indices_paged.py', 12, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_swa_write_paged.py', 15, "Skipped: requires full atom import env: cannot import name 'QuantizationConfig' from 'atom.config' (unknown location)")('/home/mohbasit@amd.com/ATOM/tests/test_transfer_engine.py', 30, 'Skipped: kv_transfer_engine was split into the moriio subpackage (#690); test imports need path updates by the disaggregation owner') \ No newline at end of file