feat(rocprofiler-sdk): Add delayed start options

projects/rocprofiler-sdk/source/bin/rocprofv3.py

@@ -52,6 +52,7 @@
 PERFETTO_BUFFER_SIZE_KB_MIN = 1
 PERFETTO_BUFFER_SIZE_KB_MAX = ((1 << 32) - 1) // 1024
 DEPRECATED_DIRECT_OUTPUT_FORMATS = ("csv", "pftrace", "otf2")
+ATT_RESOURCE_MODES = ("default", "hsa", "code-object")
 
 
 class dotdict(dict):
@@ -1238,6 +1239,13 @@ def add_parser_bool_argument(gparser, *args, **kwargs):
         type=str,
     )
 
+    att_options.add_argument(
+        "--att-resource-mode",
+        help="When to allocate thread trace resources: profiler-selected, HSA initialization, or the first code object per GPU. Default: code-object.",
+        default=None,
+        choices=ATT_RESOURCE_MODES,
+    )
+
     att_options.add_argument(
         "--att-shader-engine-mask",
         help="Bitmask of shader engines to enable. Default 0x1",
@@ -2380,6 +2388,18 @@ def log_config(_env):
         update_env("ROCPROF_ADVANCED_THREAD_TRACE", True, overwrite=True)
         update_env("ROCPROF_ATT_NO_INTERCEPT", args.att_no_intercept, overwrite=True)
 
+        if args.att_resource_mode is not None:
+            if args.att_resource_mode not in ATT_RESOURCE_MODES:
+                fatal_error(
+                    f"Invalid att_resource_mode: {args.att_resource_mode}. "
+                    f"Valid choices are: {', '.join(ATT_RESOURCE_MODES)}"
+                )
+            update_env(
+                "ROCPROF_ATT_PARAM_RESOURCE_MODE",
+                args.att_resource_mode,
+                overwrite=True,
+            )
+
         if args.att_target_cu is not None:
             update_env(
                 "ROCPROF_ATT_PARAM_TARGET_CU",

projects/rocprofiler-sdk/source/bin/tests/CMakeLists.txt

@@ -9,7 +9,7 @@ if(NOT Python3_EXECUTABLE)
 endif()
 
 foreach(_FILE IN ITEMS conftest.py pytest.ini test_multipass_parsing.py
-                       test_rocprof_attach.py)
+                       test_att_resource_mode.py test_rocprof_attach.py)
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${_FILE}
                    ${CMAKE_CURRENT_BINARY_DIR}/${_FILE} COPYONLY)
 endforeach()
@@ -20,7 +20,7 @@ endforeach()
 # instead of the integration-tests/validation labels that macro applies.
 rocprofiler_sdk_pytest_discover_tests(
     rocprofv3-launcher-tests
-    TEST_PATHS test_multipass_parsing.py
+    TEST_PATHS test_multipass_parsing.py test_att_resource_mode.py
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     PYTHON_EXECUTABLE ${Python3_EXECUTABLE}
     EXTRA_ARGS --rocprofv3-path ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/rocprofv3

projects/rocprofiler-sdk/source/bin/tests/test_att_resource_mode.py

@@ -0,0 +1,99 @@
+# MIT License
+#
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+"""GPU-free tests for ATT resource-mode selection and environment forwarding."""
+
+import json
+
+import pytest
+
+
+@pytest.fixture
+def launch(rocprofv3, monkeypatch):
+    environments = []
+    monkeypatch.delenv("ROCPROF_ATT_PARAM_RESOURCE_MODE", raising=False)
+    monkeypatch.setattr(rocprofv3, "resolve_library_path", lambda path, args: path)
+    monkeypatch.setattr(rocprofv3, "check_att_capability", lambda args: True)
+    monkeypatch.setattr(
+        rocprofv3.os,
+        "execvpe",
+        lambda executable, argv, env: environments.append(env),
+    )
+    monkeypatch.setattr(
+        rocprofv3.subprocess,
+        "check_call",
+        lambda argv, env: environments.append(env) or 0,
+    )
+
+    def run(*args):
+        rocprofv3.main(["--att", *args, "--", "/bin/true"])
+        assert len(environments) == 1
+        return environments[0]
+
+    return run
+
+
+@pytest.mark.parametrize("mode", ["default", "hsa", "code-object"])
+def test_resource_mode_cli(launch, monkeypatch, mode):
+    monkeypatch.setenv("ROCPROF_ATT_PARAM_RESOURCE_MODE", "hsa")
+    env = launch("--att-resource-mode", mode)
+    assert env["ROCPROF_ATT_PARAM_RESOURCE_MODE"] == mode
+
+
+def test_resource_mode_omitted_uses_tool_default(launch):
+    assert "ROCPROF_ATT_PARAM_RESOURCE_MODE" not in launch()
+
+
+@pytest.mark.parametrize("mode", ["default", "hsa", "code-object"])
+def test_resource_mode_omitted_preserves_environment(launch, monkeypatch, mode):
+    monkeypatch.setenv("ROCPROF_ATT_PARAM_RESOURCE_MODE", mode)
+    assert launch()["ROCPROF_ATT_PARAM_RESOURCE_MODE"] == mode
+
+
+@pytest.mark.parametrize("mode", ["hip", "all", "dispatch", "invalid", "4"])
+def test_resource_mode_invalid_cli(rocprofv3, mode):
+    with pytest.raises(SystemExit) as exc:
+        rocprofv3.parse_arguments(["--att", "--att-resource-mode", mode])
+    assert exc.value.code == 2
+
+
+@pytest.mark.parametrize("mode", ["default", "hsa", "code-object"])
+def test_resource_mode_json(launch, tmp_path, monkeypatch, mode):
+    monkeypatch.setenv("ROCPROF_ATT_PARAM_RESOURCE_MODE", "hsa")
+    path = tmp_path / "input.json"
+    path.write_text(json.dumps({"jobs": [{"att_resource_mode": mode}]}))
+    assert launch("-i", str(path))["ROCPROF_ATT_PARAM_RESOURCE_MODE"] == mode
+
+
+def test_resource_mode_yaml(launch, tmp_path):
+    pytest.importorskip("yaml")
+    path = tmp_path / "input.yaml"
+    path.write_text("jobs:\n  - att_resource_mode: code-object\n")
+    assert launch("-i", str(path))["ROCPROF_ATT_PARAM_RESOURCE_MODE"] == "code-object"
+
+
+def test_resource_mode_invalid_json(launch, tmp_path):
+    path = tmp_path / "input.json"
+    path.write_text(json.dumps({"jobs": [{"att_resource_mode": "invalid"}]}))
+    with pytest.raises(SystemExit) as exc:
+        launch("-i", str(path))
+    assert exc.value.code != 0

projects/rocprofiler-sdk/source/docs/api-reference/thread_trace.rst

@@ -89,9 +89,33 @@ The configuration parameters are described here:
 
 - ROCPROFILER_THREAD_TRACE_PARAMETER_BUFFER_SIZE: Configures the buffer size. This buffer is shared among all SEs specified in ROCPROFILER_THREAD_TRACE_PARAMETER_SHADER_ENGINE_MASK. There is a minimal side effect to specifying a larger buffer size, except for increased VRAM usage.
 
+- ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE: Controls when the profiler allocates thread trace queues, signals, and memory for each configured GPU. See `Resource initialization modes`_ below.
+
 
 The thread trace can be configured in two primary modes: device-wide or per-dispatch, as described in the following sections.
 
+Resource initialization modes
++++++++++++++++++++++++++++++
+
+``ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE`` applies to both device and dispatch thread trace services. The following modes are currently supported:
+
+- ``ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DEFAULT``: Currently equivalent to ``ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_CODE_OBJECT``. Omitting the parameter has the same behavior.
+
+- ``ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_HSA``: Allocates resources during HSA initialization for configured GPUs visible to ROCr, respecting ``ROCR_VISIBLE_DEVICES``.
+
+- ``ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_CODE_OBJECT``: Defers allocation until the first code object is registered for each configured GPU visible to ROCr. GPUs that never load a code object do not allocate thread trace resources, even when ``ROCR_VISIBLE_DEVICES`` is unset. Code objects already loaded when the service initializes also trigger allocation.
+
+The ``HIP``, ``ALL``, and ``DISPATCH`` resource modes currently return ``ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED`` from either configuration API. Invalid resource mode values return ``ROCPROFILER_STATUS_ERROR_INVALID_ARGUMENT``.
+
+To defer resource allocation until a GPU loads a code object, add:
+
+.. code-block:: cpp
+
+    params.push_back({ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE,
+                      {ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_CODE_OBJECT}});
+
+In code-object mode, starting a device thread trace context before a GPU has loaded a code object records a pending start without allocating GPU resources. Tracing starts for that GPU when its first code object is registered, provided the context is still active. Stopping the context before that load cancels the pending start. Once allocated, resources are reused across subsequent code-object loads and context starts; unloading code objects does not release them.
+
 Device thread trace
 +++++++++++++++++++
 
@@ -156,7 +180,7 @@ To enable selective thread trace based on specific kernel dispatches, use the di
             "thread trace service configure");
     }
 
-For device-wide thread trace, starting the context automatically begins data capture. Some application warmup is recommended before starting the device thread trace. For the dispatch thread trace, this step is not necessary as tracing doesn't start automatically.
+For device-wide thread trace, starting the context begins data capture once resources are initialized. Some application warmup is recommended before starting the device thread trace. For the dispatch thread trace, this step is not necessary as tracing doesn't start automatically.
 
 To start the context after all services are configured, use:
 

projects/rocprofiler-sdk/source/docs/how-to/using-thread-trace.rst

@@ -110,6 +110,9 @@ The following table lists the parameters relevant to thread tracing:
 | att-buffer-size             | Bytes   | 1MB-2GB | 96MB      | Specifies the trace buffer size. This is shared for all SEs. |
 |                             |         |         |           | Increase this value if the buffer tends to get full.         |
 +-----------------------------+---------+---------+-----------+--------------------------------------------------------------+
+| att-resource-mode           | String  |         |code-object| Selects when thread trace resources are allocated per GPU.   |
+|                             |         |         |           | Values: default, hsa, code-object. See below for details.    |
++-----------------------------+---------+---------+-----------+--------------------------------------------------------------+
 | att-serialize-all           | Bool    |         | False     | If set to "True", turns on serialization for untraced kernels|
 +-----------------------------+---------+---------+-----------+--------------------------------------------------------------+
 | att-no-detail               | Bool    |         | False     | Collects occupancy data without instruction-level detail.    |
@@ -142,6 +145,20 @@ The following table lists the parameters relevant to thread tracing:
 |                             |         |         |           | --selected-regions.                                          |
 +-----------------------------+---------+---------+-----------+--------------------------------------------------------------+
 
+Use ``--att-resource-mode`` to choose when thread trace queues, signals, and memory are allocated for each GPU. It applies to both dispatch tracing and device-wide tracing, including ``--att-no-intercept``. Supported values are:
+
+- ``default``: Lets the profiler select the mode; currently equivalent to ``code-object``.
+- ``hsa``: Allocates resources during HSA initialization for configured GPUs visible to ROCr.
+- ``code-object``: Defers allocation until a code object is registered for each configured GPU. GPUs that never load a code object do not allocate thread trace resources, even when ``ROCR_VISIBLE_DEVICES`` is unset. This is used when neither the option nor the environment variable is set.
+
+For example:
+
+.. code-block:: bash
+
+  rocprofv3 --att --att-resource-mode code-object -- <application_path>
+
+In JSON or YAML input, set ``att_resource_mode`` to the same string value. The equivalent environment variable is ``ROCPROF_ATT_PARAM_RESOURCE_MODE``; an explicit command-line or input-file value overrides it. For resource lifetime and deferred device-trace starts, see :ref:`thread-trace`.
+
 For AMD Instinct accelerators, enable perfmon streaming using:
 
 .. code-block:: bash

projects/rocprofiler-sdk/source/docs/rocprofv3_input_schema.json

@@ -11,6 +11,12 @@
              "description": "data for rocprofv3",
               "properties": {
 
+                    "att_resource_mode": {
+                      "type": "string",
+                      "enum": ["default", "hsa", "code-object"],
+                      "description": "When to allocate thread trace resources for each GPU"
+                    },
+
                     "pmc": {
                       "type" : "array",
                       "description": "list of counters to collect"

projects/rocprofiler-sdk/source/include/rocprofiler-sdk/cxx/enum_string.hpp

@@ -1747,6 +1747,11 @@ ROCPROFILER_ENUM_INFO(rocprofiler_thread_trace_parameter_type_t,
                       ROCPROFILER_THREAD_TRACE_PARAMETER_LAST,
                       false,
                       not_operation);
+ROCPROFILER_ENUM_INFO(rocprofiler_thread_trace_resource_mode_t,
+                      ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DEFAULT,
+                      ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_LAST,
+                      false,
+                      not_operation);
 
 ROCPROFILER_ENUM_INFO(rocprofiler_agent_version_t,
                       ROCPROFILER_AGENT_INFO_VERSION_NONE,
@@ -2177,11 +2182,20 @@ ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_SERIALIZE_ALL);
 ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_PERFCOUNTER_EXCLUDE_MASK);
 ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_NO_DETAIL);
 ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_NUM_BUFFERS);
-static_assert(ROCPROFILER_THREAD_TRACE_PARAMETER_LAST == 10);
+ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE);
+static_assert(ROCPROFILER_THREAD_TRACE_PARAMETER_LAST == 11);
 
 ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_CONTROL_NONE);
 ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_CONTROL_START_AND_STOP);
 
+ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DEFAULT);
+ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_HSA);
+ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_HIP);
+ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_ALL);
+ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_CODE_OBJECT);
+ROCPROFILER_ENUM_LABEL(ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DISPATCH);
+static_assert(ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_LAST == 6);
+
 // rocprofiler_agent_version_t
 ROCPROFILER_ENUM_LABEL(ROCPROFILER_AGENT_INFO_VERSION_NONE);
 ROCPROFILER_ENUM_LABEL(ROCPROFILER_AGENT_INFO_VERSION_0);

projects/rocprofiler-sdk/source/include/rocprofiler-sdk/experimental/thread-trace/core.h

@@ -61,7 +61,12 @@ typedef enum rocprofiler_thread_trace_parameter_type_t
                                                      ///< no async copy). 2 is not allowed. Values
                                                      ///< >= 3 enable the async copy pipeline and
                                                      ///< require a single shader engine.
+    ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE,
     ROCPROFILER_THREAD_TRACE_PARAMETER_LAST
+
+    /// @var ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE
+    /// @brief Chooses when the profiler should initialize thread trace resources.
+    /// One of ::rocprofiler_thread_trace_resource_mode_t.
 } rocprofiler_thread_trace_parameter_type_t;
 
 /**
@@ -110,6 +115,34 @@ typedef enum rocprofiler_thread_trace_shader_data_flags_t
     /// handling of rocprofiler_thread_trace_shader_data_callback_t takes too long.
 } rocprofiler_thread_trace_shader_data_flags_t;
 
+/**
+ * @brief Modes to be used with ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE.
+ * Defines if and when the profiler should initialize the resources.
+ */
+typedef enum rocprofiler_thread_trace_resource_mode_t
+{
+    ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DEFAULT = 0,
+    ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_HSA,
+    ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_HIP,
+    ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_ALL,
+    ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_CODE_OBJECT,
+    ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DISPATCH,
+    ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_LAST
+
+    /// @var ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DEFAULT
+    /// @brief The profiler chooses initialization mode. Default.
+    /// @var ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_ALL
+    /// @brief Initializes resources for all GPUs reported by KFD as soon as possible.
+    /// @var ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_HSA
+    /// @brief Initializes at hsa_init(), respects ROCR_VISIBLE_DEVICES
+    /// @var ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_HIP
+    /// @brief Initializes during HIP bringup, respects HIP_VISIBLE_DEVICES
+    /// @var ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_CODE_OBJECT
+    /// @brief Initializes resources only for GPUs which had a code object registered to them.
+    /// @var ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DISPATCH
+    /// @brief Initializes resources at first kernel dispatch per GPU.
+} rocprofiler_thread_trace_resource_mode_t;
+
 /**
  * @brief Bundle of shader data delivered to
  * rocprofiler_thread_trace_shader_data_callback_t.

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/config.cpp

@@ -245,6 +245,17 @@ config::config()
 
     if(kernel_filter_include.empty()) kernel_filter_include = std::string{".*"};
 
+    const std::unordered_map<std::string_view, rocprofiler_thread_trace_resource_mode_t>
+        att_resource_modes = {
+            {"default", ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_DEFAULT},
+            {"hsa", ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_HSA},
+            {"code-object", ROCPROFILER_THREAD_TRACE_PARAMETER_RESOURCE_MODE_CODE_OBJECT}};
+    auto resource_mode = att_resource_modes.find(att_resource_mode);
+    ROCP_FATAL_IF(resource_mode == att_resource_modes.end())
+        << "Invalid value for ROCPROF_ATT_PARAM_RESOURCE_MODE: " << att_resource_mode
+        << ". Valid choices are: default, hsa, code-object";
+    att_resource_mode_value = resource_mode->second;
+
     std::unordered_map<std::string_view, rocprofiler_pc_sampling_unit_t> pc_sampling_unit_map = {
         {"none", ROCPROFILER_PC_SAMPLING_UNIT_NONE},
         {"instructions", ROCPROFILER_PC_SAMPLING_UNIT_INSTRUCTIONS},