vedithal-amd · Aug 1, 2025 · Aug 14, 2025 · Aug 19, 2025 · Aug 19, 2025 · cfallows-amd
@@ -18,6 +18,8 @@ Full documentation for ROCm Compute Profiler is available at [https://rocm.docs.
   * Enable SQC_DCACHE_INFLIGHT_LEVEL counter and associated metrics
   * Enable TCP_TCP_LATENCY counter and associated counter for all GPUs except MI300
 
+* Add support for analysis report output as a sqlite database using ``--output-format db`` analysis mode option
+
 ### Changed
 
 * Add notice for change in default output format to `rocpd` in a future release
@@ -53,6 +55,12 @@ Full documentation for ROCm Compute Profiler is available at [https://rocm.docs.
   * sL1D-L2 BW Utilization (section 1401)
   * Bandwidth Utilization (section 1601)
 
+* Analysis output:
+  * Remove ``-o`` or ``--output`` analyze mode option.
+  * Remove ``--save-dfs`` analyze mode option.
+  * Add ``--output-format`` analysis mode option to select the output format of the analysis report.
+  * Add ``--output-name`` analysis mode option to override the default file/folder name.
+
 ### Resolved issues
 
 * Fixed not detecting memory clock issue when using amd-smi

@@ -22,6 +22,7 @@ RUN git config --global --add safe.directory /app
 # Install any dependencies specified in requirements.txt
 # Run interactive bash shell
  CMD ["/bin/bash", "-c", "\
-     python3 -m pip install -r docs/sphinx/requirements.txt \
+     cd /app/projects/rocprofiler-compute \
+     && python3 -m pip install -r docs/sphinx/requirements.txt \
      && exec /bin/bash \
  "]
@@ -1,9 +1,9 @@
 services:
   doctest: # service name
     build:
-      context: ../
-      dockerfile: docker/Dockerfile.doctest
+      context: ../../../
+      dockerfile: projects/rocprofiler-compute/docker/Dockerfile.doctest
     volumes:
-      - ../:/app
+      - ../../../:/app
     tty: true
     stdin_open: true
@@ -213,4 +213,4 @@
 }
 
 # Uncomment if facing rate limit exceed issue with local build
-external_projects_remote_repository = ""
+# external_projects_remote_repository = ""
@@ -320,3 +320,102 @@ Baseline comparison
   .. code-block:: shell
 
      rocprof-compute analyze -p workload1/path/ -k 0  -p workload2/path/ -k 1
+
+Analysis output format
+======================
+
+Use the ``--output-format <format>`` analyze mode option to specify the output format of the
+analysis report. Supported formats are ``stdout``, ``txt``, ``csv``, and ``db``. The default output
+format is ``stdout``.
+
+* ``stdout`` format:
+   * Print analysis report to the terminal.
+   * NOTE: This option will not generate any file or folder.
+
+* ``txt`` format:
+   * Generate a file named ``rocprof_compute_<uuid>.txt`` in the current working directory.
+   * This file contains the entire analysis report as printed on the terminal.
+   * This is useful in case of searching across long analysis reports.
+   * NOTE: This option will disable output of analysis report to terminal.
+
+* ``csv`` format:
+   * Generate a folder named ``rocprof_compute_<uuid>`` in the current working directory.
+   * This folder contains multiple csv files representing the data in each metric table in the analysis report.
+   * This is useful for further programmatic analysis of analysis reports.
+   * NOTE: This will print the analysis report to the terminal as well.
+
+* ``db`` format:
+   * NOTE: This only works when provided workload paths are created using ``--format-rocprof-output rocpd`` profile mode option.
+   * Generate a file named ``rocprof_compute_<uuid>.db`` in the current working directory.
+   * This is a SQLite database file containing all the data in the analysis report structured according to :ref:`analysis database schema <analysis-database>`.
+   * This is useful for further programmatic analysis of analysis reports.
+   * NOTE: This option will disable output of analysis report to terminal.
+
+Default file/folder name ``rocprofiler_compute_<uuid>`` can be overriden using ``--output-name <name>`` analyze mode option.
+
+.. _analysis-database:
+
+Analysis database schema
+========================
+
+Analysis database tables
+
+.. image:: ../../data/analyze/analysis_data_dump_schema.png
+   :align: center
+   :alt: Analysis database tables
+
+Analysis database views
+
+.. image:: ../../data/analyze/analysis_data_dump_views.png
+   :align: center
+   :alt: Analysis database views
+
+Analysis database example
+
+.. note::
+
+   Some metrics cannot be calculated when corresponding counters are missing as shown in the warnings below
+
+.. note::
+
+   It is possible to merge the analysis data dump for multiple workload folders (resulting from multiple profiles) by repeating ``-p`` option for each workload
+
+.. code-block:: shell-session
+
+   $ rocprof-compute analyze --verbose --db test -p workloads/vmem/MI300X_A1 -p workloads/vmem1/MI300X_A1
+   DEBUG Execution mode = analyze
+
+                                    __                                       _
+   _ __ ___   ___ _ __  _ __ ___  / _|       ___ ___  _ __ ___  _ __  _   _| |_ ___
+   | '__/ _ \ / __| '_ \| '__/ _ \| |_ _____ / __/ _ \| '_ ` _ \| '_ \| | | | __/ _ \
+   | | | (_) | (__| |_) | | | (_) |  _|_____| (_| (_) | | | | | | |_) | |_| | ||  __/
+   |_|  \___/ \___| .__/|_|  \___/|_|        \___\___/|_| |_| |_| .__/ \__,_|\__\___|
+                  |_|                                           |_|
+
+      INFO Analysis mode = db
+   DEBUG [omnisoc init]
+   DEBUG [omnisoc init]
+   DEBUG [analysis] prepping to do some analysis
+      INFO [analysis] deriving rocprofiler-compute metrics...
+   WARNING Roofline ceilings not found for /app/projects/rocprofiler-compute/workloads/vmem/MI300X_A1.
+   WARNING Roofline ceilings not found for /app/projects/rocprofiler-compute/workloads/vmem1/MI300X_A1.
+   WARNING PC sampling data not found for /app/projects/rocprofiler-compute/workloads/vmem/MI300X_A1.
+   WARNING PC sampling data not found for /app/projects/rocprofiler-compute/workloads/vmem1/MI300X_A1.
+   DEBUG Collected dispatch data
+   DEBUG Applied analysis mode filters
+   DEBUG Calculated dispatch data
+   DEBUG Collected metrics data
+   WARNING Failed to evaluate expression for 3.1.25 - Value: to_round(to_avg(
+   (pmc_df.get("TCP_TCP_LATENCY_sum") / pmc_df.get("TCP_TA_TCP_STATE_READ_sum")).where((pmc_df.get("TCP_TA_TCP_STATE_READ_sum") != 0), None)), 0) - unsupported operand type(s) for /: 'NoneType' and 'float'
+   WARNING Failed to evaluate expression for 3.1.39 - Value: to_round((to_avg(
+   (pmc_df.get("pmc_perf_ACCUM") / pmc_df.get("SQC_ICACHE_REQ")).where((pmc_df.get("SQC_ICACHE_REQ") != 0), None)) * 100), 0) - unsupported operand type(s) for /: 'NoneType' and 'float'
+   WARNING Failed to evaluate expression for 3.1.25 - Value: to_round(to_avg(
+   (pmc_df.get("TCP_TCP_LATENCY_sum") / pmc_df.get("TCP_TA_TCP_STATE_READ_sum")).where((pmc_df.get("TCP_TA_TCP_STATE_READ_sum") != 0), None)), 0) - unsupported operand type(s) for /: 'NoneType' and 'float'
+   WARNING Failed to evaluate expression for 3.1.39 - Value: to_round((to_avg(
+   (pmc_df.get("pmc_perf_ACCUM") / pmc_df.get("SQC_ICACHE_REQ")).where((pmc_df.get("SQC_ICACHE_REQ") != 0), None)) * 100), 0) - unsupported operand type(s) for /: 'NoneType' and 'float'
+   DEBUG Calculated metric values
+   DEBUG Calculated roofline data points
+   DEBUG [analysis] generating analysis
+   DEBUG SQLite database initialized with name: test.db
+   DEBUG Initialized database: test.db
+   DEBUG Completed writing database
@@ -22,7 +22,7 @@ options.
 * :doc:`cli`
 * :doc:`grafana-gui`
 * :doc:`standalone-gui`
-* :doc:`text-based user interface (TUI)`
+* :doc:`tui`
 
 .. note::
 

@@ -24,6 +24,15 @@ Profiling with ROCm Compute Profiler yields the following benefits.
 * :ref:`Automate counter collection <profiling-routine>`: ROCm Compute Profiler handles all
   of your profiling via pre-configured input files.
 
+* :ref:`Profiling output format <profiling-output-format>`: ROCm Compute Profile can adjust the
+   output format of underlying rocprof tool which changes the output format of raw performance
+   counter data in the workload folder created during profiling. Supported output formats are
+   ``json``, ``csv``, and ``rocpd``. The default output format is ``csv``.
+
+.. note::
+
+   The default output format will be changed to ``rocpd`` in the future releases of ROCm Compute Profiler.
+
 * :ref:`Filtering <filtering>`: Apply runtime filters to speed up the profiling
   process.
 
@@ -217,6 +226,32 @@ an Instinct MI210 vs an Instinct MI250.
    -rw-r--r-- 1 auser agroup   650 Mar  1 15:15 sysinfo.csv
    -rw-r--r-- 1 auser agroup   399 Mar  1 15:15 timestamps.csv
 
+.. _profiling-output-format:
+
+Profiling output format
+-----------------------
+
+Use the ``--format-rocprof-output <format>`` profile mode option to specify the output format
+of the underlying ``rocprof`` tool. The following formats are supported:
+
+* ``csv`` format:
+   * Ask underlying rocprof tool to dump raw performance counter data in csv format.
+   * The generated csv files across multiple runs of rocprof are processed and dumped into the workload directory as csv files.
+   * Multiple csv files are merged into single pmc_perf.csv file in workload directory.
+
+* ``json`` format:
+   * Ask underlying rocprof tool to dump raw performance counter data in json format.
+   * The generated json files across multiple runs of rocprof are processed and dumped into the workload directory as csv files.
+   * Multiple csv files are merged into single pmc_perf.csv file in workload directory.
+
+* ``rocpd`` format:
+   * Ask underlying rocprof tool to dump raw performance counter data in rocpd format.
+   * Multiple ``rocpd`` database files containding counter collection data are merged into a single csv under the workload folder.
+     The database files are then removed.
+   * Use ``--retain-rocpd-output`` profile mode option to preserve the ``rocpd`` database(s) in the workload folder.
+     This is useful for custom analysis of profiling data.
+
+
 .. _filtering:
 
 Filtering

@@ -12,6 +12,7 @@ plotille
 pymongo
 pyyaml
 setuptools
+sqlalchemy==2.0.42
 tabulate
 textual
 textual_plotext

@@ -228,7 +228,10 @@ def validate_block(value):
         "--set",
         default=None,
         dest="set_selected",
-        help="\t\t\tProfile a set of metrics of topic of interest by collecting counters in a single pass.\n\t\t\tFor available sets, see --list-sets",
+        help=(
+            "\t\t\tProfile a set of metrics of topic of interest by collecting counters"
+            " in a single pass.\n\t\t\tFor available sets, see --list-sets"
+        ),
     )
 
     profile_group.add_argument(
@@ -307,7 +310,7 @@ def validate_block(value):
             "\t\t\t  For stochastic sampling, the interval is in cycles.\n"
             "\t\t\t  For host_trap sampling, the interval is in microsecond "
             "(DEFAULT: 1048576)."
-        )
+        ),
     )
 
     profile_group.add_argument(
@@ -632,11 +635,27 @@ def validate_block(value):
         help="\t\tMode of spatial multiplexing.",
     )
     analyze_group.add_argument(
-        "-o",
-        "--output",
+        "--output-format",
+        metavar="",
+        dest="output_format",
+        choices=["stdout", "txt", "csv", "db"],
+        default="stdout",
+        help=(
+            "\t\tSet the format of output file or folder containing analysis data.\n"
+            "\t\tBy default, file or folder created will have the name rocprof_compute_<uuid>.\n"
+            "\t\tFile or folder name can be overriden using --output-name.\n"
+            "\t\tDefault output format is stdout which will not generate any file/folder.\n"
+        ),
+    )
+    analyze_group.add_argument(
+        "--output-name",
         metavar="",
-        dest="output_file",
-        help="\t\tSpecify an output file to save analysis results.",
+        dest="output_name",
+        help="\t\tOverride the default output file name rocprof_compue_<uuid> with the specified name.",
+        help=(
+            "\t\tOverride the default output file name rocprof_compue_<uuid> with the specified name.\n"
+            "\t\tThis is only applicable when --output-format txt/csv/db is used.\n"
+        ),
     )
     analyze_group.add_argument(
         "--gui",
@@ -755,12 +774,6 @@ def validate_block(value):
         help="\t\tSpecify the directory of customized configs.",
         default=rocprof_compute_home.joinpath("rocprof_compute_soc/analysis_configs/"),
     )
-    analyze_advanced_group.add_argument(
-        "--save-dfs",
-        dest="df_file_dir",
-        metavar="",
-        help="\t\tSpecify the dirctory to save analysis dataframe csv files.",
-    )
     analyze_advanced_group.add_argument(
         "--cols",
         type=int,

@@ -32,7 +32,7 @@
 
 from utils import file_io, parser, schema
 from utils.logger import console_debug, console_error, console_log, demarcate
-from utils.utils import is_workload_empty, merge_counters_spatial_multiplex
+from utils.utils import is_workload_empty, merge_counters_spatial_multiplex, get_uuid
 
 
 class OmniAnalyze_Base:
@@ -258,6 +258,21 @@ def sanitize(self):
             print("Node list:", "  ".join(nodes))
             sys.exit(0)
 
+        # Ensure analysis output does not overwrite existing files
+        if self.__args.output_name:
+            if not self.__args.output_name.isalnum():
+                console_error(
+                    "Analysis output file/folder name must contain only alphanumeric characters."
+                )
+            path_to_check = self.__args.output_name
+            if self.__args.output_format in ("txt", "db"):
+                path_to_check += f".{self.__args.output_format}"
+            if Path(path_to_check).exists():
+                console_error(
+                    f"Analysis output file/folder {path_to_check} already exists. "
+                    "Please choose a different name."
+                )
+
     # ----------------------------------------------------
     # Required methods to be implemented by child classes
     # ----------------------------------------------------
@@ -267,11 +282,14 @@ def pre_processing(self):
         console_debug("analysis", "prepping to do some analysis")
         console_log("analysis", "deriving rocprofiler-compute metrics...")
         # initalize output file
-        self._output = (
-            open(self.__args.output_file, "w+")
-            if self.__args.output_file
-            else sys.stdout
-        )
+        if self.__args.output_format == "txt":
+            if self.__args.output_name:
+                self._output = open(f"{self.__args.output_name}.txt", "w+")
+                print(f"Created file: {self.__args.output_name}.txt")
+            else:
+                self._output = open(f"rocprof_compute_{get_uuid()}.txt", "w+")
+        elif self.__args.output_format == "stdout":
+            self._output = sys.stdout
 
         # Read profiling config
         self._profiling_config = file_io.load_profiling_config(self.__args.path[0][0])