[rocprof-compute] Generalize Roofline

projects/rocprofiler-compute/src/config.py

@@ -32,6 +32,6 @@
 HIDDEN_COLUMNS = ["coll_level"]
 HIDDEN_COLUMNS_CLI = ["Description", "coll_level"]
 HIDDEN_COLUMNS_TUI = ["Description", "coll_level"]
-HIDDEN_SECTIONS = [400, 1900, 2000]
+HIDDEN_SECTIONS = [1900, 2000]
 
 TIME_UNITS = {"s": 10**9, "ms": 10**6, "us": 10**3, "ns": 1}

projects/rocprofiler-compute/src/rocprof_compute_analyze/analysis_base.py

@@ -31,7 +31,13 @@
 from pathlib import Path
 
 from utils import file_io, parser, schema
-from utils.logger import console_debug, console_error, console_log, demarcate
+from utils.logger import (
+    console_debug,
+    console_error,
+    console_log,
+    console_warning,
+    demarcate,
+)
 from utils.utils import is_workload_empty, merge_counters_spatial_multiplex
 
 
@@ -189,6 +195,21 @@ def initalize_runs(self, normalization_filter=None):
                 else file_io.find_1st_sub_dir(d[0])
             )
             w.sys_info = file_io.load_sys_info(sysinfo_path.joinpath("sysinfo.csv"))
+
+            if not getattr(self.get_args(), "no_roof", False):
+                try:
+                    roofline_path = sysinfo_path.joinpath("roofline.csv")
+                    roofline_df = file_io.load_roofline_peaks(roofline_path)
+
+                    #use original column names from roofline.csv directly
+                    w.roofline_peaks = roofline_df
+
+                except FileNotFoundError:
+                    console_warning("roofline.csv not found.")
+                    w.roofline_peaks = file_io.create_empty_dataframe()
+            else:
+                w.roofline_peaks = file_io.create_empty_dataframe()
+
             arch = w.sys_info.iloc[0]["gpu_arch"]
             mspec = self.get_socs()[arch]._mspec
             if self.__args.specs_correction:

projects/rocprofiler-compute/src/rocprof_compute_analyze/analysis_cli.py

@@ -40,8 +40,9 @@ def pre_processing(self):
         if self.get_args().random_port:
             console_error("--gui flag is required to enable --random-port")
         for d in self.get_args().path:
+            workload = self._runs[d[0]]
             # create 'mega dataframe'
-            self._runs[d[0]].raw_pmc = file_io.create_df_pmc(
+            workload.raw_pmc = file_io.create_df_pmc(
                 d[0],
                 self.get_args().nodes,
                 self.get_args().spatial_multiplexing,
@@ -51,29 +52,32 @@ def pre_processing(self):
             )
 
             if self.get_args().spatial_multiplexing:
-                self._runs[d[0]].raw_pmc = self.spatial_multiplex_merge_counters(
-                    self._runs[d[0]].raw_pmc
+                workload.raw_pmc = self.spatial_multiplex_merge_counters(
+                    workload.raw_pmc
                 )
 
             file_io.create_df_kernel_top_stats(
-                df_in=self._runs[d[0]].raw_pmc,
+                df_in=workload.raw_pmc,
                 raw_data_dir=d[0],
-                filter_gpu_ids=self._runs[d[0]].filter_gpu_ids,
-                filter_dispatch_ids=self._runs[d[0]].filter_dispatch_ids,
-                filter_nodes=self._runs[d[0]].filter_nodes,
+                filter_gpu_ids=workload.filter_gpu_ids,
+                filter_dispatch_ids=workload.filter_dispatch_ids,
+                filter_nodes=workload.filter_nodes,
                 time_unit=self.get_args().time_unit,
                 max_stat_num=self.get_args().max_stat_num,
                 kernel_verbose=self.get_args().kernel_verbose,
             )
 
             # demangle and overwrite original 'Kernel_Name'
             kernel_name_shortener(
-                self._runs[d[0]].raw_pmc, self.get_args().kernel_verbose
+                workload.raw_pmc, self.get_args().kernel_verbose
             )
 
+            #update path
+            self._runs[d[0]].path = d[0]
+
             # create the loaded table
             parser.load_table_data(
-                workload=self._runs[d[0]],
+                workload=workload,
                 dir=d[0],
                 is_gui=False,
                 args=self.get_args(),
@@ -97,30 +101,30 @@ def run_analysis(self):
         else:
             roof_plot = None
             # 1. check if not baseline && compatible soc:
-            if (len(self.get_args().path)) == 1 and self._runs[
-                self.get_args().path[0][0]
-            ].sys_info.iloc[0]["gpu_arch"] in [
-                "gfx90a",
-                "gfx940",
-                "gfx941",
-                "gfx942",
-                "gfx950",
-            ]:
-                # add roofline plot to cli output
-                roof_obj = self.get_socs()[
-                    self._runs[self.get_args().path[0][0]].sys_info.iloc[0]["gpu_arch"]
-                ].roofline_obj
+            if (len(self.get_args().path)) == 1:
+                workload_path = self.get_args().path[0][0]
+                workload = self._runs[workload_path]
+                gpu_arch = workload.sys_info.iloc[0]["gpu_arch"]
+
+                if gpu_arch in ["gfx90a", "gfx940", "gfx941", "gfx942", "gfx950"]:
+                    roof_obj = self.get_socs()[gpu_arch].roofline_obj
+
+                    if roof_obj:
+                        #store path in workload for calc_ai_analyze
+                        workload.path = workload_path
 
-                if roof_obj:
-                    # NOTE: using default data type
-                    roof_plot = roof_obj.cli_generate_plot(roof_obj.get_dtype()[0])
+                        # NOTE: using default data type
+                        roof_plot = roof_obj.cli_generate_plot(
+                            dtype=roof_obj.get_dtype()[0],
+                            workload=workload,
+                            config=self._profiling_config,
+                            arch_config=self._arch_configs[gpu_arch]
+                        )
 
             tty.show_all(
                 self.get_args(),
                 self._runs,
-                self._arch_configs[
-                    self._runs[self.get_args().path[0][0]].sys_info.iloc[0]["gpu_arch"]
-                ],
+                self._arch_configs[self._runs[self.get_args().path[0][0]].sys_info.iloc[0]["gpu_arch"]],
                 self._output,
                 self._profiling_config,
                 roof_plot=roof_plot,

projects/rocprofiler-compute/src/rocprof_compute_base.py

@@ -289,7 +289,7 @@ def list_sets(self):
         if sets_info:
             first_set = next(iter(sets_info.keys()))
             print(f"  rocprof-compute profile --set {first_set}  # Profile this set")
-        print(f"  rocprof-compute profile --list-sets        # Show this help")
+        print("  rocprof-compute profile --list-sets        # Show this help")
         print()
 
         sys.exit(0)

projects/rocprofiler-compute/src/rocprof_compute_soc/analysis_configs/gfx908/0400_roofline.yaml

@@ -2,8 +2,199 @@
 Panel Config:
   id: 400
   title: Roofline
-  metrics_description: {}
+  metrics_description:
+    VALU FLOPs: 'The total floating-point operations executed per second on the VALU.
+      This is also presented as a percent of the peak theoretical FLOPs achievable
+      on the specific accelerator. Note: this does not include any floating-point
+      operations from MFMA instructions.'
+    MFMA FLOPs (F8): The total number of 8-bit brain floating point MFMA operations
+      executed per second. This does not include any 16-bit brain floating point operations
+      from VALU instructions. The peak empirically measured F8 MFMA operations achievable
+      on the specific accelerator is displayed alongside for comparison. It is supported
+      on AMD Instinct MI300 series and later only.
+    MFMA FLOPs (BF16): 'The total number of 16-bit brain floating point MFMA operations
+      executed per second. Note: this does not include any 16-bit brain floating point
+      operations from VALU instructions. The peak empirically measured BF16 MFMA operations
+      achievable on the specific accelerator is displayed alongside for comparison.'
+    MFMA FLOPs (F16): 'The total number of 16-bit floating point MFMA operations executed
+      per second. Note: this does not include any 16-bit floating point operations
+      from VALU instructions. The peak empirically measured F16 MFMA operations achievable
+      on the specific accelerator is displayed alongside for comparison.'
+    MFMA FLOPs (F32): 'The total number of 32-bit floating point MFMA operations executed
+      per second. Note: this does not include any 32-bit floating point operations
+      from VALU instructions. The peak empirically measured F32 MFMA operations achievable
+      on the specific accelerator is displayed alongside for comparison.'
+    MFMA FLOPs (F64): 'The total number of 64-bit floating point MFMA operations executed
+      per second. Note: this does not include any 64-bit floating point operations
+      from VALU instructions. The peak empirically measured F64 MFMA operations achievable
+      on the specific accelerator is displayed alongside for comparison.'
+    MFMA IOPs (Int8): 'The total number of 8-bit integer MFMA operations executed
+      per second. Note: this does not include any 8-bit integer operations from VALU
+      instructions. The peak empirically measured INT8 MFMA operations achievable
+      on the specific accelerator is displayed alongside for comparison.'
+    HBM Bandwidth: The total number of bytes read from and written to High-Bandwidth
+      Memory (HBM) per second. The peak empirically measured bandwidth achievable
+      on the specific accelerator is displayed alongside for comparison.
+    L2 Cache Bandwidth: The number of bytes looked up in the L2 cache per unit time.
+      The number of bytes is calculated as the number of cache lines requested multiplied
+      by the cache line size. This value does not consider partial requests, so e.g.,
+      if only a single value is requested in a cache line, the data movement will
+      still be counted as a full cache line. The peak empirically measured bandwidth
+      achievable on the specific accelerator is displayed alongside for comparison.
+    L1 Cache Bandwidth: The number of bytes looked up in the vL1D cache as a result
+      of VMEM instructions per unit time. The number of bytes is calculated as the
+      number of cache lines requested multiplied by the cache line size. This value
+      does not consider partial requests, so e.g., if only a single value is requested
+      in a cache line, the data movement will still be counted as a full cache line.
+      The peak empirically measured bandwidth achievable on the specific accelerator
+      is displayed alongside for comparison.
+    LDS Bandwidth: Indicates the maximum amount of bytes that could have been loaded
+      from, stored to, or atomically updated in the LDS per unit time (see LDS Bandwidth
+      example for more detail). The peak empirically measured LDS bandwidth achievable
+      on the specific accelerator is displayed alongside for comparison.
+    AI L1: The Arithmetic Intensity (AI) relative to the L1 Cache. It is the ratio
+      of total floating-point operations (FLOPs) to total bytes transferred between
+      the L1 cache and the processing units. This value is used as the x-coordinate
+      for the L1 roofline.
+    AI L2: The Arithmetic Intensity (AI) relative to the L2 Cache. It is the ratio
+      of total floating-point operations (FLOPs) to total bytes transferred between
+      the L2 cache and the L1 cache. This value is used as the x-coordinate for the
+      L2 roofline.
+    AI HBM: The Arithmetic Intensity (AI) relative to High-Bandwidth Memory (HBM).
+      It is the ratio of total floating-point operations (FLOPs) to total bytes transferred
+      between HBM and the L2 cache. This value is used as the x-coordinate for the
+      HBM roofline.
+    Performance GFLOPs: The overall achieved performance, measured in GigaFLOPs per
+      second (GFLOP/s). This is calculated as the sum of all VALU and MFMA floating-point
+      operations divided by the total execution time. This value is used as the y-coordinate
+      for the kernel's point on the Roofline plot.
   data source:
-  - None:
+  - metric_table:
       id: 401
-      title: Roofline
+      title: Roofline Performance Rates
+      cli_style: Roofline
+      tui_style: Roofline
+      header:
+        metric: Metric
+        value: Value
+        unit: Unit
+        peak: Peak (Empirical)
+      metric:
+        VALU FLOPs:
+          value: AVG((($wave_size * ( (SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16
+            + (2 * SQ_INSTS_VALU_FMA_F16) + SQ_INSTS_VALU_TRANS_F16) + (SQ_INSTS_VALU_ADD_F32
+            + SQ_INSTS_VALU_MUL_F32 + (2 * SQ_INSTS_VALU_FMA_F32) + SQ_INSTS_VALU_TRANS_F32)
+            + (SQ_INSTS_VALU_ADD_F64 + SQ_INSTS_VALU_MUL_F64 + (2 * SQ_INSTS_VALU_FMA_F64)
+            + SQ_INSTS_VALU_TRANS_F64) )) / ((End_Timestamp - Start_Timestamp) / 1e9))
+            / 1e9)
+          unit: GFLOP/s
+          peak: (((($max_sclk * $cu_per_gpu) * 64) * 2) / 1000)
+        MFMA FLOPs (F64):
+          value: AVG((((SQ_INSTS_VALU_MFMA_MOPS_F64 * 512)) / ((End_Timestamp - Start_Timestamp)
+            / 1e9)) / 1e9)
+          unit: GFLOP/s
+          peak: $MFMAF64Flops_empirical_peak
+        MFMA FLOPs (F32):
+          value: AVG((((SQ_INSTS_VALU_MFMA_MOPS_F32 * 512)) / ((End_Timestamp - Start_Timestamp)
+            / 1e9)) / 1e9)
+          unit: GFLOP/s
+          peak: $MFMAF32Flops_empirical_peak
+        MFMA FLOPs (F16):
+          value: AVG((((SQ_INSTS_VALU_MFMA_MOPS_F16 * 512)) / ((End_Timestamp - Start_Timestamp)
+            / 1e9)) / 1e9)
+          unit: GFLOP/s
+          peak: $MFMAF16Flops_empirical_peak
+        MFMA FLOPs (BF16):
+          value: AVG((((SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512)) / ((End_Timestamp - Start_Timestamp)
+            / 1e9)) / 1e9)
+          unit: GFLOP/s
+          peak: $MFMABF16Flops_empirical_peak
+        MFMA FLOPs (F8):
+          value: 0
+          unit: GFLOP/s
+          peak: 0
+        MFMA FLOPs (F6F4):
+          value: 0
+          unit: GFLOP/s
+          peak: 0
+        MFMA IOPs (Int8):
+          value: AVG((((SQ_INSTS_VALU_MFMA_MOPS_I8 * 512)) / ((End_Timestamp - Start_Timestamp)
+            / 1e9)) / 1e9)
+          unit: GIOP/s
+          peak: $MFMAI8Ops_empirical_peak
+        HBM Bandwidth:
+          value: AVG((( (TCC_BUBBLE_sum * 128) + (TCC_EA0_RDREQ_32B_sum * 32) + ((TCC_EA0_RDREQ_sum
+            - TCC_BUBBLE_sum - TCC_EA0_RDREQ_32B_sum) * 64) + ((TCC_EA0_WRREQ_sum
+            - TCC_EA0_WRREQ_64B_sum) * 32) + (TCC_EA0_WRREQ_64B_sum * 64)) / ((End_Timestamp
+            - Start_Timestamp) / 1e9)) / 1e9)
+          unit: GB/s
+          peak: $HBMBw_empirical_peak
+        L2 Cache Bandwidth:
+          value: AVG(((((TCP_TCC_WRITE_REQ_sum + TCP_TCC_ATOMIC_WITH_RET_REQ_sum +
+            TCP_TCC_ATOMIC_WITHOUT_RET_REQ_sum + TCP_TCC_READ_REQ_sum) * 64)) / ((End_Timestamp
+            - Start_Timestamp) / 1e9)) / 1e9)
+          unit: GB/s
+          peak: $L2Bw_empirical_peak
+        L1 Cache Bandwidth:
+          value: AVG((((TCP_TOTAL_CACHE_ACCESSES_sum * 64)) / ((End_Timestamp - Start_Timestamp)
+            / 1e9)) / 1e9)
+          unit: GB/s
+          peak: $L1Bw_empirical_peak
+        LDS Bandwidth:
+          value: AVG(((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4 * $lds_banks_per_cu))
+            / ((End_Timestamp - Start_Timestamp) / 1e9)) / 1e9)
+          unit: GB/s
+          peak: $LDSBw_empirical_peak
+  - metric_table:
+      id: 402
+      title: Roofline Plot Points
+      cli_style: Roofline
+      tui_style: Roofline
+      header:
+        metric: Metric
+        value: Value
+        unit: Unit
+      metric:
+        AI HBM:
+          value: ( SUM( ($wave_size * ( (SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16
+            + (2 * SQ_INSTS_VALU_FMA_F16) + SQ_INSTS_VALU_TRANS_F16) + (SQ_INSTS_VALU_ADD_F32
+            + SQ_INSTS_VALU_MUL_F32 + (2 * SQ_INSTS_VALU_FMA_F32) + SQ_INSTS_VALU_TRANS_F32)
+            + (SQ_INSTS_VALU_ADD_F64 + SQ_INSTS_VALU_MUL_F64 + (2 * SQ_INSTS_VALU_FMA_F64)
+            + SQ_INSTS_VALU_TRANS_F64) )) + (SQ_INSTS_VALU_MFMA_MOPS_F16 * 512) +
+            (SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512) + (SQ_INSTS_VALU_MFMA_MOPS_F32 *
+            512) + (SQ_INSTS_VALU_MFMA_MOPS_F64 * 512) ) / SUM( (TCC_BUBBLE_sum *
+            128) + (TCC_EA0_RDREQ_32B_sum * 32) + ((TCC_EA0_RDREQ_sum - TCC_BUBBLE_sum
+            - TCC_EA0_RDREQ_32B_sum) * 64) + ((TCC_EA0_WRREQ_sum - TCC_EA0_WRREQ_64B_sum)
+            * 32) + (TCC_EA0_WRREQ_64B_sum * 64) ) )
+          unit: FLOPs/Byte
+        AI L2:
+          value: ( SUM( ($wave_size * ( (SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16
+            + (2 * SQ_INSTS_VALU_FMA_F16) + SQ_INSTS_VALU_TRANS_F16) + (SQ_INSTS_VALU_ADD_F32
+            + SQ_INSTS_VALU_MUL_F32 + (2 * SQ_INSTS_VALU_FMA_F32) + SQ_INSTS_VALU_TRANS_F32)
+            + (SQ_INSTS_VALU_ADD_F64 + SQ_INSTS_VALU_MUL_F64 + (2 * SQ_INSTS_VALU_FMA_F64)
+            + SQ_INSTS_VALU_TRANS_F64) )) + (SQ_INSTS_VALU_MFMA_MOPS_F16 * 512) +
+            (SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512) + (SQ_INSTS_VALU_MFMA_MOPS_F32 *
+            512) + (SQ_INSTS_VALU_MFMA_MOPS_F64 * 512) ) / SUM( (TCP_TCC_WRITE_REQ_sum
+            + TCP_TCC_ATOMIC_WITH_RET_REQ_sum + TCP_TCC_ATOMIC_WITHOUT_RET_REQ_sum
+            + TCP_TCC_READ_REQ_sum) * 64 ) )
+          unit: FLOPs/Byte
+        AI L1:
+          value: ( SUM( ($wave_size * ( (SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16
+            + (2 * SQ_INSTS_VALU_FMA_F16) + SQ_INSTS_VALU_TRANS_F16) + (SQ_INSTS_VALU_ADD_F32
+            + SQ_INSTS_VALU_MUL_F32 + (2 * SQ_INSTS_VALU_FMA_F32) + SQ_INSTS_VALU_TRANS_F32)
+            + (SQ_INSTS_VALU_ADD_F64 + SQ_INSTS_VALU_MUL_F64 + (2 * SQ_INSTS_VALU_FMA_F64)
+            + SQ_INSTS_VALU_TRANS_F64) )) + (SQ_INSTS_VALU_MFMA_MOPS_F16 * 512) +
+            (SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512) + (SQ_INSTS_VALU_MFMA_MOPS_F32 *
+            512) + (SQ_INSTS_VALU_MFMA_MOPS_F64 * 512) ) / SUM(TCP_TOTAL_CACHE_ACCESSES_sum
+            * 64) )
+          unit: FLOPs/Byte
+        Performance GFLOPs:
+          value: ( SUM( ($wave_size * ( (SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16
+            + (2 * SQ_INSTS_VALU_FMA_F16) + SQ_INSTS_VALU_TRANS_F16) + (SQ_INSTS_VALU_ADD_F32
+            + SQ_INSTS_VALU_MUL_F32 + (2 * SQ_INSTS_VALU_FMA_F32) + SQ_INSTS_VALU_TRANS_F32)
+            + (SQ_INSTS_VALU_ADD_F64 + SQ_INSTS_VALU_MUL_F64 + (2 * SQ_INSTS_VALU_FMA_F64)
+            + SQ_INSTS_VALU_TRANS_F64) )) + (SQ_INSTS_VALU_MFMA_MOPS_F16 * 512) +
+            (SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512) + (SQ_INSTS_VALU_MFMA_MOPS_F32 *
+            512) + (SQ_INSTS_VALU_MFMA_MOPS_F64 * 512) ) / (SUM(End_Timestamp - Start_Timestamp)
+            / 1e9) ) / 1e9
+          unit: GFLOP/s