Bottleneck Characterization

requirements.txt

@@ -13,3 +13,4 @@ dash-bootstrap-components
 kaleido
 setuptools
 plotille
+perfetto

src/argparser.py

@@ -196,6 +196,13 @@ def omniarg_parser(parser, omniperf_home, supported_archs, omniperf_version):
         profile_group.add_argument(
             "--kernel-summaries", default=False, dest="summaries", help=argparse.SUPPRESS
         )
+    profile_group.add_argument(
+        "--no-trace",
+        required=False,
+        default=False,
+        action="store_true",
+        help="\t\t\tExplicitly disable binary instrumentation via Omnitrace. Used for Bottleneck Characterization.",
+    )
     profile_group.add_argument(
         "--join-type",
         metavar="",
@@ -456,6 +463,12 @@ def omniarg_parser(parser, omniperf_home, supported_archs, omniperf_version):
         const=8050,
         help="\t\tActivate a GUI to interate with Omniperf metrics.\n\t\tOptionally, specify port to launch application (DEFAULT: 8050)",
     )
+    analyze_group.add_argument(
+        "--trace",
+        required=False,
+        metavar="",
+        help="\t\tSpecify path to Omnitrace (.proto) output to be used in bottleneck characterization.",
+    )
     analyze_advanced_group.add_argument(
         "--random-port",
         action="store_true",

src/omniperf_analyze/analysis_base.py

@@ -26,6 +26,7 @@
 import os
 import sys
 import copy
+import time
 from collections import OrderedDict
 from pathlib import Path
 from utils.utils import (
@@ -36,6 +37,7 @@
     console_error,
 )
 from utils import schema, file_io, parser
+from utils.workload_characterization import Bottleneck_Classification
 
 
 class OmniAnalyze_Base:
@@ -46,6 +48,7 @@ def __init__(self, args, supported_archs):
         self.__supported_archs = supported_archs
         self._output = None
         self.__socs: dict = None  # available OmniSoC objs
+        self._bottleneck_characterization: Bottleneck_Classification = None
 
     def get_args(self):
         return self.__args
@@ -230,3 +233,51 @@ def pre_processing(self):
     def run_analysis(self):
         """Run analysis."""
         console_debug("analysis", "generating analysis")
+        self._bottleneck_characterization = get_bottleneck_characterization(
+            self.get_args()
+        )
+
+
+def get_bottleneck_characterization(args):
+    # Default path for omnitrace output (can be overriden using --trace flag)
+    default_omnitrace_proto = args.path[0][0] + "/omnitrace/instrumentation-output.proto"
+    # Instantiate the Bottleneck Classification object
+    if args.trace:
+        bc = Bottleneck_Classification(
+            omniperf_dir=args.path[0][0],
+            omnitrace_dir=args.trace,
+            treshold_ratio=0.8,
+        )
+    elif os.path.isfile(default_omnitrace_proto):
+        bc = Bottleneck_Classification(
+            omniperf_dir=args.path[0][0],
+            omnitrace_dir=default_omnitrace_proto,
+            treshold_ratio=0.8,
+        )
+    else:
+        return None
+    # make sure that the gpu ids match between omnitrace and omniperf
+    if not (
+        list(bc.omnitrace_data["gpu_ids"]).sort()
+        == list(bc.omniperf_data["gpu_bounds_time"].keys()).sort()
+    ):
+        console_error(
+            "Bottleneck Characterization",
+            "GPU ids in omnitrace and omniperf do not match",
+        )
+    # create and save plots (Re-save to remove loading MathJax pop up)
+    bc.create_output_plots()
+    bc.end_to_end_plot.write_image(
+        bc.input_dirs["omniperf"] + "/characterization-e2e_time.pdf"
+    )
+    bc.gpu_bottleneck_plot.write_image(
+        bc.input_dirs["omniperf"] + "/characterization-gpu_time.pdf"
+    )
+    time.sleep(1)
+    bc.end_to_end_plot.write_image(
+        bc.input_dirs["omniperf"] + "/characterization-e2e_time.pdf"
+    )
+    bc.gpu_bottleneck_plot.write_image(
+        bc.input_dirs["omniperf"] + "/characterization-gpu_time.pdf"
+    )
+    return bc

src/omniperf_analyze/analysis_webui.py

@@ -77,7 +77,12 @@ def build_layout(self, input_filters, arch_configs):
             children=[
                 dbc.Spinner(
                     children=[
-                        get_header(base_data.raw_pmc, input_filters, filt_kernel_names),
+                        get_header(
+                            base_data.raw_pmc,
+                            input_filters,
+                            self._bottleneck_characterization,
+                            filt_kernel_names,
+                        ),
                         html.Div(id="container", children=[]),
                     ],
                     fullscreen=True,
@@ -296,7 +301,7 @@ def pre_processing(self):
 
     @demarcate
     def run_analysis(self):
-        """Run CLI analysis."""
+        """Run GUI analysis."""
         super().run_analysis()
         args = self.get_args()
         input_filters = {

src/omniperf_analyze/assets/layout.css

@@ -58,7 +58,7 @@ header .banner {
    display: inline-block;
    vertical-align: middle;
    margin: 0 auto;
-   width: 85%;
+   width: 95%;
    padding-bottom: 30px;
    text-align: center;
 }

src/omniperf_profile/profiler_base.py

@@ -29,6 +29,7 @@
 import sys
 import os
 import re
+import shutil
 from utils.utils import (
     capture_subprocess_output,
     run_prof,
@@ -39,6 +40,7 @@
     console_debug,
     console_error,
     console_warning,
+    detect_omnitrace,
     print_status,
 )
 import config
@@ -53,6 +55,7 @@ def __init__(self, args, profiler_mode, soc):
         self.__perfmon_dir = os.path.join(
             str(config.omniperf_home), "omniperf_soc", "profile_configs"
         )
+        self.__omnitrace_path = None
 
     def get_args(self):
         return self.__args
@@ -265,72 +268,9 @@ def join_prof(self, out=None):
         else:
             return df
 
-    # ----------------------------------------------------
-    # Required methods to be implemented by child classes
-    # ----------------------------------------------------
-    @abstractmethod
-    def pre_processing(self):
-        """Perform any pre-processing steps prior to profiling."""
-        console_debug("profiling", "pre-processing using %s profiler" % self.__profiler)
-
-        # verify soc compatibility
-        if self.__profiler not in self._soc.get_compatible_profilers():
-            console_error(
-                "%s is not enabled in %s. Available profilers include: %s"
-                % (
-                    self._soc.get_arch(),
-                    self.__profiler,
-                    self._soc.get_compatible_profilers(),
-                )
-            )
-        # verify not accessing parent directories
-        if ".." in str(self.__args.path):
-            console_error(
-                "Access denied. Cannot access parent directories in path (i.e. ../)"
-            )
-
-        # verify correct formatting for application binary
-        self.__args.remaining = self.__args.remaining[1:]
-        if self.__args.remaining:
-            if not os.path.isfile(self.__args.remaining[0]):
-                console_error(
-                    "Your command %s doesn't point to a executable. Please verify."
-                    % self.__args.remaining[0]
-                )
-            self.__args.remaining = " ".join(self.__args.remaining)
-        else:
-            console_error(
-                "Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256"
-            )
-
-        # verify name meets MongoDB length requirements and no illegal chars
-        if len(self.__args.name) > 35:
-            console_error("-n/--name exceeds 35 character limit. Try again.")
-        if self.__args.name.find(".") != -1 or self.__args.name.find("-") != -1:
-            console_error("'-' and '.' are not permitted in -n/--name")
-
-    @abstractmethod
-    def run_profiling(self, version: str, prog: str):
-        """Run profiling."""
-        console_debug(
-            "profiling", "performing profiling using %s profiler" % self.__profiler
-        )
-
-        # log basic info
-        console_log(str(prog).title() + " version: " + str(version))
-        console_log("Profiler choice: %s" % self.__profiler)
-        console_log("Path: " + str(os.path.abspath(self.__args.path)))
-        console_log("Target: " + str(self._soc._mspec.gpu_model))
-        console_log("Command: " + str(self.__args.remaining))
-        console_log("Kernel Selection: " + str(self.__args.kernel))
-        console_log("Dispatch Selection: " + str(self.__args.dispatch))
-        if self.__args.ipblocks == None:
-            console_log("Hardware Blocks: All")
-        else:
-            console_log("Hardware Blocks: " + str(self.__args.ipblocks))
-
-        print_status("Collecting Performance Counters")
-
+    @demarcate
+    def run_profiler(self):
+        """Profile the application using selected profiler"""
         # show status bar in error-only mode
         disable_tqdm = True
         if self.__args.loglevel >= logging.ERROR:
@@ -400,6 +340,89 @@ def run_profiling(self, version: str, prog: str):
                 # TODO: Finish logic
                 console_error("Profiler not supported")
 
+    # ----------------------------------------------------
+    # Required methods to be implemented by child classes
+    # ----------------------------------------------------
+    @abstractmethod
+    def pre_processing(self):
+        """Perform any pre-processing steps prior to profiling."""
+        console_debug("profiling", "pre-processing using %s profiler" % self.__profiler)
+
+        # verify soc compatibility
+        if self.__profiler not in self._soc.get_compatible_profilers():
+            console_error(
+                "%s is not enabled in %s. Available profilers include: %s"
+                % (
+                    self._soc.get_arch(),
+                    self.__profiler,
+                    self._soc.get_compatible_profilers(),
+                )
+            )
+        # verify not accessing parent directories
+        if ".." in str(self.__args.path):
+            console_error(
+                "Access denied. Cannot access parent directories in path (i.e. ../)"
+            )
+
+        # verify correct formatting for application binary
+        self.__args.remaining = self.__args.remaining[1:]
+        if self.__args.remaining:
+            if not os.path.isfile(self.__args.remaining[0]):
+                console_error(
+                    "Your command %s doesn't point to a executable. Please verify."
+                    % self.__args.remaining[0]
+                )
+            self.__args.remaining = " ".join(self.__args.remaining)
+        else:
+            console_error(
+                "Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256"
+            )
+
+        # verify name meets MongoDB length requirements and no illegal chars
+        if len(self.__args.name) > 35:
+            console_error("-n/--name exceeds 35 character limit. Try again.")
+        if self.__args.name.find(".") != -1 or self.__args.name.find("-") != -1:
+            console_error("'-' and '.' are not permitted in -n/--name")
+
+        # verify Omnitrace installation
+        if not self.__args.no_trace:
+            self.__omnitrace_path = detect_omnitrace()
+
+    @abstractmethod
+    def run_profiling(self, version: str, prog: str):
+        """Run profiling."""
+        # log basic info
+        console_log(str(prog).title() + " version: " + str(version))
+        console_log("Profiler choice: %s" % self.__profiler)
+        console_log("Path: " + str(os.path.abspath(self.__args.path)))
+        console_log("Target: " + str(self._soc._mspec.gpu_model))
+        console_log("Command: " + str(self.__args.remaining))
+        (
+            console_log("Binary Instumentation: ON")
+            if self.__omnitrace_path
+            else console_log("Binary Instumentation: OFF")
+        )
+        console_log("Kernel Selection: " + str(self.__args.kernel))
+        console_log("Dispatch Selection: " + str(self.__args.dispatch))
+        (
+            console_log("Hardware Blocks: All")
+            if self.__args.ipblocks == None
+            else console_log("Hardware Blocks: " + str(self.__args.ipblocks))
+        )
+
+        # Instrument the application via Omnitrace if available
+        if self.__omnitrace_path:
+            console_debug("profiling", "performing instrumentation")
+            print_status("Running Binary Instrumentation")
+            run_omnitrace(self.__args.path, self.__omnitrace_path, self.__args.remaining)
+
+        # Collect required perfmon
+        console_debug(
+            "profiling", "performing profiling using %s profiler" % self.__profiler
+        )
+        print_status(f"Collecting Performance Counters")
+        self.run_profiler()
+
     @abstractmethod
     def post_processing(self):
         """Perform any post-processing steps prior to profiling."""
@@ -421,3 +444,26 @@ def post_processing(self):
 
 def test_df_column_equality(df):
     return df.eq(df.iloc[:, 0], axis=0).all(1).all()
+
+
+def run_omnitrace(path, omnitrace, cmd):
+    """Instrument the application via Omnitrace"""
+    success, output = capture_subprocess_output(
+        [
+            omnitrace,
+            "--env",
+            f"OMNITRACE_OUTPUT_PATH={path}/omnitrace",
+            f"OMNITRACE_PERFETTO_FILE=instrumentation-output.proto",
+            "OMNITRACE_TIME_OUTPUT=OFF",
+            "OMNITRACE_USE_PID=OFF",
+            "--",
+            cmd,
+        ],
+        profile_type="trace",
+    )
+    if not success:
+        for line in output.split("\n"):
+            console_warning(line)
+        console_error(
+            "Instrumentation failed. Note: Instrumentation can be disabled with the '--no-trace' option."
+        )

src/omniperf_profile/profiler_rocprof_v2.py

@@ -28,7 +28,6 @@
     demarcate,
     console_log,
     replace_timestamps,
-    fixup_rocprofv2_dispatch_ids,
 )
 
 
@@ -48,10 +47,7 @@ def get_profiler_options(self, fname):
             # v2 requires output directory argument
             "-d",
             self.get_args().path + "/" + "out",
-            # v2 does not require csv extension
-            "-o",
-            fbase,
-            # v2 doen not require quotes on cmd
+            # v2 does not require quotes on cmd
             app_cmd,
         ]
         return args
@@ -87,7 +83,5 @@ def post_processing(self):
         if self.ready_to_profile:
             # Manually join each pmc_perf*.csv output
             self.join_prof()
-            # Correct dispatch ids
-            fixup_rocprofv2_dispatch_ids(self.get_args().path)
             # Replace timestamp data to solve a known rocprof bug
             replace_timestamps(self.get_args().path)