diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index d83a869cbf99f1..cf38cb8547640f 100755 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -554,22 +554,19 @@ def __init__(self, coreclr_args): if coreclr_args.host_os == "OSX": self.collection_shim_name = "libsuperpmi-shim-collector.dylib" - self.mcs_tool_name = "mcs" self.corerun_tool_name = "corerun" elif coreclr_args.host_os == "Linux": self.collection_shim_name = "libsuperpmi-shim-collector.so" - self.mcs_tool_name = "mcs" self.corerun_tool_name = "corerun" elif coreclr_args.host_os == "Windows_NT": self.collection_shim_name = "superpmi-shim-collector.dll" - self.mcs_tool_name = "mcs.exe" self.corerun_tool_name = "corerun.exe" else: raise RuntimeError("Unsupported OS.") self.jit_path = os.path.join(coreclr_args.core_root, determine_jit_name(coreclr_args)) self.superpmi_path = determine_superpmi_tool_path(coreclr_args) - self.mcs_path = os.path.join(coreclr_args.core_root, self.mcs_tool_name) + self.mcs_path = determine_mcs_tool_path(coreclr_args) self.core_root = coreclr_args.core_root @@ -1577,6 +1574,9 @@ def determine_coredis_tools(coreclr_args): coredistools_location (str) : path of [lib]coredistools.dylib|so|dll """ + if not hasattr(coreclr_args, "core_root") or coreclr_args.core_root is None: + raise RuntimeError("Core_Root not set properly") + coredistools_dll_name = None if coreclr_args.host_os.lower() == "osx": coredistools_dll_name = "libcoredistools.dylib" @@ -1591,6 +1591,13 @@ def determine_coredis_tools(coreclr_args): if os.path.isfile(coredistools_location): print("Using coredistools found at {}".format(coredistools_location)) else: + # Often, Core_Root will already exist. However, you can do a product build without + # creating a Core_Root, and successfully run replay or asm diffs, if we just create Core_Root + # and copy coredistools there. Note that our replays all depend on Core_Root existing, as we + # set the current directory to Core_Root before running superpmi. + if not os.path.isdir(coreclr_args.core_root): + print("Warning: Core_Root does not exist at \"{}\"; creating it now".format(coreclr_args.core_root)) + os.makedirs(coreclr_args.core_root) coredistools_uri = az_blob_storage_superpmi_container_uri + "/libcoredistools/{}-{}/{}".format(coreclr_args.host_os.lower(), coreclr_args.arch.lower(), coredistools_dll_name) print("Download: {} -> {}".format(coredistools_uri, coredistools_location)) urllib.request.urlretrieve(coredistools_uri, coredistools_location) @@ -1660,6 +1667,46 @@ def determine_jit_name(coreclr_args): else: raise RuntimeError("Unknown OS.") +def find_tool(coreclr_args, tool_name, search_core_root=True, search_product_location=True, search_PATH=True): + """ Find a tool or any specified file (e.g., clrjit.dll) and return the full path to that tool if found. + + Args: + coreclr_args (CoreclrArguments): parsed args + tool_name (str): tool to find, e.g., "superpmi.exe" + search_core_root (bool): True to search the Core_Root folder + search_product_location: True to search the build product folder + search_PATH: True to search along the PATH + + Return: + (str) Full path of the tool, or None if not found. + """ + + # First, look in Core_Root, if there is one. + if search_core_root and hasattr(coreclr_args, "core_root") and coreclr_args.core_root is not None and os.path.isdir(coreclr_args.core_root): + tool_path = os.path.join(coreclr_args.core_root, tool_name) + if os.path.isfile(tool_path): + print("Using {} from Core_Root: {}".format(tool_name, tool_path)) + return tool_path + + # Next, look in the built product directory, if it exists. We can use superpmi/mcs directly from the + # product build directory instead from Core_Root because they don't depend on managed code libraries. + if search_product_location and hasattr(coreclr_args, "product_location") and coreclr_args.product_location is not None and os.path.isdir(coreclr_args.product_location): + tool_path = os.path.join(coreclr_args.product_location, tool_name) + if os.path.isfile(tool_path): + print("Using {} from product build location: {}".format(tool_name, tool_path)) + return tool_path + + # Finally, look on the PATH + if search_PATH: + search_path = os.environ.get("PATH") + if search_path is not None: + tool_path = find_file(tool_name, search_path.split(";")) + if tool_path is not None: + print("Using {} from PATH: {}".format(tool_name, tool_path)) + return tool_path + + raise RuntimeError("Tool " + tool_name + " not found. Have you built the runtime repo and created a Core_Root, or put it on your PATH?") + def determine_superpmi_tool_name(coreclr_args): """ Determine the superpmi tool name based on the OS @@ -1670,9 +1717,7 @@ def determine_superpmi_tool_name(coreclr_args): (str) Name of the superpmi tool to use """ - if coreclr_args.host_os == "OSX": - return "superpmi" - elif coreclr_args.host_os == "Linux": + if coreclr_args.host_os == "OSX" or coreclr_args.host_os == "Linux": return "superpmi" elif coreclr_args.host_os == "Windows_NT": return "superpmi.exe" @@ -1690,16 +1735,94 @@ def determine_superpmi_tool_path(coreclr_args): """ superpmi_tool_name = determine_superpmi_tool_name(coreclr_args) - superpmi_tool_path = os.path.join(coreclr_args.core_root, superpmi_tool_name) - if not os.path.isfile(superpmi_tool_path): - # We couldn't find superpmi in core_root. This is probably fatal. - # However, just in case, check for it on the PATH. - search_path = os.environ.get("PATH") - superpmi_tool_path = find_file(superpmi_tool_name, search_path.split(";")) if search_path is not None else None - if superpmi_tool_path is None: - raise RuntimeError("Superpmi tool not found. Have you built the runtime repo and created a Core_Root?") + return find_tool(coreclr_args, superpmi_tool_name) + +def determine_mcs_tool_name(coreclr_args): + """ Determine the mcs tool name based on the OS + + Args: + coreclr_args (CoreclrArguments): parsed args + + Return: + (str) Name of the mcs tool to use + """ + + if coreclr_args.host_os == "OSX" or coreclr_args.host_os == "Linux": + return "mcs" + elif coreclr_args.host_os == "Windows_NT": + return "mcs.exe" + else: + raise RuntimeError("Unsupported OS.") + +def determine_mcs_tool_path(coreclr_args): + """ Determine the mcs tool full path + + Args: + coreclr_args (CoreclrArguments): parsed args + + Return: + (str) Path of the mcs tool to use + """ + + mcs_tool_name = determine_mcs_tool_name(coreclr_args) + return find_tool(coreclr_args, mcs_tool_name) + +def determine_jit_ee_version(coreclr_args): + """ Determine the JIT-EE version to use, for determining which MCH files to download and use, as follows: + 1. Try to parse it out of the source code. If we can find src\coreclr\src\inc\corinfo.h in the source + tree (and we're already assuming we can find the repo root from the relative path of this script), + then the JIT-EE version lives in corinfo.h as follows: + + constexpr GUID JITEEVersionIdentifier = { /* a5eec3a4-4176-43a7-8c2b-a05b551d4f49 */ + 0xa5eec3a4, + 0x4176, + 0x43a7, + {0x8c, 0x2b, 0xa0, 0x5b, 0x55, 0x1d, 0x4f, 0x49} + }; + + We want the string between the /* */ comments. + 2. Find the mcs tool and run "mcs -printJITEEVersion". + 3. Otherwise, just use "unknown-jit-ee-version", which will probably cause downstream failures. + + NOTE: When using mcs, we need to run the tool. So we need a version that will run. If a user specifies + an "-arch" argument that creates a Core_Root path that won't run, like an arm32 Core_Root on an + x64 machine, this won't work. This could happen if doing "upload" or "list-collections" on + collections from a machine that didn't create the native collections. We should create a "native" + Core_Root and use that in case there are "cross-arch" scenarios. + + Args: + coreclr_args (CoreclrArguments): parsed args + + Return: + (str) The JIT-EE version to use + """ + + corinfo_h_path = os.path.join(coreclr_args.coreclr_dir, "src", "inc", "corinfo.h") + if os.path.isfile(corinfo_h_path): + # The string is near the beginning of the somewhat large file, so just read a line at a time when searching. + with open(corinfo_h_path, 'r') as file_handle: + for line in file_handle: + match_obj = re.search(r'JITEEVersionIdentifier *= *{ */\* *([^ ]*) *\*/', line) + if match_obj is not None: + corinfo_h_jit_ee_version = match_obj.group(1) + print("Using JIT/EE Version from corinfo.h: {}".format(corinfo_h_jit_ee_version)) + return corinfo_h_jit_ee_version + print("Warning: couldn't find JITEEVersionIdentifier in {}; is the file corrupt?".format(corinfo_h_path)) + + mcs_path = determine_mcs_tool_path(coreclr_args) + command = [mcs_path, "-printJITEEVersion"] + proc = subprocess.Popen(command, stdout=subprocess.PIPE) + stdout_jit_ee_version, stderr_output = proc.communicate() + return_code = proc.returncode + if return_code == 0: + mcs_jit_ee_version = stdout_jit_ee_version.decode('utf-8').strip() + print("Using JIT/EE Version from mcs: {}".format(mcs_jit_ee_version)) + return mcs_jit_ee_version - return superpmi_tool_path + # Otherwise, use the default "unknown" version. + default_jit_ee_version = "unknown-jit-ee-version" + print("Using default JIT/EE Version: {}".format(default_jit_ee_version)) + return default_jit_ee_version def print_platform_specific_environment_vars(coreclr_args, var, value): """ Print environment variables as set {}={} or export {}={} @@ -2333,41 +2456,12 @@ def setup_jit_ee_version_arg(jit_ee_version): if jit_ee_version is not None: # The user passed a specific jit_ee_version on the command-line, so use that return jit_ee_version + return determine_jit_ee_version(coreclr_args) - # Try to find the mcs tool, and run "mcs -printJITEEVersion" to find the version. - # NOTE: we need to run this tool. So we need a version that will run. If a user specifies a "-arch" that creates - # a core_root path that won't run, like an arm32 core_root on an x64 machine, this won't work. This could happen - # if doing "upload" or "list-collections" on collections from a machine that didn't create the native collections. - # We should create a "native" core_root and use that in case there are "cross-arch" scenarios. - - # NOTE: there's some code duplication between here and SuperPMICollect::__init__, for finding the mcs path. - - if coreclr_args.host_os == "OSX" or coreclr_args.host_os == "Linux": - mcs_tool_name = "mcs" - elif coreclr_args.host_os == "Windows_NT": - mcs_tool_name = "mcs.exe" - else: - raise RuntimeError("Unsupported OS.") - - mcs_path = os.path.join(coreclr_args.core_root, mcs_tool_name) - if os.path.isfile(mcs_path): - command = [mcs_path, "-printJITEEVersion"] - proc = subprocess.Popen(command, stdout=subprocess.PIPE) - stdout_jit_ee_version, stderr_output = proc.communicate() - return_code = proc.returncode - if return_code == 0: - mcs_jit_ee_version = stdout_jit_ee_version.decode('utf-8').strip() - print("Using JIT/EE Version: {}".format(mcs_jit_ee_version)) - return mcs_jit_ee_version - else: - print("Note: \"{}\" failed".format(" ".join(command))) - else: - print("Note: \"{}\" not found".format(mcs_path)) - - # Otherwise, use the default "unknown" version. - default_jit_ee_version = "unknown-jit-ee-version" - print("Using JIT/EE Version: {}".format(default_jit_ee_version)) - return default_jit_ee_version + def setup_jit_path_arg(jit_path): + if jit_path is not None: + return os.path.abspath(jit_path) + return find_tool(coreclr_args, determine_jit_name(coreclr_args), search_PATH=False) # It doesn't make sense to search PATH for the JIT dll. def verify_superpmi_common_args(): @@ -2512,11 +2606,6 @@ def verify_replay_common_args(): lambda unused: True, "Unable to set use_zapdisable") - coreclr_args.verify(False, # Force it to false. TODO: support altjit collections? - "altjit", - lambda unused: True, - "Unable to set altjit.") - if args.collection_command is None and args.merge_mch_files is not True: assert args.collection_args is None assert args.pmi is True @@ -2535,15 +2624,15 @@ def verify_replay_common_args(): "jit_path", lambda jit_path: os.path.isfile(jit_path), lambda jit_path: "Error: JIT not found at jit_path {}".format(jit_path), - modify_arg=lambda arg: os.path.join(coreclr_args.core_root, determine_jit_name(coreclr_args)) if arg is None else os.path.abspath(arg)) + modify_arg=lambda arg: setup_jit_path_arg(arg)) - standard_location = False + jit_in_product_location = False if coreclr_args.product_location.lower() in coreclr_args.jit_path.lower(): - standard_location = True + jit_in_product_location = True determined_arch = None determined_build_type = None - if standard_location: + if jit_in_product_location: # Get os/arch/flavor directory, e.g. split "F:\gh\runtime\artifacts\bin\coreclr\Windows_NT.x64.Checked" with "F:\gh\runtime\artifacts\bin\coreclr" # yielding # [0]: "" @@ -2565,7 +2654,7 @@ def verify_replay_common_args(): # Make a more intelligent decision about the arch and build type # based on the path of the jit passed - if standard_location and not coreclr_args.build_type in coreclr_args.jit_path: + if jit_in_product_location and coreclr_args.build_type not in coreclr_args.jit_path: coreclr_args.verify(determined_arch.lower(), "arch", lambda unused: True, @@ -2589,8 +2678,8 @@ def verify_replay_common_args(): coreclr_args.verify(args, "diff_jit_path", lambda jit_path: os.path.isfile(jit_path), - "Unable to set diff_jit_path", - modify_arg=lambda arg: os.path.join(coreclr_args.core_root, determine_jit_name(coreclr_args)) if arg is None else os.path.abspath(arg)) + lambda diff_jit_path: "Error: JIT not found at diff_jit_path {}".format(diff_jit_path), + modify_arg=lambda arg: setup_jit_path_arg(arg)) coreclr_args.verify(args, "git_hash", @@ -2643,13 +2732,13 @@ def verify_replay_common_args(): process_base_jit_path_arg(coreclr_args) - standard_location = False + jit_in_product_location = False if coreclr_args.product_location.lower() in coreclr_args.base_jit_path.lower(): - standard_location = True + jit_in_product_location = True determined_arch = None determined_build_type = None - if standard_location: + if jit_in_product_location: # Get os/arch/flavor directory, e.g. split "F:\gh\runtime\artifacts\bin\coreclr\Windows_NT.x64.Checked" with "F:\gh\runtime\artifacts\bin\coreclr" # yielding # [0]: "" @@ -2671,13 +2760,13 @@ def verify_replay_common_args(): # Make a more intelligent decision about the arch and build type # based on the path of the jit passed - if standard_location and not coreclr_args.build_type in coreclr_args.base_jit_path: + if jit_in_product_location and coreclr_args.build_type not in coreclr_args.base_jit_path: coreclr_args.verify(determined_build_type, "build_type", coreclr_args.check_build_type, "Invalid build_type") - if standard_location and not coreclr_args.arch in coreclr_args.base_jit_path: + if jit_in_product_location and coreclr_args.arch not in coreclr_args.base_jit_path: coreclr_args.verify(determined_arch.lower(), "arch", lambda unused: True,