From e0b3fb8309bbbaf034422e914549896a16fe4196 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 8 May 2024 11:43:50 -0400 Subject: [PATCH 1/3] make use of archspec and psutil to get system details and this information will be printed with 'buildtest info' command --- buildtest/cli/info.py | 25 ++++++++++++++++++------- buildtest/tools/cpu.py | 24 ++++++++++++++++++++++++ pyproject.toml | 2 ++ 3 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 buildtest/tools/cpu.py diff --git a/buildtest/cli/info.py b/buildtest/cli/info.py index 93a2a1335..0e9d8ba38 100644 --- a/buildtest/cli/info.py +++ b/buildtest/cli/info.py @@ -11,6 +11,7 @@ console, ) from buildtest.executors.setup import BuildExecutor +from buildtest.tools.cpu import cpuinfo from buildtest.utils.command import BuildTestCommand from buildtest.utils.file import is_dir, is_file @@ -25,6 +26,7 @@ def buildtest_info(configuration, buildtest_system): """ be = BuildExecutor(configuration) + cpu_details = cpuinfo() buildtest_details = [ f"[red]Buildtest Version:[/red] [green]{BUILDTEST_VERSION}[/green]", @@ -35,13 +37,22 @@ def buildtest_info(configuration, buildtest_system): f"[red]Available Executors:[/red] [green]{be.names()}[/green]", ] system_details = [ - f"[red]Python Path:[/red] [green]{buildtest_system.system['python']}[/green]", - f"[red]Python Version:[/red] [green]{buildtest_system.system['pyver']}[/green]", - f"[red]Processor:[/red] [green]{buildtest_system.system['processor']}[/green]", - f"[red]Host:[/red] [green]{buildtest_system.system['host']}[/green]", - f"[red]Machine:[/red] [green]{buildtest_system.system['machine']}[/green]", - f"[red]Operating System:[/red] [green]{buildtest_system.system['os']}[/green]", - f"[red]Module System:[/red] [green]{buildtest_system.system['moduletool']}[/green]", + f"[red]Python Path:[/red] [green]{buildtest_system.system['python']}[/green]", + f"[red]Python Version:[/red] [green]{buildtest_system.system['pyver']}[/green]", + f"[red]Processor:[/red] [green]{buildtest_system.system['processor']}[/green]", + f"[red]Host:[/red] [green]{buildtest_system.system['host']}[/green]", + f"[red]Operating System:[/red] [green]{buildtest_system.system['os']}[/green]", + f"[red]Module System:[/red] [green]{buildtest_system.system['moduletool']}[/green]", + f"[red]Architecture:[/red] [green]{cpu_details['arch']}[/green]", + f"[red]Vendor:[/red] [green]{cpu_details['vendor']}[/green]", + f"[red]Model:[/red] [green]{cpu_details['model']}[/green]", + f"[red]Platform:[/red] [green]{cpu_details['platform']}[/green]", + f"[red]CPU:[/red] [green]{cpu_details['cpu']}[/green]", + f"[red]Virtual CPU:[/red] [green]{cpu_details['vcpu']}[/green]", + f"[red]Virtual Memory Total:[/red] [green]{cpu_details['virtualmemory']['total']}[/green]", + f"[red]Virtual Memory Used:[/red] [green]{cpu_details['virtualmemory']['used']}[/green]", + f"[red]Virtual Memory Available:[/red] [green]{cpu_details['virtualmemory']['available']}[/green]", + f"[red]Virtual Memory Free:[/red] [green]{cpu_details['virtualmemory']['free']}[/green]", ] if is_dir(BUILD_HISTORY_DIR): diff --git a/buildtest/tools/cpu.py b/buildtest/tools/cpu.py new file mode 100644 index 000000000..6402236dd --- /dev/null +++ b/buildtest/tools/cpu.py @@ -0,0 +1,24 @@ +import platform + +import archspec.cpu +import psutil + + +def cpuinfo(): + """Return CPU information using archspec and psutil library""" + + cpu_details = {} + + cpu_details["arch"] = archspec.cpu.host().name + cpu_details["vendor"] = archspec.cpu.host().vendor + cpu_details["model"] = archspec.cpu.brand_string() + cpu_details["platform"] = platform.machine() + cpu_details["cpu"] = psutil.cpu_count(logical=False) + cpu_details["vcpu"] = psutil.cpu_count(logical=True) + cpu_details["virtualmemory"] = {} + cpu_details["virtualmemory"]["used"] = psutil.virtual_memory().used + cpu_details["virtualmemory"]["total"] = psutil.virtual_memory().total + cpu_details["virtualmemory"]["available"] = psutil.virtual_memory().available + cpu_details["virtualmemory"]["free"] = psutil.virtual_memory().free + + return cpu_details diff --git a/pyproject.toml b/pyproject.toml index 4765550fa..5db677404 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ dependencies=[ 'black==24.3.0', 'pyflakes', 'isort', + 'archspec', + 'psutil', ] From 07592d43e8649152123da48c3c99bfebfa425df7 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 8 May 2024 13:06:14 -0400 Subject: [PATCH 2/3] add methods for detecting system topology information --- buildtest/cli/info.py | 10 +- buildtest/tools/cpu.py | 298 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 300 insertions(+), 8 deletions(-) diff --git a/buildtest/cli/info.py b/buildtest/cli/info.py index 0e9d8ba38..93975a2e0 100644 --- a/buildtest/cli/info.py +++ b/buildtest/cli/info.py @@ -49,10 +49,12 @@ def buildtest_info(configuration, buildtest_system): f"[red]Platform:[/red] [green]{cpu_details['platform']}[/green]", f"[red]CPU:[/red] [green]{cpu_details['cpu']}[/green]", f"[red]Virtual CPU:[/red] [green]{cpu_details['vcpu']}[/green]", - f"[red]Virtual Memory Total:[/red] [green]{cpu_details['virtualmemory']['total']}[/green]", - f"[red]Virtual Memory Used:[/red] [green]{cpu_details['virtualmemory']['used']}[/green]", - f"[red]Virtual Memory Available:[/red] [green]{cpu_details['virtualmemory']['available']}[/green]", - f"[red]Virtual Memory Free:[/red] [green]{cpu_details['virtualmemory']['free']}[/green]", + f"[red]Sockets:[/red] [green]{cpu_details['num_sockets']}[/green]", + f"[red]Cores per Socket:[/red] [green]{cpu_details['num_cpus_per_socket']}[/green]", + f"[red]Virtual Memory Total:[/red] [green]{cpu_details['virtualmemory']['total']} MB[/green]", + f"[red]Virtual Memory Used:[/red] [green]{cpu_details['virtualmemory']['used']} MB[/green]", + f"[red]Virtual Memory Available:[/red] [green]{cpu_details['virtualmemory']['available']} MB[/green]", + f"[red]Virtual Memory Free:[/red] [green]{cpu_details['virtualmemory']['free']} MB[/green]", ] if is_dir(BUILD_HISTORY_DIR): diff --git a/buildtest/tools/cpu.py b/buildtest/tools/cpu.py index 6402236dd..b12e751f3 100644 --- a/buildtest/tools/cpu.py +++ b/buildtest/tools/cpu.py @@ -1,9 +1,283 @@ +import contextlib +import glob +import os import platform +import re +import subprocess import archspec.cpu import psutil +def _bits_from_str(mask_s): + """Return the set bits from a string representing a bit array.""" + + bits = [] + mask = int(mask_s, 0) + pos = 0 + while mask: + if mask & 1: + bits.append(pos) + + pos += 1 + mask >>= 1 + + return bits + + +def _str_from_bits(bits): + """Return a string representation of a bit array with ``bits`` set.""" + + ret = 0 + for b in bits: + ret |= 1 << b + + return hex(ret).lower() + + +def _sysfs_topo(): + cache_units = {"K": 1024, "M": 1024 * 1024, "G": 1024 * 1024 * 1024} + cpuinfo = {"topology": {}} + cpu_dirs = glob.glob(r"/sys/devices/system/cpu/cpu[0-9]*") + nodes = glob.glob(r"/sys/devices/system/node/node[0-9]*") + cores = set() + for cpu in cpu_dirs: + core_cpus_path = os.path.join(cpu, "topology/core_cpus") + thread_siblings_path = os.path.join(cpu, "topology/thread_siblings") + if glob.glob(core_cpus_path): + cores_path = core_cpus_path + elif glob.glob(thread_siblings_path): + cores_path = thread_siblings_path + else: + # Information cannot be retrieved + continue + + with contextlib.suppress(IOError): + with open(cores_path) as fp: + core_cpus = fp.read() + core_cpus = re.sub(r"[\s,]", "", core_cpus) + core_cpus = f"0x{core_cpus.lower()}" + cores.add(core_cpus) + + sockets = set() + for cpu in cpu_dirs: + package_cpus_path = os.path.join(cpu, "topology/package_cpus") + core_siblings_path = os.path.join(cpu, "topology/core_siblings") + if glob.glob(package_cpus_path): + sockets_path = package_cpus_path + elif glob.glob(core_siblings_path): + sockets_path = core_siblings_path + else: + # Information cannot be retrieved + continue + + with contextlib.suppress(IOError): + with open(sockets_path) as fp: + package_cpus = fp.read() + package_cpus = re.sub(r"[\s,]", "", package_cpus) + package_cpus = f"0x{package_cpus.lower()}" + sockets.add(package_cpus) + + numa_nodes = [] + for node in nodes: + with contextlib.suppress(IOError): + with open(os.path.join(node, "cpumap")) as fp: + cpumap = fp.read() + cpumap = re.sub(r"[\s,]", "", cpumap) + cpumap = f"0x{cpumap.lower()}" + numa_nodes.append(cpumap) + + numa_nodes.sort() + caches = {} + for cpu in cpu_dirs: + cache_dirs = glob.glob(cpu + r"/cache/index[0-9]*") + for cache in cache_dirs: + cache_level = 0 + cache_size = 0 + cache_linesize = 0 + cache_associativity = 0 + cache_cpuset = "" + with contextlib.suppress(IOError): + with open(os.path.join(cache, "level")) as fp: + cache_level = int(fp.read()) + + with contextlib.suppress(IOError): + # Skip L1 instruction cache + with open(os.path.join(cache, "type")) as fp: + if cache_level == 1 and fp.read() == "Instruction\n": + continue + + with contextlib.suppress(IOError): + with open(os.path.join(cache, "size")) as fp: + cache_size = fp.read() + m = re.match(r"(?P\d+)(?P\S)", cache_size) + if m: + value = int(m.group("val")) + unit = cache_units.get(m.group("unit"), 1) + cache_size = value * unit + + with contextlib.suppress(IOError): + with open(os.path.join(cache, "coherency_line_size")) as fp: + cache_linesize = int(fp.read()) + + # Don't take the associativity directly from + # "ways_of_associativity" file because some archs (ia64, ppc) + # put 0 there when fully-associative, while others (x86) + # put something like -1. + with contextlib.suppress(IOError): + with open(os.path.join(cache, "number_of_sets")) as fp: + cache_number_of_sets = int(fp.read()) + + with open(os.path.join(cache, "physical_line_partition")) as fp: + cache_physical_line_partition = int(fp.read()) + + if ( + cache_linesize + and cache_physical_line_partition + and cache_number_of_sets + ): + cache_associativity = ( + cache_size + // cache_linesize + // cache_physical_line_partition + // cache_number_of_sets + ) + + with contextlib.suppress(IOError): + with open(os.path.join(cache, "shared_cpu_map")) as fp: + cache_cpuset = fp.read() + cache_cpuset = re.sub(r"[\s,]", "", cache_cpuset) + cache_cpuset = f"0x{cache_cpuset.lower()}" + + num_cpus = len(_bits_from_str(cache_cpuset)) + caches.setdefault( + ( + cache_level, + cache_size, + cache_linesize, + cache_associativity, + num_cpus, + ), + set(), + ) + caches[ + (cache_level, cache_size, cache_linesize, cache_associativity, num_cpus) + ].add(cache_cpuset) + + num_cpus = len(cpu_dirs) + num_cores = len(cores) + num_sockets = len(sockets) + num_cpus_per_core = num_cpus // num_cores if num_cores else 0 + num_cpus_per_socket = num_cpus // num_sockets if num_sockets else 0 + + # Fill in the cpuinfo + cpuinfo["num_cpus"] = num_cpus + cpuinfo["num_cpus_per_core"] = num_cpus_per_core + cpuinfo["num_cpus_per_socket"] = num_cpus_per_socket + cpuinfo["num_sockets"] = num_sockets + cpuinfo["topology"]["numa_nodes"] = numa_nodes + cpuinfo["topology"]["sockets"] = sorted(list(sockets)) + cpuinfo["topology"]["cores"] = sorted(list(cores)) + cpuinfo["topology"]["caches"] = [] + for cache_type, cpusets in caches.items(): + (cache_level, cache_size, cache_linesize, cache_associativity, num_cpus) = ( + cache_type + ) + c = { + "type": f"L{cache_level}", + "size": cache_size, + "linesize": cache_linesize, + "associativity": cache_associativity, + "num_cpus": num_cpus, + "cpusets": sorted(list(cpusets)), + } + cpuinfo["topology"]["caches"].append(c) + + return cpuinfo + + +def _sysctl_topo(): + + exec_output = subprocess.run( + ["sysctl hw machdep.cpu"], shell=True, capture_output=True + ) + exec_output = exec_output.stdout.decode() + # extract stdout and convert to string since its in bytes + + cpuinfo = {"topology": {}} + match = re.search(r"hw\.ncpu: (?P\d+)", exec_output) + if match: + num_cpus = int(match.group("num_cpus")) + + match = re.search(r"hw\.physicalcpu: (?P\d+)", exec_output) + if match: + num_cores = int(match.group("num_cores")) + + match = re.search(r"hw\.packages: (?P\d+)", exec_output) + + if match: + num_sockets = int(match.group("num_sockets")) + cpuinfo["num_sockets"] = num_sockets + + match = re.search(r"hw\.cacheconfig:(?P(\s\d+)*)", exec_output) + if match: + cacheconfig = list(map(int, match.group("cacheconfig").split())) + + match = re.search(r"hw\.cachesize:(?P(\s\d+)*)", exec_output) + if match: + cachesize = list(map(int, match.group("cachesize").split())) + + match = re.search(r"hw\.cachelinesize: (?P\d+)", exec_output) + if match: + linesize = int(match.group("linesize")) + + # index 0 is referring to memory + cache_associativity = [0] + for i in range(1, len(cachesize)): + if cachesize[i] == 0: + break + + match = re.search( + rf"machdep\.cpu\.cache\.L{i}_associativity: " rf"(?P\d+)", + exec_output, + ) + assoc = int(match.group("associativity")) if match else 0 + cache_associativity.append(assoc) + + num_cpus_per_socket = num_cpus // num_sockets + num_cpus_per_core = num_cpus // num_cores + + # Fill in the cpuinfo + cpuinfo["num_cpus"] = num_cpus + cpuinfo["num_cpus_per_socket"] = num_cpus_per_socket + cpuinfo["num_cpus_per_core"] = num_cpus_per_core + cpuinfo["topology"]["numa_nodes"] = [_str_from_bits(range(num_cpus))] + cpuinfo["topology"]["sockets"] = [ + _str_from_bits(range(start, start + num_cpus_per_socket)) + for start in range(0, num_cpus, num_cpus_per_socket) + ] + cpuinfo["topology"]["cores"] = [ + _str_from_bits(range(start, start + num_cpus_per_core)) + for start in range(0, num_cpus, num_cpus_per_core) + ] + cpuinfo["topology"]["caches"] = [] + for i in range(1, len(cache_associativity)): + t = { + "type": f"L{i}", + "size": cachesize[i], + "linesize": linesize, + "associativity": cache_associativity[i], + "num_cpus": cacheconfig[i], + "cpusets": [ + _str_from_bits(range(start, start + cacheconfig[i])) + for start in range(0, num_cpus, cacheconfig[i]) + ], + } + cpuinfo["topology"]["caches"].append(t) + + return cpuinfo + + def cpuinfo(): """Return CPU information using archspec and psutil library""" @@ -16,9 +290,25 @@ def cpuinfo(): cpu_details["cpu"] = psutil.cpu_count(logical=False) cpu_details["vcpu"] = psutil.cpu_count(logical=True) cpu_details["virtualmemory"] = {} - cpu_details["virtualmemory"]["used"] = psutil.virtual_memory().used - cpu_details["virtualmemory"]["total"] = psutil.virtual_memory().total - cpu_details["virtualmemory"]["available"] = psutil.virtual_memory().available - cpu_details["virtualmemory"]["free"] = psutil.virtual_memory().free + cpu_details["virtualmemory"]["used"] = round( + psutil.virtual_memory().used / (1024 * 1024), 2 + ) + cpu_details["virtualmemory"]["total"] = round( + psutil.virtual_memory().total / (1024 * 1024), 2 + ) + cpu_details["virtualmemory"]["available"] = round( + psutil.virtual_memory().available / (1024 * 1024), 2 + ) + cpu_details["virtualmemory"]["free"] = round( + psutil.virtual_memory().free / (1024 * 1024), 2 + ) + + # Try first to get information from the filesystem + if os.path.isdir("/sys"): + topology = _sysfs_topo() + cpu_details.update(topology) + else: + topology = _sysctl_topo() + cpu_details.update(topology) return cpu_details From edc95489dbdae34c756d6b0b5ffff0389655d904 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 8 May 2024 13:13:58 -0400 Subject: [PATCH 3/3] remove platform record from system entry and remove from output --- buildtest/cli/info.py | 1 - buildtest/system.py | 1 - 2 files changed, 2 deletions(-) diff --git a/buildtest/cli/info.py b/buildtest/cli/info.py index 93975a2e0..f49342ce5 100644 --- a/buildtest/cli/info.py +++ b/buildtest/cli/info.py @@ -39,7 +39,6 @@ def buildtest_info(configuration, buildtest_system): system_details = [ f"[red]Python Path:[/red] [green]{buildtest_system.system['python']}[/green]", f"[red]Python Version:[/red] [green]{buildtest_system.system['pyver']}[/green]", - f"[red]Processor:[/red] [green]{buildtest_system.system['processor']}[/green]", f"[red]Host:[/red] [green]{buildtest_system.system['host']}[/green]", f"[red]Operating System:[/red] [green]{buildtest_system.system['os']}[/green]", f"[red]Module System:[/red] [green]{buildtest_system.system['moduletool']}[/green]", diff --git a/buildtest/system.py b/buildtest/system.py index 41922d2f4..da81f4628 100644 --- a/buildtest/system.py +++ b/buildtest/system.py @@ -52,7 +52,6 @@ def check(self): self.system["user"] = getpass.getuser() self.system["python"] = os.getenv("BUILDTEST_PYTHON") self.system["pyver"] = platform.python_version() - self.system["processor"] = platform.processor() self.system["host"] = socket.getfqdn() self.system["machine"] = platform.machine()