diff --git a/README.md b/README.md index 49c2eff9..7f2791a9 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,6 @@ probe export --help - `probe_py/probe_py`: Main package to be imported or run. - `probe_py/pyproject.toml`: Definition of main package and dependencies. - `probe_py/tests`: Python unittests, i.e., `from probe_py import foobar; test_foobar()`; Run `just test-py`. - - `probe_py/mypy_stubs`: "Stub" files that tell Mypy how to check untyped library code. Should be added to `$MYPYPATH` by `nix develop`. - `tests`: End-to-end opaque-box tests. They will be run with Pytest, but they will not test Python directly; they should always `subprocess.run(["probe", ...])`. Additionally, some tests have to be manually invoked. - `docs`: Documentation and papers. - `benchmark`: Programs and infrastructure for benchmarking. diff --git a/flake.nix b/flake.nix index 042fe256..5d08d410 100644 --- a/flake.nix +++ b/flake.nix @@ -55,6 +55,18 @@ old-stdenv = pkgs.overrideCC pkgs.stdenv new-clang-old-glibc; in rec { packages = rec { + types-networkx = python.pkgs.buildPythonPackage rec { + pname = "types-networkx"; + version = "3.5.0.20251001"; + src = pkgs.fetchPypi { + pname = "types_networkx"; + inherit version; + sha256 = "8e3c5c491ba5870d75e175751d70ddeac81df43caf2a64bae161e181f5e8ea7a"; + }; + pyproject = true; + nativeBuildInputs = [python.pkgs.setuptools]; + propagatedBuildInputs = [python.pkgs.numpy]; + }; inherit (cli-wrapper-pkgs) cargoArtifacts probe-cli; libprobe = old-stdenv.mkDerivation rec { pname = "libprobe"; @@ -83,13 +95,13 @@ doCheck = true; nativeCheckInputs = [ old-pkgs.criterion - pkgs.include-what-you-use + pkgs.clang pkgs.clang-analyzer pkgs.clang-tools - pkgs.clang pkgs.compiledb pkgs.cppcheck pkgs.cppclean + pkgs.include-what-you-use ]; checkPhase = '' # When a user buidls this WITHOUT build sandbox isolation, the libc files appear to come from somewhere different. @@ -151,28 +163,26 @@ }; propagatedBuildInputs = [ python.pkgs.networkx - python.pkgs.pygraphviz + python.pkgs.numpy python.pkgs.pydot + python.pkgs.pygraphviz python.pkgs.rich - python.pkgs.typer - python.pkgs.xdg-base-dirs python.pkgs.sqlalchemy - python.pkgs.pyyaml - python.pkgs.numpy python.pkgs.tqdm + python.pkgs.typer + python.pkgs.xdg-base-dirs ]; nativeCheckInputs = [ + packages.types-networkx + pkgs.ruff python.pkgs.mypy - python.pkgs.types-pyyaml python.pkgs.types-tqdm - pkgs.ruff ]; checkPhase = '' runHook preCheck #ruff format --check probe_src # TODO: uncomment ruff check . - python -c 'import probe_py' - MYPYPATH=$src/mypy_stubs:$MYPYPATH mypy --strict --package probe_py + mypy --strict --package probe_py runHook postCheck ''; }; @@ -190,11 +200,11 @@ checks = { inherit (cli-wrapper.checks."${system}") + probe-workspace-audit probe-workspace-clippy + probe-workspace-deny probe-workspace-doc probe-workspace-fmt - probe-workspace-audit - probe-workspace-deny probe-workspace-nextest ; fmt-nix = pkgs.stdenv.mkDerivation { @@ -213,18 +223,18 @@ packages.probe (python.withPackages (ps: with ps; [ + packages.probe-py pytest - pytest-timeout pytest-asyncio - packages.probe-py + pytest-timeout ])) pkgs.buildah - pkgs.podman - pkgs.docker + pkgs.clang pkgs.coreutils # so we can `probe record head ...`, etc. + pkgs.docker pkgs.gnumake - pkgs.clang pkgs.nix + pkgs.podman ] ++ pkgs.lib.lists.optional (system != "i686-linux" && system != "armv7l-linux") pkgs.jdk23_headless; buildPhase = '' @@ -244,23 +254,22 @@ probe-python = python.withPackages (pypkgs: [ # probe_py.manual runtime requirements pypkgs.networkx + pypkgs.numpy pypkgs.pydot pypkgs.rich - pypkgs.typer pypkgs.sqlalchemy - pypkgs.xdg-base-dirs - pypkgs.pyyaml - pypkgs.numpy pypkgs.tqdm + pypkgs.typer + pypkgs.xdg-base-dirs # probe_py.manual "dev time" requirements - pypkgs.types-tqdm - pypkgs.types-pyyaml - pypkgs.pytest - pypkgs.pytest-timeout - pypkgs.mypy + packages.types-networkx pypkgs.ipython + pypkgs.mypy + pypkgs.pytest pypkgs.pytest-asyncio + pypkgs.pytest-timeout + pypkgs.types-tqdm # libprobe build time requirement pypkgs.pycparser @@ -276,10 +285,10 @@ shellPackages = [ # Rust tools - pkgs.cargo-deny pkgs.cargo-audit - pkgs.cargo-machete + pkgs.cargo-deny pkgs.cargo-hakari + pkgs.cargo-machete # Replay tools pkgs.buildah @@ -292,14 +301,14 @@ pkgs.clang-analyzer pkgs.clang-tools # must go after clang-analyzer pkgs.cppcheck - pkgs.gnumake pkgs.git + pkgs.gnumake pkgs.include-what-you-use old-pkgs.criterion # unit testing framework # Programs for testing - pkgs.nix pkgs.coreutils + pkgs.nix # For other lints pkgs.alejandra diff --git a/libprobe/src/env.c b/libprobe/src/env.c index 101652ab..d00437cc 100644 --- a/libprobe/src/env.c +++ b/libprobe/src/env.c @@ -158,8 +158,10 @@ char* const* arena_copy_cmdline(struct ArenaDir* arena_dir, result_sized_mem cmd ptr += length + 1; } +#ifndef NDEBUG ptr -= 1; ASSERTF(!*ptr, "'%s'", ptr); +#endif argv_copy[argc] = NULL; return argv_copy; diff --git a/probe_py/mypy_stubs/networkx/__init__.pyi b/probe_py/mypy_stubs/networkx/__init__.pyi deleted file mode 100644 index 12333315..00000000 --- a/probe_py/mypy_stubs/networkx/__init__.pyi +++ /dev/null @@ -1,122 +0,0 @@ -import abc -import typing - -from . import drawing as drawing - -_Node = typing.TypeVar("_Node", bound=typing.Hashable) -_dict: typing.TypeAlias = dict[str, typing.Any] - -class DiGraph(typing.Generic[_Node]): - def add_node(self, node: _Node, **kwargs: typing.Any) -> None: ... - def add_nodes_from(self, nodes: typing.Iterable[_Node], **kwargs: typing.Any) -> None: ... - def remove_node(self, node: _Node) -> None: ... - def has_node(self, node: _Node) -> bool: ... - - def add_edge(self, src: _Node, dst: _Node, **kwargs: typing.Any) -> None: ... - def add_edges_from(self, edges: typing.Iterable[tuple[_Node, _Node]], **kwargs: typing.Any) -> None: ... - def remove_edge(self, src: _Node, dst: _Node) -> None: ... - def has_edge(self, src: _Node, dst: _Node) -> bool: ... - - def successors(self, node: _Node) -> typing.Iterable[_Node]: ... - def predecessors(self, node: _Node) -> typing.Iterable[_Node]: ... - - def in_degree(self, node: _Node) -> int: ... - def out_degree(self, node: _Node) -> int: ... - - def reverse(self) -> DiGraph[_Node]: ... - - def __len__(self) -> int: ... - - # graph.get_edge_data(a, b) is better than graph.edges[a, b] because graph.edges is already overloaded. - def get_edge_data(self, src: _Node, dst: _Node) -> dict[str, typing.Any]: ... - - @typing.overload - def nodes(self) -> NodeView[_Node]: ... - - @typing.overload - def nodes(self, data: typing.Literal[False]) -> NodeView[_Node]: ... - - @typing.overload - def nodes(self, data: typing.Literal[True]) -> NodeDataView[_Node]: ... - - @typing.overload - def edges(self) -> typing.Iterable[tuple[_Node, _Node]]: ... - - @typing.overload - def edges(self, data: typing.Literal[False]) -> typing.Iterable[tuple[_Node, _Node]]: ... - - @typing.overload - def edges(self, data: typing.Literal[True]) -> typing.Iterable[tuple[_Node, _Node, _dict]]: ... - - def copy(self, as_view: bool = ...) -> DiGraph[_Node]: ... - - -class NodeView(typing.Iterable[_Node], typing.Generic[_Node], metaclass=abc.ABCMeta): ... - - -class NodeDataView(typing.Generic[_Node], metaclass=abc.ABCMeta): - def __iter__(self) -> typing.Iterator[tuple[_Node, _dict]]: ... - def __getitem__(self, node: _Node) -> _dict: ... - - -def bfs_edges(G: DiGraph[_Node], source: _Node, reverse: bool = ...) -> typing.Iterator[list[_Node]]: ... - -def bfs_layers(digraph: DiGraph[_Node], source: _Node = ...) -> typing.Iterator[list[_Node]]: ... - - -def dfs_edges(digraph: DiGraph[_Node], source: _Node = ...) -> typing.Iterator[tuple[_Node, _Node]]: ... - - -def dfs_preorder_nodes(digraph: DiGraph[_Node], source: _Node = ...) -> typing.Iterator[_Node]: ... - - -def dfs_postorder_nodes(digraph: DiGraph[_Node], source: _Node = ...) -> typing.Iterator[_Node]: ... - - -def bfs_successors(digraph: DiGraph[_Node], source: _Node) -> typing.Iterator[_Node]: ... - - -def bfs_predecessors(digraph: DiGraph[_Node], source: _Node) -> typing.Iterator[_Node]: ... - - -def topological_sort(digraph: DiGraph[_Node]) -> typing.Iterator[_Node]: ... - -def topological_generations(digraph: DiGraph[_Node]) -> typing.Iterator[set[_Node]]: ... - - -def find_cycle(digraph: DiGraph[_Node]) -> typing.Iterator[tuple[_Node, _Node]]: ... - - -def is_directed_acyclic_graph(digraph: DiGraph[_Node]) -> bool: ... - - -def is_weakly_connected(digraph: DiGraph[_Node]) -> bool: ... - - -def weakly_connected_components(digraph: DiGraph[_Node]) -> typing.Iterator[frozenset[_Node]]: ... - - -def descendants(digraph: DiGraph[_Node], source: _Node) -> typing.Iterable[_Node]: ... - - -_Node2 = typing.TypeVar("_Node2") -def relabel_nodes(digraph: DiGraph[_Node], mapping: typing.Mapping[_Node, _Node2], copy: bool = ...) -> DiGraph[_Node2]: ... - -class NetworkXNoCycle(Exception): ... - - -def transitive_closure( - digraph: DiGraph[_Node], - reflexive: bool = ..., -) -> DiGraph[_Node]: ... - - -def transitive_reduction( - digraph: DiGraph[_Node], -) -> DiGraph[_Node]: ... - - -def quotient_graph( - digraph: DiGraph[_Node], - partition: typing.Callable[[_Node, _Node], bool], -) -> DiGraph[frozenset[_Node]]: ... diff --git a/probe_py/mypy_stubs/networkx/drawing/__init__.pyi b/probe_py/mypy_stubs/networkx/drawing/__init__.pyi deleted file mode 100644 index 4ab0d482..00000000 --- a/probe_py/mypy_stubs/networkx/drawing/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from . import nx_pydot as nx_pydot diff --git a/probe_py/mypy_stubs/networkx/drawing/nx_pydot.pyi b/probe_py/mypy_stubs/networkx/drawing/nx_pydot.pyi deleted file mode 100644 index 2fa9c0cb..00000000 --- a/probe_py/mypy_stubs/networkx/drawing/nx_pydot.pyi +++ /dev/null @@ -1,9 +0,0 @@ -import typing -from .. import DiGraph -import pydot - - -_Node = typing.TypeVar("_Node") - - -def to_pydot(graph: DiGraph[_Node]) -> pydot.Dot: ... diff --git a/probe_py/mypy_stubs/scipy/__init__.pyi b/probe_py/mypy_stubs/scipy/__init__.pyi deleted file mode 100644 index a6b81e3d..00000000 --- a/probe_py/mypy_stubs/scipy/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from . import cluster as cluster diff --git a/probe_py/mypy_stubs/scipy/cluster/__init__.pyi b/probe_py/mypy_stubs/scipy/cluster/__init__.pyi deleted file mode 100644 index 348008bf..00000000 --- a/probe_py/mypy_stubs/scipy/cluster/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from . import hierarchy as hierarchy diff --git a/probe_py/mypy_stubs/scipy/cluster/hierarchy.pyi b/probe_py/mypy_stubs/scipy/cluster/hierarchy.pyi deleted file mode 100644 index df6f0687..00000000 --- a/probe_py/mypy_stubs/scipy/cluster/hierarchy.pyi +++ /dev/null @@ -1,10 +0,0 @@ -import typing - - -_T = typing.TypeVar("_T") - - -class DisjointSet(typing.Generic[_T]): - def __init__(self, elems: typing.Iterable[_T]) -> None: ... - def merge(self, i0: _T, i1: _T) -> bool: ... - def subsets(self) -> typing.Sequence[frozenset[_T]]: ... diff --git a/probe_py/probe_py/dataflow_graph.py b/probe_py/probe_py/dataflow_graph.py index abb79aba..aaaefe4d 100644 --- a/probe_py/probe_py/dataflow_graph.py +++ b/probe_py/probe_py/dataflow_graph.py @@ -301,4 +301,4 @@ def shorten_path(input: pathlib.Path) -> str: data["shape"] = "rectangle" data["id"] = str(hash(node)) for a, b in cycle: - dataflow_graph.edges[a, b]["color"] = "red" # type: ignore + dataflow_graph.edges[a, b]["color"] = "red" diff --git a/probe_py/probe_py/graph_utils.py b/probe_py/probe_py/graph_utils.py index 01241da5..431f5c83 100644 --- a/probe_py/probe_py/graph_utils.py +++ b/probe_py/probe_py/graph_utils.py @@ -86,7 +86,7 @@ def map_nodes( ) -> networkx.DiGraph[_Node2]: dct = {node: function(node) for node in graph.nodes()} assert util.all_unique(dct.values()), util.duplicates(dct.values()) - return networkx.relabel_nodes(graph, dct) + return networkx.relabel_nodes(graph, dct) # type: ignore def serialize_graph( @@ -96,10 +96,8 @@ def serialize_graph( cluster_labels: collections.abc.Mapping[str, str] = {}, ) -> None: if name_mapper is None: - name_mapper = typing.cast( - typing.Callable[[_Node], str], - lambda node: graph.nodes(data=True)[node].get("id", str(node)), - ) + def name_mapper(node: _Node) -> str: + return str(graph.nodes(data=True)[node].get("id", node)) graph2 = map_nodes(name_mapper, graph) pydot_graph = networkx.drawing.nx_pydot.to_pydot(graph2) @@ -376,7 +374,7 @@ def add_edge(self, source: _Node, target: _Node) -> None: def get_faces( - planar_graph: networkx.PlanarEmbedding[_Node], # type: ignore + planar_graph: networkx.PlanarEmbedding[_Node], ) -> frozenset[tuple[_Node, ...]]: faces = set() covered_half_edges = set() diff --git a/probe_py/probe_py/remote_access.py b/probe_py/probe_py/remote_access.py index 9e3be3b6..e0135179 100644 --- a/probe_py/probe_py/remote_access.py +++ b/probe_py/probe_py/remote_access.py @@ -16,7 +16,6 @@ import shlex import subprocess import pathlib -import yaml import typing PROBE_HOME = xdg_base_dirs.xdg_data_home() / "PROBE" @@ -160,7 +159,7 @@ def create_directories_on_remote(remote_home: pathlib.Path, remote: Host, ssh_op mkdir_command.pop() -def get_stat_results_remote(remote: Host, file_path: pathlib.Path, ssh_options: list[str]) -> bytes: +def get_stat_results_remote(remote: Host, file_path: pathlib.Path, ssh_options: list[str]) -> tuple[int, int]: remote_scp_address = remote.get_address() ssh_command = [ "ssh", @@ -169,16 +168,10 @@ def get_stat_results_remote(remote: Host, file_path: pathlib.Path, ssh_options: for option in ssh_options: ssh_command.insert(-1, option) - ssh_command.append(f'stat -c "size: %s\nmode: 0x%f\n" {file_path}') - try: - result = subprocess.run(ssh_command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = result.stdout - stats = yaml.safe_load(output) - except subprocess.CalledProcessError as e: - raise ValueError(f"Error retrieving stat for {file_path}: {e.stderr.decode()}") - - file_size = stats["size"] - return bytes(file_size) + ssh_command.append(f'stat -c "%s\n%f" {file_path}') + result = subprocess.run(ssh_command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + size_str, mode_str = result.stdout.strip().split(b"\n") + return int(size_str), int(mode_str, 16) def generate_random_pid() -> int: min_pid = 1 diff --git a/setup_devshell.sh b/setup_devshell.sh index a20f204f..44b0c504 100644 --- a/setup_devshell.sh +++ b/setup_devshell.sh @@ -35,7 +35,5 @@ export PATH="$PROBE_ROOT/cli-wrapper/target/debug:$PATH" # Add probe_py to the Python path # PYTHONPATH gets consumed by Python tooling # PROBE_PYTHONPATH gets consumed by `probe py` (works in situations where the environment needs a different `PYTHONPATH`) -# MYPYPATH gets consumed by Mypy, which may be slightly different than the PYTHONPATH export PYTHONPATH="$PROBE_ROOT/probe_py/:$PYTHONPATH" -export PROBE_PYTHONPATH=$PYTHONPATH -export MYPYPATH="$PROBE_ROOT/probe_py/mypy_stubs:$PROBE_ROOT/probe_py/:$MYPYPATH" +export PROBE_PYTHONPATH="$PYTHONPATH"