diff --git a/README.md b/README.md index 49c2eff9..7f2791a9 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,6 @@ probe export --help - `probe_py/probe_py`: Main package to be imported or run. - `probe_py/pyproject.toml`: Definition of main package and dependencies. - `probe_py/tests`: Python unittests, i.e., `from probe_py import foobar; test_foobar()`; Run `just test-py`. - - `probe_py/mypy_stubs`: "Stub" files that tell Mypy how to check untyped library code. Should be added to `$MYPYPATH` by `nix develop`. - `tests`: End-to-end opaque-box tests. They will be run with Pytest, but they will not test Python directly; they should always `subprocess.run(["probe", ...])`. Additionally, some tests have to be manually invoked. - `docs`: Documentation and papers. - `benchmark`: Programs and infrastructure for benchmarking. diff --git a/flake.nix b/flake.nix index 042fe256..6b51ad62 100644 --- a/flake.nix +++ b/flake.nix @@ -55,6 +55,18 @@ old-stdenv = pkgs.overrideCC pkgs.stdenv new-clang-old-glibc; in rec { packages = rec { + types-networkx = python.pkgs.buildPythonPackage rec { + pname = "types-networkx"; + version = "3.5.0.20251104"; + src = pkgs.fetchPypi { + pname = "types_networkx"; + inherit version; + sha256 = "0ae55ff562126dcb7dc6bf0b716a2333b69d5ff6fbcb7f768916eb8bd09fc0c9"; + }; + pyproject = true; + nativeBuildInputs = [python.pkgs.setuptools]; + propagatedBuildInputs = [python.pkgs.numpy]; + }; inherit (cli-wrapper-pkgs) cargoArtifacts probe-cli; libprobe = old-stdenv.mkDerivation rec { pname = "libprobe"; @@ -151,20 +163,21 @@ }; propagatedBuildInputs = [ python.pkgs.networkx - python.pkgs.pygraphviz + python.pkgs.numpy python.pkgs.pydot python.pkgs.rich - python.pkgs.typer - python.pkgs.xdg-base-dirs python.pkgs.sqlalchemy - python.pkgs.pyyaml - python.pkgs.numpy python.pkgs.tqdm + python.pkgs.typer + python.pkgs.xdg-base-dirs ]; nativeCheckInputs = [ python.pkgs.mypy - python.pkgs.types-pyyaml + python.pkgs.pytest + python.pkgs.pytest-asyncio + python.pkgs.pytest-timeout python.pkgs.types-tqdm + types-networkx pkgs.ruff ]; checkPhase = '' @@ -172,7 +185,7 @@ #ruff format --check probe_src # TODO: uncomment ruff check . python -c 'import probe_py' - MYPYPATH=$src/mypy_stubs:$MYPYPATH mypy --strict --package probe_py + mypy --strict --package probe_py runHook postCheck ''; }; @@ -190,11 +203,11 @@ checks = { inherit (cli-wrapper.checks."${system}") + probe-workspace-audit probe-workspace-clippy + probe-workspace-deny probe-workspace-doc probe-workspace-fmt - probe-workspace-audit - probe-workspace-deny probe-workspace-nextest ; fmt-nix = pkgs.stdenv.mkDerivation { @@ -213,6 +226,7 @@ packages.probe (python.withPackages (ps: with ps; [ + packages.probe-py pytest pytest-timeout pytest-asyncio @@ -244,23 +258,22 @@ probe-python = python.withPackages (pypkgs: [ # probe_py.manual runtime requirements pypkgs.networkx + pypkgs.numpy pypkgs.pydot pypkgs.rich - pypkgs.typer pypkgs.sqlalchemy - pypkgs.xdg-base-dirs - pypkgs.pyyaml - pypkgs.numpy pypkgs.tqdm + pypkgs.typer + pypkgs.xdg-base-dirs # probe_py.manual "dev time" requirements - pypkgs.types-tqdm - pypkgs.types-pyyaml - pypkgs.pytest - pypkgs.pytest-timeout - pypkgs.mypy pypkgs.ipython + pypkgs.mypy + pypkgs.pytest pypkgs.pytest-asyncio + pypkgs.pytest-timeout + pypkgs.types-tqdm + packages.types-networkx # libprobe build time requirement pypkgs.pycparser @@ -276,10 +289,10 @@ shellPackages = [ # Rust tools - pkgs.cargo-deny pkgs.cargo-audit - pkgs.cargo-machete + pkgs.cargo-deny pkgs.cargo-hakari + pkgs.cargo-machete # Replay tools pkgs.buildah @@ -298,8 +311,8 @@ old-pkgs.criterion # unit testing framework # Programs for testing - pkgs.nix pkgs.coreutils + pkgs.nix # For other lints pkgs.alejandra diff --git a/probe_py/mypy_stubs/networkx/__init__.pyi b/probe_py/mypy_stubs/networkx/__init__.pyi deleted file mode 100644 index 12333315..00000000 --- a/probe_py/mypy_stubs/networkx/__init__.pyi +++ /dev/null @@ -1,122 +0,0 @@ -import abc -import typing - -from . import drawing as drawing - -_Node = typing.TypeVar("_Node", bound=typing.Hashable) -_dict: typing.TypeAlias = dict[str, typing.Any] - -class DiGraph(typing.Generic[_Node]): - def add_node(self, node: _Node, **kwargs: typing.Any) -> None: ... - def add_nodes_from(self, nodes: typing.Iterable[_Node], **kwargs: typing.Any) -> None: ... - def remove_node(self, node: _Node) -> None: ... - def has_node(self, node: _Node) -> bool: ... - - def add_edge(self, src: _Node, dst: _Node, **kwargs: typing.Any) -> None: ... - def add_edges_from(self, edges: typing.Iterable[tuple[_Node, _Node]], **kwargs: typing.Any) -> None: ... - def remove_edge(self, src: _Node, dst: _Node) -> None: ... - def has_edge(self, src: _Node, dst: _Node) -> bool: ... - - def successors(self, node: _Node) -> typing.Iterable[_Node]: ... - def predecessors(self, node: _Node) -> typing.Iterable[_Node]: ... - - def in_degree(self, node: _Node) -> int: ... - def out_degree(self, node: _Node) -> int: ... - - def reverse(self) -> DiGraph[_Node]: ... - - def __len__(self) -> int: ... - - # graph.get_edge_data(a, b) is better than graph.edges[a, b] because graph.edges is already overloaded. - def get_edge_data(self, src: _Node, dst: _Node) -> dict[str, typing.Any]: ... - - @typing.overload - def nodes(self) -> NodeView[_Node]: ... - - @typing.overload - def nodes(self, data: typing.Literal[False]) -> NodeView[_Node]: ... - - @typing.overload - def nodes(self, data: typing.Literal[True]) -> NodeDataView[_Node]: ... - - @typing.overload - def edges(self) -> typing.Iterable[tuple[_Node, _Node]]: ... - - @typing.overload - def edges(self, data: typing.Literal[False]) -> typing.Iterable[tuple[_Node, _Node]]: ... - - @typing.overload - def edges(self, data: typing.Literal[True]) -> typing.Iterable[tuple[_Node, _Node, _dict]]: ... - - def copy(self, as_view: bool = ...) -> DiGraph[_Node]: ... - - -class NodeView(typing.Iterable[_Node], typing.Generic[_Node], metaclass=abc.ABCMeta): ... - - -class NodeDataView(typing.Generic[_Node], metaclass=abc.ABCMeta): - def __iter__(self) -> typing.Iterator[tuple[_Node, _dict]]: ... - def __getitem__(self, node: _Node) -> _dict: ... - - -def bfs_edges(G: DiGraph[_Node], source: _Node, reverse: bool = ...) -> typing.Iterator[list[_Node]]: ... - -def bfs_layers(digraph: DiGraph[_Node], source: _Node = ...) -> typing.Iterator[list[_Node]]: ... - - -def dfs_edges(digraph: DiGraph[_Node], source: _Node = ...) -> typing.Iterator[tuple[_Node, _Node]]: ... - - -def dfs_preorder_nodes(digraph: DiGraph[_Node], source: _Node = ...) -> typing.Iterator[_Node]: ... - - -def dfs_postorder_nodes(digraph: DiGraph[_Node], source: _Node = ...) -> typing.Iterator[_Node]: ... - - -def bfs_successors(digraph: DiGraph[_Node], source: _Node) -> typing.Iterator[_Node]: ... - - -def bfs_predecessors(digraph: DiGraph[_Node], source: _Node) -> typing.Iterator[_Node]: ... - - -def topological_sort(digraph: DiGraph[_Node]) -> typing.Iterator[_Node]: ... - -def topological_generations(digraph: DiGraph[_Node]) -> typing.Iterator[set[_Node]]: ... - - -def find_cycle(digraph: DiGraph[_Node]) -> typing.Iterator[tuple[_Node, _Node]]: ... - - -def is_directed_acyclic_graph(digraph: DiGraph[_Node]) -> bool: ... - - -def is_weakly_connected(digraph: DiGraph[_Node]) -> bool: ... - - -def weakly_connected_components(digraph: DiGraph[_Node]) -> typing.Iterator[frozenset[_Node]]: ... - - -def descendants(digraph: DiGraph[_Node], source: _Node) -> typing.Iterable[_Node]: ... - - -_Node2 = typing.TypeVar("_Node2") -def relabel_nodes(digraph: DiGraph[_Node], mapping: typing.Mapping[_Node, _Node2], copy: bool = ...) -> DiGraph[_Node2]: ... - -class NetworkXNoCycle(Exception): ... - - -def transitive_closure( - digraph: DiGraph[_Node], - reflexive: bool = ..., -) -> DiGraph[_Node]: ... - - -def transitive_reduction( - digraph: DiGraph[_Node], -) -> DiGraph[_Node]: ... - - -def quotient_graph( - digraph: DiGraph[_Node], - partition: typing.Callable[[_Node, _Node], bool], -) -> DiGraph[frozenset[_Node]]: ... diff --git a/probe_py/mypy_stubs/networkx/drawing/__init__.pyi b/probe_py/mypy_stubs/networkx/drawing/__init__.pyi deleted file mode 100644 index 4ab0d482..00000000 --- a/probe_py/mypy_stubs/networkx/drawing/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from . import nx_pydot as nx_pydot diff --git a/probe_py/mypy_stubs/networkx/drawing/nx_pydot.pyi b/probe_py/mypy_stubs/networkx/drawing/nx_pydot.pyi deleted file mode 100644 index 2fa9c0cb..00000000 --- a/probe_py/mypy_stubs/networkx/drawing/nx_pydot.pyi +++ /dev/null @@ -1,9 +0,0 @@ -import typing -from .. import DiGraph -import pydot - - -_Node = typing.TypeVar("_Node") - - -def to_pydot(graph: DiGraph[_Node]) -> pydot.Dot: ... diff --git a/probe_py/mypy_stubs/scipy/__init__.pyi b/probe_py/mypy_stubs/scipy/__init__.pyi deleted file mode 100644 index a6b81e3d..00000000 --- a/probe_py/mypy_stubs/scipy/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from . import cluster as cluster diff --git a/probe_py/mypy_stubs/scipy/cluster/__init__.pyi b/probe_py/mypy_stubs/scipy/cluster/__init__.pyi deleted file mode 100644 index 348008bf..00000000 --- a/probe_py/mypy_stubs/scipy/cluster/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from . import hierarchy as hierarchy diff --git a/probe_py/mypy_stubs/scipy/cluster/hierarchy.pyi b/probe_py/mypy_stubs/scipy/cluster/hierarchy.pyi deleted file mode 100644 index df6f0687..00000000 --- a/probe_py/mypy_stubs/scipy/cluster/hierarchy.pyi +++ /dev/null @@ -1,10 +0,0 @@ -import typing - - -_T = typing.TypeVar("_T") - - -class DisjointSet(typing.Generic[_T]): - def __init__(self, elems: typing.Iterable[_T]) -> None: ... - def merge(self, i0: _T, i1: _T) -> bool: ... - def subsets(self) -> typing.Sequence[frozenset[_T]]: ... diff --git a/probe_py/probe_py/dataflow_graph.py b/probe_py/probe_py/dataflow_graph.py index abb79aba..aaaefe4d 100644 --- a/probe_py/probe_py/dataflow_graph.py +++ b/probe_py/probe_py/dataflow_graph.py @@ -301,4 +301,4 @@ def shorten_path(input: pathlib.Path) -> str: data["shape"] = "rectangle" data["id"] = str(hash(node)) for a, b in cycle: - dataflow_graph.edges[a, b]["color"] = "red" # type: ignore + dataflow_graph.edges[a, b]["color"] = "red" diff --git a/probe_py/probe_py/graph_utils.py b/probe_py/probe_py/graph_utils.py index 01241da5..0017fc13 100644 --- a/probe_py/probe_py/graph_utils.py +++ b/probe_py/probe_py/graph_utils.py @@ -15,14 +15,11 @@ _Node = typing.TypeVar("_Node") -_CoNode = typing.TypeVar("_CoNode", covariant=True) - - @dataclasses.dataclass(frozen=True) -class Segment(typing.Generic[_CoNode]): - dag_tc: ReachabilityOracle[_CoNode] - upper_bound: frozenset[_CoNode] - lower_bound: frozenset[_CoNode] +class Segment(typing.Generic[_Node]): + dag_tc: ReachabilityOracle[_Node] + upper_bound: frozenset[_Node] + lower_bound: frozenset[_Node] def __post_init__(self) -> None: assert self.upper_bound @@ -38,15 +35,15 @@ def __post_init__(self) -> None: assert not unbounded, \ f"{unbounded} in self.lower_bound is not a descendant of any in {self.upper_bound=}" - def nodes(self) -> collections.abc.Iterable[_CoNode]: + def nodes(self) -> collections.abc.Iterable[_Node]: return self.dag_tc.nodes_between(self.upper_bound, self.lower_bound) - def overlaps(self, other: Segment[_CoNode]) -> bool: + def overlaps(self, other: Segment[_Node]) -> bool: assert self.dag_tc is other.dag_tc return bool(frozenset(self.nodes()) & frozenset(other.nodes())) @staticmethod - def union(segments: typing.Sequence[Segment[_CoNode]]) -> Segment[_CoNode]: + def union(segments: typing.Sequence[Segment[_Node]]) -> Segment[_Node]: assert segments dag_tc = segments[0].dag_tc assert all(segment.dag_tc is dag_tc for segment in segments) @@ -86,7 +83,8 @@ def map_nodes( ) -> networkx.DiGraph[_Node2]: dct = {node: function(node) for node in graph.nodes()} assert util.all_unique(dct.values()), util.duplicates(dct.values()) - return networkx.relabel_nodes(graph, dct) + ret = typing.cast("networkx.DiGraph[_Node2]", networkx.relabel_nodes(graph, dct)) + return ret def serialize_graph( @@ -233,7 +231,7 @@ def create(dag: networkx.DiGraph[_Node]) -> ReachabilityOracle[_Node]: @abc.abstractmethod def is_reachable(self, u: _Node, v: _Node) -> bool: - pass + ... @abc.abstractmethod def nodes_between( @@ -321,6 +319,7 @@ def non_ancestors( candidates: collections.abc.Iterable[_Node], lower_bounds: collections.abc.Iterable[_Node], ) -> collections.abc.Iterable[_Node]: + "Return all candidates that are not ancestors of any element in lower_bounds." return frozenset({ candidate for candidate in candidates @@ -335,6 +334,7 @@ def non_descendants( candidates: collections.abc.Iterable[_Node], upper_bounds: collections.abc.Iterable[_Node], ) -> collections.abc.Iterable[_Node]: + "Return all candidates that are not descendent of any element in upper_bounds." return frozenset({ candidate for candidate in candidates @@ -376,7 +376,7 @@ def add_edge(self, source: _Node, target: _Node) -> None: def get_faces( - planar_graph: networkx.PlanarEmbedding[_Node], # type: ignore + planar_graph: networkx.PlanarEmbedding[_Node], ) -> frozenset[tuple[_Node, ...]]: faces = set() covered_half_edges = set() diff --git a/probe_py/probe_py/remote_access.py b/probe_py/probe_py/remote_access.py index 9e3be3b6..e0135179 100644 --- a/probe_py/probe_py/remote_access.py +++ b/probe_py/probe_py/remote_access.py @@ -16,7 +16,6 @@ import shlex import subprocess import pathlib -import yaml import typing PROBE_HOME = xdg_base_dirs.xdg_data_home() / "PROBE" @@ -160,7 +159,7 @@ def create_directories_on_remote(remote_home: pathlib.Path, remote: Host, ssh_op mkdir_command.pop() -def get_stat_results_remote(remote: Host, file_path: pathlib.Path, ssh_options: list[str]) -> bytes: +def get_stat_results_remote(remote: Host, file_path: pathlib.Path, ssh_options: list[str]) -> tuple[int, int]: remote_scp_address = remote.get_address() ssh_command = [ "ssh", @@ -169,16 +168,10 @@ def get_stat_results_remote(remote: Host, file_path: pathlib.Path, ssh_options: for option in ssh_options: ssh_command.insert(-1, option) - ssh_command.append(f'stat -c "size: %s\nmode: 0x%f\n" {file_path}') - try: - result = subprocess.run(ssh_command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = result.stdout - stats = yaml.safe_load(output) - except subprocess.CalledProcessError as e: - raise ValueError(f"Error retrieving stat for {file_path}: {e.stderr.decode()}") - - file_size = stats["size"] - return bytes(file_size) + ssh_command.append(f'stat -c "%s\n%f" {file_path}') + result = subprocess.run(ssh_command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + size_str, mode_str = result.stdout.strip().split(b"\n") + return int(size_str), int(mode_str, 16) def generate_random_pid() -> int: min_pid = 1 diff --git a/setup_devshell.sh b/setup_devshell.sh index a20f204f..91fd047a 100644 --- a/setup_devshell.sh +++ b/setup_devshell.sh @@ -37,5 +37,4 @@ export PATH="$PROBE_ROOT/cli-wrapper/target/debug:$PATH" # PROBE_PYTHONPATH gets consumed by `probe py` (works in situations where the environment needs a different `PYTHONPATH`) # MYPYPATH gets consumed by Mypy, which may be slightly different than the PYTHONPATH export PYTHONPATH="$PROBE_ROOT/probe_py/:$PYTHONPATH" -export PROBE_PYTHONPATH=$PYTHONPATH -export MYPYPATH="$PROBE_ROOT/probe_py/mypy_stubs:$PROBE_ROOT/probe_py/:$MYPYPATH" +export PROBE_PYTHONPATH="$PYTHONPATH"