Support rust extensions for PyPy via cpyext

.travis.yml

@@ -23,6 +23,10 @@ matrix:
       python: "3.7"
       # Keep this synced up with build.rs
       env: TRAVIS_RUST_VERSION=nightly-2019-02-07
+      # Tested via anaconda PyPy (since travis's PyPy version is too old)
+    - name: PyPy3.5 7.0
+      python: "3.7"
+      env: FEATURES="pypy" PATH="$PATH:/opt/anaconda/envs/pypy3/bin"
   allow_failures:
     - python: "3.8-dev"
 

README.md

@@ -18,6 +18,9 @@ A comparison with rust-cpython can be found [in the guide](https://pyo3.rs/maste
 
 PyO3 supports Python 3.5 and up. The minimum required rust version is 1.34.0-nightly 2019-02-06.
 
+PyPy is also supported (via cpyext) for Python 3.5 only, targeted PyPy version is 7.0.0.
+Please refer to the guide for installation instruction against PyPy.
+
 You can either write a native Python module in rust or use Python from a Rust binary.
 
 However, on some OSs, you need some additional packages. E.g. if you are on *Ubuntu 18.04*, please run

build.rs

@@ -18,11 +18,18 @@ use version_check::{is_min_date, is_min_version, supports_features};
 const MIN_DATE: &'static str = "2019-02-06";
 const MIN_VERSION: &'static str = "1.34.0-nightly";
 
+#[derive(Debug, Clone, PartialEq)]
+pub enum PythonInterpreterKind {
+    CPython,
+    PyPy,
+}
+
 #[derive(Debug)]
 struct PythonVersion {
     major: u8,
     // minor == None means any minor version will do
     minor: Option<u8>,
+    implementation: PythonInterpreterKind,
 }
 
 impl PartialEq for PythonVersion {
@@ -142,6 +149,7 @@ fn load_cross_compile_info() -> Result<(PythonVersion, HashMap<String, String>,
     let python_version = PythonVersion {
         major,
         minor: Some(minor),
+        implementation: PythonInterpreterKind::CPython,
     };
 
     let config_map = parse_header_defines(python_include_dir.join("pyconfig.h"))?;
@@ -280,17 +288,43 @@ fn run_python_script(interpreter: &str, script: &str) -> Result<String, String>
     Ok(String::from_utf8(out.stdout).unwrap())
 }
 
+fn get_library_link_name(version: &PythonVersion, ld_version: &str) -> String {
+    if cfg!(target_os = "windows") {
+        let minor_or_empty_string = match version.minor {
+            Some(minor) => format!("{}", minor),
+            None => String::new(),
+        };
+        match version.implementation {
+            PythonInterpreterKind::CPython => {
+                format!("python{}{}", version.major, minor_or_empty_string)
+            }
+            PythonInterpreterKind::PyPy => format!("pypy{}-c", version.major),
+        }
+    } else {
+        match version.implementation {
+            PythonInterpreterKind::CPython => format!("python{}", ld_version),
+            PythonInterpreterKind::PyPy => format!("pypy{}-c", version.major),
+        }
+    }
+}
+
 #[cfg(not(target_os = "macos"))]
 #[cfg(not(target_os = "windows"))]
 fn get_rustc_link_lib(
-    _: &PythonVersion,
+    version: &PythonVersion,
     ld_version: &str,
     enable_shared: bool,
 ) -> Result<String, String> {
     if enable_shared {
-        Ok(format!("cargo:rustc-link-lib=python{}", ld_version))
+        Ok(format!(
+            "cargo:rustc-link-lib={}",
+            get_library_link_name(&version, ld_version)
+        ))
     } else {
-        Ok(format!("cargo:rustc-link-lib=static=python{}", ld_version))
+        Ok(format!(
+            "cargo:rustc-link-lib=static={}",
+            get_library_link_name(&version, ld_version)
+        ))
     }
 }
 
@@ -311,42 +345,63 @@ else:
 }
 
 #[cfg(target_os = "macos")]
-fn get_rustc_link_lib(_: &PythonVersion, ld_version: &str, _: bool) -> Result<String, String> {
+fn get_rustc_link_lib(
+    version: &PythonVersion,
+    ld_version: &str,
+    _: bool,
+) -> Result<String, String> {
     // os x can be linked to a framework or static or dynamic, and
     // Py_ENABLE_SHARED is wrong; framework means shared library
     match get_macos_linkmodel().unwrap().as_ref() {
-        "static" => Ok(format!("cargo:rustc-link-lib=static=python{}", ld_version)),
-        "shared" => Ok(format!("cargo:rustc-link-lib=python{}", ld_version)),
-        "framework" => Ok(format!("cargo:rustc-link-lib=python{}", ld_version)),
+        "static" => Ok(format!(
+            "cargo:rustc-link-lib=static={}",
+            get_library_link_name(&version, ld_version)
+        )),
+        "shared" => Ok(format!(
+            "cargo:rustc-link-lib={}",
+            get_library_link_name(&version, ld_version)
+        )),
+        "framework" => Ok(format!(
+            "cargo:rustc-link-lib={}",
+            get_library_link_name(&version, ld_version)
+        )),
         other => Err(format!("unknown linkmodel {}", other)),
     }
 }
 
+#[cfg(target_os = "windows")]
+fn get_rustc_link_lib(
+    version: &PythonVersion,
+    ld_version: &str,
+    _: bool,
+) -> Result<String, String> {
+    // Py_ENABLE_SHARED doesn't seem to be present on windows.
+    Ok(format!(
+        "cargo:rustc-link-lib=pythonXY:{}",
+        get_library_link_name(&version, ld_version)
+    ))
+}
+
 /// Parse string as interpreter version.
-fn get_interpreter_version(line: &str) -> Result<PythonVersion, String> {
+fn get_interpreter_version(line: &str, implementation: &str) -> Result<PythonVersion, String> {
     let version_re = Regex::new(r"\((\d+), (\d+)\)").unwrap();
     match version_re.captures(&line) {
         Some(cap) => Ok(PythonVersion {
             major: cap.get(1).unwrap().as_str().parse().unwrap(),
             minor: Some(cap.get(2).unwrap().as_str().parse().unwrap()),
+            implementation: match implementation {
+                "CPython" => PythonInterpreterKind::CPython,
+                "PyPy" => PythonInterpreterKind::PyPy,
+                _ => panic!(format!(
+                    "Unsupported python implementation `{}`",
+                    implementation
+                )),
+            },
         }),
         None => Err(format!("Unexpected response to version query {}", line)),
     }
 }
 
-#[cfg(target_os = "windows")]
-fn get_rustc_link_lib(version: &PythonVersion, _: &str, _: bool) -> Result<String, String> {
-    // Py_ENABLE_SHARED doesn't seem to be present on windows.
-    Ok(format!(
-        "cargo:rustc-link-lib=pythonXY:python{}{}",
-        version.major,
-        match version.minor {
-            Some(minor) => minor.to_string(),
-            None => "".to_owned(),
-        }
-    ))
-}
-
 /// Locate a suitable python interpreter and extract config from it.
 ///
 /// The following locations are checked in the order listed:
@@ -387,10 +442,17 @@ fn find_interpreter_and_get_config(
     let expected_version = version.unwrap_or(PythonVersion {
         major: 3,
         minor: None,
+        implementation: PythonInterpreterKind::CPython,
     });
 
+    let binary_name = match expected_version.implementation {
+        PythonInterpreterKind::CPython => "python",
+        PythonInterpreterKind::PyPy => "pypy",
+    };
+
     // check default python
-    let interpreter_path = "python";
+    let interpreter_path = binary_name;
+
     let (interpreter_version, lines) = get_config_from_interpreter(interpreter_path)?;
     if expected_version == interpreter_version {
         return Ok((
@@ -430,20 +492,45 @@ fn get_config_from_interpreter(interpreter: &str) -> Result<(PythonVersion, Vec<
     let script = r#"
 import sys
 import sysconfig
+import platform
+
+PYPY = platform.python_implementation() == "PyPy"
 
 print(sys.version_info[0:2])
 print(sysconfig.get_config_var('LIBDIR'))
-print(sysconfig.get_config_var('Py_ENABLE_SHARED'))
+if PYPY:
+    print("1")
+else:
+    print(sysconfig.get_config_var('Py_ENABLE_SHARED'))
 print(sysconfig.get_config_var('LDVERSION') or sysconfig.get_config_var('py_version_short'))
 print(sys.exec_prefix)
+print(platform.python_implementation())
 "#;
     let out = run_python_script(interpreter, script)?;
     let lines: Vec<String> = out.lines().map(|line| line.to_owned()).collect();
-    let interpreter_version = get_interpreter_version(&lines[0])?;
+    let interpreter_version = get_interpreter_version(&lines[0], &lines[5])?;
     Ok((interpreter_version, lines))
 }
 
+fn ensure_python_version_is_supported(version: &PythonVersion) -> Result<(), String> {
+    match (&version.implementation, version.major, version.minor) {
+        (PythonInterpreterKind::PyPy, 2, _) => {
+            Err("PyPy cpyext bindings is only supported for Python3".to_string())
+        }
+        (_, 3, Some(minor)) if minor < PY3_MIN_MINOR => Err(format!(
+            "Python 3 required version is 3.{}, current version is 3.{}",
+            PY3_MIN_MINOR, minor
+        )),
+        _ => Ok(()),
+    }
+}
+
 fn configure(interpreter_version: &PythonVersion, lines: Vec<String>) -> Result<(String), String> {
+    ensure_python_version_is_supported(&interpreter_version).expect(&format!(
+        "Unsupported interpreter {:?}",
+        interpreter_version
+    ));
+
     let libpath: &str = &lines[1];
     let enable_shared: &str = &lines[2];
     let ld_version: &str = &lines[3];
@@ -464,26 +551,28 @@ fn configure(interpreter_version: &PythonVersion, lines: Vec<String>) -> Result<
 
     let mut flags = String::new();
 
+    if interpreter_version.implementation == PythonInterpreterKind::PyPy {
+        println!("cargo:rustc-cfg=PyPy");
+        flags += format!("CFG_PyPy").as_ref();
+    };
+
     if let PythonVersion {
         major: 3,
         minor: some_minor,
+        implementation: _,
     } = interpreter_version
     {
         if env::var_os("CARGO_FEATURE_ABI3").is_some() {
             println!("cargo:rustc-cfg=Py_LIMITED_API");
         }
+
         if let Some(minor) = some_minor {
-            if minor < &PY3_MIN_MINOR {
-                return Err(format!(
-                    "Python 3 required version is 3.{}, current version is 3.{}",
-                    PY3_MIN_MINOR, minor
-                ));
-            }
             for i in 5..(minor + 1) {
                 println!("cargo:rustc-cfg=Py_3_{}", i);
                 flags += format!("CFG_Py_3_{},", i).as_ref();
             }
         }
+        println!("cargo:rustc-cfg=Py_3");
     } else {
         // fail PYTHON_SYS_EXECUTABLE=python2 cargo ...
         return Err("Python 2 is not supported".to_string());
@@ -512,6 +601,7 @@ fn version_from_env() -> Option<PythonVersion> {
                         Some(s) => Some(s.as_str().parse().unwrap()),
                         None => None,
                     },
+                    implementation: PythonInterpreterKind::CPython,
                 });
             }
             None => (),
@@ -589,6 +679,15 @@ fn main() -> Result<(), String> {
         }
     }
 
+    // These flags need to be enabled manually for PyPy, because it does not expose
+    // them in `sysconfig.get_config_vars()`
+    if interpreter_version.implementation == PythonInterpreterKind::PyPy {
+        config_map.insert("WITH_THREAD".to_owned(), "1".to_owned());
+        config_map.insert("Py_USING_UNICODE".to_owned(), "1".to_owned());
+        config_map.insert("Py_UNICODE_SIZE".to_owned(), "4".to_owned());
+        config_map.insert("Py_UNICODE_WIDE".to_owned(), "1".to_owned());
+    }
+
     // WITH_THREAD is always on for 3.7
     if interpreter_version.major == 3 && interpreter_version.minor.unwrap_or(0) >= 7 {
         config_map.insert("WITH_THREAD".to_owned(), "1".to_owned());

ci/travis/cover.sh

@@ -2,6 +2,12 @@
 
 set -ex
 
+### PyPy does not run the test suite ###########################################
+
+if ! [[ $FEATURES == *"pypy"* ]]; then
+  exit 0
+fi
+
 ### Run kcov in parallel #######################################################
 
 rm -f target/debug/pyo3*.d

ci/travis/setup.sh

@@ -11,9 +11,28 @@ if [ "$TRAVIS_JOB_NAME" = "Minimum nightly" ]; then
     rustup component add rustfmt
 fi
 
+### Setup PyPy ################################################################
+
+if [[ $FEATURES == *"pypy"* ]]; then
+    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    /bin/bash Miniconda3-latest-Linux-x86_64.sh -f -b -p /opt/anaconda && \
+    /opt/anaconda/bin/conda install --quiet --yes conda && \
+    /opt/anaconda/bin/conda config --system --add channels conda-forge && \
+    /opt/anaconda/bin/conda init bash && \
+    /opt/anaconda/bin/conda create -n pypy3 pypy3.5 -y && \
+    /opt/anaconda/envs/pypy3/bin/pypy3 -m ensurepip && \
+    /opt/anaconda/envs/pypy3/bin/pypy3 -m pip install setuptools-rust pytest pytest-benchmark tox
+fi
+
 ### Setup python linker flags ##################################################
 
-PYTHON_LIB=$(python -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))")
+if [[ $FEATURES == *"pypy"* ]]; then
+    PYTHON_BINARY="pypy3"
+else
+    PYTHON_BINARY="python"
+fi
+
+PYTHON_LIB=$($PYTHON_BINARY -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))")
 
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PYTHON_LIB:$HOME/rust/lib"
 
@@ -32,4 +51,4 @@ if [ ! -f "$HOME/.cargo/bin/kcov" ]; then
     make
     install src/kcov $HOME/.cargo/bin/kcov
     cd $TRAVIS_BUILD_DIR
-fi
+fi

ci/travis/test.sh

@@ -2,13 +2,25 @@
 set -ex
 
 cargo clean
-cargo test --features "$FEATURES num-complex"
-( cd pyo3-derive-backend; cargo test )
+
+# run `cargo test` only if testing against cpython.
+if ! [[ $FEATURES == *"pypy"* ]]; then
+  cargo test --features "$FEATURES num-complex"
+  ( cd pyo3-derive-backend; cargo test )
+else
+  # check that pypy at least builds
+  PYTHON_SYS_EXECUTABLE="/opt/anaconda/envs/pypy3/bin/pypy3" cargo build;
+fi
+
 if [ "$TRAVIS_JOB_NAME" = "Minimum nightly" ]; then
     cargo fmt --all -- --check
     cargo clippy --features "$FEATURES num-complex"
 fi
 
 for example_dir in examples/*; do
-    tox -c "$example_dir/tox.ini" -e py
+    if [[ $FEATURES == *"pypy"* ]]; then
+        tox -c "$example_dir/tox.ini" -e pypy3
+    else
+        tox -c "$example_dir/tox.ini" -e py
+    fi
 done

examples/rustapi_module/setup.py

@@ -1,4 +1,5 @@
 import sys
+import platform
 
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
@@ -25,6 +26,9 @@ def get_py_version_cfgs():
     for minor in range(py3_min, version[1] + 1):
         out_cfg.append("--cfg=Py_3_%d" % minor)
 
+    if platform.python_implementation() == "PyPy":
+        out_cfg.append("--cfg=PyPy")
+
     return out_cfg