Add cffi package (pyodide#1761)

Author: Hood Chatham

.gitignore

@@ -25,6 +25,6 @@ emsdk/emsdk
 geckodriver.log
 node_modules
 packages/.artifacts
-packages/*/build.log
+packages/*/build.log*
 packages/build-logs
 dist/

conftest.py

@@ -133,6 +133,10 @@ def javascript_setup(self):
             pyodide_checks=False,
         )
 
+    @property
+    def pyodide_loaded(self):
+        return self.run_js("return !!(self.pyodide && self.pyodide.runPython);")
+
     @property
     def logs(self):
         logs = self.run_js("return self.logs;", pyodide_checks=False)
@@ -401,11 +405,11 @@ def pytest_runtest_call(item):
     https://github.com/pytest-dev/pytest/issues/5044
     """
     selenium = None
-    if "selenium" in item._fixtureinfo.argnames:
-        selenium = item.funcargs["selenium"]
-    if "selenium_standalone" in item._fixtureinfo.argnames:
-        selenium = item.funcargs["selenium_standalone"]
-    if selenium:
+    for fixture in item._fixtureinfo.argnames:
+        if fixture.startswith("selenium"):
+            selenium = item.funcargs[fixture]
+            break
+    if selenium and selenium.pyodide_loaded:
         trace_pyproxies = pytest.mark.skip_pyproxy_check.mark not in item.own_markers
         trace_hiwire_refs = (
             trace_pyproxies

packages/Makefile

@@ -1,5 +1,6 @@
 export PYODIDE_ROOT=$(abspath ..)
 PYODIDE_LIBRARIES=$(abspath ./.artifacts)
+CPYTHONROOT=$(PYODIDE_ROOT)/cpython
 include ../Makefile.envs
 
 export NUMPY_LIB=$(PYODIDE_ROOT)/packages/numpy/build/numpy-1.17.5/install/lib/python$(PYMAJOR).$(PYMINOR)/site-packages/numpy/core/lib/

packages/cffi/meta.yaml

@@ -0,0 +1,19 @@
+package:
+  name: cffi
+  version: 1.14.6
+requirements:
+  run:
+    - pycparser
+source:
+  url: https://files.pythonhosted.org/packages/2e/92/87bb61538d7e60da8a7ec247dc048f7671afe17016cd0008b3b710012804/cffi-1.14.6.tar.gz
+  sha256: c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd
+  patches:
+    - patches/libffi-config.patch
+test:
+  imports:
+    - cffi
+about:
+  home: http://cffi.readthedocs.org
+  PyPi: https://pypi.org/project/cffi
+  summary: Foreign Function Interface for Python calling C code.
+  license: MIT

packages/cffi/patches/libffi-config.patch

@@ -0,0 +1,31 @@
+# HG changeset patch
+# User hoodmane
+# Date 1627635284 -7200
+#      Fri Jul 30 10:54:44 2021 +0200
+# Branch patch
+# Node ID 8d1dd7efc05383a1ff7430ec3c64154a0ecd4be1
+# Parent  e0e3f0fc068ab19d2a8f6daf1f89dea6a7912b8e
+Change config settings
+
+diff -r e0e3f0fc068a -r 8d1dd7efc053 c/_cffi_backend.c
+--- a/c/_cffi_backend.c	Thu Jul 22 17:44:04 2021 +0200
++++ b/c/_cffi_backend.c	Fri Jul 30 10:54:44 2021 +0200
+@@ -107,12 +107,12 @@
+ 
+ #else
+ 
+-# define CFFI_CHECK_FFI_CLOSURE_ALLOC 0
+-# define CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE 0
+-# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC 0
+-# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE 0
+-# define CFFI_CHECK_FFI_PREP_CIF_VAR 0
+-# define CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE 0
++# define CFFI_CHECK_FFI_CLOSURE_ALLOC 1
++# define CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE 1
++# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC 1
++# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE 1
++# define CFFI_CHECK_FFI_PREP_CIF_VAR 1
++# define CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE 1
+ 
+ #endif
+ 

packages/cffi/test_cffi.py

@@ -0,0 +1,17 @@
+from pyodide_build.testing import run_in_pyodide
+
+
+@run_in_pyodide(packages=["cffi"])
+def test_blah():
+    from cffi import FFI
+
+    ffi = FFI()
+    ffi.cdef(
+        """int asprintf(char** buf, const char *format, ...);   // copy-pasted from the man page"""
+    )
+    C = ffi.dlopen(None)  # loads the entire C namespace
+    buf = ffi.new("char**")
+    arg1 = ffi.new("char[]", b"wo")
+    arg2 = ffi.new("char[]", b"ld")
+    C.asprintf(buf, b"hello %sr%s", arg1, arg2)
+    assert ffi.string(buf[0]).decode() == "hello world"

packages/cffi_example/meta.yaml

@@ -0,0 +1,12 @@
+package:
+  name: cffi_example
+  version: "0.1"
+requirements:
+  run:
+    - cffi
+source:
+  url: https://github.com/wolever/python-cffi-example/archive/ec14da61ee0de770750ff3781b412a3b25f020eb.zip
+  sha256: 4d7844ff45e5e870dc7996b4cc61636eabd33c32cc8663cf2cd21e3f4f893abc
+test:
+  imports:
+    - cffi_example

packages/cffi_example/test_cffi_example.py

@@ -0,0 +1,35 @@
+import pytest
+from pyodide_build.testing import run_in_pyodide
+
+
+@pytest.mark.parametrize(
+    "pattern,name,flags,expected",
+    [
+        ("foo", "bar", 0, False),
+        ("f*", "foo", 0, True),
+        ("f*bar", "f/bar", 0, True),
+        ("f*bar", "f/bar", "fnmatch.FNM_PATHNAME", False),
+    ],
+)
+def test_fnmatch(selenium_module_scope, pattern, name, flags, expected):
+    selenium = selenium_module_scope
+    selenium.load_package("cffi_example")
+    result = selenium.run(
+        f"""
+        from cffi_example import fnmatch
+        fnmatch.fnmatch({pattern!r}, {name!r}, {flags})
+        """
+    )
+    assert result == expected
+
+
+@run_in_pyodide(packages=["cffi_example"], module_scope=True)
+def test_person():
+    from cffi_example.person import Person
+
+    p = Person("Alex", "Smith", 72)
+    assert p.get_age() == 72
+    assert p.get_full_name() == "Alex Smith"
+
+    p = Person("x" * 100, "y" * 100, 72)
+    assert p.get_full_name() == "x" * 100

packages/libyaml/meta.yaml

@@ -4,6 +4,7 @@ package:
 
 source:
   url: https://github.com/yaml/libyaml/archive/0.2.1.zip
+  sha256: 56070c9d4bf244a8dcc68e04613e5bbce5c8411ed97cdccc1f4b5fb46aebe5a8
 
 build:
   library: true

packages/micropip/test_micropip.py

@@ -26,17 +26,17 @@ def test_install_simple(selenium_standalone_micropip):
     assert (
         selenium.run_js(
             """
-            let result = await pyodide.runPythonAsync(`
+            return await pyodide.runPythonAsync(`
                 import os
                 import micropip
+                from pyodide import to_js
                 # Package 'pyodide-micropip-test' has dependency on 'snowballstemmer'
                 # It is used to test markers support
                 await micropip.install('pyodide-micropip-test')
                 import snowballstemmer
                 stemmer = snowballstemmer.stemmer('english')
-                stemmer.stemWords('go going goes gone'.split())
+                to_js(stemmer.stemWords('go going goes gone'.split()))
             `);
-            return result.toJs();
             """
         )
         == ["go", "go", "goe", "gone"]

packages/pycparser/meta.yaml

@@ -0,0 +1,14 @@
+package:
+  name: pycparser
+  version: "2.20"
+source:
+  url: https://files.pythonhosted.org/packages/0f/86/e19659527668d70be91d0369aeaa055b4eb396b0f387a4f92293a20035bd/pycparser-2.20.tar.gz
+  sha256: 2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0
+test:
+  imports:
+    - pycparser
+about:
+  home: https://github.com/eliben/pycparser
+  PyPi: https://pypi.org/project/pycparser
+  summary: C parser in Python
+  license: BSD

packages/pyyaml/meta.yaml

@@ -4,6 +4,7 @@ package:
 
 source:
   url: https://github.com/yaml/pyyaml/archive/refs/tags/5.4.1.tar.gz
+  sha256: 75f966559c5f262dfc44da0f958cc2aa18953ae5021f2c3657b415c5a370045f
 
   patches:
     - patches/Makefile.patch

pyodide-build/pyodide_build/buildpkg.py

@@ -99,7 +99,7 @@ def download_and_extract(
         srcdir = Path(pkg["source"]["path"])
 
         if not srcdir.is_dir():
-            raise ValueError("'path' must point to a path")
+            raise ValueError(f"path={srcdir} must point to a directory that exists")
 
         if not srcpath.is_dir():
             shutil.copytree(srcdir, srcpath)

pyodide-build/pyodide_build/pywasmcross.py

@@ -327,6 +327,8 @@ def handle_command(line, args, dryrun=False):
         # On Mac, we need to omit some darwin-specific arguments
         if arg in ["-bundle", "-undefined", "dynamic_lookup"]:
             continue
+        if arg == "-lffi":
+            continue
         # The native build is possibly multithreaded, but the emscripten one
         # definitely isn't
         arg = re.sub(r"/python([0-9]\.[0-9]+)m", r"/python\1", arg)

pyodide-build/pyodide_build/testing.py

@@ -29,6 +29,7 @@ def run_in_pyodide(
     _function: Optional[Callable] = None,
     *,
     standalone: bool = False,
+    module_scope: bool = False,
     packages: List[str] = [],
     xfail_browsers: Dict[str, str] = {},
     driver_timeout: Optional[Union[str, int]] = None,
@@ -107,17 +108,20 @@ def inner(selenium):
 
         if standalone:
 
-            def wrapped_standalone(selenium_standalone):
+            def wrapped(selenium_standalone):  # type: ignore
                 inner(selenium_standalone)
 
-            return wrapped_standalone
+        elif module_scope:
+
+            def wrapped(selenium_module_scope):  # type: ignore
+                inner(selenium_module_scope)
 
         else:
 
-            def wrapped(selenium):
+            def wrapped(selenium):  # type: ignore
                 inner(selenium)
 
-            return wrapped
+        return wrapped
 
     if _function is not None:
         return decorator(_function)

src/tests/test_typeconversions.py

@@ -11,6 +11,7 @@
 def test_string_conversion(selenium_module_scope, s):
     with selenium_context_manager(selenium_module_scope) as selenium:
         # careful string escaping here -- hypothesis will fuzz it.
+        # Note: using base 64 encoding would be much more compact...
         sbytes = list(s.encode())
         selenium.run_js(
             f"""
@@ -111,6 +112,8 @@ def test_bigint_conversions(selenium_module_scope, n):
 
 
 # Generate an object of any type
+@pytest.mark.skip_refcount_check
+@pytest.mark.skip_pyproxy_check
 @given(obj=from_type(type).flatmap(from_type))
 @settings(deadline=2000)
 def test_hyp_py2js2py(selenium_module_scope, obj):
@@ -178,6 +181,8 @@ def test_big_integer_py2js2py(selenium):
 
 
 # Generate an object of any type
+@pytest.mark.skip_refcount_check
+@pytest.mark.skip_pyproxy_check
 @given(obj=from_type(type).flatmap(from_type))
 @settings(deadline=2000)
 def test_hyp_tojs_no_crash(selenium_module_scope, obj):
@@ -270,8 +275,6 @@ def test_python2js5(selenium):
     )
 
 
-# @pytest.mark.skip_refcount_check
-# @pytest.mark.skip_pyproxy_check
 def test_python2js_track_proxies(selenium):
     selenium.run_js(
         """