diff --git a/.appveyor.yml b/.appveyor.yml index 2d37fc7a..360760ac 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,67 +1,35 @@ version: 1.0.{build} image: - Visual Studio 2017 -- Visual Studio 2015 test: off +skip_branch_with_pr: true build: parallel: true platform: -- x64 - x86 environment: matrix: - PYTHON: 36 - CPP: 14 CONFIG: Debug - - PYTHON: 27 - CPP: 14 - CONFIG: Debug - - CONDA: 36 - CPP: latest - CONFIG: Release -matrix: - exclude: - - image: Visual Studio 2015 - platform: x86 - - image: Visual Studio 2015 - CPP: latest - - image: Visual Studio 2017 - CPP: latest - platform: x86 install: - ps: | - if ($env:PLATFORM -eq "x64") { $env:CMAKE_ARCH = "x64" } - if ($env:APPVEYOR_JOB_NAME -like "*Visual Studio 2017*") { - $env:CMAKE_GENERATOR = "Visual Studio 15 2017" - $env:CMAKE_INCLUDE_PATH = "C:\Libraries\boost_1_64_0" - $env:CXXFLAGS = "-permissive-" - } else { - $env:CMAKE_GENERATOR = "Visual Studio 14 2015" - } - if ($env:PYTHON) { - if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } - $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" - pip install --disable-pip-version-check --user --upgrade pip wheel - pip install pytest numpy - } elseif ($env:CONDA) { - if ($env:CONDA -eq "27") { $env:CONDA = "" } - if ($env:PLATFORM -eq "x64") { $env:CONDA = "$env:CONDA-x64" } - $env:PATH = "C:\Miniconda$env:CONDA\;C:\Miniconda$env:CONDA\Scripts\;$env:PATH" - $env:PYTHONHOME = "C:\Miniconda$env:CONDA" - conda install -y -q pytest numpy scipy - } + $env:CMAKE_GENERATOR = "Visual Studio 15 2017" + if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } + $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" + python -W ignore -m pip install --upgrade pip wheel + python -W ignore -m pip install pytest numpy --no-warn-script-location pytest-timeout - ps: | - Start-FileDownload 'http://bitbucket.org/eigen/eigen/get/3.3.3.zip' - 7z x 3.3.3.zip -y > $null - $env:CMAKE_INCLUDE_PATH = "eigen-eigen-67e894c6cd8f;$env:CMAKE_INCLUDE_PATH" + Start-FileDownload 'https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.zip' + 7z x eigen-3.3.7.zip -y > $null + $env:CMAKE_INCLUDE_PATH = "eigen-3.3.7;$env:CMAKE_INCLUDE_PATH" build_script: - cmake -G "%CMAKE_GENERATOR%" -A "%CMAKE_ARCH%" - -DPYBIND11_CPP_STANDARD=/std:c++%CPP% + -DCMAKE_CXX_STANDARD=14 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DCMAKE_SUPPRESS_REGENERATION=1 + . - set MSBuildLogger="C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - cmake --build . --config %CONFIG% --target pytest -- /m /v:m /logger:%MSBuildLogger% - cmake --build . --config %CONFIG% --target cpptest -- /m /v:m /logger:%MSBuildLogger% -- if "%CPP%"=="latest" (cmake --build . --config %CONFIG% --target test_cmake_build -- /m /v:m /logger:%MSBuildLogger%) on_failure: if exist "tests\test_cmake_build" type tests\test_cmake_build\*.log* diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..b477a160 --- /dev/null +++ b/.clang-format @@ -0,0 +1,38 @@ +--- +# See all possible options and defaults with: +# clang-format --style=llvm --dump-config +BasedOnStyle: LLVM +AccessModifierOffset: -4 +AllowShortLambdasOnASingleLine: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: false +BinPackParameters: false +BreakBeforeBinaryOperators: All +BreakConstructorInitializers: BeforeColon +ColumnLimit: 99 +CommentPragmas: 'NOLINT:.*|^ IWYU pragma:' +IncludeBlocks: Regroup +IndentCaseLabels: true +IndentPPDirectives: AfterHash +IndentWidth: 4 +Language: Cpp +SpaceAfterCStyleCast: true +Standard: Cpp11 +StatementMacros: ['PyObject_HEAD'] +TabWidth: 4 +IncludeCategories: + - Regex: '' + Priority: 4 + - Regex: '.*' + Priority: 5 +... diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000..23018386 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,77 @@ +FormatStyle: file + +Checks: | + *bugprone*, + *performance*, + clang-analyzer-optin.cplusplus.VirtualCall, + clang-analyzer-optin.performance.Padding, + cppcoreguidelines-init-variables, + cppcoreguidelines-prefer-member-initializer, + cppcoreguidelines-pro-type-static-cast-downcast, + cppcoreguidelines-slicing, + google-explicit-constructor, + llvm-namespace-comment, + misc-definitions-in-headers, + misc-misplaced-const, + misc-non-copyable-objects, + misc-static-assert, + misc-throw-by-value-catch-by-reference, + misc-uniqueptr-reset-release, + misc-unused-parameters, + modernize-avoid-bind, + modernize-loop-convert, + modernize-make-shared, + modernize-redundant-void-arg, + modernize-replace-auto-ptr, + modernize-replace-disallow-copy-and-assign-macro, + modernize-replace-random-shuffle, + modernize-shrink-to-fit, + modernize-use-auto, + modernize-use-bool-literals, + modernize-use-default-member-init, + modernize-use-emplace, + modernize-use-equals-default, + modernize-use-equals-delete, + modernize-use-noexcept, + modernize-use-nullptr, + modernize-use-override, + modernize-use-using, + readability-avoid-const-params-in-decls, + readability-braces-around-statements, + readability-const-return-type, + readability-container-size-empty, + readability-delete-null-pointer, + readability-else-after-return, + readability-implicit-bool-conversion, + readability-inconsistent-declaration-parameter-name, + readability-make-member-function-const, + readability-misplaced-array-index, + readability-non-const-parameter, + readability-qualified-auto, + readability-redundant-function-ptr-dereference, + readability-redundant-smartptr-get, + readability-redundant-string-cstr, + readability-simplify-subscript-expr, + readability-static-accessed-through-instance, + readability-static-definition-in-anonymous-namespace, + readability-string-compare, + readability-suspicious-call-argument, + readability-uniqueptr-delete-release, + -bugprone-easily-swappable-parameters, + -bugprone-exception-escape, + -bugprone-reserved-identifier, + -bugprone-unused-raii, + +CheckOptions: +- key: modernize-use-equals-default.IgnoreMacros + value: false +- key: performance-for-range-copy.WarnOnAllAutoCopies + value: true +- key: performance-inefficient-string-concatenation.StrictMode + value: true +- key: performance-unnecessary-value-param.AllowedTypes + value: 'exception_ptr$;' +- key: readability-implicit-bool-conversion.AllowPointerConditions + value: true + +HeaderFilterRegex: 'pybind11/.*h' diff --git a/.cmake-format.yaml b/.cmake-format.yaml new file mode 100644 index 00000000..a2a69f3f --- /dev/null +++ b/.cmake-format.yaml @@ -0,0 +1,73 @@ +parse: + additional_commands: + pybind11_add_module: + flags: + - THIN_LTO + - MODULE + - SHARED + - NO_EXTRAS + - EXCLUDE_FROM_ALL + - SYSTEM + +format: + line_width: 99 + tab_size: 2 + + # If an argument group contains more than this many sub-groups + # (parg or kwarg groups) then force it to a vertical layout. + max_subgroups_hwrap: 2 + + # If a positional argument group contains more than this many + # arguments, then force it to a vertical layout. + max_pargs_hwrap: 6 + + # If a cmdline positional group consumes more than this many + # lines without nesting, then invalidate the layout (and nest) + max_rows_cmdline: 2 + separate_ctrl_name_with_space: false + separate_fn_name_with_space: false + dangle_parens: false + + # If the trailing parenthesis must be 'dangled' on its on + # 'line, then align it to this reference: `prefix`: the start' + # 'of the statement, `prefix-indent`: the start of the' + # 'statement, plus one indentation level, `child`: align to' + # the column of the arguments + dangle_align: prefix + # If the statement spelling length (including space and + # parenthesis) is smaller than this amount, then force reject + # nested layouts. + min_prefix_chars: 4 + + # If the statement spelling length (including space and + # parenthesis) is larger than the tab width by more than this + # amount, then force reject un-nested layouts. + max_prefix_chars: 10 + + # If a candidate layout is wrapped horizontally but it exceeds + # this many lines, then reject the layout. + max_lines_hwrap: 2 + + line_ending: unix + + # Format command names consistently as 'lower' or 'upper' case + command_case: canonical + + # Format keywords consistently as 'lower' or 'upper' case + # unchanged is valid too + keyword_case: 'upper' + + # A list of command names which should always be wrapped + always_wrap: [] + + # If true, the argument lists which are known to be sortable + # will be sorted lexicographically + enable_sort: true + + # If true, the parsers may infer whether or not an argument + # list is sortable (without annotation). + autosort: false + +# Causes a few issues - can be solved later, possibly. +markup: + enable_markup: false diff --git a/.codespell-ignore-lines b/.codespell-ignore-lines new file mode 100644 index 00000000..2a01d63e --- /dev/null +++ b/.codespell-ignore-lines @@ -0,0 +1,24 @@ +template + template + auto &this_ = static_cast(*this); + if (load_impl(temp, false)) { + ssize_t nd = 0; + auto trivial = broadcast(buffers, nd, shape); + auto ndim = (size_t) nd; + int nd; + ssize_t ndim() const { return detail::array_proxy(m_ptr)->nd; } + using op = op_impl; +template + template + class_ &def(const detail::op_ &op, const Extra &...extra) { + class_ &def_cast(const detail::op_ &op, const Extra &...extra) { +@pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) +struct IntStruct { + explicit IntStruct(int v) : value(v){}; + ~IntStruct() { value = -value; } + IntStruct(const IntStruct &) = default; + IntStruct &operator=(const IntStruct &) = default; + py::class_(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); })); + py::implicitly_convertible(); + m.def("test", [](int expected, const IntStruct &in) { + [](int expected, const IntStruct &in) { diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d611e149 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +docs/*.svg binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..4e2c6690 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,9 @@ +*.cmake @henryiii +CMakeLists.txt @henryiii +*.yml @henryiii +*.yaml @henryiii +/tools/ @henryiii +/pybind11/ @henryiii +noxfile.py @henryiii +.clang-format @henryiii +.clang-tidy @henryiii diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 00000000..ad797439 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,388 @@ +Thank you for your interest in this project! Please refer to the following +sections on how to contribute code and bug reports. + +### Reporting bugs + +Before submitting a question or bug report, please take a moment of your time +and ensure that your issue isn't already discussed in the project documentation +provided at [pybind11.readthedocs.org][] or in the [issue tracker][]. You can +also check [gitter][] to see if it came up before. + +Assuming that you have identified a previously unknown problem or an important +question, it's essential that you submit a self-contained and minimal piece of +code that reproduces the problem. In other words: no external dependencies, +isolate the function(s) that cause breakage, submit matched and complete C++ +and Python snippets that can be easily compiled and run in isolation; or +ideally make a small PR with a failing test case that can be used as a starting +point. + +## Pull requests + +Contributions are submitted, reviewed, and accepted using GitHub pull requests. +Please refer to [this article][using pull requests] for details and adhere to +the following rules to make the process as smooth as possible: + +* Make a new branch for every feature you're working on. +* Make small and clean pull requests that are easy to review but make sure they + do add value by themselves. +* Add tests for any new functionality and run the test suite (`cmake --build + build --target pytest`) to ensure that no existing features break. +* Please run [`pre-commit`][pre-commit] to check your code matches the + project style. (Note that `gawk` is required.) Use `pre-commit run + --all-files` before committing (or use installed-mode, check pre-commit docs) + to verify your code passes before pushing to save time. +* This project has a strong focus on providing general solutions using a + minimal amount of code, thus small pull requests are greatly preferred. + +### Licensing of contributions + +pybind11 is provided under a BSD-style license that can be found in the +``LICENSE`` file. By using, distributing, or contributing to this project, you +agree to the terms and conditions of this license. + +You are under no obligation whatsoever to provide any bug fixes, patches, or +upgrades to the features, functionality or performance of the source code +("Enhancements") to anyone; however, if you choose to make your Enhancements +available either publicly, or directly to the author of this software, without +imposing a separate written license agreement for such Enhancements, then you +hereby grant the following license: a non-exclusive, royalty-free perpetual +license to install, use, modify, prepare derivative works, incorporate into +other computer software, distribute, and sublicense such enhancements or +derivative works thereof, in binary and source code form. + + +## Development of pybind11 + +### Quick setup + +To setup a quick development environment, use [`nox`](https://nox.thea.codes). +This will allow you to do some common tasks with minimal setup effort, but will +take more time to run and be less flexible than a full development environment. +If you use [`pipx run nox`](https://pipx.pypa.io), you don't even need to +install `nox`. Examples: + +```bash +# List all available sessions +nox -l + +# Run linters +nox -s lint + +# Run tests on Python 3.9 +nox -s tests-3.9 + +# Build and preview docs +nox -s docs -- serve + +# Build SDists and wheels +nox -s build +``` + +### Full setup + +To setup an ideal development environment, run the following commands on a +system with CMake 3.14+: + +```bash +python3 -m venv venv +source venv/bin/activate +pip install -r tests/requirements.txt +cmake -S . -B build -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON +cmake --build build -j4 +``` + +Tips: + +* You can use `virtualenv` (faster, from PyPI) instead of `venv`. +* You can select any name for your environment folder; if it contains "env" it + will be ignored by git. +* If you don't have CMake 3.14+, just add "cmake" to the pip install command. +* You can use `-DPYBIND11_FINDPYTHON=ON` to use FindPython on CMake 3.12+ +* In classic mode, you may need to set `-DPYTHON_EXECUTABLE=/path/to/python`. + FindPython uses `-DPython_ROOT_DIR=/path/to` or + `-DPython_EXECUTABLE=/path/to/python`. + +### Configuration options + +In CMake, configuration options are given with "-D". Options are stored in the +build directory, in the `CMakeCache.txt` file, so they are remembered for each +build directory. Two selections are special - the generator, given with `-G`, +and the compiler, which is selected based on environment variables `CXX` and +similar, or `-DCMAKE_CXX_COMPILER=`. Unlike the others, these cannot be changed +after the initial run. + +The valid options are: + +* `-DCMAKE_BUILD_TYPE`: Release, Debug, MinSizeRel, RelWithDebInfo +* `-DPYBIND11_FINDPYTHON=ON`: Use CMake 3.12+'s FindPython instead of the + classic, deprecated, custom FindPythonLibs +* `-DPYBIND11_NOPYTHON=ON`: Disable all Python searching (disables tests) +* `-DBUILD_TESTING=ON`: Enable the tests +* `-DDOWNLOAD_CATCH=ON`: Download catch to build the C++ tests +* `-DDOWNLOAD_EIGEN=ON`: Download Eigen for the NumPy tests +* `-DPYBIND11_INSTALL=ON/OFF`: Enable the install target (on by default for the + master project) +* `-DUSE_PYTHON_INSTALL_DIR=ON`: Try to install into the python dir + + +
A few standard CMake tricks: (click to expand)

+ +* Use `cmake --build build -v` to see the commands used to build the files. +* Use `cmake build -LH` to list the CMake options with help. +* Use `ccmake` if available to see a curses (terminal) gui, or `cmake-gui` for + a completely graphical interface (not present in the PyPI package). +* Use `cmake --build build -j12` to build with 12 cores (for example). +* Use `-G` and the name of a generator to use something different. `cmake + --help` lists the generators available. + - On Unix, setting `CMAKE_GENERATER=Ninja` in your environment will give + you automatic mulithreading on all your CMake projects! +* Open the `CMakeLists.txt` with QtCreator to generate for that IDE. +* You can use `-DCMAKE_EXPORT_COMPILE_COMMANDS=ON` to generate the `.json` file + that some tools expect. + +

+ + +To run the tests, you can "build" the check target: + +```bash +cmake --build build --target check +``` + +`--target` can be spelled `-t` in CMake 3.15+. You can also run individual +tests with these targets: + +* `pytest`: Python tests only, using the +[pytest](https://docs.pytest.org/en/stable/) framework +* `cpptest`: C++ tests only +* `test_cmake_build`: Install / subdirectory tests + +If you want to build just a subset of tests, use +`-DPYBIND11_TEST_OVERRIDE="test_callbacks;test_pickling"`. If this is +empty, all tests will be built. Tests are specified without an extension if they need both a .py and +.cpp file. + +You may also pass flags to the `pytest` target by editing `tests/pytest.ini` or +by using the `PYTEST_ADDOPTS` environment variable +(see [`pytest` docs](https://docs.pytest.org/en/2.7.3/customize.html#adding-default-options)). As an example: + +```bash +env PYTEST_ADDOPTS="--capture=no --exitfirst" \ + cmake --build build --target pytest +# Or using abbreviated flags +env PYTEST_ADDOPTS="-s -x" cmake --build build --target pytest +``` + +### Formatting + +All formatting is handled by pre-commit. + +Install with brew (macOS) or pip (any OS): + +```bash +# Any OS +python3 -m pip install pre-commit + +# OR macOS with homebrew: +brew install pre-commit +``` + +Then, you can run it on the items you've added to your staging area, or all +files: + +```bash +pre-commit run +# OR +pre-commit run --all-files +``` + +And, if you want to always use it, you can install it as a git hook (hence the +name, pre-commit): + +```bash +pre-commit install +``` + +### Clang-Format + +As of v2.6.2, pybind11 ships with a [`clang-format`][clang-format] +configuration file at the top level of the repo (the filename is +`.clang-format`). Currently, formatting is NOT applied automatically, but +manually using `clang-format` for newly developed files is highly encouraged. +To check if a file needs formatting: + +```bash +clang-format -style=file --dry-run some.cpp +``` + +The output will show things to be fixed, if any. To actually format the file: + +```bash +clang-format -style=file -i some.cpp +``` + +Note that the `-style-file` option searches the parent directories for the +`.clang-format` file, i.e. the commands above can be run in any subdirectory +of the pybind11 repo. + +### Clang-Tidy + +[`clang-tidy`][clang-tidy] performs deeper static code analyses and is +more complex to run, compared to `clang-format`, but support for `clang-tidy` +is built into the pybind11 CMake configuration. To run `clang-tidy`, the +following recipe should work. Run the `docker` command from the top-level +directory inside your pybind11 git clone. Files will be modified in place, +so you can use git to monitor the changes. + +```bash +docker run --rm -v $PWD:/mounted_pybind11 -it silkeh/clang:15-bullseye +apt-get update && apt-get install -y git python3-dev python3-pytest +cmake -S /mounted_pybind11/ -B build -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--use-color" -DDOWNLOAD_EIGEN=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=17 +cmake --build build -j 2 +``` + +You can add `--fix` to the options list if you want. + +### Include what you use + +To run include what you use, install (`brew install include-what-you-use` on +macOS), then run: + +```bash +cmake -S . -B build-iwyu -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=$(which include-what-you-use) +cmake --build build +``` + +The report is sent to stderr; you can pipe it into a file if you wish. + +### Build recipes + +This builds with the Intel compiler (assuming it is in your path, along with a +recent CMake and Python): + +```bash +python3 -m venv venv +. venv/bin/activate +pip install pytest +cmake -S . -B build-intel -DCMAKE_CXX_COMPILER=$(which icpc) -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DPYBIND11_WERROR=ON +``` + +This will test the PGI compilers: + +```bash +docker run --rm -it -v $PWD:/pybind11 nvcr.io/hpc/pgi-compilers:ce +apt-get update && apt-get install -y python3-dev python3-pip python3-pytest +wget -qO- "https://cmake.org/files/v3.18/cmake-3.18.2-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local +cmake -S pybind11/ -B build +cmake --build build +``` + +### Explanation of the SDist/wheel building design + +> These details below are _only_ for packaging the Python sources from git. The +> SDists and wheels created do not have any extra requirements at all and are +> completely normal. + +The main objective of the packaging system is to create SDists (Python's source +distribution packages) and wheels (Python's binary distribution packages) that +include everything that is needed to work with pybind11, and which can be +installed without any additional dependencies. This is more complex than it +appears: in order to support CMake as a first class language even when using +the PyPI package, they must include the _generated_ CMake files (so as not to +require CMake when installing the `pybind11` package itself). They should also +provide the option to install to the "standard" location +(`/include/pybind11` and `/share/cmake/pybind11`) so they are +easy to find with CMake, but this can cause problems if you are not an +environment or using ``pyproject.toml`` requirements. This was solved by having +two packages; the "nice" pybind11 package that stores the includes and CMake +files inside the package, that you get access to via functions in the package, +and a `pybind11-global` package that can be included via `pybind11[global]` if +you want the more invasive but discoverable file locations. + +If you want to install or package the GitHub source, it is best to have Pip 10 +or newer on Windows, macOS, or Linux (manylinux1 compatible, includes most +distributions). You can then build the SDists, or run any procedure that makes +SDists internally, like making wheels or installing. + + +```bash +# Editable development install example +python3 -m pip install -e . +``` + +Since Pip itself does not have an `sdist` command (it does have `wheel` and +`install`), you may want to use the upcoming `build` package: + +```bash +python3 -m pip install build + +# Normal package +python3 -m build -s . + +# Global extra +PYBIND11_GLOBAL_SDIST=1 python3 -m build -s . +``` + +If you want to use the classic "direct" usage of `python setup.py`, you will +need CMake 3.15+ and either `make` or `ninja` preinstalled (possibly via `pip +install cmake ninja`), since directly running Python on `setup.py` cannot pick +up and install `pyproject.toml` requirements. As long as you have those two +things, though, everything works the way you would expect: + +```bash +# Normal package +python3 setup.py sdist + +# Global extra +PYBIND11_GLOBAL_SDIST=1 python3 setup.py sdist +``` + +A detailed explanation of the build procedure design for developers wanting to +work on or maintain the packaging system is as follows: + +#### 1. Building from the source directory + +When you invoke any `setup.py` command from the source directory, including +`pip wheel .` and `pip install .`, you will activate a full source build. This +is made of the following steps: + +1. If the tool is PEP 518 compliant, like Pip 10+, it will create a temporary + virtual environment and install the build requirements (mostly CMake) into + it. (if you are not on Windows, macOS, or a manylinux compliant system, you + can disable this with `--no-build-isolation` as long as you have CMake 3.15+ + installed) +2. The environment variable `PYBIND11_GLOBAL_SDIST` is checked - if it is set + and truthy, this will be make the accessory `pybind11-global` package, + instead of the normal `pybind11` package. This package is used for + installing the files directly to your environment root directory, using + `pybind11[global]`. +2. `setup.py` reads the version from `pybind11/_version.py` and verifies it + matches `includes/pybind11/detail/common.h`. +3. CMake is run with `-DCMAKE_INSTALL_PREIFX=pybind11`. Since the CMake install + procedure uses only relative paths and is identical on all platforms, these + files are valid as long as they stay in the correct relative position to the + includes. `pybind11/share/cmake/pybind11` has the CMake files, and + `pybind11/include` has the includes. The build directory is discarded. +4. Simpler files are placed in the SDist: `tools/setup_*.py.in`, + `tools/pyproject.toml` (`main` or `global`) +5. The package is created by running the setup function in the + `tools/setup_*.py`. `setup_main.py` fills in Python packages, and + `setup_global.py` fills in only the data/header slots. +6. A context manager cleans up the temporary CMake install directory (even if + an error is thrown). + +### 2. Building from SDist + +Since the SDist has the rendered template files in `tools` along with the +includes and CMake files in the correct locations, the builds are completely +trivial and simple. No extra requirements are required. You can even use Pip 9 +if you really want to. + + +[pre-commit]: https://pre-commit.com +[clang-format]: https://clang.llvm.org/docs/ClangFormat.html +[clang-tidy]: https://clang.llvm.org/extra/clang-tidy/ +[pybind11.readthedocs.org]: http://pybind11.readthedocs.org/en/latest +[issue tracker]: https://github.com/pybind/pybind11/issues +[gitter]: https://gitter.im/pybind/Lobby +[using pull requests]: https://help.github.com/articles/using-pull-requests diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..4f1e78f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,61 @@ +name: Bug Report +description: File an issue about a bug +title: "[BUG]: " +labels: [triage] +body: + - type: markdown + attributes: + value: | + Please do your best to make the issue as easy to act on as possible, and only submit here if there is clearly a problem with pybind11 (ask first if unsure). **Note that a reproducer in a PR is much more likely to get immediate attention.** + + - type: checkboxes + id: steps + attributes: + label: Required prerequisites + description: Make sure you've completed the following steps before submitting your issue -- thank you! + options: + - label: Make sure you've read the [documentation](https://pybind11.readthedocs.io). Your issue may be addressed there. + required: true + - label: Search the [issue tracker](https://github.com/pybind/pybind11/issues) and [Discussions](https:/pybind/pybind11/discussions) to verify that this hasn't already been reported. +1 or comment there if it has. + required: true + - label: Consider asking first in the [Gitter chat room](https://gitter.im/pybind/Lobby) or in a [Discussion](https:/pybind/pybind11/discussions/new). + required: false + + - type: input + id: version + attributes: + label: What version (or hash if on master) of pybind11 are you using? + validations: + required: true + + - type: textarea + id: description + attributes: + label: Problem description + placeholder: >- + Provide a short description, state the expected behavior and what + actually happens. Include relevant information like what version of + pybind11 you are using, what system you are on, and any useful commands + / output. + validations: + required: true + + - type: textarea + id: code + attributes: + label: Reproducible example code + placeholder: >- + The code should be minimal, have no external dependencies, isolate the + function(s) that cause breakage. Submit matched and complete C++ and + Python snippets that can be easily compiled and run to diagnose the + issue. — Note that a reproducer in a PR is much more likely to get + immediate attention: failing tests in the pybind11 CI are the best + starting point for working out fixes. + render: text + + - type: input + id: regression + attributes: + label: Is this a regression? Put the last known working version here if it is. + description: Put the last known working version here if this is a regression. + value: Not a regression diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..27f9a804 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Ask a question + url: https://github.com/pybind/pybind11/discussions/new + about: Please ask and answer questions here, or propose new ideas. + - name: Gitter room + url: https://gitter.im/pybind/Lobby + about: A room for discussing pybind11 with an active community diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..2c7d1708 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..abb0d05a --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,8 @@ +docs: +- any: + - 'docs/**/*.rst' + - '!docs/changelog.rst' + - '!docs/upgrade.rst' + +ci: +- '.github/workflows/*.yml' diff --git a/.github/labeler_merged.yml b/.github/labeler_merged.yml new file mode 100644 index 00000000..2374ad42 --- /dev/null +++ b/.github/labeler_merged.yml @@ -0,0 +1,3 @@ +needs changelog: +- all: + - '!docs/changelog.rst' diff --git a/.github/matchers/pylint.json b/.github/matchers/pylint.json new file mode 100644 index 00000000..e3a6bd16 --- /dev/null +++ b/.github/matchers/pylint.json @@ -0,0 +1,32 @@ +{ + "problemMatcher": [ + { + "severity": "warning", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d+): ([A-DF-Z]\\d+): \\033\\[[\\d;]+m([^\\033]+).*$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ], + "owner": "pylint-warning" + }, + { + "severity": "error", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d+): (E\\d+): \\033\\[[\\d;]+m([^\\033]+).*$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ], + "owner": "pylint-error" + } + ] +} diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..54b7f510 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ + +## Description + + + + +## Suggested changelog entry: + + + +```rst + +``` + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c88f0797 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,1159 @@ +name: CI + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + - stable + - v* + +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +env: + PIP_ONLY_BINARY: numpy + FORCE_COLOR: 3 + PYTEST_TIMEOUT: 300 + # For cmake: + VERBOSE: 1 + +jobs: + # This is the "main" test suite, which tests a large number of different + # versions of default compilers and Python versions in GitHub Actions. + standard: + strategy: + fail-fast: false + matrix: + runs-on: [ubuntu-20.04, windows-2022, macos-latest] + python: + - '3.6' + - '3.9' + - '3.10' + - '3.11' + - 'pypy-3.7' + - 'pypy-3.8' + - 'pypy-3.9' + + # Items in here will either be added to the build matrix (if not + # present), or add new keys to an existing matrix element if all the + # existing keys match. + # + # We support an optional key: args, for cmake args + include: + # Just add a key + - runs-on: ubuntu-20.04 + python: '3.6' + args: > + -DPYBIND11_FINDPYTHON=ON + -DCMAKE_CXX_FLAGS="-D_=1" + - runs-on: ubuntu-20.04 + python: 'pypy-3.8' + args: > + -DPYBIND11_FINDPYTHON=ON + - runs-on: windows-2019 + python: '3.6' + args: > + -DPYBIND11_FINDPYTHON=ON + # Inject a couple Windows 2019 runs + - runs-on: windows-2019 + python: '3.9' + + name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}" + runs-on: ${{ matrix.runs-on }} + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Setup Boost (Linux) + # Can't use boost + define _ + if: runner.os == 'Linux' && matrix.python != '3.6' + run: sudo apt-get install libboost-dev + + - name: Setup Boost (macOS) + if: runner.os == 'macOS' + run: brew install boost + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Cache wheels + if: runner.os == 'macOS' + uses: actions/cache@v3 + with: + # This path is specific to macOS - we really only need it for PyPy NumPy wheels + # See https://github.com/actions/cache/blob/master/examples.md#python---pip + # for ways to do this more generally + path: ~/Library/Caches/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ matrix.python }}-x64-${{ hashFiles('tests/requirements.txt') }} + + - name: Prepare env + run: | + python -m pip install -r tests/requirements.txt + + - name: Setup annotations on Linux + if: runner.os == 'Linux' + run: python -m pip install pytest-github-actions-annotate-failures + + # First build - C++11 mode and inplace + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON here. + - name: Configure C++11 ${{ matrix.args }} + run: > + cmake -S . -B . + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=11 + ${{ matrix.args }} + + - name: Build C++11 + run: cmake --build . -j 2 + + - name: Python tests C++11 + run: cmake --build . --target pytest -j 2 + + - name: C++11 tests + # TODO: Figure out how to load the DLL on Python 3.8+ + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" + run: cmake --build . --target cpptest -j 2 + + - name: Interface test C++11 + run: cmake --build . --target test_cmake_build + + - name: Clean directory + run: git clean -fdx + + # Second build - C++17 mode and in a build directory + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here. + - name: Configure C++17 + run: > + cmake -S . -B build2 + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=17 + ${{ matrix.args }} + + - name: Build + run: cmake --build build2 -j 2 + + - name: Python tests + run: cmake --build build2 --target pytest + + - name: C++ tests + # TODO: Figure out how to load the DLL on Python 3.8+ + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" + run: cmake --build build2 --target cpptest + + # Third build - C++17 mode with unstable ABI + - name: Configure (unstable ABI) + run: > + cmake -S . -B build3 + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=17 + -DPYBIND11_INTERNALS_VERSION=10000000 + ${{ matrix.args }} + + - name: Build (unstable ABI) + run: cmake --build build3 -j 2 + + - name: Python tests (unstable ABI) + run: cmake --build build3 --target pytest + + - name: Interface test + run: cmake --build build2 --target test_cmake_build + + # This makes sure the setup_helpers module can build packages using + # setuptools + - name: Setuptools helpers test + run: pytest tests/extra_setuptools + if: "!(matrix.runs-on == 'windows-2022')" + + + deadsnakes: + strategy: + fail-fast: false + matrix: + include: + # TODO: Fails on 3.10, investigate + - python-version: "3.9" + python-debug: true + valgrind: true + - python-version: "3.11" + python-debug: false + + name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python ${{ matrix.python-version }} (deadsnakes) + uses: deadsnakes/action@v3.0.0 + with: + python-version: ${{ matrix.python-version }} + debug: ${{ matrix.python-debug }} + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Valgrind cache + if: matrix.valgrind + uses: actions/cache@v3 + id: cache-valgrind + with: + path: valgrind + key: 3.16.1 # Valgrind version + + - name: Compile Valgrind + if: matrix.valgrind && steps.cache-valgrind.outputs.cache-hit != 'true' + run: | + VALGRIND_VERSION=3.16.1 + curl https://sourceware.org/pub/valgrind/valgrind-$VALGRIND_VERSION.tar.bz2 -o - | tar xj + mv valgrind-$VALGRIND_VERSION valgrind + cd valgrind + ./configure + make -j 2 > /dev/null + + - name: Install Valgrind + if: matrix.valgrind + working-directory: valgrind + run: | + sudo make install + sudo apt-get update + sudo apt-get install libc6-dbg # Needed by Valgrind + + - name: Prepare env + run: | + python -m pip install -r tests/requirements.txt + + - name: Configure + env: + SETUPTOOLS_USE_DISTUTILS: stdlib + run: > + cmake -S . -B build + -DCMAKE_BUILD_TYPE=Debug + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=17 + + - name: Build + run: cmake --build build -j 2 + + - name: Python tests + run: cmake --build build --target pytest + + - name: C++ tests + run: cmake --build build --target cpptest + + - name: Run Valgrind on Python tests + if: matrix.valgrind + run: cmake --build build --target memcheck + + + # Testing on clang using the excellent silkeh clang docker images + clang: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + clang: + - 3.6 + - 3.7 + - 3.9 + - 7 + - 9 + - dev + std: + - 11 + container_suffix: + - "" + include: + - clang: 5 + std: 14 + - clang: 10 + std: 17 + - clang: 11 + std: 20 + - clang: 12 + std: 20 + - clang: 13 + std: 20 + - clang: 14 + std: 20 + - clang: 15 + std: 20 + container_suffix: "-bullseye" + + name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64" + container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" + + steps: + - uses: actions/checkout@v3 + + - name: Add wget and python3 + run: apt-get update && apt-get install -y python3-dev python3-numpy python3-pytest libeigen3-dev + + - name: Configure + shell: bash + run: > + cmake -S . -B build + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build build -j 2 + + - name: Python tests + run: cmake --build build --target pytest + + - name: C++ tests + run: cmake --build build --target cpptest + + - name: Interface test + run: cmake --build build --target test_cmake_build + + + # Testing NVCC; forces sources to behave like .cu files + cuda: + runs-on: ubuntu-latest + name: "🐍 3.10 • CUDA 11.7 • Ubuntu 22.04" + container: nvidia/cuda:11.7.0-devel-ubuntu22.04 + + steps: + - uses: actions/checkout@v3 + + # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND + - name: Install 🐍 3 + run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake git python3-dev python3-pytest python3-numpy + + - name: Configure + run: cmake -S . -B build -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON + + - name: Build + run: cmake --build build -j2 --verbose + + - name: Python tests + run: cmake --build build --target pytest + + +# TODO: Internal compiler error - report to NVidia +# # Testing CentOS 8 + PGI compilers +# centos-nvhpc8: +# runs-on: ubuntu-latest +# name: "🐍 3 • CentOS8 / PGI 20.11 • x64" +# container: centos:8 +# +# steps: +# - uses: actions/checkout@v3 +# +# - name: Add Python 3 and a few requirements +# run: yum update -y && yum install -y git python3-devel python3-numpy python3-pytest make environment-modules +# +# - name: Install CMake with pip +# run: | +# python3 -m pip install --upgrade pip +# python3 -m pip install cmake --prefer-binary +# +# - name: Install NVidia HPC SDK +# run: > +# yum -y install +# https://developer.download.nvidia.com/hpc-sdk/20.11/nvhpc-20-11-20.11-1.x86_64.rpm +# https://developer.download.nvidia.com/hpc-sdk/20.11/nvhpc-2020-20.11-1.x86_64.rpm +# +# - name: Configure +# shell: bash +# run: | +# source /etc/profile.d/modules.sh +# module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.11 +# cmake -S . -B build -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=14 -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") +# +# - name: Build +# run: cmake --build build -j 2 --verbose +# +# - name: Python tests +# run: cmake --build build --target pytest +# +# - name: C++ tests +# run: cmake --build build --target cpptest +# +# - name: Interface test +# run: cmake --build build --target test_cmake_build + + + # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds + centos-nvhpc7: + runs-on: ubuntu-latest + name: "🐍 3 • CentOS7 / PGI 22.9 • x64" + container: centos:7 + + steps: + - uses: actions/checkout@v3 + + - name: Add Python 3 and a few requirements + run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 yum-utils + + - name: Install NVidia HPC SDK + run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.9 + + # On CentOS 7, we have to filter a few tests (compiler internal error) + # and allow deeper template recursion (not needed on CentOS 8 with a newer + # standard library). On some systems, you many need further workarounds: + # https://github.com/pybind/pybind11/pull/2475 + - name: Configure + shell: bash + run: | + source /etc/profile.d/modules.sh + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.9 + cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ + -DCMAKE_CXX_STANDARD=11 \ + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ + -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ + -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp" + + # Building before installing Pip should produce a warning but not an error + - name: Build + run: cmake3 --build build -j 2 --verbose + + - name: Install CMake with pip + run: | + python3 -m pip install --upgrade pip + python3 -m pip install pytest + + - name: Python tests + run: cmake3 --build build --target pytest + + - name: C++ tests + run: cmake3 --build build --target cpptest + + - name: Interface test + run: cmake3 --build build --target test_cmake_build + + + # Testing on GCC using the GCC docker images (only recent images supported) + gcc: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { gcc: 7, std: 11 } + - { gcc: 7, std: 17 } + - { gcc: 8, std: 14 } + - { gcc: 8, std: 17 } + - { gcc: 10, std: 17 } + - { gcc: 11, std: 20 } + - { gcc: 12, std: 20 } + + name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" + container: "gcc:${{ matrix.gcc }}" + + steps: + - uses: actions/checkout@v3 + + - name: Add Python 3 + run: apt-get update; apt-get install -y python3-dev python3-numpy python3-pytest python3-pip libeigen3-dev + + - name: Update pip + run: python3 -m pip install --upgrade pip + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Configure + shell: bash + run: > + cmake -S . -B build + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build build -j 2 + + - name: Python tests + run: cmake --build build --target pytest + + - name: C++ tests + run: cmake --build build --target cpptest + + - name: Interface test + run: cmake --build build --target test_cmake_build + + - name: Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + shell: bash + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial --target pytest + + # Testing on ICC using the oneAPI apt repo + icc: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + + name: "🐍 3 • ICC latest • x64" + + steps: + - uses: actions/checkout@v3 + + - name: Add apt repo + run: | + sudo apt-get update + sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg + wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB + sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB + echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list + + - name: Add ICC & Python 3 + run: sudo apt-get update; sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic cmake python3-dev python3-numpy python3-pytest python3-pip + + - name: Update pip + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + python3 -m pip install --upgrade pip + + - name: Install dependencies + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + python3 -m pip install -r tests/requirements.txt + + - name: Configure C++11 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + cmake -S . -B build-11 \ + -DPYBIND11_WERROR=ON \ + -DDOWNLOAD_CATCH=ON \ + -DDOWNLOAD_EIGEN=OFF \ + -DCMAKE_CXX_STANDARD=11 \ + -DCMAKE_CXX_COMPILER=$(which icpc) \ + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build C++11 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + cmake --build build-11 -j 2 -v + + - name: Python tests C++11 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + sudo service apport stop + cmake --build build-11 --target check + + - name: C++ tests C++11 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + cmake --build build-11 --target cpptest + + - name: Interface test C++11 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + cmake --build build-11 --target test_cmake_build + + - name: Configure C++17 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + cmake -S . -B build-17 \ + -DPYBIND11_WERROR=ON \ + -DDOWNLOAD_CATCH=ON \ + -DDOWNLOAD_EIGEN=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_CXX_COMPILER=$(which icpc) \ + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build C++17 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + cmake --build build-17 -j 2 -v + + - name: Python tests C++17 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + sudo service apport stop + cmake --build build-17 --target check + + - name: C++ tests C++17 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + cmake --build build-17 --target cpptest + + - name: Interface test C++17 + run: | + set +e; source /opt/intel/oneapi/setvars.sh; set -e + cmake --build build-17 --target test_cmake_build + + + # Testing on CentOS (manylinux uses a centos base, and this is an easy way + # to get GCC 4.8, which is the manylinux1 compiler). + centos: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + container: + - "centos:7" # GCC 4.8 + - "almalinux:8" + - "almalinux:9" + + name: "🐍 3 • ${{ matrix.container }} • x64" + container: "${{ matrix.container }}" + + steps: + - uses: actions/checkout@v3 + + - name: Add Python 3 (RHEL 7) + if: matrix.container == 'centos:7' + run: yum update -y && yum install -y python3-devel gcc-c++ make git + + - name: Add Python 3 (RHEL 8+) + if: matrix.container != 'centos:7' + run: dnf update -y && dnf install -y python3-devel gcc-c++ make git + + - name: Update pip + run: python3 -m pip install --upgrade pip + + - name: Install dependencies + run: | + python3 -m pip install cmake -r tests/requirements.txt + + - name: Configure + shell: bash + run: > + cmake -S . -B build + -DCMAKE_BUILD_TYPE=MinSizeRel + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=11 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build build -j 2 + + - name: Python tests + run: cmake --build build --target pytest + + - name: C++ tests + run: cmake --build build --target cpptest + + - name: Interface test + run: cmake --build build --target test_cmake_build + + + # This tests an "install" with the CMake tools + install-classic: + name: "🐍 3.7 • Debian • x86 • Install" + runs-on: ubuntu-latest + container: i386/debian:buster + + steps: + - uses: actions/checkout@v1 # Required to run inside docker + + - name: Install requirements + run: | + apt-get update + apt-get install -y git make cmake g++ libeigen3-dev python3-dev python3-pip + pip3 install "pytest==6.*" + + - name: Configure for install + run: > + cmake . + -DPYBIND11_INSTALL=1 -DPYBIND11_TEST=0 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Make and install + run: make install + + - name: Copy tests to new directory + run: cp -a tests /pybind11-tests + + - name: Make a new test directory + run: mkdir /build-tests + + - name: Configure tests + run: > + cmake ../pybind11-tests + -DDOWNLOAD_CATCH=ON + -DPYBIND11_WERROR=ON + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + working-directory: /build-tests + + - name: Python tests + run: make pytest -j 2 + working-directory: /build-tests + + + # This verifies that the documentation is not horribly broken, and does a + # basic validation check on the SDist. + doxygen: + name: "Documentation build test" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Install Doxygen + run: sudo apt-get install -y doxygen librsvg2-bin # Changed to rsvg-convert in 20.04 + + - name: Build docs + run: pipx run nox -s docs + + - name: Make SDist + run: pipx run nox -s build -- --sdist + + - run: git status --ignored + + - name: Check local include dir + run: > + ls pybind11; + python3 -c "import pybind11, pathlib; assert (a := pybind11.get_include()) == (b := str(pathlib.Path('include').resolve())), f'{a} != {b}'" + + - name: Compare Dists (headers only) + working-directory: include + run: | + python3 -m pip install --user -U ../dist/*.tar.gz + installed=$(python3 -c "import pybind11; print(pybind11.get_include() + '/pybind11')") + diff -rq $installed ./pybind11 + + win32: + strategy: + fail-fast: false + matrix: + python: + - 3.6 + - 3.7 + - 3.8 + - 3.9 + + include: + - python: 3.9 + args: -DCMAKE_CXX_STANDARD=20 + - python: 3.8 + args: -DCMAKE_CXX_STANDARD=17 + - python: 3.7 + args: -DCMAKE_CXX_STANDARD=14 + + + name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}" + runs-on: windows-2019 + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + architecture: x86 + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Prepare MSVC + uses: ilammy/msvc-dev-cmd@v1.12.1 + with: + arch: x86 + + - name: Prepare env + run: | + python -m pip install -r tests/requirements.txt + + # First build - C++11 mode and inplace + - name: Configure ${{ matrix.args }} + run: > + cmake -S . -B build + -G "Visual Studio 16 2019" -A Win32 + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + ${{ matrix.args }} + - name: Build C++11 + run: cmake --build build -j 2 + + - name: Python tests + run: cmake --build build -t pytest + + win32-debug: + strategy: + fail-fast: false + matrix: + python: + - 3.8 + - 3.9 + + include: + - python: 3.9 + args: -DCMAKE_CXX_STANDARD=20 + - python: 3.8 + args: -DCMAKE_CXX_STANDARD=17 + + name: "🐍 ${{ matrix.python }} • MSVC 2019 (Debug) • x86 ${{ matrix.args }}" + runs-on: windows-2019 + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + architecture: x86 + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Prepare MSVC + uses: ilammy/msvc-dev-cmd@v1.12.1 + with: + arch: x86 + + - name: Prepare env + run: | + python -m pip install -r tests/requirements.txt + + # First build - C++11 mode and inplace + - name: Configure ${{ matrix.args }} + run: > + cmake -S . -B build + -G "Visual Studio 16 2019" -A Win32 + -DCMAKE_BUILD_TYPE=Debug + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + ${{ matrix.args }} + - name: Build C++11 + run: cmake --build build --config Debug -j 2 + + - name: Python tests + run: cmake --build build --config Debug -t pytest + + + windows-2022: + strategy: + fail-fast: false + matrix: + python: + - 3.9 + + name: "🐍 ${{ matrix.python }} • MSVC 2022 C++20 • x64" + runs-on: windows-2022 + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Prepare env + run: | + python3 -m pip install -r tests/requirements.txt + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Configure C++20 + run: > + cmake -S . -B build + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=20 + + - name: Build C++20 + run: cmake --build build -j 2 + + - name: Python tests + run: cmake --build build --target pytest + + - name: C++20 tests + run: cmake --build build --target cpptest -j 2 + + - name: Interface test C++20 + run: cmake --build build --target test_cmake_build + + - name: Configure C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=20 + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest + + mingw: + name: "🐍 3 • windows-latest • ${{ matrix.sys }}" + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + strategy: + fail-fast: false + matrix: + include: + - { sys: mingw64, env: x86_64 } + - { sys: mingw32, env: i686 } + steps: + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{matrix.sys}} + install: >- + git + mingw-w64-${{matrix.env}}-gcc + mingw-w64-${{matrix.env}}-python-pip + mingw-w64-${{matrix.env}}-python-numpy + mingw-w64-${{matrix.env}}-python-scipy + mingw-w64-${{matrix.env}}-cmake + mingw-w64-${{matrix.env}}-make + mingw-w64-${{matrix.env}}-python-pytest + mingw-w64-${{matrix.env}}-eigen3 + mingw-w64-${{matrix.env}}-boost + mingw-w64-${{matrix.env}}-catch + + - uses: actions/checkout@v3 + + - name: Configure C++11 + # LTO leads to many undefined reference like + # `pybind11::detail::function_call::function_call(pybind11::detail::function_call&&) + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build + + - name: Build C++11 + run: cmake --build build -j 2 + + - name: Python tests C++11 + run: cmake --build build --target pytest -j 2 + + - name: C++11 tests + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build --target cpptest -j 2 + + - name: Interface test C++11 + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build --target test_cmake_build + + - name: Clean directory + run: git clean -fdx + + - name: Configure C++14 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build2 + + - name: Build C++14 + run: cmake --build build2 -j 2 + + - name: Python tests C++14 + run: cmake --build build2 --target pytest -j 2 + + - name: C++14 tests + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build2 --target cpptest -j 2 + + - name: Interface test C++14 + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build2 --target test_cmake_build + + - name: Clean directory + run: git clean -fdx + + - name: Configure C++17 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build3 + + - name: Build C++17 + run: cmake --build build3 -j 2 + + - name: Python tests C++17 + run: cmake --build build3 --target pytest -j 2 + + - name: C++17 tests + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target cpptest -j 2 + + - name: Interface test C++17 + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target test_cmake_build + + windows_clang: + + strategy: + matrix: + os: [windows-latest] + python: ['3.10'] + + runs-on: "${{ matrix.os }}" + + name: "🐍 ${{ matrix.python }} • ${{ matrix.os }} • clang-latest" + + steps: + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Install ninja-build tool + uses: seanmiddleditch/gha-setup-ninja@v3 + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Show Clang++ version + run: clang++ --version + + - name: Show CMake version + run: cmake --version + + # TODO: WERROR=ON + - name: Configure Clang + run: > + cmake -G Ninja -S . -B . + -DPYBIND11_WERROR=OFF + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: Clean directory + run: git clean -fdx + + macos_brew_install_llvm: + name: "macos-latest • brew install llvm" + runs-on: macos-latest + + env: + # https://apple.stackexchange.com/questions/227026/how-to-install-recent-clang-with-homebrew + LDFLAGS: '-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib' + + steps: + - name: Update PATH + run: echo "/usr/local/opt/llvm/bin" >> $GITHUB_PATH + + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Show Clang++ version before brew install llvm + run: clang++ --version + + - name: brew install llvm + run: brew install llvm + + - name: Show Clang++ version after brew install llvm + run: clang++ --version + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Run pip installs + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r tests/requirements.txt + python3 -m pip install numpy + python3 -m pip install scipy + + - name: Show CMake version + run: cmake --version + + - name: CMake Configure + run: > + cmake -S . -B . + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: CMake Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest -j 2 + + - name: Clean directory + run: git clean -fdx diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml new file mode 100644 index 00000000..b469a69d --- /dev/null +++ b/.github/workflows/configure.yml @@ -0,0 +1,84 @@ +name: Config + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + - stable + - v* + +env: + # For cmake: + VERBOSE: 1 + +jobs: + # This tests various versions of CMake in various combinations, to make sure + # the configure step passes. + cmake: + strategy: + fail-fast: false + matrix: + runs-on: [ubuntu-20.04, macos-latest, windows-latest] + arch: [x64] + cmake: ["3.23"] + + include: + - runs-on: ubuntu-20.04 + arch: x64 + cmake: 3.4 + + - runs-on: macos-latest + arch: x64 + cmake: 3.7 + + - runs-on: windows-2019 + arch: x64 # x86 compilers seem to be missing on 2019 image + cmake: 3.18 + + name: 🐍 3.7 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }} + runs-on: ${{ matrix.runs-on }} + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python 3.7 + uses: actions/setup-python@v4 + with: + python-version: 3.7 + architecture: ${{ matrix.arch }} + + - name: Prepare env + run: python -m pip install -r tests/requirements.txt + + # An action for adding a specific version of CMake: + # https://github.com/jwlawson/actions-setup-cmake + - name: Setup CMake ${{ matrix.cmake }} + uses: jwlawson/actions-setup-cmake@v1.14 + with: + cmake-version: ${{ matrix.cmake }} + + # These steps use a directory with a space in it intentionally + - name: Make build directories + run: mkdir "build dir" + + - name: Configure + working-directory: build dir + shell: bash + run: > + cmake .. + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + + # Only build and test if this was manually triggered in the GitHub UI + - name: Build + working-directory: build dir + if: github.event_name == 'workflow_dispatch' + run: cmake --build . --config Release + + - name: Test + working-directory: build dir + if: github.event_name == 'workflow_dispatch' + run: cmake --build . --config Release --target check diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 00000000..46489feb --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,57 @@ +# This is a format job. Pre-commit has a first-party GitHub action, so we use +# that: https://github.com/pre-commit/action + +name: Format + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + - stable + - "v*" + +env: + FORCE_COLOR: 3 + # For cmake: + VERBOSE: 1 + +jobs: + pre-commit: + name: Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Add matchers + run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/matchers/pylint.json" + - uses: pre-commit/action@v3.0.0 + with: + # Slow hooks are marked with manual - slow is okay here, run them too + extra_args: --hook-stage manual --all-files + + clang-tidy: + # When making changes here, please also review the "Clang-Tidy" section + # in .github/CONTRIBUTING.md and update as needed. + name: Clang-Tidy + runs-on: ubuntu-latest + container: silkeh/clang:15-bullseye + steps: + - uses: actions/checkout@v3 + + - name: Install requirements + run: apt-get update && apt-get install -y git python3-dev python3-pytest + + - name: Configure + run: > + cmake -S . -B build + -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--use-color;--warnings-as-errors=*" + -DDOWNLOAD_EIGEN=ON + -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=17 + + - name: Build + run: cmake --build build -j 2 -- --keep-going diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 00000000..165a2fd8 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,20 @@ +name: Labeler +on: + pull_request_target: + types: [closed] + +jobs: + label: + name: Labeler + runs-on: ubuntu-latest + steps: + + - uses: actions/labeler@main + if: > + github.event.pull_request.merged == true && + !startsWith(github.event.pull_request.title, 'chore(deps):') && + !startsWith(github.event.pull_request.title, 'ci(fix):') && + !startsWith(github.event.pull_request.title, 'docs(changelog):') + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + configuration-path: .github/labeler_merged.yml diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml new file mode 100644 index 00000000..7f686b13 --- /dev/null +++ b/.github/workflows/pip.yml @@ -0,0 +1,110 @@ +name: Pip + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + - stable + - v* + release: + types: + - published + +env: + PIP_ONLY_BINARY: numpy + +jobs: + # This builds the sdists and wheels and makes sure the files are exactly as + # expected. Using Windows and Python 3.6, since that is often the most + # challenging matrix element. + test-packaging: + name: 🐍 3.6 • 📦 tests • windows-latest + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + + - name: Setup 🐍 3.6 + uses: actions/setup-python@v4 + with: + python-version: 3.6 + + - name: Prepare env + run: | + python -m pip install -r tests/requirements.txt + + - name: Python Packaging tests + run: pytest tests/extra_python_package/ + + + # This runs the packaging tests and also builds and saves the packages as + # artifacts. + packaging: + name: 🐍 3.8 • 📦 & 📦 tests • ubuntu-latest + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Setup 🐍 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Prepare env + run: | + python -m pip install -r tests/requirements.txt build twine + + - name: Python Packaging tests + run: pytest tests/extra_python_package/ + + - name: Build SDist and wheels + run: | + python -m build + PYBIND11_GLOBAL_SDIST=1 python -m build + + - name: Check metadata + run: twine check dist/* + + - name: Save standard package + uses: actions/upload-artifact@v3 + with: + name: standard + path: dist/pybind11-* + + - name: Save global package + uses: actions/upload-artifact@v3 + with: + name: global + path: dist/pybind11_global-* + + + + # When a GitHub release is made, upload the artifacts to PyPI + upload: + name: Upload to PyPI + runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' + needs: [packaging] + + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + + # Downloads all to directories matching the artifact names + - uses: actions/download-artifact@v3 + + - name: Publish standard package + uses: pypa/gh-action-pypi-publish@v1.8.5 + with: + password: ${{ secrets.pypi_password }} + packages-dir: standard/ + + - name: Publish global package + uses: pypa/gh-action-pypi-publish@v1.8.5 + with: + password: ${{ secrets.pypi_password_global }} + packages-dir: global/ diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml new file mode 100644 index 00000000..be3cd405 --- /dev/null +++ b/.github/workflows/upstream.yml @@ -0,0 +1,112 @@ + +name: Upstream + +on: + workflow_dispatch: + pull_request: + +concurrency: + group: upstream-${{ github.ref }} + cancel-in-progress: true + +env: + PIP_ONLY_BINARY: ":all:" + # For cmake: + VERBOSE: 1 + +jobs: + standard: + name: "🐍 3.12 latest • ubuntu-latest • x64" + runs-on: ubuntu-latest + # Only runs when the 'python dev' label is selected + if: "contains(github.event.pull_request.labels.*.name, 'python dev')" + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: "3.12-dev" + + - name: Setup Boost + run: sudo apt-get install libboost-dev + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Show platform info + run: | + python -m platform + cmake --version + pip list + + # First build - C++11 mode and inplace + - name: Configure C++11 + run: > + cmake -S . -B build11 + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=11 + -DCMAKE_BUILD_TYPE=Debug + + - name: Build C++11 + run: cmake --build build11 -j 2 + + - name: Python tests C++11 + run: cmake --build build11 --target pytest -j 2 + + # - name: C++11 tests + # run: cmake --build build11 --target cpptest -j 2 + + - name: Interface test C++11 + run: cmake --build build11 --target test_cmake_build + + # Second build - C++17 mode and in a build directory + - name: Configure C++17 + run: > + cmake -S . -B build17 + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=17 + + - name: Build + run: cmake --build build17 -j 2 + + - name: Python tests + run: cmake --build build17 --target pytest + + # - name: C++ tests + # run: cmake --build build17 --target cpptest + + # Third build - C++17 mode with unstable ABI + - name: Configure (unstable ABI) + run: > + cmake -S . -B build17max + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=17 + -DPYBIND11_INTERNALS_VERSION=10000000 + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build (unstable ABI) + run: cmake --build build17max -j 2 + + - name: Python tests (unstable ABI) + run: cmake --build build17max --target pytest + + - name: Interface test + run: cmake --build build17max --target test_cmake_build + + # This makes sure the setup_helpers module can build packages using + # setuptools + - name: Setuptools helpers test + run: pytest tests/extra_setuptools diff --git a/.gitignore b/.gitignore index c444c17e..43d5094c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ CMakeCache.txt CMakeFiles Makefile cmake_install.cmake +cmake_uninstall.cmake .DS_Store *.so *.pyd @@ -10,6 +11,7 @@ cmake_install.cmake *.sdf *.opensdf *.vcxproj +*.vcxproj.user *.filters example.dir Win32 @@ -27,11 +29,18 @@ MANIFEST *.py[co] *.egg-info *~ +.*.swp .DS_Store /dist -/build -/cmake/ +/*build* .cache/ sosize-*.txt pybind11Config*.cmake pybind11Targets.cmake +/*env* +/.vscode +/pybind11/include/* +/pybind11/share/* +/docs/_build/* +.ipynb_checkpoints/ +tests/main.cpp diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 5191885e..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "tools/clang"] - path = tools/clang - url = https://github.com/wjakob/clang-cindex-python3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..7c431619 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,149 @@ +# To use: +# +# pre-commit run -a +# +# Or: +# +# pre-commit install # (runs every time you commit in git) +# +# To update this file: +# +# pre-commit autoupdate +# +# See https://github.com/pre-commit/pre-commit + + +ci: + autoupdate_commit_msg: "chore(deps): update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + autoupdate_schedule: monthly + +# third-party content +exclude: ^tools/JoinPaths.cmake$ + +repos: +# Standard hooks +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.4.0" + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-docstring-first + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: requirements-txt-fixer + - id: trailing-whitespace + +# Black, the code formatter, natively supports pre-commit +- repo: https://github.com/psf/black + rev: "23.1.0" # Keep in sync with blacken-docs + hooks: + - id: black + +# Also code format the docs +- repo: https://github.com/asottile/blacken-docs + rev: "1.13.0" + hooks: + - id: blacken-docs + additional_dependencies: + - black==23.1.0 # keep in sync with black hook + +# Changes tabs to spaces +- repo: https://github.com/Lucas-C/pre-commit-hooks + rev: "v1.4.2" + hooks: + - id: remove-tabs + +# Avoid directional quotes +- repo: https://github.com/sirosen/texthooks + rev: "0.5.0" + hooks: + - id: fix-ligatures + - id: fix-smartquotes + +# Ruff, the Python auto-correcting linter written in Rust +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.254 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + +# Checking for common mistakes +- repo: https://github.com/pre-commit/pygrep-hooks + rev: "v1.10.0" + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + + +# PyLint has native support - not always usable, but works for us +- repo: https://github.com/PyCQA/pylint + rev: "v2.16.4" + hooks: + - id: pylint + files: ^pybind11 + +# CMake formatting +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: "v0.6.13" + hooks: + - id: cmake-format + additional_dependencies: [pyyaml] + types: [file] + files: (\.cmake|CMakeLists.txt)(.in)?$ + +# Check static types with mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.0.1" + hooks: + - id: mypy + args: [] + exclude: ^(tests|docs)/ + additional_dependencies: [nox, rich] + +# Checks the manifest for missing files (native support) +- repo: https://github.com/mgedmin/check-manifest + rev: "0.49" + hooks: + - id: check-manifest + # This is a slow hook, so only run this if --hook-stage manual is passed + stages: [manual] + additional_dependencies: [cmake, ninja] + +# Check for spelling +# Use tools/codespell_ignore_lines_from_errors.py +# to rebuild .codespell-ignore-lines +- repo: https://github.com/codespell-project/codespell + rev: "v2.2.2" + hooks: + - id: codespell + exclude: ".supp$" + args: ["-x", ".codespell-ignore-lines"] + +# Check for common shell mistakes +- repo: https://github.com/shellcheck-py/shellcheck-py + rev: "v0.9.0.2" + hooks: + - id: shellcheck + +# Disallow some common capitalization mistakes +- repo: local + hooks: + - id: disallow-caps + name: Disallow improper capitalization + language: pygrep + entry: PyBind|Numpy|Cmake|CCache|PyTest + exclude: ^\.pre-commit-config.yaml$ + +# Clang format the codebase automatically +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v15.0.7" + hooks: + - id: clang-format + types_or: [c++, c, cuda] diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d1ebdb0b..00000000 --- a/.travis.yml +++ /dev/null @@ -1,213 +0,0 @@ -language: cpp -dist: trusty -sudo: false -matrix: - include: - # This config does a few things: - # - Checks C++ and Python code styles (check-style.sh and flake8). - # - Makes sure sphinx can build the docs without any errors or warnings. - # - Tests setup.py sdist and install (all header files should be present). - # - Makes sure that everything still works without optional deps (numpy/scipy/eigen) and - # also tests the automatic discovery functions in CMake (Python version, C++ standard). - - os: linux - env: STYLE DOCS PIP - cache: false - before_install: - - pyenv global $(pyenv whence 2to3) # activate all python versions - - PY_CMD=python3 - - $PY_CMD -m pip install --user --upgrade pip wheel - install: - - $PY_CMD -m pip install --user --upgrade sphinx sphinx_rtd_theme breathe flake8 pep8-naming pytest - - curl -fsSL ftp://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.12.linux.bin.tar.gz | tar xz - - export PATH="$PWD/doxygen-1.8.12/bin:$PATH" - script: - - tools/check-style.sh - - flake8 - - $PY_CMD -m sphinx -W -b html docs docs/.build - - | - # Make sure setup.py distributes and installs all the headers - $PY_CMD setup.py sdist - $PY_CMD -m pip install --user -U ./dist/* - installed=$($PY_CMD -c "import pybind11; print(pybind11.get_include(True) + '/pybind11')") - diff -rq $installed ./include/pybind11 - - | - # Barebones build - cmake -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON - make pytest -j 2 - make cpptest -j 2 - # The following are regular test configurations, including optional dependencies. - # With regard to each other they differ in Python version, C++ standard and compiler. - - os: linux - env: PYTHON=2.7 CPP=11 GCC=4.8 - addons: - apt: - packages: [cmake=2.\*, cmake-data=2.\*] - - os: linux - env: PYTHON=3.6 CPP=11 GCC=4.8 - addons: - apt: - sources: [deadsnakes] - packages: [python3.6-dev python3.6-venv, cmake=2.\*, cmake-data=2.\*] - - sudo: true - services: docker - env: PYTHON=2.7 CPP=14 GCC=6 CMAKE=1 - - sudo: true - services: docker - env: PYTHON=3.5 CPP=14 GCC=6 DEBUG=1 - - sudo: true - services: docker - env: PYTHON=3.6 CPP=17 GCC=7 - - os: linux - env: PYTHON=3.6 CPP=17 CLANG=5.0 - addons: - apt: - sources: [deadsnakes, llvm-toolchain-trusty-5.0, ubuntu-toolchain-r-test] - packages: [python3.6-dev python3.6-venv clang-5.0 llvm-5.0-dev, lld-5.0] - - os: osx - osx_image: xcode7.3 - env: PYTHON=2.7 CPP=14 CLANG CMAKE=1 - - os: osx - osx_image: xcode9 - env: PYTHON=3.6 CPP=14 CLANG DEBUG=1 - # Test a PyPy 2.7 build - - os: linux - env: PYPY=5.8 PYTHON=2.7 CPP=11 GCC=4.8 - addons: - apt: - packages: [libblas-dev, liblapack-dev, gfortran] - # Build in 32-bit mode and tests against the CMake-installed version - - sudo: true - services: docker - env: ARCH=i386 PYTHON=3.5 CPP=14 GCC=6 INSTALL=1 - script: - - | - $SCRIPT_RUN_PREFIX sh -c "set -e - cmake ${CMAKE_EXTRA_ARGS} -DPYBIND11_INSTALL=1 -DPYBIND11_TEST=0 - make install - cp -a tests /pybind11-tests - mkdir /build-tests && cd /build-tests - cmake ../pybind11-tests ${CMAKE_EXTRA_ARGS} -DPYBIND11_WERROR=ON - make pytest -j 2" -cache: - directories: - - $HOME/.local/bin - - $HOME/.local/lib - - $HOME/.local/include - - $HOME/Library/Python -before_install: -- | - # Configure build variables - if [ "$TRAVIS_OS_NAME" = "linux" ]; then - if [ -n "$CLANG" ]; then - export CXX=clang++-$CLANG CC=clang-$CLANG - EXTRA_PACKAGES+=" clang-$CLANG llvm-$CLANG-dev" - else - if [ -z "$GCC" ]; then GCC=4.8 - else EXTRA_PACKAGES+=" g++-$GCC" - fi - export CXX=g++-$GCC CC=gcc-$GCC - fi - if [ "$GCC" = "6" ]; then DOCKER=${ARCH:+$ARCH/}debian:stretch - elif [ "$GCC" = "7" ]; then DOCKER=debian:buster EXTRA_PACKAGES+=" catch" DOWNLOAD_CATCH=OFF - fi - elif [ "$TRAVIS_OS_NAME" = "osx" ]; then - export CXX=clang++ CC=clang; - fi - if [ -n "$CPP" ]; then CPP=-std=c++$CPP; fi - if [ "${PYTHON:0:1}" = "3" ]; then PY=3; fi - if [ -n "$DEBUG" ]; then CMAKE_EXTRA_ARGS+=" -DCMAKE_BUILD_TYPE=Debug"; fi -- | - # Initialize environment - set -e - if [ -n "$DOCKER" ]; then - docker pull $DOCKER - - containerid=$(docker run --detach --tty \ - --volume="$PWD":/pybind11 --workdir=/pybind11 \ - --env="CC=$CC" --env="CXX=$CXX" --env="DEBIAN_FRONTEND=$DEBIAN_FRONTEND" \ - --env=GCC_COLORS=\ \ - $DOCKER) - SCRIPT_RUN_PREFIX="docker exec --tty $containerid" - $SCRIPT_RUN_PREFIX sh -c 'for s in 0 15; do sleep $s; apt-get update && apt-get -qy dist-upgrade && break; done' - else - if [ "$PYPY" = "5.8" ]; then - curl -fSL https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux64.tar.bz2 | tar xj - PY_CMD=$(echo `pwd`/pypy2-v5.8.0-linux64/bin/pypy) - CMAKE_EXTRA_ARGS+=" -DPYTHON_EXECUTABLE:FILEPATH=$PY_CMD" - else - PY_CMD=python$PYTHON - if [ "$TRAVIS_OS_NAME" = "osx" ]; then - if [ "$PY" = "3" ]; then - brew update && brew install python$PY; - else - curl -fsSL https://bootstrap.pypa.io/get-pip.py | $PY_CMD - --user - fi - fi - fi - if [ "$PY" = 3 ] || [ -n "$PYPY" ]; then - $PY_CMD -m ensurepip --user - fi - $PY_CMD -m pip install --user --upgrade pip wheel - fi - set +e -install: -- | - # Install dependencies - set -e - if [ -n "$DOCKER" ]; then - if [ -n "$DEBUG" ]; then - PY_DEBUG="python$PYTHON-dbg python$PY-scipy-dbg" - CMAKE_EXTRA_ARGS+=" -DPYTHON_EXECUTABLE=/usr/bin/python${PYTHON}dm" - fi - $SCRIPT_RUN_PREFIX sh -c "for s in 0 15; do sleep \$s; \ - apt-get -qy --no-install-recommends install \ - $PY_DEBUG python$PYTHON-dev python$PY-pytest python$PY-scipy \ - libeigen3-dev libboost-dev cmake make ${EXTRA_PACKAGES} && break; done" - else - - if [ "$CLANG" = "5.0" ]; then - if ! [ -d ~/.local/include/c++/v1 ]; then - # Neither debian nor llvm provide a libc++ 5.0 deb; luckily it's fairly quick - # to build, install (and cache), so do it ourselves: - git clone --depth=1 https://github.com/llvm-mirror/llvm.git llvm-source - git clone https://github.com/llvm-mirror/libcxx.git llvm-source/projects/libcxx -b release_50 - git clone https://github.com/llvm-mirror/libcxxabi.git llvm-source/projects/libcxxabi -b release_50 - mkdir llvm-build && cd llvm-build - # Building llvm requires a newer cmake than is provided by the trusty container: - CMAKE_VER=cmake-3.8.0-Linux-x86_64 - curl https://cmake.org/files/v3.8/$CMAKE_VER.tar.gz | tar xz - ./$CMAKE_VER/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=~/.local ../llvm-source - make -j2 install-cxxabi install-cxx - cp -a include/c++/v1/*cxxabi*.h ~/.local/include/c++/v1 - cd .. - fi - export CXXFLAGS="-isystem $HOME/.local/include/c++/v1 -stdlib=libc++" - export LDFLAGS="-L$HOME/.local/lib -fuse-ld=lld-$CLANG" - export LD_LIBRARY_PATH="$HOME/.local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - if [ "$CPP" = "-std=c++17" ]; then CPP="-std=c++1z"; fi - fi - - export NPY_NUM_BUILD_JOBS=2 - echo "Installing pytest, numpy, scipy..." - ${PYPY:+travis_wait 30} $PY_CMD -m pip install --user --upgrade pytest numpy scipy \ - ${PYPY:+--extra-index-url https://imaginary.ca/trusty-pypi} - echo "done." - - mkdir eigen - curl -fsSL https://bitbucket.org/eigen/eigen/get/3.3.4.tar.bz2 | \ - tar --extract -j --directory=eigen --strip-components=1 - export CMAKE_INCLUDE_PATH="${CMAKE_INCLUDE_PATH:+$CMAKE_INCLUDE_PATH:}$PWD/eigen" - fi - set +e -script: -- $SCRIPT_RUN_PREFIX cmake ${CMAKE_EXTRA_ARGS} - -DPYBIND11_PYTHON_VERSION=$PYTHON - -DPYBIND11_CPP_STANDARD=$CPP - -DPYBIND11_WERROR=${WERROR:-ON} - -DDOWNLOAD_CATCH=${DOWNLOAD_CATCH:-ON} -- $SCRIPT_RUN_PREFIX make pytest -j 2 -- $SCRIPT_RUN_PREFIX make cpptest -j 2 -- if [ -n "$CMAKE" ]; then $SCRIPT_RUN_PREFIX make test_cmake_build; fi -after_failure: cat tests/test_cmake_build/*.log* -after_script: -- if [ -n "$DOCKER" ]; then docker stop "$containerid"; docker rm "$containerid"; fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 4280ba74..0d932038 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,151 +5,316 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 3.4) -if (POLICY CMP0048) - # cmake warns if loaded from a min-3.0-required parent dir, so silence the warning: - cmake_policy(SET CMP0048 NEW) +# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with +# some versions of VS that have a patched CMake 3.11. This forces us to emulate +# the behavior using the following workaround: +if(${CMAKE_VERSION} VERSION_LESS 3.22) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +else() + cmake_policy(VERSION 3.22) endif() -# CMake versions < 3.4.0 do not support try_compile/pthread checks without C as active language. -if(CMAKE_VERSION VERSION_LESS 3.4.0) - project(pybind11) -else() - project(pybind11 CXX) +# Avoid infinite recursion if tests include this as a subdirectory +if(DEFINED PYBIND11_MASTER_PROJECT) + return() +endif() + +# Extract project version from source +file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/pybind11/detail/common.h" + pybind11_version_defines REGEX "#define PYBIND11_VERSION_(MAJOR|MINOR|PATCH) ") + +foreach(ver ${pybind11_version_defines}) + if(ver MATCHES [[#define PYBIND11_VERSION_(MAJOR|MINOR|PATCH) +([^ ]+)$]]) + set(PYBIND11_VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}") + endif() +endforeach() + +if(PYBIND11_VERSION_PATCH MATCHES [[\.([a-zA-Z0-9]+)$]]) + set(pybind11_VERSION_TYPE "${CMAKE_MATCH_1}") +endif() +string(REGEX MATCH "^[0-9]+" PYBIND11_VERSION_PATCH "${PYBIND11_VERSION_PATCH}") + +project( + pybind11 + LANGUAGES CXX + VERSION "${PYBIND11_VERSION_MAJOR}.${PYBIND11_VERSION_MINOR}.${PYBIND11_VERSION_PATCH}") + +# Standard includes +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) +include(CMakeDependentOption) + +if(NOT pybind11_FIND_QUIETLY) + message(STATUS "pybind11 v${pybind11_VERSION} ${pybind11_VERSION_TYPE}") endif() # Check if pybind11 is being used directly or via add_subdirectory -set(PYBIND11_MASTER_PROJECT OFF) -if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + ### Warn if not an out-of-source builds + if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) + set(lines + "You are building in-place. If that is not what you intended to " + "do, you can clean the source directory with:\n" + "rm -r CMakeCache.txt CMakeFiles/ cmake_uninstall.cmake pybind11Config.cmake " + "pybind11ConfigVersion.cmake tests/CMakeFiles/\n") + message(AUTHOR_WARNING ${lines}) + endif() + set(PYBIND11_MASTER_PROJECT ON) + + if(OSX AND CMAKE_VERSION VERSION_LESS 3.7) + # Bug in macOS CMake < 3.7 is unable to download catch + message(WARNING "CMAKE 3.7+ needed on macOS to download catch, and newer HIGHLY recommended") + elseif(WINDOWS AND CMAKE_VERSION VERSION_LESS 3.8) + # Only tested with 3.8+ in CI. + message(WARNING "CMAKE 3.8+ tested on Windows, previous versions untested") + endif() + + message(STATUS "CMake ${CMAKE_VERSION}") + + if(CMAKE_CXX_STANDARD) + set(CMAKE_CXX_EXTENSIONS OFF) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + endif() + + set(pybind11_system "") + + set_property(GLOBAL PROPERTY USE_FOLDERS ON) +else() + set(PYBIND11_MASTER_PROJECT OFF) + set(pybind11_system SYSTEM) endif() +# Options option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) -option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) +option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) +option(PYBIND11_NOPYTHON "Disable search for Python" OFF) +option(PYBIND11_SIMPLE_GIL_MANAGEMENT + "Use simpler GIL management logic that does not support disassociation" OFF) +set(PYBIND11_INTERNALS_VERSION + "" + CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/tools") +if(PYBIND11_SIMPLE_GIL_MANAGEMENT) + add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT) +endif() -include(pybind11Tools) +cmake_dependent_option( + USE_PYTHON_INCLUDE_DIR + "Install pybind11 headers in Python include directory instead of default installation prefix" + OFF "PYBIND11_INSTALL" OFF) -# Cache variables so pybind11_add_module can be used in parent projects -set(PYBIND11_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}/include" CACHE INTERNAL "") -set(PYTHON_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS} CACHE INTERNAL "") -set(PYTHON_LIBRARIES ${PYTHON_LIBRARIES} CACHE INTERNAL "") -set(PYTHON_MODULE_PREFIX ${PYTHON_MODULE_PREFIX} CACHE INTERNAL "") -set(PYTHON_MODULE_EXTENSION ${PYTHON_MODULE_EXTENSION} CACHE INTERNAL "") +cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF + "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF) # NB: when adding a header don't forget to also add it to setup.py set(PYBIND11_HEADERS - include/pybind11/detail/class.h - include/pybind11/detail/common.h - include/pybind11/detail/descr.h - include/pybind11/detail/init.h - include/pybind11/detail/internals.h - include/pybind11/detail/typeid.h - include/pybind11/attr.h - include/pybind11/buffer_info.h - include/pybind11/cast.h - include/pybind11/chrono.h - include/pybind11/common.h - include/pybind11/complex.h - include/pybind11/options.h - include/pybind11/eigen.h - include/pybind11/embed.h - include/pybind11/eval.h - include/pybind11/functional.h - include/pybind11/numpy.h - include/pybind11/operators.h - include/pybind11/pybind11.h - include/pybind11/pytypes.h - include/pybind11/stl.h - include/pybind11/stl_bind.h -) -string(REPLACE "include/" "${CMAKE_CURRENT_SOURCE_DIR}/include/" - PYBIND11_HEADERS "${PYBIND11_HEADERS}") - -if (PYBIND11_TEST) - add_subdirectory(tests) + include/pybind11/detail/class.h + include/pybind11/detail/common.h + include/pybind11/detail/descr.h + include/pybind11/detail/init.h + include/pybind11/detail/internals.h + include/pybind11/detail/type_caster_base.h + include/pybind11/detail/typeid.h + include/pybind11/attr.h + include/pybind11/buffer_info.h + include/pybind11/cast.h + include/pybind11/chrono.h + include/pybind11/common.h + include/pybind11/complex.h + include/pybind11/options.h + include/pybind11/eigen.h + include/pybind11/eigen/matrix.h + include/pybind11/eigen/tensor.h + include/pybind11/embed.h + include/pybind11/eval.h + include/pybind11/gil.h + include/pybind11/iostream.h + include/pybind11/functional.h + include/pybind11/numpy.h + include/pybind11/operators.h + include/pybind11/pybind11.h + include/pybind11/pytypes.h + include/pybind11/stl.h + include/pybind11/stl_bind.h + include/pybind11/stl/filesystem.h) + +# Compare with grep and warn if mismatched +if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) + file( + GLOB_RECURSE _pybind11_header_check + LIST_DIRECTORIES false + RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" + CONFIGURE_DEPENDS "include/pybind11/*.h") + set(_pybind11_here_only ${PYBIND11_HEADERS}) + set(_pybind11_disk_only ${_pybind11_header_check}) + list(REMOVE_ITEM _pybind11_here_only ${_pybind11_header_check}) + list(REMOVE_ITEM _pybind11_disk_only ${PYBIND11_HEADERS}) + if(_pybind11_here_only) + message(AUTHOR_WARNING "PYBIND11_HEADERS has extra files:" ${_pybind11_here_only}) + endif() + if(_pybind11_disk_only) + message(AUTHOR_WARNING "PYBIND11_HEADERS is missing files:" ${_pybind11_disk_only}) + endif() endif() -include(GNUInstallDirs) -include(CMakePackageConfigHelpers) +# CMake 3.12 added list(TRANSFORM PREPEND +# But we can't use it yet +string(REPLACE "include/" "${CMAKE_CURRENT_SOURCE_DIR}/include/" PYBIND11_HEADERS + "${PYBIND11_HEADERS}") -# extract project version from source -file(STRINGS "${PYBIND11_INCLUDE_DIR}/pybind11/detail/common.h" pybind11_version_defines - REGEX "#define PYBIND11_VERSION_(MAJOR|MINOR|PATCH) ") -foreach(ver ${pybind11_version_defines}) - if (ver MATCHES "#define PYBIND11_VERSION_(MAJOR|MINOR|PATCH) +([^ ]+)$") - set(PYBIND11_VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}" CACHE INTERNAL "") +# Cache variable so this can be used in parent projects +set(pybind11_INCLUDE_DIR + "${CMAKE_CURRENT_LIST_DIR}/include" + CACHE INTERNAL "Directory where pybind11 headers are located") + +# Backward compatible variable for add_subdirectory mode +if(NOT PYBIND11_MASTER_PROJECT) + set(PYBIND11_INCLUDE_DIR + "${pybind11_INCLUDE_DIR}" + CACHE INTERNAL "") +endif() + +# Note: when creating targets, you cannot use if statements at configure time - +# you need generator expressions, because those will be placed in the target file. +# You can also place ifs *in* the Config.in, but not here. + +# This section builds targets, but does *not* touch Python +# Non-IMPORT targets cannot be defined twice +if(NOT TARGET pybind11_headers) + # Build the headers-only target (no Python included): + # (long name used here to keep this from clashing in subdirectory mode) + add_library(pybind11_headers INTERFACE) + add_library(pybind11::pybind11_headers ALIAS pybind11_headers) # to match exported target + add_library(pybind11::headers ALIAS pybind11_headers) # easier to use/remember + + target_include_directories( + pybind11_headers ${pybind11_system} INTERFACE $ + $) + + target_compile_features(pybind11_headers INTERFACE cxx_inheriting_constructors cxx_user_literals + cxx_right_angle_brackets) + if(NOT "${PYBIND11_INTERNALS_VERSION}" STREQUAL "") + target_compile_definitions( + pybind11_headers INTERFACE "PYBIND11_INTERNALS_VERSION=${PYBIND11_INTERNALS_VERSION}") endif() -endforeach() -set(${PROJECT_NAME}_VERSION ${PYBIND11_VERSION_MAJOR}.${PYBIND11_VERSION_MINOR}.${PYBIND11_VERSION_PATCH}) -message(STATUS "pybind11 v${${PROJECT_NAME}_VERSION}") +else() + # It is invalid to install a target twice, too. + set(PYBIND11_INSTALL OFF) +endif() -option (USE_PYTHON_INCLUDE_DIR "Install pybind11 headers in Python include directory instead of default installation prefix" OFF) -if (USE_PYTHON_INCLUDE_DIR) - file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIRS}) +include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake") +# https://github.com/jtojnar/cmake-snips/#concatenating-paths-when-building-pkg-config-files +# TODO: cmake 3.20 adds the cmake_path() function, which obsoletes this snippet +include("${CMAKE_CURRENT_SOURCE_DIR}/tools/JoinPaths.cmake") + +# Relative directory setting +if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS) + file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${Python_INCLUDE_DIRS}) +elseif(USE_PYTHON_INCLUDE_DIR AND DEFINED PYTHON_INCLUDE_DIR) + file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIRS}) endif() -if(NOT (CMAKE_VERSION VERSION_LESS 3.0)) # CMake >= 3.0 - # Build an interface library target: - add_library(pybind11 INTERFACE) - add_library(pybind11::pybind11 ALIAS pybind11) # to match exported target - target_include_directories(pybind11 INTERFACE $ - $ - $) - target_compile_options(pybind11 INTERFACE $) - - add_library(module INTERFACE) - add_library(pybind11::module ALIAS module) - if(NOT MSVC) - target_compile_options(module INTERFACE -fvisibility=hidden) +if(PYBIND11_INSTALL) + install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + set(PYBIND11_CMAKECONFIG_INSTALL_DIR + "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}" + CACHE STRING "install path for pybind11Config.cmake") + + if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}") + set(pybind11_INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}") + else() + set(pybind11_INCLUDEDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_INCLUDEDIR}") + endif() + + configure_package_config_file( + tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" + INSTALL_DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) + + if(CMAKE_VERSION VERSION_LESS 3.14) + # Remove CMAKE_SIZEOF_VOID_P from ConfigVersion.cmake since the library does + # not depend on architecture specific settings or libraries. + set(_PYBIND11_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P}) + unset(CMAKE_SIZEOF_VOID_P) + + write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion) + + set(CMAKE_SIZEOF_VOID_P ${_PYBIND11_CMAKE_SIZEOF_VOID_P}) + else() + # CMake 3.14+ natively supports header-only libraries + write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT) + endif() + + install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + tools/FindPythonLibsNew.cmake + tools/pybind11Common.cmake + tools/pybind11Tools.cmake + tools/pybind11NewTools.cmake + DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) + + if(NOT PYBIND11_EXPORT_NAME) + set(PYBIND11_EXPORT_NAME "${PROJECT_NAME}Targets") endif() - target_link_libraries(module INTERFACE pybind11::pybind11) - if(WIN32 OR CYGWIN) - target_link_libraries(module INTERFACE $) - elseif(APPLE) - target_link_libraries(module INTERFACE "-undefined dynamic_lookup") + + install(TARGETS pybind11_headers EXPORT "${PYBIND11_EXPORT_NAME}") + + install( + EXPORT "${PYBIND11_EXPORT_NAME}" + NAMESPACE "pybind11::" + DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) + + # pkg-config support + if(NOT prefix_for_pc_file) + set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") endif() + join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/") - add_library(embed INTERFACE) - add_library(pybind11::embed ALIAS embed) - target_link_libraries(embed INTERFACE pybind11::pybind11 $) + # Uninstall target + if(PYBIND11_MASTER_PROJECT) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY) + + add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P + ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) + endif() endif() -if (PYBIND11_INSTALL) - install(DIRECTORY ${PYBIND11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - # GNUInstallDirs "DATADIR" wrong here; CMake search path wants "share". - set(PYBIND11_CMAKECONFIG_INSTALL_DIR "share/cmake/${PROJECT_NAME}" CACHE STRING "install path for pybind11Config.cmake") - - configure_package_config_file(tools/${PROJECT_NAME}Config.cmake.in - "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" - INSTALL_DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) - # Remove CMAKE_SIZEOF_VOID_P from ConfigVersion.cmake since the library does - # not depend on architecture specific settings or libraries. - set(_PYBIND11_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P}) - unset(CMAKE_SIZEOF_VOID_P) - write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - VERSION ${${PROJECT_NAME}_VERSION} - COMPATIBILITY AnyNewerVersion) - set(CMAKE_SIZEOF_VOID_P ${_PYBIND11_CMAKE_SIZEOF_VOID_P}) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - tools/FindPythonLibsNew.cmake - tools/pybind11Tools.cmake - DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) - - if(NOT (CMAKE_VERSION VERSION_LESS 3.0)) - if(NOT PYBIND11_EXPORT_NAME) - set(PYBIND11_EXPORT_NAME "${PROJECT_NAME}Targets") +# BUILD_TESTING takes priority, but only if this is the master project +if(PYBIND11_MASTER_PROJECT AND DEFINED BUILD_TESTING) + if(BUILD_TESTING) + if(_pybind11_nopython) + message(FATAL_ERROR "Cannot activate tests in NOPYTHON mode") + else() + add_subdirectory(tests) endif() - - install(TARGETS pybind11 module embed - EXPORT "${PYBIND11_EXPORT_NAME}") - if(PYBIND11_MASTER_PROJECT) - install(EXPORT "${PYBIND11_EXPORT_NAME}" - NAMESPACE "${PROJECT_NAME}::" - DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) + endif() +else() + if(PYBIND11_TEST) + if(_pybind11_nopython) + message(FATAL_ERROR "Cannot activate tests in NOPYTHON mode") + else() + add_subdirectory(tests) endif() endif() endif() + +# Better symmetry with find_package(pybind11 CONFIG) mode. +if(NOT PYBIND11_MASTER_PROJECT) + set(pybind11_FOUND + TRUE + CACHE INTERNAL "True if pybind11 and all required components found on the system") +endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 375735f6..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,47 +0,0 @@ -Thank you for your interest in this project! Please refer to the following -sections on how to contribute code and bug reports. - -### Reporting bugs - -At the moment, this project is run in the spare time of a single person -([Wenzel Jakob](http://rgl.epfl.ch/people/wjakob)) with very limited resources -for issue tracker tickets. Thus, before submitting a question or bug report, -please take a moment of your time and ensure that your issue isn't already -discussed in the project documentation provided at -[http://pybind11.readthedocs.org/en/latest](http://pybind11.readthedocs.org/en/latest). - -Assuming that you have identified a previously unknown problem or an important -question, it's essential that you submit a self-contained and minimal piece of -code that reproduces the problem. In other words: no external dependencies, -isolate the function(s) that cause breakage, submit matched and complete C++ -and Python snippets that can be easily compiled and run on my end. - -## Pull requests -Contributions are submitted, reviewed, and accepted using Github pull requests. -Please refer to [this -article](https://help.github.com/articles/using-pull-requests) for details and -adhere to the following rules to make the process as smooth as possible: - -* Make a new branch for every feature you're working on. -* Make small and clean pull requests that are easy to review but make sure they - do add value by themselves. -* Add tests for any new functionality and run the test suite (``make pytest``) - to ensure that no existing features break. -* This project has a strong focus on providing general solutions using a - minimal amount of code, thus small pull requests are greatly preferred. - -### Licensing of contributions - -pybind11 is provided under a BSD-style license that can be found in the -``LICENSE`` file. By using, distributing, or contributing to this project, you -agree to the terms and conditions of this license. - -You are under no obligation whatsoever to provide any bug fixes, patches, or -upgrades to the features, functionality or performance of the source code -("Enhancements") to anyone; however, if you choose to make your Enhancements -available either publicly, or directly to the author of this software, without -imposing a separate written license agreement for such Enhancements, then you -hereby grant the following license: a non-exclusive, royalty-free perpetual -license to install, use, modify, prepare derivative works, incorporate into -other computer software, distribute, and sublicense such enhancements or -derivative works thereof, in binary and source code form. diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md deleted file mode 100644 index 75df3998..00000000 --- a/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,17 +0,0 @@ -Make sure you've completed the following steps before submitting your issue -- thank you! - -1. Check if your question has already been answered in the [FAQ](http://pybind11.readthedocs.io/en/latest/faq.html) section. -2. Make sure you've read the [documentation](http://pybind11.readthedocs.io/en/latest/). Your issue may be addressed there. -3. If those resources didn't help and you only have a short question (not a bug report), consider asking in the [Gitter chat room](https://gitter.im/pybind/Lobby). -4. If you have a genuine bug report or a more complex question which is not answered in the previous items (or not suitable for chat), please fill in the details below. -5. Include a self-contained and minimal piece of code that reproduces the problem. If that's not possible, try to make the description as clear as possible. - -*After reading, remove this checklist and the template text in parentheses below.* - -## Issue description - -(Provide a short description, state the expected behavior and what actually happens.) - -## Reproducible example code - -(The code should be minimal, have no external dependencies, isolate the function(s) that cause breakage. Submit matched and complete C++ and Python snippets that can be easily compiled and run to diagnose the issue.) diff --git a/LICENSE b/LICENSE index 6f15578c..e466b0df 100644 --- a/LICENSE +++ b/LICENSE @@ -25,5 +25,5 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Please also refer to the file CONTRIBUTING.md, which clarifies licensing of +Please also refer to the file .github/CONTRIBUTING.md, which clarifies licensing of external contributions to this project including patches, pull requests, etc. diff --git a/MANIFEST.in b/MANIFEST.in index 6e57baee..31632acc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,6 @@ -recursive-include include/pybind11 *.h -include LICENSE README.md CONTRIBUTING.md +prune tests +recursive-include pybind11/include/pybind11 *.h +recursive-include pybind11 *.py +recursive-include pybind11 py.typed +include pybind11/share/cmake/pybind11/*.cmake +include LICENSE README.rst pyproject.toml setup.py setup.cfg diff --git a/README.md b/README.md deleted file mode 100644 index 04568ace..00000000 --- a/README.md +++ /dev/null @@ -1,128 +0,0 @@ -![pybind11 logo](https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png) - -# pybind11 — Seamless operability between C++11 and Python - -[![Documentation Status](https://readthedocs.org/projects/pybind11/badge/?version=master)](http://pybind11.readthedocs.org/en/master/?badge=master) -[![Documentation Status](https://readthedocs.org/projects/pybind11/badge/?version=stable)](http://pybind11.readthedocs.org/en/stable/?badge=stable) -[![Gitter chat](https://img.shields.io/gitter/room/gitterHQ/gitter.svg)](https://gitter.im/pybind/Lobby) -[![Build Status](https://travis-ci.org/pybind/pybind11.svg?branch=master)](https://travis-ci.org/pybind/pybind11) -[![Build status](https://ci.appveyor.com/api/projects/status/riaj54pn4h08xy40?svg=true)](https://ci.appveyor.com/project/wjakob/pybind11) - -**pybind11** is a lightweight header-only library that exposes C++ types in Python -and vice versa, mainly to create Python bindings of existing C++ code. Its -goals and syntax are similar to the excellent -[Boost.Python](http://www.boost.org/doc/libs/1_58_0/libs/python/doc/) library -by David Abrahams: to minimize boilerplate code in traditional extension -modules by inferring type information using compile-time introspection. - -The main issue with Boost.Python—and the reason for creating such a similar -project—is Boost. Boost is an enormously large and complex suite of utility -libraries that works with almost every C++ compiler in existence. This -compatibility has its cost: arcane template tricks and workarounds are -necessary to support the oldest and buggiest of compiler specimens. Now that -C++11-compatible compilers are widely available, this heavy machinery has -become an excessively large and unnecessary dependency. - -Think of this library as a tiny self-contained version of Boost.Python with -everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~4K lines of code and depend on -Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This -compact implementation was possible thanks to some of the new C++11 language -features (specifically: tuples, lambda functions and variadic templates). Since -its creation, this library has grown beyond Boost.Python in many ways, leading -to dramatically simpler binding code in many common situations. - -Tutorial and reference documentation is provided at -[http://pybind11.readthedocs.org/en/master](http://pybind11.readthedocs.org/en/master). -A PDF version of the manual is available -[here](https://media.readthedocs.org/pdf/pybind11/master/pybind11.pdf). - -## Core features -pybind11 can map the following core C++ features to Python - -- Functions accepting and returning custom data structures per value, reference, or pointer -- Instance methods and static methods -- Overloaded functions -- Instance attributes and static attributes -- Arbitrary exception types -- Enumerations -- Callbacks -- Iterators and ranges -- Custom operators -- Single and multiple inheritance -- STL data structures -- Iterators and ranges -- Smart pointers with reference counting like ``std::shared_ptr`` -- Internal references with correct reference counting -- C++ classes with virtual (and pure virtual) methods can be extended in Python - -## Goodies -In addition to the core functionality, pybind11 provides some extra goodies: - -- Python 2.7, 3.x, and PyPy (PyPy2.7 >= 5.7) are supported with an - implementation-agnostic interface. - -- It is possible to bind C++11 lambda functions with captured variables. The - lambda capture data is stored inside the resulting Python function object. - -- pybind11 uses C++11 move constructors and move assignment operators whenever - possible to efficiently transfer custom data types. - -- It's easy to expose the internal storage of custom data types through - Pythons' buffer protocols. This is handy e.g. for fast conversion between - C++ matrix classes like Eigen and NumPy without expensive copy operations. - -- pybind11 can automatically vectorize functions so that they are transparently - applied to all entries of one or more NumPy array arguments. - -- Python's slice-based access and assignment operations can be supported with - just a few lines of code. - -- Everything is contained in just a few header files; there is no need to link - against any additional libraries. - -- Binaries are generally smaller by a factor of at least 2 compared to - equivalent bindings generated by Boost.Python. A recent pybind11 conversion - of PyRosetta, an enormous Boost.Python binding project, - [reported](http://graylab.jhu.edu/RosettaCon2016/PyRosetta-4.pdf) a binary - size reduction of **5.4x** and compile time reduction by **5.8x**. - -- Function signatures are precomputed at compile time (using ``constexpr``), - leading to smaller binaries. - -- With little extra effort, C++ types can be pickled and unpickled similar to - regular Python objects. - -## Supported compilers - -1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or newer) -2. GCC 4.8 or newer -3. Microsoft Visual Studio 2015 Update 3 or newer -4. Intel C++ compiler 16 or newer (15 with a [workaround](https://github.com/pybind/pybind11/issues/276)) -5. Cygwin/GCC (tested on 2.5.1) - -## About - -This project was created by [Wenzel Jakob](http://rgl.epfl.ch/people/wjakob). -Significant features and/or improvements to the code were contributed by -Jonas Adler, -Sylvain Corlay, -Trent Houliston, -Axel Huebl, -@hulucc, -Sergey Lyskov -Johan Mabille, -Tomasz Miąsko, -Dean Moldovan, -Ben Pritchard, -Jason Rhinelander, -Boris Schäling, -Pim Schellart, -Ivan Smirnov, and -Patrick Stewart. - -### License - -pybind11 is provided under a BSD-style license that can be found in the -``LICENSE`` file. By using, distributing, or contributing to this project, -you agree to the terms and conditions of this license. diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..3c75edb5 --- /dev/null +++ b/README.rst @@ -0,0 +1,180 @@ +.. figure:: https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png + :alt: pybind11 logo + +**pybind11 — Seamless operability between C++11 and Python** + +|Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |GitHub Discussions| |CI| |Build status| + +|Repology| |PyPI package| |Conda-forge| |Python Versions| + +`Setuptools example `_ +• `Scikit-build example `_ +• `CMake example `_ + +.. start + + +**pybind11** is a lightweight header-only library that exposes C++ types +in Python and vice versa, mainly to create Python bindings of existing +C++ code. Its goals and syntax are similar to the excellent +`Boost.Python `_ +library by David Abrahams: to minimize boilerplate code in traditional +extension modules by inferring type information using compile-time +introspection. + +The main issue with Boost.Python—and the reason for creating such a +similar project—is Boost. Boost is an enormously large and complex suite +of utility libraries that works with almost every C++ compiler in +existence. This compatibility has its cost: arcane template tricks and +workarounds are necessary to support the oldest and buggiest of compiler +specimens. Now that C++11-compatible compilers are widely available, +this heavy machinery has become an excessively large and unnecessary +dependency. + +Think of this library as a tiny self-contained version of Boost.Python +with everything stripped away that isn't relevant for binding +generation. Without comments, the core header files only require ~4K +lines of code and depend on Python (3.6+, or PyPy) and the C++ +standard library. This compact implementation was possible thanks to +some of the new C++11 language features (specifically: tuples, lambda +functions and variadic templates). Since its creation, this library has +grown beyond Boost.Python in many ways, leading to dramatically simpler +binding code in many common situations. + +Tutorial and reference documentation is provided at +`pybind11.readthedocs.io `_. +A PDF version of the manual is available +`here `_. +And the source code is always available at +`github.com/pybind/pybind11 `_. + + +Core features +------------- + + +pybind11 can map the following core C++ features to Python: + +- Functions accepting and returning custom data structures per value, + reference, or pointer +- Instance methods and static methods +- Overloaded functions +- Instance attributes and static attributes +- Arbitrary exception types +- Enumerations +- Callbacks +- Iterators and ranges +- Custom operators +- Single and multiple inheritance +- STL data structures +- Smart pointers with reference counting like ``std::shared_ptr`` +- Internal references with correct reference counting +- C++ classes with virtual (and pure virtual) methods can be extended + in Python + +Goodies +------- + +In addition to the core functionality, pybind11 provides some extra +goodies: + +- Python 3.6+, and PyPy3 7.3 are supported with an implementation-agnostic + interface (pybind11 2.9 was the last version to support Python 2 and 3.5). + +- It is possible to bind C++11 lambda functions with captured + variables. The lambda capture data is stored inside the resulting + Python function object. + +- pybind11 uses C++11 move constructors and move assignment operators + whenever possible to efficiently transfer custom data types. + +- It's easy to expose the internal storage of custom data types through + Pythons' buffer protocols. This is handy e.g. for fast conversion + between C++ matrix classes like Eigen and NumPy without expensive + copy operations. + +- pybind11 can automatically vectorize functions so that they are + transparently applied to all entries of one or more NumPy array + arguments. + +- Python's slice-based access and assignment operations can be + supported with just a few lines of code. + +- Everything is contained in just a few header files; there is no need + to link against any additional libraries. + +- Binaries are generally smaller by a factor of at least 2 compared to + equivalent bindings generated by Boost.Python. A recent pybind11 + conversion of PyRosetta, an enormous Boost.Python binding project, + `reported `_ + a binary size reduction of **5.4x** and compile time reduction by + **5.8x**. + +- Function signatures are precomputed at compile time (using + ``constexpr``), leading to smaller binaries. + +- With little extra effort, C++ types can be pickled and unpickled + similar to regular Python objects. + +Supported compilers +------------------- + +1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or + newer) +2. GCC 4.8 or newer +3. Microsoft Visual Studio 2017 or newer +4. Intel classic C++ compiler 18 or newer (ICC 20.2 tested in CI) +5. Cygwin/GCC (previously tested on 2.5.1) +6. NVCC (CUDA 11.0 tested in CI) +7. NVIDIA PGI (20.9 tested in CI) + +About +----- + +This project was created by `Wenzel +Jakob `_. Significant features and/or +improvements to the code were contributed by Jonas Adler, Lori A. Burns, +Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel +Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz Miąsko, +Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim +Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. + +We thank Google for a generous financial contribution to the continuous +integration infrastructure used by this project. + + +Contributing +~~~~~~~~~~~~ + +See the `contributing +guide `_ +for information on building and contributing to pybind11. + +License +~~~~~~~ + +pybind11 is provided under a BSD-style license that can be found in the +`LICENSE `_ +file. By using, distributing, or contributing to this project, you agree +to the terms and conditions of this license. + +.. |Latest Documentation Status| image:: https://readthedocs.org/projects/pybind11/badge?version=latest + :target: http://pybind11.readthedocs.org/en/latest +.. |Stable Documentation Status| image:: https://img.shields.io/badge/docs-stable-blue.svg + :target: http://pybind11.readthedocs.org/en/stable +.. |Gitter chat| image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg + :target: https://gitter.im/pybind/Lobby +.. |CI| image:: https://github.com/pybind/pybind11/workflows/CI/badge.svg + :target: https://github.com/pybind/pybind11/actions +.. |Build status| image:: https://ci.appveyor.com/api/projects/status/riaj54pn4h08xy40?svg=true + :target: https://ci.appveyor.com/project/wjakob/pybind11 +.. |PyPI package| image:: https://img.shields.io/pypi/v/pybind11.svg + :target: https://pypi.org/project/pybind11/ +.. |Conda-forge| image:: https://img.shields.io/conda/vn/conda-forge/pybind11.svg + :target: https://github.com/conda-forge/pybind11-feedstock +.. |Repology| image:: https://repology.org/badge/latest-versions/python:pybind11.svg + :target: https://repology.org/project/python:pybind11/versions +.. |Python Versions| image:: https://img.shields.io/pypi/pyversions/pybind11.svg + :target: https://pypi.org/project/pybind11/ +.. |GitHub Discussions| image:: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github + :target: https://github.com/pybind/pybind11/discussions diff --git a/docs/Doxyfile b/docs/Doxyfile index 1b9d1297..09138db3 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -18,3 +18,4 @@ ALIASES += "endrst=\endverbatim" QUIET = YES WARNINGS = YES WARN_IF_UNDOCUMENTED = NO +PREDEFINED = PYBIND11_NOINLINE diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css new file mode 100644 index 00000000..7a49a6ac --- /dev/null +++ b/docs/_static/css/custom.css @@ -0,0 +1,3 @@ +.highlight .go { + color: #707070; +} diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css deleted file mode 100644 index 1071809f..00000000 --- a/docs/_static/theme_overrides.css +++ /dev/null @@ -1,11 +0,0 @@ -.wy-table-responsive table td, -.wy-table-responsive table th { - white-space: initial !important; -} -.rst-content table.docutils td { - vertical-align: top !important; -} -div[class^='highlight'] pre { - white-space: pre; - white-space: pre-wrap; -} diff --git a/docs/advanced/cast/chrono.rst b/docs/advanced/cast/chrono.rst index 8c6b3d7e..fbd46057 100644 --- a/docs/advanced/cast/chrono.rst +++ b/docs/advanced/cast/chrono.rst @@ -59,7 +59,7 @@ Provided conversions .. rubric:: Python to C++ -- ``datetime.datetime`` → ``std::chrono::system_clock::time_point`` +- ``datetime.datetime`` or ``datetime.date`` or ``datetime.time`` → ``std::chrono::system_clock::time_point`` Date/time objects are converted into system clock timepoints. Any timezone information is ignored and the type is treated as a naive object. diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index e4f99ac5..8138cac6 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -26,17 +26,19 @@ The following Python snippet demonstrates the intended usage from the Python sid def __int__(self): return 123 + from example import print + print(A()) -To register the necessary conversion routines, it is necessary to add -a partial overload to the ``pybind11::detail::type_caster`` template. -Although this is an implementation detail, adding partial overloads to this +To register the necessary conversion routines, it is necessary to add an +instantiation of the ``pybind11::detail::type_caster`` template. +Although this is an implementation detail, adding an instantiation of this type is explicitly allowed. .. code-block:: cpp - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster { public: /** @@ -44,7 +46,7 @@ type is explicitly allowed. * function signatures and declares a local variable * 'value' of type inty */ - PYBIND11_TYPE_CASTER(inty, _("inty")); + PYBIND11_TYPE_CASTER(inty, const_name("inty")); /** * Conversion part 1 (Python->C++): convert a PyObject into a inty @@ -76,7 +78,7 @@ type is explicitly allowed. return PyLong_FromLong(src.long_value); } }; - }} // namespace pybind11::detail + }} // namespace PYBIND11_NAMESPACE::detail .. note:: diff --git a/docs/advanced/cast/eigen.rst b/docs/advanced/cast/eigen.rst index 9c7cbd22..a5c11a3f 100644 --- a/docs/advanced/cast/eigen.rst +++ b/docs/advanced/cast/eigen.rst @@ -37,7 +37,7 @@ that maps into the source ``numpy.ndarray`` data: this requires both that the data types are the same (e.g. ``dtype='float64'`` and ``MatrixType::Scalar`` is ``double``); and that the storage is layout compatible. The latter limitation is discussed in detail in the section below, and requires careful -consideration: by default, numpy matrices and eigen matrices are *not* storage +consideration: by default, numpy matrices and Eigen matrices are *not* storage compatible. If the numpy matrix cannot be used as is (either because its types differ, e.g. @@ -52,7 +52,7 @@ can be mapped *and* if the numpy array is writeable (that is the passed variable will be transparently carried out directly on the ``numpy.ndarray``. -This means you can can write code such as the following and have it work as +This means you can write code such as the following and have it work as expected: .. code-block:: cpp @@ -112,7 +112,7 @@ example: .. code-block:: python a = MyClass() - m = a.get_matrix() # flags.writeable = True, flags.owndata = False + m = a.get_matrix() # flags.writeable = True, flags.owndata = False v = a.view_matrix() # flags.writeable = False, flags.owndata = False c = a.copy_matrix() # flags.writeable = True, flags.owndata = True # m[5,6] and v[5,6] refer to the same element, c[5,6] does not. @@ -203,7 +203,7 @@ adding the ``order='F'`` option when creating an array: .. code-block:: python - myarray = np.array(source, order='F') + myarray = np.array(source, order="F") Such an object will be passable to a bound function accepting an ``Eigen::Ref`` (or similar column-major Eigen type). @@ -226,7 +226,7 @@ order. Failing rather than copying =========================== -The default behaviour when binding ``Eigen::Ref`` eigen +The default behaviour when binding ``Eigen::Ref`` Eigen references is to copy matrix values when passed a numpy array that does not conform to the element type of ``MatrixType`` or does not have a compatible stride layout. If you want to explicitly avoid copying in such a case, you @@ -274,8 +274,8 @@ Vectors versus column/row matrices Eigen and numpy have fundamentally different notions of a vector. In Eigen, a vector is simply a matrix with the number of columns or rows set to 1 at -compile time (for a column vector or row vector, respectively). Numpy, in -contast, has comparable 2-dimensional 1xN and Nx1 arrays, but *also* has +compile time (for a column vector or row vector, respectively). NumPy, in +contrast, has comparable 2-dimensional 1xN and Nx1 arrays, but *also* has 1-dimensional arrays of size N. When passing a 2-dimensional 1xN or Nx1 array to Eigen, the Eigen type must @@ -289,13 +289,13 @@ will be passed as such a column vector. If not, but the Eigen type constraints will accept a row vector, it will be passed as a row vector. (The column vector takes precedence when both are supported, for example, when passing a 1D numpy array to a MatrixXd argument). Note that the type need not be -expicitly a vector: it is permitted to pass a 1D numpy array of size 5 to an +explicitly a vector: it is permitted to pass a 1D numpy array of size 5 to an Eigen ``Matrix``: you would end up with a 1x5 Eigen matrix. Passing the same to an ``Eigen::MatrixXd`` would result in a 5x1 Eigen matrix. -When returning an eigen vector to numpy, the conversion is ambiguous: a row +When returning an Eigen vector to numpy, the conversion is ambiguous: a row vector of length 4 could be returned as either a 1D array of length 4, or as a -2D array of size 1x4. When encoutering such a situation, pybind11 compromises +2D array of size 1x4. When encountering such a situation, pybind11 compromises by considering the returned Eigen type: if it is a compile-time vector--that is, the type has either the number of rows or columns set to 1 at compile time--pybind11 converts to a 1D numpy array when returning the value. For diff --git a/docs/advanced/cast/index.rst b/docs/advanced/cast/index.rst index 54c10570..3ce9ea02 100644 --- a/docs/advanced/cast/index.rst +++ b/docs/advanced/cast/index.rst @@ -1,3 +1,5 @@ +.. _type-conversions: + Type conversions ################ @@ -39,4 +41,3 @@ the last case of the above list. chrono eigen custom - diff --git a/docs/advanced/cast/overview.rst b/docs/advanced/cast/overview.rst index 2ac7d300..011bd4c7 100644 --- a/docs/advanced/cast/overview.rst +++ b/docs/advanced/cast/overview.rst @@ -75,89 +75,96 @@ The following basic data types are supported out of the box (some may require an additional extension header to be included). To pass other data structures as arguments and return values, refer to the section on binding :ref:`classes`. -+------------------------------------+---------------------------+-------------------------------+ -| Data type | Description | Header file | -+====================================+===========================+===============================+ -| ``int8_t``, ``uint8_t`` | 8-bit integers | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``int16_t``, ``uint16_t`` | 16-bit integers | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``int32_t``, ``uint32_t`` | 32-bit integers | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``int64_t``, ``uint64_t`` | 64-bit integers | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``ssize_t``, ``size_t`` | Platform-dependent size | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``float``, ``double`` | Floating point types | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``bool`` | Two-state Boolean type | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``char`` | Character literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``char16_t`` | UTF-16 character literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``char32_t`` | UTF-32 character literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``wchar_t`` | Wide character literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``const char *`` | UTF-8 string literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``const char16_t *`` | UTF-16 string literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``const char32_t *`` | UTF-32 string literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``const wchar_t *`` | Wide string literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::string`` | STL dynamic UTF-8 string | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::u16string`` | STL dynamic UTF-16 string | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::u32string`` | STL dynamic UTF-32 string | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::wstring`` | STL dynamic wide string | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::string_view``, | STL C++17 string views | :file:`pybind11/pybind11.h` | -| ``std::u16string_view``, etc. | | | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::pair`` | Pair of two custom types | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::tuple<...>`` | Arbitrary tuple of types | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::reference_wrapper<...>`` | Reference type wrapper | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::complex`` | Complex numbers | :file:`pybind11/complex.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::array`` | STL static array | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::vector`` | STL dynamic array | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::valarray`` | STL value array | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::list`` | STL linked list | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::map`` | STL ordered map | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::unordered_map`` | STL unordered map | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::set`` | STL ordered set | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::unordered_set`` | STL unordered set | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::optional`` | STL optional type (C++17) | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::experimental::optional`` | STL optional type (exp.) | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::variant<...>`` | Type-safe union (C++17) | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::function<...>`` | STL polymorphic function | :file:`pybind11/functional.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::chrono::duration<...>`` | STL time duration | :file:`pybind11/chrono.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::chrono::time_point<...>`` | STL date/time | :file:`pybind11/chrono.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``Eigen::Matrix<...>`` | Eigen: dense matrix | :file:`pybind11/eigen.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``Eigen::Map<...>`` | Eigen: mapped memory | :file:`pybind11/eigen.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``Eigen::SparseMatrix<...>`` | Eigen: sparse matrix | :file:`pybind11/eigen.h` | -+------------------------------------+---------------------------+-------------------------------+ ++------------------------------------+---------------------------+-----------------------------------+ +| Data type | Description | Header file | ++====================================+===========================+===================================+ +| ``int8_t``, ``uint8_t`` | 8-bit integers | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``int16_t``, ``uint16_t`` | 16-bit integers | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``int32_t``, ``uint32_t`` | 32-bit integers | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``int64_t``, ``uint64_t`` | 64-bit integers | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``ssize_t``, ``size_t`` | Platform-dependent size | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``float``, ``double`` | Floating point types | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``bool`` | Two-state Boolean type | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``char`` | Character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``char16_t`` | UTF-16 character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``char32_t`` | UTF-32 character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``wchar_t`` | Wide character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``const char *`` | UTF-8 string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``const char16_t *`` | UTF-16 string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``const char32_t *`` | UTF-32 string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``const wchar_t *`` | Wide string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::string`` | STL dynamic UTF-8 string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::u16string`` | STL dynamic UTF-16 string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::u32string`` | STL dynamic UTF-32 string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::wstring`` | STL dynamic wide string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::string_view``, | STL C++17 string views | :file:`pybind11/pybind11.h` | +| ``std::u16string_view``, etc. | | | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::pair`` | Pair of two custom types | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::tuple<...>`` | Arbitrary tuple of types | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::reference_wrapper<...>`` | Reference type wrapper | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::complex`` | Complex numbers | :file:`pybind11/complex.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::array`` | STL static array | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::vector`` | STL dynamic array | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::deque`` | STL double-ended queue | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::valarray`` | STL value array | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::list`` | STL linked list | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::map`` | STL ordered map | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::unordered_map`` | STL unordered map | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::set`` | STL ordered set | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::unordered_set`` | STL unordered set | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::optional`` | STL optional type (C++17) | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::experimental::optional`` | STL optional type (exp.) | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::variant<...>`` | Type-safe union (C++17) | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::filesystem::path`` | STL path (C++17) [#]_ | :file:`pybind11/stl/filesystem.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::function<...>`` | STL polymorphic function | :file:`pybind11/functional.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::chrono::duration<...>`` | STL time duration | :file:`pybind11/chrono.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::chrono::time_point<...>`` | STL date/time | :file:`pybind11/chrono.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``Eigen::Matrix<...>`` | Eigen: dense matrix | :file:`pybind11/eigen.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``Eigen::Map<...>`` | Eigen: mapped memory | :file:`pybind11/eigen.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``Eigen::SparseMatrix<...>`` | Eigen: sparse matrix | :file:`pybind11/eigen.h` | ++------------------------------------+---------------------------+-----------------------------------+ + +.. [#] ``std::filesystem::path`` is converted to ``pathlib.Path`` and + ``os.PathLike`` is converted to ``std::filesystem::path``. diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index 3f30c029..03d49b29 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -5,7 +5,7 @@ Automatic conversion ==================== When including the additional header file :file:`pybind11/stl.h`, conversions -between ``std::vector<>``/``std::list<>``/``std::array<>``, +between ``std::vector<>``/``std::deque<>``/``std::list<>``/``std::array<>``/``std::valarray<>``, ``std::set<>``/``std::unordered_set<>``, and ``std::map<>``/``std::unordered_map<>`` and the Python ``list``, ``set`` and ``dict`` data structures are automatically enabled. The types ``std::pair<>`` @@ -42,7 +42,7 @@ types: .. code-block:: cpp // `boost::optional` as an example -- can be any `std::optional`-like container - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : optional_caster> {}; }} @@ -54,7 +54,7 @@ for custom variant types: .. code-block:: cpp // `boost::variant` as an example -- can be any `std::variant`-like container - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : variant_caster> {}; @@ -66,18 +66,27 @@ for custom variant types: return boost::apply_visitor(args...); } }; - }} // namespace pybind11::detail + }} // namespace PYBIND11_NAMESPACE::detail The ``visit_helper`` specialization is not required if your ``name::variant`` provides a ``name::visit()`` function. For any other function name, the specialization must be included to tell pybind11 how to visit the variant. +.. warning:: + + When converting a ``variant`` type, pybind11 follows the same rules as when + determining which function overload to call (:ref:`overload_resolution`), and + so the same caveats hold. In particular, the order in which the ``variant``'s + alternatives are listed is important, since pybind11 will try conversions in + this order. This means that, for example, when converting ``variant``, + the ``bool`` variant will never be selected, as any Python ``bool`` is already + an ``int`` and is convertible to a C++ ``int``. Changing the order of alternatives + (and using ``variant``, in this example) provides a solution. + .. note:: pybind11 only supports the modern implementation of ``boost::variant`` which makes use of variadic templates. This requires Boost 1.56 or newer. - Additionally, on Windows, MSVC 2017 is required because ``boost::variant`` - falls back to the old non-variadic implementation on MSVC 2015. .. _opaque: @@ -157,7 +166,7 @@ the declaration before any binding code (e.g. invocations to ``class_::def()``, etc.). This macro must be specified at the top level (and outside of any namespaces), since -it instantiates a partial template overload. If your binding code consists of +it adds a template instantiation of ``type_caster``. If your binding code consists of multiple compilation units, it must be present in every file (typically via a common header) preceding any usage of ``std::vector``. Opaque types must also have a corresponding ``class_`` declaration to associate them with a name @@ -175,9 +184,6 @@ in Python, and to define a set of available operations, e.g.: }, py::keep_alive<0, 1>()) /* Keep vector alive while iterator is used */ // .... -Please take a look at the :ref:`macro_notes` before using the -``PYBIND11_MAKE_OPAQUE`` macro. - .. seealso:: The file :file:`tests/test_opaque_types.cpp` contains a complete diff --git a/docs/advanced/cast/strings.rst b/docs/advanced/cast/strings.rst index 2cdbade3..e246c521 100644 --- a/docs/advanced/cast/strings.rst +++ b/docs/advanced/cast/strings.rst @@ -1,14 +1,6 @@ Strings, bytes and Unicode conversions ###################################### -.. note:: - - This section discusses string handling in terms of Python 3 strings. For - Python 2.7, replace all occurrences of ``str`` with ``unicode`` and - ``bytes`` with ``str``. Python 2.7 users may find it best to use ``from - __future__ import unicode_literals`` to avoid unintentionally using ``str`` - instead of ``unicode``. - Passing Python strings to C++ ============================= @@ -36,13 +28,13 @@ everywhere `_. } ); -.. code-block:: python +.. code-block:: pycon - >>> utf8_test('🎂') + >>> utf8_test("🎂") utf-8 is icing on the cake. 🎂 - >>> utf8_charptr('🍕') + >>> utf8_charptr("🍕") My favorite food is 🍕 @@ -58,7 +50,9 @@ Passing bytes to C++ -------------------- A Python ``bytes`` object will be passed to C++ functions that accept -``std::string`` or ``char*`` *without* conversion. +``std::string`` or ``char*`` *without* conversion. In order to make a function +*only* accept ``bytes`` (and not ``str``), declare it as taking a ``py::bytes`` +argument. Returning C++ strings to Python @@ -78,7 +72,7 @@ raise a ``UnicodeDecodeError``. } ); -.. code-block:: python +.. code-block:: pycon >>> isinstance(example.std_string_return(), str) True @@ -112,7 +106,7 @@ conversion has the same overhead as implicit conversion. } ); -.. code-block:: python +.. code-block:: pycon >>> str_output() 'Send your résumé to Alice in HR' @@ -141,7 +135,7 @@ returned to Python as ``bytes``, then one can return the data as a } ); -.. code-block:: python +.. code-block:: pycon >>> example.return_bytes() b'\xba\xd0\xba\xd0' @@ -158,7 +152,7 @@ encoding, but cannot convert ``std::string`` back to ``bytes`` implicitly. } ); -.. code-block:: python +.. code-block:: pycon >>> isinstance(example.asymmetry(b"have some bytes"), str) True @@ -202,11 +196,6 @@ decoded to Python ``str``. } ); -.. warning:: - - Wide character strings may not work as described on Python 2.7 or Python - 3.3 compiled with ``--enable-unicode=ucs2``. - Strings in multibyte encodings such as Shift-JIS must transcoded to a UTF-8/16/32 before being returned to Python. @@ -227,16 +216,16 @@ character. m.def("pass_char", [](char c) { return c; }); m.def("pass_wchar", [](wchar_t w) { return w; }); -.. code-block:: python +.. code-block:: pycon - >>> example.pass_char('A') + >>> example.pass_char("A") 'A' While C++ will cast integers to character types (``char c = 0x65;``), pybind11 does not convert Python integers to characters implicitly. The Python function ``chr()`` can be used to convert integers to characters. -.. code-block:: python +.. code-block:: pycon >>> example.pass_char(0x65) TypeError @@ -257,17 +246,17 @@ a combining acute accent). The combining character will be lost if the two-character sequence is passed as an argument, even though it renders as a single grapheme. -.. code-block:: python +.. code-block:: pycon - >>> example.pass_wchar('é') + >>> example.pass_wchar("é") 'é' - >>> combining_e_acute = 'e' + '\u0301' + >>> combining_e_acute = "e" + "\u0301" >>> combining_e_acute 'é' - >>> combining_e_acute == 'é' + >>> combining_e_acute == "é" False >>> example.pass_wchar(combining_e_acute) @@ -276,9 +265,9 @@ single grapheme. Normalizing combining characters before passing the character literal to C++ may resolve *some* of these issues: -.. code-block:: python +.. code-block:: pycon - >>> example.pass_wchar(unicodedata.normalize('NFC', combining_e_acute)) + >>> example.pass_wchar(unicodedata.normalize("NFC", combining_e_acute)) 'é' In some languages (Thai for example), there are `graphemes that cannot be diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 93deeec6..01a490b7 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -9,7 +9,7 @@ that you are already familiar with the basics from :doc:`/classes`. Overriding virtual functions in Python ====================================== -Suppose that a C++ class or interface has a virtual function that we'd like to +Suppose that a C++ class or interface has a virtual function that we'd like to override from within Python (we'll focus on the class ``Animal``; ``Dog`` is given as a specific example of how one would do this with traditional C++ code). @@ -46,11 +46,10 @@ Normally, the binding code for these classes would look as follows: .. code-block:: cpp PYBIND11_MODULE(example, m) { - py::class_ animal(m, "Animal"); - animal + py::class_(m, "Animal") .def("go", &Animal::go); - py::class_(m, "Dog", animal) + py::class_(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); @@ -72,7 +71,7 @@ helper class that is defined as follows: /* Trampoline (need one for each virtual function) */ std::string go(int n_times) override { - PYBIND11_OVERLOAD_PURE( + PYBIND11_OVERRIDE_PURE( std::string, /* Return type */ Animal, /* Parent class */ go, /* Name of function in C++ (must match Python name) */ @@ -81,10 +80,10 @@ helper class that is defined as follows: } }; -The macro :func:`PYBIND11_OVERLOAD_PURE` should be used for pure virtual -functions, and :func:`PYBIND11_OVERLOAD` should be used for functions which have +The macro :c:macro:`PYBIND11_OVERRIDE_PURE` should be used for pure virtual +functions, and :c:macro:`PYBIND11_OVERRIDE` should be used for functions which have a default implementation. There are also two alternate macros -:func:`PYBIND11_OVERLOAD_PURE_NAME` and :func:`PYBIND11_OVERLOAD_NAME` which +:c:macro:`PYBIND11_OVERRIDE_PURE_NAME` and :c:macro:`PYBIND11_OVERRIDE_NAME` which take a string-valued name argument between the *Parent class* and *Name of the function* slots, which defines the name of function in Python. This is required when the C++ and Python versions of the @@ -93,15 +92,14 @@ function have different names, e.g. ``operator()`` vs ``__call__``. The binding code also needs a few minor adaptations (highlighted): .. code-block:: cpp - :emphasize-lines: 2,4,5 + :emphasize-lines: 2,3 PYBIND11_MODULE(example, m) { - py::class_ animal(m, "Animal"); - animal + py::class_(m, "Animal") .def(py::init<>()) .def("go", &Animal::go); - py::class_(m, "Dog", animal) + py::class_(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); @@ -116,15 +114,15 @@ define a constructor as usual. Bindings should be made against the actual class, not the trampoline helper class. .. code-block:: cpp + :emphasize-lines: 3 - py::class_ animal(m, "Animal"); - animal - .def(py::init<>()) - .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */ + py::class_(m, "Animal"); + .def(py::init<>()) + .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */ Note, however, that the above is sufficient for allowing python classes to extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the -necessary steps required to providing proper overload support for inherited +necessary steps required to providing proper overriding support for inherited classes. The Python session below shows how to override ``Animal::go`` and invoke it via @@ -135,14 +133,14 @@ a virtual method call. >>> from example import * >>> d = Dog() >>> call_go(d) - u'woof! woof! woof! ' + 'woof! woof! woof! ' >>> class Cat(Animal): ... def go(self, n_times): - ... return "meow! " * n_times + ... return "meow! " * n_times ... >>> c = Cat() >>> call_go(c) - u'meow! meow! meow! ' + 'meow! meow! meow! ' If you are defining a custom constructor in a derived Python class, you *must* ensure that you explicitly call the bound C++ constructor using ``__init__``, @@ -151,14 +149,19 @@ memory for the C++ portion of the instance will be left uninitialized, which will generally leave the C++ instance in an invalid state and cause undefined behavior if the C++ instance is subsequently used. +.. versionchanged:: 2.6 + The default pybind11 metaclass will throw a ``TypeError`` when it detects + that ``__init__`` was not called by a derived class. + Here is an example: .. code-block:: python - class Dachschund(Dog): + class Dachshund(Dog): def __init__(self, name): - Dog.__init__(self) # Without this, undefind behavior may occur if the C++ portions are referenced. + Dog.__init__(self) # Without this, a TypeError is raised. self.name = name + def bark(self): return "yap!" @@ -179,15 +182,24 @@ Please take a look at the :ref:`macro_notes` before using this feature. - because in these cases there is no C++ variable to reference (the value is stored in the referenced Python variable), pybind11 provides one in - the PYBIND11_OVERLOAD macros (when needed) with static storage duration. - Note that this means that invoking the overloaded method on *any* + the PYBIND11_OVERRIDE macros (when needed) with static storage duration. + Note that this means that invoking the overridden method on *any* instance will change the referenced value stored in *all* instances of that type. - Attempts to modify a non-const reference will not have the desired effect: it will change only the static cache variable, but this change will not propagate to underlying Python instance, and the change will be - replaced the next time the overload is invoked. + replaced the next time the override is invoked. + +.. warning:: + + The :c:macro:`PYBIND11_OVERRIDE` and accompanying macros used to be called + ``PYBIND11_OVERLOAD`` up until pybind11 v2.5.0, and :func:`get_override` + used to be called ``get_overload``. This naming was corrected and the older + macro and function names may soon be deprecated, in order to reduce + confusion with overloaded functions and methods and ``py::overload_cast`` + (see :ref:`classes`). .. seealso:: @@ -235,20 +247,20 @@ override the ``name()`` method): class PyAnimal : public Animal { public: using Animal::Animal; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, Animal, go, n_times); } - std::string name() override { PYBIND11_OVERLOAD(std::string, Animal, name, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, Animal, go, n_times); } + std::string name() override { PYBIND11_OVERRIDE(std::string, Animal, name, ); } }; class PyDog : public Dog { public: using Dog::Dog; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, Dog, go, n_times); } - std::string name() override { PYBIND11_OVERLOAD(std::string, Dog, name, ); } - std::string bark() override { PYBIND11_OVERLOAD(std::string, Dog, bark, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE(std::string, Dog, go, n_times); } + std::string name() override { PYBIND11_OVERRIDE(std::string, Dog, name, ); } + std::string bark() override { PYBIND11_OVERRIDE(std::string, Dog, bark, ); } }; .. note:: - Note the trailing commas in the ``PYBIND11_OVERLOAD`` calls to ``name()`` + Note the trailing commas in the ``PYBIND11_OVERRIDE`` calls to ``name()`` and ``bark()``. These are needed to portably implement a trampoline for a function that does not take any arguments. For functions that take a nonzero number of arguments, the trailing comma must be omitted. @@ -263,9 +275,9 @@ declare or override any virtual methods itself: class PyHusky : public Husky { public: using Husky::Husky; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, Husky, go, n_times); } - std::string name() override { PYBIND11_OVERLOAD(std::string, Husky, name, ); } - std::string bark() override { PYBIND11_OVERLOAD(std::string, Husky, bark, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, Husky, go, n_times); } + std::string name() override { PYBIND11_OVERRIDE(std::string, Husky, name, ); } + std::string bark() override { PYBIND11_OVERRIDE(std::string, Husky, bark, ); } }; There is, however, a technique that can be used to avoid this duplication @@ -278,15 +290,15 @@ follows: template class PyAnimal : public AnimalBase { public: using AnimalBase::AnimalBase; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, AnimalBase, go, n_times); } - std::string name() override { PYBIND11_OVERLOAD(std::string, AnimalBase, name, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, AnimalBase, go, n_times); } + std::string name() override { PYBIND11_OVERRIDE(std::string, AnimalBase, name, ); } }; template class PyDog : public PyAnimal { public: using PyAnimal::PyAnimal; // Inherit constructors // Override PyAnimal's pure virtual go() with a non-pure one: - std::string go(int n_times) override { PYBIND11_OVERLOAD(std::string, DogBase, go, n_times); } - std::string bark() override { PYBIND11_OVERLOAD(std::string, DogBase, bark, ); } + std::string go(int n_times) override { PYBIND11_OVERRIDE(std::string, DogBase, go, n_times); } + std::string bark() override { PYBIND11_OVERRIDE(std::string, DogBase, bark, ); } }; This technique has the advantage of requiring just one trampoline method to be @@ -300,8 +312,8 @@ The classes are then registered with pybind11 using: .. code-block:: cpp py::class_> animal(m, "Animal"); - py::class_> dog(m, "Dog"); - py::class_> husky(m, "Husky"); + py::class_> dog(m, "Dog"); + py::class_> husky(m, "Husky"); // ... add animal, dog, husky definitions Note that ``Husky`` did not require a dedicated trampoline template class at @@ -327,6 +339,10 @@ can now create a python class that inherits from ``Dog``: Extended trampoline class functionality ======================================= +.. _extended_class_functionality_forced_trampoline: + +Forced trampoline class initialisation +-------------------------------------- The trampoline classes described in the previous sections are, by default, only initialized when needed. More specifically, they are initialized when a python class actually inherits from a registered type (instead of merely creating an @@ -335,7 +351,7 @@ valid for the trampoline class but not the registered class. This is primarily for performance reasons: when the trampoline class is not needed for anything except virtual method dispatching, not initializing the trampoline class improves performance by avoiding needing to do a run-time check to see if the -inheriting python instance has an overloaded method. +inheriting python instance has an overridden method. Sometimes, however, it is useful to always initialize a trampoline class as an intermediate class that does more than just handle virtual method dispatching. @@ -354,6 +370,45 @@ ensuring member initialization and (eventual) destruction. See the file :file:`tests/test_virtual_functions.cpp` for complete examples showing both normal and forced trampoline instantiation. +Different method signatures +--------------------------- +The macro's introduced in :ref:`overriding_virtuals` cover most of the standard +use cases when exposing C++ classes to Python. Sometimes it is hard or unwieldy +to create a direct one-on-one mapping between the arguments and method return +type. + +An example would be when the C++ signature contains output arguments using +references (See also :ref:`faq_reference_arguments`). Another way of solving +this is to use the method body of the trampoline class to do conversions to the +input and return of the Python method. + +The main building block to do so is the :func:`get_override`, this function +allows retrieving a method implemented in Python from within the trampoline's +methods. Consider for example a C++ method which has the signature +``bool myMethod(int32_t& value)``, where the return indicates whether +something should be done with the ``value``. This can be made convenient on the +Python side by allowing the Python function to return ``None`` or an ``int``: + +.. code-block:: cpp + + bool MyClass::myMethod(int32_t& value) + { + pybind11::gil_scoped_acquire gil; // Acquire the GIL while in this scope. + // Try to look up the overridden method on the Python side. + pybind11::function override = pybind11::get_override(this, "myMethod"); + if (override) { // method is found + auto obj = override(value); // Call the Python function. + if (py::isinstance(obj)) { // check if it returned a Python integer type + value = obj.cast(); // Cast it and assign it to the value. + return true; // Return true; value should be used. + } else { + return false; // Python returned none, return false. + } + } + return false; // Alternatively return MyClass::myMethod(value); + } + + .. _custom_constructors: Custom constructors @@ -513,6 +568,46 @@ crucial that instances are deallocated on the C++ side to avoid memory leaks. py::class_>(m, "MyClass") .def(py::init<>()) +.. _destructors_that_call_python: + +Destructors that call Python +============================ + +If a Python function is invoked from a C++ destructor, an exception may be thrown +of type :class:`error_already_set`. If this error is thrown out of a class destructor, +``std::terminate()`` will be called, terminating the process. Class destructors +must catch all exceptions of type :class:`error_already_set` to discard the Python +exception using :func:`error_already_set::discard_as_unraisable`. + +Every Python function should be treated as *possibly throwing*. When a Python generator +stops yielding items, Python will throw a ``StopIteration`` exception, which can pass +though C++ destructors if the generator's stack frame holds the last reference to C++ +objects. + +For more information, see :ref:`the documentation on exceptions `. + +.. code-block:: cpp + + class MyClass { + public: + ~MyClass() { + try { + py::print("Even printing is dangerous in a destructor"); + py::exec("raise ValueError('This is an unraisable exception')"); + } catch (py::error_already_set &e) { + // error_context should be information about where/why the occurred, + // e.g. use __func__ to get the name of the current function + e.discard_as_unraisable(__func__); + } + } + }; + +.. note:: + + pybind11 does not support C++ destructors marked ``noexcept(false)``. + +.. versionadded:: 2.6 + .. _implicit_conversions: Implicit conversions @@ -621,6 +716,7 @@ to Python. .def(py::self *= float()) .def(float() * py::self) .def(py::self * float()) + .def(-py::self) .def("__repr__", &Vector2::toString); } @@ -709,7 +805,7 @@ to bind these two functions: } )); -The ``__setstate__`` part of the ``py::picke()`` definition follows the same +The ``__setstate__`` part of the ``py::pickle()`` definition follows the same rules as the single-argument version of ``py::init()``. The return type can be a value, pointer or holder type. See :ref:`custom_constructors` for details. @@ -717,22 +813,21 @@ An instance can now be pickled as follows: .. code-block:: python - try: - import cPickle as pickle # Use cPickle on Python 2.7 - except ImportError: - import pickle + import pickle p = Pickleable("test_value") p.setExtra(15) - data = pickle.dumps(p, 2) + data = pickle.dumps(p) -Note that only the cPickle module is supported on Python 2.7. The second -argument to ``dumps`` is also crucial: it selects the pickle protocol version -2, since the older version 1 is not supported. Newer versions are also fine—for -instance, specify ``-1`` to always use the latest available version. Beware: -failure to follow these instructions will cause important pybind11 memory -allocation routines to be skipped during unpickling, which will likely lead to -memory corruption and/or segmentation faults. + +.. note:: + If given, the second argument to ``dumps`` must be 2 or larger - 0 and 1 are + not supported. Newer versions are also fine; for instance, specify ``-1`` to + always use the latest available version. Beware: failure to follow these + instructions will cause important pybind11 memory allocation routines to be + skipped during unpickling, which will likely lead to memory corruption + and/or segmentation faults. Python defaults to version 3 (Python 3-3.7) and + version 4 for Python 3.8+. .. seealso:: @@ -742,6 +837,36 @@ memory corruption and/or segmentation faults. .. [#f3] http://docs.python.org/3/library/pickle.html#pickling-class-instances +Deepcopy support +================ + +Python normally uses references in assignments. Sometimes a real copy is needed +to prevent changing all copies. The ``copy`` module [#f5]_ provides these +capabilities. + +A class with pickle support is automatically also (deep)copy +compatible. However, performance can be improved by adding custom +``__copy__`` and ``__deepcopy__`` methods. + +For simple classes (deep)copy can be enabled by using the copy constructor, +which should look as follows: + +.. code-block:: cpp + + py::class_(m, "Copyable") + .def("__copy__", [](const Copyable &self) { + return Copyable(self); + }) + .def("__deepcopy__", [](const Copyable &self, py::dict) { + return Copyable(self); + }, "memo"_a); + +.. note:: + + Dynamic attributes will not be copied in this example. + +.. [#f5] https://docs.python.org/3/library/copy.html + Multiple Inheritance ==================== @@ -760,7 +885,7 @@ document)---pybind11 will automatically find out which is which. The only requirement is that the first template argument is the type to be declared. It is also permitted to inherit multiply from exported C++ classes in Python, -as well as inheriting from multiple Python and/or pybind-exported classes. +as well as inheriting from multiple Python and/or pybind11-exported classes. There is one caveat regarding the implementation of this feature: @@ -781,7 +906,7 @@ are listed. Module-local class bindings =========================== -When creating a binding for a class, pybind by default makes that binding +When creating a binding for a class, pybind11 by default makes that binding "global" across modules. What this means is that a type defined in one module can be returned from any module resulting in the same Python type. For example, this allows the following: @@ -982,7 +1107,7 @@ described trampoline: class Trampoline : public A { public: - int foo() const override { PYBIND11_OVERLOAD(int, A, foo, ); } + int foo() const override { PYBIND11_OVERRIDE(int, A, foo, ); } }; class Publicist : public A { @@ -993,9 +1118,218 @@ described trampoline: py::class_(m, "A") // <-- `Trampoline` here .def("foo", &Publicist::foo); // <-- `Publicist` here, not `Trampoline`! +Binding final classes +===================== + +Some classes may not be appropriate to inherit from. In C++11, classes can +use the ``final`` specifier to ensure that a class cannot be inherited from. +The ``py::is_final`` attribute can be used to ensure that Python classes +cannot inherit from a specified type. The underlying C++ type does not need +to be declared final. + +.. code-block:: cpp + + class IsFinal final {}; + + py::class_(m, "IsFinal", py::is_final()); + +When you try to inherit from such a class in Python, you will now get this +error: + +.. code-block:: pycon + + >>> class PyFinalChild(IsFinal): + ... pass + ... + TypeError: type 'IsFinal' is not an acceptable base type + +.. note:: This attribute is currently ignored on PyPy + +.. versionadded:: 2.6 + +Binding classes with template parameters +======================================== + +pybind11 can also wrap classes that have template parameters. Consider these classes: + +.. code-block:: cpp + + struct Cat {}; + struct Dog {}; + + template + struct Cage { + Cage(PetType& pet); + PetType& get(); + }; + +C++ templates may only be instantiated at compile time, so pybind11 can only +wrap instantiated templated classes. You cannot wrap a non-instantiated template: + +.. code-block:: cpp + + // BROKEN (this will not compile) + py::class_(m, "Cage"); + .def("get", &Cage::get); + +You must explicitly specify each template/type combination that you want to +wrap separately. + +.. code-block:: cpp + + // ok + py::class_>(m, "CatCage") + .def("get", &Cage::get); + + // ok + py::class_>(m, "DogCage") + .def("get", &Cage::get); + +If your class methods have template parameters you can wrap those as well, +but once again each instantiation must be explicitly specified: + +.. code-block:: cpp + + typename + struct MyClass { + template + T fn(V v); + }; + + py::class>(m, "MyClassT") + .def("fn", &MyClass::fn); + +Custom automatic downcasters +============================ + +As explained in :ref:`inheritance`, pybind11 comes with built-in +understanding of the dynamic type of polymorphic objects in C++; that +is, returning a Pet to Python produces a Python object that knows it's +wrapping a Dog, if Pet has virtual methods and pybind11 knows about +Dog and this Pet is in fact a Dog. Sometimes, you might want to +provide this automatic downcasting behavior when creating bindings for +a class hierarchy that does not use standard C++ polymorphism, such as +LLVM [#f4]_. As long as there's some way to determine at runtime +whether a downcast is safe, you can proceed by specializing the +``pybind11::polymorphic_type_hook`` template: + +.. code-block:: cpp + + enum class PetKind { Cat, Dog, Zebra }; + struct Pet { // Not polymorphic: has no virtual methods + const PetKind kind; + int age = 0; + protected: + Pet(PetKind _kind) : kind(_kind) {} + }; + struct Dog : Pet { + Dog() : Pet(PetKind::Dog) {} + std::string sound = "woof!"; + std::string bark() const { return sound; } + }; + + namespace PYBIND11_NAMESPACE { + template<> struct polymorphic_type_hook { + static const void *get(const Pet *src, const std::type_info*& type) { + // note that src may be nullptr + if (src && src->kind == PetKind::Dog) { + type = &typeid(Dog); + return static_cast(src); + } + return src; + } + }; + } // namespace PYBIND11_NAMESPACE + +When pybind11 wants to convert a C++ pointer of type ``Base*`` to a +Python object, it calls ``polymorphic_type_hook::get()`` to +determine if a downcast is possible. The ``get()`` function should use +whatever runtime information is available to determine if its ``src`` +parameter is in fact an instance of some class ``Derived`` that +inherits from ``Base``. If it finds such a ``Derived``, it sets ``type += &typeid(Derived)`` and returns a pointer to the ``Derived`` object +that contains ``src``. Otherwise, it just returns ``src``, leaving +``type`` at its default value of nullptr. If you set ``type`` to a +type that pybind11 doesn't know about, no downcasting will occur, and +the original ``src`` pointer will be used with its static type +``Base*``. + +It is critical that the returned pointer and ``type`` argument of +``get()`` agree with each other: if ``type`` is set to something +non-null, the returned pointer must point to the start of an object +whose type is ``type``. If the hierarchy being exposed uses only +single inheritance, a simple ``return src;`` will achieve this just +fine, but in the general case, you must cast ``src`` to the +appropriate derived-class pointer (e.g. using +``static_cast(src)``) before allowing it to be returned as a +``void*``. + +.. [#f4] https://llvm.org/docs/HowToSetUpLLVMStyleRTTI.html + .. note:: - MSVC 2015 has a compiler bug (fixed in version 2017) which - requires a more explicit function binding in the form of - ``.def("foo", static_cast(&Publicist::foo));`` - where ``int (A::*)() const`` is the type of ``A::foo``. + pybind11's standard support for downcasting objects whose types + have virtual methods is implemented using + ``polymorphic_type_hook`` too, using the standard C++ ability to + determine the most-derived type of a polymorphic object using + ``typeid()`` and to cast a base pointer to that most-derived type + (even if you don't know what it is) using ``dynamic_cast``. + +.. seealso:: + + The file :file:`tests/test_tagbased_polymorphic.cpp` contains a + more complete example, including a demonstration of how to provide + automatic downcasting for an entire class hierarchy without + writing one get() function for each class. + +Accessing the type object +========================= + +You can get the type object from a C++ class that has already been registered using: + +.. code-block:: cpp + + py::type T_py = py::type::of(); + +You can directly use ``py::type::of(ob)`` to get the type object from any python +object, just like ``type(ob)`` in Python. + +.. note:: + + Other types, like ``py::type::of()``, do not work, see :ref:`type-conversions`. + +.. versionadded:: 2.6 + +Custom type setup +================= + +For advanced use cases, such as enabling garbage collection support, you may +wish to directly manipulate the ``PyHeapTypeObject`` corresponding to a +``py::class_`` definition. + +You can do that using ``py::custom_type_setup``: + +.. code-block:: cpp + + struct OwnsPythonObjects { + py::object value = py::none(); + }; + py::class_ cls( + m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { + auto *type = &heap_type->ht_type; + type->tp_flags |= Py_TPFLAGS_HAVE_GC; + type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) { + auto &self = py::cast(py::handle(self_base)); + Py_VISIT(self.value.ptr()); + return 0; + }; + type->tp_clear = [](PyObject *self_base) { + auto &self = py::cast(py::handle(self_base)); + self.value = py::none(); + return 0; + }; + })); + cls.def(py::init<>()); + cls.def_readwrite("value", &OwnsPythonObjects::value); + +.. versionadded:: 2.8 diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 39303160..dd980d48 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.0) + cmake_minimum_required(VERSION 3.4) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` @@ -40,15 +40,15 @@ The essential structure of the ``main.cpp`` file looks like this: } The interpreter must be initialized before using any Python API, which includes -all the functions and classes in pybind11. The RAII guard class `scoped_interpreter` +all the functions and classes in pybind11. The RAII guard class ``scoped_interpreter`` takes care of the interpreter lifetime. After the guard is destroyed, the interpreter shuts down and clears its memory. No Python functions can be called after this. Executing Python code ===================== -There are a few different ways to run Python code. One option is to use `eval`, -`exec` or `eval_file`, as explained in :ref:`eval`. Here is a quick example in +There are a few different ways to run Python code. One option is to use ``eval``, +``exec`` or ``eval_file``, as explained in :ref:`eval`. Here is a quick example in the context of an executable with an embedded interpreter: .. code-block:: cpp @@ -108,11 +108,11 @@ The two approaches can also be combined: Importing modules ================= -Python modules can be imported using `module::import()`: +Python modules can be imported using ``module_::import()``: .. code-block:: cpp - py::module sys = py::module::import("sys"); + py::module_ sys = py::module_::import("sys"); py::print(sys.attr("path")); For convenience, the current working directory is included in ``sys.path`` when @@ -122,18 +122,19 @@ embedding the interpreter. This makes it easy to import local Python files: """calc.py located in the working directory""" + def add(i, j): return i + j .. code-block:: cpp - py::module calc = py::module::import("calc"); + py::module_ calc = py::module_::import("calc"); py::object result = calc.attr("add")(1, 2); int n = result.cast(); assert(n == 3); -Modules can be reloaded using `module::reload()` if the source is modified e.g. +Modules can be reloaded using ``module_::reload()`` if the source is modified e.g. by an external process. This can be useful in scenarios where the application imports a user defined data processing script which needs to be updated after changes by the user. Note that this function does not reload modules recursively. @@ -143,7 +144,7 @@ changes by the user. Note that this function does not reload modules recursively Adding embedded modules ======================= -Embedded binary modules can be added using the `PYBIND11_EMBEDDED_MODULE` macro. +Embedded binary modules can be added using the ``PYBIND11_EMBEDDED_MODULE`` macro. Note that the definition must be placed at global scope. They can be imported like any other module. @@ -153,7 +154,7 @@ like any other module. namespace py = pybind11; PYBIND11_EMBEDDED_MODULE(fast_calc, m) { - // `m` is a `py::module` which is used to bind functions and classes + // `m` is a `py::module_` which is used to bind functions and classes m.def("add", [](int i, int j) { return i + j; }); @@ -162,14 +163,14 @@ like any other module. int main() { py::scoped_interpreter guard{}; - auto fast_calc = py::module::import("fast_calc"); + auto fast_calc = py::module_::import("fast_calc"); auto result = fast_calc.attr("add")(1, 2).cast(); assert(result == 3); } Unlike extension modules where only a single binary module can be created, on the embedded side an unlimited number of modules can be added using multiple -`PYBIND11_EMBEDDED_MODULE` definitions (as long as they have unique names). +``PYBIND11_EMBEDDED_MODULE`` definitions (as long as they have unique names). These modules are added to Python's list of builtins, so they can also be imported in pure Python files loaded by the interpreter. Everything interacts @@ -196,7 +197,7 @@ naturally: int main() { py::scoped_interpreter guard{}; - auto py_module = py::module::import("py_module"); + auto py_module = py::module_::import("py_module"); auto locals = py::dict("fmt"_a="{} + {} = {}", **py_module.attr("__dict__")); assert(locals["a"].cast() == 1); @@ -215,9 +216,9 @@ naturally: Interpreter lifetime ==================== -The Python interpreter shuts down when `scoped_interpreter` is destroyed. After +The Python interpreter shuts down when ``scoped_interpreter`` is destroyed. After this, creating a new instance will restart the interpreter. Alternatively, the -`initialize_interpreter` / `finalize_interpreter` pair of functions can be used +``initialize_interpreter`` / ``finalize_interpreter`` pair of functions can be used to directly set the state at any time. Modules created with pybind11 can be safely re-initialized after the interpreter @@ -229,8 +230,8 @@ global data. All the details can be found in the CPython documentation. .. warning:: - Creating two concurrent `scoped_interpreter` guards is a fatal error. So is - calling `initialize_interpreter` for a second time after the interpreter + Creating two concurrent ``scoped_interpreter`` guards is a fatal error. So is + calling ``initialize_interpreter`` for a second time after the interpreter has already been initialized. Do not use the raw CPython API functions ``Py_Initialize`` and @@ -241,7 +242,7 @@ global data. All the details can be found in the CPython documentation. Sub-interpreter support ======================= -Creating multiple copies of `scoped_interpreter` is not possible because it +Creating multiple copies of ``scoped_interpreter`` is not possible because it represents the main Python interpreter. Sub-interpreters are something different and they do permit the existence of multiple interpreters. This is an advanced feature of the CPython API and should be handled with care. pybind11 does not @@ -257,5 +258,5 @@ We'll just mention a couple of caveats the sub-interpreters support in pybind11: 2. Managing multiple threads, multiple interpreters and the GIL can be challenging and there are several caveats here, even within the pure CPython API (please refer to the Python docs for details). As for - pybind11, keep in mind that `gil_scoped_release` and `gil_scoped_acquire` + pybind11, keep in mind that ``gil_scoped_release`` and ``gil_scoped_acquire`` do not take sub-interpreters into account. diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 3122c372..53981dc0 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -1,59 +1,72 @@ Exceptions ########## -Built-in exception translation -============================== +Built-in C++ to Python exception translation +============================================ -When C++ code invoked from Python throws an ``std::exception``, it is -automatically converted into a Python ``Exception``. pybind11 defines multiple -special exception classes that will map to different types of Python -exceptions: +When Python calls C++ code through pybind11, pybind11 provides a C++ exception handler +that will trap C++ exceptions, translate them to the corresponding Python exception, +and raise them so that Python code can handle them. + +pybind11 defines translations for ``std::exception`` and its standard +subclasses, and several special exception classes that translate to specific +Python exceptions. Note that these are not actually Python exceptions, so they +cannot be examined using the Python C API. Instead, they are pure C++ objects +that pybind11 will translate the corresponding Python exception when they arrive +at its exception handler. .. tabularcolumns:: |p{0.5\textwidth}|p{0.45\textwidth}| -+--------------------------------------+------------------------------+ -| C++ exception type | Python exception type | -+======================================+==============================+ -| :class:`std::exception` | ``RuntimeError`` | -+--------------------------------------+------------------------------+ -| :class:`std::bad_alloc` | ``MemoryError`` | -+--------------------------------------+------------------------------+ -| :class:`std::domain_error` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`std::invalid_argument` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`std::length_error` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`std::out_of_range` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`std::range_error` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`pybind11::stop_iteration` | ``StopIteration`` (used to | -| | implement custom iterators) | -+--------------------------------------+------------------------------+ -| :class:`pybind11::index_error` | ``IndexError`` (used to | -| | indicate out of bounds | -| | accesses in ``__getitem__``, | -| | ``__setitem__``, etc.) | -+--------------------------------------+------------------------------+ -| :class:`pybind11::value_error` | ``ValueError`` (used to | -| | indicate wrong value passed | -| | in ``container.remove(...)`` | -+--------------------------------------+------------------------------+ -| :class:`pybind11::key_error` | ``KeyError`` (used to | -| | indicate out of bounds | -| | accesses in ``__getitem__``, | -| | ``__setitem__`` in dict-like | -| | objects, etc.) | -+--------------------------------------+------------------------------+ -| :class:`pybind11::error_already_set` | Indicates that the Python | -| | exception flag has already | -| | been initialized | -+--------------------------------------+------------------------------+ - -When a Python function invoked from C++ throws an exception, it is converted -into a C++ exception of type :class:`error_already_set` whose string payload -contains a textual summary. ++--------------------------------------+--------------------------------------+ +| Exception thrown by C++ | Translated to Python exception type | ++======================================+======================================+ +| :class:`std::exception` | ``RuntimeError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::bad_alloc` | ``MemoryError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::domain_error` | ``ValueError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::invalid_argument` | ``ValueError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::length_error` | ``ValueError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::out_of_range` | ``IndexError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::range_error` | ``ValueError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::overflow_error` | ``OverflowError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::stop_iteration` | ``StopIteration`` (used to implement | +| | custom iterators) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::index_error` | ``IndexError`` (used to indicate out | +| | of bounds access in ``__getitem__``, | +| | ``__setitem__``, etc.) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::key_error` | ``KeyError`` (used to indicate out | +| | of bounds access in ``__getitem__``, | +| | ``__setitem__`` in dict-like | +| | objects, etc.) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::value_error` | ``ValueError`` (used to indicate | +| | wrong value passed in | +| | ``container.remove(...)``) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::type_error` | ``TypeError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::buffer_error` | ``BufferError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::import_error` | ``ImportError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::attribute_error` | ``AttributeError`` | ++--------------------------------------+--------------------------------------+ +| Any other exception | ``RuntimeError`` | ++--------------------------------------+--------------------------------------+ + +Exception translation is not bidirectional. That is, *catching* the C++ +exceptions defined above will not trap exceptions that originate from +Python. For that, catch :class:`pybind11::error_already_set`. See :ref:`below +` for further details. There is also a special exception :class:`cast_error` that is thrown by :func:`handle::call` when the input arguments cannot be converted to Python @@ -64,9 +77,10 @@ Registering custom translators If the default exception conversion policy described above is insufficient, pybind11 also provides support for registering custom exception translators. -To register a simple exception conversion that translates a C++ exception into -a new Python exception using the C++ exception's ``what()`` method, a helper -function is available: +Similar to pybind11 classes, exception translators can be local to the module +they are defined in or global to the entire python session. To register a simple +exception conversion that translates a C++ exception into a new Python exception +using the C++ exception's ``what()`` method, a helper function is available: .. code-block:: cpp @@ -76,16 +90,39 @@ This call creates a Python exception class with the name ``PyExp`` in the given module and automatically converts any encountered exceptions of type ``CppExp`` into Python exceptions of type ``PyExp``. -When more advanced exception translation is needed, the function -``py::register_exception_translator(translator)`` can be used to register +A matching function is available for registering a local exception translator: + +.. code-block:: cpp + + py::register_local_exception(module, "PyExp"); + + +It is possible to specify base class for the exception using the third +parameter, a ``handle``: + +.. code-block:: cpp + + py::register_exception(module, "PyExp", PyExc_RuntimeError); + py::register_local_exception(module, "PyExp", PyExc_RuntimeError); + +Then ``PyExp`` can be caught both as ``PyExp`` and ``RuntimeError``. + +The class objects of the built-in Python exceptions are listed in the Python +documentation on `Standard Exceptions `_. +The default base class is ``PyExc_Exception``. + +When more advanced exception translation is needed, the functions +``py::register_exception_translator(translator)`` and +``py::register_local_exception_translator(translator)`` can be used to register functions that can translate arbitrary exception types (and which may include -additional logic to do so). The function takes a stateless callable (e.g. a +additional logic to do so). The functions takes a stateless callable (e.g. a function pointer or a lambda function without captured variables) with the call signature ``void(std::exception_ptr)``. When a C++ exception is thrown, the registered exception translators are tried in reverse order of registration (i.e. the last registered translator gets the -first shot at handling the exception). +first shot at handling the exception). All local translators will be tried +before a global translator is tried. Inside the translator, ``std::rethrow_exception`` should be used within a try block to re-throw the exception. One or more catch clauses to catch @@ -98,7 +135,6 @@ and use this in the associated exception translator (note: it is often useful to make this a static declaration when using it inside a lambda expression without requiring capturing). - The following example demonstrates this for a hypothetical exception classes ``MyCustomException`` and ``OtherException``: the first is translated to a custom python exception ``MyCustomError``, while the second is translated to a @@ -132,7 +168,7 @@ section. .. note:: - You must call either ``PyErr_SetString`` or a custom exception's call + Call either ``PyErr_SetString`` or a custom exception's call operator (``exc(string)``) for every exception caught in a custom exception translator. Failure to do so will cause Python to crash with ``SystemError: error return without exception set``. @@ -140,3 +176,226 @@ section. Exceptions that you do not plan to handle should simply not be caught, or may be explicitly (re-)thrown to delegate it to the other, previously-declared existing exception translators. + + Note that ``libc++`` and ``libstdc++`` `behave differently under macOS + `_ + with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI + boundaries need to be explicitly exported, as exercised in + ``tests/test_exceptions.h``. See also: + "Problems with C++ exceptions" under `GCC Wiki `_. + + +Local vs Global Exception Translators +===================================== + +When a global exception translator is registered, it will be applied across all +modules in the reverse order of registration. This can create behavior where the +order of module import influences how exceptions are translated. + +If module1 has the following translator: + +.. code-block:: cpp + + py::register_exception_translator([](std::exception_ptr p) { + try { + if (p) std::rethrow_exception(p); + } catch (const std::invalid_argument &e) { + PyErr_SetString("module1 handled this") + } + } + +and module2 has the following similar translator: + +.. code-block:: cpp + + py::register_exception_translator([](std::exception_ptr p) { + try { + if (p) std::rethrow_exception(p); + } catch (const std::invalid_argument &e) { + PyErr_SetString("module2 handled this") + } + } + +then which translator handles the invalid_argument will be determined by the +order that module1 and module2 are imported. Since exception translators are +applied in the reverse order of registration, which ever module was imported +last will "win" and that translator will be applied. + +If there are multiple pybind11 modules that share exception types (either +standard built-in or custom) loaded into a single python instance and +consistent error handling behavior is needed, then local translators should be +used. + +Changing the previous example to use ``register_local_exception_translator`` +would mean that when invalid_argument is thrown in the module2 code, the +module2 translator will always handle it, while in module1, the module1 +translator will do the same. + +.. _handling_python_exceptions_cpp: + +Handling exceptions from Python in C++ +====================================== + +When C++ calls Python functions, such as in a callback function or when +manipulating Python objects, and Python raises an ``Exception``, pybind11 +converts the Python exception into a C++ exception of type +:class:`pybind11::error_already_set` whose payload contains a C++ string textual +summary and the actual Python exception. ``error_already_set`` is used to +propagate Python exception back to Python (or possibly, handle them in C++). + +.. tabularcolumns:: |p{0.5\textwidth}|p{0.45\textwidth}| + ++--------------------------------------+--------------------------------------+ +| Exception raised in Python | Thrown as C++ exception type | ++======================================+======================================+ +| Any Python ``Exception`` | :class:`pybind11::error_already_set` | ++--------------------------------------+--------------------------------------+ + +For example: + +.. code-block:: cpp + + try { + // open("missing.txt", "r") + auto file = py::module_::import("io").attr("open")("missing.txt", "r"); + auto text = file.attr("read")(); + file.attr("close")(); + } catch (py::error_already_set &e) { + if (e.matches(PyExc_FileNotFoundError)) { + py::print("missing.txt not found"); + } else if (e.matches(PyExc_PermissionError)) { + py::print("missing.txt found but not accessible"); + } else { + throw; + } + } + +Note that C++ to Python exception translation does not apply here, since that is +a method for translating C++ exceptions to Python, not vice versa. The error raised +from Python is always ``error_already_set``. + +This example illustrates this behavior: + +.. code-block:: cpp + + try { + py::eval("raise ValueError('The Ring')"); + } catch (py::value_error &boromir) { + // Boromir never gets the ring + assert(false); + } catch (py::error_already_set &frodo) { + // Frodo gets the ring + py::print("I will take the ring"); + } + + try { + // py::value_error is a request for pybind11 to raise a Python exception + throw py::value_error("The ball"); + } catch (py::error_already_set &cat) { + // cat won't catch the ball since + // py::value_error is not a Python exception + assert(false); + } catch (py::value_error &dog) { + // dog will catch the ball + py::print("Run Spot run"); + throw; // Throw it again (pybind11 will raise ValueError) + } + +Handling errors from the Python C API +===================================== + +Where possible, use :ref:`pybind11 wrappers ` instead of calling +the Python C API directly. When calling the Python C API directly, in +addition to manually managing reference counts, one must follow the pybind11 +error protocol, which is outlined here. + +After calling the Python C API, if Python returns an error, +``throw py::error_already_set();``, which allows pybind11 to deal with the +exception and pass it back to the Python interpreter. This includes calls to +the error setting functions such as ``PyErr_SetString``. + +.. code-block:: cpp + + PyErr_SetString(PyExc_TypeError, "C API type error demo"); + throw py::error_already_set(); + + // But it would be easier to simply... + throw py::type_error("pybind11 wrapper type error"); + +Alternately, to ignore the error, call `PyErr_Clear +`_. + +Any Python error must be thrown or cleared, or Python/pybind11 will be left in +an invalid state. + +Chaining exceptions ('raise from') +================================== + +Python has a mechanism for indicating that exceptions were caused by other +exceptions: + +.. code-block:: py + + try: + print(1 / 0) + except Exception as exc: + raise RuntimeError("could not divide by zero") from exc + +To do a similar thing in pybind11, you can use the ``py::raise_from`` function. It +sets the current python error indicator, so to continue propagating the exception +you should ``throw py::error_already_set()``. + +.. code-block:: cpp + + try { + py::eval("print(1 / 0")); + } catch (py::error_already_set &e) { + py::raise_from(e, PyExc_RuntimeError, "could not divide by zero"); + throw py::error_already_set(); + } + +.. versionadded:: 2.8 + +.. _unraisable_exceptions: + +Handling unraisable exceptions +============================== + +If a Python function invoked from a C++ destructor or any function marked +``noexcept(true)`` (collectively, "noexcept functions") throws an exception, there +is no way to propagate the exception, as such functions may not throw. +Should they throw or fail to catch any exceptions in their call graph, +the C++ runtime calls ``std::terminate()`` to abort immediately. + +Similarly, Python exceptions raised in a class's ``__del__`` method do not +propagate, but are logged by Python as an unraisable error. In Python 3.8+, a +`system hook is triggered +`_ +and an auditing event is logged. + +Any noexcept function should have a try-catch block that traps +class:`error_already_set` (or any other exception that can occur). Note that +pybind11 wrappers around Python exceptions such as +:class:`pybind11::value_error` are *not* Python exceptions; they are C++ +exceptions that pybind11 catches and converts to Python exceptions. Noexcept +functions cannot propagate these exceptions either. A useful approach is to +convert them to Python exceptions and then ``discard_as_unraisable`` as shown +below. + +.. code-block:: cpp + + void nonthrowing_func() noexcept(true) { + try { + // ... + } catch (py::error_already_set &eas) { + // Discard the Python error using Python APIs, using the C++ magic + // variable __func__. Python already knows the type and value and of the + // exception object. + eas.discard_as_unraisable(__func__); + } catch (const std::exception &e) { + // Log and discard C++ exceptions. + third_party::log(e); + } + } + +.. versionadded:: 2.6 diff --git a/docs/advanced/functions.rst b/docs/advanced/functions.rst index e3acff06..69e3d8a1 100644 --- a/docs/advanced/functions.rst +++ b/docs/advanced/functions.rst @@ -17,7 +17,7 @@ bindings for functions that return a non-trivial type. Just by looking at the type information, it is not clear whether Python should take charge of the returned value and eventually free its resources, or if this is handled on the C++ side. For this reason, pybind11 provides a several *return value policy* -annotations that can be passed to the :func:`module::def` and +annotations that can be passed to the :func:`module_::def` and :func:`class_::def` functions. The default policy is :enum:`return_value_policy::automatic`. @@ -50,7 +50,7 @@ implied transfer of ownership, i.e.: .. code-block:: cpp - m.def("get_data", &get_data, return_value_policy::reference); + m.def("get_data", &get_data, py::return_value_policy::reference); On the other hand, this is not the right policy for many other situations, where ignoring ownership could lead to resource leaks. @@ -90,17 +90,18 @@ The following table provides an overview of available policies: | | return value is referenced by Python. This is the default policy for | | | property getters created via ``def_property``, ``def_readwrite``, etc. | +--------------------------------------------------+----------------------------------------------------------------------------+ -| :enum:`return_value_policy::automatic` | **Default policy.** This policy falls back to the policy | +| :enum:`return_value_policy::automatic` | This policy falls back to the policy | | | :enum:`return_value_policy::take_ownership` when the return value is a | | | pointer. Otherwise, it uses :enum:`return_value_policy::move` or | | | :enum:`return_value_policy::copy` for rvalue and lvalue references, | | | respectively. See above for a description of what all of these different | -| | policies do. | +| | policies do. This is the default policy for ``py::class_``-wrapped types. | +--------------------------------------------------+----------------------------------------------------------------------------+ | :enum:`return_value_policy::automatic_reference` | As above, but use policy :enum:`return_value_policy::reference` when the | | | return value is a pointer. This is the default conversion policy for | | | function arguments when calling Python functions manually from C++ code | -| | (i.e. via handle::operator()). You probably won't need to use this. | +| | (i.e. via ``handle::operator()``) and the casters in ``pybind11/stl.h``. | +| | You probably won't need to use this explicitly. | +--------------------------------------------------+----------------------------------------------------------------------------+ Return value policies can also be applied to properties: @@ -119,7 +120,7 @@ targeted arguments can be passed through the :class:`cpp_function` constructor: .. code-block:: cpp class_(m, "MyClass") - .def_property("data" + .def_property("data", py::cpp_function(&MyClass::getData, py::return_value_policy::copy), py::cpp_function(&MyClass::setData) ); @@ -182,6 +183,9 @@ relies on the ability to create a *weak reference* to the nurse object. When the nurse object is not a pybind11-registered type and does not support weak references, an exception will be thrown. +If you use an incorrect argument index, you will get a ``RuntimeError`` saying +``Could not activate keep_alive!``. You should review the indices you're using. + Consider the following example: here, the binding code for a list append operation ties the lifetime of the newly added element to the underlying container: @@ -228,7 +232,7 @@ is equivalent to the following pseudocode: }); The only requirement is that ``T`` is default-constructible, but otherwise any -scope guard will work. This is very useful in combination with `gil_scoped_release`. +scope guard will work. This is very useful in combination with ``gil_scoped_release``. See :ref:`gil`. Multiple guards can also be specified as ``py::call_guard``. The @@ -251,7 +255,7 @@ For instance, the following statement iterates over a Python ``dict``: .. code-block:: cpp - void print_dict(py::dict dict) { + void print_dict(const py::dict& dict) { /* Easily interact with Python types */ for (auto item : dict) std::cout << "key=" << std::string(py::str(item.first)) << ", " @@ -268,7 +272,7 @@ And used in Python as usual: .. code-block:: pycon - >>> print_dict({'foo': 123, 'bar': 'hello'}) + >>> print_dict({"foo": 123, "bar": "hello"}) key=foo, value=123 key=bar, value=hello @@ -289,7 +293,7 @@ Such functions can also be created using pybind11: .. code-block:: cpp - void generic(py::args args, py::kwargs kwargs) { + void generic(py::args args, const py::kwargs& kwargs) { /// .. do something with args if (kwargs) /// .. do something with kwargs @@ -302,8 +306,9 @@ The class ``py::args`` derives from ``py::tuple`` and ``py::kwargs`` derives from ``py::dict``. You may also use just one or the other, and may combine these with other -arguments as long as the ``py::args`` and ``py::kwargs`` arguments are the last -arguments accepted by the function. +arguments. Note, however, that ``py::kwargs`` must always be the last argument +of the function, and ``py::args`` implies that any further arguments are +keyword-only (see :ref:`keyword_only_arguments`). Please refer to the other examples for details on how to iterate over these, and on how to cast their entries into C++ objects. A demonstration is also @@ -360,7 +365,66 @@ like so: .. code-block:: cpp py::class_("MyClass") - .def("myFunction", py::arg("arg") = (SomeType *) nullptr); + .def("myFunction", py::arg("arg") = static_cast(nullptr)); + +.. _keyword_only_arguments: + +Keyword-only arguments +====================== + +Python implements keyword-only arguments by specifying an unnamed ``*`` +argument in a function definition: + +.. code-block:: python + + def f(a, *, b): # a can be positional or via keyword; b must be via keyword + pass + + + f(a=1, b=2) # good + f(b=2, a=1) # good + f(1, b=2) # good + f(1, 2) # TypeError: f() takes 1 positional argument but 2 were given + +Pybind11 provides a ``py::kw_only`` object that allows you to implement +the same behaviour by specifying the object between positional and keyword-only +argument annotations when registering the function: + +.. code-block:: cpp + + m.def("f", [](int a, int b) { /* ... */ }, + py::arg("a"), py::kw_only(), py::arg("b")); + +.. versionadded:: 2.6 + +A ``py::args`` argument implies that any following arguments are keyword-only, +as if ``py::kw_only()`` had been specified in the same relative location of the +argument list as the ``py::args`` argument. The ``py::kw_only()`` may be +included to be explicit about this, but is not required. + +.. versionchanged:: 2.9 + This can now be combined with ``py::args``. Before, ``py::args`` could only + occur at the end of the argument list, or immediately before a ``py::kwargs`` + argument at the end. + + +Positional-only arguments +========================= + +Python 3.8 introduced a new positional-only argument syntax, using ``/`` in the +function definition (note that this has been a convention for CPython +positional arguments, such as in ``pow()``, since Python 2). You can +do the same thing in any version of Python using ``py::pos_only()``: + +.. code-block:: cpp + + m.def("f", [](int a, int b) { /* ... */ }, + py::arg("a"), py::pos_only(), py::arg("b")); + +You now cannot give argument ``a`` by keyword. This can be combined with +keyword-only arguments, as well. + +.. versionadded:: 2.6 .. _nonconverting_arguments: @@ -438,7 +502,7 @@ To explicitly enable or disable this behaviour, using the py::class_(m, "Cat").def(py::init<>()); m.def("bark", [](Dog *dog) -> std::string { if (dog) return "woof!"; /* Called with a Dog instance */ - else return "(no dog)"; /* Called with None, d == nullptr */ + else return "(no dog)"; /* Called with None, dog == nullptr */ }, py::arg("dog").none(true)); m.def("meow", [](Cat *cat) -> std::string { // Can't be called with None argument @@ -467,6 +531,17 @@ dog)"``, while attempting to call ``meow(None)`` will raise a ``TypeError``: The default behaviour when the tag is unspecified is to allow ``None``. +.. note:: + + Even when ``.none(true)`` is specified for an argument, ``None`` will be converted to a + ``nullptr`` *only* for custom and :ref:`opaque ` types. Pointers to built-in types + (``double *``, ``int *``, ...) and STL types (``std::vector *``, ...; if ``pybind11/stl.h`` + is included) are copied when converted to C++ (see :doc:`/advanced/cast/overview`) and will + not allow ``None`` as argument. To pass optional argument of these copied types consider + using ``std::optional`` + +.. _overload_resolution: + Overload resolution order ========================= @@ -483,11 +558,13 @@ an explicit ``py::arg().noconvert()`` attribute in the function definition). If the second pass also fails a ``TypeError`` is raised. Within each pass, overloads are tried in the order they were registered with -pybind11. +pybind11. If the ``py::prepend()`` tag is added to the definition, a function +can be placed at the beginning of the overload sequence instead, allowing user +overloads to proceed built in functions. What this means in practice is that pybind11 will prefer any overload that does -not require conversion of arguments to an overload that does, but otherwise prefers -earlier-defined overloads to later-defined ones. +not require conversion of arguments to an overload that does, but otherwise +prefers earlier-defined overloads to later-defined ones. .. note:: @@ -496,3 +573,42 @@ earlier-defined overloads to later-defined ones. requiring one conversion over one requiring three, but only prioritizes overloads requiring no conversion at all to overloads that require conversion of at least one argument. + +.. versionadded:: 2.6 + + The ``py::prepend()`` tag. + +Binding functions with template parameters +========================================== + +You can bind functions that have template parameters. Here's a function: + +.. code-block:: cpp + + template + void set(T t); + +C++ templates cannot be instantiated at runtime, so you cannot bind the +non-instantiated function: + +.. code-block:: cpp + + // BROKEN (this will not compile) + m.def("set", &set); + +You must bind each instantiated function template separately. You may bind +each instantiation with the same name, which will be treated the same as +an overloaded function: + +.. code-block:: cpp + + m.def("set", &set); + m.def("set", &set); + +Sometimes it's more clear to bind them with separate names, which is also +an option: + +.. code-block:: cpp + + m.def("setInt", &set); + m.def("setString", &set); diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 5faf11f0..805ec838 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -7,28 +7,74 @@ General notes regarding convenience macros ========================================== pybind11 provides a few convenience macros such as -:func:`PYBIND11_MAKE_OPAQUE` and :func:`PYBIND11_DECLARE_HOLDER_TYPE`, and -``PYBIND11_OVERLOAD_*``. Since these are "just" macros that are evaluated -in the preprocessor (which has no concept of types), they *will* get confused -by commas in a template argument such as ``PYBIND11_OVERLOAD(MyReturnValue, myFunc)``. In this case, the preprocessor assumes that the comma indicates -the beginning of the next parameter. Use a ``typedef`` to bind the template to -another name and use it in the macro to avoid this problem. +:func:`PYBIND11_DECLARE_HOLDER_TYPE` and ``PYBIND11_OVERRIDE_*``. Since these +are "just" macros that are evaluated in the preprocessor (which has no concept +of types), they *will* get confused by commas in a template argument; for +example, consider: + +.. code-block:: cpp + + PYBIND11_OVERRIDE(MyReturnType, Class, func) + +The limitation of the C preprocessor interprets this as five arguments (with new +arguments beginning after each comma) rather than three. To get around this, +there are two alternatives: you can use a type alias, or you can wrap the type +using the ``PYBIND11_TYPE`` macro: + +.. code-block:: cpp + + // Version 1: using a type alias + using ReturnType = MyReturnType; + using ClassType = Class; + PYBIND11_OVERRIDE(ReturnType, ClassType, func); + + // Version 2: using the PYBIND11_TYPE macro: + PYBIND11_OVERRIDE(PYBIND11_TYPE(MyReturnType), + PYBIND11_TYPE(Class), func) + +The ``PYBIND11_MAKE_OPAQUE`` macro does *not* require the above workarounds. .. _gil: Global Interpreter Lock (GIL) ============================= -When calling a C++ function from Python, the GIL is always held. +The Python C API dictates that the Global Interpreter Lock (GIL) must always +be held by the current thread to safely access Python objects. As a result, +when Python calls into C++ via pybind11 the GIL must be held, and pybind11 +will never implicitly release the GIL. + +.. code-block:: cpp + + void my_function() { + /* GIL is held when this function is called from Python */ + } + + PYBIND11_MODULE(example, m) { + m.def("my_function", &my_function); + } + +pybind11 will ensure that the GIL is held when it knows that it is calling +Python code. For example, if a Python callback is passed to C++ code via +``std::function``, when C++ code calls the function the built-in wrapper +will acquire the GIL before calling the Python callback. Similarly, the +``PYBIND11_OVERRIDE`` family of macros will acquire the GIL before calling +back into Python. + +When writing C++ code that is called from other C++ code, if that code accesses +Python state, it must explicitly acquire and release the GIL. + The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to acquire and release the global interpreter lock in the body of a C++ function call. In this way, long-running C++ code can be parallelized using -multiple Python threads. Taking :ref:`overriding_virtuals` as an example, this +multiple Python threads, **but great care must be taken** when any +:class:`gil_scoped_release` appear: if there is any way that the C++ code +can access Python objects, :class:`gil_scoped_acquire` should be used to +reacquire the GIL. Taking :ref:`overriding_virtuals` as an example, this could be realized as follows (important changes highlighted): .. code-block:: cpp - :emphasize-lines: 8,9,31,32 + :emphasize-lines: 8,30,31 class PyAnimal : public Animal { public: @@ -37,10 +83,8 @@ could be realized as follows (important changes highlighted): /* Trampoline (need one for each virtual function) */ std::string go(int n_times) { - /* Acquire GIL before calling Python code */ - py::gil_scoped_acquire acquire; - - PYBIND11_OVERLOAD_PURE( + /* PYBIND11_OVERRIDE_PURE will acquire the GIL before accessing Python state */ + PYBIND11_OVERRIDE_PURE( std::string, /* Return type */ Animal, /* Parent class */ go, /* Name of function */ @@ -59,13 +103,14 @@ could be realized as follows (important changes highlighted): .def(py::init<>()); m.def("call_go", [](Animal *animal) -> std::string { - /* Release GIL before calling into (potentially long-running) C++ code */ + // GIL is held when called from Python code. Release GIL before + // calling into (potentially long-running) C++ code py::gil_scoped_release release; return call_go(animal); }); } -The ``call_go`` wrapper can also be simplified using the `call_guard` policy +The ``call_go`` wrapper can also be simplified using the ``call_guard`` policy (see :ref:`call_policies`) which yields the same result: .. code-block:: cpp @@ -73,6 +118,34 @@ The ``call_go`` wrapper can also be simplified using the `call_guard` policy m.def("call_go", &call_go, py::call_guard()); +Common Sources Of Global Interpreter Lock Errors +================================================================== + +Failing to properly hold the Global Interpreter Lock (GIL) is one of the +more common sources of bugs within code that uses pybind11. If you are +running into GIL related errors, we highly recommend you consult the +following checklist. + +- Do you have any global variables that are pybind11 objects or invoke + pybind11 functions in either their constructor or destructor? You are generally + not allowed to invoke any Python function in a global static context. We recommend + using lazy initialization and then intentionally leaking at the end of the program. + +- Do you have any pybind11 objects that are members of other C++ structures? One + commonly overlooked requirement is that pybind11 objects have to increase their reference count + whenever their copy constructor is called. Thus, you need to be holding the GIL to invoke + the copy constructor of any C++ class that has a pybind11 member. This can sometimes be very + tricky to track for complicated programs Think carefully when you make a pybind11 object + a member in another struct. + +- C++ destructors that invoke Python functions can be particularly troublesome as + destructors can sometimes get invoked in weird and unexpected circumstances as a result + of exceptions. + +- You should try running your code in a debug build. That will enable additional assertions + within pybind11 that will throw exceptions on certain GIL handling errors + (reference counting operations). + Binding sequence data types, iterators, the slicing protocol, etc. ================================================================== @@ -113,7 +186,7 @@ However, it can be acquired as follows: .. code-block:: cpp - py::object pet = (py::object) py::module::import("basic").attr("Pet"); + py::object pet = (py::object) py::module_::import("basic").attr("Pet"); py::class_(m, "Dog", pet) .def(py::init()) @@ -127,7 +200,7 @@ has been executed: .. code-block:: cpp - py::module::import("basic"); + py::module_::import("basic"); py::class_(m, "Dog") .def(py::init()) @@ -137,7 +210,7 @@ Naturally, both methods will fail when there are cyclic dependencies. Note that pybind11 code compiled with hidden-by-default symbol visibility (e.g. via the command line flag ``-fvisibility=hidden`` on GCC/Clang), which is -required proper pybind11 functionality, can interfere with the ability to +required for proper pybind11 functionality, can interfere with the ability to access types defined in another extension module. Working around this requires manually exporting types that are accessed by multiple extension modules; pybind11 provides a macro to do just this: @@ -157,9 +230,9 @@ pybind11 version. Consider the following example: .. code-block:: cpp - auto data = (MyData *) py::get_shared_data("mydata"); + auto data = reinterpret_cast(py::get_shared_data("mydata")); if (!data) - data = (MyData *) py::set_shared_data("mydata", new MyData(42)); + data = static_cast(py::set_shared_data("mydata", new MyData(42))); If the above snippet was used in several separately compiled extension modules, the first one to be imported would create a ``MyData`` instance and associate @@ -199,12 +272,12 @@ collected: Both approaches also expose a potentially dangerous ``_cleanup`` attribute in Python, which may be undesirable from an API standpoint (a premature explicit -call from Python might lead to undefined behavior). Yet another approach that +call from Python might lead to undefined behavior). Yet another approach that avoids this issue involves weak reference with a cleanup callback: .. code-block:: cpp - // Register a callback function that is invoked when the BaseClass object is colelcted + // Register a callback function that is invoked when the BaseClass object is collected py::cpp_function cleanup_callback( [](py::handle weakref) { // perform cleanup here -- this function is called with the GIL held @@ -218,13 +291,13 @@ avoids this issue involves weak reference with a cleanup callback: .. note:: - PyPy (at least version 5.9) does not garbage collect objects when the - interpreter exits. An alternative approach (which also works on CPython) is to use - the :py:mod:`atexit` module [#f7]_, for example: + PyPy does not garbage collect objects when the interpreter exits. An alternative + approach (which also works on CPython) is to use the :py:mod:`atexit` module [#f7]_, + for example: .. code-block:: cpp - auto atexit = py::module::import("atexit"); + auto atexit = py::module_::import("atexit"); atexit.attr("register")(py::cpp_function([]() { // perform cleanup here -- this function is called with the GIL held })); @@ -264,9 +337,9 @@ work, it is important that all lines are indented consistently, i.e.: ---------- )mydelimiter"); -By default, pybind11 automatically generates and prepends a signature to the docstring of a function -registered with ``module::def()`` and ``class_::def()``. Sometimes this -behavior is not desirable, because you want to provide your own signature or remove +By default, pybind11 automatically generates and prepends a signature to the docstring of a function +registered with ``module_::def()`` and ``class_::def()``. Sometimes this +behavior is not desirable, because you want to provide your own signature or remove the docstring completely to exclude the function from the Sphinx documentation. The class ``options`` allows you to selectively suppress auto-generated signatures: @@ -279,9 +352,49 @@ The class ``options`` allows you to selectively suppress auto-generated signatur m.def("add", [](int a, int b) { return a + b; }, "A function which adds two numbers"); } -Note that changes to the settings affect only function bindings created during the -lifetime of the ``options`` instance. When it goes out of scope at the end of the module's init function, +pybind11 also appends all members of an enum to the resulting enum docstring. +This default behavior can be disabled by using the ``disable_enum_members_docstring()`` +function of the ``options`` class. + +With ``disable_user_defined_docstrings()`` all user defined docstrings of +``module_::def()``, ``class_::def()`` and ``enum_()`` are disabled, but the +function signatures and enum members are included in the docstring, unless they +are disabled separately. + +Note that changes to the settings affect only function bindings created during the +lifetime of the ``options`` instance. When it goes out of scope at the end of the module's init function, the default settings are restored to prevent unwanted side effects. .. [#f4] http://www.sphinx-doc.org .. [#f5] http://github.com/pybind/python_example + +.. _avoiding-cpp-types-in-docstrings: + +Avoiding C++ types in docstrings +================================ + +Docstrings are generated at the time of the declaration, e.g. when ``.def(...)`` is called. +At this point parameter and return types should be known to pybind11. +If a custom type is not exposed yet through a ``py::class_`` constructor or a custom type caster, +its C++ type name will be used instead to generate the signature in the docstring: + +.. code-block:: text + + | __init__(...) + | __init__(self: example.Foo, arg0: ns::Bar) -> None + ^^^^^^^ + + +This limitation can be circumvented by ensuring that C++ classes are registered with pybind11 +before they are used as a parameter or return type of a function: + +.. code-block:: cpp + + PYBIND11_MODULE(example, m) { + + auto pyFoo = py::class_(m, "Foo"); + auto pyBar = py::class_(m, "Bar"); + + pyFoo.def(py::init()); + pyBar.def(py::init()); + } diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index 18eff80b..07c96930 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -57,17 +57,17 @@ specification. struct buffer_info { void *ptr; - ssize_t itemsize; + py::ssize_t itemsize; std::string format; - ssize_t ndim; - std::vector shape; - std::vector strides; + py::ssize_t ndim; + std::vector shape; + std::vector strides; }; To create a C++ function that can take a Python buffer object as an argument, simply use the type ``py::buffer`` as one of its arguments. Buffers can exist in a great variety of configurations, hence some safety checks are usually -necessary in the function body. Below, you can see an basic example on how to +necessary in the function body. Below, you can see a basic example on how to define a custom constructor for the Eigen double precision matrix (``Eigen::MatrixXd``) type, which supports initialization from compatible buffer objects (e.g. a NumPy matrix). @@ -81,13 +81,13 @@ buffer objects (e.g. a NumPy matrix). constexpr bool rowMajor = Matrix::Flags & Eigen::RowMajorBit; py::class_(m, "Matrix", py::buffer_protocol()) - .def("__init__", [](Matrix &m, py::buffer b) { + .def(py::init([](py::buffer b) { typedef Eigen::Stride Strides; /* Request a buffer descriptor from Python */ py::buffer_info info = b.request(); - /* Some sanity checks ... */ + /* Some basic validation checks ... */ if (info.format != py::format_descriptor::format()) throw std::runtime_error("Incompatible format: expected a double array!"); @@ -101,8 +101,8 @@ buffer objects (e.g. a NumPy matrix). auto map = Eigen::Map( static_cast(info.ptr), info.shape[0], info.shape[1], strides); - new (&m) Matrix(map); - }); + return Matrix(map); + })); For reference, the ``def_buffer()`` call for this Eigen data type should look as follows: @@ -150,8 +150,10 @@ NumPy array containing double precision values. When it is invoked with a different type (e.g. an integer or a list of integers), the binding code will attempt to cast the input into a NumPy array -of the requested type. Note that this feature requires the -:file:`pybind11/numpy.h` header to be included. +of the requested type. This feature requires the :file:`pybind11/numpy.h` +header to be included. Note that :file:`pybind11/numpy.h` does not depend on +the NumPy headers, and thus can be used without declaring a build-time +dependency on NumPy; NumPy>=1.7.0 is a runtime dependency. Data in NumPy arrays is not guaranteed to packed in a dense manner; furthermore, entries can be separated by arbitrary column and row strides. @@ -169,6 +171,31 @@ template parameter, and it ensures that non-conforming arguments are converted into an array satisfying the specified requirements instead of trying the next function overload. +There are several methods on arrays; the methods listed below under references +work, as well as the following functions based on the NumPy API: + +- ``.dtype()`` returns the type of the contained values. + +- ``.strides()`` returns a pointer to the strides of the array (optionally pass + an integer axis to get a number). + +- ``.flags()`` returns the flag settings. ``.writable()`` and ``.owndata()`` + are directly available. + +- ``.offset_at()`` returns the offset (optionally pass indices). + +- ``.squeeze()`` returns a view with length-1 axes removed. + +- ``.view(dtype)`` returns a view of the array with a different dtype. + +- ``.reshape({i, j, ...})`` returns a view of the array with a different shape. + ``.resize({...})`` is also available. + +- ``.index_at(i, j, ...)`` gets the count from the beginning to a given index. + + +There are also several methods for getting references (described below). + Structured types ================ @@ -231,8 +258,8 @@ by the compiler. The result is returned as a NumPy array of type .. code-block:: pycon - >>> x = np.array([[1, 3],[5, 7]]) - >>> y = np.array([[2, 4],[6, 8]]) + >>> x = np.array([[1, 3], [5, 7]]) + >>> y = np.array([[2, 4], [6, 8]]) >>> z = 3 >>> result = vectorized_func(x, y, z) @@ -261,7 +288,7 @@ simply using ``vectorize``). namespace py = pybind11; py::array_t add_arrays(py::array_t input1, py::array_t input2) { - auto buf1 = input1.request(), buf2 = input2.request(); + py::buffer_info buf1 = input1.request(), buf2 = input2.request(); if (buf1.ndim != 1 || buf2.ndim != 1) throw std::runtime_error("Number of dimensions must be one"); @@ -272,11 +299,11 @@ simply using ``vectorize``). /* No pointer is passed, so NumPy will allocate the buffer */ auto result = py::array_t(buf1.size); - auto buf3 = result.request(); + py::buffer_info buf3 = result.request(); - double *ptr1 = (double *) buf1.ptr, - *ptr2 = (double *) buf2.ptr, - *ptr3 = (double *) buf3.ptr; + double *ptr1 = static_cast(buf1.ptr); + double *ptr2 = static_cast(buf2.ptr); + double *ptr3 = static_cast(buf3.ptr); for (size_t idx = 0; idx < buf1.shape[0]; idx++) ptr3[idx] = ptr1[idx] + ptr2[idx]; @@ -309,17 +336,17 @@ where ``N`` gives the required dimensionality of the array: m.def("sum_3d", [](py::array_t x) { auto r = x.unchecked<3>(); // x must have ndim = 3; can be non-writeable double sum = 0; - for (ssize_t i = 0; i < r.shape(0); i++) - for (ssize_t j = 0; j < r.shape(1); j++) - for (ssize_t k = 0; k < r.shape(2); k++) + for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t k = 0; k < r.shape(2); k++) sum += r(i, j, k); return sum; }); m.def("increment_3d", [](py::array_t x) { auto r = x.mutable_unchecked<3>(); // Will throw if ndim != 3 or flags.writeable is false - for (ssize_t i = 0; i < r.shape(0); i++) - for (ssize_t j = 0; j < r.shape(1); j++) - for (ssize_t k = 0; k < r.shape(2); k++) + for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t j = 0; j < r.shape(1); j++) + for (py::ssize_t k = 0; k < r.shape(2); k++) r(i, j, k) += 1.0; }, py::arg().noconvert()); @@ -343,24 +370,86 @@ The returned proxy object supports some of the same methods as ``py::array`` so that it can be used as a drop-in replacement for some existing, index-checked uses of ``py::array``: -- ``r.ndim()`` returns the number of dimensions +- ``.ndim()`` returns the number of dimensions -- ``r.data(1, 2, ...)`` and ``r.mutable_data(1, 2, ...)``` returns a pointer to +- ``.data(1, 2, ...)`` and ``r.mutable_data(1, 2, ...)``` returns a pointer to the ``const T`` or ``T`` data, respectively, at the given indices. The latter is only available to proxies obtained via ``a.mutable_unchecked()``. -- ``itemsize()`` returns the size of an item in bytes, i.e. ``sizeof(T)``. +- ``.itemsize()`` returns the size of an item in bytes, i.e. ``sizeof(T)``. -- ``ndim()`` returns the number of dimensions. +- ``.ndim()`` returns the number of dimensions. -- ``shape(n)`` returns the size of dimension ``n`` +- ``.shape(n)`` returns the size of dimension ``n`` -- ``size()`` returns the total number of elements (i.e. the product of the shapes). +- ``.size()`` returns the total number of elements (i.e. the product of the shapes). -- ``nbytes()`` returns the number of bytes used by the referenced elements +- ``.nbytes()`` returns the number of bytes used by the referenced elements (i.e. ``itemsize()`` times ``size()``). .. seealso:: The file :file:`tests/test_numpy_array.cpp` contains additional examples demonstrating the use of this feature. + +Ellipsis +======== + +Python provides a convenient ``...`` ellipsis notation that is often used to +slice multidimensional arrays. For instance, the following snippet extracts the +middle dimensions of a tensor with the first and last index set to zero. + +.. code-block:: python + + a = ... # a NumPy array + b = a[0, ..., 0] + +The function ``py::ellipsis()`` function can be used to perform the same +operation on the C++ side: + +.. code-block:: cpp + + py::array a = /* A NumPy array */; + py::array b = a[py::make_tuple(0, py::ellipsis(), 0)]; + + +Memory view +=========== + +For a case when we simply want to provide a direct accessor to C/C++ buffer +without a concrete class object, we can return a ``memoryview`` object. Suppose +we wish to expose a ``memoryview`` for 2x4 uint8_t array, we can do the +following: + +.. code-block:: cpp + + const uint8_t buffer[] = { + 0, 1, 2, 3, + 4, 5, 6, 7 + }; + m.def("get_memoryview2d", []() { + return py::memoryview::from_buffer( + buffer, // buffer pointer + { 2, 4 }, // shape (rows, cols) + { sizeof(uint8_t) * 4, sizeof(uint8_t) } // strides in bytes + ); + }); + +This approach is meant for providing a ``memoryview`` for a C/C++ buffer not +managed by Python. The user is responsible for managing the lifetime of the +buffer. Using a ``memoryview`` created in this way after deleting the buffer in +C++ side results in undefined behavior. + +We can also use ``memoryview::from_memory`` for a simple 1D contiguous buffer: + +.. code-block:: cpp + + m.def("get_memoryview1d", []() { + return py::memoryview::from_memory( + buffer, // buffer pointer + sizeof(uint8_t) * 8 // buffer size + ); + }); + +.. versionchanged:: 2.6 + ``memoryview::from_memory`` added. diff --git a/docs/advanced/pycpp/object.rst b/docs/advanced/pycpp/object.rst index 117131ed..93e1a94d 100644 --- a/docs/advanced/pycpp/object.rst +++ b/docs/advanced/pycpp/object.rst @@ -1,6 +1,8 @@ Python types ############ +.. _wrappers: + Available wrappers ================== @@ -13,6 +15,47 @@ Available types include :class:`handle`, :class:`object`, :class:`bool_`, :class:`iterable`, :class:`iterator`, :class:`function`, :class:`buffer`, :class:`array`, and :class:`array_t`. +.. warning:: + + Be sure to review the :ref:`pytypes_gotchas` before using this heavily in + your C++ API. + +.. _instantiating_compound_types: + +Instantiating compound Python types from C++ +============================================ + +Dictionaries can be initialized in the :class:`dict` constructor: + +.. code-block:: cpp + + using namespace pybind11::literals; // to bring in the `_a` literal + py::dict d("spam"_a=py::none(), "eggs"_a=42); + +A tuple of python objects can be instantiated using :func:`py::make_tuple`: + +.. code-block:: cpp + + py::tuple tup = py::make_tuple(42, py::none(), "spam"); + +Each element is converted to a supported Python type. + +A `simple namespace`_ can be instantiated using + +.. code-block:: cpp + + using namespace pybind11::literals; // to bring in the `_a` literal + py::object SimpleNamespace = py::module_::import("types").attr("SimpleNamespace"); + py::object ns = SimpleNamespace("spam"_a=py::none(), "eggs"_a=42); + +Attributes on a namespace can be modified with the :func:`py::delattr`, +:func:`py::getattr`, and :func:`py::setattr` functions. Simple namespaces can +be useful as lightweight stand-ins for class instances. + +.. _simple namespace: https://docs.python.org/3/library/types.html#types.SimpleNamespace + +.. _casting_back_and_forth: + Casting back and forth ====================== @@ -21,7 +64,7 @@ types to Python, which can be done using :func:`py::cast`: .. code-block:: cpp - MyClass *cls = ..; + MyClass *cls = ...; py::object obj = py::cast(cls); The reverse direction uses the following syntax: @@ -47,20 +90,21 @@ This example obtains a reference to the Python ``Decimal`` class. .. code-block:: cpp // Equivalent to "from decimal import Decimal" - py::object Decimal = py::module::import("decimal").attr("Decimal"); + py::object Decimal = py::module_::import("decimal").attr("Decimal"); .. code-block:: cpp // Try to import scipy - py::object scipy = py::module::import("scipy"); + py::object scipy = py::module_::import("scipy"); return scipy.attr("__version__"); + .. _calling_python_functions: Calling Python functions ======================== -It is also possible to call Python classes, functions and methods +It is also possible to call Python classes, functions and methods via ``operator()``. .. code-block:: cpp @@ -71,11 +115,11 @@ via ``operator()``. .. code-block:: cpp // Use Python to make our directories - py::object os = py::module::import("os"); + py::object os = py::module_::import("os"); py::object makedirs = os.attr("makedirs"); makedirs("/tmp/path/to/somewhere"); -One can convert the result obtained from Python to a pure C++ version +One can convert the result obtained from Python to a pure C++ version if a ``py::class_`` or type conversion is defined. .. code-block:: cpp @@ -99,8 +143,8 @@ Python method. py::print(py::str(exp_pi)); In the example above ``pi.attr("exp")`` is a *bound method*: it will always call -the method for that same instance of the class. Alternately one can create an -*unbound method* via the Python class (instead of instance) and pass the ``self`` +the method for that same instance of the class. Alternately one can create an +*unbound method* via the Python class (instead of instance) and pass the ``self`` object explicitly, followed by other arguments. .. code-block:: cpp @@ -122,6 +166,7 @@ Keyword arguments are also supported. In Python, there is the usual call syntax: def f(number, say, to): ... # function code + f(1234, say="hello", to=some_instance) # keyword call in Python In C++, the same call can be made using: @@ -168,3 +213,74 @@ Generalized unpacking according to PEP448_ is also supported: Python functions from C++, including keywords arguments and unpacking. .. _PEP448: https://www.python.org/dev/peps/pep-0448/ + +.. _implicit_casting: + +Implicit casting +================ + +When using the C++ interface for Python types, or calling Python functions, +objects of type :class:`object` are returned. It is possible to invoke implicit +conversions to subclasses like :class:`dict`. The same holds for the proxy objects +returned by ``operator[]`` or ``obj.attr()``. +Casting to subtypes improves code readability and allows values to be passed to +C++ functions that require a specific subtype rather than a generic :class:`object`. + +.. code-block:: cpp + + #include + using namespace pybind11::literals; + + py::module_ os = py::module_::import("os"); + py::module_ path = py::module_::import("os.path"); // like 'import os.path as path' + py::module_ np = py::module_::import("numpy"); // like 'import numpy as np' + + py::str curdir_abs = path.attr("abspath")(path.attr("curdir")); + py::print(py::str("Current directory: ") + curdir_abs); + py::dict environ = os.attr("environ"); + py::print(environ["HOME"]); + py::array_t arr = np.attr("ones")(3, "dtype"_a="float32"); + py::print(py::repr(arr + py::int_(1))); + +These implicit conversions are available for subclasses of :class:`object`; there +is no need to call ``obj.cast()`` explicitly as for custom classes, see +:ref:`casting_back_and_forth`. + +.. note:: + If a trivial conversion via move constructor is not possible, both implicit and + explicit casting (calling ``obj.cast()``) will attempt a "rich" conversion. + For instance, ``py::list env = os.attr("environ");`` will succeed and is + equivalent to the Python code ``env = list(os.environ)`` that produces a + list of the dict keys. + +.. TODO: Adapt text once PR #2349 has landed + +Handling exceptions +=================== + +Python exceptions from wrapper classes will be thrown as a ``py::error_already_set``. +See :ref:`Handling exceptions from Python in C++ +` for more information on handling exceptions +raised when calling C++ wrapper classes. + +.. _pytypes_gotchas: + +Gotchas +======= + +Default-Constructed Wrappers +---------------------------- + +When a wrapper type is default-constructed, it is **not** a valid Python object (i.e. it is not ``py::none()``). It is simply the same as +``PyObject*`` null pointer. To check for this, use +``static_cast(my_wrapper)``. + +Assigning py::none() to wrappers +-------------------------------- + +You may be tempted to use types like ``py::str`` and ``py::dict`` in C++ +signatures (either pure C++, or in bound signatures), and assign them default +values of ``py::none()``. However, in a best case scenario, it will fail fast +because ``None`` is not convertible to that type (e.g. ``py::dict``), or in a +worse case scenario, it will silently work but corrupt the types you want to +work with (e.g. ``py::str(py::none())`` will yield ``"None"`` in Python). diff --git a/docs/advanced/pycpp/utilities.rst b/docs/advanced/pycpp/utilities.rst index 369e7c94..af0f9cb2 100644 --- a/docs/advanced/pycpp/utilities.rst +++ b/docs/advanced/pycpp/utilities.rst @@ -28,7 +28,7 @@ Capturing standard output from ostream Often, a library will use the streams ``std::cout`` and ``std::cerr`` to print, but this does not play well with Python's standard ``sys.stdout`` and ``sys.stderr`` -redirection. Replacing a library's printing with `py::print ` may not +redirection. Replacing a library's printing with ``py::print `` may not be feasible. This can be fixed using a guard around the library function that redirects output to the corresponding Python streams: @@ -42,20 +42,31 @@ redirects output to the corresponding Python streams: m.def("noisy_func", []() { py::scoped_ostream_redirect stream( std::cout, // std::ostream& - py::module::import("sys").attr("stdout") // Python output + py::module_::import("sys").attr("stdout") // Python output ); call_noisy_func(); }); +.. warning:: + + The implementation in ``pybind11/iostream.h`` is NOT thread safe. Multiple + threads writing to a redirected ostream concurrently cause data races + and potentially buffer overflows. Therefore it is currently a requirement + that all (possibly) concurrent redirected ostream writes are protected by + a mutex. #HelpAppreciated: Work on iostream.h thread safety. For more + background see the discussions under + `PR #2982 `_ and + `PR #2995 `_. + This method respects flushes on the output streams and will flush if needed when the scoped guard is destroyed. This allows the output to be redirected in real time, such as to a Jupyter notebook. The two arguments, the C++ stream and the Python output, are optional, and default to standard output if not given. An -extra type, `py::scoped_estream_redirect `, is identical +extra type, ``py::scoped_estream_redirect ``, is identical except for defaulting to ``std::cerr`` and ``sys.stderr``; this can be useful with -`py::call_guard`, which allows multiple items, but uses the default constructor: +``py::call_guard``, which allows multiple items, but uses the default constructor: -.. code-block:: py +.. code-block:: cpp // Alternative: Call single function using call guard m.def("noisy_func", &call_noisy_function, @@ -63,7 +74,7 @@ except for defaulting to ``std::cerr`` and ``sys.stderr``; this can be useful wi py::scoped_estream_redirect>()); The redirection can also be done in Python with the addition of a context -manager, using the `py::add_ostream_redirect() ` function: +manager, using the ``py::add_ostream_redirect() `` function: .. code-block:: cpp @@ -92,7 +103,7 @@ arguments to disable one of the streams if needed. Evaluating Python expressions from strings and files ==================================================== -pybind11 provides the `eval`, `exec` and `eval_file` functions to evaluate +pybind11 provides the ``eval``, ``exec`` and ``eval_file`` functions to evaluate Python expressions and statements. The following example illustrates how they can be used. @@ -104,7 +115,7 @@ can be used. ... // Evaluate in scope of main module - py::object scope = py::module::import("__main__").attr("__dict__"); + py::object scope = py::module_::import("__main__").attr("__dict__"); // Evaluate an isolated expression int result = py::eval("my_variable + 10", scope).cast(); diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index da57748c..3c40ce12 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -77,6 +77,7 @@ segmentation fault). .. code-block:: python from example import Parent + print(Parent().get_child()) The problem is that ``Parent::get_child()`` returns a pointer to an instance of @@ -156,7 +157,7 @@ specialized: PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr); // Only needed if the type's `.get()` goes by another name - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct holder_helper> { // <-- specialization static const T *get(const SmartPtr &p) { return p.getPointer(); } diff --git a/docs/basics.rst b/docs/basics.rst index 447250ed..e9b24c7f 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -11,11 +11,11 @@ included set of test cases. Compiling the test cases ======================== -Linux/MacOS +Linux/macOS ----------- On Linux you'll need to install the **python-dev** or **python3-dev** packages as -well as **cmake**. On Mac OS, the included python version works out of the box, +well as **cmake**. On macOS, the included python version works out of the box, but **cmake** must still be installed. After installing the prerequisites, run @@ -32,8 +32,15 @@ The last line will both compile and run the tests. Windows ------- -On Windows, only **Visual Studio 2015** and newer are supported since pybind11 relies -on various C++11 language features that break older versions of Visual Studio. +On Windows, only **Visual Studio 2017** and newer are supported. + +.. Note:: + + To use the C++17 in Visual Studio 2017 (MSVC 14.1), pybind11 requires the flag + ``/permissive-`` to be passed to the compiler `to enforce standard conformance`_. When + building with Visual Studio 2019, this is not strictly necessary, but still advised. + +.. _`to enforce standard conformance`: https://docs.microsoft.com/en-us/cpp/build/reference/permissive-standards-conformance?view=vs-2017 To compile and run the tests: @@ -101,7 +108,7 @@ a file named :file:`example.cpp` with the following contents: PYBIND11_MODULE(example, m) { m.doc() = "pybind11 example plugin"; // optional module docstring - m.def("add", &add, "A function which adds two numbers"); + m.def("add", &add, "A function that adds two numbers"); } .. [#f1] In practice, implementation and binding code will generally be located @@ -110,8 +117,8 @@ a file named :file:`example.cpp` with the following contents: The :func:`PYBIND11_MODULE` macro creates a function that will be called when an ``import`` statement is issued from within Python. The module name (``example``) is given as the first macro argument (it should not be in quotes). The second -argument (``m``) defines a variable of type :class:`py::module ` which -is the main interface for creating bindings. The method :func:`module::def` +argument (``m``) defines a variable of type :class:`py::module_ ` which +is the main interface for creating bindings. The method :func:`module_::def` generates binding code that exposes the ``add()`` function to Python. .. note:: @@ -128,9 +135,16 @@ On Linux, the above example can be compiled using the following command: .. code-block:: bash - $ c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` example.cpp -o example`python3-config --extension-suffix` + $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix) -For more details on the required compiler flags on Linux and MacOS, see +.. note:: + + If you used :ref:`include_as_a_submodule` to get the pybind11 source, then + use ``$(python3-config --includes) -Iextern/pybind11/include`` instead of + ``$(python3 -m pybind11 --includes)`` in the above compilation, as + explained in :ref:`building_manually`. + +For more details on the required compiler flags on Linux and macOS, see :ref:`building_manually`. For complete cross-platform compilation instructions, refer to the :ref:`compiling` page. @@ -151,12 +165,12 @@ load and execute the example: .. code-block:: pycon $ python - Python 2.7.10 (default, Aug 22 2015, 20:33:39) - [GCC 4.2.1 Compatible Apple LLVM 7.0.0 (clang-700.0.59.1)] on darwin + Python 3.9.10 (main, Jan 15 2022, 11:48:04) + [Clang 13.0.0 (clang-1300.0.29.3)] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> import example >>> example.add(1, 2) - 3L + 3 >>> .. _keyword_args: @@ -164,7 +178,7 @@ load and execute the example: Keyword arguments ================= -With a simple modification code, it is possible to inform Python about the +With a simple code modification, it is possible to inform Python about the names of the arguments ("i" and "j" in this case). .. code-block:: cpp @@ -173,7 +187,7 @@ names of the arguments ("i" and "j" in this case). py::arg("i"), py::arg("j")); :class:`arg` is one of several special tag classes which can be used to pass -metadata into :func:`module::def`. With this modified binding code, we can now +metadata into :func:`module_::def`. With this modified binding code, we can now call the function using keyword arguments, which is a more readable alternative particularly for functions taking many parameters: diff --git a/docs/benchmark.py b/docs/benchmark.py index 6dc0604e..2150b6ca 100644 --- a/docs/benchmark.py +++ b/docs/benchmark.py @@ -1,7 +1,6 @@ -import random -import os -import time import datetime as dt +import os +import random nfns = 4 # Functions per class nargs = 4 # Arguments per function @@ -12,27 +11,26 @@ def generate_dummy_code_pybind11(nclasses=10): bindings = "" for cl in range(nclasses): - decl += "class cl%03i;\n" % cl - decl += '\n' + decl += f"class cl{cl:03};\n" + decl += "\n" for cl in range(nclasses): - decl += "class cl%03i {\n" % cl + decl += f"class {cl:03} {{\n" decl += "public:\n" - bindings += ' py::class_(m, "cl%03i")\n' % (cl, cl) + bindings += f' py::class_(m, "cl{cl:03}")\n' for fn in range(nfns): ret = random.randint(0, nclasses - 1) - params = [random.randint(0, nclasses - 1) for i in range(nargs)] - decl += " cl%03i *fn_%03i(" % (ret, fn) - decl += ", ".join("cl%03i *" % p for p in params) + params = [random.randint(0, nclasses - 1) for i in range(nargs)] + decl += f" cl{ret:03} *fn_{fn:03}(" + decl += ", ".join(f"cl{p:03} *" for p in params) decl += ");\n" - bindings += ' .def("fn_%03i", &cl%03i::fn_%03i)\n' % \ - (fn, cl, fn) + bindings += f' .def("fn_{fn:03}", &cl{cl:03}::fn_{fn:03})\n' decl += "};\n\n" - bindings += ' ;\n' + bindings += " ;\n" result = "#include \n\n" result += "namespace py = pybind11;\n\n" - result += decl + '\n' + result += decl + "\n" result += "PYBIND11_MODULE(example, m) {\n" result += bindings result += "}" @@ -44,27 +42,26 @@ def generate_dummy_code_boost(nclasses=10): bindings = "" for cl in range(nclasses): - decl += "class cl%03i;\n" % cl - decl += '\n' + decl += f"class cl{cl:03};\n" + decl += "\n" for cl in range(nclasses): decl += "class cl%03i {\n" % cl decl += "public:\n" - bindings += ' py::class_("cl%03i")\n' % (cl, cl) + bindings += f' py::class_("cl{cl:03}")\n' for fn in range(nfns): ret = random.randint(0, nclasses - 1) - params = [random.randint(0, nclasses - 1) for i in range(nargs)] - decl += " cl%03i *fn_%03i(" % (ret, fn) - decl += ", ".join("cl%03i *" % p for p in params) + params = [random.randint(0, nclasses - 1) for i in range(nargs)] + decl += f" cl{ret:03} *fn_{fn:03}(" + decl += ", ".join(f"cl{p:03} *" for p in params) decl += ");\n" - bindings += ' .def("fn_%03i", &cl%03i::fn_%03i, py::return_value_policy())\n' % \ - (fn, cl, fn) + bindings += f' .def("fn_{fn:03}", &cl{cl:03}::fn_{fn:03}, py::return_value_policy())\n' decl += "};\n\n" - bindings += ' ;\n' + bindings += " ;\n" result = "#include \n\n" result += "namespace py = boost::python;\n\n" - result += decl + '\n' + result += decl + "\n" result += "BOOST_PYTHON_MODULE(example) {\n" result += bindings result += "}" @@ -72,17 +69,19 @@ def generate_dummy_code_boost(nclasses=10): for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]: - print ("{") + print("{") for i in range(0, 10): - nclasses = 2 ** i + nclasses = 2**i with open("test.cpp", "w") as f: f.write(codegen(nclasses)) n1 = dt.datetime.now() - os.system("g++ -Os -shared -rdynamic -undefined dynamic_lookup " + os.system( + "g++ -Os -shared -rdynamic -undefined dynamic_lookup " "-fvisibility=hidden -std=c++14 test.cpp -I include " - "-I /System/Library/Frameworks/Python.framework/Headers -o test.so") + "-I /System/Library/Frameworks/Python.framework/Headers -o test.so" + ) n2 = dt.datetime.now() elapsed = (n2 - n1).total_seconds() - size = os.stat('test.so').st_size + size = os.stat("test.so").st_size print(" {%i, %f, %i}," % (nclasses * nfns, elapsed, size)) - print ("}") + print("}") diff --git a/docs/benchmark.rst b/docs/benchmark.rst index 59d533df..02c2ccde 100644 --- a/docs/benchmark.rst +++ b/docs/benchmark.rst @@ -93,5 +93,3 @@ favor. .. only:: latex .. image:: pybind11_vs_boost_python2.png - - diff --git a/docs/changelog.rst b/docs/changelog.rst index 2397056e..58cc4098 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,1587 @@ Changelog Starting with version 1.8.0, pybind11 releases use a `semantic versioning `_ policy. -v2.3.0 (Not yet released) +Changes will be added here periodically from the "Suggested changelog entry" +block in pull request descriptions. + + +IN DEVELOPMENT +-------------- + +Changes will be summarized here periodically. + +Changes: + +* ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & + ``dec_ref()`` are now enabled by default again. + `#4246 `_ + +* ``py::initialize_interpreter()`` using ``PyConfig_InitPythonConfig()`` + instead of ``PyConfig_InitIsolatedConfig()``, to obtain complete + ``sys.path``. + `#4473 `_ + +* Cast errors now always include Python type information, even if + ``PYBIND11_DETAILED_ERROR_MESSAGES`` is not defined. This increases binary + sizes slightly (~1.5%) but the error messages are much more informative. + `#4463 `_ + + +Build system improvements: + +* Update clang-tidy to 15 in CI. + `#4387 `_ + +* Moved the linting framework over to Ruff. + `#4483 `_ + +Version 2.10.4 (Mar 16, 2023) +---------------------------- + +Changes: + +* ``python3 -m pybind11`` gained a ``--version`` option (prints the version and + exits). + `#4526 `_ + +Bug Fixes: + +* Fix a warning when pydebug is enabled on Python 3.11. + `#4461 `_ + +* Ensure ``gil_scoped_release`` RAII is non-copyable. + `#4490 `_ + +* Ensure the tests dir does not show up with new versions of setuptools. + `#4510 `_ + +* Better stacklevel for a warning in setuptools helpers. + `#4516 `_ + +Version 2.10.3 (Jan 3, 2023) +---------------------------- + +Changes: + +* Temporarily made our GIL status assertions (added in 2.10.2) disabled by + default (re-enable manually by defining + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, will be enabled in 2.11). + `#4432 `_ + +* Improved error messages when ``inc_ref``/``dec_ref`` are called with an + invalid GIL state. + `#4427 `_ + `#4436 `_ + +Bug Fixes: + +* Some minor touchups found by static analyzers. + `#4440 `_ + + +Version 2.10.2 (Dec 20, 2022) +----------------------------- + +Changes: + +* ``scoped_interpreter`` constructor taking ``PyConfig``. + `#4330 `_ + +* ``pybind11/eigen/tensor.h`` adds converters to and from ``Eigen::Tensor`` and + ``Eigen::TensorMap``. + `#4201 `_ + +* ``PyGILState_Check()``'s were integrated to ``pybind11::handle`` + ``inc_ref()`` & ``dec_ref()``. The added GIL checks are guarded by + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, which is the default only if + ``NDEBUG`` is not defined. (Made non-default in 2.10.3, will be active in 2.11) + `#4246 `_ + +* Add option for enable/disable enum members in docstring. + `#2768 `_ + +* Fixed typing of ``KeysView``, ``ValuesView`` and ``ItemsView`` in ``bind_map``. + `#4353 `_ + +Bug fixes: + +* Bug fix affecting only Python 3.6 under very specific, uncommon conditions: + move ``PyEval_InitThreads()`` call to the correct location. + `#4350 `_ + +* Fix segfault bug when passing foreign native functions to functional.h. + `#4254 `_ + +Build system improvements: + +* Support setting PYTHON_LIBRARIES manually for Windows ARM cross-compilation + (classic mode). + `#4406 `_ + +* Extend IPO/LTO detection for ICX (a.k.a IntelLLVM) compiler. + `#4402 `_ + +* Allow calling ``find_package(pybind11 CONFIG)`` multiple times from separate + directories in the same CMake project and properly link Python (new mode). + `#4401 `_ + +* ``multiprocessing_set_spawn`` in pytest fixture for added safety. + `#4377 `_ + +* Fixed a bug in two pybind11/tools cmake scripts causing "Unknown arguments specified" errors. + `#4327 `_ + + + +Version 2.10.1 (Oct 31, 2022) +----------------------------- + +This is the first version to fully support embedding the newly released Python 3.11. + +Changes: + +* Allow ``pybind11::capsule`` constructor to take null destructor pointers. + `#4221 `_ + +* ``embed.h`` was changed so that ``PYTHONPATH`` is used also with Python 3.11 + (established behavior). + `#4119 `_ + +* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define), + along with many additional tests in ``test_gil_scoped.py``. The option may be + useful to try when debugging GIL-related issues, to determine if the more + complex default implementation is or is not to blame. See #4216 for + background. WARNING: Please be careful to not create ODR violations when + using the option: everything that is linked together with mutual symbol + visibility needs to be rebuilt. + `#4216 `_ + +* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes + Linux builds safer, and enables the removal of warning suppression pragmas for + Windows. + `#4298 `_ + +Bug fixes: + +* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various + ``py::str`` ctors when decoding surrogate utf characters. + `#4294 `_ + +* Revert perfect forwarding for ``make_iterator``. This broke at least one + valid use case. May revisit later. + `#4234 `_ + +* Fix support for safe casts to ``void*`` (regression in 2.10.0). + `#4275 `_ + +* Fix ``char8_t`` support (regression in 2.9). + `#4278 `_ + +* Unicode surrogate character in Python exception message leads to process + termination in ``error_already_set::what()``. + `#4297 `_ + +* Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. + `#4188 `_ + +* Make augmented assignment operators non-const for the object-api. Behavior + was previously broken for augmented assignment operators. + `#4065 `_ + +* Add proper error checking to C++ bindings for Python list append and insert. + `#4208 `_ + +* Work-around for Nvidia's CUDA nvcc compiler in versions 11.4.0 - 11.8.0. + `#4220 `_ + +* A workaround for PyPy was added in the ``py::error_already_set`` + implementation, related to PR `#1895 `_ + released with v2.10.0. + `#4079 `_ + +* Fixed compiler errors when C++23 ``std::forward_like`` is available. + `#4136 `_ + +* Properly raise exceptions in contains methods (like when an object in unhashable). + `#4209 `_ + +* Further improve another error in exception handling. + `#4232 `_ + +* ``get_local_internals()`` was made compatible with + ``finalize_interpreter()``, fixing potential freezes during interpreter + finalization. + `#4192 `_ + +Performance and style: + +* Reserve space in set and STL map casters if possible. This will prevent + unnecessary rehashing / resizing by knowing the number of keys ahead of time + for Python to C++ casting. This improvement will greatly speed up the casting + of large unordered maps and sets. + `#4194 `_ + +* GIL RAII scopes are non-copyable to avoid potential bugs. + `#4183 `_ + +* Explicitly default all relevant ctors for pytypes in the ``PYBIND11_OBJECT`` + macros and enforce the clang-tidy checks ``modernize-use-equals-default`` in + macros as well. + `#4017 `_ + +* Optimize iterator advancement in C++ bindings. + `#4237 `_ + +* Use the modern ``PyObject_GenericGetDict`` and ``PyObject_GenericSetDict`` + for handling dynamic attribute dictionaries. + `#4106 `_ + +* Document that users should use ``PYBIND11_NAMESPACE`` instead of using ``pybind11`` when + opening namespaces. Using namespace declarations and namespace qualification + remain the same as ``pybind11``. This is done to ensure consistent symbol + visibility. + `#4098 `_ + +* Mark ``detail::forward_like`` as constexpr. + `#4147 `_ + +* Optimize unpacking_collector when processing ``arg_v`` arguments. + `#4219 `_ + +* Optimize casting C++ object to ``None``. + `#4269 `_ + + +Build system improvements: + +* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``. + `#4195 `_ + +* Include a pkg-config file when installing pybind11, such as in the Python + package. + `#4077 `_ + +* Avoid stripping debug symbols when ``CMAKE_BUILD_TYPE`` is set to ``DEBUG`` + instead of ``Debug``. + `#4078 `_ + +* Followup to `#3948 `_, fixing vcpkg again. + `#4123 `_ + +Version 2.10.0 (Jul 15, 2022) +----------------------------- + +Removed support for Python 2.7, Python 3.5, and MSVC 2015. Support for MSVC +2017 is limited due to availability of CI runners; we highly recommend MSVC +2019 or 2022 be used. Initial support added for Python 3.11. + +New features: + +* ``py::anyset`` & ``py::frozenset`` were added, with copying (cast) to + ``std::set`` (similar to ``set``). + `#3901 `_ + +* Support bytearray casting to string. + `#3707 `_ + +* ``type_caster`` was added. ``std::monostate`` is a tag type + that allows ``std::variant`` to act as an optional, or allows default + construction of a ``std::variant`` holding a non-default constructible type. + `#3818 `_ + +* ``pybind11::capsule::set_name`` added to mutate the name of the capsule instance. + `#3866 `_ + +* NumPy: dtype constructor from type number added, accessors corresponding to + Python API ``dtype.num``, ``dtype.byteorder``, ``dtype.flags`` and + ``dtype.alignment`` added. + `#3868 `_ + + +Changes: + +* Python 3.6 is now the minimum supported version. + `#3688 `_ + `#3719 `_ + +* The minimum version for MSVC is now 2017. + `#3722 `_ + +* Fix issues with CPython 3.11 betas and add to supported test matrix. + `#3923 `_ + +* ``error_already_set`` is now safer and more performant, especially for + exceptions with long tracebacks, by delaying computation. + `#1895 `_ + +* Improve exception handling in python ``str`` bindings. + `#3826 `_ + +* The bindings for capsules now have more consistent exception handling. + `#3825 `_ + +* ``PYBIND11_OBJECT_CVT`` and ``PYBIND11_OBJECT_CVT_DEFAULT`` macro can now be + used to define classes in namespaces other than pybind11. + `#3797 `_ + +* Error printing code now uses ``PYBIND11_DETAILED_ERROR_MESSAGES`` instead of + requiring ``NDEBUG``, allowing use with release builds if desired. + `#3913 `_ + +* Implicit conversion of the literal ``0`` to ``pybind11::handle`` is now disabled. + `#4008 `_ + + +Bug fixes: + +* Fix exception handling when ``pybind11::weakref()`` fails. + `#3739 `_ + +* ``module_::def_submodule`` was missing proper error handling. This is fixed now. + `#3973 `_ + +* The behavior or ``error_already_set`` was made safer and the highly opaque + "Unknown internal error occurred" message was replaced with a more helpful + message. + `#3982 `_ + +* ``error_already_set::what()`` now handles non-normalized exceptions correctly. + `#3971 `_ + +* Support older C++ compilers where filesystem is not yet part of the standard + library and is instead included in ``std::experimental::filesystem``. + `#3840 `_ + +* Fix ``-Wfree-nonheap-object`` warnings produced by GCC by avoiding returning + pointers to static objects with ``return_value_policy::take_ownership``. + `#3946 `_ + +* Fix cast from pytype rvalue to another pytype. + `#3949 `_ + +* Ensure proper behavior when garbage collecting classes with dynamic attributes in Python >=3.9. + `#4051 `_ + +* A couple long-standing ``PYBIND11_NAMESPACE`` + ``__attribute__((visibility("hidden")))`` inconsistencies are now fixed + (affects only unusual environments). + `#4043 `_ + +* ``pybind11::detail::get_internals()`` is now resilient to in-flight Python + exceptions. + `#3981 `_ + +* Arrays with a dimension of size 0 are now properly converted to dynamic Eigen + matrices (more common in NumPy 1.23). + `#4038 `_ + +* Avoid catching unrelated errors when importing NumPy. + `#3974 `_ + +Performance and style: + +* Added an accessor overload of ``(object &&key)`` to reference steal the + object when using python types as keys. This prevents unnecessary reference + count overhead for attr, dictionary, tuple, and sequence look ups. Added + additional regression tests. Fixed a performance bug the caused accessor + assignments to potentially perform unnecessary copies. + `#3970 `_ + +* Perfect forward all args of ``make_iterator``. + `#3980 `_ + +* Avoid potential bug in pycapsule destructor by adding an ``error_guard`` to + one of the dtors. + `#3958 `_ + +* Optimize dictionary access in ``strip_padding`` for numpy. + `#3994 `_ + +* ``stl_bind.h`` bindings now take slice args as a const-ref. + `#3852 `_ + +* Made slice constructor more consistent, and improve performance of some + casters by allowing reference stealing. + `#3845 `_ + +* Change numpy dtype from_args method to use const ref. + `#3878 `_ + +* Follow rule of three to ensure ``PyErr_Restore`` is called only once. + `#3872 `_ + +* Added missing perfect forwarding for ``make_iterator`` functions. + `#3860 `_ + +* Optimize c++ to python function casting by using the rvalue caster. + `#3966 `_ + +* Optimize Eigen sparse matrix casting by removing unnecessary temporary. + `#4064 `_ + +* Avoid potential implicit copy/assignment constructors causing double free in + ``strdup_gaurd``. + `#3905 `_ + +* Enable clang-tidy checks ``misc-definitions-in-headers``, + ``modernize-loop-convert``, and ``modernize-use-nullptr``. + `#3881 `_ + `#3988 `_ + + +Build system improvements: + +* CMake: Fix file extension on Windows with cp36 and cp37 using FindPython. + `#3919 `_ + +* CMake: Support multiple Python targets (such as on vcpkg). + `#3948 `_ + +* CMake: Fix issue with NVCC on Windows. + `#3947 `_ + +* CMake: Drop the bitness check on cross compiles (like targeting WebAssembly + via Emscripten). + `#3959 `_ + +* Add MSVC builds in debug mode to CI. + `#3784 `_ + +* MSVC 2022 C++20 coverage was added to GitHub Actions, including Eigen. + `#3732 `_, + `#3741 `_ + + +Backend and tidying up: + +* New theme for the documentation. + `#3109 `_ + +* Remove idioms in code comments. Use more inclusive language. + `#3809 `_ + +* ``#include `` was removed from the ``pybind11/stl.h`` header. Your + project may break if it has a transitive dependency on this include. The fix + is to "Include What You Use". + `#3928 `_ + +* Avoid ``setup.py `` usage in internal tests. + `#3734 `_ + + +Version 2.9.2 (Mar 29, 2022) +---------------------------- + +Changes: + +* Enum now has an ``__index__`` method on Python <3.8 too. + `#3700 `_ + +* Local internals are now cleared after finalizing the interpreter. + `#3744 `_ + +Bug fixes: + +* Better support for Python 3.11 alphas. + `#3694 `_ + +* ``PYBIND11_TYPE_CASTER`` now uses fully qualified symbols, so it can be used + outside of ``pybind11::detail``. + `#3758 `_ + +* Some fixes for PyPy 3.9. + `#3768 `_ + +* Fixed a potential memleak in PyPy in ``get_type_override``. + `#3774 `_ + +* Fix usage of ``VISIBILITY_INLINES_HIDDEN``. + `#3721 `_ + + +Build system improvements: + +* Uses ``sysconfig`` module to determine installation locations on Python >= + 3.10, instead of ``distutils`` which has been deprecated. + `#3764 `_ + +* Support Catch 2.13.5+ (supporting GLIBC 2.34+). + `#3679 `_ + +* Fix test failures with numpy 1.22 by ignoring whitespace when comparing + ``str()`` of dtypes. + `#3682 `_ + + +Backend and tidying up: + +* clang-tidy: added ``readability-qualified-auto``, + ``readability-braces-around-statements``, + ``cppcoreguidelines-prefer-member-initializer``, + ``clang-analyzer-optin.performance.Padding``, + ``cppcoreguidelines-pro-type-static-cast-downcast``, and + ``readability-inconsistent-declaration-parameter-name``. + `#3702 `_, + `#3699 `_, + `#3716 `_, + `#3709 `_ + +* clang-format was added to the pre-commit actions, and the entire code base + automatically reformatted (after several iterations preparing for this leap). + `#3713 `_ + + +Version 2.9.1 (Feb 2, 2022) +--------------------------- + +Changes: + +* If possible, attach Python exception with ``py::raise_from`` to ``TypeError`` + when casting from C++ to Python. This will give additional info if Python + exceptions occur in the caster. Adds a test case of trying to convert a set + from C++ to Python when the hash function is not defined in Python. + `#3605 `_ + +* Add a mapping of C++11 nested exceptions to their Python exception + equivalent using ``py::raise_from``. This attaches the nested exceptions in + Python using the ``__cause__`` field. + `#3608 `_ + +* Propagate Python exception traceback using ``raise_from`` if a pybind11 + function runs out of overloads. + `#3671 `_ + +* ``py::multiple_inheritance`` is now only needed when C++ bases are hidden + from pybind11. + `#3650 `_ and + `#3659 `_ + + +Bug fixes: + +* Remove a boolean cast in ``numpy.h`` that causes MSVC C4800 warnings when + compiling against Python 3.10 or newer. + `#3669 `_ + +* Render ``py::bool_`` and ``py::float_`` as ``bool`` and ``float`` + respectively. + `#3622 `_ + +Build system improvements: + +* Fix CMake extension suffix computation on Python 3.10+. + `#3663 `_ + +* Allow ``CMAKE_ARGS`` to override CMake args in pybind11's own ``setup.py``. + `#3577 `_ + +* Remove a few deprecated c-headers. + `#3610 `_ + +* More uniform handling of test targets. + `#3590 `_ + +* Add clang-tidy readability check to catch potentially swapped function args. + `#3611 `_ + + +Version 2.9.0 (Dec 28, 2021) +---------------------------- + +This is the last version to support Python 2.7 and 3.5. + +New Features: + +* Allow ``py::args`` to be followed by other arguments; the remaining arguments + are implicitly keyword-only, as if a ``py::kw_only{}`` annotation had been + used. + `#3402 `_ + +Changes: + +* Make str/bytes/memoryview more interoperable with ``std::string_view``. + `#3521 `_ + +* Replace ``_`` with ``const_name`` in internals, avoid defining ``pybind::_`` + if ``_`` defined as macro (common gettext usage) + `#3423 `_ + + +Bug fixes: + +* Fix a rare warning about extra copy in an Eigen constructor. + `#3486 `_ + +* Fix caching of the C++ overrides. + `#3465 `_ + +* Add missing ``std::forward`` calls to some ``cpp_function`` overloads. + `#3443 `_ + +* Support PyPy 7.3.7 and the PyPy3.8 beta. Test python-3.11 on PRs with the + ``python dev`` label. + `#3419 `_ + +* Replace usage of deprecated ``Eigen::MappedSparseMatrix`` with + ``Eigen::Map>`` for Eigen 3.3+. + `#3499 `_ + +* Tweaks to support Microsoft Visual Studio 2022. + `#3497 `_ + +Build system improvements: + +* Nicer CMake printout and IDE organisation for pybind11's own tests. + `#3479 `_ + +* CMake: report version type as part of the version string to avoid a spurious + space in the package status message. + `#3472 `_ + +* Flags starting with ``-g`` in ``$CFLAGS`` and ``$CPPFLAGS`` are no longer + overridden by ``.Pybind11Extension``. + `#3436 `_ + +* Ensure ThreadPool is closed in ``setup_helpers``. + `#3548 `_ + +* Avoid LTS on ``mips64`` and ``ppc64le`` (reported broken). + `#3557 `_ + + +v2.8.1 (Oct 27, 2021) +--------------------- + +Changes and additions: + +* The simple namespace creation shortcut added in 2.8.0 was deprecated due to + usage of CPython internal API, and will be removed soon. Use + ``py::module_::import("types").attr("SimpleNamespace")``. + `#3374 `_ + +* Add C++ Exception type to throw and catch ``AttributeError``. Useful for + defining custom ``__setattr__`` and ``__getattr__`` methods. + `#3387 `_ + +Fixes: + +* Fixed the potential for dangling references when using properties with + ``std::optional`` types. + `#3376 `_ + +* Modernize usage of ``PyCodeObject`` on Python 3.9+ (moving toward support for + Python 3.11a1) + `#3368 `_ + +* A long-standing bug in ``eigen.h`` was fixed (originally PR #3343). The bug + was unmasked by newly added ``static_assert``'s in the Eigen 3.4.0 release. + `#3352 `_ + +* Support multiple raw inclusion of CMake helper files (Conan.io does this for + multi-config generators). + `#3420 `_ + +* Fix harmless warning on upcoming CMake 3.22. + `#3368 `_ + +* Fix 2.8.0 regression with MSVC 2017 + C++17 mode + Python 3. + `#3407 `_ + +* Fix 2.8.0 regression that caused undefined behavior (typically + segfaults) in ``make_key_iterator``/``make_value_iterator`` if dereferencing + the iterator returned a temporary value instead of a reference. + `#3348 `_ + + +v2.8.0 (Oct 4, 2021) +-------------------- + +New features: + +* Added ``py::raise_from`` to enable chaining exceptions. + `#3215 `_ + +* Allow exception translators to be optionally registered local to a module + instead of applying globally across all pybind11 modules. Use + ``register_local_exception_translator(ExceptionTranslator&& translator)`` + instead of ``register_exception_translator(ExceptionTranslator&& + translator)`` to keep your exception remapping code local to the module. + `#2650 `_ + +* Add ``make_simple_namespace`` function for instantiating Python + ``SimpleNamespace`` objects. **Deprecated in 2.8.1.** + `#2840 `_ + +* ``pybind11::scoped_interpreter`` and ``initialize_interpreter`` have new + arguments to allow ``sys.argv`` initialization. + `#2341 `_ + +* Allow Python builtins to be used as callbacks in CPython. + `#1413 `_ + +* Added ``view`` to view arrays with a different datatype. + `#987 `_ + +* Implemented ``reshape`` on arrays. + `#984 `_ + +* Enable defining custom ``__new__`` methods on classes by fixing bug + preventing overriding methods if they have non-pybind11 siblings. + `#3265 `_ + +* Add ``make_value_iterator()``, and fix ``make_key_iterator()`` to return + references instead of copies. + `#3293 `_ + +* Improve the classes generated by ``bind_map``: `#3310 `_ + + * Change ``.items`` from an iterator to a dictionary view. + * Add ``.keys`` and ``.values`` (both dictionary views). + * Allow ``__contains__`` to take any object. + +* ``pybind11::custom_type_setup`` was added, for customizing the + ``PyHeapTypeObject`` corresponding to a class, which may be useful for + enabling garbage collection support, among other things. + `#3287 `_ + + +Changes: + +* Set ``__file__`` constant when running ``eval_file`` in an embedded interpreter. + `#3233 `_ + +* Python objects and (C++17) ``std::optional`` now accepted in ``py::slice`` + constructor. + `#1101 `_ + +* The pybind11 proxy types ``str``, ``bytes``, ``bytearray``, ``tuple``, + ``list`` now consistently support passing ``ssize_t`` values for sizes and + indexes. Previously, only ``size_t`` was accepted in several interfaces. + `#3219 `_ + +* Avoid evaluating ``PYBIND11_TLS_REPLACE_VALUE`` arguments more than once. + `#3290 `_ + +Fixes: + +* Bug fix: enum value's ``__int__`` returning non-int when underlying type is + bool or of char type. + `#1334 `_ + +* Fixes bug in setting error state in Capsule's pointer methods. + `#3261 `_ + +* A long-standing memory leak in ``py::cpp_function::initialize`` was fixed. + `#3229 `_ + +* Fixes thread safety for some ``pybind11::type_caster`` which require lifetime + extension, such as for ``std::string_view``. + `#3237 `_ + +* Restore compatibility with gcc 4.8.4 as distributed by ubuntu-trusty, linuxmint-17. + `#3270 `_ + + +Build system improvements: + +* Fix regression in CMake Python package config: improper use of absolute path. + `#3144 `_ + +* Cached Python version information could become stale when CMake was re-run + with a different Python version. The build system now detects this and + updates this information. + `#3299 `_ + +* Specified UTF8-encoding in setup.py calls of open(). + `#3137 `_ + +* Fix a harmless warning from CMake 3.21 with the classic Python discovery. + `#3220 `_ + +* Eigen repo and version can now be specified as cmake options. + `#3324 `_ + + +Backend and tidying up: + +* Reduced thread-local storage required for keeping alive temporary data for + type conversion to one key per ABI version, rather than one key per extension + module. This makes the total thread-local storage required by pybind11 2 + keys per ABI version. + `#3275 `_ + +* Optimize NumPy array construction with additional moves. + `#3183 `_ + +* Conversion to ``std::string`` and ``std::string_view`` now avoids making an + extra copy of the data on Python >= 3.3. + `#3257 `_ + +* Remove const modifier from certain C++ methods on Python collections + (``list``, ``set``, ``dict``) such as (``clear()``, ``append()``, + ``insert()``, etc...) and annotated them with ``py-non-const``. + +* Enable readability ``clang-tidy-const-return`` and remove useless consts. + `#3254 `_ + `#3194 `_ + +* The clang-tidy ``google-explicit-constructor`` option was enabled. + `#3250 `_ + +* Mark a pytype move constructor as noexcept (perf). + `#3236 `_ + +* Enable clang-tidy check to guard against inheritance slicing. + `#3210 `_ + +* Legacy warning suppression pragma were removed from eigen.h. On Unix + platforms, please use -isystem for Eigen include directories, to suppress + compiler warnings originating from Eigen headers. Note that CMake does this + by default. No adjustments are needed for Windows. + `#3198 `_ + +* Format pybind11 with isort consistent ordering of imports + `#3195 `_ + +* The warnings-suppression "pragma clamp" at the top/bottom of pybind11 was + removed, clearing the path to refactoring and IWYU cleanup. + `#3186 `_ + +* Enable most bugprone checks in clang-tidy and fix the found potential bugs + and poor coding styles. + `#3166 `_ + +* Add ``clang-tidy-readability`` rules to make boolean casts explicit improving + code readability. Also enabled other misc and readability clang-tidy checks. + `#3148 `_ + +* Move object in ``.pop()`` for list. + `#3116 `_ + + + + +v2.7.1 (Aug 3, 2021) +--------------------- + +Minor missing functionality added: + +* Allow Python builtins to be used as callbacks in CPython. + `#1413 `_ + +Bug fixes: + +* Fix regression in CMake Python package config: improper use of absolute path. + `#3144 `_ + +* Fix Mingw64 and add to the CI testing matrix. + `#3132 `_ + +* Specified UTF8-encoding in setup.py calls of open(). + `#3137 `_ + +* Add clang-tidy-readability rules to make boolean casts explicit improving + code readability. Also enabled other misc and readability clang-tidy checks. + `#3148 `_ + +* Move object in ``.pop()`` for list. + `#3116 `_ + +Backend and tidying up: + +* Removed and fixed warning suppressions. + `#3127 `_ + `#3129 `_ + `#3135 `_ + `#3141 `_ + `#3142 `_ + `#3150 `_ + `#3152 `_ + `#3160 `_ + `#3161 `_ + + +v2.7.0 (Jul 16, 2021) +--------------------- + +New features: + +* Enable ``py::implicitly_convertible`` for + ``py::class_``-wrapped types. + `#3059 `_ + +* Allow function pointer extraction from overloaded functions. + `#2944 `_ + +* NumPy: added ``.char_()`` to type which gives the NumPy public ``char`` + result, which also distinguishes types by bit length (unlike ``.kind()``). + `#2864 `_ + +* Add ``pybind11::bytearray`` to manipulate ``bytearray`` similar to ``bytes``. + `#2799 `_ + +* ``pybind11/stl/filesystem.h`` registers a type caster that, on C++17/Python + 3.6+, converts ``std::filesystem::path`` to ``pathlib.Path`` and any + ``os.PathLike`` to ``std::filesystem::path``. + `#2730 `_ + +* A ``PYBIND11_VERSION_HEX`` define was added, similar to ``PY_VERSION_HEX``. + `#3120 `_ + + + +Changes: + +* ``py::str`` changed to exclusively hold ``PyUnicodeObject``. Previously + ``py::str`` could also hold ``bytes``, which is probably surprising, was + never documented, and can mask bugs (e.g. accidental use of ``py::str`` + instead of ``py::bytes``). + `#2409 `_ + +* Add a safety guard to ensure that the Python GIL is held when C++ calls back + into Python via ``object_api<>::operator()`` (e.g. ``py::function`` + ``__call__``). (This feature is available for Python 3.6+ only.) + `#2919 `_ + +* Catch a missing ``self`` argument in calls to ``__init__()``. + `#2914 `_ + +* Use ``std::string_view`` if available to avoid a copy when passing an object + to a ``std::ostream``. + `#3042 `_ + +* An important warning about thread safety was added to the ``iostream.h`` + documentation; attempts to make ``py::scoped_ostream_redirect`` thread safe + have been removed, as it was only partially effective. + `#2995 `_ + + +Fixes: + +* Performance: avoid unnecessary strlen calls. + `#3058 `_ + +* Fix auto-generated documentation string when using ``const T`` in + ``pyarray_t``. + `#3020 `_ + +* Unify error messages thrown by ``simple_collector``/``unpacking_collector``. + `#3013 `_ + +* ``pybind11::builtin_exception`` is now explicitly exported, which means the + types included/defined in different modules are identical, and exceptions + raised in different modules can be caught correctly. The documentation was + updated to explain that custom exceptions that are used across module + boundaries need to be explicitly exported as well. + `#2999 `_ + +* Fixed exception when printing UTF-8 to a ``scoped_ostream_redirect``. + `#2982 `_ + +* Pickle support enhancement: ``setstate`` implementation will attempt to + ``setattr`` ``__dict__`` only if the unpickled ``dict`` object is not empty, + to not force use of ``py::dynamic_attr()`` unnecessarily. + `#2972 `_ + +* Allow negative timedelta values to roundtrip. + `#2870 `_ + +* Fix unchecked errors could potentially swallow signals/other exceptions. + `#2863 `_ + +* Add null pointer check with ``std::localtime``. + `#2846 `_ + +* Fix the ``weakref`` constructor from ``py::object`` to create a new + ``weakref`` on conversion. + `#2832 `_ + +* Avoid relying on exceptions in C++17 when getting a ``shared_ptr`` holder + from a ``shared_from_this`` class. + `#2819 `_ + +* Allow the codec's exception to be raised instead of :code:`RuntimeError` when + casting from :code:`py::str` to :code:`std::string`. + `#2903 `_ + + +Build system improvements: + +* In ``setup_helpers.py``, test for platforms that have some multiprocessing + features but lack semaphores, which ``ParallelCompile`` requires. + `#3043 `_ + +* Fix ``pybind11_INCLUDE_DIR`` in case ``CMAKE_INSTALL_INCLUDEDIR`` is + absolute. + `#3005 `_ + +* Fix bug not respecting ``WITH_SOABI`` or ``WITHOUT_SOABI`` to CMake. + `#2938 `_ + +* Fix the default ``Pybind11Extension`` compilation flags with a Mingw64 python. + `#2921 `_ + +* Clang on Windows: do not pass ``/MP`` (ignored flag). + `#2824 `_ + +* ``pybind11.setup_helpers.intree_extensions`` can be used to generate + ``Pybind11Extension`` instances from cpp files placed in the Python package + source tree. + `#2831 `_ + +Backend and tidying up: + +* Enable clang-tidy performance, readability, and modernization checks + throughout the codebase to enforce best coding practices. + `#3046 `_, + `#3049 `_, + `#3051 `_, + `#3052 `_, + `#3080 `_, and + `#3094 `_ + + +* Checks for common misspellings were added to the pre-commit hooks. + `#3076 `_ + +* Changed ``Werror`` to stricter ``Werror-all`` for Intel compiler and fixed + minor issues. + `#2948 `_ + +* Fixed compilation with GCC < 5 when the user defines ``_GLIBCXX_USE_CXX11_ABI``. + `#2956 `_ + +* Added nox support for easier local testing and linting of contributions. + `#3101 `_ and + `#3121 `_ + +* Avoid RTD style issue with docutils 0.17+. + `#3119 `_ + +* Support pipx run, such as ``pipx run pybind11 --include`` for a quick compile. + `#3117 `_ + + + +v2.6.2 (Jan 26, 2021) +--------------------- + +Minor missing functionality added: + +* enum: add missing Enum.value property. + `#2739 `_ + +* Allow thread termination to be avoided during shutdown for CPython 3.7+ via + ``.disarm`` for ``gil_scoped_acquire``/``gil_scoped_release``. + `#2657 `_ + +Fixed or improved behavior in a few special cases: + +* Fix bug where the constructor of ``object`` subclasses would not throw on + being passed a Python object of the wrong type. + `#2701 `_ + +* The ``type_caster`` for integers does not convert Python objects with + ``__int__`` anymore with ``noconvert`` or during the first round of trying + overloads. + `#2698 `_ + +* When casting to a C++ integer, ``__index__`` is always called and not + considered as conversion, consistent with Python 3.8+. + `#2801 `_ + +Build improvements: + +* Setup helpers: ``extra_compile_args`` and ``extra_link_args`` automatically set by + Pybind11Extension are now prepended, which allows them to be overridden + by user-set ``extra_compile_args`` and ``extra_link_args``. + `#2808 `_ + +* Setup helpers: Don't trigger unused parameter warning. + `#2735 `_ + +* CMake: Support running with ``--warn-uninitialized`` active. + `#2806 `_ + +* CMake: Avoid error if included from two submodule directories. + `#2804 `_ + +* CMake: Fix ``STATIC`` / ``SHARED`` being ignored in FindPython mode. + `#2796 `_ + +* CMake: Respect the setting for ``CMAKE_CXX_VISIBILITY_PRESET`` if defined. + `#2793 `_ + +* CMake: Fix issue with FindPython2/FindPython3 not working with ``pybind11::embed``. + `#2662 `_ + +* CMake: mixing local and installed pybind11's would prioritize the installed + one over the local one (regression in 2.6.0). + `#2716 `_ + + +Bug fixes: + +* Fixed segfault in multithreaded environments when using + ``scoped_ostream_redirect``. + `#2675 `_ + +* Leave docstring unset when all docstring-related options are disabled, rather + than set an empty string. + `#2745 `_ + +* The module key in builtins that pybind11 uses to store its internals changed + from std::string to a python str type (more natural on Python 2, no change on + Python 3). + `#2814 `_ + +* Fixed assertion error related to unhandled (later overwritten) exception in + CPython 3.8 and 3.9 debug builds. + `#2685 `_ + +* Fix ``py::gil_scoped_acquire`` assert with CPython 3.9 debug build. + `#2683 `_ + +* Fix issue with a test failing on pytest 6.2. + `#2741 `_ + +Warning fixes: + +* Fix warning modifying constructor parameter 'flag' that shadows a field of + 'set_flag' ``[-Wshadow-field-in-constructor-modified]``. + `#2780 `_ + +* Suppressed some deprecation warnings about old-style + ``__init__``/``__setstate__`` in the tests. + `#2759 `_ + +Valgrind work: + +* Fix invalid access when calling a pybind11 ``__init__`` on a non-pybind11 + class instance. + `#2755 `_ + +* Fixed various minor memory leaks in pybind11's test suite. + `#2758 `_ + +* Resolved memory leak in cpp_function initialization when exceptions occurred. + `#2756 `_ + +* Added a Valgrind build, checking for leaks and memory-related UB, to CI. + `#2746 `_ + +Compiler support: + +* Intel compiler was not activating C++14 support due to a broken define. + `#2679 `_ + +* Support ICC and NVIDIA HPC SDK in C++17 mode. + `#2729 `_ + +* Support Intel OneAPI compiler (ICC 20.2) and add to CI. + `#2573 `_ + + + +v2.6.1 (Nov 11, 2020) +--------------------- + +* ``py::exec``, ``py::eval``, and ``py::eval_file`` now add the builtins module + as ``"__builtins__"`` to their ``globals`` argument, better matching ``exec`` + and ``eval`` in pure Python. + `#2616 `_ + +* ``setup_helpers`` will no longer set a minimum macOS version higher than the + current version. + `#2622 `_ + +* Allow deleting static properties. + `#2629 `_ + +* Seal a leak in ``def_buffer``, cleaning up the ``capture`` object after the + ``class_`` object goes out of scope. + `#2634 `_ + +* ``pybind11_INCLUDE_DIRS`` was incorrect, potentially causing a regression if + it was expected to include ``PYTHON_INCLUDE_DIRS`` (please use targets + instead). + `#2636 `_ + +* Added parameter names to the ``py::enum_`` constructor and methods, avoiding + ``arg0`` in the generated docstrings. + `#2637 `_ + +* Added ``needs_recompile`` optional function to the ``ParallelCompiler`` + helper, to allow a recompile to be skipped based on a user-defined function. + `#2643 `_ + + +v2.6.0 (Oct 21, 2020) +--------------------- + +See :ref:`upgrade-guide-2.6` for help upgrading to the new version. + +New features: + +* Keyword-only arguments supported in Python 2 or 3 with ``py::kw_only()``. + `#2100 `_ + +* Positional-only arguments supported in Python 2 or 3 with ``py::pos_only()``. + `#2459 `_ + +* ``py::is_final()`` class modifier to block subclassing (CPython only). + `#2151 `_ + +* Added ``py::prepend()``, allowing a function to be placed at the beginning of + the overload chain. + `#1131 `_ + +* Access to the type object now provided with ``py::type::of()`` and + ``py::type::of(h)``. + `#2364 `_ + +* Perfect forwarding support for methods. + `#2048 `_ + +* Added ``py::error_already_set::discard_as_unraisable()``. + `#2372 `_ + +* ``py::hash`` is now public. + `#2217 `_ + +* ``py::class_`` is now supported. Note that writing to one data + member of the union and reading another (type punning) is UB in C++. Thus + pybind11-bound enums should never be used for such conversions. + `#2320 `_. + +* Classes now check local scope when registering members, allowing a subclass + to have a member with the same name as a parent (such as an enum). + `#2335 `_ + +Code correctness features: + +* Error now thrown when ``__init__`` is forgotten on subclasses. + `#2152 `_ + +* Throw error if conversion to a pybind11 type if the Python object isn't a + valid instance of that type, such as ``py::bytes(o)`` when ``py::object o`` + isn't a bytes instance. + `#2349 `_ + +* Throw if conversion to ``str`` fails. + `#2477 `_ + + +API changes: + +* ``py::module`` was renamed ``py::module_`` to avoid issues with C++20 when + used unqualified, but an alias ``py::module`` is provided for backward + compatibility. + `#2489 `_ + +* Public constructors for ``py::module_`` have been deprecated; please use + ``pybind11::module_::create_extension_module`` if you were using the public + constructor (fairly rare after ``PYBIND11_MODULE`` was introduced). + `#2552 `_ + +* ``PYBIND11_OVERLOAD*`` macros and ``get_overload`` function replaced by + correctly-named ``PYBIND11_OVERRIDE*`` and ``get_override``, fixing + inconsistencies in the presence of a closing ``;`` in these macros. + ``get_type_overload`` is deprecated. + `#2325 `_ + +Packaging / building improvements: + +* The Python package was reworked to be more powerful and useful. + `#2433 `_ + + * :ref:`build-setuptools` is easier thanks to a new + ``pybind11.setup_helpers`` module, which provides utilities to use + setuptools with pybind11. It can be used via PEP 518, ``setup_requires``, + or by directly importing or copying ``setup_helpers.py`` into your project. + + * CMake configuration files are now included in the Python package. Use + ``pybind11.get_cmake_dir()`` or ``python -m pybind11 --cmakedir`` to get + the directory with the CMake configuration files, or include the + site-packages location in your ``CMAKE_MODULE_PATH``. Or you can use the + new ``pybind11[global]`` extra when you install ``pybind11``, which + installs the CMake files and headers into your base environment in the + standard location. + + * ``pybind11-config`` is another way to write ``python -m pybind11`` if you + have your PATH set up. + + * Added external typing support to the helper module, code from + ``import pybind11`` can now be type checked. + `#2588 `_ + +* Minimum CMake required increased to 3.4. + `#2338 `_ and + `#2370 `_ + + * Full integration with CMake's C++ standard system and compile features + replaces ``PYBIND11_CPP_STANDARD``. + + * Generated config file is now portable to different Python/compiler/CMake + versions. + + * Virtual environments prioritized if ``PYTHON_EXECUTABLE`` is not set + (``venv``, ``virtualenv``, and ``conda``) (similar to the new FindPython + mode). + + * Other CMake features now natively supported, like + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION``, ``set(CMAKE_CXX_VISIBILITY_PRESET + hidden)``. + + * ``CUDA`` as a language is now supported. + + * Helper functions ``pybind11_strip``, ``pybind11_extension``, + ``pybind11_find_import`` added, see :doc:`cmake/index`. + + * Optional :ref:`find-python-mode` and :ref:`nopython-mode` with CMake. + `#2370 `_ + +* Uninstall target added. + `#2265 `_ and + `#2346 `_ + +* ``pybind11_add_module()`` now accepts an optional ``OPT_SIZE`` flag that + switches the binding target to size-based optimization if the global build + type can not always be fixed to ``MinSizeRel`` (except in debug mode, where + optimizations remain disabled). ``MinSizeRel`` or this flag reduces binary + size quite substantially (~25% on some platforms). + `#2463 `_ + +Smaller or developer focused features and fixes: + +* Moved ``mkdoc.py`` to a new repo, `pybind11-mkdoc`_. There are no longer + submodules in the main repo. + +* ``py::memoryview`` segfault fix and update, with new + ``py::memoryview::from_memory`` in Python 3, and documentation. + `#2223 `_ + +* Fix for ``buffer_info`` on Python 2. + `#2503 `_ + +* If ``__eq__`` defined but not ``__hash__``, ``__hash__`` is now set to + ``None``. + `#2291 `_ + +* ``py::ellipsis`` now also works on Python 2. + `#2360 `_ + +* Pointer to ``std::tuple`` & ``std::pair`` supported in cast. + `#2334 `_ + +* Small fixes in NumPy support. ``py::array`` now uses ``py::ssize_t`` as first + argument type. + `#2293 `_ + +* Added missing signature for ``py::array``. + `#2363 `_ + +* ``unchecked_mutable_reference`` has access to operator ``()`` and ``[]`` when + const. + `#2514 `_ + +* ``py::vectorize`` is now supported on functions that return void. + `#1969 `_ + +* ``py::capsule`` supports ``get_pointer`` and ``set_pointer``. + `#1131 `_ + +* Fix crash when different instances share the same pointer of the same type. + `#2252 `_ + +* Fix for ``py::len`` not clearing Python's error state when it fails and throws. + `#2575 `_ + +* Bugfixes related to more extensive testing, new GitHub Actions CI. + `#2321 `_ + +* Bug in timezone issue in Eastern hemisphere midnight fixed. + `#2438 `_ + +* ``std::chrono::time_point`` now works when the resolution is not the same as + the system. + `#2481 `_ + +* Bug fixed where ``py::array_t`` could accept arrays that did not match the + requested ordering. + `#2484 `_ + +* Avoid a segfault on some compilers when types are removed in Python. + `#2564 `_ + +* ``py::arg::none()`` is now also respected when passing keyword arguments. + `#2611 `_ + +* PyPy fixes, PyPy 7.3.x now supported, including PyPy3. (Known issue with + PyPy2 and Windows `#2596 `_). + `#2146 `_ + +* CPython 3.9.0 workaround for undefined behavior (macOS segfault). + `#2576 `_ + +* CPython 3.9 warning fixes. + `#2253 `_ + +* Improved C++20 support, now tested in CI. + `#2489 `_ + `#2599 `_ + +* Improved but still incomplete debug Python interpreter support. + `#2025 `_ + +* NVCC (CUDA 11) now supported and tested in CI. + `#2461 `_ + +* NVIDIA PGI compilers now supported and tested in CI. + `#2475 `_ + +* At least Intel 18 now explicitly required when compiling with Intel. + `#2577 `_ + +* Extensive style checking in CI, with `pre-commit`_ support. Code + modernization, checked by clang-tidy. + +* Expanded docs, including new main page, new installing section, and CMake + helpers page, along with over a dozen new sections on existing pages. + +* In GitHub, new docs for contributing and new issue templates. + +.. _pre-commit: https://pre-commit.com + +.. _pybind11-mkdoc: https://github.com/pybind/pybind11-mkdoc + +v2.5.0 (Mar 31, 2020) +----------------------------------------------------- + +* Use C++17 fold expressions in type casters, if available. This can + improve performance during overload resolution when functions have + multiple arguments. + `#2043 `_. + +* Changed include directory resolution in ``pybind11/__init__.py`` + and installation in ``setup.py``. This fixes a number of open issues + where pybind11 headers could not be found in certain environments. + `#1995 `_. + +* C++20 ``char8_t`` and ``u8string`` support. `#2026 + `_. + +* CMake: search for Python 3.9. `bb9c91 + `_. + +* Fixes for MSYS-based build environments. + `#2087 `_, + `#2053 `_. + +* STL bindings for ``std::vector<...>::clear``. `#2074 + `_. + +* Read-only flag for ``py::buffer``. `#1466 + `_. + +* Exception handling during module initialization. + `bf2b031 `_. + +* Support linking against a CPython debug build. + `#2025 `_. + +* Fixed issues involving the availability and use of aligned ``new`` and + ``delete``. `#1988 `_, + `759221 `_. + +* Fixed a resource leak upon interpreter shutdown. + `#2020 `_. + +* Fixed error handling in the boolean caster. + `#1976 `_. + +v2.4.3 (Oct 15, 2019) +----------------------------------------------------- + +* Adapt pybind11 to a C API convention change in Python 3.8. `#1950 + `_. + +v2.4.2 (Sep 21, 2019) +----------------------------------------------------- + +* Replaced usage of a C++14 only construct. `#1929 + `_. + +* Made an ifdef future-proof for Python >= 4. `f3109d + `_. + +v2.4.1 (Sep 20, 2019) +----------------------------------------------------- + +* Fixed a problem involving implicit conversion from enumerations to integers + on Python 3.8. `#1780 `_. + +v2.4.0 (Sep 19, 2019) +----------------------------------------------------- + +* Try harder to keep pybind11-internal data structures separate when there + are potential ABI incompatibilities. Fixes crashes that occurred when loading + multiple pybind11 extensions that were e.g. compiled by GCC (libstdc++) + and Clang (libc++). + `#1588 `_ and + `c9f5a `_. + +* Added support for ``__await__``, ``__aiter__``, and ``__anext__`` protocols. + `#1842 `_. + +* ``pybind11_add_module()``: don't strip symbols when compiling in + ``RelWithDebInfo`` mode. `#1980 + `_. + +* ``enum_``: Reproduce Python behavior when comparing against invalid values + (e.g. ``None``, strings, etc.). Add back support for ``__invert__()``. + `#1912 `_, + `#1907 `_. + +* List insertion operation for ``py::list``. + Added ``.empty()`` to all collection types. + Added ``py::set::contains()`` and ``py::dict::contains()``. + `#1887 `_, + `#1884 `_, + `#1888 `_. + +* ``py::details::overload_cast_impl`` is available in C++11 mode, can be used + like ``overload_cast`` with an additional set of parentheses. + `#1581 `_. + +* Fixed ``get_include()`` on Conda. + `#1877 `_. + +* ``stl_bind.h``: negative indexing support. + `#1882 `_. + +* Minor CMake fix to add MinGW compatibility. + `#1851 `_. + +* GIL-related fixes. + `#1836 `_, + `8b90b `_. + +* Other very minor/subtle fixes and improvements. + `#1329 `_, + `#1910 `_, + `#1863 `_, + `#1847 `_, + `#1890 `_, + `#1860 `_, + `#1848 `_, + `#1821 `_, + `#1837 `_, + `#1833 `_, + `#1748 `_, + `#1852 `_. + +v2.3.0 (June 11, 2019) ----------------------------------------------------- * Significantly reduced module binary size (10-20%) when compiled in C++11 mode @@ -15,11 +1595,157 @@ v2.3.0 (Not yet released) for non-MSVC compilers). `#934 `_. -* Added support for write only properties. - `#1144 `_. +* Add basic support for tag-based static polymorphism, where classes + provide a method to returns the desired type of an instance. + `#1326 `_. + +* Python type wrappers (``py::handle``, ``py::object``, etc.) + now support map Python's number protocol onto C++ arithmetic + operators such as ``operator+``, ``operator/=``, etc. + `#1511 `_. + +* A number of improvements related to enumerations: + + 1. The ``enum_`` implementation was rewritten from scratch to reduce + code bloat. Rather than instantiating a full implementation for each + enumeration, most code is now contained in a generic base class. + `#1511 `_. + + 2. The ``value()`` method of ``py::enum_`` now accepts an optional + docstring that will be shown in the documentation of the associated + enumeration. `#1160 `_. + + 3. check for already existing enum value and throw an error if present. + `#1453 `_. + +* Support for over-aligned type allocation via C++17's aligned ``new`` + statement. `#1582 `_. + +* Added ``py::ellipsis()`` method for slicing of multidimensional NumPy arrays + `#1502 `_. + +* Numerous Improvements to the ``mkdoc.py`` script for extracting documentation + from C++ header files. + `#1788 `_. + +* ``pybind11_add_module()``: allow including Python as a ``SYSTEM`` include path. + `#1416 `_. + +* ``pybind11/stl.h`` does not convert strings to ``vector`` anymore. + `#1258 `_. + +* Mark static methods as such to fix auto-generated Sphinx documentation. + `#1732 `_. + +* Re-throw forced unwind exceptions (e.g. during pthread termination). + `#1208 `_. + +* Added ``__contains__`` method to the bindings of maps (``std::map``, + ``std::unordered_map``). + `#1767 `_. + +* Improvements to ``gil_scoped_acquire``. + `#1211 `_. + +* Type caster support for ``std::deque``. + `#1609 `_. + +* Support for ``std::unique_ptr`` holders, whose deleters differ between a base and derived + class. `#1353 `_. -* The ``value()`` method of ``py::enum_`` now accepts an optional docstring - that will be shown in the documentation of the associated enumeration. +* Construction of STL array/vector-like data structures from + iterators. Added an ``extend()`` operation. + `#1709 `_, + +* CMake build system improvements for projects that include non-C++ + files (e.g. plain C, CUDA) in ``pybind11_add_module`` et al. + `#1678 `_. + +* Fixed asynchronous invocation and deallocation of Python functions + wrapped in ``std::function``. + `#1595 `_. + +* Fixes regarding return value policy propagation in STL type casters. + `#1603 `_. + +* Fixed scoped enum comparisons. + `#1571 `_. + +* Fixed iostream redirection for code that releases the GIL. + `#1368 `_, + +* A number of CI-related fixes. + `#1757 `_, + `#1744 `_, + `#1670 `_. + +v2.2.4 (September 11, 2018) +----------------------------------------------------- + +* Use new Python 3.7 Thread Specific Storage (TSS) implementation if available. + `#1454 `_, + `#1517 `_. + +* Fixes for newer MSVC versions and C++17 mode. + `#1347 `_, + `#1462 `_. + +* Propagate return value policies to type-specific casters + when casting STL containers. + `#1455 `_. + +* Allow ostream-redirection of more than 1024 characters. + `#1479 `_. + +* Set ``Py_DEBUG`` define when compiling against a debug Python build. + `#1438 `_. + +* Untangle integer logic in number type caster to work for custom + types that may only be castable to a restricted set of builtin types. + `#1442 `_. + +* CMake build system: Remember Python version in cache file. + `#1434 `_. + +* Fix for custom smart pointers: use ``std::addressof`` to obtain holder + address instead of ``operator&``. + `#1435 `_. + +* Properly report exceptions thrown during module initialization. + `#1362 `_. + +* Fixed a segmentation fault when creating empty-shaped NumPy array. + `#1371 `_. + +* The version of Intel C++ compiler must be >= 2017, and this is now checked by + the header files. `#1363 `_. + +* A few minor typo fixes and improvements to the test suite, and + patches that silence compiler warnings. + +* Vectors now support construction from generators, as well as ``extend()`` from a + list or generator. + `#1496 `_. + + +v2.2.3 (April 29, 2018) +----------------------------------------------------- + +* The pybind11 header location detection was replaced by a new implementation + that no longer depends on ``pip`` internals (the recently released ``pip`` + 10 has restricted access to this API). + `#1190 `_. + +* Small adjustment to an implementation detail to work around a compiler segmentation fault in Clang 3.3/3.4. + `#1350 `_. + +* The minimal supported version of the Intel compiler was >= 17.0 since + pybind11 v2.1. This check is now explicit, and a compile-time error is raised + if the compiler meet the requirement. + `#1363 `_. + +* Fixed an endianness-related fault in the test suite. + `#1287 `_. v2.2.2 (February 7, 2018) ----------------------------------------------------- @@ -89,7 +1815,7 @@ v2.2.2 (February 7, 2018) v2.2.1 (September 14, 2017) ----------------------------------------------------- -* Added ``py::module::reload()`` member function for reloading a module. +* Added ``py::module_::reload()`` member function for reloading a module. `#1040 `_. * Fixed a reference leak in the number converter. @@ -152,6 +1878,7 @@ v2.2.0 (August 31, 2017) from cpp_module import CppBase1, CppBase2 + class PyDerived(CppBase1, CppBase2): def __init__(self): CppBase1.__init__(self) # C++ bases must be initialized explicitly @@ -264,7 +1991,7 @@ v2.2.0 (August 31, 2017) in reference cycles. `#856 `_. -* Numpy and buffer protocol related improvements: +* NumPy and buffer protocol related improvements: 1. Support for negative strides in Python buffer objects/numpy arrays. This required changing integers from unsigned to signed for the related C++ APIs. @@ -311,6 +2038,9 @@ v2.2.0 (August 31, 2017) * Fixed overriding static properties in derived classes. `#784 `_. +* Added support for write only properties. + `#1144 `_. + * Improved deduction of member functions of a derived class when its bases aren't registered with pybind11. `#855 `_. @@ -361,7 +2091,7 @@ v2.2.0 (August 31, 2017) * Intel C++ compiler compatibility fixes. `#937 `_. -* Fixed implicit conversion of `py::enum_` to integer types on Python 2.7. +* Fixed implicit conversion of ``py::enum_`` to integer types on Python 2.7. `#821 `_. * Added ``py::hash`` to fetch the hash value of Python objects, and @@ -578,7 +2308,7 @@ Happy Christmas! being (notably dynamic attributes in custom types). `#527 `_. -* Significant work on the documentation -- in particular, the monolitic +* Significant work on the documentation -- in particular, the monolithic ``advanced.rst`` file was restructured into a easier to read hierarchical organization. `#448 `_. @@ -646,8 +2376,8 @@ Happy Christmas! `_. - 3. This version of pybind11 uses a redesigned mechnism for instantiating - trempoline classes that are used to override virtual methods from within + 3. This version of pybind11 uses a redesigned mechanism for instantiating + trampoline classes that are used to override virtual methods from within Python. This led to the following user-visible syntax change: instead of .. code-block:: cpp @@ -992,7 +2722,7 @@ Happy Christmas! * Improved support for ``std::shared_ptr<>`` conversions * Initial support for ``std::set<>`` conversions * Fixed type resolution issue for types defined in a separate plugin module -* Cmake build system improvements +* CMake build system improvements * Factored out generic functionality to non-templated code (smaller code size) * Added a code size / compile time benchmark vs Boost.Python * Added an appveyor CI script diff --git a/docs/classes.rst b/docs/classes.rst index 890257d5..c0c53135 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -44,14 +44,14 @@ interactive Python session demonstrating this example is shown below: % python >>> import example - >>> p = example.Pet('Molly') + >>> p = example.Pet("Molly") >>> print(p) >>> p.getName() - u'Molly' - >>> p.setName('Charly') + 'Molly' + >>> p.setName("Charly") >>> p.getName() - u'Charly' + 'Charly' .. seealso:: @@ -74,7 +74,7 @@ Note how ``print(p)`` produced a rather useless summary of our data structure in >>> print(p) -To address this, we could bind an utility function that returns a human-readable +To address this, we could bind a utility function that returns a human-readable summary to the special method slot named ``__repr__``. Unfortunately, there is no suitable functionality in the ``Pet`` data structure, and it would be nice if we did not have to change it. This can easily be accomplished by binding a @@ -122,12 +122,12 @@ This makes it possible to write .. code-block:: pycon - >>> p = example.Pet('Molly') + >>> p = example.Pet("Molly") >>> p.name - u'Molly' - >>> p.name = 'Charly' + 'Molly' + >>> p.name = "Charly" >>> p.name - u'Charly' + 'Charly' Now suppose that ``Pet::name`` was a private internal variable that can only be accessed via setters and getters. @@ -174,10 +174,10 @@ Native Python classes can pick up new attributes dynamically: .. code-block:: pycon >>> class Pet: - ... name = 'Molly' + ... name = "Molly" ... >>> p = Pet() - >>> p.name = 'Charly' # overwrite existing + >>> p.name = "Charly" # overwrite existing >>> p.age = 2 # dynamically add a new attribute By default, classes exported from C++ do not support this and the only writable @@ -195,7 +195,7 @@ Trying to set any other attribute results in an error: .. code-block:: pycon >>> p = example.Pet() - >>> p.name = 'Charly' # OK, attribute defined in C++ + >>> p.name = "Charly" # OK, attribute defined in C++ >>> p.age = 2 # fail AttributeError: 'Pet' object has no attribute 'age' @@ -213,7 +213,7 @@ Now everything works as expected: .. code-block:: pycon >>> p = example.Pet() - >>> p.name = 'Charly' # OK, overwrite value in C++ + >>> p.name = "Charly" # OK, overwrite value in C++ >>> p.age = 2 # OK, dynamically add a new attribute >>> p.__dict__ # just like a native Python class {'age': 2} @@ -228,8 +228,8 @@ just brings them on par. .. _inheritance: -Inheritance and automatic upcasting -=================================== +Inheritance and automatic downcasting +===================================== Suppose now that the example consists of two data structures with an inheritance relationship: @@ -280,11 +280,11 @@ expose fields and methods of both types: .. code-block:: pycon - >>> p = example.Dog('Molly') + >>> p = example.Dog("Molly") >>> p.name - u'Molly' + 'Molly' >>> p.bark() - u'woof!' + 'woof!' The C++ classes defined above are regular non-polymorphic types with an inheritance relationship. This is reflected in Python: @@ -298,7 +298,7 @@ inheritance relationship. This is reflected in Python: >>> p = example.pet_store() >>> type(p) # `Dog` instance behind `Pet` pointer - Pet # no pointer upcasting for regular non-polymorphic types + Pet # no pointer downcasting for regular non-polymorphic types >>> p.bark() AttributeError: 'Pet' object has no attribute 'bark' @@ -330,11 +330,11 @@ will automatically recognize this: >>> p = example.pet_store2() >>> type(p) - PolymorphicDog # automatically upcast + PolymorphicDog # automatically downcast >>> p.bark() - u'woof!' + 'woof!' -Given a pointer to a polymorphic base, pybind11 performs automatic upcasting +Given a pointer to a polymorphic base, pybind11 performs automatic downcasting to the actual derived type. Note that this goes beyond the usual situation in C++: we don't just get access to the virtual functions of the base, we get the concrete derived type including functions and attributes that the base type may @@ -373,8 +373,8 @@ sequence. py::class_(m, "Pet") .def(py::init()) - .def("set", (void (Pet::*)(int)) &Pet::set, "Set the pet's age") - .def("set", (void (Pet::*)(const std::string &)) &Pet::set, "Set the pet's name"); + .def("set", static_cast(&Pet::set), "Set the pet's age") + .def("set", static_cast(&Pet::set), "Set the pet's name"); The overload signatures are also visible in the method's docstring: @@ -422,9 +422,19 @@ on constness, the ``py::const_`` tag should be used: .def("foo_mutable", py::overload_cast(&Widget::foo)) .def("foo_const", py::overload_cast(&Widget::foo, py::const_)); +If you prefer the ``py::overload_cast`` syntax but have a C++11 compatible compiler only, +you can use ``py::detail::overload_cast_impl`` with an additional set of parentheses: + +.. code-block:: cpp -.. [#cpp14] A compiler which supports the ``-std=c++14`` flag - or Visual Studio 2015 Update 2 and newer. + template + using overload_cast_ = pybind11::detail::overload_cast_impl; + + py::class_(m, "Pet") + .def("set", overload_cast_()(&Pet::set), "Set the pet's age") + .def("set", overload_cast_()(&Pet::set), "Set the pet's name"); + +.. [#cpp14] A compiler which supports the ``-std=c++14`` flag. .. note:: @@ -435,8 +445,7 @@ on constness, the ``py::const_`` tag should be used: Enumerations and internal types =============================== -Let's now suppose that the example class contains an internal enumeration type, -e.g.: +Let's now suppose that the example class contains internal types like enumerations, e.g.: .. code-block:: cpp @@ -446,10 +455,15 @@ e.g.: Cat }; + struct Attributes { + float age = 0; + }; + Pet(const std::string &name, Kind type) : name(name), type(type) { } std::string name; Kind type; + Attributes attr; }; The binding code for this example looks as follows: @@ -460,22 +474,28 @@ The binding code for this example looks as follows: pet.def(py::init()) .def_readwrite("name", &Pet::name) - .def_readwrite("type", &Pet::type); + .def_readwrite("type", &Pet::type) + .def_readwrite("attr", &Pet::attr); py::enum_(pet, "Kind") .value("Dog", Pet::Kind::Dog) .value("Cat", Pet::Kind::Cat) .export_values(); -To ensure that the ``Kind`` type is created within the scope of ``Pet``, the -``pet`` :class:`class_` instance must be supplied to the :class:`enum_`. + py::class_(pet, "Attributes") + .def(py::init<>()) + .def_readwrite("age", &Pet::Attributes::age); + + +To ensure that the nested types ``Kind`` and ``Attributes`` are created within the scope of ``Pet``, the +``pet`` :class:`class_` instance must be supplied to the :class:`enum_` and :class:`class_` constructor. The :func:`enum_::export_values` function exports the enum entries into the parent scope, which should be skipped for newer C++11-style strongly typed enums. .. code-block:: pycon - >>> p = Pet('Lucy', Pet.Cat) + >>> p = Pet("Lucy", Pet.Cat) >>> p.type Kind.Cat >>> int(p.type) @@ -488,6 +508,24 @@ The entries defined by the enumeration type are exposed in the ``__members__`` p >>> Pet.Kind.__members__ {'Dog': Kind.Dog, 'Cat': Kind.Cat} +The ``name`` property returns the name of the enum value as a unicode string. + +.. note:: + + It is also possible to use ``str(enum)``, however these accomplish different + goals. The following shows how these two approaches differ. + + .. code-block:: pycon + + >>> p = Pet("Lucy", Pet.Cat) + >>> pet_type = p.type + >>> pet_type + Pet.Cat + >>> str(pet_type) + 'Pet.Cat' + >>> pet_type.name + 'Cat' + .. note:: When the special tag ``py::arithmetic()`` is specified to the ``enum_`` diff --git a/docs/cmake/index.rst b/docs/cmake/index.rst new file mode 100644 index 00000000..eaf66d70 --- /dev/null +++ b/docs/cmake/index.rst @@ -0,0 +1,8 @@ +CMake helpers +------------- + +Pybind11 can be used with ``add_subdirectory(extern/pybind11)``, or from an +install with ``find_package(pybind11 CONFIG)``. The interface provided in +either case is functionally identical. + +.. cmake-module:: ../../tools/pybind11Config.cmake.in diff --git a/docs/compiling.rst b/docs/compiling.rst index b5d6ce94..2b543be0 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -3,6 +3,8 @@ Build systems ############# +.. _build-setuptools: + Building with setuptools ======================== @@ -13,6 +15,212 @@ the [python_example]_ repository. .. [python_example] https://github.com/pybind/python_example +A helper file is provided with pybind11 that can simplify usage with setuptools. + +To use pybind11 inside your ``setup.py``, you have to have some system to +ensure that ``pybind11`` is installed when you build your package. There are +four possible ways to do this, and pybind11 supports all four: You can ask all +users to install pybind11 beforehand (bad), you can use +:ref:`setup_helpers-pep518` (good, but very new and requires Pip 10), +:ref:`setup_helpers-setup_requires` (discouraged by Python packagers now that +PEP 518 is available, but it still works everywhere), or you can +:ref:`setup_helpers-copy-manually` (always works but you have to manually sync +your copy to get updates). + +An example of a ``setup.py`` using pybind11's helpers: + +.. code-block:: python + + from glob import glob + from setuptools import setup + from pybind11.setup_helpers import Pybind11Extension + + ext_modules = [ + Pybind11Extension( + "python_example", + sorted(glob("src/*.cpp")), # Sort source files for reproducibility + ), + ] + + setup(..., ext_modules=ext_modules) + +If you want to do an automatic search for the highest supported C++ standard, +that is supported via a ``build_ext`` command override; it will only affect +``Pybind11Extensions``: + +.. code-block:: python + + from glob import glob + from setuptools import setup + from pybind11.setup_helpers import Pybind11Extension, build_ext + + ext_modules = [ + Pybind11Extension( + "python_example", + sorted(glob("src/*.cpp")), + ), + ] + + setup(..., cmdclass={"build_ext": build_ext}, ext_modules=ext_modules) + +If you have single-file extension modules that are directly stored in the +Python source tree (``foo.cpp`` in the same directory as where a ``foo.py`` +would be located), you can also generate ``Pybind11Extensions`` using +``setup_helpers.intree_extensions``: ``intree_extensions(["path/to/foo.cpp", +...])`` returns a list of ``Pybind11Extensions`` which can be passed to +``ext_modules``, possibly after further customizing their attributes +(``libraries``, ``include_dirs``, etc.). By doing so, a ``foo.*.so`` extension +module will be generated and made available upon installation. + +``intree_extension`` will automatically detect if you are using a ``src``-style +layout (as long as no namespace packages are involved), but you can also +explicitly pass ``package_dir`` to it (as in ``setuptools.setup``). + +Since pybind11 does not require NumPy when building, a light-weight replacement +for NumPy's parallel compilation distutils tool is included. Use it like this: + +.. code-block:: python + + from pybind11.setup_helpers import ParallelCompile + + # Optional multithreaded build + ParallelCompile("NPY_NUM_BUILD_JOBS").install() + + setup(...) + +The argument is the name of an environment variable to control the number of +threads, such as ``NPY_NUM_BUILD_JOBS`` (as used by NumPy), though you can set +something different if you want; ``CMAKE_BUILD_PARALLEL_LEVEL`` is another choice +a user might expect. You can also pass ``default=N`` to set the default number +of threads (0 will take the number of threads available) and ``max=N``, the +maximum number of threads; if you have a large extension you may want set this +to a memory dependent number. + +If you are developing rapidly and have a lot of C++ files, you may want to +avoid rebuilding files that have not changed. For simple cases were you are +using ``pip install -e .`` and do not have local headers, you can skip the +rebuild if an object file is newer than its source (headers are not checked!) +with the following: + +.. code-block:: python + + from pybind11.setup_helpers import ParallelCompile, naive_recompile + + ParallelCompile("NPY_NUM_BUILD_JOBS", needs_recompile=naive_recompile).install() + + +If you have a more complex build, you can implement a smarter function and pass +it to ``needs_recompile``, or you can use [Ccache]_ instead. ``CXX="cache g++" +pip install -e .`` would be the way to use it with GCC, for example. Unlike the +simple solution, this even works even when not compiling in editable mode, but +it does require Ccache to be installed. + +Keep in mind that Pip will not even attempt to rebuild if it thinks it has +already built a copy of your code, which it deduces from the version number. +One way to avoid this is to use [setuptools_scm]_, which will generate a +version number that includes the number of commits since your last tag and a +hash for a dirty directory. Another way to force a rebuild is purge your cache +or use Pip's ``--no-cache-dir`` option. + +.. [Ccache] https://ccache.dev + +.. [setuptools_scm] https://github.com/pypa/setuptools_scm + +.. _setup_helpers-pep518: + +PEP 518 requirements (Pip 10+ required) +--------------------------------------- + +If you use `PEP 518's `_ +``pyproject.toml`` file, you can ensure that ``pybind11`` is available during +the compilation of your project. When this file exists, Pip will make a new +virtual environment, download just the packages listed here in ``requires=``, +and build a wheel (binary Python package). It will then throw away the +environment, and install your wheel. + +Your ``pyproject.toml`` file will likely look something like this: + +.. code-block:: toml + + [build-system] + requires = ["setuptools>=42", "wheel", "pybind11~=2.6.1"] + build-backend = "setuptools.build_meta" + +.. note:: + + The main drawback to this method is that a `PEP 517`_ compliant build tool, + such as Pip 10+, is required for this approach to work; older versions of + Pip completely ignore this file. If you distribute binaries (called wheels + in Python) using something like `cibuildwheel`_, remember that ``setup.py`` + and ``pyproject.toml`` are not even contained in the wheel, so this high + Pip requirement is only for source builds, and will not affect users of + your binary wheels. If you are building SDists and wheels, then + `pypa-build`_ is the recommended official tool. + +.. _PEP 517: https://www.python.org/dev/peps/pep-0517/ +.. _cibuildwheel: https://cibuildwheel.readthedocs.io +.. _pypa-build: https://pypa-build.readthedocs.io/en/latest/ + +.. _setup_helpers-setup_requires: + +Classic ``setup_requires`` +-------------------------- + +If you want to support old versions of Pip with the classic +``setup_requires=["pybind11"]`` keyword argument to setup, which triggers a +two-phase ``setup.py`` run, then you will need to use something like this to +ensure the first pass works (which has not yet installed the ``setup_requires`` +packages, since it can't install something it does not know about): + +.. code-block:: python + + try: + from pybind11.setup_helpers import Pybind11Extension + except ImportError: + from setuptools import Extension as Pybind11Extension + + +It doesn't matter that the Extension class is not the enhanced subclass for the +first pass run; and the second pass will have the ``setup_requires`` +requirements. + +This is obviously more of a hack than the PEP 518 method, but it supports +ancient versions of Pip. + +.. _setup_helpers-copy-manually: + +Copy manually +------------- + +You can also copy ``setup_helpers.py`` directly to your project; it was +designed to be usable standalone, like the old example ``setup.py``. You can +set ``include_pybind11=False`` to skip including the pybind11 package headers, +so you can use it with git submodules and a specific git version. If you use +this, you will need to import from a local file in ``setup.py`` and ensure the +helper file is part of your MANIFEST. + + +Closely related, if you include pybind11 as a subproject, you can run the +``setup_helpers.py`` inplace. If loaded correctly, this should even pick up +the correct include for pybind11, though you can turn it off as shown above if +you want to input it manually. + +Suggested usage if you have pybind11 as a submodule in ``extern/pybind11``: + +.. code-block:: python + + DIR = os.path.abspath(os.path.dirname(__file__)) + + sys.path.append(os.path.join(DIR, "extern", "pybind11")) + from pybind11.setup_helpers import Pybind11Extension # noqa: E402 + + del sys.path[-1] + + +.. versionchanged:: 2.6 + + Added ``setup_helpers`` file. + Building with cppimport ======================== @@ -33,8 +241,8 @@ extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 2.8.12) - project(example) + cmake_minimum_required(VERSION 3.4...3.18) + project(example LANGUAGES CXX) add_subdirectory(pybind11) pybind11_add_module(example example.cpp) @@ -50,6 +258,11 @@ PyPI integration, can be found in the [cmake_example]_ repository. .. [cmake_example] https://github.com/pybind/cmake_example +.. versionchanged:: 2.6 + CMake 3.4+ is required. + +Further information can be found at :doc:`cmake/index`. + pybind11_add_module ------------------- @@ -59,7 +272,7 @@ function with the following signature: .. code-block:: cmake pybind11_add_module( [MODULE | SHARED] [EXCLUDE_FROM_ALL] - [NO_EXTRAS] [THIN_LTO] source1 [source2 ...]) + [NO_EXTRAS] [THIN_LTO] [OPT_SIZE] source1 [source2 ...]) This function behaves very much like CMake's builtin ``add_library`` (in fact, it's a wrapper function around that command). It will add a library target @@ -89,46 +302,65 @@ will result in code bloat and is generally not recommended. As stated above, LTO is enabled by default. Some newer compilers also support different flavors of LTO such as `ThinLTO`_. Setting ``THIN_LTO`` will cause the function to prefer this flavor if available. The function falls back to -regular LTO if ``-flto=thin`` is not available. +regular LTO if ``-flto=thin`` is not available. If +``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is set (either ``ON`` or ``OFF``), then +that will be respected instead of the built-in flag search. + +.. note:: + + If you want to set the property form on targets or the + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION_`` versions of this, you should + still use ``set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF)`` (otherwise a + no-op) to disable pybind11's ipo flags. + +The ``OPT_SIZE`` flag enables size-based optimization equivalent to the +standard ``/Os`` or ``-Os`` compiler flags and the ``MinSizeRel`` build type, +which avoid optimizations that that can substantially increase the size of the +resulting binary. This flag is particularly useful in projects that are split +into performance-critical parts and associated bindings. In this case, we can +compile the project in release mode (and hence, optimize performance globally), +and specify ``OPT_SIZE`` for the binding target, where size might be the main +concern as performance is often less critical here. A ~25% size reduction has +been observed in practice. This flag only changes the optimization behavior at +a per-target level and takes precedence over the global CMake build type +(``Release``, ``RelWithDebInfo``) except for ``Debug`` builds, where +optimizations remain disabled. .. _ThinLTO: http://clang.llvm.org/docs/ThinLTO.html Configuration variables ----------------------- -By default, pybind11 will compile modules with the C++14 standard, if available -on the target compiler, falling back to C++11 if C++14 support is not -available. Note, however, that this default is subject to change: future -pybind11 releases are expected to migrate to newer C++ standards as they become -available. To override this, the standard flag can be given explicitly in -``PYBIND11_CPP_STANDARD``: +By default, pybind11 will compile modules with the compiler default or the +minimum standard required by pybind11, whichever is higher. You can set the +standard explicitly with +`CMAKE_CXX_STANDARD `_: .. code-block:: cmake - # Use just one of these: - # GCC/clang: - set(PYBIND11_CPP_STANDARD -std=c++11) - set(PYBIND11_CPP_STANDARD -std=c++14) - set(PYBIND11_CPP_STANDARD -std=c++1z) # Experimental C++17 support - # MSVC: - set(PYBIND11_CPP_STANDARD /std:c++14) - set(PYBIND11_CPP_STANDARD /std:c++latest) # Enables some MSVC C++17 features - - add_subdirectory(pybind11) # or find_package(pybind11) + set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ version selection") # or 11, 14, 17, 20 + set(CMAKE_CXX_STANDARD_REQUIRED ON) # optional, ensure standard is supported + set(CMAKE_CXX_EXTENSIONS OFF) # optional, keep compiler extensions off -Note that this and all other configuration variables must be set **before** the -call to ``add_subdirectory`` or ``find_package``. The variables can also be set -when calling CMake from the command line using the ``-D=`` flag. +The variables can also be set when calling CMake from the command line using +the ``-D=`` flag. You can also manually set ``CXX_STANDARD`` +on a target or use ``target_compile_features`` on your targets - anything that +CMake supports. -The target Python version can be selected by setting ``PYBIND11_PYTHON_VERSION`` -or an exact Python installation can be specified with ``PYTHON_EXECUTABLE``. -For example: +Classic Python support: The target Python version can be selected by setting +``PYBIND11_PYTHON_VERSION`` or an exact Python installation can be specified +with ``PYTHON_EXECUTABLE``. For example: .. code-block:: bash cmake -DPYBIND11_PYTHON_VERSION=3.6 .. - # or - cmake -DPYTHON_EXECUTABLE=path/to/python .. + + # Another method: + cmake -DPYTHON_EXECUTABLE=/path/to/python .. + + # This often is a good way to get the current Python, works in environments: + cmake -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") .. + find_package vs. add_subdirectory --------------------------------- @@ -139,12 +371,31 @@ See the `Config file`_ docstring for details of relevant CMake variables. .. code-block:: cmake - cmake_minimum_required(VERSION 2.8.12) - project(example) + cmake_minimum_required(VERSION 3.4...3.18) + project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) pybind11_add_module(example example.cpp) +Note that ``find_package(pybind11)`` will only work correctly if pybind11 +has been correctly installed on the system, e. g. after downloading or cloning +the pybind11 repository : + +.. code-block:: bash + + # Classic CMake + cd pybind11 + mkdir build + cd build + cmake .. + make install + + # CMake 3.15+ + cd pybind11 + cmake -S . -B build + cmake --build build -j 2 # Build on 2 cores + cmake --install build + Once detected, the aforementioned ``pybind11_add_module`` can be employed as before. The function usage and configuration variables are identical no matter if pybind11 is added as a subdirectory or found as an installed package. You @@ -153,41 +404,142 @@ can refer to the same [cmake_example]_ repository for a full sample project .. _Config file: https://github.com/pybind/pybind11/blob/master/tools/pybind11Config.cmake.in -Advanced: interface library target ----------------------------------- -When using a version of CMake greater than 3.0, pybind11 can additionally -be used as a special *interface library* . The target ``pybind11::module`` -is available with pybind11 headers, Python headers and libraries as needed, -and C++ compile definitions attached. This target is suitable for linking -to an independently constructed (through ``add_library``, not -``pybind11_add_module``) target in the consuming project. +.. _find-python-mode: + +FindPython mode +--------------- + +CMake 3.12+ (3.15+ recommended, 3.18.2+ ideal) added a new module called +FindPython that had a highly improved search algorithm and modern targets +and tools. If you use FindPython, pybind11 will detect this and use the +existing targets instead: + +.. code-block:: cmake + + cmake_minimum_required(VERSION 3.15...3.22) + project(example LANGUAGES CXX) + + find_package(Python 3.6 COMPONENTS Interpreter Development REQUIRED) + find_package(pybind11 CONFIG REQUIRED) + # or add_subdirectory(pybind11) + + pybind11_add_module(example example.cpp) + +You can also use the targets (as listed below) with FindPython. If you define +``PYBIND11_FINDPYTHON``, pybind11 will perform the FindPython step for you +(mostly useful when building pybind11's own tests, or as a way to change search +algorithms from the CMake invocation, with ``-DPYBIND11_FINDPYTHON=ON``. + +.. warning:: + + If you use FindPython to multi-target Python versions, use the individual + targets listed below, and avoid targets that directly include Python parts. + +There are `many ways to hint or force a discovery of a specific Python +installation `_), +setting ``Python_ROOT_DIR`` may be the most common one (though with +virtualenv/venv support, and Conda support, this tends to find the correct +Python version more often than the old system did). + +.. warning:: + + When the Python libraries (i.e. ``libpythonXX.a`` and ``libpythonXX.so`` + on Unix) are not available, as is the case on a manylinux image, the + ``Development`` component will not be resolved by ``FindPython``. When not + using the embedding functionality, CMake 3.18+ allows you to specify + ``Development.Module`` instead of ``Development`` to resolve this issue. + +.. versionadded:: 2.6 + +Advanced: interface library targets +----------------------------------- + +Pybind11 supports modern CMake usage patterns with a set of interface targets, +available in all modes. The targets provided are: + + ``pybind11::headers`` + Just the pybind11 headers and minimum compile requirements + + ``pybind11::pybind11`` + Python headers + ``pybind11::headers`` + + ``pybind11::python_link_helper`` + Just the "linking" part of pybind11:module + + ``pybind11::module`` + Everything for extension modules - ``pybind11::pybind11`` + ``Python::Module`` (FindPython CMake 3.15+) or ``pybind11::python_link_helper`` + + ``pybind11::embed`` + Everything for embedding the Python interpreter - ``pybind11::pybind11`` + ``Python::Python`` (FindPython) or Python libs + + ``pybind11::lto`` / ``pybind11::thin_lto`` + An alternative to `INTERPROCEDURAL_OPTIMIZATION` for adding link-time optimization. + + ``pybind11::windows_extras`` + ``/bigobj`` and ``/mp`` for MSVC. + + ``pybind11::opt_size`` + ``/Os`` for MSVC, ``-Os`` for other compilers. Does nothing for debug builds. + +Two helper functions are also provided: + + ``pybind11_strip(target)`` + Strips a target (uses ``CMAKE_STRIP`` after the target is built) + + ``pybind11_extension(target)`` + Sets the correct extension (with SOABI) for a target. + +You can use these targets to build complex applications. For example, the +``add_python_module`` function is identical to: .. code-block:: cmake - cmake_minimum_required(VERSION 3.0) - project(example) + cmake_minimum_required(VERSION 3.4) + project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) add_library(example MODULE main.cpp) - target_link_libraries(example PRIVATE pybind11::module) - set_target_properties(example PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") + + target_link_libraries(example PRIVATE pybind11::module pybind11::lto pybind11::windows_extras) + + pybind11_extension(example) + if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Strip unnecessary sections of the binary on Linux/macOS + pybind11_strip(example) + endif() + + set_target_properties(example PROPERTIES CXX_VISIBILITY_PRESET "hidden" + CUDA_VISIBILITY_PRESET "hidden") + +Instead of setting properties, you can set ``CMAKE_*`` variables to initialize these correctly. .. warning:: Since pybind11 is a metatemplate library, it is crucial that certain compiler flags are provided to ensure high quality code generation. In contrast to the ``pybind11_add_module()`` command, the CMake interface - library only provides the *minimal* set of parameters to ensure that the - code using pybind11 compiles, but it does **not** pass these extra compiler - flags (i.e. this is up to you). + provides a *composable* set of targets to ensure that you retain flexibility. + It can be especially important to provide or set these properties; the + :ref:`FAQ ` contains an explanation on why these are needed. - These include Link Time Optimization (``-flto`` on GCC/Clang/ICPC, ``/GL`` - and ``/LTCG`` on Visual Studio) and .OBJ files with many sections on Visual - Studio (``/bigobj``). The :ref:`FAQ ` contains an - explanation on why these are needed. +.. versionadded:: 2.6 + +.. _nopython-mode: + +Advanced: NOPYTHON mode +----------------------- + +If you want complete control, you can set ``PYBIND11_NOPYTHON`` to completely +disable Python integration (this also happens if you run ``FindPython2`` and +``FindPython3`` without running ``FindPython``). This gives you complete +freedom to integrate into an existing system (like `Scikit-Build's +`_ ``PythonExtensions``). +``pybind11_add_module`` and ``pybind11_extension`` will be unavailable, and the +targets will be missing any Python specific behavior. + +.. versionadded:: 2.6 Embedding the Python interpreter -------------------------------- @@ -201,8 +553,8 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.0) - project(example) + cmake_minimum_required(VERSION 3.4...3.18) + project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -222,10 +574,7 @@ On Linux, you can compile an example such as the one given in .. code-block:: bash - $ c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` example.cpp -o example`python3-config --extension-suffix` - -The flags given here assume that you're using Python 3. For Python 2, just -change the executable appropriately (to ``python`` or ``python2``). + $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix) The ``python3 -m pybind11 --includes`` command fetches the include paths for both pybind11 and Python headers. This assumes that pybind11 has been installed @@ -233,19 +582,13 @@ using ``pip`` or ``conda``. If it hasn't, you can also manually specify ``-I /include`` together with the Python includes path ``python3-config --includes``. -Note that Python 2.7 modules don't use a special suffix, so you should simply -use ``example.so`` instead of ``example`python3-config --extension-suffix```. -Besides, the ``--extension-suffix`` option may or may not be available, depending -on the distribution; in the latter case, the module extension can be manually -set to ``.so``. - -On Mac OS: the build command is almost the same but it also requires passing +On macOS: the build command is almost the same but it also requires passing the ``-undefined dynamic_lookup`` flag so as to ignore missing symbols when building the module: .. code-block:: bash - $ c++ -O3 -Wall -shared -std=c++11 -undefined dynamic_lookup `python3 -m pybind11 --includes` example.cpp -o example`python3-config --extension-suffix` + $ c++ -O3 -Wall -shared -std=c++11 -undefined dynamic_lookup $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix) In general, it is advisable to include several additional build parameters that can considerably reduce the size of the created binary. Refer to section @@ -263,6 +606,13 @@ build system that works on all platforms including Windows. of possibly importing a second Python library into a process that already contains one (which will lead to a segfault). + +Building with Bazel +=================== + +You can build with the Bazel build system using the `pybind11_bazel +`_ repository. + Generating binding code automatically ===================================== @@ -271,3 +621,18 @@ code by introspecting existing C++ codebases using LLVM/Clang. See the [binder]_ documentation for details. .. [binder] http://cppbinder.readthedocs.io/en/latest/about.html + +[AutoWIG]_ is a Python library that wraps automatically compiled libraries into +high-level languages. It parses C++ code using LLVM/Clang technologies and +generates the wrappers using the Mako templating engine. The approach is automatic, +extensible, and applies to very complex C++ libraries, composed of thousands of +classes or incorporating modern meta-programming constructs. + +.. [AutoWIG] https://github.com/StatisKit/AutoWIG + +[robotpy-build]_ is a is a pure python, cross platform build tool that aims to +simplify creation of python wheels for pybind11 projects, and provide +cross-project dependency management. Additionally, it is able to autogenerate +customizable pybind11-based wrappers by parsing C++ header files. + +.. [robotpy-build] https://robotpy-build.readthedocs.io diff --git a/docs/conf.py b/docs/conf.py index 868f9609..6e24751e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # pybind11 documentation build configuration file, created by # sphinx-quickstart on Sun Oct 11 19:23:48 2015. @@ -13,57 +12,69 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os -import shlex +import re import subprocess +import sys +from pathlib import Path + +DIR = Path(__file__).parent.resolve() # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['breathe'] +extensions = [ + "breathe", + "sphinx_copybutton", + "sphinxcontrib.rsvgconverter", + "sphinxcontrib.moderncmakedomain", +] -breathe_projects = {'pybind11': '.build/doxygenxml/'} -breathe_default_project = 'pybind11' -breathe_domain_by_extension = {'h': 'cpp'} +breathe_projects = {"pybind11": ".build/doxygenxml/"} +breathe_default_project = "pybind11" +breathe_domain_by_extension = {"h": "cpp"} # Add any paths that contain templates here, relative to this directory. -templates_path = ['.templates'] +templates_path = [".templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'pybind11' -copyright = '2017, Wenzel Jakob' -author = 'Wenzel Jakob' +project = "pybind11" +copyright = "2017, Wenzel Jakob" +author = "Wenzel Jakob" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. -# -# The short X.Y version. -version = '2.3' + +# Read the listed version +with open("../pybind11/_version.py") as f: + code = compile(f.read(), "../pybind11/_version.py", "exec") +loc = {} +exec(code, loc) + # The full version, including alpha/beta/rc tags. -release = '2.3.dev0' +version = loc["__version__"] # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -74,37 +85,37 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['.build', 'release.rst'] +exclude_patterns = [".build", "release.rst"] # The reST default role (used for this markup: `text`) to use for all # documents. -default_role = 'any' +default_role = "any" # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -#pygments_style = 'monokai' +# pygments_style = 'monokai' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -115,141 +126,138 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' - -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - - html_context = { - 'css_files': [ - '_static/theme_overrides.css' - ] - } -else: - html_context = { - 'css_files': [ - '//media.readthedocs.org/css/sphinx_rtd_theme.css', - '//media.readthedocs.org/css/readthedocs-doc-embed.css', - '_static/theme_overrides.css' - ] - } +html_theme = "furo" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None +# " v documentation". +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] + +html_css_files = [ + "css/custom.css", +] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'pybind11doc' +htmlhelp_basename = "pybind11doc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -'preamble': '\DeclareUnicodeCharacter{00A0}{}', +latex_engine = "pdflatex" -# Latex figure (float) alignment -#'figure_align': 'htbp', +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # + # Additional stuff for the LaTeX preamble. + # remove blank pages (between the title page and the TOC, etc.) + "classoptions": ",openany,oneside", + "preamble": r""" +\usepackage{fontawesome} +\usepackage{textgreek} +\DeclareUnicodeCharacter{00A0}{} +\DeclareUnicodeCharacter{2194}{\faArrowsH} +\DeclareUnicodeCharacter{1F382}{\faBirthdayCake} +\DeclareUnicodeCharacter{1F355}{\faAdjust} +\DeclareUnicodeCharacter{0301}{'} +\DeclareUnicodeCharacter{03C0}{\textpi} + +""", + # Latex figure (float) alignment + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pybind11.tex', 'pybind11 Documentation', - 'Wenzel Jakob', 'manual'), + (master_doc, "pybind11.tex", "pybind11 Documentation", "Wenzel Jakob", "manual"), ] # The name of an image file (relative to this directory) to place at the top of @@ -258,32 +266,29 @@ # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pybind11', 'pybind11 Documentation', - [author], 1) -] +man_pages = [(master_doc, "pybind11", "pybind11 Documentation", [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -292,41 +297,72 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pybind11', 'pybind11 Documentation', - author, 'pybind11', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "pybind11", + "pybind11 Documentation", + author, + "pybind11", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False -primary_domain = 'cpp' -highlight_language = 'cpp' +primary_domain = "cpp" +highlight_language = "cpp" def generate_doxygen_xml(app): - build_dir = os.path.join(app.confdir, '.build') + build_dir = os.path.join(app.confdir, ".build") if not os.path.exists(build_dir): os.mkdir(build_dir) try: - subprocess.call(['doxygen', '--version']) - retcode = subprocess.call(['doxygen'], cwd=app.confdir) + subprocess.call(["doxygen", "--version"]) + retcode = subprocess.call(["doxygen"], cwd=app.confdir) if retcode < 0: - sys.stderr.write("doxygen error code: {}\n".format(-retcode)) + sys.stderr.write(f"doxygen error code: {-retcode}\n") except OSError as e: - sys.stderr.write("doxygen execution failed: {}\n".format(e)) + sys.stderr.write(f"doxygen execution failed: {e}\n") + + +def prepare(app): + with open(DIR.parent / "README.rst") as f: + contents = f.read() + + if app.builder.name == "latex": + # Remove badges and stuff from start + contents = contents[contents.find(r".. start") :] + + # Filter out section titles for index.rst for LaTeX + contents = re.sub(r"^(.*)\n[-~]{3,}$", r"**\1**", contents, flags=re.MULTILINE) + + with open(DIR / "readme.rst", "w") as f: + f.write(contents) + + +def clean_up(app, exception): # noqa: ARG001 + (DIR / "readme.rst").unlink() def setup(app): - """Add hook for building doxygen xml when needed""" + # Add hook for building doxygen xml when needed app.connect("builder-inited", generate_doxygen_xml) + + # Copy the readme in + app.connect("builder-inited", prepare) + + # Clean up the generated readme + app.connect("build-finished", clean_up) diff --git a/docs/faq.rst b/docs/faq.rst index bfe83036..28498e7d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -4,9 +4,11 @@ Frequently asked questions "ImportError: dynamic module does not define init function" =========================================================== -You are likely using an incompatible version of Python (for instance, the -extension library was compiled against Python 2, while the interpreter is -running on top of some version of Python 3, or vice versa). +1. Make sure that the name specified in PYBIND11_MODULE is identical to the +filename of the extension library (without suffixes such as ``.so``). + +2. If the above did not fix the issue, you are likely using an incompatible +version of Python that does not match what you compiled with. "Symbol not found: ``__Py_ZeroStruct`` / ``_PyInstanceMethod_Type``" ======================================================================== @@ -23,17 +25,7 @@ The Python interpreter immediately crashes when importing my module See the first answer. -CMake doesn't detect the right Python version -============================================= - -The CMake-based build system will try to automatically detect the installed -version of Python and link against that. When this fails, or when there are -multiple versions of Python and it finds the wrong one, delete -``CMakeCache.txt`` and then invoke CMake as follows: - -.. code-block:: bash - - cmake -DPYTHON_EXECUTABLE:FILEPATH= . +.. _faq_reference_arguments: Limitations involving reference arguments ========================================= @@ -60,7 +52,7 @@ provided by the caller -- in fact, it does nothing at all. .. code-block:: python def increment(i): - i += 1 # nope.. + i += 1 # nope.. pybind11 is also affected by such language-level conventions, which means that binding ``increment`` or ``increment_ptr`` will also create Python functions @@ -94,8 +86,8 @@ following example: .. code-block:: cpp - void init_ex1(py::module &); - void init_ex2(py::module &); + void init_ex1(py::module_ &); + void init_ex2(py::module_ &); /* ... */ PYBIND11_MODULE(example, m) { @@ -108,7 +100,7 @@ following example: .. code-block:: cpp - void init_ex1(py::module &m) { + void init_ex1(py::module_ &m) { m.def("add", [](int a, int b) { return a + b; }); } @@ -116,7 +108,7 @@ following example: .. code-block:: cpp - void init_ex1(py::module &m) { + void init_ex2(py::module_ &m) { m.def("sub", [](int a, int b) { return a - b; }); } @@ -153,7 +145,7 @@ using C++14 template metaprogramming. .. _`faq:hidden_visibility`: -"‘SomeClass’ declared with greater visibility than the type of its field ‘SomeClass::member’ [-Wattributes]" +"'SomeClass' declared with greater visibility than the type of its field 'SomeClass::member' [-Wattributes]" ============================================================================================================ This error typically indicates that you are compiling without the required @@ -175,8 +167,8 @@ can be changed, but even if it isn't it is not always enough to guarantee complete independence of the symbols involved when not using ``-fvisibility=hidden``. -Additionally, ``-fvisiblity=hidden`` can deliver considerably binary size -savings. (See the following section for more details). +Additionally, ``-fvisibility=hidden`` can deliver considerably binary size +savings. (See the following section for more details.) .. _`faq:symhidden`: @@ -186,7 +178,7 @@ How can I create smaller binaries? To do its job, pybind11 extensively relies on a programming technique known as *template metaprogramming*, which is a way of performing computation at compile -time using type information. Template metaprogamming usually instantiates code +time using type information. Template metaprogramming usually instantiates code involving significant numbers of deeply nested types that are either completely removed or reduced to just a few instructions during the compiler's optimization phase. However, due to the nested nature of these types, the resulting symbol @@ -228,19 +220,76 @@ In addition to decreasing binary size, ``-fvisibility=hidden`` also avoids potential serious issues when loading multiple modules and is required for proper pybind operation. See the previous FAQ entry for more details. -Working with ancient Visual Studio 2009 builds on Windows -========================================================= +How can I properly handle Ctrl-C in long-running functions? +=========================================================== + +Ctrl-C is received by the Python interpreter, and holds it until the GIL +is released, so a long-running function won't be interrupted. -The official Windows distributions of Python are compiled using truly -ancient versions of Visual Studio that lack good C++11 support. Some users -implicitly assume that it would be impossible to load a plugin built with -Visual Studio 2015 into a Python distribution that was compiled using Visual -Studio 2009. However, no such issue exists: it's perfectly legitimate to -interface DLLs that are built with different compilers and/or C libraries. -Common gotchas to watch out for involve not ``free()``-ing memory region -that that were ``malloc()``-ed in another shared library, using data -structures with incompatible ABIs, and so on. pybind11 is very careful not -to make these types of mistakes. +To interrupt from inside your function, you can use the ``PyErr_CheckSignals()`` +function, that will tell if a signal has been raised on the Python side. This +function merely checks a flag, so its impact is negligible. When a signal has +been received, you must either explicitly interrupt execution by throwing +``py::error_already_set`` (which will propagate the existing +``KeyboardInterrupt``), or clear the error (which you usually will not want): + +.. code-block:: cpp + + PYBIND11_MODULE(example, m) + { + m.def("long running_func", []() + { + for (;;) { + if (PyErr_CheckSignals() != 0) + throw py::error_already_set(); + // Long running iteration + } + }); + } + +CMake doesn't detect the right Python version +============================================= + +The CMake-based build system will try to automatically detect the installed +version of Python and link against that. When this fails, or when there are +multiple versions of Python and it finds the wrong one, delete +``CMakeCache.txt`` and then add ``-DPYTHON_EXECUTABLE=$(which python)`` to your +CMake configure line. (Replace ``$(which python)`` with a path to python if +your prefer.) + +You can alternatively try ``-DPYBIND11_FINDPYTHON=ON``, which will activate the +new CMake FindPython support instead of pybind11's custom search. Requires +CMake 3.12+, and 3.15+ or 3.18.2+ are even better. You can set this in your +``CMakeLists.txt`` before adding or finding pybind11, as well. + +Inconsistent detection of Python version in CMake and pybind11 +============================================================== + +The functions ``find_package(PythonInterp)`` and ``find_package(PythonLibs)`` +provided by CMake for Python version detection are modified by pybind11 due to +unreliability and limitations that make them unsuitable for pybind11's needs. +Instead pybind11 provides its own, more reliable Python detection CMake code. +Conflicts can arise, however, when using pybind11 in a project that *also* uses +the CMake Python detection in a system with several Python versions installed. + +This difference may cause inconsistencies and errors if *both* mechanisms are +used in the same project. + +There are three possible solutions: + +1. Avoid using ``find_package(PythonInterp)`` and ``find_package(PythonLibs)`` + from CMake and rely on pybind11 in detecting Python version. If this is not + possible, the CMake machinery should be called *before* including pybind11. +2. Set ``PYBIND11_FINDPYTHON`` to ``True`` or use ``find_package(Python + COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better, + 3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead + of the old, deprecated search tools, and these modules are much better at + finding the correct Python. +3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python. + However, you will have to use the target-based system, and do more setup + yourself, because it does not know about or include things that depend on + Python, like ``pybind11_add_module``. This might be ideal for integrating + into an existing system, like scikit-build's Python helpers. How to cite this project? ========================= @@ -256,4 +305,3 @@ discourse: note = {https://github.com/pybind/pybind11}, title = {pybind11 -- Seamless operability between C++11 and Python} } - diff --git a/docs/index.rst b/docs/index.rst index d236611b..4e2e8ca3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,18 +1,17 @@ -.. only: not latex +.. only:: latex - .. image:: pybind11-logo.png + Intro + ===== -pybind11 --- Seamless operability between C++11 and Python -========================================================== +.. include:: readme.rst -.. only: not latex +.. only:: not latex Contents: .. toctree:: :maxdepth: 1 - intro changelog upgrade @@ -20,6 +19,7 @@ pybind11 --- Seamless operability between C++11 and Python :caption: The Basics :maxdepth: 2 + installing basics classes compiling @@ -45,3 +45,4 @@ pybind11 --- Seamless operability between C++11 and Python benchmark limitations reference + cmake/index diff --git a/docs/installing.rst b/docs/installing.rst new file mode 100644 index 00000000..30b9f185 --- /dev/null +++ b/docs/installing.rst @@ -0,0 +1,105 @@ +.. _installing: + +Installing the library +###################### + +There are several ways to get the pybind11 source, which lives at +`pybind/pybind11 on GitHub `_. The pybind11 +developers recommend one of the first three ways listed here, submodule, PyPI, +or conda-forge, for obtaining pybind11. + +.. _include_as_a_submodule: + +Include as a submodule +====================== + +When you are working on a project in Git, you can use the pybind11 repository +as a submodule. From your git repository, use: + +.. code-block:: bash + + git submodule add -b stable ../../pybind/pybind11 extern/pybind11 + git submodule update --init + +This assumes you are placing your dependencies in ``extern/``, and that you are +using GitHub; if you are not using GitHub, use the full https or ssh URL +instead of the relative URL ``../../pybind/pybind11`` above. Some other servers +also require the ``.git`` extension (GitHub does not). + +From here, you can now include ``extern/pybind11/include``, or you can use +the various integration tools (see :ref:`compiling`) pybind11 provides directly +from the local folder. + +Include with PyPI +================= + +You can download the sources and CMake files as a Python package from PyPI +using Pip. Just use: + +.. code-block:: bash + + pip install pybind11 + +This will provide pybind11 in a standard Python package format. If you want +pybind11 available directly in your environment root, you can use: + +.. code-block:: bash + + pip install "pybind11[global]" + +This is not recommended if you are installing with your system Python, as it +will add files to ``/usr/local/include/pybind11`` and +``/usr/local/share/cmake/pybind11``, so unless that is what you want, it is +recommended only for use in virtual environments or your ``pyproject.toml`` +file (see :ref:`compiling`). + +Include with conda-forge +======================== + +You can use pybind11 with conda packaging via `conda-forge +`_: + +.. code-block:: bash + + conda install -c conda-forge pybind11 + + +Include with vcpkg +================== +You can download and install pybind11 using the Microsoft `vcpkg +`_ dependency manager: + +.. code-block:: bash + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + vcpkg install pybind11 + +The pybind11 port in vcpkg is kept up to date by Microsoft team members and +community contributors. If the version is out of date, please `create an issue +or pull request `_ on the vcpkg +repository. + +Global install with brew +======================== + +The brew package manager (Homebrew on macOS, or Linuxbrew on Linux) has a +`pybind11 package +`_. +To install: + +.. code-block:: bash + + brew install pybind11 + +.. We should list Conan, and possibly a few other C++ package managers (hunter, +.. perhaps). Conan has a very clean CMake integration that would be good to show. + +Other options +============= + +Other locations you can find pybind11 are `listed here +`_; these are maintained +by various packagers and the community. diff --git a/docs/intro.rst b/docs/intro.rst deleted file mode 100644 index 3e9420ec..00000000 --- a/docs/intro.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. image:: pybind11-logo.png - -About this project -================== -**pybind11** is a lightweight header-only library that exposes C++ types in Python -and vice versa, mainly to create Python bindings of existing C++ code. Its -goals and syntax are similar to the excellent `Boost.Python`_ library by David -Abrahams: to minimize boilerplate code in traditional extension modules by -inferring type information using compile-time introspection. - -.. _Boost.Python: http://www.boost.org/doc/libs/release/libs/python/doc/index.html - -The main issue with Boost.Python—and the reason for creating such a similar -project—is Boost. Boost is an enormously large and complex suite of utility -libraries that works with almost every C++ compiler in existence. This -compatibility has its cost: arcane template tricks and workarounds are -necessary to support the oldest and buggiest of compiler specimens. Now that -C++11-compatible compilers are widely available, this heavy machinery has -become an excessively large and unnecessary dependency. -Think of this library as a tiny self-contained version of Boost.Python with -everything stripped away that isn't relevant for binding generation. Without -comments, the core header files only require ~4K lines of code and depend on -Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This -compact implementation was possible thanks to some of the new C++11 language -features (specifically: tuples, lambda functions and variadic templates). Since -its creation, this library has grown beyond Boost.Python in many ways, leading -to dramatically simpler binding code in many common situations. - -Core features -************* -The following core C++ features can be mapped to Python - -- Functions accepting and returning custom data structures per value, reference, or pointer -- Instance methods and static methods -- Overloaded functions -- Instance attributes and static attributes -- Arbitrary exception types -- Enumerations -- Callbacks -- Iterators and ranges -- Custom operators -- Single and multiple inheritance -- STL data structures -- Iterators and ranges -- Smart pointers with reference counting like ``std::shared_ptr`` -- Internal references with correct reference counting -- C++ classes with virtual (and pure virtual) methods can be extended in Python - -Goodies -******* -In addition to the core functionality, pybind11 provides some extra goodies: - -- Python 2.7, 3.x, and PyPy (PyPy2.7 >= 5.7) are supported with an - implementation-agnostic interface. - -- It is possible to bind C++11 lambda functions with captured variables. The - lambda capture data is stored inside the resulting Python function object. - -- pybind11 uses C++11 move constructors and move assignment operators whenever - possible to efficiently transfer custom data types. - -- It's easy to expose the internal storage of custom data types through - Pythons' buffer protocols. This is handy e.g. for fast conversion between - C++ matrix classes like Eigen and NumPy without expensive copy operations. - -- pybind11 can automatically vectorize functions so that they are transparently - applied to all entries of one or more NumPy array arguments. - -- Python's slice-based access and assignment operations can be supported with - just a few lines of code. - -- Everything is contained in just a few header files; there is no need to link - against any additional libraries. - -- Binaries are generally smaller by a factor of at least 2 compared to - equivalent bindings generated by Boost.Python. A recent pybind11 conversion - of `PyRosetta`_, an enormous Boost.Python binding project, reported a binary - size reduction of **5.4x** and compile time reduction by **5.8x**. - -- Function signatures are precomputed at compile time (using ``constexpr``), - leading to smaller binaries. - -- With little extra effort, C++ types can be pickled and unpickled similar to - regular Python objects. - -.. _PyRosetta: http://graylab.jhu.edu/RosettaCon2016/PyRosetta-4.pdf - -Supported compilers -******************* - -1. Clang/LLVM (any non-ancient version with C++11 support) -2. GCC 4.8 or newer -3. Microsoft Visual Studio 2015 or newer -4. Intel C++ compiler v15 or newer diff --git a/docs/limitations.rst b/docs/limitations.rst index a1a4f1af..def5ad65 100644 --- a/docs/limitations.rst +++ b/docs/limitations.rst @@ -1,6 +1,9 @@ Limitations ########### +Design choices +^^^^^^^^^^^^^^ + pybind11 strives to be a general solution to binding generation, but it also has certain limitations: @@ -11,10 +14,59 @@ certain limitations: - The NumPy interface ``pybind11::array`` greatly simplifies accessing numerical data from C++ (and vice versa), but it's not a full-blown array - class like ``Eigen::Array`` or ``boost.multi_array``. + class like ``Eigen::Array`` or ``boost.multi_array``. ``Eigen`` objects are + directly supported, however, with ``pybind11/eigen.h``. + +Large but useful features could be implemented in pybind11 but would lead to a +significant increase in complexity. Pybind11 strives to be simple and compact. +Users who require large new features are encouraged to write an extension to +pybind11; see `pybind11_json `_ for an +example. + + +Known bugs +^^^^^^^^^^ + +These are issues that hopefully will one day be fixed, but currently are +unsolved. If you know how to help with one of these issues, contributions +are welcome! + +- Intel 20.2 is currently having an issue with the test suite. + `#2573 `_ + +- Debug mode Python does not support 1-5 tests in the test suite currently. + `#2422 `_ + +- PyPy3 7.3.1 and 7.3.2 have issues with several tests on 32-bit Windows. + +Known limitations +^^^^^^^^^^^^^^^^^ + +These are issues that are probably solvable, but have not been fixed yet. A +clean, well written patch would likely be accepted to solve them. + +- Type casters are not kept alive recursively. + `#2527 `_ + One consequence is that containers of ``char *`` are currently not supported. + `#2245 `_ + +- The ``cpptest`` does not run on Windows with Python 3.8 or newer, due to DLL + loader changes. User code that is correctly installed should not be affected. + `#2560 `_ + +Python 3.9.0 warning +^^^^^^^^^^^^^^^^^^^^ -These features could be implemented but would lead to a significant increase in -complexity. I've decided to draw the line here to keep this project simple and -compact. Users who absolutely require these features are encouraged to fork -pybind11. +Combining older versions of pybind11 (< 2.6.0) with Python on exactly 3.9.0 +will trigger undefined behavior that typically manifests as crashes during +interpreter shutdown (but could also destroy your data. **You have been +warned**). +This issue was `fixed in Python `_. +As a mitigation for this bug, pybind11 2.6.0 or newer includes a workaround +specifically when Python 3.9.0 is detected at runtime, leaking about 50 bytes +of memory when a callback function is garbage collected. For reference, the +pybind11 test suite has about 2,000 such callbacks, but only 49 are garbage +collected before the end-of-process. Wheels (even if built with Python 3.9.0) +will correctly avoid the leak when run in Python 3.9.1, and this does not +affect other 3.X versions. diff --git a/docs/pybind11-logo.png b/docs/pybind11-logo.png index 4cbad54f..2d633a4d 100644 Binary files a/docs/pybind11-logo.png and b/docs/pybind11-logo.png differ diff --git a/docs/reference.rst b/docs/reference.rst index e41141bd..e64a0351 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -46,12 +46,26 @@ With reference counting Convenience classes for specific Python types ============================================= -.. doxygenclass:: module +.. doxygenclass:: module_ :members: .. doxygengroup:: pytypes :members: +Convenience functions converting to Python types +================================================ + +.. doxygenfunction:: make_tuple(Args&&...) + +.. doxygenfunction:: make_iterator(Iterator, Sentinel, Extra &&...) +.. doxygenfunction:: make_iterator(Type &, Extra&&...) + +.. doxygenfunction:: make_key_iterator(Iterator, Sentinel, Extra &&...) +.. doxygenfunction:: make_key_iterator(Type &, Extra&&...) + +.. doxygenfunction:: make_value_iterator(Iterator, Sentinel, Extra &&...) +.. doxygenfunction:: make_value_iterator(Type &, Extra&&...) + .. _extras: Passing extra arguments to ``def`` or ``class_`` @@ -80,12 +94,27 @@ Redirecting C++ streams .. doxygenfunction:: add_ostream_redirect -Python build-in functions +Python built-in functions ========================= .. doxygengroup:: python_builtins :members: +Inheritance +=========== + +See :doc:`/classes` and :doc:`/advanced/classes` for more detail. + +.. doxygendefine:: PYBIND11_OVERRIDE + +.. doxygendefine:: PYBIND11_OVERRIDE_PURE + +.. doxygendefine:: PYBIND11_OVERRIDE_NAME + +.. doxygendefine:: PYBIND11_OVERRIDE_PURE_NAME + +.. doxygenfunction:: get_override + Exceptions ========== @@ -95,7 +124,6 @@ Exceptions .. doxygenclass:: builtin_exception :members: - Literals ======== diff --git a/docs/release.rst b/docs/release.rst index b31bbe97..e761cdf7 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1,25 +1,97 @@ +On version numbers +^^^^^^^^^^^^^^^^^^ + +The two version numbers (C++ and Python) must match when combined (checked when +you build the PyPI package), and must be a valid `PEP 440 +`_ version when combined. + +For example: + +.. code-block:: C++ + + #define PYBIND11_VERSION_MAJOR X + #define PYBIND11_VERSION_MINOR Y + #define PYBIND11_VERSION_PATCH Z.dev1 + +For beta, ``PYBIND11_VERSION_PATCH`` should be ``Z.b1``. RC's can be ``Z.rc1``. +Always include the dot (even though PEP 440 allows it to be dropped). For a +final release, this must be a simple integer. There is also a HEX version of +the version just below. + + To release a new version of pybind11: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you don't have nox, you should either use ``pipx run nox`` instead, or use +``pipx install nox`` or ``brew install nox`` (Unix). -- Update the version number and push to pypi - - Update ``pybind11/_version.py`` (set release version, remove 'dev'). - - Update ``PYBIND11_VERSION_MAJOR`` etc. in ``include/pybind11/detail/common.h``. - - Ensure that all the information in ``setup.py`` is up-to-date. - - Update version in ``docs/conf.py``. - - Tag release date in ``docs/changelog.rst``. - - ``git add`` and ``git commit``. - - if new minor version: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` +- Update the version number + - Update ``PYBIND11_VERSION_MAJOR`` etc. in + ``include/pybind11/detail/common.h``. PATCH should be a simple integer. + - Update the version HEX just below, as well. + - Update ``pybind11/_version.py`` (match above) + - Run ``nox -s tests_packaging`` to ensure this was done correctly. + - Ensure that all the information in ``setup.cfg`` is up-to-date, like + supported Python versions. + - Add release date in ``docs/changelog.rst``. + - Check to make sure + `needs-changelog `_ + issues are entered in the changelog (clear the label when done). + - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it + fails due to a known flake issue, either ignore or restart CI.) +- Add a release branch if this is a new minor version, or update the existing release branch if it is a patch version + - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` + - Update branch: ``git checkout vX.Y``, ``git merge ``, ``git push`` +- Update tags (optional; if you skip this, the GitHub release makes a + non-annotated tag for you) - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'``. - - ``git push`` - ``git push --tags``. - - ``python setup.py sdist upload``. - - ``python setup.py bdist_wheel upload``. -- Update conda-forge (https://github.com/conda-forge/pybind11-feedstock) via PR - - download release package from Github: ``wget https://github.com/pybind/pybind11/archive/vX.Y.Z.tar.gz`` - - compute checksum: ``shasum -a 256 vX.Y.Z.tar.gz`` - - change version number and checksum in ``recipe/meta.yml`` +- Update stable + - ``git checkout stable`` + - ``git merge master`` + - ``git push`` +- Make a GitHub release (this shows up in the UI, sends new release + notifications to users watching releases, and also uploads PyPI packages). + (Note: if you do not use an existing tag, this creates a new lightweight tag + for you, so you could skip the above step.) + - GUI method: Under `releases `_ + click "Draft a new release" on the far right, fill in the tag name + (if you didn't tag above, it will be made here), fill in a release name + like "Version X.Y.Z", and copy-and-paste the markdown-formatted (!) changelog + into the description (usually ``cat docs/changelog.rst | pandoc -f rst -t gfm``). + Check "pre-release" if this is a beta/RC. + - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"`` + If this is a pre-release, add ``-p``. + - Get back to work - - Update ``_version.py`` (add 'dev' and increment minor). - - Update version in ``docs/conf.py`` - - Update version macros in ``include/pybind11/common.h`` - - ``git add`` and ``git commit``. - ``git push`` + - Make sure you are on master, not somewhere else: ``git checkout master`` + - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to + ``0.dev1`` and increment MINOR). + - Update ``_version.py`` to match + - Run ``nox -s tests_packaging`` to ensure this was done correctly. + - Add a spot for in-development updates in ``docs/changelog.rst``. + - ``git add``, ``git commit``, ``git push`` + +If a version branch is updated, remember to set PATCH to ``1.dev1``. + +If you'd like to bump homebrew, run: + +.. code-block:: console + + brew bump-formula-pr --url https://github.com/pybind/pybind11/archive/vX.Y.Z.tar.gz + +Conda-forge should automatically make a PR in a few hours, and automatically +merge it if there are no issues. + + +Manual packaging +^^^^^^^^^^^^^^^^ + +If you need to manually upload releases, you can download the releases from the job artifacts and upload them with twine. You can also make the files locally (not recommended in general, as your local directory is more likely to be "dirty" and SDists love picking up random unrelated/hidden files); this is the procedure: + +.. code-block:: bash + + nox -s build + twine upload dist/* + +This makes SDists and wheels, and the final line uploads them. diff --git a/docs/requirements.txt b/docs/requirements.txt index 3818fe80..d2a9ae16 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,6 @@ -breathe == 4.5.0 +breathe==4.34.0 +furo==2022.6.21 +sphinx==5.0.2 +sphinx-copybutton==0.5.0 +sphinxcontrib-moderncmakedomain==3.21.4 +sphinxcontrib-svg2pdfconverter==1.2.0 diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 3f569739..6a9db2d0 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,154 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +.. _upgrade-guide-2.9: + +v2.9 +==== + +* Any usage of the recently added ``py::make_simple_namespace`` should be + converted to using ``py::module_::import("types").attr("SimpleNamespace")`` + instead. + +* The use of ``_`` in custom type casters can now be replaced with the more + readable ``const_name`` instead. The old ``_`` shortcut has been retained + unless it is being used as a macro (like for gettext). + + +.. _upgrade-guide-2.7: + +v2.7 +==== + +*Before* v2.7, ``py::str`` can hold ``PyUnicodeObject`` or ``PyBytesObject``, +and ``py::isinstance()`` is ``true`` for both ``py::str`` and +``py::bytes``. Starting with v2.7, ``py::str`` exclusively holds +``PyUnicodeObject`` (`#2409 `_), +and ``py::isinstance()`` is ``true`` only for ``py::str``. To help in +the transition of user code, the ``PYBIND11_STR_LEGACY_PERMISSIVE`` macro +is provided as an escape hatch to go back to the legacy behavior. This macro +will be removed in future releases. Two types of required fixes are expected +to be common: + +* Accidental use of ``py::str`` instead of ``py::bytes``, masked by the legacy + behavior. These are probably very easy to fix, by changing from + ``py::str`` to ``py::bytes``. + +* Reliance on py::isinstance(obj) being ``true`` for + ``py::bytes``. This is likely to be easy to fix in most cases by adding + ``|| py::isinstance(obj)``, but a fix may be more involved, e.g. if + ``py::isinstance`` appears in a template. Such situations will require + careful review and custom fixes. + + +.. _upgrade-guide-2.6: + +v2.6 +==== + +Usage of the ``PYBIND11_OVERLOAD*`` macros and ``get_overload`` function should +be replaced by ``PYBIND11_OVERRIDE*`` and ``get_override``. In the future, the +old macros may be deprecated and removed. + +``py::module`` has been renamed ``py::module_``, but a backward compatible +typedef has been included. This change was to avoid a language change in C++20 +that requires unqualified ``module`` not be placed at the start of a logical +line. Qualified usage is unaffected and the typedef will remain unless the +C++ language rules change again. + +The public constructors of ``py::module_`` have been deprecated. Use +``PYBIND11_MODULE`` or ``module_::create_extension_module`` instead. + +An error is now thrown when ``__init__`` is forgotten on subclasses. This was +incorrect before, but was not checked. Add a call to ``__init__`` if it is +missing. + +A ``py::type_error`` is now thrown when casting to a subclass (like +``py::bytes`` from ``py::object``) if the conversion is not valid. Make a valid +conversion instead. + +The undocumented ``h.get_type()`` method has been deprecated and replaced by +``py::type::of(h)``. + +Enums now have a ``__str__`` method pre-defined; if you want to override it, +the simplest fix is to add the new ``py::prepend()`` tag when defining +``"__str__"``. + +If ``__eq__`` defined but not ``__hash__``, ``__hash__`` is now set to +``None``, as in normal CPython. You should add ``__hash__`` if you intended the +class to be hashable, possibly using the new ``py::hash`` shortcut. + +The constructors for ``py::array`` now always take signed integers for size, +for consistency. This may lead to compiler warnings on some systems. Cast to +``py::ssize_t`` instead of ``std::size_t``. + +The ``tools/clang`` submodule and ``tools/mkdoc.py`` have been moved to a +standalone package, `pybind11-mkdoc`_. If you were using those tools, please +use them via a pip install from the new location. + +The ``pybind11`` package on PyPI no longer fills the wheel "headers" slot - if +you were using the headers from this slot, they are available by requesting the +``global`` extra, that is, ``pip install "pybind11[global]"``. (Most users will +be unaffected, as the ``pybind11/include`` location is reported by ``python -m +pybind11 --includes`` and ``pybind11.get_include()`` is still correct and has +not changed since 2.5). + +.. _pybind11-mkdoc: https://github.com/pybind/pybind11-mkdoc + +CMake support: +-------------- + +The minimum required version of CMake is now 3.4. Several details of the CMake +support have been deprecated; warnings will be shown if you need to change +something. The changes are: + +* ``PYBIND11_CPP_STANDARD=`` is deprecated, please use + ``CMAKE_CXX_STANDARD=`` instead, or any other valid CMake CXX or CUDA + standard selection method, like ``target_compile_features``. + +* If you do not request a standard, pybind11 targets will compile with the + compiler default, but not less than C++11, instead of forcing C++14 always. + If you depend on the old behavior, please use ``set(CMAKE_CXX_STANDARD 14 CACHE STRING "")`` + instead. + +* Direct ``pybind11::module`` usage should always be accompanied by at least + ``set(CMAKE_CXX_VISIBILITY_PRESET hidden)`` or similar - it used to try to + manually force this compiler flag (but not correctly on all compilers or with + CUDA). + +* ``pybind11_add_module``'s ``SYSTEM`` argument is deprecated and does nothing; + linking now behaves like other imported libraries consistently in both + config and submodule mode, and behaves like a ``SYSTEM`` library by + default. + +* If ``PYTHON_EXECUTABLE`` is not set, virtual environments (``venv``, + ``virtualenv``, and ``conda``) are prioritized over the standard search + (similar to the new FindPython mode). + +In addition, the following changes may be of interest: + +* ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` will be respected by + ``pybind11_add_module`` if set instead of linking to ``pybind11::lto`` or + ``pybind11::thin_lto``. + +* Using ``find_package(Python COMPONENTS Interpreter Development)`` before + pybind11 will cause pybind11 to use the new Python mechanisms instead of its + own custom search, based on a patched version of classic ``FindPythonInterp`` + / ``FindPythonLibs``. In the future, this may become the default. A recent + (3.15+ or 3.18.2+) version of CMake is recommended. + + + +v2.5 +==== + +The Python package now includes the headers as data in the package itself, as +well as in the "headers" wheel slot. ``pybind11 --includes`` and +``pybind11.get_include()`` report the new location, which is always correct +regardless of how pybind11 was installed, making the old ``user=`` argument +meaningless. If you are not using the function to get the location already, you +are encouraged to switch to the package location. + v2.2 ==== @@ -84,7 +232,7 @@ way to get and set object state. See :ref:`pickling` for details. ... .def(py::pickle( [](const Foo &self) { // __getstate__ - return py::make_tuple(f.value1(), f.value2(), ...); // unchanged + return py::make_tuple(self.value1(), self.value2(), ...); // unchanged }, [](py::tuple t) { // __setstate__, note: no `self` argument return new Foo(t[0].cast(), ...); @@ -148,7 +296,7 @@ Within pybind11's CMake build system, ``pybind11_add_module`` has always been setting the ``-fvisibility=hidden`` flag in release mode. From now on, it's being applied unconditionally, even in debug mode and it can no longer be opted out of with the ``NO_EXTRAS`` option. The ``pybind11::module`` target now also -adds this flag to it's interface. The ``pybind11::embed`` target is unchanged. +adds this flag to its interface. The ``pybind11::embed`` target is unchanged. The most significant change here is for the ``pybind11::module`` target. If you were previously relying on default visibility, i.e. if your Python module was @@ -376,7 +524,7 @@ include a declaration of the form: PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr) -Continuing to do so won’t cause an error or even a deprecation warning, +Continuing to do so won't cause an error or even a deprecation warning, but it's completely redundant. diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index dce875a6..b5e3b7b2 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -10,65 +10,113 @@ #pragma once +#include "detail/common.h" #include "cast.h" -NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) /// \addtogroup annotations /// @{ /// Annotation for methods -struct is_method { handle class_; is_method(const handle &c) : class_(c) { } }; +struct is_method { + handle class_; + explicit is_method(const handle &c) : class_(c) {} +}; /// Annotation for operators -struct is_operator { }; +struct is_operator {}; + +/// Annotation for classes that cannot be subclassed +struct is_final {}; /// Annotation for parent scope -struct scope { handle value; scope(const handle &s) : value(s) { } }; +struct scope { + handle value; + explicit scope(const handle &s) : value(s) {} +}; /// Annotation for documentation -struct doc { const char *value; doc(const char *value) : value(value) { } }; +struct doc { + const char *value; + explicit doc(const char *value) : value(value) {} +}; /// Annotation for function names -struct name { const char *value; name(const char *value) : value(value) { } }; +struct name { + const char *value; + explicit name(const char *value) : value(value) {} +}; /// Annotation indicating that a function is an overload associated with a given "sibling" -struct sibling { handle value; sibling(const handle &value) : value(value.ptr()) { } }; +struct sibling { + handle value; + explicit sibling(const handle &value) : value(value.ptr()) {} +}; /// Annotation indicating that a class derives from another given type -template struct base { - PYBIND11_DEPRECATED("base() was deprecated in favor of specifying 'T' as a template argument to class_") - base() { } +template +struct base { + + PYBIND11_DEPRECATED( + "base() was deprecated in favor of specifying 'T' as a template argument to class_") + base() = default; }; /// Keep patient alive while nurse lives -template struct keep_alive { }; +template +struct keep_alive {}; /// Annotation indicating that a class is involved in a multiple inheritance relationship -struct multiple_inheritance { }; +struct multiple_inheritance {}; /// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class -struct dynamic_attr { }; +struct dynamic_attr {}; /// Annotation which enables the buffer protocol for a type -struct buffer_protocol { }; +struct buffer_protocol {}; /// Annotation which requests that a special metaclass is created for a type struct metaclass { handle value; PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.") - metaclass() {} + metaclass() = default; /// Override pybind11's default metaclass - explicit metaclass(handle value) : value(value) { } + explicit metaclass(handle value) : value(value) {} +}; + +/// Specifies a custom callback with signature `void (PyHeapTypeObject*)` that +/// may be used to customize the Python type. +/// +/// The callback is invoked immediately before `PyType_Ready`. +/// +/// Note: This is an advanced interface, and uses of it may require changes to +/// work with later versions of pybind11. You may wish to consult the +/// implementation of `make_new_python_type` in `detail/classes.h` to understand +/// the context in which the callback will be run. +struct custom_type_setup { + using callback = std::function; + + explicit custom_type_setup(callback value) : value(std::move(value)) {} + + callback value; }; /// Annotation that marks a class as local to the module: -struct module_local { const bool value; constexpr module_local(bool v = true) : value(v) { } }; +struct module_local { + const bool value; + constexpr explicit module_local(bool v = true) : value(v) {} +}; /// Annotation to mark enums as an arithmetic type -struct arithmetic { }; +struct arithmetic {}; + +/// Mark a function for addition at the beginning of the existing overload chain instead of the end +struct prepend {}; /** \rst A call policy which places one or more guard variables (``Ts...``) around the function call. @@ -88,9 +136,13 @@ struct arithmetic { }; return foo(args...); // forwarded arguments }); \endrst */ -template struct call_guard; +template +struct call_guard; -template <> struct call_guard<> { using type = detail::void_type; }; +template <> +struct call_guard<> { + using type = detail::void_type; +}; template struct call_guard { @@ -110,13 +162,14 @@ struct call_guard { /// @} annotations -NAMESPACE_BEGIN(detail) +PYBIND11_NAMESPACE_BEGIN(detail) /* Forward declarations */ enum op_id : int; enum op_type : int; struct undefined_t; -template struct op_; -inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); +template +struct op_; +void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); /// Internal data structure which holds metadata about a keyword argument struct argument_record { @@ -127,14 +180,16 @@ struct argument_record { bool none : 1; ///< True if None is allowed when loading argument_record(const char *name, const char *descr, handle value, bool convert, bool none) - : name(name), descr(descr), value(value), convert(convert), none(none) { } + : name(name), descr(descr), value(value), convert(convert), none(none) {} }; -/// Internal data structure which holds metadata about a bound function (signature, overloads, etc.) +/// Internal data structure which holds metadata about a bound function (signature, overloads, +/// etc.) struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), has_args(false), has_kwargs(false), is_method(false) { } + is_operator(false), is_method(false), has_args(false), has_kwargs(false), + prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -149,13 +204,13 @@ struct function_record { std::vector args; /// Pointer to lambda function which converts arguments and performs the actual call - handle (*impl) (function_call &) = nullptr; + handle (*impl)(function_call &) = nullptr; /// Storage for the wrapped function pointer and captured data, if any - void *data[3] = { }; + void *data[3] = {}; /// Pointer to custom destructor for 'data' (if needed) - void (*free_data) (function_record *ptr) = nullptr; + void (*free_data)(function_record *ptr) = nullptr; /// Return value policy associated with this function return_value_policy policy = return_value_policy::automatic; @@ -172,18 +227,28 @@ struct function_record { /// True if this is an operator (__add__), etc. bool is_operator : 1; + /// True if this is a method + bool is_method : 1; + /// True if the function has a '*args' argument bool has_args : 1; /// True if the function has a '**kwargs' argument bool has_kwargs : 1; - /// True if this is a method - bool is_method : 1; + /// True if this function is to be inserted at the beginning of the overload resolution chain + bool prepend : 1; /// Number of arguments (including py::args and/or py::kwargs, if present) std::uint16_t nargs; + /// Number of leading positional arguments, which are terminated by a py::args or py::kwargs + /// argument or by a py::kw_only annotation. + std::uint16_t nargs_pos = 0; + + /// Number of leading arguments (counted in `nargs`) that are positional-only + std::uint16_t nargs_pos_only = 0; + /// Python method object PyMethodDef *def = nullptr; @@ -200,7 +265,8 @@ struct function_record { /// Special data structure which (temporarily) holds metadata about a bound class struct type_record { PYBIND11_NOINLINE type_record() - : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), module_local(false) { } + : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), + default_holder(true), module_local(false), is_final(false) {} /// Handle to the parent scope handle scope; @@ -214,11 +280,14 @@ struct type_record { /// How large is the underlying C++ type? size_t type_size = 0; + /// What is the alignment of the underlying C++ type? + size_t type_align = 0; + /// How large is the type's holder? size_t holder_size = 0; /// The global operator new can be overridden with a class-specific variant - void *(*operator_new)(size_t) = ::operator new; + void *(*operator_new)(size_t) = nullptr; /// Function pointer to class_<..>::init_instance void (*init_instance)(instance *, const void *) = nullptr; @@ -235,6 +304,9 @@ struct type_record { /// Custom metaclass (optional) handle metaclass; + /// Custom type setup. + custom_type_setup::callback custom_type_setup_callback; + /// Multiple inheritance marker bool multiple_inheritance : 1; @@ -250,42 +322,48 @@ struct type_record { /// Is the class definition local to the module shared object? bool module_local : 1; - PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *)) { - auto base_info = detail::get_type_info(base, false); + /// Is the class inheritable from python classes? + bool is_final : 1; + + PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) { + auto *base_info = detail::get_type_info(base, false); if (!base_info) { std::string tname(base.name()); detail::clean_type_id(tname); - pybind11_fail("generic_type: type \"" + std::string(name) + - "\" referenced unknown base type \"" + tname + "\""); + pybind11_fail("generic_type: type \"" + std::string(name) + + "\" referenced unknown base type \"" + tname + "\""); } if (default_holder != base_info->default_holder) { std::string tname(base.name()); detail::clean_type_id(tname); - pybind11_fail("generic_type: type \"" + std::string(name) + "\" " + - (default_holder ? "does not have" : "has") + - " a non-default holder type while its base \"" + tname + "\" " + - (base_info->default_holder ? "does not" : "does")); + pybind11_fail("generic_type: type \"" + std::string(name) + "\" " + + (default_holder ? "does not have" : "has") + + " a non-default holder type while its base \"" + tname + "\" " + + (base_info->default_holder ? "does not" : "does")); } bases.append((PyObject *) base_info->type); - if (base_info->type->tp_dictoffset != 0) - dynamic_attr = true; +#if PY_VERSION_HEX < 0x030B0000 + dynamic_attr |= base_info->type->tp_dictoffset != 0; +#else + dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; +#endif - if (caster) + if (caster) { base_info->implicit_casts.emplace_back(type, caster); + } } }; -inline function_call::function_call(function_record &f, handle p) : - func(f), parent(p) { +inline function_call::function_call(const function_record &f, handle p) : func(f), parent(p) { args.reserve(f.nargs); args_convert.reserve(f.nargs); } /// Tag for a new-style `__init__` defined in `detail/init.h` -struct is_new_style_constructor { }; +struct is_new_style_constructor {}; /** * Partial template specializations to process custom attributes provided to @@ -293,105 +371,177 @@ struct is_new_style_constructor { }; * fields in the type_record and function_record data structures or executed at * runtime to deal with custom call policies (e.g. keep_alive). */ -template struct process_attribute; +template +struct process_attribute; -template struct process_attribute_default { +template +struct process_attribute_default { /// Default implementation: do nothing - static void init(const T &, function_record *) { } - static void init(const T &, type_record *) { } - static void precall(function_call &) { } - static void postcall(function_call &, handle) { } + static void init(const T &, function_record *) {} + static void init(const T &, type_record *) {} + static void precall(function_call &) {} + static void postcall(function_call &, handle) {} }; /// Process an attribute specifying the function's name -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const name &n, function_record *r) { r->name = const_cast(n.value); } }; /// Process an attribute specifying the function's docstring -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const doc &n, function_record *r) { r->doc = const_cast(n.value); } }; /// Process an attribute specifying the function's docstring (provided as a C-style string) -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const char *d, function_record *r) { r->doc = const_cast(d); } - static void init(const char *d, type_record *r) { r->doc = const_cast(d); } + static void init(const char *d, type_record *r) { r->doc = d; } }; -template <> struct process_attribute : process_attribute { }; +template <> +struct process_attribute : process_attribute {}; /// Process an attribute indicating the function's return value policy -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const return_value_policy &p, function_record *r) { r->policy = p; } }; -/// Process an attribute which indicates that this is an overloaded function associated with a given sibling -template <> struct process_attribute : process_attribute_default { +/// Process an attribute which indicates that this is an overloaded function associated with a +/// given sibling +template <> +struct process_attribute : process_attribute_default { static void init(const sibling &s, function_record *r) { r->sibling = s.value; } }; /// Process an attribute which indicates that this function is a method -template <> struct process_attribute : process_attribute_default { - static void init(const is_method &s, function_record *r) { r->is_method = true; r->scope = s.class_; } +template <> +struct process_attribute : process_attribute_default { + static void init(const is_method &s, function_record *r) { + r->is_method = true; + r->scope = s.class_; + } }; /// Process an attribute which indicates the parent scope of a method -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const scope &s, function_record *r) { r->scope = s.value; } }; /// Process an attribute which indicates that this function is an operator -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const is_operator &, function_record *r) { r->is_operator = true; } }; -template <> struct process_attribute : process_attribute_default { - static void init(const is_new_style_constructor &, function_record *r) { r->is_new_style_constructor = true; } +template <> +struct process_attribute + : process_attribute_default { + static void init(const is_new_style_constructor &, function_record *r) { + r->is_new_style_constructor = true; + } }; +inline void check_kw_only_arg(const arg &a, function_record *r) { + if (r->args.size() > r->nargs_pos && (!a.name || a.name[0] == '\0')) { + pybind11_fail("arg(): cannot specify an unnamed argument after a kw_only() annotation or " + "args() argument"); + } +} + +inline void append_self_arg_if_needed(function_record *r) { + if (r->is_method && r->args.empty()) { + r->args.emplace_back("self", nullptr, handle(), /*convert=*/true, /*none=*/false); + } +} + /// Process a keyword argument attribute (*without* a default value) -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const arg &a, function_record *r) { - if (r->is_method && r->args.empty()) - r->args.emplace_back("self", nullptr, handle(), true /*convert*/, false /*none not allowed*/); + append_self_arg_if_needed(r); r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert, a.flag_none); + + check_kw_only_arg(a, r); } }; /// Process a keyword argument attribute (*with* a default value) -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const arg_v &a, function_record *r) { - if (r->is_method && r->args.empty()) - r->args.emplace_back("self", nullptr /*descr*/, handle() /*parent*/, true /*convert*/, false /*none not allowed*/); + if (r->is_method && r->args.empty()) { + r->args.emplace_back( + "self", /*descr=*/nullptr, /*parent=*/handle(), /*convert=*/true, /*none=*/false); + } if (!a.value) { -#if !defined(NDEBUG) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) std::string descr("'"); - if (a.name) descr += std::string(a.name) + ": "; + if (a.name) { + descr += std::string(a.name) + ": "; + } descr += a.type + "'"; if (r->is_method) { - if (r->name) - descr += " in method '" + (std::string) str(r->scope) + "." + (std::string) r->name + "'"; - else + if (r->name) { + descr += " in method '" + (std::string) str(r->scope) + "." + + (std::string) r->name + "'"; + } else { descr += " in method of '" + (std::string) str(r->scope) + "'"; + } } else if (r->name) { descr += " in function '" + (std::string) r->name + "'"; } - pybind11_fail("arg(): could not convert default argument " - + descr + " into a Python object (type not registered yet?)"); + pybind11_fail("arg(): could not convert default argument " + descr + + " into a Python object (type not registered yet?)"); #else pybind11_fail("arg(): could not convert default argument " "into a Python object (type not registered yet?). " - "Compile in debug mode for more information."); + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "more information."); #endif } r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert, a.flag_none); + + check_kw_only_arg(a, r); + } +}; + +/// Process a keyword-only-arguments-follow pseudo argument +template <> +struct process_attribute : process_attribute_default { + static void init(const kw_only &, function_record *r) { + append_self_arg_if_needed(r); + if (r->has_args && r->nargs_pos != static_cast(r->args.size())) { + pybind11_fail("Mismatched args() and kw_only(): they must occur at the same relative " + "argument location (or omit kw_only() entirely)"); + } + r->nargs_pos = static_cast(r->args.size()); + } +}; + +/// Process a positional-only-argument maker +template <> +struct process_attribute : process_attribute_default { + static void init(const pos_only &, function_record *r) { + append_self_arg_if_needed(r); + r->nargs_pos_only = static_cast(r->args.size()); + if (r->nargs_pos_only > r->nargs_pos) { + pybind11_fail("pos_only(): cannot follow a py::args() argument"); + } + // It also can't follow a kw_only, but a static_assert in pybind11.h checks that } }; -/// Process a parent class attribute. Single inheritance only (class_ itself already guarantees that) +/// Process a parent class attribute. Single inheritance only (class_ itself already guarantees +/// that) template -struct process_attribute::value>> : process_attribute_default { +struct process_attribute::value>> + : process_attribute_default { static void init(const handle &h, type_record *r) { r->bases.append(h); } }; @@ -404,7 +554,9 @@ struct process_attribute> : process_attribute_default> { /// Process a multiple inheritance attribute template <> struct process_attribute : process_attribute_default { - static void init(const multiple_inheritance &, type_record *r) { r->multiple_inheritance = true; } + static void init(const multiple_inheritance &, type_record *r) { + r->multiple_inheritance = true; + } }; template <> @@ -412,6 +564,18 @@ struct process_attribute : process_attribute_default static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; } }; +template <> +struct process_attribute { + static void init(const custom_type_setup &value, type_record *r) { + r->custom_type_setup_callback = value.value; + } +}; + +template <> +struct process_attribute : process_attribute_default { + static void init(const is_final &, type_record *r) { r->is_final = true; } +}; + template <> struct process_attribute : process_attribute_default { static void init(const buffer_protocol &, type_record *r) { r->buffer_protocol = true; } @@ -427,46 +591,70 @@ struct process_attribute : process_attribute_default static void init(const module_local &l, type_record *r) { r->module_local = l.value; } }; +/// Process a 'prepend' attribute, putting this at the beginning of the overload chain +template <> +struct process_attribute : process_attribute_default { + static void init(const prepend &, function_record *r) { r->prepend = true; } +}; + /// Process an 'arithmetic' attribute for enums (does nothing here) template <> struct process_attribute : process_attribute_default {}; template -struct process_attribute> : process_attribute_default> { }; +struct process_attribute> : process_attribute_default> {}; /** * Process a keep_alive call policy -- invokes keep_alive_impl during the * pre-call handler if both Nurse, Patient != 0 and use the post-call handler * otherwise */ -template struct process_attribute> : public process_attribute_default> { +template +struct process_attribute> + : public process_attribute_default> { template = 0> - static void precall(function_call &call) { keep_alive_impl(Nurse, Patient, call, handle()); } + static void precall(function_call &call) { + keep_alive_impl(Nurse, Patient, call, handle()); + } template = 0> - static void postcall(function_call &, handle) { } + static void postcall(function_call &, handle) {} template = 0> - static void precall(function_call &) { } + static void precall(function_call &) {} template = 0> - static void postcall(function_call &call, handle ret) { keep_alive_impl(Nurse, Patient, call, ret); } + static void postcall(function_call &call, handle ret) { + keep_alive_impl(Nurse, Patient, call, ret); + } }; /// Recursively iterate over variadic template arguments -template struct process_attributes { - static void init(const Args&... args, function_record *r) { - int unused[] = { 0, (process_attribute::type>::init(args, r), 0) ... }; - ignore_unused(unused); +template +struct process_attributes { + static void init(const Args &...args, function_record *r) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(r); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(r); + using expander = int[]; + (void) expander{ + 0, ((void) process_attribute::type>::init(args, r), 0)...}; } - static void init(const Args&... args, type_record *r) { - int unused[] = { 0, (process_attribute::type>::init(args, r), 0) ... }; - ignore_unused(unused); + static void init(const Args &...args, type_record *r) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(r); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(r); + using expander = int[]; + (void) expander{0, + (process_attribute::type>::init(args, r), 0)...}; } static void precall(function_call &call) { - int unused[] = { 0, (process_attribute::type>::precall(call), 0) ... }; - ignore_unused(unused); + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(call); + using expander = int[]; + (void) expander{0, + (process_attribute::type>::precall(call), 0)...}; } static void postcall(function_call &call, handle fn_ret) { - int unused[] = { 0, (process_attribute::type>::postcall(call, fn_ret), 0) ... }; - ignore_unused(unused); + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(call, fn_ret); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(fn_ret); + using expander = int[]; + (void) expander{ + 0, (process_attribute::type>::postcall(call, fn_ret), 0)...}; } }; @@ -480,10 +668,11 @@ using extract_guard_t = typename exactly_one_t, Extr /// Check the number of named arguments at compile time template ::value...), - size_t self = constexpr_sum(std::is_same::value...)> + size_t self = constexpr_sum(std::is_same::value...)> constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { - return named == 0 || (self + named + has_args + has_kwargs) == nargs; + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(nargs, has_args, has_kwargs); + return named == 0 || (self + named + size_t(has_args) + size_t(has_kwargs)) == nargs; } -NAMESPACE_END(detail) -NAMESPACE_END(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index 9f072fa7..06120d55 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -11,56 +11,122 @@ #include "detail/common.h" -NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) + +// Default, C-style strides +inline std::vector c_strides(const std::vector &shape, ssize_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim, itemsize); + if (ndim > 0) { + for (size_t i = ndim - 1; i > 0; --i) { + strides[i - 1] = strides[i] * shape[i]; + } + } + return strides; +} + +// F-style strides; default when constructing an array_t with `ExtraFlags & f_style` +inline std::vector f_strides(const std::vector &shape, ssize_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim, itemsize); + for (size_t i = 1; i < ndim; ++i) { + strides[i] = strides[i - 1] * shape[i - 1]; + } + return strides; +} + +PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object struct buffer_info { void *ptr = nullptr; // Pointer to the underlying storage ssize_t itemsize = 0; // Size of individual items in bytes ssize_t size = 0; // Total number of entries - std::string format; // For homogeneous buffers, this should be set to format_descriptor::format() + std::string format; // For homogeneous buffers, this should be set to + // format_descriptor::format() ssize_t ndim = 0; // Number of dimensions std::vector shape; // Shape of the tensor (1 entry per dimension) - std::vector strides; // Number of entries between adjacent entries (for each per dimension) - - buffer_info() { } - - buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, - detail::any_container shape_in, detail::any_container strides_in) - : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), - shape(std::move(shape_in)), strides(std::move(strides_in)) { - if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) + std::vector strides; // Number of bytes between adjacent entries + // (for each per dimension) + bool readonly = false; // flag to indicate if the underlying storage may be written to + + buffer_info() = default; + + buffer_info(void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t ndim, + detail::any_container shape_in, + detail::any_container strides_in, + bool readonly = false) + : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), + shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { + if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) { pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length"); - for (size_t i = 0; i < (size_t) ndim; ++i) + } + for (size_t i = 0; i < (size_t) ndim; ++i) { size *= shape[i]; + } } template - buffer_info(T *ptr, detail::any_container shape_in, detail::any_container strides_in) - : buffer_info(private_ctr_tag(), ptr, sizeof(T), format_descriptor::format(), static_cast(shape_in->size()), std::move(shape_in), std::move(strides_in)) { } + buffer_info(T *ptr, + detail::any_container shape_in, + detail::any_container strides_in, + bool readonly = false) + : buffer_info(private_ctr_tag(), + ptr, + sizeof(T), + format_descriptor::format(), + static_cast(shape_in->size()), + std::move(shape_in), + std::move(strides_in), + readonly) {} + + buffer_info(void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t size, + bool readonly = false) + : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}, readonly) {} - buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t size) - : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}) { } + template + buffer_info(T *ptr, ssize_t size, bool readonly = false) + : buffer_info(ptr, sizeof(T), format_descriptor::format(), size, readonly) {} template - buffer_info(T *ptr, ssize_t size) - : buffer_info(ptr, sizeof(T), format_descriptor::format(), size) { } + buffer_info(const T *ptr, ssize_t size, bool readonly = true) + : buffer_info( + const_cast(ptr), sizeof(T), format_descriptor::format(), size, readonly) {} explicit buffer_info(Py_buffer *view, bool ownview = true) - : buffer_info(view->buf, view->itemsize, view->format, view->ndim, - {view->shape, view->shape + view->ndim}, {view->strides, view->strides + view->ndim}) { - this->view = view; + : buffer_info( + view->buf, + view->itemsize, + view->format, + view->ndim, + {view->shape, view->shape + view->ndim}, + /* Though buffer::request() requests PyBUF_STRIDES, ctypes objects + * ignore this flag and return a view with NULL strides. + * When strides are NULL, build them manually. */ + view->strides + ? std::vector(view->strides, view->strides + view->ndim) + : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), + (view->readonly != 0)) { + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + this->m_view = view; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) this->ownview = ownview; } buffer_info(const buffer_info &) = delete; - buffer_info& operator=(const buffer_info &) = delete; + buffer_info &operator=(const buffer_info &) = delete; - buffer_info(buffer_info &&other) { - (*this) = std::move(other); - } + buffer_info(buffer_info &&other) noexcept { (*this) = std::move(other); } - buffer_info& operator=(buffer_info &&rhs) { + buffer_info &operator=(buffer_info &&rhs) noexcept { ptr = rhs.ptr; itemsize = rhs.itemsize; size = rhs.size; @@ -68,41 +134,60 @@ struct buffer_info { ndim = rhs.ndim; shape = std::move(rhs.shape); strides = std::move(rhs.strides); - std::swap(view, rhs.view); + std::swap(m_view, rhs.m_view); std::swap(ownview, rhs.ownview); + readonly = rhs.readonly; return *this; } ~buffer_info() { - if (view && ownview) { PyBuffer_Release(view); delete view; } + if (m_view && ownview) { + PyBuffer_Release(m_view); + delete m_view; + } } -private: - struct private_ctr_tag { }; + Py_buffer *view() const { return m_view; } + Py_buffer *&view() { return m_view; } - buffer_info(private_ctr_tag, void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, - detail::any_container &&shape_in, detail::any_container &&strides_in) - : buffer_info(ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in)) { } - - Py_buffer *view = nullptr; +private: + struct private_ctr_tag {}; + + buffer_info(private_ctr_tag, + void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t ndim, + detail::any_container &&shape_in, + detail::any_container &&strides_in, + bool readonly) + : buffer_info( + ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) {} + + Py_buffer *m_view = nullptr; bool ownview = false; }; -NAMESPACE_BEGIN(detail) +PYBIND11_NAMESPACE_BEGIN(detail) -template struct compare_buffer_info { - static bool compare(const buffer_info& b) { +template +struct compare_buffer_info { + static bool compare(const buffer_info &b) { return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); } }; -template struct compare_buffer_info::value>> { - static bool compare(const buffer_info& b) { - return (size_t) b.itemsize == sizeof(T) && (b.format == format_descriptor::value || - ((sizeof(T) == sizeof(long)) && b.format == (std::is_unsigned::value ? "L" : "l")) || - ((sizeof(T) == sizeof(size_t)) && b.format == (std::is_unsigned::value ? "N" : "n"))); +template +struct compare_buffer_info::value>> { + static bool compare(const buffer_info &b) { + return (size_t) b.itemsize == sizeof(T) + && (b.format == format_descriptor::value + || ((sizeof(T) == sizeof(long)) + && b.format == (std::is_unsigned::value ? "L" : "l")) + || ((sizeof(T) == sizeof(size_t)) + && b.format == (std::is_unsigned::value ? "N" : "n"))); } }; -NAMESPACE_END(detail) -NAMESPACE_END(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index f96d6aec..9d1ce15d 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -10,964 +10,187 @@ #pragma once -#include "pytypes.h" -#include "detail/typeid.h" +#include "detail/common.h" #include "detail/descr.h" -#include "detail/internals.h" +#include "detail/type_caster_base.h" +#include "detail/typeid.h" +#include "pytypes.h" + #include -#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include -#if defined(PYBIND11_CPP17) -# if defined(__has_include) -# if __has_include() -# define PYBIND11_HAS_STRING_VIEW -# endif -# elif defined(_MSC_VER) -# define PYBIND11_HAS_STRING_VIEW -# endif -#endif -#ifdef PYBIND11_HAS_STRING_VIEW -#include -#endif - -NAMESPACE_BEGIN(PYBIND11_NAMESPACE) -NAMESPACE_BEGIN(detail) - -/// A life support system for temporary objects created by `type_caster::load()`. -/// Adding a patient will keep it alive up until the enclosing function returns. -class loader_life_support { -public: - /// A new patient frame is created when a function is entered - loader_life_support() { - get_internals().loader_patient_stack.push_back(nullptr); - } - - /// ... and destroyed after it returns - ~loader_life_support() { - auto &stack = get_internals().loader_patient_stack; - if (stack.empty()) - pybind11_fail("loader_life_support: internal error"); - - auto ptr = stack.back(); - stack.pop_back(); - Py_CLEAR(ptr); - - // A heuristic to reduce the stack's capacity (e.g. after long recursive calls) - if (stack.capacity() > 16 && stack.size() != 0 && stack.capacity() / stack.size() > 2) - stack.shrink_to_fit(); - } - - /// This can only be used inside a pybind11-bound function, either by `argument_loader` - /// at argument preparation time or by `py::cast()` at execution time. - PYBIND11_NOINLINE static void add_patient(handle h) { - auto &stack = get_internals().loader_patient_stack; - if (stack.empty()) - throw cast_error("When called outside a bound function, py::cast() cannot " - "do Python -> C++ conversions which require the creation " - "of temporary values"); - - auto &list_ptr = stack.back(); - if (list_ptr == nullptr) { - list_ptr = PyList_New(1); - if (!list_ptr) - pybind11_fail("loader_life_support: error allocating list"); - PyList_SET_ITEM(list_ptr, 0, h.inc_ref().ptr()); - } else { - auto result = PyList_Append(list_ptr, h.ptr()); - if (result == -1) - pybind11_fail("loader_life_support: error adding patient"); - } - } -}; - -// Gets the cache entry for the given type, creating it if necessary. The return value is the pair -// returned by emplace, i.e. an iterator for the entry and a bool set to `true` if the entry was -// just created. -inline std::pair all_type_info_get_cache(PyTypeObject *type); - -// Populates a just-created cache entry. -PYBIND11_NOINLINE inline void all_type_info_populate(PyTypeObject *t, std::vector &bases) { - std::vector check; - for (handle parent : reinterpret_borrow(t->tp_bases)) - check.push_back((PyTypeObject *) parent.ptr()); - - auto const &type_dict = get_internals().registered_types_py; - for (size_t i = 0; i < check.size(); i++) { - auto type = check[i]; - // Ignore Python2 old-style class super type: - if (!PyType_Check((PyObject *) type)) continue; - - // Check `type` in the current set of registered python types: - auto it = type_dict.find(type); - if (it != type_dict.end()) { - // We found a cache entry for it, so it's either pybind-registered or has pre-computed - // pybind bases, but we have to make sure we haven't already seen the type(s) before: we - // want to follow Python/virtual C++ rules that there should only be one instance of a - // common base. - for (auto *tinfo : it->second) { - // NB: Could use a second set here, rather than doing a linear search, but since - // having a large number of immediate pybind11-registered types seems fairly - // unlikely, that probably isn't worthwhile. - bool found = false; - for (auto *known : bases) { - if (known == tinfo) { found = true; break; } - } - if (!found) bases.push_back(tinfo); - } - } - else if (type->tp_bases) { - // It's some python type, so keep follow its bases classes to look for one or more - // registered types - if (i + 1 == check.size()) { - // When we're at the end, we can pop off the current element to avoid growing - // `check` when adding just one base (which is typical--i.e. when there is no - // multiple inheritance) - check.pop_back(); - i--; - } - for (handle parent : reinterpret_borrow(type->tp_bases)) - check.push_back((PyTypeObject *) parent.ptr()); - } - } -} - -/** - * Extracts vector of type_info pointers of pybind-registered roots of the given Python type. Will - * be just 1 pybind type for the Python type of a pybind-registered class, or for any Python-side - * derived class that uses single inheritance. Will contain as many types as required for a Python - * class that uses multiple inheritance to inherit (directly or indirectly) from multiple - * pybind-registered classes. Will be empty if neither the type nor any base classes are - * pybind-registered. - * - * The value is cached for the lifetime of the Python type. - */ -inline const std::vector &all_type_info(PyTypeObject *type) { - auto ins = all_type_info_get_cache(type); - if (ins.second) - // New cache entry: populate it - all_type_info_populate(type, ins.first->second); - - return ins.first->second; -} - -/** - * Gets a single pybind11 type info for a python type. Returns nullptr if neither the type nor any - * ancestors are pybind11-registered. Throws an exception if there are multiple bases--use - * `all_type_info` instead if you want to support multiple bases. - */ -PYBIND11_NOINLINE inline detail::type_info* get_type_info(PyTypeObject *type) { - auto &bases = all_type_info(type); - if (bases.size() == 0) - return nullptr; - if (bases.size() > 1) - pybind11_fail("pybind11::detail::get_type_info: type has multiple pybind11-registered bases"); - return bases.front(); -} - -inline detail::type_info *get_local_type_info(const std::type_index &tp) { - auto &locals = registered_local_types_cpp(); - auto it = locals.find(tp); - if (it != locals.end()) - return it->second; - return nullptr; -} - -inline detail::type_info *get_global_type_info(const std::type_index &tp) { - auto &types = get_internals().registered_types_cpp; - auto it = types.find(tp); - if (it != types.end()) - return it->second; - return nullptr; -} - -/// Return the type info for a given C++ type; on lookup failure can either throw or return nullptr. -PYBIND11_NOINLINE inline detail::type_info *get_type_info(const std::type_index &tp, - bool throw_if_missing = false) { - if (auto ltype = get_local_type_info(tp)) - return ltype; - if (auto gtype = get_global_type_info(tp)) - return gtype; - - if (throw_if_missing) { - std::string tname = tp.name(); - detail::clean_type_id(tname); - pybind11_fail("pybind11::detail::get_type_info: unable to find type info for \"" + tname + "\""); - } - return nullptr; -} - -PYBIND11_NOINLINE inline handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { - detail::type_info *type_info = get_type_info(tp, throw_if_missing); - return handle(type_info ? ((PyObject *) type_info->type) : nullptr); -} - -struct value_and_holder { - instance *inst; - size_t index; - const detail::type_info *type; - void **vh; - - // Main constructor for a found value/holder: - value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) : - inst{i}, index{index}, type{type}, - vh{inst->simple_layout ? inst->simple_value_holder : &inst->nonsimple.values_and_holders[vpos]} - {} - - // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) - value_and_holder() : inst{nullptr} {} - - // Used for past-the-end iterator - value_and_holder(size_t index) : index{index} {} - - template V *&value_ptr() const { - return reinterpret_cast(vh[0]); - } - // True if this `value_and_holder` has a non-null value pointer - explicit operator bool() const { return value_ptr(); } - - template H &holder() const { - return reinterpret_cast(vh[1]); - } - bool holder_constructed() const { - return inst->simple_layout - ? inst->simple_holder_constructed - : inst->nonsimple.status[index] & instance::status_holder_constructed; - } - void set_holder_constructed(bool v = true) { - if (inst->simple_layout) - inst->simple_holder_constructed = v; - else if (v) - inst->nonsimple.status[index] |= instance::status_holder_constructed; - else - inst->nonsimple.status[index] &= (uint8_t) ~instance::status_holder_constructed; - } - bool instance_registered() const { - return inst->simple_layout - ? inst->simple_instance_registered - : inst->nonsimple.status[index] & instance::status_instance_registered; - } - void set_instance_registered(bool v = true) { - if (inst->simple_layout) - inst->simple_instance_registered = v; - else if (v) - inst->nonsimple.status[index] |= instance::status_instance_registered; - else - inst->nonsimple.status[index] &= (uint8_t) ~instance::status_instance_registered; - } -}; - -// Container for accessing and iterating over an instance's values/holders -struct values_and_holders { -private: - instance *inst; - using type_vec = std::vector; - const type_vec &tinfo; - -public: - values_and_holders(instance *inst) : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} - - struct iterator { - private: - instance *inst; - const type_vec *types; - value_and_holder curr; - friend struct values_and_holders; - iterator(instance *inst, const type_vec *tinfo) - : inst{inst}, types{tinfo}, - curr(inst /* instance */, - types->empty() ? nullptr : (*types)[0] /* type info */, - 0, /* vpos: (non-simple types only): the first vptr comes first */ - 0 /* index */) - {} - // Past-the-end iterator: - iterator(size_t end) : curr(end) {} - public: - bool operator==(const iterator &other) { return curr.index == other.curr.index; } - bool operator!=(const iterator &other) { return curr.index != other.curr.index; } - iterator &operator++() { - if (!inst->simple_layout) - curr.vh += 1 + (*types)[curr.index]->holder_size_in_ptrs; - ++curr.index; - curr.type = curr.index < types->size() ? (*types)[curr.index] : nullptr; - return *this; - } - value_and_holder &operator*() { return curr; } - value_and_holder *operator->() { return &curr; } - }; - - iterator begin() { return iterator(inst, &tinfo); } - iterator end() { return iterator(tinfo.size()); } - - iterator find(const type_info *find_type) { - auto it = begin(), endit = end(); - while (it != endit && it->type != find_type) ++it; - return it; - } - - size_t size() { return tinfo.size(); } -}; - -/** - * Extracts C++ value and holder pointer references from an instance (which may contain multiple - * values/holders for python-side multiple inheritance) that match the given type. Throws an error - * if the given type (or ValueType, if omitted) is not a pybind11 base of the given instance. If - * `find_type` is omitted (or explicitly specified as nullptr) the first value/holder are returned, - * regardless of type (and the resulting .type will be nullptr). - * - * The returned object should be short-lived: in particular, it must not outlive the called-upon - * instance. - */ -PYBIND11_NOINLINE inline value_and_holder instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, bool throw_if_missing /*= true in common.h*/) { - // Optimize common case: - if (!find_type || Py_TYPE(this) == find_type->type) - return value_and_holder(this, find_type, 0, 0); - - detail::values_and_holders vhs(this); - auto it = vhs.find(find_type); - if (it != vhs.end()) - return *it; - - if (!throw_if_missing) - return value_and_holder(); - -#if defined(NDEBUG) - pybind11_fail("pybind11::detail::instance::get_value_and_holder: " - "type is not a pybind11 base of the given instance " - "(compile in debug mode for type details)"); -#else - pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" + - std::string(find_type->type->tp_name) + "' is not a pybind11 base of the given `" + - std::string(Py_TYPE(this)->tp_name) + "' instance"); -#endif -} - -PYBIND11_NOINLINE inline void instance::allocate_layout() { - auto &tinfo = all_type_info(Py_TYPE(this)); - - const size_t n_types = tinfo.size(); - - if (n_types == 0) - pybind11_fail("instance allocation failed: new instance has no pybind11-registered base types"); - - simple_layout = - n_types == 1 && tinfo.front()->holder_size_in_ptrs <= instance_simple_holder_in_ptrs(); - - // Simple path: no python-side multiple inheritance, and a small-enough holder - if (simple_layout) { - simple_value_holder[0] = nullptr; - simple_holder_constructed = false; - simple_instance_registered = false; - } - else { // multiple base types or a too-large holder - // Allocate space to hold: [v1*][h1][v2*][h2]...[bb...] where [vN*] is a value pointer, - // [hN] is the (uninitialized) holder instance for value N, and [bb...] is a set of bool - // values that tracks whether each associated holder has been initialized. Each [block] is - // padded, if necessary, to an integer multiple of sizeof(void *). - size_t space = 0; - for (auto t : tinfo) { - space += 1; // value pointer - space += t->holder_size_in_ptrs; // holder instance - } - size_t flags_at = space; - space += size_in_ptrs(n_types); // status bytes (holder_constructed and instance_registered) - - // Allocate space for flags, values, and holders, and initialize it to 0 (flags and values, - // in particular, need to be 0). Use Python's memory allocation functions: in Python 3.6 - // they default to using pymalloc, which is designed to be efficient for small allocations - // like the one we're doing here; in earlier versions (and for larger allocations) they are - // just wrappers around malloc. -#if PY_VERSION_HEX >= 0x03050000 - nonsimple.values_and_holders = (void **) PyMem_Calloc(space, sizeof(void *)); - if (!nonsimple.values_and_holders) throw std::bad_alloc(); -#else - nonsimple.values_and_holders = (void **) PyMem_New(void *, space); - if (!nonsimple.values_and_holders) throw std::bad_alloc(); - std::memset(nonsimple.values_and_holders, 0, space * sizeof(void *)); -#endif - nonsimple.status = reinterpret_cast(&nonsimple.values_and_holders[flags_at]); - } - owned = true; -} - -PYBIND11_NOINLINE inline void instance::deallocate_layout() { - if (!simple_layout) - PyMem_Free(nonsimple.values_and_holders); -} - -PYBIND11_NOINLINE inline bool isinstance_generic(handle obj, const std::type_info &tp) { - handle type = detail::get_type_handle(tp, false); - if (!type) - return false; - return isinstance(obj, type); -} - -PYBIND11_NOINLINE inline std::string error_string() { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "Unknown internal error occurred"); - return "Unknown internal error occurred"; - } - - error_scope scope; // Preserve error state - - std::string errorString; - if (scope.type) { - errorString += handle(scope.type).attr("__name__").cast(); - errorString += ": "; - } - if (scope.value) - errorString += (std::string) str(scope.value); - - PyErr_NormalizeException(&scope.type, &scope.value, &scope.trace); - -#if PY_MAJOR_VERSION >= 3 - if (scope.trace != nullptr) - PyException_SetTraceback(scope.value, scope.trace); -#endif - -#if !defined(PYPY_VERSION) - if (scope.trace) { - PyTracebackObject *trace = (PyTracebackObject *) scope.trace; - - /* Get the deepest trace possible */ - while (trace->tb_next) - trace = trace->tb_next; - - PyFrameObject *frame = trace->tb_frame; - errorString += "\n\nAt:\n"; - while (frame) { - int lineno = PyFrame_GetLineNumber(frame); - errorString += - " " + handle(frame->f_code->co_filename).cast() + - "(" + std::to_string(lineno) + "): " + - handle(frame->f_code->co_name).cast() + "\n"; - frame = frame->f_back; - } - } -#endif - - return errorString; -} - -PYBIND11_NOINLINE inline handle get_object_handle(const void *ptr, const detail::type_info *type ) { - auto &instances = get_internals().registered_instances; - auto range = instances.equal_range(ptr); - for (auto it = range.first; it != range.second; ++it) { - for (auto vh : values_and_holders(it->second)) { - if (vh.type == type) - return handle((PyObject *) it->second); - } - } - return handle(); -} - -inline PyThreadState *get_thread_state_unchecked() { -#if defined(PYPY_VERSION) - return PyThreadState_GET(); -#elif PY_VERSION_HEX < 0x03000000 - return _PyThreadState_Current; -#elif PY_VERSION_HEX < 0x03050000 - return (PyThreadState*) _Py_atomic_load_relaxed(&_PyThreadState_Current); -#elif PY_VERSION_HEX < 0x03050200 - return (PyThreadState*) _PyThreadState_Current.value; -#else - return _PyThreadState_UncheckedGet(); -#endif -} - -// Forward declarations -inline void keep_alive_impl(handle nurse, handle patient); -inline PyObject *make_new_instance(PyTypeObject *type); - -class type_caster_generic { -public: - PYBIND11_NOINLINE type_caster_generic(const std::type_info &type_info) - : typeinfo(get_type_info(type_info)), cpptype(&type_info) { } - - type_caster_generic(const type_info *typeinfo) - : typeinfo(typeinfo), cpptype(typeinfo ? typeinfo->cpptype : nullptr) { } - - bool load(handle src, bool convert) { - return load_impl(src, convert); - } - - PYBIND11_NOINLINE static handle cast(const void *_src, return_value_policy policy, handle parent, - const detail::type_info *tinfo, - void *(*copy_constructor)(const void *), - void *(*move_constructor)(const void *), - const void *existing_holder = nullptr) { - if (!tinfo) // no type info: error will be set already - return handle(); - - void *src = const_cast(_src); - if (src == nullptr) - return none().release(); - - auto it_instances = get_internals().registered_instances.equal_range(src); - for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { - for (auto instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { - if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) - return handle((PyObject *) it_i->second).inc_ref(); - } - } - - auto inst = reinterpret_steal(make_new_instance(tinfo->type)); - auto wrapper = reinterpret_cast(inst.ptr()); - wrapper->owned = false; - void *&valueptr = values_and_holders(wrapper).begin()->value_ptr(); - - switch (policy) { - case return_value_policy::automatic: - case return_value_policy::take_ownership: - valueptr = src; - wrapper->owned = true; - break; - - case return_value_policy::automatic_reference: - case return_value_policy::reference: - valueptr = src; - wrapper->owned = false; - break; - - case return_value_policy::copy: - if (copy_constructor) { - valueptr = copy_constructor(src); - wrapper->owned = true; - } else { // Instead of throwing an error, treat as if return_value_policy::reference. - valueptr = src; - wrapper->owned = false; - } - break; - - case return_value_policy::move: - if (move_constructor) - valueptr = move_constructor(src); - else if (copy_constructor) - valueptr = copy_constructor(src); - else - throw cast_error("return_value_policy = move, but the " - "object is neither movable nor copyable!"); - wrapper->owned = true; - break; - - case return_value_policy::reference_internal: - valueptr = src; - wrapper->owned = false; - keep_alive_impl(inst, parent); - break; - - default: - throw cast_error("unhandled return_value_policy: should not happen!"); - } - - tinfo->init_instance(wrapper, existing_holder); - - return inst.release(); - } - - // Base methods for generic caster; there are overridden in copyable_holder_caster - void load_value(value_and_holder &&v_h) { - auto *&vptr = v_h.value_ptr(); - // Lazy allocation for unallocated values: - if (vptr == nullptr) { - auto *type = v_h.type ? v_h.type : typeinfo; - vptr = type->operator_new(type->type_size); - } - value = vptr; - } - bool try_implicit_casts(handle src, bool convert) { - for (auto &cast : typeinfo->implicit_casts) { - type_caster_generic sub_caster(*cast.first); - if (sub_caster.load(src, convert)) { - value = cast.second(sub_caster.value); - return true; - } - } - return false; - } - bool try_direct_conversions(handle src) { - for (auto &converter : *typeinfo->direct_conversions) { - if (converter(src.ptr(), value)) - return true; - } - return false; - } - void check_holder_compat() {} - - PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { - auto caster = type_caster_generic(ti); - if (caster.load(src, false)) - return caster.value; - return nullptr; - } - - /// Try to load with foreign typeinfo, if available. Used when there is no - /// native typeinfo, or when the native one wasn't able to produce a value. - PYBIND11_NOINLINE bool try_load_foreign_module_local(handle src) { - constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; - const auto pytype = src.get_type(); - if (!hasattr(pytype, local_key)) - return false; - - type_info *foreign_typeinfo = reinterpret_borrow(getattr(pytype, local_key)); - // Only consider this foreign loader if actually foreign and is a loader of the correct cpp type - if (foreign_typeinfo->module_local_load == &local_load - || (cpptype && !same_type(*cpptype, *foreign_typeinfo->cpptype))) - return false; - - if (auto result = foreign_typeinfo->module_local_load(src.ptr(), foreign_typeinfo)) { - value = result; - return true; - } - return false; - } - - // Implementation of `load`; this takes the type of `this` so that it can dispatch the relevant - // bits of code between here and copyable_holder_caster where the two classes need different - // logic (without having to resort to virtual inheritance). - template - PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { - if (!src) return false; - if (!typeinfo) return try_load_foreign_module_local(src); - if (src.is_none()) { - // Defer accepting None to other overloads (if we aren't in convert mode): - if (!convert) return false; - value = nullptr; - return true; - } - - auto &this_ = static_cast(*this); - this_.check_holder_compat(); - - PyTypeObject *srctype = Py_TYPE(src.ptr()); - - // Case 1: If src is an exact type match for the target type then we can reinterpret_cast - // the instance's value pointer to the target type: - if (srctype == typeinfo->type) { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder()); - return true; - } - // Case 2: We have a derived class - else if (PyType_IsSubtype(srctype, typeinfo->type)) { - auto &bases = all_type_info(srctype); - bool no_cpp_mi = typeinfo->simple_type; - - // Case 2a: the python type is a Python-inherited derived class that inherits from just - // one simple (no MI) pybind11 class, or is an exact match, so the C++ instance is of - // the right type and we can use reinterpret_cast. - // (This is essentially the same as case 2b, but because not using multiple inheritance - // is extremely common, we handle it specially to avoid the loop iterator and type - // pointer lookup overhead) - if (bases.size() == 1 && (no_cpp_mi || bases.front()->type == typeinfo->type)) { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder()); - return true; - } - // Case 2b: the python type inherits from multiple C++ bases. Check the bases to see if - // we can find an exact match (or, for a simple C++ type, an inherited match); if so, we - // can safely reinterpret_cast to the relevant pointer. - else if (bases.size() > 1) { - for (auto base : bases) { - if (no_cpp_mi ? PyType_IsSubtype(base->type, typeinfo->type) : base->type == typeinfo->type) { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder(base)); - return true; - } - } - } - - // Case 2c: C++ multiple inheritance is involved and we couldn't find an exact type match - // in the registered bases, above, so try implicit casting (needed for proper C++ casting - // when MI is involved). - if (this_.try_implicit_casts(src, convert)) - return true; - } - - // Perform an implicit conversion - if (convert) { - for (auto &converter : typeinfo->implicit_conversions) { - auto temp = reinterpret_steal(converter(src.ptr(), typeinfo->type)); - if (load_impl(temp, false)) { - loader_life_support::add_patient(temp); - return true; - } - } - if (this_.try_direct_conversions(src)) - return true; - } - - // Failed to match local typeinfo. Try again with global. - if (typeinfo->module_local) { - if (auto gtype = get_global_type_info(*typeinfo->cpptype)) { - typeinfo = gtype; - return load(src, false); - } - } - - // Global typeinfo has precedence over foreign module_local - return try_load_foreign_module_local(src); - } - - - // Called to do type lookup and wrap the pointer and type in a pair when a dynamic_cast - // isn't needed or can't be used. If the type is unknown, sets the error and returns a pair - // with .second = nullptr. (p.first = nullptr is not an error: it becomes None). - PYBIND11_NOINLINE static std::pair src_and_type( - const void *src, const std::type_info &cast_type, const std::type_info *rtti_type = nullptr) { - if (auto *tpi = get_type_info(cast_type)) - return {src, const_cast(tpi)}; - - // Not found, set error: - std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); - detail::clean_type_id(tname); - std::string msg = "Unregistered type : " + tname; - PyErr_SetString(PyExc_TypeError, msg.c_str()); - return {nullptr, nullptr}; - } - - const type_info *typeinfo = nullptr; - const std::type_info *cpptype = nullptr; - void *value = nullptr; -}; - -/** - * Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster - * needs to provide `operator T*()` and `operator T&()` operators. - * - * If the type supports moving the value away via an `operator T&&() &&` method, it should use - * `movable_cast_op_type` instead. - */ -template -using cast_op_type = - conditional_t>::value, - typename std::add_pointer>::type, - typename std::add_lvalue_reference>::type>; - -/** - * Determine suitable casting operator for a type caster with a movable value. Such a type caster - * needs to provide `operator T*()`, `operator T&()`, and `operator T&&() &&`. The latter will be - * called in appropriate contexts where the value can be moved rather than copied. - * - * These operator are automatically provided when using the PYBIND11_TYPE_CASTER macro. - */ -template -using movable_cast_op_type = - conditional_t::type>::value, - typename std::add_pointer>::type, - conditional_t::value, - typename std::add_rvalue_reference>::type, - typename std::add_lvalue_reference>::type>>; - -// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template struct is_copy_constructible : std::is_copy_constructible {}; - -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. -template struct is_copy_constructible, - std::is_same - >::value>> : is_copy_constructible {}; - -#if !defined(PYBIND11_CPP17) -// Likewise for std::pair before C++17 (which mandates that the copy constructor not exist when the -// two types aren't themselves copy constructible). -template struct is_copy_constructible> - : all_of, is_copy_constructible> {}; -#endif - -/// Generic type caster for objects stored on the heap -template class type_caster_base : public type_caster_generic { - using itype = intrinsic_t; -public: - static constexpr auto name = _(); - - type_caster_base() : type_caster_base(typeid(type)) { } - explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) { } - - static handle cast(const itype &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) - policy = return_value_policy::copy; - return cast(&src, policy, parent); - } - - static handle cast(itype &&src, return_value_policy, handle parent) { - return cast(&src, return_value_policy::move, parent); - } - - // Returns a (pointer, type_info) pair taking care of necessary RTTI type lookup for a - // polymorphic type. If the instance isn't derived, returns the non-RTTI base version. - template ::value, int> = 0> - static std::pair src_and_type(const itype *src) { - const void *vsrc = src; - auto &cast_type = typeid(itype); - const std::type_info *instance_type = nullptr; - if (vsrc) { - instance_type = &typeid(*src); - if (!same_type(cast_type, *instance_type)) { - // This is a base pointer to a derived type; if it is a pybind11-registered type, we - // can get the correct derived pointer (which may be != base pointer) by a - // dynamic_cast to most derived type: - if (auto *tpi = get_type_info(*instance_type)) - return {dynamic_cast(src), const_cast(tpi)}; - } - } - // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, so - // don't do a cast - return type_caster_generic::src_and_type(vsrc, cast_type, instance_type); - } - - // Non-polymorphic type, so no dynamic casting; just call the generic version directly - template ::value, int> = 0> - static std::pair src_and_type(const itype *src) { - return type_caster_generic::src_and_type(src, typeid(itype)); - } - - static handle cast(const itype *src, return_value_policy policy, handle parent) { - auto st = src_and_type(src); - return type_caster_generic::cast( - st.first, policy, parent, st.second, - make_copy_constructor(src), make_move_constructor(src)); - } - - static handle cast_holder(const itype *src, const void *holder) { - auto st = src_and_type(src); - return type_caster_generic::cast( - st.first, return_value_policy::take_ownership, {}, st.second, - nullptr, nullptr, holder); - } +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) - template using cast_op_type = detail::cast_op_type; +PYBIND11_WARNING_DISABLE_MSVC(4127) - operator itype*() { return (type *) value; } - operator itype&() { if (!value) throw reference_cast_error(); return *((itype *) value); } +PYBIND11_NAMESPACE_BEGIN(detail) -protected: - using Constructor = void *(*)(const void *); - - /* Only enabled when the types are {copy,move}-constructible *and* when the type - does not have a private operator new implementation. */ - template ::value>> - static auto make_copy_constructor(const T *x) -> decltype(new T(*x), Constructor{}) { - return [](const void *arg) -> void * { - return new T(*reinterpret_cast(arg)); - }; - } - - template ::value>> - static auto make_move_constructor(const T *x) -> decltype(new T(std::move(*const_cast(x))), Constructor{}) { - return [](const void *arg) -> void * { - return new T(std::move(*const_cast(reinterpret_cast(arg)))); - }; - } - - static Constructor make_copy_constructor(...) { return nullptr; } - static Constructor make_move_constructor(...) { return nullptr; } -}; - -template class type_caster : public type_caster_base { }; -template using make_caster = type_caster>; +template +class type_caster : public type_caster_base {}; +template +using make_caster = type_caster>; // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T -template typename make_caster::template cast_op_type cast_op(make_caster &caster) { +template +typename make_caster::template cast_op_type cast_op(make_caster &caster) { return caster.operator typename make_caster::template cast_op_type(); } -template typename make_caster::template cast_op_type::type> +template +typename make_caster::template cast_op_type::type> cast_op(make_caster &&caster) { - return std::move(caster).operator - typename make_caster::template cast_op_type::type>(); + return std::move(caster).operator typename make_caster:: + template cast_op_type::type>(); } -template class type_caster> { +template +class type_caster> { private: using caster_t = make_caster; caster_t subcaster; - using subcaster_cast_op_type = typename caster_t::template cast_op_type; - static_assert(std::is_same::type &, subcaster_cast_op_type>::value, - "std::reference_wrapper caster requires T to have a caster with an `T &` operator"); + using reference_t = type &; + using subcaster_cast_op_type = typename caster_t::template cast_op_type; + + static_assert( + std::is_same::type &, subcaster_cast_op_type>::value + || std::is_same::value, + "std::reference_wrapper caster requires T to have a caster with an " + "`operator T &()` or `operator const T &()`"); + public: bool load(handle src, bool convert) { return subcaster.load(src, convert); } static constexpr auto name = caster_t::name; - static handle cast(const std::reference_wrapper &src, return_value_policy policy, handle parent) { + static handle + cast(const std::reference_wrapper &src, return_value_policy policy, handle parent) { // It is definitely wrong to take ownership of this pointer, so mask that rvp - if (policy == return_value_policy::take_ownership || policy == return_value_policy::automatic) + if (policy == return_value_policy::take_ownership + || policy == return_value_policy::automatic) { policy = return_value_policy::automatic_reference; + } return caster_t::cast(&src.get(), policy, parent); } - template using cast_op_type = std::reference_wrapper; - operator std::reference_wrapper() { return subcaster.operator subcaster_cast_op_type&(); } + template + using cast_op_type = std::reference_wrapper; + explicit operator std::reference_wrapper() { return cast_op(subcaster); } }; -#define PYBIND11_TYPE_CASTER(type, py_name) \ - protected: \ - type value; \ - public: \ - static constexpr auto name = py_name; \ - template >::value, int> = 0> \ - static handle cast(T_ *src, return_value_policy policy, handle parent) { \ - if (!src) return none().release(); \ - if (policy == return_value_policy::take_ownership) { \ - auto h = cast(std::move(*src), policy, parent); delete src; return h; \ - } else { \ - return cast(*src, policy, parent); \ - } \ - } \ - operator type*() { return &value; } \ - operator type&() { return value; } \ - operator type&&() && { return std::move(value); } \ - template using cast_op_type = pybind11::detail::movable_cast_op_type - - -template using is_std_char_type = any_of< - std::is_same, /* std::string */ - std::is_same, /* std::u16string */ - std::is_same, /* std::u32string */ - std::is_same /* std::wstring */ ->; +#define PYBIND11_TYPE_CASTER(type, py_name) \ +protected: \ + type value; \ + \ +public: \ + static constexpr auto name = py_name; \ + template >::value, \ + int> \ + = 0> \ + static ::pybind11::handle cast( \ + T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ + if (!src) \ + return ::pybind11::none().release(); \ + if (policy == ::pybind11::return_value_policy::take_ownership) { \ + auto h = cast(std::move(*src), policy, parent); \ + delete src; \ + return h; \ + } \ + return cast(*src, policy, parent); \ + } \ + operator type *() { return &value; } /* NOLINT(bugprone-macro-parentheses) */ \ + operator type &() { return value; } /* NOLINT(bugprone-macro-parentheses) */ \ + operator type &&() && { return std::move(value); } /* NOLINT(bugprone-macro-parentheses) */ \ + template \ + using cast_op_type = ::pybind11::detail::movable_cast_op_type + +template +using is_std_char_type = any_of, /* std::string */ +#if defined(PYBIND11_HAS_U8STRING) + std::is_same, /* std::u8string */ +#endif + std::is_same, /* std::u16string */ + std::is_same, /* std::u32string */ + std::is_same /* std::wstring */ + >; template struct type_caster::value && !is_std_char_type::value>> { using _py_type_0 = conditional_t; - using _py_type_1 = conditional_t::value, _py_type_0, typename std::make_unsigned<_py_type_0>::type>; + using _py_type_1 = conditional_t::value, + _py_type_0, + typename std::make_unsigned<_py_type_0>::type>; using py_type = conditional_t::value, double, _py_type_1>; -public: +public: bool load(handle src, bool convert) { py_type py_value; - if (!src) + if (!src) { return false; + } + +#if !defined(PYPY_VERSION) + auto index_check = [](PyObject *o) { return PyIndex_Check(o); }; +#else + // In PyPy 7.3.3, `PyIndex_Check` is implemented by calling `__index__`, + // while CPython only considers the existence of `nb_index`/`__index__`. + auto index_check = [](PyObject *o) { return hasattr(o, "__index__"); }; +#endif if (std::is_floating_point::value) { - if (convert || PyFloat_Check(src.ptr())) + if (convert || PyFloat_Check(src.ptr())) { py_value = (py_type) PyFloat_AsDouble(src.ptr()); - else + } else { return false; - } else if (PyFloat_Check(src.ptr())) { + } + } else if (PyFloat_Check(src.ptr()) + || (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr()))) { return false; - } else if (std::is_unsigned::value) { - py_value = as_unsigned(src.ptr()); - } else { // signed integer: - py_value = sizeof(T) <= sizeof(long) - ? (py_type) PyLong_AsLong(src.ptr()) - : (py_type) PYBIND11_LONG_AS_LONGLONG(src.ptr()); + } else { + handle src_or_index = src; + // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls. +#if PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) + object index; + if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) + index = reinterpret_steal(PyNumber_Index(src.ptr())); + if (!index) { + PyErr_Clear(); + if (!convert) + return false; + } else { + src_or_index = index; + } + } +#endif + if (std::is_unsigned::value) { + py_value = as_unsigned(src_or_index.ptr()); + } else { // signed integer: + py_value = sizeof(T) <= sizeof(long) + ? (py_type) PyLong_AsLong(src_or_index.ptr()) + : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr()); + } } + // Python API reported an error bool py_err = py_value == (py_type) -1 && PyErr_Occurred(); - if (py_err || (std::is_integral::value && sizeof(py_type) != sizeof(T) && - (py_value < (py_type) std::numeric_limits::min() || - py_value > (py_type) std::numeric_limits::max()))) { - bool type_error = py_err && PyErr_ExceptionMatches( -#if PY_VERSION_HEX < 0x03000000 && !defined(PYPY_VERSION) - PyExc_SystemError -#else - PyExc_TypeError -#endif - ); + + // Check to see if the conversion is valid (integers should match exactly) + // Signed/unsigned checks happen elsewhere + if (py_err + || (std::is_integral::value && sizeof(py_type) != sizeof(T) + && py_value != (py_type) (T) py_value)) { PyErr_Clear(); - if (type_error && convert && PyNumber_Check(src.ptr())) { + if (py_err && convert && (PyNumber_Check(src.ptr()) != 0)) { auto tmp = reinterpret_steal(std::is_floating_point::value - ? PyNumber_Float(src.ptr()) - : PyNumber_Long(src.ptr())); + ? PyNumber_Float(src.ptr()) + : PyNumber_Long(src.ptr())); PyErr_Clear(); return load(tmp, false); } @@ -978,49 +201,75 @@ struct type_caster::value && !is_std_char_t return true; } - static handle cast(T src, return_value_policy /* policy */, handle /* parent */) { - if (std::is_floating_point::value) { - return PyFloat_FromDouble((double) src); - } else if (sizeof(T) <= sizeof(ssize_t)) { - // This returns a long automatically if needed - if (std::is_signed::value) - return PYBIND11_LONG_FROM_SIGNED(src); - else - return PYBIND11_LONG_FROM_UNSIGNED(src); - } else { - if (std::is_signed::value) - return PyLong_FromLongLong((long long) src); - else - return PyLong_FromUnsignedLongLong((unsigned long long) src); - } + template + static typename std::enable_if::value, handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyFloat_FromDouble((double) src); + } + + template + static typename std::enable_if::value && std::is_signed::value + && (sizeof(U) <= sizeof(long)), + handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PYBIND11_LONG_FROM_SIGNED((long) src); + } + + template + static typename std::enable_if::value && std::is_unsigned::value + && (sizeof(U) <= sizeof(unsigned long)), + handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PYBIND11_LONG_FROM_UNSIGNED((unsigned long) src); + } + + template + static typename std::enable_if::value && std::is_signed::value + && (sizeof(U) > sizeof(long)), + handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyLong_FromLongLong((long long) src); } - PYBIND11_TYPE_CASTER(T, _::value>("int", "float")); + template + static typename std::enable_if::value && std::is_unsigned::value + && (sizeof(U) > sizeof(unsigned long)), + handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyLong_FromUnsignedLongLong((unsigned long long) src); + } + + PYBIND11_TYPE_CASTER(T, const_name::value>("int", "float")); }; -template struct void_caster { +template +struct void_caster { public: bool load(handle src, bool) { - if (src && src.is_none()) + if (src && src.is_none()) { return true; + } return false; } static handle cast(T, return_value_policy /* policy */, handle /* parent */) { - return none().inc_ref(); + return none().release(); } - PYBIND11_TYPE_CASTER(T, _("None")); + PYBIND11_TYPE_CASTER(T, const_name("None")); }; -template <> class type_caster : public void_caster {}; +template <> +class type_caster : public void_caster {}; -template <> class type_caster : public type_caster { +template <> +class type_caster : public type_caster { public: using type_caster::cast; bool load(handle h, bool) { if (!h) { return false; - } else if (h.is_none()) { + } + if (h.is_none()) { value = nullptr; return true; } @@ -1032,7 +281,7 @@ template <> class type_caster : public type_caster { } /* Check if this is a C++ type */ - auto &bases = all_type_info((PyTypeObject *) h.get_type().ptr()); + const auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); if (bases.size() == 1) { // Only allowing loading from a single-value type value = values_and_holders(reinterpret_cast(h.ptr())).begin()->value_ptr(); return true; @@ -1043,188 +292,247 @@ template <> class type_caster : public type_caster { } static handle cast(const void *ptr, return_value_policy /* policy */, handle /* parent */) { - if (ptr) + if (ptr) { return capsule(ptr).release(); - else - return none().inc_ref(); + } + return none().release(); } - template using cast_op_type = void*&; - operator void *&() { return value; } - static constexpr auto name = _("capsule"); + template + using cast_op_type = void *&; + explicit operator void *&() { return value; } + static constexpr auto name = const_name("capsule"); + private: void *value = nullptr; }; -template <> class type_caster : public void_caster { }; +template <> +class type_caster : public void_caster {}; -template <> class type_caster { +template <> +class type_caster { public: bool load(handle src, bool convert) { - if (!src) return false; - else if (src.ptr() == Py_True) { value = true; return true; } - else if (src.ptr() == Py_False) { value = false; return true; } - else if (convert || !strcmp("numpy.bool_", Py_TYPE(src.ptr())->tp_name)) { + if (!src) { + return false; + } + if (src.ptr() == Py_True) { + value = true; + return true; + } + if (src.ptr() == Py_False) { + value = false; + return true; + } + if (convert || (std::strcmp("numpy.bool_", Py_TYPE(src.ptr())->tp_name) == 0)) { // (allow non-implicit conversion for numpy booleans) Py_ssize_t res = -1; if (src.is_none()) { - res = 0; // None is implicitly converted to False + res = 0; // None is implicitly converted to False } - #if defined(PYPY_VERSION) - // On PyPy, check that "__bool__" (or "__nonzero__" on Python 2.7) attr exists +#if defined(PYPY_VERSION) + // On PyPy, check that "__bool__" attr exists else if (hasattr(src, PYBIND11_BOOL_ATTR)) { res = PyObject_IsTrue(src.ptr()); } - #else +#else // Alternate approach for CPython: this does the same as the above, but optimized // using the CPython API so as to avoid an unneeded attribute lookup. - else if (auto tp_as_number = src.ptr()->ob_type->tp_as_number) { + else if (auto *tp_as_number = src.ptr()->ob_type->tp_as_number) { if (PYBIND11_NB_BOOL(tp_as_number)) { res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr()); } } - #endif +#endif if (res == 0 || res == 1) { - value = (bool) res; + value = (res != 0); return true; } + PyErr_Clear(); } return false; } static handle cast(bool src, return_value_policy /* policy */, handle /* parent */) { return handle(src ? Py_True : Py_False).inc_ref(); } - PYBIND11_TYPE_CASTER(bool, _("bool")); + PYBIND11_TYPE_CASTER(bool, const_name("bool")); }; // Helper class for UTF-{8,16,32} C++ stl strings: -template struct string_caster { +template +struct string_caster { using CharT = typename StringType::value_type; // Simplify life by being able to assume standard char sizes (the standard only guarantees // minimums, but Python requires exact sizes) - static_assert(!std::is_same::value || sizeof(CharT) == 1, "Unsupported char size != 1"); - static_assert(!std::is_same::value || sizeof(CharT) == 2, "Unsupported char16_t size != 2"); - static_assert(!std::is_same::value || sizeof(CharT) == 4, "Unsupported char32_t size != 4"); + static_assert(!std::is_same::value || sizeof(CharT) == 1, + "Unsupported char size != 1"); +#if defined(PYBIND11_HAS_U8STRING) + static_assert(!std::is_same::value || sizeof(CharT) == 1, + "Unsupported char8_t size != 1"); +#endif + static_assert(!std::is_same::value || sizeof(CharT) == 2, + "Unsupported char16_t size != 2"); + static_assert(!std::is_same::value || sizeof(CharT) == 4, + "Unsupported char32_t size != 4"); // wchar_t can be either 16 bits (Windows) or 32 (everywhere else) static_assert(!std::is_same::value || sizeof(CharT) == 2 || sizeof(CharT) == 4, - "Unsupported wchar_t size != 2/4"); + "Unsupported wchar_t size != 2/4"); static constexpr size_t UTF_N = 8 * sizeof(CharT); bool load(handle src, bool) { -#if PY_MAJOR_VERSION < 3 - object temp; -#endif handle load_src = src; if (!src) { return false; - } else if (!PyUnicode_Check(load_src.ptr())) { -#if PY_MAJOR_VERSION >= 3 - return load_bytes(load_src); -#else - if (sizeof(CharT) == 1) { - return load_bytes(load_src); - } + } + if (!PyUnicode_Check(load_src.ptr())) { + return load_raw(load_src); + } - // The below is a guaranteed failure in Python 3 when PyUnicode_Check returns false - if (!PYBIND11_BYTES_CHECK(load_src.ptr())) + // For UTF-8 we avoid the need for a temporary `bytes` object by using + // `PyUnicode_AsUTF8AndSize`. + if (UTF_N == 8) { + Py_ssize_t size = -1; + const auto *buffer + = reinterpret_cast(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); + if (!buffer) { + PyErr_Clear(); return false; - - temp = reinterpret_steal(PyUnicode_FromObject(load_src.ptr())); - if (!temp) { PyErr_Clear(); return false; } - load_src = temp; -#endif + } + value = StringType(buffer, static_cast(size)); + return true; } - object utfNbytes = reinterpret_steal(PyUnicode_AsEncodedString( - load_src.ptr(), UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr)); - if (!utfNbytes) { PyErr_Clear(); return false; } + auto utfNbytes + = reinterpret_steal(PyUnicode_AsEncodedString(load_src.ptr(), + UTF_N == 8 ? "utf-8" + : UTF_N == 16 ? "utf-16" + : "utf-32", + nullptr)); + if (!utfNbytes) { + PyErr_Clear(); + return false; + } - const CharT *buffer = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); + const auto *buffer + = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); - if (UTF_N > 8) { buffer++; length--; } // Skip BOM for UTF-16/32 + // Skip BOM for UTF-16/32 + if (UTF_N > 8) { + buffer++; + length--; + } value = StringType(buffer, length); // If we're loading a string_view we need to keep the encoded Python object alive: - if (IsView) + if (IsView) { loader_life_support::add_patient(utfNbytes); + } return true; } - static handle cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { + static handle + cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { const char *buffer = reinterpret_cast(src.data()); - ssize_t nbytes = ssize_t(src.size() * sizeof(CharT)); + auto nbytes = ssize_t(src.size() * sizeof(CharT)); handle s = decode_utfN(buffer, nbytes); - if (!s) throw error_already_set(); + if (!s) { + throw error_already_set(); + } return s; } - PYBIND11_TYPE_CASTER(StringType, _(PYBIND11_STRING_NAME)); + PYBIND11_TYPE_CASTER(StringType, const_name(PYBIND11_STRING_NAME)); private: static handle decode_utfN(const char *buffer, ssize_t nbytes) { #if !defined(PYPY_VERSION) - return - UTF_N == 8 ? PyUnicode_DecodeUTF8(buffer, nbytes, nullptr) : - UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) : - PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); + return UTF_N == 8 ? PyUnicode_DecodeUTF8(buffer, nbytes, nullptr) + : UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) + : PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); #else - // PyPy seems to have multiple problems related to PyUnicode_UTF*: the UTF8 version - // sometimes segfaults for unknown reasons, while the UTF16 and 32 versions require a - // non-const char * arguments, which is also a nuissance, so bypass the whole thing by just - // passing the encoding as a string value, which works properly: - return PyUnicode_Decode(buffer, nbytes, UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr); + // PyPy segfaults when on PyUnicode_DecodeUTF16 (and possibly on PyUnicode_DecodeUTF32 as + // well), so bypass the whole thing by just passing the encoding as a string value, which + // works properly: + return PyUnicode_Decode(buffer, + nbytes, + UTF_N == 8 ? "utf-8" + : UTF_N == 16 ? "utf-16" + : "utf-32", + nullptr); #endif } - // When loading into a std::string or char*, accept a bytes object as-is (i.e. + // When loading into a std::string or char*, accept a bytes/bytearray object as-is (i.e. // without any encoding/decoding attempt). For other C++ char sizes this is a no-op. // which supports loading a unicode from a str, doesn't take this path. template - bool load_bytes(enable_if_t src) { + bool load_raw(enable_if_t::value, handle> src) { if (PYBIND11_BYTES_CHECK(src.ptr())) { - // We were passed a Python 3 raw bytes; accept it into a std::string or char* + // We were passed raw bytes; accept it into a std::string or char* // without any encoding attempt. const char *bytes = PYBIND11_BYTES_AS_STRING(src.ptr()); - if (bytes) { - value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); - return true; + if (!bytes) { + pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); + } + value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); + return true; + } + if (PyByteArray_Check(src.ptr())) { + // We were passed a bytearray; accept it into a std::string or char* + // without any encoding attempt. + const char *bytearray = PyByteArray_AsString(src.ptr()); + if (!bytearray) { + pybind11_fail("Unexpected PyByteArray_AsString() failure."); } + value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); + return true; } return false; } template - bool load_bytes(enable_if_t) { return false; } + bool load_raw(enable_if_t::value, handle>) { + return false; + } }; template -struct type_caster, enable_if_t::value>> +struct type_caster, + enable_if_t::value>> : string_caster> {}; #ifdef PYBIND11_HAS_STRING_VIEW template -struct type_caster, enable_if_t::value>> +struct type_caster, + enable_if_t::value>> : string_caster, true> {}; #endif // Type caster for C-style strings. We basically use a std::string type caster, but also add the // ability to use None as a nullptr char* (which the string caster doesn't allow). -template struct type_caster::value>> { +template +struct type_caster::value>> { using StringType = std::basic_string; - using StringCaster = type_caster; + using StringCaster = make_caster; StringCaster str_caster; bool none = false; CharT one_char = 0; + public: bool load(handle src, bool convert) { - if (!src) return false; + if (!src) { + return false; + } if (src.is_none()) { // Defer accepting None to other overloads (if we aren't in convert mode): - if (!convert) return false; + if (!convert) { + return false; + } none = true; return true; } @@ -1232,45 +540,58 @@ template struct type_caster::value) { handle s = PyUnicode_DecodeLatin1((const char *) &src, 1, nullptr); - if (!s) throw error_already_set(); + if (!s) { + throw error_already_set(); + } return s; } return StringCaster::cast(StringType(1, src), policy, parent); } - operator CharT*() { return none ? nullptr : const_cast(static_cast(str_caster).c_str()); } - operator CharT&() { - if (none) + explicit operator CharT *() { + return none ? nullptr : const_cast(static_cast(str_caster).c_str()); + } + explicit operator CharT &() { + if (none) { throw value_error("Cannot convert None to a character"); + } auto &value = static_cast(str_caster); size_t str_len = value.size(); - if (str_len == 0) + if (str_len == 0) { throw value_error("Cannot convert empty string to a character"); + } // If we're in UTF-8 mode, we have two possible failures: one for a unicode character that - // is too high, and one for multiple unicode characters (caught later), so we need to figure - // out how long the first encoded character is in bytes to distinguish between these two - // errors. We also allow want to allow unicode characters U+0080 through U+00FF, as those - // can fit into a single char value. + // is too high, and one for multiple unicode characters (caught later), so we need to + // figure out how long the first encoded character is in bytes to distinguish between these + // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as + // those can fit into a single char value. if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { - unsigned char v0 = static_cast(value[0]); - size_t char0_bytes = !(v0 & 0x80) ? 1 : // low bits only: 0-127 - (v0 & 0xE0) == 0xC0 ? 2 : // 0b110xxxxx - start of 2-byte sequence - (v0 & 0xF0) == 0xE0 ? 3 : // 0b1110xxxx - start of 3-byte sequence - 4; // 0b11110xxx - start of 4-byte sequence + auto v0 = static_cast(value[0]); + // low bits only: 0-127 + // 0b110xxxxx - start of 2-byte sequence + // 0b1110xxxx - start of 3-byte sequence + // 0b11110xxx - start of 4-byte sequence + size_t char0_bytes = (v0 & 0x80) == 0 ? 1 + : (v0 & 0xE0) == 0xC0 ? 2 + : (v0 & 0xF0) == 0xE0 ? 3 + : 4; if (char0_bytes == str_len) { // If we have a 128-255 value, we can decode it into a single char: if (char0_bytes == 2 && (v0 & 0xFC) == 0xC0) { // 0x110000xx 0x10xxxxxx - one_char = static_cast(((v0 & 3) << 6) + (static_cast(value[1]) & 0x3F)); + one_char = static_cast(((v0 & 3) << 6) + + (static_cast(value[1]) & 0x3F)); return one_char; } // Otherwise we have a single character, but it's > U+00FF @@ -1283,34 +604,40 @@ template struct type_caster(value[0]); - if (one_char >= 0xD800 && one_char < 0xE000) + if (one_char >= 0xD800 && one_char < 0xE000) { throw value_error("Character code point not in range(0x10000)"); + } } - if (str_len != 1) + if (str_len != 1) { throw value_error("Expected a character, but multi-character string found"); + } one_char = value[0]; return one_char; } - static constexpr auto name = _(PYBIND11_STRING_NAME); - template using cast_op_type = pybind11::detail::cast_op_type<_T>; + static constexpr auto name = const_name(PYBIND11_STRING_NAME); + template + using cast_op_type = pybind11::detail::cast_op_type<_T>; }; // Base implementation for std::tuple and std::pair -template class Tuple, typename... Ts> class tuple_caster { +template