From 3e6c313077f763a4af7d6a83097c95ba3e9e054c Mon Sep 17 00:00:00 2001 From: Justin Carpentier Date: Sat, 4 Sep 2021 15:38:05 +0200 Subject: [PATCH 1/8] core: add option to also parse Boost.Python librairies --- pybind11_stubgen/__init__.py | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/pybind11_stubgen/__init__.py b/pybind11_stubgen/__init__.py index ba7d85a6..6015631f 100644 --- a/pybind11_stubgen/__init__.py +++ b/pybind11_stubgen/__init__.py @@ -85,6 +85,68 @@ def __init__(self, name, args='*args, **kwargs', rtype='None', validate=True): for invalid_default in invalid_defaults: logger.log(lvl, " {}".format(invalid_default)) + if USE_BOOST_PYTHON: + if args: + find_optional_args = re.findall('\[(.*?)\]$', args) + optional_args = None + if find_optional_args: + optional_args = find_optional_args[0] + if optional_args: + nominal_args = args.replace("[" + optional_args + "]","") + else: + nominal_args = args + + num_nominal_args = 0 + if nominal_args: + nominal_args = nominal_args.split(",") + num_nominal_args = len(nominal_args) + + num_optional_args = 0 + if optional_args: + optional_args = optional_args.split("[,") + num_optional_args = len(optional_args) + if num_optional_args > 1: + optional_args[-1] = re.sub(']'*(num_optional_args-1)+'$', '', optional_args[-1]) # Replace at the end + new_args = "" + + if nominal_args: + for k,arg in enumerate(nominal_args): + type_name = re.findall('\((.*?)\)', arg)[0] + arg_name = arg.split(")")[1] + arg_name = arg_name.replace(' ','_') + + new_args += arg_name + ": " + type_name + if k < num_nominal_args-1: + new_args += ", " + + if num_optional_args > 0 and num_nominal_args > 0: + new_args += ", " + + if optional_args and True: + for k,arg in enumerate(optional_args): + # Check for default value + split_arg_equal = arg.split('=',maxsplit=1) + main_arg = split_arg_equal[0] + type_name = re.findall('\((.*?)\)', main_arg)[0] + + arg_name = main_arg.split(")")[1] + arg_name = arg_name.replace(' ','_') + new_args += arg_name + ": " + type_name + optional_value = None + if len(split_arg_equal) > 1: + optional_value = split_arg_equal[1] + new_args += " = " + optional_value + + if k < num_optional_args-1: + new_args += ", " + + new_args = new_args.replace(" ,", ",") + self.args = new_args + args = new_args + + rtype = rtype.split(" :")[0] + self.rtype = rtype + function_def_str = "def {sig.name}({sig.args}) -> {sig.rtype}: ...".format(sig=self) try: ast.parse(function_def_str) @@ -189,6 +251,9 @@ def replace_typing_types(match): return "typing." + match.group('type').capitalize() +# If true, parse BOOST_PYTHON signature +USE_BOOST_PYTHON = False + class StubsGenerator(object): INDENT = " " * 4 @@ -869,6 +934,7 @@ def main(args=None): help="Render `numpy.ndarray` without (non-standardized) bracket-enclosed type and shape info") parser.add_argument("module_names", nargs="+", metavar="MODULE_NAME", type=str, help="modules names") parser.add_argument("--log-level", default="INFO", help="Set output log level") + parser.add_argument("--boost-python", action="store_true") sys_args = parser.parse_args(args or sys.argv[1:]) @@ -880,6 +946,10 @@ def main(args=None): global BARE_NUPMY_NDARRAY BARE_NUPMY_NDARRAY = True + if sys_args.boost_python: + global USE_BOOST_PYTHON + USE_BOOST_PYTHON = True + if 'all' in sys_args.ignore_invalid: FunctionSignature.ignore_invalid_signature = True FunctionSignature.ignore_invalid_defaultarg = True From 413855ed37ee7d3b0509c2de0729ff7d6a358926 Mon Sep 17 00:00:00 2001 From: Justin Carpentier Date: Mon, 6 Sep 2021 14:12:42 +0200 Subject: [PATCH 2/8] core: remove useless indents --- pybind11_stubgen/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pybind11_stubgen/__init__.py b/pybind11_stubgen/__init__.py index 6015631f..f7bbde91 100644 --- a/pybind11_stubgen/__init__.py +++ b/pybind11_stubgen/__init__.py @@ -86,7 +86,7 @@ def __init__(self, name, args='*args, **kwargs', rtype='None', validate=True): logger.log(lvl, " {}".format(invalid_default)) if USE_BOOST_PYTHON: - if args: + if args: find_optional_args = re.findall('\[(.*?)\]$', args) optional_args = None if find_optional_args: @@ -95,17 +95,17 @@ def __init__(self, name, args='*args, **kwargs', rtype='None', validate=True): nominal_args = args.replace("[" + optional_args + "]","") else: nominal_args = args - + num_nominal_args = 0 - if nominal_args: + if nominal_args: nominal_args = nominal_args.split(",") num_nominal_args = len(nominal_args) - + num_optional_args = 0 if optional_args: - optional_args = optional_args.split("[,") + optional_args = optional_args.split("[,") num_optional_args = len(optional_args) - if num_optional_args > 1: + if num_optional_args > 1: optional_args[-1] = re.sub(']'*(num_optional_args-1)+'$', '', optional_args[-1]) # Replace at the end new_args = "" @@ -120,7 +120,7 @@ def __init__(self, name, args='*args, **kwargs', rtype='None', validate=True): new_args += ", " if num_optional_args > 0 and num_nominal_args > 0: - new_args += ", " + new_args += ", " if optional_args and True: for k,arg in enumerate(optional_args): @@ -132,18 +132,18 @@ def __init__(self, name, args='*args, **kwargs', rtype='None', validate=True): arg_name = main_arg.split(")")[1] arg_name = arg_name.replace(' ','_') new_args += arg_name + ": " + type_name - optional_value = None + optional_value = None if len(split_arg_equal) > 1: optional_value = split_arg_equal[1] new_args += " = " + optional_value if k < num_optional_args-1: new_args += ", " - + new_args = new_args.replace(" ,", ",") self.args = new_args args = new_args - + rtype = rtype.split(" :")[0] self.rtype = rtype From 68aa08552a65f2b2121c068dcb1c950db0f3c669 Mon Sep 17 00:00:00 2001 From: Justin Carpentier Date: Mon, 6 Sep 2021 14:13:59 +0200 Subject: [PATCH 3/8] core: also consider classes within the imported scope --- pybind11_stubgen/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pybind11_stubgen/__init__.py b/pybind11_stubgen/__init__.py index f7bbde91..2a28c4d6 100644 --- a/pybind11_stubgen/__init__.py +++ b/pybind11_stubgen/__init__.py @@ -767,6 +767,8 @@ def parse(self): self.classes.append(ClassStubsGenerator(member)) else: self.imported_classes[name] = member + self.classes.append(ClassStubsGenerator(member)) + self.classes[-1].parse() elif name == "__doc__": self.doc_string = member elif name not in self.attributes_blacklist: From 44706594dcbc755742f05555602d983425b8a3f8 Mon Sep 17 00:00:00 2001 From: Justin Carpentier Date: Mon, 6 Sep 2021 14:23:11 +0200 Subject: [PATCH 4/8] core: import module --- pybind11_stubgen/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pybind11_stubgen/__init__.py b/pybind11_stubgen/__init__.py index 2a28c4d6..de4cd499 100644 --- a/pybind11_stubgen/__init__.py +++ b/pybind11_stubgen/__init__.py @@ -767,6 +767,7 @@ def parse(self): self.classes.append(ClassStubsGenerator(member)) else: self.imported_classes[name] = member + importlib.import_module(member.__module__) self.classes.append(ClassStubsGenerator(member)) self.classes[-1].parse() elif name == "__doc__": From f8b6c441fe1dd325e968631b0306b47a7f5c1aee Mon Sep 17 00:00:00 2001 From: Justin Carpentier Date: Mon, 6 Sep 2021 14:30:50 +0200 Subject: [PATCH 5/8] core: skip type types --- pybind11_stubgen/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pybind11_stubgen/__init__.py b/pybind11_stubgen/__init__.py index de4cd499..be827866 100644 --- a/pybind11_stubgen/__init__.py +++ b/pybind11_stubgen/__init__.py @@ -761,6 +761,9 @@ def parse(self): logger.debug("Skip '%s' module while parsing '%s' " % (m.module.__name__, self.module.__name__)) elif inspect.isbuiltin(member) or inspect.isfunction(member): self.free_functions.append(FreeFunctionStubsGenerator(name, member, self.module.__name__)) + elif type(member) is type: + logger.debug("Skip '%s' type while parsing '%s' " % (name, self.module.__name__)) + pass elif inspect.isclass(member): if member.__module__ == self.module.__name__: if member.__name__ not in self.class_name_blacklist: From 48987a0c9967e721a0dda4dcc03ddaba473bb0d1 Mon Sep 17 00:00:00 2001 From: Justin Carpentier Date: Mon, 6 Sep 2021 14:51:55 +0200 Subject: [PATCH 6/8] core: improve export --- pybind11_stubgen/__init__.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/pybind11_stubgen/__init__.py b/pybind11_stubgen/__init__.py index be827866..5f71bfc5 100644 --- a/pybind11_stubgen/__init__.py +++ b/pybind11_stubgen/__init__.py @@ -833,15 +833,20 @@ def to_lines(self): # type: () -> List[str] "import typing" ] - for name, class_ in self.imported_classes.items(): - class_name = getattr(class_, "__qualname__", class_.__name__) - if name == class_name: - suffix = "" - else: - suffix = " as {}".format(name) - result += [ - 'from {} import {}{}'.format(class_.__module__, class_name, suffix) - ] + globals_ = {} + exec("from {} import *".format(self.module.__name__), globals_) + + result += [""] + all_ = set(globals_.keys()) - {"__builtins__"} + result.append("__all__ = [\n " + ",\n ".join(map(lambda s: '"%s"' % s, sorted(all_))) + "\n]\n") + + for x in itertools.chain(self.classes, + self.free_functions): + result.extend(x.to_lines()) + result += [""] + + for x in itertools.chain(self.attributes): + result.extend(x.to_lines()) # import used packages used_modules = sorted(self.get_involved_modules_names()) @@ -858,15 +863,7 @@ def to_lines(self): # type: () -> List[str] # add space between imports and rest of module result += [""] - globals_ = {} - exec("from {} import *".format(self.module.__name__), globals_) - all_ = set(globals_.keys()) - {"__builtins__"} - result.append("__all__ = [\n " + ",\n ".join(map(lambda s: '"%s"' % s, sorted(all_))) + "\n]\n\n") - for x in itertools.chain(self.classes, - self.free_functions, - self.attributes): - result.extend(x.to_lines()) result.append("") # Newline at EOF return result From e95e8aa04030a2ca95fadc3936e19b70c1831d2a Mon Sep 17 00:00:00 2001 From: Justin Carpentier Date: Mon, 6 Sep 2021 20:56:53 +0200 Subject: [PATCH 7/8] core: add codec to fix bugs on Windows --- pybind11_stubgen/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pybind11_stubgen/__init__.py b/pybind11_stubgen/__init__.py index 5f71bfc5..717393a0 100644 --- a/pybind11_stubgen/__init__.py +++ b/pybind11_stubgen/__init__.py @@ -872,12 +872,13 @@ def short_name(self): return self.module.__name__.split(".")[-1] def write(self): + import codecs if not os.path.exists(self.short_name + self.stub_suffix): logger.debug("mkdir `%s`" % (self.short_name + self.stub_suffix)) os.mkdir(self.short_name + self.stub_suffix) with DirectoryWalkerGuard(self.short_name + self.stub_suffix): - with open("__init__.pyi", "w") as init_pyi: + with codecs.open("__init__.pyi", "w", "utf8") as init_pyi: init_pyi.write("\n".join(self.to_lines())) for m in self.submodules: m.write() From f44c3801cde87f98084e49244da0839a09d56dd4 Mon Sep 17 00:00:00 2001 From: Justin Carpentier Date: Mon, 6 Sep 2021 20:59:52 +0200 Subject: [PATCH 8/8] core: small fix --- pybind11_stubgen/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pybind11_stubgen/__init__.py b/pybind11_stubgen/__init__.py index 717393a0..78d0743f 100644 --- a/pybind11_stubgen/__init__.py +++ b/pybind11_stubgen/__init__.py @@ -872,13 +872,12 @@ def short_name(self): return self.module.__name__.split(".")[-1] def write(self): - import codecs if not os.path.exists(self.short_name + self.stub_suffix): logger.debug("mkdir `%s`" % (self.short_name + self.stub_suffix)) os.mkdir(self.short_name + self.stub_suffix) with DirectoryWalkerGuard(self.short_name + self.stub_suffix): - with codecs.open("__init__.pyi", "w", "utf8") as init_pyi: + with open("__init__.pyi", "w", encoding="utf-8") as init_pyi: init_pyi.write("\n".join(self.to_lines())) for m in self.submodules: m.write()