From 44ba2676e4b15f9422bca4e5e8e197b22561db83 Mon Sep 17 00:00:00 2001 From: James Mitchell Date: Sun, 13 Sep 2026 10:22:53 +0100 Subject: [PATCH] Add draft stub generation and Action typing Generate stubs from runtime bindings, source annotations, and documented signatures. Require an explicit module, print to stdout by default, and select public native classes plus typing dependencies for extension targets. Add sample Action and BMat8 helper stubs, bind Action's element and point types to the class, and accept CxxWrapper subclasses in copy_cxx_mem_fns. AI disclosure: OpenAI Codex helped prepare the implementation, tests, documentation, commit message, and PR description, and ran validation. Assisted-by: Codex Co-authored-by: Codex --- CONTRIBUTING.rst | 83 ++ etc/generate-stubs.py | 802 ++++++++++++++++++ src/libsemigroups_pybind11/action.py | 18 +- src/libsemigroups_pybind11/action.pyi | 119 +++ src/libsemigroups_pybind11/bmat8.pyi | 35 + .../detail/cxx_wrapper.py | 2 +- tests/test_generate_stubs.py | 552 ++++++++++++ 7 files changed, 1603 insertions(+), 8 deletions(-) create mode 100755 etc/generate-stubs.py create mode 100644 src/libsemigroups_pybind11/action.pyi create mode 100644 src/libsemigroups_pybind11/bmat8.pyi create mode 100644 tests/test_generate_stubs.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 401166c7..548bd0dc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -91,6 +91,89 @@ the output. If you followed the guidance output by the script, the class will now be accessible in ``_libsemigroups_pybind11``. You must now check the contents of that file adheres to the styles set out in this guide. +Generating type stubs +--------------------- + +``etc/generate-stubs.py`` generates draft ``.pyi`` files from the importable +package. Use an environment containing the built extension and its Python +dependencies. The module argument is required, and the stub is printed to +standard output by default. For example, to generate the ``action`` module stub:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py libsemigroups_pybind11.action + +Use ``--output-dir`` to write files under a directory. Package targets include +discovered submodules and require directory output when producing multiple stubs:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py libsemigroups_pybind11.action --output-dir _generate/stubs + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py libsemigroups_pybind11 --output-dir _generate/stubs + +Diagnostics and generation summaries go to standard error, leaving standard +output suitable for redirection or piping. A failed generation emits no stub +text. Targets producing multiple stubs are rejected when using standard output; +select an individual module or use ``--output-dir`` instead. ``--stdout`` can +explicitly select the default behavior and cannot be combined with +``--output-dir``. ``--force`` requires ``--output-dir``. + +Classes defined directly in the compiled extension, such as ``BMat8``, require +the extension module as the target. This target defaults to ``--public-only``, +which generates a smaller stub covering the public native classes:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py _libsemigroups_pybind11 + +Add ``--output-dir _generate/stubs`` to save this as +``_generate/stubs/_libsemigroups_pybind11.pyi``. Selection follows +``__all__`` in ``libsemigroups_pybind11`` and its publicly exported submodules +(or non-private names for modules without ``__all__``). Classes are matched by +object identity, so native classes exported under aliases are included. The +selection adapts to newly exported classes and the active build's optional +features, without maintaining a separate list of class names. + +Only selected classes and their typing dependencies are inspected. Required +base classes and types referenced in their signatures are included automatically +and reported. Python wrappers such as ``Alphabet`` still need their own module +stubs; their internal specializations, such as ``AlphabetWord``, are omitted +unless a retained signature requires them. + +This mode does not generate standalone helper functions or constants except +where needed as dependencies. It marks the extension stub as partial with a +module-level ``__getattr__`` returning ``_typeshed.Incomplete``, so omitted names +remain untyped. Use the full-extension mode when those declarations are needed:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py _libsemigroups_pybind11 --no-public-only --output-dir _generate/stubs + +Both extension modes use the same filename with directory output; ``--force`` +is required to replace a previous output. Other module targets retain their +normal discovery behavior; ``--public-only`` requires the extension target. +The ``libsemigroups_pybind11.bmat8`` target generates the helper module instead. +Invalid C++ base-class names are reported +and replaced by ``typing.Any`` in drafts. Parameters named after Python keywords +are renamed and marked positional-only; ``--strict`` rejects these cases. + +Generation happens in a temporary directory first, so failures during discovery +or parsing do not overwrite existing stubs. To run without uv, install +``pybind11-stubgen==2.5.5`` in your development environment and invoke the script +with that environment's Python interpreter. + +Shell redirection can also save a single stub to a chosen filename:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py libsemigroups_pybind11.action > action.pyi + +The generator combines pybind11-stubgen's runtime discovery with source +annotations, generic bases, and the project's ``:sig=...:`` docstrings. It also +accounts for the positional arguments accepted by dynamically copied methods. +The output needs review: missing annotations, unsupported type spellings, and +dynamic dispatch can still produce incomplete or inaccurate types. Inherited +untyped wrappers also limit static checking. Optional build features, such as +HPCombi, can cause discovery diagnostics. Use ``--strict`` to stop on those +diagnostics instead of emitting drafts. + +No ``py.typed`` marker is added, and existing source stubs are not replaced by +the default command. Run the generator tests with:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 --with mypy==2.3.1 pytest tests/test_generate_stubs.py + +AI assistance: OpenAI Codex helped implement, document, and test this generator. + The bindings ------------ diff --git a/etc/generate-stubs.py b/etc/generate-stubs.py new file mode 100755 index 00000000..35103f44 --- /dev/null +++ b/etc/generate-stubs.py @@ -0,0 +1,802 @@ +#!/usr/bin/env python3 +# Distributed under the terms of the GPL license version 3. +# The full license is in the file LICENSE, distributed with this software. +# AI assistance: OpenAI Codex helped implement and test this generator. + +r"""Generate draft stubs from the built package, source annotations, and docstrings. + +Run in an environment containing the built extension, for example:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py \ + libsemigroups_pybind11.action + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py \ + libsemigroups_pybind11.sims --output-dir _generate/stubs + +The module argument is required. A single stub is printed to stdout by default, +with diagnostics on standard error. Use --output-dir to write files instead. +Package targets include the submodules discovered by pybind11-stubgen and need +--output-dir when generating multiple stubs. Existing files require --force. +Extension targets default to the classes exported by the public package and its +public submodules, together with their typing dependencies. Use --no-public-only +to generate the full extension stub. +Use --strict to stop on pybind11-stubgen diagnostics instead of emitting drafts. +Output is a draft: missing annotations, dynamic dispatch, and C++ conversions +still require review. This script does not install dependencies or add py.typed. +""" + +from __future__ import annotations + +import argparse +import ast +import builtins +import contextlib +import copy +import enum +import importlib +import importlib.util +import inspect +import json +import keyword +import re +import subprocess +import sys +import tempfile +import types +import typing +from pathlib import Path + +EXTENSION_MODULE = "_libsemigroups_pybind11" + +HEADER = """# Generated by etc/generate-stubs.py using pybind11-stubgen. +# Draft only: review signatures and incomplete types before publishing. +# AI assistance: OpenAI Codex helped implement the generator. + +""" + + +def expression(text: str) -> ast.expr: + """Parse a type expression without executing it.""" + return ast.parse(text, mode="eval").body + + +def parse_stub(text: str, module_name: str, *, strict: bool) -> ast.Module: + """Handle native names that cannot be written in Python stub syntax.""" + lines = text.splitlines(keepends=True) + renamed = {} + while True: + try: + tree = ast.parse("".join(lines)) + except SyntaxError as error: + if strict or error.lineno is None: + raise + index = error.lineno - 1 + match = re.fullmatch(r"(\s*class \w+)\((.+)\):\s*", lines[index]) + if match is None: + # A binding can accept a keyword named "from", but Python syntax cannot. + line = lines[index] + offset = (error.offset or 1) - 1 + name = re.match(r"(\w+)(?=\s*[:=,)])", line[offset:]) + if ( + line.lstrip().startswith("def ") + and line[:offset].rstrip().endswith(("(", ",")) + and name + and keyword.iskeyword(name[1]) + ): + replacement = name[1] + "_" + lines[index] = line[:offset] + replacement + line[offset + len(name[1]) :] + renamed.setdefault(error.lineno, set()).add(replacement) + print( + f"{module_name}: representing keyword parameter {name[1]!r} " + f"as positional-only {replacement!r}", + file=sys.stderr, + ) + continue + raise + try: + expression(match[2]) + except SyntaxError: + # Some extension types have names such as detail::CongruenceCommon. + # Any records the unknown base without inventing a Python class. + print( + f"{module_name}: replacing invalid base {match[2]!r} with typing.Any", + file=sys.stderr, + ) + lines[index] = f"{match[1]}(typing.Any):\n" + else: + raise error + else: + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.lineno in renamed: + names = renamed[node.lineno] + if any(arg.arg in names for arg in node.args.kwonlyargs): + raise SyntaxError( + "Cannot represent a keyword-only parameter named after a keyword" + ) + count = max( + (i + 1 for i, arg in enumerate(node.args.args) if arg.arg in names), + default=0, + ) + node.args.posonlyargs += node.args.args[:count] + node.args.args = node.args.args[count:] + return tree + + +def documented_signatures(name: str, doc: str) -> list[ast.FunctionDef]: + """Prefer project signature overrides, then pybind11 signature lines.""" + overrides = re.findall(r"^\s*:sig=[^(\n]*(\(.*):\s*$", doc, re.MULTILINE) + signatures = overrides or re.findall( + rf"^\s*(?:\d+\. )?{re.escape(name)}(\(.*)$", doc, re.MULTILINE + ) + result = [] + seen = set() + for signature in signatures: + try: + node = ast.parse(f"def {name}{signature}: ...").body[0] + except SyntaxError: + continue # C++ spellings and enum defaults need manual review. + key = ast.dump(node) + if key not in seen and not (node.args.vararg and node.args.kwarg and not node.args.args): + result.append(typing.cast(ast.FunctionDef, node)) + seen.add(key) + return result + + +def bindings(nodes: list[ast.stmt]) -> set[str]: + """Collect names bound at module scope, including imported aliases.""" + names = set() + for node in nodes: + if isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + names.add(alias.asname or alias.name.split(".")[0]) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + names.add(node.target.id) + elif isinstance(node, ast.Assign): + names.update(target.id for target in node.targets if isinstance(target, ast.Name)) + return names + + +def public_native_classes(public: types.ModuleType, native: types.ModuleType) -> set[str]: + """Find native classes by identity, following the public modules' exports.""" + exported_ids = set() + pending = [public] + seen = set() + while pending: + module = pending.pop() + if id(module) in seen: + continue + seen.add(id(module)) + names = getattr( + module, "__all__", [name for name in vars(module) if not name.startswith("_")] + ) + for name in names: + value = getattr(module, name, None) + if inspect.isclass(value): + exported_ids.add(id(value)) + elif inspect.ismodule(value) and value.__name__.startswith(public.__name__ + "."): + pending.append(value) + return { + name + for name, value in vars(native).items() + if inspect.isclass(value) and id(value) in exported_ids + } + + +def native_dependencies(tree: ast.Module, native: types.ModuleType) -> set[str]: + """Find native names used by bases, annotations, aliases, and default values.""" + names = set() + pending = [tree] + for node in ast.walk(tree): + annotation = None + if isinstance(node, (ast.arg, ast.AnnAssign)): + annotation = node.annotation + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + annotation = node.returns + if annotation is not None: + # Forward references inside annotations can also name native types. + for part in ast.walk(annotation): + if isinstance(part, ast.Constant) and isinstance(part.value, str): + with contextlib.suppress(SyntaxError): + pending.append(expression(part.value)) + for root in pending: + for node in ast.walk(root): + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + names.add(node.id) + elif isinstance(node, ast.Attribute): + parts = ast.unparse(node).split(".") + if parts[0] == native.__name__ and len(parts) > 1: + names.add(parts[1]) + elif isinstance(node, ast.ImportFrom) and node.module == native.__name__: + names.update(alias.name for alias in node.names) + return { + name + for name in names + if name in vars(native) and not name.startswith("__") and name.isidentifier() + } + + +def generate_public_raw(module_name: str, names_json: str, output_dir: str, strict: bool) -> None: + """Run in a subprocess so inspection cannot change the caller's imported modules.""" + import pybind11_stubgen + + native = importlib.import_module(module_name) + selected = set(json.loads(names_json)) + roots = selected.copy() + # dir() controls discovery; getattr() still resolves every real native name. + # Importing the public package here would rename some native classes for Sphinx. + view = types.ModuleType(module_name) + vars(view).update(vars(native)) + view.__dir__ = lambda: sorted(selected) + sys.modules[module_name] = view + try: + while True: + command = [module_name, "-o", output_dir] + if strict: + command.append("--exit-code") + pybind11_stubgen.main(command) + path = Path(output_dir) / f"{module_name}.pyi" + tree = parse_stub(path.read_text(encoding="utf-8"), module_name, strict=strict) + dependencies = native_dependencies(tree, native) - selected + if not dependencies: + break + selected.update(dependencies) + finally: + sys.modules[module_name] = native + print( + f"Selected {len(roots)} public native classes and {len(selected - roots)} supporting names." + ) + if selected - roots: + print("Supporting names: " + ", ".join(sorted(selected - roots))) + + +def accepts_integer_index(annotation: ast.expr | None) -> bool: + """Identify integer overloads used by Python's implicit sequence iteration.""" + if isinstance(annotation, ast.BinOp) and isinstance(annotation.op, ast.BitOr): + return accepts_integer_index(annotation.left) or accepts_integer_index(annotation.right) + return isinstance(annotation, (ast.Name, ast.Attribute)) and ast.unparse(annotation).split(".")[ + -1 + ] in {"int", "SupportsInt", "SupportsIndex"} + + +class TypeNames(ast.NodeTransformer): + """Resolve documentation names in the public package and the owning class.""" + + def __init__(self, refiner: Refiner, parameters: tuple, *, specialize: bool = True): + self.refiner = refiner + self.parameters = {param.__name__.lstrip("_"): param.__name__ for param in parameters} + self.specialize = specialize + + def visit_Subscript(self, node: ast.Subscript) -> ast.Subscript: + previous = self.specialize + self.specialize = False + node.value = self.visit(node.value) + self.specialize = previous + node.slice = self.visit(node.slice) + return node + + def visit_Attribute(self, node: ast.Attribute) -> ast.expr: + if ast.unparse(node) == "typing.Self": + return expression("typing_extensions.Self") + parts = ast.unparse(node).split(".") + value = getattr(self.refiner.module, parts[0], None) + if inspect.isclass(value): + for part in parts[1:]: + value = getattr(value, part, None) + if inspect.isclass(value) and value.__module__ != self.refiner.module.__name__: + self.refiner.imports.add(value.__module__) + return expression(f"{value.__module__}.{value.__qualname__}") + # Qualified names already supplied by pybind11-stubgen have imports. + return node + + def visit_Name(self, node: ast.Name) -> ast.expr: + name = node.id + if name in self.parameters: + return ast.Name(id=self.parameters[name], ctx=ast.Load()) + value = getattr(self.refiner.module, name, None) + if inspect.isclass(value) and self.specialize and getattr(value, "__parameters__", ()): + params = ", ".join(param.__name__ for param in value.__parameters__) + return expression(f"{name}[{params}]") + if name in self.refiner.names or hasattr(builtins, name): + return node + if name == "Self": + return expression("typing_extensions.Self") + if hasattr(typing, name): + return expression(f"typing.{name}") + value = getattr(self.refiner.public, name, None) + if inspect.isclass(value): + self.refiner.imports.add(value.__module__) + return expression(f"{value.__module__}.{value.__qualname__}") + self.refiner.incomplete.add(name) + return expression("_typeshed.Incomplete") + + +class Refiner: + """Combine runtime discovery with information lost by generic stub tools.""" + + def __init__(self, module: types.ModuleType, tree: ast.Module, *, native_classes: bool = False): + self.module = module + self.native_classes = native_classes + self.public = importlib.import_module("libsemigroups_pybind11") + try: + self.source = ast.parse(inspect.getsource(module)) + except (OSError, TypeError): + self.source = ast.Module(body=[], type_ignores=[]) + self.names = bindings(tree.body) | bindings(self.source.body) + self.imports = {"collections.abc", "datetime", "typing", "typing_extensions", "_typeshed"} + if native_classes: + self.imports.add(module.__name__) + self.incomplete: set[str] = set() + + def annotation(self, node: ast.expr | None, parameters: tuple) -> ast.expr | None: + if node is None: + return None + return TypeNames(self, parameters).visit(copy.deepcopy(node)) + + def function(self, node: ast.FunctionDef, parameters: tuple) -> ast.FunctionDef: + node = copy.deepcopy(node) + args = node.args + for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs, args.vararg, args.kwarg]: + if arg is not None: + arg.annotation = self.annotation(arg.annotation, parameters) + if arg.arg in {"self", "cls"}: + arg.annotation = None + node.returns = self.annotation(node.returns, parameters) + node.decorator_list = [ + dec + for dec in node.decorator_list + if ast.unparse(dec).split(".")[-1] + in {"overload", "staticmethod", "classmethod", "property", "setter", "abstractmethod"} + ] + node.body = [ast.Expr(value=ast.Constant(value=Ellipsis))] + return node + + def dispatch_annotations(self, owner: object, source: ast.FunctionDef) -> dict[str, ast.expr]: + """Recognize class-valued constructor keywords used directly in dispatch keys.""" + mapping = getattr(owner, "_py_template_params_to_cxx_type", {}) + result = {} + if source.name != "__init__" or not isinstance(mapping, dict) or not mapping: + return result + for node in ast.walk(source): + if not ( + isinstance(node, ast.Assign) + and isinstance(node.value, ast.Tuple) + and any( + isinstance(target, ast.Attribute) and target.attr == "py_template_params" + for target in node.targets + ) + ): + continue + for index, item in enumerate(node.value.elts): + if not isinstance(item, ast.Name): + continue + if not all(isinstance(key, tuple) and len(key) > index for key in mapping): + continue + values = {key[index] for key in mapping} + if not all(inspect.isclass(value) for value in values): + continue + for value in values: + self.imports.add(value.__module__) + result[item.id] = expression( + " | ".join( + sorted(f"type[{value.__module__}.{value.__qualname__}]" for value in values) + ) + ) + return result + + def functions( + self, + node: ast.FunctionDef, + obj: object, + source: ast.FunctionDef | None, + parameters: tuple, + owner: object, + ) -> list[ast.FunctionDef]: + doc = inspect.getdoc(obj) or "" + documented = ( + documented_signatures(node.name, doc) if isinstance(obj, types.FunctionType) else [] + ) + if source is not None: + result = copy.deepcopy(source) + # Preserve actual parameter kinds/defaults; fill only missing annotations. + if documented: + candidates = { + arg.arg: arg.annotation + for sig in documented + for arg in [*sig.args.posonlyargs, *sig.args.args, *sig.args.kwonlyargs] + } + candidates.update(self.dispatch_annotations(owner, source)) + positional = [*result.args.posonlyargs, *result.args.args] + defaults = [None] * ( + len(positional) - len(result.args.defaults) + ) + result.args.defaults + for arg, default in zip( + [*positional, *result.args.kwonlyargs], + [*defaults, *result.args.kw_defaults], + strict=True, + ): + if arg.annotation is None and candidates.get(arg.arg) is not None: + arg.annotation = copy.deepcopy(candidates[arg.arg]) + if isinstance(default, ast.Constant) and default.value is None: + arg.annotation = expression(f"{ast.unparse(arg.annotation)} | None") + if result.returns is None and len(documented) == 1: + result.returns = documented[0].returns + return [self.function(result, parameters)] + if not documented: + return [self.function(node, parameters)] + try: + actual = inspect.signature(obj, follow_wrapped=False) + except (TypeError, ValueError): + actual = None + for sig in documented: + if ( + isinstance(sig.returns, ast.Name) + and sig.returns.id == getattr(owner, "__name__", None) + and re.search(r":returns?:\s*\*self\*", doc) + ): + sig.returns = expression("typing_extensions.Self") + # CxxWrapper's forwarding closure accepts *args, without **kwargs. + if ( + actual + and any(p.kind == p.VAR_POSITIONAL for p in actual.parameters.values()) + and not any(p.kind == p.VAR_KEYWORD for p in actual.parameters.values()) + ): + sig.args.posonlyargs += sig.args.args + sig.args.args = [] + if len(documented) > 1: + sig.decorator_list = [expression("typing.overload")] + return [self.function(sig, parameters) for sig in documented] + + def scope( + self, body: list[ast.stmt], owner: object, source: list[ast.stmt], parameters: tuple = () + ) -> list[ast.stmt]: + definitions = { + node.name: node for node in source if isinstance(node, (ast.ClassDef, ast.FunctionDef)) + } + result = [] + rebuilt = set() + for node in body: + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and inspect.isclass(owner) + and issubclass(owner, enum.Enum) + and node.target.id in owner.__members__ + ): + # Enum members in stubs are assignments, not annotated attributes. + original = next( + ( + item + for item in source + if isinstance(item, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == node.target.id + for target in item.targets + ) + ), + None, + ) + result.append( + copy.deepcopy(original) + if original is not None + else ast.Assign(targets=[node.target], value=ast.Constant(value=Ellipsis)) + ) + continue + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and ( + (node.target.id.startswith("_") and not node.target.id.endswith("__")) + or node.target.id in {"__orig_bases__", "__parameters__"} + ) + and not isinstance(owner, types.ModuleType) + ): + continue + if ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + continue + if isinstance(node, ast.ClassDef): + cls = getattr(owner, node.name, None) + if ( + inspect.isclass(cls) + and cls.__module__ != self.module.__name__ + and not self.native_classes + ): + self.imports.add(cls.__module__) + result.append( + ast.parse(f"{node.name} = {cls.__module__}.{cls.__qualname__}").body[0] + ) + continue + original = definitions.get(node.name) + class_source = original.body if isinstance(original, ast.ClassDef) else [] + if isinstance(original, ast.ClassDef): + node.bases = copy.deepcopy(original.bases) + node.body = self.scope( + node.body, cls, class_source, getattr(cls, "__parameters__", ()) + ) + if cls is not None and "__getitem__" in vars(cls) and not hasattr(cls, "__iter__"): + getitem = next( + ( + item + for item in node.body + if isinstance(item, ast.FunctionDef) + and item.name == "__getitem__" + and len(args := [*item.args.posonlyargs, *item.args.args]) == 2 + and accepts_integer_index(args[1].annotation) + ), + None, + ) + if getitem and getitem.returns: + node.body.append( + ast.parse( + "@typing.type_check_only\n" + "def __iter__(self) -> collections.abc.Iterator[" + + ast.unparse(getitem.returns) + + "]: ..." + ).body[0] + ) + result.append(node) + elif isinstance(node, ast.FunctionDef): + if node.name in rebuilt: + continue + obj = getattr(owner, node.name, None) + if isinstance(obj, property): + accessors = [ + item + for item in source + if isinstance(item, ast.FunctionDef) and item.name == node.name + ] + if accessors: + result.extend(self.function(item, parameters) for item in accessors) + rebuilt.add(node.name) + continue + original = definitions.get(node.name) + original = original if isinstance(original, ast.FunctionDef) else None + functions = self.functions(node, obj, original, parameters, owner) + if original or ( + isinstance(obj, types.FunctionType) + and documented_signatures(node.name, inspect.getdoc(obj) or "") + ): + rebuilt.add(node.name) + result.extend(functions) + else: + result.append(node) + return result + + def refine(self, tree: ast.Module) -> str: + tree.body = self.scope(tree.body, self.module, self.source.body) + source_imports = [ + node for node in self.source.body if isinstance(node, (ast.Import, ast.ImportFrom)) + ] + imported_names = bindings(source_imports) + typevars = { + node.targets[0].id: node + for node in self.source.body + if isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and isinstance(getattr(self.module, node.targets[0].id, None), typing.TypeVar) + } + body = [] + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name in imported_names: + # Preserve source re-exports even when __module__ was changed for Sphinx. + continue + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and (node.target.id in imported_names or node.target.id in typevars) + ): + continue + if isinstance(node, (ast.Import, ast.ImportFrom)): + node.names = [ + alias + for alias in node.names + if (alias.asname or alias.name.split(".")[0]) not in imported_names + ] + if not node.names: + continue + body.append(node) + imports = [ + *source_imports, + *[node for node in body if isinstance(node, (ast.Import, ast.ImportFrom))], + ] + imports += [ast.Import(names=[ast.alias(name=name)]) for name in sorted(self.imports)] + # Future imports must come first; deduplicate imports from both sources. + unique_imports = {ast.unparse(node): node for node in imports} + tree.body = [ + unique_imports[key] + for key in sorted( + unique_imports, key=lambda key: (not key.startswith("from __future__"), key) + ) + ] + tree.body += list(typevars.values()) + tree.body += [node for node in body if not isinstance(node, (ast.Import, ast.ImportFrom))] + exports = set(getattr(self.module, "__all__", ())) + for node in tree.body: + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "__all__" + and isinstance(node.value, (ast.List, ast.Tuple)) + ): + exports.update(ast.literal_eval(node.value)) + used = { + node.id + for node in ast.walk(tree) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) + } | exports + cleaned = [] + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)) and not ( + isinstance(node, ast.ImportFrom) and node.module == "__future__" + ): + node.names = [ + alias + for alias in node.names + if (alias.asname or alias.name.split(".")[0]) in used or alias.name == "*" + ] + if not node.names: + continue + cleaned.append(node) + tree.body = cleaned + text = HEADER + ast.unparse(ast.fix_missing_locations(tree)) + "\n" + text = re.sub(r"(\n[ \t]*(?:async )?def [^\n]+:)\n[ \t]+\.\.\.", r"\1 ...", text) + ast.parse(text, feature_version=(3, 10)) + return text + + +def render_stubs( + module_name: str, *, strict: bool = False, public_only: bool = False +) -> dict[Path, str]: + """Generate validated drafts in memory, keyed by their relative module paths.""" + if public_only and module_name != EXTENSION_MODULE: + raise ValueError(f"--public-only requires the {EXTENSION_MODULE} module") + if importlib.util.find_spec("pybind11_stubgen") is None: + raise RuntimeError( + "Missing pybind11-stubgen. Run with: uv run --no-sync " + f"--with pybind11-stubgen==2.5.5 python etc/generate-stubs.py {module_name}" + ) + with tempfile.TemporaryDirectory(prefix="libsemigroups-stubs-") as temp: + command = [sys.executable, "-m", "pybind11_stubgen", module_name, "-o", temp] + if public_only: + native = importlib.import_module(module_name) + public = importlib.import_module("libsemigroups_pybind11") + names = public_native_classes(public, native) + if not names: + raise RuntimeError("No publicly exported native classes found") + command = [ + sys.executable, + "-c", + "import runpy, sys; " + "runpy.run_path(sys.argv[1])['generate_public_raw'](" + "sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] == '1')", + str(Path(__file__).resolve()), + module_name, + json.dumps(sorted(names)), + temp, + "1" if strict else "0", + ] + elif strict: + command.append("--exit-code") + subprocess.run(command, check=True, stdout=sys.stderr) + drafts = {} + for path in sorted(Path(temp).rglob("*.pyi")): + relative = path.relative_to(temp) + parts = list(relative.with_suffix("").parts) + if parts[-1] == "__init__": + parts.pop() + module = importlib.import_module(".".join(parts)) + tree = parse_stub(path.read_text(encoding="utf-8"), module.__name__, strict=strict) + if public_only: + tree.body = [node for node in tree.body if "__all__" not in bindings([node])] + # This deliberately partial native stub leaves omitted objects untyped. + tree.body += ast.parse( + "def __getattr__(name: str) -> _typeshed.Incomplete: ..." + ).body + refiner = Refiner(module, tree, native_classes=public_only) + drafts[relative] = refiner.refine(tree) + if refiner.incomplete: + print( + f"{module.__name__}: unresolved documentation types: " + + ", ".join(sorted(refiner.incomplete)), + file=sys.stderr, + ) + if not drafts: + raise RuntimeError(f"No stubs generated for {module_name}") + return drafts + + +def generate( + module_name: str, + output_dir: Path, + *, + force: bool = False, + strict: bool = False, + public_only: bool = False, +) -> list[Path]: + """Generate and validate drafts before writing any destination files.""" + drafts = { + output_dir / relative: text + for relative, text in render_stubs( + module_name, strict=strict, public_only=public_only + ).items() + } + conflicts = [str(path) for path in drafts if path.exists()] + if conflicts and not force: + raise FileExistsError( + "Existing stubs would be overwritten (use --force): " + ", ".join(conflicts) + ) + for path, text in drafts.items(): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w" if force else "x", encoding="utf-8") as stream: + stream.write(text) + return list(drafts) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("module", help="importable module or package to generate stubs for") + parser.add_argument( + "--public-only", + action=argparse.BooleanOptionalAction, + default=None, + help="select public native classes and dependencies (default for the extension)", + ) + output = parser.add_mutually_exclusive_group() + output.add_argument("-o", "--output-dir", type=Path, help="write stubs under this directory") + output.add_argument( + "--stdout", action="store_true", help="print one stub to standard output (default)" + ) + parser.add_argument("--force", action="store_true", help="replace existing generated files") + parser.add_argument( + "--strict", action="store_true", help="stop on pybind11-stubgen diagnostics" + ) + args = parser.parse_args() + module_name = args.module + public_only = args.public_only + if public_only is None: + public_only = module_name == EXTENSION_MODULE + if public_only and module_name != EXTENSION_MODULE: + parser.error(f"--public-only requires the {EXTENSION_MODULE} module") + if args.output_dir is None and args.force: + parser.error("--force requires --output-dir") + try: + if args.output_dir is None: + with contextlib.redirect_stdout(sys.stderr): + drafts = render_stubs(module_name, strict=args.strict, public_only=public_only) + if len(drafts) != 1: + raise RuntimeError( + f"Standard output requires exactly one stub, but {module_name} " + f"generated {len(drafts)}; " + "target an individual module or use --output-dir" + ) + sys.stdout.write(next(iter(drafts.values()))) + else: + for path in generate( + module_name, + args.output_dir, + force=args.force, + strict=args.strict, + public_only=public_only, + ): + print(path) + except ( + ImportError, + RuntimeError, + OSError, + SyntaxError, + subprocess.CalledProcessError, + ) as error: + print(f"Stub generation failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/libsemigroups_pybind11/action.py b/src/libsemigroups_pybind11/action.py index 238d5359..3d1395d9 100644 --- a/src/libsemigroups_pybind11/action.py +++ b/src/libsemigroups_pybind11/action.py @@ -9,7 +9,7 @@ """ from collections.abc import Iterator -from typing import TypeVar as _TypeVar +from typing import Generic as _Generic, TypeVar as _TypeVar from typing_extensions import Self as _Self @@ -56,7 +56,11 @@ ######################################################################## -class Action(_CxxWrapper): +_Element = _TypeVar("_Element") +_Point = _TypeVar("_Point") + + +class Action(_CxxWrapper, _Generic[_Element, _Point]): __doc__ = _RightActionPPerm1List.__doc__ Element = _TypeVar("Element") @@ -159,13 +163,13 @@ def __init__(self: _Self, *args, generators=None, seeds=None, func=None, side=No for x in seeds: self.add_seed(x) - def __getitem__(self: _Self, i: int) -> Point: + def __getitem__(self: _Self, i: int) -> _Point: return _to_py(_to_cxx(self)[i]) def __len__(self: _Self) -> int: return _to_cxx(self).size() - def __contains__(self: _Self, pt: Point) -> bool: + def __contains__(self: _Self, pt: _Point) -> bool: return self.position(pt) != _UNDEFINED ######################################################################## @@ -173,7 +177,7 @@ def __contains__(self: _Self, pt: Point) -> bool: ######################################################################## @_copydoc(_RightActionPPerm1PPerm1.generators) - def generators(self: _Self) -> Iterator[Element]: + def generators(self: _Self) -> Iterator[_Element]: # pylint: disable=missing-function-docstring return (_to_py(x) for x in _to_cxx(self).generators()) @@ -194,7 +198,7 @@ def generators(self: _Self) -> Iterator[Element]: ######################################################################## -class RightAction(Action): +class RightAction(Action[_Element, _Point]): """Class representing a right action of a semigroup or monoid on a set. This page contains the documentation for the class :any:`RightAction`, @@ -235,7 +239,7 @@ def __init__(self: _Self, *args, generators=None, seeds=None) -> None: ######################################################################## -class LeftAction(Action): +class LeftAction(Action[_Element, _Point]): """Class representing a left action of a semigroup or monoid on a set. This page contains the documentation for the class ``LeftAction``, which diff --git a/src/libsemigroups_pybind11/action.pyi b/src/libsemigroups_pybind11/action.pyi new file mode 100644 index 00000000..90c9be2d --- /dev/null +++ b/src/libsemigroups_pybind11/action.pyi @@ -0,0 +1,119 @@ +# Distributed under the terms of the GPL license version 3. +# The full license is in the file LICENSE, distributed with this software. +# AI assistance: OpenAI Codex drafted and checked this sample stub. + +# Sample types for the public action module. This does not opt the package into +# installed typing support via py.typed; related extension types remain untyped. +# The type parameters track elements and points; supported combinations of +# element, point, function, and side are still checked at runtime by action.py. + +from collections.abc import Callable, Iterator +from datetime import datetime, timedelta +from typing import ClassVar, Generic, TypeVar, overload, type_check_only + +from typing_extensions import Self + +from _libsemigroups_pybind11 import ( + Gabow as _Gabow, + Reporter as _Reporter, + Runner as _Runner, + Undefined as _Undefined, + WordGraph as _WordGraph, + side as _side, +) + +from .adapters import ImageLeftAction as _ImageLeftAction, ImageRightAction as _ImageRightAction + +__all__ = ["Action", "LeftAction", "RightAction"] + +_Element = TypeVar("_Element") +_Point = TypeVar("_Point") + +class Action(Generic[_Element, _Point]): + def __init__( + self, + *, + generators: list[_Element] | None = None, + seeds: list[_Point] | None = None, + func: type[_ImageLeftAction] | type[_ImageRightAction] | None = None, + side: _side | None = None, + ) -> None: ... + def __getitem__(self, i: int) -> _Point: ... + # Python iterates via __getitem__; this method only models that for typing. + @type_check_only + def __iter__(self) -> Iterator[_Point]: ... + def __len__(self) -> int: ... + def __contains__(self, pt: _Point) -> bool: ... + def __repr__(self) -> str: ... + def __eq__(self, that: object) -> bool: ... + def __copy__(self) -> Action[_Element, _Point]: ... + def generators(self) -> Iterator[_Element]: ... + + # Methods attached by copy_cxx_mem_fns accept positional arguments only. + def add_generator(self, gen: _Element, /) -> Self: ... + def add_seed(self, seed: _Point, /) -> Self: ... + def apply(self, pt: _Point, x: _Element, /) -> _Point: ... + @overload + def cache_scc_multipliers(self) -> bool: ... + @overload + def cache_scc_multipliers(self, val: bool, /) -> Self: ... + def copy(self) -> Action[_Element, _Point]: ... + def current_size(self) -> int: ... + def empty(self) -> bool: ... + def init(self) -> Self: ... + def multiplier_from_scc_root(self, pos: int, /) -> _Element: ... + def multiplier_to_scc_root(self, pos: int, /) -> _Element: ... + def number_of_generators(self) -> int: ... + def position(self, pt: _Point, /) -> int | _Undefined: ... + def reserve(self, val: int, /) -> Self: ... + def root_of_scc(self, x: _Point | int, /) -> _Point: ... + def scc(self) -> _Gabow: ... + def size(self) -> int: ... + def word_graph(self) -> _WordGraph: ... + + # Runner methods are forwarded; the Python wrapper is not a Runner subclass. + state: ClassVar[type[_Runner.state]] + def current_state(self) -> _Runner.state: ... + def dead(self) -> bool: ... + def finished(self) -> bool: ... + def kill(self) -> None: ... + def report_why_we_stopped(self) -> None: ... + def run(self) -> None: ... + def run_for(self, t: timedelta, /) -> None: ... + def run_until(self, func: Callable[[], bool], /) -> None: ... + def running(self) -> bool: ... + def running_for(self) -> bool: ... + def running_until(self) -> bool: ... + def started(self) -> bool: ... + def stopped(self) -> bool: ... + def stopped_by_predicate(self) -> bool: ... + def success(self) -> bool: ... + def timed_out(self) -> bool: ... + + # The current bindings return Reporter objects from these setters. + def last_report(self) -> datetime: ... + def report(self) -> bool: ... + @overload + def report_every(self) -> timedelta: ... + @overload + def report_every(self, val: timedelta, /) -> _Reporter: ... + @overload + def report_prefix(self) -> str: ... + @overload + def report_prefix(self, val: str, /) -> _Reporter: ... + def reset_last_report(self) -> _Reporter: ... + def reset_start_time(self) -> _Reporter: ... + def start_time(self) -> datetime: ... + +class RightAction(Action[_Element, _Point]): + def __init__( + self, *, generators: list[_Element] | None = None, seeds: list[_Point] | None = None + ) -> None: ... + +class LeftAction(Action[_Element, _Point]): + def __init__( + self, + *args: object, + generators: list[_Element] | None = None, + seeds: list[_Point] | None = None, + ) -> None: ... diff --git a/src/libsemigroups_pybind11/bmat8.pyi b/src/libsemigroups_pybind11/bmat8.pyi new file mode 100644 index 00000000..bc3b9a74 --- /dev/null +++ b/src/libsemigroups_pybind11/bmat8.pyi @@ -0,0 +1,35 @@ +# Generated by etc/generate-stubs.py using pybind11-stubgen. +# Draft only: review signatures and incomplete types before publishing. +# AI assistance: OpenAI Codex helped implement the generator. + +from __future__ import annotations + +from _libsemigroups_pybind11 import ( + bmat8_col_space_basis as col_space_basis, + bmat8_col_space_size as col_space_size, + bmat8_is_regular_element as is_regular_element, + bmat8_minimum_dim as minimum_dim, + bmat8_number_of_cols as number_of_cols, + bmat8_number_of_rows as number_of_rows, + bmat8_one as one, + bmat8_random as random, + bmat8_row_space_basis as row_space_basis, + bmat8_row_space_size as row_space_size, + bmat8_rows as rows, + bmat8_transpose as transpose, +) + +__all__: list = [ + "col_space_basis", + "col_space_size", + "is_regular_element", + "minimum_dim", + "number_of_cols", + "number_of_rows", + "one", + "random", + "row_space_basis", + "row_space_size", + "rows", + "transpose", +] diff --git a/src/libsemigroups_pybind11/detail/cxx_wrapper.py b/src/libsemigroups_pybind11/detail/cxx_wrapper.py index 31564011..7368c6c4 100644 --- a/src/libsemigroups_pybind11/detail/cxx_wrapper.py +++ b/src/libsemigroups_pybind11/detail/cxx_wrapper.py @@ -203,7 +203,7 @@ def cxx_free_fn_wrapper(*args, **kwargs): return cxx_free_fn_wrapper -def copy_cxx_mem_fns(cxx_class: Pybind11Type, py_class: CxxWrapper) -> None: +def copy_cxx_mem_fns(cxx_class: Pybind11Type, py_class: type[CxxWrapper]) -> None: """Copy all the non-special methods of *cxx_class* into methods of *py_class* that call the cxx member function on the _cxx_obj. """ diff --git a/tests/test_generate_stubs.py b/tests/test_generate_stubs.py new file mode 100644 index 00000000..d5eaf800 --- /dev/null +++ b/tests/test_generate_stubs.py @@ -0,0 +1,552 @@ +# Distributed under the terms of the GPL license version 3. +# The full license is in the file LICENSE, distributed with this software. +# AI assistance: OpenAI Codex helped write these generator checks. + +"""Check stub generation, including signatures attached to wrappers at runtime.""" + +# pylint: disable=missing-function-docstring + +import ast +import importlib.util +import subprocess +import sys +import types +from pathlib import Path + +import pytest + + +@pytest.fixture(name="stub_generator", scope="module") +def fixture_stub_generator(): + path = Path(__file__).resolve().parents[1] / "etc" / "generate-stubs.py" + spec = importlib.util.spec_from_file_location("generate_stubs", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(name="generated_action", scope="module") +def fixture_generated_action(stub_generator, tmp_path_factory): + pytest.importorskip("pybind11_stubgen") + output = tmp_path_factory.mktemp("action_stubs") + original = Path(__file__).resolve().parents[1] / "src/libsemigroups_pybind11/action.pyi" + before = original.read_bytes() if original.exists() else None + paths = stub_generator.generate("libsemigroups_pybind11.action", output, strict=True) + assert paths == [output / "libsemigroups_pybind11/action.pyi"] + assert (original.read_bytes() if original.exists() else None) == before + return paths[0] + + +def test_documented_signature_precedence(stub_generator): + doc = """size(self: SomeCppType, unused: int) -> str + :sig=(self: Action) -> int: + :sig=(self: Action) -> int: + """ + signatures = stub_generator.documented_signatures("size", doc) + assert len(signatures) == 1 + assert [arg.arg for arg in signatures[0].args.args] == ["self"] + assert ast.unparse(signatures[0].returns) == "int" + + +def test_documented_overloads(stub_generator): + signatures = stub_generator.documented_signatures( + "setting", + "setting(*args, **kwargs)\nOverloaded function.\n\n" + "1. setting(self: Cpp) -> bool\n" + "2. setting(self: Cpp, value: bool) -> Cpp\n", + ) + assert len(signatures) == 2 + assert [len(sig.args.args) for sig in signatures] == [1, 2] + + +def test_signature_defaults_are_not_executed(stub_generator): + signatures = stub_generator.documented_signatures( + "f", ":sig=(x: str = this_function_does_not_exist()) -> None:" + ) + assert len(signatures) == 1 + assert isinstance(signatures[0].args.defaults[0], ast.Call) + + +def test_invalid_native_base(stub_generator, capsys): + text = "class BoundClass(detail::Base):\n def size(self) -> int: ...\n" + tree = stub_generator.parse_stub(text, "example", strict=False) + cls = next(node for node in tree.body if isinstance(node, ast.ClassDef)) + assert ast.unparse(cls.bases[0]) == "typing.Any" + assert cls.body[0].name == "size" + assert "detail::Base" in capsys.readouterr().err + with pytest.raises(SyntaxError): + stub_generator.parse_stub(text, "example", strict=True) + + +def test_unrelated_syntax_errors_are_not_suppressed(stub_generator): + with pytest.raises(SyntaxError): + stub_generator.parse_stub("def broken(: ...", "example", strict=False) + + +def test_native_keyword_parameter(stub_generator): + tree = stub_generator.parse_stub( + "def f(graph: object, from: int, path: list[int]) -> int: ...", "example", strict=False + ) + args = tree.body[0].args + assert [arg.arg for arg in args.posonlyargs] == ["graph", "from_"] + assert [arg.arg for arg in args.args] == ["path"] + ast.parse(ast.unparse(tree)) + tree = stub_generator.parse_stub("def f(from: int, /): ...", "example", strict=False) + assert tree.body[0].args.posonlyargs[0].arg == "from_" + with pytest.raises(SyntaxError): + stub_generator.parse_stub("def f(*, from: int): ...", "example", strict=False) + + +def test_native_bmat8(stub_generator, tmp_path): + pytest.importorskip("pybind11_stubgen") + paths = stub_generator.generate("_libsemigroups_pybind11", tmp_path) + path = tmp_path / "_libsemigroups_pybind11.pyi" + assert path in paths + tree = ast.parse(path.read_text(encoding="utf-8"), feature_version=(3, 10)) + cls = next( + node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "BMat8" + ) + methods = [node for node in cls.body if isinstance(node, ast.FunctionDef)] + assert {node.name for node in methods} >= {"__init__", "__getitem__", "__mul__", "to_int"} + assert ast.unparse(next(node.returns for node in methods if node.name == "to_int")) == "int" + assert sum(node.name == "__init__" for node in methods) > 1 + + +def test_public_class_selection_uses_exports_and_identity(stub_generator): + native = types.ModuleType("native") + public = types.ModuleType("public") + helper = types.ModuleType("public.helper") + native.RealName = type("RenamedForDocs", (), {}) + native.Hidden = type("Hidden", (), {}) + public.Alias = native.RealName + public.AccidentallyImported = native.Hidden + public.Wrapper = type("RealName", (), {}) + public.helper = helper + public.__all__ = ["Alias", "Wrapper", "helper"] + helper.__all__ = ["Root"] + helper.Root = public # A cycle in exported modules must terminate. + assert stub_generator.public_native_classes(public, native) == {"RealName"} + helper.AlsoPublic = native.Hidden + helper.__all__.append("AlsoPublic") + assert stub_generator.public_native_classes(public, native) == {"RealName", "Hidden"} + + +def test_native_typing_dependencies(stub_generator): + native = types.ModuleType("native") + for name in ("Base", "Result", "Nested", "Unused", "Hidden"): + setattr(native, name, type(name, (), {})) + native.DEFAULT = 0 + tree = ast.parse( + "class Public(Base):\n" + ' """Hidden is mentioned only in documentation."""\n' + ' def f(self, x: list["Result"] = DEFAULT) -> native.Nested.Inner: ...\n' + ) + assert stub_generator.native_dependencies(tree, native) == { + "Base", + "Result", + "Nested", + "DEFAULT", + } + + +def test_public_only_rejects_wrapper_target(stub_generator, tmp_path): + with pytest.raises(ValueError, match="--public-only requires"): + stub_generator.generate("libsemigroups_pybind11.action", tmp_path, public_only=True) + assert not any(tmp_path.iterdir()) + + +def test_public_native_stub(stub_generator, tmp_path, capfd): + pytest.importorskip("pybind11_stubgen") + native = importlib.import_module("_libsemigroups_pybind11") + public = importlib.import_module("libsemigroups_pybind11") + + before = {name: value for name, value in vars(native).items() if isinstance(value, type)} + paths = stub_generator.generate("_libsemigroups_pybind11", tmp_path, public_only=True) + assert paths == [tmp_path / "_libsemigroups_pybind11.pyi"] + text = paths[0].read_text(encoding="utf-8") + tree = ast.parse(text, feature_version=(3, 10)) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + assert stub_generator.public_native_classes(public, native) <= classes.keys() + assert {"BMat8", "Runner", "forest_PathsFromRoots"} <= classes.keys() + assert { + "AlphabetWord", + "AlphabetString", + "RightActionPPerm1List", + "FroidurePinBMat8", + }.isdisjoint(classes) + # An unexported base is kept because public PathsFromRoots inherits from it. + assert "forest_PathsFromToRootsCommon" in classes + assert stub_generator.native_dependencies(tree, native) <= stub_generator.bindings(tree.body) + assert "__all__" not in stub_generator.bindings(tree.body) + assert "def __getattr__(name: str) -> _typeshed.Incomplete:" in text + iterator = next( + node + for node in classes["BMat8"].body + if isinstance(node, ast.FunctionDef) and node.name == "__iter__" + ) + assert ast.unparse(iterator.returns) == "collections.abc.Iterator[list[bool]]" + assert all(getattr(native, name) is value for name, value in before.items()) + assert "AlphabetWord" not in capfd.readouterr().err + stub_generator.generate("_libsemigroups_pybind11", tmp_path, public_only=True, force=True) + assert paths[0].read_text(encoding="utf-8") == text + + +def test_public_bmat8_with_mypy(stub_generator, tmp_path): + pytest.importorskip("pybind11_stubgen") + pytest.importorskip("mypy") + stub_generator.generate("_libsemigroups_pybind11", tmp_path, public_only=True) + # Isolate the selected BMat8 section: other public classes still have malformed + # native signatures. The selection/dependency test checks the complete output. + stub = tmp_path / "_libsemigroups_pybind11.pyi" + tree = ast.parse(stub.read_text(encoding="utf-8")) + tree.body = [ + node + for node in tree.body + if isinstance(node, (ast.Import, ast.ImportFrom)) + or isinstance(node, ast.ClassDef) + and node.name == "BMat8" + ] + stub.write_text(ast.unparse(tree), encoding="utf-8") + package = tmp_path / "libsemigroups_pybind11" + package.mkdir() + (package / "__init__.pyi").write_text( + "from _libsemigroups_pybind11 import BMat8 as BMat8\n", encoding="utf-8" + ) + client = tmp_path / "client.py" + client.write_text( + "from typing_extensions import assert_type\n" + "from libsemigroups_pybind11 import BMat8\n" + "matrix = BMat8([[True, False], [False, True]])\n" + "assert_type(matrix.to_int(), int)\n" + "assert_type(matrix[0, 1], bool)\n" + "assert_type(matrix[0], list[bool])\n" + "for row in matrix:\n" + " assert_type(row, list[bool])\n" + "matrix.nonexistent() # type: ignore[attr-defined]\n" + "BMat8('bad') # type: ignore[arg-type]\n", + encoding="utf-8", + ) + # Check client inference while unrelated draft-native annotations remain incomplete. + config = tmp_path / "mypy.ini" + config.write_text( + "[mypy]\nfollow_imports = silent\nfollow_imports_for_stubs = True\n", encoding="utf-8" + ) + result = subprocess.run( + [ + sys.executable, + "-m", + "mypy", + "--python-version=3.10", + "--ignore-missing-imports", + "--config-file", + str(config), + "--warn-unused-ignores", + "--cache-dir", + str(tmp_path / "cache"), + str(client), + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_action_generic_signatures(generated_action): + text = generated_action.read_text(encoding="utf-8") + tree = ast.parse(text, feature_version=(3, 10)) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + action = classes["Action"] + methods = [node for node in action.body if isinstance(node, ast.FunctionDef)] + by_name = {node.name: node for node in methods} + assert ast.unparse(action.bases[-1]) == "_Generic[_Element, _Point]" + assert ast.unparse(by_name["__getitem__"].returns) == "_Point" + assert ast.unparse(by_name["generators"].returns) == "Iterator[_Element]" + assert ast.unparse(by_name["add_seed"].returns) == "typing_extensions.Self" + assert ast.unparse(by_name["__iter__"].returns) == "collections.abc.Iterator[_Point]" + assert len(by_name["add_seed"].args.posonlyargs) == 2 + assert not by_name["add_seed"].args.args + assert ast.unparse(classes["RightAction"].bases[0]) == "Action[_Element, _Point]" + assert ast.unparse(classes["LeftAction"].bases[0]) == "Action[_Element, _Point]" + constructor = {arg.arg: arg for arg in by_name["__init__"].args.kwonlyargs} + assert ast.unparse(constructor["seeds"].annotation) == "list[_Point] | None" + assert "type[libsemigroups_pybind11.adapters.ImageRightAction]" in ast.unparse( + constructor["func"].annotation + ) + overloads = [node for node in methods if node.name == "cache_scc_multipliers"] + assert [ast.unparse(node.returns) for node in overloads] == ["bool", "typing_extensions.Self"] + assert all(ast.unparse(node.decorator_list[0]) == "typing.overload" for node in overloads) + assert "_all_wrapped_cxx_types" not in text + assert ":sig=" not in text + + +def test_regeneration_is_deterministic(stub_generator, generated_action): + before = generated_action.read_bytes() + stub_generator.generate( + "libsemigroups_pybind11.action", generated_action.parents[1], force=True, strict=True + ) + assert generated_action.read_bytes() == before + + +def test_overwrite_requires_force(stub_generator, generated_action, tmp_path): + destination = tmp_path / "libsemigroups_pybind11/action.pyi" + destination.parent.mkdir() + destination.write_text("# hand edited stub\n", encoding="utf-8") + with pytest.raises(FileExistsError, match="--force"): + stub_generator.generate("libsemigroups_pybind11.action", tmp_path) + assert destination.read_text(encoding="utf-8") == "# hand edited stub\n" + assert generated_action.exists() + + +def test_another_wrapper_module(stub_generator, tmp_path): + pytest.importorskip("pybind11_stubgen") + paths = stub_generator.generate("libsemigroups_pybind11.sims", tmp_path, strict=True) + tree = ast.parse(paths[0].read_text(encoding="utf-8"), feature_version=(3, 10)) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + assert {"Sims1", "Sims2", "RepOrc"} <= classes.keys() + assert any( + isinstance(node, ast.FunctionDef) and node.name == "find_if" + for node in classes["Sims1"].body + ) + + +@pytest.mark.parametrize("name", ["presentation", "matrix", "forest", "detail.dot"]) +def test_properties_enums_and_reexports(stub_generator, tmp_path, name): + pytest.importorskip("pybind11_stubgen") + paths = stub_generator.generate(f"libsemigroups_pybind11.{name}", tmp_path, strict=True) + path = next(path for path in paths if path.name != "examples.pyi") + tree = ast.parse(path.read_text(encoding="utf-8")) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + if name == "presentation": + accessors = [ + node + for node in classes["Presentation"].body + if isinstance(node, ast.FunctionDef) and node.name == "rules" + ] + assert [ast.unparse(node.decorator_list[0]) for node in accessors] == [ + "property", + "rules.setter", + ] + assert [len(node.args.args) for node in accessors] == [1, 2] + elif name == "matrix": + members = [node for node in classes["MatrixKind"].body if isinstance(node, ast.Assign)] + assert {node.targets[0].id for node in members} >= {"Boolean", "Integer", "MaxPlus"} + else: + assert not classes + imports = [node for node in tree.body if isinstance(node, ast.ImportFrom)] + assert any(node.module == "_libsemigroups_pybind11" for node in imports) + if name == "detail.dot": + assert any(alias.name == "Dot" for node in imports for alias in node.names) + + +def test_generated_stub_with_mypy(generated_action, tmp_path): + pytest.importorskip("mypy") + # Isolate this check from the hand-written Action stub in the source tree. + package = tmp_path / "libsemigroups_pybind11" + package.mkdir() + (package / "__init__.pyi").write_text("", encoding="utf-8") + stub = package / "action.pyi" + stub.write_bytes(generated_action.read_bytes()) + client = tmp_path / "client.py" + client.write_text( + "from typing_extensions import assert_type\n" + "from libsemigroups_pybind11.action import RightAction\n" + "def check(action: RightAction[object, list[int]]) -> None:\n" + " assert_type(action[0], list[int])\n" + " assert_type(action.cache_scc_multipliers(), bool)\n" + " assert_type(action.cache_scc_multipliers(True), RightAction[object, list[int]])\n" + " for point in action:\n" + " assert_type(point, list[int])\n" + " action.add_seed('bad') # type: ignore[arg-type]\n" + " action.reserve(val=10) # type: ignore[call-arg]\n", + encoding="utf-8", + ) + result = subprocess.run( + [ + sys.executable, + "-m", + "mypy", + "--python-version", + "3.10", + "--ignore-missing-imports", + "--follow-imports=silent", + "--warn-unused-ignores", + "--cache-dir", + str(tmp_path / "cache"), + str(stub), + str(client), + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_missing_generator_dependency(stub_generator, monkeypatch, tmp_path): + monkeypatch.setattr(stub_generator.importlib.util, "find_spec", lambda _: None) + with pytest.raises(RuntimeError, match="--with pybind11-stubgen"): + stub_generator.generate("libsemigroups_pybind11.action", tmp_path) + assert not any(tmp_path.iterdir()) + + +@pytest.mark.parametrize( + "options, class_name", + [ + (["libsemigroups_pybind11.action"], "Action"), + (["_libsemigroups_pybind11"], "BMat8"), + (["_libsemigroups_pybind11", "--public-only"], "BMat8"), + ], +) +@pytest.mark.parametrize("stdout_options", [[], ["--stdout"]]) +def test_stdout_cli(stub_generator, tmp_path, options, class_name, stdout_options): + pytest.importorskip("pybind11_stubgen") + sentinel = tmp_path / "keep.pyi" + sentinel.write_text("# existing stub\n", encoding="utf-8") + result = subprocess.run( + [sys.executable, stub_generator.__file__, *options, *stdout_options], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.startswith(stub_generator.HEADER) + tree = ast.parse(result.stdout, feature_version=(3, 10)) + assert any(isinstance(node, ast.ClassDef) and node.name == class_name for node in tree.body) + assert "Selected " not in result.stdout + if options[0] == "_libsemigroups_pybind11": + assert "Selected " in result.stderr + assert not any( + isinstance(node, ast.ClassDef) and node.name == "AlphabetWord" for node in tree.body + ) + assert list(tmp_path.iterdir()) == [sentinel] + assert sentinel.read_text(encoding="utf-8") == "# existing stub\n" + + +def test_explicit_output_directory_cli(stub_generator, generated_action, tmp_path): + output = tmp_path / "stubs" + result = subprocess.run( + [ + sys.executable, + stub_generator.__file__, + "libsemigroups_pybind11.action", + "--output-dir", + str(output), + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + stub = output / "libsemigroups_pybind11/action.pyi" + assert stub.read_bytes() == generated_action.read_bytes() + assert result.stdout == f"{stub}\n" + + +def test_stdout_rejects_multiple_stubs(stub_generator, monkeypatch, capsys, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", "package"]) + monkeypatch.setattr( + stub_generator, + "render_stubs", + lambda *_args, **_kwargs: { + Path("a.pyi"): "class A: ...\n", + Path("b.pyi"): "class B: ...\n", + }, + ) + assert stub_generator.main() == 1 + result = capsys.readouterr() + assert result.out == "" + assert "Standard output requires exactly one stub" in result.err + assert not any(tmp_path.iterdir()) + + +def test_stdout_failure_keeps_output_empty(stub_generator, monkeypatch, capsys): + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", "example", "--strict"]) + + def fail(*_args, **kwargs): + assert kwargs["strict"] + print("Import diagnostic") + raise RuntimeError("generation failed") + + monkeypatch.setattr(stub_generator, "render_stubs", fail) + assert stub_generator.main() == 1 + result = capsys.readouterr() + assert result.out == "" + assert "Import diagnostic" in result.err + assert "generation failed" in result.err + + +@pytest.mark.parametrize( + "options", [["--stdout", "--output-dir", "stubs"], ["--stdout", "--force"], ["--force"]] +) +def test_stdout_rejects_file_options(stub_generator, monkeypatch, capsys, options): + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", "example", *options]) + with pytest.raises(SystemExit) as error: + stub_generator.main() + assert error.value.code == 2 + assert capsys.readouterr().out == "" + + +@pytest.mark.parametrize("options", [[], ["--public-only"]]) +def test_module_argument_is_required(stub_generator, monkeypatch, capsys, options): + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", *options]) + with pytest.raises(SystemExit) as error: + stub_generator.main() + assert error.value.code == 2 + result = capsys.readouterr() + assert result.out == "" + assert "the following arguments are required: module" in result.err + + +@pytest.mark.parametrize( + "options, public_only", + [ + (["_libsemigroups_pybind11"], True), + (["_libsemigroups_pybind11", "--public-only"], True), + (["_libsemigroups_pybind11", "--no-public-only"], False), + (["libsemigroups_pybind11.action"], False), + ], +) +def test_cli_public_only_selection(stub_generator, monkeypatch, capsys, options, public_only): + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", *options]) + + def render(module_name, **kwargs): + assert module_name == options[0] + assert kwargs["public_only"] is public_only + return {Path("example.pyi"): "class Example: ...\n"} + + monkeypatch.setattr(stub_generator, "render_stubs", render) + assert stub_generator.main() == 0 + assert capsys.readouterr().out == "class Example: ...\n" + + +@pytest.mark.parametrize("public_only", [False, True]) +def test_failed_generation_preserves_destination( + stub_generator, monkeypatch, tmp_path, public_only +): + monkeypatch.setattr(stub_generator.importlib.util, "find_spec", lambda _: object()) + destination = tmp_path / "action.pyi" + destination.write_text("# keep this stub\n", encoding="utf-8") + + def fail(command, **_kwargs): + if public_only: + assert command[-1] == "1" # Strict mode is passed to the isolated backend. + else: + assert "--exit-code" in command + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr(stub_generator.subprocess, "run", fail) + with pytest.raises(subprocess.CalledProcessError): + stub_generator.generate( + "_libsemigroups_pybind11" if public_only else "libsemigroups_pybind11.action", + tmp_path, + strict=True, + public_only=public_only, + ) + assert list(tmp_path.iterdir()) == [destination] + assert destination.read_text(encoding="utf-8") == "# keep this stub\n"