diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 401166c7..548bd0dc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -91,6 +91,89 @@ the output. If you followed the guidance output by the script, the class will now be accessible in ``_libsemigroups_pybind11``. You must now check the contents of that file adheres to the styles set out in this guide. +Generating type stubs +--------------------- + +``etc/generate-stubs.py`` generates draft ``.pyi`` files from the importable +package. Use an environment containing the built extension and its Python +dependencies. The module argument is required, and the stub is printed to +standard output by default. For example, to generate the ``action`` module stub:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py libsemigroups_pybind11.action + +Use ``--output-dir`` to write files under a directory. Package targets include +discovered submodules and require directory output when producing multiple stubs:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py libsemigroups_pybind11.action --output-dir _generate/stubs + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py libsemigroups_pybind11 --output-dir _generate/stubs + +Diagnostics and generation summaries go to standard error, leaving standard +output suitable for redirection or piping. A failed generation emits no stub +text. Targets producing multiple stubs are rejected when using standard output; +select an individual module or use ``--output-dir`` instead. ``--stdout`` can +explicitly select the default behavior and cannot be combined with +``--output-dir``. ``--force`` requires ``--output-dir``. + +Classes defined directly in the compiled extension, such as ``BMat8``, require +the extension module as the target. This target defaults to ``--public-only``, +which generates a smaller stub covering the public native classes:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py _libsemigroups_pybind11 + +Add ``--output-dir _generate/stubs`` to save this as +``_generate/stubs/_libsemigroups_pybind11.pyi``. Selection follows +``__all__`` in ``libsemigroups_pybind11`` and its publicly exported submodules +(or non-private names for modules without ``__all__``). Classes are matched by +object identity, so native classes exported under aliases are included. The +selection adapts to newly exported classes and the active build's optional +features, without maintaining a separate list of class names. + +Only selected classes and their typing dependencies are inspected. Required +base classes and types referenced in their signatures are included automatically +and reported. Python wrappers such as ``Alphabet`` still need their own module +stubs; their internal specializations, such as ``AlphabetWord``, are omitted +unless a retained signature requires them. + +This mode does not generate standalone helper functions or constants except +where needed as dependencies. It marks the extension stub as partial with a +module-level ``__getattr__`` returning ``_typeshed.Incomplete``, so omitted names +remain untyped. Use the full-extension mode when those declarations are needed:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py _libsemigroups_pybind11 --no-public-only --output-dir _generate/stubs + +Both extension modes use the same filename with directory output; ``--force`` +is required to replace a previous output. Other module targets retain their +normal discovery behavior; ``--public-only`` requires the extension target. +The ``libsemigroups_pybind11.bmat8`` target generates the helper module instead. +Invalid C++ base-class names are reported +and replaced by ``typing.Any`` in drafts. Parameters named after Python keywords +are renamed and marked positional-only; ``--strict`` rejects these cases. + +Generation happens in a temporary directory first, so failures during discovery +or parsing do not overwrite existing stubs. To run without uv, install +``pybind11-stubgen==2.5.5`` in your development environment and invoke the script +with that environment's Python interpreter. + +Shell redirection can also save a single stub to a chosen filename:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py libsemigroups_pybind11.action > action.pyi + +The generator combines pybind11-stubgen's runtime discovery with source +annotations, generic bases, and the project's ``:sig=...:`` docstrings. It also +accounts for the positional arguments accepted by dynamically copied methods. +The output needs review: missing annotations, unsupported type spellings, and +dynamic dispatch can still produce incomplete or inaccurate types. Inherited +untyped wrappers also limit static checking. Optional build features, such as +HPCombi, can cause discovery diagnostics. Use ``--strict`` to stop on those +diagnostics instead of emitting drafts. + +No ``py.typed`` marker is added, and existing source stubs are not replaced by +the default command. Run the generator tests with:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 --with mypy==2.3.1 pytest tests/test_generate_stubs.py + +AI assistance: OpenAI Codex helped implement, document, and test this generator. + The bindings ------------ diff --git a/etc/generate-stubs.py b/etc/generate-stubs.py new file mode 100755 index 00000000..35103f44 --- /dev/null +++ b/etc/generate-stubs.py @@ -0,0 +1,802 @@ +#!/usr/bin/env python3 +# Distributed under the terms of the GPL license version 3. +# The full license is in the file LICENSE, distributed with this software. +# AI assistance: OpenAI Codex helped implement and test this generator. + +r"""Generate draft stubs from the built package, source annotations, and docstrings. + +Run in an environment containing the built extension, for example:: + + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py \ + libsemigroups_pybind11.action + uv run --no-sync --with pybind11-stubgen==2.5.5 python etc/generate-stubs.py \ + libsemigroups_pybind11.sims --output-dir _generate/stubs + +The module argument is required. A single stub is printed to stdout by default, +with diagnostics on standard error. Use --output-dir to write files instead. +Package targets include the submodules discovered by pybind11-stubgen and need +--output-dir when generating multiple stubs. Existing files require --force. +Extension targets default to the classes exported by the public package and its +public submodules, together with their typing dependencies. Use --no-public-only +to generate the full extension stub. +Use --strict to stop on pybind11-stubgen diagnostics instead of emitting drafts. +Output is a draft: missing annotations, dynamic dispatch, and C++ conversions +still require review. This script does not install dependencies or add py.typed. +""" + +from __future__ import annotations + +import argparse +import ast +import builtins +import contextlib +import copy +import enum +import importlib +import importlib.util +import inspect +import json +import keyword +import re +import subprocess +import sys +import tempfile +import types +import typing +from pathlib import Path + +EXTENSION_MODULE = "_libsemigroups_pybind11" + +HEADER = """# Generated by etc/generate-stubs.py using pybind11-stubgen. +# Draft only: review signatures and incomplete types before publishing. +# AI assistance: OpenAI Codex helped implement the generator. + +""" + + +def expression(text: str) -> ast.expr: + """Parse a type expression without executing it.""" + return ast.parse(text, mode="eval").body + + +def parse_stub(text: str, module_name: str, *, strict: bool) -> ast.Module: + """Handle native names that cannot be written in Python stub syntax.""" + lines = text.splitlines(keepends=True) + renamed = {} + while True: + try: + tree = ast.parse("".join(lines)) + except SyntaxError as error: + if strict or error.lineno is None: + raise + index = error.lineno - 1 + match = re.fullmatch(r"(\s*class \w+)\((.+)\):\s*", lines[index]) + if match is None: + # A binding can accept a keyword named "from", but Python syntax cannot. + line = lines[index] + offset = (error.offset or 1) - 1 + name = re.match(r"(\w+)(?=\s*[:=,)])", line[offset:]) + if ( + line.lstrip().startswith("def ") + and line[:offset].rstrip().endswith(("(", ",")) + and name + and keyword.iskeyword(name[1]) + ): + replacement = name[1] + "_" + lines[index] = line[:offset] + replacement + line[offset + len(name[1]) :] + renamed.setdefault(error.lineno, set()).add(replacement) + print( + f"{module_name}: representing keyword parameter {name[1]!r} " + f"as positional-only {replacement!r}", + file=sys.stderr, + ) + continue + raise + try: + expression(match[2]) + except SyntaxError: + # Some extension types have names such as detail::CongruenceCommon. + # Any records the unknown base without inventing a Python class. + print( + f"{module_name}: replacing invalid base {match[2]!r} with typing.Any", + file=sys.stderr, + ) + lines[index] = f"{match[1]}(typing.Any):\n" + else: + raise error + else: + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.lineno in renamed: + names = renamed[node.lineno] + if any(arg.arg in names for arg in node.args.kwonlyargs): + raise SyntaxError( + "Cannot represent a keyword-only parameter named after a keyword" + ) + count = max( + (i + 1 for i, arg in enumerate(node.args.args) if arg.arg in names), + default=0, + ) + node.args.posonlyargs += node.args.args[:count] + node.args.args = node.args.args[count:] + return tree + + +def documented_signatures(name: str, doc: str) -> list[ast.FunctionDef]: + """Prefer project signature overrides, then pybind11 signature lines.""" + overrides = re.findall(r"^\s*:sig=[^(\n]*(\(.*):\s*$", doc, re.MULTILINE) + signatures = overrides or re.findall( + rf"^\s*(?:\d+\. )?{re.escape(name)}(\(.*)$", doc, re.MULTILINE + ) + result = [] + seen = set() + for signature in signatures: + try: + node = ast.parse(f"def {name}{signature}: ...").body[0] + except SyntaxError: + continue # C++ spellings and enum defaults need manual review. + key = ast.dump(node) + if key not in seen and not (node.args.vararg and node.args.kwarg and not node.args.args): + result.append(typing.cast(ast.FunctionDef, node)) + seen.add(key) + return result + + +def bindings(nodes: list[ast.stmt]) -> set[str]: + """Collect names bound at module scope, including imported aliases.""" + names = set() + for node in nodes: + if isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + names.add(alias.asname or alias.name.split(".")[0]) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + names.add(node.target.id) + elif isinstance(node, ast.Assign): + names.update(target.id for target in node.targets if isinstance(target, ast.Name)) + return names + + +def public_native_classes(public: types.ModuleType, native: types.ModuleType) -> set[str]: + """Find native classes by identity, following the public modules' exports.""" + exported_ids = set() + pending = [public] + seen = set() + while pending: + module = pending.pop() + if id(module) in seen: + continue + seen.add(id(module)) + names = getattr( + module, "__all__", [name for name in vars(module) if not name.startswith("_")] + ) + for name in names: + value = getattr(module, name, None) + if inspect.isclass(value): + exported_ids.add(id(value)) + elif inspect.ismodule(value) and value.__name__.startswith(public.__name__ + "."): + pending.append(value) + return { + name + for name, value in vars(native).items() + if inspect.isclass(value) and id(value) in exported_ids + } + + +def native_dependencies(tree: ast.Module, native: types.ModuleType) -> set[str]: + """Find native names used by bases, annotations, aliases, and default values.""" + names = set() + pending = [tree] + for node in ast.walk(tree): + annotation = None + if isinstance(node, (ast.arg, ast.AnnAssign)): + annotation = node.annotation + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + annotation = node.returns + if annotation is not None: + # Forward references inside annotations can also name native types. + for part in ast.walk(annotation): + if isinstance(part, ast.Constant) and isinstance(part.value, str): + with contextlib.suppress(SyntaxError): + pending.append(expression(part.value)) + for root in pending: + for node in ast.walk(root): + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + names.add(node.id) + elif isinstance(node, ast.Attribute): + parts = ast.unparse(node).split(".") + if parts[0] == native.__name__ and len(parts) > 1: + names.add(parts[1]) + elif isinstance(node, ast.ImportFrom) and node.module == native.__name__: + names.update(alias.name for alias in node.names) + return { + name + for name in names + if name in vars(native) and not name.startswith("__") and name.isidentifier() + } + + +def generate_public_raw(module_name: str, names_json: str, output_dir: str, strict: bool) -> None: + """Run in a subprocess so inspection cannot change the caller's imported modules.""" + import pybind11_stubgen + + native = importlib.import_module(module_name) + selected = set(json.loads(names_json)) + roots = selected.copy() + # dir() controls discovery; getattr() still resolves every real native name. + # Importing the public package here would rename some native classes for Sphinx. + view = types.ModuleType(module_name) + vars(view).update(vars(native)) + view.__dir__ = lambda: sorted(selected) + sys.modules[module_name] = view + try: + while True: + command = [module_name, "-o", output_dir] + if strict: + command.append("--exit-code") + pybind11_stubgen.main(command) + path = Path(output_dir) / f"{module_name}.pyi" + tree = parse_stub(path.read_text(encoding="utf-8"), module_name, strict=strict) + dependencies = native_dependencies(tree, native) - selected + if not dependencies: + break + selected.update(dependencies) + finally: + sys.modules[module_name] = native + print( + f"Selected {len(roots)} public native classes and {len(selected - roots)} supporting names." + ) + if selected - roots: + print("Supporting names: " + ", ".join(sorted(selected - roots))) + + +def accepts_integer_index(annotation: ast.expr | None) -> bool: + """Identify integer overloads used by Python's implicit sequence iteration.""" + if isinstance(annotation, ast.BinOp) and isinstance(annotation.op, ast.BitOr): + return accepts_integer_index(annotation.left) or accepts_integer_index(annotation.right) + return isinstance(annotation, (ast.Name, ast.Attribute)) and ast.unparse(annotation).split(".")[ + -1 + ] in {"int", "SupportsInt", "SupportsIndex"} + + +class TypeNames(ast.NodeTransformer): + """Resolve documentation names in the public package and the owning class.""" + + def __init__(self, refiner: Refiner, parameters: tuple, *, specialize: bool = True): + self.refiner = refiner + self.parameters = {param.__name__.lstrip("_"): param.__name__ for param in parameters} + self.specialize = specialize + + def visit_Subscript(self, node: ast.Subscript) -> ast.Subscript: + previous = self.specialize + self.specialize = False + node.value = self.visit(node.value) + self.specialize = previous + node.slice = self.visit(node.slice) + return node + + def visit_Attribute(self, node: ast.Attribute) -> ast.expr: + if ast.unparse(node) == "typing.Self": + return expression("typing_extensions.Self") + parts = ast.unparse(node).split(".") + value = getattr(self.refiner.module, parts[0], None) + if inspect.isclass(value): + for part in parts[1:]: + value = getattr(value, part, None) + if inspect.isclass(value) and value.__module__ != self.refiner.module.__name__: + self.refiner.imports.add(value.__module__) + return expression(f"{value.__module__}.{value.__qualname__}") + # Qualified names already supplied by pybind11-stubgen have imports. + return node + + def visit_Name(self, node: ast.Name) -> ast.expr: + name = node.id + if name in self.parameters: + return ast.Name(id=self.parameters[name], ctx=ast.Load()) + value = getattr(self.refiner.module, name, None) + if inspect.isclass(value) and self.specialize and getattr(value, "__parameters__", ()): + params = ", ".join(param.__name__ for param in value.__parameters__) + return expression(f"{name}[{params}]") + if name in self.refiner.names or hasattr(builtins, name): + return node + if name == "Self": + return expression("typing_extensions.Self") + if hasattr(typing, name): + return expression(f"typing.{name}") + value = getattr(self.refiner.public, name, None) + if inspect.isclass(value): + self.refiner.imports.add(value.__module__) + return expression(f"{value.__module__}.{value.__qualname__}") + self.refiner.incomplete.add(name) + return expression("_typeshed.Incomplete") + + +class Refiner: + """Combine runtime discovery with information lost by generic stub tools.""" + + def __init__(self, module: types.ModuleType, tree: ast.Module, *, native_classes: bool = False): + self.module = module + self.native_classes = native_classes + self.public = importlib.import_module("libsemigroups_pybind11") + try: + self.source = ast.parse(inspect.getsource(module)) + except (OSError, TypeError): + self.source = ast.Module(body=[], type_ignores=[]) + self.names = bindings(tree.body) | bindings(self.source.body) + self.imports = {"collections.abc", "datetime", "typing", "typing_extensions", "_typeshed"} + if native_classes: + self.imports.add(module.__name__) + self.incomplete: set[str] = set() + + def annotation(self, node: ast.expr | None, parameters: tuple) -> ast.expr | None: + if node is None: + return None + return TypeNames(self, parameters).visit(copy.deepcopy(node)) + + def function(self, node: ast.FunctionDef, parameters: tuple) -> ast.FunctionDef: + node = copy.deepcopy(node) + args = node.args + for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs, args.vararg, args.kwarg]: + if arg is not None: + arg.annotation = self.annotation(arg.annotation, parameters) + if arg.arg in {"self", "cls"}: + arg.annotation = None + node.returns = self.annotation(node.returns, parameters) + node.decorator_list = [ + dec + for dec in node.decorator_list + if ast.unparse(dec).split(".")[-1] + in {"overload", "staticmethod", "classmethod", "property", "setter", "abstractmethod"} + ] + node.body = [ast.Expr(value=ast.Constant(value=Ellipsis))] + return node + + def dispatch_annotations(self, owner: object, source: ast.FunctionDef) -> dict[str, ast.expr]: + """Recognize class-valued constructor keywords used directly in dispatch keys.""" + mapping = getattr(owner, "_py_template_params_to_cxx_type", {}) + result = {} + if source.name != "__init__" or not isinstance(mapping, dict) or not mapping: + return result + for node in ast.walk(source): + if not ( + isinstance(node, ast.Assign) + and isinstance(node.value, ast.Tuple) + and any( + isinstance(target, ast.Attribute) and target.attr == "py_template_params" + for target in node.targets + ) + ): + continue + for index, item in enumerate(node.value.elts): + if not isinstance(item, ast.Name): + continue + if not all(isinstance(key, tuple) and len(key) > index for key in mapping): + continue + values = {key[index] for key in mapping} + if not all(inspect.isclass(value) for value in values): + continue + for value in values: + self.imports.add(value.__module__) + result[item.id] = expression( + " | ".join( + sorted(f"type[{value.__module__}.{value.__qualname__}]" for value in values) + ) + ) + return result + + def functions( + self, + node: ast.FunctionDef, + obj: object, + source: ast.FunctionDef | None, + parameters: tuple, + owner: object, + ) -> list[ast.FunctionDef]: + doc = inspect.getdoc(obj) or "" + documented = ( + documented_signatures(node.name, doc) if isinstance(obj, types.FunctionType) else [] + ) + if source is not None: + result = copy.deepcopy(source) + # Preserve actual parameter kinds/defaults; fill only missing annotations. + if documented: + candidates = { + arg.arg: arg.annotation + for sig in documented + for arg in [*sig.args.posonlyargs, *sig.args.args, *sig.args.kwonlyargs] + } + candidates.update(self.dispatch_annotations(owner, source)) + positional = [*result.args.posonlyargs, *result.args.args] + defaults = [None] * ( + len(positional) - len(result.args.defaults) + ) + result.args.defaults + for arg, default in zip( + [*positional, *result.args.kwonlyargs], + [*defaults, *result.args.kw_defaults], + strict=True, + ): + if arg.annotation is None and candidates.get(arg.arg) is not None: + arg.annotation = copy.deepcopy(candidates[arg.arg]) + if isinstance(default, ast.Constant) and default.value is None: + arg.annotation = expression(f"{ast.unparse(arg.annotation)} | None") + if result.returns is None and len(documented) == 1: + result.returns = documented[0].returns + return [self.function(result, parameters)] + if not documented: + return [self.function(node, parameters)] + try: + actual = inspect.signature(obj, follow_wrapped=False) + except (TypeError, ValueError): + actual = None + for sig in documented: + if ( + isinstance(sig.returns, ast.Name) + and sig.returns.id == getattr(owner, "__name__", None) + and re.search(r":returns?:\s*\*self\*", doc) + ): + sig.returns = expression("typing_extensions.Self") + # CxxWrapper's forwarding closure accepts *args, without **kwargs. + if ( + actual + and any(p.kind == p.VAR_POSITIONAL for p in actual.parameters.values()) + and not any(p.kind == p.VAR_KEYWORD for p in actual.parameters.values()) + ): + sig.args.posonlyargs += sig.args.args + sig.args.args = [] + if len(documented) > 1: + sig.decorator_list = [expression("typing.overload")] + return [self.function(sig, parameters) for sig in documented] + + def scope( + self, body: list[ast.stmt], owner: object, source: list[ast.stmt], parameters: tuple = () + ) -> list[ast.stmt]: + definitions = { + node.name: node for node in source if isinstance(node, (ast.ClassDef, ast.FunctionDef)) + } + result = [] + rebuilt = set() + for node in body: + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and inspect.isclass(owner) + and issubclass(owner, enum.Enum) + and node.target.id in owner.__members__ + ): + # Enum members in stubs are assignments, not annotated attributes. + original = next( + ( + item + for item in source + if isinstance(item, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == node.target.id + for target in item.targets + ) + ), + None, + ) + result.append( + copy.deepcopy(original) + if original is not None + else ast.Assign(targets=[node.target], value=ast.Constant(value=Ellipsis)) + ) + continue + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and ( + (node.target.id.startswith("_") and not node.target.id.endswith("__")) + or node.target.id in {"__orig_bases__", "__parameters__"} + ) + and not isinstance(owner, types.ModuleType) + ): + continue + if ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + continue + if isinstance(node, ast.ClassDef): + cls = getattr(owner, node.name, None) + if ( + inspect.isclass(cls) + and cls.__module__ != self.module.__name__ + and not self.native_classes + ): + self.imports.add(cls.__module__) + result.append( + ast.parse(f"{node.name} = {cls.__module__}.{cls.__qualname__}").body[0] + ) + continue + original = definitions.get(node.name) + class_source = original.body if isinstance(original, ast.ClassDef) else [] + if isinstance(original, ast.ClassDef): + node.bases = copy.deepcopy(original.bases) + node.body = self.scope( + node.body, cls, class_source, getattr(cls, "__parameters__", ()) + ) + if cls is not None and "__getitem__" in vars(cls) and not hasattr(cls, "__iter__"): + getitem = next( + ( + item + for item in node.body + if isinstance(item, ast.FunctionDef) + and item.name == "__getitem__" + and len(args := [*item.args.posonlyargs, *item.args.args]) == 2 + and accepts_integer_index(args[1].annotation) + ), + None, + ) + if getitem and getitem.returns: + node.body.append( + ast.parse( + "@typing.type_check_only\n" + "def __iter__(self) -> collections.abc.Iterator[" + + ast.unparse(getitem.returns) + + "]: ..." + ).body[0] + ) + result.append(node) + elif isinstance(node, ast.FunctionDef): + if node.name in rebuilt: + continue + obj = getattr(owner, node.name, None) + if isinstance(obj, property): + accessors = [ + item + for item in source + if isinstance(item, ast.FunctionDef) and item.name == node.name + ] + if accessors: + result.extend(self.function(item, parameters) for item in accessors) + rebuilt.add(node.name) + continue + original = definitions.get(node.name) + original = original if isinstance(original, ast.FunctionDef) else None + functions = self.functions(node, obj, original, parameters, owner) + if original or ( + isinstance(obj, types.FunctionType) + and documented_signatures(node.name, inspect.getdoc(obj) or "") + ): + rebuilt.add(node.name) + result.extend(functions) + else: + result.append(node) + return result + + def refine(self, tree: ast.Module) -> str: + tree.body = self.scope(tree.body, self.module, self.source.body) + source_imports = [ + node for node in self.source.body if isinstance(node, (ast.Import, ast.ImportFrom)) + ] + imported_names = bindings(source_imports) + typevars = { + node.targets[0].id: node + for node in self.source.body + if isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and isinstance(getattr(self.module, node.targets[0].id, None), typing.TypeVar) + } + body = [] + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name in imported_names: + # Preserve source re-exports even when __module__ was changed for Sphinx. + continue + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and (node.target.id in imported_names or node.target.id in typevars) + ): + continue + if isinstance(node, (ast.Import, ast.ImportFrom)): + node.names = [ + alias + for alias in node.names + if (alias.asname or alias.name.split(".")[0]) not in imported_names + ] + if not node.names: + continue + body.append(node) + imports = [ + *source_imports, + *[node for node in body if isinstance(node, (ast.Import, ast.ImportFrom))], + ] + imports += [ast.Import(names=[ast.alias(name=name)]) for name in sorted(self.imports)] + # Future imports must come first; deduplicate imports from both sources. + unique_imports = {ast.unparse(node): node for node in imports} + tree.body = [ + unique_imports[key] + for key in sorted( + unique_imports, key=lambda key: (not key.startswith("from __future__"), key) + ) + ] + tree.body += list(typevars.values()) + tree.body += [node for node in body if not isinstance(node, (ast.Import, ast.ImportFrom))] + exports = set(getattr(self.module, "__all__", ())) + for node in tree.body: + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "__all__" + and isinstance(node.value, (ast.List, ast.Tuple)) + ): + exports.update(ast.literal_eval(node.value)) + used = { + node.id + for node in ast.walk(tree) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) + } | exports + cleaned = [] + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)) and not ( + isinstance(node, ast.ImportFrom) and node.module == "__future__" + ): + node.names = [ + alias + for alias in node.names + if (alias.asname or alias.name.split(".")[0]) in used or alias.name == "*" + ] + if not node.names: + continue + cleaned.append(node) + tree.body = cleaned + text = HEADER + ast.unparse(ast.fix_missing_locations(tree)) + "\n" + text = re.sub(r"(\n[ \t]*(?:async )?def [^\n]+:)\n[ \t]+\.\.\.", r"\1 ...", text) + ast.parse(text, feature_version=(3, 10)) + return text + + +def render_stubs( + module_name: str, *, strict: bool = False, public_only: bool = False +) -> dict[Path, str]: + """Generate validated drafts in memory, keyed by their relative module paths.""" + if public_only and module_name != EXTENSION_MODULE: + raise ValueError(f"--public-only requires the {EXTENSION_MODULE} module") + if importlib.util.find_spec("pybind11_stubgen") is None: + raise RuntimeError( + "Missing pybind11-stubgen. Run with: uv run --no-sync " + f"--with pybind11-stubgen==2.5.5 python etc/generate-stubs.py {module_name}" + ) + with tempfile.TemporaryDirectory(prefix="libsemigroups-stubs-") as temp: + command = [sys.executable, "-m", "pybind11_stubgen", module_name, "-o", temp] + if public_only: + native = importlib.import_module(module_name) + public = importlib.import_module("libsemigroups_pybind11") + names = public_native_classes(public, native) + if not names: + raise RuntimeError("No publicly exported native classes found") + command = [ + sys.executable, + "-c", + "import runpy, sys; " + "runpy.run_path(sys.argv[1])['generate_public_raw'](" + "sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] == '1')", + str(Path(__file__).resolve()), + module_name, + json.dumps(sorted(names)), + temp, + "1" if strict else "0", + ] + elif strict: + command.append("--exit-code") + subprocess.run(command, check=True, stdout=sys.stderr) + drafts = {} + for path in sorted(Path(temp).rglob("*.pyi")): + relative = path.relative_to(temp) + parts = list(relative.with_suffix("").parts) + if parts[-1] == "__init__": + parts.pop() + module = importlib.import_module(".".join(parts)) + tree = parse_stub(path.read_text(encoding="utf-8"), module.__name__, strict=strict) + if public_only: + tree.body = [node for node in tree.body if "__all__" not in bindings([node])] + # This deliberately partial native stub leaves omitted objects untyped. + tree.body += ast.parse( + "def __getattr__(name: str) -> _typeshed.Incomplete: ..." + ).body + refiner = Refiner(module, tree, native_classes=public_only) + drafts[relative] = refiner.refine(tree) + if refiner.incomplete: + print( + f"{module.__name__}: unresolved documentation types: " + + ", ".join(sorted(refiner.incomplete)), + file=sys.stderr, + ) + if not drafts: + raise RuntimeError(f"No stubs generated for {module_name}") + return drafts + + +def generate( + module_name: str, + output_dir: Path, + *, + force: bool = False, + strict: bool = False, + public_only: bool = False, +) -> list[Path]: + """Generate and validate drafts before writing any destination files.""" + drafts = { + output_dir / relative: text + for relative, text in render_stubs( + module_name, strict=strict, public_only=public_only + ).items() + } + conflicts = [str(path) for path in drafts if path.exists()] + if conflicts and not force: + raise FileExistsError( + "Existing stubs would be overwritten (use --force): " + ", ".join(conflicts) + ) + for path, text in drafts.items(): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w" if force else "x", encoding="utf-8") as stream: + stream.write(text) + return list(drafts) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("module", help="importable module or package to generate stubs for") + parser.add_argument( + "--public-only", + action=argparse.BooleanOptionalAction, + default=None, + help="select public native classes and dependencies (default for the extension)", + ) + output = parser.add_mutually_exclusive_group() + output.add_argument("-o", "--output-dir", type=Path, help="write stubs under this directory") + output.add_argument( + "--stdout", action="store_true", help="print one stub to standard output (default)" + ) + parser.add_argument("--force", action="store_true", help="replace existing generated files") + parser.add_argument( + "--strict", action="store_true", help="stop on pybind11-stubgen diagnostics" + ) + args = parser.parse_args() + module_name = args.module + public_only = args.public_only + if public_only is None: + public_only = module_name == EXTENSION_MODULE + if public_only and module_name != EXTENSION_MODULE: + parser.error(f"--public-only requires the {EXTENSION_MODULE} module") + if args.output_dir is None and args.force: + parser.error("--force requires --output-dir") + try: + if args.output_dir is None: + with contextlib.redirect_stdout(sys.stderr): + drafts = render_stubs(module_name, strict=args.strict, public_only=public_only) + if len(drafts) != 1: + raise RuntimeError( + f"Standard output requires exactly one stub, but {module_name} " + f"generated {len(drafts)}; " + "target an individual module or use --output-dir" + ) + sys.stdout.write(next(iter(drafts.values()))) + else: + for path in generate( + module_name, + args.output_dir, + force=args.force, + strict=args.strict, + public_only=public_only, + ): + print(path) + except ( + ImportError, + RuntimeError, + OSError, + SyntaxError, + subprocess.CalledProcessError, + ) as error: + print(f"Stub generation failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/libsemigroups_pybind11/action.py b/src/libsemigroups_pybind11/action.py index 238d5359..3d1395d9 100644 --- a/src/libsemigroups_pybind11/action.py +++ b/src/libsemigroups_pybind11/action.py @@ -9,7 +9,7 @@ """ from collections.abc import Iterator -from typing import TypeVar as _TypeVar +from typing import Generic as _Generic, TypeVar as _TypeVar from typing_extensions import Self as _Self @@ -56,7 +56,11 @@ ######################################################################## -class Action(_CxxWrapper): +_Element = _TypeVar("_Element") +_Point = _TypeVar("_Point") + + +class Action(_CxxWrapper, _Generic[_Element, _Point]): __doc__ = _RightActionPPerm1List.__doc__ Element = _TypeVar("Element") @@ -159,13 +163,13 @@ def __init__(self: _Self, *args, generators=None, seeds=None, func=None, side=No for x in seeds: self.add_seed(x) - def __getitem__(self: _Self, i: int) -> Point: + def __getitem__(self: _Self, i: int) -> _Point: return _to_py(_to_cxx(self)[i]) def __len__(self: _Self) -> int: return _to_cxx(self).size() - def __contains__(self: _Self, pt: Point) -> bool: + def __contains__(self: _Self, pt: _Point) -> bool: return self.position(pt) != _UNDEFINED ######################################################################## @@ -173,7 +177,7 @@ def __contains__(self: _Self, pt: Point) -> bool: ######################################################################## @_copydoc(_RightActionPPerm1PPerm1.generators) - def generators(self: _Self) -> Iterator[Element]: + def generators(self: _Self) -> Iterator[_Element]: # pylint: disable=missing-function-docstring return (_to_py(x) for x in _to_cxx(self).generators()) @@ -194,7 +198,7 @@ def generators(self: _Self) -> Iterator[Element]: ######################################################################## -class RightAction(Action): +class RightAction(Action[_Element, _Point]): """Class representing a right action of a semigroup or monoid on a set. This page contains the documentation for the class :any:`RightAction`, @@ -235,7 +239,7 @@ def __init__(self: _Self, *args, generators=None, seeds=None) -> None: ######################################################################## -class LeftAction(Action): +class LeftAction(Action[_Element, _Point]): """Class representing a left action of a semigroup or monoid on a set. This page contains the documentation for the class ``LeftAction``, which diff --git a/src/libsemigroups_pybind11/action.pyi b/src/libsemigroups_pybind11/action.pyi new file mode 100644 index 00000000..90c9be2d --- /dev/null +++ b/src/libsemigroups_pybind11/action.pyi @@ -0,0 +1,119 @@ +# Distributed under the terms of the GPL license version 3. +# The full license is in the file LICENSE, distributed with this software. +# AI assistance: OpenAI Codex drafted and checked this sample stub. + +# Sample types for the public action module. This does not opt the package into +# installed typing support via py.typed; related extension types remain untyped. +# The type parameters track elements and points; supported combinations of +# element, point, function, and side are still checked at runtime by action.py. + +from collections.abc import Callable, Iterator +from datetime import datetime, timedelta +from typing import ClassVar, Generic, TypeVar, overload, type_check_only + +from typing_extensions import Self + +from _libsemigroups_pybind11 import ( + Gabow as _Gabow, + Reporter as _Reporter, + Runner as _Runner, + Undefined as _Undefined, + WordGraph as _WordGraph, + side as _side, +) + +from .adapters import ImageLeftAction as _ImageLeftAction, ImageRightAction as _ImageRightAction + +__all__ = ["Action", "LeftAction", "RightAction"] + +_Element = TypeVar("_Element") +_Point = TypeVar("_Point") + +class Action(Generic[_Element, _Point]): + def __init__( + self, + *, + generators: list[_Element] | None = None, + seeds: list[_Point] | None = None, + func: type[_ImageLeftAction] | type[_ImageRightAction] | None = None, + side: _side | None = None, + ) -> None: ... + def __getitem__(self, i: int) -> _Point: ... + # Python iterates via __getitem__; this method only models that for typing. + @type_check_only + def __iter__(self) -> Iterator[_Point]: ... + def __len__(self) -> int: ... + def __contains__(self, pt: _Point) -> bool: ... + def __repr__(self) -> str: ... + def __eq__(self, that: object) -> bool: ... + def __copy__(self) -> Action[_Element, _Point]: ... + def generators(self) -> Iterator[_Element]: ... + + # Methods attached by copy_cxx_mem_fns accept positional arguments only. + def add_generator(self, gen: _Element, /) -> Self: ... + def add_seed(self, seed: _Point, /) -> Self: ... + def apply(self, pt: _Point, x: _Element, /) -> _Point: ... + @overload + def cache_scc_multipliers(self) -> bool: ... + @overload + def cache_scc_multipliers(self, val: bool, /) -> Self: ... + def copy(self) -> Action[_Element, _Point]: ... + def current_size(self) -> int: ... + def empty(self) -> bool: ... + def init(self) -> Self: ... + def multiplier_from_scc_root(self, pos: int, /) -> _Element: ... + def multiplier_to_scc_root(self, pos: int, /) -> _Element: ... + def number_of_generators(self) -> int: ... + def position(self, pt: _Point, /) -> int | _Undefined: ... + def reserve(self, val: int, /) -> Self: ... + def root_of_scc(self, x: _Point | int, /) -> _Point: ... + def scc(self) -> _Gabow: ... + def size(self) -> int: ... + def word_graph(self) -> _WordGraph: ... + + # Runner methods are forwarded; the Python wrapper is not a Runner subclass. + state: ClassVar[type[_Runner.state]] + def current_state(self) -> _Runner.state: ... + def dead(self) -> bool: ... + def finished(self) -> bool: ... + def kill(self) -> None: ... + def report_why_we_stopped(self) -> None: ... + def run(self) -> None: ... + def run_for(self, t: timedelta, /) -> None: ... + def run_until(self, func: Callable[[], bool], /) -> None: ... + def running(self) -> bool: ... + def running_for(self) -> bool: ... + def running_until(self) -> bool: ... + def started(self) -> bool: ... + def stopped(self) -> bool: ... + def stopped_by_predicate(self) -> bool: ... + def success(self) -> bool: ... + def timed_out(self) -> bool: ... + + # The current bindings return Reporter objects from these setters. + def last_report(self) -> datetime: ... + def report(self) -> bool: ... + @overload + def report_every(self) -> timedelta: ... + @overload + def report_every(self, val: timedelta, /) -> _Reporter: ... + @overload + def report_prefix(self) -> str: ... + @overload + def report_prefix(self, val: str, /) -> _Reporter: ... + def reset_last_report(self) -> _Reporter: ... + def reset_start_time(self) -> _Reporter: ... + def start_time(self) -> datetime: ... + +class RightAction(Action[_Element, _Point]): + def __init__( + self, *, generators: list[_Element] | None = None, seeds: list[_Point] | None = None + ) -> None: ... + +class LeftAction(Action[_Element, _Point]): + def __init__( + self, + *args: object, + generators: list[_Element] | None = None, + seeds: list[_Point] | None = None, + ) -> None: ... diff --git a/src/libsemigroups_pybind11/bmat8.pyi b/src/libsemigroups_pybind11/bmat8.pyi new file mode 100644 index 00000000..bc3b9a74 --- /dev/null +++ b/src/libsemigroups_pybind11/bmat8.pyi @@ -0,0 +1,35 @@ +# Generated by etc/generate-stubs.py using pybind11-stubgen. +# Draft only: review signatures and incomplete types before publishing. +# AI assistance: OpenAI Codex helped implement the generator. + +from __future__ import annotations + +from _libsemigroups_pybind11 import ( + bmat8_col_space_basis as col_space_basis, + bmat8_col_space_size as col_space_size, + bmat8_is_regular_element as is_regular_element, + bmat8_minimum_dim as minimum_dim, + bmat8_number_of_cols as number_of_cols, + bmat8_number_of_rows as number_of_rows, + bmat8_one as one, + bmat8_random as random, + bmat8_row_space_basis as row_space_basis, + bmat8_row_space_size as row_space_size, + bmat8_rows as rows, + bmat8_transpose as transpose, +) + +__all__: list = [ + "col_space_basis", + "col_space_size", + "is_regular_element", + "minimum_dim", + "number_of_cols", + "number_of_rows", + "one", + "random", + "row_space_basis", + "row_space_size", + "rows", + "transpose", +] diff --git a/src/libsemigroups_pybind11/detail/cxx_wrapper.py b/src/libsemigroups_pybind11/detail/cxx_wrapper.py index 31564011..7368c6c4 100644 --- a/src/libsemigroups_pybind11/detail/cxx_wrapper.py +++ b/src/libsemigroups_pybind11/detail/cxx_wrapper.py @@ -203,7 +203,7 @@ def cxx_free_fn_wrapper(*args, **kwargs): return cxx_free_fn_wrapper -def copy_cxx_mem_fns(cxx_class: Pybind11Type, py_class: CxxWrapper) -> None: +def copy_cxx_mem_fns(cxx_class: Pybind11Type, py_class: type[CxxWrapper]) -> None: """Copy all the non-special methods of *cxx_class* into methods of *py_class* that call the cxx member function on the _cxx_obj. """ diff --git a/tests/test_generate_stubs.py b/tests/test_generate_stubs.py new file mode 100644 index 00000000..d5eaf800 --- /dev/null +++ b/tests/test_generate_stubs.py @@ -0,0 +1,552 @@ +# Distributed under the terms of the GPL license version 3. +# The full license is in the file LICENSE, distributed with this software. +# AI assistance: OpenAI Codex helped write these generator checks. + +"""Check stub generation, including signatures attached to wrappers at runtime.""" + +# pylint: disable=missing-function-docstring + +import ast +import importlib.util +import subprocess +import sys +import types +from pathlib import Path + +import pytest + + +@pytest.fixture(name="stub_generator", scope="module") +def fixture_stub_generator(): + path = Path(__file__).resolve().parents[1] / "etc" / "generate-stubs.py" + spec = importlib.util.spec_from_file_location("generate_stubs", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(name="generated_action", scope="module") +def fixture_generated_action(stub_generator, tmp_path_factory): + pytest.importorskip("pybind11_stubgen") + output = tmp_path_factory.mktemp("action_stubs") + original = Path(__file__).resolve().parents[1] / "src/libsemigroups_pybind11/action.pyi" + before = original.read_bytes() if original.exists() else None + paths = stub_generator.generate("libsemigroups_pybind11.action", output, strict=True) + assert paths == [output / "libsemigroups_pybind11/action.pyi"] + assert (original.read_bytes() if original.exists() else None) == before + return paths[0] + + +def test_documented_signature_precedence(stub_generator): + doc = """size(self: SomeCppType, unused: int) -> str + :sig=(self: Action) -> int: + :sig=(self: Action) -> int: + """ + signatures = stub_generator.documented_signatures("size", doc) + assert len(signatures) == 1 + assert [arg.arg for arg in signatures[0].args.args] == ["self"] + assert ast.unparse(signatures[0].returns) == "int" + + +def test_documented_overloads(stub_generator): + signatures = stub_generator.documented_signatures( + "setting", + "setting(*args, **kwargs)\nOverloaded function.\n\n" + "1. setting(self: Cpp) -> bool\n" + "2. setting(self: Cpp, value: bool) -> Cpp\n", + ) + assert len(signatures) == 2 + assert [len(sig.args.args) for sig in signatures] == [1, 2] + + +def test_signature_defaults_are_not_executed(stub_generator): + signatures = stub_generator.documented_signatures( + "f", ":sig=(x: str = this_function_does_not_exist()) -> None:" + ) + assert len(signatures) == 1 + assert isinstance(signatures[0].args.defaults[0], ast.Call) + + +def test_invalid_native_base(stub_generator, capsys): + text = "class BoundClass(detail::Base):\n def size(self) -> int: ...\n" + tree = stub_generator.parse_stub(text, "example", strict=False) + cls = next(node for node in tree.body if isinstance(node, ast.ClassDef)) + assert ast.unparse(cls.bases[0]) == "typing.Any" + assert cls.body[0].name == "size" + assert "detail::Base" in capsys.readouterr().err + with pytest.raises(SyntaxError): + stub_generator.parse_stub(text, "example", strict=True) + + +def test_unrelated_syntax_errors_are_not_suppressed(stub_generator): + with pytest.raises(SyntaxError): + stub_generator.parse_stub("def broken(: ...", "example", strict=False) + + +def test_native_keyword_parameter(stub_generator): + tree = stub_generator.parse_stub( + "def f(graph: object, from: int, path: list[int]) -> int: ...", "example", strict=False + ) + args = tree.body[0].args + assert [arg.arg for arg in args.posonlyargs] == ["graph", "from_"] + assert [arg.arg for arg in args.args] == ["path"] + ast.parse(ast.unparse(tree)) + tree = stub_generator.parse_stub("def f(from: int, /): ...", "example", strict=False) + assert tree.body[0].args.posonlyargs[0].arg == "from_" + with pytest.raises(SyntaxError): + stub_generator.parse_stub("def f(*, from: int): ...", "example", strict=False) + + +def test_native_bmat8(stub_generator, tmp_path): + pytest.importorskip("pybind11_stubgen") + paths = stub_generator.generate("_libsemigroups_pybind11", tmp_path) + path = tmp_path / "_libsemigroups_pybind11.pyi" + assert path in paths + tree = ast.parse(path.read_text(encoding="utf-8"), feature_version=(3, 10)) + cls = next( + node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "BMat8" + ) + methods = [node for node in cls.body if isinstance(node, ast.FunctionDef)] + assert {node.name for node in methods} >= {"__init__", "__getitem__", "__mul__", "to_int"} + assert ast.unparse(next(node.returns for node in methods if node.name == "to_int")) == "int" + assert sum(node.name == "__init__" for node in methods) > 1 + + +def test_public_class_selection_uses_exports_and_identity(stub_generator): + native = types.ModuleType("native") + public = types.ModuleType("public") + helper = types.ModuleType("public.helper") + native.RealName = type("RenamedForDocs", (), {}) + native.Hidden = type("Hidden", (), {}) + public.Alias = native.RealName + public.AccidentallyImported = native.Hidden + public.Wrapper = type("RealName", (), {}) + public.helper = helper + public.__all__ = ["Alias", "Wrapper", "helper"] + helper.__all__ = ["Root"] + helper.Root = public # A cycle in exported modules must terminate. + assert stub_generator.public_native_classes(public, native) == {"RealName"} + helper.AlsoPublic = native.Hidden + helper.__all__.append("AlsoPublic") + assert stub_generator.public_native_classes(public, native) == {"RealName", "Hidden"} + + +def test_native_typing_dependencies(stub_generator): + native = types.ModuleType("native") + for name in ("Base", "Result", "Nested", "Unused", "Hidden"): + setattr(native, name, type(name, (), {})) + native.DEFAULT = 0 + tree = ast.parse( + "class Public(Base):\n" + ' """Hidden is mentioned only in documentation."""\n' + ' def f(self, x: list["Result"] = DEFAULT) -> native.Nested.Inner: ...\n' + ) + assert stub_generator.native_dependencies(tree, native) == { + "Base", + "Result", + "Nested", + "DEFAULT", + } + + +def test_public_only_rejects_wrapper_target(stub_generator, tmp_path): + with pytest.raises(ValueError, match="--public-only requires"): + stub_generator.generate("libsemigroups_pybind11.action", tmp_path, public_only=True) + assert not any(tmp_path.iterdir()) + + +def test_public_native_stub(stub_generator, tmp_path, capfd): + pytest.importorskip("pybind11_stubgen") + native = importlib.import_module("_libsemigroups_pybind11") + public = importlib.import_module("libsemigroups_pybind11") + + before = {name: value for name, value in vars(native).items() if isinstance(value, type)} + paths = stub_generator.generate("_libsemigroups_pybind11", tmp_path, public_only=True) + assert paths == [tmp_path / "_libsemigroups_pybind11.pyi"] + text = paths[0].read_text(encoding="utf-8") + tree = ast.parse(text, feature_version=(3, 10)) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + assert stub_generator.public_native_classes(public, native) <= classes.keys() + assert {"BMat8", "Runner", "forest_PathsFromRoots"} <= classes.keys() + assert { + "AlphabetWord", + "AlphabetString", + "RightActionPPerm1List", + "FroidurePinBMat8", + }.isdisjoint(classes) + # An unexported base is kept because public PathsFromRoots inherits from it. + assert "forest_PathsFromToRootsCommon" in classes + assert stub_generator.native_dependencies(tree, native) <= stub_generator.bindings(tree.body) + assert "__all__" not in stub_generator.bindings(tree.body) + assert "def __getattr__(name: str) -> _typeshed.Incomplete:" in text + iterator = next( + node + for node in classes["BMat8"].body + if isinstance(node, ast.FunctionDef) and node.name == "__iter__" + ) + assert ast.unparse(iterator.returns) == "collections.abc.Iterator[list[bool]]" + assert all(getattr(native, name) is value for name, value in before.items()) + assert "AlphabetWord" not in capfd.readouterr().err + stub_generator.generate("_libsemigroups_pybind11", tmp_path, public_only=True, force=True) + assert paths[0].read_text(encoding="utf-8") == text + + +def test_public_bmat8_with_mypy(stub_generator, tmp_path): + pytest.importorskip("pybind11_stubgen") + pytest.importorskip("mypy") + stub_generator.generate("_libsemigroups_pybind11", tmp_path, public_only=True) + # Isolate the selected BMat8 section: other public classes still have malformed + # native signatures. The selection/dependency test checks the complete output. + stub = tmp_path / "_libsemigroups_pybind11.pyi" + tree = ast.parse(stub.read_text(encoding="utf-8")) + tree.body = [ + node + for node in tree.body + if isinstance(node, (ast.Import, ast.ImportFrom)) + or isinstance(node, ast.ClassDef) + and node.name == "BMat8" + ] + stub.write_text(ast.unparse(tree), encoding="utf-8") + package = tmp_path / "libsemigroups_pybind11" + package.mkdir() + (package / "__init__.pyi").write_text( + "from _libsemigroups_pybind11 import BMat8 as BMat8\n", encoding="utf-8" + ) + client = tmp_path / "client.py" + client.write_text( + "from typing_extensions import assert_type\n" + "from libsemigroups_pybind11 import BMat8\n" + "matrix = BMat8([[True, False], [False, True]])\n" + "assert_type(matrix.to_int(), int)\n" + "assert_type(matrix[0, 1], bool)\n" + "assert_type(matrix[0], list[bool])\n" + "for row in matrix:\n" + " assert_type(row, list[bool])\n" + "matrix.nonexistent() # type: ignore[attr-defined]\n" + "BMat8('bad') # type: ignore[arg-type]\n", + encoding="utf-8", + ) + # Check client inference while unrelated draft-native annotations remain incomplete. + config = tmp_path / "mypy.ini" + config.write_text( + "[mypy]\nfollow_imports = silent\nfollow_imports_for_stubs = True\n", encoding="utf-8" + ) + result = subprocess.run( + [ + sys.executable, + "-m", + "mypy", + "--python-version=3.10", + "--ignore-missing-imports", + "--config-file", + str(config), + "--warn-unused-ignores", + "--cache-dir", + str(tmp_path / "cache"), + str(client), + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_action_generic_signatures(generated_action): + text = generated_action.read_text(encoding="utf-8") + tree = ast.parse(text, feature_version=(3, 10)) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + action = classes["Action"] + methods = [node for node in action.body if isinstance(node, ast.FunctionDef)] + by_name = {node.name: node for node in methods} + assert ast.unparse(action.bases[-1]) == "_Generic[_Element, _Point]" + assert ast.unparse(by_name["__getitem__"].returns) == "_Point" + assert ast.unparse(by_name["generators"].returns) == "Iterator[_Element]" + assert ast.unparse(by_name["add_seed"].returns) == "typing_extensions.Self" + assert ast.unparse(by_name["__iter__"].returns) == "collections.abc.Iterator[_Point]" + assert len(by_name["add_seed"].args.posonlyargs) == 2 + assert not by_name["add_seed"].args.args + assert ast.unparse(classes["RightAction"].bases[0]) == "Action[_Element, _Point]" + assert ast.unparse(classes["LeftAction"].bases[0]) == "Action[_Element, _Point]" + constructor = {arg.arg: arg for arg in by_name["__init__"].args.kwonlyargs} + assert ast.unparse(constructor["seeds"].annotation) == "list[_Point] | None" + assert "type[libsemigroups_pybind11.adapters.ImageRightAction]" in ast.unparse( + constructor["func"].annotation + ) + overloads = [node for node in methods if node.name == "cache_scc_multipliers"] + assert [ast.unparse(node.returns) for node in overloads] == ["bool", "typing_extensions.Self"] + assert all(ast.unparse(node.decorator_list[0]) == "typing.overload" for node in overloads) + assert "_all_wrapped_cxx_types" not in text + assert ":sig=" not in text + + +def test_regeneration_is_deterministic(stub_generator, generated_action): + before = generated_action.read_bytes() + stub_generator.generate( + "libsemigroups_pybind11.action", generated_action.parents[1], force=True, strict=True + ) + assert generated_action.read_bytes() == before + + +def test_overwrite_requires_force(stub_generator, generated_action, tmp_path): + destination = tmp_path / "libsemigroups_pybind11/action.pyi" + destination.parent.mkdir() + destination.write_text("# hand edited stub\n", encoding="utf-8") + with pytest.raises(FileExistsError, match="--force"): + stub_generator.generate("libsemigroups_pybind11.action", tmp_path) + assert destination.read_text(encoding="utf-8") == "# hand edited stub\n" + assert generated_action.exists() + + +def test_another_wrapper_module(stub_generator, tmp_path): + pytest.importorskip("pybind11_stubgen") + paths = stub_generator.generate("libsemigroups_pybind11.sims", tmp_path, strict=True) + tree = ast.parse(paths[0].read_text(encoding="utf-8"), feature_version=(3, 10)) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + assert {"Sims1", "Sims2", "RepOrc"} <= classes.keys() + assert any( + isinstance(node, ast.FunctionDef) and node.name == "find_if" + for node in classes["Sims1"].body + ) + + +@pytest.mark.parametrize("name", ["presentation", "matrix", "forest", "detail.dot"]) +def test_properties_enums_and_reexports(stub_generator, tmp_path, name): + pytest.importorskip("pybind11_stubgen") + paths = stub_generator.generate(f"libsemigroups_pybind11.{name}", tmp_path, strict=True) + path = next(path for path in paths if path.name != "examples.pyi") + tree = ast.parse(path.read_text(encoding="utf-8")) + classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)} + if name == "presentation": + accessors = [ + node + for node in classes["Presentation"].body + if isinstance(node, ast.FunctionDef) and node.name == "rules" + ] + assert [ast.unparse(node.decorator_list[0]) for node in accessors] == [ + "property", + "rules.setter", + ] + assert [len(node.args.args) for node in accessors] == [1, 2] + elif name == "matrix": + members = [node for node in classes["MatrixKind"].body if isinstance(node, ast.Assign)] + assert {node.targets[0].id for node in members} >= {"Boolean", "Integer", "MaxPlus"} + else: + assert not classes + imports = [node for node in tree.body if isinstance(node, ast.ImportFrom)] + assert any(node.module == "_libsemigroups_pybind11" for node in imports) + if name == "detail.dot": + assert any(alias.name == "Dot" for node in imports for alias in node.names) + + +def test_generated_stub_with_mypy(generated_action, tmp_path): + pytest.importorskip("mypy") + # Isolate this check from the hand-written Action stub in the source tree. + package = tmp_path / "libsemigroups_pybind11" + package.mkdir() + (package / "__init__.pyi").write_text("", encoding="utf-8") + stub = package / "action.pyi" + stub.write_bytes(generated_action.read_bytes()) + client = tmp_path / "client.py" + client.write_text( + "from typing_extensions import assert_type\n" + "from libsemigroups_pybind11.action import RightAction\n" + "def check(action: RightAction[object, list[int]]) -> None:\n" + " assert_type(action[0], list[int])\n" + " assert_type(action.cache_scc_multipliers(), bool)\n" + " assert_type(action.cache_scc_multipliers(True), RightAction[object, list[int]])\n" + " for point in action:\n" + " assert_type(point, list[int])\n" + " action.add_seed('bad') # type: ignore[arg-type]\n" + " action.reserve(val=10) # type: ignore[call-arg]\n", + encoding="utf-8", + ) + result = subprocess.run( + [ + sys.executable, + "-m", + "mypy", + "--python-version", + "3.10", + "--ignore-missing-imports", + "--follow-imports=silent", + "--warn-unused-ignores", + "--cache-dir", + str(tmp_path / "cache"), + str(stub), + str(client), + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_missing_generator_dependency(stub_generator, monkeypatch, tmp_path): + monkeypatch.setattr(stub_generator.importlib.util, "find_spec", lambda _: None) + with pytest.raises(RuntimeError, match="--with pybind11-stubgen"): + stub_generator.generate("libsemigroups_pybind11.action", tmp_path) + assert not any(tmp_path.iterdir()) + + +@pytest.mark.parametrize( + "options, class_name", + [ + (["libsemigroups_pybind11.action"], "Action"), + (["_libsemigroups_pybind11"], "BMat8"), + (["_libsemigroups_pybind11", "--public-only"], "BMat8"), + ], +) +@pytest.mark.parametrize("stdout_options", [[], ["--stdout"]]) +def test_stdout_cli(stub_generator, tmp_path, options, class_name, stdout_options): + pytest.importorskip("pybind11_stubgen") + sentinel = tmp_path / "keep.pyi" + sentinel.write_text("# existing stub\n", encoding="utf-8") + result = subprocess.run( + [sys.executable, stub_generator.__file__, *options, *stdout_options], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.startswith(stub_generator.HEADER) + tree = ast.parse(result.stdout, feature_version=(3, 10)) + assert any(isinstance(node, ast.ClassDef) and node.name == class_name for node in tree.body) + assert "Selected " not in result.stdout + if options[0] == "_libsemigroups_pybind11": + assert "Selected " in result.stderr + assert not any( + isinstance(node, ast.ClassDef) and node.name == "AlphabetWord" for node in tree.body + ) + assert list(tmp_path.iterdir()) == [sentinel] + assert sentinel.read_text(encoding="utf-8") == "# existing stub\n" + + +def test_explicit_output_directory_cli(stub_generator, generated_action, tmp_path): + output = tmp_path / "stubs" + result = subprocess.run( + [ + sys.executable, + stub_generator.__file__, + "libsemigroups_pybind11.action", + "--output-dir", + str(output), + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + stub = output / "libsemigroups_pybind11/action.pyi" + assert stub.read_bytes() == generated_action.read_bytes() + assert result.stdout == f"{stub}\n" + + +def test_stdout_rejects_multiple_stubs(stub_generator, monkeypatch, capsys, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", "package"]) + monkeypatch.setattr( + stub_generator, + "render_stubs", + lambda *_args, **_kwargs: { + Path("a.pyi"): "class A: ...\n", + Path("b.pyi"): "class B: ...\n", + }, + ) + assert stub_generator.main() == 1 + result = capsys.readouterr() + assert result.out == "" + assert "Standard output requires exactly one stub" in result.err + assert not any(tmp_path.iterdir()) + + +def test_stdout_failure_keeps_output_empty(stub_generator, monkeypatch, capsys): + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", "example", "--strict"]) + + def fail(*_args, **kwargs): + assert kwargs["strict"] + print("Import diagnostic") + raise RuntimeError("generation failed") + + monkeypatch.setattr(stub_generator, "render_stubs", fail) + assert stub_generator.main() == 1 + result = capsys.readouterr() + assert result.out == "" + assert "Import diagnostic" in result.err + assert "generation failed" in result.err + + +@pytest.mark.parametrize( + "options", [["--stdout", "--output-dir", "stubs"], ["--stdout", "--force"], ["--force"]] +) +def test_stdout_rejects_file_options(stub_generator, monkeypatch, capsys, options): + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", "example", *options]) + with pytest.raises(SystemExit) as error: + stub_generator.main() + assert error.value.code == 2 + assert capsys.readouterr().out == "" + + +@pytest.mark.parametrize("options", [[], ["--public-only"]]) +def test_module_argument_is_required(stub_generator, monkeypatch, capsys, options): + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", *options]) + with pytest.raises(SystemExit) as error: + stub_generator.main() + assert error.value.code == 2 + result = capsys.readouterr() + assert result.out == "" + assert "the following arguments are required: module" in result.err + + +@pytest.mark.parametrize( + "options, public_only", + [ + (["_libsemigroups_pybind11"], True), + (["_libsemigroups_pybind11", "--public-only"], True), + (["_libsemigroups_pybind11", "--no-public-only"], False), + (["libsemigroups_pybind11.action"], False), + ], +) +def test_cli_public_only_selection(stub_generator, monkeypatch, capsys, options, public_only): + monkeypatch.setattr(sys, "argv", ["generate-stubs.py", *options]) + + def render(module_name, **kwargs): + assert module_name == options[0] + assert kwargs["public_only"] is public_only + return {Path("example.pyi"): "class Example: ...\n"} + + monkeypatch.setattr(stub_generator, "render_stubs", render) + assert stub_generator.main() == 0 + assert capsys.readouterr().out == "class Example: ...\n" + + +@pytest.mark.parametrize("public_only", [False, True]) +def test_failed_generation_preserves_destination( + stub_generator, monkeypatch, tmp_path, public_only +): + monkeypatch.setattr(stub_generator.importlib.util, "find_spec", lambda _: object()) + destination = tmp_path / "action.pyi" + destination.write_text("# keep this stub\n", encoding="utf-8") + + def fail(command, **_kwargs): + if public_only: + assert command[-1] == "1" # Strict mode is passed to the isolated backend. + else: + assert "--exit-code" in command + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr(stub_generator.subprocess, "run", fail) + with pytest.raises(subprocess.CalledProcessError): + stub_generator.generate( + "_libsemigroups_pybind11" if public_only else "libsemigroups_pybind11.action", + tmp_path, + strict=True, + public_only=public_only, + ) + assert list(tmp_path.iterdir()) == [destination] + assert destination.read_text(encoding="utf-8") == "# keep this stub\n"