From 475c9e5ed71acdd738da3866b6864caf3be3c500 Mon Sep 17 00:00:00 2001 From: Tobias Langer Date: Mon, 24 Nov 2025 16:07:47 +0100 Subject: [PATCH 1/3] Fix type issues in meta_model downloader script --- dev_scripts/codegen/download_aas_core_meta_model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev_scripts/codegen/download_aas_core_meta_model.py b/dev_scripts/codegen/download_aas_core_meta_model.py index 5590ffd7..d30391cc 100644 --- a/dev_scripts/codegen/download_aas_core_meta_model.py +++ b/dev_scripts/codegen/download_aas_core_meta_model.py @@ -4,6 +4,7 @@ import os import argparse import sys +from typing import Dict, Union import requests @@ -24,7 +25,7 @@ def _latest_commit_sha_for_path( Uses: GET /repos/{owner}/{repo}/commits?path=...&sha=...&per_page=1 """ url = f"{GITHUB_API}/repos/{owner}/{repo}/commits" - params = {"path": path, "sha": ref, "per_page": 1} + params: Dict[str, Union[str, int]] = {"path": path, "sha": ref, "per_page": 1} resp = requests.get(url, params=params, timeout=timeout) try: resp.raise_for_status() @@ -43,6 +44,7 @@ def _latest_commit_sha_for_path( sha = commits[0].get("sha") if not sha or not isinstance(sha, str): raise RuntimeError("API did not return a valid commit SHA.") + assert isinstance(sha, str) return sha From fd84c86158430f86940d57eb15353677c618686f Mon Sep 17 00:00:00 2001 From: Tobias Langer Date: Mon, 24 Nov 2025 16:08:11 +0100 Subject: [PATCH 2/3] Fix lint and style issues in generator script --- dev_scripts/update_to_aas_core_meta_codegen.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dev_scripts/update_to_aas_core_meta_codegen.py b/dev_scripts/update_to_aas_core_meta_codegen.py index f62335a2..4d3c3c2c 100644 --- a/dev_scripts/update_to_aas_core_meta_codegen.py +++ b/dev_scripts/update_to_aas_core_meta_codegen.py @@ -115,7 +115,7 @@ def _regenerate_code( target_dir = our_repo - print(f"Starting to run codegen script") + print("Starting to run codegen script") start = time.perf_counter() proc = subprocess.run( @@ -128,6 +128,7 @@ def _regenerate_code( str(target_dir), ], cwd=str(codegen_dir), + check=True, ) if proc.returncode != 0: @@ -146,9 +147,9 @@ def _semantically_patch(our_repo: pathlib.Path) -> Optional[int]: verification_java = ( our_repo / "src/main/java/aas_core/aas3_0/verification/Verification.java" ) - assert verification_java.exists() and verification_java.is_file(), ( - f"No Verification.java found to patch: {verification_java=}" - ) + assert ( + verification_java.exists() and verification_java.is_file() + ), f"No Verification.java found to patch: {verification_java=}" exit_code = _execute(cmd=["mvn", "package"], cwd=str(cwd)) if exit_code is not None: @@ -271,7 +272,7 @@ def _get_codegen_revision(our_repo: pathlib.Path) -> str | None: print(f"Cannot read codegen revision: {os_error}.") if codegen_sha is None: - print(f"Cannot read codegen revision.") + print("Cannot read codegen revision.") return codegen_sha @@ -303,7 +304,7 @@ def _get_meta_model_revision(our_repo: pathlib.Path) -> str | None: print(f"Cannot read meta model revision: {os_error}.") if meta_model_sha is None: - print(f"Cannot read meta model revision.") + print("Cannot read meta model revision.") return meta_model_sha @@ -320,7 +321,7 @@ def _get_testgen_revision(our_repo: pathlib.Path) -> str | None: print(f"Cannot read testgen revision: {os_error}.") if testgen_rev is None: - print(f"Cannot read testgen revision.") + print("Cannot read testgen revision.") return testgen_rev From 996f04613f4bfa4f7596b6156fae6c38f599150a Mon Sep 17 00:00:00 2001 From: Tobias Langer Date: Mon, 24 Nov 2025 16:08:31 +0100 Subject: [PATCH 3/3] Add CI scripts for dev_scripts --- .github/workflows/ci-for-dev-scripts.yml | 36 ++++ .../__init__.py | 0 .../mypy.ini | 1 + .../precommit.py | 200 ++++++++++++++++++ .../pylint.rc | 10 + dev_scripts/pyproject.toml | 10 +- 6 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-for-dev-scripts.yml create mode 100644 dev_scripts/continuous_integration_of_dev_scripts/__init__.py create mode 100644 dev_scripts/continuous_integration_of_dev_scripts/mypy.ini create mode 100644 dev_scripts/continuous_integration_of_dev_scripts/precommit.py create mode 100644 dev_scripts/continuous_integration_of_dev_scripts/pylint.rc diff --git a/.github/workflows/ci-for-dev-scripts.yml b/.github/workflows/ci-for-dev-scripts.yml new file mode 100644 index 00000000..e353bb33 --- /dev/null +++ b/.github/workflows/ci-for-dev-scripts.yml @@ -0,0 +1,36 @@ +name: CI-for-dev-scripts + +on: + push: + branches: [ master, main ] + tags: [ "**" ] + paths: + - _dev_scripts/** + pull_request: + branches: [ "**" ] + paths: + - _dev_scripts/** + +jobs: + Execute-continuous-integration: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10'] + + steps: + - uses: actions/checkout@master + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install -e _dev_scripts --group _dev_scripts/pyproject.toml:dev + + - name: Run checks + run: | + python3 _dev_scripts/continuous_integration_of_dev_scripts/precommit.py diff --git a/dev_scripts/continuous_integration_of_dev_scripts/__init__.py b/dev_scripts/continuous_integration_of_dev_scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev_scripts/continuous_integration_of_dev_scripts/mypy.ini b/dev_scripts/continuous_integration_of_dev_scripts/mypy.ini new file mode 100644 index 00000000..f8b1844b --- /dev/null +++ b/dev_scripts/continuous_integration_of_dev_scripts/mypy.ini @@ -0,0 +1 @@ +[mypy] diff --git a/dev_scripts/continuous_integration_of_dev_scripts/precommit.py b/dev_scripts/continuous_integration_of_dev_scripts/precommit.py new file mode 100644 index 00000000..0f1cb06e --- /dev/null +++ b/dev_scripts/continuous_integration_of_dev_scripts/precommit.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 + +"""Run pre-commit checks on the repository.""" + +import argparse +import enum +import os +import pathlib +import shlex +import subprocess +import sys +from typing import Optional, Mapping, Sequence + + +# pylint: disable=unnecessary-comprehension + + +class Step(enum.Enum): + """Enumerate different pre-commit steps.""" + + REFORMAT = "reformat" + MYPY = "mypy" + PYLINT = "pylint" + TEST = "test" + DOCTEST = "doctest" + + +def call_and_report( + verb: str, + cmd: Sequence[str], + cwd: Optional[pathlib.Path] = None, + env: Optional[Mapping[str, str]] = None, +) -> int: + """ + Wrap a subprocess call with the reporting to STDERR if it failed. + + Return 1 if there is an error and 0 otherwise. + """ + cmd_str = " ".join(shlex.quote(part) for part in cmd) + + if cwd is not None: + print(f"Executing from {cwd}: {cmd_str}") + else: + print(f"Executing: {cmd_str}") + + exit_code = subprocess.call(cmd, cwd=str(cwd) if cwd is not None else None, env=env) + + if exit_code != 0: + print( + f"Failed to {verb} with exit code {exit_code}: {cmd_str}", file=sys.stderr + ) + + return exit_code + + +def main() -> int: + """Execute entry_point routine.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--overwrite", + help="Try to automatically fix the offending files (e.g., by re-formatting).", + action="store_true", + ) + parser.add_argument( + "--select", + help=( + "If set, only the selected steps are executed. " + "This is practical if some of the steps failed and you want to " + "fix them in isolation. " + "The steps are given as a space-separated list of: " + + " ".join(value.value for value in Step) + ), + metavar="", + nargs="+", + choices=[value.value for value in Step], + ) + parser.add_argument( + "--skip", + help=( + "If set, skips the specified steps. " + "This is practical if some of the steps passed and " + "you want to fix the remainder in isolation. " + "The steps are given as a space-separated list of: " + + " ".join(value.value for value in Step) + ), + metavar="", + nargs="+", + choices=[value.value for value in Step], + ) + + args = parser.parse_args() + + overwrite = bool(args.overwrite) + + selects = ( + [Step(value) for value in args.select] + if args.select is not None + else [value for value in Step] + ) + skips = [Step(value) for value in args.skip] if args.skip is not None else [] + + src_root = pathlib.Path(os.path.realpath(__file__)).parent.parent + + if Step.REFORMAT in selects and Step.REFORMAT not in skips: + print("Re-formatting...") + reformat_targets = [ + "codegen/codegen.py", + "codegen/download_aas_core_meta_model.py", + "continuous_integration_of_dev_scripts", + "update_to_aas_core_meta_codegen.py", + ] + if overwrite: + exit_code = call_and_report( + verb="black", + cmd=["black"] + reformat_targets, + cwd=src_root, + ) + if exit_code != 0: + return 1 + else: + exit_code = call_and_report( + verb="check with black", + cmd=["black", "--check"] + reformat_targets, + cwd=src_root, + ) + if exit_code != 0: + return 1 + else: + print("Skipped re-formatting.") + + if Step.MYPY in selects and Step.MYPY not in skips: + print("Mypy'ing...") + mypy_targets = [ + "codegen/codegen.py", + "codegen/download_aas_core_meta_model.py", + "continuous_integration_of_dev_scripts", + "update_to_aas_core_meta_codegen.py", + ] + config_file = pathlib.Path("continuous_integration_of_dev_scripts") / "mypy.ini" + + exit_code = call_and_report( + verb="mypy", + cmd=["mypy", "--strict", "--config-file", str(config_file)] + mypy_targets, + cwd=src_root, + ) + if exit_code != 0: + return 1 + else: + print("Skipped mypy'ing.") + + if Step.PYLINT in selects and Step.PYLINT not in skips: + print("Pylint'ing...") + pylint_targets = [ + "codegen/codegen.py", + "codegen/download_aas_core_meta_model.py", + "continuous_integration_of_dev_scripts", + "update_to_aas_core_meta_codegen.py", + ] + rcfile = pathlib.Path("continuous_integration_of_dev_scripts") / "pylint.rc" + + exit_code = call_and_report( + verb="pylint", + cmd=["pylint", f"--rcfile={rcfile}"] + pylint_targets, + cwd=src_root, + ) + if exit_code != 0: + return 1 + else: + print("Skipped pylint'ing.") + + if Step.DOCTEST in selects and Step.DOCTEST not in skips: + print("Doctest'ing...") + + for module_name in [ + "continuous_integration_of_dev_scripts", + ]: + for pth in (src_root / module_name).glob("**/*.py"): + if pth.name == "__main__.py": + continue + + # NOTE (mristin, 2022-12-08): + # The subprocess calls are expensive, call only if there is an actual + # doctest + text = pth.read_text(encoding="utf-8") + if ">>>" in text: + exit_code = call_and_report( + verb="doctest", + cmd=[sys.executable, "-m", "doctest", str(pth)], + cwd=src_root, + ) + if exit_code != 0: + return 1 + else: + print("Skipped doctest'ing.") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev_scripts/continuous_integration_of_dev_scripts/pylint.rc b/dev_scripts/continuous_integration_of_dev_scripts/pylint.rc new file mode 100644 index 00000000..e3249629 --- /dev/null +++ b/dev_scripts/continuous_integration_of_dev_scripts/pylint.rc @@ -0,0 +1,10 @@ +[TYPECHECK] +ignored-modules = numpy +ignored-classes = numpy,PurePath +generated-members=bottle\.request\.forms\.decode,bottle\.request\.query\.decode + +[FORMAT] +max-line-length=120 + +[MESSAGES CONTROL] +disable=too-few-public-methods,len-as-condition,duplicate-code,no-else-raise,no-else-return,too-many-locals,too-many-branches,too-many-nested-blocks,too-many-return-statements,unsubscriptable-object,not-an-iterable,broad-except,too-many-statements,protected-access,unnecessary-pass,too-many-statements,too-many-arguments,no-member,too-many-instance-attributes,too-many-lines,undefined-variable,unnecessary-lambda,assignment-from-none,useless-return,unused-argument,too-many-boolean-expressions,consider-using-f-string,use-dict-literal,invalid-name,no-else-continue,no-else-break,unneeded-not,too-many-public-methods,line-too-long diff --git a/dev_scripts/pyproject.toml b/dev_scripts/pyproject.toml index 65d77ad9..821d78c2 100644 --- a/dev_scripts/pyproject.toml +++ b/dev_scripts/pyproject.toml @@ -26,5 +26,13 @@ dependencies = [ "requests>=2.32.5", ] +[dependency-groups] +dev = [ + "black==25.9.0", + "mypy==1.18.2", + "pylint==3.3.8", + "types-requests>=2.32.4", +] + [tool.setuptools.packages.find] -exclude = ["tests", "continuous_integration", "dev_scripts"] +exclude = ["continuous_integration_of_dev_scripts"]