Add CI for dev_scripts

.github/workflows/ci-for-dev-scripts.yml

@@ -0,0 +1,36 @@
+name: CI-for-dev-scripts
+
+on:
+  push:
+    branches: [ master, main ]
+    tags: [ "**" ]
+    paths:
+      - _dev_scripts/**
+  pull_request:
+    branches: [ "**" ]
+    paths:
+      - _dev_scripts/**
+
+jobs:
+  Execute-continuous-integration:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          pip3 install -e _dev_scripts --group _dev_scripts/pyproject.toml:dev
+
+      - name: Run checks
+        run: |
+          python3 _dev_scripts/continuous_integration_of_dev_scripts/precommit.py

dev_scripts/codegen/download_aas_core_meta_model.py

@@ -4,6 +4,7 @@
 import os
 import argparse
 import sys
+from typing import Dict, Union
 
 import requests
 
@@ -24,7 +25,7 @@ def _latest_commit_sha_for_path(
     Uses: GET /repos/{owner}/{repo}/commits?path=...&sha=...&per_page=1
     """
     url = f"{GITHUB_API}/repos/{owner}/{repo}/commits"
-    params = {"path": path, "sha": ref, "per_page": 1}
+    params: Dict[str, Union[str, int]] = {"path": path, "sha": ref, "per_page": 1}
     resp = requests.get(url, params=params, timeout=timeout)
     try:
         resp.raise_for_status()
@@ -43,6 +44,7 @@ def _latest_commit_sha_for_path(
     sha = commits[0].get("sha")
     if not sha or not isinstance(sha, str):
         raise RuntimeError("API did not return a valid commit SHA.")
+    assert isinstance(sha, str)
     return sha
 
 

dev_scripts/continuous_integration_of_dev_scripts/mypy.ini

@@ -0,0 +1 @@
+[mypy]

dev_scripts/continuous_integration_of_dev_scripts/precommit.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+
+"""Run pre-commit checks on the repository."""
+
+import argparse
+import enum
+import os
+import pathlib
+import shlex
+import subprocess
+import sys
+from typing import Optional, Mapping, Sequence
+
+
+# pylint: disable=unnecessary-comprehension
+
+
+class Step(enum.Enum):
+    """Enumerate different pre-commit steps."""
+
+    REFORMAT = "reformat"
+    MYPY = "mypy"
+    PYLINT = "pylint"
+    TEST = "test"
+    DOCTEST = "doctest"
+
+
+def call_and_report(
+    verb: str,
+    cmd: Sequence[str],
+    cwd: Optional[pathlib.Path] = None,
+    env: Optional[Mapping[str, str]] = None,
+) -> int:
+    """
+    Wrap a subprocess call with the reporting to STDERR if it failed.
+
+    Return 1 if there is an error and 0 otherwise.
+    """
+    cmd_str = " ".join(shlex.quote(part) for part in cmd)
+
+    if cwd is not None:
+        print(f"Executing from {cwd}: {cmd_str}")
+    else:
+        print(f"Executing: {cmd_str}")
+
+    exit_code = subprocess.call(cmd, cwd=str(cwd) if cwd is not None else None, env=env)
+
+    if exit_code != 0:
+        print(
+            f"Failed to {verb} with exit code {exit_code}: {cmd_str}", file=sys.stderr
+        )
+
+    return exit_code
+
+
+def main() -> int:
+    """Execute entry_point routine."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--overwrite",
+        help="Try to automatically fix the offending files (e.g., by re-formatting).",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--select",
+        help=(
+            "If set, only the selected steps are executed. "
+            "This is practical if some of the steps failed and you want to "
+            "fix them in isolation. "
+            "The steps are given as a space-separated list of: "
+            + " ".join(value.value for value in Step)
+        ),
+        metavar="",
+        nargs="+",
+        choices=[value.value for value in Step],
+    )
+    parser.add_argument(
+        "--skip",
+        help=(
+            "If set, skips the specified steps. "
+            "This is practical if some of the steps passed and "
+            "you want to fix the remainder in isolation. "
+            "The steps are given as a space-separated list of: "
+            + " ".join(value.value for value in Step)
+        ),
+        metavar="",
+        nargs="+",
+        choices=[value.value for value in Step],
+    )
+
+    args = parser.parse_args()
+
+    overwrite = bool(args.overwrite)
+
+    selects = (
+        [Step(value) for value in args.select]
+        if args.select is not None
+        else [value for value in Step]
+    )
+    skips = [Step(value) for value in args.skip] if args.skip is not None else []
+
+    src_root = pathlib.Path(os.path.realpath(__file__)).parent.parent
+
+    if Step.REFORMAT in selects and Step.REFORMAT not in skips:
+        print("Re-formatting...")
+        reformat_targets = [
+            "codegen/codegen.py",
+            "codegen/download_aas_core_meta_model.py",
+            "continuous_integration_of_dev_scripts",
+            "update_to_aas_core_meta_codegen.py",
+        ]
+        if overwrite:
+            exit_code = call_and_report(
+                verb="black",
+                cmd=["black"] + reformat_targets,
+                cwd=src_root,
+            )
+            if exit_code != 0:
+                return 1
+        else:
+            exit_code = call_and_report(
+                verb="check with black",
+                cmd=["black", "--check"] + reformat_targets,
+                cwd=src_root,
+            )
+            if exit_code != 0:
+                return 1
+    else:
+        print("Skipped re-formatting.")
+
+    if Step.MYPY in selects and Step.MYPY not in skips:
+        print("Mypy'ing...")
+        mypy_targets = [
+            "codegen/codegen.py",
+            "codegen/download_aas_core_meta_model.py",
+            "continuous_integration_of_dev_scripts",
+            "update_to_aas_core_meta_codegen.py",
+        ]
+        config_file = pathlib.Path("continuous_integration_of_dev_scripts") / "mypy.ini"
+
+        exit_code = call_and_report(
+            verb="mypy",
+            cmd=["mypy", "--strict", "--config-file", str(config_file)] + mypy_targets,
+            cwd=src_root,
+        )
+        if exit_code != 0:
+            return 1
+    else:
+        print("Skipped mypy'ing.")
+
+    if Step.PYLINT in selects and Step.PYLINT not in skips:
+        print("Pylint'ing...")
+        pylint_targets = [
+            "codegen/codegen.py",
+            "codegen/download_aas_core_meta_model.py",
+            "continuous_integration_of_dev_scripts",
+            "update_to_aas_core_meta_codegen.py",
+        ]
+        rcfile = pathlib.Path("continuous_integration_of_dev_scripts") / "pylint.rc"
+
+        exit_code = call_and_report(
+            verb="pylint",
+            cmd=["pylint", f"--rcfile={rcfile}"] + pylint_targets,
+            cwd=src_root,
+        )
+        if exit_code != 0:
+            return 1
+    else:
+        print("Skipped pylint'ing.")
+
+    if Step.DOCTEST in selects and Step.DOCTEST not in skips:
+        print("Doctest'ing...")
+
+        for module_name in [
+            "continuous_integration_of_dev_scripts",
+        ]:
+            for pth in (src_root / module_name).glob("**/*.py"):
+                if pth.name == "__main__.py":
+                    continue
+
+                # NOTE (mristin, 2022-12-08):
+                # The subprocess calls are expensive, call only if there is an actual
+                # doctest
+                text = pth.read_text(encoding="utf-8")
+                if ">>>" in text:
+                    exit_code = call_and_report(
+                        verb="doctest",
+                        cmd=[sys.executable, "-m", "doctest", str(pth)],
+                        cwd=src_root,
+                    )
+                    if exit_code != 0:
+                        return 1
+    else:
+        print("Skipped doctest'ing.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev_scripts/continuous_integration_of_dev_scripts/pylint.rc

@@ -0,0 +1,10 @@
+[TYPECHECK]
+ignored-modules = numpy
+ignored-classes = numpy,PurePath
+generated-members=bottle\.request\.forms\.decode,bottle\.request\.query\.decode
+
+[FORMAT]
+max-line-length=120
+
+[MESSAGES CONTROL]
+disable=too-few-public-methods,len-as-condition,duplicate-code,no-else-raise,no-else-return,too-many-locals,too-many-branches,too-many-nested-blocks,too-many-return-statements,unsubscriptable-object,not-an-iterable,broad-except,too-many-statements,protected-access,unnecessary-pass,too-many-statements,too-many-arguments,no-member,too-many-instance-attributes,too-many-lines,undefined-variable,unnecessary-lambda,assignment-from-none,useless-return,unused-argument,too-many-boolean-expressions,consider-using-f-string,use-dict-literal,invalid-name,no-else-continue,no-else-break,unneeded-not,too-many-public-methods,line-too-long

dev_scripts/pyproject.toml

@@ -26,5 +26,13 @@ dependencies = [
     "requests>=2.32.5",
 ]
 
+[dependency-groups]
+dev = [
+    "black==25.9.0",
+    "mypy==1.18.2",
+    "pylint==3.3.8",
+    "types-requests>=2.32.4",
+]
+
 [tool.setuptools.packages.find]
-exclude = ["tests", "continuous_integration", "dev_scripts"]
+exclude = ["continuous_integration_of_dev_scripts"]

dev_scripts/update_to_aas_core_meta_codegen.py

@@ -115,7 +115,7 @@ def _regenerate_code(
 
     target_dir = our_repo
 
-    print(f"Starting to run codegen script")
+    print("Starting to run codegen script")
     start = time.perf_counter()
 
     proc = subprocess.run(
@@ -128,6 +128,7 @@ def _regenerate_code(
             str(target_dir),
         ],
         cwd=str(codegen_dir),
+        check=True,
     )
 
     if proc.returncode != 0:
@@ -146,9 +147,9 @@ def _semantically_patch(our_repo: pathlib.Path) -> Optional[int]:
     verification_java = (
         our_repo / "src/main/java/aas_core/aas3_0/verification/Verification.java"
     )
-    assert verification_java.exists() and verification_java.is_file(), (
-        f"No Verification.java found to patch: {verification_java=}"
-    )
+    assert (
+        verification_java.exists() and verification_java.is_file()
+    ), f"No Verification.java found to patch: {verification_java=}"
 
     exit_code = _execute(cmd=["mvn", "package"], cwd=str(cwd))
     if exit_code is not None:
@@ -271,7 +272,7 @@ def _get_codegen_revision(our_repo: pathlib.Path) -> str | None:
         print(f"Cannot read codegen revision: {os_error}.")
 
     if codegen_sha is None:
-        print(f"Cannot read codegen revision.")
+        print("Cannot read codegen revision.")
 
     return codegen_sha
 
@@ -303,7 +304,7 @@ def _get_meta_model_revision(our_repo: pathlib.Path) -> str | None:
         print(f"Cannot read meta model revision: {os_error}.")
 
     if meta_model_sha is None:
-        print(f"Cannot read meta model revision.")
+        print("Cannot read meta model revision.")
 
     return meta_model_sha
 
@@ -320,7 +321,7 @@ def _get_testgen_revision(our_repo: pathlib.Path) -> str | None:
         print(f"Cannot read testgen revision: {os_error}.")
 
     if testgen_rev is None:
-        print(f"Cannot read testgen revision.")
+        print("Cannot read testgen revision.")
 
     return testgen_rev