diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 5e9e767c383..14b1f546e04 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -30,47 +30,8 @@ runs: requirements_doc.txt requirements.txt - - name: Install Linux System Deps. - if: runner.os == 'Linux' - shell: bash - run: | - sudo apt-get update - sudo apt-get install -y build-essential python3-setuptools python3-tk libgtk2.0 - - - name: SWIG Install (Linux) - if: runner.os == 'Linux' - uses: mmomtchev/setup-swig@v4 - with: - version: v4.2.1 - - - name: SWIG Install (macOS) - if: runner.os == 'macOS' - shell: bash - env: - HOMEBREW_NO_AUTO_UPDATE: 1 - HOMEBREW_NO_INSTALL_UPGRADE: 1 - HOMEBREW_NO_ANALYTICS: 1 - run: brew install swig || true - - - name: SWIG Install (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: | - $swigDir = "C:\Program Files\SWIG" - if (!(Test-Path $swigDir)) {New-Item -ItemType Directory -Path $swigDir | Out-Null} - $swigZip = "$swigDir\swigwin-4.2.1.zip" - $swigUrl = "https://sourceforge.net/projects/swig/files/swigwin/swigwin-4.2.1/swigwin-4.2.1.zip/download" - Start-Process -NoNewWindow -Wait -FilePath "curl.exe" -ArgumentList "-L -o `"$swigZip`" `"$swigUrl`"" - if (!(Test-Path $swigZip) -or ((Get-Item $swigZip).Length -lt 500KB)) { Write-Host "Download failed or file is corrupted." } - Expand-Archive -Path $swigZip -DestinationPath $swigDir -Force - - - name: "Add Basilisk and SWIG paths" - if: runner.os == 'Windows' - shell: pwsh - run: | - $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path - $newPath = “C:\Program Files\SWIG\swigwin-4.2.1;$oldpath;${{ env.GITHUB_WORKSPACE }}\dist3\Basilisk” - echo "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Setup system dependencies + uses: ./.github/actions/setup - name: Install requirements shell: bash diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000000..d58f05eca9e --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,48 @@ +name: setup +description: > + Sets up the required system dependencies to perform a Basilisk build + +runs: + using: "composite" + steps: + - name: Install Linux System Deps. + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y build-essential python3-setuptools python3-tk libgtk2.0 + + - name: SWIG Install (Linux) + if: runner.os == 'Linux' + uses: mmomtchev/setup-swig@v4 + with: + version: v4.2.1 + + - name: SWIG Install (macOS) + if: runner.os == 'macOS' + shell: bash + env: + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_UPGRADE: 1 + HOMEBREW_NO_ANALYTICS: 1 + run: brew install swig || true + + - name: SWIG Install (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $swigDir = "C:\Program Files\SWIG" + if (!(Test-Path $swigDir)) {New-Item -ItemType Directory -Path $swigDir | Out-Null} + $swigZip = "$swigDir\swigwin-4.2.1.zip" + $swigUrl = "https://sourceforge.net/projects/swig/files/swigwin/swigwin-4.2.1/swigwin-4.2.1.zip/download" + Start-Process -NoNewWindow -Wait -FilePath "curl.exe" -ArgumentList "-L -o `"$swigZip`" `"$swigUrl`"" + if (!(Test-Path $swigZip) -or ((Get-Item $swigZip).Length -lt 500KB)) { Write-Host "Download failed or file is corrupted." } + Expand-Archive -Path $swigZip -DestinationPath $swigDir -Force + + - name: "Add Basilisk and SWIG paths" + if: runner.os == 'Windows' + shell: pwsh + run: | + $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path + $newPath = “C:\Program Files\SWIG\swigwin-4.2.1;$oldpath;${{ env.GITHUB_WORKSPACE }}\dist3\Basilisk” + echo "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 6d9c4dc94b4..2bddf083669 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -26,6 +26,7 @@ jobs: token: ${{ secrets.BOT_ACCESS_TOKEN }} - name: Bump version + id: bump run: ./.github/workflows/version-bumper.sh ./docs/source/bskVersion.txt - name: Commit and push @@ -35,35 +36,12 @@ jobs: git commit -a -m "[AUTO] Bump version number" || echo "No changes" git push || true - build-ubuntu-latest-wheels: - name: Build ubuntu-latest wheels - needs: bump_version - # Allow for manual runs to generate new wheels - if: ${{ always() }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - # The 'Build wheel' step will perform the actual build. However, we want - # all pre build setup to still be performed - - uses: ./.github/actions/build - with: - python-version: ${{ matrix.python-version }} - extra-apt: "cmake" - skip-build: true - - - name: Build wheel + - name: Create tag + if: ${{ steps.bump.outputs.updated_version != '' }} run: | - python -m pip wheel . -v --wheel-dir /tmp/wheelhouse - - - uses: actions/upload-artifact@v4 - with: - name: basilisk-wheels_ubuntu-22.04_python${{ matrix['python-version'] }} - path: /tmp/wheelhouse/**/*asilisk*.whl + git tag -a "v${{ steps.bump.outputs.updated_version }}" \ + -m "Release v${{ steps.bump.outputs.updated_version }}" + git push --tags build_documentation: name: macOS Docs Deployment diff --git a/.github/workflows/publish-wheels.yml b/.github/workflows/publish-wheels.yml new file mode 100644 index 00000000000..f9ccbb409d2 --- /dev/null +++ b/.github/workflows/publish-wheels.yml @@ -0,0 +1,144 @@ +name: Publish Wheels + +on: + push: + tags: + - "v[0-9]*" + - "test*" + +jobs: + build-wheels: + name: Build Basilisk Wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-13 # x86_64 + - macos-latest # ARM64 + - ubuntu-latest # x86_64 + - ubuntu-22.04-arm # ARM64 + - windows-latest # x86_64 + + steps: + - name: Checkout Code + uses: actions/checkout@v5 + with: + ref: ${{ github.ref }} + # Workaround for macos-13 (Intel). Ensures the conan build uses the right + # SSL cert. + - name: Set CA bundle (macos-13 only) + if: ${{ matrix.os == 'macos-13' }} + shell: bash + run: | + python -m pip install -U certifi + CA="$(python -c 'import certifi; print(certifi.where())')" + { + echo "REQUESTS_CA_BUNDLE=$CA" # Python requests / Conan + echo "PIP_CERT=$CA" # pip + echo "SSL_CERT_FILE=$CA" # OpenSSL consumers + echo "CURL_CA_BUNDLE=$CA" # curl / CMake file(DOWNLOAD) + echo "CMAKE_TLS_CAINFO=$CA" # CMake + } >> "$GITHUB_ENV" + echo "Using CA: $CA" + + - name: Setup system dependencies + uses: ./.github/actions/setup + + - name: Build wheels + uses: pypa/cibuildwheel@v3.1.4 + env: + CONAN_ARGS: "--opNav True --mujoco True --mujocoReplay True --recorderPropertyRollback True" + CIBW_TEST_REQUIRES_WINDOWS: "numpy>=2.1" + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + make_sdist: + name: Make SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.ref }} + + - uses: actions/setup-python@v5 + with: + python-version: 3.13 + + - name: Build SDist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: cibw-sdist + path: dist/*.tar.gz + + publish: + name: Publish to PyPI + needs: [build-wheels, make_sdist] + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Checkout tagged source + uses: actions/checkout@v5 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + + - name: Download wheels + uses: actions/download-artifact@v4 + with: + pattern: cibw-wheels-* + merge-multiple: true + path: dist + + - name: Download sdist + uses: actions/download-artifact@v4 + with: + name: cibw-sdist + path: dist + + - name: Verify versions match tag + shell: bash + if: startsWith(github.ref, 'refs/tags/v') + run: | + TAG="${GITHUB_REF##*/}" + VER="${TAG#v}" + echo "Tag: $TAG Version: $VER" + shopt -s nullglob + [[ -e dist/*"$VER"*.whl ]] || { echo "No wheel with $VER"; exit 1; } + [[ -e dist/*"$VER"*.tar.gz ]] || { echo "No sdist with $VER"; exit 1; } + echo "OK — artifacts match $VER" + + - name: Show release info + shell: bash + run: | + echo "Release Info:" + TAG="${GITHUB_REF##*/}" + VER="${TAG#v}" + echo "Release tag: $TAG" + echo "Release ver: $VER" + echo + echo "Artifacts in dist/:" + ls -lh dist || true + + - name: Publish to TestPyPI (test tags) + if: startsWith(github.ref, 'refs/tags/test') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: dist + skip-existing: true + verbose: true + + - name: Publish to PyPI (real releases) + if: startsWith(github.ref, 'refs/tags/v') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist diff --git a/.github/workflows/version-bumper.sh b/.github/workflows/version-bumper.sh index c843b4bb794..66ce714a233 100755 --- a/.github/workflows/version-bumper.sh +++ b/.github/workflows/version-bumper.sh @@ -17,4 +17,12 @@ while IFS= read -r version || [[ -n "$version" ]]; do fi done < $1 -echo "$updated_version" > $1 \ No newline at end of file +echo "$updated_version" > $1 + +# Expose the update versions to GitHub Actions +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "version=$version" + echo "updated_version=$updated_version" + } >> "$GITHUB_OUTPUT" +fi diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000000..e21cfb89a66 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,13 @@ +prune examples +prune docs +prune src/tests +prune **/_UnitTest +prune **/_Documentation + +global-exclude *.pdf *.png *.jpg *.jpeg *.gif *.JPG *.svg *.psd +global-exclude *.ipynb *.bib *.tex +global-exclude *.mex* *.o *.a *.so *.dylib *.dll +global-exclude build/** CMakeFiles/** .git/** .github/** .venv/** + +# This file is used by SWIG +include src/fswAlgorithms/effectorInterfaces/thrForceMapping/_UnitTest/Support/Results_thrForceMapping.py diff --git a/conanfile.py b/conanfile.py index 705d47b7602..ec34cb6113d 100644 --- a/conanfile.py +++ b/conanfile.py @@ -7,6 +7,7 @@ import sys from datetime import datetime from typing import Optional, Callable +from glob import glob import importlib.metadata from packaging.requirements import Requirement @@ -252,6 +253,14 @@ def layout(self): self.folders.build = str(self.options.get_safe("buildFolder")) def generate(self): + if self.settings.os == "Windows": + # Ensure dependent DLLs are copied into the Basilisk package + # directory inside the build folder so they can be discovered by + # packaging tools (delvewheel) and included in wheels. + basilisk_dst = os.path.join(self.build_folder, "Basilisk") + for dep in self.dependencies.values(): + for bindir in dep.cpp_info.bindirs: + copy(self, "*.dll", bindir, basilisk_dst) if self.settings.os == "Windows": for dep in self.dependencies.values(): for libdir in dep.cpp_info.bindirs: @@ -309,6 +318,8 @@ def generate(self): # Set the minimum buildable MacOS version. # tc.cache_variables["CMAKE_OSX_DEPLOYMENT_TARGET"] = "10.13" tc.parallel = True + if self.options.get_safe("pyLimitedAPI"): + tc.cache_variables["PY_LIMITED_API"] = str(self.options.pyLimitedAPI) # Generate! tc.generate() @@ -329,6 +340,41 @@ def build(self): cmake.build() print("Total Build Time: " + str(datetime.now() - start)) print(f"{statusColor}The Basilisk build is successful and the scripts are ready to run{endColor}") + # On Windows, copy project-built DLLs next to the Python extension modules + # so they are bundled in the wheel and resolvable at runtime without PATH tweaks. + if self.settings.os == "Windows": + basilisk_dst_root = os.path.join(self.build_folder, "Basilisk") + common_srcs = [ + os.path.join(self.build_folder, "bin"), + os.path.join(self.build_folder, "Release"), + os.path.join(self.build_folder, "Debug"), + ] + for src in common_srcs: + if os.path.isdir(src): + try: + copy(self, "*.dll", src, basilisk_dst_root) + except Exception as e: + self.output.warning(f"Failed to copy DLLs from {src}: {e}") + + # As a fallback, scan the build tree for any remaining DLLs. + for root, _dirs, files in os.walk(self.build_folder): + # Skip the destination to avoid self-copy + if os.path.commonpath([root, basilisk_dst_root]) == basilisk_dst_root: + continue + if any(f.lower().endswith(".dll") for f in files): + try: + copy(self, "*.dll", root, basilisk_dst_root) + except Exception as e: + self.output.warning(f"Failed to copy DLLs from {root}: {e}") + + # Rename DLLs to lowercase + for path in glob(os.path.join(basilisk_dst_root, "*.dll")): + base = os.path.basename(path) + lower = base.lower() + if base != lower: + tmp = os.path.join(basilisk_dst_root, f".{lower}.tmp") + os.replace(path, tmp) + os.replace(tmp, os.path.join(basilisk_dst_root, lower)) else: print(f"{statusColor}Finished configuring the Basilisk project.{endColor}") if self.settings.os != "Linux": diff --git a/examples/OpNavScenarios/modelsOpNav/BSK_OpNavFsw.py b/examples/OpNavScenarios/modelsOpNav/BSK_OpNavFsw.py index 0cca00a3be1..cf51ffcf702 100644 --- a/examples/OpNavScenarios/modelsOpNav/BSK_OpNavFsw.py +++ b/examples/OpNavScenarios/modelsOpNav/BSK_OpNavFsw.py @@ -30,6 +30,7 @@ import math +from pathlib import Path import numpy as np from Basilisk import __path__ @@ -64,6 +65,15 @@ centerRadiusCNNIncluded = False +def get_repo_root(start: Path = Path(__file__).resolve()) -> Path: + for parent in [start] + list(start.parents): + if (parent / ".git").exists(): + return parent + if (parent / "pyproject.toml").exists() or (parent / "setup.cfg").exists(): + return parent + raise RuntimeError("Repo root not found") + + class BSKFswModels(): """ OpNav BSK FSW Models @@ -521,7 +531,14 @@ def SetCNNOpNav(self, SimBase): self.opNavCNN.imageInMsg.subscribeTo(SimBase.DynModels.cameraMod.imageOutMsg) self.opNavCNN.opnavCirclesOutMsg = self.opnavCirclesMsg self.opNavCNN.pixelNoise = [5,5,5] - self.opNavCNN.pathToNetwork = bskPath + "/../../src/fswAlgorithms/imageProcessing/centerRadiusCNN/CAD.onnx" + self.opNavCNN.pathToNetwork = str( + get_repo_root() + / "src" + / "fswAlgorithms" + / "imageProcessing" + / "centerRadiusCNN" + / "CAD.onnx" + ) def SetImageProcessing(self, SimBase): self.imageProcessing.imageInMsg.subscribeTo(SimBase.DynModels.cameraMod.imageOutMsg) diff --git a/pyproject.toml b/pyproject.toml index da017a57008..d5c20f02912 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,11 +3,12 @@ build-backend = "setuptools.build_meta" requires = [ "setuptools>=70.1.0,<=78.1.0", # Required for "bdist_wheel" to work correctly. "setuptools-scm>=8.0,<=8.2.1", # Automatically include all Git-controlled files in sdist - "packaging>=22,<=24.2", # Due to incompatibility: https://github.com/pypa/setuptools/issues/4483 + "packaging>=24.0,<26", # needed to find the numpy include headers, # pin to lowest runtime allowed version to take advantage of backwards compatibility - "numpy<=2.2.4,>=1.24.4", + "numpy>=1.24.4,<2.4.0; python_version < '3.13'", + "numpy>=2.0,<2.4.0; python_version >= '3.13'", # Requirements for building Basilisk through conanfile "conan>=2.0.5,<=2.15.1", @@ -16,7 +17,7 @@ requires = [ ] [project] -name = 'Basilisk' +name = 'bsk' dynamic = ["version", "dependencies"] requires-python = ">=3.8, <3.14" @@ -55,3 +56,36 @@ Basilisk = [ [tool.setuptools.dynamic] version = {file = "docs/source/bskVersion.txt"} dependencies = {file = "requirements.txt"} + +[project.optional-dependencies] +test = [ + "psutil", + "pytest-error-for-skips", + "pytest", +] + +[tool.cibuildwheel] +build = ["cp39-*", "cp310-*", "cp311-*", "cp312-*", "cp313-*"] +skip = ["*-win32", "cp38-macosx_arm64", "*-musllinux_*"] +build-verbosity = 1 +test-extras = ["test"] +test-command = "bskLargeData && pytest -v {project}/src/tests -m \"not ciSkip and not scenarioTest\"" + +[tool.cibuildwheel.environment] +CMAKE_BUILD_TYPE = "Release" + +[tool.cibuildwheel.macos.environment] +MACOSX_DEPLOYMENT_TARGET = "11.0" +CMAKE_OSX_DEPLOYMENT_TARGET = "11.0" + +[tool.cibuildwheel.windows] +before-build = "pip install delvewheel" +repair-wheel-command = "python -m delvewheel repair -v --ignore-existing --analyze-existing -w {dest_dir} {wheel}" + +[tool.cibuildwheel.linux] +archs = ["native"] +repair-wheel-command = [ + "auditwheel repair --strip -w {dest_dir} {wheel}" +] +manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" diff --git a/requirements.txt b/requirements.txt index 69c66b4e504..53700bb2a09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ pandas<=2.2.3,>=2.0.3 matplotlib<=3.10.1,>=3.7.5 -numpy<=2.2.4,>=1.24.4 +numpy>=1.24.4,<2.4.0; python_version < "3.13" +numpy>=2.0,<2.4.0; python_version >= "3.13" colorama==0.4.6 tqdm==4.67.1 pillow<=11.2.1,>=10.4.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index 459461ff40a..52552f009b5 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,7 +1,7 @@ wheel>=0.45.1,<=0.46.1 cmake>=3.26,<4.0 conan>=2.0.5,<=2.15.1 -packaging>=22,<=24.2 +packaging>=24,<26 setuptools>=70.1.0,<=78.1.0 setuptools-scm>=8.0,<=8.2.1 diff --git a/setup.py b/setup.py index dba17adcc96..6d93ea07abc 100644 --- a/setup.py +++ b/setup.py @@ -105,6 +105,9 @@ def run(self) -> None: self.distribution.packages.append(pkg) self.distribution.package_dir[pkg] = os.path.relpath(pkg_dir, start=HERE) + pd = self.distribution.package_data.setdefault(pkg, []) + pd += ["*.dll", "**/*.dll", "*.pyd", "**/*.pyd"] + if self.editable_mode and len(self.distribution.packages) == 0: raise Exception("Tried to install in editable mode, but packages have not been prepared yet! " \ "Please install via `python conanfile.py` instead!") @@ -146,4 +149,6 @@ def run(self) -> None: # XXX: Override build_ext with ConanExtension builder. cmdclass={'build_ext': BuildConanExtCommand}, + zip_safe=False, + include_package_data=True, ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15945b0eb58..0c4f38f4259 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -206,7 +206,6 @@ function(generate_package_libraries INIT_DIRECTORY AllLibs) # Link all necessary libraries target_link_libraries(${LIB_NAME} PRIVATE ModuleIdGenerator) - target_link_libraries(${LIB_NAME} PRIVATE ${PYTHON3_MODULE}) target_link_libraries(${LIB_NAME} PRIVATE Eigen3::Eigen3) if(${PARENT_DIR_NAME} STREQUAL "mujocoDynamics") target_link_libraries(${LIB_NAME} PUBLIC dynamicsLib) @@ -362,6 +361,11 @@ function(generate_package_targets TARGET_LIST LIB_DEP_LIST MODULE_DIR) target_link_libraries(${TARGET_NAME} PRIVATE ${LIB}) endforeach() + target_link_libraries(${TARGET_NAME} PRIVATE ${PYTHON3_MODULE}) + if(PY_LIMITED_API AND NOT PY_LIMITED_API STREQUAL "") + target_compile_definitions(${TARGET_NAME} PRIVATE "Py_LIMITED_API=${PY_LIMITED_API}") + endif() + if(${MODULE_DIR} STREQUAL "ExternalModules") set_target_properties(${TARGET_NAME} PROPERTIES FOLDER ${MODULE_DIR}) else() diff --git a/src/tests/test_scenarioSepMomentumManagement.py b/src/tests/test_scenarioSepMomentumManagement.py index 4273f5b00c5..512b66ed569 100644 --- a/src/tests/test_scenarioSepMomentumManagement.py +++ b/src/tests/test_scenarioSepMomentumManagement.py @@ -52,6 +52,7 @@ @pytest.mark.parametrize("withFunction", [False, True]) +@pytest.mark.scenarioTest # provide a unique test method name, starting with test_ def test_sepMomentumManagement(withFunction): """This function is called by the pytest environment."""