diff --git a/.github/workflows/openstudio-server-tests.yml b/.github/workflows/openstudio-server-tests.yml index 4c61a5a26..7195c0c14 100644 --- a/.github/workflows/openstudio-server-tests.yml +++ b/.github/workflows/openstudio-server-tests.yml @@ -98,6 +98,41 @@ jobs: with: name: openstudio-server-gems-darwin path: build/NREL/export/*.tar.gz + macos-arm64-test: + runs-on: macos-15 + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: setup + shell: bash + run: ./ci/github-actions/setup.sh + env: + BUILD_TYPE: test + - name: unit-test + shell: bash + run: ./ci/github-actions/test.sh + env: + BUILD_TYPE: test + - name: integration + shell: bash + run: ./ci/github-actions/test.sh + env: + BUILD_TYPE: integration + SKIP_URBANOPT_ALGO: true # Set this when excluding urbanopt-cli gem from server. + - name: logs + if: ${{ failure() }} + shell: bash + run: ./ci/github-actions/print_logs.sh + - name: build gem package + if: ${{ success() }} + shell: bash + run: ./ci/github-actions/export_build_osx.sh + - name: upload gem package + if: ${{ success() }} + uses: actions/upload-artifact@v4 + with: + name: openstudio-server-gems-darwin-arm64 + path: build/NREL/export/*.tar.gz docker: runs-on: ubuntu-24.04 steps: diff --git a/bin/openstudio_meta b/bin/openstudio_meta index 4a226cd99..46be6c938 100755 --- a/bin/openstudio_meta +++ b/bin/openstudio_meta @@ -393,6 +393,8 @@ class InstallGems puts "Copying files from '#{src}' to '#{dest}'" if options[:debug] Dir.glob(src + '/*').each do |f| next if /\.git/ =~ f + next if File.basename(f) == 'build' + next if f.end_with?('.tar.gz', '.tgz') puts "Copying files from '#{f}' to '#{dest}/.'" if options[:debug] FileUtils.cp_r(f, dest + '/.') end @@ -410,7 +412,7 @@ class InstallGems Dir.glob(dest + '/**/OpenStudio-workflow*/test/').each {|p| FileUtils.rm_rf(p)} platform = 'win32' if /darwin/.match(RUBY_PLATFORM) - platform = 'darwin' + platform = /arm64|aarch64/.match(RUBY_PLATFORM) ? 'darwin-arm64' : 'darwin' elsif /linux/.match(RUBY_PLATFORM) platform = 'linux' end @@ -421,6 +423,7 @@ class InstallGems server_sha = `git -C \"#{src}\" rev-parse --short=10 HEAD`.strip puts "Server SHA = #{server_sha}" if options[:debug] + ENV['COPYFILE_DISABLE'] = '1' if /darwin/.match(RUBY_PLATFORM) sys_call = "tar -czf OpenStudio-server-#{server_sha}-#{platform}.tar.gz OpenStudio-server" puts "Creating tar.gz in #{options[:export_dir]}" puts sys_call if options[:debug] diff --git a/ci/github-actions/export_build_osx.sh b/ci/github-actions/export_build_osx.sh index 881fe1c4a..d718dd77c 100755 --- a/ci/github-actions/export_build_osx.sh +++ b/ci/github-actions/export_build_osx.sh @@ -1,17 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + #empty dir for export mkdir -p $GITHUB_WORKSPACE/build/NREL/export mkdir -p $HOME/build/NREL/export +ARCH=$(uname -m) +case "${ARCH}" in + arm64) + OS_ARCH="arm64" + TARBALL_PLATFORM="darwin-arm64" + ;; + x86_64) + OS_ARCH="x86_64" + TARBALL_PLATFORM="darwin" + ;; + *) + echo "Error: Unsupported macOS architecture '${ARCH}'" >&2 + exit 1 + ;; +esac -export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64 +export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-${OS_ARCH} export PATH="$GITHUB_WORKSPACE/gems/bin:/usr/local/ruby/bin:$HOME/$OS_NAME_WITH_PLUS/bin:$PATH" export RUBYLIB="$HOME/$OS_NAME_WITH_PLUS/Ruby" export GEM_HOME="$GITHUB_WORKSPACE/gems" export GEM_PATH="$GITHUB_WORKSPACE/gems:$GITHUB_WORKSPACE/gems/bundler/gems" -oss_filename="OpenStudio-server-$(git -C "${GITHUB_WORKSPACE}" rev-parse --short=10 HEAD)-darwin.tar.gz" +oss_filename="OpenStudio-server-$(git -C "${GITHUB_WORKSPACE}" rev-parse --short=10 HEAD)-${TARBALL_PLATFORM}.tar.gz" # Build the gems and output to tar gz file. You need to specify a dir outside of the repo to export or # openstudio_meta will error. Then for sake of using relative dirs with GitHub upload-artifacts, cp the file to # the repo working directory. /usr/local/ruby/bin/ruby "${GITHUB_WORKSPACE}/bin/openstudio_meta" install_gems --export="${HOME}/build/NREL/export" -cp $HOME/build/NREL/export/$oss_filename $GITHUB_WORKSPACE/build/NREL/export/$oss_filename \ No newline at end of file +cp "$HOME/build/NREL/export/$oss_filename" "$GITHUB_WORKSPACE/build/NREL/export/$oss_filename" + +# Verify exported tarball structure, architecture, and dylib portability for OpenStudio-PAT +verify_dir=$(mktemp -d) +trap 'rm -rf "${verify_dir}"' EXIT +tar -xzf "$GITHUB_WORKSPACE/build/NREL/export/$oss_filename" -C "${verify_dir}" + +for required_dir in \ + "${verify_dir}/OpenStudio-server/bin" \ + "${verify_dir}/OpenStudio-server/server" \ + "${verify_dir}/OpenStudio-server/gems"; do + if [ ! -d "${required_dir}" ]; then + echo "FAIL: Missing required directory: ${required_dir}" >&2 + exit 1 + fi +done + +for required_file in \ + "${verify_dir}/OpenStudio-server/bin/openstudio_meta" \ + "${verify_dir}/OpenStudio-server/Gemfile" \ + "${verify_dir}/OpenStudio-server/server/Gemfile" \ + "${verify_dir}/OpenStudio-server/server/Gemfile.lock"; do + if [ ! -f "${required_file}" ]; then + echo "FAIL: Missing required file: ${required_file}" >&2 + exit 1 + fi +done + +dot_under=$(find "${verify_dir}" -name '._*' | wc -l | tr -d ' ') +if [ "${dot_under}" -gt 0 ]; then + echo "FAIL: Found ${dot_under} AppleDouble ._* file(s) in archive" >&2 + exit 1 +fi + +nested_tarballs=$(find "${verify_dir}" -maxdepth 3 -name 'OpenStudio-server-*.tar.gz' | wc -l | tr -d ' ') +if [ "${nested_tarballs}" -gt 0 ] || [ -d "${verify_dir}/OpenStudio-server/build" ]; then + echo "FAIL: Found nested build artifacts inside OpenStudio-server archive" >&2 + exit 1 +fi + +bundle_files=$(find "${verify_dir}/OpenStudio-server/gems" -name '*.bundle' -type f) +bundle_count=$(echo "${bundle_files}" | grep -c '.' || true) +if [ "${bundle_count}" -eq 0 ]; then + echo "FAIL: No .bundle native extensions found in gems/" >&2 + exit 1 +fi + +while IFS= read -r bfile; do + [ -z "${bfile}" ] && continue + file_info=$(file "${bfile}") + if ! echo "${file_info}" | grep -q "Mach-O.*${OS_ARCH}"; then + echo "FAIL: Wrong architecture for ${bfile}: ${file_info}" >&2 + exit 1 + fi + bad_load=$(otool -L "${bfile}" | tail -n +2 | awk '{print $1}' | grep -E '^(/opt/homebrew|/usr/local/|/Users/|/private/)' || true) + if [ -n "${bad_load}" ]; then + echo "FAIL: Non-portable dylib reference in ${bfile}: ${bad_load}" >&2 + exit 1 + fi +done <<< "${bundle_files}" + +echo "Verified ${oss_filename} (${bundle_count} native .bundle extensions for ${OS_ARCH})" \ No newline at end of file diff --git a/ci/github-actions/setup.sh b/ci/github-actions/setup.sh index 65199d1f0..f235761e9 100755 --- a/ci/github-actions/setup.sh +++ b/ci/github-actions/setup.sh @@ -15,37 +15,122 @@ if [ "${ImageOS}" == "ubuntu24" ] && [ "${BUILD_TYPE}" == "docker" ]; then else # sudo rvm implode --force # rvm PATH rewriting interferes with portable Ruby. if [ "${ImageOS}" == "macos15" ]; then + ARCH=$(uname -m) + case "${ARCH}" in + arm64) + RUBY_TARBALL="ruby-3.2.2-darwin-arm64.tar.gz" + MONGO_ARCHIVE="mongodb-macos-arm64-8.0.12" + MONGO_EXTRACT_DIR="mongodb-macos-aarch64-8.0.12" + OS_ARCH="arm64" + ;; + x86_64) + RUBY_TARBALL="ruby-3.2.2-darwin.tar.gz" + MONGO_ARCHIVE="mongodb-macos-x86_64-8.0.12" + MONGO_EXTRACT_DIR="mongodb-macos-x86_64-8.0.12" + OS_ARCH="x86_64" + ;; + *) + echo "Error: Unsupported macOS architecture '${ARCH}'" >&2 + exit 1 + ;; + esac brew update > $GITHUB_WORKSPACE/spec/files/logs/brew-update.log brew install pv tree coreutils shared-mime-info # install portable ruby - required for build that will eventually be published # see https://github.com/NatLabRockies/OpenStudio-PAT/wiki/Pat-Build-Notes - curl -SLO --insecure https://openstudio-resources.s3.amazonaws.com/pat-dependencies3/ruby-3.2.2-darwin.tar.gz - tar xzf ruby-3.2.2-darwin.tar.gz + curl -SLO --insecure "https://openstudio-resources.s3.amazonaws.com/pat-dependencies3/${RUBY_TARBALL}" + tar xzf "${RUBY_TARBALL}" exit_status_tar=$? if [ $exit_status_tar -ne 0 ]; then echo "Error: Failed to extract Ruby 3.2.2 archive" exit $exit_status_tar fi + sudo rm -rf /usr/local/ruby sudo mv ruby /usr/local/ otool -L /usr/local/ruby/bin/ruby - rm ruby-3.2.2-darwin.tar.gz + rm "${RUBY_TARBALL}" + + # Ensure static archives and headers for gmp, libyaml, and openssl@3 are staged in + # /usr/local/ruby so native gem extensions (psych, puma, etc.) link statically + # without embedding non-portable /opt/homebrew or /usr/local/opt dylib paths. + for pkg in gmp libyaml openssl@3; do + brew list "${pkg}" &>/dev/null || brew install "${pkg}" + pkg_prefix=$(brew --prefix "${pkg}") + cp -f "${pkg_prefix}"/lib/*.a /usr/local/ruby/lib/ + cp -Rf "${pkg_prefix}"/include/* /usr/local/ruby/include/ + done + + # Replace extinit.o and encinit.o in libruby.3.2-static.a (present on arm64) with dummy + # Init_ext/Init_enc so mkmf try_link0 checks against -lruby.3.2-static succeed without + # undefined symbols. + if ar -t /usr/local/ruby/lib/libruby.3.2-static.a | grep -Eq '^(extinit|encinit)\.o$'; then + dmyext_dir=$(mktemp -d) + printf 'void Init_ext(void) {}\nvoid Init_enc(void) {}\n' > "${dmyext_dir}/dmyext.c" + clang -c -arch "${OS_ARCH}" "${dmyext_dir}/dmyext.c" -o "${dmyext_dir}/dmyext.o" + ar -d /usr/local/ruby/lib/libruby.3.2-static.a extinit.o encinit.o + ar -r /usr/local/ruby/lib/libruby.3.2-static.a "${dmyext_dir}/dmyext.o" + ranlib /usr/local/ruby/lib/libruby.3.2-static.a + rm -rf "${dmyext_dir}" + if ar -t /usr/local/ruby/lib/libruby.3.2-static.a | grep -Eq '^(extinit|encinit)\.o$'; then + echo "Error: extinit.o/encinit.o still present in libruby.3.2-static.a" >&2 + exit 1 + fi + fi + + # Sanitize Conan/Homebrew build-runner paths in rbconfig.rb and disable pkg-config lookups + # in mkmf so native extensions only link against /usr/local/ruby/lib static archives. + /usr/local/ruby/bin/ruby -e ' + rbconfig = Dir.glob("/usr/local/ruby/lib/ruby/3.2.0/*-darwin*/rbconfig.rb").first + raise "Could not find rbconfig.rb under /usr/local/ruby" unless rbconfig + content = File.read(rbconfig) + replaced_conan = content.gsub!(/\/Users\/runner\/work\/conan-recipes\/conan-recipes\/conan-cache\/\.conan2\/p(?:\/b)?\/[^\/\s\x27":]+\/p/, "/usr/local/ruby") + replaced_brew = content.gsub!(/\/usr\/local\/opt\/(?:openssl@3|libyaml|gmp)/, "/usr/local/ruby") + raise "Expected Conan or Homebrew paths in #{rbconfig}" unless replaced_conan || replaced_brew + raise "Remaining Conan paths in #{rbconfig}" if content.include?(".conan2") + raise "Remaining Homebrew opt paths in #{rbconfig}" if content.match?(/\/usr\/local\/opt\/(?:openssl@3|libyaml|gmp)/) + unless content.include?("--without-pkg-config") + patched_args = content.sub!("CONFIG[\"configure_args\"] = \" ", "CONFIG[\"configure_args\"] = \" \x27--without-pkg-config\x27 ") + raise "Failed to inject --without-pkg-config into #{rbconfig}" unless patched_args + end + File.write(rbconfig, content) + + rubygems = "/usr/local/ruby/lib/ruby/3.2.0/rubygems.rb" + rg_content = File.read(rubygems) + { + "require \x27rubygems/dependency_list\x27" => "autoload :DependencyList, File.expand_path(\"rubygems/dependency_list\", __dir__)", + "require \x27rubygems/installer\x27" => "autoload :Installer, File.expand_path(\"rubygems/installer\", __dir__)", + "require \x27rubygems/util/licenses\x27" => "autoload :Licenses, File.expand_path(\"rubygems/util/licenses\", __dir__)", + "require \x27rubygems/name_tuple\x27" => "autoload :NameTuple, File.expand_path(\"rubygems/name_tuple\", __dir__)", + "require \x27rubygems/request_set\x27" => "autoload :RequestSet, File.expand_path(\"rubygems/request_set\", __dir__)", + "require \x27rubygems/resolver\x27" => "autoload :Resolver, File.expand_path(\"rubygems/resolver\", __dir__)", + "require \x27rubygems/source\x27" => "autoload :Source, File.expand_path(\"rubygems/source\", __dir__)", + "require \x27rubygems/source_list\x27" => "autoload :SourceList, File.expand_path(\"rubygems/source_list\", __dir__)", + "require \x27rubygems/spec_fetcher\x27" => "autoload :SpecFetcher, File.expand_path(\"rubygems/spec_fetcher\", __dir__)" + }.each do |old_line, new_line| + if rg_content.include?(old_line) + raise "Failed to restore #{new_line} in #{rubygems}" unless rg_content.sub!(old_line, new_line) + end + end + File.write(rubygems, rg_content) + ' # Install mongodb from a download. Brew is hanging and requires building mongo. This also speeds up the builds. - curl -SLO https://fastdl.mongodb.org/osx/mongodb-macos-x86_64-8.0.12.tgz - tar xvzf mongodb-macos-x86_64-8.0.12.tgz + curl -SLO "https://fastdl.mongodb.org/osx/${MONGO_ARCHIVE}.tgz" + tar xvzf "${MONGO_ARCHIVE}.tgz" exit_status_tar=$? if [ $exit_status_tar -ne 0 ]; then echo "Error: Failed to extract Mongo 8.0.12 archive" exit $exit_status_tar fi - sudo cp mongodb-macos-x86_64-8.0.12/bin/* /usr/local/bin/ + sudo mkdir -p /usr/local/bin + sudo cp "${MONGO_EXTRACT_DIR}"/bin/* /usr/local/bin/ rm -r mongodb-macos* # Install openstudio -- Use the install script that is in this repo now, the one on OpenStudio/develop has changed - export OS_NAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64 - export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64 + export OS_NAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-${OS_ARCH} + export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-${OS_ARCH} #curl -SL --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/develop/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz #curl -SL --insecure https://github.com/NatLabRockies/OpenStudio/releases/download/v3.8.0/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz #curl -SL --insecure https://github.com/NatLabRockies/OpenStudio/releases/download/v${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz diff --git a/ci/github-actions/test.sh b/ci/github-actions/test.sh index 62cb6e310..6128c7781 100755 --- a/ci/github-actions/test.sh +++ b/ci/github-actions/test.sh @@ -2,8 +2,21 @@ # platform-specific config here (also in setup.sh): if [ "${ImageOS}" == "macos15" ]; then + ARCH=$(uname -m) + case "${ARCH}" in + arm64) + OS_ARCH="arm64" + ;; + x86_64) + OS_ARCH="x86_64" + ;; + *) + echo "Error: Unsupported macOS architecture '${ARCH}'" >&2 + exit 1 + ;; + esac # Dir containing openstudio - export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64 + export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-${OS_ARCH} export RUBYLIB="$HOME/$OS_NAME_WITH_PLUS/Ruby" export OPENSTUDIO_TEST_EXE="$HOME/$OS_NAME_WITH_PLUS/bin/openstudio" # re-export PATH, even though it's set in setup.sh.