From 8e830d6833af89a37af1f32db9791569ccc6058e Mon Sep 17 00:00:00 2001 From: Lynd Stringer Date: Tue, 4 Aug 2026 10:27:10 -0500 Subject: [PATCH 01/10] Added run_HeCBench.sh to aomp/bin. --- bin/run_HeCBench.sh | 193 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100755 bin/run_HeCBench.sh diff --git a/bin/run_HeCBench.sh b/bin/run_HeCBench.sh new file mode 100755 index 000000000..ee885ed0f --- /dev/null +++ b/bin/run_HeCBench.sh @@ -0,0 +1,193 @@ +#!/bin/bash + +# run_HeCBench.sh - runs HeCBench benchmarks in the $AOMP_REPOS_TEST dir. +# User can set RUN_OPTIONS to control what variants(openmp, hip) are selected. +# +# Verbose debug: ./run_HeCBench.sh -v +# VERBOSE=1 ./run_HeCBench.sh + +# --- Start standard header to set AOMP environment variables ---- +realpath=`realpath $0` +thisdir=`dirname $realpath` +export AOMP_USE_CCACHE=0 + +. $thisdir/aomp_common_vars +# --- end standard header ---- + +VERBOSE=${VERBOSE:-0} +while [ $# -gt 0 ]; do + case "$1" in + -v|--verbose) + VERBOSE=1 + shift + ;; + -h|--help) + echo "Usage: $0 [-v|--verbose]" + echo " RUN_OPTIONS openmp hip (default: both)" + echo " HECBENCH_LIST space-separated benchmark dirs to run" + echo " HECBENCH_TIMEOUT per-benchmark timeout seconds (default: 180)" + echo " LAUNCHER passed to Makefile run target" + echo " VERBOSE=1 same as -v" + exit 0 + ;; + *) + echo "ERROR: Unknown option: $1 (try -h)" + exit 1 + ;; + esac +done + +vlog() { + if [ "$VERBOSE" == 1 ]; then + echo "[verbose] $*" + fi +} + +# Setup AOMP variables +AOMP=${AOMP:-/usr/lib/aomp} +AOMPHIP=${AOMPHIP:-$AOMP} + +# Use function to set and test AOMP_GPU +setaompgpu + +RUN_OPTIONS=${RUN_OPTIONS:-"openmp hip"} +HECBENCH_TIMEOUT=${HECBENCH_TIMEOUT:-180} +HECBENCH_LIST=${HECBENCH_LIST:-""} +LAUNCHER=${LAUNCHER:-} + +hecbench_root=$AOMP_REPOS_TEST/HeCBench +hecbench_src=$hecbench_root/src + +vlog "AOMP=$AOMP" +vlog "AOMP_GPU=$AOMP_GPU" +vlog "AOMP_REPOS_TEST=$AOMP_REPOS_TEST" +vlog "hecbench_root=$hecbench_root" +vlog "hecbench_src=$hecbench_src" +vlog "PWD(before cd)=$(pwd)" + +if [ -d "$hecbench_src" ]; then + cd "$hecbench_src" || exit 1 + vlog "PWD(after cd)=$(pwd)" +elif [ -d "$hecbench_root" ]; then + vlog "WARN: $hecbench_src missing; listing $hecbench_root:" + if [ "$VERBOSE" == 1 ]; then + ls -la "$hecbench_root" + fi + echo "ERROR: HeCBench src not found: $hecbench_src" + exit 1 +else + echo "ERROR: HeCBench not found in $AOMP_REPOS_TEST." + vlog "Expected: $hecbench_root" + exit 1 +fi + +results=$hecbench_root/results.txt +rm -f "$results" + +export PATH=$AOMP/bin:$PATH +export LD_LIBRARY_PATH=$AOMP/lib:$LD_LIBRARY_PATH + +vlog "PATH=$PATH" +vlog "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" +vlog "which clang++=$(which clang++ 2>/dev/null || echo not-found)" +vlog "which hipcc=$(which hipcc 2>/dev/null || echo not-found)" +vlog "which make=$(which make 2>/dev/null || echo not-found)" +vlog "results=$results" + +echo RUN_OPTIONS: $RUN_OPTIONS +if [ "$VERBOSE" == 1 ]; then + echo "VERBOSE: enabled" +fi +for option in $RUN_OPTIONS; do + if [ "$option" == "openmp" ]; then + suffix="-omp" + makefile="Makefile.aomp" + elif [ "$option" == "hip" ]; then + suffix="-hip" + makefile="Makefile" + else + echo "ERROR: Option not recognized: $option." + exit 1 + fi + + if [ -n "$HECBENCH_LIST" ]; then + dirs="$HECBENCH_LIST" + vlog "Using HECBENCH_LIST ($option): $dirs" + else + dirs=$(find . -maxdepth 1 -type d -name "*$suffix" | sort | sed 's|^\./||') + dir_count=$(echo "$dirs" | wc -w) + vlog "Discovered $dir_count *$suffix dirs under $(pwd)" + if [ "$VERBOSE" == 1 ] && [ -n "$dirs" ]; then + vlog "Dirs: $dirs" + fi + fi + + if [ -z "$dirs" ]; then + echo "WARNING: No benchmark dirs found for option=$option suffix=$suffix in $(pwd)" + vlog "find pattern: *$suffix" + continue + fi + + ran=0 + skipped=0 + for d in $dirs; do + if [ ! -d "$d" ]; then + vlog "SKIP (not a directory): $d" + skipped=$((skipped + 1)) + continue + fi + if [ ! -f "$d/$makefile" ]; then + vlog "SKIP (missing $makefile): $d" + skipped=$((skipped + 1)) + continue + fi + ran=$((ran + 1)) + echo "=== [$option] $d ===" | tee -a "$results" + ( + cd "$d" || exit 1 + vlog "PWD=$(pwd)" + if [ "$option" == "openmp" ]; then + export EXTRA_CFLAGS='-fopenmp-offload-mandatory -fopenmp-target-fast' + make_clean=(make -f "$makefile" "ARCH=$AOMP_GPU" clean) + make_run=(make -f "$makefile" "ARCH=$AOMP_GPU" "LAUNCHER=$LAUNCHER" run) + else + unset EXTRA_CFLAGS + make_clean=(make -f "$makefile" clean) + make_run=(make -f "$makefile" "LAUNCHER=$LAUNCHER" run) + fi + vlog "${make_clean[*]}" + if [ "$VERBOSE" == 1 ]; then + "${make_clean[@]}" + else + "${make_clean[@]}" >/dev/null 2>&1 + fi + vlog "${make_run[*]}" + if [ "$VERBOSE" == 1 ]; then + set -o pipefail + timeout $HECBENCH_TIMEOUT "${make_run[@]}" 2>&1 | tee -a "$results" + rc=${PIPESTATUS[0]} + set +o pipefail + else + if timeout $HECBENCH_TIMEOUT "${make_run[@]}" >>"$results" 2>&1; then + rc=0 + else + rc=$? + fi + fi + if [ $rc -eq 0 ]; then + echo "STATUS $d: PASS" | tee -a "$results" + if [ "$VERBOSE" == 1 ]; then + "${make_clean[@]}" + else + "${make_clean[@]}" >/dev/null 2>&1 + fi + else + echo "STATUS $d: FAIL(rc=$rc)" | tee -a "$results" + fi + ) + done + vlog "option=$option: ran=$ran skipped=$skipped" + echo >> "$results" +done + +vlog "Done. Results: $results" From 223d6c1bf956f937ae17a05021452bb0da3e0c86 Mon Sep 17 00:00:00 2001 From: Lynd Stringer Date: Wed, 5 Aug 2026 14:44:46 -0500 Subject: [PATCH 02/10] Changes, AOMP and ROCM. --- bin/run_HeCBench.sh | 199 +++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 104 deletions(-) diff --git a/bin/run_HeCBench.sh b/bin/run_HeCBench.sh index ee885ed0f..20339a1e6 100755 --- a/bin/run_HeCBench.sh +++ b/bin/run_HeCBench.sh @@ -1,52 +1,95 @@ -#!/bin/bash +#!/usr/bin/env bash + +# +#Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +#SPDX-License-Identifier: MIT +# # run_HeCBench.sh - runs HeCBench benchmarks in the $AOMP_REPOS_TEST dir. -# User can set RUN_OPTIONS to control what variants(openmp, hip) are selected. # -# Verbose debug: ./run_HeCBench.sh -v -# VERBOSE=1 ./run_HeCBench.sh +# Environment variables (set before running; none are required unless noted): +# +# Compiler / ROCm layout: +# AOMP LLVM compiler tree (clang++, libomp) +# if unset: /opt/rocm/lib/llvm +# if set: use as-is +# ROCM HIP/ROCm install root (hipcc, libamdhip64) +# if unset and AOMP set: realpath(AOMP/../..) +# if unset and AOMP unset: /opt/rocm +# AOMP_GPU GPU arch for OpenMP builds (ARCH= in Makefile.aomp); +# auto-detected via rocm_agent_enumerator if unset +# +# Test tree (from aomp_common_vars): +# AOMP_REPOS_TEST parent of cloned HeCBench; default: $HOME/git/aomp-test +# expects: $AOMP_REPOS_TEST/HeCBench/src/-{omp,hip} +# +# Run control: +# RUN_OPTIONS space-separated list of build variants to run; +# default: "openmp hip" (both) +# openmp - src/*-omp dirs, build with Makefile.aomp +# (clang++, ARCH=$AOMP_GPU) +# hip - src/*-hip dirs, build with Makefile (hipcc) +# examples: +# RUN_OPTIONS=openmp +# RUN_OPTIONS="openmp hip" +# RUN_OPTIONS=hip +# HECBENCH_LIST space-separated benchmark dirs to run (default: all) +# HECBENCH_TIMEOUT per-benchmark timeout in seconds (default: 180) +# LAUNCHER passed to Makefile run target (e.g. "gpurun time -p") +# +# Compiler flags: +# EXTRA_CFLAGS extra compiler flags (Makefile.aomp / Makefile); not set +# by this script — export before running, e.g.: +# export EXTRA_CFLAGS='-fopenmp-target-fast' # --- Start standard header to set AOMP environment variables ---- -realpath=`realpath $0` -thisdir=`dirname $realpath` +realpath=$(realpath "$0") +thisdir=$(dirname "$realpath") export AOMP_USE_CCACHE=0 -. $thisdir/aomp_common_vars +# shellcheck source=aomp_common_vars +_AOMP_USER_SET=0 +_ROCM_USER_SET=0 +[ -n "${AOMP+x}" ] && _AOMP_USER_SET=1 +[ -n "${ROCM+x}" ] && _ROCM_USER_SET=1 +# shellcheck disable=SC1091 +. "$thisdir/aomp_common_vars" # --- end standard header ---- -VERBOSE=${VERBOSE:-0} -while [ $# -gt 0 ]; do - case "$1" in - -v|--verbose) - VERBOSE=1 - shift - ;; - -h|--help) - echo "Usage: $0 [-v|--verbose]" - echo " RUN_OPTIONS openmp hip (default: both)" - echo " HECBENCH_LIST space-separated benchmark dirs to run" - echo " HECBENCH_TIMEOUT per-benchmark timeout seconds (default: 180)" - echo " LAUNCHER passed to Makefile run target" - echo " VERBOSE=1 same as -v" - exit 0 - ;; - *) - echo "ERROR: Unknown option: $1 (try -h)" - exit 1 - ;; - esac -done - -vlog() { - if [ "$VERBOSE" == 1 ]; then - echo "[verbose] $*" +# Setup AOMP / ROCM (see header for rules) +if [ "$_AOMP_USER_SET" -eq 1 ]; then + if [ "$_ROCM_USER_SET" -eq 0 ]; then + ROCM=$(realpath -m "$(realpath -m "$AOMP")"/../..) + fi +else + export AOMP=/opt/rocm/lib/llvm + if [ "$_ROCM_USER_SET" -eq 0 ]; then + export ROCM=/opt/rocm + fi +fi +AOMPTOP=$(echo "$AOMP" | sed -e 's|/lib[/]*llvm||' -e 's|/llvm||') +export AOMP AOMPTOP ROCM + +warn_hipcc_clang_mismatch() { + local hipcc_bin clang_bin hipcc_ver clang_ver hipcc_clang_line + hipcc_bin=$(PATH="$ROCM/bin:$AOMPTOP/bin:$PATH" command -v hipcc 2>/dev/null) + clang_bin=$(PATH="$AOMP/bin:$PATH" command -v clang 2>/dev/null) + if [ -z "$hipcc_bin" ] || [ -z "$clang_bin" ]; then + return 0 + fi + hipcc_ver=$("$hipcc_bin" --version 2>/dev/null | head -2) + clang_ver=$("$clang_bin" --version 2>/dev/null | head -1) + hipcc_clang_line=$(echo "$hipcc_ver" | grep -i 'clang version' | head -1) + if [ -n "$hipcc_clang_line" ] && [ -n "$clang_ver" ] && + [ "$hipcc_clang_line" != "$clang_ver" ]; then + echo "WARNING: hipcc and clang report different compiler versions:" >&2 + echo " hipcc ($hipcc_bin):" >&2 + printf ' %s\n' "$hipcc_ver" >&2 + echo " clang ($clang_bin): $clang_ver" >&2 fi } -# Setup AOMP variables -AOMP=${AOMP:-/usr/lib/aomp} -AOMPHIP=${AOMPHIP:-$AOMP} - # Use function to set and test AOMP_GPU setaompgpu @@ -58,46 +101,25 @@ LAUNCHER=${LAUNCHER:-} hecbench_root=$AOMP_REPOS_TEST/HeCBench hecbench_src=$hecbench_root/src -vlog "AOMP=$AOMP" -vlog "AOMP_GPU=$AOMP_GPU" -vlog "AOMP_REPOS_TEST=$AOMP_REPOS_TEST" -vlog "hecbench_root=$hecbench_root" -vlog "hecbench_src=$hecbench_src" -vlog "PWD(before cd)=$(pwd)" - if [ -d "$hecbench_src" ]; then cd "$hecbench_src" || exit 1 - vlog "PWD(after cd)=$(pwd)" elif [ -d "$hecbench_root" ]; then - vlog "WARN: $hecbench_src missing; listing $hecbench_root:" - if [ "$VERBOSE" == 1 ]; then - ls -la "$hecbench_root" - fi echo "ERROR: HeCBench src not found: $hecbench_src" exit 1 else echo "ERROR: HeCBench not found in $AOMP_REPOS_TEST." - vlog "Expected: $hecbench_root" exit 1 fi results=$hecbench_root/results.txt rm -f "$results" -export PATH=$AOMP/bin:$PATH -export LD_LIBRARY_PATH=$AOMP/lib:$LD_LIBRARY_PATH +export PATH=$AOMP/bin:$AOMPTOP/bin:$ROCM/bin:$PATH +export LD_LIBRARY_PATH=$AOMP/lib:$AOMPTOP/lib:$ROCM/lib:$LD_LIBRARY_PATH -vlog "PATH=$PATH" -vlog "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" -vlog "which clang++=$(which clang++ 2>/dev/null || echo not-found)" -vlog "which hipcc=$(which hipcc 2>/dev/null || echo not-found)" -vlog "which make=$(which make 2>/dev/null || echo not-found)" -vlog "results=$results" +warn_hipcc_clang_mismatch -echo RUN_OPTIONS: $RUN_OPTIONS -if [ "$VERBOSE" == 1 ]; then - echo "VERBOSE: enabled" -fi +echo RUN_OPTIONS: "$RUN_OPTIONS" for option in $RUN_OPTIONS; do if [ "$option" == "openmp" ]; then suffix="-omp" @@ -112,82 +134,51 @@ for option in $RUN_OPTIONS; do if [ -n "$HECBENCH_LIST" ]; then dirs="$HECBENCH_LIST" - vlog "Using HECBENCH_LIST ($option): $dirs" else dirs=$(find . -maxdepth 1 -type d -name "*$suffix" | sort | sed 's|^\./||') - dir_count=$(echo "$dirs" | wc -w) - vlog "Discovered $dir_count *$suffix dirs under $(pwd)" - if [ "$VERBOSE" == 1 ] && [ -n "$dirs" ]; then - vlog "Dirs: $dirs" - fi fi if [ -z "$dirs" ]; then echo "WARNING: No benchmark dirs found for option=$option suffix=$suffix in $(pwd)" - vlog "find pattern: *$suffix" continue fi - ran=0 - skipped=0 + NumTestsRun=0 + NumTestsSkipped=0 for d in $dirs; do if [ ! -d "$d" ]; then - vlog "SKIP (not a directory): $d" - skipped=$((skipped + 1)) + NumTestsSkipped=$((NumTestsSkipped + 1)) continue fi if [ ! -f "$d/$makefile" ]; then - vlog "SKIP (missing $makefile): $d" - skipped=$((skipped + 1)) + NumTestsSkipped=$((NumTestsSkipped + 1)) continue fi - ran=$((ran + 1)) + NumTestsRun=$((NumTestsRun + 1)) echo "=== [$option] $d ===" | tee -a "$results" ( cd "$d" || exit 1 - vlog "PWD=$(pwd)" if [ "$option" == "openmp" ]; then - export EXTRA_CFLAGS='-fopenmp-offload-mandatory -fopenmp-target-fast' make_clean=(make -f "$makefile" "ARCH=$AOMP_GPU" clean) make_run=(make -f "$makefile" "ARCH=$AOMP_GPU" "LAUNCHER=$LAUNCHER" run) else - unset EXTRA_CFLAGS make_clean=(make -f "$makefile" clean) make_run=(make -f "$makefile" "LAUNCHER=$LAUNCHER" run) fi - vlog "${make_clean[*]}" - if [ "$VERBOSE" == 1 ]; then - "${make_clean[@]}" - else - "${make_clean[@]}" >/dev/null 2>&1 - fi - vlog "${make_run[*]}" - if [ "$VERBOSE" == 1 ]; then - set -o pipefail - timeout $HECBENCH_TIMEOUT "${make_run[@]}" 2>&1 | tee -a "$results" - rc=${PIPESTATUS[0]} - set +o pipefail + "${make_clean[@]}" >/dev/null 2>&1 + if timeout "$HECBENCH_TIMEOUT" "${make_run[@]}" >>"$results" 2>&1; then + rc=0 else - if timeout $HECBENCH_TIMEOUT "${make_run[@]}" >>"$results" 2>&1; then - rc=0 - else - rc=$? - fi + rc=$? fi if [ $rc -eq 0 ]; then echo "STATUS $d: PASS" | tee -a "$results" - if [ "$VERBOSE" == 1 ]; then - "${make_clean[@]}" - else - "${make_clean[@]}" >/dev/null 2>&1 - fi + "${make_clean[@]}" >/dev/null 2>&1 else echo "STATUS $d: FAIL(rc=$rc)" | tee -a "$results" fi ) done - vlog "option=$option: ran=$ran skipped=$skipped" + echo "[$option] NumTestsRun=$NumTestsRun NumTestsSkipped=$NumTestsSkipped" echo >> "$results" done - -vlog "Done. Results: $results" From ad90a9ab362fbe66a13e519c6e125f9528e70d49 Mon Sep 17 00:00:00 2001 From: Lynd Stringer Date: Mon, 10 Aug 2026 13:47:05 -0500 Subject: [PATCH 03/10] Simplified logic and ROCM to ROCM_PATH. --- bin/run_HeCBench.sh | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/bin/run_HeCBench.sh b/bin/run_HeCBench.sh index 20339a1e6..50ba476c1 100755 --- a/bin/run_HeCBench.sh +++ b/bin/run_HeCBench.sh @@ -14,9 +14,8 @@ # AOMP LLVM compiler tree (clang++, libomp) # if unset: /opt/rocm/lib/llvm # if set: use as-is -# ROCM HIP/ROCm install root (hipcc, libamdhip64) -# if unset and AOMP set: realpath(AOMP/../..) -# if unset and AOMP unset: /opt/rocm +# ROCM_PATH HIP/ROCm install root (hipcc, libamdhip64) +# if unset: realpath(AOMP/../..) (e.g. /opt/rocm) # AOMP_GPU GPU arch for OpenMP builds (ARCH= in Makefile.aomp); # auto-detected via rocm_agent_enumerator if unset # @@ -49,31 +48,24 @@ thisdir=$(dirname "$realpath") export AOMP_USE_CCACHE=0 # shellcheck source=aomp_common_vars -_AOMP_USER_SET=0 -_ROCM_USER_SET=0 -[ -n "${AOMP+x}" ] && _AOMP_USER_SET=1 -[ -n "${ROCM+x}" ] && _ROCM_USER_SET=1 +_had_aomp=${AOMP+1} +_had_rocm_path=${ROCM_PATH+1} # shellcheck disable=SC1091 . "$thisdir/aomp_common_vars" # --- end standard header ---- -# Setup AOMP / ROCM (see header for rules) -if [ "$_AOMP_USER_SET" -eq 1 ]; then - if [ "$_ROCM_USER_SET" -eq 0 ]; then - ROCM=$(realpath -m "$(realpath -m "$AOMP")"/../..) - fi -else - export AOMP=/opt/rocm/lib/llvm - if [ "$_ROCM_USER_SET" -eq 0 ]; then - export ROCM=/opt/rocm - fi +# Setup AOMP / ROCM_PATH (see header for rules) +[ -z "$_had_aomp" ] && export AOMP=/opt/rocm/lib/llvm +if [ -z "$_had_rocm_path" ]; then + ROCM_PATH=$(realpath -m "$(realpath -m "$AOMP")"/../..) + export ROCM_PATH fi AOMPTOP=$(echo "$AOMP" | sed -e 's|/lib[/]*llvm||' -e 's|/llvm||') -export AOMP AOMPTOP ROCM +export AOMP AOMPTOP ROCM_PATH warn_hipcc_clang_mismatch() { local hipcc_bin clang_bin hipcc_ver clang_ver hipcc_clang_line - hipcc_bin=$(PATH="$ROCM/bin:$AOMPTOP/bin:$PATH" command -v hipcc 2>/dev/null) + hipcc_bin=$(PATH="$ROCM_PATH/bin:$AOMPTOP/bin:$PATH" command -v hipcc 2>/dev/null) clang_bin=$(PATH="$AOMP/bin:$PATH" command -v clang 2>/dev/null) if [ -z "$hipcc_bin" ] || [ -z "$clang_bin" ]; then return 0 @@ -114,8 +106,8 @@ fi results=$hecbench_root/results.txt rm -f "$results" -export PATH=$AOMP/bin:$AOMPTOP/bin:$ROCM/bin:$PATH -export LD_LIBRARY_PATH=$AOMP/lib:$AOMPTOP/lib:$ROCM/lib:$LD_LIBRARY_PATH +export PATH=$AOMP/bin:$AOMPTOP/bin:$ROCM_PATH/bin:$PATH +export LD_LIBRARY_PATH=$AOMP/lib:$AOMPTOP/lib:$ROCM_PATH/lib:$LD_LIBRARY_PATH warn_hipcc_clang_mismatch From bd0b0fb005b4dbd4dfa42bc6cc873da7ce9ccb93 Mon Sep 17 00:00:00 2001 From: Lynd Stringer Date: Tue, 11 Aug 2026 11:51:10 -0500 Subject: [PATCH 04/10] More simplifications, deleted unneeded. --- bin/run_HeCBench.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bin/run_HeCBench.sh b/bin/run_HeCBench.sh index 50ba476c1..f919938e5 100755 --- a/bin/run_HeCBench.sh +++ b/bin/run_HeCBench.sh @@ -45,7 +45,6 @@ # --- Start standard header to set AOMP environment variables ---- realpath=$(realpath "$0") thisdir=$(dirname "$realpath") -export AOMP_USE_CCACHE=0 # shellcheck source=aomp_common_vars _had_aomp=${AOMP+1} @@ -60,12 +59,10 @@ if [ -z "$_had_rocm_path" ]; then ROCM_PATH=$(realpath -m "$(realpath -m "$AOMP")"/../..) export ROCM_PATH fi -AOMPTOP=$(echo "$AOMP" | sed -e 's|/lib[/]*llvm||' -e 's|/llvm||') -export AOMP AOMPTOP ROCM_PATH warn_hipcc_clang_mismatch() { local hipcc_bin clang_bin hipcc_ver clang_ver hipcc_clang_line - hipcc_bin=$(PATH="$ROCM_PATH/bin:$AOMPTOP/bin:$PATH" command -v hipcc 2>/dev/null) + hipcc_bin=$(PATH="$ROCM_PATH/bin:$PATH" command -v hipcc 2>/dev/null) clang_bin=$(PATH="$AOMP/bin:$PATH" command -v clang 2>/dev/null) if [ -z "$hipcc_bin" ] || [ -z "$clang_bin" ]; then return 0 @@ -106,8 +103,8 @@ fi results=$hecbench_root/results.txt rm -f "$results" -export PATH=$AOMP/bin:$AOMPTOP/bin:$ROCM_PATH/bin:$PATH -export LD_LIBRARY_PATH=$AOMP/lib:$AOMPTOP/lib:$ROCM_PATH/lib:$LD_LIBRARY_PATH +export PATH=$AOMP/bin:$ROCM_PATH/bin:$PATH +export LD_LIBRARY_PATH=$AOMP/lib:$ROCM_PATH/lib:$LD_LIBRARY_PATH warn_hipcc_clang_mismatch From 1204614d83088033e400117af748da260bdb4d4f Mon Sep 17 00:00:00 2001 From: Lynd Stringer Date: Tue, 11 Aug 2026 16:41:17 -0500 Subject: [PATCH 05/10] Cleanup, simplification. --- bin/run_HeCBench.sh | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/bin/run_HeCBench.sh b/bin/run_HeCBench.sh index f919938e5..952330da5 100755 --- a/bin/run_HeCBench.sh +++ b/bin/run_HeCBench.sh @@ -24,15 +24,15 @@ # expects: $AOMP_REPOS_TEST/HeCBench/src/-{omp,hip} # # Run control: -# RUN_OPTIONS space-separated list of build variants to run; +# PROGRAMMING_MODELS space-separated list of build variants to run; # default: "openmp hip" (both) # openmp - src/*-omp dirs, build with Makefile.aomp # (clang++, ARCH=$AOMP_GPU) # hip - src/*-hip dirs, build with Makefile (hipcc) # examples: -# RUN_OPTIONS=openmp -# RUN_OPTIONS="openmp hip" -# RUN_OPTIONS=hip +# PROGRAMMING_MODELS=openmp +# PROGRAMMING_MODELS="openmp hip" +# PROGRAMMING_MODELS=hip # HECBENCH_LIST space-separated benchmark dirs to run (default: all) # HECBENCH_TIMEOUT per-benchmark timeout in seconds (default: 180) # LAUNCHER passed to Makefile run target (e.g. "gpurun time -p") @@ -46,19 +46,13 @@ realpath=$(realpath "$0") thisdir=$(dirname "$realpath") -# shellcheck source=aomp_common_vars -_had_aomp=${AOMP+1} -_had_rocm_path=${ROCM_PATH+1} # shellcheck disable=SC1091 . "$thisdir/aomp_common_vars" -# --- end standard header ---- -# Setup AOMP / ROCM_PATH (see header for rules) -[ -z "$_had_aomp" ] && export AOMP=/opt/rocm/lib/llvm -if [ -z "$_had_rocm_path" ]; then - ROCM_PATH=$(realpath -m "$(realpath -m "$AOMP")"/../..) - export ROCM_PATH -fi +# If AOMP and ROCM_PATH are already set, use them. If not, use defaults. +# The default for AOMP is /opt/rocm/llvm. The default for ROCM_PATH is $AOMP/../.. +export AOMP="${AOMP:-/opt/rocm/lib/llvm}" +export ROCM_PATH="${ROCM_PATH:-$(realpath -m "${AOMP}/../..")}" warn_hipcc_clang_mismatch() { local hipcc_bin clang_bin hipcc_ver clang_ver hipcc_clang_line @@ -82,7 +76,7 @@ warn_hipcc_clang_mismatch() { # Use function to set and test AOMP_GPU setaompgpu -RUN_OPTIONS=${RUN_OPTIONS:-"openmp hip"} +PROGRAMMING_MODELS=${PROGRAMMING_MODELS:-"openmp hip"} HECBENCH_TIMEOUT=${HECBENCH_TIMEOUT:-180} HECBENCH_LIST=${HECBENCH_LIST:-""} LAUNCHER=${LAUNCHER:-} @@ -108,8 +102,8 @@ export LD_LIBRARY_PATH=$AOMP/lib:$ROCM_PATH/lib:$LD_LIBRARY_PATH warn_hipcc_clang_mismatch -echo RUN_OPTIONS: "$RUN_OPTIONS" -for option in $RUN_OPTIONS; do +echo PROGRAMMING_MODELS: "$PROGRAMMING_MODELS" +for option in $PROGRAMMING_MODELS; do if [ "$option" == "openmp" ]; then suffix="-omp" makefile="Makefile.aomp" From 5bdaa0ca8979aa67d7597fbb0d1f5c571fbce1e5 Mon Sep 17 00:00:00 2001 From: Lynd Stringer Date: Wed, 12 Aug 2026 11:34:39 -0500 Subject: [PATCH 06/10] Deleted the variable rc, not needed. --- bin/run_HeCBench.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bin/run_HeCBench.sh b/bin/run_HeCBench.sh index 952330da5..1db4c1cb6 100755 --- a/bin/run_HeCBench.sh +++ b/bin/run_HeCBench.sh @@ -150,15 +150,10 @@ for option in $PROGRAMMING_MODELS; do fi "${make_clean[@]}" >/dev/null 2>&1 if timeout "$HECBENCH_TIMEOUT" "${make_run[@]}" >>"$results" 2>&1; then - rc=0 - else - rc=$? - fi - if [ $rc -eq 0 ]; then echo "STATUS $d: PASS" | tee -a "$results" "${make_clean[@]}" >/dev/null 2>&1 else - echo "STATUS $d: FAIL(rc=$rc)" | tee -a "$results" + echo "STATUS $d: FAIL(rc=$?)" | tee -a "$results" fi ) done From 46b8aa7c60c6ec43250e952115ec8803bec54f36 Mon Sep 17 00:00:00 2001 From: dpalermo Date: Wed, 19 Aug 2026 11:19:21 -0500 Subject: [PATCH 07/10] [srock] Add -DTHEROCK_FLAG_LLVM_ENABLE_ASSERTIONS=ON (#2443) --- srock-bin/srock_common_vars | 1 + 1 file changed, 1 insertion(+) diff --git a/srock-bin/srock_common_vars b/srock-bin/srock_common_vars index c55b400c5..82848353f 100644 --- a/srock-bin/srock_common_vars +++ b/srock-bin/srock_common_vars @@ -226,6 +226,7 @@ _cmake_args=(-B build -GNinja -DTHEROCK_AMDGPU_DIST_BUNDLE_NAME=srock -DTHEROCK_BACKGROUND_BUILD_JOBS=1 -DTHEROCK_BUILD_LLVM_TESTS=1 + -DTHEROCK_FLAG_LLVM_ENABLE_ASSERTIONS=ON "${_cmake_enable[@]}" $SROCK_CMAKE_EXTRA "$SROCK_THEROCK_DIR" From e053820f9e08b85f899a3bbc0f18aec7d0bd6607 Mon Sep 17 00:00:00 2001 From: Lynd Stringer Date: Wed, 19 Aug 2026 13:48:59 -0500 Subject: [PATCH 08/10] Fixes for Michael. --- bin/run_HeCBench.sh | 62 +++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/bin/run_HeCBench.sh b/bin/run_HeCBench.sh index 1db4c1cb6..4ed850846 100755 --- a/bin/run_HeCBench.sh +++ b/bin/run_HeCBench.sh @@ -53,19 +53,28 @@ thisdir=$(dirname "$realpath") # The default for AOMP is /opt/rocm/llvm. The default for ROCM_PATH is $AOMP/../.. export AOMP="${AOMP:-/opt/rocm/lib/llvm}" export ROCM_PATH="${ROCM_PATH:-$(realpath -m "${AOMP}/../..")}" +export PATH=$AOMP/bin:$ROCM_PATH/bin:$PATH +export LD_LIBRARY_PATH=$AOMP/lib:$ROCM_PATH/lib:$LD_LIBRARY_PATH + +PROGRAMMING_MODELS=${PROGRAMMING_MODELS:-"openmp hip"} +HECBENCH_TIMEOUT=${HECBENCH_TIMEOUT:-180} +HECBENCH_LIST=${HECBENCH_LIST:-""} +LAUNCHER=${LAUNCHER:-} + +hecbench_root=$AOMP_REPOS_TEST/HeCBench +hecbench_src=$hecbench_root/src warn_hipcc_clang_mismatch() { - local hipcc_bin clang_bin hipcc_ver clang_ver hipcc_clang_line - hipcc_bin=$(PATH="$ROCM_PATH/bin:$PATH" command -v hipcc 2>/dev/null) + local hipcc_bin clang_bin hipcc_ver clang_ver + hipcc_bin=$(PATH="$AOMP/bin:$ROCM_PATH/bin:$PATH" command -v hipcc 2>/dev/null) clang_bin=$(PATH="$AOMP/bin:$PATH" command -v clang 2>/dev/null) if [ -z "$hipcc_bin" ] || [ -z "$clang_bin" ]; then return 0 fi - hipcc_ver=$("$hipcc_bin" --version 2>/dev/null | head -2) - clang_ver=$("$clang_bin" --version 2>/dev/null | head -1) - hipcc_clang_line=$(echo "$hipcc_ver" | grep -i 'clang version' | head -1) - if [ -n "$hipcc_clang_line" ] && [ -n "$clang_ver" ] && - [ "$hipcc_clang_line" != "$clang_ver" ]; then + hipcc_ver=$("$hipcc_bin" --version 2>&1 | grep -i 'clang version') + clang_ver=$("$clang_bin" --version 2>&1 | grep -i 'clang version') + if [ -n "$hipcc_ver" ] && [ -n "$clang_ver" ] && + [ "$hipcc_ver" != "$clang_ver" ]; then echo "WARNING: hipcc and clang report different compiler versions:" >&2 echo " hipcc ($hipcc_bin):" >&2 printf ' %s\n' "$hipcc_ver" >&2 @@ -76,42 +85,32 @@ warn_hipcc_clang_mismatch() { # Use function to set and test AOMP_GPU setaompgpu -PROGRAMMING_MODELS=${PROGRAMMING_MODELS:-"openmp hip"} -HECBENCH_TIMEOUT=${HECBENCH_TIMEOUT:-180} -HECBENCH_LIST=${HECBENCH_LIST:-""} -LAUNCHER=${LAUNCHER:-} - -hecbench_root=$AOMP_REPOS_TEST/HeCBench -hecbench_src=$hecbench_root/src - -if [ -d "$hecbench_src" ]; then - cd "$hecbench_src" || exit 1 -elif [ -d "$hecbench_root" ]; then +if [ ! -d "$hecbench_src" ]; then echo "ERROR: HeCBench src not found: $hecbench_src" exit 1 -else +elif [ ! -d "$hecbench_root" ]; then echo "ERROR: HeCBench not found in $AOMP_REPOS_TEST." exit 1 fi +cd "$hecbench_src" || exit 1 + results=$hecbench_root/results.txt rm -f "$results" -export PATH=$AOMP/bin:$ROCM_PATH/bin:$PATH -export LD_LIBRARY_PATH=$AOMP/lib:$ROCM_PATH/lib:$LD_LIBRARY_PATH - +# Check for a mismatch. warn_hipcc_clang_mismatch echo PROGRAMMING_MODELS: "$PROGRAMMING_MODELS" -for option in $PROGRAMMING_MODELS; do - if [ "$option" == "openmp" ]; then +for model in $PROGRAMMING_MODELS; do + if [ "$model" == "openmp" ]; then suffix="-omp" makefile="Makefile.aomp" - elif [ "$option" == "hip" ]; then + elif [ "$model" == "hip" ]; then suffix="-hip" makefile="Makefile" else - echo "ERROR: Option not recognized: $option." + echo "ERROR: Option not recognized: $model." exit 1 fi @@ -122,7 +121,7 @@ for option in $PROGRAMMING_MODELS; do fi if [ -z "$dirs" ]; then - echo "WARNING: No benchmark dirs found for option=$option suffix=$suffix in $(pwd)" + echo "WARNING: No benchmark dirs found for model=$model suffix=$suffix in $(pwd)" continue fi @@ -138,10 +137,10 @@ for option in $PROGRAMMING_MODELS; do continue fi NumTestsRun=$((NumTestsRun + 1)) - echo "=== [$option] $d ===" | tee -a "$results" + echo "=== [$model] $d ===" | tee -a "$results" ( cd "$d" || exit 1 - if [ "$option" == "openmp" ]; then + if [ "$model" == "openmp" ]; then make_clean=(make -f "$makefile" "ARCH=$AOMP_GPU" clean) make_run=(make -f "$makefile" "ARCH=$AOMP_GPU" "LAUNCHER=$LAUNCHER" run) else @@ -157,6 +156,9 @@ for option in $PROGRAMMING_MODELS; do fi ) done - echo "[$option] NumTestsRun=$NumTestsRun NumTestsSkipped=$NumTestsSkipped" + echo "[$model] NumTestsRun=$NumTestsRun NumTestsSkipped=$NumTestsSkipped" echo >> "$results" + echo "=== SUMMARY [$model] ===" | tee -a "$results" + echo "NumTestsRun=$NumTestsRun" | tee -a "$results" + echo "NumTestsSkipped=$NumTestsSkipped" | tee -a "$results" done From 7769d1a61930e16af1b29b7d1645a54f15310a31 Mon Sep 17 00:00:00 2001 From: Dominik Adamski Date: Thu, 20 Aug 2026 15:55:18 +0200 Subject: [PATCH 09/10] [smoke-fort-fails] Add test case to check performance of use_dev_addr (#2281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flang should emit only two map entries: the descriptor for x and the allocation that holds x’s data. Using use_device_addr must not add any further map entries. --- .../flang-use-dev-addr-performance/Makefile | 26 ++++++++ .../flang-use-dev-addr-performance/bar.hip | 27 ++++++++ .../flang-use-dev-addr-performance/check.txt | 13 ++++ .../flang-use-dev-addr-performance/main.f90 | 65 +++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 test/smoke-fort-fails/flang-use-dev-addr-performance/Makefile create mode 100644 test/smoke-fort-fails/flang-use-dev-addr-performance/bar.hip create mode 100644 test/smoke-fort-fails/flang-use-dev-addr-performance/check.txt create mode 100644 test/smoke-fort-fails/flang-use-dev-addr-performance/main.f90 diff --git a/test/smoke-fort-fails/flang-use-dev-addr-performance/Makefile b/test/smoke-fort-fails/flang-use-dev-addr-performance/Makefile new file mode 100644 index 000000000..c2150aaf2 --- /dev/null +++ b/test/smoke-fort-fails/flang-use-dev-addr-performance/Makefile @@ -0,0 +1,26 @@ +include ../../Makefile.defs + +TESTNAME = use-dev-addr-performance +TESTSRC_MAIN = main.f90 +TESTSRC_AUX = bar.o +TESTSRC_ALL = $(TESTSRC_MAIN) $(TESTSRC_AUX) +AOMPHIP ?= $(AOMP) +HIPCC ?= $(AOMPHIP)/bin/hipcc +HIP_CLANG_PATH ?= $(AOMP)/bin + +CFLAGS = -O3 +FLANG ?= flang +OMP_BIN = $(AOMP)/bin/$(FLANG) +CC = $(OMP_BIN) $(VERBOSE) +EXTRA_CFLAGS = -L$(AOMPHIP)/lib -lamdhip64 -Wl,-rpath,$(AOMPHIP)/lib -fPIC +#-ccc-print-phases +#"-\#\#\#" + +include ../Makefile.rules +all: $(TESTNAME) + +bar.o : bar.hip + HIP_CLANG_PATH=$(HIP_CLANG_PATH) $(HIPCC) -c --offload-arch=$(AOMP_GPU) -fPIC $^ -o $@ + +run: $(TESTNAME) + LIBOMPTARGET_INFO=8 ./$(TESTNAME) 2>&1 | $(AOMP)/bin/FileCheck check.txt diff --git a/test/smoke-fort-fails/flang-use-dev-addr-performance/bar.hip b/test/smoke-fort-fails/flang-use-dev-addr-performance/bar.hip new file mode 100644 index 000000000..5ca383c51 --- /dev/null +++ b/test/smoke-fort-fails/flang-use-dev-addr-performance/bar.hip @@ -0,0 +1,27 @@ +#include +#include + +__global__ void bar_kernel(int *x, int n) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + + if (i < n) { + x[i]++; + } +} + +extern "C" { + +void bar_GPU(int *x, int n) +{ + int num_threads = 256; + int num_blocks = n / num_threads + 1; + hipLaunchKernelGGL(bar_kernel, dim3(num_blocks), dim3(num_threads), 0, 0, x, n); + (void)hipDeviceSynchronize(); +} + +void print_ptr(int * x) { + static int cnt; + printf("pointer x %lx number of calls: %d\n",(ulong)x, ++cnt); +} +} /* extern "C" */ diff --git a/test/smoke-fort-fails/flang-use-dev-addr-performance/check.txt b/test/smoke-fort-fails/flang-use-dev-addr-performance/check.txt new file mode 100644 index 000000000..c2c05551f --- /dev/null +++ b/test/smoke-fort-fails/flang-use-dev-addr-performance/check.txt @@ -0,0 +1,13 @@ +; CHECK: Creating new map entry +; CHECK: Creating new map entry +; CHECK-NOT: Creating new map entry +; CHECK: Removing map entry +; CHECK: Removing map entry +; CHECK-NOT: Removing map entry +; CHECK: Success +; CHECK: pointer x [[ADDR:[0-9a-f]+]] number of calls: 1 +; CHECK: pointer x [[ADDR]] number of calls: 2 +; CHECK: pointer x [[GPU_ADDR:[0-9a-f]+]] number of calls: 3 +; CHECK: pointer x [[GPU_ADDR]] number of calls: 4 +; CHECK: pointer x [[ADDR]] number of calls: 5 + diff --git a/test/smoke-fort-fails/flang-use-dev-addr-performance/main.f90 b/test/smoke-fort-fails/flang-use-dev-addr-performance/main.f90 new file mode 100644 index 000000000..2ff3e3d00 --- /dev/null +++ b/test/smoke-fort-fails/flang-use-dev-addr-performance/main.f90 @@ -0,0 +1,65 @@ +MODULE foo + USE iso_c_binding + USE omp_lib + IMPLICIT NONE + PRIVATE + PUBLIC :: bar_device_addr, print_ptr, bar + + INTERFACE + SUBROUTINE bar(x, n) BIND(C, name="bar_GPU") + USE iso_c_binding + TYPE(C_PTR), VALUE, INTENT(IN) :: x + INTEGER(C_INT), VALUE, INTENT(IN) :: n + END SUBROUTINE + SUBROUTINE print_ptr(x) BIND(C, name="print_ptr") + USE iso_c_binding + TYPE(C_PTR), VALUE, INTENT(IN) :: x + END SUBROUTINE + + END INTERFACE + +CONTAINS + + SUBROUTINE bar_device_addr(x, n) + INTEGER, TARGET, INTENT(IN) :: x(:) + INTEGER(C_INT), INTENT(IN) :: n + !$omp target data use_device_addr (x) + CALL print_ptr(c_loc(x)) + CALL bar(c_loc(x), n) + !$omp end target data + END SUBROUTINE + +END MODULE foo + +PROGRAM test_ptr + USE iso_c_binding + USE, intrinsic :: iso_fortran_env, only: error_unit + USE omp_lib + USE foo + IMPLICIT NONE + + INTEGER, ALLOCATABLE, TARGET :: x(:) + INTEGER(C_INT) :: i, n + n = 1000 + ALLOCATE(x(n)) + x = 1 + CALL print_ptr(c_loc(x)) + !$omp target enter data map(to: x) + CALL print_ptr(c_loc(x)) + !$omp target data use_device_addr (x) + CALL print_ptr(c_loc(x)) + CALL bar(c_loc(x), n) + !$omp end target data + CALL bar_device_addr(x,n) + !$omp target exit data map(from: x) + CALL print_ptr(c_loc(x)) + DO i = 1,n + IF (x(i) .ne. 3) then + PRINT *, "Bad result for use_device_addr!" + STOP 1 + ENDIF + END DO + DEALLOCATE(x) + write(error_unit, *) 'Success' +END PROGRAM test_ptr + From ea6fcb654829663dbe8b9215485699bef122759b Mon Sep 17 00:00:00 2001 From: Lynd Stringer Date: Thu, 20 Aug 2026 12:06:54 -0500 Subject: [PATCH 10/10] Fixes for Michael. 2 --- bin/run_HeCBench.sh | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/bin/run_HeCBench.sh b/bin/run_HeCBench.sh index 4ed850846..1c434bae3 100755 --- a/bin/run_HeCBench.sh +++ b/bin/run_HeCBench.sh @@ -64,33 +64,38 @@ LAUNCHER=${LAUNCHER:-} hecbench_root=$AOMP_REPOS_TEST/HeCBench hecbench_src=$hecbench_root/src -warn_hipcc_clang_mismatch() { - local hipcc_bin clang_bin hipcc_ver clang_ver +check_hipcc_clang_mismatch() { + local hipcc_bin clang_bin hipcc_clang_line clang_ver hipcc_bin=$(PATH="$AOMP/bin:$ROCM_PATH/bin:$PATH" command -v hipcc 2>/dev/null) clang_bin=$(PATH="$AOMP/bin:$PATH" command -v clang 2>/dev/null) if [ -z "$hipcc_bin" ] || [ -z "$clang_bin" ]; then return 0 fi - hipcc_ver=$("$hipcc_bin" --version 2>&1 | grep -i 'clang version') + hipcc_clang_line=$("$hipcc_bin" --version 2>&1 | grep -i 'clang version') clang_ver=$("$clang_bin" --version 2>&1 | grep -i 'clang version') - if [ -n "$hipcc_ver" ] && [ -n "$clang_ver" ] && - [ "$hipcc_ver" != "$clang_ver" ]; then - echo "WARNING: hipcc and clang report different compiler versions:" >&2 - echo " hipcc ($hipcc_bin):" >&2 - printf ' %s\n' "$hipcc_ver" >&2 - echo " clang ($clang_bin): $clang_ver" >&2 + if [ -n "$hipcc_clang_line" ] && [ -n "$clang_ver" ]; then + if [ "$hipcc_clang_line" == "$clang_ver" ]; then + echo "INFO: hipcc and clang compiler versions match." >&2 + else + echo "WARNING: hipcc and clang report different compiler versions:" >&2 + echo " hipcc ($hipcc_bin):" >&2 + printf ' %s\n' "$hipcc_clang_line" >&2 + echo " clang ($clang_bin): $clang_ver" >&2 + fi + else + echo "WARNING: hipcc and clang compiler versions unverified." >&2 fi } # Use function to set and test AOMP_GPU setaompgpu -if [ ! -d "$hecbench_src" ]; then - echo "ERROR: HeCBench src not found: $hecbench_src" - exit 1 -elif [ ! -d "$hecbench_root" ]; then +if [ ! -d "$hecbench_root" ]; then echo "ERROR: HeCBench not found in $AOMP_REPOS_TEST." exit 1 +elif [ ! -d "$hecbench_src" ]; then + echo "ERROR: HeCBench src not found: $hecbench_src" + exit 1 fi cd "$hecbench_src" || exit 1 @@ -99,7 +104,7 @@ results=$hecbench_root/results.txt rm -f "$results" # Check for a mismatch. -warn_hipcc_clang_mismatch +check_hipcc_clang_mismatch echo PROGRAMMING_MODELS: "$PROGRAMMING_MODELS" for model in $PROGRAMMING_MODELS; do