Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 0 additions & 33 deletions .github/workflows/docker_build_tpls.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,6 @@ jobs:
GCC_VERSION: 13
SPEC: "~pygeosx +docs %gcc-13"

- name: Ubuntu 24.04 - gcc 13 (+docs +hypredrive)
RUNS_ON: ubuntu-latest
TPL_DOCKERFILE: docker/tpl-ubuntu.Dockerfile
DOCKER_BASE_IMAGE_TAG: 24.04-gcc13
DOCKER_BASE_IMAGE_REPO: geosx/ubuntu
DOCKER_REPOSITORY: geosx/ubuntu24.04-gcc13-hypredrive
GCC_VERSION: 13
SPEC: "~pygeosx +docs +hypredrive %gcc-13"

- name: Ubuntu 24.04 - clang 19
RUNS_ON: ubuntu-latest
TPL_DOCKERFILE: docker/tpl-ubuntu.Dockerfile
Expand Down Expand Up @@ -86,14 +77,6 @@ jobs:
DOCKER_REPOSITORY: geosx/rockylinux8-gcc13
SPEC: "~pygeosx ~docs %gcc-13"

- name: Rocky Linux 8 - gcc 13 (+hypredrive)
RUNS_ON: ubuntu-latest
TPL_DOCKERFILE: docker/tpl-rockylinux.Dockerfile
DOCKER_BASE_IMAGE_TAG: 8-gcc13
DOCKER_BASE_IMAGE_REPO: geosx/rockylinux
DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-hypredrive
SPEC: "~pygeosx ~docs +hypredrive %gcc-13"

- name: Rocky Linux 8 - clang 19
RUNS_ON: ubuntu-latest
TPL_DOCKERFILE: docker/tpl-rockylinux.Dockerfile
Expand All @@ -120,14 +103,6 @@ jobs:
GCC_VERSION: 13
SPEC: "+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers"

- name: Ubuntu 24.04 - gcc 13 + CUDA 12.9.1 (+hypredrive)
RUNS_ON: streak2
TPL_DOCKERFILE: docker/tpl-ubuntu.Dockerfile
DOCKER_BASE_IMAGE_TAG: 24.04-gcc13-cuda12.9.1
DOCKER_BASE_IMAGE_REPO: geosx/ubuntu
DOCKER_REPOSITORY: geosx/ubuntu24.04-gcc13-cuda12.9.1-hypredrive
GCC_VERSION: 13
SPEC: "+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs +hypredrive %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers"

# CUDA 13 is blocked by the pinned RAJA package:
# raja: '^cuda@13:' conflicts with '+cuda'
Expand Down Expand Up @@ -166,14 +141,6 @@ jobs:
DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1
SPEC: "+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers"

- name: Rocky Linux 8 - gcc 13 + CUDA 12.9.1 (+hypredrive)
RUNS_ON: streak2
TPL_DOCKERFILE: docker/tpl-rockylinux.Dockerfile
DOCKER_BASE_IMAGE_TAG: 8-gcc13-cuda12.9.1
DOCKER_BASE_IMAGE_REPO: geosx/rockylinux
DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1-hypredrive
SPEC: "+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs +hypredrive %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers"

# Rocky8 clang19 CUDA 12.9.1 is blocked by SuperLU_DIST using
# C++11 with CUDA 12.9 + clang19 + GCC 14 libstdc++.
# - name: Rocky Linux 8 - clang 19 + CUDA 12.9.1
Expand Down
5 changes: 5 additions & 0 deletions scripts/docker-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ EXTRA_BUILD_ARGS=()
if [ -n "${GCC_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "GCC_VERSION=${GCC_VERSION}"); fi
if [ -n "${CLANG_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "CLANG_VERSION=${CLANG_VERSION}"); fi

BUILDER_ARGS=()
if [ -n "${DOCKER_BUILDER:-}" ]; then BUILDER_ARGS+=(--builder "${DOCKER_BUILDER}"); fi
if [ "${DOCKER_LOAD:-0}" = 1 ]; then BUILDER_ARGS+=(--load); fi

docker build --progress=plain \
"${BUILDER_ARGS[@]}" \
--build-arg HOST_CONFIG=${HOST_CONFIG} \
--build-arg DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} \
--build-arg INSTALL_DIR=${INSTALL_DIR} \
Expand Down
3 changes: 0 additions & 3 deletions scripts/setupLC-TPL-uberenv.bash
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ function launch_jobs() {
ALLOC_CMD="srun -N 1 --exclusive -t 60 -A vortex"
"${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %%gcc-12 ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %%gcc-13 ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13-hypredrive "+docs +hypredrive %%gcc-13 ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" dane llvm-14 "+docs %%clang-14 ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" dane llvm-19 "+docs %%clang-19 ${COMMON}" "${ALLOC_CMD}" "$@" &
;;
Expand All @@ -100,7 +99,6 @@ function launch_jobs() {
ALLOC_CMD="srun -N 1 --exclusive -t 60 -A vortex"
"${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda ~uncrustify cuda_arch=90 %%gcc-12 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda ~uncrustify cuda_arch=90 %%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9-hypredrive "+cuda ~uncrustify +hypredrive cuda_arch=90 %%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" matrix llvm-14-cuda-12.6 "+cuda ~uncrustify cuda_arch=90 %%clang-14 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" matrix llvm-19-cuda-12.9 "+cuda ~uncrustify cuda_arch=90 %%clang-19 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" &
;;
Expand All @@ -109,7 +107,6 @@ function launch_jobs() {
ALLOC_CMD="salloc -N 1 --exclusive -p pdebug -t 60 -A vortex"
"${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.3 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %%cce-20 ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne llvm-amdgpu-6.4.3 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %%llvm-amdgpu_6_4_3 ${COMMON}" "${ALLOC_CMD}" "$@" &
"${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne llvm-amdgpu-6.4.3-hypredrive "+rocm~pygeosx~trilinos~petsc~docs+hypredrive amdgpu_target=gfx942 %%llvm-amdgpu_6_4_3 ${COMMON}" "${ALLOC_CMD}" "$@" &
;;

*)
Expand Down
8 changes: 4 additions & 4 deletions scripts/spack_configs/versions.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@
# This file lists the package versions for geos's dependencies
#
packages:
# master - 01/23/26
# master - 08/01/26
hypre:
require: "@git.8b0093306228fef1b92384d9face7fbe5a63b460=master"
require: "@git.8df038ce669c623cd403ab77f3b4850fbb622dc9=master"

# master - 03/28/26
# master - 08/01/26
hypredrive:
require: "@git.d062c48cc8d26a9bee92f33f3c588d071c230d30=master"
require: "@git.21a2e3abe2a6f5cd0c6db89ff419f851c3f6c13f=master"

# master - 04/12/20
uncrustify:
Expand Down
2 changes: 1 addition & 1 deletion scripts/spack_packages/packages/geosx/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage):
variant('vtk', default=True, description='Build VTK support.')
variant('trilinos', default=True, description='Build Trilinos support.')
variant('hypre', default=True, description='Build HYPRE support.')
variant('hypredrive', default=False, description='Build hypredrive support.')
variant('hypredrive', default=True, description='Build hypredrive support.')
variant('petsc', default=False, description='Build PETSc support.')
variant('scotch', default=True, description='Build Scotch support.')
variant('uncrustify', default=True, description='Build Uncrustify support.')
Expand Down
3 changes: 2 additions & 1 deletion scripts/spack_packages/packages/hypredrive/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,15 @@ class Hypredrive(CMakePackage):
def cmake_args(self):
spec = self.spec
from_variant = self.define_from_variant
pic_enabled = "+pic" in spec or "+shared" in spec

args = [
from_variant("BUILD_SHARED_LIBS", "shared"),
from_variant("HYPREDRV_ENABLE_EXAMPLES", "examples"),
from_variant("HYPREDRV_ENABLE_HWLOC", "hwloc"),
from_variant("HYPREDRV_ENABLE_CALIPER", "caliper"),
from_variant("HYPREDRV_ENABLE_COMPRESSION", "compression"),
from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
self.define("CMAKE_POSITION_INDEPENDENT_CODE", pic_enabled),
self.define("HYPRE_ROOT", spec["hypre"].prefix),
self.define("HYPREDRV_ENABLE_TESTING", self.run_tests),
self.define("HYPREDRV_ENABLE_COVERAGE", False),
Expand Down
228 changes: 228 additions & 0 deletions scripts/spack_packages/packages/trilinos/fix_ml_metis_idx_t.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
diff -Naur a/packages/ml/src/Coarsen/ml_agg_METIS.c b/packages/ml/src/Coarsen/ml_agg_METIS.c
--- a/packages/ml/src/Coarsen/ml_agg_METIS.c 2025-03-19 14:59:00
+++ b/packages/ml/src/Coarsen/ml_agg_METIS.c 2026-05-30 14:31:46
@@ -82,7 +82,7 @@
int graph_decomposition[],
int mypid);
*/
-static int ML_LocalReorder_with_METIS( int Nrows, int xadj[], int adjncy[] ,
+static int ML_LocalReorder_with_METIS( int Nrows, indextype xadj[], indextype adjncy[] ,
int Nparts, indextype part[], int level,
ML_Comm *comm );

@@ -441,7 +441,7 @@
*/
/* ------------------------------------------------------------------------ */

-static int ML_LocalReorder_with_METIS( int Nrows, int xadj[], int adjncy[] ,
+static int ML_LocalReorder_with_METIS( int Nrows, indextype xadj[], indextype adjncy[] ,
int Nparts, indextype part[], int level,
ML_Comm *comm )
{
@@ -451,7 +451,11 @@
int MaxNnzRow;
indextype * xadj2 = NULL, * adjncy2 = NULL;
indextype * perm = NULL, * iperm = NULL;
+#if defined(HAVE_ML_METIS) && (METIS_VER_MAJOR >= 5)
+ indextype options[METIS_NOPTIONS];
+#else
indextype options[8];
+#endif
size_t used_mem;
int bandwidth_orig, bandwidth_perm;
int mypid = comm->ML_mypid;
@@ -616,7 +620,11 @@
exit( EXIT_FAILURE );
}

+#if defined(HAVE_ML_METIS) && (METIS_VER_MAJOR >= 5)
+ METIS_SetDefaultOptions(options);
+#else
options[0] = 0; /* default values */
+#endif
/* those are instead possible non-default values. No idea about how
to pick them up...
options[0] = 1;
@@ -663,7 +671,14 @@

i = 0; /* offset, C or FORTRAN */
#ifdef HAVE_ML_METIS
+# if METIS_VER_MAJOR < 5
METIS_NodeND( &Nparts, xadj2, adjncy2, &i, options, perm, iperm );
+# else
+ {
+ indextype metis_nparts = (indextype) Nparts;
+ METIS_NodeND( &metis_nparts, xadj2, adjncy2, NULL, options, perm, iperm );
+ }
+# endif
#else
fprintf( stderr,
"*ERR*ML* This function has been compiled without -DHAVE_ML_METIS\n"
@@ -750,7 +765,8 @@

int i, j,jj, count, count2, count_start;
int Nrows, Nrows_global,NrowsMETIS, N_nonzeros, N_bdry_nodes;
- int *wgtflag=NULL, numflag;
+ indextype *wgtflag=NULL;
+ int numflag;
indextype *xadj=NULL, *adjncy=NULL;
#ifdef HAVE_ML_METIS
indextype *vwgt=NULL, *adjwgt=NULL;
@@ -970,8 +986,12 @@
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
- METIS_PartGraphRecursive (&NrowsMETIS, &numBalancingConstraints, xadj, adjncy,
- vwgt, vsize, adjwgt, &N_parts, tpwgts, ubvec, options, &edgecut, part);
+ {
+ indextype metis_nrows = (indextype) NrowsMETIS;
+ indextype metis_nparts = (indextype) N_parts;
+ METIS_PartGraphRecursive (&metis_nrows, &numBalancingConstraints, xadj, adjncy,
+ vwgt, vsize, adjwgt, &metis_nparts, tpwgts, ubvec, options, &edgecut, part);
+ }
# endif
} else {

@@ -990,8 +1010,12 @@
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
- METIS_PartGraphKway (&NrowsMETIS, &numBalancingConstraints, xadj, adjncy,
- vwgt, vsize, adjwgt, &N_parts, tpwgts, ubvec, options, &edgecut, part);
+ {
+ indextype metis_nrows = (indextype) NrowsMETIS;
+ indextype metis_nparts = (indextype) N_parts;
+ METIS_PartGraphKway (&metis_nrows, &numBalancingConstraints, xadj, adjncy,
+ vwgt, vsize, adjwgt, &metis_nparts, tpwgts, ubvec, options, &edgecut, part);
+ }
# endif
}
#else
diff -Naur a/packages/ml/src/Coarsen/ml_agg_ParMETIS.c b/packages/ml/src/Coarsen/ml_agg_ParMETIS.c
--- a/packages/ml/src/Coarsen/ml_agg_ParMETIS.c 2025-03-19 14:59:00
+++ b/packages/ml/src/Coarsen/ml_agg_ParMETIS.c 2026-05-30 14:31:56
@@ -472,7 +472,7 @@

int i, j,jj, count;
int Nrows;
- int *wgtflag=NULL, numflag, *options=NULL, edgecut;
+ indextype *wgtflag=NULL, numflag, *options=NULL, edgecut;
indextype *xadj=NULL, *adjncy=NULL;
#if defined(ML_MPI)
#if defined(HAVE_ML_PARMETIS)
@@ -490,7 +490,7 @@
int mypid = Amatrix->comm->ML_mypid;
int * offsets = NULL;
indextype * vtxdist = NULL;
- int ncon = 1;
+ indextype ncon = 1;
float * tpwgts = NULL;
float ubvec; /* size = ncon */
int * proc_with_parmetis = NULL;
@@ -670,7 +670,7 @@
/* set parameters for ParMETIS */

wgtflag = (indextype *) ML_allocate (4*sizeof(indextype));
- options = (int *) ML_allocate (4*sizeof(int));
+ options = (indextype *) ML_allocate (4*sizeof(indextype));

wgtflag[0] = 0; /* no weights */
numflag = 0; /* C style */
@@ -734,10 +734,13 @@

#if defined(HAVE_ML_PARMETIS) && defined(ML_MPI)
/* never tested right now... */
- ParMETIS_V3_PartKway (vtxdist, xadj, adjncy, vwgt, adjwgt,
- wgtflag, &numflag, &ncon, &N_parts, tpwgts,
- &ubvec,options,
- &edgecut, part, &ParMETISComm);
+ {
+ indextype metis_nparts = (indextype) N_parts;
+ ParMETIS_V3_PartKway (vtxdist, xadj, adjncy, vwgt, adjwgt,
+ wgtflag, &numflag, &ncon, &metis_nparts, tpwgts,
+ &ubvec, options,
+ &edgecut, part, &ParMETISComm);
+ }
#else
if( Amatrix->comm->ML_mypid == 0 ) {
fprintf( stderr,
diff -Naur a/packages/ml/src/Coarsen/ml_agg_VBMETIS.c b/packages/ml/src/Coarsen/ml_agg_VBMETIS.c
--- a/packages/ml/src/Coarsen/ml_agg_VBMETIS.c 2025-03-19 14:59:00
+++ b/packages/ml/src/Coarsen/ml_agg_VBMETIS.c 2026-05-30 14:31:46
@@ -2081,7 +2081,8 @@

int i, j,jj, count, count2;
int Nrows, Nrows_global,NrowsMETIS, N_nonzeros, N_bdry_nodes;
- int *wgtflag=NULL, numflag;
+ indextype *wgtflag=NULL;
+ int numflag;
indextype *xadj=NULL, *adjncy=NULL;
#ifdef HAVE_ML_METIS
indextype *vwgt=NULL, *adjwgt=NULL;
@@ -2325,8 +2326,12 @@
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
- METIS_PartGraphRecursive (&NrowsMETIS, &numBalancingConstraints, xadj, adjncy,
- vwgt, vsize, adjwgt, &N_parts, tpwgts, ubvec, options, &edgecut, part);
+ {
+ indextype metis_nrows = (indextype) NrowsMETIS;
+ indextype metis_nparts = (indextype) N_parts;
+ METIS_PartGraphRecursive (&metis_nrows, &numBalancingConstraints, xadj, adjncy,
+ vwgt, vsize, adjwgt, &metis_nparts, tpwgts, ubvec, options, &edgecut, part);
+ }
# endif
} else {

@@ -2345,8 +2350,12 @@
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
- METIS_PartGraphKway (&NrowsMETIS, &numBalancingConstraints, xadj, adjncy,
- vwgt, vsize, adjwgt, &N_parts, tpwgts, ubvec, options, &edgecut, part);
+ {
+ indextype metis_nrows = (indextype) NrowsMETIS;
+ indextype metis_nparts = (indextype) N_parts;
+ METIS_PartGraphKway (&metis_nrows, &numBalancingConstraints, xadj, adjncy,
+ vwgt, vsize, adjwgt, &metis_nparts, tpwgts, ubvec, options, &edgecut, part);
+ }
# endif
}
#else
diff -Naur a/packages/ml/src/Operator/ml_op_utils.c b/packages/ml/src/Operator/ml_op_utils.c
--- a/packages/ml/src/Operator/ml_op_utils.c 2025-03-19 14:59:00
+++ b/packages/ml/src/Operator/ml_op_utils.c 2026-05-30 15:11:07
@@ -461,3 +461,4 @@
- indextype *vtxdist = NULL, *tpwts = NULL, *adjncy = NULL, *xadj = NULL, *node_wt = NULL;
+ indextype *vtxdist = NULL, *adjncy = NULL, *xadj = NULL, *node_wt = NULL;
int *map = NULL, *bindx = NULL, *blks = NULL, nprocs, myid, j, ii, jj;
+ int *vtxdist_int = NULL, *tpwts_int = NULL;
int allocated = 0, row_length, itemp1, itemp2, Nrows;
@@ -741,7 +742,13 @@
- tpwts = (indextype *) ML_allocate(sizeof(indextype)*ML_max(*nblks,nprocs));
vtxdist = (indextype *) ML_allocate(sizeof(indextype)*(nprocs+1));
+ vtxdist_int = (int *) ML_allocate(sizeof(int)*nprocs);
+ tpwts_int = (int *) ML_allocate(sizeof(int)*nprocs);
for (ii = 0; ii <= nprocs; ii++) vtxdist[ii] = 0;
- vtxdist[myid] = n;
- ML_gsum_vec_int(&vtxdist,&tpwts,nprocs,matrix->comm);
-
+ for (ii = 0; ii < nprocs; ii++) vtxdist_int[ii] = 0;
+ vtxdist_int[myid] = n;
+ ML_gsum_vec_int(&vtxdist_int,&tpwts_int,nprocs,matrix->comm);
+ for (ii = 0; ii < nprocs; ii++) vtxdist[ii] = (indextype) vtxdist_int[ii];
+ ML_free(vtxdist_int);
+ ML_free(tpwts_int);
+ vtxdist_int = NULL;
+ tpwts_int = NULL;
itemp1 = 0;
@@ -865,8 +873,9 @@
if (bindx != NULL) ML_free(bindx);
if (blks != NULL) ML_free(blks);
if (vtxdist != NULL) ML_free(vtxdist);
- if (tpwts != NULL) ML_free(tpwts);
+ if (vtxdist_int != NULL) ML_free(vtxdist_int);
+ if (tpwts_int != NULL) ML_free(tpwts_int);
if (adjncy != NULL) ML_free(adjncy);
if (xadj != NULL) ML_free(xadj);
if (node_wt != NULL) ML_free(node_wt);
1 change: 1 addition & 0 deletions scripts/spack_packages/packages/trilinos/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,7 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage):
# patch('xlf_seacas.patch', when='@12.10.1:12.12.1 %' + _compiler)
# patch('xlf_tpetra.patch', when='@12.12.1 %' + _compiler)
patch('fix_clang_errors_12_18_1.patch', when='@12.18.1%clang')
patch('fix_ml_metis_idx_t.patch', when='@16.1.0 +ml ^metis+int64')
#patch('cray_secas_12_12_1.patch', when='@12.12.1%cce')
patch('cray_secas.patch', when='@12.14.1:%cce')

Expand Down