19d22c58dSStefano Zampini#!/usr/bin/python3 29d22c58dSStefano Zampini 3*6ea4635bSSatish Balay# Modules loaded by default (on login to spock): 49d22c58dSStefano Zampini# 5*6ea4635bSSatish Balay# 1) craype-x86-rome 8) cce/12.0.3 6*6ea4635bSSatish Balay# 2) libfabric/1.11.0.4.75 9) craype/2.7.11 7*6ea4635bSSatish Balay# 3) craype-network-ofi 10) cray-dsmml/0.2.2 8*6ea4635bSSatish Balay# 4) perftools-base/21.10.0 11) cray-mpich/8.1.10 9*6ea4635bSSatish Balay# 5) xpmem/2.2.40-2.1_2.44__g3cf3325.shasta 12) cray-libsci/21.08.1.2 10*6ea4635bSSatish Balay# 6) cray-pmi/6.0.14 13) PrgEnv-cray/8.2.0 11*6ea4635bSSatish Balay# 7) cray-pmi-lib/6.0.14 14) DefApps/default 12*6ea4635bSSatish Balay# 13*6ea4635bSSatish Balay# Need to load additional rocm module to build with hip 14*6ea4635bSSatish Balay# 15*6ea4635bSSatish Balay# module load rocm/4.3.0 16*6ea4635bSSatish Balay# 17*6ea4635bSSatish Balay# Note: LIBS option below is needed to use GPU enabled MPI. It also requires 18*6ea4635bSSatish Balay# the following env variables to be set at runtime 19*6ea4635bSSatish Balay# 209d22c58dSStefano Zampini# export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0 219d22c58dSStefano Zampini# export MPICH_GPU_SUPPORT_ENABLED=1 229d22c58dSStefano Zampini# export MPICH_SMP_SINGLE_COPY_MODE=CMA 239d22c58dSStefano Zampini# 24*6ea4635bSSatish Balay# Additional note: If "craype-accel-amd-gfx908" module is loaded (that is 25*6ea4635bSSatish Balay# needed for "OpenMP offload") - it causes link errors when using 'cc or hipcc' 26*6ea4635bSSatish Balay# with fortran objs, hence not used 27*6ea4635bSSatish Balay# 289d22c58dSStefano Zampini 299d22c58dSStefano Zampiniif __name__ == '__main__': 309d22c58dSStefano Zampini import sys 319d22c58dSStefano Zampini import os 329d22c58dSStefano Zampini sys.path.insert(0, os.path.abspath('config')) 339d22c58dSStefano Zampini import configure 349d22c58dSStefano Zampini configure_options = [ 359d22c58dSStefano Zampini '--with-cc=cc', 369d22c58dSStefano Zampini '--with-cxx=CC', 379d22c58dSStefano Zampini '--with-fc=ftn', 38*6ea4635bSSatish Balay 'LIBS=-L{x}/gtl/lib -lmpi_gtl_hsa'.format(x=os.environ['CRAY_MPICH_ROOTDIR']), 39*6ea4635bSSatish Balay '--with-debugging=0', 40*6ea4635bSSatish Balay '--with-mpiexec=srun -p ecp -N 1 -A csc314 -t 00:10:00', 419d22c58dSStefano Zampini '--with-hip=1', 429d22c58dSStefano Zampini '--with-hipc=hipcc', 43*6ea4635bSSatish Balay '--download-kokkos=1', 44*6ea4635bSSatish Balay '--download-kokkos-kernels=1', 45*6ea4635bSSatish Balay '--download-magma=1', 469d22c58dSStefano Zampini '--with-magma-gputarget=gfx908', 479d22c58dSStefano Zampini ] 489d22c58dSStefano Zampini configure.petsc_configure(configure_options) 49