1#!/usr/bin/python3 2 3# Modules loaded by default (on login to Crusher): 4# 5# 1) craype-x86-trento 9) craype/2.7.15 6# 2) libfabric/1.15.0.0 10) cray-dsmml/0.2.2 7# 3) craype-network-ofi 11) cray-mpich/8.1.16 8# 4) perftools-base/22.05.0 12) cray-libsci/21.08.1.2 9# 5) xpmem/2.3.2-2.2_7.8__g93dd7ee.shasta 13) PrgEnv-cray/8.3.3 10# 6) cray-pmi/6.1.2 14) xalt/1.3.0 11# 7) cray-pmi-lib/6.0.17 15) DefApps/default 12# 8) cce/14.0.0 13# 14# Need to load additional rocm module to build with hip 15# 16# module load rocm/5.1.0 17# 18# We use Cray Programming Environment, Cray compilers, Cray-mpich. 19# To enable GPU-aware MPI, one has to also set this runtime environment variable 20# 21# export MPICH_GPU_SUPPORT_ENABLED=1 22# 23# Additional note: "craype-accel-amd-gfx90a" module is recommended for 24# "OpenMP offload" or "GPU enabled MPI". It requires "--with-openmp" option. 25# [otherwise building c examples gives link errors (when fortran bindings are enabled)] 26# Alternative is to use "-lmpi_gtl_hsa" as used below. 27# 28# ld.lld: error: /autofs/nccs-svm1_home1/balay/petsc/arch-olcf-crusher/lib/libpetsc.so: undefined reference to .omp_offloading.img_start.cray_amdgcn-amd-amdhsa [--no-allow-shlib-undefined] 29# 30 31if __name__ == '__main__': 32 import sys 33 import os 34 sys.path.insert(0, os.path.abspath('config')) 35 import configure 36 configure_options = [ 37 '--with-cc=cc', 38 '--with-cxx=CC', 39 '--with-fc=ftn', 40 'LIBS=-L{x}/gtl/lib -lmpi_gtl_hsa'.format(x=os.environ['CRAY_MPICH_ROOTDIR']), 41 #'--with-openmp=1', # enable if using "craype-accel-amd-gfx90a" module 42 '--with-debugging=0', 43 '--with-mpiexec=srun -p batch -N 1 -A csc314_crusher -t 00:10:00', 44 '--with-hip', 45 '--with-hipc=hipcc', 46 '--download-kokkos', 47 '--download-kokkos-kernels', 48 ] 49 configure.petsc_configure(configure_options) 50