1#!/usr/bin/python3 2 3# Use GNU compilers 4# module load PrgEnv-gnu 5# module load cray-mpich 6# module load craype-accel-amd-gfx90a 7# module load rocm 8# 9# To enable GPU-aware MPI, one has to also set this runtime environment variable 10# 11# export MPICH_GPU_SUPPORT_ENABLED=1 12# 13# To use hipcc with GPU-aware Cray MPICH, use the following environment variables to setup the needed header files and libraries. 14# 15# -I${MPICH_DIR}/include 16# -L${MPICH_DIR}/lib -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a} 17# 18# See also https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#gpu-aware-mpi 19# 20if __name__ == '__main__': 21 import sys 22 import os 23 sys.path.insert(0, os.path.abspath('config')) 24 import configure 25 configure_options = [ 26 '--with-debugging=0', 27 '--with-cc=cc', 28 '--with-cxx=CC', 29 '--with-fc=ftn', 30 '--with-mpiexec=srun -p batch -N 1 -A csc314 -t 00:10:00', 31 '--with-batch', 32 '--with-hip', 33 '--with-hipc=hipcc', 34 'LIBS={GTLDIR} {GTLLIBS}'.format(GTLDIR=os.environ['PE_MPICH_GTL_DIR_amd_gfx90a'], GTLLIBS=os.environ['PE_MPICH_GTL_LIBS_amd_gfx90a']), 35 '--download-metis', 36 '--download-parmetis', 37 '--download-kokkos', 38 '--download-kokkos-kernels', 39 '--download-superlu_dist', 40 '--download-umpire', 41 '--download-hypre' 42 ] 43 configure.petsc_configure(configure_options) 44 45# Use Cray compilers 46# module load PrgEnv-cray 47# module load cray-mpich 48# module load amd-mixed/5.4.0 49 50# To enable GPU-aware MPI, one has to also set this runtime environment variable 51# 52# export MPICH_GPU_SUPPORT_ENABLED=1 53# 54# Additional note: "craype-accel-amd-gfx90a" module is recommended for 55# "OpenMP offload" or "GPU enabled MPI". It requires "--with-openmp" option. 56# [otherwise building c examples gives link errors (when fortran bindings are enabled)] 57# Alternative is to use "-lmpi_gtl_hsa" as shown below. 58# 59# ld.lld: error: lib/libpetsc.so: undefined reference to .omp_offloading.img_start.cray_amdgcn-amd-amdhsa [--no-allow-shlib-undefined] 60# 61# Also, please ignore warnings like this. If you don't use Fortran, use '--with-fc=0' to get rid of them. 62# 63# ftn-878 ftn: WARNING PETSC, File = ../../../autofs/nccs-svm1_home1/jczhang/petsc/src/tao/ftn-mod/petsctaomod.F90, Line = 37, Column = 13 64# A module named "PETSCVECDEFDUMMY" has already been directly or indirectly use associated into this scope. 65 66# if __name__ == '__main__': 67# import sys 68# import os 69# sys.path.insert(0, os.path.abspath('config')) 70# import configure 71# configure_options = [ 72# '--with-debugging=0', 73# '--with-cc=cc', 74# '--with-cxx=CC', 75# '--with-fc=ftn', 76# # -std=c2x is a workaround for this hipsparse problem 77# # /opt/rocm-5.4.0/include/hipsparse/hipsparse.h:8741:28: error: expected '= constant-expression' or end of enumerator definition 78# # HIPSPARSE_ORDER_COLUMN [[deprecated("Please use HIPSPARSE_ORDER_COL instead")]] = 1, 79# # -Wno-constant-logical-operand is a workaround to suppress excessive warnings caused by -std=c2x in PETSc source which we don't want to address, see MR !6287 80# '--CFLAGS=-std=c2x -Wno-constant-logical-operand', 81# 'LIBS={GTLDIR} {GTLLIBS}'.format(GTLDIR=os.environ['PE_MPICH_GTL_DIR_amd_gfx90a'], GTLLIBS=os.environ['PE_MPICH_GTL_LIBS_amd_gfx90a']), 82# #'--with-openmp=1', # enable if using "craype-accel-amd-gfx90a" module 83# '--with-mpiexec=srun -p batch -N 1 -A csc314 -t 00:10:00', 84# '--with-batch', 85# '--with-hip', 86# '--with-hipc=hipcc', 87# '--download-kokkos', 88# '--download-kokkos-kernels', 89# ] 90# configure.petsc_configure(configure_options) 91