1#!/usr/bin/env python3 2 3# Example configure script for Perlmutter, the HPE Cray EX system at NERSC/LBNL equipped with 4# AMD EPYC CPUS and NVIDIA A100 GPUS. Here we target the GPU compute nodes and builds with 5# support for the CUDA/cuSPARSE, Kokkos, and ViennaCL back-ends. 6# 7# Currently, configuring PETSc on the system does not require loading many , if any, non-default modules. 8# As documented at https://docs.nersc.gov/systems/perlmutter/software/#mpi, typical settings might be 9# 10# export MPICH_GPU_SUPPORT_ENABLED=1 11# module load cudatoolkit 12# module load PrgEnv-gnu 13# module load craype-accel-nvidia80 14# module load cray-python 15# 16# The above are currently present in the default environment. Users may wish to 'module load' a 17# different programming environment (which will generally force a reload of certain related modules, 18# such as the one corresponding to the MPI implementation). 19 20if __name__ == '__main__': 21 import sys 22 import os 23 sys.path.insert(0, os.path.abspath('config')) 24 import configure 25 configure_options = [ 26 '--with-make-np=8', # Must limit size of parallel build to stay within resource limitations imposed by the center 27 # '-G4' requests all four GPUs present on a Perlmutter GPU compute node. 28 # --gpu-bind=none to avoid the gpu-aware mpi runtime error: (GTL DEBUG: 0) cuIpcOpenMemHandle: invalid argument, CUDA_ERROR_INVALID_VALUE, line no 360 29 '--with-mpiexec=srun -G4 --gpu-bind=none', 30 '--with-batch=0', 31 32 # Use the Cray compiler wrappers, regardless of the underlying compilers loaded by the programming environment module: 33 '--with-cc=cc', 34 '--with-cxx=CC', 35 '--with-fc=ftn', 36 37 # Build with aggressive optimization ('-O3') but also include debugging symbols ('-g') to support detailed profiling. 38 # If you are doing development, using no optimization ('-O0') can be a good idea. Also note that some compilers (GNU 39 # is one) support the '-g3' debug flag, which allows macro expansion in some debuggers; this can be very useful when 40 # debugging PETSc code, as PETSc makes extensive use of macros. 41 '--COPTFLAGS= -g -O3', 42 '--CXXOPTFLAGS= -g -O3', 43 '--FOPTFLAGS= -g -O3', 44 '--CUDAFLAGS= -g -O3', 45 '--with-debugging=0', # Disable debugging for production builds; use '--with-debugging=1' for development work. 46 47 # Build with support for CUDA/cuSPARSE, Kokkos/Kokkos Kernels, and ViennaCL back-ends: 48 '--with-cuda=1', 49 '--with-cuda-arch=80', 50 '--download-viennacl', 51 '--download-kokkos', 52 '--download-kokkos-kernels', 53 54 # Download and build a few commonly-used packages: 55 '--download-umpire', 56 '--download-hypre', 57 '--download-metis', 58 '--download-parmetis', 59 '--download-hdf5', # Note that NERSC does provide an HDF5 module, but using our own is generally reliable. 60 '--download-hdf5-fortran-bindings', 61 ] 62 configure.petsc_configure(configure_options) 63