1#!/usr/bin/env python3 2 3# Example configure script for Perlmutter, the HPE Cray EX system at NERSC/LBNL equipped with 4# AMD EPYC CPUS and NVIDIA A100 GPUS. Here we target the GPU compute nodes and builds with 5# support for the CUDA/cuSPARSE, Kokkos, and ViennaCL back-ends. 6# 7# Currently, configuring PETSc on the system does not require loading many , if any, non-default modules. 8# As documented at https://docs.nersc.gov/systems/perlmutter/software/#mpi, typical settings might be 9# 10# export MPICH_GPU_SUPPORT_ENABLED=1 11# module load cudatoolkit 12# module load PrgEnv-gnu 13# module load craype-accel-nvidia80 14# 15# The above are currently present in the default environment. Users may wish to 'module load' a 16# different programming environment (which will generally force a reload of certain related modules, 17# such as the one corresponding to the MPI implementation). 18 19if __name__ == '__main__': 20 import sys 21 import os 22 sys.path.insert(0, os.path.abspath('config')) 23 import configure 24 configure_options = [ 25 '--with-make-np=8', # Must limit size of parallel build to stay within resource limitations imposed by the center 26 '--with-mpiexec=srun -G4', # '-G4' requests all four GPUs present on a Perlmutter GPU compute node. 27 '--with-batch=0', 28 29 # Use the Cray compiler wrappers, regardless of the underlying compilers loaded by the programming environment module: 30 '--with-cc=cc', 31 '--with-cxx=CC', 32 '--with-fc=ftn', 33 34 # Build with aggressive optimization ('-O3') but also include debugging symbols ('-g') to support detailed profiling. 35 # If you are doing development, using no optimization ('-O0') can be a good idea. Also note that some compilers (GNU 36 # is one) support the '-g3' debug flag, which allows macro expansion in some debuggers; this can be very useful when 37 # debugging PETSc code, as PETSc makes extensive use of macros. 38 '--COPTFLAGS= -g -O3', 39 '--CXXOPTFLAGS= -g -O3', 40 '--FOPTFLAGS= -g -O3', 41 '--CUDAFLAGS= -g -O3', 42 '--with-debugging=0', # Disable debugging for production builds; use '--with-debugging=1' for development work. 43 44 # Build with support for CUDA/cuSPARSE, Kokkos/Kokkos Kernels, and ViennaCL back-ends: 45 '--with-cuda=1', 46 '--with-cuda-arch=80', 47 '--download-viennacl', 48 '--download-kokkos', 49 '--download-kokkos-kernels', 50 51 # Download and build a few commonly-used packages: 52 '--download-hypre', 53 '--download-metis', 54 '--download-parmetis', 55 '--download-hdf5', # Note that NERSC does provide an HDF5 module, but using our own is generally reliable. 56 '--download-hdf5-fortran-bindings', 57 ] 58 configure.petsc_configure(configure_options) 59