1#!/usr/bin/env python3 2 3# Example configure script for Perlmutter, the HPE Cray EX system at NERSC/LBNL equipped with 4# wth AMD EPYC CPUS and NVIDIA A100 GPUS. Here we target the GPU compute nodes and builds with 5# support for the CUDA/cuSPARSE, Kokkos, and ViennaCL back-ends. 6# 7# Currently, configuring PETSc on the system does not require loading many , if any, non-default modules. 8# As documented at https://docs.nersc.gov/systems/perlmutter/software/#mpi, typical settings might be 9# 10# export MPICH_GPU_SUPPORT_ENABLED=1 11# module load cudatoolkit 12# module load PrgEnv-gnu 13# module load craype-accel-nvidia80 14# 15# The above are currently present in the default environment. Users may wish to 'module load' a 16# different programming environment (which will generally force a reload of certain related modules, 17# such as the one corresponding to the MPI implementation). 18 19if __name__ == '__main__': 20 import sys 21 import os 22 sys.path.insert(0, os.path.abspath('config')) 23 import configure 24 configure_options = [ 25 '--with-make-np=8', # Must limit size of parallel build to stay within resource limitations imposed by the center 26 '--with-mpiexec=srun -G4', # '-G4' requests all four GPUs present on a Perlmutter GPU compute node. 27 '--with-batch=0', 28 29 # Use the Cray compiler wrappers, regardless of the underlying compilers loaded by the programming environment module: 30 '--with-cc=cc', 31 '--with-cxx=CC', 32 '--with-fc=ftn', 33 34 # Build with aggressive optimization ('-O3') but also include debugging symbols ('-g') to support detailed profiling. 35 # If you are doing development, using no optimization ('-O0') can be a good idea. Also note that some compilers (GNU 36 # is one) support the '-g3' debug flag, which allows macro expansion in some debuggers; this can be very useful when 37 # debugging PETSc code, as PETSc makes extensive use of macros. 38 '--COPTFLAGS= -g -O3', 39 '--CXXOPTFLAGS= -g -O3', 40 '--FOPTFLAGS= -g -O3', 41 '--CUDAFLAGS= -g -O3', 42 '--with-debugging=0', # Disable debugging for production builds; use '--with-debugging=1' for development work. 43 44 # Set sowing-cc and sowing-cxx explicitly, as this prevents errors caused by compiling sowing with GCC when a 45 # programming environment other than PrgEnv-gnu has been loaded. If there is this compiler mismatch, we will see 46 # errors like 47 # 48 # /opt/nvidia/hpc_sdk/Linux_x86_64/22.5/compilers/include/bits/floatn.h:60:17: error: two or more data types in declaration specifiers 49 # typedef float _Float32; 50 # ^~~~~~~~ 51 '--download-sowing-cc=cc', # Note that sowing is only needed when Fortran bindings are required. 52 '--download-sowing-cc=CC', 53 54 55 # Build with support for CUDA/cuSPARSE, Kokkos/Kokkos Kernels, and ViennaCL back-ends: 56 '--with-cuda=1', 57 '--with-cuda-arch=80', 58 '--download-viennacl', 59 '--download-kokkos', 60 '--download-kokkos-kernels', 61 '--with-kokkos-kernels-tpl=0', # Use native Kokkos kernels, rather than NVIDIA-provided ones. 62 63 # Download and build a few commonly-used packages: 64 '--download-hypre', 65 '--download-metis', 66 '--download-parmetis', 67 '--download-hdf5', # Note that NERSC does provide an HDF5 module, but using our own is generally reliable. 68 '--download-hdf5-fortran-bindings', 69 ] 70 configure.petsc_configure(configure_options) 71