12e294c49SRichard Tran Mills#!/usr/bin/env python3 22e294c49SRichard Tran Mills 32e294c49SRichard Tran Mills# Example configure script for Perlmutter, the HPE Cray EX system at NERSC/LBNL equipped with 45804573cSPierre Jolivet# AMD EPYC CPUS and NVIDIA A100 GPUS. Here we target the GPU compute nodes and builds with 52e294c49SRichard Tran Mills# support for the CUDA/cuSPARSE, Kokkos, and ViennaCL back-ends. 62e294c49SRichard Tran Mills# 72e294c49SRichard Tran Mills# Currently, configuring PETSc on the system does not require loading many , if any, non-default modules. 82e294c49SRichard Tran Mills# As documented at https://docs.nersc.gov/systems/perlmutter/software/#mpi, typical settings might be 92e294c49SRichard Tran Mills# 102e294c49SRichard Tran Mills# export MPICH_GPU_SUPPORT_ENABLED=1 112e294c49SRichard Tran Mills# module load cudatoolkit 122e294c49SRichard Tran Mills# module load PrgEnv-gnu 132e294c49SRichard Tran Mills# module load craype-accel-nvidia80 145c20b387SJunchao Zhang# module load cray-python 152e294c49SRichard Tran Mills# 162e294c49SRichard Tran Mills# The above are currently present in the default environment. Users may wish to 'module load' a 172e294c49SRichard Tran Mills# different programming environment (which will generally force a reload of certain related modules, 182e294c49SRichard Tran Mills# such as the one corresponding to the MPI implementation). 192e294c49SRichard Tran Mills 202e294c49SRichard Tran Millsif __name__ == '__main__': 212e294c49SRichard Tran Mills import sys 222e294c49SRichard Tran Mills import os 232e294c49SRichard Tran Mills sys.path.insert(0, os.path.abspath('config')) 242e294c49SRichard Tran Mills import configure 252e294c49SRichard Tran Mills configure_options = [ 262e294c49SRichard Tran Mills '--with-make-np=8', # Must limit size of parallel build to stay within resource limitations imposed by the center 275c20b387SJunchao Zhang # '-G4' requests all four GPUs present on a Perlmutter GPU compute node. 285c20b387SJunchao Zhang # --gpu-bind=none to avoid the gpu-aware mpi runtime error: (GTL DEBUG: 0) cuIpcOpenMemHandle: invalid argument, CUDA_ERROR_INVALID_VALUE, line no 360 295c20b387SJunchao Zhang '--with-mpiexec=srun -G4 --gpu-bind=none', 302e294c49SRichard Tran Mills '--with-batch=0', 312e294c49SRichard Tran Mills 322e294c49SRichard Tran Mills # Use the Cray compiler wrappers, regardless of the underlying compilers loaded by the programming environment module: 332e294c49SRichard Tran Mills '--with-cc=cc', 342e294c49SRichard Tran Mills '--with-cxx=CC', 352e294c49SRichard Tran Mills '--with-fc=ftn', 362e294c49SRichard Tran Mills 372e294c49SRichard Tran Mills # Build with aggressive optimization ('-O3') but also include debugging symbols ('-g') to support detailed profiling. 382e294c49SRichard Tran Mills # If you are doing development, using no optimization ('-O0') can be a good idea. Also note that some compilers (GNU 392e294c49SRichard Tran Mills # is one) support the '-g3' debug flag, which allows macro expansion in some debuggers; this can be very useful when 402e294c49SRichard Tran Mills # debugging PETSc code, as PETSc makes extensive use of macros. 412e294c49SRichard Tran Mills '--COPTFLAGS= -g -O3', 422e294c49SRichard Tran Mills '--CXXOPTFLAGS= -g -O3', 432e294c49SRichard Tran Mills '--FOPTFLAGS= -g -O3', 442e294c49SRichard Tran Mills '--CUDAFLAGS= -g -O3', 452e294c49SRichard Tran Mills '--with-debugging=0', # Disable debugging for production builds; use '--with-debugging=1' for development work. 462e294c49SRichard Tran Mills 472e294c49SRichard Tran Mills # Build with support for CUDA/cuSPARSE, Kokkos/Kokkos Kernels, and ViennaCL back-ends: 482e294c49SRichard Tran Mills '--with-cuda=1', 492e294c49SRichard Tran Mills '--with-cuda-arch=80', 502e294c49SRichard Tran Mills '--download-viennacl', 512e294c49SRichard Tran Mills '--download-kokkos', 522e294c49SRichard Tran Mills '--download-kokkos-kernels', 532e294c49SRichard Tran Mills 542e294c49SRichard Tran Mills # Download and build a few commonly-used packages: 55*04ba64a0SVictor A. P. Magri '--download-umpire', 562e294c49SRichard Tran Mills '--download-hypre', 572e294c49SRichard Tran Mills '--download-metis', 582e294c49SRichard Tran Mills '--download-parmetis', 592e294c49SRichard Tran Mills '--download-hdf5', # Note that NERSC does provide an HDF5 module, but using our own is generally reliable. 602e294c49SRichard Tran Mills '--download-hdf5-fortran-bindings', 612e294c49SRichard Tran Mills ] 622e294c49SRichard Tran Mills configure.petsc_configure(configure_options) 63