1#!/bin/bash 2 3# Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at 4# the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights 5# reserved. See files LICENSE and NOTICE for details. 6# 7# This file is part of CEED, a collection of benchmarks, miniapps, software 8# libraries and APIs for efficient high-order finite element and spectral 9# element discretizations for exascale applications. For more information and 10# source code availability see http://github.com/ceed. 11# 12# The CEED research is supported by the Exascale Computing Project (17-SC-20-SC) 13# a collaborative effort of two U.S. Department of Energy organizations (Office 14# of Science and the National Nuclear Security Administration) responsible for 15# the planning and preparation of a capable exascale ecosystem, including 16# software, applications, hardware, advanced system engineering and early 17# testbed platforms, in support of the nation's exascale computing imperative. 18############################################################################### 19# Script for Building and Running Nek5000 examples 20############################################################################### 21## Nek5000 path 22#NEK5K_DIR= 23 24## NEKTOOLS path 25#NEK5K_TOOLS_DIR= 26 27## CEED path 28#CEED_DIR= 29 30## Fortran compiler 31#FC= 32 33## C compiler 34#CC= 35 36############################################################################### 37# DONT'T TOUCH WHAT FOLLOWS !!! 38############################################################################### 39# Set defaults for the parameters 40: ${NEK5K_DIR:=`cd "../../../Nek5000"; pwd`} 41: ${NEK5K_TOOLS_DIR:=`cd "${NEK5K_DIR}/bin"; pwd`} 42: ${CEED_DIR:=`cd "../../"; pwd`} 43: ${FC:="mpif77"} 44: ${CC:="mpicc"} 45: ${MPI:=1} 46 47# Exit if being sourced 48if [[ "${#BASH_ARGV[@]}" -ne "$#" ]]; then 49 nek_exit_cmd=return 50else 51 nek_exit_cmd=exit 52fi 53 54# Read in parameter values 55nek_examples=("bp1") 56nek_spec=/cpu/self 57nek_np=1 58nek_box= 59nek_clean="false" 60nek_make="false" 61nek_run="true" 62nek_test="notest" 63# Won't work if there is a symlink. 64nek_box_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/boxes" 65 66# Set constants 67nek_this_file="${BASH_SOURCE[0]}" 68nek_help_msg=" 69$nek_this_file [options] 70 71options: 72 -h|-help Print this usage information and exit 73 -c|-ceed Ceed backend to be used for the run (optional, default: /cpu/self) 74 -e|-example Example name (optional, default: bp1) 75 -n|-np Specify number of MPI ranks for the run (optional, default: 1) 76 -t|-test Run in test mode (not on by default) 77 -b|-box Box case in boxes sub-directory found along with this script (default: 2x2x2) 78 -clean clean the examples directory 79 -m|-make Make the examples 80 81Example: 82 Build examples with: 83 ./nek-examples.sh -m -e \"bp1 bp3\" 84 Run them with: 85 ./nek-examples.sh -c /cpu/self -e \"bp1 bp3\" -n 4 -b 3 86 Clean the examples directory with: 87 ./nek-examples.sh -clean 88" 89 90nek_verbose="true" 91nek_mpi="true" 92 93while [ $# -gt 0 ]; do 94 case "$1" in 95 -h|-help) 96 echo "$nek_help_msg" 97 $nek_exit_cmd 98 ;; 99 -e|-example) 100 shift 101 nek_examples=($1) 102 ;; 103 -c|-ceed) 104 shift 105 nek_spec="$1" 106 ;; 107 -n|-np) 108 shift 109 nek_np="$1" 110 ;; 111 -b|-box) 112 shift 113 nek_box="$1" 114 ;; 115 -t|-test) 116 nek_verbose="false" 117 nek_mpi="false" 118 nek_test="test" 119 ;; 120 -clean) 121 nek_clean="true" 122 nek_run="false" 123 ;; 124 -m|-make) 125 nek_make="true" 126 nek_run="false" 127 ;; 128 esac 129 shift 130done 131 132function make() { 133 # Set flags 134 FFLAGS="-g -std=legacy -I${CEED_DIR}/include -DEXAMPLE_DIR='\"${PWD}/\"'" 135 USR_LFLAGS="-g -L${CEED_DIR}/lib -Wl,-rpath,${CEED_DIR}/lib -lceed" 136 137 # Build examples 138 echo "Building examples:" 139 140 # BP dir 141 if [[ ! -d build ]]; then 142 mkdir build 143 fi 144 145 # Copy makenek from NEK5K_DIR/bin/ 146 if [[ ! -f build/makenek ]]; then 147 cp $NEK5K_DIR/bin/makenek build/ 148 fi 149 150 # Copy BP files 151 cp bps/bps.* build/ 152 153 # Copy SIZE file 154 if [[ ! -f build/SIZE ]]; then 155 cp SIZE.in build/SIZE 156 fi 157 158 # Change to build directory 159 cd build 160 161 # Attempt build 162 CC=$CC FC=$FC MPI=$MPI NEK_SOURCE_ROOT="${NEK5K_DIR}" FFLAGS="$FFLAGS" \ 163 USR_LFLAGS="$USR_LFLAGS" ./makenek bps >> bps.build.log 2>&1 164 165 # Check and report 166 if [ ! -f ./nek5000 ]; then 167 echo " Building examples failed. See build/bps.build.log for details." 168 cd ../.. 169 ${nek_exit_cmd} 1 170 elif [ ${nek_verbose} = "true" ]; then 171 mv nek5000 bps 172 cd ../ 173 echo " Built examples successfully. See build/bps.build.log for details." 174 fi 175} 176 177# Function to clean 178function clean() { 179 if [ ${nek_verbose} = "true" ]; then 180 echo "Cleaning ..." 181 fi 182 183 # Run clean 184 if [[ -f ./build/makenek ]]; then 185 cd build 186 printf "y\n" | NEK_SOURCE_ROOT=${NEK5K_DIR} ./makenek clean 2>&1 >> /dev/null 187 cd .. 188 fi 189 190 # Remove build dir 191 rm -rf build 192 193 # Remove BPs 194 rm -f bps *log* SESSION.NAME 2> /dev/null 195 196 # Clean box dir 197 find ${nek_box_dir} -type d -regex ".*/b[0-9]+" -exec rm -rf "{}" \; 2>/dev/null 198} 199 200# Functions needed for creating box meshes 201function xyz() 202{ 203 prod=$1 204 split=$((prod/3)) 205 206 nex=$split 207 nez=$split 208 ney=$split 209 210 if [ $((prod%3)) -ne 0 ]; then 211 nex=$((split + 1)) 212 fi 213 if [ $((prod%3)) -eq 2 ]; then 214 ney=$((split + 1)) 215 fi 216 217 nex=$((2**nex)) 218 ney=$((2**ney)) 219 nez=$((2**nez)) 220 221 echo "$nex $ney $nez" 222} 223 224function genbb() 225{ 226 cp $1.box ttt.box 227 if [ ${nek_verbose} = "true" ]; then 228 echo "Running genbox ..." 229 fi 230 231 if [ -z ${NEK5K_TOOLS_DIR} ]; then 232 echo "Required variable NEKTOOLS_DIR not found." 233 ${nek_exit_cmd} 1 234 fi 235 236 printf "ttt.box\n" | $NEK5K_TOOLS_DIR/genbox 2>&1 1>>box.log || return 1 237 238 if [ ${nek_verbose} = "true" ]; then 239 echo "Running genmap ..." 240 fi 241 printf "box\n.1\n" | $NEK5K_TOOLS_DIR/genmap 2>&1 1>>box.log || return 1 242 243 if [ ${nek_verbose} = "true" ]; then 244 echo "Running reatore2 ..." 245 fi 246 printf "box\n$1\n" | $NEK5K_TOOLS_DIR/reatore2 2>&1 1>>box.log || return 1 247 248 rm ttt.box 2>/dev/null 249 rm box.rea 2>/dev/null 250 rm box.tmp 2>/dev/null 251 mv box.map $1.map 2>/dev/null 252} 253 254function generate_boxes() 255{ 256 if [ $# -ne 2 ]; then 257 echo "Error: should be called with two parameters. See syntax below." 258 echo "Syntax: generate_boxes log_2(<min_elem>) log_2(<max_elem>)." 259 echo "Example: generate-boxes 2 4" 260 ${nek_exit_cmd} 1 261 fi 262 local min_elem=$1 263 local max_elem=$2 264 local pwd_=`pwd` 265 266 mkdir -p ${nek_box_dir} && cd ${nek_box_dir} 267 # Run thorugh the box sizes 268 for i in `seq $min_elem 1 $max_elem`; do 269 # Generate the boxes only if they have not 270 # been generated before. 271 if [ ! -f b$i/b$i.map ]; then 272 # Set the number of elements in box file. 273 xyz=$(xyz $i) 274 nex=$( echo $xyz | cut -f 1 -d ' ' ) 275 ney=$( echo $xyz | cut -f 2 -d ' ' ) 276 nez=$( echo $xyz | cut -f 3 -d ' ' ) 277 278 mkdir -p b$i 279 sed "5s/.*/-$nex -$ney -$nez/" ${CEED_DIR}/examples/nek/boxes/b.box > b$i/b$i.box 280 cp ${CEED_DIR}/examples/nek/boxes/b1e.rea b$i/ 281 282 cd b$i 283 genbb b$i 284 genbb b$i &> log || { 285 echo "Error generating box. See $PWD/log for details." 286 return 1 287 } 288 cd .. 289 fi 290 done 291 cd $pwd_ 292} 293 294function run() { 295 for nek_ex in "${nek_examples[@]}"; do 296 if [ ${nek_verbose} = "true" ]; then 297 echo "Running Nek5000 Example: $nek_ex" 298 fi 299 300 # Check for build 301 if [ ! -f $bps ]; then 302 echo " Examples file does not exist. Build it with nek-examples.sh -m" 303 ${nek_exit_cmd} 1 304 fi 305 306 # Generate boxes, if needed 307 if [ ! -f ${nek_box_dir}/b${nek_box}/b${nek_box}.rea ] || \ 308 [ ! -f ${nek_box_dir}/b${nek_box}/b${nek_box}.map ]; then 309 if [ -z ${nek_box} ]; then 310 nek_box=3 311 fi 312 generate_boxes ${nek_box} ${nek_box} 313 fi 314 315 # Get CEED spec 316 nek_spec_short=${nek_spec//[\/]} 317 318 # Logging 319 echo b${nek_box} > SESSION.NAME 320 echo `cd ${nek_box_dir}/b${nek_box}; pwd`'/' >> SESSION.NAME 321 rm -f logfile 322 rm -f ioinfo 323 mv ${nek_ex}.${nek_spec_short}.log.${nek_np}.b${nek_box} ${nek_ex}.${nek_spec_short}.log1.${nek_np}.b${nek_box} 2>/dev/null 324 325 # Run example 326 if [ ${nek_mpi} = "false" ]; then 327 ./build/bps ${nek_ex} ${nek_spec} ${nek_test} > ${nek_ex}.${nek_spec_short}.log.${nek_np}.b${nek_box} 328 wait $! 329 else 330 ${MPIEXEC:-mpiexec} -np ${nek_np} ./build/bps ${nek_ex} ${nek_spec} ${nek_test} > \ 331 ${nek_ex}.${nek_spec_short}.log.${nek_np}.b${nek_box} 332 wait $! 333 fi 334 335 # Log output location 336 if [ ${nek_verbose} = "true" ]; then 337 echo " Run finished. Output was written to ${nek_ex}.${nek_spec_short}.log.${nek_np}.b${nek_box}" 338 fi 339 340 # Check error 341 if [[ $(grep 'ERROR IS TOO LARGE' ${nek_ex}.${nek_spec_short}.log*) ]]; then 342 echo "ERROR IS TOO LARGE" 343 ${nek_exit_cmd} 1 344 elif [ ${nek_verbose} != "true" ]; then # Cleanup if test mode 345 rm -f ${nek_ex}.${nek_spec_short}.log* 346 fi 347 done 348 349 ${nek_exit_cmd} 0 350} 351 352if [ "${nek_clean}" = "true" ]; then 353 clean 354fi 355if [ "${nek_make}" = "true" ]; then 356 make 357fi 358if [ "${nek_run}" = "true" ]; then 359 run 360fi 361${nek_exit_cmd} 0 362