1#!/bin/bash 2 3# Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at 4# the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights 5# reserved. See files LICENSE and NOTICE for details. 6# 7# This file is part of CEED, a collection of benchmarks, miniapps, software 8# libraries and APIs for efficient high-order finite element and spectral 9# element discretizations for exascale applications. For more information and 10# source code availability see http://github.com/ceed. 11# 12# The CEED research is supported by the Exascale Computing Project (17-SC-20-SC) 13# a collaborative effort of two U.S. Department of Energy organizations (Office 14# of Science and the National Nuclear Security Administration) responsible for 15# the planning and preparation of a capable exascale ecosystem, including 16# software, applications, hardware, advanced system engineering and early 17# testbed platforms, in support of the nation's exascale computing imperative. 18############################################################################### 19# Script for Building and Running Nek5000 examples 20############################################################################### 21## Nek5000 path 22#NEK5K_DIR= 23 24## NEKTOOLS path 25#NEK5K_TOOLS_DIR= 26 27## CEED path 28#CEED_DIR= 29 30## Fortran compiler 31#FC= 32 33## C compiler 34#CC= 35 36############################################################################### 37# DONT'T TOUCH WHAT FOLLOWS !!! 38############################################################################### 39# Set defaults for the parameters 40: ${NEK5K_DIR:=`cd "../../../Nek5000"; pwd`} 41: ${NEK5K_TOOLS_DIR:=`cd "${NEK5K_DIR}/bin"; pwd`} 42: ${CEED_DIR:=`cd "../../"; pwd`} 43: ${FC:="mpif77"} 44: ${CC:="mpicc"} 45: ${MPI:=1} 46 47# Exit if being sourced 48if [[ "${#BASH_ARGV[@]}" -ne "$#" ]]; then 49 nek_exit_cmd=return 50else 51 nek_exit_cmd=exit 52fi 53 54# Read in parameter values 55nek_examples=("bp1") 56nek_spec=/cpu/self 57nek_np=1 58nek_box= 59nek_clean="false" 60nek_make="false" 61nek_run="true" 62nek_test="notest" 63# Won't work if there is a symlink. 64nek_box_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/boxes" 65 66# Set constants 67nek_this_file="${BASH_SOURCE[0]}" 68nek_help_msg=" 69$nek_this_file [options] 70 71options: 72 -h|-help Print this usage information and exit 73 -c|-ceed Ceed backend to be used for the run (optional, default: /cpu/self) 74 -e|-example Example name (optional, default: bp1) 75 -n|-np Specify number of MPI ranks for the run (optional, default: 1) 76 -t|-test Run in test mode (not on by default) 77 -b|-box Box case in boxes sub-directory found along with this script (default: 2x2x2) 78 -clean clean the examples directory 79 -m|-make Make the examples 80 81Example: 82 Build examples with: 83 ./nek-examples.sh -m -e \"bp1 bp3\" 84 Run them with: 85 ./nek-examples.sh -c /cpu/self -e \"bp1 bp3\" -n 4 -b 3 86 Clean the examples directory with: 87 ./nek-examples.sh -clean 88" 89 90nek_verbose="true" 91nek_mpi="true" 92 93while [ $# -gt 0 ]; do 94 case "$1" in 95 -h|-help) 96 echo "$nek_help_msg" 97 $nek_exit_cmd 98 ;; 99 -e|-example) 100 shift 101 nek_examples=($1) 102 ;; 103 -c|-ceed) 104 shift 105 nek_spec="$1" 106 ;; 107 -n|-np) 108 shift 109 nek_np="$1" 110 ;; 111 -b|-box) 112 shift 113 nek_box="$1" 114 ;; 115 -t|-test) 116 nek_verbose="false" 117 nek_mpi="false" 118 nek_test="test" 119 ;; 120 -clean) 121 nek_clean="true" 122 nek_run="false" 123 ;; 124 -m|-make) 125 nek_make="true" 126 nek_run="false" 127 ;; 128 esac 129 shift 130done 131 132function make() { 133 # Set flags 134 CFLAGS="-fPIC" 135 FFLAGS="-g -std=legacy -I${CEED_DIR}/include -ffixed-line-length-132 -DEXAMPLE_DIR='\"${PWD}/\"' -fPIC" 136 USR_LFLAGS="-g -L${CEED_DIR}/lib -Wl,-rpath,${CEED_DIR}/lib -lceed -fPIC" 137 138 # Build examples 139 echo "Building examples:" 140 141 # BP dir 142 if [[ ! -d build ]]; then 143 mkdir build 144 fi 145 146 # Copy makenek from NEK5K_DIR/bin/ 147 if [[ ! -f build/makenek ]]; then 148 cp $NEK5K_DIR/bin/makenek build/ 149 fi 150 151 # Copy BP files 152 cp bps/bps.* build/ 153 154 # Copy SIZE file 155 if [[ ! -f build/SIZE ]]; then 156 cp SIZE.in build/SIZE 157 fi 158 159 # Change to build directory 160 cd build 161 162 # Attempt build 163 CC=$CC FC=$FC MPI=$MPI NEK_SOURCE_ROOT="${NEK5K_DIR}" CFLAGS="$CFLAGS" \ 164 FFLAGS="$FFLAGS" USR_LFLAGS="$USR_LFLAGS" ./makenek bps >> bps.build.log 2>&1 165 166 # Check and report 167 if [ ! -f ./nek5000 ]; then 168 echo " Building examples failed. See build/bps.build.log for details." 169 cd ../.. 170 ${nek_exit_cmd} 1 171 elif [ ${nek_verbose} = "true" ]; then 172 mv nek5000 bps 173 cd ../ 174 echo " Built examples successfully. See build/bps.build.log for details." 175 fi 176} 177 178# Function to clean 179function clean() { 180 if [ ${nek_verbose} = "true" ]; then 181 echo "Cleaning ..." 182 fi 183 184 # Run clean 185 if [[ -f ./build/makenek ]]; then 186 cd build 187 printf "y\n" | NEK_SOURCE_ROOT=${NEK5K_DIR} ./makenek clean 2>&1 >> /dev/null 188 cd .. 189 fi 190 191 # Remove build dir 192 rm -rf build 193 194 # Remove BPs 195 rm -f bps *log* SESSION.NAME 2> /dev/null 196 197 # Clean box dir 198 find ${nek_box_dir} -type d -regex ".*/b[0-9]+" -exec rm -rf "{}" \; 2>/dev/null 199} 200 201# Functions needed for creating box meshes 202function xyz() 203{ 204 prod=$1 205 split=$((prod/3)) 206 207 nex=$split 208 nez=$split 209 ney=$split 210 211 if [ $((prod%3)) -ne 0 ]; then 212 nex=$((split + 1)) 213 fi 214 if [ $((prod%3)) -eq 2 ]; then 215 ney=$((split + 1)) 216 fi 217 218 nex=$((2**nex)) 219 ney=$((2**ney)) 220 nez=$((2**nez)) 221 222 echo "$nex $ney $nez" 223} 224 225function genbb() 226{ 227 cp $1.box ttt.box 228 if [ ${nek_verbose} = "true" ]; then 229 echo "Running genbox ..." 230 fi 231 232 if [ -z ${NEK5K_TOOLS_DIR} ]; then 233 echo "Required variable NEKTOOLS_DIR not found." 234 ${nek_exit_cmd} 1 235 fi 236 237 printf "ttt.box\n" | $NEK5K_TOOLS_DIR/genbox 2>&1 1>>box.log || return 1 238 239 if [ ${nek_verbose} = "true" ]; then 240 echo "Running genmap ..." 241 fi 242 printf "box\n.1\n" | $NEK5K_TOOLS_DIR/genmap 2>&1 1>>box.log || return 1 243 244 if [ ${nek_verbose} = "true" ]; then 245 echo "Running reatore2 ..." 246 fi 247 printf "box\n$1\n" | $NEK5K_TOOLS_DIR/reatore2 2>&1 1>>box.log || return 1 248 249 rm ttt.box 2>/dev/null 250 rm box.rea 2>/dev/null 251 rm box.tmp 2>/dev/null 252 mv box.map $1.map 2>/dev/null 253} 254 255function generate_boxes() 256{ 257 if [ $# -ne 2 ]; then 258 echo "Error: should be called with two parameters. See syntax below." 259 echo "Syntax: generate_boxes log_2(<min_elem>) log_2(<max_elem>)." 260 echo "Example: generate-boxes 2 4" 261 ${nek_exit_cmd} 1 262 fi 263 local min_elem=$1 264 local max_elem=$2 265 local pwd_=`pwd` 266 267 mkdir -p ${nek_box_dir} && cd ${nek_box_dir} 268 # Run thorugh the box sizes 269 for i in `seq $min_elem 1 $max_elem`; do 270 # Generate the boxes only if they have not 271 # been generated before. 272 if [ ! -f b$i/b$i.map ]; then 273 # Set the number of elements in box file. 274 xyz=$(xyz $i) 275 nex=$( echo $xyz | cut -f 1 -d ' ' ) 276 ney=$( echo $xyz | cut -f 2 -d ' ' ) 277 nez=$( echo $xyz | cut -f 3 -d ' ' ) 278 279 mkdir -p b$i 280 sed "5s/.*/-$nex -$ney -$nez/" ${CEED_DIR}/examples/nek/boxes/b.box > b$i/b$i.box 281 cp ${CEED_DIR}/examples/nek/boxes/b1e.rea b$i/ 282 283 cd b$i 284 genbb b$i 285 genbb b$i &> log || { 286 echo "Error generating box. See $PWD/log for details." 287 return 1 288 } 289 cd .. 290 fi 291 done 292 cd $pwd_ 293} 294 295function run() { 296 for nek_ex in "${nek_examples[@]}"; do 297 if [ ${nek_verbose} = "true" ]; then 298 echo "Running Nek5000 Example: $nek_ex" 299 fi 300 301 # Check for build 302 if [ ! -f $bps ]; then 303 echo " Examples file does not exist. Build it with nek-examples.sh -m" 304 ${nek_exit_cmd} 1 305 fi 306 307 # Generate boxes, if needed 308 if [ ! -f ${nek_box_dir}/b${nek_box}/b${nek_box}.rea ] || \ 309 [ ! -f ${nek_box_dir}/b${nek_box}/b${nek_box}.map ]; then 310 if [ -z ${nek_box} ]; then 311 nek_box=3 312 fi 313 generate_boxes ${nek_box} ${nek_box} 314 fi 315 316 # Get CEED spec 317 nek_spec_short=${nek_spec//[\/]} 318 319 # Logging 320 echo b${nek_box} > SESSION.NAME 321 echo `cd ${nek_box_dir}/b${nek_box}; pwd`'/' >> SESSION.NAME 322 rm -f logfile 323 rm -f ioinfo 324 mv ${nek_ex}.${nek_spec_short}.log.${nek_np}.b${nek_box} ${nek_ex}.${nek_spec_short}.log1.${nek_np}.b${nek_box} 2>/dev/null 325 326 # Run example 327 if [ ${nek_mpi} = "false" ]; then 328 ./build/bps ${nek_ex} ${nek_spec} ${nek_test} > ${nek_ex}.${nek_spec_short}.log.${nek_np}.b${nek_box} 329 wait $! 330 else 331 ${MPIEXEC:-mpiexec} -np ${nek_np} ./build/bps ${nek_ex} ${nek_spec} ${nek_test} > \ 332 ${nek_ex}.${nek_spec_short}.log.${nek_np}.b${nek_box} 333 wait $! 334 fi 335 336 # Log output location 337 if [ ${nek_verbose} = "true" ]; then 338 echo " Run finished. Output was written to ${nek_ex}.${nek_spec_short}.log.${nek_np}.b${nek_box}" 339 fi 340 341 # Check error 342 if [[ $(grep 'ERROR IS TOO LARGE' ${nek_ex}.${nek_spec_short}.log*) ]]; then 343 echo "ERROR IS TOO LARGE" 344 ${nek_exit_cmd} 1 345 elif [ ${nek_verbose} != "true" ]; then # Cleanup if test mode 346 rm -f ${nek_ex}.${nek_spec_short}.log* 347 fi 348 done 349 350 ${nek_exit_cmd} 0 351} 352 353if [ "${nek_clean}" = "true" ]; then 354 clean 355fi 356if [ "${nek_make}" = "true" ]; then 357 make 358fi 359if [ "${nek_run}" = "true" ]; then 360 run 361fi 362${nek_exit_cmd} 0 363