1#!/bin/bash 2 3# Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 4# Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 5# All Rights reserved. See files LICENSE and NOTICE for details. 6# 7# This file is part of CEED, a collection of benchmarks, miniapps, software 8# libraries and APIs for efficient high-order finite element and spectral 9# element discretizations for exascale applications. For more information and 10# source code availability see http://github.com/ceed. 11# 12# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 13# a collaborative effort of two U.S. Department of Energy organizations (Office 14# of Science and the National Nuclear Security Administration) responsible for 15# the planning and preparation of a capable exascale ecosystem, including 16# software, applications, hardware, advanced system engineering and early 17# testbed platforms, in support of the nation's exascale computing imperative. 18 19this_file="${BASH_SOURCE[0]}" 20if [[ "${#BASH_ARGV[@]}" -ne "$#" ]]; then 21 script_is_sourced="yes" 22 exit_cmd=return 23else 24 script_is_sourced="" 25 exit_cmd=exit 26fi 27test_file="" 28backend_list="/cpu/self" 29bp_list="bp1 bp3" 30run="" 31num_proc_run=${num_proc_run:-""} 32num_proc_node=${num_proc_node:-""} 33dry_run="" # empty string = NO 34start_shell="" 35verbose="" 36cur_dir="$PWD" 37 38mpiexec="mpirun" 39mpiexec_np="-np" 40mpiexec_opts="" 41mpiexec_post_opts="" 42profiler="" 43 44function abspath() 45{ 46 local outvar="$1" path="$2" cur_dir="$PWD" 47 cd "$path" && path="$PWD" && cd "$cur_dir" && eval "$outvar=\"$path\"" 48} 49 50abspath root_dir ".." || $exit_cmd 1 51build_root="$root_dir/build" 52 53if [[ -t 1 ]]; then 54 # ANSI color codes 55 none=$'\E[0m' 56 red=$'\E[0;31m' 57 green=$'\E[0;32m' 58 yellow=$'\E[0;33m' 59 blue=$'\E[0;34m' 60 bblue=$'\E[1;34m' 61 magenta=$'\E[0;35m' 62 cyan=$'\E[0;36m' 63 clear="$(tput sgr0)" 64fi 65 66help_msg=" 67$this_file [options] 68 69Options: 70 -h|--help print this usage information and exit 71 -b|--bp \"list\" choose the benchmark problems to run 72 -c|--ceed \"list\" choose the libCEED backends to benchmark 73 -r|--run <name> run the tests in the script <name> 74 -n|--num-proc \"list\" total number of MPI tasks to use in the tests 75 -p|--proc-node \"list\" number of MPI tasks per node to use in the tests 76 -d|--dry-run show (but do not run) the commands for the tests 77 -s|--shell execute bash shell commands before running the test 78 -v|--verbose print additional messages 79 -x enable script tracing with 'set -x' 80 var=value define shell variables; evaluated with 'eval' 81 82This script builds and runs a set of benchmarks for a list of specified 83backends. 84 85Example usage: 86 $this_file --run petsc-bpsraw.sh 87 $this_file --run petsc-bps.sh 88" 89 90function build_examples() 91{ 92 for example; do 93 # We require the examples to be already built because we do not know what 94 # options to use when building the library + examples. 95 if [ ! -e $build_root/$example ]; then 96 echo "Error: example is not built: $example" 97 return 1 98 fi 99 done 100} 101 102function compose_mpi_run_command() 103{ 104 mpi_run="${mpiexec:-mpirun} ${mpiexec_opts}" 105 mpi_run+=" ${mpiexec_np:--np} ${num_proc_run} ${mpiexec_post_opts}" 106 if [[ -n "$profiler" ]]; then 107 mpi_run+=" $profiler" 108 fi 109} 110 111function quoted_echo() 112{ 113 local arg= string= 114 for arg; do 115 if [[ -z "${arg##* *}" ]]; then 116 string+=" \"${arg//\"/\\\"}\"" 117 else 118 string+=" $arg" 119 fi 120 done 121 printf "%s\n" "${string# }" 122} 123 124function set_num_nodes() 125{ 126 if [[ -n "$num_proc_node" ]]; then 127 ((num_proc_run % num_proc_node != 0)) && { 128 echo "The total number of tasks ($num_proc_run) must be a multiple of" 129 echo "the number of tasks per node ($num_proc_node). Stop." 130 return 1 131 } 132 ((num_nodes = num_proc_run / num_proc_node)) 133 else 134 num_proc_node="unknown number of" 135 num_nodes="" 136 fi 137 echo "Running the tests using a total of $num_proc_run MPI tasks ..." | tee -a $output_file 138 echo "... with $num_proc_node tasks per node ..." | tee -a $output_file 139 echo | tee -a $output_file 140} 141 142### Process command line parameters 143 144while [ $# -gt 0 ]; do 145 146case "$1" in 147 -h|--help) 148 # Echo usage information 149 echo "$help_msg" 150 $exit_cmd 151 ;; 152 -b|--bp) 153 shift 154 [ $# -gt 0 ] || { 155 echo "Missing \"list\" in --bp \"list\""; $exit_cmd 1; } 156 bp_list="$1" 157 ;; 158 -c|--ceed) 159 shift 160 [ $# -gt 0 ] || { 161 echo "Missing \"list\" in --ceed \"list\""; $exit_cmd 1; } 162 backend_list="$1" 163 ;; 164 -r|--run) 165 run=on 166 shift 167 [ $# -gt 0 ] || { echo "Missing <name> in --run <name>"; $exit_cmd 1; } 168 test_file="$1" 169 [[ -r "$test_file" ]] || { 170 echo "Test script not found: '$1'"; $exit_cmd 1 171 } 172 ;; 173 -n|--num-proc) 174 shift 175 [ $# -gt 0 ] || { 176 echo "Missing \"list\" in --num-proc \"list\""; $exit_cmd 1; } 177 num_proc_run="$1" 178 ;; 179 -p|--proc-node) 180 shift 181 [ $# -gt 0 ] || { 182 echo "Missing \"list\" in --proc-node \"list\""; $exit_cmd 1; } 183 num_proc_node="$1" 184 ;; 185 -d|--dry-run) 186 dry_run="quoted_echo" 187 ;; 188 -s|--shell) 189 start_shell="yes" 190 ;; 191 -v|--verbose) 192 verbose="yes" 193 ;; 194 -x) 195 set -x 196 ;; 197 *=*) 198 eval "$1" || { echo "Error evaluating argument: $1"; $exit_cmd 1; } 199 ;; 200 *) 201 echo "Unknown option: '$1'" 202 $exit_cmd 1 203 ;; 204esac 205 206shift 207done # while ... 208# Done processing command line parameters 209 210num_proc_list=(${num_proc_run:-4}) 211num_proc_list_size=${#num_proc_list[@]} 212num_proc_node_list=(${num_proc_node:-4}) 213num_proc_node_list_size=${#num_proc_node_list[@]} 214(( num_proc_list_size != num_proc_node_list_size )) && { 215 echo " 216The size of the number-of-processors list (option --num-proc) must be the same 217as the size of the number-of-processors-per-node list (option --proc-node)." 218 echo 219 $exit_cmd 1 220} 221 222### Loop over BPs 223 224for bp in $bp_list; do 225 226### Loop over backends 227 228for backend in $backend_list; do 229( ## Run each backend in its own environment 230 231### Setup output 232### Test name 233cd "$cur_dir" 234abspath test_dir "$(dirname "$test_file")" || $exit_cmd 1 235test_basename="$(basename "$test_file")" 236test_file="${test_dir}/${test_basename}" 237### Backend name 238short_backend=${backend//[\/]} 239### Output file 240output_file="${test_file%%.*}-$bp-$short_backend-output.txt" 241rm -rf output_file 242 243### Setup the environment based on $backend 244 245echo 246echo "Using backend $backend ..." | tee $output_file 247 248### Run the tests (building and running $test_file) 249 250[ -n "$run" ] && { 251 252[[ "$verbose" = "yes" ]] && { 253 echo "Test problem file, $test_basename:" | tee -a $output_file 254 echo "------------------------------------------------" | tee -a $output_file 255 cat $test_file | tee -a $output_file 256 echo "------------------------------------------------" | tee -a $output_file 257 echo | tee -a $output_file 258} 259 260test_exe_dir="$build_root" 261 262trap 'printf "\nScript interrupted.\n"; '$exit_cmd' 33' INT 263 264## Source the test script file. 265echo "Reading test file: $test_file" | tee -a $output_file 266echo | tee -a $output_file 267test_required_examples="" 268. "$test_file" || $exit_cmd 1 269 270## Build files required by the test 271echo "Example(s) required by the test: $test_required_examples" | tee -a $output_file 272build_examples $test_required_examples || $exit_cmd 1 273echo | tee -a $output_file 274 275## Loop over the number-of-processors list. 276for (( num_proc_idx = 0; num_proc_idx < num_proc_list_size; num_proc_idx++ )) 277do 278 279num_proc_run="${num_proc_list[$num_proc_idx]}" 280num_proc_node="${num_proc_node_list[$num_proc_idx]}" 281 282set_num_nodes || $exit_cmd 1 283compose_mpi_run_command 284 285if [[ "$start_shell" = "yes" ]]; then 286 if [[ ! -t 1 ]]; then 287 echo "Standard output is not a terminal. Stop." | tee -a $output_file 288 $exit_cmd 1 289 fi 290 echo "Reading shell commands, type 'c' to continue, 'exit' to stop ..." | tee -a $output_file 291 echo | tee -a $output_file 292 cd "$cur_dir" 293 set -o emacs 294 PS1='$ ' 295 [[ -r $HOME/.bashrc ]] && source $HOME/.bashrc 296 HISTFILE="$root_dir/.bash_history" 297 history -c 298 history -r 299 # bind '"\\C-i": menu-complete' 300 alias c='break' 301 while cwd="$PWD/" cwd="${cwd#${root_dir}/}" cwd="${cwd%/}" \ 302 prompt="[${cyan}benchmarks$none:$blue$cwd$clear]\$ " && \ 303 read -p "$prompt" -e line; do 304 history -s "$line" 305 history -w 306 shopt -q -s expand_aliases 307 eval "$line" 308 shopt -q -u expand_aliases 309 done 310 [[ "${#line}" -eq 0 ]] && { echo; $exit_cmd 0; } 311 shopt -q -u expand_aliases 312 echo "Continuing ..." | tee -a $output_file 313fi 314 315# Call the function run_tests defined inside the $test_file 316ceed=$backend 317if [ -z "$dry_run" ]; then 318 run_tests >> $output_file 319else 320 run_tests 321fi 322echo 323 324done ## End of loop over processor numbers 325 326trap - INT 327 328} ## run is on 329 330$exit_cmd 0 331 332) || { 333 echo "Sub-shell for backend '$backend' returned error code $?. Stop." 334 $exit_cmd 1 335} 336done ## Loop over $backend_list 337 338done ## Loop over $bp_list 339 340$exit_cmd 0 341 342