1#!/bin/bash 2 3# Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 4# Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 5# All Rights reserved. See files LICENSE and NOTICE for details. 6# 7# This file is part of CEED, a collection of benchmarks, miniapps, software 8# libraries and APIs for efficient high-order finite element and spectral 9# element discretizations for exascale applications. For more information and 10# source code availability see http://github.com/ceed. 11# 12# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 13# a collaborative effort of two U.S. Department of Energy organizations (Office 14# of Science and the National Nuclear Security Administration) responsible for 15# the planning and preparation of a capable exascale ecosystem, including 16# software, applications, hardware, advanced system engineering and early 17# testbed platforms, in support of the nation's exascale computing imperative. 18 19this_file="${BASH_SOURCE[0]}" 20if [[ "${#BASH_ARGV[@]}" -ne "$#" ]]; then 21 script_is_sourced="yes" 22 exit_cmd=return 23else 24 script_is_sourced="" 25 exit_cmd=exit 26fi 27test_file="" 28backend_list="/cpu/self" 29run="" 30num_proc_run=${num_proc_run:-""} 31num_proc_node=${num_proc_node:-""} 32dry_run="" # empty string = NO 33start_shell="" 34verbose="" 35cur_dir="$PWD" 36 37mpiexec="mpirun" 38mpiexec_np="-np" 39mpiexec_opts="" 40mpiexec_post_opts="" 41profiler="" 42 43function abspath() 44{ 45 local outvar="$1" path="$2" cur_dir="$PWD" 46 cd "$path" && path="$PWD" && cd "$cur_dir" && eval "$outvar=\"$path\"" 47} 48 49abspath root_dir ".." || $exit_cmd 1 50build_root="$root_dir/build" 51 52if [[ -t 1 ]]; then 53 # ANSI color codes 54 none=$'\E[0m' 55 red=$'\E[0;31m' 56 green=$'\E[0;32m' 57 yellow=$'\E[0;33m' 58 blue=$'\E[0;34m' 59 bblue=$'\E[1;34m' 60 magenta=$'\E[0;35m' 61 cyan=$'\E[0;36m' 62 clear="$(tput sgr0)" 63fi 64 65help_msg=" 66$this_file [options] 67 68Options: 69 -h|--help print this usage information and exit 70 -c|--ceed \"list\" choose the libCEED backends to benchmark 71 -r|--run <name> run the tests in the script <name> 72 -n|--num-proc \"list\" total number of MPI tasks to use in the tests 73 -p|--proc-node \"list\" number of MPI tasks per node to use in the tests 74 -d|--dry-run show (but do not run) the commands for the tests 75 -s|--shell execute bash shell commands before running the test 76 -v|--verbose print additional messages 77 -x enable script tracing with 'set -x' 78 var=value define shell variables; evaluated with 'eval' 79 80This script builds and runs a set of benchmarks for a list of specified 81backends. 82 83Example usage: 84 $this_file --run petsc-bp1.sh 85" 86 87 88function build_examples() 89{ 90 for example; do 91 # We require the examples to be already built because we do not know what 92 # options to use when building the library + examples. 93 if [ ! -e $build_root/$example ]; then 94 echo "Error: example is not built: $example" 95 return 1 96 fi 97 done 98} 99 100 101function compose_mpi_run_command() 102{ 103 mpi_run="${mpiexec:-mpirun} ${mpiexec_opts}" 104 mpi_run+=" ${mpiexec_np:--np} ${num_proc_run} ${mpiexec_post_opts}" 105 if [[ -n "$profiler" ]]; then 106 mpi_run+=" $profiler" 107 fi 108} 109 110 111function quoted_echo() 112{ 113 local arg= string= 114 for arg; do 115 if [[ -z "${arg##* *}" ]]; then 116 string+=" \"${arg//\"/\\\"}\"" 117 else 118 string+=" $arg" 119 fi 120 done 121 printf "%s\n" "${string# }" 122} 123 124 125function set_num_nodes() 126{ 127 if [[ -n "$num_proc_node" ]]; then 128 ((num_proc_run % num_proc_node != 0)) && { 129 echo "The total number of tasks ($num_proc_run) must be a multiple of" 130 echo "the number of tasks per node ($num_proc_node). Stop." 131 return 1 132 } 133 ((num_nodes = num_proc_run / num_proc_node)) 134 else 135 num_proc_node="unknown number of" 136 num_nodes="" 137 fi 138 echo "Running the tests using a total of $num_proc_run MPI tasks ..." | tee -a $output_file 139 echo "... with $num_proc_node tasks per node ..." | tee -a $output_file 140 echo | tee -a $output_file 141} 142 143 144### Process command line parameters 145 146while [ $# -gt 0 ]; do 147 148case "$1" in 149 -h|--help) 150 # Echo usage information 151 echo "$help_msg" 152 $exit_cmd 153 ;; 154 -c|--ceed) 155 shift 156 [ $# -gt 0 ] || { 157 echo "Missing \"list\" in --ceed \"list\""; $exit_cmd 1; } 158 backend_list="$1" 159 ;; 160 -r|--run) 161 run=on 162 shift 163 [ $# -gt 0 ] || { echo "Missing <name> in --run <name>"; $exit_cmd 1; } 164 test_file="$1" 165 [[ -r "$test_file" ]] || { 166 echo "Test script not found: '$1'"; $exit_cmd 1 167 } 168 ;; 169 -n|--num-proc) 170 shift 171 [ $# -gt 0 ] || { 172 echo "Missing \"list\" in --num-proc \"list\""; $exit_cmd 1; } 173 num_proc_run="$1" 174 ;; 175 -p|--proc-node) 176 shift 177 [ $# -gt 0 ] || { 178 echo "Missing \"list\" in --proc-node \"list\""; $exit_cmd 1; } 179 num_proc_node="$1" 180 ;; 181 -d|--dry-run) 182 dry_run="quoted_echo" 183 ;; 184 -s|--shell) 185 start_shell="yes" 186 ;; 187 -v|--verbose) 188 verbose="yes" 189 ;; 190 -x) 191 set -x 192 ;; 193 *=*) 194 eval "$1" || { echo "Error evaluating argument: $1"; $exit_cmd 1; } 195 ;; 196 *) 197 echo "Unknown option: '$1'" 198 $exit_cmd 1 199 ;; 200esac 201 202shift 203done # while ... 204# Done processing command line parameters 205 206 207num_proc_list=(${num_proc_run:-4}) 208num_proc_list_size=${#num_proc_list[@]} 209num_proc_node_list=(${num_proc_node:-4}) 210num_proc_node_list_size=${#num_proc_node_list[@]} 211(( num_proc_list_size != num_proc_node_list_size )) && { 212 echo " 213The size of the number-of-processors list (option --num-proc) must be the same 214as the size of the number-of-processors-per-node list (option --proc-node)." 215 echo 216 $exit_cmd 1 217} 218 219 220### Loop over backends 221 222for backend in $backend_list; do 223( ## Run each backend in its own environment 224 225### Setup output 226### Test name 227cd "$cur_dir" 228abspath test_dir "$(dirname "$test_file")" || $exit_cmd 1 229test_basename="$(basename "$test_file")" 230test_file="${test_dir}/${test_basename}" 231### Backend name 232short_backend=${backend//[\/]} 233### Output file 234output_file="${test_file%%.*}-$short_backend-output.txt" 235rm -rf output_file 236 237### Setup the environment based on $backend 238 239echo 240echo "Using backend $backend ..." | tee $output_file 241 242### Run the tests (building and running $test_file) 243 244[ -n "$run" ] && { 245 246[[ "$verbose" = "yes" ]] && { 247 echo "Test problem file, $test_basename:" | tee -a $output_file 248 echo "------------------------------------------------" | tee -a $output_file 249 cat $test_file | tee -a $output_file 250 echo "------------------------------------------------" | tee -a $output_file 251 echo | tee -a $output_file 252} 253 254test_exe_dir="$build_root" 255 256trap 'printf "\nScript interrupted.\n"; '$exit_cmd' 33' INT 257 258## Source the test script file. 259echo "Reading test file: $test_file" | tee -a $output_file 260echo | tee -a $output_file 261test_required_examples="" 262. "$test_file" || $exit_cmd 1 263 264## Build files required by the test 265echo "Example(s) required by the test: $test_required_examples" | tee -a $output_file 266build_examples $test_required_examples || $exit_cmd 1 267echo | tee -a $output_file 268 269## Loop over the number-of-processors list. 270for (( num_proc_idx = 0; num_proc_idx < num_proc_list_size; num_proc_idx++ )) 271do 272 273num_proc_run="${num_proc_list[$num_proc_idx]}" 274num_proc_node="${num_proc_node_list[$num_proc_idx]}" 275 276set_num_nodes || $exit_cmd 1 277compose_mpi_run_command 278 279if [[ "$start_shell" = "yes" ]]; then 280 if [[ ! -t 1 ]]; then 281 echo "Standard output is not a terminal. Stop." | tee -a $output_file 282 $exit_cmd 1 283 fi 284 echo "Reading shell commands, type 'c' to continue, 'exit' to stop ..." | tee -a $output_file 285 echo | tee -a $output_file 286 cd "$cur_dir" 287 set -o emacs 288 PS1='$ ' 289 [[ -r $HOME/.bashrc ]] && source $HOME/.bashrc 290 HISTFILE="$root_dir/.bash_history" 291 history -c 292 history -r 293 # bind '"\\C-i": menu-complete' 294 alias c='break' 295 while cwd="$PWD/" cwd="${cwd#${root_dir}/}" cwd="${cwd%/}" \ 296 prompt="[${cyan}benchmarks$none:$blue$cwd$clear]\$ " && \ 297 read -p "$prompt" -e line; do 298 history -s "$line" 299 history -w 300 shopt -q -s expand_aliases 301 eval "$line" 302 shopt -q -u expand_aliases 303 done 304 [[ "${#line}" -eq 0 ]] && { echo; $exit_cmd 0; } 305 shopt -q -u expand_aliases 306 echo "Continuing ..." | tee -a $output_file 307fi 308 309# Call the function run_tests defined inside the $test_file 310ceed=$backend 311if [ -z "$dry_run" ]; then 312 run_tests >> $output_file 313else 314 run_tests 315fi 316echo 317 318done ## End of loop over processor numbers 319 320trap - INT 321 322} ## run is on 323 324$exit_cmd 0 325 326) || { 327 echo "Sub-shell for backend '$backend' returned error code $?. Stop." 328 $exit_cmd 1 329} 330done ## Loop over $backend_list 331 332 333$exit_cmd 0 334 335