1#!/bin/bash 2 3# Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 4# Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 5# All Rights reserved. See files LICENSE and NOTICE for details. 6# 7# This file is part of CEED, a collection of benchmarks, miniapps, software 8# libraries and APIs for efficient high-order finite element and spectral 9# element discretizations for exascale applications. For more information and 10# source code availability see http://github.com/ceed. 11# 12# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 13# a collaborative effort of two U.S. Department of Energy organizations (Office 14# of Science and the National Nuclear Security Administration) responsible for 15# the planning and preparation of a capable exascale ecosystem, including 16# software, applications, hardware, advanced system engineering and early 17# testbed platforms, in support of the nation's exascale computing imperative. 18 19this_file="${BASH_SOURCE[0]}" 20if [[ "${#BASH_ARGV[@]}" -ne "$#" ]]; then 21 script_is_sourced="yes" 22 exit_cmd=return 23else 24 script_is_sourced="" 25 exit_cmd=exit 26fi 27test_file="" 28backend_list="/cpu/self" 29bp_list="bp1 bp3" 30run="" 31num_proc_run=${num_proc_run:-""} 32num_proc_node=${num_proc_node:-""} 33dry_run="" # empty string = NO 34start_shell="" 35verbose="" 36cur_dir="$PWD" 37 38mpiexec="mpirun" 39mpiexec_np="-np" 40mpiexec_opts="" 41mpiexec_post_opts="" 42profiler="" 43 44function abspath() 45{ 46 local outvar="$1" path="$2" cur_dir="$PWD" 47 cd "$path" && path="$PWD" && cd "$cur_dir" && eval "$outvar=\"$path\"" 48} 49 50abspath root_dir ".." || $exit_cmd 1 51build_root="$root_dir/build" 52 53if [[ -t 1 ]]; then 54 # ANSI color codes 55 none=$'\E[0m' 56 red=$'\E[0;31m' 57 green=$'\E[0;32m' 58 yellow=$'\E[0;33m' 59 blue=$'\E[0;34m' 60 bblue=$'\E[1;34m' 61 magenta=$'\E[0;35m' 62 cyan=$'\E[0;36m' 63 clear="$(tput sgr0)" 64fi 65 66help_msg=" 67$this_file [options] 68 69Options: 70 -h|--help print this usage information and exit 71 -b|--bp \"list\" choose the benchmark problems to run 72 -c|--ceed \"list\" choose the libCEED backends to benchmark 73 -r|--run <name> run the tests in the script <name> 74 -n|--num-proc \"list\" total number of MPI tasks to use in the tests 75 -p|--proc-node \"list\" number of MPI tasks per node to use in the tests 76 -d|--dry-run show (but do not run) the commands for the tests 77 -s|--shell execute bash shell commands before running the test 78 -v|--verbose print additional messages 79 -x enable script tracing with 'set -x' 80 var=value define shell variables; evaluated with 'eval' 81 82This script builds and runs a set of benchmarks for a list of specified 83backends. 84 85Example usage: 86 $this_file --run petsc-bps.sh 87" 88 89 90function build_examples() 91{ 92 for example; do 93 # We require the examples to be already built because we do not know what 94 # options to use when building the library + examples. 95 if [ ! -e $build_root/$example ]; then 96 echo "Error: example is not built: $example" 97 return 1 98 fi 99 done 100} 101 102 103function compose_mpi_run_command() 104{ 105 mpi_run="${mpiexec:-mpirun} ${mpiexec_opts}" 106 mpi_run+=" ${mpiexec_np:--np} ${num_proc_run} ${mpiexec_post_opts}" 107 if [[ -n "$profiler" ]]; then 108 mpi_run+=" $profiler" 109 fi 110} 111 112 113function quoted_echo() 114{ 115 local arg= string= 116 for arg; do 117 if [[ -z "${arg##* *}" ]]; then 118 string+=" \"${arg//\"/\\\"}\"" 119 else 120 string+=" $arg" 121 fi 122 done 123 printf "%s\n" "${string# }" 124} 125 126 127function set_num_nodes() 128{ 129 if [[ -n "$num_proc_node" ]]; then 130 ((num_proc_run % num_proc_node != 0)) && { 131 echo "The total number of tasks ($num_proc_run) must be a multiple of" 132 echo "the number of tasks per node ($num_proc_node). Stop." 133 return 1 134 } 135 ((num_nodes = num_proc_run / num_proc_node)) 136 else 137 num_proc_node="unknown number of" 138 num_nodes="" 139 fi 140 echo "Running the tests using a total of $num_proc_run MPI tasks ..." | tee -a $output_file 141 echo "... with $num_proc_node tasks per node ..." | tee -a $output_file 142 echo | tee -a $output_file 143} 144 145 146### Process command line parameters 147 148while [ $# -gt 0 ]; do 149 150case "$1" in 151 -h|--help) 152 # Echo usage information 153 echo "$help_msg" 154 $exit_cmd 155 ;; 156 -b|--bp) 157 shift 158 [ $# -gt 0 ] || { 159 echo "Missing \"list\" in --bp \"list\""; $exit_cmd 1; } 160 bp_list="$1" 161 ;; 162 -c|--ceed) 163 shift 164 [ $# -gt 0 ] || { 165 echo "Missing \"list\" in --ceed \"list\""; $exit_cmd 1; } 166 backend_list="$1" 167 ;; 168 -r|--run) 169 run=on 170 shift 171 [ $# -gt 0 ] || { echo "Missing <name> in --run <name>"; $exit_cmd 1; } 172 test_file="$1" 173 [[ -r "$test_file" ]] || { 174 echo "Test script not found: '$1'"; $exit_cmd 1 175 } 176 ;; 177 -n|--num-proc) 178 shift 179 [ $# -gt 0 ] || { 180 echo "Missing \"list\" in --num-proc \"list\""; $exit_cmd 1; } 181 num_proc_run="$1" 182 ;; 183 -p|--proc-node) 184 shift 185 [ $# -gt 0 ] || { 186 echo "Missing \"list\" in --proc-node \"list\""; $exit_cmd 1; } 187 num_proc_node="$1" 188 ;; 189 -d|--dry-run) 190 dry_run="quoted_echo" 191 ;; 192 -s|--shell) 193 start_shell="yes" 194 ;; 195 -v|--verbose) 196 verbose="yes" 197 ;; 198 -x) 199 set -x 200 ;; 201 *=*) 202 eval "$1" || { echo "Error evaluating argument: $1"; $exit_cmd 1; } 203 ;; 204 *) 205 echo "Unknown option: '$1'" 206 $exit_cmd 1 207 ;; 208esac 209 210shift 211done # while ... 212# Done processing command line parameters 213 214 215num_proc_list=(${num_proc_run:-4}) 216num_proc_list_size=${#num_proc_list[@]} 217num_proc_node_list=(${num_proc_node:-4}) 218num_proc_node_list_size=${#num_proc_node_list[@]} 219(( num_proc_list_size != num_proc_node_list_size )) && { 220 echo " 221The size of the number-of-processors list (option --num-proc) must be the same 222as the size of the number-of-processors-per-node list (option --proc-node)." 223 echo 224 $exit_cmd 1 225} 226 227### Loop over BPs 228 229for bp in $bp_list; do 230 231### Loop over backends 232 233for backend in $backend_list; do 234( ## Run each backend in its own environment 235 236### Setup output 237### Test name 238cd "$cur_dir" 239abspath test_dir "$(dirname "$test_file")" || $exit_cmd 1 240test_basename="$(basename "$test_file")" 241test_file="${test_dir}/${test_basename}" 242### Backend name 243short_backend=${backend//[\/]} 244### Output file 245output_file="${test_file%%.*}-$bp-$short_backend-output.txt" 246rm -rf output_file 247 248### Setup the environment based on $backend 249 250echo 251echo "Using backend $backend ..." | tee $output_file 252 253### Run the tests (building and running $test_file) 254 255[ -n "$run" ] && { 256 257[[ "$verbose" = "yes" ]] && { 258 echo "Test problem file, $test_basename:" | tee -a $output_file 259 echo "------------------------------------------------" | tee -a $output_file 260 cat $test_file | tee -a $output_file 261 echo "------------------------------------------------" | tee -a $output_file 262 echo | tee -a $output_file 263} 264 265test_exe_dir="$build_root" 266 267trap 'printf "\nScript interrupted.\n"; '$exit_cmd' 33' INT 268 269## Source the test script file. 270echo "Reading test file: $test_file" | tee -a $output_file 271echo | tee -a $output_file 272test_required_examples="" 273. "$test_file" || $exit_cmd 1 274 275## Build files required by the test 276echo "Example(s) required by the test: $test_required_examples" | tee -a $output_file 277build_examples $test_required_examples || $exit_cmd 1 278echo | tee -a $output_file 279 280## Loop over the number-of-processors list. 281for (( num_proc_idx = 0; num_proc_idx < num_proc_list_size; num_proc_idx++ )) 282do 283 284num_proc_run="${num_proc_list[$num_proc_idx]}" 285num_proc_node="${num_proc_node_list[$num_proc_idx]}" 286 287set_num_nodes || $exit_cmd 1 288compose_mpi_run_command 289 290if [[ "$start_shell" = "yes" ]]; then 291 if [[ ! -t 1 ]]; then 292 echo "Standard output is not a terminal. Stop." | tee -a $output_file 293 $exit_cmd 1 294 fi 295 echo "Reading shell commands, type 'c' to continue, 'exit' to stop ..." | tee -a $output_file 296 echo | tee -a $output_file 297 cd "$cur_dir" 298 set -o emacs 299 PS1='$ ' 300 [[ -r $HOME/.bashrc ]] && source $HOME/.bashrc 301 HISTFILE="$root_dir/.bash_history" 302 history -c 303 history -r 304 # bind '"\\C-i": menu-complete' 305 alias c='break' 306 while cwd="$PWD/" cwd="${cwd#${root_dir}/}" cwd="${cwd%/}" \ 307 prompt="[${cyan}benchmarks$none:$blue$cwd$clear]\$ " && \ 308 read -p "$prompt" -e line; do 309 history -s "$line" 310 history -w 311 shopt -q -s expand_aliases 312 eval "$line" 313 shopt -q -u expand_aliases 314 done 315 [[ "${#line}" -eq 0 ]] && { echo; $exit_cmd 0; } 316 shopt -q -u expand_aliases 317 echo "Continuing ..." | tee -a $output_file 318fi 319 320# Call the function run_tests defined inside the $test_file 321ceed=$backend 322if [ -z "$dry_run" ]; then 323 run_tests >> $output_file 324else 325 run_tests 326fi 327echo 328 329done ## End of loop over processor numbers 330 331trap - INT 332 333} ## run is on 334 335$exit_cmd 0 336 337) || { 338 echo "Sub-shell for backend '$backend' returned error code $?. Stop." 339 $exit_cmd 1 340} 341done ## Loop over $backend_list 342 343done ## Loop over $bp_list 344 345 346$exit_cmd 0 347 348