1#!/bin/bash 2 3# Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 4# Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 5# All Rights reserved. See files LICENSE and NOTICE for details. 6# 7# This file is part of CEED, a collection of benchmarks, miniapps, software 8# libraries and APIs for efficient high-order finite element and spectral 9# element discretizations for exascale applications. For more information and 10# source code availability see http://github.com/ceed. 11# 12# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 13# a collaborative effort of two U.S. Department of Energy organizations (Office 14# of Science and the National Nuclear Security Administration) responsible for 15# the planning and preparation of a capable exascale ecosystem, including 16# software, applications, hardware, advanced system engineering and early 17# testbed platforms, in support of the nation's exascale computing imperative. 18 19this_file="${BASH_SOURCE[0]}" 20if [[ "${#BASH_ARGV[@]}" -ne "$#" ]]; then 21 script_is_sourced="yes" 22 exit_cmd=return 23else 24 script_is_sourced="" 25 exit_cmd=exit 26fi 27test_file="" 28backend_list="/cpu/self" 29bp_list="bp1 bp3" 30run="" 31num_proc_run=${num_proc_run:-""} 32num_proc_node=${num_proc_node:-""} 33dry_run="" # empty string = NO 34start_shell="" 35verbose="" 36cur_dir="$PWD" 37 38mpiexec="mpirun" 39mpiexec_np="-np" 40mpiexec_opts="" 41mpiexec_post_opts="" 42profiler="" 43 44function abspath() 45{ 46 local outvar="$1" path="$2" cur_dir="$PWD" 47 cd "$path" && path="$PWD" && cd "$cur_dir" && eval "$outvar=\"$path\"" 48} 49 50abspath root_dir ".." || $exit_cmd 1 51build_root="$root_dir/build" 52 53if [[ -t 1 ]]; then 54 # ANSI color codes 55 none=$'\E[0m' 56 red=$'\E[0;31m' 57 green=$'\E[0;32m' 58 yellow=$'\E[0;33m' 59 blue=$'\E[0;34m' 60 bblue=$'\E[1;34m' 61 magenta=$'\E[0;35m' 62 cyan=$'\E[0;36m' 63 clear="$(tput sgr0)" 64fi 65 66help_msg=" 67$this_file [options] 68 69Options: 70 -h|--help print this usage information and exit 71 -b|--bp \"list\" choose the benchmark problems to run 72 -c|--ceed \"list\" choose the libCEED backends to benchmark 73 -r|--run <name> run the tests in the script <name> 74 -n|--num-proc \"list\" total number of MPI tasks to use in the tests 75 -p|--proc-node \"list\" number of MPI tasks per node to use in the tests 76 -d|--dry-run show (but do not run) the commands for the tests 77 -s|--shell execute bash shell commands before running the test 78 -v|--verbose print additional messages 79 -x enable script tracing with 'set -x' 80 var=value define shell variables; evaluated with 'eval' 81 82This script builds and runs a set of benchmarks for a list of specified 83backends. 84 85Example usage: 86 $this_file --run petsc-bpsraw.sh 87 $this_file --run petsc-bps.sh 88" 89 90 91function build_examples() 92{ 93 for example; do 94 # We require the examples to be already built because we do not know what 95 # options to use when building the library + examples. 96 if [ ! -e $build_root/$example ]; then 97 echo "Error: example is not built: $example" 98 return 1 99 fi 100 done 101} 102 103 104function compose_mpi_run_command() 105{ 106 mpi_run="${mpiexec:-mpirun} ${mpiexec_opts}" 107 mpi_run+=" ${mpiexec_np:--np} ${num_proc_run} ${mpiexec_post_opts}" 108 if [[ -n "$profiler" ]]; then 109 mpi_run+=" $profiler" 110 fi 111} 112 113 114function quoted_echo() 115{ 116 local arg= string= 117 for arg; do 118 if [[ -z "${arg##* *}" ]]; then 119 string+=" \"${arg//\"/\\\"}\"" 120 else 121 string+=" $arg" 122 fi 123 done 124 printf "%s\n" "${string# }" 125} 126 127 128function set_num_nodes() 129{ 130 if [[ -n "$num_proc_node" ]]; then 131 ((num_proc_run % num_proc_node != 0)) && { 132 echo "The total number of tasks ($num_proc_run) must be a multiple of" 133 echo "the number of tasks per node ($num_proc_node). Stop." 134 return 1 135 } 136 ((num_nodes = num_proc_run / num_proc_node)) 137 else 138 num_proc_node="unknown number of" 139 num_nodes="" 140 fi 141 echo "Running the tests using a total of $num_proc_run MPI tasks ..." | tee -a $output_file 142 echo "... with $num_proc_node tasks per node ..." | tee -a $output_file 143 echo | tee -a $output_file 144} 145 146 147### Process command line parameters 148 149while [ $# -gt 0 ]; do 150 151case "$1" in 152 -h|--help) 153 # Echo usage information 154 echo "$help_msg" 155 $exit_cmd 156 ;; 157 -b|--bp) 158 shift 159 [ $# -gt 0 ] || { 160 echo "Missing \"list\" in --bp \"list\""; $exit_cmd 1; } 161 bp_list="$1" 162 ;; 163 -c|--ceed) 164 shift 165 [ $# -gt 0 ] || { 166 echo "Missing \"list\" in --ceed \"list\""; $exit_cmd 1; } 167 backend_list="$1" 168 ;; 169 -r|--run) 170 run=on 171 shift 172 [ $# -gt 0 ] || { echo "Missing <name> in --run <name>"; $exit_cmd 1; } 173 test_file="$1" 174 [[ -r "$test_file" ]] || { 175 echo "Test script not found: '$1'"; $exit_cmd 1 176 } 177 ;; 178 -n|--num-proc) 179 shift 180 [ $# -gt 0 ] || { 181 echo "Missing \"list\" in --num-proc \"list\""; $exit_cmd 1; } 182 num_proc_run="$1" 183 ;; 184 -p|--proc-node) 185 shift 186 [ $# -gt 0 ] || { 187 echo "Missing \"list\" in --proc-node \"list\""; $exit_cmd 1; } 188 num_proc_node="$1" 189 ;; 190 -d|--dry-run) 191 dry_run="quoted_echo" 192 ;; 193 -s|--shell) 194 start_shell="yes" 195 ;; 196 -v|--verbose) 197 verbose="yes" 198 ;; 199 -x) 200 set -x 201 ;; 202 *=*) 203 eval "$1" || { echo "Error evaluating argument: $1"; $exit_cmd 1; } 204 ;; 205 *) 206 echo "Unknown option: '$1'" 207 $exit_cmd 1 208 ;; 209esac 210 211shift 212done # while ... 213# Done processing command line parameters 214 215 216num_proc_list=(${num_proc_run:-4}) 217num_proc_list_size=${#num_proc_list[@]} 218num_proc_node_list=(${num_proc_node:-4}) 219num_proc_node_list_size=${#num_proc_node_list[@]} 220(( num_proc_list_size != num_proc_node_list_size )) && { 221 echo " 222The size of the number-of-processors list (option --num-proc) must be the same 223as the size of the number-of-processors-per-node list (option --proc-node)." 224 echo 225 $exit_cmd 1 226} 227 228### Loop over BPs 229 230for bp in $bp_list; do 231 232### Loop over backends 233 234for backend in $backend_list; do 235( ## Run each backend in its own environment 236 237### Setup output 238### Test name 239cd "$cur_dir" 240abspath test_dir "$(dirname "$test_file")" || $exit_cmd 1 241test_basename="$(basename "$test_file")" 242test_file="${test_dir}/${test_basename}" 243### Backend name 244short_backend=${backend//[\/]} 245### Output file 246output_file="${test_file%%.*}-$bp-$short_backend-output.txt" 247rm -rf output_file 248 249### Setup the environment based on $backend 250 251echo 252echo "Using backend $backend ..." | tee $output_file 253 254### Run the tests (building and running $test_file) 255 256[ -n "$run" ] && { 257 258[[ "$verbose" = "yes" ]] && { 259 echo "Test problem file, $test_basename:" | tee -a $output_file 260 echo "------------------------------------------------" | tee -a $output_file 261 cat $test_file | tee -a $output_file 262 echo "------------------------------------------------" | tee -a $output_file 263 echo | tee -a $output_file 264} 265 266test_exe_dir="$build_root" 267 268trap 'printf "\nScript interrupted.\n"; '$exit_cmd' 33' INT 269 270## Source the test script file. 271echo "Reading test file: $test_file" | tee -a $output_file 272echo | tee -a $output_file 273test_required_examples="" 274. "$test_file" || $exit_cmd 1 275 276## Build files required by the test 277echo "Example(s) required by the test: $test_required_examples" | tee -a $output_file 278build_examples $test_required_examples || $exit_cmd 1 279echo | tee -a $output_file 280 281## Loop over the number-of-processors list. 282for (( num_proc_idx = 0; num_proc_idx < num_proc_list_size; num_proc_idx++ )) 283do 284 285num_proc_run="${num_proc_list[$num_proc_idx]}" 286num_proc_node="${num_proc_node_list[$num_proc_idx]}" 287 288set_num_nodes || $exit_cmd 1 289compose_mpi_run_command 290 291if [[ "$start_shell" = "yes" ]]; then 292 if [[ ! -t 1 ]]; then 293 echo "Standard output is not a terminal. Stop." | tee -a $output_file 294 $exit_cmd 1 295 fi 296 echo "Reading shell commands, type 'c' to continue, 'exit' to stop ..." | tee -a $output_file 297 echo | tee -a $output_file 298 cd "$cur_dir" 299 set -o emacs 300 PS1='$ ' 301 [[ -r $HOME/.bashrc ]] && source $HOME/.bashrc 302 HISTFILE="$root_dir/.bash_history" 303 history -c 304 history -r 305 # bind '"\\C-i": menu-complete' 306 alias c='break' 307 while cwd="$PWD/" cwd="${cwd#${root_dir}/}" cwd="${cwd%/}" \ 308 prompt="[${cyan}benchmarks$none:$blue$cwd$clear]\$ " && \ 309 read -p "$prompt" -e line; do 310 history -s "$line" 311 history -w 312 shopt -q -s expand_aliases 313 eval "$line" 314 shopt -q -u expand_aliases 315 done 316 [[ "${#line}" -eq 0 ]] && { echo; $exit_cmd 0; } 317 shopt -q -u expand_aliases 318 echo "Continuing ..." | tee -a $output_file 319fi 320 321# Call the function run_tests defined inside the $test_file 322ceed=$backend 323if [ -z "$dry_run" ]; then 324 run_tests >> $output_file 325else 326 run_tests 327fi 328echo 329 330done ## End of loop over processor numbers 331 332trap - INT 333 334} ## run is on 335 336$exit_cmd 0 337 338) || { 339 echo "Sub-shell for backend '$backend' returned error code $?. Stop." 340 $exit_cmd 1 341} 342done ## Loop over $backend_list 343 344done ## Loop over $bp_list 345 346 347$exit_cmd 0 348 349