1 2 3scriptname=`basename $0` 4rundir=${scriptname%.sh} 5TIMEOUT=60 6 7if test "$PWD"!=`dirname $0`; then 8 cd `dirname $0` 9 abspath_scriptdir=$PWD 10fi 11if test -d "${rundir}" && test -n "${rundir}"; then 12 rm -f ${rundir}/*.tmp ${rundir}/*.err ${rundir}/*.out 13fi 14mkdir -p ${rundir} 15if test -n "${runfiles}"; then 16 for runfile in ${runfiles}; do 17 subdir=`dirname ${runfile}` 18 mkdir -p ${rundir}/${subdir} 19 cp -r ${runfile} ${rundir}/${subdir} 20 done 21fi 22cd ${rundir} 23 24# 25# Method to print out general and script specific options 26# 27print_usage() { 28 29cat >&2 <<EOF 30Usage: $0 [options] 31 32OPTIONS 33 -a <args> ......... Override default arguments 34 -c ................ Cleanup (remove generated files) 35 -C ................ Compile 36 -d ................ Launch in debugger 37 -e <args> ......... Add extra arguments to default 38 -f ................ force attempt to run test that would otherwise be skipped 39 -h ................ help: print this message 40 -n <integer> ...... Override the number of processors to use 41 -j ................ Pass -j to petscdiff (just use diff) 42 -J <arg> .......... Pass -J to petscdiff (just use diff with arg) 43 -m ................ Update results using petscdiff 44 -M ................ Update alt files using petscdiff 45 -o <arg> .......... Output format: 'interactive', 'err_only' 46 -p ................ Print command: Print first command and exit 47 -t ................ Override the default timeout (default=$TIMEOUT sec) 48 -U ................ run cUda-memcheck 49 -V ................ run Valgrind 50 -v ................ Verbose: Print commands 51EOF 52 53 if declare -f extrausage > /dev/null; then extrausage; fi 54 exit $1 55} 56### 57## Arguments for overriding things 58# 59output_fmt="interactive" 60verbose=false 61cleanup=false 62compile=false 63debugger=false 64printcmd=false 65force=false 66diff_flags="" 67while getopts "a:cCde:fhjJ:mMn:o:pt:UvV" arg 68do 69 case $arg in 70 a ) args="$OPTARG" ;; 71 c ) cleanup=true ;; 72 C ) compile=true ;; 73 d ) debugger=true ;; 74 e ) extra_args="$OPTARG" ;; 75 f ) force=true ;; 76 h ) print_usage; exit ;; 77 n ) nsize="$OPTARG" ;; 78 j ) diff_flags=$diff_flags" -j" ;; 79 J ) diff_flags=$diff_flags" -J $OPTARG" ;; 80 m ) diff_flags=$diff_flags" -m" ;; 81 M ) diff_flags=$diff_flags" -M" ;; 82 o ) output_fmt=$OPTARG ;; 83 p ) printcmd=true ;; 84 t ) TIMEOUT=$OPTARG ;; 85 U ) mpiexec="petsc_mpiexec_cudamemcheck $mpiexec" ;; 86 V ) mpiexec="petsc_mpiexec_valgrind $mpiexec" ;; 87 v ) verbose=true ;; 88 *) # To take care of any extra args 89 if test -n "$OPTARG"; then 90 eval $arg=\"$OPTARG\" 91 else 92 eval $arg=found 93 fi 94 ;; 95 esac 96done 97shift $(( $OPTIND - 1 )) 98 99# Individual tests can extend the default 100export MPIEXEC_TIMEOUT=$((TIMEOUT*timeoutfactor)) 101STARTTIME=`date +%s` 102 103if test -n "$extra_args"; then 104 args="$args $extra_args" 105fi 106if $debugger; then 107 args="-start_in_debugger $args" 108fi 109if test -n "$filter"; then 110 diff_flags=$diff_flags" -F \$'$filter'" 111fi 112if test -n "$filter_output"; then 113 diff_flags=$diff_flags" -f \$'$filter_output'" 114fi 115 116 117# Init 118success=0; failed=0; failures=""; rmfiles="" 119total=0 120todo=-1; skip=-1 121job_level=0 122 123function petsc_report_tapoutput() { 124 notornot=$1 125 test_label=$2 126 comment=$3 127 if test -n "$comment"; then 128 comment=" # ${comment}" 129 fi 130 131 tap_message="${notornot} ok ${test_label}${comment}" 132 133 # Log messages 134 printf "${tap_message}\n" >> ${testlogtapfile} 135 136 if test ${output_fmt} == "err_only"; then 137 if test -n "${notornot}"; then 138 printf "${tap_message}\n" | tee -a ${testlogerrfile} 139 fi 140 else 141 printf "${tap_message}\n" 142 fi 143} 144 145function printcmd() { 146 # Print command that can be run from PETSC_DIR 147 cmd="$1" 148 basedir=`dirname ${PWD} | sed "s#${petsc_dir}/##"` 149 modcmd=`echo ${cmd} | sed -e "s#\.\.#${basedir}#" | sed s#\>.*##` 150 printf "${modcmd}\n" 151 exit 152} 153 154function petsc_testrun() { 155 # First arg = Basic command 156 # Second arg = stdout file 157 # Third arg = stderr file 158 # Fourth arg = label for reporting 159 rmfiles="${rmfiles} $2 $3" 160 tlabel=$4 161 error=$5 162 cmd="$1 > $2 2> $3" 163 if test -n "$error"; then 164 cmd="$1 2>&1 | cat > $2" 165 fi 166 echo "$cmd" > ${tlabel}.sh; chmod 755 ${tlabel}.sh 167 if $printcmd; then 168 printcmd "$cmd" 169 fi 170 171 eval "{ time -p $cmd ; } 2>> timing.out" 172 cmd_res=$? 173 # If it is a lack of GPU resources, then try once more 174 # See: src/sys/error/err.c 175 if [ $cmd_res -eq 96 ]; then 176 eval "{ time -p $cmd ; } 2>> timing.out" 177 cmd_res=$? 178 fi 179 touch "$2" "$3" 180 # ETIMEDOUT=110 on most systems (used by Open MPI 3.0). MPICH uses 181 # 255. Earlier Open MPI returns 1 but outputs about MPIEXEC_TIMEOUT. 182 if [ $cmd_res -eq 110 -o $cmd_res -eq 255 ] || \ 183 fgrep -q -s 'APPLICATION TIMED OUT' "$2" "$3" || \ 184 fgrep -q -s MPIEXEC_TIMEOUT "$2" "$3" || \ 185 fgrep -q -s 'APPLICATION TERMINATED WITH THE EXIT STRING: job ending due to timeout' "$2" "$3" || \ 186 grep -q -s "Timeout after [0-9]* seconds. Terminating job" "$2" "$3"; then 187 timed_out=1 188 # If timed out, then ensure non-zero error code 189 if [ $cmd_res -eq 0 ]; then 190 cmd_res=1 191 fi 192 fi 193 194 # Report errors 195 comment="" 196 if test $cmd_res == 0; then 197 if "${verbose}"; then 198 comment="${cmd}" 199 fi 200 petsc_report_tapoutput "" "$tlabel" "$comment" 201 let success=$success+1 202 else 203 if [ -n "$timed_out" ]; then 204 comment="Exceeded timeout limit of $MPIEXEC_TIMEOUT s" 205 else 206 comment="Error code: ${cmd_res}" 207 fi 208 petsc_report_tapoutput "not" "$tlabel" "$comment" 209 210 # Report errors in detail 211 if [ -z "$timed_out" ]; then 212 # We've had tests fail but stderr->stdout, as well as having 213 # mpi_abort go to stderr which throws this test off. Show both 214 # with stdout first 215 awk '{print "#\t" $0}' < $2 | tee -a ${testlogerrfile} 216 # if statement is for diff tests 217 if test "$2" != "$3"; then 218 awk '{print "#\t" $0}' < $3 | tee -a ${testlogerrfile} 219 fi 220 fi 221 let failed=$failed+1 222 failures="$failures $tlabel" 223 fi 224 let total=$success+$failed 225 return $cmd_res 226} 227 228function petsc_testend() { 229 logfile=$1/counts/${label}.counts 230 logdir=`dirname $logfile` 231 if ! test -d "$logdir"; then 232 mkdir -p $logdir 233 fi 234 if ! test -e "$logfile"; then 235 touch $logfile 236 fi 237 printf "total $total\n" > $logfile 238 printf "success $success\n" >> $logfile 239 printf "failed $failed\n" >> $logfile 240 printf "failures $failures\n" >> $logfile 241 if test ${todo} -gt 0; then 242 printf "todo $todo\n" >> $logfile 243 fi 244 if test ${skip} -gt 0; then 245 printf "skip $skip\n" >> $logfile 246 fi 247 ENDTIME=`date +%s` 248 timing=`touch timing.out && egrep '(user|sys)' timing.out | awk '{if( sum1 == "" || $2 > sum1 ) { sum1=sprintf("%.2f",$2) } ; sum2 += sprintf("%.2f",$2)} END {printf "%.2f %.2f\n",sum1,sum2}'` 249 printf "time $timing\n" >> $logfile 250 if $cleanup; then 251 echo "Cleaning up" 252 /bin/rm -f $rmfiles 253 fi 254} 255 256function petsc_mpiexec_cudamemcheck() { 257 _mpiexec=$1;shift 258 npopt=$1;shift 259 np=$1;shift 260 261 cudamemchk="cuda-memcheck" 262 263 $_mpiexec $npopt $np $cudamemchk $* 264} 265export LC_ALL=C 266 267if $compile; then 268 curexec=`basename ${exec}` 269 (cd $petsc_dir && make -f gmakefile.test ${abspath_scriptdir}/${curexec}) 270fi 271function petsc_mpiexec_valgrind() { 272 _mpiexec=$1;shift 273 npopt=$1;shift 274 np=$1;shift 275 276 valgrind="valgrind -q --tool=memcheck --leak-check=yes --num-callers=20 --track-origins=yes --suppressions=$petsc_bindir/maint/petsc-val.supp --error-exitcode=10" 277 278 $_mpiexec $npopt $np $valgrind "$@" 279} 280export LC_ALL=C 281 282if $compile; then 283 curexec=`basename ${exec}` 284 (cd $petsc_dir && make -f gmakefile.test ${abspath_scriptdir}/${curexec}) 285fi 286 287