1#!/usr/bin/env bash 2 3# Copyright 2017-2018 4# 5# Karlsruhe Institute of Technology 6# Universitat Jaume I 7# University of Tennessee 8# 9# Redistribution and use in source and binary forms, with or without 10# modification, are permitted provided that the following conditions are met: 11# 12# 1. Redistributions of source code must retain the above copyright notice, 13# this list of conditions and the following disclaimer. 14# 15# 2. Redistributions in binary form must reproduce the above copyright notice, 16# this list of conditions and the following disclaimer in the documentation 17# and/or other materials provided with the distribution. 18# 19# 3. Neither the name of the copyright holder nor the names of its contributors 20# may be used to endorse or promote products derived from this software 21# without specific prior written permission. 22# 23# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33# POSSIBILITY OF SUCH DAMAGE. 34 35set -e 36 37ARCHIVE_LOCATION=${ARCHIVE_LOCATION:="./ssget_archive"} 38THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd ) 39SS_URL="https://sparse.tamu.edu" 40 41mkdir -p "${ARCHIVE_LOCATION}" 42 43COMMAND=get_database_version 44MATRIX_TYPE="MM" 45MATRIX_ID=1 46PROP_NAMES=( 47 "group" 48 "name" 49 "rows" 50 "cols" 51 "nonzeros" 52 "real" 53 "binary" 54 "2d3d" 55 "posdef" 56 "psym" 57 "nsym" 58 "kind") 59 60 61redownload_info() { 62 curl -Lo "${ARCHIVE_LOCATION}/ssstats.csv" "${SS_URL}/files/ssstats.csv" 63} 64 65 66download_info() { 67 if [ ! -f "${ARCHIVE_LOCATION}/ssstats.csv" ]; then 68 redownload_info 69 fi 70} 71 72 73PROPS="" 74get_properties() { 75 download_info 76 if [ "${PROPS}" == "" ]; then 77 PROPS=$(head -$((${MATRIX_ID} + 2)) "${ARCHIVE_LOCATION}/ssstats.csv" \ 78 | tail -1) 79 fi 80 echo "${PROPS}" 81} 82 83 84get_property() { 85 IFS="," read -ra PROPS <<< "$(get_properties)" 86 NPARAM=${#PROP_NAMES[@]} 87 for (( i=0; i < ${NPARAM}; ++i)); do 88 if [ "$1" = "${PROP_NAMES[$i]}" ]; then 89 echo ${PROPS[$i]} 90 return 91 fi 92 done 93} 94 95 96get_as_json() { 97 BOOL_MAP=("false" "true") 98 IFS="," read -ra PROPS <<< "$(get_properties)" 99 cat << JSON 100{ 101 "id": ${MATRIX_ID}, 102 "group": "${PROPS[0]}", 103 "name": "${PROPS[1]}", 104 "rows": ${PROPS[2]}, 105 "cols": ${PROPS[3]}, 106 "nonzeros": ${PROPS[4]}, 107 "real": ${BOOL_MAP[${PROPS[5]}]}, 108 "binary": ${BOOL_MAP[${PROPS[6]}]}, 109 "2d3d": ${BOOL_MAP[${PROPS[7]}]}, 110 "posdef": ${BOOL_MAP[${PROPS[8]}]}, 111 "psym": ${PROPS[9]}, 112 "nsym": ${PROPS[10]}, 113 "kind": "${PROPS[11]}" 114} 115JSON 116} 117 118 119get_path_info() { 120 GROUP=$(get_property group) 121 NAME=$(get_property name) 122 if [ "${MATRIX_TYPE}" = "mat" ]; then 123 EXT="mat" 124 else 125 EXT="tar.gz" 126 fi 127 MATRIX_URI="${MATRIX_TYPE}/${GROUP}/${NAME}" 128 UPSTREAM_URL="${SS_URL}/${MATRIX_URI}.${EXT}" 129 DOWNLOAD_PATH="${ARCHIVE_LOCATION}/${MATRIX_URI}.${EXT}" 130 EXTRACT_PATH="${ARCHIVE_LOCATION}/${MATRIX_URI}" 131 cat <<- EOT 132 ${GROUP} 133 ${NAME} 134 ${MATRIX_URI} 135 ${UPSTREAM_URL} 136 ${DOWNLOAD_PATH} 137 ${EXTRACT_PATH} 138EOT 139} 140 141 142download_archive() { 143 PATH_INFO=($(get_path_info)) 144 if [ ! -f ${PATH_INFO[4]} ]; then 145 mkdir -p $(dirname ${PATH_INFO[4]}) 146 curl -Lo ${PATH_INFO[4]} ${PATH_INFO[3]} 147 fi 148 echo ${PATH_INFO[4]} 149} 150 151 152extract_archive() { 153 PATH_INFO=($(get_path_info)) 154 download_archive >/dev/null 155 if [ "${MATRIX_TYPE}" = "mat" ]; then 156 echo ${PATH_INFO[4]} 157 return 158 fi 159 mkdir -p ${PATH_INFO[5]} 160 tar -xzf ${PATH_INFO[4]} -C ${PATH_INFO[5]} --strip-components=1 161 if [ "${MATRIX_TYPE}" = "RB" ]; then 162 EXT="rb" 163 else 164 EXT="mtx" 165 fi 166 echo "${PATH_INFO[5]}/${PATH_INFO[1]}.${EXT}" 167} 168 169 170clean_extracted() { 171 PATH_INFO=($(get_path_info)) 172 if [ ! -e ${PATH_INFO[5]} ]; then 173 echo 0 174 return 175 fi 176 177 SIZE=($(du -b ${PATH_INFO[5]})) 178 rm -rf ${PATH_INFO[5]} 179 echo ${SIZE[0]} 180} 181 182 183remove_archive() { 184 PATH_INFO=($(get_path_info)) 185 SIZE=($(du -b ${PATH_INFO[4]})) 186 rm -rf ${PATH_INFO[4]} 187 echo ${SIZE[0]} 188} 189 190 191get_collection_size() { 192 download_info 193 head -1 "${ARCHIVE_LOCATION}/ssstats.csv" 194} 195 196 197get_database_version() { 198 download_info 199 echo -n "${ARCHIVE_LOCATION}/ssstats.csv " 200 echo "$(head -2 "${ARCHIVE_LOCATION}/ssstats.csv" | tail -1)" 201} 202 203 204CONDITION="" 205TEMP_PROPS="" 206replace_placeholder() { 207 BOOL_MAP=("false" "true") 208 EVAL_COND=${CONDITION} 209 # s/@kind/${TEMP_PROPS[11]}/g is failed because s/@kind/2D/3D Problem/g has 210 # too many slashes 211 REPLACE="s/@group/${TEMP_PROPS[0]}/g;" 212 REPLACE="${REPLACE}s/@name/${TEMP_PROPS[1]}/g;" 213 REPLACE="${REPLACE}s/@rows/${TEMP_PROPS[2]}/g;" 214 REPLACE="${REPLACE}s/@cols/${TEMP_PROPS[3]}/g;" 215 REPLACE="${REPLACE}s/@nonzeros/${TEMP_PROPS[4]}/g;" 216 REPLACE="${REPLACE}s/@real/${BOOL_MAP[${TEMP_PROPS[5]}]}/g;" 217 REPLACE="${REPLACE}s/@binary/${BOOL_MAP[${TEMP_PROPS[6]}]}/g;" 218 REPLACE="${REPLACE}s/@2d3d/${BOOL_MAP[${TEMP_PROPS[7]}]}/g;" 219 REPLACE="${REPLACE}s/@posdef/${BOOL_MAP[${TEMP_PROPS[8]}]}/g;" 220 REPLACE="${REPLACE}s/@psym/${TEMP_PROPS[9]}/g;" 221 REPLACE="${REPLACE}s/@nsym/${TEMP_PROPS[10]}/g;" 222 REPLACE="${REPLACE}s~@kind~${TEMP_PROPS[11]}~g" 223 EVAL_COND=$(echo $EVAL_COND | sed -e "$REPLACE") 224} 225 226 227search_database() { 228 download_info 229 INDEX=-2; 230 while IFS='' read -r LINE || [[ -n "$LINE" ]]; do 231 INDEX=$(( $INDEX + 1 )) 232 if [[ $INDEX -gt 0 ]]; then 233 IFS="," read -ra TEMP_PROPS <<< "$LINE" 234 EVAL_COND="" 235 replace_placeholder 236 if eval $EVAL_COND ; then 237 echo $INDEX 238 fi 239 fi 240 done < "${ARCHIVE_LOCATION}/ssstats.csv" 241} 242 243 244print_usage_and_exit() { 245 cat 1>&2 << EOT 246Usage: $0 [options] 247 248Available options: 249 -c clean files extracted from archive 250 -d (re)download matrix info file 251 -e download matrix and extract archive 252 -f download matrix and get path to archive 253 -h show this help 254 -i ID matrix id 255 -j print information about the matrix in JSON format 256 -n get number of matrices in collection 257 -p PROPERTY print information about the matrix, PROPERTY is the property to 258 print, one of group, name, rows, cols, nonzeros, real, binary, 259 2d3d, posdef, psym, nsym, kind 260 -r remove archive 261 -t TYPE matrix type, TYPE is one of: MM (matrix market, '.mtx'), RB 262 (Rutherford Boeing, '.rb'), mat (MATLAB, '.mat') 263 -v get database version 264 -s search database with conditions. It uses @PROPERTY as the 265 placeholder 266 267Calling $0 without arguments is equivalent to: $0 -i 0 -t MM -v 268EOT 269 exit $1 270} 271 272 273while getopts ":cdefhi:jnp:rt:vs:" opt; do 274 case ${opt} in 275 :) 276 echo 1>&2 "Option -${OPTARG} provided without an argument" 277 print_usage_and_exit 2 278 ;; 279 \?) 280 echo 1>&2 "Unknown option: -${OPTARG}" 281 print_usage_and_exit 1 282 ;; 283 c) 284 COMMAND=clean_extracted 285 ;; 286 d) 287 COMMAND=redownload_info 288 ;; 289 e) 290 COMMAND=extract_archive 291 ;; 292 f) 293 COMMAND=download_archive 294 ;; 295 h) 296 print_usage_and_exit 0 297 ;; 298 i) 299 if [[ ! "${OPTARG}" =~ ^([0-9]+)$ ]]; then 300 echo 1>&2 "Matrix ID has to be a number, got: ${OPTARG}" 301 print_usage_and_exit 4 302 fi 303 MATRIX_ID=${OPTARG} 304 ;; 305 j) 306 COMMAND=get_as_json 307 ;; 308 n) 309 COMMAND=get_collection_size 310 ;; 311 p) 312 PROP_LIST="group|name|rows|cols|nonzeros" 313 PROP_LIST="${PROP_LIST}|real|binary|2d3d|posdef|psym|nsym|kind" 314 if [[ ! "${OPTARG}" =~ ^(${PROP_LIST})$ ]]; then 315 echo 1>&2 "Unknown property: ${OPTARG}" 316 print_usage_and_exit 5 317 fi 318 COMMAND="get_property ${OPTARG}" 319 ;; 320 r) 321 COMMAND=remove_archive 322 ;; 323 t) 324 if [[ ! "${OPTARG}" =~ ^(MM|RB|mat)$ ]]; then 325 echo 1>&2 "Wrong matrix type: ${OPTARG}" 326 print_usage_and_exit 3 327 fi 328 MATRIX_TYPE=${OPTARG} 329 ;; 330 v) 331 COMMAND=get_database_version 332 ;; 333 s) 334 COMMAND=search_database 335 CONDITION=${OPTARG} 336 ;; 337 esac 338done 339 340 341${COMMAND} 342