1# ---------------------------------------------------------------------------------------- 2# HONEE GitLab CI 3# ---------------------------------------------------------------------------------------- 4stages: 5 - test:stage-lint 6 - test:stage-full 7 - test:docs 8 - deploy 9 10.test-basic: 11 interruptible: true 12 only: 13 refs: 14 - web 15 - merge_requests 16 17.test: 18 extends: .test-basic 19 only: 20 refs: 21 - web 22 - merge_requests 23 - main 24 - release 25 except: 26 variables: 27 # Skip if the No-Code label is attached to a merge request (i.e., documentation only) 28 - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/ 29 needs: 30 - job: docker-build 31 optional: true 32 33.docs: 34 image: python:3.10 35 before_script: 36 - pip install -r doc/requirements.txt 37 - apt-get update 38 - apt-get install -y doxygen librsvg2-bin 39 40 41# ---------------------------------------------------------------------------------------- 42# Test formatting and static analysis 43# ---------------------------------------------------------------------------------------- 44noether-lint: 45 stage: test:stage-lint 46 extends: .test 47 tags: 48 - noether 49 - shell 50 needs: [] 51 script: 52 - rm -f .SUCCESS 53 # Environment 54 - export CC=gcc HIPCC=hipcc 55 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 56 - echo "-------------- CC ------------------" && $CC --version 57 - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 58 # Libraries 59 # -- libCEED 60 - echo "-------------- libCEED -------------" 61 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 62 # -- PETSc 63 - echo "-------------- PETSc ---------------" 64 - export PETSC_DIR=/projects/honee/petsc 65 - export PETSC_ARCH=arch-serial-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 66 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 67 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 68 # make with Werror, Wall, supress loop vectorization warnings 69 - echo "-------------- make Werror ---------" 70 - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU 71 # make format 72 - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version 73 - make -j$NPROC_CPU format && git diff --color=always --exit-code 74 # Clang-tidy 75 - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version 76 - PETSC_ARCH=arch-serial-hip make -j$NPROC_CPU tidy 77 # Report status 78 - touch .SUCCESS 79 80 81# ---------------------------------------------------------------------------------------- 82# Test memory access assumptions 83# ---------------------------------------------------------------------------------------- 84noether-memcheck: 85 stage: test:stage-lint 86 extends: .test 87 tags: 88 - noether 89 - shell 90 needs: [] 91 script: 92 - rm -f .SUCCESS 93 # Environment 94 # -- NOTE: Coverage disabled because it doesn't play nice with the ASAN options 95 - export CC=clang-15 96 - export NPROC_POOL=8 97 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 98 - echo "-------------- CC ------------------" && $CC --version 99 # Libraries 100 # -- libCEED 101 - echo "-------------- libCEED -------------" 102 - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 103 # -- PETSc 104 - echo "-------------- PETSc ---------------" 105 - export PETSC_DIR=/projects/honee/petsc 106 - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 107 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 108 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 109 # ASAN 110 - echo "-------------- ASAN ----------------" 111 - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" 112 - echo $AFLAGS 113 # HONEE 114 - echo "-------------- HONEE ---------------" && make info 115 - make clean 116 - make -j$NPROC_CPU 117 # Test suite 118 - echo "-------------- HONEE tests ---------" 119 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 120 # -- Memcheck libCEED CPU backend, serial 121 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes 122 # Report status 123 - touch .SUCCESS 124 artifacts: 125 paths: 126 - build/*.junit 127 reports: 128 junit: build/*.junit 129 performance: performance.json 130 expire_in: 28 days 131 132 133# ---------------------------------------------------------------------------------------- 134# CPU testing on Noether 135# ---------------------------------------------------------------------------------------- 136noether-cpu: 137 stage: test:stage-full 138 extends: .test 139 tags: 140 - noether 141 - shell 142 script: 143 - rm -f .SUCCESS 144 # Environment 145 - export COVERAGE=1 CC=gcc 146 - export NPROC_POOL=4 147 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 148 - echo "-------------- CC ------------------" && $CC --version 149 - echo "-------------- GCOV ----------------" && gcov --version 150 # Libraries 151 # -- libCEED 152 - echo "-------------- libCEED -------------" 153 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 154 # -- PETSc 155 - echo "-------------- PETSc ---------------" 156 - export PETSC_DIR=/projects/honee/petsc 157 - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 158 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 159 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 160 # HONEE 161 - echo "-------------- HONEE ---------------" && make info 162 - make clean 163 - make -j$NPROC_CPU 164 # Test suite 165 - echo "-------------- HONEE tests ---------" 166 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 167 # -- Fastest libCEED CPU backend, parallel 168 - echo "Parallel tests skipped for now" 169 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 170 - export SMARTREDIS_DIR=/home/phypid/SmartSimTestingSoftware/smartredis/install 171 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes 172 - spack unload py-torch@2.3+cuda && export USE_TORCH=0 173 - source /home/phypid/SmartSimTestingSoftware/bin/activate 174 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim" 175 # Report status 176 - touch .SUCCESS 177 after_script: 178 - | 179 if [ -f .SUCCESS ]; then 180 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 181 fi 182 coverage: '/^lines:\s+(\d+.\d\%)/' 183 artifacts: 184 paths: 185 - coverage.xml 186 - build/*.junit 187 reports: 188 coverage_report: 189 coverage_format: cobertura 190 path: coverage.xml 191 junit: build/*.junit 192 performance: performance.json 193 expire_in: 28 days 194 195 196# ---------------------------------------------------------------------------------------- 197# CPU Int64 testing on Noether 198# ---------------------------------------------------------------------------------------- 199noether-cpu-int64: 200 stage: test:stage-full 201 extends: .test 202 tags: 203 - noether 204 - shell 205 script: 206 - rm -f .SUCCESS 207 # Environment 208 - export COVERAGE=1 CC=gcc 209 - export NPROC_POOL=4 210 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 211 - echo "-------------- CC ------------------" && $CC --version 212 - echo "-------------- GCOV ----------------" && gcov --version 213 # Libraries 214 # -- libCEED 215 - echo "-------------- libCEED -------------" 216 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 217 # -- PETSc 218 - echo "-------------- PETSc ---------------" 219 - export PETSC_DIR=/projects/honee/petsc 220 - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 221 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 222 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 223 # HONEE 224 - echo "-------------- HONEE ---------------" && make info 225 - make clean 226 - make -j$NPROC_CPU 227 # Test suite 228 - echo "-------------- HONEE tests ---------" 229 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 230 # -- Fastest libCEED CPU backend, serial 231 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 232 - export SMARTREDIS_DIR=/home/phypid/SmartSimTestingSoftware/smartredis/install 233 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes 234 - spack unload py-torch@2.3+cuda && export USE_TORCH=0 235 - source /home/phypid/SmartSimTestingSoftware/bin/activate 236 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-py-smartsim_regression_framework" 237 # Report status 238 - touch .SUCCESS 239 after_script: 240 - | 241 if [ -f .SUCCESS ]; then 242 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 243 fi 244 coverage: '/^lines:\s+(\d+.\d\%)/' 245 artifacts: 246 paths: 247 - coverage.xml 248 - build/*.junit 249 reports: 250 coverage_report: 251 coverage_format: cobertura 252 path: coverage.xml 253 junit: build/*.junit 254 performance: performance.json 255 expire_in: 28 days 256 257 258# ---------------------------------------------------------------------------------------- 259# GPU testing on Noether 260# ---------------------------------------------------------------------------------------- 261noether-hip: 262 stage: test:stage-full 263 extends: .test 264 tags: 265 - noether 266 - shell 267 script: 268 - rm -f .SUCCESS 269 # Environment 270 - export COVERAGE=1 CC=gcc HIPCC=hipcc 271 - export NPROC_POOL=4 272 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 273 - echo "-------------- CC ------------------" && $CC --version 274 - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 275 - echo "-------------- GCOV ----------------" && gcov --version 276 # Libraries 277 # -- libCEED 278 - echo "-------------- libCEED -------------" 279 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 280 # -- PETSc 281 - echo "-------------- PETSc ---------------" 282 - export PETSC_DIR=/projects/honee/petsc 283 - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 284 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 285 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 286 # HONEE 287 - echo "-------------- HONEE ---------------" && make info 288 - make clean 289 - make -j$NPROC_CPU 290 # Test suite 291 - echo "-------------- HONEE tests ---------" 292 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 293 # -- Fastest libCEED HIP backend, serial 294 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 295 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes 296 # Report status 297 - touch .SUCCESS 298 after_script: 299 - | 300 if [ -f .SUCCESS ]; then 301 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 302 fi 303 coverage: '/^lines:\s+(\d+.\d\%)/' 304 artifacts: 305 paths: 306 - coverage.xml 307 - build/*.junit 308 reports: 309 coverage_report: 310 coverage_format: cobertura 311 path: coverage.xml 312 junit: build/*.junit 313 performance: performance.json 314 expire_in: 28 days 315 316 317noether-cuda: 318 stage: test:stage-full 319 extends: .test 320 tags: 321 - noether 322 - shell 323 script: 324 - rm -f .SUCCESS 325 # Environment 326 - export COVERAGE=1 CC=gcc NVCC=nvcc 327 - export NPROC_POOL=4 328 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 329 - echo "-------------- CC ------------------" && $CC --version 330 - echo "-------------- NVCC ----------------" && $NVCC --version 331 - echo "-------------- GCOV ----------------" && gcov --version 332 # Libraries 333 # -- libCEED 334 - echo "-------------- libCEED -------------" 335 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 336 # -- PETSc 337 - echo "-------------- PETSc ---------------" 338 - export PETSC_DIR=/projects/honee/petsc 339 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 340 - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests 341 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 342 # HONEE 343 - echo "-------------- HONEE ---------------" && make info 344 - make clean 345 - make -j$NPROC_CPU 346 # Test suite 347 - echo "-------------- HONEE tests ---------" 348 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 349 # -- Fastest libCEED CUDA backend, serial 350 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 351 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 352 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes 353 # Report status 354 - touch .SUCCESS 355 after_script: 356 - | 357 if [ -f .SUCCESS ]; then 358 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 359 fi 360 coverage: '/^lines:\s+(\d+.\d\%)/' 361 artifacts: 362 paths: 363 - coverage.xml 364 - build/*.junit 365 reports: 366 coverage_report: 367 coverage_format: cobertura 368 path: coverage.xml 369 junit: build/*.junit 370 performance: performance.json 371 expire_in: 28 days 372 373 374# ---------------------------------------------------------------------------------------- 375# Build documentation 376# ---------------------------------------------------------------------------------------- 377docs-review: 378 stage: test:docs 379 tags: 380 - noether 381 - docker 382 extends: 383 - .docs 384 - .test-basic 385 interruptible: true 386 script: 387 - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-hip CEED_DIR=/projects/honee/libCEED 388 - git submodule update --init 389 - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W 390 - mv doc/build/html public 391 artifacts: 392 paths: 393 - public 394 expire_in: 28 days 395 environment: 396 name: review/$CI_COMMIT_REF_NAME 397 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html 398 399 400# ---------------------------------------------------------------------------------------- 401# Deploy documentation using GitLab pages 402# ---------------------------------------------------------------------------------------- 403pages: # this job name has special meaning to GitLab 404 stage: deploy 405 tags: 406 - noether 407 - docker 408 extends: .docs 409 interruptible: false 410 script: 411 - git submodule update --init 412 - make doc-dirhtml pkgconf=true DOXYGENOPTS= 413 - mv doc/build/dirhtml public 414 only: 415 - main 416 artifacts: 417 paths: 418 - public 419