1# ---------------------------------------------------------------------------------------- 2# HONEE GitLab CI 3# ---------------------------------------------------------------------------------------- 4stages: 5 - test:stage-lint 6 - test:stage-full 7 - test:post 8 - test:docs 9 - deploy 10 11workflow: 12 auto_cancel: 13 on_job_failure: all 14 15.test-basic: 16 interruptible: true 17 only: 18 refs: 19 - web 20 - merge_requests 21 22.test: 23 extends: .test-basic 24 only: 25 refs: 26 - web 27 - merge_requests 28 - main 29 - release 30 except: 31 variables: 32 # Skip if the No-Code label is attached to a merge request (i.e., documentation only) 33 - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/ 34 35.docs: 36 image: python:3.10 37 before_script: 38 - pip install -r doc/requirements.txt 39 - apt-get update 40 - apt-get install -y doxygen librsvg2-bin 41 42 43# ---------------------------------------------------------------------------------------- 44# Test formatting 45# ---------------------------------------------------------------------------------------- 46noether-format: 47 stage: test:stage-lint 48 extends: 49 - .test-basic 50 tags: 51 - noether 52 - shell 53 script: 54 - rm -f .SUCCESS 55 - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version 56 - make -j$NPROC_CPU format && git diff --color=always --exit-code 57 - echo "-------------- make checkbadSource ---------" 58 - make -j$NPROC_CPU checkbadSource 59 - touch .SUCCESS 60 61 62# ---------------------------------------------------------------------------------------- 63# Test static analysis 64# ---------------------------------------------------------------------------------------- 65noether-lint: 66 stage: test:stage-lint 67 extends: .test 68 tags: 69 - noether 70 - shell 71 script: 72 - rm -f .SUCCESS 73 # Environment 74 - export CC=gcc 75 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 76 - echo "-------------- CC ------------------" && $CC --version 77 # Libraries 78 # -- libCEED 79 - echo "-------------- libCEED -------------" 80 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 81 # -- PETSc 82 - echo "-------------- PETSc ---------------" 83 - export PETSC_DIR=/projects/honee/petsc 84 - export PETSC_ARCH=arch-serial-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 85 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 86 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 87 # make with Werror, Wall, supress loop vectorization warnings 88 - echo "-------------- make Werror ---------" 89 - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU 90 # Clang-tidy 91 - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version 92 - PETSC_ARCH=arch-serial-cuda make -j$NPROC_CPU tidy 93 # Report status 94 - touch .SUCCESS 95 96 97# ---------------------------------------------------------------------------------------- 98# Test memory access assumptions 99# ---------------------------------------------------------------------------------------- 100noether-memcheck: 101 stage: test:stage-lint 102 extends: .test 103 tags: 104 - noether 105 - shell 106 script: 107 - rm -f .SUCCESS 108 # Environment 109 - export COVERAGE=1 CC=clang-15 110 - export NPROC_POOL=8 111 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 112 - echo "-------------- CC ------------------" && $CC --version 113 # Libraries 114 # -- libCEED 115 - echo "-------------- libCEED -------------" 116 - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 117 # -- PETSc 118 - echo "-------------- PETSc ---------------" 119 - export PETSC_DIR=/projects/honee/petsc 120 - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 121 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 122 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 123 # ASAN 124 - echo "-------------- ASAN ----------------" 125 - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" 126 - echo $AFLAGS 127 # HONEE 128 - echo "-------------- HONEE ---------------" && make info 129 - make clean 130 - make -j$NPROC_CPU 131 # Test suite 132 - echo "-------------- HONEE tests ---------" 133 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 134 # -- Memcheck libCEED CPU backend, serial 135 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes 136 # Report status 137 - touch .SUCCESS 138 after_script: 139 - | 140 if [ -f .SUCCESS ]; then 141 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 142 fi 143 artifacts: 144 paths: 145 - coverage-$CI_JOB_ID.json 146 - build/*.junit 147 reports: 148 junit: build/*.junit 149 performance: performance.json 150 expire_in: 28 days 151 152 153# ---------------------------------------------------------------------------------------- 154# CPU testing on Noether 155# ---------------------------------------------------------------------------------------- 156noether-cpu: 157 stage: test:stage-full 158 extends: .test 159 tags: 160 - noether 161 - shell 162 script: 163 - rm -f .SUCCESS 164 # Environment 165 - export COVERAGE=1 CC=gcc 166 - export NPROC_POOL=4 167 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 168 - echo "-------------- CC ------------------" && $CC --version 169 - echo "-------------- GCOV ----------------" && gcov --version 170 # Libraries 171 # -- libCEED 172 - echo "-------------- libCEED -------------" 173 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 174 # -- PETSc 175 - echo "-------------- PETSc ---------------" 176 - export PETSC_DIR=/projects/honee/petsc 177 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 178 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 179 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 180 # HONEE 181 - echo "-------------- HONEE ---------------" && make info 182 - make clean 183 - make -j$NPROC_CPU 184 # Test suite 185 - echo "-------------- HONEE tests ---------" 186 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 187 # -- Fastest libCEED CPU backend, parallel 188 - echo "Parallel tests skipped for now" 189 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 190 - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install 191 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes 192 # - spack unload py-torch@2.3+cuda && export USE_TORCH=0 193 - source /projects/honee/SmartSim/venv/bin/activate 194 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim" 195 # Report status 196 - touch .SUCCESS 197 after_script: 198 - | 199 if [ -f .SUCCESS ]; then 200 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 201 fi 202 artifacts: 203 paths: 204 - coverage-$CI_JOB_ID.json 205 - build/*.junit 206 reports: 207 junit: build/*.junit 208 performance: performance.json 209 expire_in: 28 days 210 211 212# ---------------------------------------------------------------------------------------- 213# CPU Int64 testing on Noether 214# ---------------------------------------------------------------------------------------- 215noether-cpu-int64: 216 stage: test:stage-full 217 extends: .test 218 tags: 219 - noether 220 - shell 221 script: 222 - rm -f .SUCCESS 223 # Environment 224 - export COVERAGE=1 CC=gcc 225 - export NPROC_POOL=4 226 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 227 - echo "-------------- CC ------------------" && $CC --version 228 - echo "-------------- GCOV ----------------" && gcov --version 229 # Libraries 230 # -- libCEED 231 - echo "-------------- libCEED -------------" 232 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 233 # -- PETSc 234 - echo "-------------- PETSc ---------------" 235 - export PETSC_DIR=/projects/honee/petsc 236 - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 237 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 238 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 239 # HONEE 240 - echo "-------------- HONEE ---------------" && make info 241 - make clean 242 - make -j$NPROC_CPU 243 # Test suite 244 - echo "-------------- HONEE tests ---------" 245 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 246 # -- Fastest libCEED CPU backend, serial 247 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 248 - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install 249 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes 250 # - spack unload py-torch@2.3+cuda && export USE_TORCH=0 251 - source /projects/honee/SmartSim/venv/bin/activate 252 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search="test-smartsim" 253 # Report status 254 - touch .SUCCESS 255 after_script: 256 - | 257 if [ -f .SUCCESS ]; then 258 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 259 fi 260 artifacts: 261 paths: 262 - coverage-$CI_JOB_ID.json 263 - build/*.junit 264 reports: 265 junit: build/*.junit 266 performance: performance.json 267 expire_in: 28 days 268 269 270#### Disable HIP temporarily 271 272# # ---------------------------------------------------------------------------------------- 273# # GPU testing on Noether 274# # ---------------------------------------------------------------------------------------- 275# noether-hip: 276# stage: test:stage-full 277# extends: .test 278# tags: 279# - noether 280# - shell 281# script: 282# - rm -f .SUCCESS 283# # Environment 284# - export COVERAGE=1 CC=gcc HIPCC=hipcc 285# - export NPROC_POOL=4 286# - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 287# - echo "-------------- CC ------------------" && $CC --version 288# - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 289# - echo "-------------- GCOV ----------------" && gcov --version 290# # Libraries 291# # -- libCEED 292# - echo "-------------- libCEED -------------" 293# - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 294# # -- PETSc 295# - echo "-------------- PETSc ---------------" 296# - export PETSC_DIR=/projects/honee/petsc 297# - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 298# - export PETSC_OPTIONS='-malloc_debug no' # faster tests 299# - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 300# # HONEE 301# - echo "-------------- HONEE ---------------" && make info 302# - make clean 303# - make -j$NPROC_CPU 304# # Test suite 305# - echo "-------------- HONEE tests ---------" 306# - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 307# # -- Fastest libCEED HIP backend, serial 308# # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 309# - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes 310# # Report status 311# - touch .SUCCESS 312# after_script: 313# - | 314# if [ -f .SUCCESS ]; then 315# gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 316# fi 317# after_script: 318# - | 319# if [ -f .SUCCESS ]; then 320# gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 321# fi 322# artifacts: 323# paths: 324# - coverage-$CI_JOB_ID.json 325# - build/*.junit 326# reports: 327# junit: build/*.junit 328# performance: performance.json 329# expire_in: 28 days 330 331 332noether-cuda: 333 stage: test:stage-full 334 extends: .test 335 tags: 336 - noether 337 - shell 338 script: 339 - rm -f .SUCCESS 340 # Environment 341 - export COVERAGE=1 CC=gcc NVCC=nvcc 342 - export NPROC_POOL=4 343 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 344 - echo "-------------- CC ------------------" && $CC --version 345 - echo "-------------- NVCC ----------------" && $NVCC --version 346 - echo "-------------- GCOV ----------------" && gcov --version 347 # Libraries 348 # -- libCEED 349 - echo "-------------- libCEED -------------" 350 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 351 # -- PETSc 352 - echo "-------------- PETSc ---------------" 353 - export PETSC_DIR=/projects/honee/petsc 354 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 355 - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests 356 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 357 # HONEE 358 - echo "-------------- HONEE ---------------" && make info 359 - make clean 360 - make -j$NPROC_CPU 361 # Test suite 362 - echo "-------------- HONEE tests ---------" 363 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 364 # -- Fastest libCEED CUDA backend, serial 365 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 366 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 367 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes 368 # Report status 369 - touch .SUCCESS 370 after_script: 371 - | 372 if [ -f .SUCCESS ]; then 373 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 374 fi 375 artifacts: 376 paths: 377 - coverage-$CI_JOB_ID.json 378 - build/*.junit 379 reports: 380 junit: build/*.junit 381 performance: performance.json 382 expire_in: 28 days 383 384 385test-coverage: 386 stage: test:post 387 extends: .test 388 tags: 389 - noether 390 - shell 391 when: always 392 needs: [noether-memcheck, noether-cpu, noether-cpu-int64, noether-cuda, ] 393 script: 394 - ls coverage-* 395 - mkdir coverage 396 - gcovr --json-add-tracefile "coverage-*.json" --xml-pretty -o coverage.xml --print-summary --html-nested coverage/index.html --html-theme github.dark-blue 397 coverage: '/^lines:\s+(\d+.\d\%)/' 398 artifacts: 399 paths: 400 - coverage.xml 401 - coverage 402 when: always 403 reports: 404 coverage_report: 405 coverage_format: cobertura 406 path: coverage.xml 407 expire_in: 28 days 408 environment: 409 name: coverage/$CI_COMMIT_REF_NAME 410 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/coverage/index.html 411 412 413# ---------------------------------------------------------------------------------------- 414# Build documentation 415# ---------------------------------------------------------------------------------------- 416docs-review: 417 stage: test:docs 418 tags: 419 - noether 420 - docker 421 extends: 422 - .docs 423 - .test-basic 424 interruptible: true 425 script: 426 - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-cuda CEED_DIR=/projects/honee/libCEED 427 - git -c safe.directory=/builds/phypid/honee submodule update --init 428 - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W 429 - mv doc/build/html public 430 artifacts: 431 paths: 432 - public 433 expire_in: 28 days 434 environment: 435 name: review/$CI_COMMIT_REF_NAME 436 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html 437 438 439# ---------------------------------------------------------------------------------------- 440# Deploy documentation using GitLab pages 441# ---------------------------------------------------------------------------------------- 442pages: # this job name has special meaning to GitLab 443 stage: deploy 444 tags: 445 - noether 446 - docker 447 extends: .docs 448 interruptible: false 449 script: 450 - git -c safe.directory=/builds/phypid/honee submodule update --init 451 - make doc-dirhtml pkgconf=true DOXYGENOPTS= 452 - mv doc/build/dirhtml public 453 only: 454 - main 455 artifacts: 456 paths: 457 - public 458