1# ---------------------------------------------------------------------------------------- 2# HONEE GitLab CI 3# ---------------------------------------------------------------------------------------- 4stages: 5 - test:stage-lint 6 - test:stage-full 7 - test:post 8 - test:docs 9 - deploy 10 11workflow: 12 auto_cancel: 13 on_job_failure: all 14 15.test-basic: 16 interruptible: true 17 only: 18 refs: 19 - web 20 - merge_requests 21 22.test: 23 extends: .test-basic 24 only: 25 refs: 26 - web 27 - merge_requests 28 - main 29 - release 30 except: 31 variables: 32 # Skip if the No-Code label is attached to a merge request (i.e., documentation only) 33 - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/ 34 35.docs: 36 image: python:3.10 37 before_script: 38 - pip install -r doc/requirements.txt 39 - apt-get update 40 - apt-get install -y doxygen librsvg2-bin 41 42 43# ---------------------------------------------------------------------------------------- 44# Test formatting 45# ---------------------------------------------------------------------------------------- 46noether-format: 47 stage: test:stage-lint 48 extends: 49 - .test-basic 50 tags: 51 - noether 52 - shell 53 script: 54 - rm -f .SUCCESS 55 - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version 56 - make -j$NPROC_CPU format && git diff --color=always --exit-code 57 - touch .SUCCESS 58 59 60# ---------------------------------------------------------------------------------------- 61# Test static analysis 62# ---------------------------------------------------------------------------------------- 63noether-lint: 64 stage: test:stage-lint 65 extends: .test 66 tags: 67 - noether 68 - shell 69 script: 70 - rm -f .SUCCESS 71 # Environment 72 - export CC=gcc 73 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 74 - echo "-------------- CC ------------------" && $CC --version 75 # Libraries 76 # -- libCEED 77 - echo "-------------- libCEED -------------" 78 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 79 # -- PETSc 80 - echo "-------------- PETSc ---------------" 81 - export PETSC_DIR=/projects/honee/petsc 82 - export PETSC_ARCH=arch-serial-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 83 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 84 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 85 # make with Werror, Wall, supress loop vectorization warnings 86 - echo "-------------- make Werror ---------" 87 - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU 88 # Clang-tidy 89 - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version 90 - PETSC_ARCH=arch-serial-cuda make -j$NPROC_CPU tidy 91 # Report status 92 - touch .SUCCESS 93 94 95# ---------------------------------------------------------------------------------------- 96# Test memory access assumptions 97# ---------------------------------------------------------------------------------------- 98noether-memcheck: 99 stage: test:stage-lint 100 extends: .test 101 tags: 102 - noether 103 - shell 104 script: 105 - rm -f .SUCCESS 106 # Environment 107 - export COVERAGE=1 CC=clang-15 108 - export NPROC_POOL=8 109 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 110 - echo "-------------- CC ------------------" && $CC --version 111 # Libraries 112 # -- libCEED 113 - echo "-------------- libCEED -------------" 114 - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 115 # -- PETSc 116 - echo "-------------- PETSc ---------------" 117 - export PETSC_DIR=/projects/honee/petsc 118 - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 119 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 120 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 121 # ASAN 122 - echo "-------------- ASAN ----------------" 123 - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" 124 - echo $AFLAGS 125 # HONEE 126 - echo "-------------- HONEE ---------------" && make info 127 - make clean 128 - make -j$NPROC_CPU 129 # Test suite 130 - echo "-------------- HONEE tests ---------" 131 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 132 # -- Memcheck libCEED CPU backend, serial 133 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes 134 # Report status 135 - touch .SUCCESS 136 after_script: 137 - | 138 if [ -f .SUCCESS ]; then 139 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 140 fi 141 artifacts: 142 paths: 143 - coverage-$CI_JOB_ID.json 144 - build/*.junit 145 reports: 146 junit: build/*.junit 147 performance: performance.json 148 expire_in: 28 days 149 150 151# ---------------------------------------------------------------------------------------- 152# CPU testing on Noether 153# ---------------------------------------------------------------------------------------- 154noether-cpu: 155 stage: test:stage-full 156 extends: .test 157 tags: 158 - noether 159 - shell 160 script: 161 - rm -f .SUCCESS 162 # Environment 163 - export COVERAGE=1 CC=gcc 164 - export NPROC_POOL=4 165 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 166 - echo "-------------- CC ------------------" && $CC --version 167 - echo "-------------- GCOV ----------------" && gcov --version 168 # Libraries 169 # -- libCEED 170 - echo "-------------- libCEED -------------" 171 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 172 # -- PETSc 173 - echo "-------------- PETSc ---------------" 174 - export PETSC_DIR=/projects/honee/petsc 175 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 176 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 177 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 178 # HONEE 179 - echo "-------------- HONEE ---------------" && make info 180 - make clean 181 - make -j$NPROC_CPU 182 # Test suite 183 - echo "-------------- HONEE tests ---------" 184 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 185 # -- Fastest libCEED CPU backend, parallel 186 - echo "Parallel tests skipped for now" 187 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 188 - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install 189 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes 190 # - spack unload py-torch@2.3+cuda && export USE_TORCH=0 191 - source /projects/honee/SmartSim/venv/bin/activate 192 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim" 193 # Report status 194 - touch .SUCCESS 195 after_script: 196 - | 197 if [ -f .SUCCESS ]; then 198 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 199 fi 200 artifacts: 201 paths: 202 - coverage-$CI_JOB_ID.json 203 - build/*.junit 204 reports: 205 junit: build/*.junit 206 performance: performance.json 207 expire_in: 28 days 208 209 210# ---------------------------------------------------------------------------------------- 211# CPU Int64 testing on Noether 212# ---------------------------------------------------------------------------------------- 213noether-cpu-int64: 214 stage: test:stage-full 215 extends: .test 216 tags: 217 - noether 218 - shell 219 script: 220 - rm -f .SUCCESS 221 # Environment 222 - export COVERAGE=1 CC=gcc 223 - export NPROC_POOL=4 224 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 225 - echo "-------------- CC ------------------" && $CC --version 226 - echo "-------------- GCOV ----------------" && gcov --version 227 # Libraries 228 # -- libCEED 229 - echo "-------------- libCEED -------------" 230 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 231 # -- PETSc 232 - echo "-------------- PETSc ---------------" 233 - export PETSC_DIR=/projects/honee/petsc 234 - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 235 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 236 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 237 # HONEE 238 - echo "-------------- HONEE ---------------" && make info 239 - make clean 240 - make -j$NPROC_CPU 241 # Test suite 242 - echo "-------------- HONEE tests ---------" 243 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 244 # -- Fastest libCEED CPU backend, serial 245 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 246 - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install 247 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes 248 # - spack unload py-torch@2.3+cuda && export USE_TORCH=0 249 - source /projects/honee/SmartSim/venv/bin/activate 250 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search="test-smartsim" 251 # Report status 252 - touch .SUCCESS 253 after_script: 254 - | 255 if [ -f .SUCCESS ]; then 256 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 257 fi 258 artifacts: 259 paths: 260 - coverage-$CI_JOB_ID.json 261 - build/*.junit 262 reports: 263 junit: build/*.junit 264 performance: performance.json 265 expire_in: 28 days 266 267 268#### Disable HIP temporarily 269 270# # ---------------------------------------------------------------------------------------- 271# # GPU testing on Noether 272# # ---------------------------------------------------------------------------------------- 273# noether-hip: 274# stage: test:stage-full 275# extends: .test 276# tags: 277# - noether 278# - shell 279# script: 280# - rm -f .SUCCESS 281# # Environment 282# - export COVERAGE=1 CC=gcc HIPCC=hipcc 283# - export NPROC_POOL=4 284# - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 285# - echo "-------------- CC ------------------" && $CC --version 286# - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 287# - echo "-------------- GCOV ----------------" && gcov --version 288# # Libraries 289# # -- libCEED 290# - echo "-------------- libCEED -------------" 291# - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 292# # -- PETSc 293# - echo "-------------- PETSc ---------------" 294# - export PETSC_DIR=/projects/honee/petsc 295# - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 296# - export PETSC_OPTIONS='-malloc_debug no' # faster tests 297# - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 298# # HONEE 299# - echo "-------------- HONEE ---------------" && make info 300# - make clean 301# - make -j$NPROC_CPU 302# # Test suite 303# - echo "-------------- HONEE tests ---------" 304# - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 305# # -- Fastest libCEED HIP backend, serial 306# # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 307# - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes 308# # Report status 309# - touch .SUCCESS 310# after_script: 311# - | 312# if [ -f .SUCCESS ]; then 313# gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 314# fi 315# after_script: 316# - | 317# if [ -f .SUCCESS ]; then 318# gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 319# fi 320# artifacts: 321# paths: 322# - coverage-$CI_JOB_ID.json 323# - build/*.junit 324# reports: 325# junit: build/*.junit 326# performance: performance.json 327# expire_in: 28 days 328 329 330noether-cuda: 331 stage: test:stage-full 332 extends: .test 333 tags: 334 - noether 335 - shell 336 script: 337 - rm -f .SUCCESS 338 # Environment 339 - export COVERAGE=1 CC=gcc NVCC=nvcc 340 - export NPROC_POOL=4 341 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 342 - echo "-------------- CC ------------------" && $CC --version 343 - echo "-------------- NVCC ----------------" && $NVCC --version 344 - echo "-------------- GCOV ----------------" && gcov --version 345 # Libraries 346 # -- libCEED 347 - echo "-------------- libCEED -------------" 348 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 349 # -- PETSc 350 - echo "-------------- PETSc ---------------" 351 - export PETSC_DIR=/projects/honee/petsc 352 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 353 - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests 354 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 355 # HONEE 356 - echo "-------------- HONEE ---------------" && make info 357 - make clean 358 - make -j$NPROC_CPU 359 # Test suite 360 - echo "-------------- HONEE tests ---------" 361 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 362 # -- Fastest libCEED CUDA backend, serial 363 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 364 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 365 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes 366 # Report status 367 - touch .SUCCESS 368 after_script: 369 - | 370 if [ -f .SUCCESS ]; then 371 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 372 fi 373 artifacts: 374 paths: 375 - coverage-$CI_JOB_ID.json 376 - build/*.junit 377 reports: 378 junit: build/*.junit 379 performance: performance.json 380 expire_in: 28 days 381 382 383test-coverage: 384 stage: test:post 385 extends: .test 386 tags: 387 - noether 388 - shell 389 when: always 390 needs: [noether-memcheck, noether-cpu, noether-cpu-int64, noether-cuda, ] 391 script: 392 - ls coverage-* 393 - mkdir coverage 394 - gcovr --json-add-tracefile "coverage-*.json" --xml-pretty -o coverage.xml --print-summary --html-nested coverage/index.html --html-theme github.dark-blue 395 coverage: '/^lines:\s+(\d+.\d\%)/' 396 artifacts: 397 paths: 398 - coverage.xml 399 - coverage 400 when: always 401 reports: 402 coverage_report: 403 coverage_format: cobertura 404 path: coverage.xml 405 expire_in: 28 days 406 environment: 407 name: coverage/$CI_COMMIT_REF_NAME 408 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/coverage/index.html 409 410 411# ---------------------------------------------------------------------------------------- 412# Build documentation 413# ---------------------------------------------------------------------------------------- 414docs-review: 415 stage: test:docs 416 tags: 417 - noether 418 - docker 419 extends: 420 - .docs 421 - .test-basic 422 interruptible: true 423 script: 424 - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-cuda CEED_DIR=/projects/honee/libCEED 425 - git -c safe.directory=/builds/phypid/honee submodule update --init 426 - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W 427 - mv doc/build/html public 428 artifacts: 429 paths: 430 - public 431 expire_in: 28 days 432 environment: 433 name: review/$CI_COMMIT_REF_NAME 434 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html 435 436 437# ---------------------------------------------------------------------------------------- 438# Deploy documentation using GitLab pages 439# ---------------------------------------------------------------------------------------- 440pages: # this job name has special meaning to GitLab 441 stage: deploy 442 tags: 443 - noether 444 - docker 445 extends: .docs 446 interruptible: false 447 script: 448 - git -c safe.directory=/builds/phypid/honee submodule update --init 449 - make doc-dirhtml pkgconf=true DOXYGENOPTS= 450 - mv doc/build/dirhtml public 451 only: 452 - main 453 artifacts: 454 paths: 455 - public 456