1# ---------------------------------------------------------------------------------------- 2# HONEE GitLab CI 3# ---------------------------------------------------------------------------------------- 4stages: 5 - test:stage-lint 6 - test:stage-full 7 - test:post 8 - test:docs 9 - deploy 10 11workflow: 12 auto_cancel: 13 on_job_failure: all 14 15.test-basic: 16 interruptible: true 17 only: 18 refs: 19 - web 20 - merge_requests 21 22.test: 23 extends: .test-basic 24 only: 25 refs: 26 - web 27 - merge_requests 28 - main 29 - release 30 except: 31 variables: 32 # Skip if the No-Code label is attached to a merge request (i.e., documentation only) 33 - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/ 34 35.docs: 36 image: python:3.10 37 before_script: 38 - pip install -r doc/requirements.txt 39 - apt-get update 40 - apt-get install -y doxygen librsvg2-bin 41 42 43# ---------------------------------------------------------------------------------------- 44# Test formatting 45# ---------------------------------------------------------------------------------------- 46noether-format: 47 stage: test:stage-lint 48 extends: 49 - .test-basic 50 tags: 51 - noether 52 - shell 53 script: 54 - rm -f .SUCCESS 55 - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version 56 - make -j$NPROC_CPU format && git diff --color=always --exit-code 57 - echo "-------------- make checkbadSource ---------" 58 - make -j$NPROC_CPU checkbadSource 59 - touch .SUCCESS 60 61 62# ---------------------------------------------------------------------------------------- 63# Test static analysis 64# ---------------------------------------------------------------------------------------- 65noether-lint: 66 stage: test:stage-lint 67 extends: .test 68 tags: 69 - noether 70 - shell 71 script: 72 - rm -f .SUCCESS 73 # Environment 74 - export CC=gcc 75 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 76 - echo "-------------- CC ------------------" && $CC --version 77 # Libraries 78 # -- libCEED 79 - echo "-------------- libCEED -------------" 80 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 81 # -- PETSc 82 - echo "-------------- PETSc ---------------" 83 - export PETSC_DIR=/projects/honee/petsc 84 - export PETSC_ARCH=arch-serial-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 85 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 86 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 87 - echo "-------------- HONEE ---------------" && make info 88 # make with Werror, Wall, supress loop vectorization warnings 89 - echo "-------------- make Werror ---------" 90 - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU 91 # Clang-tidy 92 - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version 93 - PETSC_ARCH=arch-serial-cuda make -j$NPROC_CPU tidy 94 # Report status 95 - touch .SUCCESS 96 97 98# ---------------------------------------------------------------------------------------- 99# Test memory access assumptions 100# ---------------------------------------------------------------------------------------- 101noether-memcheck: 102 stage: test:stage-lint 103 extends: .test 104 tags: 105 - noether 106 - shell 107 script: 108 - rm -f .SUCCESS 109 # Environment 110 - export COVERAGE=1 CC=clang-15 111 - export NPROC_POOL=8 112 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 113 - echo "-------------- CC ------------------" && $CC --version 114 # Libraries 115 # -- libCEED 116 - echo "-------------- libCEED -------------" 117 - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 118 # -- PETSc 119 - echo "-------------- PETSc ---------------" 120 - export PETSC_DIR=/projects/honee/petsc 121 - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 122 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 123 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 124 - echo "-------------- HONEE ---------------" && make info 125 # ASAN 126 - echo "-------------- ASAN ----------------" 127 - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" 128 - echo $AFLAGS 129 # HONEE 130 - echo "-------------- HONEE ---------------" && make info 131 - make clean 132 - make -j$NPROC_CPU 133 # Test suite 134 - echo "-------------- HONEE tests ---------" 135 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 136 # -- Memcheck libCEED CPU backend, serial 137 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes 138 # Report status 139 - touch .SUCCESS 140 after_script: 141 - | 142 if [ -f .SUCCESS ]; then 143 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 144 fi 145 artifacts: 146 paths: 147 - coverage-$CI_JOB_ID.json 148 - build/*.junit 149 reports: 150 junit: build/*.junit 151 performance: performance.json 152 expire_in: 28 days 153 154 155# ---------------------------------------------------------------------------------------- 156# CPU testing on Noether 157# ---------------------------------------------------------------------------------------- 158noether-cpu: 159 stage: test:stage-full 160 extends: .test 161 tags: 162 - noether 163 - shell 164 script: 165 - rm -f .SUCCESS 166 # Environment 167 - export COVERAGE=1 CC=gcc 168 - export NPROC_POOL=4 169 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 170 - echo "-------------- CC ------------------" && $CC --version 171 - echo "-------------- GCOV ----------------" && gcov --version 172 # Libraries 173 # -- libCEED 174 - echo "-------------- libCEED -------------" 175 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 176 # -- PETSc 177 - echo "-------------- PETSc ---------------" 178 - export PETSC_DIR=/projects/honee/petsc 179 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 180 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 181 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 182 # HONEE 183 - echo "-------------- HONEE ---------------" && make info 184 - make clean 185 - make -j$NPROC_CPU 186 # Test suite 187 - echo "-------------- HONEE tests ---------" 188 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 189 # -- Fastest libCEED CPU backend, parallel 190 - echo "Parallel tests skipped for now" 191 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 192 - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install 193 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes 194 # - spack unload py-torch@2.3+cuda && export USE_TORCH=0 195 - source /projects/honee/SmartSim/venv/bin/activate 196 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim" 197 # Report status 198 - touch .SUCCESS 199 after_script: 200 - | 201 if [ -f .SUCCESS ]; then 202 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 203 fi 204 artifacts: 205 paths: 206 - coverage-$CI_JOB_ID.json 207 - build/*.junit 208 reports: 209 junit: build/*.junit 210 performance: performance.json 211 expire_in: 28 days 212 213 214# ---------------------------------------------------------------------------------------- 215# CPU Int64 testing on Noether 216# ---------------------------------------------------------------------------------------- 217noether-cpu-int64: 218 stage: test:stage-full 219 extends: .test 220 tags: 221 - noether 222 - shell 223 script: 224 - rm -f .SUCCESS 225 # Environment 226 - export COVERAGE=1 CC=gcc 227 - export NPROC_POOL=4 228 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 229 - echo "-------------- CC ------------------" && $CC --version 230 - echo "-------------- GCOV ----------------" && gcov --version 231 # Libraries 232 # -- libCEED 233 - echo "-------------- libCEED -------------" 234 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 235 # -- PETSc 236 - echo "-------------- PETSc ---------------" 237 - export PETSC_DIR=/projects/honee/petsc 238 - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 239 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 240 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 241 # HONEE 242 - echo "-------------- HONEE ---------------" && make info 243 - make clean 244 - make -j$NPROC_CPU 245 # Test suite 246 - echo "-------------- HONEE tests ---------" 247 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 248 # -- Fastest libCEED CPU backend, serial 249 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 250 - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install 251 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes 252 # - spack unload py-torch@2.3+cuda && export USE_TORCH=0 253 - source /projects/honee/SmartSim/venv/bin/activate 254 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search="test-smartsim" 255 # Report status 256 - touch .SUCCESS 257 after_script: 258 - | 259 if [ -f .SUCCESS ]; then 260 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 261 fi 262 artifacts: 263 paths: 264 - coverage-$CI_JOB_ID.json 265 - build/*.junit 266 reports: 267 junit: build/*.junit 268 performance: performance.json 269 expire_in: 28 days 270 271 272#### Disable HIP temporarily 273 274# # ---------------------------------------------------------------------------------------- 275# # GPU testing on Noether 276# # ---------------------------------------------------------------------------------------- 277# noether-hip: 278# stage: test:stage-full 279# extends: .test 280# tags: 281# - noether 282# - shell 283# script: 284# - rm -f .SUCCESS 285# # Environment 286# - export COVERAGE=1 CC=gcc HIPCC=hipcc 287# - export NPROC_POOL=4 288# - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 289# - echo "-------------- CC ------------------" && $CC --version 290# - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 291# - echo "-------------- GCOV ----------------" && gcov --version 292# # Libraries 293# # -- libCEED 294# - echo "-------------- libCEED -------------" 295# - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 296# # -- PETSc 297# - echo "-------------- PETSc ---------------" 298# - export PETSC_DIR=/projects/honee/petsc 299# - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 300# - export PETSC_OPTIONS='-malloc_debug no' # faster tests 301# - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 302# # HONEE 303# - echo "-------------- HONEE ---------------" && make info 304# - make clean 305# - make -j$NPROC_CPU 306# # Test suite 307# - echo "-------------- HONEE tests ---------" 308# - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 309# # -- Fastest libCEED HIP backend, serial 310# # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 311# - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes 312# # Report status 313# - touch .SUCCESS 314# after_script: 315# - | 316# if [ -f .SUCCESS ]; then 317# gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 318# fi 319# after_script: 320# - | 321# if [ -f .SUCCESS ]; then 322# gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 323# fi 324# artifacts: 325# paths: 326# - coverage-$CI_JOB_ID.json 327# - build/*.junit 328# reports: 329# junit: build/*.junit 330# performance: performance.json 331# expire_in: 28 days 332 333 334noether-cuda: 335 stage: test:stage-full 336 extends: .test 337 tags: 338 - noether 339 - shell 340 script: 341 - rm -f .SUCCESS 342 # Environment 343 - export COVERAGE=1 CC=gcc NVCC=nvcc 344 - export NPROC_POOL=4 345 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 346 - echo "-------------- CC ------------------" && $CC --version 347 - echo "-------------- NVCC ----------------" && $NVCC --version 348 - echo "-------------- GCOV ----------------" && gcov --version 349 # Libraries 350 # -- libCEED 351 - echo "-------------- libCEED -------------" 352 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 353 # -- PETSc 354 - echo "-------------- PETSc ---------------" 355 - export PETSC_DIR=/projects/honee/petsc 356 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 357 - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests 358 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 359 # HONEE 360 - echo "-------------- HONEE ---------------" && make info 361 - make clean 362 - make -j$NPROC_CPU 363 # Test suite 364 - echo "-------------- HONEE tests ---------" 365 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 366 # -- Fastest libCEED CUDA backend, serial 367 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 368 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 369 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes 370 # Report status 371 - touch .SUCCESS 372 after_script: 373 - | 374 if [ -f .SUCCESS ]; then 375 gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json; 376 fi 377 artifacts: 378 paths: 379 - coverage-$CI_JOB_ID.json 380 - build/*.junit 381 reports: 382 junit: build/*.junit 383 performance: performance.json 384 expire_in: 28 days 385 386 387test-coverage: 388 stage: test:post 389 extends: .test 390 tags: 391 - noether 392 - shell 393 when: always 394 needs: [noether-memcheck, noether-cpu, noether-cpu-int64, noether-cuda, ] 395 script: 396 - ls coverage-* 397 - mkdir coverage 398 - gcovr --json-add-tracefile "coverage-*.json" --xml-pretty -o coverage.xml --print-summary --html-nested coverage/index.html --html-theme github.dark-blue 399 coverage: '/^lines:\s+(\d+.\d\%)/' 400 artifacts: 401 paths: 402 - coverage.xml 403 - coverage 404 when: always 405 reports: 406 coverage_report: 407 coverage_format: cobertura 408 path: coverage.xml 409 expire_in: 28 days 410 environment: 411 name: coverage/$CI_COMMIT_REF_NAME 412 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/coverage/index.html 413 414 415# ---------------------------------------------------------------------------------------- 416# Build documentation 417# ---------------------------------------------------------------------------------------- 418docs-review: 419 stage: test:docs 420 tags: 421 - noether 422 - docker 423 extends: 424 - .docs 425 - .test-basic 426 interruptible: true 427 script: 428 - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-cuda CEED_DIR=/projects/honee/libCEED 429 - git -c safe.directory=/builds/phypid/honee submodule update --init 430 - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W 431 - mv doc/build/html public 432 artifacts: 433 paths: 434 - public 435 expire_in: 28 days 436 environment: 437 name: review/$CI_COMMIT_REF_NAME 438 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html 439 440 441# ---------------------------------------------------------------------------------------- 442# Deploy documentation using GitLab pages 443# ---------------------------------------------------------------------------------------- 444pages: # this job name has special meaning to GitLab 445 stage: deploy 446 tags: 447 - noether 448 - docker 449 extends: .docs 450 interruptible: false 451 script: 452 - git -c safe.directory=/builds/phypid/honee submodule update --init 453 - make doc-dirhtml pkgconf=true DOXYGENOPTS= 454 - mv doc/build/dirhtml public 455 only: 456 - main 457 artifacts: 458 paths: 459 - public 460