1# ---------------------------------------------------------------------------------------- 2# HONEE GitLab CI 3# ---------------------------------------------------------------------------------------- 4stages: 5 - test:stage-lint 6 - test:stage-full 7 - test:docs 8 - deploy 9 10workflow: 11 auto_cancel: 12 on_job_failure: all 13 14.test-basic: 15 interruptible: true 16 only: 17 refs: 18 - web 19 - merge_requests 20 21.test: 22 extends: .test-basic 23 only: 24 refs: 25 - web 26 - merge_requests 27 - main 28 - release 29 except: 30 variables: 31 # Skip if the No-Code label is attached to a merge request (i.e., documentation only) 32 - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/ 33 34.docs: 35 image: python:3.10 36 before_script: 37 - pip install -r doc/requirements.txt 38 - apt-get update 39 - apt-get install -y doxygen librsvg2-bin 40 41 42# ---------------------------------------------------------------------------------------- 43# Test formatting 44# ---------------------------------------------------------------------------------------- 45noether-format: 46 stage: test:stage-lint 47 extends: 48 - .test-basic 49 tags: 50 - noether 51 - shell 52 script: 53 - rm -f .SUCCESS 54 - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version 55 - make -j$NPROC_CPU format && git diff --color=always --exit-code 56 - touch .SUCCESS 57 58 59# ---------------------------------------------------------------------------------------- 60# Test static analysis 61# ---------------------------------------------------------------------------------------- 62noether-lint: 63 stage: test:stage-lint 64 extends: .test 65 tags: 66 - noether 67 - shell 68 script: 69 - rm -f .SUCCESS 70 # Environment 71 - export CC=gcc 72 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 73 - echo "-------------- CC ------------------" && $CC --version 74 # Libraries 75 # -- libCEED 76 - echo "-------------- libCEED -------------" 77 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 78 # -- PETSc 79 - echo "-------------- PETSc ---------------" 80 - export PETSC_DIR=/projects/honee/petsc 81 - export PETSC_ARCH=arch-serial-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 82 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 83 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 84 # make with Werror, Wall, supress loop vectorization warnings 85 - echo "-------------- make Werror ---------" 86 - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU 87 # Clang-tidy 88 - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version 89 - PETSC_ARCH=arch-serial-cuda make -j$NPROC_CPU tidy 90 # Report status 91 - touch .SUCCESS 92 93 94# ---------------------------------------------------------------------------------------- 95# Test memory access assumptions 96# ---------------------------------------------------------------------------------------- 97noether-memcheck: 98 stage: test:stage-lint 99 extends: .test 100 tags: 101 - noether 102 - shell 103 script: 104 - rm -f .SUCCESS 105 # Environment 106 # -- NOTE: Coverage disabled because it doesn't play nice with the ASAN options 107 - export CC=clang-15 108 - export NPROC_POOL=8 109 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 110 - echo "-------------- CC ------------------" && $CC --version 111 # Libraries 112 # -- libCEED 113 - echo "-------------- libCEED -------------" 114 - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 115 # -- PETSc 116 - echo "-------------- PETSc ---------------" 117 - export PETSC_DIR=/projects/honee/petsc 118 - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 119 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 120 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 121 # ASAN 122 - echo "-------------- ASAN ----------------" 123 - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" 124 - echo $AFLAGS 125 # HONEE 126 - echo "-------------- HONEE ---------------" && make info 127 - make clean 128 - make -j$NPROC_CPU 129 # Test suite 130 - echo "-------------- HONEE tests ---------" 131 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 132 # -- Memcheck libCEED CPU backend, serial 133 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes 134 # Report status 135 - touch .SUCCESS 136 artifacts: 137 paths: 138 - build/*.junit 139 reports: 140 junit: build/*.junit 141 performance: performance.json 142 expire_in: 28 days 143 144 145# ---------------------------------------------------------------------------------------- 146# CPU testing on Noether 147# ---------------------------------------------------------------------------------------- 148noether-cpu: 149 stage: test:stage-full 150 extends: .test 151 tags: 152 - noether 153 - shell 154 script: 155 - rm -f .SUCCESS 156 # Environment 157 - export COVERAGE=1 CC=gcc 158 - export NPROC_POOL=4 159 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 160 - echo "-------------- CC ------------------" && $CC --version 161 - echo "-------------- GCOV ----------------" && gcov --version 162 # Libraries 163 # -- libCEED 164 - echo "-------------- libCEED -------------" 165 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 166 # -- PETSc 167 - echo "-------------- PETSc ---------------" 168 - export PETSC_DIR=/projects/honee/petsc 169 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 170 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 171 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 172 # HONEE 173 - echo "-------------- HONEE ---------------" && make info 174 - make clean 175 - make -j$NPROC_CPU 176 # Test suite 177 - echo "-------------- HONEE tests ---------" 178 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 179 # -- Fastest libCEED CPU backend, parallel 180 - echo "Parallel tests skipped for now" 181 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 182 - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install 183 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes 184 # - spack unload py-torch@2.3+cuda && export USE_TORCH=0 185 - source /projects/honee/SmartSim/venv/bin/activate 186 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim" 187 # Report status 188 - touch .SUCCESS 189 after_script: 190 - | 191 if [ -f .SUCCESS ]; then 192 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 193 fi 194 coverage: '/^lines:\s+(\d+.\d\%)/' 195 artifacts: 196 paths: 197 - coverage.xml 198 - build/*.junit 199 reports: 200 coverage_report: 201 coverage_format: cobertura 202 path: coverage.xml 203 junit: build/*.junit 204 performance: performance.json 205 expire_in: 28 days 206 207 208# ---------------------------------------------------------------------------------------- 209# CPU Int64 testing on Noether 210# ---------------------------------------------------------------------------------------- 211noether-cpu-int64: 212 stage: test:stage-full 213 extends: .test 214 tags: 215 - noether 216 - shell 217 script: 218 - rm -f .SUCCESS 219 # Environment 220 - export COVERAGE=1 CC=gcc 221 - export NPROC_POOL=4 222 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 223 - echo "-------------- CC ------------------" && $CC --version 224 - echo "-------------- GCOV ----------------" && gcov --version 225 # Libraries 226 # -- libCEED 227 - echo "-------------- libCEED -------------" 228 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 229 # -- PETSc 230 - echo "-------------- PETSc ---------------" 231 - export PETSC_DIR=/projects/honee/petsc 232 - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 233 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 234 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 235 # HONEE 236 - echo "-------------- HONEE ---------------" && make info 237 - make clean 238 - make -j$NPROC_CPU 239 # Test suite 240 - echo "-------------- HONEE tests ---------" 241 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 242 # -- Fastest libCEED CPU backend, serial 243 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 244 - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install 245 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes 246 # - spack unload py-torch@2.3+cuda && export USE_TORCH=0 247 - source /home/phypid/SmartSimTestingSoftware/bin/activate 248 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-py-smartsim_regression_framework" 249 # Report status 250 - touch .SUCCESS 251 after_script: 252 - | 253 if [ -f .SUCCESS ]; then 254 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 255 fi 256 coverage: '/^lines:\s+(\d+.\d\%)/' 257 artifacts: 258 paths: 259 - coverage.xml 260 - build/*.junit 261 reports: 262 coverage_report: 263 coverage_format: cobertura 264 path: coverage.xml 265 junit: build/*.junit 266 performance: performance.json 267 expire_in: 28 days 268 269 270#### Disable HIP temporarily 271 272# # ---------------------------------------------------------------------------------------- 273# # GPU testing on Noether 274# # ---------------------------------------------------------------------------------------- 275# noether-hip: 276# stage: test:stage-full 277# extends: .test 278# tags: 279# - noether 280# - shell 281# script: 282# - rm -f .SUCCESS 283# # Environment 284# - export COVERAGE=1 CC=gcc HIPCC=hipcc 285# - export NPROC_POOL=4 286# - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 287# - echo "-------------- CC ------------------" && $CC --version 288# - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 289# - echo "-------------- GCOV ----------------" && gcov --version 290# # Libraries 291# # -- libCEED 292# - echo "-------------- libCEED -------------" 293# - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 294# # -- PETSc 295# - echo "-------------- PETSc ---------------" 296# - export PETSC_DIR=/projects/honee/petsc 297# - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 298# - export PETSC_OPTIONS='-malloc_debug no' # faster tests 299# - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 300# # HONEE 301# - echo "-------------- HONEE ---------------" && make info 302# - make clean 303# - make -j$NPROC_CPU 304# # Test suite 305# - echo "-------------- HONEE tests ---------" 306# - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 307# # -- Fastest libCEED HIP backend, serial 308# # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 309# - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes 310# # Report status 311# - touch .SUCCESS 312# after_script: 313# - | 314# if [ -f .SUCCESS ]; then 315# gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 316# fi 317# coverage: '/^lines:\s+(\d+.\d\%)/' 318# artifacts: 319# paths: 320# - coverage.xml 321# - build/*.junit 322# reports: 323# coverage_report: 324# coverage_format: cobertura 325# path: coverage.xml 326# junit: build/*.junit 327# performance: performance.json 328# expire_in: 28 days 329 330 331noether-cuda: 332 stage: test:stage-full 333 extends: .test 334 tags: 335 - noether 336 - shell 337 script: 338 - rm -f .SUCCESS 339 # Environment 340 - export COVERAGE=1 CC=gcc NVCC=nvcc 341 - export NPROC_POOL=4 342 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 343 - echo "-------------- CC ------------------" && $CC --version 344 - echo "-------------- NVCC ----------------" && $NVCC --version 345 - echo "-------------- GCOV ----------------" && gcov --version 346 # Libraries 347 # -- libCEED 348 - echo "-------------- libCEED -------------" 349 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 350 # -- PETSc 351 - echo "-------------- PETSc ---------------" 352 - export PETSC_DIR=/projects/honee/petsc 353 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 354 - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests 355 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 356 # HONEE 357 - echo "-------------- HONEE ---------------" && make info 358 - make clean 359 - make -j$NPROC_CPU 360 # Test suite 361 - echo "-------------- HONEE tests ---------" 362 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 363 # -- Fastest libCEED CUDA backend, serial 364 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 365 # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 366 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes 367 # Report status 368 - touch .SUCCESS 369 after_script: 370 - | 371 if [ -f .SUCCESS ]; then 372 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 373 fi 374 coverage: '/^lines:\s+(\d+.\d\%)/' 375 artifacts: 376 paths: 377 - coverage.xml 378 - build/*.junit 379 reports: 380 coverage_report: 381 coverage_format: cobertura 382 path: coverage.xml 383 junit: build/*.junit 384 performance: performance.json 385 expire_in: 28 days 386 387 388# ---------------------------------------------------------------------------------------- 389# Build documentation 390# ---------------------------------------------------------------------------------------- 391docs-review: 392 stage: test:docs 393 tags: 394 - noether 395 - docker 396 extends: 397 - .docs 398 - .test-basic 399 interruptible: true 400 script: 401 - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-cuda CEED_DIR=/projects/honee/libCEED 402 - git -c safe.directory=/builds/phypid/honee submodule update --init 403 - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W 404 - mv doc/build/html public 405 artifacts: 406 paths: 407 - public 408 expire_in: 28 days 409 environment: 410 name: review/$CI_COMMIT_REF_NAME 411 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html 412 413 414# ---------------------------------------------------------------------------------------- 415# Deploy documentation using GitLab pages 416# ---------------------------------------------------------------------------------------- 417pages: # this job name has special meaning to GitLab 418 stage: deploy 419 tags: 420 - noether 421 - docker 422 extends: .docs 423 interruptible: false 424 script: 425 - git -c safe.directory=/builds/phypid/honee submodule update --init 426 - make doc-dirhtml pkgconf=true DOXYGENOPTS= 427 - mv doc/build/dirhtml public 428 only: 429 - main 430 artifacts: 431 paths: 432 - public 433