1# ---------------------------------------------------------------------------------------- 2# HONEE GitLab CI 3# ---------------------------------------------------------------------------------------- 4stages: 5 - test:stage-lint 6 - test:stage-full 7 - test:docs 8 - deploy 9 10workflow: 11 auto_cancel: 12 on_job_failure: all 13 14.test-basic: 15 interruptible: true 16 only: 17 refs: 18 - web 19 - merge_requests 20 21.test: 22 extends: .test-basic 23 only: 24 refs: 25 - web 26 - merge_requests 27 - main 28 - release 29 except: 30 variables: 31 # Skip if the No-Code label is attached to a merge request (i.e., documentation only) 32 - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/ 33 34.docs: 35 image: python:3.10 36 before_script: 37 - pip install -r doc/requirements.txt 38 - apt-get update 39 - apt-get install -y doxygen librsvg2-bin 40 41 42# ---------------------------------------------------------------------------------------- 43# Test formatting and static analysis 44# ---------------------------------------------------------------------------------------- 45noether-lint: 46 stage: test:stage-lint 47 extends: .test 48 tags: 49 - noether 50 - shell 51 script: 52 - rm -f .SUCCESS 53 # Environment 54 - export CC=gcc HIPCC=hipcc 55 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 56 - echo "-------------- CC ------------------" && $CC --version 57 - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 58 # Libraries 59 # -- libCEED 60 - echo "-------------- libCEED -------------" 61 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 62 # -- PETSc 63 - echo "-------------- PETSc ---------------" 64 - export PETSC_DIR=/projects/honee/petsc 65 - export PETSC_ARCH=arch-serial-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 66 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 67 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 68 # make with Werror, Wall, supress loop vectorization warnings 69 - echo "-------------- make Werror ---------" 70 - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU 71 # make format 72 - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version 73 - make -j$NPROC_CPU format && git diff --color=always --exit-code 74 # Clang-tidy 75 - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version 76 - PETSC_ARCH=arch-serial-hip make -j$NPROC_CPU tidy 77 # Report status 78 - touch .SUCCESS 79 80 81# ---------------------------------------------------------------------------------------- 82# Test memory access assumptions 83# ---------------------------------------------------------------------------------------- 84noether-memcheck: 85 stage: test:stage-lint 86 extends: .test 87 tags: 88 - noether 89 - shell 90 script: 91 - rm -f .SUCCESS 92 # Environment 93 # -- NOTE: Coverage disabled because it doesn't play nice with the ASAN options 94 - export CC=clang-15 95 - export NPROC_POOL=8 96 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 97 - echo "-------------- CC ------------------" && $CC --version 98 # Libraries 99 # -- libCEED 100 - echo "-------------- libCEED -------------" 101 - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 102 # -- PETSc 103 - echo "-------------- PETSc ---------------" 104 - export PETSC_DIR=/projects/honee/petsc 105 - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 106 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 107 - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff 108 # ASAN 109 - echo "-------------- ASAN ----------------" 110 - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" 111 - echo $AFLAGS 112 # HONEE 113 - echo "-------------- HONEE ---------------" && make info 114 - make clean 115 - make -j$NPROC_CPU 116 # Test suite 117 - echo "-------------- HONEE tests ---------" 118 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 119 # -- Memcheck libCEED CPU backend, serial 120 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes 121 # Report status 122 - touch .SUCCESS 123 artifacts: 124 paths: 125 - build/*.junit 126 reports: 127 junit: build/*.junit 128 performance: performance.json 129 expire_in: 28 days 130 131 132# ---------------------------------------------------------------------------------------- 133# CPU testing on Noether 134# ---------------------------------------------------------------------------------------- 135noether-cpu: 136 stage: test:stage-full 137 extends: .test 138 tags: 139 - noether 140 - shell 141 script: 142 - rm -f .SUCCESS 143 # Environment 144 - export COVERAGE=1 CC=gcc 145 - export NPROC_POOL=4 146 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 147 - echo "-------------- CC ------------------" && $CC --version 148 - echo "-------------- GCOV ----------------" && gcov --version 149 # Libraries 150 # -- libCEED 151 - echo "-------------- libCEED -------------" 152 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 153 # -- PETSc 154 - echo "-------------- PETSc ---------------" 155 - export PETSC_DIR=/projects/honee/petsc 156 - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 157 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 158 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 159 # HONEE 160 - echo "-------------- HONEE ---------------" && make info 161 - make clean 162 - make -j$NPROC_CPU 163 # Test suite 164 - echo "-------------- HONEE tests ---------" 165 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 166 # -- Fastest libCEED CPU backend, parallel 167 - echo "Parallel tests skipped for now" 168 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 169 - export SMARTREDIS_DIR=/home/phypid/SmartSimTestingSoftware/smartredis/install 170 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes 171 - spack unload py-torch@2.3+cuda && export USE_TORCH=0 172 - source /home/phypid/SmartSimTestingSoftware/bin/activate 173 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim" 174 # Report status 175 - touch .SUCCESS 176 after_script: 177 - | 178 if [ -f .SUCCESS ]; then 179 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 180 fi 181 coverage: '/^lines:\s+(\d+.\d\%)/' 182 artifacts: 183 paths: 184 - coverage.xml 185 - build/*.junit 186 reports: 187 coverage_report: 188 coverage_format: cobertura 189 path: coverage.xml 190 junit: build/*.junit 191 performance: performance.json 192 expire_in: 28 days 193 194 195# ---------------------------------------------------------------------------------------- 196# CPU Int64 testing on Noether 197# ---------------------------------------------------------------------------------------- 198noether-cpu-int64: 199 stage: test:stage-full 200 extends: .test 201 tags: 202 - noether 203 - shell 204 script: 205 - rm -f .SUCCESS 206 # Environment 207 - export COVERAGE=1 CC=gcc 208 - export NPROC_POOL=4 209 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 210 - echo "-------------- CC ------------------" && $CC --version 211 - echo "-------------- GCOV ----------------" && gcov --version 212 # Libraries 213 # -- libCEED 214 - echo "-------------- libCEED -------------" 215 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 216 # -- PETSc 217 - echo "-------------- PETSc ---------------" 218 - export PETSC_DIR=/projects/honee/petsc 219 - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 220 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 221 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 222 # HONEE 223 - echo "-------------- HONEE ---------------" && make info 224 - make clean 225 - make -j$NPROC_CPU 226 # Test suite 227 - echo "-------------- HONEE tests ---------" 228 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 229 # -- Fastest libCEED CPU backend, serial 230 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 231 - export SMARTREDIS_DIR=/home/phypid/SmartSimTestingSoftware/smartredis/install 232 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes 233 - spack unload py-torch@2.3+cuda && export USE_TORCH=0 234 - source /home/phypid/SmartSimTestingSoftware/bin/activate 235 - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-py-smartsim_regression_framework" 236 # Report status 237 - touch .SUCCESS 238 after_script: 239 - | 240 if [ -f .SUCCESS ]; then 241 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 242 fi 243 coverage: '/^lines:\s+(\d+.\d\%)/' 244 artifacts: 245 paths: 246 - coverage.xml 247 - build/*.junit 248 reports: 249 coverage_report: 250 coverage_format: cobertura 251 path: coverage.xml 252 junit: build/*.junit 253 performance: performance.json 254 expire_in: 28 days 255 256 257# ---------------------------------------------------------------------------------------- 258# GPU testing on Noether 259# ---------------------------------------------------------------------------------------- 260noether-hip: 261 stage: test:stage-full 262 extends: .test 263 tags: 264 - noether 265 - shell 266 script: 267 - rm -f .SUCCESS 268 # Environment 269 - export COVERAGE=1 CC=gcc HIPCC=hipcc 270 - export NPROC_POOL=4 271 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 272 - echo "-------------- CC ------------------" && $CC --version 273 - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm 274 - echo "-------------- GCOV ----------------" && gcov --version 275 # Libraries 276 # -- libCEED 277 - echo "-------------- libCEED -------------" 278 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 279 # -- PETSc 280 - echo "-------------- PETSc ---------------" 281 - export PETSC_DIR=/projects/honee/petsc 282 - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 283 - export PETSC_OPTIONS='-malloc_debug no' # faster tests 284 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 285 # HONEE 286 - echo "-------------- HONEE ---------------" && make info 287 - make clean 288 - make -j$NPROC_CPU 289 # Test suite 290 - echo "-------------- HONEE tests ---------" 291 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 292 # -- Fastest libCEED HIP backend, serial 293 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 294 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes 295 # Report status 296 - touch .SUCCESS 297 after_script: 298 - | 299 if [ -f .SUCCESS ]; then 300 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 301 fi 302 coverage: '/^lines:\s+(\d+.\d\%)/' 303 artifacts: 304 paths: 305 - coverage.xml 306 - build/*.junit 307 reports: 308 coverage_report: 309 coverage_format: cobertura 310 path: coverage.xml 311 junit: build/*.junit 312 performance: performance.json 313 expire_in: 28 days 314 315 316noether-cuda: 317 stage: test:stage-full 318 extends: .test 319 tags: 320 - noether 321 - shell 322 script: 323 - rm -f .SUCCESS 324 # Environment 325 - export COVERAGE=1 CC=gcc NVCC=nvcc 326 - export NPROC_POOL=4 327 - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU 328 - echo "-------------- CC ------------------" && $CC --version 329 - echo "-------------- NVCC ----------------" && $NVCC --version 330 - echo "-------------- GCOV ----------------" && gcov --version 331 # Libraries 332 # -- libCEED 333 - echo "-------------- libCEED -------------" 334 - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info 335 # -- PETSc 336 - echo "-------------- PETSc ---------------" 337 - export PETSC_DIR=/projects/honee/petsc 338 - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info 339 - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests 340 - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff 341 # HONEE 342 - echo "-------------- HONEE ---------------" && make info 343 - make clean 344 - make -j$NPROC_CPU 345 # Test suite 346 - echo "-------------- HONEE tests ---------" 347 - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json 348 # -- Fastest libCEED CUDA backend, serial 349 # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes 350 - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1 351 - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes 352 # Report status 353 - touch .SUCCESS 354 after_script: 355 - | 356 if [ -f .SUCCESS ]; then 357 gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml; 358 fi 359 coverage: '/^lines:\s+(\d+.\d\%)/' 360 artifacts: 361 paths: 362 - coverage.xml 363 - build/*.junit 364 reports: 365 coverage_report: 366 coverage_format: cobertura 367 path: coverage.xml 368 junit: build/*.junit 369 performance: performance.json 370 expire_in: 28 days 371 372 373# ---------------------------------------------------------------------------------------- 374# Build documentation 375# ---------------------------------------------------------------------------------------- 376docs-review: 377 stage: test:docs 378 tags: 379 - noether 380 - docker 381 extends: 382 - .docs 383 - .test-basic 384 interruptible: true 385 script: 386 - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-hip CEED_DIR=/projects/honee/libCEED 387 - git -c safe.directory=/builds/phypid/honee submodule update --init 388 - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W 389 - mv doc/build/html public 390 artifacts: 391 paths: 392 - public 393 expire_in: 28 days 394 environment: 395 name: review/$CI_COMMIT_REF_NAME 396 url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html 397 398 399# ---------------------------------------------------------------------------------------- 400# Deploy documentation using GitLab pages 401# ---------------------------------------------------------------------------------------- 402pages: # this job name has special meaning to GitLab 403 stage: deploy 404 tags: 405 - noether 406 - docker 407 extends: .docs 408 interruptible: false 409 script: 410 - git -c safe.directory=/builds/phypid/honee submodule update --init 411 - make doc-dirhtml pkgconf=true DOXYGENOPTS= 412 - mv doc/build/dirhtml public 413 only: 414 - main 415 artifacts: 416 paths: 417 - public 418