xref: /honee/.gitlab-ci.yml (revision 39a30fb814eccf34da0ac75a52572d349fee38a7) 
1# ----------------------------------------------------------------------------------------
2# HONEE GitLab CI
3# ----------------------------------------------------------------------------------------
4stages:
5  - test:stage-lint
6  - test:stage-full
7  - test:docs
8  - deploy
9
10workflow:
11  auto_cancel:
12    on_job_failure: all
13
14.test-basic:
15  interruptible: true
16  only:
17    refs:
18      - web
19      - merge_requests
20
21.test:
22  extends: .test-basic
23  only:
24    refs:
25      - web
26      - merge_requests
27      - main
28      - release
29  except:
30    variables:
31      # Skip if the No-Code label is attached to a merge request (i.e., documentation only)
32      - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/
33
34.docs:
35  image: python:3.10
36  before_script:
37    - pip install -r doc/requirements.txt
38    - apt-get update
39    - apt-get install -y doxygen librsvg2-bin
40
41
42# ----------------------------------------------------------------------------------------
43# Test formatting and static analysis
44# ----------------------------------------------------------------------------------------
45noether-lint:
46  stage: test:stage-lint
47  extends: .test
48  tags:
49    - noether
50    - shell
51  script:
52    - rm -f .SUCCESS
53    # Environment
54    - export CC=gcc HIPCC=hipcc
55    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
56    - echo "-------------- CC ------------------" && $CC --version
57    - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm
58    # Libraries
59    # -- libCEED
60    - echo "-------------- libCEED -------------"
61    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
62    # -- PETSc
63    - echo "-------------- PETSc ---------------"
64    - export PETSC_DIR=/projects/honee/petsc
65    - export PETSC_ARCH=arch-serial-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
66    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
67    - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff
68    # make with Werror, Wall, supress loop vectorization warnings
69    - echo "-------------- make Werror ---------"
70    - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU
71    # make format
72    - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version
73    - make -j$NPROC_CPU format && git diff --color=always --exit-code
74    # Clang-tidy
75    - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version
76    - PETSC_ARCH=arch-serial-hip make -j$NPROC_CPU tidy
77    # Report status
78    - touch .SUCCESS
79
80
81# ----------------------------------------------------------------------------------------
82# Test memory access assumptions
83# ----------------------------------------------------------------------------------------
84noether-memcheck:
85  stage: test:stage-lint
86  extends: .test
87  tags:
88    - noether
89    - shell
90  script:
91    - rm -f .SUCCESS
92    # Environment
93    # -- NOTE: Coverage disabled because it doesn't play nice with the ASAN options
94    - export CC=clang-15
95    - export NPROC_POOL=8
96    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
97    - echo "-------------- CC ------------------" && $CC --version
98    # Libraries
99    # -- libCEED
100    - echo "-------------- libCEED -------------"
101    - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
102    # -- PETSc
103    - echo "-------------- PETSc ---------------"
104    - export PETSC_DIR=/projects/honee/petsc
105    - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
106    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
107    - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff
108    # ASAN
109    - echo "-------------- ASAN ----------------"
110    - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak"
111    - echo $AFLAGS
112    # HONEE
113    - echo "-------------- HONEE ---------------" && make info
114    - make clean
115    - make -j$NPROC_CPU
116    # Test suite
117    - echo "-------------- HONEE tests ---------"
118    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
119    # -- Memcheck libCEED CPU backend, serial
120    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes
121    # Report status
122    - touch .SUCCESS
123  artifacts:
124    paths:
125      - build/*.junit
126    reports:
127      junit: build/*.junit
128      performance: performance.json
129    expire_in: 28 days
130
131
132# ----------------------------------------------------------------------------------------
133# CPU testing on Noether
134# ----------------------------------------------------------------------------------------
135noether-cpu:
136  stage: test:stage-full
137  extends: .test
138  tags:
139    - noether
140    - shell
141  script:
142    - rm -f .SUCCESS
143    # Environment
144    - export COVERAGE=1 CC=gcc
145    - export NPROC_POOL=4
146    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
147    - echo "-------------- CC ------------------" && $CC --version
148    - echo "-------------- GCOV ----------------" && gcov --version
149    # Libraries
150    # -- libCEED
151    - echo "-------------- libCEED -------------"
152    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
153    # -- PETSc
154    - echo "-------------- PETSc ---------------"
155    - export PETSC_DIR=/projects/honee/petsc
156    - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
157    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
158    - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff
159    # HONEE
160    - echo "-------------- HONEE ---------------" && make info
161    - make clean
162    - make -j$NPROC_CPU
163    # Test suite
164    - echo "-------------- HONEE tests ---------"
165    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
166    # -- Fastest libCEED CPU backend, parallel
167    - echo "Parallel tests skipped for now"
168    - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1
169    - export SMARTREDIS_DIR=/home/phypid/SmartSimTestingSoftware/smartredis/install
170    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes
171    - spack unload py-torch@2.3+cuda && export USE_TORCH=0
172    - source /home/phypid/SmartSimTestingSoftware/bin/activate
173    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim"
174    # Report status
175    - touch .SUCCESS
176  after_script:
177    - |
178      if [ -f .SUCCESS ]; then
179        gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml;
180      fi
181  coverage: '/^lines:\s+(\d+.\d\%)/'
182  artifacts:
183    paths:
184      - coverage.xml
185      - build/*.junit
186    reports:
187      coverage_report:
188        coverage_format: cobertura
189        path: coverage.xml
190      junit: build/*.junit
191      performance: performance.json
192    expire_in: 28 days
193
194
195# ----------------------------------------------------------------------------------------
196# CPU Int64 testing on Noether
197# ----------------------------------------------------------------------------------------
198noether-cpu-int64:
199  stage: test:stage-full
200  extends: .test
201  tags:
202    - noether
203    - shell
204  script:
205    - rm -f .SUCCESS
206    # Environment
207    - export COVERAGE=1 CC=gcc
208    - export NPROC_POOL=4
209    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
210    - echo "-------------- CC ------------------" && $CC --version
211    - echo "-------------- GCOV ----------------" && gcov --version
212    # Libraries
213    # -- libCEED
214    - echo "-------------- libCEED -------------"
215    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
216    # -- PETSc
217    - echo "-------------- PETSc ---------------"
218    - export PETSC_DIR=/projects/honee/petsc
219    - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
220    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
221    - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff
222    # HONEE
223    - echo "-------------- HONEE ---------------" && make info
224    - make clean
225    - make -j$NPROC_CPU
226    # Test suite
227    - echo "-------------- HONEE tests ---------"
228    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
229    # -- Fastest libCEED CPU backend, serial
230    - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1
231    - export SMARTREDIS_DIR=/home/phypid/SmartSimTestingSoftware/smartredis/install
232    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes
233    - spack unload py-torch@2.3+cuda && export USE_TORCH=0
234    - source /home/phypid/SmartSimTestingSoftware/bin/activate
235    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-py-smartsim_regression_framework"
236    # Report status
237    - touch .SUCCESS
238  after_script:
239    - |
240      if [ -f .SUCCESS ]; then
241        gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml;
242      fi
243  coverage: '/^lines:\s+(\d+.\d\%)/'
244  artifacts:
245    paths:
246      - coverage.xml
247      - build/*.junit
248    reports:
249      coverage_report:
250        coverage_format: cobertura
251        path: coverage.xml
252      junit: build/*.junit
253      performance: performance.json
254    expire_in: 28 days
255
256
257# ----------------------------------------------------------------------------------------
258# GPU testing on Noether
259# ----------------------------------------------------------------------------------------
260noether-hip:
261  stage: test:stage-full
262  extends: .test
263  tags:
264    - noether
265    - shell
266  script:
267    - rm -f .SUCCESS
268    # Environment
269    - export COVERAGE=1 CC=gcc HIPCC=hipcc
270    - export NPROC_POOL=4
271    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
272    - echo "-------------- CC ------------------" && $CC --version
273    - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm
274    - echo "-------------- GCOV ----------------" && gcov --version
275    # Libraries
276    # -- libCEED
277    - echo "-------------- libCEED -------------"
278    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
279    # -- PETSc
280    - echo "-------------- PETSc ---------------"
281    - export PETSC_DIR=/projects/honee/petsc
282    - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
283    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
284    - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff
285    # HONEE
286    - echo "-------------- HONEE ---------------" && make info
287    - make clean
288    - make -j$NPROC_CPU
289    # Test suite
290    - echo "-------------- HONEE tests ---------"
291    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
292    # -- Fastest libCEED HIP backend, serial
293    # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes
294    - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes
295    # Report status
296    - touch .SUCCESS
297  after_script:
298    - |
299      if [ -f .SUCCESS ]; then
300        gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml;
301      fi
302  coverage: '/^lines:\s+(\d+.\d\%)/'
303  artifacts:
304    paths:
305      - coverage.xml
306      - build/*.junit
307    reports:
308      coverage_report:
309        coverage_format: cobertura
310        path: coverage.xml
311      junit: build/*.junit
312      performance: performance.json
313    expire_in: 28 days
314
315
316noether-cuda:
317  stage: test:stage-full
318  extends: .test
319  tags:
320    - noether
321    - shell
322  script:
323    - rm -f .SUCCESS
324    # Environment
325    - export COVERAGE=1 CC=gcc NVCC=nvcc
326    - export NPROC_POOL=4
327    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
328    - echo "-------------- CC ------------------" && $CC --version
329    - echo "-------------- NVCC ----------------" && $NVCC --version
330    - echo "-------------- GCOV ----------------" && gcov --version
331    # Libraries
332    # -- libCEED
333    - echo "-------------- libCEED -------------"
334    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
335    # -- PETSc
336    - echo "-------------- PETSc ---------------"
337    - export PETSC_DIR=/projects/honee/petsc
338    - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
339    - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests
340    - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff
341    # HONEE
342    - echo "-------------- HONEE ---------------" && make info
343    - make clean
344    - make -j$NPROC_CPU
345    # Test suite
346    - echo "-------------- HONEE tests ---------"
347    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
348    # -- Fastest libCEED CUDA backend, serial
349    # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes
350    - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1
351    - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes
352    # Report status
353    - touch .SUCCESS
354  after_script:
355    - |
356      if [ -f .SUCCESS ]; then
357        gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml;
358      fi
359  coverage: '/^lines:\s+(\d+.\d\%)/'
360  artifacts:
361    paths:
362      - coverage.xml
363      - build/*.junit
364    reports:
365      coverage_report:
366        coverage_format: cobertura
367        path: coverage.xml
368      junit: build/*.junit
369      performance: performance.json
370    expire_in: 28 days
371
372
373# ----------------------------------------------------------------------------------------
374# Build documentation
375# ----------------------------------------------------------------------------------------
376docs-review:
377  stage: test:docs
378  tags:
379    - noether
380    - docker
381  extends:
382    - .docs
383    - .test-basic
384  interruptible: true
385  script:
386    - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-hip CEED_DIR=/projects/honee/libCEED
387    - git -c safe.directory=/builds/phypid/honee submodule update --init
388    - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W
389    - mv doc/build/html public
390  artifacts:
391    paths:
392      - public
393    expire_in: 28 days
394  environment:
395    name: review/$CI_COMMIT_REF_NAME
396    url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html
397
398
399# ----------------------------------------------------------------------------------------
400# Deploy documentation using GitLab pages
401# ----------------------------------------------------------------------------------------
402pages:  # this job name has special meaning to GitLab
403  stage: deploy
404  tags:
405    - noether
406    - docker
407  extends: .docs
408  interruptible: false
409  script:
410    - git -c safe.directory=/builds/phypid/honee submodule update --init
411    - make doc-dirhtml pkgconf=true DOXYGENOPTS=
412    - mv doc/build/dirhtml public
413  only:
414    - main
415  artifacts:
416    paths:
417      - public
418