xref: /libCEED/.gitlab-ci.yml (revision bdd929d2535f59f3fa3773fee8d7c44fbe1c8ed5) 
1# ----------------------------------------------------------------------------------------
2# libCEED GitLab CI
3# ----------------------------------------------------------------------------------------
4stages:
5  - test:cpu-and-tidy
6  - test:gpu-and-float
7
8workflow:
9  auto_cancel:
10    on_job_failure: all
11
12
13# ----------------------------------------------------------------------------------------
14# Memcheck backends + ASAN
15# ----------------------------------------------------------------------------------------
16noether-asan:
17  stage: test:cpu-and-tidy
18  tags:
19    - cpu
20  interruptible: true
21  before_script:
22    # Environment
23    #    Note: COVERAGE=0 is needed when using ASAN
24    - export COVERAGE=0 CC=gcc CXX=g++ FC=gfortran
25    - export NPROC_POOL=8
26    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
27    - echo "-------------- CC ------------------" && $CC --version
28    - echo "-------------- CXX -----------------" && $CXX --version
29    - echo "-------------- FC ------------------" && $FC --version
30    # ASAN
31    - echo "-------------- ASAN ----------------"
32    - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak"
33    - echo $AFLAGS
34  script:
35    - rm -f .SUCCESS
36    # libCEED
37    - make configure OPT='-O -march=native -ffp-contract=fast'
38    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
39    - echo "-------------- libCEED -------------" && make info
40    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
41    - make clean
42    - make -j$NPROC_CPU
43    # -- libCEED only tests
44    - echo "-------------- core tests ----------"
45    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
46    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
47    - export PETSC_DIR= PETSC_ARCH=
48    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="memcheck" junit realsearch=%
49    # Clang-tidy
50    - echo "-------------- clang-tidy ----------" && clang-tidy --version
51    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
52    # Report status
53    - touch .SUCCESS
54  artifacts:
55    paths:
56      - build/*.junit
57    reports:
58      junit: build/*.junit
59      performance: performance.json
60
61
62# ----------------------------------------------------------------------------------------
63# CPU backends only + make tidy
64# ----------------------------------------------------------------------------------------
65noether-cpu:
66  stage: test:cpu-and-tidy
67  tags:
68    - cpu
69  interruptible: true
70  before_script:
71    # Environment
72    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran
73    - export NPROC_POOL=8
74    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
75    - echo "-------------- CC ------------------" && $CC --version
76    - echo "-------------- CXX -----------------" && $CXX --version
77    - echo "-------------- FC ------------------" && $FC --version
78    - echo "-------------- GCOV ----------------" && gcov --version
79    # Libraries for backends
80    # -- LIBXSMM 7 April 2024
81    - cd .. && export XSMM_HASH=94ee71576870152feb62f3f0cf6b061d036dcdb5 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
82    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
83    # -- OCCA v1.6.0
84    - cd .. && export OCCA_VERSION=occa-1.6.0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.6.0 https://github.com/libocca/occa.git $OCCA_VERSION && cd $OCCA_VERSION && export ENABLE_OPENCL="OFF" ENABLE_DPCPP="OFF" ENABLE_HIP="OFF" ENABLE_CUDA="OFF" && ./configure-cmake.sh && cmake --build build --parallel $NPROC_CPU && cmake --install build && cd ..; }; } && export OCCA_DIR=$PWD/$OCCA_VERSION/install && cd libCEED
85    - echo "-------------- OCCA ----------------" && git -C $OCCA_DIR describe --tags && LD_LIBRARY_PATH=$OCCA_DIR/lib $OCCA_DIR/bin/occa info
86  script:
87    - rm -f .SUCCESS
88    # libCEED
89    - make configure OPT='-O -march=native -ffp-contract=fast'
90    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
91    - echo "-------------- libCEED -------------" && make info
92    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
93    - make clean
94    - OCCA_DIR= PEDANTIC=1 make -j$NPROC_CPU
95    - make -j$NPROC_CPU
96    # -- libCEED only tests
97    - echo "-------------- core tests ----------"
98    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
99    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
100    - export PETSC_DIR= PETSC_ARCH=
101    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit realsearch=%
102    # Libraries for examples
103    # -- PETSc (minimal)
104    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
105    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
106    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search="petsc fluids-navierstokes solids"
107    # -- MFEM v4.7
108    - cd .. && export MFEM_VERSION=mfem-4.7 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.7 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
109    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
110    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=mfem
111    # -- Nek5000 v19.0
112    - export COVERAGE=0
113    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
114    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
115    - export NPROC_POOL=1
116    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=nek NEK5K_DIR=$NEK5K_DIR
117    # -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11
118    - OCCA_DIR= BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
119    - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install
120    - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always
121    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR
122    # Report status
123    - touch .SUCCESS
124  after_script:
125    - |
126      if [ -f .SUCCESS ]; then
127        lcov --directory . --capture --output-file coverage.info --ignore-errors source,mismatch;
128        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
129        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
130        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
131        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
132        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
133      fi
134  artifacts:
135    paths:
136      - build/*.junit
137    reports:
138      junit: build/*.junit
139      performance: performance.json
140
141
142# ----------------------------------------------------------------------------------------
143# Check SYCL backends build
144# ----------------------------------------------------------------------------------------
145
146# SYCL tests currently disabled
147
148#noether-sycl:
149#  stage: test:gpu-and-float
150#  tags:
151#    - sycl
152#  interruptible: true
153#  before_script:
154#    # Environment
155#    - . /opt/intel/oneapi/setvars.sh
156#    - export COVERAGE=1 CC=icx CXX=icpx
157#    - export NPROC_POOL=8
158#    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
159#    - echo "-------------- CC ------------------" && $CC --version
160#    - echo "-------------- CXX -----------------" && $CXX --version
161#  script:
162#    - rm -f .SUCCESS
163#    # libCEED
164#    - make configure SYCL_DIR=/opt/intel/oneapi/compiler/latest OPT='-O -march=native -ffp-contract=fast'
165#    - BACKENDS_SYCL=$(make info-backends-all | grep -o '/sycl[^ ]*' | tr '\n' ' ')
166#    - echo "-------------- libCEED -------------" && make info
167#    - echo "-------------- BACKENDS_SYCL -------" && echo $BACKENDS_SYCL
168#    - make clean
169#    - make -j$NPROC_CPU
170#    # Report status
171#    - touch .SUCCESS
172
173
174# ----------------------------------------------------------------------------------------
175# Rust + CUDA
176# ----------------------------------------------------------------------------------------
177noether-rust-qfunctions:
178  stage: test:gpu-and-float
179  tags:
180    - cuda
181  interruptible: true
182  before_script:
183    # Environment
184    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran NVCC=nvcc GPU_CLANG=1
185    - export NPROC_POOL=4
186    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
187    - echo "-------------- CC ------------------" && $CC --version
188    - echo "-------------- CXX -----------------" && $CXX --version
189    - echo "-------------- FC ------------------" && $FC --version
190    - echo "-------------- NVCC ----------------" && $NVCC --version
191    - echo "-------------- Rustc ---------------" && rustc --version
192    - echo "-------------- Clang++ -------------" && clang++ --version
193    - echo "-------------- GCOV ----------------" && gcov --version
194  script:
195    - rm -f .SUCCESS
196    # Rustup
197    - rustup update nightly
198    - rustup component add rust-src --toolchain nightly
199    - rustup component add llvm-tools --toolchain nightly
200    # libCEED
201    - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr/local/cuda-12.9
202    - echo "-------------- libCEED -------------" && make info
203    - make clean
204    - make -k -j$NPROC_CPU -l$NPROC_CPU
205    # -- libCEED only tests
206    - echo "-------------- Rust QFunction tests -----"
207    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
208    - export PETSC_DIR= PETSC_ARCH=
209    - make -k -j$((NPROC_GPU / NPROC_POOL)) JUNIT_BATCH="rust-qfunction" junit search=rustqfunction
210    # Report status
211    - touch .SUCCESS
212  after_script:
213    - |
214      if [ -f .SUCCESS ]; then
215        lcov --directory . --capture --output-file coverage.info --ignore-errors source,mismatch;
216        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
217        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
218        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
219        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
220        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
221      fi
222  artifacts:
223    paths:
224      - build/*.junit
225    reports:
226      junit: build/*.junit
227
228
229# ----------------------------------------------------------------------------------------
230# CUDA backends
231# ----------------------------------------------------------------------------------------
232noether-cuda:
233  stage: test:gpu-and-float
234  tags:
235    - cuda
236  interruptible: true
237  before_script:
238    # Environment
239    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran NVCC=nvcc
240    - export NPROC_POOL=4
241    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
242    - echo "-------------- CC ------------------" && $CC --version
243    - echo "-------------- CXX -----------------" && $CXX --version
244    - echo "-------------- FC ------------------" && $FC --version
245    - echo "-------------- NVCC ----------------" && $NVCC --version
246    - echo "-------------- GCOV ----------------" && gcov --version
247    # ASAN
248    - echo "-------------- ASAN ----------------"
249    - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" ASAN_OPTIONS=protect_shadow_gap=0
250    - echo $AFLAGS
251  script:
252    - rm -f .SUCCESS
253    # libCEED
254    - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr/local/cuda-12.9
255    - echo "-------------- libCEED -------------" && make info
256    - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
257    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
258    - make clean
259    - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU
260    # -- libCEED only tests
261    - echo "-------------- core tests ----------"
262    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
263    - export PETSC_DIR= PETSC_ARCH=
264    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit realsearch=%
265    # Rebuild without ASAN
266    - unset ASAN AFLAGS ASAN_OPTIONS
267    - make clean
268    - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU
269    # Libraries for examples
270    # -- PETSc with CUDA (minimal)
271    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cuda-O PETSC_OPTIONS='-use_gpu_aware_mpi 0' && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
272    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
273    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search="petsc fluids solids"
274    # -- MFEM v4.7
275    - cd .. && export MFEM_VERSION=mfem-4.7 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.7 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
276    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
277    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=mfem
278    # -- Nek5000 v19.0
279    - export COVERAGE=0
280    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
281    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
282    - export NPROC_POOL=1
283    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=nek NEK5K_DIR=$NEK5K_DIR
284    # -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11
285    - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install
286    - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always
287    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR
288    # Clang-tidy
289    - echo "-------------- clang-tidy ----------" && clang-tidy --version
290    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
291    # Report status
292    - touch .SUCCESS
293  after_script:
294    - |
295      if [ -f .SUCCESS ]; then
296        lcov --directory . --capture --output-file coverage.info --ignore-errors source,mismatch;
297        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
298        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
299        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
300        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
301        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
302      fi
303  artifacts:
304    paths:
305      - build/*.junit
306    reports:
307      junit: build/*.junit
308
309
310# ----------------------------------------------------------------------------------------
311# ROCm backends
312# ----------------------------------------------------------------------------------------
313
314# ROCm test execution currently disabled
315
316#noether-rocm:
317#  stage: test:gpu-and-float
318#  tags:
319#    - rocm
320#  interruptible: true
321#  before_script:
322#    # Environment
323#    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
324#    - export NPROC_POOL=4
325#    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
326#    - echo "-------------- CC ------------------" && $CC --version
327#    - echo "-------------- CXX -----------------" && $CXX --version
328#    - echo "-------------- FC ------------------" && $FC --version
329#    - echo "-------------- HIPCC ---------------" && $HIPCC --version
330#    - echo "-------------- GCOV ----------------" && gcov --version
331#    # Libraries for backends
332#    # -- MAGMA from dev branch
333#    - echo "-------------- MAGMA ---------------"
334#    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
335#  script:
336#    - rm -f .SUCCESS
337#    # libCEED
338#    - make configure ROCM_DIR=/opt/rocm-6.3.0 OPT='-O -march=native -ffp-contract=fast'
339#    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
340#    - echo "-------------- libCEED -------------" && make info
341#    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
342#    - make clean
343#    - make -j$NPROC_CPU
344#    # -- libCEED only tests
345#    - echo "-------------- core tests ----------"
346#    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
347#    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
348#    - export PETSC_DIR= PETSC_ARCH=
349#    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=%
350#    # Libraries for examples
351#    # -- PETSc with HIP (minimal)
352#    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
353#    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
354#    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids"
355#    # Clang-tidy
356#    - echo "-------------- clang-tidy ----------" && clang-tidy --version
357#    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
358#    # Report status
359#    - touch .SUCCESS
360#  after_script:
361#    - |
362#      if [ -f .SUCCESS ]; then
363#        lcov --directory . --capture --output-file coverage.info --ignore-errors source,mismatch;
364#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
365#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
366#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
367#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
368#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
369#      fi
370#  artifacts:
371#    paths:
372#      - build/*.junit
373#    reports:
374#      junit: build/*.junit
375#      performance: performance.json
376
377
378noether-rocm:
379  stage: test:gpu-and-float
380  tags:
381    - rocm
382  interruptible: true
383  before_script:
384    # Environment
385    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
386    - export NPROC_POOL=4
387    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
388    - echo "-------------- CC ------------------" && $CC --version
389    - echo "-------------- CXX -----------------" && $CXX --version
390    - echo "-------------- FC ------------------" && $FC --version
391    - echo "-------------- HIPCC ---------------" && $HIPCC --version
392    - echo "-------------- GCOV ----------------" && gcov --version
393    # Libraries for backends
394    # -- MAGMA from dev branch
395    - echo "-------------- MAGMA ---------------"
396    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
397  script:
398    - rm -f .SUCCESS
399    # libCEED
400    - make configure ROCM_DIR=/opt/rocm-6.3.0 OPT='-O -march=native -ffp-contract=fast'
401    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
402    - echo "-------------- libCEED -------------" && make info
403    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
404    - make clean
405    - make -j$NPROC_CPU
406    # Clang-tidy
407    - echo "-------------- clang-tidy ----------" && clang-tidy --version
408    - make clean
409    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
410    # Report status
411    - touch .SUCCESS
412
413
414# ----------------------------------------------------------------------------------------
415# CPU + CUDA backends with CeedScalar == float (32 bit)
416# ----------------------------------------------------------------------------------------
417noether-float:
418  stage: test:gpu-and-float
419  tags:
420    - cpu
421    - cuda
422  interruptible: true
423  before_script:
424    # Environment
425    - export COVERAGE=1 CC=gcc CXX=g++ FC= NVCC=nvcc
426    - export NPROC_POOL=8
427    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
428    - echo "-------------- CC ------------------" && $CC --version
429    - echo "-------------- CXX -----------------" && $CXX --version
430    - echo "-------------- NVCC ----------------" && $NVCC --version
431    - echo "-------------- GCOV ----------------" && gcov --version
432    # Libraries for backends
433# ROCm tests currently disabled
434# -- MAGMA from dev branch
435#    - echo "-------------- MAGMA ---------------"
436#    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
437    # -- LIBXSMM 19 March 2025
438    - cd .. && export XSMM_HASH=ba9d6bc69c421c10f0597d582ea1ace6a6126308 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
439    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
440  script:
441    - rm -f .SUCCESS
442    # libCEED
443    # Change to single precision
444    - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/types.h
445    # Build libCEED
446    - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr/local/cuda-12.9
447    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
448    - echo "-------------- libCEED -------------" && make info
449    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
450    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
451    - make clean
452    - make -j$NPROC_CPU
453    # -- libCEED only tests
454    - echo "-------------- core tests ----------"
455    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
456    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
457    - export PETSC_DIR= PETSC_ARCH=
458    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="float-cpu" junit realsearch=%
459    - export NPROC_POOL=4
460    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-cuda" junit realsearch=%
461    # Report status
462    - echo "SUCCESS" > .job_status
463  after_script:
464    - |
465      if [ $(cat .job_status) == "SUCCESS" ]; then
466        lcov --directory . --capture --output-file coverage.info --ignore-errors source,mismatch;
467        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
468        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
469        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
470        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
471        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
472      fi
473    - sed -i 's/ceed-f32/ceed-f64/1' include/ceed/types.h
474  artifacts:
475    paths:
476      - build/*.junit
477    reports:
478      junit: build/*.junit
479      performance: performance.json
480