xref: /libCEED/.gitlab-ci.yml (revision 4e3038a58c29aaa28aa8eaaef94bbc7eab6ca469) 
1stages:
2  - test:cpu-and-tidy
3  - test:gpu-and-float
4
5
6noether-cpu:
7  stage: test:cpu-and-tidy
8  tags:
9    - cpu
10    - rocm
11  interruptible: true
12  before_script:
13# Environment
14    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
15    - export NPROC_POOL=8
16    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
17    - echo "-------------- CC ------------------" && $CC --version
18    - echo "-------------- CXX -----------------" && $CXX --version
19    - echo "-------------- FC ------------------" && $FC --version
20    - echo "-------------- HIPCC ---------------" && $HIPCC --version
21    - echo "-------------- GCOV ----------------" && gcov --version
22# Libraries for backends
23# -- LIBXSMM 2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8
24    - cd .. && export XSMM_HASH=2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
25    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
26# -- OCCA v1.1.0
27    - cd .. && export OCCA_VERSION=occa-1.4.0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.4.0 https://github.com/libocca/occa.git $OCCA_VERSION && cd $OCCA_VERSION && export ENABLE_OPENCL="OFF" ENABLE_DPCPP="OFF" ENABLE_HIP="OFF" ENABLE_CUDA="OFF" && ./configure-cmake.sh && cmake --build build --parallel $NPROC_CPU && cmake --install build && cd ..; }; } && export OCCA_DIR=$PWD/$OCCA_VERSION/install && cd libCEED
28    - echo "-------------- OCCA ----------------" && git -C $OCCA_DIR describe --tags && LD_LIBRARY_PATH=$OCCA_DIR/lib $OCCA_DIR/bin/occa info
29  script:
30    - rm -f .SUCCESS
31# libCEED
32    - make configure OPT='-O -march=native -ffp-contract=fast'
33    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
34    - echo "-------------- libCEED -------------" && make info
35    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
36    - OCCA_DIR= PEDANTIC=1 make -j$NPROC_CPU
37    - make -j$NPROC_CPU
38# -- libCEED only tests
39    - echo "-------------- core tests ----------"
40    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
41#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
42    - export PETSC_DIR= PETSC_ARCH=
43    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit realsearch=%
44# Libraries for examples
45# -- PETSc with HIP (minimal)
46    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
47    - source /home/jawr8143/SmartSimTestingSoftware/bin/activate && export SMARTREDIS_DIR=/home/jawr8143/SmartSimTestingSoftware/smartredis/install
48    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
49    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search="petsc fluids solids"
50# -- MFEM v4.2
51    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
52    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
53    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=mfem
54# -- Nek5000 v19.0
55    - export COVERAGE=0
56    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
57    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
58    - export NPROC_POOL=1
59    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=nek NEK5K_DIR=$NEK5K_DIR
60# -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11
61    - OCCA_DIR= BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
62    - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install
63    - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always
64    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR
65# Clang-tidy
66    - echo "-------------- clang-tidy ----------" && clang-tidy --version
67    - export OCCA_DIR= && make lib -j$NPROC_CPU
68    - make -j$NPROC_CPU tidy
69# Report status
70    - touch .SUCCESS
71  after_script:
72    - |
73      if [ -f .SUCCESS ]; then
74        lcov --directory . --capture --output-file coverage.info;
75        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
76        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
77        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
78        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
79        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
80      fi
81  artifacts:
82    paths:
83      - build/*.junit
84    reports:
85      junit: build/*.junit
86      performance: performance.json
87
88
89noether-sycl:
90  stage: test:gpu-and-float
91  tags:
92    - sycl
93  interruptible: true
94  before_script:
95# Environment
96    - . /opt/intel/oneapi/setvars.sh
97    - export COVERAGE=1 CC=icx CXX=icpx
98    - export NPROC_POOL=8
99    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
100    - echo "-------------- CC ------------------" && $CC --version
101    - echo "-------------- CXX -----------------" && $CXX --version
102  script:
103    - rm -f .SUCCESS
104# libCEED
105    - make configure SYCL_DIR=/opt/intel/oneapi/compiler/2023.2.0/linux OPT='-O -march=native -ffp-contract=fast'
106    - BACKENDS_SYCL=$(make info-backends-all | grep -o '/sycl[^ ]*' | tr '\n' ' ')
107    - echo "-------------- libCEED -------------" && make info
108    - echo "-------------- BACKENDS_SYCL -------" && echo $BACKENDS_SYCL
109    - make -j$NPROC_CPU
110# Report status
111    - touch .SUCCESS
112
113
114noether-rocm:
115  stage: test:gpu-and-float
116  tags:
117    - rocm
118  interruptible: true
119  before_script:
120# Environment
121    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
122    - export NPROC_POOL=4
123    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
124    - echo "-------------- CC ------------------" && $CC --version
125    - echo "-------------- CXX -----------------" && $CXX --version
126    - echo "-------------- FC ------------------" && $FC --version
127    - echo "-------------- HIPCC ---------------" && $HIPCC --version
128    - echo "-------------- GCOV ----------------" && gcov --version
129# Libraries for backends
130# -- MAGMA from dev branch
131    - echo "-------------- MAGMA ---------------"
132    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
133  script:
134    - rm -f .SUCCESS
135# libCEED
136    - make configure ROCM_DIR=/opt/rocm-5.6.0 OPT='-O -march=native -ffp-contract=fast'
137    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
138    - echo "-------------- libCEED -------------" && make info
139    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
140    - make -j$NPROC_CPU
141# -- libCEED only tests
142    - echo "-------------- core tests ----------"
143    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
144#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
145    - export PETSC_DIR= PETSC_ARCH=
146    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=%
147# Libraries for examples
148# -- PETSc with HIP (minimal)
149    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
150    - source /home/jawr8143/SmartSimTestingSoftware/bin/activate && export SMARTREDIS_DIR=/home/jawr8143/SmartSimTestingSoftware/smartredis/install
151    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
152    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids"
153# -- MFEM v4.2
154    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
155    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
156    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=mfem
157# -- Nek5000 v19.0
158    - export COVERAGE=0
159    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
160    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
161    - export NPROC_POOL=1
162    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=nek NEK5K_DIR=$NEK5K_DIR
163# -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11
164    - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install
165    - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always
166    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR
167# Clang-tidy
168    - echo "-------------- clang-tidy ----------" && clang-tidy --version
169    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
170# Report status
171    - touch .SUCCESS
172  after_script:
173    - |
174      if [ -f .SUCCESS ]; then
175        lcov --directory . --capture --output-file coverage.info;
176        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
177        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
178        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
179        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
180        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
181      fi
182  artifacts:
183    paths:
184      - build/*.junit
185    reports:
186      junit: build/*.junit
187      performance: performance.json
188
189
190noether-float:
191  stage: test:gpu-and-float
192  tags:
193    - cpu
194    - rocm
195  interruptible: true
196  before_script:
197# Environment
198    - export COVERAGE=1 CC=gcc CXX=g++ FC= HIPCC=hipcc
199    - export NPROC_POOL=8
200    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
201    - echo "-------------- CC ------------------" && $CC --version
202    - echo "-------------- CXX -----------------" && $CXX --version
203    - echo "-------------- HIPCC ---------------" && $HIPCC --version
204    - echo "-------------- GCOV ----------------" && gcov --version
205# Libraries for backends
206# -- MAGMA from dev branch
207    - echo "-------------- MAGMA ---------------"
208    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
209# -- LIBXSMM 2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8
210    - cd .. && export XSMM_HASH=2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
211    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
212  script:
213    - rm -f .SUCCESS
214# libCEED
215# Change to single precision
216    - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/types.h
217# Build libCEED
218    - make configure ROCM_DIR=/opt/rocm-5.6.0 OPT='-O -march=native -ffp-contract=fast'
219    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
220    - echo "-------------- libCEED -------------" && make info
221    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
222    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
223    - make -j$NPROC_CPU
224# -- libCEED only tests
225    - echo "-------------- core tests ----------"
226    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
227#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
228    - export PETSC_DIR= PETSC_ARCH=
229    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="float-cpu" junit realsearch=%
230    - export NPROC_POOL=4
231    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-hip" junit realsearch=%
232# Report status
233    - echo "SUCCESS" > .job_status
234  after_script:
235    - |
236      if [ $(cat .job_status) == "SUCCESS" ]; then
237        lcov --directory . --capture --output-file coverage.info;
238        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
239        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
240        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
241        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
242        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
243      fi
244    - sed -i 's/ceed-f32/ceed-f64/1' include/ceed/types.h
245  artifacts:
246    paths:
247      - build/*.junit
248    reports:
249      junit: build/*.junit
250      performance: performance.json
251
252
253noether-cuda:
254  stage: test:gpu-and-float
255  tags:
256    - cuda
257  interruptible: true
258  before_script:
259# Environment
260    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran NVCC=nvcc
261    - export NPROC_POOL=4
262    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
263    - echo "-------------- CC ------------------" && $CC --version
264    - echo "-------------- CXX -----------------" && $CXX --version
265    - echo "-------------- FC ------------------" && $FC --version
266    - echo "-------------- NVCC ----------------" && $NVCC --version
267    - echo "-------------- GCOV ----------------" && gcov --version
268  script:
269    - rm -f .SUCCESS
270# libCEED
271    - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr
272    - echo "-------------- libCEED -------------" && make info
273    - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
274    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
275    - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU
276# -- libCEED only tests
277    - echo "-------------- core tests ----------"
278#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
279    - export PETSC_DIR= PETSC_ARCH=
280    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit realsearch=%
281# Libraries for examples
282# -- PETSc with CUDA (minimal)
283    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cuda-O PETSC_OPTIONS='-use_gpu_aware_mpi 0' && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
284    - source /home/jawr8143/SmartSimTestingSoftware/bin/activate && export SMARTREDIS_DIR=/home/jawr8143/SmartSimTestingSoftware/smartredis/install
285    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
286    - make -k -j$((NPROC_GPU / NPROC_POOL)) JUNIT_BATCH="cuda" junit BACKENDS="$BACKENDS_GPU" search="petsc fluids solids"
287# Report status
288    - touch .SUCCESS
289  after_script:
290    - |
291      if [ -f .SUCCESS ]; then
292        lcov --directory . --capture --output-file coverage.info;
293        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
294        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
295        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
296        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
297        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
298      fi
299  artifacts:
300    paths:
301      - build/*.junit
302    reports:
303      junit: build/*.junit
304