xref: /libCEED/.gitlab-ci.yml (revision 216bbcafa310ca9c8563aabbff60855a4edc792b) 
1stages:
2  - test:cpu-and-tidy
3  - test:gpu-and-float
4
5noether-cpu:
6  stage: test:cpu-and-tidy
7  tags:
8    - cpu
9    - rocm
10  interruptible: true
11  before_script:
12# Environment
13    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
14    - export NPROC_POOL=8
15    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
16    - echo "-------------- CC ------------------" && $CC --version
17    - echo "-------------- CXX -----------------" && $CXX --version
18    - echo "-------------- FC ------------------" && $FC --version
19    - echo "-------------- HIPCC ---------------" && $HIPCC --version
20    - echo "-------------- GCOV ----------------" && gcov --version
21# Libraries for backends
22# -- LIBXSMM 2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8
23    - cd .. && export XSMM_HASH=2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
24    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
25# -- OCCA v1.1.0
26    - cd .. && export OCCA_VERSION=occa-1.4.0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.4.0 https://github.com/libocca/occa.git $OCCA_VERSION && cd $OCCA_VERSION && export ENABLE_OPENCL="OFF" ENABLE_DPCPP="OFF" ENABLE_HIP="OFF" ENABLE_CUDA="OFF" && ./configure-cmake.sh && cmake --build build --parallel $NPROC_CPU && cmake --install build && cd ..; }; } && export OCCA_DIR=$PWD/$OCCA_VERSION/install && cd libCEED
27    - echo "-------------- OCCA ----------------" && git -C $OCCA_DIR describe --tags && LD_LIBRARY_PATH=$OCCA_DIR/lib $OCCA_DIR/bin/occa info
28  script:
29    - rm -f .SUCCESS
30# libCEED
31    - make configure OPT='-O -march=native -ffp-contract=fast'
32    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
33    - echo "-------------- libCEED -------------" && make info
34    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
35    - OCCA_DIR= PEDANTIC=1 make -j$NPROC_CPU
36    - make -j$NPROC_CPU
37# -- libCEED only tests
38    - echo "-------------- core tests ----------"
39    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
40#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
41    - export PETSC_DIR= PETSC_ARCH=
42    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit realsearch=%
43# Libraries for examples
44# -- PETSc with HIP (minimal)
45    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
46    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
47    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search="petsc fluids solids"
48# -- MFEM v4.2
49    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
50    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
51    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=mfem
52# -- Nek5000 v19.0
53    - export COVERAGE=0
54    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
55    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
56    - export NPROC_POOL=1
57    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=nek NEK5K_DIR=$NEK5K_DIR
58# Clang-tidy
59    - echo "-------------- clang-tidy ----------" && clang-tidy --version
60    - export OCCA_DIR= && make lib -j$NPROC_CPU
61    - make -j$NPROC_CPU tidy
62# Report status
63    - touch .SUCCESS
64  after_script:
65    - |
66      if [ -f .SUCCESS ]; then
67        lcov --directory . --capture --output-file coverage.info;
68        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
69        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
70        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
71        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
72        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
73      fi
74  artifacts:
75    paths:
76      - build/*.junit
77    reports:
78      junit: build/*.junit
79      performance: performance.json
80
81noether-rocm:
82  stage: test:gpu-and-float
83  tags:
84    - rocm
85  interruptible: true
86  before_script:
87# Environment
88    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
89    - export NPROC_POOL=4
90    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
91    - echo "-------------- CC ------------------" && $CC --version
92    - echo "-------------- CXX -----------------" && $CXX --version
93    - echo "-------------- FC ------------------" && $FC --version
94    - echo "-------------- HIPCC ---------------" && $HIPCC --version
95    - echo "-------------- GCOV ----------------" && gcov --version
96# Libraries for backends
97# -- MAGMA from dev branch
98    - echo "-------------- MAGMA ---------------"
99    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
100  script:
101    - rm -f .SUCCESS
102# libCEED
103    - make configure ROCM_DIR=/opt/rocm-5.6.0 OPT='-O -march=native -ffp-contract=fast'
104    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
105    - echo "-------------- libCEED -------------" && make info
106    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
107    - make -j$NPROC_CPU
108# -- libCEED only tests
109    - echo "-------------- core tests ----------"
110    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
111#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
112    - export PETSC_DIR= PETSC_ARCH=
113    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=%
114# Libraries for examples
115# -- PETSc with HIP (minimal)
116    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
117    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
118    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids"
119# -- MFEM v4.2
120    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
121    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
122    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=mfem
123# -- Nek5000 v19.0
124    - export COVERAGE=0
125    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
126    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
127    - export NPROC_POOL=1
128    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=nek NEK5K_DIR=$NEK5K_DIR
129# Clang-tidy
130    - echo "-------------- clang-tidy ----------" && clang-tidy --version
131    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
132# Report status
133    - touch .SUCCESS
134  after_script:
135    - |
136      if [ -f .SUCCESS ]; then
137        lcov --directory . --capture --output-file coverage.info;
138        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
139        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
140        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
141        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
142        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
143      fi
144  artifacts:
145    paths:
146      - build/*.junit
147    reports:
148      junit: build/*.junit
149      performance: performance.json
150
151noether-float:
152  stage: test:gpu-and-float
153  tags:
154    - cpu
155    - rocm
156  interruptible: true
157  before_script:
158# Environment
159    - export COVERAGE=1 CC=gcc CXX=g++ FC= HIPCC=hipcc
160    - export NPROC_POOL=8
161    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
162    - echo "-------------- CC ------------------" && $CC --version
163    - echo "-------------- CXX -----------------" && $CXX --version
164    - echo "-------------- HIPCC ---------------" && $HIPCC --version
165    - echo "-------------- GCOV ----------------" && gcov --version
166# Libraries for backends
167# -- MAGMA from dev branch
168    - echo "-------------- MAGMA ---------------"
169    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
170# -- LIBXSMM 2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8
171    - cd .. && export XSMM_HASH=2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
172    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
173  script:
174    - rm -f .SUCCESS
175# libCEED
176# Change to single precision
177    - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/types.h
178# Build libCEED
179    - make configure ROCM_DIR=/opt/rocm-5.6.0 OPT='-O -march=native -ffp-contract=fast'
180    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
181    - echo "-------------- libCEED -------------" && make info
182    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
183    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
184    - make -j$NPROC_CPU
185# -- libCEED only tests
186    - echo "-------------- core tests ----------"
187    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
188#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
189    - export PETSC_DIR= PETSC_ARCH=
190    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="float-cpu" junit realsearch=%
191    - export NPROC_POOL=4
192    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-hip" junit realsearch=%
193# Report status
194    - echo "SUCCESS" > .job_status
195  after_script:
196    - |
197      if [ $(cat .job_status) == "SUCCESS" ]; then
198        lcov --directory . --capture --output-file coverage.info;
199        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
200        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
201        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
202        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
203        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
204      fi
205    - sed -i 's/ceed-f32/ceed-f64/1' include/ceed/types.h
206  artifacts:
207    paths:
208      - build/*.junit
209    reports:
210      junit: build/*.junit
211      performance: performance.json
212
213noether-cuda:
214  stage: test:gpu-and-float
215  tags:
216    - cuda
217  interruptible: true
218  before_script:
219# Environment
220    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran NVCC=nvcc
221    - export NPROC_POOL=4
222    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
223    - echo "-------------- CC ------------------" && $CC --version
224    - echo "-------------- CXX -----------------" && $CXX --version
225    - echo "-------------- FC ------------------" && $FC --version
226    - echo "-------------- NVCC ----------------" && $NVCC --version
227    - echo "-------------- GCOV ----------------" && gcov --version
228  script:
229    - rm -f .SUCCESS
230# libCEED
231    - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr
232    - echo "-------------- libCEED -------------" && make info
233    - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
234    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
235    - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU
236# -- libCEED only tests
237    - echo "-------------- core tests ----------"
238#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
239    - export PETSC_DIR= PETSC_ARCH=
240    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit realsearch=%
241# Libraries for examples
242# -- PETSc with CUDA (minimal)
243    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cuda-O PETSC_OPTIONS='-use_gpu_aware_mpi 0' && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
244    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
245    - make -k -j$((NPROC_GPU / NPROC_POOL)) JUNIT_BATCH="cuda" junit BACKENDS="$BACKENDS_GPU" search="petsc fluids solids"
246# Report status
247    - touch .SUCCESS
248  after_script:
249    - |
250      if [ -f .SUCCESS ]; then
251        lcov --directory . --capture --output-file coverage.info;
252        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
253        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
254        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
255        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
256        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
257      fi
258  artifacts:
259    paths:
260      - build/*.junit
261    reports:
262      junit: build/*.junit
263