xref: /libCEED/.gitlab-ci.yml (revision 58f118fde9e934489c18d94d567a96d887adab93) 
1stages:
2  - test:cpu-and-tidy
3  - test:gpu-and-float
4
5noether-cpu:
6  stage: test:cpu-and-tidy
7  tags:
8    - cpu
9    - rocm
10  interruptible: true
11  before_script:
12# Environment
13    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
14    - export NPROC_POOL=8
15    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
16    - echo "-------------- CC ------------------" && $CC --version
17    - echo "-------------- CXX -----------------" && $CXX --version
18    - echo "-------------- FC ------------------" && $FC --version
19    - echo "-------------- HIPCC ---------------" && $HIPCC --version
20    - echo "-------------- GCOV ----------------" && gcov --version
21# Libraries for backends
22# -- LIBXSMM 2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8
23    - cd .. && export XSMM_HASH=2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
24    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
25# -- OCCA v1.1.0
26    - cd .. && export OCCA_VERSION=occa-1.4.0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.4.0 https://github.com/libocca/occa.git $OCCA_VERSION && cd $OCCA_VERSION && export ENABLE_OPENCL="OFF" ENABLE_DPCPP="OFF" ENABLE_HIP="OFF" ENABLE_CUDA="OFF" && ./configure-cmake.sh && cmake --build build --parallel $NPROC_CPU && cmake --install build && cd ..; }; } && export OCCA_DIR=$PWD/$OCCA_VERSION/install && cd libCEED
27    - echo "-------------- OCCA ----------------" && git -C $OCCA_DIR describe --tags && LD_LIBRARY_PATH=$OCCA_DIR/lib $OCCA_DIR/bin/occa info
28  script:
29    - rm -f .SUCCESS
30# libCEED
31    - make configure OPT='-O -march=native -ffp-contract=fast'
32    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
33    - echo "-------------- libCEED -------------" && make info
34    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
35    - OCCA_DIR= PEDANTIC=1 make -j$NPROC_CPU
36    - make -j$NPROC_CPU
37# -- libCEED only tests
38    - echo "-------------- core tests ----------"
39    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
40#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
41    - export PETSC_DIR= PETSC_ARCH=
42    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit realsearch=%
43# Libraries for examples
44# -- PETSc with HIP (minimal)
45    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
46    - source /home/jawr8143/SmartSimTestingSoftware/bin/activate && export SMARTREDIS_DIR=/home/jawr8143/SmartSimTestingSoftware/smartredis/install
47    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
48    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search="petsc fluids solids"
49# -- MFEM v4.2
50    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
51    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
52    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=mfem
53# -- Nek5000 v19.0
54    - export COVERAGE=0
55    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
56    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
57    - export NPROC_POOL=1
58    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=nek NEK5K_DIR=$NEK5K_DIR
59# -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11
60    - OCCA_DIR= BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
61    - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install
62    - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always
63    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR
64# Clang-tidy
65    - echo "-------------- clang-tidy ----------" && clang-tidy --version
66    - export OCCA_DIR= && make lib -j$NPROC_CPU
67    - make -j$NPROC_CPU tidy
68# Report status
69    - touch .SUCCESS
70  after_script:
71    - |
72      if [ -f .SUCCESS ]; then
73        lcov --directory . --capture --output-file coverage.info;
74        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
75        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
76        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
77        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
78        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
79      fi
80  artifacts:
81    paths:
82      - build/*.junit
83    reports:
84      junit: build/*.junit
85      performance: performance.json
86
87noether-rocm:
88  stage: test:gpu-and-float
89  tags:
90    - rocm
91  interruptible: true
92  before_script:
93# Environment
94    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
95    - export NPROC_POOL=4
96    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
97    - echo "-------------- CC ------------------" && $CC --version
98    - echo "-------------- CXX -----------------" && $CXX --version
99    - echo "-------------- FC ------------------" && $FC --version
100    - echo "-------------- HIPCC ---------------" && $HIPCC --version
101    - echo "-------------- GCOV ----------------" && gcov --version
102# Libraries for backends
103# -- MAGMA from dev branch
104    - echo "-------------- MAGMA ---------------"
105    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
106  script:
107    - rm -f .SUCCESS
108# libCEED
109    - make configure ROCM_DIR=/opt/rocm-5.6.0 OPT='-O -march=native -ffp-contract=fast'
110    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
111    - echo "-------------- libCEED -------------" && make info
112    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
113    - make -j$NPROC_CPU
114# -- libCEED only tests
115    - echo "-------------- core tests ----------"
116    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
117#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
118    - export PETSC_DIR= PETSC_ARCH=
119    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=%
120# Libraries for examples
121# -- PETSc with HIP (minimal)
122    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
123    - source /home/jawr8143/SmartSimTestingSoftware/bin/activate && export SMARTREDIS_DIR=/home/jawr8143/SmartSimTestingSoftware/smartredis/install
124    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
125    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids"
126# -- MFEM v4.2
127    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
128    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
129    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=mfem
130# -- Nek5000 v19.0
131    - export COVERAGE=0
132    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
133    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
134    - export NPROC_POOL=1
135    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=nek NEK5K_DIR=$NEK5K_DIR
136# -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11
137    - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install
138    - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always
139    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR
140# Clang-tidy
141    - echo "-------------- clang-tidy ----------" && clang-tidy --version
142    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
143# Report status
144    - touch .SUCCESS
145  after_script:
146    - |
147      if [ -f .SUCCESS ]; then
148        lcov --directory . --capture --output-file coverage.info;
149        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
150        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
151        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
152        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
153        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
154      fi
155  artifacts:
156    paths:
157      - build/*.junit
158    reports:
159      junit: build/*.junit
160      performance: performance.json
161
162noether-float:
163  stage: test:gpu-and-float
164  tags:
165    - cpu
166    - rocm
167  interruptible: true
168  before_script:
169# Environment
170    - export COVERAGE=1 CC=gcc CXX=g++ FC= HIPCC=hipcc
171    - export NPROC_POOL=8
172    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
173    - echo "-------------- CC ------------------" && $CC --version
174    - echo "-------------- CXX -----------------" && $CXX --version
175    - echo "-------------- HIPCC ---------------" && $HIPCC --version
176    - echo "-------------- GCOV ----------------" && gcov --version
177# Libraries for backends
178# -- MAGMA from dev branch
179    - echo "-------------- MAGMA ---------------"
180    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
181# -- LIBXSMM 2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8
182    - cd .. && export XSMM_HASH=2c145a109b5a8ad4e15f60ea42a86b9056bdc8b8 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
183    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
184  script:
185    - rm -f .SUCCESS
186# libCEED
187# Change to single precision
188    - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/types.h
189# Build libCEED
190    - make configure ROCM_DIR=/opt/rocm-5.6.0 OPT='-O -march=native -ffp-contract=fast'
191    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
192    - echo "-------------- libCEED -------------" && make info
193    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
194    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
195    - make -j$NPROC_CPU
196# -- libCEED only tests
197    - echo "-------------- core tests ----------"
198    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
199#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
200    - export PETSC_DIR= PETSC_ARCH=
201    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="float-cpu" junit realsearch=%
202    - export NPROC_POOL=4
203    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-hip" junit realsearch=%
204# Report status
205    - echo "SUCCESS" > .job_status
206  after_script:
207    - |
208      if [ $(cat .job_status) == "SUCCESS" ]; then
209        lcov --directory . --capture --output-file coverage.info;
210        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
211        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
212        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
213        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
214        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
215      fi
216    - sed -i 's/ceed-f32/ceed-f64/1' include/ceed/types.h
217  artifacts:
218    paths:
219      - build/*.junit
220    reports:
221      junit: build/*.junit
222      performance: performance.json
223
224noether-cuda:
225  stage: test:gpu-and-float
226  tags:
227    - cuda
228  interruptible: true
229  before_script:
230# Environment
231    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran NVCC=nvcc
232    - export NPROC_POOL=4
233    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
234    - echo "-------------- CC ------------------" && $CC --version
235    - echo "-------------- CXX -----------------" && $CXX --version
236    - echo "-------------- FC ------------------" && $FC --version
237    - echo "-------------- NVCC ----------------" && $NVCC --version
238    - echo "-------------- GCOV ----------------" && gcov --version
239  script:
240    - rm -f .SUCCESS
241# libCEED
242    - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr
243    - echo "-------------- libCEED -------------" && make info
244    - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
245    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
246    - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU
247# -- libCEED only tests
248    - echo "-------------- core tests ----------"
249#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
250    - export PETSC_DIR= PETSC_ARCH=
251    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit realsearch=%
252# Libraries for examples
253# -- PETSc with CUDA (minimal)
254    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cuda-O PETSC_OPTIONS='-use_gpu_aware_mpi 0' && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
255    - source /home/jawr8143/SmartSimTestingSoftware/bin/activate && export SMARTREDIS_DIR=/home/jawr8143/SmartSimTestingSoftware/smartredis/install
256    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
257    - make -k -j$((NPROC_GPU / NPROC_POOL)) JUNIT_BATCH="cuda" junit BACKENDS="$BACKENDS_GPU" search="petsc fluids solids"
258# Report status
259    - touch .SUCCESS
260  after_script:
261    - |
262      if [ -f .SUCCESS ]; then
263        lcov --directory . --capture --output-file coverage.info;
264        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
265        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
266        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
267        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
268        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
269      fi
270  artifacts:
271    paths:
272      - build/*.junit
273    reports:
274      junit: build/*.junit
275