xref: /libCEED/.gitlab-ci.yml (revision e2cfdb03597769a59e92151c2ed08fee92f7b662) 
1stages:
2  - test:cpu-and-tidy
3  - test:gpu-and-float
4
5noether-cpu:
6  stage: test:cpu-and-tidy
7  tags:
8    - cpu
9    - rocm
10  interruptible: true
11  before_script:
12# Environment
13    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
14    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
15    - echo "-------------- CC ------------------" && $CC --version
16    - echo "-------------- CXX -----------------" && $CXX --version
17    - echo "-------------- FC ------------------" && $FC --version
18    - echo "-------------- HIPCC ---------------" && $HIPCC --version
19    - echo "-------------- GCOV ----------------" && gcov --version
20# Libraries for backends
21# -- LIBXSMM v1.17
22    - cd .. && export XSMM_VERSION=libxsmm-1.17 && { [[ -d $XSMM_VERSION ]] || { git clone --depth 1 --branch 1.17 https://github.com/hfp/libxsmm.git $XSMM_VERSION && make -C $XSMM_VERSION -j$(nproc); }; } && export XSMM_DIR=$PWD/$XSMM_VERSION && cd libCEED
23    - echo "-------------- LIBXSMM -------------" && git -C $XSMM_DIR describe --tags
24# -- OCCA v1.1.0
25    - cd .. && export OCCA_VERSION=occa-1.1.0 OCCA_OPENCL_ENABLED=0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.1.0 https://github.com/libocca/occa.git $OCCA_VERSION && make -C $OCCA_VERSION -j$(nproc); }; } && export OCCA_DIR=$PWD/$OCCA_VERSION && cd libCEED
26    - echo "-------------- OCCA ----------------" && make -C $OCCA_DIR info
27  script:
28    - rm -f .SUCCESS
29# libCEED
30    - make configure OPT='-O -march=native -ffp-contract=fast'
31#    Note: OCCA backends currently disabled in CI
32    - BACKENDS_CPU=$(OCCA_DIR= make info-backends-all | grep -o '/cpu[^ ]*')
33    - echo "-------------- libCEED -------------" && make info
34    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
35    - make -j$NPROC_CPU
36# Remove OCCA after verifying it compiles
37# CodeCov of OCCA backend not useful since testing is intentionally disabled
38    - export OCCA_DIR= && make -j$NPROC_CPU -W build/interface/ceed-register.o
39# -- libCEED only tests
40    - echo "-------------- core tests ----------"
41    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
42#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
43    - export PETSC_DIR= PETSC_ARCH=
44    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit realsearch=%
45# Libraries for examples
46# -- PETSc with HIP (minimal)
47    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR describe
48    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
49    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search="petsc fluids solids"
50# -- MFEM v4.2
51    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
52    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
53    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=mfem
54# -- Nek5000 v19.0
55    - export COVERAGE=0
56    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
57    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
58    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=nek
59# Clang-tidy
60    - echo "-------------- clang-tidy ----------" && clang-tidy --version
61    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --exit-code
62# Report status
63    - touch .SUCCESS
64  after_script:
65    - |
66      if [ -f .SUCCESS ]; then
67        lcov --directory . --capture --output-file coverage.info;
68        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
69        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
70        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
71        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
72        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
73      fi
74  artifacts:
75    paths:
76      - build/*.junit
77    reports:
78      junit: build/*.junit
79      performance: performance.json
80
81noether-rocm:
82  stage: test:gpu-and-float
83  tags:
84    - rocm
85  interruptible: true
86  before_script:
87# Environment
88    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
89    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
90    - echo "-------------- CC ------------------" && $CC --version
91    - echo "-------------- CXX -----------------" && $CXX --version
92    - echo "-------------- FC ------------------" && $FC --version
93    - echo "-------------- HIPCC ---------------" && $HIPCC --version
94    - echo "-------------- GCOV ----------------" && gcov --version
95# Libraries for backends
96# -- MAGMA from dev branch
97    - echo "-------------- MAGMA ---------------"
98    - export MAGMA_DIR=/projects/hipMAGMA/install && git -C $MAGMA_DIR describe
99  script:
100    - rm -f .SUCCESS
101# libCEED
102    - make configure HIP_DIR=/opt/rocm OPT='-O -march=native -ffp-contract=fast'
103    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*')
104    - echo "-------------- libCEED -------------" && make info
105    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
106    - make -j$NPROC_CPU
107# -- libCEED only tests
108    - echo "-------------- core tests ----------"
109    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
110#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
111    - export PETSC_DIR= PETSC_ARCH=
112    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=%
113# Libraries for examples
114# -- PETSc with HIP (minimal)
115    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR describe
116    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
117    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids"
118# -- MFEM v4.2
119    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
120    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
121    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=mfem
122# -- Nek5000 v19.0
123    - export COVERAGE=0
124    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
125    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
126    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=nek
127# Clang-tidy
128    - echo "-------------- clang-tidy ----------" && clang-tidy --version
129    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --exit-code
130# Report status
131    - touch .SUCCESS
132  after_script:
133    - |
134      if [ -f .SUCCESS ]; then
135        lcov --directory . --capture --output-file coverage.info;
136        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
137        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
138        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
139        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
140        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
141      fi
142  artifacts:
143    paths:
144      - build/*.junit
145    reports:
146      junit: build/*.junit
147      performance: performance.json
148
149noether-float:
150  stage: test:gpu-and-float
151  tags:
152    - cpu
153    - rocm
154  interruptible: true
155  before_script:
156# Environment
157    - export COVERAGE=1 CC=gcc CXX=g++ FC= HIPCC=hipcc
158    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
159    - echo "-------------- CC ------------------" && $CC --version
160    - echo "-------------- CXX -----------------" && $CXX --version
161    - echo "-------------- HIPCC ---------------" && $HIPCC --version
162    - echo "-------------- GCOV ----------------" && gcov --version
163# Libraries for backends
164# -- MAGMA from dev branch
165    - echo "-------------- MAGMA ---------------"
166    - export MAGMA_DIR=/projects/hipMAGMA/install && git -C $MAGMA_DIR describe
167# -- LIBXSMM v1.17
168    - cd .. && export XSMM_VERSION=libxsmm-1.17 && { [[ -d $XSMM_VERSION ]] || { git clone --depth 1 --branch 1.17 https://github.com/hfp/libxsmm.git $XSMM_VERSION && make -C $XSMM_VERSION -j$(nproc); }; } && export XSMM_DIR=$PWD/$XSMM_VERSION && cd libCEED
169    - echo "-------------- LIBXSMM -------------" && git -C $XSMM_DIR describe --tags
170  script:
171    - rm -f .SUCCESS
172# libCEED
173# Change to single precision
174    - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/ceed.h
175# Build libCEED
176    - make configure HIP_DIR=/opt/rocm OPT='-O -march=native -ffp-contract=fast'
177    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*')
178    - echo "-------------- libCEED -------------" && make info
179    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
180    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
181    - make -j$NPROC_CPU
182# -- libCEED only tests
183    - echo "-------------- core tests ----------"
184    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
185#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
186    - export PETSC_DIR= PETSC_ARCH=
187    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="float-cpu" junit realsearch=%
188    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-hip" junit realsearch=%
189# Report status
190    - echo "SUCCESS" > .job_status
191  after_script:
192    - |
193      if [ $(cat .job_status) == "SUCCESS" ]; then
194        lcov --directory . --capture --output-file coverage.info;
195        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
196        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
197        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
198        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
199        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
200      fi
201    - sed -i 's/ceed-f32/ceed-f64/1' include/ceed/ceed.h
202  artifacts:
203    paths:
204      - build/*.junit
205    reports:
206      junit: build/*.junit
207      performance: performance.json
208
209lv-cuda:
210  stage: test:gpu-and-float
211  tags:
212    - cuda
213  interruptible: true
214  before_script:
215# Environment
216    - ulimit -v $[1024*1024*32] # 32 GiB in units of 1024 bytes
217    - . /opt/rh/gcc-toolset-10/enable
218    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran
219    - export CUDA_DIR=/usr/local/cuda-11.6
220    - export CUDA_VISIBLE_DEVICES=GPU-c4529365-8229-f689-b43d-ccd7f1677079 # our RTX 2080 Super via nvidia-smi -L
221    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=4 && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
222    - echo "-------------- CC ------------------" && $CC --version
223    - echo "-------------- CXX -----------------" && $CXX --version
224    - echo "-------------- FC ------------------" && $FC --version
225    - echo "-------------- NVCC ----------------" && $CUDA_DIR/bin/nvcc --version
226  script:
227    - rm -f .SUCCESS
228# libCEED
229    - make configure OPT='-O -march=native -ffp-contract=fast'
230    - echo "-------------- libCEED -------------" && make info
231    - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*')
232    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
233    - nice make -k -j$NPROC_CPU -l$NPROC_CPU
234# -- libCEED only tests
235    - echo "-------------- core tests ----------"
236#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
237    - export PETSC_DIR= PETSC_ARCH=
238    - nice make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit realsearch=%
239# Libraries for examples
240# -- PETSc with CUDA (minimal)
241    - export PETSC_DIR=/home/jeth8984/petsc PETSC_ARCH=cuda-O && git -C $PETSC_DIR describe
242    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
243#     Note: Skipping fluids and solids due to CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES on RTX 2080 Super
244    - nice make -k -j$NPROC_GPU JUNIT_BATCH="cuda" junit BACKENDS="$BACKENDS_GPU" search="petsc"
245# Report status
246    - touch .SUCCESS
247  after_script:
248    - |
249      if [ -f .SUCCESS ]; then
250        lcov --directory . --capture --output-file coverage.info;
251        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
252        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
253        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
254        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
255        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
256      fi
257  artifacts:
258    paths:
259      - build/*.junit
260    reports:
261      junit: build/*.junit
262