xref: /libCEED/.gitlab-ci.yml (revision f849ae497c092d3a29613f4bb564326c8fd73f8a) 
1stages:
2  - test
3
4noether-rocm:
5  stage: test
6  tags:
7    - rocm
8  interruptible: true
9  script:
10    - rm -f .SUCCESS
11# Compilers
12    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
13    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
14    - echo "-------------- CC ------------------" && $CC --version
15    - echo "-------------- CXX -----------------" && $CXX --version
16    - echo "-------------- FC ------------------" && $FC --version
17    - echo "-------------- HIPCC ---------------" && $HIPCC --version
18    - echo "-------------- GCOV ----------------" && gcov --version
19# Libraries for backends
20# -- MAGMA from dev branch
21    - echo "-------------- MAGMA ---------------"
22    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR describe
23# -- LIBXSMM v1.16.1
24    - cd .. && export XSMM_VERSION=libxsmm-1.16.1 && { [[ -d $XSMM_VERSION ]] || { git clone --depth 1 --branch 1.16.1 https://github.com/hfp/libxsmm.git $XSMM_VERSION && make -C $XSMM_VERSION -j$(nproc); }; } && export XSMM_DIR=$PWD/$XSMM_VERSION && cd libCEED
25    - echo "-------------- LIBXSMM -------------" && git -C $XSMM_DIR describe --tags
26# -- OCCA v1.1.0
27    - cd .. && export OCCA_VERSION=occa-1.1.0 OCCA_OPENCL_ENABLED=0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.1.0 https://github.com/libocca/occa.git $OCCA_VERSION && make -C $OCCA_VERSION -j$(nproc); }; } && export OCCA_DIR=$PWD/$OCCA_VERSION && cd libCEED
28    - echo "-------------- OCCA ----------------" && make -C $OCCA_DIR info
29# libCEED
30    - make configure HIP_DIR=/opt/rocm OPT='-O -march=native -ffp-contract=fast'
31#    Note: OCCA backends currently disabled in CI
32    - BACKENDS_CPU=$(OCCA_DIR= make info-backends-all | grep -o '/cpu[^ ]*') && BACKENDS_GPU=$(OCCA_DIR= make info-backends | grep -o '/gpu[^ ]*')
33    - echo "-------------- libCEED -------------" && make info
34    - echo "-------------- BACKENDS_CPU---------" && echo $BACKENDS_CPU
35    - echo "-------------- BACKENDS_GPU---------" && echo $BACKENDS_GPU
36    - make -j$NPROC_CPU
37# -- libCEED only tests
38    - echo "-------------- core tests ----------"
39    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
40#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
41    - export PETSC_DIR= PETSC_ARCH=
42    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit realsearch=%
43    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit realsearch=%
44# Libraries for examples
45# -- PETSc with HIP (minimal)
46    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR describe
47    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
48    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit search="petsc fluids solids"
49    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit search="petsc fluids solids"
50# -- MFEM v4.2
51    - cd .. && export MFEM_VERSION=mfem-4.2 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.2 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
52    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
53    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit search=mfem
54    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit search=mfem
55# -- Nek5000 v19.0
56    - export COVERAGE=0
57    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
58    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
59    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit search=nek
60    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit search=nek
61# Clang-tidy
62    - echo "-------------- clang-tidy ----------" && clang-tidy --version
63    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --exit-code
64# Report status
65    - touch .SUCCESS
66  after_script:
67    - |
68      if [ -f .SUCCESS ]; then
69        lcov --directory . --capture --output-file coverage.info;
70        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
71        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
72        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
73        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
74        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
75      fi
76  artifacts:
77    paths:
78      - build/*.junit
79    reports:
80      junit: build/*.junit
81      performance: performance.json
82
83noether-float:
84  stage: test
85  tags:
86    - rocm
87  interruptible: true
88  script:
89# Compilers
90    - export COVERAGE=1 CC=gcc CXX=g++ FC= HIPCC=hipcc
91    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
92    - echo "-------------- CC ------------------" && $CC --version
93    - echo "-------------- CXX -----------------" && $CXX --version
94    - echo "-------------- HIPCC ---------------" && $HIPCC --version
95    - echo "-------------- GCOV ----------------" && gcov --version
96# Libraries for backends
97# -- MAGMA from dev branch
98    - echo "-------------- MAGMA ---------------"
99    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR describe
100# -- LIBXSMM v1.16.1
101    - cd .. && export XSMM_VERSION=libxsmm-1.16.1 && { [[ -d $XSMM_VERSION ]] || { git clone --depth 1 --branch 1.16.1 https://github.com/hfp/libxsmm.git $XSMM_VERSION && make -C $XSMM_VERSION -j$(nproc); }; } && export XSMM_DIR=$PWD/$XSMM_VERSION && cd libCEED
102    - echo "-------------- LIBXSMM -------------" && git -C $XSMM_DIR describe --tags
103# libCEED
104# First change to single precision
105    - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/ceed.h
106# Build libCEED
107    - make configure HIP_DIR=/opt/rocm OPT='-O -march=native -ffp-contract=fast'
108    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*')
109    - echo "-------------- libCEED -------------" && make info
110    - echo "-------------- BACKENDS_CPU---------" && echo $BACKENDS_CPU
111    - echo "-------------- BACKENDS_GPU---------" && echo $BACKENDS_GPU
112    - make -j$NPROC_CPU
113# -- libCEED only tests
114    - echo "-------------- core tests ----------"
115    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
116#    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
117    - export PETSC_DIR= PETSC_ARCH=
118    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" junit realsearch=%
119    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" junit realsearch=%
120# Report status
121    - echo "SUCCESS" > .job_status
122  after_script:
123    - |
124      if [ $(cat .job_status) == "SUCCESS" ]; then
125        lcov --directory . --capture --output-file coverage.info;
126        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
127        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
128        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
129        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
130        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
131      fi
132    - sed -i 's/ceed-f32/ceed-f64/1' include/ceed/ceed.h
133  artifacts:
134    paths:
135      - build/*.junit
136    reports:
137      junit: build/*.junit
138      performance: performance.json
139
140lv-cuda:
141  stage: test
142  tags:
143    - cuda
144  interruptible: true
145  before_script:
146# Compilers
147    - ulimit -v $[1024*1024*32] # 32 GiB in units of 1024 bytes
148    - . /opt/rh/gcc-toolset-10/enable
149    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran
150    - export CUDA_DIR=/usr/local/cuda-11.4
151    - export CUDA_VISIBLE_DEVICES=GPU-c4529365-8229-f689-b43d-ccd7f1677079 # our RTX 2080 Super via nvidia-smi -L
152    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=1 && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
153    - echo "-------------- CC ------------------" && $CC --version
154    - echo "-------------- CXX -----------------" && $CXX --version
155    - echo "-------------- FC ------------------" && $FC --version
156    - echo "-------------- NVCC ----------------" && $CUDA_DIR/bin/nvcc --version
157# libCEED
158    - make configure OPT='-O -march=native -ffp-contract=fast'
159    - echo "-------------- libCEED -------------" && make info
160    - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*')
161    - echo "-------------- BACKENDS_GPU---------" && echo $BACKENDS_GPU
162    - export PETSC_DIR=/home/jeth8984/petsc PETSC_ARCH=cuda-O && git -C $PETSC_DIR describe
163  script:
164    - rm -f .SUCCESS
165    - nice make -k -j$NPROC_CPU -l$NPROC_CPU
166    - echo "-------------- quick CPU tests to get everything built (no junit) --------------"
167    - nice make -k -j$NPROC_CPU -l$NPROC_CPU test BACKENDS=/cpu/self/avx/blocked realsearch=%
168    - echo "-------------- CUDA tests without PETSc (makes *.junit) --------------"
169    - nice make -k -j$NPROC_GPU junit PETSC_DIR= BACKENDS="$BACKENDS_GPU" realsearch=%
170    - echo "-------------- CUDA tests with PETSc (makes *.junit) --------------"
171# skip fluids and solids due to CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES on RTX 2080 Super
172    - nice make -k -j$NPROC_GPU junit BACKENDS="$BACKENDS_GPU" search="petsc"
173# Report status
174    - touch .SUCCESS
175  after_script:
176    - |
177      if [ -f .SUCCESS ]; then
178        lcov --directory . --capture --output-file coverage.info;
179        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
180        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
181        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
182        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
183        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
184      fi
185  artifacts:
186    paths:
187      - build/*.junit
188    reports:
189      junit: build/*.junit
190