xref: /libCEED/.gitlab-ci.yml (revision ed66640a7bddfba6ec5e31af0b66705275f047c9)

# ----------------------------------------------------------------------------------------
# libCEED GitLab CI
# ----------------------------------------------------------------------------------------
stages:
  - test:cpu-and-tidy
  - test:gpu-and-float

workflow:
  auto_cancel:
    on_job_failure: all


# ----------------------------------------------------------------------------------------
# Memcheck backends + ASAN
# ----------------------------------------------------------------------------------------
noether-asan:
  stage: test:cpu-and-tidy
  tags:
    - cpu
  interruptible: true
  before_script:
    # Environment
    #    Note: COVERAGE=0 is needed when using ASAN
    - export COVERAGE=0 CC=gcc CXX=g++ FC=gfortran
    - export NPROC_POOL=8
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    - echo "-------------- CXX -----------------" && $CXX --version
    - echo "-------------- FC ------------------" && $FC --version
    # ASAN
    - echo "-------------- ASAN ----------------"
    - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak"
    - echo $AFLAGS
  script:
    - rm -f .SUCCESS
    # libCEED
    - make configure OPT='-O -march=native -ffp-contract=fast'
    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
    - echo "-------------- libCEED -------------" && make info
    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
    - make clean
    - make -j$NPROC_CPU
    # -- libCEED only tests
    - echo "-------------- core tests ----------"
    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
    - export PETSC_DIR= PETSC_ARCH=
    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="memcheck" junit realsearch=%
    # Clang-tidy
    - echo "-------------- clang-tidy ----------" && clang-tidy --version
    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
    # Report status
    - touch .SUCCESS
  artifacts:
    paths:
      - build/*.junit
    reports:
      junit: build/*.junit
      performance: performance.json


# ----------------------------------------------------------------------------------------
# CPU backends only + make tidy
# ----------------------------------------------------------------------------------------
noether-cpu:
  stage: test:cpu-and-tidy
  tags:
    - cpu
  interruptible: true
  before_script:
    # Environment
    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran
    - export NPROC_POOL=8
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    - echo "-------------- CXX -----------------" && $CXX --version
    - echo "-------------- FC ------------------" && $FC --version
    - echo "-------------- GCOV ----------------" && gcov --version
    # Libraries for backends
    # -- LIBXSMM 7 April 2024
    - cd .. && export XSMM_HASH=94ee71576870152feb62f3f0cf6b061d036dcdb5 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
    # -- OCCA v1.6.0
    - cd .. && export OCCA_VERSION=occa-1.6.0 && { [[ -d $OCCA_VERSION ]] || { git clone --depth 1 --branch v1.6.0 https://github.com/libocca/occa.git $OCCA_VERSION && cd $OCCA_VERSION && export ENABLE_OPENCL="OFF" ENABLE_DPCPP="OFF" ENABLE_HIP="OFF" ENABLE_CUDA="OFF" && ./configure-cmake.sh && cmake --build build --parallel $NPROC_CPU && cmake --install build && cd ..; }; } && export OCCA_DIR=$PWD/$OCCA_VERSION/install && cd libCEED
    - echo "-------------- OCCA ----------------" && git -C $OCCA_DIR describe --tags && LD_LIBRARY_PATH=$OCCA_DIR/lib $OCCA_DIR/bin/occa info
  script:
    - rm -f .SUCCESS
    # libCEED
    - make configure OPT='-O -march=native -ffp-contract=fast'
    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
    - echo "-------------- libCEED -------------" && make info
    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
    - make clean
    - OCCA_DIR= PEDANTIC=1 make -j$NPROC_CPU
    - make -j$NPROC_CPU
    # -- libCEED only tests
    - echo "-------------- core tests ----------"
    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
    - export PETSC_DIR= PETSC_ARCH=
    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit realsearch=%
    # Libraries for examples
    # -- PETSc (minimal)
    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search="petsc fluids-navierstokes solids"
    # -- MFEM v4.7
    - cd .. && export MFEM_VERSION=mfem-4.7 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.7 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=mfem
    # -- Nek5000 v19.0
    - export COVERAGE=0
    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
    - export NPROC_POOL=1
    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=nek NEK5K_DIR=$NEK5K_DIR
    # -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11
    - OCCA_DIR= BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ')
    - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install
    - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always
    - make -k -j$NPROC_CPU BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR
    # Report status
    - touch .SUCCESS
  after_script:
    - |
      if [ -f .SUCCESS ]; then
        lcov --directory . --capture --output-file coverage.info;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
      fi
  artifacts:
    paths:
      - build/*.junit
    reports:
      junit: build/*.junit
      performance: performance.json


# ----------------------------------------------------------------------------------------
# Check SYCL backends build
# ----------------------------------------------------------------------------------------

# SYCL tests currently disabled

#noether-sycl:
#  stage: test:gpu-and-float
#  tags:
#    - sycl
#  interruptible: true
#  before_script:
#    # Environment
#    - . /opt/intel/oneapi/setvars.sh
#    - export COVERAGE=1 CC=icx CXX=icpx
#    - export NPROC_POOL=8
#    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
#    - echo "-------------- CC ------------------" && $CC --version
#    - echo "-------------- CXX -----------------" && $CXX --version
#  script:
#    - rm -f .SUCCESS
#    # libCEED
#    - make configure SYCL_DIR=/opt/intel/oneapi/compiler/latest OPT='-O -march=native -ffp-contract=fast'
#    - BACKENDS_SYCL=$(make info-backends-all | grep -o '/sycl[^ ]*' | tr '\n' ' ')
#    - echo "-------------- libCEED -------------" && make info
#    - echo "-------------- BACKENDS_SYCL -------" && echo $BACKENDS_SYCL
#    - make clean
#    - make -j$NPROC_CPU
#    # Report status
#    - touch .SUCCESS


# ----------------------------------------------------------------------------------------
# CUDA backends
# ----------------------------------------------------------------------------------------
noether-cuda:
  stage: test:gpu-and-float
  tags:
    - cuda
  interruptible: true
  before_script:
    # Environment
    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran NVCC=nvcc
    - export NPROC_POOL=4
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    - echo "-------------- CXX -----------------" && $CXX --version
    - echo "-------------- FC ------------------" && $FC --version
    - echo "-------------- NVCC ----------------" && $NVCC --version
    - echo "-------------- GCOV ----------------" && gcov --version
    # ASAN
    - echo "-------------- ASAN ----------------"
    - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak" ASAN_OPTIONS=protect_shadow_gap=0
    - echo $AFLAGS
  script:
    - rm -f .SUCCESS
    # libCEED
    - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr
    - echo "-------------- libCEED -------------" && make info
    - BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
    - make clean
    - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU
    # -- libCEED only tests
    - echo "-------------- core tests ----------"
    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
    - export PETSC_DIR= PETSC_ARCH=
    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit realsearch=%
    # Rebuild without ASAN
    - unset ASAN AFLAGS ASAN_OPTIONS
    - make clean
    - PEDANTIC=1 make -k -j$NPROC_CPU -l$NPROC_CPU
    # Libraries for examples
    # -- PETSc with CUDA (minimal)
    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-cuda-O PETSC_OPTIONS='-use_gpu_aware_mpi 0' && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search="petsc fluids solids"
    # -- MFEM v4.7
    - cd .. && export MFEM_VERSION=mfem-4.7 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.7 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
    - echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=mfem
    # -- Nek5000 v19.0
    - export COVERAGE=0
    - cd .. && export NEK5K_VERSION=Nek5000-19.0 && { [[ -d $NEK5K_VERSION ]] || { git clone --depth 1 --branch v19.0 https://github.com/Nek5000/Nek5000.git $NEK5K_VERSION && cd $NEK5K_VERSION/tools && ./maketools genbox genmap reatore2 && cd ../..; }; } && export NEK5K_DIR=$PWD/$NEK5K_VERSION && export PATH=$NEK5K_DIR/bin:$PATH MPI=0 && cd libCEED
    - echo "-------------- Nek5000 -------------" && git -C $NEK5K_DIR describe --tags
    - export NPROC_POOL=1
    - make -k -j$NPROC_GPU BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=nek NEK5K_DIR=$NEK5K_DIR
    # -- deal.II 8bd5c262f13e15793aa206b6eed8774a9b25ce11
    - export DEAL_II_ROOT_DIR=/projects/dealii DEAL_II_DIR=/projects/dealii/install
    - echo "-------------- deal.II -------------" && git -C $DEAL_II_ROOT_DIR -c safe.directory=$DEAL_II_ROOT_DIR describe --always
    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="cuda" junit search=dealii DEAL_II_DIR=$DEAL_II_DIR
    # Clang-tidy
    - echo "-------------- clang-tidy ----------" && clang-tidy --version
    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
    # Report status
    - touch .SUCCESS
  after_script:
    - |
      if [ -f .SUCCESS ]; then
        lcov --directory . --capture --output-file coverage.info;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
      fi
  artifacts:
    paths:
      - build/*.junit
    reports:
      junit: build/*.junit


# ----------------------------------------------------------------------------------------
# ROCm backends
# ----------------------------------------------------------------------------------------

# ROCm test execution currently disabled

#noether-rocm:
#  stage: test:gpu-and-float
#  tags:
#    - rocm
#  interruptible: true
#  before_script:
#    # Environment
#    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
#    - export NPROC_POOL=4
#    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
#    - echo "-------------- CC ------------------" && $CC --version
#    - echo "-------------- CXX -----------------" && $CXX --version
#    - echo "-------------- FC ------------------" && $FC --version
#    - echo "-------------- HIPCC ---------------" && $HIPCC --version
#    - echo "-------------- GCOV ----------------" && gcov --version
#    # Libraries for backends
#    # -- MAGMA from dev branch
#    - echo "-------------- MAGMA ---------------"
#    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
#  script:
#    - rm -f .SUCCESS
#    # libCEED
#    - make configure ROCM_DIR=/opt/rocm-6.1.0 OPT='-O -march=native -ffp-contract=fast'
#    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
#    - echo "-------------- libCEED -------------" && make info
#    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
#    - make clean
#    - make -j$NPROC_CPU
#    # -- libCEED only tests
#    - echo "-------------- core tests ----------"
#    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
#    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
#    - export PETSC_DIR= PETSC_ARCH=
#    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit realsearch=%
#    # Libraries for examples
#    # -- PETSc with HIP (minimal)
#    - export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
#    - echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
#    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids"
#    # Clang-tidy
#    - echo "-------------- clang-tidy ----------" && clang-tidy --version
#    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
#    # Report status
#    - touch .SUCCESS
#  after_script:
#    - |
#      if [ -f .SUCCESS ]; then
#        lcov --directory . --capture --output-file coverage.info;
#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
#        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
#      fi
#  artifacts:
#    paths:
#      - build/*.junit
#    reports:
#      junit: build/*.junit
#      performance: performance.json


noether-rocm:
  stage: test:gpu-and-float
  tags:
    - rocm
  interruptible: true
  before_script:
    # Environment
    - export COVERAGE=1 CC=gcc CXX=g++ FC=gfortran HIPCC=hipcc
    - export NPROC_POOL=4
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    - echo "-------------- CXX -----------------" && $CXX --version
    - echo "-------------- FC ------------------" && $FC --version
    - echo "-------------- HIPCC ---------------" && $HIPCC --version
    - echo "-------------- GCOV ----------------" && gcov --version
    # Libraries for backends
    # -- MAGMA from dev branch
    - echo "-------------- MAGMA ---------------"
    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
  script:
    - rm -f .SUCCESS
    # libCEED
    - make configure ROCM_DIR=/opt/rocm-6.1.0 OPT='-O -march=native -ffp-contract=fast'
    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
    - echo "-------------- libCEED -------------" && make info
    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
    - make clean
    - make -j$NPROC_CPU
    # Clang-tidy
    - echo "-------------- clang-tidy ----------" && clang-tidy --version
    - make clean
    - TIDY_OPTS="-fix-errors" make -j$NPROC_CPU tidy && git diff --color=always --exit-code
    # Report status
    - touch .SUCCESS


# ----------------------------------------------------------------------------------------
# CPU + CUDA backends with CeedScalar == float (32 bit)
# ----------------------------------------------------------------------------------------
noether-float:
  stage: test:gpu-and-float
  tags:
    - cpu
    - cuda
  interruptible: true
  before_script:
    # Environment
    - export COVERAGE=1 CC=gcc CXX=g++ FC= NVCC=nvcc
    - export NPROC_POOL=8
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    - echo "-------------- CXX -----------------" && $CXX --version
    - echo "-------------- NVCC ----------------" && $NVCC --version
    - echo "-------------- GCOV ----------------" && gcov --version
    # Libraries for backends
# ROCm tests currently disabled
# -- MAGMA from dev branch
#    - echo "-------------- MAGMA ---------------"
#    - export MAGMA_DIR=/projects/hipMAGMA && git -C $MAGMA_DIR -c safe.directory=$MAGMA_DIR describe
    # -- LIBXSMM 19 March 2025
    - cd .. && export XSMM_HASH=ba9d6bc69c421c10f0597d582ea1ace6a6126308 && { [[ -d libxsmm-$XSMM_HASH ]] || { curl -L https://github.com/libxsmm/libxsmm/archive/$XSMM_HASH.tar.gz -o xsmm.tar.gz && tar zvxf xsmm.tar.gz && rm xsmm.tar.gz && make -C libxsmm-$XSMM_HASH -j$(nproc); }; } && export XSMM_DIR=$PWD/libxsmm-$XSMM_HASH && cd libCEED
    - echo "-------------- LIBXSMM -------------" && basename $XSMM_DIR
  script:
    - rm -f .SUCCESS
    # libCEED
    # Change to single precision
    - sed -i 's/ceed-f64/ceed-f32/1' include/ceed/types.h
    # Build libCEED
    - make configure OPT='-O -march=native -ffp-contract=fast' CUDA_DIR=/usr
    - BACKENDS_CPU=$(make info-backends-all | grep -o '/cpu[^ ]*' | tr '\n' ' ') && BACKENDS_GPU=$(make info-backends | grep -o '/gpu[^ ]*' | tr '\n' ' ')
    - echo "-------------- libCEED -------------" && make info
    - echo "-------------- BACKENDS_CPU --------" && echo $BACKENDS_CPU
    - echo "-------------- BACKENDS_GPU --------" && echo $BACKENDS_GPU
    - make clean
    - make -j$NPROC_CPU
    # -- libCEED only tests
    - echo "-------------- core tests ----------"
    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
    #    Note: PETSC_DIR is set by default in GitLab runner env, unsetting to isolate core tests
    - export PETSC_DIR= PETSC_ARCH=
    - make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="float-cpu" junit realsearch=%
    - export NPROC_POOL=4
    - make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="float-cuda" junit realsearch=%
    # Report status
    - echo "SUCCESS" > .job_status
  after_script:
    - |
      if [ $(cat .job_status) == "SUCCESS" ]; then
        lcov --directory . --capture --output-file coverage.info;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F interface;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F gallery;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F backends;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F tests;
        bash <(curl -s https://codecov.io/bash) -f coverage.info -t ${CODECOV_ACCESS_TOKEN} -F examples;
      fi
    - sed -i 's/ceed-f32/ceed-f64/1' include/ceed/types.h
  artifacts:
    paths:
      - build/*.junit
    reports:
      junit: build/*.junit
      performance: performance.json