# ----------------------------------------------------------------------------------------
# HONEE GitLab CI
# ----------------------------------------------------------------------------------------
stages:
  - test:stage-lint
  - test:stage-full
  - test:post
  - test:docs
  - deploy

workflow:
  auto_cancel:
    on_job_failure: all

.test-basic:
  interruptible: true
  only:
    refs:
      - web
      - merge_requests

.test:
  extends: .test-basic
  only:
    refs:
      - web
      - merge_requests
      - main
      - release
  except:
    variables:
      # Skip if the No-Code label is attached to a merge request (i.e., documentation only)
      - $CI_MERGE_REQUEST_LABELS =~ /(^|,)No-Code($|,)/

.docs:
  image: python:3.10
  before_script:
    - pip install -r doc/requirements.txt
    - apt-get update
    - apt-get install -y doxygen librsvg2-bin


# ----------------------------------------------------------------------------------------
# Test formatting
# ----------------------------------------------------------------------------------------
noether-format:
  stage: test:stage-lint
  extends: 
    - .test-basic
  tags:
    - noether
    - shell
  script:
    - rm -f .SUCCESS
    - echo "-------------- make format ---------" && export CLANG_FORMAT=clang-format-15 && $CLANG_FORMAT --version
    - make -j$NPROC_CPU format && git diff --color=always --exit-code
    - echo "-------------- make checkbadSource ---------"
    - make -j$NPROC_CPU checkbadSource
    - touch .SUCCESS


# ----------------------------------------------------------------------------------------
# Test static analysis
# ----------------------------------------------------------------------------------------
noether-lint:
  stage: test:stage-lint
  extends: .test
  tags:
    - noether
    - shell
  script:
    - rm -f .SUCCESS
    # Environment
    - export CC=gcc
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    # Libraries
    # -- libCEED
    - echo "-------------- libCEED -------------"
    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
    # -- PETSc
    - echo "-------------- PETSc ---------------"
    - export PETSC_DIR=/projects/honee/petsc
    - export PETSC_ARCH=arch-serial-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
    - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff
    - echo "-------------- HONEE ---------------" && make info
    # make with Werror, Wall, supress loop vectorization warnings
    - echo "-------------- make Werror ---------"
    - PEDANTIC=1 PEDANTICFLAGS="-Werror -Wall -Wno-pass-failed" make -j$NPROC_CPU
    # Clang-tidy
    - echo "-------------- clang-tidy ----------" && export CLANG_TIDY=clang-tidy-15 && $CLANG_TIDY --version
    - PETSC_ARCH=arch-serial-cuda make -j$NPROC_CPU tidy
    # Report status
    - touch .SUCCESS


# ----------------------------------------------------------------------------------------
# Test memory access assumptions
# ----------------------------------------------------------------------------------------
noether-memcheck:
  stage: test:stage-lint
  extends: .test
  tags:
    - noether
    - shell
  script:
    - rm -f .SUCCESS
    # Environment
    - export COVERAGE=1 CC=clang-15
    - export NPROC_POOL=8
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    # Libraries
    # -- libCEED
    - echo "-------------- libCEED -------------"
    - export CEED_DIR=/projects/honee/libCEED-cpu && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
    # -- PETSc
    - echo "-------------- PETSc ---------------"
    - export PETSC_DIR=/projects/honee/petsc
    - export PETSC_ARCH=arch-serial-cpu-clang && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
    - export LD_LIBRARY_PATH=$PETSC_DIR/lib PATH="$PATH:$PETSC_DIR/bin" # cgnsdiff
    - echo "-------------- HONEE ---------------" && make info
    # ASAN
    - echo "-------------- ASAN ----------------"
    - export ASAN=1 AFLAGS="-fsanitize=address -fsanitize=leak"
    - echo $AFLAGS
    # HONEE
    - echo "-------------- HONEE ---------------" && make info
    - make clean
    - make -j$NPROC_CPU
    # Test suite
    - echo "-------------- HONEE tests ---------"
    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
    # -- Memcheck libCEED CPU backend, serial
    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL)) CEED_BACKENDS="/cpu/self/memcheck" JUNIT_BATCH="cpu-serial-memcheck" junit search=navierstokes
    # Report status
    - touch .SUCCESS
  after_script:
    - |
      if [ -f .SUCCESS ]; then
        gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json;
      fi
  artifacts:
    paths:
      - coverage-$CI_JOB_ID.json
      - build/*.junit
      - build/test_failure_artifacts
    when: always
    reports:
      junit: build/*.junit
      performance: performance.json
    expire_in: 28 days


# ----------------------------------------------------------------------------------------
# CPU testing on Noether
# ----------------------------------------------------------------------------------------
noether-cpu:
  stage: test:stage-full
  extends: .test
  tags:
    - noether
    - shell
  script:
    - rm -f .SUCCESS
    # Environment
    - export COVERAGE=1 CC=gcc
    - export NPROC_POOL=4
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    - echo "-------------- GCOV ----------------" && gcov --version
    # Libraries
    # -- libCEED
    - echo "-------------- libCEED -------------"
    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
    # -- PETSc
    - echo "-------------- PETSc ---------------"
    - export PETSC_DIR=/projects/honee/petsc
    - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
    - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff
    # HONEE
    - echo "-------------- HONEE ---------------" && make info
    - make clean
    - make -j$NPROC_CPU
    # Test suite
    - echo "-------------- HONEE tests ---------"
    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
    # -- Fastest libCEED CPU backend, parallel
    - echo "Parallel tests skipped for now"
    # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1
    - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install
    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search=navierstokes
    # - spack unload py-torch@2.3+cuda && export USE_TORCH=0
    - source /projects/honee/SmartSim/venv/bin/activate
    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial" junit search="test-smartsim"
    # Report status
    - touch .SUCCESS
  after_script:
    - |
      if [ -f .SUCCESS ]; then
        gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json;
      fi
  artifacts:
    paths:
      - coverage-$CI_JOB_ID.json
      - build/*.junit
      - build/test_failure_artifacts
    when: always
    reports:
      junit: build/*.junit
      performance: performance.json
    expire_in: 28 days


# ----------------------------------------------------------------------------------------
# CPU Int64 testing on Noether
# ----------------------------------------------------------------------------------------
noether-cpu-int64:
  stage: test:stage-full
  extends: .test
  tags:
    - noether
    - shell
  script:
    - rm -f .SUCCESS
    # Environment
    - export COVERAGE=1 CC=gcc
    - export NPROC_POOL=4
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    - echo "-------------- GCOV ----------------" && gcov --version
    # Libraries
    # -- libCEED
    - echo "-------------- libCEED -------------"
    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
    # -- PETSc
    - echo "-------------- PETSc ---------------"
    - export PETSC_DIR=/projects/honee/petsc
    - export PETSC_ARCH=arch-serial-cpu-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
    - export PETSC_OPTIONS='-malloc_debug no' # faster tests
    - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff
    # HONEE
    - echo "-------------- HONEE ---------------" && make info
    - make clean
    - make -j$NPROC_CPU
    # Test suite
    - echo "-------------- HONEE tests ---------"
    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
    # -- Fastest libCEED CPU backend, serial
    # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1
    - export SMARTREDIS_DIR=/projects/honee/SmartSim/smartredis/install
    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search=navierstokes
    # - spack unload py-torch@2.3+cuda && export USE_TORCH=0
    - source /projects/honee/SmartSim/venv/bin/activate
    - NPROC_TEST=1 make -k -j$((NPROC_CPU / NPROC_POOL / 1)) CEED_BACKENDS="/cpu/self" JUNIT_BATCH="cpu-serial-int64" junit search="test-smartsim"
    # Report status
    - touch .SUCCESS
  after_script:
    - |
      if [ -f .SUCCESS ]; then
        gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json;
      fi
  artifacts:
    paths:
      - coverage-$CI_JOB_ID.json
      - build/*.junit
      - build/test_failure_artifacts
    when: always
    reports:
      junit: build/*.junit
      performance: performance.json
    expire_in: 28 days


#### Disable HIP temporarily

# # ----------------------------------------------------------------------------------------
# # GPU testing on Noether
# # ----------------------------------------------------------------------------------------
# noether-hip:
#   stage: test:stage-full
#   extends: .test
#   tags:
#     - noether
#     - shell
#   script:
#     - rm -f .SUCCESS
#     # Environment
#     - export COVERAGE=1 CC=gcc HIPCC=hipcc
#     - export NPROC_POOL=4
#     - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
#     - echo "-------------- CC ------------------" && $CC --version
#     - echo "-------------- HIPCC ---------------" && $HIPCC --version && export HIP_DIR=/opt/rocm
#     - echo "-------------- GCOV ----------------" && gcov --version
#     # Libraries
#     # -- libCEED
#     - echo "-------------- libCEED -------------"
#     - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
#     # -- PETSc
#     - echo "-------------- PETSc ---------------"
#     - export PETSC_DIR=/projects/honee/petsc
#     - export PETSC_ARCH=arch-parallel-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
#     - export PETSC_OPTIONS='-malloc_debug no' # faster tests
#     - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff
#     # HONEE
#     - echo "-------------- HONEE ---------------" && make info
#     - make clean
#     - make -j$NPROC_CPU
#     # Test suite
#     - echo "-------------- HONEE tests ---------"
#     - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
#     # -- Fastest libCEED HIP backend, serial
#     # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes
#     - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/hip/shared" JUNIT_BATCH="hip-serial" junit search=navierstokes
#     # Report status
#     - touch .SUCCESS
#   after_script:
#     - |
#       if [ -f .SUCCESS ]; then
#         gcovr --xml-pretty --exclude-lines-by-pattern '^\s*SETERR.*' --exclude-unreachable-branches --print-summary -o coverage.xml;
#       fi
#   after_script:
#     - |
#       if [ -f .SUCCESS ]; then
#         gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json;
#       fi
#   artifacts:
#     paths:
#       - coverage-$CI_JOB_ID.json
#       - build/*.junit
#       - build/test_failure_artifacts
#     when: always
#     reports:
#       junit: build/*.junit
#       performance: performance.json
#     expire_in: 28 days


noether-cuda:
  stage: test:stage-full
  extends: .test
  tags:
    - noether
    - shell
  script:
    - rm -f .SUCCESS
    # Environment
    - export COVERAGE=1 CC=gcc NVCC=nvcc
    - export NPROC_POOL=4
    - echo "-------------- nproc ---------------" && NPROC_CPU=$(nproc) && NPROC_GPU=$(($(nproc)<8?$(nproc):8)) && echo "NPROC_CPU" $NPROC_CPU && echo "NPROC_GPU" $NPROC_GPU
    - echo "-------------- CC ------------------" && $CC --version
    - echo "-------------- NVCC ----------------" && $NVCC --version
    - echo "-------------- GCOV ----------------" && gcov --version
    # Libraries
    # -- libCEED
    - echo "-------------- libCEED -------------"
    - export CEED_DIR=/projects/honee/libCEED && git -C $CEED_DIR -c safe.directory=$CEED_DIR describe && make -C $CEED_DIR info
    # -- PETSc
    - echo "-------------- PETSc ---------------"
    - export PETSC_DIR=/projects/honee/petsc
    - export PETSC_ARCH=arch-parallel-cuda && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe && make -C $PETSC_DIR info
    - export PETSC_OPTIONS='-malloc_debug no -use_gpu_aware_mpi 0' # faster tests
    - export LD_LIBRARY_PATH=$PETSC_DIR/$PETSC_ARCH/lib PATH="$PATH:$PETSC_DIR/$PETSC_ARCH/bin" # cgnsdiff
    # HONEE
    - echo "-------------- HONEE ---------------" && make info
    - make clean
    - make -j$NPROC_CPU
    # Test suite
    - echo "-------------- HONEE tests ---------"
    - echo '[{"subject":"/","metrics":[{"name":"Transfer Size (KB)","value":"19.5","desiredSize":"smaller"},{"name":"Speed Index","value":0,"desiredSize":"smaller"},{"name":"Total Score","value":92,"desiredSize":"larger"},{"name":"Requests","value":4,"desiredSize":"smaller"}]}]' > performance.json
    # -- Fastest libCEED CUDA backend, serial
    # Note: /shared is faster due to /gen JiT time for CeedOperators overwhelming runtime improvements at these problem sizes
    # - source /home/phypid/spack/share/spack/setup-env.sh && spack load py-torch@2.3+cuda && export USE_TORCH=1
    - NPROC_TEST=1 make -k -j$((NPROC_GPU / NPROC_POOL / 1)) CEED_BACKENDS="/gpu/cuda/shared" JUNIT_BATCH="cuda-serial" junit search=navierstokes
    # Report status
    - touch .SUCCESS
  after_script:
    - |
      if [ -f .SUCCESS ]; then
        gcovr --gcov-executable "llvm-cov-15 gcov" --json-pretty --print-summary -o coverage-$CI_JOB_ID.json;
      fi
  artifacts:
    paths:
      - coverage-$CI_JOB_ID.json
      - build/*.junit
      - build/test_failure_artifacts
    when: always
    reports:
      junit: build/*.junit
      performance: performance.json
    expire_in: 28 days


test-coverage:
  stage: test:post
  extends: .test
  tags:
    - noether
    - shell
  when: always
  needs: [noether-memcheck, noether-cpu, noether-cpu-int64, noether-cuda, ]
  script:
    - ls coverage-*
    - mkdir coverage
    - gcovr --json-add-tracefile "coverage-*.json" --xml-pretty -o coverage.xml --print-summary --html-nested coverage/index.html --html-theme github.dark-blue
  coverage: '/^lines:\s+(\d+.\d\%)/'
  artifacts:
    paths:
      - coverage.xml
      - coverage
    when: always
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
    expire_in: 28 days
  environment:
    name: coverage/$CI_COMMIT_REF_NAME
    url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/coverage/index.html


# ----------------------------------------------------------------------------------------
# Build documentation
# ----------------------------------------------------------------------------------------
docs-review:
  stage: test:docs
  tags:
    - noether
    - docker
  extends:
    - .docs
    - .test-basic
  interruptible: true
  script:
    - export PETSC_DIR=/projects/honee/petsc PETSC_ARCH=arch-parallel-cuda CEED_DIR=/projects/honee/libCEED
    - git -c safe.directory=/builds/phypid/honee submodule update --init
    - make doc-html pkgconf=true DOXYGENOPTS= SPHINXOPTS=-W
    - mv doc/build/html public
  artifacts:
    paths:
      - public
    expire_in: 28 days
  environment:
    name: review/$CI_COMMIT_REF_NAME
    url: https://$CI_PROJECT_NAMESPACE.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html


# ----------------------------------------------------------------------------------------
# Deploy documentation using GitLab pages
# ----------------------------------------------------------------------------------------
pages:  # this job name has special meaning to GitLab
  stage: deploy
  tags:
    - noether
    - docker
  extends: .docs
  interruptible: false
  script:
    - git -c safe.directory=/builds/phypid/honee submodule update --init
    - make doc-dirhtml pkgconf=true DOXYGENOPTS=
    - mv doc/build/dirhtml public
  only:
    - main
  artifacts:
    paths:
      - public