GitLab CI configuration#
The Continuous Integration (CI) pipeline for the CIF is defined in the file
.gitlab-ci.yml at the root of the repository.
It controls which Docker images are used for each test suite and which test markers are executed on each push.
stages:
- build
- general-tests
- model-tests
- post-tests
- publish
# Do not run the pipeline for merge request events (avoid duplicate jobs)
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
when: never
- when: always
variables:
PYTHON_VERSION: "3.11" # Default Python version to use
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PYCIF_DATATEST: "/tmp/PYCIF_DATA_TEST/"
# CI_DEBUG_TRACE: "true" # For debugging
default:
image:
name: pycif/pycif-ubuntu:1.0
entrypoint: [""]
.python-venv:
before_script:
- python$PYTHON_VERSION -m venv .venv
- source .venv/bin/activate
- pip install --upgrade pip
- python --version ; pip --version # For debugging
# --- Python package related jobs ---------------------------------------------
# Build wheel and sources archive for release
build:
extends: .python-venv
stage: build
rules:
- if: $CI_COMMIT_TAG
script:
- pip install build wheel setuptools setuptools_scm
- python -m build
- echo "BUILD_JOB_ID=$(echo $CI_JOB_ID)" >> build.env
- echo "BUILD_VERSION=$(python -m setuptools_scm)" >> build.env
artifacts:
reports:
dotenv: build.env
paths:
- dist/
# Install PyCIF for multiple Python versions
install:
extends: .python-venv
stage: build
script:
- pip install .
- pip freeze
- python -c "import pycif"
parallel:
matrix:
- PYTHON_VERSION:
- "3.9"
- "3.10"
- "3.11"
cache:
key: pip
paths:
- .cache/pip
# --- Documentation build jobs ------------------------------------------------
# Build the documentation as a test
docs:
extends: .python-venv
stage: general-tests
rules:
- if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_BRANCH != "devel"
script:
- pip install .[docs]
- if [ -d examples_artifact ]; then rsync -az examples_artifact/* examples/; fi
- cd docs
- make html
cache:
key: pip
paths:
- .cache/pip
# Build the documentation and publish it
pages:
extends: docs
stage: publish
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH == "devel" || $CI_COMMIT_BRANCH == "claude"
after_script:
- pwd
- mv docs/build/html/ public/
artifacts:
paths:
- public
# --- Unit tests jobs ---------------------------------------------------------
.pytest:
stage: model-tests
needs: ["install", "basic"] # Do not wait for previous stages to run
dependencies: [] # Prevent downloading artifacts from previous stages
variables:
MARKS: "" # Marks used to select tests, mandatory
TOX_ENVS: "3.9" # Tox environment(s) to uses, optional, defaults to 3.9
TOX_EXTRA_ARGS: "" # Extra arguments to pass to tox, optional
script:
- pip install --upgrade numpy # Upgrade system Numpy so it does not mess GDAL installation
- tox $TOX_EXTRA_ARGS -e $TOX_ENVS -- -m "$MARKS"
after_script:
- mkdir -p coverage
- mv .coverage coverage/.coverage.$CI_JOB_NAME_SLUG
coverage: /TOTAL.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
artifacts:
reports:
junit: report.xml
coverage_report:
coverage_format: cobertura
path: coverage.xml
paths:
- examples_artifact
- coverage
expire_in: 7 day
cache:
- key: pip
paths:
- .cache/pip
# Cache for tox environment "3.9"
# You can add cache for other tox environments if needed
- key:
files:
# .lock file generated by tox with pip-tools
# the corresponding cache re-generated when this file changes
- .tox/3.9/requirements.lock
prefix: $CI_COMMIT_REF_SLUG # Share cache within a branch
paths:
- .tox/3.9
# Template for new tests
# test-template:
# extends: .pytest
# variables:
# MARKS: "foo and bar" # Marks to select tests
# before_script:
# - cmd ... # Commands to be run before the tests (download test data, etc.)
# Basic tests
basic:
extends: .pytest
stage: build
needs: []
variables:
MARKS: "basic"
TOX_ENVS: "3.9,3.10,3.11"
TOX_EXTRA_ARGS: "--parallel auto"
# before_script:
# # Clear cache, might be need if GDAL does not install properly
# - rm -rf .cache/pip
# - rm -rf .tox
cache:
- key: pip
paths:
- .cache/pip
# Cache for tox environments "3.9", "3.10", "3.11"
- key:
files:
- .tox/3.9/requirements.lock
prefix: $CI_COMMIT_REF_SLUG
paths:
- .tox/3.9
- key:
files:
- .tox/3.10/requirements.lock
prefix: $CI_COMMIT_REF_SLUG
paths:
- .tox/3.10
- key:
files:
- .tox/3.11/requirements.lock
prefix: $CI_COMMIT_REF_SLUG
paths:
- .tox/3.11
# Tutorials tests
tutorials:
extends: .pytest
stage: general-tests
variables:
MARKS: "tuto"
before_script:
- bin/download-test-data 0lpOhIa1J3iT6ja -d /tmp/PYCIF_DATA_TEST/RAW/EMISSIONS/
# Generate figures and artifacts for the website
article:
extends: .pytest
stage: general-tests
rules:
- if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
variables:
MARKS: "(dummy and article and inversion and not adjtltest and not uncertainties and bands) or (fwd and ref_config)"
article:default-branch:
extends: .pytest
stage: general-tests
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
MARKS: "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config) or (allsimulations)"
article-uncertainties:
extends: .pytest
stage: general-tests
rules:
- if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
variables:
MARKS: "(dummy and article and inversion and not adjtltest and uncertainties and bands) or (fwd and ref_config)"
article-uncertainties:default-branch:
extends: .pytest
stage: general-tests
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
MARKS: "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config) or (allsimulations)"
# Tests for the gridded_NetCDF plugin
tests-gridded-netcdf:
extends: .pytest
stage: general-tests
variables:
MARKS: "gridded_netcdf"
# Tests for regrid methods
tests-regrid:
extends: .pytest
stage: general-tests
variables:
MARKS: "test_in_ci and regrid"
# Tests for the dummy model
tests-dummy:
extends: .pytest
variables:
MARKS: "test_in_ci and dummy"
# Tests for Chimere (basic)
tests-chimere-basic:
extends: .pytest
variables:
MARKS: "test_in_ci and chimere and (fwd or ref_chimere)"
before_script:
- bin/download-test-data zUpoUXODKFLZJrf -d /tmp/PYCIF_DATA_TEST/
- bin/download-test-data GgTDXL8peF4BOuV -d /tmp/PYCIF_DATA_TEST/
# Tests for Chimere (inputs)
tests-chimere-inputs:
extends: .pytest
variables:
MARKS: "test_in_ci and chimere and inputs"
before_script:
- bin/download-test-data zUpoUXODKFLZJrf -d /tmp/PYCIF_DATA_TEST/
- bin/download-test-data GgTDXL8peF4BOuV -d /tmp/PYCIF_DATA_TEST/
# Tests for Chimere (melchior-short)
tests-chimere-melchior-short:
extends: .pytest
variables:
MARKS: "test_in_ci and chimere and not fwd and melchior and not long"
before_script:
- bin/download-test-data zUpoUXODKFLZJrf -d /tmp/PYCIF_DATA_TEST/
- bin/download-test-data GgTDXL8peF4BOuV -d /tmp/PYCIF_DATA_TEST/
# Tests for LMDZ
tests-lmdz:
extends: .pytest
variables:
MARKS: "test_in_ci and lmdz and (lmdz_acc or lmdz_reg_ico) and acad"
before_script:
- bin/download-test-data Z0KBiONVQLCWHrV -d /tmp/PYCIF_DATA_TEST/LMDZ/
- bin/download-test-data oIemGgBNNbrq6mF -d /tmp/PYCIF_DATA_TEST/LMDZ/
# Tests for FLEXPART
# Tests are split in two jobs.
# Tox is very unstable with FLEXPART for some reason. It might be related to the
# fact that it uses a lot of memory that is not released properly between tests.
# See https://gitlab.in2p3.fr/satinv/cif/-/issues/79
tests-flexpart:
extends: .pytest
retry: 2
variables:
MARKS: "test_in_ci and flexpart and (fwd or flexpart_empa or (not long))"
before_script:
- bin/download-test-data k2LnQLHVUkefbnr -d /tmp/PYCIF_DATA_TEST/
# Second half of FLEXPART tests
tests-flexpart-long-1:
extends: .pytest
retry: 2
variables:
MARKS: "test_in_ci and flexpart and (not fwd) and (not flexpart_empa) and long and constant_increment and control_space"
before_script:
- bin/download-test-data k2LnQLHVUkefbnr -d /tmp/PYCIF_DATA_TEST/
tests-flexpart-long-2:
extends: .pytest
retry: 2
variables:
MARKS: "test_in_ci and flexpart and (not fwd) and (not flexpart_empa) and long and chi_space"
before_script:
- bin/download-test-data k2LnQLHVUkefbnr -d /tmp/PYCIF_DATA_TEST/
# Tests for TM5
tests-tm5:
extends: .pytest
variables:
MARKS: "test_in_ci and tm5 and fwd"
before_script:
- bin/download-test-data qBWaoGcTWqV7fhG -d /tmp/PYCIF_DATA_TEST/TM5/
# --- Jobs to be run after all tests are done ---------------------------------
# Combine coverage from all tests
coverage:
stage: post-tests
script:
- pip install coverage
- cd coverage
- coverage combine
- coverage xml
- coverage report
coverage: /TOTAL.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage/coverage.xml
# Prepare to commit changes on the repository
.prepare-commit:
variables:
USER: ci-bot
GITLAB_BOT_ID: f970f908e8fe61f5524a3becfefbc831
before_script:
- git config user.name "project_${CI_PROJECT_ID}_bot_${GITLAB_BOT_ID}"
- git config user.email "project_${CI_PROJECT_ID}_bot_${GITLAB_BOT_ID}@noreply.${CI_SERVER_HOST}"
- git remote remove gitlab_origin || true
- git remote add gitlab_origin "https://oauth2:${ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
# Commit changes in example files
update-examples:
stage: post-tests
extends: .prepare-commit
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH == "devel"
script:
- if [ -d examples_artifact ]; then rsync -az examples_artifact/* examples/; fi
- git add examples
- git commit -m "Updating examples [skip ci]" || true
- git push gitlab_origin HEAD:$CI_COMMIT_BRANCH -o ci.skip
# --- Release jobs ------------------------------------------------------------
# # Make release notes based on changelog file when a tag is created
# release-notes:
# stage: publish
# rules:
# - if: $CI_COMMIT_TAG # Run this job when a tag is created
# script:
# - git cat-file HEAD~1:CHANGELOG.md
# - |
# if [ $? -ne 0 ]; then
# git show HEAD~1:CHANGELOG.md | python bin/release-notes.py
# else
# cp CHANGELOG.md release_notes.md
# fi
# artifacts:
# paths:
# - release_notes.md
# expire_in: 1 day
# Make a release when a tag is created
release:
stage: publish
image: registry.gitlab.com/gitlab-org/release-cli:latest
rules:
- if: $CI_COMMIT_TAG # Run this job when a tag is created
before_script:
- echo "running release job for ${CI_COMMIT_TAG}"
script:
- BUILD_VERSION_SLUG="${BUILD_VERSION/+/-}" # Replace '+' with '-'
release:
name: "PyCIF ${CI_COMMIT_TAG}"
description: "Automatic release of PyCIF ${CI_COMMIT_TAG}"
tag_name: $CI_COMMIT_TAG
assets:
links:
- name: "PyCIF ${CI_COMMIT_TAG} wheel"
filepath: "/pycif-py3-none-any.whl"
url: "https://${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${BUILD_JOB_ID}/artifacts/raw/dist/pycif-${BUILD_VERSION}-py3-none-any.whl"
link_type: package
- name: "PyCIF ${CI_COMMIT_TAG} sources"
filepath: "/pycif.tar.gz"
url: "https://${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${BUILD_JOB_ID}/artifacts/raw/dist/pycif-${BUILD_VERSION}.tar.gz"
link_type: package