# This file is a template, and might need editing before it works on your project.
# Full project: https://gitlab.com/pages/plain-html
# build:
# stage: build
# script:
# - cd docs
# - make html
# artifacts:
# paths:
# - build/html
#
# Include a template used in the code_quality job
# include:
# - template: Code-Quality.gitlab-ci.yml
# Define stages
stages:
- build
- test
- style
- coverage
- store_max_coverage
- deploy
- release
# Install environment then PyCIF
install:
stage: build
script:
- echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories
- echo http://dl-cdn.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories
- apk --no-cache add --update py2-pip
- apk --no-cache add --update alpine-sdk build-base gcc python-dev linux-headers
- apk --no-cache add --update geos geos-dev
- apk --no-cache add --update proj proj-dev proj-util proj-doc proj-static
- apk --no-cache add --update hdf5 hdf5-dev hdf5-doc hdf5-static
- apk --no-cache add --update netcdf netcdf-dev netcdf-doc netcdf-utils
- apk --no-cache add --update gdal gdal-dev py-gdal
- apk --no-cache add --update freetype freetype-dev freetype-doc freetype-static
- apk --no-cache add --update libpng libpng-dev libpng-doc libpng-static libpng-utils
- apk --no-cache add --update openblas openblas-dev openblas-doc openblas-static openblas-ilp64
- apk --no-cache add --update psutils psutils-dev psutils-doc
- apk --no-cache add --update ethtool ethtool-doc py-ethtool py-ethtool-doc
- apk --no-cache add --update py2-yaml py2-dateutil py2-six py2-tz
- apk --no-cache add --update py2-pillow
- pip install psutil
- pip install "numpy<1.17"
- pip install "scipy<=1.2"
- pip install cftime
- pip install netCDF4
- pip install "pandas==0.24"
- pip install matplotlib
- pip install pyproj
- pip install setuptools
- pip install cython
- pip install shapely
- pip list
- pip install .
only:
- testinstall
# Build the documentation and publish it
pages:
stage: deploy
image:
name: pycif/pycif-ubuntu:0.3
entrypoint: [""]
script:
- ls -R pycif/plugins/models
- pip install .
- rsync -avz examples_artifact/* examples/
- make html
- cd ..
- mkdir .public
- cp -r docs/build/html/* .public/
- mv .public public
artifacts:
paths:
- public
only:
- LSCE
- devel
- doc
release_tagging:
stage: release
image:
name: pycif/pycif-ubuntu:0.3
entrypoint: [""]
before_script:
- ls -R
- mkdir -p ~/.ssh
# Settings > Repository > Deploy Keys > "DEPLOY_KEY_PUBLIC" is the public key of the utitlized SSH pair
# Settings > CI/CD > Variables > "DEPLOY_KEY_PRIVATE" is the private key of the utitlized SSH pair, type is 'File' and ends with empty line
- mv "$DEPLOY_KEY_PRIVATE" ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- "which ssh-agent || (apt-get update -y && apt-get install openssh-client git -y) > /dev/null 2>&1"
- eval "$(ssh-agent -s)"
- ssh-add ~/.ssh/id_rsa > /dev/null 2>&1
- (ssh-keyscan -H $CI_SERVER_HOST >> ~/.ssh/known_hosts) > /dev/null 2>&1
script:
- export
# .gitconfig
- touch ~/.gitconfig
- git config --global user.name $GITLAB_USER_NAME
- git config --global user.email $GITLAB_USER_EMAIL
# fresh clone
- ref_dir=`pwd`
- mkdir ~/source && cd $_
- git clone -b ${CI_COMMIT_REF_NAME} git@$CI_SERVER_HOST:$CI_PROJECT_PATH.git ./
- git branch
# Copy yml for releasing
- pwd
- rsync -avz ${ref_dir}/examples_artifact/* examples/
- git add examples
- git commit -m "Updating examples [skip ci]" || echo "No changes, nothing to commit!"
- git push
only:
- devel
# Check that coverage did not decreased
improved_coverage:
stage: coverage
image:
name: pycif/pycif-ubuntu:0.3
entrypoint: [""]
before_script:
- pip freeze
- pip install coverage
script:
- python3 -m coverage combine coverage_raw/.coverage*
- coverage html -d reports/coverage
- coverage xml -o reports/coverage.xml
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
# - MAX=`if [ -f coverage/.master_cov ] ; then cat coverage/.master_cov ; else echo 0.00 ; fi`
# - CURRENT=`cat coverage/.current_coverage`
# - echo $CURRENT
# - echo $MAX
# - if [[ $CURRENT < $MAX ]] ; then echo "Coverage decreased!!!"; exit 1 ; else echo "Coverage did not decrease, good job!"; exit 0 ; fi;
artifacts:
when: always
paths:
- reports/coverage/
- coverage/.current_coverage
# - coverage_raw/
# cache:
# paths:
# - coverage/.master_cov
# Basic tests
basic:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.3
entrypoint: [""]
before_script:
- pip freeze
- rm -rf /tmp/PYCIF_DATA_TEST/
- mkdir -p /tmp/PYCIF_DATA_TEST/RAW/EMISSIONS/
- wget https://sharebox.lsce.ipsl.fr/index.php/s/0lpOhIa1J3iT6ja/download -O GCP-CH4_EMISSIONS.tar > /dev/null 2>&1
- tar -xf GCP-CH4_EMISSIONS.tar -C /tmp/PYCIF_DATA_TEST/RAW/EMISSIONS/
- wget https://sharebox.lsce.ipsl.fr/index.php/s/Z0KBiONVQLCWHrV/download -O LMDZ.tar > /dev/null 2>&1
- mkdir -p /tmp/PYCIF_DATA_TEST/LMDZ/
- tar -xf LMDZ.tar -C /tmp/PYCIF_DATA_TEST/LMDZ/
script:
- tox -e py38 -e coverage -- -m "basic or tuto";
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
- figures_artifact
# Generate figures and artifacts for the website
article:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
script:
- |
if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config) or (allsimulations)";
else
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties and bands) or (fwd and ref_config)";
fi;
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
- figures_artifact
article_uncertainties:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
script:
- |
if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config) or (allsimulations)";
else
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties and bands) or (fwd and ref_config)";
fi;
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article_uncertainties
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
- figures_artifact
# Run the tests for the dummy model
tests_dummy:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
script:
- tox -e py38 -e coverage -- -m "test_in_ci and dummy"
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.dummy
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
# Run the tests for chimere (include downloading data)
tests_chimere:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.3
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
- rm -rf /tmp/PYCIF_DATA_TEST/
- mkdir -p /tmp/PYCIF_DATA_TEST/
- wget https://sharebox.lsce.ipsl.fr/index.php/s/zUpoUXODKFLZJrf/download -O CHIMERE.tar > /dev/null 2>&1
- tar -xf CHIMERE.tar -C /tmp/PYCIF_DATA_TEST/
- wget https://sharebox.lsce.ipsl.fr/index.php/s/GgTDXL8peF4BOuV/download -O RAW.tar > /dev/null 2>&1
- tar -xf RAW.tar -C /tmp/PYCIF_DATA_TEST/
script:
- tox -e py38 -e coverage -- -m "test_in_ci and chimere"
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.chimere
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
# Run the tests for chimere (include downloading data)
tests_lmdz:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.3
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
- rm -rf /tmp/PYCIF_DATA_TEST/
- wget https://sharebox.lsce.ipsl.fr/index.php/s/Z0KBiONVQLCWHrV/download -O LMDZ.tar > /dev/null 2>&1
- mkdir -p /tmp/PYCIF_DATA_TEST/LMDZ/
- tar -xf LMDZ.tar -C /tmp/PYCIF_DATA_TEST/LMDZ/
script:
- tox -e py38 -e coverage -- -m "test_in_ci and lmdz and acad"
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.chimere
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
# Run the tests for flexpart (include downloading data)
tests_flexpart:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
- rm -rf /tmp/PYCIF_DATA_TEST/
- mkdir -p /tmp/PYCIF_DATA_TEST/
- wget https://sharebox.lsce.ipsl.fr/index.php/s/k2LnQLHVUkefbnr/download > /dev/null 2>&1
- tar -xf download -C /tmp/PYCIF_DATA_TEST/
# - ls -R /tmp/PYCIF_DATA_TEST/
script:
- tox -e py38 -e coverage -- -m "test_in_ci and flexpart"
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.flexpart
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
# Run the tests for flexpart (include downloading data)
tests_tm5:
stage: test
retry: 2
image:
name: pycif/pycif-tm5:0.2
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
- rm -rf /tmp/PYCIF_DATA_TEST/
- mkdir -p /tmp/PYCIF_DATA_TEST/TM5/
- wget https://sharebox.lsce.ipsl.fr/index.php/s/qBWaoGcTWqV7fhG/download
- tar -xf download -C /tmp/PYCIF_DATA_TEST/TM5/
- ls -R /tmp/PYCIF_DATA_TEST/
script:
- tox -e py38 -e coverage -- -m "test_in_ci and tm5 and fwd"
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.tm5
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
# Run the tests for the gridded_NetCDF plugin
tests_gridded_netcdf:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.3
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
script:
- tox -e py38 -e coverage -- -m gridded_netcdf
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.chimere
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
#
## Run the tests for LMDZ: run on LSCE machines
#tests_lmdz:
# stage: test
# tags:
# - lsce
## image:
## name: pycif/pycif-ubuntu:0.1
## entrypoint: [""]
## before_script:
## # - apt-get update
## - pip freeze
## - rm -rf /tmp/PYCIF_DATA_TEST/
## - mkdir -p /tmp/PYCIF_DATA_TEST/
## - wget https://sharebox.lsce.ipsl.fr/index.php/s/k2LnQLHVUkefbnr/download
## - tar -xf download -C /tmp/PYCIF_DATA_TEST/
## - ls -R /tmp/PYCIF_DATA_TEST/
# script:
# - hostname
# - whoami
# - ssh -t obelix 'cd ~/CIF/bin; qsub pycif_test_batch;'
## - tox -e py38 -e coverage -- -m "test_in_ci and flexpart"
## after_script:
## - mkdir -p coverage
## - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
## - echo 'TOTAL COVERAGE:'" $(cat coverage/.current_coverage)%"
## coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
## artifacts:
## when: always
## paths:
## - reports/pytest*.html
## - reports/coverage/
## - coverage/.current_coverage
## - examples_artifact
# Check the code quality using the Code-Quality template
# code_quality:
# except:
# - master
# - devel
# artifacts:
# paths: [gl-code-quality-report.json]
# Check the code format using flake8
# code_format:
# stage: style
# image: python:3.6
# before_script:
# - pip install tox > /dev/null
# script:
# - tox -e style
# artifacts:
# when: always
# paths:
# - reports/flake8/
# Store the new coverage value only for the master and devel branches
# store_coverage:
# stage: store_max_coverage
# image: bashell/alpine-bash
# only:
# - master
# - devel
# before_script:
# - ls -al coverage
# script:
# - mkdir -p coverage
# - OLD_MAX=`if [ -f coverage/.master_cov ] ; then cat coverage/.master_cov ; else echo 0.00 ; fi`
# - NEW_MAX=`cat coverage/.current_coverage`
# - echo "old max => $OLD_MAX"
# - echo "new max => $NEW_MAX"
# - echo $NEW_MAX > coverage/.master_cov
# after_script:
# - ls -al coverage
# - cat coverage/.master_cov
# cache:
# paths:
# - coverage/.master_cov
