Gitlab CI file .gitlab-ci.yml

Gitlab CI file .gitlab-ci.yml#

# This file is a template, and might need editing before it works on your project.
# Full project: https://gitlab.com/pages/plain-html
# build:
#   stage: build
#   script:
#     - cd docs
#     - make html
#   artifacts:
#     paths:
#       - build/html
#
# Include a template used in the code_quality job
# include:
#   - template: Code-Quality.gitlab-ci.yml

# Define stages
stages:
  - build
  - test
  - style
  - coverage
  - store_max_coverage
  - deploy
  - release

# Install environment then PyCIF
install:
  stage: build
  script:
    - echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories
    - echo http://dl-cdn.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories
    - apk --no-cache add --update py2-pip
    - apk --no-cache add --update alpine-sdk build-base gcc python-dev linux-headers
    - apk --no-cache add --update geos geos-dev
    - apk --no-cache add --update proj proj-dev proj-util proj-doc proj-static
    - apk --no-cache add --update hdf5 hdf5-dev hdf5-doc hdf5-static
    - apk --no-cache add --update netcdf netcdf-dev netcdf-doc netcdf-utils
    - apk --no-cache add --update gdal gdal-dev py-gdal
    - apk --no-cache add --update freetype freetype-dev freetype-doc freetype-static
    - apk --no-cache add --update libpng libpng-dev libpng-doc libpng-static libpng-utils
    - apk --no-cache add --update openblas openblas-dev openblas-doc openblas-static openblas-ilp64
    - apk --no-cache add --update psutils psutils-dev psutils-doc
    - apk --no-cache add --update ethtool ethtool-doc py-ethtool py-ethtool-doc
    - apk --no-cache add --update py2-yaml py2-dateutil py2-six py2-tz
    - apk --no-cache add --update py2-pillow
    - pip install psutil
    - pip install "numpy<1.17"
    - pip install "scipy<=1.2"
    - pip install cftime
    - pip install netCDF4
    - pip install "pandas==0.24"
    - pip install matplotlib
    - pip install pyproj
    - pip install setuptools
    - pip install cython
    - pip install shapely
    - pip list
    - python setup.py install
  only:
    - testinstall

# Build the documentation and publish it
pages:
  stage: deploy
  image:
    name: pycif/pycif-ubuntu:0.3
    entrypoint: [""]
  script:
    - ls -R pycif/plugins/models
    - python3 setup.py install
    - rsync -avz examples_artifact/* examples/
    - make html
    - cd ..
    - mkdir .public
    - cp -r docs/build/html/* .public/
    - mv .public public
  artifacts:
    paths:
      - public
  only:
    - LSCE
    - devel
    - doc

release_tagging:
  stage: release
  image:
    name: pycif/pycif-ubuntu:0.3
    entrypoint: [""]
  before_script:
    - ls -R
    - mkdir -p ~/.ssh
    # Settings > Repository > Deploy Keys > "DEPLOY_KEY_PUBLIC" is the public key of the utitlized SSH pair
    # Settings > CI/CD > Variables > "DEPLOY_KEY_PRIVATE" is the private key of the utitlized SSH pair, type is 'File' and ends with empty line
    - mv "$DEPLOY_KEY_PRIVATE" ~/.ssh/id_rsa
    - chmod 600 ~/.ssh/id_rsa
    - "which ssh-agent || (apt-get update -y && apt-get install openssh-client git -y) > /dev/null 2>&1"
    - eval "$(ssh-agent -s)"
    - ssh-add ~/.ssh/id_rsa > /dev/null 2>&1
    - (ssh-keyscan -H $CI_SERVER_HOST >> ~/.ssh/known_hosts) > /dev/null 2>&1
  script:
    - export
    # .gitconfig
    - touch ~/.gitconfig
    - git config --global user.name $GITLAB_USER_NAME
    - git config --global user.email $GITLAB_USER_EMAIL
    # fresh clone
    - ref_dir=`pwd`
    - mkdir ~/source && cd $_
    - git clone -b ${CI_COMMIT_REF_NAME} git@$CI_SERVER_HOST:$CI_PROJECT_PATH.git ./
    - git branch
    # Copy yml for releasing
    - pwd
    - rsync -avz ${ref_dir}/examples_artifact/* examples/
    - git add examples
    - git commit -m "Updating examples [skip ci]" || echo "No changes, nothing to commit!"
    - git push
  only:
    - devel

# Check that coverage did not decreased
improved_coverage:
  stage: coverage
  image:
    name: pycif/pycif-ubuntu:0.3
    entrypoint: [""]
  before_script:
    - pip freeze
    - pip install coverage
  script:
    - python3 -m coverage combine coverage_raw/.coverage*
    - coverage html -d reports/coverage
    - coverage xml -o reports/coverage.xml
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'

  #     - MAX=`if [ -f coverage/.master_cov ] ; then cat coverage/.master_cov ; else echo 0.00 ; fi`
  #     - CURRENT=`cat coverage/.current_coverage`
  #     - echo $CURRENT
  #     - echo $MAX
  #     - if [[ $CURRENT < $MAX ]] ; then echo "Coverage decreased!!!"; exit 1 ; else echo "Coverage did not decrease, good job!";  exit 0 ; fi;
  artifacts:
    when: always
    paths:
      - reports/coverage/
      - coverage/.current_coverage
#      - coverage_raw/
#  cache:
#    paths:
#      - coverage/.master_cov

# Basic tests
basic:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-ubuntu:0.3
    entrypoint: [""]
  before_script:
    - pip freeze
    - rm -rf /tmp/PYCIF_DATA_TEST/
    - mkdir -p /tmp/PYCIF_DATA_TEST/RAW/EMISSIONS/
    - wget https://sharebox.lsce.ipsl.fr/index.php/s/0lpOhIa1J3iT6ja/download -O GCP-CH4_EMISSIONS.tar  > /dev/null 2>&1
    - tar -xf GCP-CH4_EMISSIONS.tar -C /tmp/PYCIF_DATA_TEST/RAW/EMISSIONS/
    - wget https://sharebox.lsce.ipsl.fr/index.php/s/Z0KBiONVQLCWHrV/download -O LMDZ.tar  > /dev/null 2>&1
    - mkdir -p /tmp/PYCIF_DATA_TEST/LMDZ/
    - tar -xf LMDZ.tar -C /tmp/PYCIF_DATA_TEST/LMDZ/
  script:
    - tox -e py38 -e coverage -- -m "basic or tuto";
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact
      - figures_artifact

# Generate figures and artifacts for the website
article:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-ubuntu:0.1
    entrypoint: [""]
  before_script:
    #    - apt-get update
    - pip freeze
  script:
    - |
      if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
        tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config) or (allsimulations)";
      else
        tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties and bands) or (fwd and ref_config)";
      fi;
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact
      - figures_artifact

article_uncertainties:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-ubuntu:0.1
    entrypoint: [""]
  before_script:
    #    - apt-get update
    - pip freeze
  script:
    - |
      if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
        tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config) or (allsimulations)";
      else
        tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties and bands) or (fwd and ref_config)";
      fi;
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article_uncertainties
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact
      - figures_artifact

# Run the tests for the dummy model
tests_dummy:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-ubuntu:0.1
    entrypoint: [""]
  before_script:
    #    - apt-get update
    - pip freeze
  script:
    - tox -e py38 -e coverage -- -m "test_in_ci and dummy"
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.dummy
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact

# Run the tests for chimere (include downloading data)
tests_chimere:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-ubuntu:0.3
    entrypoint: [""]
  before_script:
    #    - apt-get update
    - pip freeze
    - rm -rf /tmp/PYCIF_DATA_TEST/
    - mkdir -p /tmp/PYCIF_DATA_TEST/
    - wget https://sharebox.lsce.ipsl.fr/index.php/s/zUpoUXODKFLZJrf/download -O CHIMERE.tar  > /dev/null 2>&1
    - tar -xf CHIMERE.tar -C /tmp/PYCIF_DATA_TEST/
    - wget https://sharebox.lsce.ipsl.fr/index.php/s/GgTDXL8peF4BOuV/download -O RAW.tar  > /dev/null 2>&1
    - tar -xf RAW.tar -C /tmp/PYCIF_DATA_TEST/
  script:
    - tox -e py38 -e coverage -- -m "test_in_ci and chimere"
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.chimere
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact

# Run the tests for chimere (include downloading data)
tests_lmdz:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-ubuntu:0.3
    entrypoint: [""]
  before_script:
    #    - apt-get update
    - pip freeze
    - rm -rf /tmp/PYCIF_DATA_TEST/
    - wget https://sharebox.lsce.ipsl.fr/index.php/s/Z0KBiONVQLCWHrV/download -O LMDZ.tar  > /dev/null 2>&1
    - mkdir -p /tmp/PYCIF_DATA_TEST/LMDZ/
    - tar -xf LMDZ.tar -C /tmp/PYCIF_DATA_TEST/LMDZ/
  script:
    - tox -e py38 -e coverage -- -m "test_in_ci and lmdz and acad"
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.chimere
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact

# Run the tests for flexpart (include downloading data)
tests_flexpart:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-ubuntu:0.1
    entrypoint: [""]
  before_script:
    #    - apt-get update
    - pip freeze
    - rm -rf /tmp/PYCIF_DATA_TEST/
    - mkdir -p /tmp/PYCIF_DATA_TEST/
    - wget https://sharebox.lsce.ipsl.fr/index.php/s/k2LnQLHVUkefbnr/download  > /dev/null 2>&1
    - tar -xf download -C /tmp/PYCIF_DATA_TEST/
    # - ls -R /tmp/PYCIF_DATA_TEST/
  script:
    - tox -e py38 -e coverage -- -m "test_in_ci and flexpart"
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.flexpart
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact

# Run the tests for flexpart (include downloading data)
tests_tm5:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-tm5:0.2
    entrypoint: [""]
  before_script:
    #    - apt-get update
    - pip freeze
    - rm -rf /tmp/PYCIF_DATA_TEST/
    - mkdir -p /tmp/PYCIF_DATA_TEST/TM5/
    - wget https://sharebox.lsce.ipsl.fr/index.php/s/qBWaoGcTWqV7fhG/download
    - tar -xf download -C /tmp/PYCIF_DATA_TEST/TM5/
    - ls -R /tmp/PYCIF_DATA_TEST/
  script:
    - tox -e py38 -e coverage -- -m "test_in_ci and tm5 and fwd"
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.tm5
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact

# Run the tests for the gridded_NetCDF plugin
tests_gridded_netcdf:
  stage: test
  retry: 2
  image:
    name: pycif/pycif-ubuntu:0.3
    entrypoint: [""]
  before_script:
    #    - apt-get update
    - pip freeze
  script:
    - tox -e py38 -e coverage -- -m gridded_netcdf
  after_script:
    - mkdir -p coverage
    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
    - calc() { awk "BEGIN{print $*}"; }
    - percent_coverage=`cat coverage/.current_coverage`
    - tot_coverage=`calc ${percent_coverage}*100`
    - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
    - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.chimere
  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
  artifacts:
    when: always
    paths:
      - reports/pytest*.html
      - reports/coverage/
      - coverage/.current_coverage
      - coverage_raw
      - examples_artifact
#
## Run the tests for LMDZ: run on LSCE machines
#tests_lmdz:
#  stage: test
#  tags:
#    - lsce
##  image:
##    name: pycif/pycif-ubuntu:0.1
##    entrypoint: [""]
##  before_script:
##    #    - apt-get update
##    - pip freeze
##    - rm -rf /tmp/PYCIF_DATA_TEST/
##    - mkdir -p /tmp/PYCIF_DATA_TEST/
##    - wget https://sharebox.lsce.ipsl.fr/index.php/s/k2LnQLHVUkefbnr/download
##    - tar -xf download -C /tmp/PYCIF_DATA_TEST/
##    - ls -R /tmp/PYCIF_DATA_TEST/
#  script:
#    - hostname
#    - whoami
#    - ssh -t obelix 'cd ~/CIF/bin; qsub pycif_test_batch;'
##    - tox -e py38 -e coverage -- -m "test_in_ci and flexpart"
##  after_script:
##    - mkdir -p coverage
##    - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
##    - echo 'TOTAL COVERAGE:'" $(cat coverage/.current_coverage)%"
##  coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
##  artifacts:
##    when: always
##    paths:
##      - reports/pytest*.html
##      - reports/coverage/
##      - coverage/.current_coverage
##      - examples_artifact

# Check the code quality using the Code-Quality template
# code_quality:
#   except:
#     - master
#     - devel
#   artifacts:
#     paths: [gl-code-quality-report.json]

# Check the code format using flake8
# code_format:
#   stage: style
#   image: python:3.6
#   before_script:
#     - pip install tox > /dev/null
#   script:
#     - tox -e style
#   artifacts:
#     when: always
#     paths:
#       - reports/flake8/

# Store the new coverage value only for the master and devel branches
# store_coverage:
#   stage: store_max_coverage
#   image: bashell/alpine-bash
#   only:
#     - master
#     - devel
#   before_script:
#     - ls -al coverage
#   script:
#     - mkdir -p coverage
#     - OLD_MAX=`if [ -f coverage/.master_cov ] ; then cat coverage/.master_cov ; else echo 0.00 ; fi`
#     - NEW_MAX=`cat coverage/.current_coverage`
#     - echo "old max => $OLD_MAX"
#     - echo "new max => $NEW_MAX"
#     - echo $NEW_MAX > coverage/.master_cov
#   after_script:
#     - ls -al coverage
#     - cat coverage/.master_cov
#   cache:
#     paths:
#       - coverage/.master_cov