#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import pandas as pd
import numpy as np
from .....utils import path
from logging import info, debug
# JvP 20210517: added import statements, MODULE_NAME module level logger
import logging
import sys
import subprocess
from .....utils.dates import date_range
from .....utils.check.errclass import CifRuntimeError
MODULE_NAME = __name__[__name__.index(
'TM5'):] if 'TM5' in __name__ else __name__
logger = logging.getLogger(MODULE_NAME)
# Original fetch function by A. Berchet
[docs]
def fetch_AB(
ref_dir,
ref_file,
input_interval,
target_dir,
tracer=None,
component=None,
**kwargs
):
"""Link individual TM5 meteo files to the working directory (original implementation).
Superseded by :func:`fetch` below, which copies the whole meteo
directory tree instead. For each month spanned by ``input_interval``
and each of a fixed set of file categories/prefixes (albedo/sr/veg
under ``an0tr1``; blh/ci/cp under ``h06h18tr1``; cld/convec/mfw under
``h06h18tr3``), links the individual source file (named via
``component.dir_<filedir>``) into ``target_dir``. A missing source
file is silently skipped (only debug-logged).
Args:
ref_dir: directory holding the TM5 meteo subdirectories.
ref_file: unused (kept for interface consistency).
input_interval: mapping of period-start dates to the list of
dates within each period; only its dates are used, expanded
to the first of each covered month.
target_dir: directory where matching files are symlinked.
tracer: unused (kept for interface consistency).
component: object exposing ``dir_an0tr1``/``dir_h06h18tr1``/
``dir_h06h18tr3`` attributes giving the sub-path (under
``ref_dir``) of each file category.
Returns:
tuple: ``(list_files, list_dates)``, both dicts keyed by the keys
of ``input_interval`` and mapping to empty lists — this
implementation does not report the dates/files it actually
linked.
"""
info(f"Copying meteo files from {ref_dir} to {target_dir}")
# File directories and prefixes
filedirs = {"an0tr1": ["albedo_%Y%m%d_00p01.nc",
"sr_%Y%m%d_00p01.nc",
"veg_%Y%m%d_00p01.nc"],
"h06h18tr1": ["blh_%Y%m%d_00p01.nc",
"ci_%Y%m%d_00p01.nc",
"cp_%Y%m%d_00p01.nc"],
"h06h18tr3": ["cld_%Y%m%d_00p03.nc",
"convec_%Y%m%d_00p03.nc",
"mfw_%Y%m%d_00p03.nc"]}
# Create the sub-directory to store meteo files
path.init_dir(target_dir)
# Loop over dates and file types
all_dates = (
pd.DatetimeIndex(
[dd for di in input_interval for dd in input_interval[di]])
+ pd.offsets.MonthBegin(0)).unique()
for date in all_dates:
for filedir in filedirs:
for filetype in filedirs[filedir]:
source = date.strftime(f"{ref_dir}/{getattr(component, f'dir_{filedir}')}/{filetype}")
target = date.strftime(
f"{target_dir}/{filetype}")
try:
path.link(source, target)
except OSError:
debug(f"Could not find file: {source}")
list_files = {datei: [] for datei in input_interval}
list_dates = {datei: [] for datei in input_interval}
return list_files, list_dates
# New fetch function by J.C.A. van Peet
[docs]
def fetch(ref_dir, ref_file, input_interval, target_dir,
tracer=None, component=None, **kwargs):
"""Link the whole TM5 meteo directory tree to the working directory.
Clears ``target_dir``, then bulk-copies the ``ref_dir`` tree into it
with ``cp -Rsn`` (directory names are recreated, files become
symlinks; ``-n`` avoids errors on already-existing targets).
Note:
Version history: 2.0 (17-05-2021, J.C.A. van Peet) replaced the
original per-file linking implementation (``fetch_AB`` above,
1.0, 28-04-2021, A. Berchet) with this bulk directory copy.
Args:
ref_dir: directory holding the TM5 meteo directory tree to copy.
ref_file: unused (kept for interface consistency).
input_interval: 2-element sequence ``(ddi, ddf)`` giving the
requested date range.
target_dir: directory where the meteo tree is copied as symlinks;
cleared before copying.
tracer: unused (kept for interface consistency).
component: unused (kept for interface consistency).
Returns:
tuple: ``(list_files, list_dates)``. ``list_files`` is a dict
with a single key (``ddi``) mapping to an empty list. ``list_dates``
maps that same key to an array of consecutive monthly
``[start, end]`` date pairs spanning ``(ddi, ddf)``.
Raises:
CifRuntimeError: if the ``cp -Rsn`` subprocess exits with a
non-zero return code.
"""
# Set the name of this function
PROG_NAME = MODULE_NAME+".fetch"
# Local logger
logger = logging.getLogger(PROG_NAME)
logger.setLevel(logging.DEBUG)
# Print debug statements
logger.debug("")
logger.debug("*"*30)
logger.debug(PROG_NAME+" => DEBUG:")
logger.debug(" ref_dir = %s", ref_dir)
logger.debug(" ref_file = %s", ref_file)
logger.debug(" input_interval = %s", input_interval)
logger.debug(" target_dir = %s", target_dir)
logger.debug(" tracer = %s", tracer)
logger.debug(" component = %s", component)
logger.debug(" kwargs = %s", kwargs)
logger.debug("*"*30)
logger.debug("")
# First clean the target directory
if os.path.isdir(target_dir):
path.remove(f"{target_dir}/*")
# Copy the meteo directory with 'cp -as' so that the files are copied
# as symbolic links, while retaining the original directory structure.
# The -n option prevents error messages if the target file already exists.
with subprocess.Popen(f'cp -Rsn {ref_dir}/* {target_dir}', shell=True, cwd='.',
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True, bufsize=1) as p:
for line in p.stdout:
logger.debug(line.strip())
# end with
#
# Check return code
if (p.returncode != 0):
print("")
print(f"{PROG_NAME} => SOME ERROR IN CMD!")
print(f" p.returncode = {p.returncode}")
print(" Computer says no!")
print("")
raise CifRuntimeError
# end if
# logger.debug("")
# logger.debug("*"*30)
# logger.debug("JvP: Computer says no!")
# logger.debug("*"*30)
# logger.debug("")
# try:
# raise RuntimeError
# except RuntimeError as e:
# #logger.exception("OOPS!")
# logger.critical(e, exc_info=True)
# #raise # => Will display the traceback on screen a second time
# sys.exit() # => Just exit.
# end try
# Apparently, this function is supposed to return two lists:
# list_files and list_dates. Not sure what the format is though,
# I should ask Antoine at some point...
ddi, ddf = input_interval
list_files = {ddi: []}
list_dates = {ddi:
np.append(
date_range(ddi, ddf, period="1MS")[:-1, np.newaxis],
date_range(ddi, ddf, period="1MS")[1:, np.newaxis],
axis=1)
}
return list_files, list_dates
# end function fetch
