Source code for pycif.plugins.datastreams.fluxes.VPRM1km_nc.fetch

import datetime
import glob
import os
from logging import debug, info

import numpy as np
import pandas as pd

from .....utils import path
from .utils import find_valid_file




def fetch(ref_dir, ref_file, input_interval, target_dir, tracer=None, **kwargs):
    """Fetch VPRM1km daily flux files and build the hourly date intervals they cover.

    Lists candidate daily files at `tracer.file_freq`, builds 24 hourly
    ``[start, end]`` intervals per day, and links each existing file into
    `target_dir`.

    Args:
        ref_dir (str): Directory containing the reference input files.
        ref_file (str): Filename pattern of the input files (a ``strftime``
            format string).
        input_interval (list[datetime.datetime]): ``[date_i, date_f]``
            simulation interval to cover.
        target_dir (str): Directory where the resolved files are linked.
        tracer: The flux tracer plugin, providing ``file_freq``.

    Returns:
        tuple[dict, dict]: ``(list_files, list_dates)``, each keyed by the
        candidate file date, mapping to a single-element list with the
        resolved file path and the list of hourly ``[start, end]``
        date-interval pairs within that day.
    """

    # Inputs:
    # ---------
    # ref_dir: directory where the original files are found
    # ref_file: (template) name of the original files
    # input_interval: list of the periods to simulate, each item is the list of the dates of the period
    # target_dir: directory where the links to the orginal files are created
    #
    # Ouputs:
    # ---------
    # list_files: for each date that begins a period, an array containing the names of the files that are available
    # for the dates within this period
    # list_dates: for each date that begins a period, an array containing the names of the dates mathcin the files
    # listed in list_files
    list_period_dates = pd.date_range(
        input_interval[0], input_interval[1], freq=tracer.file_freq
    )
    list_dates = {}
    list_files = {}
    for dd in list_period_dates:
        file = dd.strftime(f"{ref_dir}/{ref_file}")
        file_hours = pd.date_range(dd, dd + pd.to_timedelta('23h'), freq="1h")
        list_dates[dd] = [[hh, hh + datetime.timedelta(hours=1)] for hh in file_hours]
        list_files[dd] = [file]

        # Fetching
        if os.path.isfile(file):
            target_file = f"{target_dir}/{os.path.basename(file)}"
            path.link(file, target_file)

    debug(
        "Fetched files and dates as follows:\n"
        "Dates: {\n"
        + "\n".join(
            [
                "\n".join(
                    [f"  {ddi}:"]
                    + [f"     {dd}" for dd in list_dates[ddi]]
                )
                for ddi in list_dates
            ]
        )
        + "\n}\n\n"
        + "Files: {\n"
        + "\n".join(
            [
                "\n".join(
                    [f"  {ddi}:"]
                    + [f"     {dd}" for dd in list_files[ddi]]
                )
                for ddi in list_files
            ]
        )
        + "\n}"
    )

    return list_files, list_dates