Source code for pycif.plugins.datastreams.fluxes.TNO_nc.fetch

import datetime
import glob
import os
import pandas as pd

import numpy as np

from pycif.utils import path
from .utils import find_valid_file




def fetch(ref_dir, ref_file, input_interval, target_dir, tracer=None, **kwargs):
    """Fetch TNO flux files and build hourly date intervals for each valid day.

    For each day in `input_interval`, uses `find_valid_file` to locate the
    nearest available (forecast-style) file bracketing the day, considering
    the current directory as well as the previous/next month's directory
    when the day is near a month boundary. Links the nearest file into
    `target_dir` and, if it exists, builds 24 hourly ``[start, end]``
    intervals for that day.

    :param ref_dir: directory where the original files are found
    :param ref_file: (template) name of the original files
    :param input_interval: list of the periods to simulate,
                        each item is the list of the dates of the period
    :param target_dir: directory where the links to the orginal files are created
    :param tracer: the flux tracer plugin (unused directly, kept for
        interface consistency)
    :param kwargs: unused, kept for interface consistency

    :return:

    list_files: for each date that begins a period, an array containing
        the names of the files that are available for the dates within this period
    list_dates: for each date that begins a period, an array containing
        the names of the dates mathcin the files listed in list_files

    """

    list_period_dates = pd.date_range(input_interval[0], input_interval[1], freq="1D")
    list_dates = {}
    list_files = {}
    for dd in list_period_dates:
        dir_dd = dd.strftime(ref_dir)
        dir_dd_next = (dd + datetime.timedelta(hours=1)).strftime(ref_dir)
        dir_dd_previous = (dd - datetime.timedelta(hours=1)).strftime(ref_dir)
        files_3d, dates_3d = find_valid_file(dir_dd, ref_file, dd, dir_dd_next,
                                             ref_dir_previous=dir_dd_previous)
        list_hours = pd.date_range(dd, dd + datetime.timedelta(hours=23),
                                   freq="1h")

        if os.path.isfile(files_3d[0]):
            # list_dates[dd] = [[dd, dd + datetime.timedelta(hours=1)]]
            # list_files[dd] = [files_3d]* len(list_dates[dd])
            list_dates[dd] = [[hh, hh + datetime.timedelta(hours=1)]
                              for hh in list_hours]
            list_files[dd] = len(list_hours) * [files_3d]
            # the  to fetch is a forecast
            local_files = []
            target_file = f"{target_dir}/{dd.strftime(ref_file)}"
            path.link(files_3d[0], target_file)
            local_files.append(target_file)

    return list_files, list_dates