Source code for pycif.plugins.datastreams.fluxes.EDGARv8_EYECLIMA_nc.fetch

import datetime
import glob
import os
import pandas as pd

import numpy as np
from logging import info

from pycif.utils import path
from .utils import find_valid_file




def fetch(ref_dir, ref_file, input_interval, target_dir, tracer=None, **kwargs):
    """
    :param ref_dir: directory where the original files are found
    :param ref_file: (template) name of the original files
    :param input_interval: list of the periods to simulate,
                        each item is the list of the dates of the period
    :param target_dir: directory where the links to the orginal files are created
    :param tracer:
    :param kwargs:

    :return:

    list_files: for each date that begins a period, an array containing
        the names of the files that are available for the dates within this period
    list_dates: for each date that begins a period, an array containing
        the names of the dates mathcin the files listed in list_files

    """

    list_period_dates = pd.date_range(input_interval[0], input_interval[1], freq="1D")
#    list_period_dates = pd.date_range(input_interval[0], input_interval[1], freq="31D")
    list_dates = {}
    list_files = {}
    step = 1
    for dd in list_period_dates:
        dir_dd = dd.strftime(ref_dir)
        dir_dd_next = (dd + datetime.timedelta(hours=1)).strftime(ref_dir)
        dir_dd_previous = (dd - datetime.timedelta(hours=1)).strftime(ref_dir)
        files_3d, dates_3d = find_valid_file(dir_dd, ref_file, dd, dir_dd_next,
                                             ref_dir_previous=dir_dd_previous)
#        list_hours = pd.date_range(dd, dd + datetime.timedelta(hours=23),
        if step < len(list_period_dates):
            list_hours = pd.date_range(dd, dd + datetime.timedelta(hours=23),
                                       freq="1H")
#           list_hours = pd.date_range(dd, dd + datetime.timedelta(days=31),
#                                   freq="1H")
            step += 1
        else:
            list_hours = pd.date_range(dd, dd + datetime.timedelta(hours=input_interval[1].hour),
                                       freq="1H")
#           staydays = (input_interval[1]-dd).days
#           info('nombre de jours dans la derniere periode du fetch')
#           info(staydays)
#           list_hours = pd.date_range(dd, dd + datetime.timedelta(days=staydays),
#                                   freq="1H")
#           info(list_hours[0])
#           info(list_hours[-1])

        if os.path.isfile(files_3d[0]):
            # list_dates[dd] = [[dd, dd + datetime.timedelta(hours=1)]]
            # list_files[dd] = [files_3d]* len(list_dates[dd])
            list_dates[dd] = [[hh, hh + datetime.timedelta(hours=1)]
                              for hh in list_hours]
            list_files[dd] = len(list_hours) * [files_3d]
            # the  to fetch is a forecast
            local_files = []
            target_file = "{}/{}".format(target_dir, dd.strftime(ref_file))
            path.link(files_3d[0], target_file)
            local_files.append(target_file)
    return list_files, list_dates