Source code for pycif.plugins.datastreams.fluxes.GridFED.fetch

import datetime
import glob
import os
from netCDF4 import Dataset
import pandas as pd
import xarray as xr
import numpy as np

from .....utils import path
from .....utils.dates import date_range
from logging import debug




def fetch(ref_dir, ref_file, input_interval, target_dir,
          tracer=None, component=None, **kwargs):
    """
    Fetch files and dates for GridFED.


    Args:
        ref_dir (str): the path to the input files
        ref_file (str): format of the input files
        input_interval (list): simulation interval (start and end dates)
        target_dir (str): where to copy
        tracer: the tracer Plugin, corresponding to the paragraph
            :bash:`datavect/components/fluxes/parameters/my_species` in the
            configuration yaml; can be needed to fetch extra information
            given by the user
        component: the component Plugin, same as tracer; corresponds to the paragraph
            :bash:`datavect/components/fluxes` in the configuration yaml

    Return:
        list_files: for each date that begins a period, an array containing
            the names of the files that are available for the dates within this period
        list_dates: for each date that begins a period, an array containing
            the names of the dates matching the files listed in list_files

    """

    # List of possible dates
    datei, datef = input_interval
    list_period_dates = \
        date_range(datei, datef, period=tracer.file_freq, close="")

    # Loop over dates
    list_files = {}
    list_dates = {}
    valid_files = []
    for dd in list_period_dates:
        file = dd.strftime("{}/{}".format(ref_dir, ref_file))
        if not os.path.isfile(file) or file in valid_files:
            continue

        # Force dates to be full years
        dates = xr.open_dataset(file)["time"].values[:, np.newaxis]
        dates = np.append(
            dates,
            dates[-1] + pd.DatetimeIndex(dates[-1]).days_in_month[0]
            * np.timedelta64(1, "D"))

        # Build the output dictionary
        list_dates[dd] = [[dd0, dd1] for dd0, dd1 in zip(dates[:-1], dates[1:])]
        list_files[dd] = len(dates[:-1]) * [file]

        # Fetching
        target_file = "{}/{}".format(target_dir, os.path.basename(file))
        path.link(file, target_file)

        valid_files.append(file)

    return list_files, list_dates