Source code for pycif.plugins.datastreams.fluxes.CarbonMonitor.fetch

import datetime
import glob
import os
import pandas as pd

import numpy as np

from .....utils import path
from logging import debug




def fetch(ref_dir, ref_file, input_interval, target_dir,
          tracer=None, component=None, **kwargs):

    datemin = input_interval[0] - datetime.timedelta(days=input_interval[0].day - 1)
    list_period_dates = \
        pd.date_range(datemin, input_interval[1],
                      freq=tracer.file_freq)
    list_dates = {}
    list_files = {}
    monthmax = input_interval[1].month
    daymax = input_interval[1].day
    hourmax = input_interval[1].hour
    for dd in list_period_dates:
        file = dd.strftime("{}/{}".format(ref_dir, ref_file))
        if dd.month == monthmax:
            dim = input_interval[1].day - 1
            hid = hourmax
        else:
            dim = (datetime.date(dd.year, dd.month + 1, 1)
                   - datetime.date(dd.year, dd.month, 1)).days
            hid = 0

        file_hours = pd.date_range(
            dd, dd + datetime.timedelta(hours=dim * 24 + hid), freq="1H")
        list_dates[dd] = [[hh, hh + datetime.timedelta(hours=1)]
                          for hh in file_hours]
        # list_dates[dd] = [[file_hours]]
        list_files[dd] = len(file_hours) * [file]

        # Fetching
        if os.path.isfile(file):
            target_file = "{}/{}".format(target_dir, os.path.basename(file))
            path.link(file, target_file)

    debug(
        "Fetched files and dates as follows:\n"
        "Dates: {\n" +
        "\n".join(["\n".join(["  {}:".format(ddi)]
                             + ["     {}".format(dd)
                                for dd in list_dates[ddi]])
                   for ddi in list_dates])
        + "\n}\n\n" +
        "Files: {\n" +
        "\n".join(["\n".join(["  {}:".format(ddi)]
                             + ["     {}".format(dd)
                                for dd in list_files[ddi]])
                   for ddi in list_files])
        + "\n}"
    )
    return list_files, list_dates