Source code for pycif.plugins.datastreams.fluxes.CarbonMonitor.fetch

import datetime
import glob
import os
import pandas as pd

import numpy as np

from .....utils import path
from logging import debug




def fetch(ref_dir, ref_file, input_interval, target_dir,
          tracer=None, component=None, **kwargs):
    """Fetch Carbon Monitor files and build the corresponding hourly dates.

    Builds a date range at ``tracer.file_freq`` starting from the first day
    of ``input_interval``'s start month; for each period, links the
    corresponding file into ``target_dir`` (if it exists) and expands the
    period into hourly ``[start, end]`` sub-intervals up to the end of the
    period (or up to ``input_interval[1]`` for the last period).

    Args:
        ref_dir (str): directory where the original files are found.
        ref_file (str): (template) name of the original files.
        input_interval (list): simulation interval, as a list of the two
            bounding dates.
        target_dir (str): directory where links to the original files are
            created.
        tracer: the tracer Plugin, giving access to ``file_freq``.
        component: the component Plugin; unused, kept for interface
            compatibility.
        **kwargs: unused, kept for interface compatibility.

    Returns:
        (dict, dict): ``list_files`` and ``list_dates``.

        list_files: for each date that begins a period, a list containing
            the names of the files that are available for the dates within
            this period.
        list_dates: for each date that begins a period, a list containing
            the date intervals matching the files listed in ``list_files``.
    """

    datemin = input_interval[0] - datetime.timedelta(days=input_interval[0].day - 1)
    list_period_dates = \
        pd.date_range(datemin, input_interval[1],
                      freq=tracer.file_freq)
    list_dates = {}
    list_files = {}
    monthmax = input_interval[1].month
    daymax = input_interval[1].day
    hourmax = input_interval[1].hour
    for dd in list_period_dates:
        file = dd.strftime(f"{ref_dir}/{ref_file}")
        if dd.month == monthmax:
            dim = input_interval[1].day - 1
            hid = hourmax
        else:
            dim = (datetime.date(dd.year, dd.month + 1, 1)
                   - datetime.date(dd.year, dd.month, 1)).days
            hid = 0

        file_hours = pd.date_range(
            dd, dd + datetime.timedelta(hours=dim * 24 + hid), freq="1h")
        list_dates[dd] = [[hh, hh + datetime.timedelta(hours=1)]
                          for hh in file_hours]
        # list_dates[dd] = [[file_hours]]
        list_files[dd] = len(file_hours) * [file]

        # Fetching
        if os.path.isfile(file):
            target_file = f"{target_dir}/{os.path.basename(file)}"
            path.link(file, target_file)

    debug(
        "Fetched files and dates as follows:\n"
        "Dates: {\n" +
        "\n".join(["\n".join([f"  {ddi}:"]
                             + [f"     {dd}"
                                for dd in list_dates[ddi]])
                   for ddi in list_dates])
        + "\n}\n\n" +
        "Files: {\n" +
        "\n".join(["\n".join([f"  {ddi}:"]
                             + [f"     {dd}"
                                for dd in list_files[ddi]])
                   for ddi in list_files])
        + "\n}"
    )
    return list_files, list_dates