Source code for pycif.plugins.obsoperators.standard.transforms.utils.fetch_inoutputs
from logging import debug
print_all = False
transfor2print = [
# 'time_interpolation_std_00009',
# 'loadfromoutputs_std_00044'
# # 'dump2inputs_std_00038'
]
# transfor2print = [
# 'dump2inputs_std_00030',
# 'time_interpolation_std_00007',
# 'loadfromoutputs_std_00043'
# ]
# transfor2print = [
# 'fromcontrol_std_00011',
# 'time_interpolation_std_00012',
# 'dump2inputs_std_00006'
# ]
[docs]
def fetch_inputs_outputs(
transform, ddi, transform_pipe, tmp_datastore,
subsimus, successors, precursors, transf_mapper,
mapper,
return_links=False
):
# Fetch outputs from successors and put them in outputs
# For all successors, fetch the corresponding sub-simulations needed
# for the present one
used_outputs = {}
orig_outputs = tmp_datastore["outputs"]
tmp_outputs = {**orig_outputs}
if transform in transfor2print or print_all:
print('fetchinout: orig_out:', transform, ddi, tmp_outputs)
# Loop over trids available in outputs of this period
for trid in subsimus["outputs"]:
trid_dict = orig_outputs.get(trid, {})
# Loop over the sub-periods for this given output
# This basically happens when several sub-periods are needed to compute
# the present period
# So far, this is typical to time_interpolation, but could be extended
for di in subsimus["outputs"][trid]:
di_dict = trid_dict.get(di, {})
# Looping over successor and checking if trid and sub-periods are available
for successor in successors.get(trid, []):
if trid not in successors:
successor_dict = None
continue
successor_dict = {}
subsimus_in = mapper[successor]["subsimus"]
for successor_di in subsimus_in:
if di not in subsimus_in[successor_di]['inputs'].get(trid, []):
continue
successor_inputs = \
transform_pipe.datastore[
successor][successor_di]["inputs"].get(trid, {})
successor_dict[successor_di] = \
successor_inputs.get(di, {}).get(
transform, {}
).get(di, {})
if len(successor_dict[successor_di]) == 0:
del successor_dict[successor_di]
# Skip if successor is empty
if successor_dict == {}:
continue
# Update used_outputs
for successor_di in successor_dict:
if successor not in used_outputs:
used_outputs[successor] = {}
if di not in used_outputs[successor]:
used_outputs[successor][di] = {}
if trid not in used_outputs[successor][di]:
used_outputs[successor][di][trid] = []
used_outputs[successor][di][trid].append(successor_di)
# Exclude redundant occurrences
di_dict[successor] = successor_dict
trid_dict[di] = di_dict
tmp_outputs[trid] = trid_dict
# if transform in ['run_model', 'loadfromoutputs_std_00045'] and trid == ('pressure', '13CH4'):
# print(tmp_outputs[trid])
# print(__file__)
# import code
# code.interact(local=dict(locals(), **globals()))
if transform in transfor2print or print_all:
print('fetchinout: target_out:', transform, ddi, tmp_outputs)
# Fetch inputs from precursors and put them in inputs
orig_inputs = tmp_datastore["inputs"]
tmp_inputs = {**orig_inputs}
if transform in transfor2print or print_all:
print('fetchinout: orig_in:', transform, ddi, orig_inputs)
print(__file__)
import code
code.interact(local=dict(locals(), **globals()))
# Loop over trids available in inputs for this period
for trid in subsimus["inputs"]:
trid_dict = orig_inputs.get(trid, {})
for di in subsimus["inputs"][trid]:
di_dict = trid_dict.get(di, {})
for precursor in precursors.get(trid, {}):
precursor_dict = {}
subsimus_out = mapper[precursor]["subsimus"]
for precursor_di in subsimus_out:
if di not in subsimus_out[precursor_di]['outputs'].get(trid, []):
continue
precursor_inputs = \
transform_pipe.datastore[
precursor][precursor_di]["outputs"].get(trid, {})
precursor_dict[precursor_di] = \
precursor_inputs.get(di, {}).get(
transform, {}
).get(ddi, {})
if transform in transfor2print or print_all:
print("AAAAAAAAAAAAAAAAA")
print(trid, di, precursor,
precursor_di,
precursor_inputs.keys(),
precursor_inputs.get(di, {}).get(
transform, {}
).keys())
print("BBBBBBBBBBBBBBBBB")
if len(precursor_dict[precursor_di]) == 0:
del precursor_dict[precursor_di]
# Skip if precursor is empty
if precursor_dict == {}:
continue
# Exclude redundant occurrences
di_dict[precursor] = precursor_dict
trid_dict[di] = di_dict
tmp_inputs[trid] = trid_dict
# try:
# orig_inputs = tmp_datastore["inputs"]
# if transform in transfor2print or print_all:
# print('fetchinout: orig_in:', transform, ddi, orig_inputs)
# print(__file__)
# import code
# code.interact(local=dict(locals(), **globals()))
# di not in subsimus_in[successor_di]['inputs'].get(trid, [])
# tmp_inputs = {**orig_inputs, **{
# trid: {**orig_inputs.get(trid, {}), **{
# di: {**orig_inputs.get(trid, {}).get(di, {}), **{
# precursor: [
# transform_pipe.datastore[precursor][precursor_di]
# ["outputs"].get(trid, {}).get(di, {})
# .get(transform, {})
# for precursor_di in mapper[precursor]["subsimus"]
# in transform_pipe.datastore[precursor]
# if di in mapper[precursor]["subsimus"][precursor_di][trid]
# transform_pipe.datastore[precursor][precursor_di]
# ["outputs"].get(trid, {})
# ][0] if trid in precursors else None
# for precursor in precursors.get(trid, [])
# }}
# for di in subsimus["inputs"][trid]
# }}
# for trid in subsimus["inputs"]
# }}
# except IndexError as e:
# debug(
# "This error happens when using batch ensemble incorrectly. "
# "Check the mappers of your transforms to see if expected sample "
# "species are there."
# )
# raise e
if transform in transfor2print or print_all:
print('fetchinout: target_in:', transform, ddi, tmp_inputs)
# if transform in ['time_interpolation_std_00079']:
# print(tmp_inputs)
# print(__file__)
# import code
# code.interact(local=dict(locals(), **globals()))
# Define used_inputs
used_inputs = {}
for trid in tmp_inputs:
for di in tmp_inputs[trid]:
for precursor in tmp_inputs[trid][di]:
if transf_mapper["inputs"][trid].get("sampled", False):
list_precursors_dates = [di]
else:
list_precursors_dates = tmp_inputs[trid][di][precursor].keys(
)
for ddd in list_precursors_dates:
if precursor not in used_inputs:
used_inputs[precursor] = {}
if di not in used_inputs[precursor]:
used_inputs[precursor][di] = {}
if trid not in used_inputs[precursor][di]:
used_inputs[precursor][di][trid] = []
used_inputs[precursor][di][trid].append(ddd)
# if transform in ['time_interpolation_std_00009']:
# print(tmp_inputs)
# print(tmp_outputs)
# print(__file__)
# import code
# code.interact(local=dict(locals(), **globals()))
if return_links:
return used_inputs, used_outputs
else:
return tmp_inputs, tmp_outputs