import os
import subprocess
from logging import debug, info
from shutil import copy, copytree, ignore_patterns, rmtree
from ....utils.check.errclass import CifIOError, CifRuntimeError
[docs]
def compile(self, runsubdir):
"""Compile (or copy) the D&B forward executable (``runmodel.x``).
Follows the same two-strategy approach as the ``chimere`` plugin:
1. **Copy a pre-built executable** (default) -- copies ``runmodel.x``
from ``self.direxec`` into ``$workdir/model/``. Skipped when
``force-recompile = True``.
2. **Compile from sources** -- triggered when ``auto-recompile = True``
(or when no pre-built executable is found). Clones the source tree
from ``self.dir_sources`` (which should point to
``model_sources/dalecbethy``, i.e. the *unmodified* mirror of
``/home/chimereges/aberchet/DB/model/sites``), links the compiler
configuration file (``mk.compile`` -> ``config/mk.compile-<compiler>``,
see the D&B ``README``), links ``runsubdir``'s already-staged
``input/{static,dyn}forcing.nc`` into the source tree, and runs
``make runmodel.x DOMAIN=<domain_name>``.
.. note::
D&B's ``runmodel.x`` target does not merely link object files: it
transitively depends on ``src/dimensions.f90``, which the Makefile
regenerates (via ``util/model_setup.py``) from the *actual*
``$(INDIR)/{static,dyn}forcing.nc`` files, i.e. ``input/staticforcing.nc``
/ ``input/dynforcing.nc`` (see the Makefile rule for
``$(SRCDIR)/dimensions.f90`` and the ``$(FORCE_STATIC)``/
``$(FORCE_DYN)`` targets). Those are exactly the files
:func:`~.io.native2inputs.native2inputs` already wrote to
``runsubdir/input/`` before :func:`~.run.run` gets here, so this
function links them straight into ``sources/input/`` under the
same names, which pre-satisfies the ``$(FORCE_STATIC)``/
``$(FORCE_DYN)`` targets and short-circuits the Makefile's own
``forcing/<domain_name>_{static,dyn}forcing*.nc`` lookup. This is
also why :func:`~.run.run` -- not the generic, input-agnostic
``run_model`` transform init -- is what triggers this function:
``runsubdir/input`` must already hold the real, resolved forcing
files for the compile to succeed.
The prior parameter CSVs (``parameters/<domain_name>-*-params.csv``)
do not need the same treatment: they are small enough to be part of
the tracked ``model_sources/dalecbethy/parameters`` mirror already,
so the Makefile's own ``$(INDIR)/%params.csv`` rule resolves them
directly from the copied source tree.
.. warning::
TODO / open question for a D&B developer: the tangent-linear/adjoint
executable is only built by the ``xassi``/``libs`` targets
(``libadstack-$(FC).a``, ``libmini-$(FC).a``, ``cost_bw.f90``); this
is not wired up here yet since D&B's own TL/adjoint currently only
feeds its internal L-BFGS-B minimiser (``make xassi``), not a
pyCIF-driven tl/adj run.
Args:
self (Plugin): dalecbethy model plugin instance (carries
``workdir``, ``direxec``, ``dir_sources``, ``domain_name``,
``force-recompile`` / ``auto-recompile`` flags).
runsubdir (str): sub-directory of the simulation that triggered
this (lazy, first-call-only) compilation; its ``input/``
directory is expected to already hold
``staticforcing.nc``/``dynforcing.nc``, as staged by
:func:`~.io.native2inputs.native2inputs`.
Raises:
CifRuntimeError: if ``auto-recompile`` is False and no executable
can be found in ``direxec``.
"""
comp_dir = f"{self.workdir}/model"
os.makedirs(comp_dir, exist_ok=True)
# Copying a pre-built executable
try:
if getattr(self, "force-recompile"):
raise CifIOError
source = f"{self.direxec}/runmodel.x"
copy(source, comp_dir)
info(f"Using pre-built D&B executable from {source}")
return
except IOError as e:
if not getattr(self, "auto-recompile"):
raise CifRuntimeError(
f"D&B could not find an executable ({self.direxec}) and "
"was not asked to compile it; specify auto-recompile = "
"True in the Yaml to do so"
) from e
# Otherwise, (re-)compile from sources
if os.path.isdir(f"{comp_dir}/sources"):
rmtree(f"{comp_dir}/sources")
dir_sources = self.dir_sources if self.dir_sources != "" else self.direxec
# Excluding heavy/generated content that should never be part of the
# tracked sources anyway (see model_sources/dalecbethy, which already
# excludes input/forcing/observations/diagout).
copytree(
dir_sources,
f"{comp_dir}/sources",
ignore=ignore_patterns("*.o", "*.a", "*.x", "__pycache__"),
)
# Link the compiler-specific include file expected by the Makefile
# (see D&B README: `ln -fs config/mk.compile-gfortran mk.compile`)
compiler_config = getattr(self, "compiler_config", "config/mk.compile-gfortran")
mk_compile = f"{comp_dir}/sources/mk.compile"
if os.path.lexists(mk_compile):
os.remove(mk_compile)
os.symlink(compiler_config, mk_compile)
# Link the (real, already-staged) forcing files that native2inputs
# wrote to 'runsubdir/input/' straight into the source tree's own
# 'input/' directory, under the names the Makefile's FORCE_STATIC/
# FORCE_DYN targets expect: 'make runmodel.x' regenerates
# src/dimensions.f90 from these files for the requested DOMAIN (see
# note above), so they must be reachable *before* make is invoked.
sources_input_dir = f"{comp_dir}/sources/input"
os.makedirs(sources_input_dir, exist_ok=True)
for forcing_file in ("staticforcing.nc", "dynforcing.nc"):
source = os.path.join(runsubdir, "input", forcing_file)
if not os.path.isfile(source):
raise CifRuntimeError(
f"D&B compilation requires '{source}' (it is needed to "
f"(re-)generate src/dimensions.f90 for domain "
f"'{self.domain_name}'); it should have been staged by "
"native2inputs before the first call to run()."
)
target = os.path.join(sources_input_dir, forcing_file)
if os.path.lexists(target):
os.remove(target)
os.symlink(os.path.abspath(source), target)
debug(f"Compiling D&B for domain '{self.domain_name}'")
with open(f"{comp_dir}/DB_compiling.log", "w") as log:
process = subprocess.Popen(
f"make runmodel.x DOMAIN={self.domain_name}",
shell=True,
stdout=log,
cwd=f"{comp_dir}/sources/",
stderr=subprocess.PIPE,
)
_, stderr = process.communicate()
exe_file = f"{comp_dir}/sources/runmodel.x"
if not os.path.isfile(exe_file):
debug("D&B returned errors during compiling.")
debug("### START OF D&B ERROR MESSAGE ###")
for ln in stderr.decode().split("\n"):
debug(ln)
debug("### END OF D&B ERROR MESSAGE ###")
raise CifRuntimeError(
"D&B compilation failed; see DB_compiling.log for details"
)
copy(exe_file, comp_dir)
