From ecc05ceba57f33a27c9a1c8952e2b052b4da93ee Mon Sep 17 00:00:00 2001
From: lkugler <lukas.kugler@gmail.com>
Date: Tue, 26 Mar 2024 12:34:05 +0100
Subject: [PATCH] docs
---
dartwrf/workflows.py | 70 ++++++++++++++++++++++++++------------------
1 file changed, 41 insertions(+), 29 deletions(-)
diff --git a/dartwrf/workflows.py b/dartwrf/workflows.py
index 3dce204..a985553 100644
--- a/dartwrf/workflows.py
+++ b/dartwrf/workflows.py
@@ -14,13 +14,12 @@ class WorkFlows(object):
def __init__(self, exp_config='cfg.py', server_config='server.py'):
"""Set up the experiment folder in `archivedir`.
- 1. Copy the DART-WRF scripts to `archivedir`
- 2. Copy the config files to `archivedir/dartwrf`
- 3. Set log paths
- 4. Write obskind.py file (dictionary of observation types)
-
- Note:
- Later, the config is loaded from dartwrf/config/exp_config.py and dartwrf/config/server_config.py
+ 1. Copy the selected config files
+ 2. Import configurations
+ 3. Prepare obskind.py file (dictionary of observation types)
+ 4. Copy the scripts and configs to `archivedir`
+ 5. Set python path
+ 6. Set log path and slurm scripts path
Args:
exp_config (str): Path to exp config file
@@ -75,11 +74,11 @@ class WorkFlows(object):
def _dict_to_py(d, outfile):
"""Write a python dictionary to a .py file
-
+
Args:
d (dict): dictionary to write
outfile (str): path to output file
-
+
Returns:
None
"""
@@ -92,40 +91,49 @@ class WorkFlows(object):
print('------ start exp from ', exp_config, ' and ', server_config, ' ------')
- # experiment starts, we dont know where the code shall run
- # => read the configuration file
-
- # copy the config files to this folder
- this_dir = '/'.join(__file__.split('/')[:-1])
+ #### 1
+ # copy the selected config files (arguments to Workflows(...)) to the scripts directory
+ # ./DART-WRF/dartwrf/server_config.py and ./DART-WRF/dartwrf/exp_config.py
+ # these config files will be used later, and no others!
+ original_scripts_dir = '/'.join(__file__.split('/')[:-1]) # usually /home/DART-WRF/dartwrf/
try:
- shutil.copyfile('config/'+server_config, this_dir+'/server_config.py')
+ shutil.copyfile('config/'+server_config, original_scripts_dir+'/server_config.py')
except shutil.SameFileError:
pass
try:
- shutil.copyfile('config/'+exp_config, this_dir+'/exp_config.py')
+ shutil.copyfile('config/'+exp_config, original_scripts_dir+'/exp_config.py')
except shutil.SameFileError:
pass
- sys.path.append(this_dir)
+ #### 2
+ # import the configuration files from where we copied them just before
+ sys.path.append(original_scripts_dir)
from server_config import cluster
self.cluster = cluster
from exp_config import exp
self.exp = exp
- # copy obs kind def to config, we will read a table from there
- # file needs to exist within package so sphinx can read it
- _dict_to_py(_obskind_read(), this_dir+'/obs/obskind.py')
+ #### 3
+ # to be able to generate obs_seq.in files, we need a dictionary to convert obs kinds to numbers
+ # a) we read the obs kind definitions (obs_kind_mod.f90 from DART code)
+ # b) we generate a python file with this dictionary
+ # Note: to include it in the documentary, the file needs to exist also in the repository
+ # (so the documentation generator SPHINX can read it)
+ _dict_to_py(_obskind_read(), original_scripts_dir+'/obs/obskind.py')
- _copy_dartwrf_to_archive() # includes config files
+ #### 4
+ # Copy scripts and config files to `self.cluster.archivedir` folder
+ _copy_dartwrf_to_archive()
+ #### 5
# we set the path from where python should import dartwrf modules
- # pythonpath is the folder above the dartwrf folder
+ # every python command then imports DART-WRF from self.cluster.archivedir+'/DART-WRF/dartwrf/'
self.cluster.python = 'export PYTHONPATH='+self.cluster.scripts_rundir+'/../; '+self.cluster.python
+ #### 6
# Set paths and backup scripts
self.cluster.log_dir = self.cluster.archivedir+'/logs/'
print('logging to', self.cluster.log_dir)
-
if self.cluster.use_slurm:
self.cluster.slurm_scripts_dir = self.cluster.archivedir+'/slurm-scripts/'
print('SLURM scripts will be in', self.cluster.slurm_scripts_dir)
@@ -282,8 +290,8 @@ class WorkFlows(object):
runtime_wallclock_mins_expected = int(8+time_in_simulation_hours*9) # usually below 9 min/hour
cfg_update = {"array": "1-"+str(self.cluster.size_WRF_jobarray), "ntasks": "10", "nodes": "1",
"time": str(runtime_wallclock_mins_expected), "mem": "40G", }
- #if runtime_wallclock_mins_expected > 10:
- # cfg_update.update({"nodelist": "jet08"})
+ # if runtime_wallclock_mins_expected > 10:
+ # cfg_update.update({"nodelist": "jet05"})
id = self.cluster.run_job(wrf_cmd, "WRF-"+self.exp.expname, cfg_update=cfg_update, depends_on=[id])
return id
@@ -375,13 +383,17 @@ class WorkFlows(object):
depends_on=[depends_on])
return id
- def evaluate_obs_posterior_after_analysis(self, init_valid_tuples, depends_on=None):
- arg = ' '.join([init.strftime('%Y-%m-%d_%H:%M,')+valid.strftime('%Y-%m-%d_%H:%M:%S') for (init, valid) in init_valid_tuples])
-
- cmd = self.cluster.python+' '+self.cluster.scripts_rundir+'/evaluate_obs_space.py '+arg
+ def evaluate_obs_posterior_after_analysis(self, init, valid, depends_on=None):
+
+ cmd = self.cluster.python+' '+self.cluster.scripts_rundir+'/evaluate_obs_space.py '+init.strftime('%Y-%m-%d_%H:%M,') + valid.strftime('%Y-%m-%d_%H:%M:%S')
id = self.cluster.run_job(cmd, 'eval+1'+self.exp.expname, cfg_update={"ntasks": "12", "mem": "50G", "ntasks-per-node": "12", "ntasks-per-core": "2",
"time": "15", "mail-type": "FAIL"},
depends_on=[depends_on])
+
+ cmd = self.cluster.python+' '+self.cluster.scripts_rundir+'/calc_linear_posterior.py '+init.strftime('%Y-%m-%d_%H:%M')
+ id = self.cluster.run_job(cmd, 'linpost'+self.exp.expname, cfg_update={"ntasks": "12", "mem": "50G", "ntasks-per-node": "12", "ntasks-per-core": "2",
+ "time": "15", "mail-type": "FAIL"},
+ depends_on=[id])
return id
def verify_sat(self, depends_on=None):
--
GitLab