diff --git a/config/cfg.py b/config/cfg.py
index 2f158be85d82cba84943d0a155e4e6fc8522884a..973314411765c7e882ae5448be980bb01c78f945 100755
--- a/config/cfg.py
+++ b/config/cfg.py
@@ -10,7 +10,7 @@ exp.use_existing_obsseq = False # False or pathname (use precomputed obs_seq.ou
#exp.use_existing_obsseq = '/users/students/lehre/advDA_s2023/dartwrf_tutorial/very_cold_observation.out'
# path to the nature run, where we take observations from
-exp.nature_wrfout = '/jetfs/home/lkugler/data/sim_archive/exp_v1.18_P1_nature/2008-07-30_06:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
+exp.nature_wrfout = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.18_P1_nature/2008-07-30_06:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
exp.input_profile = '/mnt/jetfs/home/lkugler/data/initial_profiles/wrf/ens/2022-03-31/raso.fc.<iens>.wrfprof'
diff --git a/config/srvx1.py b/config/srvx1.py
index afbc74bcc2a1271fa2cc5b1793a97d86a450b49f..c1c779118d5b01ba92626a27d268c7c414480a74 100755
--- a/config/srvx1.py
+++ b/config/srvx1.py
@@ -53,6 +53,8 @@ cluster.python_verif = '/users/staff/lkugler/miniconda3/bin/python'
cluster.ncks = '/home/swd/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/nco-5.0.1-ntu44aoxlvwtr2tsrobfr4lht7cpvccf/bin/ncks'
cluster.ideal = '' #/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
cluster.wrfexe = '' #/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
+cluster.dart_modules = ''
+cluster.wrf_modules = ''
# paths for data output
cluster.wrf_rundir_base = utils.userhome+'/AdvDA23/run_WRF/' # path for temporary files
diff --git a/dartwrf/assim_synth_obs.py b/dartwrf/assim_synth_obs.py
index 7e5b40f9232e413d3b38266f485996a7b0b6f67c..b952978bc76128174d00babb71516a6da5576bcf 100755
--- a/dartwrf/assim_synth_obs.py
+++ b/dartwrf/assim_synth_obs.py
@@ -23,13 +23,13 @@ def link_nature_to_dart_truth(time):
"""
# get wrfout_d01 from nature run
- shutil.copy(time.strftime(exp.nature+'/'+wrfout_format),
+ shutil.copy(time.strftime(exp.nature_wrfout),
cluster.dart_rundir + "/wrfout_d01")
# DART may need a wrfinput file as well, which serves as a template for dimension sizes
symlink(cluster.dart_rundir + "/wrfout_d01",
cluster.dart_rundir + "/wrfinput_d01")
- print("linked", time.strftime(exp.nature+'/'+wrfout_format),
+ print("linked", time.strftime(exp.nature_wrfout),
"to", cluster.dart_rundir + "/wrfout_d01")
@@ -437,6 +437,46 @@ def archive_inflation_2(time):
copy(f_output, f_archive)
print(f_archive, 'saved')
+def link_DART_binaries_and_RTTOV_files():
+ joinp = os.path.join
+
+ # link DART binaries
+ bins = ['perfect_model_obs', 'filter', 'obs_diag', 'obs_seq_to_netcdf']
+ for b in bins:
+ symlink(joinp(cluster.dart_srcdir, b),
+ joinp(cluster.dart_rundir, b))
+
+ # link RTTOV files
+ rttov_files = ['rttov13pred54L/rtcoef_msg_4_seviri_o3.dat',
+ 'mfasis_lut/rttov_mfasis_cld_msg_4_seviri_deff.H5',
+ 'cldaer_visir/sccldcoef_msg_4_seviri.dat']
+
+ try: # may fail quietly if we dont need RTTOV
+ for f_src in rttov_files:
+ destname = os.path.basename(f_src)
+ if 'rtcoef' in f_src:
+ destname = 'rtcoef_msg_4_seviri.dat'
+
+ symlink(cluster.rttov_srcdir + f_src,
+ cluster.dart_rundir+'/'+destname)
+
+ symlink(cluster.dart_rundir+'/rttov_mfasis_cld_msg_4_seviri_deff.H5',
+ cluster.dart_rundir+'/rttov_mfasis_cld_msg_4_seviri.H5') # use deff, not OPAC
+
+ symlink(cluster.dart_srcdir+'/../../../observations/forward_operators/rttov_sensor_db.csv',
+ cluster.dart_rundir+'/rttov_sensor_db.csv')
+
+ symlink(cluster.dart_srcdir+'/../../../assimilation_code/programs/gen_sampling_err_table/'
+ +'work/sampling_error_correction_table.nc',
+ cluster.dart_rundir+'/sampling_error_correction_table.nc')
+
+ print('prepared DART & RTTOV links in', cluster.dart_rundir)
+ except Exception as e:
+ if any([hasattr(obscfg, 'sat_channel') for obscfg in exp.observations]):
+ raise e
+ else:
+ pass # we dont need RTTOV anyway
+
def main(time, prior_init_time, prior_valid_time, prior_path_exp):
"""Assimilate observations
@@ -464,8 +504,7 @@ def main(time, prior_init_time, prior_valid_time, prior_path_exp):
os.makedirs(cluster.dart_rundir, exist_ok=True) # create directory to run DART in
os.chdir(cluster.dart_rundir)
- # link DART binaries to run_DART
- os.system(cluster.python + " " + cluster.scripts_rundir + "/link_dart_rttov.py")
+ link_DART_binaries_and_RTTOV_files()
# remove any existing observation files
os.system("rm -f input.nml obs_seq.in obs_seq.out obs_seq.out-orig obs_seq.final")
diff --git a/dartwrf/dart_nml.py b/dartwrf/dart_nml.py
index 729930457dc6b2034603b21787086a70b13789dc..c11c9a99181f5bbb64881e1ffc24d90525a1ecff 100644
--- a/dartwrf/dart_nml.py
+++ b/dartwrf/dart_nml.py
@@ -12,7 +12,7 @@ def read_namelist(filepath):
filepath (str): Path to namelist file
Returns:
- dict: keys are namelist sections, values are dictionaries of namelist variables
+ dict: namelist[section][parameter] = [[arg1, arg2,], [arg3, arg4]]
"""
d = dict()
@@ -24,52 +24,93 @@ def read_namelist(filepath):
# skip whitespace
line = line.strip()
- # skip comments
- if not line.startswith('#'):
-
- # skip empty lines
- if len(line) > 0:
-
- # namelist section
- if line.startswith('&'):
- section = line
- d[section] = dict()
- continue
-
- if '/' in line:
- continue # skip end of namelist section
-
- try:
- # split line into variable name and value
- var, val = line.split('=')
- val = val.strip().strip(',').strip()
-
- except ValueError:
- # If the above split failed, we are still, we are still in the previous variable
- nextline_values = line.strip().split(',').strip()
- val = val + nextline_values
-
- # add variable to dictionary
- d[section][var] = val
- return d
+ if line.startswith('#') or line.startswith('!'):
+ continue # skip this line
+
+ # skip empty lines
+ if len(line) > 0:
+
+ # namelist section
+ if line.startswith('&'):
+ section = line
+ d[section] = dict()
+ continue
+
+ if '/' in line:
+ continue # skip end of namelist section
+
+ line = line.strip().strip(',')
+
+ try:
+ # split line into variable name and value
+ param, val = line.split('=')
+ param = param.strip()
+
+ param_data = []
+ except ValueError:
+ # If the above split failed,
+ # then there is additional data for the previous variable
+ val = line # in this line, there is only param_data
+ # param is still the one from previously
+
+ val = val.strip().strip(',').split(',')
+
+ # ensure that we have strings
+ if isinstance(val, list):
+ val = [str(v) for v in val]
+ else:
+ val = [str(val)]
+
+ param_data.append(val)
+
+ print('this iteration var, val ...', {param: param_data})
+
+ # add variable to dictionary
+ d[section][param] = param_data
+ return d
def write_namelist_from_dict(d, filepath):
"""Write a DART namelist dictionary to a file.
Args:
- d (dict): keys are namelist sections, values are dictionaries of namelist variables
+ d (dict): keys are namelist sections, values are dictionaries.
+ these dictionaries contain keys (=parameters) and values (list type)
+ every item in values is a line (list type)
+ every line contains possibly multiple entries
filepath (str): Path to namelist file
"""
with open(filepath, 'w') as f:
for section in d:
f.write(section+'\n')
- for var in d[section]:
- val = d[section][var]
- if isinstance(val, list):
- val = ', '.join(val)
- f.write('\t '+var+' = '+str(val)+'\n')
- f.write('\t /\n\n')
+
+ try:
+ parameters = d[section].keys()
+ print(parameters, [len(p) for p in parameters])
+ max_width_of_parameter_name = max([len(p) for p in parameters])
+ width = max_width_of_parameter_name + 1
+ except:
+ width = None
+
+ for parameter in parameters:
+ lines = d[section][parameter]
+
+ if isinstance(lines, str):
+ lines = [lines,]
+
+ for i, line in enumerate(lines):
+
+ try:
+ line = ', '.join(line) # write line (is a list)
+ except:
+ pass
+
+
+ if i == 0:
+ f.write(' '+parameter.ljust(width)+' = '+line+'\n')
+ else:
+ f.write(' '+' '*width+' = '+line+'\n')
+ f.write(' /\n\n')
def _get_list_of_localizations():
@@ -142,13 +183,24 @@ def _get_list_of_localizations():
# set the other (unused) list to a dummy value
if len(l_loc_vert_km) > 0:
- l_loc_vert_scaleheight = [-1,]
+ l_loc_vert_scaleheight = ["-1",]
else:
- l_loc_vert_km = [-1,]
+ l_loc_vert_km = ["-1",]
return l_obstypes, l_loc_horiz_rad, l_loc_vert_km, l_loc_vert_scaleheight
+def _to_fortran_list(l):
+ """Ensure formatting as "arg1", "arg2", """
+ assert isinstance(l, list)
+
+ if len(l) > 1: # multiple entries
+ return ', '.join(['"'+v+'"' for v in l])
+ elif len(l) == 1: # single entry
+ return '"'+l[0]+'"'
+ else: # no entry
+ return ''
+
def write_namelist(just_prior_values=False):
"""Set DART namelist variables in 'input.nml' file.
@@ -171,7 +223,7 @@ def write_namelist(just_prior_values=False):
nml = read_namelist(cluster.dart_srcdir + "/input.nml")
# make sure that observations defined in `exp.observations` are assimilated
- nml['&obs_kind_nml']['assimilate_these_obs_types'] = list_obstypes
+ nml['&obs_kind_nml']['assimilate_these_obs_types'] = _to_fortran_list(list_obstypes)
# dont compute posterior, just evaluate prior
if just_prior_values:
@@ -180,32 +232,39 @@ def write_namelist(just_prior_values=False):
nml['&filter_nml']['output_mean'] = '.false.'
nml['&filter_nml']['output_sd'] = '.false.'
nml['&obs_kind_nml']['assimilate_these_obs_types'] = []
- nml['&obs_kind_nml']['evaluate_these_obs_types'] = list_obstypes
+ nml['&obs_kind_nml']['evaluate_these_obs_types'] = [_to_fortran_list(list_obstypes)]
# write localization variables
- nml['&assim_tools_nml']['special_localization_obs_types'] = list_obstypes
- nml['&assim_tools_nml']['special_localization_cutoffs'] = list_loc_horiz_rad
+ nml['&assim_tools_nml']['special_localization_obs_types'] = [_to_fortran_list(list_obstypes)]
+ nml['&assim_tools_nml']['special_localization_cutoffs'] = [_to_fortran_list(list_loc_horiz_rad)]
+
+ nml['&location_nml']['special_vert_normalization_obs_types'] = [_to_fortran_list(list_obstypes)]
+ nml['&location_nml']['special_vert_normalization_heights'] = [_to_fortran_list(list_loc_vert_km)]
+ nml['&location_nml']['special_vert_normalization_scale_heights'] = [_to_fortran_list(list_loc_vert_scaleheight)]
- nml['&location_nml']['special_vert_normalization_obs_types'] = list_obstypes
- nml['&location_nml']['special_vert_normalization_heights'] = list_loc_vert_km
- nml['&location_nml']['special_vert_normalization_scale_heights'] = list_loc_vert_scaleheight
+ print(nml['&location_nml']['special_vert_normalization_obs_types'])
+ # overwrite namelist with experiment configuration
+ for section, sdata in exp.dart_nml.items():
- # overwrite namelist with DART-WRF/config/ configuration
- for key, value in exp.dart_nml.items():
+ # if section is not in namelist, add it
+ if section not in nml:
+ nml[section] = {}
- # if key is not in namelist, add it
- if key not in nml:
- nml[key] = {}
+ for parameter, value in sdata.items():
- # overwrite entry in each dictionary
- nml[key] = value
+ if isinstance(value, list) and len(value) > 1: # it is a real list
+ value = [value] # value was a list of parameter values, but just one line
+ else:
+ value = [[value]] # value was a single entry
+ # overwrite entry in each dictionary
+ nml[section][parameter] = value # every entry in this list is one line
# final checks
# fail if horiz_dist_only == false but observations contain a satellite channel
- if nml['&location_nml']['horiz_dist_only'] == '.false.':
+ if nml['&location_nml']['horiz_dist_only'][0] == '.false.':
for obscfg in exp.observations:
if hasattr(obscfg, "sat_channel"):
raise ValueError("Selected vertical localization, but observations contain satellite obs -> Not possible.")
diff --git a/dartwrf/link_dart_rttov.py b/dartwrf/link_dart_rttov.py
deleted file mode 100644
index 4b1e0cfd6062ea0d59a74ac951ca3459523edd4f..0000000000000000000000000000000000000000
--- a/dartwrf/link_dart_rttov.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import os
-from config.cfg import exp
-from config.cluster import cluster
-from dartwrf.utils import symlink, copy, sed_inplace
-
-joinp = os.path.join
-
-if __name__ == "__main__":
-
- # DART executables
- bins = ['perfect_model_obs', 'filter', 'obs_diag', 'obs_seq_to_netcdf']
- for b in bins:
- symlink(joinp(cluster.dart_srcdir, b),
- joinp(cluster.dart_rundir, b))
- print(joinp(cluster.dart_rundir, b), 'created')
-
- rttov_files = ['rttov13pred54L/rtcoef_msg_4_seviri_o3.dat',
- #'mfasis_lut/rttov_mfasis_cld_msg_4_seviri_deff.dat',
- 'mfasis_lut/rttov_mfasis_cld_msg_4_seviri_deff.H5',
- 'cldaer_visir/sccldcoef_msg_4_seviri.dat']
-
- for f_src in rttov_files:
- destname = os.path.basename(f_src)
- if 'rtcoef' in f_src:
- destname = 'rtcoef_msg_4_seviri.dat'
-
- symlink(cluster.rttov_srcdir + f_src,
- cluster.dart_rundir+'/'+destname)
-
- ##################
- symlink(cluster.dart_rundir+'/rttov_mfasis_cld_msg_4_seviri_deff.H5',
- cluster.dart_rundir+'/rttov_mfasis_cld_msg_4_seviri.H5')
-
- symlink(cluster.dart_srcdir+'/../../../observations/forward_operators/rttov_sensor_db.csv',
- cluster.dart_rundir+'/rttov_sensor_db.csv')
-
- symlink(cluster.dart_srcdir+'/../../../assimilation_code/programs/gen_sampling_err_table/'
- +'work/sampling_error_correction_table.nc',
- cluster.dart_rundir+'/sampling_error_correction_table.nc')
-
- print('prepared DART & RTTOV links in', cluster.dart_rundir)
diff --git a/dartwrf/utils.py b/dartwrf/utils.py
index 1e81157ce8dadf489f5c7a7bbca8054597b98e84..8e28b3d2c3f8e061ca97e20029077c67ec6acb5f 100755
--- a/dartwrf/utils.py
+++ b/dartwrf/utils.py
@@ -14,7 +14,11 @@ class ClusterConfig(object):
"""Collection of variables regarding the cluster configuration"""
def __init__(self, exp):
self.exp = exp # makes derived properties available
- self.dart_modules = '' # default value
+
+ # defaults
+ self.dart_modules = ''
+ self.wrf_modules = ''
+ self.size_jobarray = '1'
@property
def archivedir(self):
@@ -71,8 +75,9 @@ class ClusterConfig(object):
else:
print(cmd)
returncode = os.system(cmd)
+ print(cmd, 'returncode', returncode)
if returncode != 0:
- raise Exception('Error running command: '+cmd)
+ raise Exception('Error running command >>> '+cmd)
userhome = os.path.expanduser('~')