From ba58e3619ab0ca4b991e3fda4ca002f8f761bea9 Mon Sep 17 00:00:00 2001
From: lkugler <lukas.kugler@gmail.com>
Date: Tue, 21 Feb 2023 17:53:21 +0100
Subject: [PATCH] new config import style
---
analysis_only.py | 16 +-
config/cfg.py | 27 +-
config/clusters.py | 155 ----------
config/jet.py | 79 +++++
config/srvx1.py | 76 +++++
config/vsc.py | 77 +++++
dartwrf/assim_synth_obs.py | 38 ++-
dartwrf/create_obs_upfront.py | 9 +-
dartwrf/create_obsseq.py | 2 +-
dartwrf/create_wbubble_wrfinput.py | 2 +-
dartwrf/evaluate_posterior.py | 2 +-
dartwrf/evaluate_prior.py | 2 +-
dartwrf/link_dart_rttov.py | 2 +-
dartwrf/obs/calculate_obs_locations.py | 2 +-
dartwrf/obsseq.py | 2 +-
dartwrf/obsseq_2dim.py | 2 +-
dartwrf/obsseq_to_netcdf.py | 2 +-
dartwrf/prep_IC_prior.py | 2 +-
dartwrf/prepare_namelist.py | 2 +-
dartwrf/prepare_wrfrundir.py | 2 +-
dartwrf/run_obs_diag.py | 2 +-
dartwrf/update_IC.py | 2 +-
dartwrf/utils.py | 104 +++----
dartwrf/workflows.py | 404 ++++++++++++++-----------
dartwrf/wrfinput_add_geo.py | 2 +-
dartwrf/wrfout_add_geo.py | 2 +-
26 files changed, 562 insertions(+), 455 deletions(-)
delete mode 100755 config/clusters.py
create mode 100755 config/jet.py
create mode 100755 config/srvx1.py
create mode 100755 config/vsc.py
diff --git a/analysis_only.py b/analysis_only.py
index 976c9b0..c1ae8b9 100755
--- a/analysis_only.py
+++ b/analysis_only.py
@@ -6,27 +6,17 @@ import os, sys, shutil
import datetime as dt
from dartwrf import utils
-from config.cfg import exp
-from config.clusters import cluster
+from dartwrf.workflows import WorkFlows
-###############################
-
prior_path_exp = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_noDA'
prior_init_time = dt.datetime(2008,7,30,12)
prior_valid_time = dt.datetime(2008,7,30,12,30)
assim_time = prior_valid_time
+w = WorkFlows(exp_config='cfg.py', server_config='srvx1.py')
-cluster.setup()
-
-os.system(
- cluster.python+' '+cluster.scripts_rundir+'/assim_synth_obs.py '
- +assim_time.strftime('%Y-%m-%d_%H:%M ')
- +prior_init_time.strftime('%Y-%m-%d_%H:%M ')
- +prior_valid_time.strftime('%Y-%m-%d_%H:%M ')
- +prior_path_exp
- )
+w.assimilate(assim_time, prior_init_time, prior_valid_time, prior_path_exp)
diff --git a/config/cfg.py b/config/cfg.py
index f70d034..46ea5e7 100755
--- a/config/cfg.py
+++ b/config/cfg.py
@@ -1,13 +1,13 @@
from dartwrf import utils
-exp = utils.ExperimentConfiguration()
-exp.expname = "exp_v1.22_P2_rr_VIS_obs10_loc20_inf5"
+exp = utils.Experiment()
+exp.expname = "test_newcode"
exp.model_dx = 2000
-exp.n_ens = 40
+exp.n_ens = 10
exp.filter_kind = 1
-exp.prior_inflation = 5
-exp.post_inflation = 0
+exp.prior_inflation = 0
+exp.post_inflation = 4
exp.sec = True
exp.reject_smallFGD = False
exp.cov_loc_vert_km_horiz_km = (3, 20)
@@ -20,13 +20,10 @@ exp.use_existing_obsseq = False # False or pathname (use precomputed obs_seq.ou
#exp.use_existing_obsseq = '/gpfs/data/fs71386/lkugler/sim_archive/exp_v1.21_P3_wbub7_REFL2D_obs10_loc20_oe5/obs_seq_out/2008-07-30_%H:%M_obs_seq.out'
#exp.use_existing_obsseq = '/gpfs/data/fs71386/lkugler/sim_archive/exp_v1.21_P2_rr_VIS_obs20_loc4/obs_seq_out/2008-07-30_%H:%M_obs_seq.out'
-#exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_P5+su_nat2/2008-07-30_07:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
-#exp.nature_wrfout = '/jetfs/home/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_nat/2008-07-30_12:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
-#exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_Pwbub5_nat/2008-07-30_12:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
-exp.nature_wrfout = '/jetfs/home/lkugler/data/sim_archive/exp_v1.18_P1_nature/2008-07-30_06:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
-#exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_P4_nat/2008-07-30_07:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
+#exp.nature = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_nat/2008-07-30_12:00/1'
+exp.nature = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.18_P1_nature/2008-07-30_06:00/1'
-exp.input_profile = '/jetfs/home/lkugler/data/initial_profiles/wrf/ens/2022-03-31/raso.fc.<iens>.wrfprof'
+exp.input_profile = '/mnt/jetfs/home/lkugler/data/initial_profiles/wrf/ens/2022-03-31/raso.fc.<iens>.wrfprof'
#exp.input_profile = '/gpfs/data/fs71386/lkugler/initial_profiles/wrf/ens/2022-03-31/raso.nat.<iens>.wrfprof'
#exp.input_profile = '/gpfs/data/fs71386/lkugler/initial_profiles/wrf/ens/2022-05-18/raso.fc.<iens>.wrfprof'
@@ -71,11 +68,11 @@ radar = dict(plotname='Radar reflectivity', plotunits='[dBz]',
t = dict(plotname='Temperature', plotunits='[K]',
kind='RADIOSONDE_TEMPERATURE',
- n_obs=22500, obs_locations='square_array_evenly_on_grid',
- # n_obs=1, obs_locations=[(45., 0.)],
+ #n_obs=22500, obs_locations='square_array_evenly_on_grid',
+ n_obs=1, obs_locations=[(45., 0.)],
error_generate=0.2, error_assimilate=0.2,
heights=[1000,], #range(1000, 17001, 2000),
- cov_loc_radius_km=1.5)
+ cov_loc_radius_km=50)
q = dict(plotname='Specific humidity', plotunits='[kg/kg]',
kind='RADIOSONDE_SPECIFIC_HUMIDITY', n_obs=1,
@@ -93,7 +90,7 @@ psfc = dict(plotname='SYNOP Pressure', plotunits='[Pa]',
error_generate=50., error_assimilate=100.,
cov_loc_radius_km=32)
-exp.observations = [vis]
+exp.observations = [t]
exp.update_vars = ['U', 'V', 'W', 'THM', 'PH', 'MU', 'QVAPOR', 'QCLOUD', 'QICE', 'PSFC']
#exp.update_vars = ['U', 'V', 'W', 'T', 'PH', 'MU', 'QVAPOR', 'PSFC']
diff --git a/config/clusters.py b/config/clusters.py
deleted file mode 100755
index f87748a..0000000
--- a/config/clusters.py
+++ /dev/null
@@ -1,155 +0,0 @@
-import os, sys
-import datetime as dt
-from dartwrf import utils
-from config.cfg import exp
-
-"""Configuration name docs
-
-When coding, use attributes of a dictionary like this:
-$ from cfg import exp, cluster
-$ path = cluster.archivedir
-
-
-attribute name | description
-------------------------------------------------------
-name any string (currently unused)
-
-python path of python version to use
-python_enstools path of python version to use for verification script (not provided)
-ncks path to 'ncks' program; type 'which ncks' to find the path,
- if it doesn't exist, try to load the module first ('module load nco')
-ideal path to WRF's ideal.exe
-wrfexe path to WRF's wrf.exe
-
-wrf_rundir_base path for temporary files for WRF
-dart_rundir_base path for temporary files for DART
-archive_base path for long-time output storage
-
-srcdir path to where WRF has been compiled, including the 'run' folder of WRF, e.g. /home/WRF-4.3/run
-dart_srcdir path to DART compile directory, e.g. /home/DART-9.11.9/models/wrf/work
-rttov_srcdir path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
-scriptsdir path where DART-WRF scripts reside, e.g. /home/DART-WRF/scripts
-
-namelist path to a namelist template; strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
-run_WRF path to script which runs WRF on a node of the cluster
-obs_impact_filename path to obs_impact_filename (see DART guide; module assim_tools_mod and program obs_impact_tool)
-geo_em path to NetCDF file of WRF domain (see WRF guide)
-
-slurm_cfg python dictionary, containing options of SLURM
- defined in SLURM docs (https://slurm.schedmd.com/sbatch.html)
- this configuration can be overwritten later on, for example:
- 'dict(cluster.slurm_cfg, **cfg_update)' where
- 'cfg_update = {"nodes": "2"}'
-"""
-
-
-vsc = utils.ClusterConfig(exp)
-vsc.name = 'vsc'
-vsc.max_nproc = 20
-vsc.size_jobarray = 10 # 10 jobs with each 4 WRF processes per node
-vsc.use_slurm = True
-
-# binaries
-vsc.python = '/home/fs71386/lkugler/miniconda3/envs/DART/bin/python'
-vsc.python_enstools = '/home/fs71386/lkugler/miniconda3/envs/enstools/bin/python'
-vsc.ncks = '/home/fs71386/lkugler/miniconda3/envs/DART/bin/ncks'
-vsc.ideal = '/home/fs71386/lkugler/compile/bin/ideal-v4.2.2_v1.22.exe'
-vsc.wrfexe = '/home/fs71386/lkugler/compile/bin/wrf-v4.3_v1.22.exe'
-vsc.container = '/home/fs71386/lkugler/run_container.sh python.gcc9.5.0.vsc4.sif'
-
-# paths for data output
-vsc.wrf_rundir_base = '/gpfs/data/fs71386/lkugler/run_WRF/' # path for temporary files
-vsc.dart_rundir_base = '/gpfs/data/fs71386/lkugler/run_DART/' # path for temporary files
-vsc.archive_base = '/gpfs/data/fs71386/lkugler/sim_archive/'
-
-# paths used as input
-vsc.srcdir = '/gpfs/data/fs71386/lkugler/compile/WRF/WRF-4.3/run'
-vsc.dart_srcdir = '/gpfs/data/fs71386/lkugler/compile/DART/DART/models/wrf/work'
-vsc.rttov_srcdir = '/gpfs/data/fs71386/lkugler/compile/RTTOV13/rtcoef_rttov13/'
-vsc.scriptsdir = '/home/fs71386/lkugler/DART-WRF/dartwrf/'
-
-# templates/run scripts
-vsc.namelist = vsc.scriptsdir+'/../templates/namelist.input'
-vsc.run_WRF = '/home/fs71386/lkugler/DART-WRF/dartwrf/run_ens.vsc.sh'
-
-vsc.slurm_cfg = {"account": "p71386", "partition": "skylake_0384", "qos": "p71386_0384",
- "nodes": "1", "ntasks": "1", "ntasks-per-node": "48", "ntasks-per-core": "1",
- "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
-
-jet = utils.ClusterConfig(exp)
-jet.name = 'jet'
-jet.max_nproc = 12
-jet.use_slurm = True
-jet.size_jobarray = 40
-
-# binaries
-jet.python = '/jetfs/home/lkugler/miniconda3/envs/DART/bin/python'
-jet.python_verif = '/jetfs/home/lkugler/miniconda3/envs/enstools/bin/python'
-jet.ncks = '/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/nco-4.9.3-dhlqiyog7howjmaleyfhm6lkt7ra37xf/bin/ncks'
-jet.ideal = '/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
-jet.wrfexe = '/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
-jet.container = ''
-
-# paths for data output
-jet.wrf_rundir_base = '/jetfs/home/lkugler/data/run_WRF/' # path for temporary files
-jet.dart_rundir_base = '/jetfs/home/lkugler/data/run_DART/' # path for temporary files
-jet.archive_base = '/jetfs/home/lkugler/data/sim_archive/'
-
-# paths used as input
-jet.srcdir = '/jetfs/home/lkugler/data/compile/WRF-4.3/run'
-jet.dart_srcdir = '/jetfs/home/lkugler/data/compile/DART/DART-10.5.3/models/wrf/work'
-jet.rttov_srcdir = '/jetfs/home/lkugler/data/compile/RTTOV13/rtcoef_rttov13/'
-jet.scriptsdir = '/jetfs/home/lkugler/DART-WRF/dartwrf/'
-
-# other inputs
-jet.geo_em = '/jetfs/home/lkugler/data/geo_em.d01.nc'
-jet.obs_impact_filename = jet.scriptsdir+'/../templates/impactfactor_T.txt'
-jet.namelist = jet.scriptsdir+'/../templates/namelist.input'
-jet.run_WRF = '/jetfs/home/lkugler/DART-WRF/dartwrf/run_ens.jet.sh'
-
-jet.slurm_cfg = {"account": "lkugler", "partition": "compute", #"nodelist": "jet07",
- "ntasks": "1", "ntasks-per-core": "1", "mem": "50G",
- "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
-
-
-srvx1 = utils.ClusterConfig(exp)
-srvx1.name = 'srvx1'
-srvx1.max_nproc = 6
-srvx1.use_slurm = False
-
-# binaries
-srvx1.python = '/users/staff/lkugler/miniconda3/bin/python'
-srvx1.python_verif = '/users/staff/lkugler/miniconda3/bin/python'
-srvx1.ncks = '/home/swd/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/nco-5.0.1-ntu44aoxlvwtr2tsrobfr4lht7cpvccf/bin/ncks'
-srvx1.ideal = '' #/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
-srvx1.wrfexe = '' #/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
-srvx1.container = ''
-
-# paths for data output
-srvx1.wrf_rundir_base = '/mnt/jetfs/home/lkugler/data/run_WRF/' # path for temporary files
-srvx1.dart_rundir_base = '/users/staff/lkugler/AdvDA23/run_DART/' # path for temporary files
-srvx1.archive_base = '/mnt/jetfs/scratch/lkugler/data/sim_archive/'
-
-# paths used as input
-srvx1.srcdir = '/users/staff/lkugler/AdvDA23/DART/WRF-4.3/run'
-srvx1.dart_srcdir = '/users/staff/lkugler/AdvDA23/DART/models/wrf/work'
-srvx1.rttov_srcdir = '/users/staff/lkugler/AdvDA23/RTTOV13/rtcoef_rttov13/'
-srvx1.scriptsdir = '/users/staff/lkugler/AdvDA23/DART-WRF/dartwrf/'
-srvx1.geo_em = '/mnt/jetfs/scratch/lkugler/data/geo_em.d01.nc'
-
-# templates/run scripts
-srvx1.namelist = srvx1.scriptsdir+'/../templates/namelist.input'
-srvx1.run_WRF = srvx1.scriptsdir+'/run_ens.jet.sh'
-
-srvx1.slurm_cfg = {"account": "lkugler", "partition": "compute",
- "ntasks": "1", "ntasks-per-core": "1", "mem": "50G",
- "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
-
-#################################
-# select cluster configuration
-if 'jet' in os.uname().nodename:
- cluster = jet
-elif 'srvx1' in os.uname().nodename:
- cluster = srvx1
-else:
- cluster = vsc
diff --git a/config/jet.py b/config/jet.py
new file mode 100755
index 0000000..96cffd3
--- /dev/null
+++ b/config/jet.py
@@ -0,0 +1,79 @@
+import os, sys
+import datetime as dt
+from dartwrf import utils
+from config.cfg import exp
+
+"""Configuration name docs
+
+When coding, use attributes of a dictionary like this:
+$ from cfg import exp, cluster
+$ path = cluster.archivedir
+
+
+attribute name | description
+------------------------------------------------------
+name any string (currently unused)
+
+python path of python version to use
+python_enstools path of python version to use for verification script (not provided)
+ncks path to 'ncks' program; type 'which ncks' to find the path,
+ if it doesn't exist, try to load the module first ('module load nco')
+ideal path to WRF's ideal.exe
+wrfexe path to WRF's wrf.exe
+
+wrf_rundir_base path for temporary files for WRF
+dart_rundir_base path for temporary files for DART
+archive_base path for long-time output storage
+
+srcdir path to where WRF has been compiled, including the 'run' folder of WRF, e.g. /home/WRF-4.3/run
+dart_srcdir path to DART compile directory, e.g. /home/DART-9.11.9/models/wrf/work
+rttov_srcdir path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
+scriptsdir path where DART-WRF scripts reside, e.g. /home/DART-WRF/scripts
+
+namelist path to a namelist template; strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
+run_WRF path to script which runs WRF on a node of the cluster
+obs_impact_filename path to obs_impact_filename (see DART guide; module assim_tools_mod and program obs_impact_tool)
+geo_em path to NetCDF file of WRF domain (see WRF guide)
+
+slurm_cfg python dictionary, containing options of SLURM
+ defined in SLURM docs (https://slurm.schedmd.com/sbatch.html)
+ this configuration can be overwritten later on, for example:
+ 'dict(cluster.slurm_cfg, **cfg_update)' where
+ 'cfg_update = {"nodes": "2"}'
+"""
+
+
+cluster = utils.ClusterConfig(exp)
+cluster.name = 'jet'
+cluster.max_nproc = 12
+cluster.use_slurm = True
+cluster.size_jobarray = 40
+
+# binaries
+cluster.python = '/jetfs/home/lkugler/miniconda3/envs/DART/bin/python'
+cluster.python_verif = '/jetfs/home/lkugler/miniconda3/envs/enstools/bin/python'
+cluster.ncks = '/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/nco-4.9.3-dhlqiyog7howjmaleyfhm6lkt7ra37xf/bin/ncks'
+cluster.ideal = '/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
+cluster.wrfexe = '/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
+cluster.container = ''
+
+# paths for data output
+cluster.wrf_rundir_base = '/jetfs/home/lkugler/data/run_WRF/' # path for temporary files
+cluster.dart_rundir_base = '/jetfs/home/lkugler/data/run_DART/' # path for temporary files
+cluster.archive_base = '/jetfs/home/lkugler/data/sim_archive/'
+
+# paths used as input
+cluster.srcdir = '/jetfs/home/lkugler/data/compile/WRF-4.3/run'
+cluster.dart_srcdir = '/jetfs/home/lkugler/data/compile/DART/DART-10.5.3/models/wrf/work'
+cluster.rttov_srcdir = '/jetfs/home/lkugler/data/compile/RTTOV13/rtcoef_rttov13/'
+cluster.scriptsdir = '/jetfs/home/lkugler/DART-WRF/dartwrf/'
+
+# other inputs
+cluster.geo_em = '/jetfs/home/lkugler/data/geo_em.d01.nc'
+cluster.obs_impact_filename = cluster.scriptsdir+'/../templates/impactfactor_T.txt'
+cluster.namelist = cluster.scriptsdir+'/../templates/namelist.input'
+cluster.run_WRF = '/jetfs/home/lkugler/DART-WRF/dartwrf/run_ens.jet.sh'
+
+cluster.slurm_cfg = {"account": "lkugler", "partition": "compute", #"nodelist": "jet07",
+ "ntasks": "1", "ntasks-per-core": "1", "mem": "50G",
+ "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
diff --git a/config/srvx1.py b/config/srvx1.py
new file mode 100755
index 0000000..9a60872
--- /dev/null
+++ b/config/srvx1.py
@@ -0,0 +1,76 @@
+import os, sys
+import datetime as dt
+from dartwrf import utils
+from config.cfg import exp
+
+"""Configuration name docs
+
+When coding, use attributes of a dictionary like this:
+$ from cfg import exp, cluster
+$ path = cluster.archivedir
+
+
+attribute name | description
+------------------------------------------------------
+name any string (currently unused)
+
+python path of python version to use
+python_enstools path of python version to use for verification script (not provided)
+ncks path to 'ncks' program; type 'which ncks' to find the path,
+ if it doesn't exist, try to load the module first ('module load nco')
+ideal path to WRF's ideal.exe
+wrfexe path to WRF's wrf.exe
+
+wrf_rundir_base path for temporary files for WRF
+dart_rundir_base path for temporary files for DART
+archive_base path for long-time output storage
+
+srcdir path to where WRF has been compiled, including the 'run' folder of WRF, e.g. /home/WRF-4.3/run
+dart_srcdir path to DART compile directory, e.g. /home/DART-9.11.9/models/wrf/work
+rttov_srcdir path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
+scriptsdir path where DART-WRF scripts reside, e.g. /home/DART-WRF/scripts
+
+namelist path to a namelist template; strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
+run_WRF path to script which runs WRF on a node of the cluster
+obs_impact_filename path to obs_impact_filename (see DART guide; module assim_tools_mod and program obs_impact_tool)
+geo_em path to NetCDF file of WRF domain (see WRF guide)
+
+slurm_cfg python dictionary, containing options of SLURM
+ defined in SLURM docs (https://slurm.schedmd.com/sbatch.html)
+ this configuration can be overwritten later on, for example:
+ 'dict(cluster.slurm_cfg, **cfg_update)' where
+ 'cfg_update = {"nodes": "2"}'
+"""
+
+cluster = utils.ClusterConfig(exp)
+cluster.name = 'srvx1'
+cluster.max_nproc = 6
+cluster.use_slurm = False
+
+# binaries
+cluster.python = '/users/staff/lkugler/miniconda3/bin/python'
+cluster.python_verif = '/users/staff/lkugler/miniconda3/bin/python'
+cluster.ncks = '/home/swd/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/nco-5.0.1-ntu44aoxlvwtr2tsrobfr4lht7cpvccf/bin/ncks'
+cluster.ideal = '' #/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
+cluster.wrfexe = '' #/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
+cluster.container = ''
+
+# paths for data output
+cluster.wrf_rundir_base = '/users/staff/lkugler/AdvDA23/run_WRF/' # path for temporary files
+cluster.dart_rundir_base = '/users/staff/lkugler/AdvDA23/run_DART/' # path for temporary files
+cluster.archive_base = '/mnt/jetfs/scratch/lkugler/data/sim_archive/'
+
+# paths used as input
+cluster.srcdir = '/users/staff/lkugler/AdvDA23/DART/WRF-4.3/run'
+cluster.dart_srcdir = '/users/staff/lkugler/AdvDA23/DART/models/wrf/work'
+cluster.rttov_srcdir = '/users/staff/lkugler/AdvDA23/RTTOV13/rtcoef_rttov13/'
+cluster.scriptsdir = '/users/staff/lkugler/AdvDA23/DART-WRF/dartwrf/'
+cluster.geo_em = '/mnt/jetfs/scratch/lkugler/data/geo_em.d01.nc'
+
+# templates/run scripts
+cluster.namelist = cluster.scriptsdir+'/../templates/namelist.input'
+cluster.run_WRF = cluster.scriptsdir+'/run_ens.jet.sh'
+
+cluster.slurm_cfg = {"account": "lkugler", "partition": "compute",
+ "ntasks": "1", "ntasks-per-core": "1", "mem": "50G",
+ "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
diff --git a/config/vsc.py b/config/vsc.py
new file mode 100755
index 0000000..7f58398
--- /dev/null
+++ b/config/vsc.py
@@ -0,0 +1,77 @@
+import os, sys
+import datetime as dt
+from dartwrf import utils
+from config.cfg import exp
+
+"""Configuration name docs
+
+When coding, use attributes of a dictionary like this:
+$ from cfg import exp, cluster
+$ path = cluster.archivedir
+
+
+attribute name | description
+------------------------------------------------------
+name any string (currently unused)
+
+python path of python version to use
+python_enstools path of python version to use for verification script (not provided)
+ncks path to 'ncks' program; type 'which ncks' to find the path,
+ if it doesn't exist, try to load the module first ('module load nco')
+ideal path to WRF's ideal.exe
+wrfexe path to WRF's wrf.exe
+
+wrf_rundir_base path for temporary files for WRF
+dart_rundir_base path for temporary files for DART
+archive_base path for long-time output storage
+
+srcdir path to where WRF has been compiled, including the 'run' folder of WRF, e.g. /home/WRF-4.3/run
+dart_srcdir path to DART compile directory, e.g. /home/DART-9.11.9/models/wrf/work
+rttov_srcdir path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
+scriptsdir path where DART-WRF scripts reside, e.g. /home/DART-WRF/scripts
+
+namelist path to a namelist template; strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
+run_WRF path to script which runs WRF on a node of the cluster
+obs_impact_filename path to obs_impact_filename (see DART guide; module assim_tools_mod and program obs_impact_tool)
+geo_em path to NetCDF file of WRF domain (see WRF guide)
+
+slurm_cfg python dictionary, containing options of SLURM
+ defined in SLURM docs (https://slurm.schedmd.com/sbatch.html)
+ this configuration can be overwritten later on, for example:
+ 'dict(cluster.slurm_cfg, **cfg_update)' where
+ 'cfg_update = {"nodes": "2"}'
+"""
+
+
+cluster = utils.ClusterConfig(exp)
+cluster.name = 'VSC'
+cluster.max_nproc = 20
+cluster.size_jobarray = 10 # 10 jobs with each 4 WRF processes per node
+cluster.use_slurm = True
+
+# binaries
+cluster.python = '/home/fs71386/lkugler/miniconda3/envs/DART/bin/python'
+cluster.python_enstools = '/home/fs71386/lkugler/miniconda3/envs/enstools/bin/python'
+cluster.ncks = '/home/fs71386/lkugler/miniconda3/envs/DART/bin/ncks'
+cluster.ideal = '/home/fs71386/lkugler/compile/bin/ideal-v4.2.2_v1.22.exe'
+cluster.wrfexe = '/home/fs71386/lkugler/compile/bin/wrf-v4.3_v1.22.exe'
+cluster.container = '/home/fs71386/lkugler/run_container.sh python.gcc9.5.0.vsc4.sif'
+
+# paths for data output
+cluster.wrf_rundir_base = '/gpfs/data/fs71386/lkugler/run_WRF/' # path for temporary files
+cluster.dart_rundir_base = '/gpfs/data/fs71386/lkugler/run_DART/' # path for temporary files
+cluster.archive_base = '/gpfs/data/fs71386/lkugler/sim_archive/'
+
+# paths used as input
+cluster.srcdir = '/gpfs/data/fs71386/lkugler/compile/WRF/WRF-4.3/run'
+cluster.dart_srcdir = '/gpfs/data/fs71386/lkugler/compile/DART/DART/models/wrf/work'
+cluster.rttov_srcdir = '/gpfs/data/fs71386/lkugler/compile/RTTOV13/rtcoef_rttov13/'
+cluster.scriptsdir = '/home/fs71386/lkugler/DART-WRF/dartwrf/'
+
+# templates/run scripts
+cluster.namelist = cluster.scriptsdir+'/../templates/namelist.input'
+cluster.run_WRF = '/home/fs71386/lkugler/DART-WRF/dartwrf/run_ens.vsc.sh'
+
+cluster.slurm_cfg = {"account": "p71386", "partition": "skylake_0384", "qos": "p71386_0384",
+ "nodes": "1", "ntasks": "1", "ntasks-per-node": "48", "ntasks-per-core": "1",
+ "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
diff --git a/dartwrf/assim_synth_obs.py b/dartwrf/assim_synth_obs.py
index 53306f8..2b57fad 100755
--- a/dartwrf/assim_synth_obs.py
+++ b/dartwrf/assim_synth_obs.py
@@ -3,15 +3,16 @@ import time as time_module
import datetime as dt
import numpy as np
-from config.cfg import exp
-from config.clusters import cluster
from dartwrf.utils import symlink, copy, sed_inplace, append_file, mkdir, try_remove, print, shell
from dartwrf.obs import error_models as err
import dartwrf.create_obsseq as osq
from dartwrf import wrfout_add_geo
from dartwrf import obsseq
+from config.cfg import exp
+from config.cluster import cluster
earth_radius_km = 6370
+wrfout_format = 'wrfout_d01_%Y-%m-%d_%H:%M:%S'
def set_DART_nml(just_prior_values=False):
@@ -71,14 +72,21 @@ def set_DART_nml(just_prior_values=False):
def link_nature_to_dart_truth(time):
+ """Set a symlink from the WRFout file to be used as nature to the run_DART folder
+
+ Args:
+ time (dt.datetime): Time at which observations will be made
+ """
+
# get wrfout_d01 from nature run
- shutil.copy(time.strftime(exp.nature_wrfout), cluster.dartrundir + "/wrfout_d01")
+ shutil.copy(time.strftime(exp.nature+'/'+wrfout_format),
+ cluster.dartrundir + "/wrfout_d01")
+
# DART may need a wrfinput file as well, which serves as a template for dimension sizes
- symlink(cluster.dartrundir + "/wrfout_d01", cluster.dartrundir + "/wrfinput_d01")
- print(
- "linked", time.strftime(exp.nature_wrfout),
- "to", cluster.dartrundir + "/wrfout_d01",
- )
+ symlink(cluster.dartrundir + "/wrfout_d01",
+ cluster.dartrundir + "/wrfinput_d01")
+ print("linked", time.strftime(exp.nature+'/'+wrfout_format),
+ "to", cluster.dartrundir + "/wrfout_d01")
def prepare_nature_dart(time):
@@ -103,7 +111,7 @@ def prepare_prior_ensemble(assim_time, prior_init_time, prior_valid_time, prior_
prior_path_exp
+ prior_init_time.strftime("/%Y-%m-%d_%H:%M/")
+ str(iens)
- + prior_valid_time.strftime("/wrfout_d01_%Y-%m-%d_%H:%M:%S")
+ + prior_valid_time.strftime("/"+wrfout_format)
)
dart_ensdir = cluster.dartrundir + "/prior_ens" + str(iens)
wrfout_dart = dart_ensdir + "/wrfout_d01"
@@ -484,27 +492,25 @@ def main(time, prior_init_time, prior_valid_time, prior_path_exp):
print("prepare prior ensemble")
prepare_prior_ensemble(time, prior_init_time, prior_valid_time, prior_path_exp)
- ################################################
print(" 1) get observations with specified obs-error")
oso = get_obsseq_out(time)
- ################################################
- print('3.1) evaluate prior') # evaluate prior observations for all locations
+ print(" 2.1) evaluate prior for all observations (incl rejected)")
osf_prior = evaluate(time, output_format="%Y-%m-%d_%H:%M_obs_seq.final-eval_prior_allobs")
- print(" 3.2) assign observation-errors for assimilation ")
+ print(" 2.2) assign observation-errors for assimilation ")
set_obserr_assimilate_in_obsseqout(oso, osf_prior, outfile=cluster.dartrundir + "/obs_seq.out")
if getattr(exp, "reject_smallFGD", False):
- print(" 3.3) reject observations? ")
+ print(" 2.3) reject observations? ")
qc_obs(time, oso, osf_prior)
- print(" 3.4) assimilate (run filter) ")
+ print(" 3) run filter ")
set_DART_nml()
filter(nproc=nproc)
archive_filteroutput(time)
- # evaluate posterior observations for all locations
+ print(" 4) evaluate posterior observations for all observations (incl rejected)")
write_list_of_inputfiles_posterior(time)
if getattr(exp, "reject_smallFGD", False):
copy(cluster.archivedir+'/obs_seq_out/'+time.strftime('%Y-%m-%d_%H:%M_obs_seq.out-beforeQC'),
diff --git a/dartwrf/create_obs_upfront.py b/dartwrf/create_obs_upfront.py
index 3e107cf..3ccd5a2 100755
--- a/dartwrf/create_obs_upfront.py
+++ b/dartwrf/create_obs_upfront.py
@@ -5,7 +5,7 @@ import datetime as dt
import numpy as np
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from dartwrf.utils import copy, print
import dartwrf.create_obsseq as osq
from dartwrf import obsseq
@@ -24,9 +24,12 @@ if __name__ == "__main__":
for tstr in args:
times.append(dt.datetime.strptime(tstr, tformat))
- dir_for_obsseqout = exp.nature_wrfout + '/../../../obs_seq_out/'+exp.use_existing_obsseq
- os.makedirs(dir_for_obsseqout, exist_ok=True) # create directory to run DART in
+ # strange path?
+ # dir_for_obsseqout = exp.nature_wrfout + '/../../../obs_seq_out/'+exp.use_existing_obsseq
+ raise NotImplementedError('where to save obsseq to?')
+ dir_for_obsseqout = '' # TODO: solve this when necessary
print('will save obsseq to', dir_for_obsseqout)
+ os.makedirs(dir_for_obsseqout, exist_ok=True)
os.chdir(cluster.dartrundir)
diff --git a/dartwrf/create_obsseq.py b/dartwrf/create_obsseq.py
index 6caaf7e..1789a9e 100755
--- a/dartwrf/create_obsseq.py
+++ b/dartwrf/create_obsseq.py
@@ -8,7 +8,7 @@ import datetime as dt
from pysolar.solar import get_altitude, get_azimuth
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from dartwrf.obs import calculate_obs_locations as col
def obskind_read():
diff --git a/dartwrf/create_wbubble_wrfinput.py b/dartwrf/create_wbubble_wrfinput.py
index 5aed424..f586102 100644
--- a/dartwrf/create_wbubble_wrfinput.py
+++ b/dartwrf/create_wbubble_wrfinput.py
@@ -4,7 +4,7 @@ import datetime as dt
import numpy as np
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
import netCDF4 as nc
dx_km = 2
diff --git a/dartwrf/evaluate_posterior.py b/dartwrf/evaluate_posterior.py
index dcc40b5..59c5c02 100755
--- a/dartwrf/evaluate_posterior.py
+++ b/dartwrf/evaluate_posterior.py
@@ -4,7 +4,7 @@ import datetime as dt
import numpy as np
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from dartwrf import assim_synth_obs as aso
diff --git a/dartwrf/evaluate_prior.py b/dartwrf/evaluate_prior.py
index f7d63c7..21751ac 100755
--- a/dartwrf/evaluate_prior.py
+++ b/dartwrf/evaluate_prior.py
@@ -4,7 +4,7 @@ import datetime as dt
import numpy as np
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from dartwrf.utils import symlink, copy, sed_inplace, append_file, mkdir, try_remove, print, shell
from dartwrf import assim_synth_obs as aso
diff --git a/dartwrf/link_dart_rttov.py b/dartwrf/link_dart_rttov.py
index 3eddcef..6c73355 100644
--- a/dartwrf/link_dart_rttov.py
+++ b/dartwrf/link_dart_rttov.py
@@ -1,6 +1,6 @@
import os
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from utils import symlink, copy_scp_srvx8, copy, sed_inplace
joinp = os.path.join
diff --git a/dartwrf/obs/calculate_obs_locations.py b/dartwrf/obs/calculate_obs_locations.py
index 1245cb5..43e09c9 100755
--- a/dartwrf/obs/calculate_obs_locations.py
+++ b/dartwrf/obs/calculate_obs_locations.py
@@ -8,7 +8,7 @@ import datetime as dt
import xarray as xr
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
#####################
# Global variables
diff --git a/dartwrf/obsseq.py b/dartwrf/obsseq.py
index 0e36216..a5b0264 100755
--- a/dartwrf/obsseq.py
+++ b/dartwrf/obsseq.py
@@ -8,7 +8,7 @@ import numpy as np
import pandas as pd
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from dartwrf.utils import symlink, copy, sed_inplace, append_file, mkdir, try_remove
diff --git a/dartwrf/obsseq_2dim.py b/dartwrf/obsseq_2dim.py
index f4c675a..966d709 100755
--- a/dartwrf/obsseq_2dim.py
+++ b/dartwrf/obsseq_2dim.py
@@ -9,7 +9,7 @@ import datetime as dt
import numpy as np
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from dartwrf import assim_synth_obs as aso
from dartwrf import obsseq
diff --git a/dartwrf/obsseq_to_netcdf.py b/dartwrf/obsseq_to_netcdf.py
index fe3cc4a..4076595 100644
--- a/dartwrf/obsseq_to_netcdf.py
+++ b/dartwrf/obsseq_to_netcdf.py
@@ -1,7 +1,7 @@
import os, sys, glob, warnings
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
import run_obs_diag as rod
def listdir_dirs(path):
diff --git a/dartwrf/prep_IC_prior.py b/dartwrf/prep_IC_prior.py
index 15e57e9..c832ffd 100755
--- a/dartwrf/prep_IC_prior.py
+++ b/dartwrf/prep_IC_prior.py
@@ -3,7 +3,7 @@ import datetime as dt
import numpy as np
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from utils import copy, clean_wrfdir, try_remove
"""
diff --git a/dartwrf/prepare_namelist.py b/dartwrf/prepare_namelist.py
index ac2ff1a..8b5a05e 100755
--- a/dartwrf/prepare_namelist.py
+++ b/dartwrf/prepare_namelist.py
@@ -13,7 +13,7 @@ import datetime as dt
from docopt import docopt
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from utils import sed_inplace, copy, symlink, mkdir
def run(iens, begin, end, hist_interval=5, radt=5, archive=True,
diff --git a/dartwrf/prepare_wrfrundir.py b/dartwrf/prepare_wrfrundir.py
index 3cfe2f9..87c2245 100755
--- a/dartwrf/prepare_wrfrundir.py
+++ b/dartwrf/prepare_wrfrundir.py
@@ -2,7 +2,7 @@ import os, sys, shutil
import datetime as dt
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from utils import symlink, copy, link_contents
import prepare_namelist
diff --git a/dartwrf/run_obs_diag.py b/dartwrf/run_obs_diag.py
index d292292..9598a8e 100644
--- a/dartwrf/run_obs_diag.py
+++ b/dartwrf/run_obs_diag.py
@@ -1,7 +1,7 @@
import os, sys, shutil, glob
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
from utils import symlink, copy, sed_inplace, append_file, shell
rundir_program = '/home/fs71386/lkugler/data/run_DART/'
diff --git a/dartwrf/update_IC.py b/dartwrf/update_IC.py
index 3247030..60cec76 100755
--- a/dartwrf/update_IC.py
+++ b/dartwrf/update_IC.py
@@ -3,7 +3,7 @@ import datetime as dt
import netCDF4 as nc
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
def update_initials_in_WRF_rundir(time):
"""Updates wrfrst-files in `/run_WRF/` directory
diff --git a/dartwrf/utils.py b/dartwrf/utils.py
index 4da09e3..b45b79f 100755
--- a/dartwrf/utils.py
+++ b/dartwrf/utils.py
@@ -2,35 +2,18 @@ import os, sys, shutil, glob, warnings
import builtins as __builtin__
import subprocess
import datetime as dt
+import re, tempfile
+import importlib
-class ExperimentConfiguration(object):
- """Collection of variables to use in code later on"""
+class Experiment(object):
+ """Collection of variables regarding the experiment configuration"""
def __init__(self):
pass
-class Shellslurm():
- """Like Slurmpy class, but runs locally"""
- def __init__(self, *args, **kwargs):
- pass
- def run(self, *args, **kwargs):
- print(args[0])
- os.system(args[0])
-
class ClusterConfig(object):
- """Collection of variables to use in code later on"""
+ """Collection of variables regarding the cluster configuration"""
def __init__(self, exp):
self.exp = exp
- self.set_up = False
-
- def setup(self):
- # Set paths and backup scripts
- self.log_dir = self.archivedir+'/logs/'
- self.slurm_scripts_dir = self.archivedir+'/slurm-scripts/'
- print('logging to', self.log_dir)
- print('scripts, which are submitted to SLURM:', self.slurm_scripts_dir)
-
- self.backup_scripts()
- self.set_up = True
@property
def archivedir(self):
@@ -43,49 +26,51 @@ class ClusterConfig(object):
@property
def scripts_rundir(self):
+ """Path to the directory where the DART-WRF scripts are executed
+
+ Example:
+ `/user/data/sim_archive/DART-WRF/`
+ """
return self.archivedir+'/DART-WRF/'
@property
def dartrundir(self):
+ """Path to the directory where DART programs will run
+ Includes the experiment name
+ """
return self.dart_rundir_base+'/'+self.exp.expname+'/'
def wrf_rundir(self, iens):
+ """Path to the directory where an ensemble member will run WRF
+ Includes the experiment name and the ensemble member index
+ """
return self.wrf_rundir_base+'/'+self.exp.expname+'/'+str(iens)
- def create_job(self, *args, cfg_update=dict(), **kwargs):
- """Shortcut to slurmpy's class; keep certain default kwargs
- and only update some with kwarg `cfg_update`
- see https://github.com/brentp/slurmpy
+ def run_job(self, cmd, jobname='', cfg_update=dict(), depends_on=None):
+ """Run scripts in a shell
- depending on cluster config : run either locally or via SLURM
- """
- if not self.set_up:
- self.setup()
+ If not using SLURM: calls scripts through shell
+ if using SLURM: uses slurmpy to submit jobs, keep certain default kwargs and only update some with kwarg `overwrite_these_configurations`
+ Args:
+ cmd (str): Bash command(s) to run
+ jobname (str, optional): Name of SLURM job
+ cfg_update (dict): The config keywords will be overwritten with values
+ depends_on (int or None): SLURM job id of dependency, job will start after this id finished.
+
+ Returns
+ None
+ """
if self.use_slurm:
from slurmpy import Slurm
- return Slurm(*args, slurm_kwargs=dict(self.slurm_cfg, **cfg_update),
- log_dir=self.log_dir,
- scripts_dir=self.slurm_scripts_dir,
- **kwargs)
+ Slurm(jobname, slurm_kwargs=dict(self.slurm_cfg, **cfg_update),
+ log_dir=self.log_dir,
+ scripts_dir=self.slurm_scripts_dir,
+ **kwargs
+ ).run(cmd, depends_on=depends_on)
else:
- return Shellslurm(*args)
-
- def backup_scripts(self):
- """Copies scripts and configuration to cluster.archivedir folder"""
- os.makedirs(self.archivedir, exist_ok=True)
-
- try:
- shutil.copytree(self.scriptsdir, self.scripts_rundir)
- print('scripts have been copied to', self.archivedir)
- except FileExistsError:
- pass
- except:
- raise
- # try:
- # copy(os.path.basename(__file__), self.scripts_rundir+'/')
- # except Exception as e:
- # warnings.warn(str(e))
+ print(cmd)
+ os.system(cmd)
def shell(args):
print(args)
@@ -145,10 +130,19 @@ def copy_scp_srvx8(src, dst):
os.system('scp '+src+' a1254888@srvx8.img.univie.ac.at:'+dst)
def sed_inplace(filename, pattern, repl):
- import re, tempfile
- '''
- Perform the pure-Python equivalent of in-place `sed` substitution: e.g.,
- `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
+ '''Perform the pure-Python equivalent of in-place `sed` substitution
+ Like `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
+
+ Args:
+ filename (str): path to file
+ pattern (str): string that will be replaced
+ repl (str): what shall be written instead of pattern?
+
+ Example:
+ sed_inplace('namelist.input', '<dx>', str(int(exp.model_dx)))
+
+ Returns:
+ None
'''
# For efficiency, precompile the passed regular expression.
pattern_compiled = re.compile(pattern)
diff --git a/dartwrf/workflows.py b/dartwrf/workflows.py
index bc60a01..acc8a8e 100644
--- a/dartwrf/workflows.py
+++ b/dartwrf/workflows.py
@@ -1,192 +1,232 @@
#!/usr/bin/python3
"""
-Run a cycled OSSE with WRF and DART.
+These functions mostly call python scripts via the shell,
+e.g. assimilate() calls dartwrf/assim_synth_obs.py through the shell.
+
+This would not be necessary, but some users might want to use queueing systems (e.g. SLURM) which must call scripts.
"""
import os, sys, shutil, glob, warnings
import datetime as dt
+import importlib
from dartwrf.utils import script_to_str
from config.cfg import exp
-from config.clusters import cluster
-def prepare_WRFrundir(init_time):
- """Create WRF/run directories and wrfinput files
- """
- cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
- print(cmd)
- os.system(cmd)
-
-def run_ideal(depends_on=None):
- """Run ideal for every ensemble member"""
- cmd = """# run ideal.exe in parallel, then add geodata
-export SLURM_STEP_GRES=none
-for ((n=1; n<="""+str(exp.n_ens)+"""; n++))
-do
- rundir="""+cluster.wrf_rundir_base+'/'+exp.expname+"""/$n
- echo $rundir
- cd $rundir
- mpirun -np 1 ./ideal.exe &
-done
-wait
-for ((n=1; n<="""+str(exp.n_ens)+"""; n++))
-do
- rundir="""+cluster.wrf_rundir_base+'/'+exp.expname+"""/$n
- mv $rundir/rsl.out.0000 $rundir/rsl.out.input
-done
-"""
- s = cluster.create_job("ideal", cfg_update={"ntasks": str(exp.n_ens),
- "time": "10", "mem": "100G"})
- id = s.run(cmd, depends_on=[depends_on])
- return id
-
-def wrfinput_insert_wbubble(perturb=True, depends_on=None):
- """Given that directories with wrfinput files exist,
- update these wrfinput files with warm bubbles
+class WorkFlows(object):
+ def __init__(self, exp_config='cfg.py', server_config='server.py'):
+ """Set up the experiment folder in `archivedir`.
+
+ Args:
+ exp (str): Path to exp config file
+ config (str): Path to the cluster config file
+ """
+ # At first, load config from present folder (later from scripts_rundir)
+ # exp = __import__('config/'+exp_config)
+ self.cluster = importlib.import_module('config.'+server_config.strip('.py')).cluster
+
+ # Set paths and backup scripts
+ self.cluster.log_dir = self.cluster.archivedir+'/logs/'
+ print('logging to', self.cluster.log_dir)
+
+ if self.cluster.use_slurm:
+ self.cluster.slurm_scripts_dir = self.cluster.archivedir+'/slurm-scripts/'
+ print('scripts, which are submitted to SLURM:', self.cluster.slurm_scripts_dir)
+
+ # Copy scripts to self.cluster.archivedir folder
+ os.makedirs(self.cluster.archivedir, exist_ok=True)
+ try:
+ shutil.copytree(self.cluster.scriptsdir, self.cluster.scripts_rundir)
+ print('scripts have been copied to', self.cluster.archivedir)
+ except FileExistsError as e:
+ warnings.warn(str(e))
+ except:
+ raise
+
+ # Copy config files
+ shutil.copy('config/'+exp_config, self.cluster.scripts_rundir+'/cfg.py')
+ shutil.copy('config/'+server_config, self.cluster.scripts_rundir+'/cluster.py') # whatever server, the config name is always the same!
+ shutil.copy('config/'+server_config, 'config/cluster.py') # whatever server, the config name is always the same!
+
+
+ def prepare_WRFrundir(self, init_time):
+ """Create WRF/run directories and wrfinput files
+ """
+ cmd = self.cluster.python+' '+self.cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
+ print(cmd)
+ os.system(cmd)
+
+ def run_ideal(self, depends_on=None):
+ """Run ideal for every ensemble member"""
+ cmd = """# run ideal.exe in parallel, then add geodata
+ export SLURM_STEP_GRES=none
+ for ((n=1; n<="""+str(exp.n_ens)+"""; n++))
+ do
+ rundir="""+self.cluster.wrf_rundir_base+'/'+exp.expname+"""/$n
+ echo $rundir
+ cd $rundir
+ mpirun -np 1 ./ideal.exe &
+ done
+ wait
+ for ((n=1; n<="""+str(exp.n_ens)+"""; n++))
+ do
+ rundir="""+self.cluster.wrf_rundir_base+'/'+exp.expname+"""/$n
+ mv $rundir/rsl.out.0000 $rundir/rsl.out.input
+ done
"""
- s = cluster.create_job("ins_wbubble", cfg_update={"time": "5"})
- pstr = ' '
- if perturb:
- pstr = ' perturb'
- id = s.run(cluster.python+' '+cluster.scripts_rundir+'/create_wbubble_wrfinput.py'+pstr,
- depends_on=[depends_on])
- return id
-
-def run_ENS(begin, end, depends_on=None, first_minute=True,
- input_is_restart=True, restart_path=False, output_restart_interval=720):
- """Run forecast for 1 minute, save output.
- Then run whole timespan with 5 minutes interval.
-
- if input_is_restart: # start WRF in restart mode
- """
- id = depends_on
- restart_flag = '.false.' if not input_is_restart else '.true.'
-
- # if False: # doesnt work with restarts at the moment# first_minute:
- # # first minute forecast (needed for validating an assimilation)
- # hist_interval = 1
- # radt = 1 # calc CFRAC also in first minute
- # begin_plus1 = begin+dt.timedelta(minutes=1)
- # s = cluster.create_job("preWRF1", cfg_update=dict(time="2"))
- # args = [cluster.python, cluster.scripts_rundir+'/prepare_namelist.py',
- # begin.strftime('%Y-%m-%d_%H:%M'),
- # begin_plus1.strftime('%Y-%m-%d_%H:%M'),
- # str(hist_interval),
- # '--radt='+str(radt),
- # '--restart='+restart_flag,]
- # id = s.run(' '.join(args), depends_on=[id])
-
- # s = cluster.create_job("runWRF1", cfg_update={"nodes": "1", "array": "1-"+str(exp.n_nodes),
- # "time": "2", "mem-per-cpu": "2G"})
- # cmd = script_to_str(cluster.run_WRF).replace('<expname>', exp.expname)
- # id = s.run(cmd, depends_on=[id])
-
- # # apply forward operator (DART filter without assimilation)
- # s = cluster.create_job("fwOP-1m", cfg_update=dict(time="10", ntasks=48))
- # id = s.run(cluster.python+' '+cluster.scripts_rundir+'/apply_obs_op_dart.py '
- # + begin.strftime('%Y-%m-%d_%H:%M')+' '
- # + begin_plus1.strftime('%Y-%m-%d_%H:%M'),
- # depends_on=[id])
-
- # whole forecast timespan
- hist_interval = 5
- radt = 5
- args = [cluster.python,
- cluster.scripts_rundir+'/prepare_namelist.py',
- begin.strftime('%Y-%m-%d_%H:%M'),
- end.strftime('%Y-%m-%d_%H:%M'),
- str(hist_interval),
- '--radt='+str(radt),
- '--restart='+restart_flag,]
- if output_restart_interval:
- args.append('--restart_interval='+str(int(float(output_restart_interval))))
-
- s = cluster.create_job("preWRF", cfg_update=dict(time="2"))
- id = s.run(' '.join(args), depends_on=[id])
-
- time_in_simulation_hours = (end-begin).total_seconds()/3600
- runtime_wallclock_mins_expected = int(8+time_in_simulation_hours*9.5) # usually below 9 min/hour
- s = cluster.create_job("WRF", cfg_update={"array": "1-"+str(cluster.size_jobarray), "ntasks": "10", "nodes": "1",
- "time": str(runtime_wallclock_mins_expected), "mem": "140G"})
- cmd = script_to_str(cluster.run_WRF).replace('<exp.expname>', exp.expname
- ).replace('<cluster.wrf_rundir_base>', cluster.wrf_rundir_base)
- id = s.run(cmd, depends_on=[id])
- return id
-
-
-def assimilate(assim_time, prior_init_time, prior_valid_time, prior_path_exp,
- depends_on=None):
- """Creates observations from a nature run and assimilates them.
-
- Args:
- assim_time (dt.datetime): timestamp of prior wrfout files
- prior_init_time (dt.datetime): timestamp to find the directory where the prior wrfout files are
- prior_path_exp (str): use this directory to get prior state (i.e. cluster.archivedir)
- """
- if not os.path.exists(prior_path_exp):
- raise IOError('prior_path_exp does not exist: '+prior_path_exp)
-
- id = cluster.create_job("Assim", cfg_update={"ntasks": "12", "time": "60",
- "mem": "200G", "ntasks-per-node": "12", "ntasks-per-core": "2"}
- ).run(cluster.python+' '+cluster.scripts_rundir+'/assim_synth_obs.py '
- +assim_time.strftime('%Y-%m-%d_%H:%M ')
- +prior_init_time.strftime('%Y-%m-%d_%H:%M ')
- +prior_valid_time.strftime('%Y-%m-%d_%H:%M ')
- +prior_path_exp, depends_on=[depends_on])
- return id
-
-
-def prepare_IC_from_prior(prior_path_exp, prior_init_time, prior_valid_time, new_start_time=None, depends_on=None):
-
- if new_start_time != None:
- tnew = new_start_time.strftime(' %Y-%m-%d_%H:%M')
- else:
- tnew = ''
-
- id = cluster.create_job("IC-prior", cfg_update=dict(time="8")
- ).run(cluster.python+' '+cluster.scripts_rundir+'/prep_IC_prior.py '
- +prior_path_exp
- +prior_init_time.strftime(' %Y-%m-%d_%H:%M')
- +prior_valid_time.strftime(' %Y-%m-%d_%H:%M')
- +tnew, depends_on=[depends_on])
- return id
-
-
-def update_IC_from_DA(assim_time, depends_on=None):
- id = cluster.create_job("IC-update", cfg_update=dict(time="8")
- ).run(cluster.python+' '+cluster.scripts_rundir+'/update_IC.py '
- +assim_time.strftime('%Y-%m-%d_%H:%M'), depends_on=[depends_on])
- return id
-
-
-def create_satimages(init_time, depends_on=None):
- s = cluster.create_job("RTTOV", cfg_update={"ntasks": "12", "time": "80", "mem": "200G"})
- id = s.run(cluster.python_verif+' ~/RTTOV-WRF/run_init.py '+cluster.archivedir
- +init_time.strftime('/%Y-%m-%d_%H:%M/'),
- depends_on=[depends_on])
- return id
-
-
-def gen_obsseq(depends_on=None):
- s = cluster.create_job("obsseq_netcdf", cfg_update={"time": "10", "mail-type": "FAIL,END"})
- id = s.run(cluster.python+' '+cluster.scripts_rundir+'/obsseq_to_netcdf.py',
- depends_on=[depends_on])
- return id
-
-
-def verify_sat(depends_on=None):
- s = cluster.create_job("verif-SAT-"+exp.expname, cfg_update={"time": "60", "mail-type": "FAIL,END", "ntasks": "20",
- "ntasks-per-node": "20", "ntasks-per-core": "1", "mem": "100G",})
- cmd = cluster.python_verif+' /jetfs/home/lkugler/osse_analysis/plot_from_raw/analyze_fc.py '+exp.expname+' has_node sat verif1d FSS BS'
- s.run(cmd, depends_on=[depends_on])
-
-def verify_wrf(depends_on=None):
- s = cluster.create_job("verif-WRF-"+exp.expname, cfg_update={"time": "120", "mail-type": "FAIL,END", "ntasks": "20",
- "ntasks-per-node": "20", "ntasks-per-core": "1", "mem": "250G"})
- cmd = cluster.python_verif+' /jetfs/home/lkugler/osse_analysis/plot_from_raw/analyze_fc.py '+exp.expname+' has_node wrf verif1d verif3d FSS BS'
- s.run(cmd, depends_on=[depends_on])
-
-def verify_fast(depends_on=None):
- s = cluster.create_job("verif-fast-"+exp.expname, cfg_update={"time": "10", "mail-type": "FAIL", "ntasks": "1",
- "ntasks-per-node": "1", "ntasks-per-core": "1"})
- cmd = cluster.python_verif+' /jetfs/home/lkugler/osse_analysis/plot_fast/plot_single_exp.py '+exp.expname
- s.run(cmd, depends_on=[depends_on])
+ id = self.cluster.run_job(cmd, "ideal", cfg_update={"ntasks": str(exp.n_ens),
+ "time": "10", "mem": "100G"}, depends_on=[depends_on])
+ return id
+
+ def wrfinput_insert_wbubble(self, perturb=True, depends_on=None):
+ """Given that directories with wrfinput files exist,
+ update these wrfinput files with warm bubbles
+ """
+
+ pstr = ' '
+ if perturb:
+ pstr = ' perturb'
+ cmd = self.cluster.python+' '+self.cluster.scripts_rundir+'/create_wbubble_wrfinput.py'+pstr
+
+ id = self.cluster.run_job(cmd, "ins_wbubble", cfg_update={"time": "5"}, depends_on=[depends_on])
+ return id
+
+ def run_ENS(self, begin, end, depends_on=None, first_minute=True,
+ input_is_restart=True, restart_path=False, output_restart_interval=720):
+ """Run forecast for 1 minute, save output.
+ Then run whole timespan with 5 minutes interval.
+
+ if input_is_restart: # start WRF in restart mode
+ """
+ id = depends_on
+ restart_flag = '.false.' if not input_is_restart else '.true.'
+
+ # if False: # doesnt work with restarts at the moment# first_minute:
+ # # first minute forecast (needed for validating an assimilation)
+ # hist_interval = 1
+ # radt = 1 # calc CFRAC also in first minute
+ # begin_plus1 = begin+dt.timedelta(minutes=1)
+ # s = self.cluster.run_job("preWRF1", cfg_update=dict(time="2"))
+ # args = [self.cluster.python, self.cluster.scripts_rundir+'/prepare_namelist.py',
+ # begin.strftime('%Y-%m-%d_%H:%M'),
+ # begin_plus1.strftime('%Y-%m-%d_%H:%M'),
+ # str(hist_interval),
+ # '--radt='+str(radt),
+ # '--restart='+restart_flag,]
+ # id = s.run(' '.join(args), depends_on=[id])
+
+ # s = self.cluster.run_job("runWRF1", cfg_update={"nodes": "1", "array": "1-"+str(exp.n_nodes),
+ # "time": "2", "mem-per-cpu": "2G"})
+ # cmd = script_to_str(self.cluster.run_WRF).replace('<expname>', exp.expname)
+ # id = s.run(cmd, depends_on=[id])
+
+ # # apply forward operator (DART filter without assimilation)
+ # s = self.cluster.run_job("fwOP-1m", cfg_update=dict(time="10", ntasks=48))
+ # id = s.run(self.cluster.python+' '+self.cluster.scripts_rundir+'/apply_obs_op_dart.py '
+ # + begin.strftime('%Y-%m-%d_%H:%M')+' '
+ # + begin_plus1.strftime('%Y-%m-%d_%H:%M'),
+ # depends_on=[id])
+
+ # whole forecast timespan
+ hist_interval = 5
+ radt = 5
+ args = [self.cluster.python,
+ self.cluster.scripts_rundir+'/prepare_namelist.py',
+ begin.strftime('%Y-%m-%d_%H:%M'),
+ end.strftime('%Y-%m-%d_%H:%M'),
+ str(hist_interval),
+ '--radt='+str(radt),
+ '--restart='+restart_flag,]
+ if output_restart_interval:
+ args.append('--restart_interval='+str(int(float(output_restart_interval))))
+
+ id = self.cluster.run_job(' '.join(args), "preWRF", cfg_update=dict(time="2"), depends_on=[id])
+
+ cmd = script_to_str(self.cluster.run_WRF).replace('<exp.expname>', exp.expname
+ ).replace('<cluster.wrf_rundir_base>', self.cluster.wrf_rundir_base)
+
+ time_in_simulation_hours = (end-begin).total_seconds()/3600
+ runtime_wallclock_mins_expected = int(8+time_in_simulation_hours*9.5) # usually below 9 min/hour
+
+ id = self.cluster.run_job(cmd, "WRF", cfg_update={"array": "1-"+str(self.cluster.size_jobarray), "ntasks": "10", "nodes": "1",
+ "time": str(runtime_wallclock_mins_expected), "mem": "140G"}, depends_on=[id])
+ return id
+
+
+ def assimilate(self, assim_time, prior_init_time, prior_valid_time, prior_path_exp,
+ depends_on=None):
+ """Creates observations from a nature run and assimilates them.
+
+ Args:
+ assim_time (dt.datetime): timestamp of prior wrfout files
+ prior_init_time (dt.datetime): timestamp to find the directory where the prior wrfout files are
+ prior_path_exp (str): use this directory to get prior state (i.e. self.cluster.archivedir)
+ """
+ if not os.path.exists(prior_path_exp):
+ raise IOError('prior_path_exp does not exist: '+prior_path_exp)
+
+ cmd = (self.cluster.python+' '+self.cluster.scripts_rundir+'/assim_synth_obs.py '
+ +assim_time.strftime('%Y-%m-%d_%H:%M ')
+ +prior_init_time.strftime('%Y-%m-%d_%H:%M ')
+ +prior_valid_time.strftime('%Y-%m-%d_%H:%M ')
+ +prior_path_exp)
+
+ id = self.cluster.run_job(cmd, "Assim", cfg_update={"ntasks": "12", "time": "60",
+ "mem": "200G", "ntasks-per-node": "12", "ntasks-per-core": "2"}, depends_on=[depends_on])
+ return id
+
+
+ def prepare_IC_from_prior(self, prior_path_exp, prior_init_time, prior_valid_time, new_start_time=None, depends_on=None):
+
+ if new_start_time != None:
+ tnew = new_start_time.strftime(' %Y-%m-%d_%H:%M')
+ else:
+ tnew = ''
+
+ cmd = (self.cluster.python+' '+self.cluster.scripts_rundir+'/prep_IC_prior.py '
+ +prior_path_exp
+ +prior_init_time.strftime(' %Y-%m-%d_%H:%M')
+ +prior_valid_time.strftime(' %Y-%m-%d_%H:%M')
+ +tnew)
+ id = self.cluster.run_job(cmd, "IC-prior", cfg_update=dict(time="8"), depends_on=[depends_on])
+ return id
+
+
+ def update_IC_from_DA(self, assim_time, depends_on=None):
+ cmd = self.cluster.python+' '+self.cluster.scripts_rundir+'/update_IC.py '+assim_time.strftime('%Y-%m-%d_%H:%M')
+ id = self.cluster.run_job(cmd, "IC-update", cfg_update=dict(time="8"), depends_on=[depends_on])
+ return id
+
+
+ def create_satimages(self, init_time, depends_on=None):
+ cmd = self.cluster.python_verif+' ~/RTTOV-WRF/run_init.py '+self.cluster.archivedir+init_time.strftime('/%Y-%m-%d_%H:%M/')
+ id = self.cluster.run_job(cmd, "RTTOV", cfg_update={"ntasks": "12", "time": "80", "mem": "200G"}, depends_on=[depends_on])
+ return id
+
+
+ def gen_obsseq(self, depends_on=None):
+ cmd = self.cluster.python+' '+self.cluster.scripts_rundir+'/obsseq_to_netcdf.py'
+ id = self.cluster.run_job("obsseq_netcdf", cfg_update={"time": "10", "mail-type": "FAIL,END"},
+ depends_on=[depends_on])
+ return id
+
+
+ def verify_sat(self, depends_on=None):
+ cmd = self.cluster.python_verif+' /jetfs/home/lkugler/osse_analysis/plot_from_raw/analyze_fc.py '+exp.expname+' has_node sat verif1d FSS BS'
+
+ self.cluster.run_job(cmd, "verif-SAT-"+exp.expname,
+ cfg_update={"time": "60", "mail-type": "FAIL,END", "ntasks": "20",
+ "ntasks-per-node": "20", "ntasks-per-core": "1", "mem": "100G",}, depends_on=[depends_on])
+
+ def verify_wrf(self, depends_on=None):
+ cmd = self.cluster.python_verif+' /jetfs/home/lkugler/osse_analysis/plot_from_raw/analyze_fc.py '+exp.expname+' has_node wrf verif1d verif3d FSS BS'
+
+ self.cluster.run_job(cmd, "verif-WRF-"+exp.expname,
+ cfg_update={"time": "120", "mail-type": "FAIL,END", "ntasks": "20",
+ "ntasks-per-node": "20", "ntasks-per-core": "1", "mem": "250G"}, depends_on=[depends_on])
+
+ def verify_fast(self, depends_on=None):
+ cmd = self.cluster.python_verif+' /jetfs/home/lkugler/osse_analysis/plot_fast/plot_single_exp.py '+exp.expname
+
+ self.cluster.run_job(cmd, "verif-fast-"+exp.expname,
+ cfg_update={"time": "10", "mail-type": "FAIL", "ntasks": "1",
+ "ntasks-per-node": "1", "ntasks-per-core": "1"}, depends_on=[depends_on])
diff --git a/dartwrf/wrfinput_add_geo.py b/dartwrf/wrfinput_add_geo.py
index fe4e155..1a2c266 100755
--- a/dartwrf/wrfinput_add_geo.py
+++ b/dartwrf/wrfinput_add_geo.py
@@ -12,7 +12,7 @@ import os, sys
import netCDF4 as nc
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
def run(geo_data_file, wrfinput_file):
geo_ds = nc.Dataset(geo_data_file, 'r')
diff --git a/dartwrf/wrfout_add_geo.py b/dartwrf/wrfout_add_geo.py
index 99e1d85..a13737c 100755
--- a/dartwrf/wrfout_add_geo.py
+++ b/dartwrf/wrfout_add_geo.py
@@ -2,7 +2,7 @@ import os, sys
import netCDF4 as nc
from config.cfg import exp
-from config.clusters import cluster
+from config.cluster import cluster
fields_old = ["XLAT_M", "XLONG_M", "CLAT",
"XLONG_U", "XLONG_V", "XLAT_U", "XLAT_V"]
--
GitLab