no significant change

config/cfg.py

@@ -7,12 +7,14 @@ class ExperimentConfiguration(object):
         pass
 
 exp = ExperimentConfiguration()
-exp.expname = "exp_v1.19_Pwbub-1_WV_obs20_loc10"
+exp.expname = "exp_v1.19_rr_WV_obs4-20_loc10"
 exp.model_dx = 2000
 exp.n_ens = 40
 exp.n_nodes = 10
 
-exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_Pwbub5_nat/2008-07-30_12:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
+#exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_Pwbub5_nat/2008-07-30_12:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
+exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.18_P1_nature/2008-07-30_06:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
+
 #exp.input_profile = '/home/fs71386/lkugler/wrf_profiles/data/wrf/ens/2021-05-04/raso.nat.001.wrfprof'
 #exp.input_profile = '/home/fs71386/lkugler/wrf_profiles/data/wrf/ens/2021-05-04/raso.nat.<iens>.wrfprof'
 exp.input_profile = '/home/fs71386/lkugler/wrf_profiles/data/wrf/ens/2021-05-04/raso.fc.<iens>.wrfprof'
@@ -22,19 +24,19 @@ exp.input_profile = '/home/fs71386/lkugler/wrf_profiles/data/wrf/ens/2021-05-04/
 # localize vertically, if it has a vertical position
 # needs a horizontal scale too, to calculate the vertical normalization
 # since you can not specify different vertical localizations for diff. variables
-exp.cov_loc_vert_km_horiz_km = (1, 10)  
-#exp.superob_km = 12
+#exp.cov_loc_vert_km_horiz_km = (2, 20)  
+exp.superob_km = 20
 
-n_obs = 256  #5776: 4km, 121: 30km, 256:16x16 (20km); 961: 10km resoltn # radar: n_obs for each observation height level
+n_obs = 5776  # 5776: 4km, 121: 30km, 256:16x16 (20km); 961: 10km resoltn # radar: n_obs for each observation height level
 
 vis = dict(plotname='VIS 0.6µm',  plotunits='[1]',
            kind='MSG_4_SEVIRI_BDRF', sat_channel=1, n_obs=n_obs, 
            error_generate=0.03, error_assimilate=0.06,
-           cov_loc_radius_km=30)
+           cov_loc_radius_km=10)
 
 wv73 = dict(plotname='Brightness temperature WV 7.3µm',  plotunits='[K]',
             kind='MSG_4_SEVIRI_TB', sat_channel=6, n_obs=n_obs, 
-            error_generate=1., error_assimilate=False, 
+            error_generate=1., error_assimilate=3., 
             cov_loc_radius_km=10)
 
 ir108 = dict(plotname='Brightness temperature IR 10.8µm', plotunits='[K]',
@@ -50,9 +52,9 @@ radar = dict(plotname='Radar reflectivity', plotunits='[dBz]',
 
 t = dict(plotname='Temperature', plotunits='[K]',
          kind='RADIOSONDE_TEMPERATURE', n_obs=n_obs,
-         error_generate=0.2, error_assimilate=0.4,
-         heights=np.arange(1000, 15001, 500),
-         cov_loc_radius_km=15)
+         error_generate=0.25, error_assimilate=0.5,
+         heights=np.arange(1000, 15001, 1000),
+         cov_loc_radius_km=10)
 
 t2m = dict(plotname='SYNOP Temperature', plotunits='[K]',
            kind='SYNOP_TEMPERATURE', n_obs=n_obs, 
@@ -65,9 +67,8 @@ psfc = dict(plotname='SYNOP Pressure', plotunits='[dBz]',
             cov_loc_radius_km=32)
 
 
-exp.observations = [wv73]  # 108, wv73, vis]
-#exp.update_vars = ['T', 'QVAPOR', 'QCLOUD', 'QICE','CLDFRA']
-exp.update_vars = ['U', 'V', 'T', 'PH', 'MU', 'QVAPOR', 'QCLOUD', 'QICE', 'CLDFRA']
+exp.observations = [wv73]
+exp.update_vars = ['U', 'V', 'T', 'PH', 'MU', 'QVAPOR', 'QCLOUD', 'QICE', 'CLDFRA', 'PSFC']
 
 # directory paths depend on the name of the experiment
 cluster.expname = exp.expname

scheduler.py

@@ -27,7 +27,7 @@ def my_Slurm(*args, cfg_update=dict(), **kwargs):
     and only update some with kwarg `cfg_update`
     see https://github.com/brentp/slurmpy
     """
-    debug = True  # run without SLURM, locally on headnode
+    debug = False  # run without SLURM, locally on headnode
     if debug:
         return Shellslurm(*args)
     return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update), 
@@ -214,15 +214,9 @@ def gen_obsseq(depends_on=None):
 
 
 def verify(depends_on=None):
-    s = my_Slurm("verify-"+exp.expname, cfg_update={"time": "240", "mail-type": "FAIL,END", 
-                 "ntasks": "40",  "ntasks-per-node": "40", "ntasks-per-core": "1"})
-    s.run(cluster.python_enstools+' /home/fs71386/lkugler/osse_analysis/analyze_fc.py '+exp.expname+' has_node',
-          depends_on=[depends_on])
-
-
-def cleanup_storage(depends_on=None):
-    my_Slurm('cleanup', cfg_update={"time": "2"}).run(
-        cluster.python+' '+cluster.scripts_rundir+'/cleanup_exp.py '+exp.expname, 
+    s = my_Slurm("verify-"+exp.expname, cfg_update={"time": "240", "mail-type": "FAIL,END", "ntasks": "1", 
+            "ntasks-per-node": "1", "ntasks-per-core": "1"})
+    s.run(cluster.python_enstools+' /home/fs71386/lkugler/osse_analysis/analyze_fc.py '+exp.expname+' has_node plot',
           depends_on=[depends_on])
 
 
@@ -237,19 +231,19 @@ if __name__ == "__main__":
     id = None
 
     init_time = dt.datetime(2008, 7, 30, 12)
-    time = dt.datetime(2008, 7, 30, 12, 30)
+    time = dt.datetime(2008, 7, 30, 13)
 
     id = prepare_WRFrundir(init_time)
     #id = run_ideal(depends_on=id)
 
     #prior_path_exp = cluster.archivedir  # 
-    prior_path_exp = '/gpfs/data/fs71386/lkugler/sim_archive/exp_v1.19_Pwbub5_40mem'
+    prior_path_exp = '/gpfs/data/fs71386/lkugler/sim_archive/exp_v1.19_P1_noDA'
     #id = wrfinput_insert_wbubble(depends_on=id)
     
     prior_init_time = init_time
     prior_valid_time = time
 
-    while time <= dt.datetime(2008, 7, 30, 13,30):
+    while time <= dt.datetime(2008, 7, 30, 14):
 
         # usually we take the prior from the current time
         # but one could use a prior from a different time from another run
@@ -257,7 +251,7 @@ if __name__ == "__main__":
         prior_valid_time = time
 
         id = assimilate(time, prior_init_time, prior_valid_time, prior_path_exp, depends_on=id)
-        sys.exit()
+
         # 1) Set posterior = prior
         id = prepare_IC_from_prior(prior_path_exp, prior_init_time, prior_valid_time, depends_on=id)
 
@@ -267,8 +261,8 @@ if __name__ == "__main__":
         # How long shall we integrate?
         timedelta_integrate = timedelta_btw_assim
         output_restart_interval = timedelta_btw_assim.total_seconds()/60
-        if time == dt.datetime(2008, 7, 30, 13,30): #this_forecast_init.minute in [0,]:  # longer forecast every full hour
-            timedelta_integrate = dt.timedelta(hours=3)
+        if time == dt.datetime(2008, 7, 30, 14): #this_forecast_init.minute in [0,]:  # longer forecast every full hour
+            timedelta_integrate = dt.timedelta(hours=1)
             output_restart_interval = 9999
 
         # 3) Run WRF ensemble
@@ -288,7 +282,5 @@ if __name__ == "__main__":
         # update time variables
         prior_init_time = time - timedelta_btw_assim
 
-
-    cleanup_storage(id)
     id = gen_obsseq(id)
     verify(id)

tests/test_assim.py

+import os, shutil
+import numpy as np
+import datetime as dt
+
+from dartwrf import obsseq, assim_synth_obs
+from config.cfg import cluster
+
+class ExperimentConfiguration(object):
+    def __init__(self):
+        pass
+
+exp = ExperimentConfiguration()
+exp.expname = "test"
+
+wv73 = dict(plotname='Brightness temperature WV 7.3µm',  plotunits='[K]',
+                kind='MSG_4_SEVIRI_TB', sat_channel=6, n_obs=4,
+                error_generate=1., error_assimilate=False,
+                cov_loc_radius_km=32)
+exp.observations = [wv73]
+
+time = dt.datetime(2008,7,30,12)
+
+
+def test_overwrite_OE_assim():
+    # checks if modified entries are correctly written to DART files
+    input1 = cluster.scriptsdir + "/../tests/obs_seq.orig.out"
+    input2 = cluster.scriptsdir + "/../tests/obs_seq.out"
+    output = cluster.scriptsdir + "/../tests/obs_seq.test.out"
+    shutil.copy(input1, input2)
+
+    oso = obsseq.ObsSeq(input2)
+
+    assim_synth_obs.set_obserr_assimilate_in_obsseqout(time, exp, oso, outfile=output)
+
+    var_orig = oso.df['variance']
+
+    oso_test = obsseq.ObsSeq(output)  # read in again
+    assert oso_test.df['variance'].iloc[0] == var_orig
+    os.remove(output)
+    os.remove(input2)
+
+if __name__ == '__main__':
+    test_overwrite_OE_assim()

tests/test_obsseq.py

-import filecmp
-from config.cfg import exp, cluster
+import os, filecmp, shutil
+import numpy as np
 
+from config.cfg import exp, cluster
 from dartwrf import obsseq
 
 
@@ -29,4 +30,4 @@ def test_osf():
     # TODO: compare with given truth
 
 if __name__ == '__main__':
-    test_osf()
+    pass
\ No newline at end of file