fixes

config/cfg.py

@@ -9,23 +9,23 @@ class ExperimentConfiguration(object):
 
 
 exp = ExperimentConfiguration()
-exp.expname = "exp_v1.12_conv1"
+exp.expname = "exp_v1.12_IR-WV-VIS"
 exp.model_dx = 2000
 exp.timestep = 10
 exp.n_ens = 40
 exp.n_nodes = 10
 
-n_obs = 64  # radar: n_obs for each observation height level
+n_obs = 1600  # radar: n_obs for each observation height level
 
 vis = dict(kind='MSG_4_SEVIRI_BDRF',
            sat_channel=1, n_obs=n_obs, err_std=0.03,
-           cov_loc_radius_km=10)
-wv = dict(kind='MSG_4_SEVIRI_TB',
+           cov_loc_radius_km=20)
+wv73 = dict(kind='MSG_4_SEVIRI_TB',
           sat_channel=6, n_obs=n_obs, err_std=False,
-          cov_loc_radius_km=10)
+          cov_loc_radius_km=20)
 ir108 = dict(kind='MSG_4_SEVIRI_TB',
              sat_channel=9, n_obs=n_obs, err_std=5.,
-             cov_loc_radius_km=10)
+             cov_loc_radius_km=20)
 
 radar = dict(kind='RADAR', n_obs=n_obs, err_std=5.,
              heights=np.arange(1000, 15001, 1000),
@@ -35,9 +35,9 @@ t2m = dict(kind='SYNOP_TEMPERATURE', n_obs=n_obs, err_std=1.0,
            cov_loc_radius_km=32, cov_loc_vert_km=1)
 psfc = dict(kind='SYNOP_SURFACE_PRESSURE', n_obs=n_obs, err_std=50.,
            cov_loc_radius_km=32, cov_loc_vert_km=5)
-           
 
-exp.observations = [t2m, psfc, ]
+
+exp.observations = [ir108, wv73, vis]
 
 # directory paths depend on the name of the experiment
 cluster.expname = exp.expname

scheduler.py

@@ -127,7 +127,7 @@ def run_ENS(begin, end, depends_on=None):
     id = s.run(cmd, depends_on=[id])
 
     # apply forward operator (DART filter without assimilation)
-    s = my_Slurm("fwOP-1m", cfg_update=dict(time="10"))
+    s = my_Slurm("fwOP-1m", cfg_update=dict(time="10", ntasks=48))
     id = s.run(cluster.python+' '+cluster.scriptsdir+'/apply_obs_op_dart.py '
                + begin.strftime('%Y-%m-%d_%H:%M')+' '
                + begin_plus1.strftime('%Y-%m-%d_%H:%M'),
@@ -188,7 +188,8 @@ def assimilate(assim_time, prior_init_time,
                +prior_path_exp, depends_on=[depends_on])
 
     # prepare nature run, generate observations
-    s = my_Slurm("Assim", cfg_update=dict(nodes="1", ntasks="48", time="50", mem="200G"))
+    s = my_Slurm("Assim", cfg_update={"nodes": "1", "ntasks": "96", "time": "30",
+                             "mem": "300G", "ntasks-per-node": "96", "ntasks-per-core": "2"})
     id = s.run(cluster.python+' '+cluster.scriptsdir+'/assim_synth_obs.py '
                +time.strftime('%Y-%m-%d_%H:%M'), depends_on=[id])
  
@@ -245,21 +246,20 @@ if is_new_run:
     first_guess = False
     
 elif start_from_existing_state:
-    #id = prepare_wrfinput()  # create initial conditions
+    id = prepare_wrfinput()  # create initial conditions
     
     # get initial conditions from archive
     init_time = dt.datetime(2008, 7, 30, 6)
     time = dt.datetime(2008, 7, 30, 10)
     exppath_arch = '/gpfs/data/fs71386/lkugler/sim_archive/exp_v1.11_LMU_filter'
-    
-    #id = update_wrfinput_from_archive(time, init_time, exppath_arch, depends_on=id)
+    id = update_wrfinput_from_archive(time, init_time, exppath_arch, depends_on=id)
 
 # values for assimilation
 assim_time = time
 prior_init_time = init_time
 prior_path_exp = exppath_arch
 
-while time <= dt.datetime(2008, 7, 30, 16):
+while time <= dt.datetime(2008, 7, 30, 17):
 
     id = assimilate(assim_time,
                     prior_init_time,
@@ -282,6 +282,6 @@ while time <= dt.datetime(2008, 7, 30, 16):
     assim_time = time
     prior_init_time = assim_time - timedelta_btw_assim
     
-    #create_satimages(depends_on=id)
+    create_satimages(depends_on=id)
 
 mailme(id)

scripts/apply_obs_op_dart.py

@@ -50,7 +50,7 @@ if __name__ == '__main__':
         wrfout_add_geo.run(cluster.dartrundir+'/geo_em.d01.nc',
                            cluster.dartrundir+'/wrfout_d01')
         aso.run_perfect_model_obs()
-        aso.assimilate()
+        aso.assimilate(nproc=48)
 
         archive_stage = savedir+kind
         # only the prior state values are of interest in this file

scripts/assim_synth_obs.py

@@ -184,13 +184,13 @@ def run_perfect_model_obs():
         raise RuntimeError('obs_seq.in does not exist in '+cluster.dartrundir)
     os.system('mpirun -np 12 ./perfect_model_obs')
 
-def assimilate():
+def assimilate(nproc=96):
     print('running filter')
     os.chdir(cluster.dartrundir)
     try_remove(cluster.dartrundir+'/obs_seq.final')
     if not os.path.exists(cluster.dartrundir+'/obs_seq.out'):
         raise RuntimeError('obs_seq.out does not exist in '+cluster.dartrundir)
-    os.system('mpirun -np 48 ./filter')
+    os.system('mpirun -genv I_MPI_PIN_PROCESSOR_LIST=0-'+str(int(nproc)-1)+' -np '+str(int(nproc))+' ./filter')
 
 def archive_diagnostics(archive_dir, time):
     print('archive obs space diagnostics')
@@ -210,7 +210,12 @@ def recycle_output():
         os.rename(cluster.dartrundir+'/filter_restart_d01.'+str(iens).zfill(4),
                   cluster.dartrundir+'/advance_temp'+str(iens)+'/wrfout_d01')
 
-def archive_output_mean(archive_stage):
+def archive_output(archive_stage):
+    print('archiving output')
+    mkdir(archive_stage)
+    copy(cluster.dartrundir+'/input.nml', archive_stage+'/input.nml')
+
+    # single members
     # for iens in range(1, exp.n_ens+1):
     #     savedir = archive_stage+'/'+str(iens)
     #     mkdir(savedir)
@@ -288,13 +293,11 @@ if __name__ == "__main__":
         if istage < n_stages-1:
             # recirculation: filter output -> input
             recycle_output()
-            copy(cluster.dartrundir+'/input.nml', archive_stage+'/input.nml')
-            archive_output_mean(archive_stage)
+            archive_output(archive_stage)
 
         elif istage == n_stages-1:
             # last assimilation, continue integration now
-            copy(cluster.dartrundir+'/input.nml', archive_stage+'/input.nml')
-            archive_output_mean(archive_stage)
+            archive_output(archive_stage)
 
         else:
             RuntimeError('this should never occur?!')

scripts/prepare_wrfinput.py

@@ -16,8 +16,8 @@ for iens in range(1, exp.n_ens+1):
     symlink(cluster.ideal, rundir+'/ideal.exe')
     symlink(cluster.wrfexe, rundir+'/wrf.exe')
 
-    prepare_namelist.run(cluster, iens, begin=dt.datetime(2008, 7, 30, 6, 0),
-                                        end=dt.datetime(2008, 7, 30, 6, 30)) # not necessary
+    prepare_namelist.run(iens, begin=dt.datetime(2008, 7, 30, 6, 0),
+                         end=dt.datetime(2008, 7, 30, 6, 30)) # not necessary
 
     symlink(input_prof, rundir+'/input_sounding')
 print('finished.')

templates/input.nml

@@ -282,9 +282,9 @@
 # tasks_per_node is set to match your hardware
 &ensemble_manager_nml
    layout = 1,
-   tasks_per_node = 48
+   tasks_per_node = 96
    communication_configuration = 1
-   debug                       = .false.
+   debug                       = .true.
    /
 
 &obs_def_gps_nml