From db30223c0566ed1830598f6543b548b21735dc90 Mon Sep 17 00:00:00 2001 From: lkugler <lukas.kugler@gmail.com> Date: Fri, 5 May 2023 03:10:52 +0200 Subject: [PATCH 1/7] inf_initial + preassim archive --- config/cfg.py | 14 +++++++------- dartwrf/assim_synth_obs.py | 11 ++++++----- templates/input.nml | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/config/cfg.py b/config/cfg.py index 8a833a6..c20d351 100755 --- a/config/cfg.py +++ b/config/cfg.py @@ -1,12 +1,13 @@ from dartwrf import utils exp = utils.Experiment() -exp.expname = "obs-cold_localization-narrow" +exp.expname = "obs-T_inflation-1.1" exp.model_dx = 2000 exp.n_ens = 10 exp.filter_kind = 1 exp.prior_inflation = 0 +exp.inf_initial = 1.1 exp.post_inflation = 4 exp.sec = True exp.cov_loc_vert_km_horiz_km = (4, 40) @@ -14,7 +15,7 @@ exp.superob_km = False # False or int (spatial averaging of observations) exp.adjust_obs_impact = False exp.use_existing_obsseq = False # False or pathname (use precomputed obs_seq.out files) -exp.use_existing_obsseq = '/users/students/lehre/advDA_s2023/dartwrf_tutorial/very_cold_observation.out' +#exp.use_existing_obsseq = '/users/students/lehre/advDA_s2023/dartwrf_tutorial/very_cold_observation.out' # path to the nature run, where we take observations from exp.nature = '/users/students/lehre/advDA_s2023/data/sample_nature/' @@ -58,11 +59,11 @@ radar = dict(plotname='Radar reflectivity', plotunits='[dBz]', t = dict(plotname='Temperature', plotunits='[K]', kind='RADIOSONDE_TEMPERATURE', - #n_obs=22500, obs_locations='square_array_evenly_on_grid', - n_obs=1, obs_locations=[(45., 0.)], + n_obs=961, obs_locations='square_array_evenly_on_grid', + #n_obs=1, obs_locations=[(45., 0.)], error_generate=0.2, error_assimilate=0.2, heights=[1000,], #range(1000, 17001, 2000), - cov_loc_radius_km=50) + cov_loc_radius_km=30) q = dict(plotname='Specific humidity', plotunits='[kg/kg]', kind='RADIOSONDE_SPECIFIC_HUMIDITY', n_obs=1, @@ -80,7 +81,6 @@ psfc = dict(plotname='SYNOP Pressure', plotunits='[Pa]', error_generate=50., error_assimilate=100., cov_loc_radius_km=32) -exp.observations = [t2m] +exp.observations = [t] exp.update_vars = ['U', 'V', 'W', 'THM', 'PH', 'MU', 'QVAPOR', 'QCLOUD', 'QICE', 'PSFC'] -#exp.update_vars = ['U', 'V', 'W', 'T', 'PH', 'MU', 'QVAPOR', 'PSFC'] diff --git a/dartwrf/assim_synth_obs.py b/dartwrf/assim_synth_obs.py index 7fcc052..b67196d 100755 --- a/dartwrf/assim_synth_obs.py +++ b/dartwrf/assim_synth_obs.py @@ -46,6 +46,7 @@ def set_DART_nml(just_prior_values=False): "<sampling_error_correction>": '.true.' if exp.sec else '.false.', "<prior_inflation>": str(exp.prior_inflation), "<post_inflation>": str(exp.post_inflation), + "<inf_initial>": str(exp.inf_initial), "<n_ens>": str(int(exp.n_ens)), "<cov_loc_radian>": "0.00000001", # dummy value, used for types not mentioned below "<list_obstypes>": "'" + "','".join(list_obstypes) + "'", @@ -220,11 +221,11 @@ def archive_filteroutput(time): ) try: # not necessary for next forecast run - for iens in range(1, exp.n_ens + 1): - copy( - cluster.dartrundir + "/postassim_member_" + str(iens).zfill(4) + ".nc", - archive_assim + "/postassim_member_" + str(iens).zfill(4) + ".nc", - ) + ftypes = ['preassim', 'postassim'] + for ftype in ftypes: + for iens in range(1, exp.n_ens + 1): + fname = "/"+ftype+"_member_" + str(iens).zfill(4) + ".nc" + copy(cluster.dartrundir + fname, archive_assim + fname) for f in ["output_mean.nc", "output_sd.nc"]: # copy mean and sd to archive copy(cluster.dartrundir + "/" + f, archive_assim + "/" + f) diff --git a/templates/input.nml b/templates/input.nml index 38edbde..3b673a7 100644 --- a/templates/input.nml +++ b/templates/input.nml @@ -60,7 +60,7 @@ inf_flavor = <prior_inflation>, <post_inflation>, inf_initial_from_restart = .true., .false., inf_sd_initial_from_restart = .true., .false., - inf_initial = 1.00, 0.90, + inf_initial = <inf_initial>, 0.90, inf_lower_bound = 1.0, 1.0, inf_upper_bound = 1000000.0, 1000000.0, inf_damping = 0.9, 1.0, -- GitLab From 5eb72fd057219bad720f7feee82c40fa9b7fcc02 Mon Sep 17 00:00:00 2001 From: lkugler <lukas.kugler@gmail.com> Date: Fri, 5 May 2023 09:43:48 +0200 Subject: [PATCH 2/7] prep tutorial --- config/cfg.py | 8 ++++---- templates/input.nml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/cfg.py b/config/cfg.py index c20d351..59b40c4 100755 --- a/config/cfg.py +++ b/config/cfg.py @@ -3,12 +3,12 @@ from dartwrf import utils exp = utils.Experiment() exp.expname = "obs-T_inflation-1.1" exp.model_dx = 2000 -exp.n_ens = 10 +exp.n_ens = 20 exp.filter_kind = 1 -exp.prior_inflation = 0 +exp.prior_inflation = 3 exp.inf_initial = 1.1 -exp.post_inflation = 4 +exp.post_inflation = 0 exp.sec = True exp.cov_loc_vert_km_horiz_km = (4, 40) exp.superob_km = False # False or int (spatial averaging of observations) @@ -63,7 +63,7 @@ t = dict(plotname='Temperature', plotunits='[K]', #n_obs=1, obs_locations=[(45., 0.)], error_generate=0.2, error_assimilate=0.2, heights=[1000,], #range(1000, 17001, 2000), - cov_loc_radius_km=30) + cov_loc_radius_km=40) q = dict(plotname='Specific humidity', plotunits='[kg/kg]', kind='RADIOSONDE_SPECIFIC_HUMIDITY', n_obs=1, diff --git a/templates/input.nml b/templates/input.nml index 3b673a7..2550215 100644 --- a/templates/input.nml +++ b/templates/input.nml @@ -58,8 +58,8 @@ write_all_stages_at_end = .false. inf_flavor = <prior_inflation>, <post_inflation>, - inf_initial_from_restart = .true., .false., - inf_sd_initial_from_restart = .true., .false., + inf_initial_from_restart = .false., .false., + inf_sd_initial_from_restart = .false., .false., inf_initial = <inf_initial>, 0.90, inf_lower_bound = 1.0, 1.0, inf_upper_bound = 1000000.0, 1000000.0, -- GitLab From 37fb1b51effa41014c69483c7283970619dada13 Mon Sep 17 00:00:00 2001 From: lkugler <lukas.kugler@gmail.com> Date: Fri, 5 May 2023 09:57:20 +0200 Subject: [PATCH 3/7] prep tutorial --- templates/input.nml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/input.nml b/templates/input.nml index 2550215..cfca979 100644 --- a/templates/input.nml +++ b/templates/input.nml @@ -51,7 +51,7 @@ output_timestamps = .false., trace_execution = .false., - stages_to_write = 'preassim', 'postassim' + stages_to_write = 'preassim', 'postassim', 'output' output_members = .true. output_mean = .true. output_sd = .true. -- GitLab From 25e0d8222f67b1ec78ca50eb3aa9c582b29c8f71 Mon Sep 17 00:00:00 2001 From: lkugler <lukas.kugler@gmail.com> Date: Fri, 19 May 2023 17:40:23 +0200 Subject: [PATCH 4/7] evaluate optional --- dartwrf/assim_synth_obs.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/dartwrf/assim_synth_obs.py b/dartwrf/assim_synth_obs.py index b67196d..738ac1b 100755 --- a/dartwrf/assim_synth_obs.py +++ b/dartwrf/assim_synth_obs.py @@ -534,22 +534,25 @@ def main(time, prior_init_time, prior_valid_time, prior_path_exp): os.system("rm -f input.nml obs_seq.in obs_seq.out obs_seq.out-orig obs_seq.final") set_DART_nml() + do_QC_here = getattr(exp, "reject_smallFGD", False) # True: triggers additional evaluations of prior & posterior + print("prepare nature") prepare_nature_dart(time) # link WRF files to DART directory print("prepare prior ensemble") prepare_prior_ensemble(time, prior_init_time, prior_valid_time, prior_path_exp) - print(" 1) get observations with specified obs-error") + print(" get observations with specified obs-error") oso = get_obsseq_out(time) - print(" 2.1) evaluate prior for all observations (incl rejected)") - osf_prior = evaluate(time, output_format="%Y-%m-%d_%H:%M_obs_seq.final-eval_prior_allobs") + if do_QC_here: + print(" (optional) evaluate prior for all observations (incl rejected)") + osf_prior = evaluate(time, output_format="%Y-%m-%d_%H:%M_obs_seq.final-eval_prior_allobs") - print(" 2.2) assign observation-errors for assimilation ") + print(" assign observation-errors for assimilation ") set_obserr_assimilate_in_obsseqout(oso, osf_prior, outfile=cluster.dartrundir + "/obs_seq.out") - if getattr(exp, "reject_smallFGD", False): + if do_QC_here: print(" 2.3) reject observations? ") qc_obs(time, oso, osf_prior) @@ -564,12 +567,13 @@ def main(time, prior_init_time, prior_valid_time, prior_path_exp): if exp.prior_inflation == 2: archive_inflation_2(time) - print(" 4) evaluate posterior observations for all observations (incl rejected)") - write_list_of_inputfiles_posterior(time) - if getattr(exp, "reject_smallFGD", False): + if do_QC_here: + print(" 4) evaluate posterior observations for all observations (incl rejected)") + write_list_of_inputfiles_posterior(time) + copy(cluster.archivedir+'/obs_seq_out/'+time.strftime('%Y-%m-%d_%H:%M_obs_seq.out-beforeQC'), cluster.dartrundir+'/obs_seq.out') - evaluate(time, output_format="%Y-%m-%d_%H:%M_obs_seq.final-eval_posterior_allobs") + evaluate(time, output_format="%Y-%m-%d_%H:%M_obs_seq.final-eval_posterior_allobs") if __name__ == "__main__": -- GitLab From 4beb45f0c75c1e2e532987c109b6dc8db881e441 Mon Sep 17 00:00:00 2001 From: lkugler <lukas.kugler@gmail.com> Date: Fri, 19 May 2023 18:07:32 +0200 Subject: [PATCH 5/7] evaluate only if necessary --- dartwrf/assim_synth_obs.py | 51 ++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/dartwrf/assim_synth_obs.py b/dartwrf/assim_synth_obs.py index 738ac1b..d1cef07 100755 --- a/dartwrf/assim_synth_obs.py +++ b/dartwrf/assim_synth_obs.py @@ -259,26 +259,32 @@ def get_parametrized_error(obscfg, osf_prior): NotImplementedError('sat_channel not implemented', obscfg.get("sat_channel")) -def set_obserr_assimilate_in_obsseqout(oso, osf_prior, outfile="./obs_seq.out"): +def set_obserr_assimilate_in_obsseqout(oso, outfile="./obs_seq.out"): """"Overwrite existing variance values in obs_seq.out files Args: oso (ObsSeq): python representation of obs_seq.out file, will be modified and written to file - osf_prior (ObsSeq): python representation of obs_seq.final (output of filter in evaluate-mode without posterior) - contains prior values; used for parameterized errors Returns: None (writes to file) + + Variables: + osf_prior (ObsSeq): python representation of obs_seq.final (output of filter in evaluate-mode without posterior) + contains prior values; used for parameterized errors """ + for obscfg in exp.observations: - kind_str = obscfg['kind'] - kind = osq.obs_kind_nrs[kind_str] + kind_str = obscfg['kind'] # e.g. 'RADIOSONDE_TEMPERATURE' + kind = osq.obs_kind_nrs[kind_str] # e.g. 263 - # modify each kind separately, one after each other + # modify observation error of each kind sequentially where_oso_iskind = oso.df.kind == kind - where_osf_iskind = osf_prior.df.kind == kind - + if obscfg["error_assimilate"] == False: + osf_prior = obsseq.ObsSeq(cluster.dartrundir + "/obs_seq.final") # this file will be generated by `evaluate()` + + where_osf_iskind = osf_prior.df.kind == kind + assim_err = get_parametrized_error(obscfg, osf_prior.df[where_osf_iskind]) oso.df.loc[where_oso_iskind, 'variance'] = assim_err**2 #assert np.allclose(assim_err, oso.df['variance']**2) # check @@ -288,7 +294,9 @@ def set_obserr_assimilate_in_obsseqout(oso, osf_prior, outfile="./obs_seq.out"): oso.to_dart(outfile) -def qc_obs(time, oso, osf_prior): +def qc_obs(time, oso): + osf_prior = obsseq.ObsSeq(cluster.dartrundir + "/obs_seq.final") + # obs should be superobbed already! for i, obscfg in enumerate(exp.observations): if i > 0: @@ -373,9 +381,6 @@ def evaluate(assim_time, copy(cluster.dartrundir + "/obs_seq.final", fout) print(fout, "saved.") - osf = obsseq.ObsSeq(cluster.dartrundir + "/obs_seq.final") - return osf - def generate_obsseq_out(time): @@ -522,9 +527,14 @@ def main(time, prior_init_time, prior_valid_time, prior_path_exp): None """ nproc = cluster.max_nproc + do_QC = getattr(exp, "reject_smallFGD", False) # True: triggers additional evaluations of prior & posterior + # for which observation type do we have a parametrized observation error? + error_is_parametrized = [obscfg["error_assimilate"] == False for obscfg in exp.observations] + + # create directory to run DART in archive_time = cluster.archivedir + time.strftime("/%Y-%m-%d_%H:%M/") - os.makedirs(cluster.dartrundir, exist_ok=True) # create directory to run DART in + os.makedirs(cluster.dartrundir, exist_ok=True) os.chdir(cluster.dartrundir) # link DART binaries to run_DART @@ -534,8 +544,6 @@ def main(time, prior_init_time, prior_valid_time, prior_path_exp): os.system("rm -f input.nml obs_seq.in obs_seq.out obs_seq.out-orig obs_seq.final") set_DART_nml() - do_QC_here = getattr(exp, "reject_smallFGD", False) # True: triggers additional evaluations of prior & posterior - print("prepare nature") prepare_nature_dart(time) # link WRF files to DART directory @@ -545,16 +553,17 @@ def main(time, prior_init_time, prior_valid_time, prior_path_exp): print(" get observations with specified obs-error") oso = get_obsseq_out(time) - if do_QC_here: + # is any observation error parametrized? + if any(error_is_parametrized) or do_QC: print(" (optional) evaluate prior for all observations (incl rejected)") - osf_prior = evaluate(time, output_format="%Y-%m-%d_%H:%M_obs_seq.final-eval_prior_allobs") + evaluate(time, output_format="%Y-%m-%d_%H:%M_obs_seq.final-eval_prior_allobs") print(" assign observation-errors for assimilation ") - set_obserr_assimilate_in_obsseqout(oso, osf_prior, outfile=cluster.dartrundir + "/obs_seq.out") + set_obserr_assimilate_in_obsseqout(oso, outfile=cluster.dartrundir + "/obs_seq.out") - if do_QC_here: + if do_QC: print(" 2.3) reject observations? ") - qc_obs(time, oso, osf_prior) + qc_obs(time, oso) if exp.prior_inflation == 2: prepare_inflation_2(time, prior_init_time) @@ -567,7 +576,7 @@ def main(time, prior_init_time, prior_valid_time, prior_path_exp): if exp.prior_inflation == 2: archive_inflation_2(time) - if do_QC_here: + if do_QC: print(" 4) evaluate posterior observations for all observations (incl rejected)") write_list_of_inputfiles_posterior(time) -- GitLab From 9cbaaa6d701443e9ca31a73e2863a5357e621830 Mon Sep 17 00:00:00 2001 From: lkugler <lukas.kugler@gmail.com> Date: Mon, 22 May 2023 10:35:18 +0200 Subject: [PATCH 6/7] . --- config/cfg.py | 22 +++++++++++----------- dartwrf/assim_synth_obs.py | 2 +- templates/input.nml | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/config/cfg.py b/config/cfg.py index 59b40c4..c4b4a29 100755 --- a/config/cfg.py +++ b/config/cfg.py @@ -1,16 +1,16 @@ from dartwrf import utils exp = utils.Experiment() -exp.expname = "obs-T_inflation-1.1" +exp.expname = "advDA_Raso-T_inf2-4" exp.model_dx = 2000 -exp.n_ens = 20 +exp.n_ens = 40 exp.filter_kind = 1 -exp.prior_inflation = 3 -exp.inf_initial = 1.1 +exp.prior_inflation = 2 +exp.inf_initial = 4 exp.post_inflation = 0 exp.sec = True -exp.cov_loc_vert_km_horiz_km = (4, 40) +exp.cov_loc_vert_km_horiz_km = (2, 100) exp.superob_km = False # False or int (spatial averaging of observations) exp.adjust_obs_impact = False @@ -41,7 +41,7 @@ wv62 = dict(plotname='Brightness temperature WV 6.2µm', plotunits='[K]', wv73 = dict(plotname='Brightness temperature WV 7.3µm', plotunits='[K]', kind='MSG_4_SEVIRI_TB', sat_channel=6, n_obs=961, obs_locations='square_array_evenly_on_grid', - error_generate=1., error_assimilate=3., + error_generate=1., error_assimilate=1., cov_loc_radius_km=20) ir108 = dict(plotname='Brightness temperature IR 10.8µm', plotunits='[K]', @@ -59,16 +59,16 @@ radar = dict(plotname='Radar reflectivity', plotunits='[dBz]', t = dict(plotname='Temperature', plotunits='[K]', kind='RADIOSONDE_TEMPERATURE', - n_obs=961, obs_locations='square_array_evenly_on_grid', - #n_obs=1, obs_locations=[(45., 0.)], + #n_obs=961, obs_locations='square_array_evenly_on_grid', + n_obs=1, obs_locations=[(45., 0.)], error_generate=0.2, error_assimilate=0.2, - heights=[1000,], #range(1000, 17001, 2000), - cov_loc_radius_km=40) + heights=range(700, 17701, 500), + cov_loc_radius_km=100) q = dict(plotname='Specific humidity', plotunits='[kg/kg]', kind='RADIOSONDE_SPECIFIC_HUMIDITY', n_obs=1, error_generate=0., error_assimilate=5*1e-5, - heights=[1000], #range(1000, 17001, 2000), + heights=[1500], #range(1000, 17001, 2000), cov_loc_radius_km=0.1) t2m = dict(plotname='SYNOP Temperature', plotunits='[K]', diff --git a/dartwrf/assim_synth_obs.py b/dartwrf/assim_synth_obs.py index d1cef07..3396f82 100755 --- a/dartwrf/assim_synth_obs.py +++ b/dartwrf/assim_synth_obs.py @@ -476,7 +476,7 @@ def prepare_inflation_2(time, prior_init_time): if os.path.isfile(f_prior): copy(f_prior, f_new) print(f_prior, 'copied to', f_new) - else: + else: # no prior inflation file at the first assimilation warnings.warn(f_prior + ' does not exist. Using default file instead.') copy(f_default, f_new) diff --git a/templates/input.nml b/templates/input.nml index cfca979..2c987da 100644 --- a/templates/input.nml +++ b/templates/input.nml @@ -63,7 +63,7 @@ inf_initial = <inf_initial>, 0.90, inf_lower_bound = 1.0, 1.0, inf_upper_bound = 1000000.0, 1000000.0, - inf_damping = 0.9, 1.0, + inf_damping = 1.0, 1.0, inf_sd_initial = 0.6, 0.0, inf_sd_lower_bound = 0.6, 0.0, inf_sd_max_change = 1.05, 1.05, -- GitLab From 2999475ad89a66a5c56f531c2e6029198295304f Mon Sep 17 00:00:00 2001 From: lkugler <lukas.kugler@gmail.com> Date: Mon, 22 May 2023 16:42:39 +0200 Subject: [PATCH 7/7] . --- config/cfg.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/config/cfg.py b/config/cfg.py index c4b4a29..a74f175 100755 --- a/config/cfg.py +++ b/config/cfg.py @@ -1,16 +1,16 @@ from dartwrf import utils exp = utils.Experiment() -exp.expname = "advDA_Raso-T_inf2-4" +exp.expname = "test_WV73_many_inf3" exp.model_dx = 2000 -exp.n_ens = 40 +exp.n_ens = 20 exp.filter_kind = 1 -exp.prior_inflation = 2 +exp.prior_inflation = 3 exp.inf_initial = 4 exp.post_inflation = 0 exp.sec = True -exp.cov_loc_vert_km_horiz_km = (2, 100) +exp.cov_loc_vert_km_horiz_km = False #(2, 10) exp.superob_km = False # False or int (spatial averaging of observations) exp.adjust_obs_impact = False @@ -40,9 +40,9 @@ wv62 = dict(plotname='Brightness temperature WV 6.2µm', plotunits='[K]', wv73 = dict(plotname='Brightness temperature WV 7.3µm', plotunits='[K]', kind='MSG_4_SEVIRI_TB', sat_channel=6, - n_obs=961, obs_locations='square_array_evenly_on_grid', - error_generate=1., error_assimilate=1., - cov_loc_radius_km=20) + n_obs=256, obs_locations='square_array_evenly_on_grid', + error_generate=2., error_assimilate=2., + cov_loc_radius_km=10) ir108 = dict(plotname='Brightness temperature IR 10.8µm', plotunits='[K]', kind='MSG_4_SEVIRI_TB', sat_channel=9, @@ -72,15 +72,17 @@ q = dict(plotname='Specific humidity', plotunits='[kg/kg]', cov_loc_radius_km=0.1) t2m = dict(plotname='SYNOP Temperature', plotunits='[K]', - kind='SYNOP_TEMPERATURE', n_obs=1, - error_generate=0.1, error_assimilate=0.1, - cov_loc_radius_km=40) + kind='SYNOP_TEMPERATURE', + #n_obs=1, obs_locations=[(45., 0.)], + n_obs=256, obs_locations='square_array_evenly_on_grid', + error_generate=1, error_assimilate=1, + cov_loc_radius_km=10) psfc = dict(plotname='SYNOP Pressure', plotunits='[Pa]', kind='SYNOP_SURFACE_PRESSURE', n_obs=1, error_generate=50., error_assimilate=100., cov_loc_radius_km=32) -exp.observations = [t] +exp.observations = [wv73] exp.update_vars = ['U', 'V', 'W', 'THM', 'PH', 'MU', 'QVAPOR', 'QCLOUD', 'QICE', 'PSFC'] -- GitLab