From a54a65f77c389209c8284374ea51e5a5374ea7c8 Mon Sep 17 00:00:00 2001
From: lkugler <lukas.kugler@gmail.com>
Date: Wed, 24 May 2023 15:13:56 +0200
Subject: [PATCH] collect all obs modules in one folder
---
dartwrf/assim_synth_obs.py | 111 ++++--------------
dartwrf/create_obs_upfront.py | 4 +-
.../create_obsseq_in.py} | 4 +-
dartwrf/obs/create_obsseq_out.py | 70 +++++++++++
dartwrf/{ => obs}/obskind.py | 0
dartwrf/{ => obs}/obsseq.py | 2 +-
dartwrf/{ => obs}/obsseq_2dim.py | 2 +-
dartwrf/{ => obs}/obsseq_to_netcdf.py | 2 +-
dartwrf/{ => obs}/run_obs_diag.py | 0
dartwrf/workflows.py | 2 +-
tests/test_assim.py | 3 +-
tests/test_dart-rttov.py | 4 +-
tests/test_obsseq.py | 2 +-
13 files changed, 104 insertions(+), 102 deletions(-)
rename dartwrf/{create_obsseq.py => obs/create_obsseq_in.py} (99%)
create mode 100644 dartwrf/obs/create_obsseq_out.py
rename dartwrf/{ => obs}/obskind.py (100%)
rename dartwrf/{ => obs}/obsseq.py (99%)
rename dartwrf/{ => obs}/obsseq_2dim.py (98%)
rename dartwrf/{ => obs}/obsseq_to_netcdf.py (95%)
rename dartwrf/{ => obs}/run_obs_diag.py (100%)
diff --git a/dartwrf/assim_synth_obs.py b/dartwrf/assim_synth_obs.py
index 48b74ac..f3cdace 100755
--- a/dartwrf/assim_synth_obs.py
+++ b/dartwrf/assim_synth_obs.py
@@ -3,17 +3,15 @@ import time as time_module
import datetime as dt
import numpy as np
-from dartwrf.utils import symlink, copy, sed_inplace, append_file, mkdir, try_remove, print, shell, write_txt
-from dartwrf.obs import error_models as err
-import dartwrf.create_obsseq as osq
+from dartwrf.utils import symlink, copy, mkdir, try_remove, print, shell, write_txt
from dartwrf import wrfout_add_geo
-from dartwrf import obsseq
+from dartwrf.obs import error_models as err
+from dartwrf.obs import obsseq
+from dartwrf.obs import create_obsseq_out as osq_out
from dartwrf import dart_nml
-from dartwrf import exp_config
-exp = exp_config.exp
-from dartwrf import server_config
-cluster = server_config.cluster
+from dartwrf.exp_config import exp
+from dartwrf.server_config import cluster
wrfout_format = 'wrfout_d01_%Y-%m-%d_%H:%M:%S'
@@ -115,19 +113,6 @@ def write_list_of_outputfiles():
files.append("./filter_restart_d01." + str(iens).zfill(4))
write_txt(files, cluster.dart_rundir+'/output_list.txt')
-def run_perfect_model_obs(nproc=12, verbose=True):
- if verbose:
- print("generating observations - running ./perfect_model_obs")
- os.chdir(cluster.dart_rundir)
-
- try_remove(cluster.dart_rundir + "/obs_seq.out")
- if not os.path.exists(cluster.dart_rundir + "/obs_seq.in"):
- raise RuntimeError("obs_seq.in does not exist in " + cluster.dart_rundir)
- shell(cluster.dart_modules+' mpirun -np '+str(nproc)+" ./perfect_model_obs > log.perfect_model_obs")
- if not os.path.exists(cluster.dart_rundir + "/obs_seq.out"):
- raise RuntimeError(
- "obs_seq.out does not exist in " + cluster.dart_rundir,
- "\n look for " + cluster.dart_rundir + "/log.perfect_model_obs")
def filter(nproc=12):
print("time now", dt.datetime.now())
@@ -219,10 +204,11 @@ def set_obserr_assimilate_in_obsseqout(oso, outfile="./obs_seq.out"):
osf_prior (ObsSeq): python representation of obs_seq.final (output of filter in evaluate-mode without posterior)
contains prior values; used for parameterized errors
"""
+ from dartwrf.obs.obskind import obs_kind_nrs
for obscfg in exp.observations:
kind_str = obscfg['kind'] # e.g. 'RADIOSONDE_TEMPERATURE'
- kind = osq.obs_kind_nrs[kind_str] # e.g. 263
+ kind = obs_kind_nrs[kind_str] # e.g. 263
# modify observation error of each kind sequentially
where_oso_iskind = oso.df.kind == kind
@@ -343,80 +329,25 @@ def evaluate(assim_time, obs_seq_out=False,
copy(cluster.dart_rundir + "/obs_seq.final", fout)
print(fout, "saved.")
-
-
-def generate_obsseq_out(time):
-
- def ensure_physical_vis(oso): # set reflectance < surface albedo to surface albedo
- print(" 2.2) removing obs below surface albedo ")
- clearsky_albedo = 0.2928
-
- if_vis_obs = oso.df['kind'].values == 262
- if_obs_below_surface_albedo = oso.df['observations'].values < clearsky_albedo
- oso.df.loc[if_vis_obs & if_obs_below_surface_albedo, ('observations')] = clearsky_albedo
- oso.to_dart(f=cluster.dart_rundir + "/obs_seq.out")
- return oso
-
-
- def apply_superobbing(oso):
- try:
- f_oso = dir_obsseq + time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out-before_superob")
- shutil.copy(cluster.dart_rundir + "/obs_seq.out-before_superob", f_oso)
- print('saved', f_oso)
- except Exception as e:
- warnings.warn(str(e))
-
- print(" 2.3) superobbing to", exp.superob_km, "km")
- oso.df = oso.df.superob(window_km=exp.superob_km)
- oso.to_dart(f=cluster.dart_rundir + "/obs_seq.out")
-
-
- ##############################
-
- dir_obsseq=cluster.archivedir + "/obs_seq_out/"
- os.makedirs(dir_obsseq, exist_ok=True)
-
- osq.create_obs_seq_in(time, exp.observations)
- run_perfect_model_obs() # generate observation, draws from gaussian
-
- print(" 2.1) obs preprocessing")
- oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out")
-
- oso = ensure_physical_vis(oso)
-
- if getattr(exp, "superob_km", False):
- oso = apply_superobbing(oso)
-
- # archive complete obsseqout
- f_oso = dir_obsseq + time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out")
- shutil.copy(cluster.dart_rundir + "/obs_seq.out", f_oso)
- print('saved', f_oso)
- return oso
-
-
def get_obsseq_out(time):
+ """Prepares an obs_seq.out file in the run_DART folder
+
+ Args:
+ time (datetime): time of assimilation
- # did we specify an obsseqout inputfile?
+ Returns:
+ obsseq.ObsSeq
+ """
if exp.use_existing_obsseq != False:
+ # use an existing obs_seq.out file
f_obsseq = time.strftime(exp.use_existing_obsseq)
copy(f_obsseq, cluster.dart_rundir+'/obs_seq.out')
print(f_obsseq, 'copied to', cluster.dart_rundir+'/obs_seq.out')
- oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out")
- else:
- # decision to NOT use existing obs_seq.out file
-
- # did we already generate observations?
- # f_oso_thisexp = cluster.archivedir+'/obs_seq_out/'+time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out")
- # if os.path.isfile(f_oso_thisexp):
- # # oso exists
- # copy(f_oso_thisexp, cluster.dart_rundir+'/obs_seq.out')
- # print('copied existing obsseqout from', f_oso_thisexp)
- # oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out")
- # else:
-
- # generate observations with new observation noise
- oso = generate_obsseq_out(time)
-
+ oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out") # read the obs_seq.out file
+ else:
+ # do NOT use an existing obs_seq.out file
+ # but generate observations with new observation noise
+ oso = osq_out.generate_obsseq_out(time)
return oso
def prepare_inflation_2(time, prior_init_time):
diff --git a/dartwrf/create_obs_upfront.py b/dartwrf/create_obs_upfront.py
index 4cf876b..c2ffc99 100755
--- a/dartwrf/create_obs_upfront.py
+++ b/dartwrf/create_obs_upfront.py
@@ -7,8 +7,8 @@ import numpy as np
from dartwrf.exp_config import exp
from dartwrf.server_config import cluster
from dartwrf.utils import copy, print
-import dartwrf.create_obsseq as osq
-from dartwrf import obsseq
+import dartwrf.obs.create_obsseq_in as osq
+from dartwrf.obs import obsseq
from dartwrf import assim_synth_obs as aso
tformat = '%Y-%m-%d_%H:%M'
diff --git a/dartwrf/create_obsseq.py b/dartwrf/obs/create_obsseq_in.py
similarity index 99%
rename from dartwrf/create_obsseq.py
rename to dartwrf/obs/create_obsseq_in.py
index b7792f1..ebcbed5 100755
--- a/dartwrf/create_obsseq.py
+++ b/dartwrf/obs/create_obsseq_in.py
@@ -8,9 +8,9 @@ import datetime as dt
from pysolar.solar import get_altitude, get_azimuth
from dartwrf.server_config import cluster
-from dartwrf.obs import calculate_obs_locations as col
from dartwrf import utils
-from dartwrf.obskind import obs_kind_nrs # dictionary string => DART internal indices
+from dartwrf.obs import calculate_obs_locations as col
+from dartwrf.obs.obskind import obs_kind_nrs # dictionary string => DART internal indices
# position on earth for RTTOV ray geometry
lat0 = 45.
diff --git a/dartwrf/obs/create_obsseq_out.py b/dartwrf/obs/create_obsseq_out.py
new file mode 100644
index 0000000..cb56e85
--- /dev/null
+++ b/dartwrf/obs/create_obsseq_out.py
@@ -0,0 +1,70 @@
+import os, shutil, warnings
+
+from dartwrf.utils import try_remove, print, shell
+import dartwrf.obs.create_obsseq_in as osi
+from dartwrf.obs import obsseq
+
+from dartwrf.exp_config import exp
+from dartwrf.server_config import cluster
+
+def generate_obsseq_out(time):
+
+ def ensure_physical_vis(oso): # set reflectance < surface albedo to surface albedo
+ print(" 2.2) removing obs below surface albedo ")
+ clearsky_albedo = 0.2928
+
+ if_vis_obs = oso.df['kind'].values == 262
+ if_obs_below_surface_albedo = oso.df['observations'].values < clearsky_albedo
+ oso.df.loc[if_vis_obs & if_obs_below_surface_albedo, ('observations')] = clearsky_albedo
+ oso.to_dart(f=cluster.dart_rundir + "/obs_seq.out")
+ return oso
+
+
+ def apply_superobbing(oso):
+ try:
+ f_oso = dir_obsseq + time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out-before_superob")
+ shutil.copy(cluster.dart_rundir + "/obs_seq.out-before_superob", f_oso)
+ print('saved', f_oso)
+ except Exception as e:
+ warnings.warn(str(e))
+
+ print(" 2.3) superobbing to", exp.superob_km, "km")
+ oso.df = oso.df.superob(window_km=exp.superob_km)
+ oso.to_dart(f=cluster.dart_rundir + "/obs_seq.out")
+
+
+ ##############################
+
+ dir_obsseq=cluster.archivedir + "/obs_seq_out/"
+ os.makedirs(dir_obsseq, exist_ok=True)
+
+ osi.create_obs_seq_in(time, exp.observations)
+ run_perfect_model_obs() # generate observation, draws from gaussian
+
+ print(" 2.1) obs preprocessing")
+ oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out")
+
+ oso = ensure_physical_vis(oso)
+
+ if getattr(exp, "superob_km", False):
+ oso = apply_superobbing(oso)
+
+ # archive complete obsseqout
+ f_oso = dir_obsseq + time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out")
+ shutil.copy(cluster.dart_rundir + "/obs_seq.out", f_oso)
+ print('saved', f_oso)
+ return oso
+
+def run_perfect_model_obs(nproc=12, verbose=True):
+ if verbose:
+ print("generating observations - running ./perfect_model_obs")
+ os.chdir(cluster.dart_rundir)
+
+ try_remove(cluster.dart_rundir + "/obs_seq.out")
+ if not os.path.exists(cluster.dart_rundir + "/obs_seq.in"):
+ raise RuntimeError("obs_seq.in does not exist in " + cluster.dart_rundir)
+ shell(cluster.dart_modules+' mpirun -np '+str(nproc)+" ./perfect_model_obs > log.perfect_model_obs")
+ if not os.path.exists(cluster.dart_rundir + "/obs_seq.out"):
+ raise RuntimeError(
+ "obs_seq.out does not exist in " + cluster.dart_rundir,
+ "\n look for " + cluster.dart_rundir + "/log.perfect_model_obs")
\ No newline at end of file
diff --git a/dartwrf/obskind.py b/dartwrf/obs/obskind.py
similarity index 100%
rename from dartwrf/obskind.py
rename to dartwrf/obs/obskind.py
diff --git a/dartwrf/obsseq.py b/dartwrf/obs/obsseq.py
similarity index 99%
rename from dartwrf/obsseq.py
rename to dartwrf/obs/obsseq.py
index fc20e51..e108e70 100755
--- a/dartwrf/obsseq.py
+++ b/dartwrf/obs/obsseq.py
@@ -703,7 +703,7 @@ class ObsSeq(object):
if __name__ == "__main__":
- from config.cluster import cluster
+ from dartwrf.server_config import cluster
# for testing purposes
# f = cluster.scriptsdir + "/../tests/obs_seq.orig.out"
diff --git a/dartwrf/obsseq_2dim.py b/dartwrf/obs/obsseq_2dim.py
similarity index 98%
rename from dartwrf/obsseq_2dim.py
rename to dartwrf/obs/obsseq_2dim.py
index c9a647c..7239906 100755
--- a/dartwrf/obsseq_2dim.py
+++ b/dartwrf/obs/obsseq_2dim.py
@@ -22,7 +22,7 @@ import numpy as np
from dartwrf.server_config import cluster
from dartwrf import utils
from dartwrf import assim_synth_obs as aso
-from dartwrf import obsseq
+from dartwrf.obs import obsseq
def _get_n_obs_per_layer(oso):
"""Get number of observations per layer"""
diff --git a/dartwrf/obsseq_to_netcdf.py b/dartwrf/obs/obsseq_to_netcdf.py
similarity index 95%
rename from dartwrf/obsseq_to_netcdf.py
rename to dartwrf/obs/obsseq_to_netcdf.py
index 14b0447..db40ff1 100644
--- a/dartwrf/obsseq_to_netcdf.py
+++ b/dartwrf/obs/obsseq_to_netcdf.py
@@ -2,7 +2,7 @@ import os, sys, glob, warnings
from dartwrf.exp_config import exp
from dartwrf.server_config import cluster
-import dartwrf.run_obs_diag as rod
+import dartwrf.obs.run_obs_diag as rod
def listdir_dirs(path):
return [a for a in os.listdir(path) if os.path.isdir(os.path.join(path, a))]
diff --git a/dartwrf/run_obs_diag.py b/dartwrf/obs/run_obs_diag.py
similarity index 100%
rename from dartwrf/run_obs_diag.py
rename to dartwrf/obs/run_obs_diag.py
diff --git a/dartwrf/workflows.py b/dartwrf/workflows.py
index bbc67b4..62e1b6a 100644
--- a/dartwrf/workflows.py
+++ b/dartwrf/workflows.py
@@ -112,7 +112,7 @@ class WorkFlows(object):
# copy obs kind def to config, we will read a table from there
# file needs to exist within package so sphinx can read it
- _dict_to_py(_obskind_read(), this_dir+'/obskind.py')
+ _dict_to_py(_obskind_read(), this_dir+'/obs/obskind.py')
_copy_dartwrf_to_archive() # includes config files
diff --git a/tests/test_assim.py b/tests/test_assim.py
index d53b53b..7f61e27 100644
--- a/tests/test_assim.py
+++ b/tests/test_assim.py
@@ -1,7 +1,8 @@
import os, shutil
import datetime as dt
-from dartwrf import obsseq, assim_synth_obs
+from dartwrf import assim_synth_obs
+from dartwrf.obs import obsseq
from dartwrf.server_config import cluster
class ExperimentConfiguration(object):
diff --git a/tests/test_dart-rttov.py b/tests/test_dart-rttov.py
index ada56bf..f5f3153 100644
--- a/tests/test_dart-rttov.py
+++ b/tests/test_dart-rttov.py
@@ -3,8 +3,8 @@ import numpy as np
import datetime as dt
import pandas as pd
-from dartwrf import obsseq
-import dartwrf.create_obsseq as osq
+from dartwrf.obs import obsseq
+import dartwrf.obs.create_obsseq_in as osq
import dartwrf.assim_synth_obs as aso
from dartwrf import wrfout_add_geo
diff --git a/tests/test_obsseq.py b/tests/test_obsseq.py
index 215df54..7769465 100644
--- a/tests/test_obsseq.py
+++ b/tests/test_obsseq.py
@@ -2,7 +2,7 @@ import os, filecmp, shutil
import numpy as np
from dartwrf.server_config import cluster
-from dartwrf import obsseq
+from dartwrf.obs import obsseq
def test_oso():
--
GitLab