diff --git a/dartwrf/assim_synth_obs.py b/dartwrf/assim_synth_obs.py
index 48b74ace071c306ded1c272f96ca4477f0ed72aa..f3cdace497ef03675a3a41dd83e09a20ee85e86d 100755
--- a/dartwrf/assim_synth_obs.py
+++ b/dartwrf/assim_synth_obs.py
@@ -3,17 +3,15 @@ import time as time_module
import datetime as dt
import numpy as np
-from dartwrf.utils import symlink, copy, sed_inplace, append_file, mkdir, try_remove, print, shell, write_txt
-from dartwrf.obs import error_models as err
-import dartwrf.create_obsseq as osq
+from dartwrf.utils import symlink, copy, mkdir, try_remove, print, shell, write_txt
from dartwrf import wrfout_add_geo
-from dartwrf import obsseq
+from dartwrf.obs import error_models as err
+from dartwrf.obs import obsseq
+from dartwrf.obs import create_obsseq_out as osq_out
from dartwrf import dart_nml
-from dartwrf import exp_config
-exp = exp_config.exp
-from dartwrf import server_config
-cluster = server_config.cluster
+from dartwrf.exp_config import exp
+from dartwrf.server_config import cluster
wrfout_format = 'wrfout_d01_%Y-%m-%d_%H:%M:%S'
@@ -115,19 +113,6 @@ def write_list_of_outputfiles():
files.append("./filter_restart_d01." + str(iens).zfill(4))
write_txt(files, cluster.dart_rundir+'/output_list.txt')
-def run_perfect_model_obs(nproc=12, verbose=True):
- if verbose:
- print("generating observations - running ./perfect_model_obs")
- os.chdir(cluster.dart_rundir)
-
- try_remove(cluster.dart_rundir + "/obs_seq.out")
- if not os.path.exists(cluster.dart_rundir + "/obs_seq.in"):
- raise RuntimeError("obs_seq.in does not exist in " + cluster.dart_rundir)
- shell(cluster.dart_modules+' mpirun -np '+str(nproc)+" ./perfect_model_obs > log.perfect_model_obs")
- if not os.path.exists(cluster.dart_rundir + "/obs_seq.out"):
- raise RuntimeError(
- "obs_seq.out does not exist in " + cluster.dart_rundir,
- "\n look for " + cluster.dart_rundir + "/log.perfect_model_obs")
def filter(nproc=12):
print("time now", dt.datetime.now())
@@ -219,10 +204,11 @@ def set_obserr_assimilate_in_obsseqout(oso, outfile="./obs_seq.out"):
osf_prior (ObsSeq): python representation of obs_seq.final (output of filter in evaluate-mode without posterior)
contains prior values; used for parameterized errors
"""
+ from dartwrf.obs.obskind import obs_kind_nrs
for obscfg in exp.observations:
kind_str = obscfg['kind'] # e.g. 'RADIOSONDE_TEMPERATURE'
- kind = osq.obs_kind_nrs[kind_str] # e.g. 263
+ kind = obs_kind_nrs[kind_str] # e.g. 263
# modify observation error of each kind sequentially
where_oso_iskind = oso.df.kind == kind
@@ -343,80 +329,25 @@ def evaluate(assim_time, obs_seq_out=False,
copy(cluster.dart_rundir + "/obs_seq.final", fout)
print(fout, "saved.")
-
-
-def generate_obsseq_out(time):
-
- def ensure_physical_vis(oso): # set reflectance < surface albedo to surface albedo
- print(" 2.2) removing obs below surface albedo ")
- clearsky_albedo = 0.2928
-
- if_vis_obs = oso.df['kind'].values == 262
- if_obs_below_surface_albedo = oso.df['observations'].values < clearsky_albedo
- oso.df.loc[if_vis_obs & if_obs_below_surface_albedo, ('observations')] = clearsky_albedo
- oso.to_dart(f=cluster.dart_rundir + "/obs_seq.out")
- return oso
-
-
- def apply_superobbing(oso):
- try:
- f_oso = dir_obsseq + time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out-before_superob")
- shutil.copy(cluster.dart_rundir + "/obs_seq.out-before_superob", f_oso)
- print('saved', f_oso)
- except Exception as e:
- warnings.warn(str(e))
-
- print(" 2.3) superobbing to", exp.superob_km, "km")
- oso.df = oso.df.superob(window_km=exp.superob_km)
- oso.to_dart(f=cluster.dart_rundir + "/obs_seq.out")
-
-
- ##############################
-
- dir_obsseq=cluster.archivedir + "/obs_seq_out/"
- os.makedirs(dir_obsseq, exist_ok=True)
-
- osq.create_obs_seq_in(time, exp.observations)
- run_perfect_model_obs() # generate observation, draws from gaussian
-
- print(" 2.1) obs preprocessing")
- oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out")
-
- oso = ensure_physical_vis(oso)
-
- if getattr(exp, "superob_km", False):
- oso = apply_superobbing(oso)
-
- # archive complete obsseqout
- f_oso = dir_obsseq + time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out")
- shutil.copy(cluster.dart_rundir + "/obs_seq.out", f_oso)
- print('saved', f_oso)
- return oso
-
-
def get_obsseq_out(time):
+ """Prepares an obs_seq.out file in the run_DART folder
+
+ Args:
+ time (datetime): time of assimilation
- # did we specify an obsseqout inputfile?
+ Returns:
+ obsseq.ObsSeq
+ """
if exp.use_existing_obsseq != False:
+ # use an existing obs_seq.out file
f_obsseq = time.strftime(exp.use_existing_obsseq)
copy(f_obsseq, cluster.dart_rundir+'/obs_seq.out')
print(f_obsseq, 'copied to', cluster.dart_rundir+'/obs_seq.out')
- oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out")
- else:
- # decision to NOT use existing obs_seq.out file
-
- # did we already generate observations?
- # f_oso_thisexp = cluster.archivedir+'/obs_seq_out/'+time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out")
- # if os.path.isfile(f_oso_thisexp):
- # # oso exists
- # copy(f_oso_thisexp, cluster.dart_rundir+'/obs_seq.out')
- # print('copied existing obsseqout from', f_oso_thisexp)
- # oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out")
- # else:
-
- # generate observations with new observation noise
- oso = generate_obsseq_out(time)
-
+ oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out") # read the obs_seq.out file
+ else:
+ # do NOT use an existing obs_seq.out file
+ # but generate observations with new observation noise
+ oso = osq_out.generate_obsseq_out(time)
return oso
def prepare_inflation_2(time, prior_init_time):
diff --git a/dartwrf/create_obs_upfront.py b/dartwrf/create_obs_upfront.py
index 4cf876bdd39e918501b5a83d826d5161f8c554bb..c2ffc99ca2c46ef8dcf6c11d892fd130b3ca77f1 100755
--- a/dartwrf/create_obs_upfront.py
+++ b/dartwrf/create_obs_upfront.py
@@ -7,8 +7,8 @@ import numpy as np
from dartwrf.exp_config import exp
from dartwrf.server_config import cluster
from dartwrf.utils import copy, print
-import dartwrf.create_obsseq as osq
-from dartwrf import obsseq
+import dartwrf.obs.create_obsseq_in as osq
+from dartwrf.obs import obsseq
from dartwrf import assim_synth_obs as aso
tformat = '%Y-%m-%d_%H:%M'
diff --git a/dartwrf/create_obsseq.py b/dartwrf/obs/create_obsseq_in.py
similarity index 99%
rename from dartwrf/create_obsseq.py
rename to dartwrf/obs/create_obsseq_in.py
index b7792f1b54ba76b6f71f7adb88537cd2912f75da..ebcbed58a36b22a5351e3603308739dfa9618f53 100755
--- a/dartwrf/create_obsseq.py
+++ b/dartwrf/obs/create_obsseq_in.py
@@ -8,9 +8,9 @@ import datetime as dt
from pysolar.solar import get_altitude, get_azimuth
from dartwrf.server_config import cluster
-from dartwrf.obs import calculate_obs_locations as col
from dartwrf import utils
-from dartwrf.obskind import obs_kind_nrs # dictionary string => DART internal indices
+from dartwrf.obs import calculate_obs_locations as col
+from dartwrf.obs.obskind import obs_kind_nrs # dictionary string => DART internal indices
# position on earth for RTTOV ray geometry
lat0 = 45.
diff --git a/dartwrf/obs/create_obsseq_out.py b/dartwrf/obs/create_obsseq_out.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb56e85dacbf2508dc98e962351878257774d7e8
--- /dev/null
+++ b/dartwrf/obs/create_obsseq_out.py
@@ -0,0 +1,70 @@
+import os, shutil, warnings
+
+from dartwrf.utils import try_remove, print, shell
+import dartwrf.obs.create_obsseq_in as osi
+from dartwrf.obs import obsseq
+
+from dartwrf.exp_config import exp
+from dartwrf.server_config import cluster
+
+def generate_obsseq_out(time):
+
+ def ensure_physical_vis(oso): # set reflectance < surface albedo to surface albedo
+ print(" 2.2) removing obs below surface albedo ")
+ clearsky_albedo = 0.2928
+
+ if_vis_obs = oso.df['kind'].values == 262
+ if_obs_below_surface_albedo = oso.df['observations'].values < clearsky_albedo
+ oso.df.loc[if_vis_obs & if_obs_below_surface_albedo, ('observations')] = clearsky_albedo
+ oso.to_dart(f=cluster.dart_rundir + "/obs_seq.out")
+ return oso
+
+
+ def apply_superobbing(oso):
+ try:
+ f_oso = dir_obsseq + time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out-before_superob")
+ shutil.copy(cluster.dart_rundir + "/obs_seq.out-before_superob", f_oso)
+ print('saved', f_oso)
+ except Exception as e:
+ warnings.warn(str(e))
+
+ print(" 2.3) superobbing to", exp.superob_km, "km")
+ oso.df = oso.df.superob(window_km=exp.superob_km)
+ oso.to_dart(f=cluster.dart_rundir + "/obs_seq.out")
+
+
+ ##############################
+
+ dir_obsseq=cluster.archivedir + "/obs_seq_out/"
+ os.makedirs(dir_obsseq, exist_ok=True)
+
+ osi.create_obs_seq_in(time, exp.observations)
+ run_perfect_model_obs() # generate observation, draws from gaussian
+
+ print(" 2.1) obs preprocessing")
+ oso = obsseq.ObsSeq(cluster.dart_rundir + "/obs_seq.out")
+
+ oso = ensure_physical_vis(oso)
+
+ if getattr(exp, "superob_km", False):
+ oso = apply_superobbing(oso)
+
+ # archive complete obsseqout
+ f_oso = dir_obsseq + time.strftime("/%Y-%m-%d_%H:%M_obs_seq.out")
+ shutil.copy(cluster.dart_rundir + "/obs_seq.out", f_oso)
+ print('saved', f_oso)
+ return oso
+
+def run_perfect_model_obs(nproc=12, verbose=True):
+ if verbose:
+ print("generating observations - running ./perfect_model_obs")
+ os.chdir(cluster.dart_rundir)
+
+ try_remove(cluster.dart_rundir + "/obs_seq.out")
+ if not os.path.exists(cluster.dart_rundir + "/obs_seq.in"):
+ raise RuntimeError("obs_seq.in does not exist in " + cluster.dart_rundir)
+ shell(cluster.dart_modules+' mpirun -np '+str(nproc)+" ./perfect_model_obs > log.perfect_model_obs")
+ if not os.path.exists(cluster.dart_rundir + "/obs_seq.out"):
+ raise RuntimeError(
+ "obs_seq.out does not exist in " + cluster.dart_rundir,
+ "\n look for " + cluster.dart_rundir + "/log.perfect_model_obs")
\ No newline at end of file
diff --git a/dartwrf/obskind.py b/dartwrf/obs/obskind.py
similarity index 100%
rename from dartwrf/obskind.py
rename to dartwrf/obs/obskind.py
diff --git a/dartwrf/obsseq.py b/dartwrf/obs/obsseq.py
similarity index 99%
rename from dartwrf/obsseq.py
rename to dartwrf/obs/obsseq.py
index fc20e51572049ad447b21f2d8363572311631ef9..e108e70491dcf26489db1958fb24fbf4e865368c 100755
--- a/dartwrf/obsseq.py
+++ b/dartwrf/obs/obsseq.py
@@ -703,7 +703,7 @@ class ObsSeq(object):
if __name__ == "__main__":
- from config.cluster import cluster
+ from dartwrf.server_config import cluster
# for testing purposes
# f = cluster.scriptsdir + "/../tests/obs_seq.orig.out"
diff --git a/dartwrf/obsseq_2dim.py b/dartwrf/obs/obsseq_2dim.py
similarity index 98%
rename from dartwrf/obsseq_2dim.py
rename to dartwrf/obs/obsseq_2dim.py
index c9a647c128ac59d26f28b20bcf9b8b2342cc3ccd..7239906206672579edf2320578784032fc0bec3a 100755
--- a/dartwrf/obsseq_2dim.py
+++ b/dartwrf/obs/obsseq_2dim.py
@@ -22,7 +22,7 @@ import numpy as np
from dartwrf.server_config import cluster
from dartwrf import utils
from dartwrf import assim_synth_obs as aso
-from dartwrf import obsseq
+from dartwrf.obs import obsseq
def _get_n_obs_per_layer(oso):
"""Get number of observations per layer"""
diff --git a/dartwrf/obsseq_to_netcdf.py b/dartwrf/obs/obsseq_to_netcdf.py
similarity index 95%
rename from dartwrf/obsseq_to_netcdf.py
rename to dartwrf/obs/obsseq_to_netcdf.py
index 14b04475990049d6a653ed555d240f359cb68481..db40ff16c015f5af7dc8083fd0b08dd3b9580b1c 100644
--- a/dartwrf/obsseq_to_netcdf.py
+++ b/dartwrf/obs/obsseq_to_netcdf.py
@@ -2,7 +2,7 @@ import os, sys, glob, warnings
from dartwrf.exp_config import exp
from dartwrf.server_config import cluster
-import dartwrf.run_obs_diag as rod
+import dartwrf.obs.run_obs_diag as rod
def listdir_dirs(path):
return [a for a in os.listdir(path) if os.path.isdir(os.path.join(path, a))]
diff --git a/dartwrf/run_obs_diag.py b/dartwrf/obs/run_obs_diag.py
similarity index 100%
rename from dartwrf/run_obs_diag.py
rename to dartwrf/obs/run_obs_diag.py
diff --git a/dartwrf/workflows.py b/dartwrf/workflows.py
index bbc67b4bdf3c0544c3e3b51d75a8f25465af1043..62e1b6a7a7042be82f0e25fdd568a4135d15a5f4 100644
--- a/dartwrf/workflows.py
+++ b/dartwrf/workflows.py
@@ -112,7 +112,7 @@ class WorkFlows(object):
# copy obs kind def to config, we will read a table from there
# file needs to exist within package so sphinx can read it
- _dict_to_py(_obskind_read(), this_dir+'/obskind.py')
+ _dict_to_py(_obskind_read(), this_dir+'/obs/obskind.py')
_copy_dartwrf_to_archive() # includes config files
diff --git a/tests/test_assim.py b/tests/test_assim.py
index d53b53bfb51b8cd0b87a6b880cb5fde67d2d65ef..7f61e27ee4c8bb929023db89c1883121d1dc0731 100644
--- a/tests/test_assim.py
+++ b/tests/test_assim.py
@@ -1,7 +1,8 @@
import os, shutil
import datetime as dt
-from dartwrf import obsseq, assim_synth_obs
+from dartwrf import assim_synth_obs
+from dartwrf.obs import obsseq
from dartwrf.server_config import cluster
class ExperimentConfiguration(object):
diff --git a/tests/test_dart-rttov.py b/tests/test_dart-rttov.py
index ada56bf197c835b338060ea095365a446bd3afa5..f5f3153317bb119559a60e05ba5ce5fbc5fd8a3f 100644
--- a/tests/test_dart-rttov.py
+++ b/tests/test_dart-rttov.py
@@ -3,8 +3,8 @@ import numpy as np
import datetime as dt
import pandas as pd
-from dartwrf import obsseq
-import dartwrf.create_obsseq as osq
+from dartwrf.obs import obsseq
+import dartwrf.obs.create_obsseq_in as osq
import dartwrf.assim_synth_obs as aso
from dartwrf import wrfout_add_geo
diff --git a/tests/test_obsseq.py b/tests/test_obsseq.py
index 215df541510027a777bf621083759e8c7f404a5d..776946537d028366cfdcce9632b53954c1c1fd67 100644
--- a/tests/test_obsseq.py
+++ b/tests/test_obsseq.py
@@ -2,7 +2,7 @@ import os, filecmp, shutil
import numpy as np
from dartwrf.server_config import cluster
-from dartwrf import obsseq
+from dartwrf.obs import obsseq
def test_oso():