diff --git a/dartwrf/utils.py b/dartwrf/utils.py
index 2e3f766019eb83bd17a74893ebd183508c4f9580..cc19b5d4dcad75b9c1758c8c0bfe40b51015a872 100755
--- a/dartwrf/utils.py
+++ b/dartwrf/utils.py
@@ -1,40 +1,40 @@
-import os, sys, shutil, glob, warnings
+"""Utility functions for DART-WRF
+
+Caution: You can not use the configuration files in here,
+because loading this would lead to a circular import.
+"""
+
+import os
+import sys
+import shutil
+import glob
+import warnings
import builtins as __builtin__
import subprocess
import datetime as dt
-import re, tempfile
-
-# class Stage(object):
-# """Collection of variables describing the assimilation stage"""
-# def __init__(self, **kwargs):
-# self.superob_km = False # False or int (spatial averaging of observations)
-# self.use_existing_obsseq = False # False or pathname (use precomputed obs_seq.out files)
-# self.__dict__.update(kwargs)
+import re
+import tempfile
+import pickle
-# # raise ValueError if attributes are not set
-# needed_attributes = ['observations', 'dart_nml',]
-# for attr in needed_attributes:
-# if not hasattr(self, attr):
-# raise ValueError('Stage.'+attr+' is not set')
class Experiment(object):
"""Collection of variables which define the experiment
-
+
Attributes:
expname (str): Name of the experiment
model_dx (int): WRF grid spacing in meters
n_ens (int): Ensemble size
do_quality_control (bool): If True, activate "quality control" function in assim_synth_obs.py
-
+
nature_wrfout_pattern (str): Path to the nature run, where we take observations from;
the path can contain wildcards (*,?), e.g. '/jetfs/exp1/*/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
input_profile (str): Path to WRF idealized input profiles;
e.g. '/data/initial_profiles/wrf/ens/raso.fc.<iens>.wrfprof';
<iens> is replaced by 001-040 for a 40-member ensemble
-
+
update_vars (list of str): Variables which will be updated after assimilation (update_IC.py)
e.g. ['U', 'V', 'W', 'THM', 'PH', 'MU', 'QVAPOR',]
-
+
observations (list of dict): Dictionaries which define an observation;
keys:
`error_generate`: measurement error standard-deviation;
@@ -42,21 +42,23 @@ class Experiment(object):
`heights`: list of integers at which observations are taken;
`loc_horiz_km`: float of horizontal localization half-width in km;
`loc_vert_km`: float of vertical localization half-width in km;
-
+
use_existing_obsseq (str, False): Path to existing obs_seq.out file (False: generate new one);
time string is replaced by actual time: /path/%Y-%m-%d_%H:%M_obs_seq.out
-
+
dart_nml (dict): updates to the default input.nml of DART (in dart_srcdir)
keys are namelist section headers (e.g. &filter_nml)
values are dictionaries of parameters and values (e.g. dict(ens_size=exp.n_ens,))
"""
+
def __init__(self):
pass
+
class ClusterConfig(object):
"""Collection of variables regarding the cluster configuration
-
+
Configuration name docs
When coding, use configuration settings like this:
@@ -89,7 +91,8 @@ class ClusterConfig(object):
rttov_srcdir (str): Path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
dartwrf_dir (str): Path where DART-WRF scripts reside, e.g. /home/DART-WRF/
- geo_em_for_WRF_ideal (str, False): Path to the geo_em.d01.nc file for idealized nature runs
+ geo_em_nature (str, False): Path to the geo_em.d01.nc file for idealized nature runs
+ geo_em_forecast (str, False): Path to the geo_em.d01.nc file for the forecast domain
obs_impact_filename
namelist (str): Path to a WRF namelist template;
strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
@@ -100,18 +103,19 @@ class ClusterConfig(object):
This configuration can be customized for any job (e.g. in workflows.py)
"""
+
def __init__(self, exp):
self.exp = exp # to access the experiment config in here
# defaults
self.dart_modules = ''
- self.wrf_modules = ''
+ self.wrf_modules = ''
self.size_jobarray = '1'
@property
def archivedir(self):
"""Path to the directory where data for the experiment is stored
-
+
Example:
`/users/abcd/data/sim_archive/experiment1/`
"""
@@ -124,7 +128,7 @@ class ClusterConfig(object):
Note:
If you want to execute scripts from the folder where you develop code, use `self.dartwrf_dir` (not sure if this works)
If you want to execute the code from a different place ('research'), then use `self.archivedir+'/DART-WRF/'`
-
+
Example:
`/user/data/sim_archive/DART-WRF/dartwrf/`
"""
@@ -160,26 +164,35 @@ class ClusterConfig(object):
"""
if self.use_slurm:
from slurmpy import Slurm
- return Slurm(jobname, slurm_kwargs=dict(self.slurm_cfg, **cfg_update),
- log_dir=self.log_dir,
- scripts_dir=self.slurm_scripts_dir,
- ).run(cmd, depends_on=depends_on)
+ return Slurm(jobname, slurm_kwargs=dict(self.slurm_cfg, **cfg_update),
+ log_dir=self.log_dir,
+ scripts_dir=self.slurm_scripts_dir,
+ ).run(cmd, depends_on=depends_on)
else:
print(cmd)
returncode = os.system(cmd)
if returncode != 0:
raise Exception('Error running command >>> '+cmd)
+
userhome = os.path.expanduser('~')
-def shell(args):
+
+def shell(args, pythonpath=None):
print(args)
- #subprocess.run(args.split(' ')) #, shell=True) #, stderr=subprocess.STDOUT)
- return os.system(args)
+ os.system(args)
+ # if pythonpath:
+ # env = os.environ.copy()
+ # env['PYTHONPATH'] = pythonpath
+ # subprocess.check_output(args.split(' '), env=env)
+ # else:
+ # subprocess.check_output(args.split(' '))
+
def print(*args):
__builtin__.print(*args, flush=True)
+
def copy(src, dst, remove_if_exists=True):
if src == dst:
return # the link already exists, nothing to do
@@ -190,54 +203,59 @@ def copy(src, dst, remove_if_exists=True):
pass
shutil.copy(src, dst)
+
def try_remove(f):
try:
os.remove(f)
except:
pass
+
def mkdir(path):
os.system('mkdir -p '+path)
+
def script_to_str(path):
return open(path, 'r').read()
+
def copy_contents(src, dst):
os.system('cp -rf '+src+'/* '+dst+'/')
+
def clean_wrfdir(dir):
for s in ['wrfout_*', 'rsl.*', 'wrfrst_*']:
for f in glob.glob(dir+'/'+s):
os.remove(f)
+
def symlink(src, dst):
"""Create a symbolic link from src to dst
+ Creates the folder if it does not exist
"""
- try:
- os.symlink(src, dst)
- except FileExistsError:
- # print('file exists')
- if os.path.realpath(dst) == src:
- pass # print('link is correct')
- else:
- os.remove(dst)
- os.symlink(src, dst)
- except Exception as e:
- raise e
+ try: # this file may not exist
+ os.remove(dst)
+ except OSError:
+ pass
+
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ os.symlink(src, dst)
+
def link_contents(src, dst):
"""Create symbolic links for all files in src to dst
-
+
Args:
src (str): Path to source directory
dst (str): Path to destination directory
-
+
Returns:
None
"""
for f in os.listdir(src):
symlink(src+'/'+f, dst+'/'+f)
+
def sed_inplace(filename, pattern, repl):
'''Perform the pure-Python equivalent of in-place `sed` substitution
Like `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
@@ -270,6 +288,7 @@ def sed_inplace(filename, pattern, repl):
shutil.copystat(filename, tmp_file.name)
shutil.move(tmp_file.name, filename)
+
def append_file(f_main, f_gets_appended):
"""Append the contents of one file to another
@@ -284,13 +303,14 @@ def append_file(f_main, f_gets_appended):
if rc != 0:
raise RuntimeError('cat '+f_gets_appended+' >> '+f_main)
+
def write_txt(lines, fpath):
"""Write a list of strings to a text file
-
+
Args:
lines (list): List of strings
fpath (str): Path to file
-
+
Returns:
None
"""
@@ -298,3 +318,13 @@ def write_txt(lines, fpath):
with open(fpath, "w") as file:
for line in lines:
file.write(line+'\n')
+
+
+def save_dict(dictionary, fpath):
+ with open(fpath, 'wb') as f:
+ pickle.dump(dictionary, f)
+
+
+def load_dict(fpath):
+ with open(fpath, 'rb') as f:
+ return pickle.load(f)