Skip to content
Snippets Groups Projects
Commit f14b7b67 authored by Lukas Kugler's avatar Lukas Kugler
Browse files

symlink did not overwrite existing links

parent 1516ad11
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 77 additions and 47 deletions
dartwrf/utils.py
+ 77
− 47
View file @ f14b7b67
import os, sys, shutil, glob, warnings
"""Utility functions for DART-WRF
Caution: You can not use the configuration files in here,
because loading this would lead to a circular import.
"""
import os
import sys
import shutil
import glob
import warnings
import builtins as __builtin__
import subprocess
import datetime as dt
import re, tempfile
# class Stage(object):
# """Collection of variables describing the assimilation stage"""
# def __init__(self, **kwargs):
# self.superob_km = False # False or int (spatial averaging of observations)
# self.use_existing_obsseq = False # False or pathname (use precomputed obs_seq.out files)
# self.__dict__.update(kwargs)
import re
import tempfile
import pickle
# # raise ValueError if attributes are not set
# needed_attributes = ['observations', 'dart_nml',]
# for attr in needed_attributes:
# if not hasattr(self, attr):
# raise ValueError('Stage.'+attr+' is not set')
class Experiment(object):
"""Collection of variables which define the experiment
Attributes:
expname (str): Name of the experiment
model_dx (int): WRF grid spacing in meters
n_ens (int): Ensemble size
do_quality_control (bool): If True, activate "quality control" function in assim_synth_obs.py
nature_wrfout_pattern (str): Path to the nature run, where we take observations from;
the path can contain wildcards (*,?), e.g. '/jetfs/exp1/*/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
input_profile (str): Path to WRF idealized input profiles;
e.g. '/data/initial_profiles/wrf/ens/raso.fc.<iens>.wrfprof';
<iens> is replaced by 001-040 for a 40-member ensemble
update_vars (list of str): Variables which will be updated after assimilation (update_IC.py)
e.g. ['U', 'V', 'W', 'THM', 'PH', 'MU', 'QVAPOR',]
observations (list of dict): Dictionaries which define an observation;
keys:
`error_generate`: measurement error standard-deviation;
......@@ -42,21 +42,23 @@ class Experiment(object):
`heights`: list of integers at which observations are taken;
`loc_horiz_km`: float of horizontal localization half-width in km;
`loc_vert_km`: float of vertical localization half-width in km;
use_existing_obsseq (str, False): Path to existing obs_seq.out file (False: generate new one);
time string is replaced by actual time: /path/%Y-%m-%d_%H:%M_obs_seq.out
dart_nml (dict): updates to the default input.nml of DART (in dart_srcdir)
keys are namelist section headers (e.g. &filter_nml)
values are dictionaries of parameters and values (e.g. dict(ens_size=exp.n_ens,))
"""
def __init__(self):
pass
class ClusterConfig(object):
"""Collection of variables regarding the cluster configuration
Configuration name docs
When coding, use configuration settings like this:
......@@ -89,7 +91,8 @@ class ClusterConfig(object):
rttov_srcdir (str): Path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
dartwrf_dir (str): Path where DART-WRF scripts reside, e.g. /home/DART-WRF/
geo_em_for_WRF_ideal (str, False): Path to the geo_em.d01.nc file for idealized nature runs
geo_em_nature (str, False): Path to the geo_em.d01.nc file for idealized nature runs
geo_em_forecast (str, False): Path to the geo_em.d01.nc file for the forecast domain
obs_impact_filename
namelist (str): Path to a WRF namelist template;
strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
......@@ -100,18 +103,19 @@ class ClusterConfig(object):
This configuration can be customized for any job (e.g. in workflows.py)
"""
def __init__(self, exp):
self.exp = exp # to access the experiment config in here
# defaults
self.dart_modules = ''
self.wrf_modules = ''
self.wrf_modules = ''
self.size_jobarray = '1'
@property
def archivedir(self):
"""Path to the directory where data for the experiment is stored
Example:
`/users/abcd/data/sim_archive/experiment1/`
"""
......@@ -124,7 +128,7 @@ class ClusterConfig(object):
Note:
If you want to execute scripts from the folder where you develop code, use `self.dartwrf_dir` (not sure if this works)
If you want to execute the code from a different place ('research'), then use `self.archivedir+'/DART-WRF/'`
Example:
`/user/data/sim_archive/DART-WRF/dartwrf/`
"""
......@@ -160,26 +164,35 @@ class ClusterConfig(object):
"""
if self.use_slurm:
from slurmpy import Slurm
return Slurm(jobname, slurm_kwargs=dict(self.slurm_cfg, **cfg_update),
log_dir=self.log_dir,
scripts_dir=self.slurm_scripts_dir,
).run(cmd, depends_on=depends_on)
return Slurm(jobname, slurm_kwargs=dict(self.slurm_cfg, **cfg_update),
log_dir=self.log_dir,
scripts_dir=self.slurm_scripts_dir,
).run(cmd, depends_on=depends_on)
else:
print(cmd)
returncode = os.system(cmd)
if returncode != 0:
raise Exception('Error running command >>> '+cmd)
userhome = os.path.expanduser('~')
def shell(args):
def shell(args, pythonpath=None):
print(args)
#subprocess.run(args.split(' ')) #, shell=True) #, stderr=subprocess.STDOUT)
return os.system(args)
os.system(args)
# if pythonpath:
# env = os.environ.copy()
# env['PYTHONPATH'] = pythonpath
# subprocess.check_output(args.split(' '), env=env)
# else:
# subprocess.check_output(args.split(' '))
def print(*args):
__builtin__.print(*args, flush=True)
def copy(src, dst, remove_if_exists=True):
if src == dst:
return # the link already exists, nothing to do
......@@ -190,54 +203,59 @@ def copy(src, dst, remove_if_exists=True):
pass
shutil.copy(src, dst)
def try_remove(f):
try:
os.remove(f)
except:
pass
def mkdir(path):
os.system('mkdir -p '+path)
def script_to_str(path):
return open(path, 'r').read()
def copy_contents(src, dst):
os.system('cp -rf '+src+'/* '+dst+'/')
def clean_wrfdir(dir):
for s in ['wrfout_*', 'rsl.*', 'wrfrst_*']:
for f in glob.glob(dir+'/'+s):
os.remove(f)
def symlink(src, dst):
"""Create a symbolic link from src to dst
Creates the folder if it does not exist
"""
try:
os.symlink(src, dst)
except FileExistsError:
# print('file exists')
if os.path.realpath(dst) == src:
pass # print('link is correct')
else:
os.remove(dst)
os.symlink(src, dst)
except Exception as e:
raise e
try: # this file may not exist
os.remove(dst)
except OSError:
pass
os.makedirs(os.path.dirname(dst), exist_ok=True)
os.symlink(src, dst)
def link_contents(src, dst):
"""Create symbolic links for all files in src to dst
Args:
src (str): Path to source directory
dst (str): Path to destination directory
Returns:
None
"""
for f in os.listdir(src):
symlink(src+'/'+f, dst+'/'+f)
def sed_inplace(filename, pattern, repl):
'''Perform the pure-Python equivalent of in-place `sed` substitution
Like `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
......@@ -270,6 +288,7 @@ def sed_inplace(filename, pattern, repl):
shutil.copystat(filename, tmp_file.name)
shutil.move(tmp_file.name, filename)
def append_file(f_main, f_gets_appended):
"""Append the contents of one file to another
......@@ -284,13 +303,14 @@ def append_file(f_main, f_gets_appended):
if rc != 0:
raise RuntimeError('cat '+f_gets_appended+' >> '+f_main)
def write_txt(lines, fpath):
"""Write a list of strings to a text file
Args:
lines (list): List of strings
fpath (str): Path to file
Returns:
None
"""
......@@ -298,3 +318,13 @@ def write_txt(lines, fpath):
with open(fpath, "w") as file:
for line in lines:
file.write(line+'\n')
def save_dict(dictionary, fpath):
with open(fpath, 'wb') as f:
pickle.dump(dictionary, f)
def load_dict(fpath):
with open(fpath, 'rb') as f:
return pickle.load(f)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment