Skip to content
Snippets Groups Projects
Commit f14b7b67 authored by Lukas Kugler's avatar Lukas Kugler
Browse files

symlink did not overwrite existing links

parent 1516ad11
No related branches found
No related tags found
No related merge requests found
import os, sys, shutil, glob, warnings """Utility functions for DART-WRF
Caution: You can not use the configuration files in here,
because loading this would lead to a circular import.
"""
import os
import sys
import shutil
import glob
import warnings
import builtins as __builtin__ import builtins as __builtin__
import subprocess import subprocess
import datetime as dt import datetime as dt
import re, tempfile import re
import tempfile
# class Stage(object): import pickle
# """Collection of variables describing the assimilation stage"""
# def __init__(self, **kwargs):
# self.superob_km = False # False or int (spatial averaging of observations)
# self.use_existing_obsseq = False # False or pathname (use precomputed obs_seq.out files)
# self.__dict__.update(kwargs)
# # raise ValueError if attributes are not set
# needed_attributes = ['observations', 'dart_nml',]
# for attr in needed_attributes:
# if not hasattr(self, attr):
# raise ValueError('Stage.'+attr+' is not set')
class Experiment(object): class Experiment(object):
"""Collection of variables which define the experiment """Collection of variables which define the experiment
...@@ -51,9 +51,11 @@ class Experiment(object): ...@@ -51,9 +51,11 @@ class Experiment(object):
values are dictionaries of parameters and values (e.g. dict(ens_size=exp.n_ens,)) values are dictionaries of parameters and values (e.g. dict(ens_size=exp.n_ens,))
""" """
def __init__(self): def __init__(self):
pass pass
class ClusterConfig(object): class ClusterConfig(object):
"""Collection of variables regarding the cluster configuration """Collection of variables regarding the cluster configuration
...@@ -89,7 +91,8 @@ class ClusterConfig(object): ...@@ -89,7 +91,8 @@ class ClusterConfig(object):
rttov_srcdir (str): Path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/ rttov_srcdir (str): Path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
dartwrf_dir (str): Path where DART-WRF scripts reside, e.g. /home/DART-WRF/ dartwrf_dir (str): Path where DART-WRF scripts reside, e.g. /home/DART-WRF/
geo_em_for_WRF_ideal (str, False): Path to the geo_em.d01.nc file for idealized nature runs geo_em_nature (str, False): Path to the geo_em.d01.nc file for idealized nature runs
geo_em_forecast (str, False): Path to the geo_em.d01.nc file for the forecast domain
obs_impact_filename obs_impact_filename
namelist (str): Path to a WRF namelist template; namelist (str): Path to a WRF namelist template;
strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
...@@ -100,6 +103,7 @@ class ClusterConfig(object): ...@@ -100,6 +103,7 @@ class ClusterConfig(object):
This configuration can be customized for any job (e.g. in workflows.py) This configuration can be customized for any job (e.g. in workflows.py)
""" """
def __init__(self, exp): def __init__(self, exp):
self.exp = exp # to access the experiment config in here self.exp = exp # to access the experiment config in here
...@@ -170,16 +174,25 @@ class ClusterConfig(object): ...@@ -170,16 +174,25 @@ class ClusterConfig(object):
if returncode != 0: if returncode != 0:
raise Exception('Error running command >>> '+cmd) raise Exception('Error running command >>> '+cmd)
userhome = os.path.expanduser('~') userhome = os.path.expanduser('~')
def shell(args):
def shell(args, pythonpath=None):
print(args) print(args)
#subprocess.run(args.split(' ')) #, shell=True) #, stderr=subprocess.STDOUT) os.system(args)
return os.system(args) # if pythonpath:
# env = os.environ.copy()
# env['PYTHONPATH'] = pythonpath
# subprocess.check_output(args.split(' '), env=env)
# else:
# subprocess.check_output(args.split(' '))
def print(*args): def print(*args):
__builtin__.print(*args, flush=True) __builtin__.print(*args, flush=True)
def copy(src, dst, remove_if_exists=True): def copy(src, dst, remove_if_exists=True):
if src == dst: if src == dst:
return # the link already exists, nothing to do return # the link already exists, nothing to do
...@@ -190,40 +203,44 @@ def copy(src, dst, remove_if_exists=True): ...@@ -190,40 +203,44 @@ def copy(src, dst, remove_if_exists=True):
pass pass
shutil.copy(src, dst) shutil.copy(src, dst)
def try_remove(f): def try_remove(f):
try: try:
os.remove(f) os.remove(f)
except: except:
pass pass
def mkdir(path): def mkdir(path):
os.system('mkdir -p '+path) os.system('mkdir -p '+path)
def script_to_str(path): def script_to_str(path):
return open(path, 'r').read() return open(path, 'r').read()
def copy_contents(src, dst): def copy_contents(src, dst):
os.system('cp -rf '+src+'/* '+dst+'/') os.system('cp -rf '+src+'/* '+dst+'/')
def clean_wrfdir(dir): def clean_wrfdir(dir):
for s in ['wrfout_*', 'rsl.*', 'wrfrst_*']: for s in ['wrfout_*', 'rsl.*', 'wrfrst_*']:
for f in glob.glob(dir+'/'+s): for f in glob.glob(dir+'/'+s):
os.remove(f) os.remove(f)
def symlink(src, dst): def symlink(src, dst):
"""Create a symbolic link from src to dst """Create a symbolic link from src to dst
Creates the folder if it does not exist
""" """
try: try: # this file may not exist
os.symlink(src, dst)
except FileExistsError:
# print('file exists')
if os.path.realpath(dst) == src:
pass # print('link is correct')
else:
os.remove(dst) os.remove(dst)
except OSError:
pass
os.makedirs(os.path.dirname(dst), exist_ok=True)
os.symlink(src, dst) os.symlink(src, dst)
except Exception as e:
raise e
def link_contents(src, dst): def link_contents(src, dst):
"""Create symbolic links for all files in src to dst """Create symbolic links for all files in src to dst
...@@ -238,6 +255,7 @@ def link_contents(src, dst): ...@@ -238,6 +255,7 @@ def link_contents(src, dst):
for f in os.listdir(src): for f in os.listdir(src):
symlink(src+'/'+f, dst+'/'+f) symlink(src+'/'+f, dst+'/'+f)
def sed_inplace(filename, pattern, repl): def sed_inplace(filename, pattern, repl):
'''Perform the pure-Python equivalent of in-place `sed` substitution '''Perform the pure-Python equivalent of in-place `sed` substitution
Like `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`. Like `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
...@@ -270,6 +288,7 @@ def sed_inplace(filename, pattern, repl): ...@@ -270,6 +288,7 @@ def sed_inplace(filename, pattern, repl):
shutil.copystat(filename, tmp_file.name) shutil.copystat(filename, tmp_file.name)
shutil.move(tmp_file.name, filename) shutil.move(tmp_file.name, filename)
def append_file(f_main, f_gets_appended): def append_file(f_main, f_gets_appended):
"""Append the contents of one file to another """Append the contents of one file to another
...@@ -284,6 +303,7 @@ def append_file(f_main, f_gets_appended): ...@@ -284,6 +303,7 @@ def append_file(f_main, f_gets_appended):
if rc != 0: if rc != 0:
raise RuntimeError('cat '+f_gets_appended+' >> '+f_main) raise RuntimeError('cat '+f_gets_appended+' >> '+f_main)
def write_txt(lines, fpath): def write_txt(lines, fpath):
"""Write a list of strings to a text file """Write a list of strings to a text file
...@@ -298,3 +318,13 @@ def write_txt(lines, fpath): ...@@ -298,3 +318,13 @@ def write_txt(lines, fpath):
with open(fpath, "w") as file: with open(fpath, "w") as file:
for line in lines: for line in lines:
file.write(line+'\n') file.write(line+'\n')
def save_dict(dictionary, fpath):
with open(fpath, 'wb') as f:
pickle.dump(dictionary, f)
def load_dict(fpath):
with open(fpath, 'rb') as f:
return pickle.load(f)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment