symlink did not overwrite existing links

dartwrf/utils.py

-import os, sys, shutil, glob, warnings
+"""Utility functions for DART-WRF
+
+Caution: You can not use the configuration files in here, 
+because loading this would lead to a circular import.
+"""
+
+import os
+import sys
+import shutil
+import glob
+import warnings
 import builtins as __builtin__
 import subprocess
 import datetime as dt
-import re, tempfile
-
-# class Stage(object):
-#     """Collection of variables describing the assimilation stage"""
-#     def __init__(self, **kwargs):
-#         self.superob_km = False  # False or int (spatial averaging of observations)
-#         self.use_existing_obsseq = False  # False or pathname (use precomputed obs_seq.out files)
-#         self.__dict__.update(kwargs)
+import re
+import tempfile
+import pickle
 
-#         # raise ValueError if attributes are not set
-#         needed_attributes = ['observations', 'dart_nml',]
-#         for attr in needed_attributes:
-#             if not hasattr(self, attr):
-#                 raise ValueError('Stage.'+attr+' is not set')
 
 class Experiment(object):
     """Collection of variables which define the experiment
@@ -51,9 +51,11 @@ class Experiment(object):
             values are dictionaries of parameters and values (e.g. dict(ens_size=exp.n_ens,))
 
     """
+
     def __init__(self):
         pass
 
+
 class ClusterConfig(object):
     """Collection of variables regarding the cluster configuration
 
@@ -89,7 +91,8 @@ class ClusterConfig(object):
         rttov_srcdir (str): Path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
         dartwrf_dir (str): Path where DART-WRF scripts reside, e.g. /home/DART-WRF/
 
-        geo_em_for_WRF_ideal (str, False): Path to the geo_em.d01.nc file for idealized nature runs
+        geo_em_nature (str, False): Path to the geo_em.d01.nc file for idealized nature runs
+        geo_em_forecast (str, False): Path to the geo_em.d01.nc file for the forecast domain
         obs_impact_filename
         namelist (str): Path to a WRF namelist template; 
                         strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
@@ -100,6 +103,7 @@ class ClusterConfig(object):
                             This configuration can be customized for any job (e.g. in workflows.py)
 
     """
+
     def __init__(self, exp):
         self.exp = exp  # to access the experiment config in here
 
@@ -170,16 +174,25 @@ class ClusterConfig(object):
             if returncode != 0:
                 raise Exception('Error running command >>> '+cmd)
 
+
 userhome = os.path.expanduser('~')
 
-def shell(args):
+
+def shell(args, pythonpath=None):
     print(args)
-    #subprocess.run(args.split(' ')) #, shell=True) #, stderr=subprocess.STDOUT) 
-    return os.system(args)
+    os.system(args)
+    # if pythonpath:
+    #     env = os.environ.copy()
+    #     env['PYTHONPATH'] = pythonpath
+    #     subprocess.check_output(args.split(' '), env=env)
+    # else:
+    #     subprocess.check_output(args.split(' '))
+
 
 def print(*args):
     __builtin__.print(*args, flush=True)
 
+
 def copy(src, dst, remove_if_exists=True):
     if src == dst:
         return  # the link already exists, nothing to do
@@ -190,40 +203,44 @@ def copy(src, dst, remove_if_exists=True):
             pass
     shutil.copy(src, dst)
 
+
 def try_remove(f):
     try:
         os.remove(f)
     except:
         pass
 
+
 def mkdir(path):
     os.system('mkdir -p '+path)
 
+
 def script_to_str(path):
     return open(path, 'r').read()
 
+
 def copy_contents(src, dst):
     os.system('cp -rf '+src+'/* '+dst+'/')
 
+
 def clean_wrfdir(dir):
     for s in ['wrfout_*', 'rsl.*', 'wrfrst_*']:
         for f in glob.glob(dir+'/'+s):
             os.remove(f)
 
+
 def symlink(src, dst):
     """Create a symbolic link from src to dst
+    Creates the folder if it does not exist
     """
-    try:
-        os.symlink(src, dst)
-    except FileExistsError:
-        # print('file exists')
-        if os.path.realpath(dst) == src:
-            pass  # print('link is correct')
-        else:
+    try:  # this file may not exist
         os.remove(dst)
+    except OSError:
+        pass
+
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
     os.symlink(src, dst)
-    except Exception as e:
-        raise e
+
 
 def link_contents(src, dst):
     """Create symbolic links for all files in src to dst
@@ -238,6 +255,7 @@ def link_contents(src, dst):
     for f in os.listdir(src):
         symlink(src+'/'+f, dst+'/'+f)
 
+
 def sed_inplace(filename, pattern, repl):
     '''Perform the pure-Python equivalent of in-place `sed` substitution
     Like `sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
@@ -270,6 +288,7 @@ def sed_inplace(filename, pattern, repl):
     shutil.copystat(filename, tmp_file.name)
     shutil.move(tmp_file.name, filename)
 
+
 def append_file(f_main, f_gets_appended):
     """Append the contents of one file to another
 
@@ -284,6 +303,7 @@ def append_file(f_main, f_gets_appended):
     if rc != 0:
         raise RuntimeError('cat '+f_gets_appended+' >> '+f_main)
 
+
 def write_txt(lines, fpath):
     """Write a list of strings to a text file
 
@@ -298,3 +318,13 @@ def write_txt(lines, fpath):
     with open(fpath, "w") as file:
         for line in lines:
             file.write(line+'\n')
+
+
+def save_dict(dictionary, fpath):
+    with open(fpath, 'wb') as f:
+        pickle.dump(dictionary, f)
+
+
+def load_dict(fpath):
+    with open(fpath, 'rb') as f:
+        return pickle.load(f)