improve loading config

dartwrf/workflows.py

@@ -25,13 +25,14 @@ def dict_to_py(d, outfile):
 
 class WorkFlows(object):
     def __init__(self, exp_config='cfg.py', server_config='server.py'):
-        """Set up the experiment folder in `archivedir`, copy config files, backup scripts.
+        """Set up the experiment folder in `archivedir`.
 
         Args:
             exp (str): Path to exp config file
             config (str): Path to the cluster config file
         """
-        # At first, load config from present folder (later from scripts_rundir)
+        # in WorkFlows, we load the config from the git cloned folder
+        # in all other dartwrf scripts, load the config from cluster.scripts_rundir
         # exp = __import__('config/'+exp_config)
         self.cluster = importlib.import_module('config.'+server_config.strip('.py')).cluster
 
@@ -84,10 +85,14 @@ class WorkFlows(object):
         except:
             raise
 
-        # Copy config files
-        shutil.copy('config/'+exp_config, self.cluster.scripts_rundir+'/cfg.py')
-        shutil.copy('config/'+server_config, self.cluster.scripts_rundir+'/cluster.py')  # whatever server, the config name is always the same!
-        shutil.copy('config/'+server_config, 'config/cluster.py')  # whatever server, the config name is always the same!
+        # later, we can load the exp cfg with `from config.cfg import exp`
+        shutil.copy('config/'+exp_config, self.cluster.scripts_rundir+'/config/cfg.py')
+
+        # later, we can load the cluster cfg with `from config.cluster import cluster`
+        shutil.copy('config/'+server_config, self.cluster.scripts_rundir+'/config/cluster.py')  # whatever server, the config name is always the same!
+        
+        # probably not needed
+        # shutil.copy('config/'+server_config, 'config/cluster.py')  # whatever server, the config name is always the same!
         
     def prepare_WRFrundir(self, init_time):
         """Create WRF/run directories and wrfinput files
@@ -138,7 +143,6 @@ class WorkFlows(object):
 
         if input_is_restart:  # start WRF in restart mode
         """
-        id = depends_on
         restart_flag = '.false.' if not input_is_restart else '.true.'
 
         # if False:  # doesnt work with restarts at the moment# first_minute:
@@ -180,7 +184,7 @@ class WorkFlows(object):
         if output_restart_interval:
             args.append('--restart_interval='+str(int(float(output_restart_interval))))
 
-        id = self.cluster.run_job(' '.join(args), "preWRF", cfg_update=dict(time="2"), depends_on=[id])
+        id = self.cluster.run_job(' '.join(args), "preWRF", cfg_update=dict(time="2"), depends_on=[depends_on])
 
         cmd = script_to_str(self.cluster.run_WRF).replace('<exp.expname>', exp.expname
                                         ).replace('<cluster.wrf_rundir_base>', self.cluster.wrf_rundir_base)
@@ -240,7 +244,7 @@ class WorkFlows(object):
 
     def create_satimages(self, init_time, depends_on=None):
         cmd = self.cluster.python_verif+' ~/RTTOV-WRF/run_init.py '+self.cluster.archivedir+init_time.strftime('/%Y-%m-%d_%H:%M/')
-        id = self.cluster.run_job(cmd, "RTTOV", cfg_update={"ntasks": "12", "time": "80", "mem": "180G"}, depends_on=[depends_on])
+        id = self.cluster.run_job(cmd, "RTTOV", cfg_update={"ntasks": "12", "time": "80", "mem": "200G"}, depends_on=[depends_on])
         return id