Skip to content
Snippets Groups Projects
Commit b29d4caf authored by lkugler's avatar lkugler
Browse files

initialize exp workflow

parent db365496
No related branches found
No related tags found
No related merge requests found
...@@ -15,22 +15,27 @@ class WorkFlows(object): ...@@ -15,22 +15,27 @@ class WorkFlows(object):
def __init__(self, exp_config='cfg.py', server_config='server.py'): def __init__(self, exp_config='cfg.py', server_config='server.py'):
"""Set up the experiment folder in `archivedir`. """Set up the experiment folder in `archivedir`.
1. Copy the DART-WRF scripts to `archivedir`
2. Copy the config files to `archivedir/dartwrf`
3. Set log paths
4. Write obskind.py file (dictionary of observation types)
we load the config from load the config from cluster.scripts_rundir/config/cfg.py
Args: Args:
exp (str): Path to exp config file exp_config (str): Path to exp config file
config (str): Path to the cluster config file server_config (str): Path to the cluster config file
Attributes: Attributes:
cluster (obj): cluster configuration as defined in server_config file cluster (obj): cluster configuration as defined in server_config file
exp (obj): experiment configuration as defined in exp_config file
we load the config from load the config from cluster.scripts_rundir/config/cfg.py
""" """
def copy_dartwrf_to_archive(): def copy_dartwrf_to_archive():
# Copy scripts to self.cluster.archivedir folder # Copy scripts to self.cluster.archivedir folder
os.makedirs(self.cluster.archivedir, exist_ok=True)
try: try:
shutil.copytree(self.cluster.dartwrf_dir, self.cluster.scripts_rundir) shutil.copytree(self.cluster.dartwrf_dir, self.cluster.archivedir+'/DART-WRF/')
print('scripts have been copied to', self.cluster.archivedir) print('DART-WRF has been copied to', self.cluster.archivedir)
except FileExistsError as e: except FileExistsError as e:
warnings.warn(str(e)) warnings.warn(str(e))
if input('The experiment name already exists! Scripts will not be overwritten. Continue? (Y/n) ') in ['Y', 'y']: if input('The experiment name already exists! Scripts will not be overwritten. Continue? (Y/n) ') in ['Y', 'y']:
...@@ -40,33 +45,42 @@ class WorkFlows(object): ...@@ -40,33 +45,42 @@ class WorkFlows(object):
except: except:
raise raise
def copy_config_to_archive(): # def copy_config_to_archive():
os.makedirs(self.cluster.scripts_rundir+'/config/', exist_ok=True) # os.makedirs(self.cluster.scripts_rundir+'/config/', exist_ok=True)
# later, we can load the exp cfg with `from config.cfg import exp` # # later, we can load the exp cfg with `from config.cfg import exp`
shutil.copyfile('config/'+exp_config, self.cluster.scripts_rundir+'/config/cfg.py') # shutil.copyfile('config/'+exp_config, self.cluster.scripts_rundir+'/config/cfg.py')
# later, we can load the cluster cfg with `from config.cluster import cluster` # # later, we can load the cluster cfg with `from config.cluster import cluster`
shutil.copyfile('config/'+server_config, self.cluster.scripts_rundir+'/config/cluster.py') # whatever server, the config name is always the same! # shutil.copyfile('config/'+server_config, self.cluster.scripts_rundir+'/config/cluster.py') # whatever server, the config name is always the same!
print('------ start exp from ', exp_config, ' and ', server_config, ' ------') print('------ start exp from ', exp_config, ' and ', server_config, ' ------')
# copy config file to current config folder # experiment starts, we dont know where the code shall run
# => read the configuration file
# copy the config files to this folder
this_dir = '/'.join(__file__.split('/')[:-1])
try: try:
shutil.copyfile('config/'+exp_config, '/'.join(__file__.split('/')[:-2])+'/config/cfg.py') shutil.copyfile('config/'+server_config, this_dir+'/server_config.py')
except shutil.SameFileError:
pass
try:
shutil.copyfile('config/'+exp_config, this_dir+'/exp_config.py')
except shutil.SameFileError: except shutil.SameFileError:
pass pass
# load python config files sys.path.append(this_dir)
self.cluster = importlib.import_module('config.'+server_config.strip('.py')).cluster from server_config import cluster
self.exp = importlib.import_module('config.'+exp_config.strip('.py')).exp self.cluster = cluster
from exp_config import exp
self.exp = exp
copy_dartwrf_to_archive() # includes config files
# we set the path from where python should import dartwrf modules # we set the path from where python should import dartwrf modules
self.cluster.python = 'export PYTHONPATH='+self.cluster.scripts_rundir+'; '+self.cluster.python self.cluster.python = 'export PYTHONPATH='+self.cluster.scripts_rundir+'; '+self.cluster.python
copy_dartwrf_to_archive()
copy_config_to_archive()
# Set paths and backup scripts # Set paths and backup scripts
self.cluster.log_dir = self.cluster.archivedir+'/logs/' self.cluster.log_dir = self.cluster.archivedir+'/logs/'
print('logging to', self.cluster.log_dir) print('logging to', self.cluster.log_dir)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment