restructure

analysis_only.py

@@ -5,6 +5,7 @@ running the forecast model without assimilation
 import os, sys, shutil
 import datetime as dt
 
+from dartwrf import utils
 from config.cfg import exp, cluster
 
 log_dir = cluster.archivedir+'/logs/'
@@ -12,10 +13,9 @@ slurm_scripts_dir = cluster.archivedir+'/slurm-scripts/'
 print('logging to', log_dir)
 print('scripts, which are submitted to SLURM:', slurm_scripts_dir)
 
-from utils import create_job, backup_scripts
 
 ###############################
-backup_scripts()
+utils.backup_scripts()
 
 
 prior_path_exp = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_noDA'
@@ -24,7 +24,7 @@ prior_valid_time = dt.datetime(2008,7,30,12,30)
 assim_time = prior_valid_time
 
 
-create_job('assim').run(
+os.system(
     cluster.python+' '+cluster.scripts_rundir+'/assim_synth_obs.py '
                 +assim_time.strftime('%Y-%m-%d_%H:%M ')
                 +prior_init_time.strftime('%Y-%m-%d_%H:%M ')

config/cfg.py

+from dartwrf import utils
 from config import clusters  # from . = problem in archivedir
 cluster = clusters.jet  # change cluster configuration here
 
-class ExperimentConfiguration(object):
-    def __init__(self):
-        pass
-
-exp = ExperimentConfiguration()
+exp = utils.ExperimentConfiguration()
 exp.expname = "test_jet" #"exp_v1.22_P3_wbub7_WV62_obs10_loc20_oe1"
 exp.model_dx = 2000
 exp.n_ens = 40
@@ -94,7 +91,7 @@ psfc = dict(plotname='SYNOP Pressure', plotunits='[Pa]',
             cov_loc_radius_km=32)
 
 
-exp.observations = [wv62]
+exp.observations = [t]
 exp.update_vars = ['U', 'V', 'W', 'THM', 'PH', 'MU', 'QVAPOR', 'QCLOUD', 'QICE', 'PSFC']
 #exp.update_vars = ['U', 'V', 'W', 'T', 'PH', 'MU', 'QVAPOR', 'PSFC']
 

config/clusters.py

 import os, sys
 import datetime as dt
+from dartwrf import utils
 
 """Configuration name docs
 
@@ -39,30 +40,12 @@ slurm_cfg               python dictionary, containing options of SLURM
 """                 
 
 
-class ClusterConfig(object):
-    """Helper class, contains useful abbreviations to use in code later on"""
-    def __init__(self):
-        pass
 
-    @property
-    def archivedir(self):
-        return self.archive_base+'/'+self.expname
 
-    def wrf_rundir(self, iens):
-        return self.wrf_rundir_base+'/'+self.expname+'/'+str(iens)
-
-    @property
-    def scripts_rundir(self):
-        return self.archivedir+'/DART-WRF/'
-
-    @property
-    def dartrundir(self):
-        return self.dart_rundir_base+'/'+self.expname+'/'
-
-
-vsc = ClusterConfig()
+vsc = utils.ClusterConfig()
 vsc.name = 'vsc' 
-srvx1.max_nproc = 20
+vsc.max_nproc = 20
+vsc.use_slurm = True
 
 # binaries
 vsc.python = '/home/fs71386/lkugler/miniconda3/envs/DART/bin/python'
@@ -91,9 +74,10 @@ vsc.slurm_cfg = {"account": "p71386", "partition": "skylake_0384", "qos": "p7138
                  "nodes": "1", "ntasks": "1", "ntasks-per-node": "48", "ntasks-per-core": "1",
                  "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
 
-jet = ClusterConfig()
+jet = utils.ClusterConfig()
 jet.name = 'jet'
-srvx1.max_nproc = 12
+jet.max_nproc = 12
+jet.use_slurm = True
 
 # binaries
 jet.python = '/jetfs/home/lkugler/miniconda3/envs/DART/bin/python'
@@ -124,9 +108,10 @@ jet.slurm_cfg = {"account": "lkugler", "partition": "compute",
                  "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
 
 
-srvx1 = ClusterConfig()
+srvx1 = utils.ClusterConfig()
 srvx1.name = 'srvx1'
 srvx1.max_nproc = 6
+srvx1.use_slurm = False
 
 # binaries
 srvx1.python = '/mnt/jetfs/home/lkugler/miniconda3/envs/DART/bin/python'

cycled_exp.py

@@ -6,9 +6,8 @@ submitting jobs into SLURM queue
 import os, sys, shutil, glob, warnings
 import datetime as dt
 
+from dartwrf.utils import script_to_str, symlink, copy, create_job
 from config.cfg import exp, cluster
-from dartwrf.utils import script_to_str, symlink, copy
-from utils import create_job
 
 log_dir = cluster.archivedir+'/logs/'
 slurm_scripts_dir = cluster.archivedir+'/slurm-scripts/'

dartwrf/utils.py

-import os, sys, shutil, glob
+import os, sys, shutil, glob, warnings
 import builtins as __builtin__
-#copy = shutil.copy
 import subprocess
+import datetime as dt
+from slurmpy import Slurm
+from config.cfg import cluster
+
+class ExperimentConfiguration(object):
+    """Collection of variables to use in code later on"""
+    def __init__(self):
+        pass
+
+class ClusterConfig(object):
+    """Collection of variables to use in code later on"""
+    def __init__(self):
+        pass
+
+    @property
+    def archivedir(self):
+        return self.archive_base+'/'+self.expname
+
+    def wrf_rundir(self, iens):
+        return self.wrf_rundir_base+'/'+self.expname+'/'+str(iens)
+
+    @property
+    def scripts_rundir(self):
+        return self.archivedir+'/DART-WRF/'
+
+    @property
+    def dartrundir(self):
+        return self.dart_rundir_base+'/'+self.expname+'/'
+
+class Shellslurm():
+    """Like Slurmpy class, but runs locally"""
+    def __init__(self, *args, **kwargs):
+        pass
+    def run(self, *args, **kwargs):
+        print(args[0])
+        os.system(args[0])
+
+def create_job(*args, cfg_update=dict(), **kwargs):
+    """Shortcut to slurmpy's class; keep certain default kwargs
+    and only update some with kwarg `cfg_update`
+    see https://github.com/brentp/slurmpy
+
+    with_slurm (bool) : if True, use SLURM, else run locally
+
+    """
+    if cluster.use_slurm:
+        return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update), 
+                 log_dir=log_dir, scripts_dir=slurm_scripts_dir, **kwargs)
+    else:
+        return Shellslurm(*args)
+
+def backup_scripts():
+    """Copies scripts and configuration to archive dir output folder"""
+    os.makedirs(cluster.archivedir, exist_ok=True)
+
+    try:
+        shutil.copytree(cluster.scriptsdir, cluster.scripts_rundir)
+    except FileExistsError:
+        pass
+    except:
+        raise
+    try:
+        copy(os.path.basename(__file__), cluster.scripts_rundir+'/')
+    except Exception as e:
+        warnings.warn(str(e))
+
+def prepare_WRFrundir(init_time):
+    """Create WRF/run directories and wrfinput files
+    """
+    cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
+    print(cmd)
+    os.system(cmd)
 
 def shell(args):
     print(args)

generate_free.py

@@ -8,15 +8,14 @@ import pandas as pd
 from slurmpy import Slurm
 
 from config.cfg import exp, cluster
-from dartwrf.utils import script_to_str, symlink
+from dartwrf.utils import script_to_str, symlink, backup_scripts
+from cycled_exp import *
 
 log_dir = cluster.archivedir+'/logs/'
 slurm_scripts_dir = cluster.archivedir+'/slurm-scripts/'
 print('logging to', log_dir)
 print('scripts, which are submitted to SLURM:', slurm_scripts_dir)
 
-from scheduler import *
-
 ################################
 print('starting osse')
 

utils.py

-import os, sys, shutil, glob, warnings
-import datetime as dt
-from slurmpy import Slurm
-
-from config.cfg import exp, cluster
-from dartwrf.utils import script_to_str, symlink, copy
-
-
-class Shellslurm():
-    """Like Slurm class, but runs locally"""
-    def __init__(self, *args, **kwargs):
-        pass
-    def run(self, *args, **kwargs):
-        print(args[0])
-        os.system(args[0])
-
-def create_job(*args, cfg_update=dict(), with_slurm=True, **kwargs):
-    """Shortcut to slurmpy's class; keep certain default kwargs
-    and only update some with kwarg `cfg_update`
-    see https://github.com/brentp/slurmpy
-
-    with_slurm (bool) : if True, use SLURM, else run locally
-
-    """
-    if with_slurm:
-        return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update), 
-                 log_dir=log_dir, scripts_dir=slurm_scripts_dir, **kwargs)
-    else:
-        return Shellslurm(*args)
-
-def backup_scripts():
-    """Copies scripts and configuration to archive dir output folder"""
-    os.makedirs(cluster.archivedir, exist_ok=True)
-
-    try:
-        shutil.copytree(cluster.scriptsdir, cluster.scripts_rundir)
-    except FileExistsError:
-        pass
-    except:
-        raise
-    try:
-        copy(os.path.basename(__file__), cluster.scripts_rundir+'/')
-    except Exception as e:
-        warnings.warn(str(e))
-
-def prepare_WRFrundir(init_time):
-    """Create WRF/run directories and wrfinput files
-    """
-    cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
-    print(cmd)
-    os.system(cmd)
-