From a830fd6b5fc80cfca2b5dc3c152595fcd1bdaa18 Mon Sep 17 00:00:00 2001
From: lkugler <lukas.kugler@gmail.com>
Date: Tue, 31 Jan 2023 15:58:43 +0100
Subject: [PATCH] restructure
---
analysis_only.py | 6 ++--
config/cfg.py | 9 ++----
config/clusters.py | 33 ++++++--------------
cycled_exp.py | 3 +-
dartwrf/utils.py | 75 ++++++++++++++++++++++++++++++++++++++++++++--
generate_free.py | 5 ++--
utils.py | 52 --------------------------------
7 files changed, 91 insertions(+), 92 deletions(-)
delete mode 100644 utils.py
diff --git a/analysis_only.py b/analysis_only.py
index c8735fe..fa735f9 100755
--- a/analysis_only.py
+++ b/analysis_only.py
@@ -5,6 +5,7 @@ running the forecast model without assimilation
import os, sys, shutil
import datetime as dt
+from dartwrf import utils
from config.cfg import exp, cluster
log_dir = cluster.archivedir+'/logs/'
@@ -12,10 +13,9 @@ slurm_scripts_dir = cluster.archivedir+'/slurm-scripts/'
print('logging to', log_dir)
print('scripts, which are submitted to SLURM:', slurm_scripts_dir)
-from utils import create_job, backup_scripts
###############################
-backup_scripts()
+utils.backup_scripts()
prior_path_exp = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_noDA'
@@ -24,7 +24,7 @@ prior_valid_time = dt.datetime(2008,7,30,12,30)
assim_time = prior_valid_time
-create_job('assim').run(
+os.system(
cluster.python+' '+cluster.scripts_rundir+'/assim_synth_obs.py '
+assim_time.strftime('%Y-%m-%d_%H:%M ')
+prior_init_time.strftime('%Y-%m-%d_%H:%M ')
diff --git a/config/cfg.py b/config/cfg.py
index d2c4c8e..b34d51c 100755
--- a/config/cfg.py
+++ b/config/cfg.py
@@ -1,11 +1,8 @@
+from dartwrf import utils
from config import clusters # from . = problem in archivedir
cluster = clusters.jet # change cluster configuration here
-class ExperimentConfiguration(object):
- def __init__(self):
- pass
-
-exp = ExperimentConfiguration()
+exp = utils.ExperimentConfiguration()
exp.expname = "test_jet" #"exp_v1.22_P3_wbub7_WV62_obs10_loc20_oe1"
exp.model_dx = 2000
exp.n_ens = 40
@@ -94,7 +91,7 @@ psfc = dict(plotname='SYNOP Pressure', plotunits='[Pa]',
cov_loc_radius_km=32)
-exp.observations = [wv62]
+exp.observations = [t]
exp.update_vars = ['U', 'V', 'W', 'THM', 'PH', 'MU', 'QVAPOR', 'QCLOUD', 'QICE', 'PSFC']
#exp.update_vars = ['U', 'V', 'W', 'T', 'PH', 'MU', 'QVAPOR', 'PSFC']
diff --git a/config/clusters.py b/config/clusters.py
index fe4d2db..27751de 100755
--- a/config/clusters.py
+++ b/config/clusters.py
@@ -1,5 +1,6 @@
import os, sys
import datetime as dt
+from dartwrf import utils
"""Configuration name docs
@@ -39,30 +40,12 @@ slurm_cfg python dictionary, containing options of SLURM
"""
-class ClusterConfig(object):
- """Helper class, contains useful abbreviations to use in code later on"""
- def __init__(self):
- pass
- @property
- def archivedir(self):
- return self.archive_base+'/'+self.expname
- def wrf_rundir(self, iens):
- return self.wrf_rundir_base+'/'+self.expname+'/'+str(iens)
-
- @property
- def scripts_rundir(self):
- return self.archivedir+'/DART-WRF/'
-
- @property
- def dartrundir(self):
- return self.dart_rundir_base+'/'+self.expname+'/'
-
-
-vsc = ClusterConfig()
+vsc = utils.ClusterConfig()
vsc.name = 'vsc'
-srvx1.max_nproc = 20
+vsc.max_nproc = 20
+vsc.use_slurm = True
# binaries
vsc.python = '/home/fs71386/lkugler/miniconda3/envs/DART/bin/python'
@@ -91,9 +74,10 @@ vsc.slurm_cfg = {"account": "p71386", "partition": "skylake_0384", "qos": "p7138
"nodes": "1", "ntasks": "1", "ntasks-per-node": "48", "ntasks-per-core": "1",
"mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
-jet = ClusterConfig()
+jet = utils.ClusterConfig()
jet.name = 'jet'
-srvx1.max_nproc = 12
+jet.max_nproc = 12
+jet.use_slurm = True
# binaries
jet.python = '/jetfs/home/lkugler/miniconda3/envs/DART/bin/python'
@@ -124,9 +108,10 @@ jet.slurm_cfg = {"account": "lkugler", "partition": "compute",
"mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
-srvx1 = ClusterConfig()
+srvx1 = utils.ClusterConfig()
srvx1.name = 'srvx1'
srvx1.max_nproc = 6
+srvx1.use_slurm = False
# binaries
srvx1.python = '/mnt/jetfs/home/lkugler/miniconda3/envs/DART/bin/python'
diff --git a/cycled_exp.py b/cycled_exp.py
index cade1c5..0fc2a61 100755
--- a/cycled_exp.py
+++ b/cycled_exp.py
@@ -6,9 +6,8 @@ submitting jobs into SLURM queue
import os, sys, shutil, glob, warnings
import datetime as dt
+from dartwrf.utils import script_to_str, symlink, copy, create_job
from config.cfg import exp, cluster
-from dartwrf.utils import script_to_str, symlink, copy
-from utils import create_job
log_dir = cluster.archivedir+'/logs/'
slurm_scripts_dir = cluster.archivedir+'/slurm-scripts/'
diff --git a/dartwrf/utils.py b/dartwrf/utils.py
index 4185b5a..d4a5560 100755
--- a/dartwrf/utils.py
+++ b/dartwrf/utils.py
@@ -1,7 +1,78 @@
-import os, sys, shutil, glob
+import os, sys, shutil, glob, warnings
import builtins as __builtin__
-#copy = shutil.copy
import subprocess
+import datetime as dt
+from slurmpy import Slurm
+from config.cfg import cluster
+
+class ExperimentConfiguration(object):
+ """Collection of variables to use in code later on"""
+ def __init__(self):
+ pass
+
+class ClusterConfig(object):
+ """Collection of variables to use in code later on"""
+ def __init__(self):
+ pass
+
+ @property
+ def archivedir(self):
+ return self.archive_base+'/'+self.expname
+
+ def wrf_rundir(self, iens):
+ return self.wrf_rundir_base+'/'+self.expname+'/'+str(iens)
+
+ @property
+ def scripts_rundir(self):
+ return self.archivedir+'/DART-WRF/'
+
+ @property
+ def dartrundir(self):
+ return self.dart_rundir_base+'/'+self.expname+'/'
+
+class Shellslurm():
+ """Like Slurmpy class, but runs locally"""
+ def __init__(self, *args, **kwargs):
+ pass
+ def run(self, *args, **kwargs):
+ print(args[0])
+ os.system(args[0])
+
+def create_job(*args, cfg_update=dict(), **kwargs):
+ """Shortcut to slurmpy's class; keep certain default kwargs
+ and only update some with kwarg `cfg_update`
+ see https://github.com/brentp/slurmpy
+
+ with_slurm (bool) : if True, use SLURM, else run locally
+
+ """
+ if cluster.use_slurm:
+ return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update),
+ log_dir=log_dir, scripts_dir=slurm_scripts_dir, **kwargs)
+ else:
+ return Shellslurm(*args)
+
+def backup_scripts():
+ """Copies scripts and configuration to archive dir output folder"""
+ os.makedirs(cluster.archivedir, exist_ok=True)
+
+ try:
+ shutil.copytree(cluster.scriptsdir, cluster.scripts_rundir)
+ except FileExistsError:
+ pass
+ except:
+ raise
+ try:
+ copy(os.path.basename(__file__), cluster.scripts_rundir+'/')
+ except Exception as e:
+ warnings.warn(str(e))
+
+def prepare_WRFrundir(init_time):
+ """Create WRF/run directories and wrfinput files
+ """
+ cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
+ print(cmd)
+ os.system(cmd)
def shell(args):
print(args)
diff --git a/generate_free.py b/generate_free.py
index b1f3cb6..acb1fc0 100755
--- a/generate_free.py
+++ b/generate_free.py
@@ -8,15 +8,14 @@ import pandas as pd
from slurmpy import Slurm
from config.cfg import exp, cluster
-from dartwrf.utils import script_to_str, symlink
+from dartwrf.utils import script_to_str, symlink, backup_scripts
+from cycled_exp import *
log_dir = cluster.archivedir+'/logs/'
slurm_scripts_dir = cluster.archivedir+'/slurm-scripts/'
print('logging to', log_dir)
print('scripts, which are submitted to SLURM:', slurm_scripts_dir)
-from scheduler import *
-
################################
print('starting osse')
diff --git a/utils.py b/utils.py
deleted file mode 100644
index d3623cf..0000000
--- a/utils.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import os, sys, shutil, glob, warnings
-import datetime as dt
-from slurmpy import Slurm
-
-from config.cfg import exp, cluster
-from dartwrf.utils import script_to_str, symlink, copy
-
-
-class Shellslurm():
- """Like Slurm class, but runs locally"""
- def __init__(self, *args, **kwargs):
- pass
- def run(self, *args, **kwargs):
- print(args[0])
- os.system(args[0])
-
-def create_job(*args, cfg_update=dict(), with_slurm=True, **kwargs):
- """Shortcut to slurmpy's class; keep certain default kwargs
- and only update some with kwarg `cfg_update`
- see https://github.com/brentp/slurmpy
-
- with_slurm (bool) : if True, use SLURM, else run locally
-
- """
- if with_slurm:
- return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update),
- log_dir=log_dir, scripts_dir=slurm_scripts_dir, **kwargs)
- else:
- return Shellslurm(*args)
-
-def backup_scripts():
- """Copies scripts and configuration to archive dir output folder"""
- os.makedirs(cluster.archivedir, exist_ok=True)
-
- try:
- shutil.copytree(cluster.scriptsdir, cluster.scripts_rundir)
- except FileExistsError:
- pass
- except:
- raise
- try:
- copy(os.path.basename(__file__), cluster.scripts_rundir+'/')
- except Exception as e:
- warnings.warn(str(e))
-
-def prepare_WRFrundir(init_time):
- """Create WRF/run directories and wrfinput files
- """
- cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
- print(cmd)
- os.system(cmd)
-
--
GitLab