utils

analysis_only.py

@@ -5,7 +5,7 @@ running the forecast model without assimilation
 import os, sys, shutil
 import datetime as dt
 
-from dartwrf import utils
+from dartwrf.utils_workflow import backup_scripts
 from config.cfg import exp, cluster
 
 log_dir = cluster.archivedir+'/logs/'
@@ -15,7 +15,7 @@ print('scripts, which are submitted to SLURM:', slurm_scripts_dir)
 
 
 ###############################
-utils.backup_scripts()
+backup_scripts()
 
 
 prior_path_exp = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_noDA'

config/cfg.py

-from dartwrf import utils
+from dartwrf.utils_workflow import ExperimentConfiguration
 from config import clusters  # from . = problem in archivedir
-cluster = clusters.jet  # change cluster configuration here
+cluster = clusters.srvx1  # change cluster configuration here
 
-exp = utils.ExperimentConfiguration()
+exp = ExperimentConfiguration()
 exp.expname = "test_srvx1" #"exp_v1.22_P3_wbub7_WV62_obs10_loc20_oe1"
 exp.model_dx = 2000
 exp.n_ens = 40
@@ -24,7 +24,7 @@ exp.use_existing_obsseq = False  # False or pathname (use precomputed obs_seq.ou
 #exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_P5+su_nat2/2008-07-30_07:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
 #exp.nature_wrfout = '/jetfs/home/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_nat/2008-07-30_12:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
 #exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_Pwbub5_nat/2008-07-30_12:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
-exp.nature_wrfout = '/jetfs/home/lkugler/data/sim_archive/exp_v1.18_P1_nature/2008-07-30_06:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
+exp.nature_wrfout = cluster.archive_base+'/exp_v1.18_P1_nature/2008-07-30_06:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
 #exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_P4_nat/2008-07-30_07:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
 
 exp.input_profile = '/jetfs/home/lkugler/data/initial_profiles/wrf/ens/2022-03-31/raso.fc.<iens>.wrfprof'

config/clusters.py

 import os, sys
 import datetime as dt
-from dartwrf import utils
+from dartwrf.utils_workflow import ClusterConfig
 
 """Configuration name docs
 
@@ -43,8 +43,7 @@ slurm_cfg               python dictionary, containing options of SLURM
 
 
 
-
-vsc = utils.ClusterConfig()
+vsc = ClusterConfig()
 vsc.name = 'vsc' 
 vsc.max_nproc = 20
 vsc.size_jobarray = 10  # 10 jobs with each 4 WRF processes per node
@@ -77,7 +76,7 @@ vsc.slurm_cfg = {"account": "p71386", "partition": "skylake_0384", "qos": "p7138
                  "nodes": "1", "ntasks": "1", "ntasks-per-node": "48", "ntasks-per-core": "1",
                  "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
 
-jet = utils.ClusterConfig()
+jet = ClusterConfig()
 jet.name = 'jet'
 jet.max_nproc = 12
 jet.use_slurm = True
@@ -113,36 +112,31 @@ jet.slurm_cfg = {"account": "lkugler", "partition": "compute", #"nodelist": "jet
                  "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
 
 
-srvx1 = utils.ClusterConfig()
+srvx1 = ClusterConfig()
 srvx1.name = 'srvx1'
 srvx1.max_nproc = 6
-srvx1.size_jobarray = 40
 srvx1.use_slurm = False
 
 # binaries
-srvx1.python = '/mnt/jetfs/home/lkugler/miniconda3/envs/DART/bin/python'
-srvx1.python_verif = '/jetfs/home/lkugler/miniconda3/envs/enstools/bin/python'
-srvx1.ncks = '/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/nco-4.9.3-dhlqiyog7howjmaleyfhm6lkt7ra37xf/bin/ncks'
-srvx1.ideal = '/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
-srvx1.wrfexe = '/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
+srvx1.python = '/users/staff/lkugler/miniconda3/bin/python'
+srvx1.python_verif = '/users/staff/lkugler/miniconda3/bin/python'
+srvx1.ncks = '/home/swd/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/nco-5.0.1-ntu44aoxlvwtr2tsrobfr4lht7cpvccf/bin/ncks'
+srvx1.ideal = '' #/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
+srvx1.wrfexe = '' #/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
 srvx1.container = ''
 
 # paths for data output
-srvx1.wrf_rundir_base = '/jetfs/home/lkugler/data/run_WRF/'  # path for temporary files
-srvx1.dart_rundir_base = '/jetfs/home/lkugler/data/run_DART/'  # path for temporary files
+srvx1.wrf_rundir_base = '/mnt/jetfs/home/lkugler/data/run_WRF/'  # path for temporary files
+srvx1.dart_rundir_base = '/users/staff/lkugler/AdvDA23/run_DART/'  # path for temporary files
 srvx1.archive_base = '/mnt/jetfs/scratch/lkugler/data/sim_archive/'
 
 # paths used as input
 srvx1.srcdir = '/users/staff/lkugler/AdvDA23/DART/WRF-4.3/run'
 srvx1.dart_srcdir = '/users/staff/lkugler/AdvDA23/DART/models/wrf/work'
 srvx1.rttov_srcdir = '/users/staff/lkugler/AdvDA23/RTTOV13/rtcoef_rttov13/'
-srvx1.scriptsdir = '/jetfs/home/lkugler/DART-WRF/dartwrf/'
+srvx1.scriptsdir = '/users/staff/lkugler/AdvDA23/DART-WRF/dartwrf/'
 srvx1.geo_em = '/mnt/jetfs/scratch/lkugler/data/geo_em.d01.nc'
 
 # templates/run scripts
 srvx1.namelist = srvx1.scriptsdir+'/../templates/namelist.input'
 srvx1.run_WRF = srvx1.scriptsdir+'/run_ens.jet.sh'
-
-srvx1.slurm_cfg = {"account": "lkugler", "partition": "compute",
-                 "ntasks": "1", "ntasks-per-core": "1", "mem": "50G",
-                 "mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}

cycled_exp.py

@@ -6,7 +6,8 @@ submitting jobs into SLURM queue
 import os, sys, shutil, glob, warnings
 import datetime as dt
 
-from dartwrf.utils import script_to_str, symlink, copy, create_job
+from utils import create_job
+from dartwrf.utils import script_to_str, symlink, copy
 from config.cfg import exp, cluster
 
 log_dir = cluster.archivedir+'/logs/'

dartwrf/utils.py

@@ -2,77 +2,6 @@ import os, sys, shutil, glob, warnings
 import builtins as __builtin__
 import subprocess
 import datetime as dt
-from slurmpy import Slurm
-from config.cfg import cluster
-
-class ExperimentConfiguration(object):
-    """Collection of variables to use in code later on"""
-    def __init__(self):
-        pass
-
-class ClusterConfig(object):
-    """Collection of variables to use in code later on"""
-    def __init__(self):
-        pass
-
-    @property
-    def archivedir(self):
-        return self.archive_base+'/'+self.expname
-
-    def wrf_rundir(self, iens):
-        return self.wrf_rundir_base+'/'+self.expname+'/'+str(iens)
-
-    @property
-    def scripts_rundir(self):
-        return self.archivedir+'/DART-WRF/'
-
-    @property
-    def dartrundir(self):
-        return self.dart_rundir_base+'/'+self.expname+'/'
-
-class Shellslurm():
-    """Like Slurmpy class, but runs locally"""
-    def __init__(self, *args, **kwargs):
-        pass
-    def run(self, *args, **kwargs):
-        print(args[0])
-        os.system(args[0])
-
-def create_job(*args, cfg_update=dict(), **kwargs):
-    """Shortcut to slurmpy's class; keep certain default kwargs
-    and only update some with kwarg `cfg_update`
-    see https://github.com/brentp/slurmpy
-
-    with_slurm (bool) : if True, use SLURM, else run locally
-
-    """
-    if cluster.use_slurm:
-        return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update), 
-                 log_dir=log_dir, scripts_dir=slurm_scripts_dir, **kwargs)
-    else:
-        return Shellslurm(*args)
-
-def backup_scripts():
-    """Copies scripts and configuration to archive dir output folder"""
-    os.makedirs(cluster.archivedir, exist_ok=True)
-
-    try:
-        shutil.copytree(cluster.scriptsdir, cluster.scripts_rundir)
-    except FileExistsError:
-        pass
-    except:
-        raise
-    try:
-        copy(os.path.basename(__file__), cluster.scripts_rundir+'/')
-    except Exception as e:
-        warnings.warn(str(e))
-
-def prepare_WRFrundir(init_time):
-    """Create WRF/run directories and wrfinput files
-    """
-    cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
-    print(cmd)
-    os.system(cmd)
 
 def shell(args):
     print(args)

dartwrf/utils_workflow.py

+import os, sys, shutil, glob, warnings
+import subprocess
+copy = shutil.copy
+
+class ExperimentConfiguration(object):
+    """Collection of variables to use in code later on"""
+    def __init__(self):
+        pass
+
+class ClusterConfig(object):
+    """Collection of variables to use in code later on"""
+    def __init__(self):
+        pass
+
+    @property
+    def archivedir(self):
+        return self.archive_base+'/'+self.expname
+
+    def wrf_rundir(self, iens):
+        return self.wrf_rundir_base+'/'+self.expname+'/'+str(iens)
+
+    @property
+    def scripts_rundir(self):
+        return self.archivedir+'/DART-WRF/'
+
+    @property
+    def dartrundir(self):
+        return self.dart_rundir_base+'/'+self.expname+'/'
+
+class Shellslurm():
+    """Like Slurmpy class, but runs locally"""
+    def __init__(self, *args, **kwargs):
+        pass
+    def run(self, *args, **kwargs):
+        print(args[0])
+        os.system(args[0])
+
+
+def create_job(*args, cfg_update=dict(), **kwargs):
+    """Shortcut to slurmpy's class; keep certain default kwargs
+    and only update some with kwarg `cfg_update`
+    see https://github.com/brentp/slurmpy
+
+    with_slurm (bool) : if True, use SLURM, else run locally
+
+    """
+    from config.cfg import cluster
+    if cluster.use_slurm:
+        from slurmpy import Slurm
+        return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update), 
+                 log_dir=log_dir, scripts_dir=slurm_scripts_dir, **kwargs)
+    else:
+        return Shellslurm(*args)
+
+def backup_scripts():
+    """Copies scripts and configuration to archive dir output folder"""
+    from config.cfg import cluster
+    os.makedirs(cluster.archivedir, exist_ok=True)
+
+    try:
+        shutil.copytree(cluster.scriptsdir, cluster.scripts_rundir)
+        shutil.copytree(cluster.scriptsdir+'/../', cluster.scripts_rundir)
+    except FileExistsError:
+        pass
+    except:
+        raise
+
+def prepare_WRFrundir(init_time):
+    """Create WRF/run directories and wrfinput files
+    """
+    from config.cfg import cluster
+    cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
+    print(cmd)
+    os.system(cmd)
\ No newline at end of file

env-minimal.yml

@@ -23,4 +23,4 @@ dependencies:
   - wrf-python
   - xarray
   - yaml
-  - pysolar
+  - pysolar>=0.10.0

generate_free.py

@@ -8,7 +8,8 @@ import pandas as pd
 from slurmpy import Slurm
 
 from config.cfg import exp, cluster
-from dartwrf.utils import script_to_str, symlink, backup_scripts
+from utils import backup_scripts
+from dartwrf.utils import script_to_str, symlink
 from cycled_exp import *
 
 log_dir = cluster.archivedir+'/logs/'

setup.py

@@ -21,7 +21,7 @@ def read_requirements(fname):
 
 setuptools.setup(
     name="dartwrf",
-    version="2022.11.02",
+    version="2023.1.31",
     author="Lukas Kugler",
     author_email="lukas.kugler@univie.ac.at",
     description="Observing system simulation experiments with WRF and DART",