Skip to content
Snippets Groups Projects
Commit b785f29f authored by lkugler's avatar lkugler
Browse files

utils

parent 5868fa84
Branches
No related tags found
No related merge requests found
......@@ -5,7 +5,7 @@ running the forecast model without assimilation
import os, sys, shutil
import datetime as dt
from dartwrf import utils
from dartwrf.utils_workflow import backup_scripts
from config.cfg import exp, cluster
log_dir = cluster.archivedir+'/logs/'
......@@ -15,7 +15,7 @@ print('scripts, which are submitted to SLURM:', slurm_scripts_dir)
###############################
utils.backup_scripts()
backup_scripts()
prior_path_exp = '/mnt/jetfs/scratch/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_noDA'
......
from dartwrf import utils
from dartwrf.utils_workflow import ExperimentConfiguration
from config import clusters # from . = problem in archivedir
cluster = clusters.jet # change cluster configuration here
cluster = clusters.srvx1 # change cluster configuration here
exp = utils.ExperimentConfiguration()
exp = ExperimentConfiguration()
exp.expname = "test_srvx1" #"exp_v1.22_P3_wbub7_WV62_obs10_loc20_oe1"
exp.model_dx = 2000
exp.n_ens = 40
......@@ -24,7 +24,7 @@ exp.use_existing_obsseq = False # False or pathname (use precomputed obs_seq.ou
#exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_P5+su_nat2/2008-07-30_07:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
#exp.nature_wrfout = '/jetfs/home/lkugler/data/sim_archive/exp_v1.19_P3_wbub7_nat/2008-07-30_12:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
#exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_Pwbub5_nat/2008-07-30_12:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
exp.nature_wrfout = '/jetfs/home/lkugler/data/sim_archive/exp_v1.18_P1_nature/2008-07-30_06:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
exp.nature_wrfout = cluster.archive_base+'/exp_v1.18_P1_nature/2008-07-30_06:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
#exp.nature_wrfout = '/home/fs71386/lkugler/data/sim_archive/exp_v1.19_P4_nat/2008-07-30_07:00/1/wrfout_d01_%Y-%m-%d_%H:%M:%S'
exp.input_profile = '/jetfs/home/lkugler/data/initial_profiles/wrf/ens/2022-03-31/raso.fc.<iens>.wrfprof'
......
import os, sys
import datetime as dt
from dartwrf import utils
from dartwrf.utils_workflow import ClusterConfig
"""Configuration name docs
......@@ -43,8 +43,7 @@ slurm_cfg python dictionary, containing options of SLURM
vsc = utils.ClusterConfig()
vsc = ClusterConfig()
vsc.name = 'vsc'
vsc.max_nproc = 20
vsc.size_jobarray = 10 # 10 jobs with each 4 WRF processes per node
......@@ -77,7 +76,7 @@ vsc.slurm_cfg = {"account": "p71386", "partition": "skylake_0384", "qos": "p7138
"nodes": "1", "ntasks": "1", "ntasks-per-node": "48", "ntasks-per-core": "1",
"mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
jet = utils.ClusterConfig()
jet = ClusterConfig()
jet.name = 'jet'
jet.max_nproc = 12
jet.use_slurm = True
......@@ -113,36 +112,31 @@ jet.slurm_cfg = {"account": "lkugler", "partition": "compute", #"nodelist": "jet
"mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
srvx1 = utils.ClusterConfig()
srvx1 = ClusterConfig()
srvx1.name = 'srvx1'
srvx1.max_nproc = 6
srvx1.size_jobarray = 40
srvx1.use_slurm = False
# binaries
srvx1.python = '/mnt/jetfs/home/lkugler/miniconda3/envs/DART/bin/python'
srvx1.python_verif = '/jetfs/home/lkugler/miniconda3/envs/enstools/bin/python'
srvx1.ncks = '/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/nco-4.9.3-dhlqiyog7howjmaleyfhm6lkt7ra37xf/bin/ncks'
srvx1.ideal = '/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
srvx1.wrfexe = '/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
srvx1.python = '/users/staff/lkugler/miniconda3/bin/python'
srvx1.python_verif = '/users/staff/lkugler/miniconda3/bin/python'
srvx1.ncks = '/home/swd/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/nco-5.0.1-ntu44aoxlvwtr2tsrobfr4lht7cpvccf/bin/ncks'
srvx1.ideal = '' #/jetfs/home/lkugler/bin/ideal-v4.3_v1.22.exe'
srvx1.wrfexe = '' #/jetfs/home/lkugler/bin/wrf-v4.3_v1.22.exe'
srvx1.container = ''
# paths for data output
srvx1.wrf_rundir_base = '/jetfs/home/lkugler/data/run_WRF/' # path for temporary files
srvx1.dart_rundir_base = '/jetfs/home/lkugler/data/run_DART/' # path for temporary files
srvx1.wrf_rundir_base = '/mnt/jetfs/home/lkugler/data/run_WRF/' # path for temporary files
srvx1.dart_rundir_base = '/users/staff/lkugler/AdvDA23/run_DART/' # path for temporary files
srvx1.archive_base = '/mnt/jetfs/scratch/lkugler/data/sim_archive/'
# paths used as input
srvx1.srcdir = '/users/staff/lkugler/AdvDA23/DART/WRF-4.3/run'
srvx1.dart_srcdir = '/users/staff/lkugler/AdvDA23/DART/models/wrf/work'
srvx1.rttov_srcdir = '/users/staff/lkugler/AdvDA23/RTTOV13/rtcoef_rttov13/'
srvx1.scriptsdir = '/jetfs/home/lkugler/DART-WRF/dartwrf/'
srvx1.scriptsdir = '/users/staff/lkugler/AdvDA23/DART-WRF/dartwrf/'
srvx1.geo_em = '/mnt/jetfs/scratch/lkugler/data/geo_em.d01.nc'
# templates/run scripts
srvx1.namelist = srvx1.scriptsdir+'/../templates/namelist.input'
srvx1.run_WRF = srvx1.scriptsdir+'/run_ens.jet.sh'
srvx1.slurm_cfg = {"account": "lkugler", "partition": "compute",
"ntasks": "1", "ntasks-per-core": "1", "mem": "50G",
"mail-type": "FAIL", "mail-user": "lukas.kugler@univie.ac.at"}
......@@ -6,7 +6,8 @@ submitting jobs into SLURM queue
import os, sys, shutil, glob, warnings
import datetime as dt
from dartwrf.utils import script_to_str, symlink, copy, create_job
from utils import create_job
from dartwrf.utils import script_to_str, symlink, copy
from config.cfg import exp, cluster
log_dir = cluster.archivedir+'/logs/'
......
......@@ -2,77 +2,6 @@ import os, sys, shutil, glob, warnings
import builtins as __builtin__
import subprocess
import datetime as dt
from slurmpy import Slurm
from config.cfg import cluster
class ExperimentConfiguration(object):
"""Collection of variables to use in code later on"""
def __init__(self):
pass
class ClusterConfig(object):
"""Collection of variables to use in code later on"""
def __init__(self):
pass
@property
def archivedir(self):
return self.archive_base+'/'+self.expname
def wrf_rundir(self, iens):
return self.wrf_rundir_base+'/'+self.expname+'/'+str(iens)
@property
def scripts_rundir(self):
return self.archivedir+'/DART-WRF/'
@property
def dartrundir(self):
return self.dart_rundir_base+'/'+self.expname+'/'
class Shellslurm():
"""Like Slurmpy class, but runs locally"""
def __init__(self, *args, **kwargs):
pass
def run(self, *args, **kwargs):
print(args[0])
os.system(args[0])
def create_job(*args, cfg_update=dict(), **kwargs):
"""Shortcut to slurmpy's class; keep certain default kwargs
and only update some with kwarg `cfg_update`
see https://github.com/brentp/slurmpy
with_slurm (bool) : if True, use SLURM, else run locally
"""
if cluster.use_slurm:
return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update),
log_dir=log_dir, scripts_dir=slurm_scripts_dir, **kwargs)
else:
return Shellslurm(*args)
def backup_scripts():
"""Copies scripts and configuration to archive dir output folder"""
os.makedirs(cluster.archivedir, exist_ok=True)
try:
shutil.copytree(cluster.scriptsdir, cluster.scripts_rundir)
except FileExistsError:
pass
except:
raise
try:
copy(os.path.basename(__file__), cluster.scripts_rundir+'/')
except Exception as e:
warnings.warn(str(e))
def prepare_WRFrundir(init_time):
"""Create WRF/run directories and wrfinput files
"""
cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
print(cmd)
os.system(cmd)
def shell(args):
print(args)
......
import os, sys, shutil, glob, warnings
import subprocess
copy = shutil.copy
class ExperimentConfiguration(object):
"""Collection of variables to use in code later on"""
def __init__(self):
pass
class ClusterConfig(object):
"""Collection of variables to use in code later on"""
def __init__(self):
pass
@property
def archivedir(self):
return self.archive_base+'/'+self.expname
def wrf_rundir(self, iens):
return self.wrf_rundir_base+'/'+self.expname+'/'+str(iens)
@property
def scripts_rundir(self):
return self.archivedir+'/DART-WRF/'
@property
def dartrundir(self):
return self.dart_rundir_base+'/'+self.expname+'/'
class Shellslurm():
"""Like Slurmpy class, but runs locally"""
def __init__(self, *args, **kwargs):
pass
def run(self, *args, **kwargs):
print(args[0])
os.system(args[0])
def create_job(*args, cfg_update=dict(), **kwargs):
"""Shortcut to slurmpy's class; keep certain default kwargs
and only update some with kwarg `cfg_update`
see https://github.com/brentp/slurmpy
with_slurm (bool) : if True, use SLURM, else run locally
"""
from config.cfg import cluster
if cluster.use_slurm:
from slurmpy import Slurm
return Slurm(*args, slurm_kwargs=dict(cluster.slurm_cfg, **cfg_update),
log_dir=log_dir, scripts_dir=slurm_scripts_dir, **kwargs)
else:
return Shellslurm(*args)
def backup_scripts():
"""Copies scripts and configuration to archive dir output folder"""
from config.cfg import cluster
os.makedirs(cluster.archivedir, exist_ok=True)
try:
shutil.copytree(cluster.scriptsdir, cluster.scripts_rundir)
shutil.copytree(cluster.scriptsdir+'/../', cluster.scripts_rundir)
except FileExistsError:
pass
except:
raise
def prepare_WRFrundir(init_time):
"""Create WRF/run directories and wrfinput files
"""
from config.cfg import cluster
cmd = cluster.python+' '+cluster.scripts_rundir+'/prepare_wrfrundir.py '+init_time.strftime('%Y-%m-%d_%H:%M')
print(cmd)
os.system(cmd)
\ No newline at end of file
......@@ -23,4 +23,4 @@ dependencies:
- wrf-python
- xarray
- yaml
- pysolar
- pysolar>=0.10.0
......@@ -8,7 +8,8 @@ import pandas as pd
from slurmpy import Slurm
from config.cfg import exp, cluster
from dartwrf.utils import script_to_str, symlink, backup_scripts
from utils import backup_scripts
from dartwrf.utils import script_to_str, symlink
from cycled_exp import *
log_dir = cluster.archivedir+'/logs/'
......
......@@ -21,7 +21,7 @@ def read_requirements(fname):
setuptools.setup(
name="dartwrf",
version="2022.11.02",
version="2023.1.31",
author="Lukas Kugler",
author_email="lukas.kugler@univie.ac.at",
description="Observing system simulation experiments with WRF and DART",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment