Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
D
DART-WRF-delete_byApril
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Deploy
Model registry
Monitor
Service Desk
Analyze
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
DataAssimilation
DART-WRF-delete_byApril
Commits
b29d4caf
Commit
b29d4caf
authored
2 years ago
by
lkugler
Browse files
Options
Downloads
Patches
Plain Diff
initialize exp workflow
parent
db365496
No related branches found
No related tags found
No related merge requests found
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
dartwrf/workflows.py
+35
-21
35 additions, 21 deletions
dartwrf/workflows.py
with
35 additions
and
21 deletions
dartwrf/workflows.py
+
35
−
21
View file @
b29d4caf
...
@@ -15,22 +15,27 @@ class WorkFlows(object):
...
@@ -15,22 +15,27 @@ class WorkFlows(object):
def
__init__
(
self
,
exp_config
=
'
cfg.py
'
,
server_config
=
'
server.py
'
):
def
__init__
(
self
,
exp_config
=
'
cfg.py
'
,
server_config
=
'
server.py
'
):
"""
Set up the experiment folder in `archivedir`.
"""
Set up the experiment folder in `archivedir`.
1. Copy the DART-WRF scripts to `archivedir`
2. Copy the config files to `archivedir/dartwrf`
3. Set log paths
4. Write obskind.py file (dictionary of observation types)
we load the config from load the config from cluster.scripts_rundir/config/cfg.py
Args:
Args:
exp (str): Path to exp config file
exp
_config
(str): Path to exp config file
config (str): Path to the cluster config file
server_
config (str): Path to the cluster config file
Attributes:
Attributes:
cluster (obj): cluster configuration as defined in server_config file
cluster (obj): cluster configuration as defined in server_config file
exp (obj): experiment configuration as defined in exp_config file
we load the config from load the config from cluster.scripts_rundir/config/cfg.py
"""
"""
def
copy_dartwrf_to_archive
():
def
copy_dartwrf_to_archive
():
# Copy scripts to self.cluster.archivedir folder
# Copy scripts to self.cluster.archivedir folder
os
.
makedirs
(
self
.
cluster
.
archivedir
,
exist_ok
=
True
)
try
:
try
:
shutil
.
copytree
(
self
.
cluster
.
dartwrf_dir
,
self
.
cluster
.
scripts_rundir
)
shutil
.
copytree
(
self
.
cluster
.
dartwrf_dir
,
self
.
cluster
.
archivedir
+
'
/DART-WRF/
'
)
print
(
'
scripts
ha
ve
been copied to
'
,
self
.
cluster
.
archivedir
)
print
(
'
DART-WRF
ha
s
been copied to
'
,
self
.
cluster
.
archivedir
)
except
FileExistsError
as
e
:
except
FileExistsError
as
e
:
warnings
.
warn
(
str
(
e
))
warnings
.
warn
(
str
(
e
))
if
input
(
'
The experiment name already exists! Scripts will not be overwritten. Continue? (Y/n)
'
)
in
[
'
Y
'
,
'
y
'
]:
if
input
(
'
The experiment name already exists! Scripts will not be overwritten. Continue? (Y/n)
'
)
in
[
'
Y
'
,
'
y
'
]:
...
@@ -40,33 +45,42 @@ class WorkFlows(object):
...
@@ -40,33 +45,42 @@ class WorkFlows(object):
except
:
except
:
raise
raise
def
copy_config_to_archive
():
#
def copy_config_to_archive():
os
.
makedirs
(
self
.
cluster
.
scripts_rundir
+
'
/config/
'
,
exist_ok
=
True
)
#
os.makedirs(self.cluster.scripts_rundir+'/config/', exist_ok=True)
# later, we can load the exp cfg with `from config.cfg import exp`
#
# later, we can load the exp cfg with `from config.cfg import exp`
shutil
.
copyfile
(
'
config/
'
+
exp_config
,
self
.
cluster
.
scripts_rundir
+
'
/config/cfg.py
'
)
#
shutil.copyfile('config/'+exp_config, self.cluster.scripts_rundir+'/config/cfg.py')
# later, we can load the cluster cfg with `from config.cluster import cluster`
#
# later, we can load the cluster cfg with `from config.cluster import cluster`
shutil
.
copyfile
(
'
config/
'
+
server_config
,
self
.
cluster
.
scripts_rundir
+
'
/config/cluster.py
'
)
# whatever server, the config name is always the same!
#
shutil.copyfile('config/'+server_config, self.cluster.scripts_rundir+'/config/cluster.py') # whatever server, the config name is always the same!
print
(
'
------ start exp from
'
,
exp_config
,
'
and
'
,
server_config
,
'
------
'
)
print
(
'
------ start exp from
'
,
exp_config
,
'
and
'
,
server_config
,
'
------
'
)
# copy config file to current config folder
# experiment starts, we dont know where the code shall run
# => read the configuration file
# copy the config files to this folder
this_dir
=
'
/
'
.
join
(
__file__
.
split
(
'
/
'
)[:
-
1
])
try
:
try
:
shutil
.
copyfile
(
'
config/
'
+
exp_config
,
'
/
'
.
join
(
__file__
.
split
(
'
/
'
)[:
-
2
])
+
'
/config/cfg.py
'
)
shutil
.
copyfile
(
'
config/
'
+
server_config
,
this_dir
+
'
/server_config.py
'
)
except
shutil
.
SameFileError
:
pass
try
:
shutil
.
copyfile
(
'
config/
'
+
exp_config
,
this_dir
+
'
/exp_config.py
'
)
except
shutil
.
SameFileError
:
except
shutil
.
SameFileError
:
pass
pass
# load python config files
sys
.
path
.
append
(
this_dir
)
self
.
cluster
=
importlib
.
import_module
(
'
config.
'
+
server_config
.
strip
(
'
.py
'
)).
cluster
from
server_config
import
cluster
self
.
exp
=
importlib
.
import_module
(
'
config.
'
+
exp_config
.
strip
(
'
.py
'
)).
exp
self
.
cluster
=
cluster
from
exp_config
import
exp
self
.
exp
=
exp
copy_dartwrf_to_archive
()
# includes config files
# we set the path from where python should import dartwrf modules
# we set the path from where python should import dartwrf modules
self
.
cluster
.
python
=
'
export PYTHONPATH=
'
+
self
.
cluster
.
scripts_rundir
+
'
;
'
+
self
.
cluster
.
python
self
.
cluster
.
python
=
'
export PYTHONPATH=
'
+
self
.
cluster
.
scripts_rundir
+
'
;
'
+
self
.
cluster
.
python
copy_dartwrf_to_archive
()
copy_config_to_archive
()
# Set paths and backup scripts
# Set paths and backup scripts
self
.
cluster
.
log_dir
=
self
.
cluster
.
archivedir
+
'
/logs/
'
self
.
cluster
.
log_dir
=
self
.
cluster
.
archivedir
+
'
/logs/
'
print
(
'
logging to
'
,
self
.
cluster
.
log_dir
)
print
(
'
logging to
'
,
self
.
cluster
.
log_dir
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment