Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
D
DART-WRF
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Package registry
Model registry
Operate
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
DataAssimilation
DART-WRF
Commits
f14b7b67
Commit
f14b7b67
authored
5 months ago
by
Lukas Kugler
Browse files
Options
Downloads
Patches
Plain Diff
symlink did not overwrite existing links
parent
1516ad11
No related branches found
No related tags found
No related merge requests found
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
dartwrf/utils.py
+77
-47
77 additions, 47 deletions
dartwrf/utils.py
with
77 additions
and
47 deletions
dartwrf/utils.py
+
77
−
47
View file @
f14b7b67
import
os
,
sys
,
shutil
,
glob
,
warnings
"""
Utility functions for DART-WRF
Caution: You can not use the configuration files in here,
because loading this would lead to a circular import.
"""
import
os
import
sys
import
shutil
import
glob
import
warnings
import
builtins
as
__builtin__
import
builtins
as
__builtin__
import
subprocess
import
subprocess
import
datetime
as
dt
import
datetime
as
dt
import
re
,
tempfile
import
re
import
tempfile
# class Stage(object):
import
pickle
# """Collection of variables describing the assimilation stage"""
# def __init__(self, **kwargs):
# self.superob_km = False # False or int (spatial averaging of observations)
# self.use_existing_obsseq = False # False or pathname (use precomputed obs_seq.out files)
# self.__dict__.update(kwargs)
# # raise ValueError if attributes are not set
# needed_attributes = ['observations', 'dart_nml',]
# for attr in needed_attributes:
# if not hasattr(self, attr):
# raise ValueError('Stage.'+attr+' is not set')
class
Experiment
(
object
):
class
Experiment
(
object
):
"""
Collection of variables which define the experiment
"""
Collection of variables which define the experiment
...
@@ -51,9 +51,11 @@ class Experiment(object):
...
@@ -51,9 +51,11 @@ class Experiment(object):
values are dictionaries of parameters and values (e.g. dict(ens_size=exp.n_ens,))
values are dictionaries of parameters and values (e.g. dict(ens_size=exp.n_ens,))
"""
"""
def
__init__
(
self
):
def
__init__
(
self
):
pass
pass
class
ClusterConfig
(
object
):
class
ClusterConfig
(
object
):
"""
Collection of variables regarding the cluster configuration
"""
Collection of variables regarding the cluster configuration
...
@@ -89,7 +91,8 @@ class ClusterConfig(object):
...
@@ -89,7 +91,8 @@ class ClusterConfig(object):
rttov_srcdir (str): Path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
rttov_srcdir (str): Path to RTTOV compile directory, e.g. /home/RTTOV13/rtcoef_rttov13/
dartwrf_dir (str): Path where DART-WRF scripts reside, e.g. /home/DART-WRF/
dartwrf_dir (str): Path where DART-WRF scripts reside, e.g. /home/DART-WRF/
geo_em_for_WRF_ideal (str, False): Path to the geo_em.d01.nc file for idealized nature runs
geo_em_nature (str, False): Path to the geo_em.d01.nc file for idealized nature runs
geo_em_forecast (str, False): Path to the geo_em.d01.nc file for the forecast domain
obs_impact_filename
obs_impact_filename
namelist (str): Path to a WRF namelist template;
namelist (str): Path to a WRF namelist template;
strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
strings like <hist_interval>, will be overwritten in scripts/prepare_namelist.py
...
@@ -100,6 +103,7 @@ class ClusterConfig(object):
...
@@ -100,6 +103,7 @@ class ClusterConfig(object):
This configuration can be customized for any job (e.g. in workflows.py)
This configuration can be customized for any job (e.g. in workflows.py)
"""
"""
def
__init__
(
self
,
exp
):
def
__init__
(
self
,
exp
):
self
.
exp
=
exp
# to access the experiment config in here
self
.
exp
=
exp
# to access the experiment config in here
...
@@ -170,16 +174,25 @@ class ClusterConfig(object):
...
@@ -170,16 +174,25 @@ class ClusterConfig(object):
if
returncode
!=
0
:
if
returncode
!=
0
:
raise
Exception
(
'
Error running command >>>
'
+
cmd
)
raise
Exception
(
'
Error running command >>>
'
+
cmd
)
userhome
=
os
.
path
.
expanduser
(
'
~
'
)
userhome
=
os
.
path
.
expanduser
(
'
~
'
)
def
shell
(
args
):
def
shell
(
args
,
pythonpath
=
None
):
print
(
args
)
print
(
args
)
#subprocess.run(args.split(' ')) #, shell=True) #, stderr=subprocess.STDOUT)
os
.
system
(
args
)
return
os
.
system
(
args
)
# if pythonpath:
# env = os.environ.copy()
# env['PYTHONPATH'] = pythonpath
# subprocess.check_output(args.split(' '), env=env)
# else:
# subprocess.check_output(args.split(' '))
def
print
(
*
args
):
def
print
(
*
args
):
__builtin__
.
print
(
*
args
,
flush
=
True
)
__builtin__
.
print
(
*
args
,
flush
=
True
)
def
copy
(
src
,
dst
,
remove_if_exists
=
True
):
def
copy
(
src
,
dst
,
remove_if_exists
=
True
):
if
src
==
dst
:
if
src
==
dst
:
return
# the link already exists, nothing to do
return
# the link already exists, nothing to do
...
@@ -190,40 +203,44 @@ def copy(src, dst, remove_if_exists=True):
...
@@ -190,40 +203,44 @@ def copy(src, dst, remove_if_exists=True):
pass
pass
shutil
.
copy
(
src
,
dst
)
shutil
.
copy
(
src
,
dst
)
def
try_remove
(
f
):
def
try_remove
(
f
):
try
:
try
:
os
.
remove
(
f
)
os
.
remove
(
f
)
except
:
except
:
pass
pass
def
mkdir
(
path
):
def
mkdir
(
path
):
os
.
system
(
'
mkdir -p
'
+
path
)
os
.
system
(
'
mkdir -p
'
+
path
)
def
script_to_str
(
path
):
def
script_to_str
(
path
):
return
open
(
path
,
'
r
'
).
read
()
return
open
(
path
,
'
r
'
).
read
()
def
copy_contents
(
src
,
dst
):
def
copy_contents
(
src
,
dst
):
os
.
system
(
'
cp -rf
'
+
src
+
'
/*
'
+
dst
+
'
/
'
)
os
.
system
(
'
cp -rf
'
+
src
+
'
/*
'
+
dst
+
'
/
'
)
def
clean_wrfdir
(
dir
):
def
clean_wrfdir
(
dir
):
for
s
in
[
'
wrfout_*
'
,
'
rsl.*
'
,
'
wrfrst_*
'
]:
for
s
in
[
'
wrfout_*
'
,
'
rsl.*
'
,
'
wrfrst_*
'
]:
for
f
in
glob
.
glob
(
dir
+
'
/
'
+
s
):
for
f
in
glob
.
glob
(
dir
+
'
/
'
+
s
):
os
.
remove
(
f
)
os
.
remove
(
f
)
def
symlink
(
src
,
dst
):
def
symlink
(
src
,
dst
):
"""
Create a symbolic link from src to dst
"""
Create a symbolic link from src to dst
Creates the folder if it does not exist
"""
"""
try
:
try
:
# this file may not exist
os
.
symlink
(
src
,
dst
)
except
FileExistsError
:
# print('file exists')
if
os
.
path
.
realpath
(
dst
)
==
src
:
pass
# print('link is correct')
else
:
os
.
remove
(
dst
)
os
.
remove
(
dst
)
except
OSError
:
pass
os
.
makedirs
(
os
.
path
.
dirname
(
dst
),
exist_ok
=
True
)
os
.
symlink
(
src
,
dst
)
os
.
symlink
(
src
,
dst
)
except
Exception
as
e
:
raise
e
def
link_contents
(
src
,
dst
):
def
link_contents
(
src
,
dst
):
"""
Create symbolic links for all files in src to dst
"""
Create symbolic links for all files in src to dst
...
@@ -238,6 +255,7 @@ def link_contents(src, dst):
...
@@ -238,6 +255,7 @@ def link_contents(src, dst):
for
f
in
os
.
listdir
(
src
):
for
f
in
os
.
listdir
(
src
):
symlink
(
src
+
'
/
'
+
f
,
dst
+
'
/
'
+
f
)
symlink
(
src
+
'
/
'
+
f
,
dst
+
'
/
'
+
f
)
def
sed_inplace
(
filename
,
pattern
,
repl
):
def
sed_inplace
(
filename
,
pattern
,
repl
):
'''
Perform the pure-Python equivalent of in-place `sed` substitution
'''
Perform the pure-Python equivalent of in-place `sed` substitution
Like `sed -i -e
'
s/
'
${pattern}
'
/
'
${repl}
'
"
${filename}
"
`.
Like `sed -i -e
'
s/
'
${pattern}
'
/
'
${repl}
'
"
${filename}
"
`.
...
@@ -270,6 +288,7 @@ def sed_inplace(filename, pattern, repl):
...
@@ -270,6 +288,7 @@ def sed_inplace(filename, pattern, repl):
shutil
.
copystat
(
filename
,
tmp_file
.
name
)
shutil
.
copystat
(
filename
,
tmp_file
.
name
)
shutil
.
move
(
tmp_file
.
name
,
filename
)
shutil
.
move
(
tmp_file
.
name
,
filename
)
def
append_file
(
f_main
,
f_gets_appended
):
def
append_file
(
f_main
,
f_gets_appended
):
"""
Append the contents of one file to another
"""
Append the contents of one file to another
...
@@ -284,6 +303,7 @@ def append_file(f_main, f_gets_appended):
...
@@ -284,6 +303,7 @@ def append_file(f_main, f_gets_appended):
if
rc
!=
0
:
if
rc
!=
0
:
raise
RuntimeError
(
'
cat
'
+
f_gets_appended
+
'
>>
'
+
f_main
)
raise
RuntimeError
(
'
cat
'
+
f_gets_appended
+
'
>>
'
+
f_main
)
def
write_txt
(
lines
,
fpath
):
def
write_txt
(
lines
,
fpath
):
"""
Write a list of strings to a text file
"""
Write a list of strings to a text file
...
@@ -298,3 +318,13 @@ def write_txt(lines, fpath):
...
@@ -298,3 +318,13 @@ def write_txt(lines, fpath):
with
open
(
fpath
,
"
w
"
)
as
file
:
with
open
(
fpath
,
"
w
"
)
as
file
:
for
line
in
lines
:
for
line
in
lines
:
file
.
write
(
line
+
'
\n
'
)
file
.
write
(
line
+
'
\n
'
)
def
save_dict
(
dictionary
,
fpath
):
with
open
(
fpath
,
'
wb
'
)
as
f
:
pickle
.
dump
(
dictionary
,
f
)
def
load_dict
(
fpath
):
with
open
(
fpath
,
'
rb
'
)
as
f
:
return
pickle
.
load
(
f
)
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment