From b6ea29ea0d65011c6566a26dbb349e688e9e9c4c Mon Sep 17 00:00:00 2001
From: Anne Tipka <anne.tipka@ctbto.org>
Date: Thu, 28 Jul 2022 10:34:17 +0000
Subject: [PATCH] Adaptations for Bologna servers in Python part
---
Source/Python/Mods/tools.py | 16 ++--------------
Source/Python/_config.py | 10 +++++-----
Source/Python/install.py | 19 +++++++------------
3 files changed, 14 insertions(+), 31 deletions(-)
diff --git a/Source/Python/Mods/tools.py b/Source/Python/Mods/tools.py
index 254af50..b16e26a 100644
--- a/Source/Python/Mods/tools.py
+++ b/Source/Python/Mods/tools.py
@@ -650,7 +650,7 @@ def make_dir(directory):
return
-def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
+def put_file_to_ecserver(ecd, filename):
'''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
Note
@@ -667,15 +667,6 @@ def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
filename : str
The name of the file to send to the ECMWF server.
- target : str
- The target queue where the file should be sent to.
-
- ecuid : str
- The user id on ECMWF server.
-
- ecgid : str
- The group id on ECMWF server.
-
Return
------
@@ -683,10 +674,7 @@ def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
try:
subprocess.check_output(['ecaccess-file-put',
- ecd + '/' + filename,
- target + ':/home/ms/' +
- ecgid + '/' + ecuid +
- '/' + filename],
+ ecd + '/' + filename],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print('... ERROR CODE: ' + str(e.returncode))
diff --git a/Source/Python/_config.py b/Source/Python/_config.py
index e6f57f7..53ced99 100644
--- a/Source/Python/_config.py
+++ b/Source/Python/_config.py
@@ -60,7 +60,7 @@ HOSTENV_READING = 'ECPLATFORM'
if os.getenv(HOSTENV_BOLOGNA) is not None:
ec_hostname = os.getenv(HOSTENV_BOLOGNA)
FLAG_ON_ECMWFSERVER = ec_hostname in HOSTNAMES_BOLOGNA_LIST
- QUEUES_LIST = ['ecs', 'ecs-login', 'hpc', 'hpc-login', 'hpc-2020']
+ QUEUES_LIST = ['ecs', 'hpc']
# Test and set ECMWF Reading server values
elif os.getenv(HOSTENV_READING) is not None:
ec_hostname = os.getenv(HOSTENV_READING)
@@ -69,9 +69,9 @@ elif os.getenv(HOSTENV_READING) is not None:
else:
FLAG_ON_ECMWFSERVER = False
-QUEUES_LIST_ALL = ['ecs', 'ecs-login', 'hpc', 'hpc-login', 'hpc-2020', 'ecgate', 'cca', 'ccb']
+QUEUES_LIST_ALL = ['ecs', 'hpc', 'ecgate', 'cca', 'ccb']
-INSTALL_TARGETS = ['local', 'syslocal', 'ecgate', 'cca', 'ccb']
+INSTALL_TARGETS = ['local', 'syslocal', 'ecgate', 'cca', 'ccb', 'ecs', 'hpc']
CDS_DATASET_ML = 'reanalysis-era5-complete'
CDS_DATASET_SFC = 'reanalysis-era5-single-levels'
@@ -87,9 +87,9 @@ FILE_MARS_REQUESTS = 'mars_requests.csv'
FORTRAN_EXECUTABLE = 'calc_etadot'
TEMPFILE_USER_ENVVARS = 'ECMWF_ENV.template'
FILE_USER_ENVVARS = 'ECMWF_ENV'
-TEMPFILE_INSTALL_COMPILEJOB = 'installscript.template'
+TEMPFILE_INSTALL_COMPILEJOB = 'installscript.template.bologna'
FILE_INSTALL_COMPILEJOB = 'compilejob.ksh'
-TEMPFILE_INSTALL_JOB = 'jobscript.template'
+TEMPFILE_INSTALL_JOB = 'jobscript.template.bologna'
TEMPFILE_JOB = 'submitscript.template'
FILE_JOB_OD = 'job.ksh'
FILE_JOB_OP = 'jopoper.ksh'
diff --git a/Source/Python/install.py b/Source/Python/install.py
index 20506cf..11e6f34 100755
--- a/Source/Python/install.py
+++ b/Source/Python/install.py
@@ -109,7 +109,7 @@ def main():
c.assign_args_to_control(args)
check_install_conditions(c)
- if c.install_target.lower() not in ['local', 'syslocal']: # ecgate or cca
+ if c.install_target.lower() not in ['local', 'syslocal']: # ecmwf servers e.g. ecgate (ecs) and hpc
install_via_gateway(c)
else: # local
install_local(c)
@@ -136,8 +136,8 @@ def get_install_cmdline_args():
parser.add_argument('--target', dest='install_target',
type=none_or_str, default=None,
- help="Valid targets: syslocal | local | ecgate | cca , \
- the latter two are at ECMWF")
+ help="Valid targets: syslocal | local | ecgate | cca | \
+ ccb | ecs | hpc , the latter 5 are at ECMWF servers")
parser.add_argument("--makefile", dest="makefile",
type=none_or_str, default=None,
help='Name of makefile for compiling the '
@@ -198,7 +198,7 @@ def install_via_gateway(c):
tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
- mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
+ mk_compilejob(c.makefile, c.ecuid, c.installdir)
mk_job_template(c.ecuid, c.ecgid, c.installdir)
@@ -206,8 +206,7 @@ def install_via_gateway(c):
mk_tarball(tar_file, c.install_target)
- put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
- c.install_target, c.ecuid, c.ecgid)
+ put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
submit_job_to_ecserver(c.install_target,
os.path.join(_config.PATH_REL_JOBSCRIPTS,
@@ -613,10 +612,10 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
return
-def mk_compilejob(makefile, ecuid, ecgid, fp_root):
+def mk_compilejob(makefile, ecuid, fp_root):
'''Modifies the original job template file so that it is specified
for the user and the environment were it will be applied. Result
- is stored in a new file "job.temp" in the python directory.
+ is stored in a new file "compilejob.ksh" in the Jobscript directory.
Parameters
----------
@@ -627,9 +626,6 @@ def mk_compilejob(makefile, ecuid, ecgid, fp_root):
ecuid : str
The user id on ECMWF server.
- ecgid : str
- The group id on ECMWF server.
-
fp_root : str
Path to the root directory of FLEXPART environment or flex_extract
environment.
@@ -651,7 +647,6 @@ def mk_compilejob(makefile, ecuid, ecgid, fp_root):
cls=NewTextTemplate)
stream = compile_template.generate(
- usergroup=ecgid,
username=ecuid,
version_number=_config._VERSION_STR,
fp_root_scripts=fp_root,
--
GitLab