diff --git a/source/python/_config.py b/source/python/_config.py
index 88998466e78f96be7c99e1e288e749e93a0a2fd1..3ce4c735631c38e0f089a3f1cc928c3d1e502db3 100644
--- a/source/python/_config.py
+++ b/source/python/_config.py
@@ -33,8 +33,9 @@ _VERSION_STR = '7.1'
FILE_MARS_REQUESTS = 'mars_requests.csv'
FORTRAN_EXECUTABLE = 'CONVERT2'
+TEMPFILE_USER_ENVVARS = 'ECMWF_ENV.template'
FILE_USER_ENVVARS = 'ECMWF_ENV'
-TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.temp'
+TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.template'
FILE_INSTALL_COMPILEJOB = 'compilejob.ksh'
TEMPFILE_INSTALL_JOB = 'job.temp.o'
TEMPFILE_JOB = 'job.temp'
diff --git a/source/python/install.py b/source/python/install.py
index b94ae31907dc481c80fccb03f5565ea63dda6674..b624592438cb3fdddb3ea8c6e0a105cdc7299ecf 100755
--- a/source/python/install.py
+++ b/source/python/install.py
@@ -319,6 +319,7 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
@Description:
Creates a file named ECMWF_ENV which contains the
necessary environmental variables at ECMWF servers.
+ It is based on the template ECMWF_ENV.template.
@Input:
ecuid: string
@@ -337,12 +338,21 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
@Return:
<nothing>
'''
+ from genshi.template.text import NewTextTemplate
+ from genshi.template import TemplateLoader
- with open(_config.PATH_REL_ECMWF_ENV, 'w') as fo:
- fo.write('ECUID ' + ecuid + '\n')
- fo.write('ECGID ' + ecgid + '\n')
- fo.write('GATEWAY ' + gateway + '\n')
- fo.write('DESTINATION ' + destination + '\n')
+ loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+ ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
+ cls=NewTextTemplate)
+
+ stream = ecmwfvars_template.generate(user_name = ecuid,
+ user_group = ecgid,
+ gateway_name = gateway,
+ destination_name = destination
+ )
+
+ with open(_config.PATH_ECMWF_ENV, 'w') as f:
+ f.write(stream.render('text'))
return
@@ -374,36 +384,60 @@ def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
@Return:
<nothing>
'''
-
- template = os.path.join(_config.PATH_REL_TEMPLATES,
- _config.TEMPFILE_INSTALL_COMPILEJOB)
- with open(template) as f:
- fdata = f.read().split('\n')
-
- compilejob = os.path.join(_config.PATH_REL_JOBSCRIPTS,
+ from genshi.template.text import NewTextTemplate
+ from genshi.template import TemplateLoader
+
+ loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+ compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
+ cls=NewTextTemplate)
+
+ if fp_root == '../':
+ fp_root = '$HOME'
+
+ stream = compile_template.generate(
+ username = ecuid,
+ usergroup = ecgid,
+ version_number = _config._VERSION_STR,
+ fp_root_scripts = fp_root,
+ makefile = makefile,
+ fortran_program = _config.FORTRAN_EXECUTABLE
+ )
+
+ compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
_config.FILE_INSTALL_COMPILEJOB)
- with open(compilejob, 'w') as fo:
- for data in fdata:
- if 'MAKEFILE=' in data:
- data = 'export MAKEFILE=' + makefile
- elif 'FLEXPART_ROOT_SCRIPTS=' in data:
- if fp_root != '../':
- data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
- else:
- data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
- elif target.lower() != 'local':
- if '--workdir' in data:
- data = '#SBATCH --workdir=/scratch/ms/' + \
- ecgid + '/' + ecuid
- elif '##PBS -o' in data:
- data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
- 'flex_ecmwf.$Jobname.$Job_ID.out'
- elif 'FLEXPART_ROOT_SCRIPTS=' in data:
- if fp_root != '../':
- data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
- else:
- data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
- fo.write(data + '\n')
+
+ with open(compilejob, 'w') as f:
+ f.write(stream.render('text'))
+
+
+ # template = os.path.join(_config.PATH_REL_TEMPLATES,
+ # _config.TEMPFILE_INSTALL_COMPILEJOB)
+ # with open(template) as f:
+ # fdata = f.read().split('\n')
+
+
+ # with open(compilejob, 'w') as fo:
+ # for data in fdata:
+ # if 'MAKEFILE=' in data:
+ # data = 'export MAKEFILE=' + makefile
+ # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
+ # if fp_root != '../':
+ # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
+ # else:
+ # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
+ # elif target.lower() != 'local':
+ # if '--workdir' in data:
+ # data = '#SBATCH --workdir=/scratch/ms/' + \
+ # ecgid + '/' + ecuid
+ # elif '##PBS -o' in data:
+ # data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
+ # 'flex_ecmwf.$Jobname.$Job_ID.out'
+ # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
+ # if fp_root != '../':
+ # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
+ # else:
+ # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
+ # fo.write(data + '\n')
return
diff --git a/source/pythontest/TestData/ECMWF_ENV.test b/source/pythontest/TestData/ECMWF_ENV.test
new file mode 100644
index 0000000000000000000000000000000000000000..dc258a940b6cacb52706c196b6ea9a5f609cc72c
--- /dev/null
+++ b/source/pythontest/TestData/ECMWF_ENV.test
@@ -0,0 +1,4 @@
+ECUID testuser
+ECGID testgroup
+GATEWAY gateway.test.ac.at
+DESTINATION user@destination
diff --git a/source/pythontest/TestData/compilejob.test b/source/pythontest/TestData/compilejob.test
new file mode 100644
index 0000000000000000000000000000000000000000..8ae61bff4614182a0efa8c4a8b173f4b96b7e6a6
--- /dev/null
+++ b/source/pythontest/TestData/compilejob.test
@@ -0,0 +1,68 @@
+#!/bin/ksh
+
+# ON ECGB:
+# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server
+# start with sbatch NAME_OF_THIS_FILE directly on machine
+
+#SBATCH --workdir=/scratch/ms/testgroup/testuser
+#SBATCH --qos=normal
+#SBATCH --job-name=flex_ecmwf
+#SBATCH --output=flex_ecmwf.%j.out
+#SBATCH --error=flex_ecmwf.%j.out
+#SBATCH --mail-type=FAIL
+#SBATCH --time=12:00:00
+
+## CRAY specific batch requests
+##PBS -N flex_ecmwf
+##PBS -q ns
+##PBS -S /usr/bin/ksh
+##PBS -o /scratch/ms/testgroup/testuser/flex_ecmwf.$Jobname.$Job_ID.out
+# job output is in .ecaccess_DO_NOT_REMOVE
+##PBS -j oe
+##PBS -V
+##PBS -l EC_threads_per_task=1
+##PBS -l EC_memory_per_task=3200MB
+
+set -x
+export VERSION=7.1
+case $HOST in
+ *ecg*)
+ module load python
+ module unload grib_api
+ module unload emos
+ module load grib_api/1.14.5
+ module load emos/437-r64
+ export FLEXPART_ROOT_SCRIPTS=fp_root_test_path
+ export MAKEFILE=Makefile.TEST
+ ;;
+ *cca*)
+ module switch PrgEnv-cray PrgEnv-intel
+ module load grib_api
+ module load emos
+ module load python
+ echo ${GROUP}
+ echo ${HOME}
+ echo $HOME | awk -F / '{print $1, $2, $3, $4}'
+ export GROUP=`echo $HOME | awk -F / '{print $4}'`
+ export SCRATCH=/scratch/ms/${GROUP}/${USER}
+ export FLEXPART_ROOT_SCRIPTS=p_root_test_path
+ export MAKEFILE=Makefile.TEST
+ ;;
+esac
+
+mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
+cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory
+tar -xvf $HOME/flex_extract_v$VERSION.tar
+cd src
+\rm *.o *.mod CONVERT2
+make -f $MAKEFILE >flexcompile 2>flexcompile
+
+ls -l CONVERT2 >>flexcompile
+if [ $? -eq 0 ]; then
+ echo 'SUCCESS!' >>flexcompile
+ mail -s flexcompile.$HOST.$$ $USER <flexcompile
+else
+ echo Environment: >>flexcompile
+ env >> flexcompile
+ mail -s "ERROR! flexcompile.$HOST.$$" $USER <flexcompile
+fi
\ No newline at end of file
diff --git a/source/pythontest/TestInstall.py b/source/pythontest/TestInstall.py
index 6691d0641d74ad09d8d7d15702a1888dea5b5fba..ddb3b2d5ca270037689b68c89ff01dacf6f05067 100644
--- a/source/pythontest/TestInstall.py
+++ b/source/pythontest/TestInstall.py
@@ -10,7 +10,7 @@ import pytest
sys.path.append('../python')
import _config
import install
-from mods.tools import make_dir
+from mods.tools import make_dir, silent_remove
class TestTools():
@@ -23,7 +23,7 @@ class TestTools():
# - install_via_gateway
#! - mk_tarball
#! - un_tarball
- # - mk_env_vars
+ #! - mk_env_vars
# - mk_compilejob
# - mk_job_template
# - delete_convert_build
@@ -115,3 +115,42 @@ class TestTools():
# clean up temp test dir
shutil.rmtree(test_dir)
+
+ def test_mk_env_vars(self):
+ import filecmp
+
+ # comparison file
+ testfile = os.path.join(_config.PATH_TEST_DIR,'TestData',
+ 'ECMWF_ENV.test')
+
+ # create test ECMWF_ENV file
+ install.mk_env_vars('testuser',
+ 'testgroup',
+ 'gateway.test.ac.at',
+ 'user@destination')
+
+ assert filecmp.cmp(testfile, _config.PATH_ECMWF_ENV, shallow=False)
+
+ # delte test file
+ silent_remove(_config.PATH_ECMWF_ENV)
+
+ def test_mk_compilejob(self):
+ import filecmp
+
+ # comparison file
+ testfile = os.path.join(_config.PATH_TEST_DIR,'TestData',
+ 'compilejob.test')
+
+ # create
+ install.mk_compilejob('Makefile.TEST',
+ '',
+ 'testuser',
+ 'testgroup',
+ 'fp_root_test_path')
+
+ finalfile = os.path.join(_config.PATH_JOBSCRIPTS,
+ _config.FILE_INSTALL_COMPILEJOB)
+ assert filecmp.cmp(testfile, finalfile, shallow=False)
+
+ # delete test file
+ silent_remove(finalfile)
diff --git a/templates/ECMWF_ENV.template b/templates/ECMWF_ENV.template
new file mode 100644
index 0000000000000000000000000000000000000000..515da08ea1d6cc2190074709f8a5bc6b22b3e3c7
--- /dev/null
+++ b/templates/ECMWF_ENV.template
@@ -0,0 +1,4 @@
+ECUID $user_name
+ECGID $user_group
+GATEWAY $gateway_name
+DESTINATION $destination_name
diff --git a/templates/compilejob.template b/templates/compilejob.template
new file mode 100644
index 0000000000000000000000000000000000000000..bb1348ec8b40310369402949a950c121f6c96063
--- /dev/null
+++ b/templates/compilejob.template
@@ -0,0 +1,68 @@
+#!/bin/ksh
+
+# ON ECGB:
+# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server
+# start with sbatch NAME_OF_THIS_FILE directly on machine
+
+#SBATCH --workdir=/scratch/ms/$usergroup/$username
+#SBATCH --qos=normal
+#SBATCH --job-name=flex_ecmwf
+#SBATCH --output=flex_ecmwf.%j.out
+#SBATCH --error=flex_ecmwf.%j.out
+#SBATCH --mail-type=FAIL
+#SBATCH --time=12:00:00
+
+## CRAY specific batch requests
+##PBS -N flex_ecmwf
+##PBS -q ns
+##PBS -S /usr/bin/ksh
+##PBS -o /scratch/ms/$usergroup/$username/flex_ecmwf.$Jobname.$Job_ID.out
+# job output is in .ecaccess_DO_NOT_REMOVE
+##PBS -j oe
+##PBS -V
+##PBS -l EC_threads_per_task=1
+##PBS -l EC_memory_per_task=3200MB
+
+set -x
+export VERSION=$version_number
+case $HOST in
+ *ecg*)
+ module load python
+ module unload grib_api
+ module unload emos
+ module load grib_api/1.14.5
+ module load emos/437-r64
+ export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts
+ export MAKEFILE=$makefile
+ ;;
+ *cca*)
+ module switch PrgEnv-cray PrgEnv-intel
+ module load grib_api
+ module load emos
+ module load python
+ echo ${GROUP}
+ echo ${HOME}
+ echo $HOME | awk -F / '{print $1, $2, $3, $4}'
+ export GROUP=`echo $HOME | awk -F / '{print $4}'`
+ export SCRATCH=/scratch/ms/${GROUP}/${USER}
+ export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts
+ export MAKEFILE=$makefile
+ ;;
+esac
+
+mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
+cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory
+tar -xvf $HOME/flex_extract_v$VERSION.tar
+cd src
+\rm *.o *.mod $fortran_program
+make -f $MAKEFILE >flexcompile 2>flexcompile
+
+ls -l $fortran_program >>flexcompile
+if [ $? -eq 0 ]; then
+ echo 'SUCCESS!' >>flexcompile
+ mail -s flexcompile.$HOST.$$ $USER <flexcompile
+else
+ echo Environment: >>flexcompile
+ env >> flexcompile
+ mail -s "ERROR! flexcompile.$HOST.$$" $USER <flexcompile
+fi
\ No newline at end of file