added writing of compilejob and ECMWF_ENV via templates and added tests

source/python/_config.py

@@ -33,8 +33,9 @@ _VERSION_STR = '7.1'
 
 FILE_MARS_REQUESTS = 'mars_requests.csv'
 FORTRAN_EXECUTABLE = 'CONVERT2'
+TEMPFILE_USER_ENVVARS = 'ECMWF_ENV.template'
 FILE_USER_ENVVARS = 'ECMWF_ENV'
-TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.temp'
+TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.template'
 FILE_INSTALL_COMPILEJOB = 'compilejob.ksh'
 TEMPFILE_INSTALL_JOB = 'job.temp.o'
 TEMPFILE_JOB = 'job.temp'

source/python/install.py

@@ -319,6 +319,7 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
     @Description:
         Creates a file named ECMWF_ENV which contains the
         necessary environmental variables at ECMWF servers.
+        It is based on the template ECMWF_ENV.template.
 
     @Input:
         ecuid: string
@@ -337,12 +338,21 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
     @Return:
         <nothing>
     '''
+    from genshi.template.text import NewTextTemplate
+    from genshi.template import  TemplateLoader
 
-    with open(_config.PATH_REL_ECMWF_ENV, 'w') as fo:
-        fo.write('ECUID ' + ecuid + '\n')
-        fo.write('ECGID ' + ecgid + '\n')
-        fo.write('GATEWAY ' + gateway + '\n')
-        fo.write('DESTINATION ' + destination + '\n')
+    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+    ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
+                                     cls=NewTextTemplate)
+
+    stream = ecmwfvars_template.generate(user_name = ecuid,
+                                       user_group = ecgid,
+                                       gateway_name = gateway,
+                                       destination_name = destination
+                                       )
+
+    with open(_config.PATH_ECMWF_ENV, 'w') as f:
+        f.write(stream.render('text'))
 
     return
 
@@ -374,36 +384,60 @@ def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
     @Return:
         <nothing>
     '''
-
-    template = os.path.join(_config.PATH_REL_TEMPLATES,
-                            _config.TEMPFILE_INSTALL_COMPILEJOB)
-    with open(template) as f:
-        fdata = f.read().split('\n')
-
-    compilejob = os.path.join(_config.PATH_REL_JOBSCRIPTS,
+    from genshi.template.text import NewTextTemplate
+    from genshi.template import  TemplateLoader
+
+    loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+    compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
+                                   cls=NewTextTemplate)
+
+    if fp_root == '../':
+        fp_root = '$HOME'
+
+    stream = compile_template.generate(
+        username = ecuid,
+        usergroup = ecgid,
+        version_number = _config._VERSION_STR,
+        fp_root_scripts = fp_root,
+        makefile = makefile,
+        fortran_program = _config.FORTRAN_EXECUTABLE
+    )
+
+    compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
                               _config.FILE_INSTALL_COMPILEJOB)
-    with open(compilejob, 'w') as fo:
-        for data in fdata:
-            if 'MAKEFILE=' in data:
-                data = 'export MAKEFILE=' + makefile
-            elif 'FLEXPART_ROOT_SCRIPTS=' in data:
-                if fp_root != '../':
-                    data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
-                else:
-                    data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
-            elif target.lower() != 'local':
-                if '--workdir' in data:
-                    data = '#SBATCH --workdir=/scratch/ms/' + \
-                            ecgid + '/' + ecuid
-                elif '##PBS -o' in data:
-                    data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
-                           'flex_ecmwf.$Jobname.$Job_ID.out'
-                elif 'FLEXPART_ROOT_SCRIPTS=' in data:
-                    if fp_root != '../':
-                        data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
-                    else:
-                        data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
-            fo.write(data + '\n')
+
+    with open(compilejob, 'w') as f:
+        f.write(stream.render('text'))
+
+
+    # template = os.path.join(_config.PATH_REL_TEMPLATES,
+                            # _config.TEMPFILE_INSTALL_COMPILEJOB)
+    # with open(template) as f:
+        # fdata = f.read().split('\n')
+
+
+    # with open(compilejob, 'w') as fo:
+        # for data in fdata:
+            # if 'MAKEFILE=' in data:
+                # data = 'export MAKEFILE=' + makefile
+            # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
+                # if fp_root != '../':
+                    # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
+                # else:
+                    # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
+            # elif target.lower() != 'local':
+                # if '--workdir' in data:
+                    # data = '#SBATCH --workdir=/scratch/ms/' + \
+                            # ecgid + '/' + ecuid
+                # elif '##PBS -o' in data:
+                    # data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
+                           # 'flex_ecmwf.$Jobname.$Job_ID.out'
+                # elif 'FLEXPART_ROOT_SCRIPTS=' in data:
+                    # if fp_root != '../':
+                        # data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
+                    # else:
+                        # data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
+            # fo.write(data + '\n')
 
     return
 

source/pythontest/TestData/ECMWF_ENV.test

+ECUID testuser
+ECGID testgroup
+GATEWAY gateway.test.ac.at
+DESTINATION user@destination

source/pythontest/TestData/compilejob.test

+#!/bin/ksh
+
+# ON ECGB:
+# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE  on gateway server
+# start with sbatch NAME_OF_THIS_FILE directly on machine
+
+#SBATCH --workdir=/scratch/ms/testgroup/testuser
+#SBATCH --qos=normal
+#SBATCH --job-name=flex_ecmwf
+#SBATCH --output=flex_ecmwf.%j.out
+#SBATCH --error=flex_ecmwf.%j.out
+#SBATCH --mail-type=FAIL
+#SBATCH --time=12:00:00
+
+## CRAY specific batch requests
+##PBS -N flex_ecmwf
+##PBS -q ns
+##PBS -S /usr/bin/ksh
+##PBS -o /scratch/ms/testgroup/testuser/flex_ecmwf.$Jobname.$Job_ID.out
+# job output is in .ecaccess_DO_NOT_REMOVE
+##PBS -j oe
+##PBS -V
+##PBS -l EC_threads_per_task=1
+##PBS -l EC_memory_per_task=3200MB
+
+set -x
+export VERSION=7.1
+case $HOST in
+  *ecg*)
+  module load python
+  module unload grib_api
+  module unload emos
+  module load grib_api/1.14.5
+  module load emos/437-r64
+  export FLEXPART_ROOT_SCRIPTS=fp_root_test_path
+  export MAKEFILE=Makefile.TEST
+  ;;
+  *cca*)
+  module switch PrgEnv-cray PrgEnv-intel
+  module load grib_api
+  module load emos
+  module load python
+  echo ${GROUP}
+  echo ${HOME}
+  echo $HOME | awk -F / '{print $1, $2, $3, $4}'
+  export GROUP=`echo $HOME | awk -F / '{print $4}'`
+  export SCRATCH=/scratch/ms/${GROUP}/${USER}
+  export FLEXPART_ROOT_SCRIPTS=p_root_test_path
+  export MAKEFILE=Makefile.TEST
+  ;;
+esac
+
+mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
+cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION   # if FLEXPART_ROOT is not set this means cd to the home directory
+tar -xvf $HOME/flex_extract_v$VERSION.tar
+cd src
+\rm *.o *.mod CONVERT2 
+make -f $MAKEFILE >flexcompile 2>flexcompile
+
+ls -l CONVERT2 >>flexcompile
+if [ $? -eq 0 ]; then
+  echo 'SUCCESS!' >>flexcompile
+  mail -s flexcompile.$HOST.$$ $USER <flexcompile
+else
+  echo Environment: >>flexcompile
+  env >> flexcompile
+  mail -s "ERROR! flexcompile.$HOST.$$" $USER <flexcompile
+fi
\ No newline at end of file

source/pythontest/TestInstall.py

@@ -10,7 +10,7 @@ import pytest
 sys.path.append('../python')
 import _config
 import install
-from mods.tools import make_dir
+from mods.tools import make_dir, silent_remove
 
 
 class TestTools():
@@ -23,7 +23,7 @@ class TestTools():
     #    - install_via_gateway
     #!    - mk_tarball
     #!    - un_tarball
-    #    - mk_env_vars
+    #!    - mk_env_vars
     #    - mk_compilejob
     #    - mk_job_template
     #    - delete_convert_build
@@ -115,3 +115,42 @@ class TestTools():
 
         # clean up temp test dir
         shutil.rmtree(test_dir)
+
+    def test_mk_env_vars(self):
+        import filecmp
+
+        # comparison file
+        testfile = os.path.join(_config.PATH_TEST_DIR,'TestData',
+                                'ECMWF_ENV.test')
+
+        # create test ECMWF_ENV file
+        install.mk_env_vars('testuser',
+                            'testgroup',
+                            'gateway.test.ac.at',
+                            'user@destination')
+
+        assert filecmp.cmp(testfile, _config.PATH_ECMWF_ENV, shallow=False)
+
+        # delte test file
+        silent_remove(_config.PATH_ECMWF_ENV)
+
+    def test_mk_compilejob(self):
+        import filecmp
+
+        # comparison file
+        testfile = os.path.join(_config.PATH_TEST_DIR,'TestData',
+                                    'compilejob.test')
+
+        # create
+        install.mk_compilejob('Makefile.TEST',
+                              '',
+                              'testuser',
+                              'testgroup',
+                              'fp_root_test_path')
+
+        finalfile = os.path.join(_config.PATH_JOBSCRIPTS,
+                              _config.FILE_INSTALL_COMPILEJOB)
+        assert filecmp.cmp(testfile, finalfile, shallow=False)
+
+        # delete test file
+        silent_remove(finalfile)

templates/ECMWF_ENV.template

+ECUID $user_name
+ECGID $user_group
+GATEWAY $gateway_name
+DESTINATION $destination_name

templates/compilejob.template

+#!/bin/ksh
+
+# ON ECGB:
+# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE  on gateway server
+# start with sbatch NAME_OF_THIS_FILE directly on machine
+
+#SBATCH --workdir=/scratch/ms/$usergroup/$username
+#SBATCH --qos=normal
+#SBATCH --job-name=flex_ecmwf
+#SBATCH --output=flex_ecmwf.%j.out
+#SBATCH --error=flex_ecmwf.%j.out
+#SBATCH --mail-type=FAIL
+#SBATCH --time=12:00:00
+
+## CRAY specific batch requests
+##PBS -N flex_ecmwf
+##PBS -q ns
+##PBS -S /usr/bin/ksh
+##PBS -o /scratch/ms/$usergroup/$username/flex_ecmwf.$Jobname.$Job_ID.out
+# job output is in .ecaccess_DO_NOT_REMOVE
+##PBS -j oe
+##PBS -V
+##PBS -l EC_threads_per_task=1
+##PBS -l EC_memory_per_task=3200MB
+
+set -x
+export VERSION=$version_number
+case $HOST in
+  *ecg*)
+  module load python
+  module unload grib_api
+  module unload emos
+  module load grib_api/1.14.5
+  module load emos/437-r64
+  export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts
+  export MAKEFILE=$makefile
+  ;;
+  *cca*)
+  module switch PrgEnv-cray PrgEnv-intel
+  module load grib_api
+  module load emos
+  module load python
+  echo ${GROUP}
+  echo ${HOME}
+  echo $HOME | awk -F / '{print $1, $2, $3, $4}'
+  export GROUP=`echo $HOME | awk -F / '{print $4}'`
+  export SCRATCH=/scratch/ms/${GROUP}/${USER}
+  export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts
+  export MAKEFILE=$makefile
+  ;;
+esac
+
+mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
+cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION   # if FLEXPART_ROOT is not set this means cd to the home directory
+tar -xvf $HOME/flex_extract_v$VERSION.tar
+cd src
+\rm *.o *.mod $fortran_program 
+make -f $MAKEFILE >flexcompile 2>flexcompile
+
+ls -l $fortran_program >>flexcompile
+if [ $? -eq 0 ]; then
+  echo 'SUCCESS!' >>flexcompile
+  mail -s flexcompile.$HOST.$$ $USER <flexcompile
+else
+  echo Environment: >>flexcompile
+  env >> flexcompile
+  mail -s "ERROR! flexcompile.$HOST.$$" $USER <flexcompile
+fi
\ No newline at end of file