diff --git a/source/python/_config.py b/source/python/_config.py
index 27491f4447afe559b9690de6ab663e51e80cc9eb..cbe125acdc13ad2cd0181c75691689d9436f80f4 100644
--- a/source/python/_config.py
+++ b/source/python/_config.py
@@ -92,7 +92,8 @@ else:
# flex_extract root directory are needed
# ------------------------------------------------------------------------------
-PATH_REL_PYTHON = os.path.relpath(PATH_LOCAL_PYTHON, PATH_FLEXEXTRACT_DIR)
+PATH_REL_PYTHON_SRC = os.path.relpath(PATH_LOCAL_PYTHON, PATH_FLEXEXTRACT_DIR)
+PATH_REL_PYTHONTEST_SRC = os.path.relpath(PATH_PYTHONTEST_SRC, PATH_FLEXEXTRACT_DIR)
PATH_REL_CONTROLFILES = os.path.relpath(PATH_CONTROLFILES, PATH_FLEXEXTRACT_DIR)
PATH_REL_TEMPLATES = os.path.relpath(PATH_TEMPLATES, PATH_FLEXEXTRACT_DIR)
PATH_REL_ECMWF_ENV = os.path.relpath(PATH_ECMWF_ENV, PATH_FLEXEXTRACT_DIR)
diff --git a/source/python/install.py b/source/python/install.py
index 3ab8218303e73a9dac59d8da30c0191264ba7421..823856b27a05ebf687d16a4b2fec0ddf20971d47 100755
--- a/source/python/install.py
+++ b/source/python/install.py
@@ -53,6 +53,7 @@ import sys
import glob
import subprocess
import inspect
+import tarfile
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
# software specific classes and modules from flex_extract
@@ -238,7 +239,6 @@ def mk_tarball(tarball_path, target):
------
'''
- import tarfile
from glob import glob
print('Create tarball ...')
@@ -251,38 +251,63 @@ def mk_tarball(tarball_path, target):
# get lists of the files to be added to the tar file
if target == 'local':
ECMWF_ENV_FILE = []
+ runfile = [os.path.relpath(x, ecd)
+ for x in UioFiles(_config.PATH_REL_RUN_DIR,
+ 'run_local.sh').files]
else:
ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
+ runfile = [os.path.relpath(x, ecd)
+ for x in UioFiles(_config.PATH_REL_RUN_DIR,
+ 'run.sh').files]
pyfiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_LOCAL_PYTHON, '*py').files]
+ for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
+ pytestfiles = [os.path.relpath(x, ecd)
+ for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
controlfiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_CONTROLFILES,
+ for x in UioFiles(_config.PATH_REL_CONTROLFILES,
'CONTROL*').files]
tempfiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_TEMPLATES , '*').files]
+ for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
+ nlfiles = [os.path.relpath(x, ecd)
+ for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
+ gribtable = [os.path.relpath(x, ecd)
+ for x in UioFiles(_config.PATH_REL_TEMPLATES , '*grib*').files]
ffiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.f*').files]
+ for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f*').files]
hfiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_FORTRAN_SRC, '*.h').files]
+ for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
makefiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_FORTRAN_SRC, 'Makefile*').files]
+ for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'Makefile*').files]
+ jobdir = [_config.PATH_REL_JOBSCRIPTS]
# concatenate single lists to one for a better looping
- filelist = pyfiles + controlfiles + tempfiles + ffiles + hfiles + \
- makefiles + ECMWF_ENV_FILE
+ filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
+ ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
+ runfile + jobdir + \
+ ['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
# create installation tar-file
+ exclude_files = [".ksh"]
try:
with tarfile.open(tarball_path, "w:gz") as tar_handle:
for file in filelist:
- tar_handle.add(file)
-
+ tar_handle.add(file, recursive=False,
+ filter=lambda tarinfo: None
+ if os.path.splitext(tarinfo.name)[1]
+ in exclude_files
+ else tarinfo)
except subprocess.CalledProcessError as e:
print('... ERROR CODE:\n ... ' + str(e.returncode))
print('... ERROR MESSAGE:\n ... ' + str(e))
- sys.exit('... could not make installation tar ball!')
+ sys.exit('\n... could not make installation tar ball!')
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
+
+ sys.exit('\n... error occured while trying to read tar-file ' +
+ str(tarball_path))
return
@@ -300,12 +325,21 @@ def un_tarball(tarball_path):
------
'''
- import tarfile
print('Untar ...')
- with tarfile.open(tarball_path) as tar_handle:
- tar_handle.extractall()
+ try:
+ with tarfile.open(tarball_path) as tar_handle:
+ tar_handle.extractall()
+ except tarfile.TarError as e:
+ sys.exit('\n... error occured while trying to read tar-file ' +
+ str(tarball_path))
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
+
+ sys.exit('\n... error occured while trying to read tar-file ' +
+ str(tarball_path))
return
@@ -335,19 +369,39 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
'''
from genshi.template.text import NewTextTemplate
from genshi.template import TemplateLoader
+ from genshi.template.eval import UndefinedError
- loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
- ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
- cls=NewTextTemplate)
+ try:
+ loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+ ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
+ cls=NewTextTemplate)
+
+ stream = ecmwfvars_template.generate(user_name = ecuid,
+ user_group = ecgid,
+ gateway_name = gateway,
+ destination_name = destination
+ )
+ except UndefinedError as e:
+ print('... ERROR ' + str(e))
+
+ sys.exit('\n... error occured while trying to generate template ' +
+ _config.PATH_ECMWF_ENV)
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
+
+ sys.exit('\n... error occured while trying to generate template ' +
+ _config.PATH_ECMWF_ENV)
- stream = ecmwfvars_template.generate(user_name = ecuid,
- user_group = ecgid,
- gateway_name = gateway,
- destination_name = destination
- )
+ try:
+ with open(_config.PATH_ECMWF_ENV, 'w') as f:
+ f.write(stream.render('text'))
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
- with open(_config.PATH_ECMWF_ENV, 'w') as f:
- f.write(stream.render('text'))
+ sys.exit('\n... error occured while trying to write ' +
+ _config.PATH_ECMWF_ENV)
return
@@ -381,28 +435,48 @@ def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
'''
from genshi.template.text import NewTextTemplate
from genshi.template import TemplateLoader
-
- loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
- compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
- cls=NewTextTemplate)
+ from genshi.template.eval import UndefinedError
if fp_root == '../':
fp_root = '$HOME'
- stream = compile_template.generate(
- usergroup = ecgid,
- username = ecuid,
- version_number = _config._VERSION_STR,
- fp_root_scripts = fp_root,
- makefile = makefile,
- fortran_program = _config.FORTRAN_EXECUTABLE
- )
+ try:
+ loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+ compile_template = loader.load(_config.TEMPFILE_INSTALL_COMPILEJOB,
+ cls=NewTextTemplate)
+
+ stream = compile_template.generate(
+ usergroup = ecgid,
+ username = ecuid,
+ version_number = _config._VERSION_STR,
+ fp_root_scripts = fp_root,
+ makefile = makefile,
+ fortran_program = _config.FORTRAN_EXECUTABLE
+ )
+ except UndefinedError as e:
+ print('... ERROR ' + str(e))
+
+ sys.exit('\n... error occured while trying to generate template ' +
+ _config.TEMPFILE_INSTALL_COMPILEJOB)
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
+
+ sys.exit('\n... error occured while trying to generate template ' +
+ _config.TEMPFILE_INSTALL_COMPILEJOB)
- compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
- _config.FILE_INSTALL_COMPILEJOB)
+ try:
+ compilejob = os.path.join(_config.PATH_JOBSCRIPTS,
+ _config.FILE_INSTALL_COMPILEJOB)
- with open(compilejob, 'w') as f:
- f.write(stream.render('text'))
+ with open(compilejob, 'w') as f:
+ f.write(stream.render('text'))
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
+
+ sys.exit('\n... error occured while trying to write ' +
+ compilejob)
return
@@ -436,27 +510,48 @@ def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
'''
from genshi.template.text import NewTextTemplate
from genshi.template import TemplateLoader
-
- loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
- compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
- cls=NewTextTemplate)
+ from genshi.template.eval import UndefinedError
fp_root_path_to_python = os.path.join(fp_root,
_config.FLEXEXTRACT_DIRNAME,
- _config.PATH_REL_PYTHON)
+ _config.PATH_REL_PYTHON_SRC)
+
+ try:
+ loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
+ compile_template = loader.load(_config.TEMPFILE_INSTALL_JOB,
+ cls=NewTextTemplate)
+
+ stream = compile_template.generate(
+ usergroup = ecgid,
+ username = ecuid,
+ version_number = _config._VERSION_STR,
+ fp_root_path = fp_root_path_to_python,
+ )
+ except UndefinedError as e:
+ print('... ERROR ' + str(e))
- stream = compile_template.generate(
- usergroup = ecgid,
- username = ecuid,
- version_number = _config._VERSION_STR,
- fp_root_path = fp_root_path_to_python,
- )
+ sys.exit('\n... error occured while trying to generate template ' +
+ _config.TEMPFILE_INSTALL_JOB)
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
+
+ sys.exit('\n... error occured while trying to generate template ' +
+ _config.TEMPFILE_INSTALL_JOB)
+
+
+ try:
+ tempjobfile = os.path.join(_config.PATH_TEMPLATES,
+ _config.TEMPFILE_JOB)
- tempjobfile = os.path.join(_config.PATH_TEMPLATES,
- _config.TEMPFILE_JOB)
+ with open(tempjobfile, 'w') as f:
+ f.write(stream.render('text'))
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
- with open(tempjobfile, 'w') as f:
- f.write(stream.render('text'))
+ sys.exit('\n... error occured while trying to write ' +
+ tempjobfile)
return
diff --git a/source/python/mods/tools.py b/source/python/mods/tools.py
index d78a5f3a8834b2462f0aa9d738ad82fc789aac98..b1e5c6073132132dc151a785ee8c0359be947491 100644
--- a/source/python/mods/tools.py
+++ b/source/python/mods/tools.py
@@ -99,6 +99,15 @@ def none_or_int(value):
return None
return int(value)
+def check_filepattern(filename):
+ '''
+ '''
+ if '.ksh' in filename:
+ return True
+ return False
+
+
+
def get_cmdline_arguments():
'''Decomposes the command line arguments and assigns them to variables.
Apply default values for non mentioned arguments.
diff --git a/source/pythontest/TestInstall.py b/source/pythontest/TestInstall.py
index c6995d25c6dc9c6b9f772dd6a9ad7fd7d5cb4bd7..fb3a643400cdd44a5bbffed685278dd75210bbde 100644
--- a/source/pythontest/TestInstall.py
+++ b/source/pythontest/TestInstall.py
@@ -5,38 +5,66 @@
import sys
import os
import inspect
+import subprocess
+import tarfile
+import errno
+import shutil
+from genshi.template import TemplateLoader
+from genshi.template.eval import UndefinedError
+from exceptions import OSError
import pytest
+from mock import patch
sys.path.append('../python')
import _config
-import install
+import _config_test
+from install import (mk_tarball, un_tarball, mk_env_vars, mk_compilejob,
+ mk_job_template)
+
from mods.tools import make_dir, silent_remove
+# - main
+# - get_install_cmdline_arguments
+# - install_via_gateway
+# - delete_convert_build
+# - make_convert_build
-class TestTools():
+class TestInstall():
'''
'''
+ @classmethod
+ def setup_class(self):
+ self.testdir = _config_test.PATH_TEST_DIR
+ self.testfilesdir = _config_test.PATH_TESTFILES_DIR
+ self.testinstalldir = _config_test.PATH_TESTINSTALL_DIR
+
+ # make test tarballs from shell script
+ subprocess.check_output([os.path.join(self.testinstalldir,
+ 'mk_install_tar.sh')])
+ # un tar the test tarballs from shell script
+ subprocess.check_output([os.path.join(self.testinstalldir,
+ 'un_install_tar.sh')])
- # - main
- # - get_install_cmdline_arguments
- # - install_via_gateway
- #! - mk_tarball
- #! - un_tarball
- #! - mk_env_vars
- #! - mk_compilejob
- #! - mk_job_template
- # - delete_convert_build
- # - make_convert_build
- def test_mk_tarball_local(self):
- import tarfile
+ @patch('tarfile.open', side_effect=[subprocess.CalledProcessError(1,'test'),
+ OSError(errno.EEXIST)])
+ def test_fail_mk_tarball_local(self, mock_open):
+ ecd = _config.PATH_FLEXEXTRACT_DIR + os.path.sep
+ # create test tarball and list its content files
+ tarballname = _config.FLEXEXTRACT_DIRNAME + '_localtest.tar'
+
+ with pytest.raises(SystemExit):
+ mk_tarball(ecd + tarballname, 'local')
+ with pytest.raises(SystemExit):
+ mk_tarball(ecd + tarballname, 'local')
+
+ def test_success_mk_tarball_local(self):
ecd = _config.PATH_FLEXEXTRACT_DIR + os.path.sep
# list comparison files for tarball content
- tar_test_dir = os.path.join(_config.PATH_TEST_DIR +
- os.path.sep + 'TestInstallTar')
+ tar_test_dir = os.path.join(self.testdir, 'InstallTar')
tar_test_file = os.path.join(tar_test_dir,
'flex_extract_v7.1_local.tar')
with tarfile.open(tar_test_file, 'r') as tar_handle:
@@ -44,7 +72,7 @@ class TestTools():
# create test tarball and list its content files
tarballname = _config.FLEXEXTRACT_DIRNAME + '_localtest.tar'
- install.mk_tarball(ecd + tarballname, 'local')
+ mk_tarball(ecd + tarballname, 'local')
with tarfile.open(ecd + tarballname, 'r') as tar_handle:
tar_content_list = tar_handle.getnames()
@@ -55,14 +83,11 @@ class TestTools():
# filelist of tarball content
assert sorted(comparison_list) == sorted(tar_content_list)
- def test_mk_tarball_ecgate(self):
- import tarfile
-
+ def test_success_mk_tarball_ecgate(self):
ecd = _config.PATH_FLEXEXTRACT_DIR + os.path.sep
# list comparison files for tarball content
- tar_test_dir = os.path.join(_config.PATH_TEST_DIR +
- os.path.sep + 'TestInstallTar')
+ tar_test_dir = os.path.join(self.testdir, 'InstallTar')
tar_test_file = os.path.join(tar_test_dir,
'flex_extract_v7.1_ecgate.tar')
with tarfile.open(tar_test_file, 'r') as tar_handle:
@@ -70,7 +95,7 @@ class TestTools():
# create test tarball and list its content files
tarballname = _config.FLEXEXTRACT_DIRNAME + '_ecgatetest.tar'
- install.mk_tarball(ecd + tarballname, 'ecgate')
+ mk_tarball(ecd + tarballname, 'ecgate')
with tarfile.open(ecd + tarballname, 'r') as tar_handle:
tar_content_list = tar_handle.getnames()
@@ -81,16 +106,18 @@ class TestTools():
# filelist of tarball content
assert sorted(comparison_list) == sorted(tar_content_list)
- def test_un_tarball(self):
- import tarfile
- import shutil
+ @patch('tarfile.open', side_effect=[tarfile.TarError, OSError])
+ def test_fail_un_tarball(self, mock_open):
+ with pytest.raises(SystemExit):
+ un_tarball('testpath')
+ def test_success_ecgate_un_tarball(self):
ecd = _config.PATH_FLEXEXTRACT_DIR + os.path.sep
# list comparison files for tarball content
- tar_test_dir = os.path.join(_config.PATH_TEST_DIR +
- os.path.sep + 'TestInstallTar')
- tar_test_fedir = os.path.join(tar_test_dir, 'flex_extract_v7.1_ecgate')
+ tar_test_dir = os.path.join(self.testdir, 'InstallTar')
+ cmp_dir = _config.FLEXEXTRACT_DIRNAME + '_ecgate'
+ tar_test_fedir = os.path.join(tar_test_dir, cmp_dir)
comparison_list = []
for path, subdirs, files in os.walk(tar_test_fedir):
for name in files:
@@ -99,11 +126,11 @@ class TestTools():
os.path.join(path, name), tar_test_fedir))
# untar in test directory
- test_dir = os.path.join(tar_test_dir, 'test_untar')
+ test_dir = os.path.join(tar_test_dir, 'test_ecgate')
make_dir(test_dir)
os.chdir(test_dir)
tarballname = _config.FLEXEXTRACT_DIRNAME + '_ecgate.tar'
- install.un_tarball(os.path.join(tar_test_dir, tarballname))
+ un_tarball(os.path.join(tar_test_dir, tarballname))
tarfiles_list = []
for path, subdirs, files in os.walk(test_dir):
for name in files:
@@ -113,66 +140,191 @@ class TestTools():
# test for equality
assert sorted(tarfiles_list) == sorted(comparison_list)
- # clean up temp test dir
- shutil.rmtree(test_dir)
+ def test_success_local_un_tarball(self):
+ ecd = _config.PATH_FLEXEXTRACT_DIR + os.path.sep
- def test_mk_env_vars(self):
+ # list comparison files for tarball content
+ tar_test_dir = os.path.join(self.testdir, 'InstallTar')
+ cmp_dir = _config.FLEXEXTRACT_DIRNAME + '_local'
+ tar_test_fedir = os.path.join(tar_test_dir, cmp_dir)
+ comparison_list = []
+ for path, subdirs, files in os.walk(tar_test_fedir):
+ for name in files:
+ if 'tar' not in name:
+ comparison_list.append(os.path.relpath(
+ os.path.join(path, name), tar_test_fedir))
+
+ # untar in test directory
+ test_dir = os.path.join(tar_test_dir, 'test_local')
+ make_dir(test_dir)
+ os.chdir(test_dir)
+ tarballname = _config.FLEXEXTRACT_DIRNAME + '_local.tar'
+ un_tarball(os.path.join(tar_test_dir, tarballname))
+ tarfiles_list = []
+ for path, subdirs, files in os.walk(test_dir):
+ for name in files:
+ tarfiles_list.append(os.path.relpath(
+ os.path.join(path, name), test_dir))
+
+ # test for equality
+ assert sorted(tarfiles_list) == sorted(comparison_list)
+
+ @patch('_config.PATH_ECMWF_ENV', _config_test.PATH_TESTFILES_DIR+'/ecmwf_test')
+ def test_success_mk_env_vars(self):
import filecmp
- # comparison file
- testfile = os.path.join(_config.PATH_TEST_DIR,'TestData',
+ cmp_file = os.path.join(self.testfilesdir,
'ECMWF_ENV.test')
- # create test ECMWF_ENV file
- install.mk_env_vars('testuser',
+ mk_env_vars('testuser',
+ 'testgroup',
+ 'gateway.test.ac.at',
+ 'user@destination')
+
+ assert filecmp.cmp(cmp_file, _config.PATH_ECMWF_ENV, shallow=False)
+
+ silent_remove(_config.PATH_ECMWF_ENV)
+
+ @patch('genshi.template.TemplateLoader', side_effect=[OSError])
+ def test_fail_load_mk_env_vars(self, mock_generate):
+ with pytest.raises(SystemExit):
+ mk_env_vars('testuser',
+ 'testgroup',
+ 'gateway.test.ac.at',
+ 'user@destination')
+
+ def test_fail_generate_mk_env_vars(self):
+ with patch('genshi.template.TemplateLoader.load') as MockHelper:
+ MockHelper.return_value.generate.side_effect = UndefinedError('undefined')
+ with pytest.raises(SystemExit):
+ mk_env_vars('testuser',
'testgroup',
'gateway.test.ac.at',
'user@destination')
- assert filecmp.cmp(testfile, _config.PATH_ECMWF_ENV, shallow=False)
-
- # delte test file
- silent_remove(_config.PATH_ECMWF_ENV)
+ @patch('__builtin__.open', side_effect=[OSError(errno.EPERM)])
+ def test_fail_open_mk_env_vars(self, mock_open):
+ with pytest.raises(SystemExit):
+ mk_env_vars('testuser',
+ 'testgroup',
+ 'gateway.test.ac.at',
+ 'user@destination')
- def test_mk_compilejob(self):
+ @patch('_config.FILE_INSTALL_COMPILEJOB', _config_test.PATH_TESTFILES_DIR+'/compilejob_test.ksh')
+ def test_success_mk_compilejob(self):
import filecmp
- # comparison file
- testfile = os.path.join(_config.PATH_TEST_DIR,'TestData',
- 'compilejob.test')
+ testfile = os.path.join(self.testfilesdir,
+ 'compilejob.test')
- # create
- install.mk_compilejob('Makefile.TEST',
- '',
- 'testuser',
- 'testgroup',
- 'fp_root_test_path')
+ mk_compilejob('Makefile.TEST',
+ '',
+ 'testuser',
+ 'testgroup',
+ 'fp_root_test_path')
finalfile = os.path.join(_config.PATH_JOBSCRIPTS,
- _config.FILE_INSTALL_COMPILEJOB)
+ _config.FILE_INSTALL_COMPILEJOB)
assert filecmp.cmp(testfile, finalfile, shallow=False)
- # delete test file
silent_remove(finalfile)
- def test_mk_job_template(self):
+ @patch('genshi.template.TemplateLoader', side_effect=[OSError])
+ def test_fail_load_mk_compilejob(self, mock_generate):
+ with pytest.raises(SystemExit):
+ mk_compilejob('Makefile.TEST',
+ '',
+ 'testuser',
+ 'testgroup',
+ 'fp_root_test_path')
+
+ def test_fail_generate_mk_compilejob(self):
+ with patch('genshi.template.TemplateLoader.load') as MockHelper:
+ MockHelper.return_value.generate.side_effect = UndefinedError('undefined')
+ with pytest.raises(SystemExit):
+ mk_compilejob('Makefile.TEST',
+ '',
+ 'testuser',
+ 'testgroup',
+ 'fp_root_test_path')
+
+ @patch('__builtin__.open', side_effect=[OSError(errno.EPERM)])
+ def test_fail_open_mk_compilejob(self, mock_open):
+ with pytest.raises(SystemExit):
+ mk_compilejob('Makefile.TEST',
+ '',
+ 'testuser',
+ 'testgroup',
+ 'fp_root_test_path')
+
+ @patch('_config.TEMPFILE_JOB', _config_test.PATH_TESTFILES_DIR+'/job_temp.test_test')
+ def test_success_mk_job_template(self):
import filecmp
- # comparison file
- testfile = os.path.join(_config.PATH_TEST_DIR,
- 'TestData',
+ testfile = os.path.join(self.testfilesdir,
'job.temp.test')
- # create
- install.mk_job_template('testuser',
+ mk_job_template('testuser',
+ 'testgroup',
+ 'gateway.test.ac.at',
+ 'dest@generic',
+ 'fp_root_test_path')
+
+ finalfile = os.path.join(_config.PATH_TEMPLATES,
+ _config.TEMPFILE_JOB)
+ assert filecmp.cmp(testfile, finalfile, shallow=False)
+
+ silent_remove(finalfile)
+
+ @patch('genshi.template.TemplateLoader', side_effect=[OSError])
+ def test_fail_load_mk_job_template(self, mock_generate):
+ with pytest.raises(SystemExit):
+ mk_job_template('testuser',
+ 'testgroup',
+ 'gateway.test.ac.at',
+ 'dest@generic',
+ 'fp_root_test_path')
+
+ def test_fail_generate_mk_job_template(self):
+ with patch('genshi.template.TemplateLoader.load') as MockHelper:
+ MockHelper.return_value.generate.side_effect = UndefinedError('undefined')
+ with pytest.raises(SystemExit):
+ mk_job_template('testuser',
'testgroup',
'gateway.test.ac.at',
'dest@generic',
'fp_root_test_path')
- finalfile = os.path.join(_config.PATH_TEMPLATES,
- _config.TEMPFILE_JOB)
- assert filecmp.cmp(testfile, finalfile, shallow=False)
+ @patch('__builtin__.open', side_effect=[OSError(errno.EPERM)])
+ def test_fail_open_mk_job_template(self, mock_open):
+ with pytest.raises(SystemExit):
+ mk_job_template('testuser',
+ 'testgroup',
+ 'gateway.test.ac.at',
+ 'dest@generic',
+ 'fp_root_test_path')
+
+ @classmethod
+ def teardown_class(self):
+
+ test_dir = os.path.join(self.testinstalldir,
+ _config.FLEXEXTRACT_DIRNAME + '_local')
+ shutil.rmtree(test_dir)
+ test_dir = os.path.join(self.testinstalldir,
+ _config.FLEXEXTRACT_DIRNAME + '_ecgate')
+ shutil.rmtree(test_dir)
+
+ test_dir = os.path.join(self.testinstalldir,
+ 'test_local')
+ shutil.rmtree(test_dir)
+ test_dir = os.path.join(self.testinstalldir,
+ 'test_ecgate')
+ shutil.rmtree(test_dir)
- # delete test file
- silent_remove(finalfile)
\ No newline at end of file
+ tar_file = os.path.join(self.testinstalldir,
+ _config.FLEXEXTRACT_DIRNAME + '_local.tar')
+ os.remove(tar_file)
+ tar_file = os.path.join(self.testinstalldir,
+ _config.FLEXEXTRACT_DIRNAME + '_ecgate.tar')
+ os.remove(tar_file)
+ pass
diff --git a/source/pythontest/_config_test.py b/source/pythontest/_config_test.py
index 9eeca9db2e201a6c971b9d5bee6f17503a3a2fb9..359add1e15bb9500352cb7e829a29b0eadb55f18 100644
--- a/source/pythontest/_config_test.py
+++ b/source/pythontest/_config_test.py
@@ -39,10 +39,10 @@ import _config
# ------------------------------------------------------------------------------
TEST_DIR = 'test'
TESTFILES_DIR = 'Testfiles'
-
+TESTINSTALL_DIR = 'InstallTar'
# ------------------------------------------------------------------------------
# PATHES
# ------------------------------------------------------------------------------
PATH_TEST_DIR = os.path.join(_config.PATH_FLEXEXTRACT_DIR, TEST_DIR)
PATH_TESTFILES_DIR = os.path.join(PATH_TEST_DIR, TESTFILES_DIR)
-
+PATH_TESTINSTALL_DIR = os.path.join(PATH_TEST_DIR, TESTINSTALL_DIR)
diff --git a/test/InstallTar/mk_install_tar.sh b/test/InstallTar/mk_install_tar.sh
new file mode 100755
index 0000000000000000000000000000000000000000..b2771ae9a08b7a87d09f7e30d3b342923fbb93bf
--- /dev/null
+++ b/test/InstallTar/mk_install_tar.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# @Author: Anne Philipp
+#
+# @Date: November, 10 2018
+#
+# @Description: Makes a tar-ball for installation
+#
+
+# path to flex_extract base directory
+path=../../
+
+tarname='flex_extract_v7.1_local.tar'
+
+tar -cvf ../../test/InstallTar/$tarname \
+ ${path}source/python/classes/*py \
+ ${path}source/python/mods/*py \
+ ${path}source/python/*py \
+ ${path}source/pythontest/*py \
+ ${path}source/fortran/*.f \
+ ${path}source/fortran/*.f90 \
+ ${path}source/fortran/*.h \
+ ${path}source/fortran/Makefile* \
+ ${path}templates/convert.nl \
+ ${path}templates/*.temp \
+ ${path}templates/ecmwf_grib1_table_128 \
+ ${path}run/run_local.sh \
+ ${path}run/control/CONTROL* \
+ ${path}run/jobscripts \
+ ${path}LICENSE.md \
+ ${path}CODE_OF_CONDUCT.md \
+ ${path}README.md \
+ --exclude=*.ksh
+
+
+
+tarname='flex_extract_v7.1_ecgate.tar'
+
+tar -cvf ../../test/InstallTar/$tarname \
+ ${path}source/python/classes/*py \
+ ${path}source/python/mods/*py \
+ ${path}source/python/*py \
+ ${path}source/pythontest/*py \
+ ${path}source/fortran/*.f \
+ ${path}source/fortran/*.f90 \
+ ${path}source/fortran/*.h \
+ ${path}source/fortran/Makefile* \
+ ${path}templates/convert.nl \
+ ${path}templates/*.temp \
+ ${path}templates/ecmwf_grib1_table_128 \
+ ${path}run/ECMWF_ENV \
+ ${path}run/run.sh \
+ ${path}run/control/CONTROL* \
+ ${path}run/jobscripts \
+ ${path}LICENSE.md \
+ ${path}CODE_OF_CONDUCT.md \
+ ${path}README.md \
+ --exclude=*.ksh
+
+
+
+
+
+
\ No newline at end of file
diff --git a/test/InstallTar/un_install_tar.sh b/test/InstallTar/un_install_tar.sh
new file mode 100755
index 0000000000000000000000000000000000000000..ac34be7cb7b19261c473512da0be7ebac68ab4ab
--- /dev/null
+++ b/test/InstallTar/un_install_tar.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# @Author: Anne Philipp
+#
+# @Date: November, 10 2018
+#
+# @Description: Untar a tar-ball for installation
+#
+
+tarname='flex_extract_v7.1_local.tar'
+dirname='flex_extract_v7.1_local'
+path=../../test/InstallTar/
+mkdir $path$dirname
+cd $path$dirname
+tar xvf ../$tarname
+cd ../../../source/pythontest
+
+tarname='flex_extract_v7.1_ecgate.tar'
+dirname='flex_extract_v7.1_ecgate'
+path=../../test/InstallTar/
+mkdir $path$dirname
+cd $path$dirname
+tar xvf ../$tarname
+cd ../../../source/pythontest
\ No newline at end of file
diff --git a/test/Testfiles/compilejob.test b/test/Testfiles/compilejob.test
new file mode 100644
index 0000000000000000000000000000000000000000..3a4646f868234a3db43d23a818ec38b89b6f1ba6
--- /dev/null
+++ b/test/Testfiles/compilejob.test
@@ -0,0 +1,69 @@
+#!/bin/ksh
+
+# ON ECGB:
+# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server
+# start with sbatch NAME_OF_THIS_FILE directly on machine
+
+#SBATCH --workdir=/scratch/ms/testgroup/testuser
+#SBATCH --qos=normal
+#SBATCH --job-name=flex_ecmwf
+#SBATCH --output=flex_ecmwf.%j.out
+#SBATCH --error=flex_ecmwf.%j.out
+#SBATCH --mail-type=FAIL
+#SBATCH --time=12:00:00
+
+## CRAY specific batch requests
+##PBS -N flex_ecmwf
+##PBS -q ns
+##PBS -S /usr/bin/ksh
+##PBS -o /scratch/ms/testgroup/testuser/flex_ecmwf.${Jobname}.${Job_ID}.out
+# job output is in .ecaccess_DO_NOT_REMOVE
+##PBS -j oe
+##PBS -V
+##PBS -l EC_threads_per_task=1
+##PBS -l EC_memory_per_task=3200MB
+
+set -x
+export VERSION=7.1
+case ${HOST} in
+ *ecg*)
+ module unload grib_api
+ module unload eccodes
+ module load eccodes
+ module load python
+ module unload emos
+ module load emos/455-r64
+ export FLEXPART_ROOT_SCRIPTS=fp_root_test_path
+ export MAKEFILE=Makefile.TEST
+ ;;
+ *cca*)
+ module switch PrgEnv-cray PrgEnv-intel
+ module load eccodes
+ module load emos
+ module load python
+ echo ${GROUP}
+ echo ${HOME}
+ echo ${HOME} | awk -F / '{print $1, $2, $3, $4}'
+ export GROUP=`echo ${HOME} | awk -F / '{print $4}'`
+ export SCRATCH=/scratch/ms/${GROUP}/${USER}
+ export FLEXPART_ROOT_SCRIPTS=fp_root_test_path
+ export MAKEFILE=Makefile.TEST
+ ;;
+esac
+
+mkdir -p ${FLEXPART_ROOT_SCRIPTS}/flex_extract_v${VERSION}
+cd ${FLEXPART_ROOT_SCRIPTS}/flex_extract_v${VERSION} # if FLEXPART_ROOT is not set this means cd to the home directory
+tar -xvf ${HOME}/flex_extract_v${VERSION}.tar
+cd source/fortran
+\rm *.o *.mod CONVERT2
+make -f ${MAKEFILE} >flexcompile 2>flexcompile
+
+ls -l CONVERT2 >>flexcompile
+if [ $? -eq 0 ]; then
+ echo 'SUCCESS!' >>flexcompile
+ mail -s flexcompile.${HOST}.$$ ${USER} <flexcompile
+else
+ echo Environment: >>flexcompile
+ env >> flexcompile
+ mail -s "ERROR! flexcompile.${HOST}.$$" ${USER} <flexcompile
+fi
diff --git a/test/Testfiles/job.temp.test b/test/Testfiles/job.temp.test
new file mode 100644
index 0000000000000000000000000000000000000000..23424a3a873daf2cecd1ec99161d3e9286a3f083
--- /dev/null
+++ b/test/Testfiles/job.temp.test
@@ -0,0 +1,77 @@
+#!/bin/ksh
+
+# ON ECGB:
+# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server
+# start with sbatch NAME_OF_THIS_FILE directly on machine
+
+#SBATCH --workdir=/scratch/ms/testgroup/testuser
+#SBATCH --qos=normal
+#SBATCH --job-name=flex_ecmwf
+#SBATCH --output=flex_ecmwf.%j.out
+#SBATCH --error=flex_ecmwf.%j.out
+#SBATCH --mail-type=FAIL
+#SBATCH --time=12:00:00
+
+## CRAY specific batch requests
+##PBS -N flex_ecmwf
+##PBS -q np
+##PBS -S /usr/bin/ksh
+## -o /scratch/ms/testgroup/testuser/flex_ecmwf.${PBS_JOBID}.out
+## job output is in .ecaccess_DO_NOT_REMOVE
+##PBS -j oe
+##PBS -V
+##PBS -l EC_threads_per_task=24
+##PBS -l EC_memory_per_task=32000MB
+
+set -x
+export VERSION=7.1
+case ${HOST} in
+ *ecg*)
+ module load python
+ module unload grib_api
+ module unload eccodes
+ module load eccodes
+ module unload emos
+ module load emos/455-r64
+ export PATH=${PATH}:fp_root_test_path/flex_extract_v7.1/source/python
+ ;;
+ *cca*)
+ module switch PrgEnv-cray PrgEnv-intel
+ module load eccodes
+ module load emos
+ module load python
+ export SCRATCH=${TMPDIR}
+ export PATH=${PATH}:fp_root_test_path/flex_extract_v7.1/source/python
+ ;;
+esac
+
+cd ${SCRATCH}
+mkdir -p python$$
+cd python$$
+
+export CONTROL=CONTROL
+
+cat >${CONTROL}<<EOF
+EOF
+
+
+submit.py --controlfile=${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1
+
+if [ $? -eq 0 ] ; then
+ l=0
+ for muser in `grep -i MAILOPS ${CONTROL}`; do
+ if [ ${l} -gt 0 ] ; then
+ mail -s flex.${HOST}.$$ ${muser} <prot
+ fi
+ l=$((${l}+1))
+ done
+else
+ l=0
+ for muser in `grep -i MAILFAIL ${CONTROL}`; do
+ if [ ${l} -gt 0 ] ; then
+ mail -s "ERROR! flex.${HOST}.$$" ${muser} <prot
+ fi
+ l=$((${l}+1))
+ done
+fi
+