Skip to content
Snippets Groups Projects
Commit 96e15333 authored by Anne Philipp's avatar Anne Philipp
Browse files

redefined test data dir and completed unittests for tools module

parent 8500ba14
No related branches found
No related tags found
No related merge requests found
Showing
with 815 additions and 220 deletions
......@@ -29,7 +29,7 @@ import inspect
_VERSION_STR = '7.1'
QUEUES_LIST = ['ecgate', 'cca']
QUEUES_LIST = ['ecgate', 'cca', 'ccb']
# ------------------------------------------------------------------------------
# FILENAMES
......@@ -77,7 +77,7 @@ PATH_ECMWF_ENV = os.path.join(PATH_RUN_DIR, FILE_USER_ENVVARS)
PATH_GRIBTABLE = os.path.join(PATH_TEMPLATES, FILE_GRIBTABLE)
PATH_JOBSCRIPTS = os.path.join(PATH_RUN_DIR, 'jobscripts')
PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'fortran')
PATH_TEST_DIR = os.path.join(PATH_SOURCES, 'pythontest')
PATH_PYTHONTEST_SRC = os.path.join(PATH_SOURCES, 'pythontest')
PATH_INPUT_DIR = os.path.join(PATH_RUN_DIR, INPUT_DIRNAME_DEFAULT)
if os.getenv('CONTROL'):
# this is only needed if gateway version with job script is used!
......
......@@ -147,6 +147,7 @@ class ControlFile(object):
self.debug = 0
self.request = 0
self.public = 0
self.ecapi = None
self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
......
......@@ -754,15 +754,12 @@ class EcFlexpart(object):
# index_vals[1]: ('0', '1200', '1800', '600') ; time
# index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
# initialize dictionaries
valsdict = {}
svalsdict = {}
# stepsdict = {}
for p in pars:
valsdict[str(p)] = []
svalsdict[str(p)] = []
# stepsdict[str(p)] = []
print('maxstep: ', c.maxstep)
# "product" genereates each possible combination between the
# values of the index keys
......@@ -827,7 +824,6 @@ class EcFlexpart(object):
values = grib_get_values(gid)
vdp = valsdict[cparamId]
svdp = svalsdict[cparamId]
# sd = stepsdict[cparamId]
if cparamId == '142' or cparamId == '143':
fak = 1. / 1000.
......@@ -1193,8 +1189,8 @@ class EcFlexpart(object):
ofile = os.path.join(self.inputdir, ofile)
if c.format.lower() == 'grib2':
p = subprocess.check_call(['grib_set', '-s', 'edition=2, \
productDefinitionTemplateNumber=8',
p = subprocess.check_call(['grib_set', '-s', 'edition=2,',
'productDefinitionTemplateNumber=8',
ofile, ofile + '_2'])
p = subprocess.check_call(['mv', ofile + '_2', ofile])
......
......@@ -41,7 +41,22 @@
# - do_retrievement
#
#*******************************************************************************
"""ToDo: Name of litte program
ToDo: Add desccription
ToDo: Add Conditions
This script requires that `...` be installed within the Python
environment you are running this script in.
This file can also be imported as a module and contains the following
functions:
* get_mars_data -
* do_retrievement -
* main - the main function of the script
"""
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
......@@ -90,7 +105,7 @@ def main():
c.check_conditions(args.queue)
get_mars_data(c)
normal_exit(c.mailfail, 'Done!')
normal_exit(c.mailops, c.queue, 'Done!')
return
......
......@@ -53,6 +53,7 @@ import sys
import glob
import subprocess
import traceback
import exceptions
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
# ------------------------------------------------------------------------------
......@@ -187,13 +188,13 @@ def get_cmdline_arguments():
return args
def read_ecenv(filename):
def read_ecenv(filepath):
'''Reads the file into a dictionary where the key values are the parameter
names.
Parameters
----------
filename : :obj:`string`
filepath : :obj:`string`
Path to file where the ECMWF environment parameters are stored.
Return
......@@ -203,16 +204,25 @@ def read_ecenv(filename):
and destination for ECMWF server environments.
'''
envs= {}
with open(filename, 'r') as f:
try:
with open(filepath, 'r') as f:
for line in f:
data = line.strip().split()
envs[str(data[0])] = str(data[1])
except OSError as e:
print('... ERROR CODE: ' + str(e.errno))
print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
sys.exit('\n... Error occured while trying to read ECMWF_ENV '
'file: ' + str(filepath))
return envs
def clean_up(c):
'''Remove all files from intermediate directory (inputdir).
'''Remove files from the intermediate directory (inputdir).
It keeps the final FLEXPART input files if program runs without
ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
Parameters
----------
......@@ -225,16 +235,19 @@ def clean_up(c):
'''
print("clean_up")
print("... clean inputdir!")
cleanlist = glob.glob(c.inputdir + "/*")
for clist in cleanlist:
if c.prefix not in clist:
silent_remove(clist)
if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
silent_remove(clist)
cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
print("Done")
if cleanlist:
for element in cleanlist:
if c.prefix not in element:
silent_remove(element)
if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
silent_remove(element)
print("... done!")
else:
print("... nothing to clean!")
return
......@@ -258,43 +271,32 @@ def my_error(users, message='ERROR'):
'''
print(message)
# comment if user does not want email notification directly from python
for user in users:
if '${USER}' in user:
user = os.getenv('USER')
try:
p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
os.path.expandvars(user)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1)
trace = '\n'.join(traceback.format_stack())
pout = p.communicate(input=message + '\n\n' + trace)[0]
except ValueError as e:
print('ERROR: ', e)
sys.exit('Email could not be sent!')
else:
print('Email sent to ' + os.path.expandvars(user) + ' ' +
pout.decode())
full_message = message + '\n\n' + trace
print(full_message)
send_mail(users, 'ERROR', full_message)
sys.exit(1)
return
def normal_exit(users, message='Done!'):
def send_mail(users, success_mode, message):
'''Prints a specific exit message which can be passed to the function.
Parameters
----------
user : :obj:`list` of :obj:`string`
users : :obj:`list` of :obj:`string`
Contains all email addresses which should be notified.
It might also contain just the ecmwf user name which wil trigger
mailing to the associated email address for this user.
success_mode : :obj:``string`
States the exit mode of the program to put into
the mail subject line.
message : :obj:`string`, optional
Message for exiting program. Default value is "Done!".
......@@ -302,32 +304,53 @@ def normal_exit(users, message='Done!'):
------
'''
print(message)
# comment if user does not want notification directly from python
for user in users:
if '${USER}' in user:
user = os.getenv('USER')
try:
p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
os.path.expandvars(user)],
p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
success_mode, os.path.expandvars(user)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1)
pout = p.communicate(input=message + '\n\n')[0]
except ValueError as e:
print('ERROR: ', e)
print('Email could not be sent!')
print('... ERROR: ' + str(e))
sys.exit('... Email could not be sent!')
except OSError as e:
print('... ERROR CODE: ' + str(e.errno))
print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
sys.exit('... Email could not be sent!')
else:
print('Email sent to ' + os.path.expandvars(user) + ' ' +
pout.decode())
print('Email sent to ' + os.path.expandvars(user))
return
def normal_exit(message='Done!'):
'''Prints a specific exit message which can be passed to the function.
Parameters
----------
message : :obj:`string`, optional
Message for exiting program. Default value is "Done!".
Return
------
'''
print(str(message))
return
def product(*args, **kwds):
'''This method combines the single characters of the passed arguments
'''Creates combinations of all passed arguments.
This method combines the single characters of the passed arguments
with each other. So that each character of each argument value
will be combined with each character of the other arguments as a tuple.
......@@ -344,7 +367,7 @@ def product(*args, **kwds):
Parameters
----------
\*args : :obj:`tuple`
\*args : :obj:`list` or :obj:`string`
Positional arguments (arbitrary number).
\*\*kwds : :obj:`dictionary`
......@@ -356,12 +379,15 @@ def product(*args, **kwds):
Return will be done with "yield". A tuple of combined arguments.
See example in description above.
'''
try:
pools = map(tuple, args) * kwds.get('repeat', 1)
result = [[]]
for pool in pools:
result = [x + [y] for x in result for y in pool]
for prod in result:
yield tuple(prod)
except TypeError as e:
sys.exit('... PRODUCT GENERATION FAILED!')
return
......@@ -382,8 +408,10 @@ def silent_remove(filename):
try:
os.remove(filename)
except OSError as e:
if e.errno != errno.ENOENT:
# errno.ENOENT = no such file or directory
if e.errno == errno.ENOENT:
pass
else:
raise # re-raise exception if a different error occured
return
......@@ -405,8 +433,16 @@ def init128(filepath):
short name of the parameter.
'''
table128 = dict()
try:
with open(filepath) as f:
fdata = f.read().split('\n')
except OSError as e:
print('... ERROR CODE: ' + str(e.errno))
print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
sys.exit('\n... Error occured while trying to read parameter '
'table file: ' + str(filepath))
else:
for data in fdata:
if data[0] != '!':
table128[data[0:3]] = data[59:64].strip()
......@@ -436,6 +472,11 @@ def to_param_id(pars, table):
List of addpar parameters from CONTROL file transformed to
parameter ids in the format of integer.
'''
if not pars:
return []
if not isinstance(pars, str):
pars=str(pars)
cpar = pars.upper().split('/')
ipar = []
for par in cpar:
......@@ -466,18 +507,22 @@ def get_list_as_string(list_obj, concatenate_sign=', '):
The content of the list as a single string.
'''
if not isinstance(list_obj, list):
list_obj = list(list_obj)
str_of_list = concatenate_sign.join(str(l) for l in list_obj)
return str_of_list
def make_dir(directory):
'''Creates a directory and gives a warning if the directory
already exists. The program stops only if there is another problem.
'''Creates a directory.
It gives a warning if the directory already exists and skips process.
The program stops only if there is another problem.
Parameters
----------
directory : :obj:`string`
The directory name including the path which should be created.
The path to directory which should be created.
Return
------
......@@ -486,11 +531,11 @@ def make_dir(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
# errno.EEXIST = directory already exists
raise # re-raise exception if a different error occured
else:
if e.errno == errno.EEXIST:
print('WARNING: Directory {0} already exists!'.format(directory))
else:
raise # re-raise exception if a different error occured
return
......@@ -522,26 +567,30 @@ def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
Return
------
rcode : :obj:`string`
Resulting code of command execution. If successful the string
will be empty.
'''
try:
rcode = subprocess.check_output(['ecaccess-file-put',
subprocess.check_output(['ecaccess-file-put',
ecd + '/' + filename,
target + ':/home/ms/' +
ecgid + '/' + ecuid +
'/' + filename],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print('... ERROR CODE:\n ... ' + str(e.returncode))
print('... ERROR MESSAGE:\n ... ' + str(e))
print('... ERROR CODE: ' + str(e.returncode))
print('... ERROR MESSAGE:\n \t ' + str(e))
print('\n... Do you have a valid ecaccess certification key?')
sys.exit('... ECACCESS-FILE-PUT FAILED!')
except OSError as e:
print('... ERROR CODE: ' + str(e.errno))
print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
return rcode
print('\n... Most likely the ECACCESS library is not available!')
sys.exit('... ECACCESS-FILE-PUT FAILED!')
return
def submit_job_to_ecserver(target, jobname):
'''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
......@@ -562,22 +611,25 @@ def submit_job_to_ecserver(target, jobname):
Return
------
rcode : :obj:`string`
Resulting code of command execution. If successful the string
will contain an integer number, representing the id of the job
at the ecmwf server.
job_id : :obj:`int`
The id number of the job as a reference at the ecmwf server.
'''
try:
rcode = subprocess.check_output(['ecaccess-job-submit',
'-queueName', target,
jobname])
except subprocess.CalledProcessError as e:
print('... ERROR CODE:\n ... ' + str(e.returncode))
print('... ERROR MESSAGE:\n ... ' + str(e))
job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
target, jobname])
except subprocess.CalledProcessError as e:
print('... ERROR CODE: ' + str(e.returncode))
print('... ERROR MESSAGE:\n \t ' + str(e))
print('\n... Do you have a valid ecaccess certification key?')
sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
except OSError as e:
print('... ERROR CODE: ' + str(e.errno))
print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
print('\n... Most likely the ECACCESS library is not available!')
sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
return rcode
return job_id
......@@ -93,13 +93,15 @@ def main():
get_mars_data(c)
if c.request == 0 or c.request == 2:
prepare_flexpart(args.ppid, c)
normal_exit(c.mailfail, 'FLEX_EXTRACT IS DONE!')
exit_message = 'FLEX_EXTRACT IS DONE!'
else:
normal_exit(c.mailfail, 'PRINTING MARS_REQUESTS DONE!')
# send files to ECMWF server and install there
exit_message = 'PRINTING MARS_REQUESTS DONE!'
# send files to ECMWF server
else:
submit(args.job_template, c, args.queue)
normal_exit(c.mailops, c.queue, exit_message)
return
def submit(jtemplate, c, queue):
......@@ -146,7 +148,7 @@ def submit(jtemplate, c, queue):
with open(job_file, 'w') as f:
f.write('\n'.join(lftextondemand))
submit_job_to_ecserver(queue, job_file)
job_id = submit_job_to_ecserver(queue, job_file)
else:
# --------- create operational job script ----------------------------------
......@@ -172,9 +174,10 @@ def submit(jtemplate, c, queue):
with open(job_file, 'w') as f:
f.write('\n'.join(lftextoper))
submit_job_to_ecserver(queue, job_file)
job_id = submit_job_to_ecserver(queue, job_file)
# --------------------------------------------------------------------------
print('The job id is: ' + str(job_id.strip()))
print('You should get an email with subject flex.hostname.pid')
return
......
#!/usr/bin/env python
#!/opt/anaconda/bin/python
# -*- coding: utf-8 -*-
# for the gateway tests, the env vars of ECUID and ECGID have to be set upfront
import os
import sys
import errno
from exceptions import OSError
import subprocess
import pipes
import pytest
from mock import patch, call
#from mock import PropertyMock
sys.path.append('../python')
import _config
from mods.tools import (get_cmdline_arguments, read_ecenv, clean_up, my_error,
normal_exit, product, silent_remove, init128,
to_param_id, get_list_as_string, make_dir,
import _config_test
from classes.ControlFile import ControlFile
from mods.tools import (none_or_str, none_or_int, get_cmdline_arguments,
read_ecenv, clean_up, my_error, send_mail,
normal_exit, product, silent_remove,
init128, to_param_id, get_list_as_string, make_dir,
put_file_to_ecserver, submit_job_to_ecserver)
class TestTools():
class TestTools(object):
'''
'''
def test_get_cmdline_arguments(self):
def setup_method(self):
self.testdir = _config_test.PATH_TEST_DIR
self.testfilesdir = _config_test.PATH_TESTFILES_DIR
self.c = ControlFile(self.testdir+'/Controls/CONTROL.test')
def test_nonestr_none_or_int(self):
assert None == none_or_int('None')
def test_intstr_none_or_int(self):
assert 42 == none_or_int('42')
def test_nonestr_none_or_str(self):
assert None == none_or_str('None')
def test_anystr_none_or_str(self):
assert 'test' == none_or_str('test')
def test_default_get_cmdline_arguments(self):
cmd_dict_control = {'start_date':None,
'end_date':None,
'date_chunk':None,
'basetime':None,
'step':None,
'levelist':None,
'area':None,
'inputdir':None,
'outputdir':None,
'flexpart_root_scripts':None,
'ppid':None,
'job_template':'job.temp',
'queue':None,
'controlfile':'CONTROL.temp',
'debug':None,
'public':None,
'request':None}
sys.argv = ['dummy.py']
results = get_cmdline_arguments()
assert cmd_dict_control == vars(results)
def test_input_get_cmdline_arguments(self):
cmd_dict_control = {'start_date':'20180101',
'end_date':'20180101',
'date_chunk':'3',
'basetime':'12',
'date_chunk':3,
'basetime':12,
'step':'1',
'levelist':'1/to/10',
'area':'50/10/60/20',
'inputdir':'../work',
'outputdir':'../work',
'outputdir':None,
'flexpart_root_scripts':'../',
'ppid':'1234',
'ppid':1234,
'job_template':'job.sh',
'queue':'ecgate',
'controlfile':'CONTROL.WORK',
'debug':'1',
'request':'0'}
'debug':1,
'public':None,
'request':0}
sys.argv = ['dummy.py',
'--start_date=20180101',
......@@ -46,146 +98,271 @@ class TestTools():
'--levelist=1/to/10',
'--area=50/10/60/20',
'--inputdir=../work',
'--outputdir=../work',
'--outputdir=None',
'--flexpart_root_scripts=../',
'--ppid=1234',
'--job_template=job.sh',
'--queue=ecgate',
'--controlfile=CONTROL.WORK',
'--debug=1',
'--public=None',
'--request=0']
results = get_cmdline_arguments()
assert cmd_dict_control == vars(results)
def test_init128(self):
def test_success_init128(self):
table128 = init128(_config.PATH_GRIBTABLE)
expected = {'078': 'TCLW', '130': 'T', '034': 'SST'}
expected_sample = {'078': 'TCLW', '130': 'T', '034': 'SST'}
# check a sample of parameters which must have been read in
result = all((k in table128 and table128[k]==v) for k,v in expected.iteritems())
assert result == True
assert all((k in table128 and table128[k]==v)
for k,v in expected_sample.iteritems())
def test_to_param_id(self):
@patch('__builtin__.open', side_effect=[OSError(errno.EEXIST)])
def test_fail_open_init128(self, mock_openfile):
with pytest.raises(SystemExit):
table128 = init128(_config.PATH_GRIBTABLE)
pars = to_param_id("T/SP/LSP/SSHF", table128)
for par in pars:
assert par in [130, 134, 142, 146]
def test_my_error(self):
with pytest.raises(SystemExit) as pytest_wrapped_e:
my_error(['${USER}', 'anne.philipp@univie.ac.at'], 'Failed!')
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 1
def test_read_ecenv(self):
envs_ref = {'ECUID': 'km4a',
'ECGID': 'at',
'GATEWAY': 'srvx8.img.univie.ac.at',
'DESTINATION': 'annep@genericSftp'
@pytest.mark.parametrize(
'ipar_str, opar_listint',
[('SP/LSP/SSHF', [134, 142, 146]),
('T', [130]),
('', []),
(None, []),
('testtest', []),
('130/142', [130, 142]),
(130, [130]),
(50.56, [])])
def test_to_param_id(self, ipar_str, opar_listint):
table128 = init128(_config.PATH_GRIBTABLE)
ipars = to_param_id(ipar_str, table128)
assert sorted(ipars) == sorted(opar_listint)
@patch('traceback.format_stack', return_value='empty trace')
@patch('mods.tools.send_mail', return_value=0)
def test_success_my_error(self, mock_mail, mock_trace, capfd):
with pytest.raises(SystemExit):
my_error(['any_user'], 'Failed!')
out, err = capfd.readouterr()
assert out == "Failed!\n\nempty_trace\n"
@patch('subprocess.Popen')
@patch('os.getenv', return_value='user')
@patch('os.path.expandvars', return_value='user')
def test_success_userenv_twouser_send_mail(self, mock_os, mock_env, mock_popen, capfd):
mock_popen.return_value = subprocess.Popen(["echo", "Hello Test!"],
stdout=subprocess.PIPE)
send_mail(['${USER}', 'any_user'], 'ERROR', message='error mail')
out, err = capfd.readouterr()
assert out == b'Email sent to user\nEmail sent to user\n'
@patch('subprocess.Popen')
@patch('os.path.expandvars', return_value='any_user')
def test_success_send_mail(self, mock_os, mock_popen, capfd):
mock_popen.return_value = subprocess.Popen(["echo", "Hello Test!"],
stdout=subprocess.PIPE)
send_mail(['any-user'], 'ERROR', message='error mail')
out, err = capfd.readouterr()
assert out == b'Email sent to any_user\n'
@patch('subprocess.Popen', side_effect=[ValueError, OSError])
@patch('os.path.expandvars', return_value='any_user')
def test_fail_valueerror_send_mail(self, mock_osvar, mock_popen):
with pytest.raises(SystemExit): # ValueError
send_mail(['any-user'], 'ERROR', message='error mail')
with pytest.raises(SystemExit): # OSError
send_mail(['any-user'], 'ERROR', message='error mail')
def test_success_read_ecenv(self):
envs_ref = {'ECUID': 'testuser',
'ECGID': 'testgroup',
'GATEWAY': 'gateway.test.ac.at',
'DESTINATION': 'user@destination'
}
envs = read_ecenv(os.getcwd() + '/TestData/ECMWF_ENV')
envs = read_ecenv(self.testfilesdir + '/ECMWF_ENV.test')
assert envs_ref == envs
def test_clean_up(self):
@patch('__builtin__.open', side_effect=[OSError(errno.EPERM)])
def test_fail_read_ecenv(self, mock_open):
with pytest.raises(SystemExit):
read_ecenv('any_file')
assert True
@patch('glob.glob', return_value=[])
@patch('mods.tools.silent_remove')
def test_empty_clean_up(self, mock_rm, mock_clean):
clean_up(self.c)
mock_rm.assert_not_called()
def test_normal_exit(self):
assert True
@patch('glob.glob', return_value=['any_file','EIfile'])
@patch('os.remove', return_value=0)
def test_success_clean_up(self, mock_rm, mock_glob):
# ectrans=0; ecstorage=0; ecapi=None; prefix not in filename
clean_up(self.c)
mock_rm.assert_has_calls([call('any_file'), call('EIfile')])
mock_rm.reset_mock()
def test_product(self):
assert True
# ectrans=0; ecstorage=0; ecapi=False; prefix in filename
self.c.prefix = 'EI'
self.c.ecapi = False
clean_up(self.c)
mock_rm.assert_has_calls([call('any_file')])
mock_rm.reset_mock()
def test_success_silent_remove(self, capfd):
testfile = 'testfile.test'
open(testfile, 'w').close()
silent_remove(testfile)
# ectrans=0; ecstorage=0; ecapi=True; prefix in filename
self.c.prefix = 'EI'
self.c.ecapi = True
clean_up(self.c)
mock_rm.assert_has_calls([call('any_file')])
mock_rm.reset_mock()
# ectrans=1; ecstorage=0; ecapi=True; prefix in filename
self.c.prefix = 'EI'
self.c.ecapi = True
self.c.ectrans = 1
clean_up(self.c)
mock_rm.assert_has_calls([call('any_file')])
mock_rm.reset_mock()
# ectrans=1; ecstorage=0; ecapi=False; prefix in filename
self.c.prefix = 'EI'
self.c.ecapi = False
self.c.ectrans = 1
clean_up(self.c)
mock_rm.assert_has_calls([call('any_file'), call('EIfile')])
mock_rm.reset_mock()
# ectrans=1; ecstorage=1; ecapi=False; prefix in filename
self.c.prefix = 'EI'
self.c.ecapi = False
self.c.ectrans = 1
self.c.ecstorage = 1
clean_up(self.c)
mock_rm.assert_has_calls([call('any_file'), call('EIfile')])
mock_rm.reset_mock()
def test_default_normal_exit(self, capfd):
normal_exit()
out, err = capfd.readouterr()
assert os.path.isfile(testfile) == False
assert out == ''
assert out == 'Done!\n'
def test_failnotexist_silent_remove(self, capfd):
testfile = 'testfile.test'
silent_remove(testfile)
def test_message_normal_exit(self, capfd):
normal_exit('Hi there!')
out, err = capfd.readouterr()
assert out == 'Hi there!\n'
def test_int_normal_exit(self, capfd):
normal_exit(42)
out, err = capfd.readouterr()
assert out == '42\n'
@pytest.mark.parametrize(
'input1, input2, output_list',
[('ABC','xy',[('A','x'),('A','y'),('B','x'),('B','y'),('C','x'),('C','y')]),
(range(1), range(1), [(0,0),(0,1),(1,0),(1,1)])])
def test_success_product(self, input1, input2, output_list):
index = 0
for prod in product(input1, input2):
assert isinstance(prod, tuple)
assert prod == output_list[index]
index += 1
@pytest.mark.parametrize(
'input1, input2, output_list',
[(1,1,(1,1))])
def test_fail_product(self, input1, input2, output_list):
index = 0
with pytest.raises(SystemExit):
for prod in product(input1, input2):
assert isinstance(prod, tuple)
assert prod == output_list[index]
index += 1
def test_success_silent_remove(self):
testfile = self.testfilesdir + 'test.txt'
open(testfile, 'w').close()
silent_remove(testfile)
assert os.path.isfile(testfile) == False
assert out == ''
@pytest.mark.skip(reason="no way of currently testing this")
def test_failany_silent_remove(self):
testfile = 'testfileany.test'
@patch('os.remove', side_effect=OSError(errno.ENOENT))
def test_fail_notexist_silent_remove(self, mock_rm):
with pytest.raises(OSError) as pytest_wrapped_e:
silent_remove(testfile)
#out, err = capfd.readouterr()
#assert os.path.isfile(testfile) == False
#assert out == ''
def test_success_get_list_as_string(self):
list_object = [1, 2, 3, '...', 'testlist']
list_as_string = '1, 2, 3, ..., testlist'
assert list_as_string == get_list_as_string(list_object)
@pytest.mark.skip(reason="no way of currently testing this")
def test_fail_get_list_as_string(self):
list_object = [1, 2, 3, '...', 'testlist']
list_as_string = '1, 2, 3, ..., testlist'
with pytest.raises(Exception) as pytest_wrapped_e:
result = get_list_as_string(list_object)
assert result == list_as_string
def test_warningexist_make_dir(self, capfd):
testdir = 'TestData'
make_dir(testdir)
out, err = capfd.readouterr()
assert out.strip() == 'WARNING: Directory {0} already exists!'.format(testdir)
silent_remove('any_dir')
assert pytest_wrapped_e.e.errno == errno.ENOENT
@patch('os.remove', side_effect=OSError(errno.EEXIST))
def test_fail_any_silent_remove(self, mock_rm):
with pytest.raises(OSError):
silent_remove('any_dir')
def test_failany_make_dir(self):
testdir = '/test' # force a permission denied error
@pytest.mark.parametrize(
'input_list, output_list',
[([],''),
([1, 2, 3.5, '...', 'testlist'], '1, 2, 3.5, ..., testlist'),
('2', '2')])
def test_success_get_list_as_string(self, input_list, output_list):
assert output_list == get_list_as_string(input_list)
@patch('os.makedirs', side_effect=[OSError(errno.EEXIST)])
def test_warning_exist_make_dir(self, mock_make):
with pytest.raises(OSError) as pytest_wrapped_e:
make_dir(testdir)
assert pytest_wrapped_e.type == OSError
make_dir('existing_dir')
assert pytest_wrapped_e.e.errno == errno.EEXIST
@patch('os.makedirs', side_effect=OSError)
def test_fail_any_make_dir(self, mock_makedir):
with pytest.raises(OSError):
make_dir('any_dir')
def test_fail_empty_make_dir(self):
with pytest.raises(OSError):
make_dir('')
def test_success_make_dir(self):
testdir = 'testing_mkdir'
make_dir(testdir)
assert os.path.exists(testdir) == True
os.rmdir(testdir)
testdir = '/testing_mkdir'
make_dir(self.testdir + testdir)
assert os.path.exists(self.testdir + testdir) == True
os.rmdir(self.testdir + testdir)
def test_fail_put_file_to_ecserver(self):
ecuid=os.environ['ECUID']
ecgid=os.environ['ECGID']
with pytest.raises(SystemExit) as pytest_wrapped_e:
put_file_to_ecserver('TestData/', 'testfil.txt',
'ecgate', ecuid, ecgid)
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == '... ECACCESS-FILE-PUT FAILED!'
def test_success_put_file_to_ecserver(self):
ecuid=os.environ['ECUID']
ecgid=os.environ['ECGID']
result = put_file_to_ecserver('TestData/', 'testfile.txt',
'ecgate', ecuid, ecgid)
assert result == ''
@patch('subprocess.check_output', side_effect=[subprocess.CalledProcessError(1,'test')])
def test_fail_put_file_to_ecserver(self, mock_co):
with pytest.raises(SystemExit):
put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt',
'ecgate', 'ex_ECUID', 'ex_ECGID')
@patch('subprocess.check_output', return_value=0)
def test_general_success_put_file_to_ecserver(self, mock_co):
result = put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt',
'ecgate', 'ex_ECUID', 'ex_ECGID')
assert result == None
@pytest.mark.msuser_pw
@pytest.mark.gateway
@pytest.mark.skip(reason="easier to ignore for now - implement in final version")
def test_fullsuccess_put_file_to_ecserver(self):
def test_full_success_put_file_to_ecserver(self):
ecuid = os.environ['ECUID']
ecgid = os.environ['ECGID']
put_file_to_ecserver('TestData/', 'testfile.txt', 'ecgate', ecuid, ecgid)
put_file_to_ecserver(self.testfilesdir, 'test_put_to_ecserver.txt',
'ecgate', ecuid, ecgid)
assert subprocess.call(['ssh', ecuid+'@ecaccess.ecmwf.int' ,
'test -e ' +
pipes.quote('/home/ms/'+ecgid+'/'+ecuid)]) == 0
def test_fail_submit_job_to_ecserver(self):
with pytest.raises(SystemExit) as pytest_wrapped_e:
submit_job_to_ecserver('ecgate', 'job.ksh')
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == '... ECACCESS-JOB-SUBMIT FAILED!'
@patch('subprocess.check_output', side_effect=[subprocess.CalledProcessError(1,'test'),
OSError])
def test_fail_submit_job_to_ecserver(self, mock_co):
with pytest.raises(SystemExit):
job_id = submit_job_to_ecserver('ecgate', 'job.ksh')
@pytest.mark.msuser_pw
@pytest.mark.gateway
@pytest.mark.skip(reason="easier to ignore for now - implement in final version")
def test_success_submit_job_to_ecserver(self):
result = submit_job_to_ecserver('ecgate', 'TestData/testfile.txt')
assert result.strip().isdigit() == True
job_id = submit_job_to_ecserver('ecgate',
os.path.join(self.testfilesdir,
'test_put_to_ecserver.txt'))
assert job_id.strip().isdigit() == True
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
*******************************************************************************
@Author: Anne Philipp (University of Vienna)
@Date: August 2018
@Change History:
@License:
(C) Copyright 2014-2018.
This software is licensed under the terms of the Apache Licence Version 2.0
which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@Description:
Contains constant value parameter for flex_extract.
*******************************************************************************
'''
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import os
import sys
sys.path.append('../python')
import _config
# ------------------------------------------------------------------------------
# FILENAMES
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# DIRECTORY NAMES
# ------------------------------------------------------------------------------
TEST_DIR = 'test'
TESTFILES_DIR = 'Testfiles'
# ------------------------------------------------------------------------------
# PATHES
# ------------------------------------------------------------------------------
PATH_TEST_DIR = os.path.join(_config.PATH_FLEXEXTRACT_DIR, TEST_DIR)
PATH_TESTFILES_DIR = os.path.join(PATH_TEST_DIR, TESTFILES_DIR)
import sys
import pytest
sys.path.append('../python')
import _config
#@pytest.fixture
#def prep_test_env():
# testdir = _config.PATH_TEST_DIR
print ''
......@@ -2,3 +2,4 @@
[pytest]
markers =
msuser_pw: Test that can be executed only as a member-state user. Password required.
gateway: Test that can be executed only in the gateway mode.
DAY1 20100809
DAY2
DTIME 3
TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
CLASS EI
STREAM OPER
NUMBER OFF
EXPVER 1
GRID 1000
LEFT -50000
LOWER -90000
UPPER 90000
RIGHT 50000
LEVEL 60
LEVELIST 1/to/60
RESOL 63
GAUSS 1
ACCURACY 16
OMEGA 0
OMEGADIFF 0
ETA 0
ETADIFF 0
DPDETA 1
SMOOTH 0
FORMAT GRIB1
ADDPAR 186/187/188/235/139/39
PREFIX EN
ECSTORAGE 1
ECTRANS 0
ECFSDIR ectmp:/${USER}/econdemand/
MAILFAIL ${USER}
MAILOPS ${USER}
GRIB2FLEXPART 0
EOF
DTIME 3
TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
CLASS EI
STREAM OPER
EXPVER 1
GRID 5000
LEFT -10000
LOWER 30000
UPPER 40000
RIGHT 10000
LEVELIST 59/to/60
RESOL 63
DAY1 20100101
DAY2
DTIME 3
TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
CLASS EI
STREAM OPER
NUMBER OFF
EXPVER 1
GRID 5000
LEFT -10000
LOWER 30000
UPPER 40000
RIGHT 10000
LEVEL 60
LEVELIST 58/to/60
RESOL 63
GAUSS 1
ACCURACY 16
OMEGA 0
OMEGADIFF 0
ETA 0
ETADIFF 0
DPDETA 1
SMOOTH 0
FORMAT GRIB1
ADDPAR 186/187/188/235/139/39
PREFIX EI
ECSTORAGE 0
ECTRANS 1
ECFSDIR ectmp:/${USER}/econdemand/
MAILFAIL ${USER}
MAILOPS ${USER}
GRIB2FLEXPART 0
EOF
DAY1
DAY2
DTIME 3
TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
CLASS EI
STREAM OPER
NUMBER OFF
EXPVER 1
GRID 1000
LEFT -15000
LOWER 30000
UPPER 75000
RIGHT 45000
LEVEL 60
LEVELIST 55/to/60
RESOL 63
GAUSS 1
ACCURACY 16
OMEGA 0
OMEGADIFF 0
ETA 0
ETADIFF 0
DPDETA 1
SMOOTH 0
FORMAT GRIB1
ADDPAR 186/187/188/235/139/39
PREFIX EI
ECSTORAGE 0
ECTRANS 1
ECFSDIR ectmp:/${USER}/econdemand/
MAILFAIL ${USER}
MAILOPS ${USER}
GRIB2FLEXPART 0
EOF
ECUID km4a
ECGID at
GATEWAY srvx8.img.univie.ac.at
DESTINATION annep@genericSftp
#!/bin/ksh
# ON ECGB:
# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server
# start with sbatch NAME_OF_THIS_FILE directly on machine
#SBATCH --workdir=/scratch/ms/testgroup/testuser
#SBATCH --qos=normal
#SBATCH --job-name=flex_ecmwf
#SBATCH --output=flex_ecmwf.%j.out
#SBATCH --error=flex_ecmwf.%j.out
#SBATCH --mail-type=FAIL
#SBATCH --time=12:00:00
## CRAY specific batch requests
##PBS -N flex_ecmwf
##PBS -q ns
##PBS -S /usr/bin/ksh
##PBS -o /scratch/ms/testgroup/testuser/flex_ecmwf.${Jobname}.${Job_ID}.out
# job output is in .ecaccess_DO_NOT_REMOVE
##PBS -j oe
##PBS -V
##PBS -l EC_threads_per_task=1
##PBS -l EC_memory_per_task=3200MB
set -x
export VERSION=7.1
case ${HOST} in
*ecg*)
module load python
module unload grib_api
module unload emos
module load grib_api/1.14.5
module load emos/437-r64
export FLEXPART_ROOT_SCRIPTS=fp_root_test_path
export MAKEFILE=Makefile.TEST
;;
*cca*)
module switch PrgEnv-cray PrgEnv-intel
module load grib_api
module load emos
module load python
echo ${GROUP}
echo ${HOME}
echo ${HOME} | awk -F / '{print $1, $2, $3, $4}'
export GROUP=`echo ${HOME} | awk -F / '{print $4}'`
export SCRATCH=/scratch/ms/${GROUP}/${USER}
export FLEXPART_ROOT_SCRIPTS=fp_root_test_path
export MAKEFILE=Makefile.TEST
;;
esac
mkdir -p ${FLEXPART_ROOT_SCRIPTS}/flex_extract_v${VERSION}
cd ${FLEXPART_ROOT_SCRIPTS}/flex_extract_v${VERSION} # if FLEXPART_ROOT is not set this means cd to the home directory
tar -xvf ${HOME}/flex_extract_v${VERSION}.tar
cd src
\rm *.o *.mod CONVERT2
make -f ${MAKEFILE} >flexcompile 2>flexcompile
ls -l CONVERT2 >>flexcompile
if [ $? -eq 0 ]; then
echo 'SUCCESS!' >>flexcompile
mail -s flexcompile.${HOST}.$$ ${USER} <flexcompile
else
echo Environment: >>flexcompile
env >> flexcompile
mail -s "ERROR! flexcompile.${HOST}.$$" ${USER} <flexcompile
fi
&NAMGEN
maxl = 61,
maxb = 46,
mlevel = 60,
mlevelist = "55/to/60",
mnauf = 63,
metapar = 77,
rlo0 = -15.0,
rlo1 = 45.0,
rla0 = 30.0,
rla1 = 75.0,
momega = 0,
momegadiff = 0,
mgauss = 1,
msmooth = 0,
meta = 0,
metadiff = 0,
mdpdeta = 1
/
#!/bin/ksh
# ON ECGB:
# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server
# start with sbatch NAME_OF_THIS_FILE directly on machine
#SBATCH --workdir=/scratch/ms/testgroup/testuser
#SBATCH --qos=normal
#SBATCH --job-name=flex_ecmwf
#SBATCH --output=flex_ecmwf.%j.out
#SBATCH --error=flex_ecmwf.%j.out
#SBATCH --mail-type=FAIL
#SBATCH --time=12:00:00
## CRAY specific batch requests
##PBS -N flex_ecmwf
##PBS -q np
##PBS -S /usr/bin/ksh
## -o /scratch/ms/testgroup/testuser/flex_ecmwf.${PBS_JOBID}.out
## job output is in .ecaccess_DO_NOT_REMOVE
##PBS -j oe
##PBS -V
##PBS -l EC_threads_per_task=24
##PBS -l EC_memory_per_task=32000MB
set -x
export VERSION=7.1
case ${HOST} in
*ecg*)
module load python
module unload grib_api
module unload emos
module load grib_api/1.14.5
module load emos/437-r64
export PATH=${PATH}:fp_root_test_path/flex_extract_v7.1/source/python
;;
*cca*)
module switch PrgEnv-cray PrgEnv-intel
module load grib_api
module load emos
module load python
export SCRATCH=${TMPDIR}
export PATH=${PATH}:fp_root_test_path/flex_extract_v7.1/source/python
;;
esac
cd ${SCRATCH}
mkdir -p python$$
cd python$$
export CONTROL=CONTROL
cat >${CONTROL}<<EOF
EOF
submit.py --controlfile=${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1
if [ $? -eq 0 ] ; then
l=0
for muser in `grep -i MAILOPS ${CONTROL}`; do
if [ ${l} -gt 0 ] ; then
mail -s flex.${HOST}.$$ ${muser} <prot
fi
l=$((${l}+1))
done
else
l=0
for muser in `grep -i MAILFAIL ${CONTROL}`; do
if [ ${l} -gt 0 ] ; then
mail -s "ERROR! flex.${HOST}.$$" ${muser} <prot
fi
l=$((${l}+1))
done
fi
File added
File added
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment