diff --git a/documentation/ECMWF_FPparameter.xlsx b/documentation/ECMWF_FPparameter.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..20207d5a2ffdfa77f4f6686d6fc3332e284e3988 Binary files /dev/null and b/documentation/ECMWF_FPparameter.xlsx differ diff --git a/documentation/ErrorDebug.ods b/documentation/ErrorDebug.ods new file mode 100644 index 0000000000000000000000000000000000000000..11547601ff905a06a7628fc10458e8c94cee0f16 Binary files /dev/null and b/documentation/ErrorDebug.ods differ diff --git a/documentation/GUIDE.odt b/documentation/GUIDE.odt new file mode 100644 index 0000000000000000000000000000000000000000..edfa22d896f30ff66f03c4c980d8d988e8ca0278 Binary files /dev/null and b/documentation/GUIDE.odt differ diff --git a/documentation/SIP.doc b/documentation/SIP.doc new file mode 100644 index 0000000000000000000000000000000000000000..9472fc7fc47a2997d2ec776483fba4b94f1ca981 Binary files /dev/null and b/documentation/SIP.doc differ diff --git a/documentation/SIP.pdf b/documentation/SIP.pdf new file mode 100644 index 0000000000000000000000000000000000000000..71e75f355e5aae9f0d9f127860ec61e1895c290c Binary files /dev/null and b/documentation/SIP.pdf differ diff --git a/documentation/SUT_ondemand.doc b/documentation/SUT_ondemand.doc new file mode 100644 index 0000000000000000000000000000000000000000..e8a7594f3227f34a19c794f5b796b63f9505340a Binary files /dev/null and b/documentation/SUT_ondemand.doc differ diff --git a/documentation/SUT_ondemand.pdf b/documentation/SUT_ondemand.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d2b75f3d69a14a0d9e4952c38b993baa6744dcdc Binary files /dev/null and b/documentation/SUT_ondemand.pdf differ diff --git a/python/pythontest/TestData/CONTROL.temp b/python/pythontest/TestData/CONTROL.temp new file mode 100644 index 0000000000000000000000000000000000000000..c3d097411cfc71a179dc87df88c0407b6f026d7c --- /dev/null +++ b/python/pythontest/TestData/CONTROL.temp @@ -0,0 +1,37 @@ +DAY1 +DAY2 +DTIME 3 +TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC +TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12 +STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11 +CLASS EI +STREAM OPER +NUMBER OFF +EXPVER 1 +GRID 5000 +LEFT -15000 +LOWER 30000 +UPPER 75000 +RIGHT 45000 +LEVEL 60 +LEVELIST 55/to/60 +RESOL 63 +GAUSS 1 +ACCURACY 16 +OMEGA 0 +OMEGADIFF 0 +ETA 0 +ETADIFF 0 +DPDETA 1 +SMOOTH 0 +FORMAT GRIB1 +ADDPAR 186/187/188/235/139/39 +PREFIX EI +ECSTORAGE 0 +ECTRANS 1 +ECFSDIR ectmp:/${USER}/econdemand/ +MAILFAIL ${USER} +MAILOPS ${USER} +GRIB2FLEXPART 0 +EOF + diff --git a/python/pythontest/TestData/ECMWF_ENV b/python/pythontest/TestData/ECMWF_ENV new file mode 100644 index 0000000000000000000000000000000000000000..5af57721445813f4a44c0b9562b8219196daeda9 --- /dev/null +++ b/python/pythontest/TestData/ECMWF_ENV @@ -0,0 +1,4 @@ +ECUID km4a +ECGID at +GATEWAY srvx8.img.univie.ac.at +DESTINATION annep@genericSftp diff --git a/python/pythontest/TestDir/FCOG__ML.20160410.40429.16424.grb b/python/pythontest/TestDir/FCOG__ML.20160410.40429.16424.grb new file mode 100644 index 0000000000000000000000000000000000000000..852a87f120f7f47d0d02897b30c7bdf4edc4ffd3 Binary files /dev/null and b/python/pythontest/TestDir/FCOG__ML.20160410.40429.16424.grb differ diff --git a/python/pythontest/TestDir/FCOG__SL.20160410.40429.16424.grb b/python/pythontest/TestDir/FCOG__SL.20160410.40429.16424.grb new file mode 100644 index 0000000000000000000000000000000000000000..e0a29285929408678276b7313097083e4ad72de1 Binary files /dev/null and b/python/pythontest/TestDir/FCOG__SL.20160410.40429.16424.grb differ diff --git a/python/pythontest/TestDir/FCSH__ML.20160410.40429.16424.grb b/python/pythontest/TestDir/FCSH__ML.20160410.40429.16424.grb new file mode 100644 index 0000000000000000000000000000000000000000..51e775c88990a5037b3a287e20674fd5d3f6aec6 Binary files /dev/null and b/python/pythontest/TestDir/FCSH__ML.20160410.40429.16424.grb differ diff --git a/python/pythontest/TestDir/FCSH__SL.20160410.40429.16424.grb b/python/pythontest/TestDir/FCSH__SL.20160410.40429.16424.grb new file mode 100644 index 0000000000000000000000000000000000000000..3d021c6a4e9eca56fe5ae95522091004f4443688 Binary files /dev/null and b/python/pythontest/TestDir/FCSH__SL.20160410.40429.16424.grb differ diff --git a/python/pythontest/TestDir/SubTestDir/OG_OROLSM__SL.20160410.40429.16424.grb b/python/pythontest/TestDir/SubTestDir/OG_OROLSM__SL.20160410.40429.16424.grb new file mode 100644 index 0000000000000000000000000000000000000000..10f1c96f45b0d5f6251546374d60d42f4b30103d Binary files /dev/null and b/python/pythontest/TestDir/SubTestDir/OG_OROLSM__SL.20160410.40429.16424.grb differ diff --git a/python/pythontest/TestDir/SubTestDir/SubSubTestDir/FCGG__SL.20160410.40429.16424.grb b/python/pythontest/TestDir/SubTestDir/SubSubTestDir/FCGG__SL.20160410.40429.16424.grb new file mode 100644 index 0000000000000000000000000000000000000000..3e87854a2ff76d6f985bf777146a3fef4af382e9 Binary files /dev/null and b/python/pythontest/TestDir/SubTestDir/SubSubTestDir/FCGG__SL.20160410.40429.16424.grb differ diff --git a/python/pythontest/TestDir/SubTestDir2/FCOG_acc_SL.20160409.40429.16424.grb b/python/pythontest/TestDir/SubTestDir2/FCOG_acc_SL.20160409.40429.16424.grb new file mode 100644 index 0000000000000000000000000000000000000000..fa7cc0a415bf4dce32f26ae427cb3f4ab3ee5259 Binary files /dev/null and b/python/pythontest/TestDir/SubTestDir2/FCOG_acc_SL.20160409.40429.16424.grb differ diff --git a/python/pythontest/TestEcFlexpart.py b/python/pythontest/TestEcFlexpart.py new file mode 100644 index 0000000000000000000000000000000000000000..d256084d7eeabf4fe0bb6cb4b6e6ca1a2ccead08 --- /dev/null +++ b/python/pythontest/TestEcFlexpart.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import sys +import EcFlexpart + +sys.path.append('../python') + +class TestEcFlexpart(unittest.TestCase): + ''' + ''' + + def setUp(self): + pass + + def test__init__(): + # create an instance of EcFlexpart and get a dictionary of the + # class attributes, compare this dict with an expected dict! + return + + def test_write_namelist(): + # simple + return + + def test_retrieve(): + # not sure how to check + return + + +# these functions should test the output and compare the results with an output +# of the old version and check if there are no differences!!! +# also check for errors?! Or check if it works for alle datasets and private public etc + + # - process_output + # - create + # - deacc_fluxes + + + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/python/pythontest/TestInput.py b/python/pythontest/TestInput.py new file mode 100644 index 0000000000000000000000000000000000000000..97fd4888e1a39f410bb79959ce28f84541086360 --- /dev/null +++ b/python/pythontest/TestInput.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import os +import sys +sys.path.append('../python') +from ControlFile import ControlFile +import tools + + +class TestInput(unittest.TestCase): + ''' + Test class to test the reading of commandline arguments and + control file. + ''' + # ToDo + # create more tests for input + # 1. nur controlfile reading + # 2. check of parameter + + def setUp(self): + ''' + ''' + # Default values for ArgumentParser + self.args = {'start_date':None, + 'end_date':None, + 'date_chunk':None, + 'basetime':None, + 'step':None, + 'levelist':None, + 'area':None, + 'inputdir':None, + 'outputdir':None, + 'flexpart_root_scripts':None, + 'ppid':None, + 'job_template':'job.temp', + 'queue':None, + 'controlfile':'CONTROL.temp', + 'debug':0, + } + #sys.argv = ['dummy.py', '--start_date=20180101', '--debug=1', + # '--step=0/to/11/BY/3', '--area=20./20./0./90.'] + sys.argv = ['dummy.py', '--start_date=20180101'] + + self.args = tools.get_commandline_arguments() + + self.c = ControlFile('TestData/CONTROL.temp') + + self.c.assign_args_to_control(self.args) + + self.c.check_conditions() + + return + + def test_args_reading(self): + + sys.argv = ['dummy.py', '--start_date=20180101', '--debug=1', + '--step=0/to/11/BY/3', '--area=20./20./0./90.'] + + arguments = tools.get_commandline_arguments() + + args_exp = {'start_date':'20180101', + 'end_date':None, + 'date_chunk':None, + 'basetime':None, + 'step':'0/to/11/BY/3', + 'levelist':None, + 'area':'20./20./0./90.', + 'inputdir':None, + 'outputdir':None, + 'flexpart_root_scripts':None, + 'ppid':None, + 'job_template':'job.temp', + 'queue':None, + 'controlfile':'CONTROL.temp', + 'debug':1, + } + + self.assertDictEqual(vars(arguments), args_exp) + return + + def test_args_assignment(self): + + import collections + + # expected parametervalue: + exp_dict = { + 'accuracy': '16', + 'addpar': ['186', '187', '188', '235', '139', '39'], + 'area': None, + 'basetime': None, + 'controlfile': 'CONTROL.temp', + 'cwc': 0, + 'date_chunk': 3, + 'debug': 0, + 'destination': None, + 'dpdeta': '1', + 'dtime': '3', + 'ecfsdir': 'ectmp:/${USER}/econdemand/', + 'ecgid': None, + 'ecmwfdatadir': '/raid60/nas/tmc/Anne/Interpolation/flexextract/flexextract/python/../', + 'ecstorage': '0', + 'ectrans': '1', + 'ecuid': None, + 'end_date': '20180101', + 'eta': '0', + 'etadiff': '0', + 'etapar': 77, + 'exedir': '/raid60/nas/tmc/Anne/Interpolation/flexextract/flexextract/python/../src/', + 'expver': '1', + 'flexpart_root_scripts': '/raid60/nas/tmc/Anne/Interpolation/flexextract/flexextract/python/../', + 'format': 'GRIB1', + 'gateway': None, + 'gauss': '1', + 'grib2flexpart': '0', + 'grid': '5000', + 'inputdir': '../work', + 'job_template': 'job.temp', + 'left': '-15000', + 'level': '60', + 'levelist': '55/to/60', + 'lower': '30000', + 'mailfail': ['${USER}'], + 'mailops': ['${USER}'], + 'makefile': None, + 'marsclass': 'EI', + 'maxstep': 11, + 'number': 'OFF', + 'omega': '0', + 'omegadiff': '0', + 'outputdir': '../work', + 'prefix': 'EI', + 'resol': '63', + 'right': '45000', + 'smooth': '0', + 'start_date': '20180101', + 'step': ['00', '01', '02', '03', '04', '05', '00', '07', '08', '09', '10', '11', '00', '01', '02', '03', '04', '05', '00', '07', '08', '09', '10', '11'], + 'stream': 'OPER', + 'target': None, + 'time': ['00', '00', '00', '00', '00', '00', '06', '00', '00', '00', '00', '00', '12', '12', '12', '12', '12', '12', '18', '12', '12', '12', '12', '12'], + 'type': ['AN', 'FC', 'FC', 'FC', 'FC', 'FC', 'AN', 'FC', 'FC', 'FC', 'FC', 'FC', 'AN', 'FC', 'FC', 'FC', 'FC', 'FC', 'AN', 'FC', 'FC', 'FC', 'FC', 'FC'], + 'upper': '75000', + 'wrf': 0} + + exp_dict = collections.OrderedDict(sorted(exp_dict.items())) + cdict = collections.OrderedDict(sorted(vars(self.c).items())) + + # remove content which isn't comparable for different users + # or different operating systems + del cdict['ecfsdir_expanded'] + del cdict['mailops_expanded'] + del cdict['mailfail_expanded'] + + #print 'cdict\n', cdict + #print 'exp_dict\n', exp_dict + + #assert cdict == exp_dict + self.assertDictEqual(cdict, exp_dict) + + return + +if __name__ == "__main__": + unittest.main() diff --git a/python/pythontest/TestInstall.py b/python/pythontest/TestInstall.py new file mode 100644 index 0000000000000000000000000000000000000000..0bc236af4e78d65a5beb19862bdda5574b73aa90 --- /dev/null +++ b/python/pythontest/TestInstall.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import sys +import os +import inspect +sys.path.append('../python') +import install + + +class TestTools(unittest.TestCase): + ''' + ''' + + def setUp(self): + pass + + + def test_mk_tarball(self): + ecd = os.path.dirname(os.path.abspath(inspect.getfile( + inspect.currentframe()))) + '/../' + #print ecd + install.mk_tarball(ecd) + + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/python/pythontest/TestPathes.py b/python/pythontest/TestPathes.py new file mode 100644 index 0000000000000000000000000000000000000000..c3ae2bb17fa97d48212a4066cff87043d7258053 --- /dev/null +++ b/python/pythontest/TestPathes.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import pytest + + +def test_vtable(): + print "vtable test" + diff --git a/python/pythontest/TestTools.py b/python/pythontest/TestTools.py new file mode 100644 index 0000000000000000000000000000000000000000..ee0d97034ee7f2419ba25f5080f24144d8f920a3 --- /dev/null +++ b/python/pythontest/TestTools.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import sys +import os +sys.path.append('../python') +from tools import init128, to_param_id, my_error, read_ecenv + + +class TestTools(unittest.TestCase): + ''' + ''' + + def setUp(self): + pass + + def test_init128(self): + ''' + ''' + table128 = init128('../grib_templates/ecmwf_grib1_table_128') + expected = {'078': 'TCLW', '130': 'T', '034': 'SST'} + # check a sample of parameters which must have been read in + result = all((k in table128 and table128[k]==v) for k,v in expected.iteritems()) + self.assertEqual(result, True) + + + def test_to_param_id(self): + ''' + ''' + table128 = init128('../grib_templates/ecmwf_grib1_table_128') + pars = to_param_id("T/SP/LSP/SSHF", table128) + for par in pars: + self.assertIn(par, [130, 134, 142, 146]) + + def test_error_notifcation(self): + ''' + ''' + with self.assertRaises(SystemExit) as re: + my_error(['${USER}', 'anne.philipp@univie.ac.at'], 'Failed!') + self.assertEqual(re.exception.code, 1) + + def test_read_ecenv(self): + + envs_ref = {'ECUID': 'km4a', + 'ECGID': 'at', + 'GATEWAY': 'srvx8.img.univie.ac.at', + 'DESTINATION': 'annep@genericSftp' + } + envs = read_ecenv(os.getcwd() + '/TestData/ECMWF_ENV') + + self.assertDictEqual(envs_ref, envs) + + + + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/python/pythontest/TestUIOFiles.py b/python/pythontest/TestUIOFiles.py new file mode 100644 index 0000000000000000000000000000000000000000..d60cbf040c3a7223502190568824040b95a03c53 --- /dev/null +++ b/python/pythontest/TestUIOFiles.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import os +import sys +sys.path.append('../python') +import UioFiles + + +class TestUioFiles(unittest.TestCase): + ''' + Test class to test the UIOFiles methods. + ''' + + def setUp(self): + ''' + @Description: + Prepare test case. Initialize comparing filelist and + the test path. + + @Input: + self: instance of TestUIOFiles + Class to test the UIOFiles methods. + + @Return: + <nothing> + ''' + self.testpath = os.path.join(os.path.dirname(__file__), 'TestDir') + self.expected = ['FCGG__SL.20160410.40429.16424.grb', + 'FCOG__ML.20160410.40429.16424.grb', + 'FCSH__ML.20160410.40429.16424.grb', + 'OG_OROLSM__SL.20160410.40429.16424.grb', + 'FCOG_acc_SL.20160409.40429.16424.grb', + 'FCOG__SL.20160410.40429.16424.grb', + 'FCSH__SL.20160410.40429.16424.grb'] + + return + + def test_listFiles(self): + ''' + @Description: + Test the listFiles method from class UIOFiles. + + @Input: + self: instance of TestClass + Class to test the UIOFiles methods. + + @Return: + <nothing> + ''' + + # Initialise and collect filenames + files = UioFiles.UioFiles(['.grb']) + files.list_files(self.testpath, '*') + # get the basename to just check for equality of filenames + filelist = [os.path.basename(f) for f in files.files] + # comparison of expected filenames against the collected ones + self.assertItemsEqual(self.expected, filelist) + + return + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/python/pythontest/__init__.py b/python/pythontest/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..749a1095b83a8b21e7836cd736cce1a0a7be4751 --- /dev/null +++ b/python/pythontest/__init__.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Author: Anne Philipp (University of Vienna) + +@Date: March 2018 + +@License: + (C) Copyright 2014 UIO. + + This software is licensed under the terms of the Apache Licence Version 2.0 + which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +""" \ No newline at end of file diff --git a/python/pythontest/_config.py b/python/pythontest/_config.py new file mode 100644 index 0000000000000000000000000000000000000000..cd24430e7a6b0f99f4bbb0611773493395a68cd8 --- /dev/null +++ b/python/pythontest/_config.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +#******************************************************************************* +# @Author: Anne Philipp (University of Vienna) +# +# @Date: August 2018 +# +# @Change History: +# +# @License: +# (C) Copyright 2014-2018. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# @Description: +# Contains global parameter for flex_extract test environment. +# +#******************************************************************************* + +# ------------------------------------------------------------------------------ +# MODULES +# ------------------------------------------------------------------------------ +import os +import sys +import inspect + + +_VERSION_STR = '7.1' + + +# add path to pythonpath +LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( + inspect.getfile(inspect.currentframe()))) +if LOCAL_PYTHON_PATH not in sys.path: + sys.path.append(LOCAL_PYTHON_PATH) +