diff --git a/Documentation/html/Documentation/Api/Fortran/css/local.css b/Documentation/html/Documentation/Api/Fortran/css/local.css
index 8ab50bcaafa5878b1fc5e2440885d87aa94a70e9..44e3dec2a0085374d61777aa4ae8688a9aa6be15 100644
--- a/Documentation/html/Documentation/Api/Fortran/css/local.css
+++ b/Documentation/html/Documentation/Api/Fortran/css/local.css
@@ -101,9 +101,9 @@ body {
}
.hl pre a {
- display: inline-block;
- height: 4.5em;
- margin: -4.5em 0 0;
+ display: inline-block;
+ height: 4.5em;
+ margin: -4.5em 0 0;
}
.codesum h3 {
margin-top: 2px;
diff --git a/For_developers/Sphinx/Makefile b/For_developers/Sphinx/Makefile
index bf53c49044b0aecc72a6a5c9003041a86b879d54..47066bbf300cfcceb1bbbd678c676f38fcba92f8 100644
--- a/For_developers/Sphinx/Makefile
+++ b/For_developers/Sphinx/Makefile
@@ -1,4 +1,4 @@
-# Minimal makefile for Sphinx documentation
+# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
diff --git a/For_developers/Sphinx/source/Documentation/Input/control.rst b/For_developers/Sphinx/source/Documentation/Input/control.rst
index 5f4ac424ed7cee480bb92a8a92f727ad3d47ad6a..cb9e500e744e6774e4d4a01b6d7691d3e32bf849 100644
--- a/For_developers/Sphinx/source/Documentation/Input/control.rst
+++ b/For_developers/Sphinx/source/Documentation/Input/control.rst
@@ -4,8 +4,8 @@ The CONTROL file
-.. MARS user documentation https://confluence.ecmwf.int/display/UDOC/MARS+user+documentation
-.. MARS keywords and explanation https://confluence.ecmwf.int/display/UDOC/MARS+keywords
+.. MARS user documentation https://confluence.ecmwf.int/display/UDOC/MARS+user+documentation
+.. MARS keywords and explanation https://confluence.ecmwf.int/display/UDOC/MARS+keywords
This file is an input file for :literal:`flex_extract's` main script :literal:`submit.py`.
diff --git a/For_developers/Sphinx/source/Documentation/disagg.rst b/For_developers/Sphinx/source/Documentation/disagg.rst
index 113143d1eac2044150d8cfde1034516cbb14ea20..deb420864c82f618b2142df0a954e4816214136c 100644
--- a/For_developers/Sphinx/source/Documentation/disagg.rst
+++ b/For_developers/Sphinx/source/Documentation/disagg.rst
@@ -103,7 +103,7 @@ Figure 3 shows an overview of the new algorithm and its components.
.. _ref-fig-IA3:
.. figure:: ../_files/IA3.png
- :figclass: align-center
+ :figclass: align-center
Fig. 3: Schematic overview of the basic notation in a precipitation interval with the original precipitation rate g (green) as a step function and the interpolated data :math:`f` (dark blue) as the piecewise linear function. The original time interval with fixed grid length :math:`\delta t` is split equidistantly in three subintervals denoted by :math:`I_i^{1,2,3}`, with the slopes in the subintervals as denoted by :math:`k_i^{1,2,3}` . The sub-grid function values :math:`f_i, f_i^{1,2}, f_{i+1}` are marked by red diamonds (Hittmeir et al. 2018).
@@ -184,4 +184,4 @@ This new point :math:`p` is used for linear interpolation of the complete timese
:hidden:
:maxdepth: 2
-
\ No newline at end of file
+
diff --git a/For_developers/Sphinx/source/evaluation.rst b/For_developers/Sphinx/source/evaluation.rst
index 48906d6fba7812702cd58fe0a0d66027d4394f90..a0d4d2a6aa7d8516b4420f448a168efcbf537dfc 100644
--- a/For_developers/Sphinx/source/evaluation.rst
+++ b/For_developers/Sphinx/source/evaluation.rst
@@ -29,11 +29,11 @@ Evaluation
- radon
Performance analysis:
- - Whole software
- - Disaggregation
- - Convert2
- - prepare_flexpart
- - Andere komponenten
+ - Whole software
+ - Disaggregation
+ - Convert2
+ - prepare_flexpart
+ - Andere komponenten
diff --git a/Source/Python/Classes/UioFiles.py b/Source/Python/Classes/UioFiles.py
index cff0ab4517686fdc0d52a81f0a2a47277697dafd..0fe9448a9b97d4192688f3ba21e14eb9236235dd 100644
--- a/Source/Python/Classes/UioFiles.py
+++ b/Source/Python/Classes/UioFiles.py
@@ -113,7 +113,7 @@ class UioFiles(object):
path = os.path.abspath(path)
# get all files in the dir and subdir as absolut path
- # pylint: disable=W0612
+ # pylint: disable=W0612
for root, dirnames, filenames in os.walk(path):
for filename in fnmatch.filter(filenames, self.pattern):
self.files.append(os.path.join(root, filename))
diff --git a/Source/Python/pylintrc b/Source/Python/pylintrc
index 75fc8ae8c78f7a6f2ae443791e1cdb6a3600c6dc..0101f44254abd52a0d6a8c9a0d4dadfb24cec707 100644
--- a/Source/Python/pylintrc
+++ b/Source/Python/pylintrc
@@ -127,10 +127,9 @@ disable=print-statement,
dict-items-not-iterating,
dict-keys-not-iterating,
dict-values-not-iterating,
- anomalous-backslash-in-string,
- relative-import,
- protected-access
-
+ anomalous-backslash-in-string,
+ relative-import,
+ protected-access
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
@@ -361,28 +360,28 @@ good-names=i,
d,
dd,
v,
- ty,
- st,
- ti,
- MR,
- pk,
- pv,
- ni,
- nj,
- nt,
- ix,
- jy,
- it,
- parId,
- g,
- h,
- xa,
- xb,
- xc,
- xd,
- IA3,
- dt,
- fi
+ ty,
+ st,
+ ti,
+ MR,
+ pk,
+ pv,
+ ni,
+ nj,
+ nt,
+ ix,
+ jy,
+ it,
+ parId,
+ g,
+ h,
+ xa,
+ xb,
+ xc,
+ xd,
+ IA3,
+ dt,
+ fi
# Include a hint for the correct naming format with invalid-name
include-naming-hint=no
diff --git a/Source/Pythontest/TestPathes.py b/Source/Pythontest/TestPathes.py
index 6174484c5cf3ac819571553863869487edf5f8b1..f02e924bdfc0732e9f15b5dd74a675d2fab54084 100644
--- a/Source/Pythontest/TestPathes.py
+++ b/Source/Pythontest/TestPathes.py
@@ -1,35 +1,35 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import sys
-import pytest
-
-sys.path.append("../python")
-import _config
-
-def test_path_localpython():
- assert os.path.exists(_config.PATH_LOCAL_PYTHON) == 1
-
-def test_path_flexextract():
- assert os.path.exists(_config.PATH_FLEXEXTRACT_DIR) == 1
-
-def test_path_flexextract_name():
- version = _config._VERSION_STR
- flexextract_name = 'flex_extract_v' + version
- assert os.path.basename(_config.PATH_FLEXEXTRACT_DIR) == flexextract_name
-
-def test_path_templates():
- assert os.path.exists(_config.PATH_TEMPLATES) == 1
-
-def test_path_vtable():
- assert os.path.exists(_config.PATH_GRIBTABLE) == 1
-
-def test_file_vtable():
- assert os.path.isfile(_config.PATH_GRIBTABLE) == 1
-
-def test_path_run():
- assert os.path.exists(_config.PATH_RUN_DIR) == 1
-
-def test_path_control():
- assert os.path.exists(_config.PATH_CONTROLFILES) == 1
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import pytest
+
+sys.path.append("../python")
+import _config
+
+def test_path_localpython():
+ assert os.path.exists(_config.PATH_LOCAL_PYTHON) == 1
+
+def test_path_flexextract():
+ assert os.path.exists(_config.PATH_FLEXEXTRACT_DIR) == 1
+
+def test_path_flexextract_name():
+ version = _config._VERSION_STR
+ flexextract_name = 'flex_extract_v' + version
+ assert os.path.basename(_config.PATH_FLEXEXTRACT_DIR) == flexextract_name
+
+def test_path_templates():
+ assert os.path.exists(_config.PATH_TEMPLATES) == 1
+
+def test_path_vtable():
+ assert os.path.exists(_config.PATH_GRIBTABLE) == 1
+
+def test_file_vtable():
+ assert os.path.isfile(_config.PATH_GRIBTABLE) == 1
+
+def test_path_run():
+ assert os.path.exists(_config.PATH_RUN_DIR) == 1
+
+def test_path_control():
+ assert os.path.exists(_config.PATH_CONTROLFILES) == 1
diff --git a/Source/Pythontest/conftest.py b/Source/Pythontest/conftest.py
index 644c77ba557acb25c75694f0712ce58265a83e0b..f9c25511ac79cd96067729c90b5fc0e4daf6b47e 100644
--- a/Source/Pythontest/conftest.py
+++ b/Source/Pythontest/conftest.py
@@ -1,10 +1,10 @@
-import sys
-import pytest
-
-sys.path.append('../Python')
-import _config
-
-#@pytest.fixture
-#def prep_test_env():
-# testdir = _config.PATH_TEST_DIR
-print('')
+import sys
+import pytest
+
+sys.path.append('../Python')
+import _config
+
+#@pytest.fixture
+#def prep_test_env():
+# testdir = _config.PATH_TEST_DIR
+print('')
diff --git a/Source/Pythontest/pytest.ini b/Source/Pythontest/pytest.ini
index 66e850a3a99e0733c970b96198f1be78c415151d..7e7fce4c544fd5f10d5bd38ef83a9430018e94ee 100644
--- a/Source/Pythontest/pytest.ini
+++ b/Source/Pythontest/pytest.ini
@@ -1,5 +1,5 @@
-# content of pytest.ini
-[pytest]
-markers =
- msuser_pw: Test that can be executed only as a member-state user. Password required.
- gateway: Test that can be executed only in the gateway mode.
+# content of pytest.ini
+[pytest]
+markers =
+ msuser_pw: Test that can be executed only as a member-state user. Password required.
+ gateway: Test that can be executed only in the gateway mode.
diff --git a/Testing/Regression/Compare_gribfiles/test_cmp_grib_file.py b/Testing/Regression/Compare_gribfiles/test_cmp_grib_file.py
index 196131ca31700cf981e652ab2836f8e8549a9db5..96200daca0fe24483403082f46f6aa72dddfcfac 100644
--- a/Testing/Regression/Compare_gribfiles/test_cmp_grib_file.py
+++ b/Testing/Regression/Compare_gribfiles/test_cmp_grib_file.py
@@ -1,270 +1,270 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Comparison of resulting Grib files of two flex_extract versions.
-
-
-
-The script should be called like:
-
- python test_cmp_grib_files.py -r <path-to-reference-files> -n <path-to-new-files> -p <file-pattern>
-
-Note
-----
-
-Licence:
---------
- (C) Copyright 2014-2019.
-
- SPDX-License-Identifier: CC-BY-4.0
-
- This work is licensed under the Creative Commons Attribution 4.0
- International License. To view a copy of this license, visit
- http://creativecommons.org/licenses/by/4.0/ or send a letter to
- Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
-
-
-Example
--------
- python test_cmp_grib_file.py -r 7.0.4/EA5/ -n 7.1/EA5/ -p 'EA*'
-"""
-
-# ------------------------------------------------------------------------------
-# MODULES
-# ------------------------------------------------------------------------------
-import os
-import sys
-from datetime import datetime
-from eccodes import GribFile, GribMessage
-
-# ------------------------------------------------------------------------------
-# FUNCTION
-# ------------------------------------------------------------------------------
-def get_cmdline_params(parlist, debug=True):
-
- import getopt
-
- iref_path = '' # e.g. Reference_files
- inew_path = '' # e.g. New_files
- smatch = '' # e.g. files matching pattern
-
- try:
- opts, pars = getopt.getopt(parlist,
- "hr:n:p:",
- ["ipref=", "ipnew=", "pattern="])
- except getopt.GetoptError:
- print('test_cmp_grib_file.py -r <ipref> -n <ipnew> -p <pattern>')
- sys.exit(2)
-
- for opt, par in opts:
- if opt == '-h':
- print('test_cmp_grib_file.py -r <ipref> -n <ipnew> -p <pattern>')
- sys.exit()
- elif opt in ("-r", "--ipref"):
- iref_path = par
- elif opt in ("-n", "--ipnew"):
- inew_path = par
- elif opt in ("-p", "--pattern"):
- smatch = par
-
- if iref_path == '':
- sys.exit('NO REFERENCE INPUT PATH SET!')
- if inew_path == '':
- sys.exit('NO "NEW" COMPARISON INPUT PATH SET!')
- if smatch == '':
- sys.exit('NO MATCHING PATTERN FOR FILES SET!')
-
- if debug:
- print("\n\nWelcome!")
- print('Reference path is: ', iref_path)
- print('New path is: ', inew_path)
- print('Filepattern is: ', smatch)
-
- return iref_path, inew_path, smatch
-
-def get_files(ipath, matchingstring, debug=True):
- """
- @Description:
- Get filenames from input path matching the
- string or regular expression and return it.
-
- @Input:
- ipath: string
- Path to the files.
-
- matchingstring: string
- A string defining the filenames,
- with or without regular exprssion.
-
- @Return
- filename: list of strings
- A list of all files matching the pattern of
- matchingstring.
- """
- import fnmatch
-
- files = []
-
- for fn in os.listdir(ipath):
- if fnmatch.fnmatch(fn, matchingstring):
- files.append(fn)
-
- filelist = sorted(files)
- if debug:
- print('The input files are: %s' %(filelist))
-
- return filelist
-
-def cmp_files_list(flist1, flist2):
- '''
- '''
-
- # 1. same length?
- length = len(flist1) == len(flist2)
- if not length:
- print('There are not the same amount of files.')
- sys.exit('Message 1')
-
- # 2. same content?
- content = [True for i, j in zip(flist1, flist2) if i == j]
- if not len(content) == len(flist1):
- print('Not the same file list')
- sys.exit('Message 2')
-
- return True
-
-
-def cmp_number_messages(iref, flist1, inew, flist2):
-
- ref_dict = {}
- new_dict = {}
- res_dict = {}
- for file in flist1:
- with GribFile(os.path.join(iref,file)) as grib:
- ref_dict[file] = len(grib)
- with GribFile(os.path.join(inew,file)) as grib:
- new_dict[file] = len(grib)
-
- res_dict[file] = ref_dict[file] == new_dict[file]
-
- for k, res in res_dict.items():
- if not res == True:
- print('LOG: Amount of messages in files {} are not the same!'.format(k))
-
- return True
-
-
-def cmp_grib_msg_header(ipath_ref, ipath_new, filelist):
- from subprocess import Popen, PIPE
- # ref_dict = {}
- # new_dict = {}
- # for file in flist1:
- # with GribFile(os.path.join(iref,file)) as grib:
- # for i in range(len(grib)):
- # msg = GribMessage(grib)
- # ref_dict[file] = {}
- # ref_dict[file][i] = [msg['shortName'],msg['level'],
- # msg['editionNumber'],msg['dataDate'],
- # msg['dataTime'],msg['marsClass'],
- # msg['type'], msg['gridType'],
- # msg['stepRange']]
- error_flag = False
- cmp_flag = False
- for file in filelist:
- try:
- res = Popen(['grib_compare', '-H',
- ipath_ref + '/' + file,
- ipath_new + '/' + file],
- stdin=PIPE, stdout=PIPE, stderr=PIPE)
- output, error = res.communicate()#.decode("utf-8"))
- if error:
- print('... ERROR: \n\t{}'.format(error.decode()))
- error_flag = True
- if output:
- print('{}'.format(output.decode()))
- cmp_flag = True
- except ValueError as e:
- print('... ERROR CODE: ' + str(e.returncode))
- print('... ERROR MESSAGE:\n \t ' + str(res))
- error_flag = True
- except OSError as e:
- print('... ERROR CODE: ' + str(e.errno))
- print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
- error_flag = True
-
- if error_flag:
- sys.exit('... ERROR IN GRIB MESSAGE COMPARISON!')
- if cmp_flag:
- sys.exit('... FILES HAVE DIFFERENCES IN GRIB MESSAGES!')
-
- return True
-
-
-def cmp_grib_msg_statistics(ipath_ref, ipath_new, filelist):
-
- from subprocess import Popen, PIPE
-
- error_flag = False
- cmp_flag = False
- for file in filelist:
- try:
- res = Popen(['grib_compare', '-c', 'statistics:n',
- ipath_ref + '/' + file,
- ipath_new + '/' + file],
- stdin=PIPE, stdout=PIPE, stderr=PIPE)
- output, error = res.communicate()#.decode("utf-8"))
- if error:
- print('... ERROR: \n\t{}'.format(error.decode()))
- error_flag = True
- if output:
- print('\nIn File: {}'.format(file))
- print('{}'.format(output.decode()))
- cmp_flag = True
- except ValueError as e:
- print('... ERROR CODE: ' + str(e.returncode))
- print('... ERROR MESSAGE:\n \t ' + str(res))
- error_flag = True
- except OSError as e:
- print('... ERROR CODE: ' + str(e.errno))
- print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
- error_flag = True
-
- if error_flag:
- sys.exit('... ERROR IN GRIB MESSAGE COMPARISON!')
- if cmp_flag:
- sys.exit('... FILES HAVE DIFFERENT STATISTICS!')
- return True
-
-if __name__ == '__main__':
-
- # get the parameter list of program call
- ref_path, new_path, fmatch = get_cmdline_params(sys.argv[1:])
-
- # get the list of files of both cases
- ref_files = get_files(ref_path, fmatch)
- new_files = get_files(new_path, fmatch)
-
- # flag to store successfull tests
- suc = True
-
- # 1. Does the 2 cases contain the same list of files?
- suc = True if suc and cmp_files_list(ref_files, new_files) else False
-
- # 2. Does each file in both cases contain the same amount of messages?
- suc = True if suc and cmp_number_messages(ref_path, ref_files, new_path, new_files) else False
-
- # 3. Does each file has the same parameters (in Header)?
- # Since we can be sure that both cases have the same files,
- # we just use 1 filelist
- suc = True if suc and cmp_grib_msg_header(ref_path, new_path, new_files) else False
-
- # 4. Are the statistics of each message the same?
- # Since we can be sure that both cases have the same files,
- # we just use 1 filelist
- suc = True if suc and cmp_grib_msg_statistics(ref_path, new_path, new_files) else False
-
- # If the program comes this far and flag "suc" = True,
- # all tests were successful
- if suc:
- exit('GRIB_COMPARISON: SUCCESSFULL!')
- else:
- exit('GRIB_COMPARISON: FAILURE!')
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Comparison of resulting Grib files of two flex_extract versions.
+
+
+
+The script should be called like:
+
+ python test_cmp_grib_files.py -r <path-to-reference-files> -n <path-to-new-files> -p <file-pattern>
+
+Note
+----
+
+Licence:
+--------
+ (C) Copyright 2014-2019.
+
+ SPDX-License-Identifier: CC-BY-4.0
+
+ This work is licensed under the Creative Commons Attribution 4.0
+ International License. To view a copy of this license, visit
+ http://creativecommons.org/licenses/by/4.0/ or send a letter to
+ Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
+
+
+Example
+-------
+ python test_cmp_grib_file.py -r 7.0.4/EA5/ -n 7.1/EA5/ -p 'EA*'
+"""
+
+# ------------------------------------------------------------------------------
+# MODULES
+# ------------------------------------------------------------------------------
+import os
+import sys
+from datetime import datetime
+from eccodes import GribFile, GribMessage
+
+# ------------------------------------------------------------------------------
+# FUNCTION
+# ------------------------------------------------------------------------------
+def get_cmdline_params(parlist, debug=True):
+
+ import getopt
+
+ iref_path = '' # e.g. Reference_files
+ inew_path = '' # e.g. New_files
+ smatch = '' # e.g. files matching pattern
+
+ try:
+ opts, pars = getopt.getopt(parlist,
+ "hr:n:p:",
+ ["ipref=", "ipnew=", "pattern="])
+ except getopt.GetoptError:
+ print('test_cmp_grib_file.py -r <ipref> -n <ipnew> -p <pattern>')
+ sys.exit(2)
+
+ for opt, par in opts:
+ if opt == '-h':
+ print('test_cmp_grib_file.py -r <ipref> -n <ipnew> -p <pattern>')
+ sys.exit()
+ elif opt in ("-r", "--ipref"):
+ iref_path = par
+ elif opt in ("-n", "--ipnew"):
+ inew_path = par
+ elif opt in ("-p", "--pattern"):
+ smatch = par
+
+ if iref_path == '':
+ sys.exit('NO REFERENCE INPUT PATH SET!')
+ if inew_path == '':
+ sys.exit('NO "NEW" COMPARISON INPUT PATH SET!')
+ if smatch == '':
+ sys.exit('NO MATCHING PATTERN FOR FILES SET!')
+
+ if debug:
+ print("\n\nWelcome!")
+ print('Reference path is: ', iref_path)
+ print('New path is: ', inew_path)
+ print('Filepattern is: ', smatch)
+
+ return iref_path, inew_path, smatch
+
+def get_files(ipath, matchingstring, debug=True):
+ """
+ @Description:
+ Get filenames from input path matching the
+ string or regular expression and return it.
+
+ @Input:
+ ipath: string
+ Path to the files.
+
+ matchingstring: string
+ A string defining the filenames,
+ with or without regular exprssion.
+
+ @Return
+ filename: list of strings
+ A list of all files matching the pattern of
+ matchingstring.
+ """
+ import fnmatch
+
+ files = []
+
+ for fn in os.listdir(ipath):
+ if fnmatch.fnmatch(fn, matchingstring):
+ files.append(fn)
+
+ filelist = sorted(files)
+ if debug:
+ print('The input files are: %s' %(filelist))
+
+ return filelist
+
+def cmp_files_list(flist1, flist2):
+ '''
+ '''
+
+ # 1. same length?
+ length = len(flist1) == len(flist2)
+ if not length:
+ print('There are not the same amount of files.')
+ sys.exit('Message 1')
+
+ # 2. same content?
+ content = [True for i, j in zip(flist1, flist2) if i == j]
+ if not len(content) == len(flist1):
+ print('Not the same file list')
+ sys.exit('Message 2')
+
+ return True
+
+
+def cmp_number_messages(iref, flist1, inew, flist2):
+
+ ref_dict = {}
+ new_dict = {}
+ res_dict = {}
+ for file in flist1:
+ with GribFile(os.path.join(iref,file)) as grib:
+ ref_dict[file] = len(grib)
+ with GribFile(os.path.join(inew,file)) as grib:
+ new_dict[file] = len(grib)
+
+ res_dict[file] = ref_dict[file] == new_dict[file]
+
+ for k, res in res_dict.items():
+ if not res == True:
+ print('LOG: Amount of messages in files {} are not the same!'.format(k))
+
+ return True
+
+
+def cmp_grib_msg_header(ipath_ref, ipath_new, filelist):
+ from subprocess import Popen, PIPE
+ # ref_dict = {}
+ # new_dict = {}
+ # for file in flist1:
+ # with GribFile(os.path.join(iref,file)) as grib:
+ # for i in range(len(grib)):
+ # msg = GribMessage(grib)
+ # ref_dict[file] = {}
+ # ref_dict[file][i] = [msg['shortName'],msg['level'],
+ # msg['editionNumber'],msg['dataDate'],
+ # msg['dataTime'],msg['marsClass'],
+ # msg['type'], msg['gridType'],
+ # msg['stepRange']]
+ error_flag = False
+ cmp_flag = False
+ for file in filelist:
+ try:
+ res = Popen(['grib_compare', '-H',
+ ipath_ref + '/' + file,
+ ipath_new + '/' + file],
+ stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ output, error = res.communicate()#.decode("utf-8"))
+ if error:
+ print('... ERROR: \n\t{}'.format(error.decode()))
+ error_flag = True
+ if output:
+ print('{}'.format(output.decode()))
+ cmp_flag = True
+ except ValueError as e:
+ print('... ERROR CODE: ' + str(e.returncode))
+ print('... ERROR MESSAGE:\n \t ' + str(res))
+ error_flag = True
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
+ error_flag = True
+
+ if error_flag:
+ sys.exit('... ERROR IN GRIB MESSAGE COMPARISON!')
+ if cmp_flag:
+ sys.exit('... FILES HAVE DIFFERENCES IN GRIB MESSAGES!')
+
+ return True
+
+
+def cmp_grib_msg_statistics(ipath_ref, ipath_new, filelist):
+
+ from subprocess import Popen, PIPE
+
+ error_flag = False
+ cmp_flag = False
+ for file in filelist:
+ try:
+ res = Popen(['grib_compare', '-c', 'statistics:n',
+ ipath_ref + '/' + file,
+ ipath_new + '/' + file],
+ stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ output, error = res.communicate()#.decode("utf-8"))
+ if error:
+ print('... ERROR: \n\t{}'.format(error.decode()))
+ error_flag = True
+ if output:
+ print('\nIn File: {}'.format(file))
+ print('{}'.format(output.decode()))
+ cmp_flag = True
+ except ValueError as e:
+ print('... ERROR CODE: ' + str(e.returncode))
+ print('... ERROR MESSAGE:\n \t ' + str(res))
+ error_flag = True
+ except OSError as e:
+ print('... ERROR CODE: ' + str(e.errno))
+ print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
+ error_flag = True
+
+ if error_flag:
+ sys.exit('... ERROR IN GRIB MESSAGE COMPARISON!')
+ if cmp_flag:
+ sys.exit('... FILES HAVE DIFFERENT STATISTICS!')
+ return True
+
+if __name__ == '__main__':
+
+ # get the parameter list of program call
+ ref_path, new_path, fmatch = get_cmdline_params(sys.argv[1:])
+
+ # get the list of files of both cases
+ ref_files = get_files(ref_path, fmatch)
+ new_files = get_files(new_path, fmatch)
+
+ # flag to store successfull tests
+ suc = True
+
+ # 1. Does the 2 cases contain the same list of files?
+ suc = True if suc and cmp_files_list(ref_files, new_files) else False
+
+ # 2. Does each file in both cases contain the same amount of messages?
+ suc = True if suc and cmp_number_messages(ref_path, ref_files, new_path, new_files) else False
+
+ # 3. Does each file has the same parameters (in Header)?
+ # Since we can be sure that both cases have the same files,
+ # we just use 1 filelist
+ suc = True if suc and cmp_grib_msg_header(ref_path, new_path, new_files) else False
+
+ # 4. Are the statistics of each message the same?
+ # Since we can be sure that both cases have the same files,
+ # we just use 1 filelist
+ suc = True if suc and cmp_grib_msg_statistics(ref_path, new_path, new_files) else False
+
+ # If the program comes this far and flag "suc" = True,
+ # all tests were successful
+ if suc:
+ exit('GRIB_COMPARISON: SUCCESSFULL!')
+ else:
+ exit('GRIB_COMPARISON: FAILURE!')
diff --git a/Testing/Regression/Mars_request/create_old_mars_requests.txt b/Testing/Regression/Mars_request/create_old_mars_requests.txt
index 9491396453a71d8705e84a648090bab5fb9823f7..84e000ee2b16ca9d13abeae63ad25e233c5e7a54 100644
--- a/Testing/Regression/Mars_request/create_old_mars_requests.txt
+++ b/Testing/Regression/Mars_request/create_old_mars_requests.txt
@@ -1,85 +1,85 @@
-#!/bin/bash
-#
-# @Author: Anne Philipp
-#
-# @Date: October, 10 2019
-#
-# @Description: Creates the mars request files with the older (7.0.4)
-# version of flex_extract and places them in the test directory
-# of mars request comparsion tests.
-#
-# @License:
-# (C) Copyright 2014-2019.
-#
-# SPDX-License-Identifier: CC-BY-4.0
-#
-# This work is licensed under the Creative Commons Attribution 4.0
-# International License. To view a copy of this license, visit
-# http://creativecommons.org/licenses/by/4.0/ or send a letter to
-# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
-#
-# @ChangeHistory:
-#
-# @Usage:
-# start script within the old flex_extract python directory
-# ./create_old_mars_request.txt <path to mars request test directory of old version>
-# e.g. ./create_old_mars_request.txt <local_path>/flex_extract_v7.1/Testing/Regression/Mars_request/7.0.4/
-#
-
-
-./submit.py --controlfile=CONTROL_CERA_mr --request=1
-mv ../work/mars_requests.csv ../work/CERA_mr.csv
-
-./submit.py --controlfile=CONTROL_CERA.public_mr --public=1 --request=1
-mv ../work/mars_requests.csv ../work/CERA.public_mr.csv
-
-./submit.py --controlfile=CONTROL_CF_mr --request=1
-mv ../work/mars_requests.csv ../work/CF_mr.csv
-
-./submit.py --controlfile=CONTROL_CV_mr --request=1
-mv ../work/mars_requests.csv ../work/CV_mr.csv
-
-./submit.py --controlfile=CONTROL_EA5.highres_mr --request=1
-mv ../work/mars_requests.csv ../work/EA5.highres_mr.csv
-
-./submit.py --controlfile=CONTROL_EA5_mr --request=1
-mv ../work/mars_requests.csv ../work/EA5_mr.csv
-
-./submit.py --controlfile=CONTROL_EA5.public_mr --public=1 --request=1
-mv ../work/mars_requests.csv ../work/EA5.public_mr.csv
-
-./submit.py --controlfile=CONTROL_EI_mr --request=1
-mv ../work/mars_requests.csv ../work/EI_mr.csv
-
-./submit.py --controlfile=CONTROL_EI.public_mr --request=1 --public=1
-mv ../work/mars_requests.csv ../work/EI.public_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.ELDA.FC.eta.ens.double_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.ELDA.FC.eta.ens.double_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.ENFO.CF_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.ENFO.CF_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.ENFO.PF_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.ENFO.PF_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.OPER.4V.operational_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.OPER.4V.operational_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.OPER.FC.36hours_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.OPER.FC.36hours_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.OPER.FC.twiceaday_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.OPER.FC.twiceaday_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.OPER.global.025_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.OPER.global.025_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.OPER.highres.eta_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.OPER.highres.eta_mr.csv
-
-./submit.py --controlfile=CONTROL_OD.OPER.highres.gauss_mr --request=1
-mv ../work/mars_requests.csv ../work/OD.OPER.highres.gauss_mr.csv
-
-
-mv ./work/*_mr.csv ${1}
+#!/bin/bash
+#
+# @Author: Anne Philipp
+#
+# @Date: October, 10 2019
+#
+# @Description: Creates the mars request files with the older (7.0.4)
+# version of flex_extract and places them in the test directory
+# of mars request comparsion tests.
+#
+# @License:
+# (C) Copyright 2014-2019.
+#
+# SPDX-License-Identifier: CC-BY-4.0
+#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
+#
+# @ChangeHistory:
+#
+# @Usage:
+# start script within the old flex_extract python directory
+# ./create_old_mars_request.txt <path to mars request test directory of old version>
+# e.g. ./create_old_mars_request.txt <local_path>/flex_extract_v7.1/Testing/Regression/Mars_request/7.0.4/
+#
+
+
+./submit.py --controlfile=CONTROL_CERA_mr --request=1
+mv ../work/mars_requests.csv ../work/CERA_mr.csv
+
+./submit.py --controlfile=CONTROL_CERA.public_mr --public=1 --request=1
+mv ../work/mars_requests.csv ../work/CERA.public_mr.csv
+
+./submit.py --controlfile=CONTROL_CF_mr --request=1
+mv ../work/mars_requests.csv ../work/CF_mr.csv
+
+./submit.py --controlfile=CONTROL_CV_mr --request=1
+mv ../work/mars_requests.csv ../work/CV_mr.csv
+
+./submit.py --controlfile=CONTROL_EA5.highres_mr --request=1
+mv ../work/mars_requests.csv ../work/EA5.highres_mr.csv
+
+./submit.py --controlfile=CONTROL_EA5_mr --request=1
+mv ../work/mars_requests.csv ../work/EA5_mr.csv
+
+./submit.py --controlfile=CONTROL_EA5.public_mr --public=1 --request=1
+mv ../work/mars_requests.csv ../work/EA5.public_mr.csv
+
+./submit.py --controlfile=CONTROL_EI_mr --request=1
+mv ../work/mars_requests.csv ../work/EI_mr.csv
+
+./submit.py --controlfile=CONTROL_EI.public_mr --request=1 --public=1
+mv ../work/mars_requests.csv ../work/EI.public_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.ELDA.FC.eta.ens.double_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.ELDA.FC.eta.ens.double_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.ENFO.CF_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.ENFO.CF_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.ENFO.PF_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.ENFO.PF_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.OPER.4V.operational_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.OPER.4V.operational_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.OPER.FC.36hours_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.OPER.FC.36hours_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.OPER.FC.twiceaday_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.OPER.FC.twiceaday_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.OPER.global.025_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.OPER.global.025_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.OPER.highres.eta_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.OPER.highres.eta_mr.csv
+
+./submit.py --controlfile=CONTROL_OD.OPER.highres.gauss_mr --request=1
+mv ../work/mars_requests.csv ../work/OD.OPER.highres.gauss_mr.csv
+
+
+mv ./work/*_mr.csv ${1}
diff --git a/Testing/Regression/Mars_request/test_cmp_mars_requests.py b/Testing/Regression/Mars_request/test_cmp_mars_requests.py
index 9bfade9ef7bd4615e90b812c2d102d962fa83c45..13039c987b10c7f7b476eefe9956d27cffebdc75 100755
--- a/Testing/Regression/Mars_request/test_cmp_mars_requests.py
+++ b/Testing/Regression/Mars_request/test_cmp_mars_requests.py
@@ -1,331 +1,331 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Comparison of the created MARS requests of two flex_extract versions.
-
-There will be comparisons for the given standard control files in the
-"Controls" - directory. The result of the comparison is stored in the
-"Log" - directory with an individual timestamp in the format %Y-%m-%d_%H-%M-%S.
-(E.g. log_2018-11-23_12-42-29)
-The MARS request files are named such that it contains the name of the
-corresponding control files "<control-identifier>.csv" (e.g. EA5_mr.csv).
-They are stored in the corresponding version directory and have the same
-name in both versions.
-
-The script should be called like:
-
- python test_cmp_mars_requests.py <old_version_number> <new_version_number>
-
-Note
-----
-The MARS request files from the older version have to be in place already.
-The request files of the new/current version will be generated automatically
-with the "run_local.sh" script.
-It is necessary to have a directory named after the version number of
-flex_extract. For example: "7.0.3" and "7.1".
-
-Licence:
---------
- (C) Copyright 2014-2019.
-
- SPDX-License-Identifier: CC-BY-4.0
-
- This work is licensed under the Creative Commons Attribution 4.0
- International License. To view a copy of this license, visit
- http://creativecommons.org/licenses/by/4.0/ or send a letter to
- Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
-
-Example
--------
- python test_cmp_mars_requests.py 7.0.4 7.1
-"""
-
-# ------------------------------------------------------------------------------
-# MODULES
-# ------------------------------------------------------------------------------
-import os
-import sys
-import pandas as pd
-import subprocess
-import shutil
-from datetime import datetime
-
-sys.path.append('../../../Source/Python')
-import _config
-from Mods.tools import init128
-
-# ------------------------------------------------------------------------------
-# FUNCTION
-# ------------------------------------------------------------------------------
-def test_mr_column_equality(mr_old, mr_new):
- '''Check if old and new version of MARS requests have the same
- amount of columns.
-
- If the number is not equal and/or the column names are not equal
- an error message is stored in global variable "err_msg".
-
- Parameters
- ----------
- mr_old : :obj:`pandas DataFrame`
- Contains the mars requests from the old version of
- flex_extract.
-
- mr_new : :obj:`pandas DataFrame`
- Contains the mars requests from the new version of
- flex_extract.
-
- Return
- ------
- bool
- True if successful, False otherwise.
- '''
- global err_msg
- if (len(mr_old.columns.values) == len(mr_new.columns.values) and
- sorted(mr_old.columns.values) == sorted(mr_new.columns.values)):
- return True
- else:
- err_msg = 'Unequal number and/or column names!\n'
- return False
-
-
-def test_mr_number_equality(mr_old, mr_new):
- '''Check if old and new version have the same number of requests.
-
- If the number is not equal an error message is stored in
- global variable "err_msg".
-
- Parameters
- ----------
- mr_old : :obj:`pandas DataFrame`
- Contains the mars requests from the old version of
- flex_extract.
-
- mr_new : :obj:`pandas DataFrame`
- Contains the mars requests from the new version of
- flex_extract.
-
- Return
- ------
- bool
- True if successful, False otherwise.
- '''
- global err_msg
- if len(mr_new.index) == len(mr_old.index):
- return True
- else:
- err_msg = 'Unequal number of mars requests!\n'
- return False
-
-def test_mr_content_equality(mr_old, mr_new):
- '''Check if old and new version have the same request contents.
-
- If the content in a column is not equal an error message is stored in
- global variable "err_msg", recording the column.
-
- Parameters
- ----------
- mr_old : :obj:`pandas DataFrame`
- Contains the mars requests from the old version of
- flex_extract.
-
- mr_new : :obj:`pandas DataFrame`
- Contains the mars requests from the new version of
- flex_extract.
-
- Return
- ------
- bool
- True if successful, False otherwise.
- '''
- global err_msg
- lresult = None
- columns = list(mr_new.columns.values)
- del columns[columns.index('target')]
- mr_new = trim_all_columns(mr_new)
- mr_old = trim_all_columns(mr_old)
- for col in columns:
- if mr_new[col].equals(mr_old[col]):
- lresult = True
- else:
- err_msg += 'Unconsistency happend to be in column: ' + col + '\n'
- print("THERE SEEMS TO BE AN ERROR:")
- print("CONTENT OF NEW VERSION:")
- print(mr_new[col])
- print("CONTENT OF OLD VERSION:")
- print(mr_old[col])
- return False
- return lresult
-
-
-def trim_all_columns(df):
- """
- Trim whitespace from ends of each value across all series in dataframe
- """
- trim_strings = lambda x: x.strip() if isinstance(x, str) else x
- return df.applymap(trim_strings)
-
-def convert_param_numbers(mr_old):
- """
- Convert the numbers parameter into integers with 3 digits.
- """
-
- if str(mr_old).strip() != "OFF" and mr_old != None and '/' in str(mr_old) :
- numbers = mr_old.split('/')
- number = str(int(numbers[0])).zfill(3)+'/TO/'+str(int(numbers[2])).zfill(3)
-
- return number
-
- return mr_old
-
-def convert_param_step(mr_old):
- """
- For pure forecast with steps greater than 23 hours, the older versions
- writes out a list of steps instead with the syntax 'to' and 'by'.
- e.g. 000/003/006/009/012/015/018/021/024/027/030/033/036
-
- Convert this to 0/to/36/by/3
- """
-
- #if 'to' in str(mr_old) and 'by' in str(mr_old):
- # steps = mr_old.split('/')
- # step = []
- # for i in range(int(steps[0]),int(steps[2]),int(steps[4])):
- # step.append(str(int(i)).zfill(2))
- # return '/'.join(step)
-
- if not mr_old.isdigit() and 'to' not in mr_old.lower():
- if int(mr_old.split('/')[-1]) > 23:
-
- steps = mr_old.split('/')
- dtime = int(steps[1]) - int(steps[0])
-
- nsteps = str(int(steps[1]))+'/to/'+str(int(steps[-1]))+'/by/'+str(int(dtime))
- return nsteps
-
- return mr_old
-
-def to_param_id(pars, table):
- '''Transform parameter names to parameter ids with ECMWF grib table 128.
-
- Parameters
- ----------
- pars : str
- Addpar argument from CONTROL file in the format of
- parameter names instead of ids. The parameter short
- names are sepearted with "/" and they are passed as
- one single string.
-
- table : dict
- Contains the ECMWF grib table 128 information.
- The key is the parameter number and the value is the
- short name of the parameter.
-
- Return
- ------
- ipar : list of int
- List of addpar parameters from CONTROL file transformed to
- parameter ids in the format of integer.
- '''
- if not pars:
- return []
- if not isinstance(pars, str):
- pars=str(pars)
-
- cpar = pars.upper().split('/')
- spar = []
- for par in cpar:
- par = par.strip()
- for k, v in table.items():
- if par.isdigit():
- par = str(int(par)).zfill(3)
- if par == k or par == v:
- spar.append(k + '.128')
- break
- else:
- print('\n\Warning: par ' + par + ' not found in table 128\n\n”')
-
- return '/'.join(str(l) for l in spar)
-
-
-
-if __name__ == '__main__':
-
- # basic values for paths and versions
- control_path = 'Controls'
- log_path = 'Log'
- old_dir = sys.argv[1] # e.g. '7.0.4'
- new_dir = sys.argv[2] # e.g. '7.1'
- mr_filename = 'mars_requests.csv'
-
- # have to be set to "True" in the beginnning
- # so it only fails if a test fails
- lfinal = True
-
- # prepare log file for this test run
- currenttime = datetime.now()
- time_str = currenttime.strftime('%Y-%m-%d_%H-%M-%S')
- logfile = os.path.join(log_path, 'log_' + time_str)
- with open(logfile, 'a') as f:
- f.write('Compare mars requests between version ' + old_dir +
- ' and version ' + new_dir + ' : \n')
-
- # list all controlfiles
- controls = os.listdir(control_path)
-
- # loop over controlfiles
- for c in controls:
- # empty error message for every controlfile
- err_msg = ''
-
- # start flex_extract with controlfiles to get mars_request files
- shutil.copy(os.path.join(control_path,c), _config.PATH_CONTROLFILES)
- subprocess.check_output(['run_local.sh', new_dir, c])
- os.remove(os.path.join(_config.PATH_CONTROLFILES,c))
-
- # cut-of "CONTROL_" string and mv mars_reqeust file
- # to a name including control specific name
- mr_name = c.split('CONTROL_')[1] + '.csv'
- shutil.move(os.path.join(new_dir,mr_filename), os.path.join(new_dir,mr_name))
-
- # read both mr files (old & new)
- mr_old = pd.read_csv(os.path.join(old_dir, mr_name))
- mr_new = pd.read_csv(os.path.join(new_dir, mr_name))
-
- mr_old.columns = mr_old.columns.str.strip()
- mr_new.columns = mr_new.columns.str.strip()
-
- # convert names in old to ids
- table128 = init128(_config.PATH_GRIBTABLE)
- #print mr_old['param']
-
- # some format corrections are necessary to compare older versions with 7.1
- mr_old['param'] = mr_old['param'].apply(to_param_id, args=[table128])
- mr_old['number'] = mr_old['number'].apply(convert_param_numbers)
- if '142' in mr_old.loc[0,'param']: # if flux request
- mr_old.loc[0,'step'] = convert_param_step(mr_old.loc[0,'step'])
-
- print('Results: ', c)
-
- # do tests on mr files
- lcoleq = test_mr_column_equality(mr_old, mr_new)
- lnoeq = test_mr_number_equality(mr_old, mr_new)
- lcoeq = test_mr_content_equality(mr_old, mr_new)
-
- # check results for mr file
- lfinal = lfinal and lcoleq and lnoeq and lcoeq
-
- # write out result to logging file
- with open(logfile, 'a') as f:
- if lcoleq and lnoeq and lcoeq:
- f.write('... ' + c + ' ... OK!' + '\n')
- else:
- f.write('... ' + c + ' ... FAILED!' + '\n')
- if err_msg:
- f.write('... \t' + err_msg + '\n')
-
- exit
-
- # exit with success or error status
- if lfinal:
- sys.exit(0) # 'SUCCESS'
- else:
- sys.exit(1) # 'FAIL'
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Comparison of the created MARS requests of two flex_extract versions.
+
+There will be comparisons for the given standard control files in the
+"Controls" - directory. The result of the comparison is stored in the
+"Log" - directory with an individual timestamp in the format %Y-%m-%d_%H-%M-%S.
+(E.g. log_2018-11-23_12-42-29)
+The MARS request files are named such that it contains the name of the
+corresponding control files "<control-identifier>.csv" (e.g. EA5_mr.csv).
+They are stored in the corresponding version directory and have the same
+name in both versions.
+
+The script should be called like:
+
+ python test_cmp_mars_requests.py <old_version_number> <new_version_number>
+
+Note
+----
+The MARS request files from the older version have to be in place already.
+The request files of the new/current version will be generated automatically
+with the "run_local.sh" script.
+It is necessary to have a directory named after the version number of
+flex_extract. For example: "7.0.3" and "7.1".
+
+Licence:
+--------
+ (C) Copyright 2014-2019.
+
+ SPDX-License-Identifier: CC-BY-4.0
+
+ This work is licensed under the Creative Commons Attribution 4.0
+ International License. To view a copy of this license, visit
+ http://creativecommons.org/licenses/by/4.0/ or send a letter to
+ Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
+
+Example
+-------
+ python test_cmp_mars_requests.py 7.0.4 7.1
+"""
+
+# ------------------------------------------------------------------------------
+# MODULES
+# ------------------------------------------------------------------------------
+import os
+import sys
+import pandas as pd
+import subprocess
+import shutil
+from datetime import datetime
+
+sys.path.append('../../../Source/Python')
+import _config
+from Mods.tools import init128
+
+# ------------------------------------------------------------------------------
+# FUNCTION
+# ------------------------------------------------------------------------------
+def test_mr_column_equality(mr_old, mr_new):
+ '''Check if old and new version of MARS requests have the same
+ amount of columns.
+
+ If the number is not equal and/or the column names are not equal
+ an error message is stored in global variable "err_msg".
+
+ Parameters
+ ----------
+ mr_old : :obj:`pandas DataFrame`
+ Contains the mars requests from the old version of
+ flex_extract.
+
+ mr_new : :obj:`pandas DataFrame`
+ Contains the mars requests from the new version of
+ flex_extract.
+
+ Return
+ ------
+ bool
+ True if successful, False otherwise.
+ '''
+ global err_msg
+ if (len(mr_old.columns.values) == len(mr_new.columns.values) and
+ sorted(mr_old.columns.values) == sorted(mr_new.columns.values)):
+ return True
+ else:
+ err_msg = 'Unequal number and/or column names!\n'
+ return False
+
+
+def test_mr_number_equality(mr_old, mr_new):
+ '''Check if old and new version have the same number of requests.
+
+ If the number is not equal an error message is stored in
+ global variable "err_msg".
+
+ Parameters
+ ----------
+ mr_old : :obj:`pandas DataFrame`
+ Contains the mars requests from the old version of
+ flex_extract.
+
+ mr_new : :obj:`pandas DataFrame`
+ Contains the mars requests from the new version of
+ flex_extract.
+
+ Return
+ ------
+ bool
+ True if successful, False otherwise.
+ '''
+ global err_msg
+ if len(mr_new.index) == len(mr_old.index):
+ return True
+ else:
+ err_msg = 'Unequal number of mars requests!\n'
+ return False
+
+def test_mr_content_equality(mr_old, mr_new):
+ '''Check if old and new version have the same request contents.
+
+ If the content in a column is not equal an error message is stored in
+ global variable "err_msg", recording the column.
+
+ Parameters
+ ----------
+ mr_old : :obj:`pandas DataFrame`
+ Contains the mars requests from the old version of
+ flex_extract.
+
+ mr_new : :obj:`pandas DataFrame`
+ Contains the mars requests from the new version of
+ flex_extract.
+
+ Return
+ ------
+ bool
+ True if successful, False otherwise.
+ '''
+ global err_msg
+ lresult = None
+ columns = list(mr_new.columns.values)
+ del columns[columns.index('target')]
+ mr_new = trim_all_columns(mr_new)
+ mr_old = trim_all_columns(mr_old)
+ for col in columns:
+ if mr_new[col].equals(mr_old[col]):
+ lresult = True
+ else:
+ err_msg += 'Inconsistency in column: ' + col + '\n'
+ print("THERE SEEMS TO BE AN ERROR:")
+ print("CONTENT OF NEW VERSION:")
+ print(mr_new[col])
+ print("CONTENT OF OLD VERSION:")
+ print(mr_old[col])
+ return False
+ return lresult
+
+
+def trim_all_columns(df):
+ """
+ Trim whitespace from ends of each value across all series in dataframe
+ """
+ trim_strings = lambda x: x.strip() if isinstance(x, str) else x
+ return df.applymap(trim_strings)
+
+def convert_param_numbers(mr_old):
+ """
+ Convert the numbers parameter into integers with 3 digits.
+ """
+
+ if str(mr_old).strip() != "OFF" and mr_old != None and '/' in str(mr_old) :
+ numbers = mr_old.split('/')
+ number = str(int(numbers[0])).zfill(3)+'/TO/'+str(int(numbers[2])).zfill(3)
+
+ return number
+
+ return mr_old
+
+def convert_param_step(mr_old):
+ """
+ For pure forecast with steps greater than 23 hours, the older versions
+ writes out a list of steps instead with the syntax 'to' and 'by'.
+ e.g. 000/003/006/009/012/015/018/021/024/027/030/033/036
+
+ Convert this to 0/to/36/by/3
+ """
+
+ #if 'to' in str(mr_old) and 'by' in str(mr_old):
+ # steps = mr_old.split('/')
+ # step = []
+ # for i in range(int(steps[0]),int(steps[2]),int(steps[4])):
+ # step.append(str(int(i)).zfill(2))
+ # return '/'.join(step)
+
+ if not mr_old.isdigit() and 'to' not in mr_old.lower():
+ if int(mr_old.split('/')[-1]) > 23:
+
+ steps = mr_old.split('/')
+ dtime = int(steps[1]) - int(steps[0])
+
+ nsteps = str(int(steps[1]))+'/to/'+str(int(steps[-1]))+'/by/'+str(int(dtime))
+ return nsteps
+
+ return mr_old
+
+def to_param_id(pars, table):
+ '''Transform parameter names to parameter ids with ECMWF grib table 128.
+
+ Parameters
+ ----------
+ pars : str
+ Addpar argument from CONTROL file in the format of
+ parameter names instead of ids. The parameter short
+ names are sepearted with "/" and they are passed as
+ one single string.
+
+ table : dict
+ Contains the ECMWF grib table 128 information.
+ The key is the parameter number and the value is the
+ short name of the parameter.
+
+ Return
+ ------
+ ipar : list of int
+ List of addpar parameters from CONTROL file transformed to
+ parameter ids in the format of integer.
+ '''
+ if not pars:
+ return []
+ if not isinstance(pars, str):
+ pars=str(pars)
+
+ cpar = pars.upper().split('/')
+ spar = []
+ for par in cpar:
+ par = par.strip()
+ for k, v in table.items():
+ if par.isdigit():
+ par = str(int(par)).zfill(3)
+ if par == k or par == v:
+ spar.append(k + '.128')
+ break
+ else:
+ print('\n\Warning: par ' + par + ' not found in table 128\n\n”')
+
+ return '/'.join(str(l) for l in spar)
+
+
+
+if __name__ == '__main__':
+
+ # basic values for paths and versions
+ control_path = 'Controls'
+ log_path = 'Log'
+ old_dir = sys.argv[1] # e.g. '7.0.4'
+ new_dir = sys.argv[2] # e.g. '7.1'
+ mr_filename = 'mars_requests.csv'
+
+ # have to be set to "True" in the beginnning
+ # so it only fails if a test fails
+ lfinal = True
+
+ # prepare log file for this test run
+ currenttime = datetime.now()
+ time_str = currenttime.strftime('%Y-%m-%d_%H-%M-%S')
+ logfile = os.path.join(log_path, 'log_' + time_str)
+ with open(logfile, 'a') as f:
+ f.write('Compare mars requests between version ' + old_dir +
+ ' and version ' + new_dir + ' : \n')
+
+ # list all controlfiles
+ controls = os.listdir(control_path)
+
+ # loop over controlfiles
+ for c in controls:
+ # empty error message for every controlfile
+ err_msg = ''
+
+ # start flex_extract with controlfiles to get mars_request files
+ shutil.copy(os.path.join(control_path,c), _config.PATH_CONTROLFILES)
+ subprocess.check_output(['run_local.sh', new_dir, c])
+ os.remove(os.path.join(_config.PATH_CONTROLFILES,c))
+
+ # cut-of "CONTROL_" string and mv mars_reqeust file
+ # to a name including control specific name
+ mr_name = c.split('CONTROL_')[1] + '.csv'
+ shutil.move(os.path.join(new_dir,mr_filename), os.path.join(new_dir,mr_name))
+
+ # read both mr files (old & new)
+ mr_old = pd.read_csv(os.path.join(old_dir, mr_name))
+ mr_new = pd.read_csv(os.path.join(new_dir, mr_name))
+
+ mr_old.columns = mr_old.columns.str.strip()
+ mr_new.columns = mr_new.columns.str.strip()
+
+ # convert names in old to ids
+ table128 = init128(_config.PATH_GRIBTABLE)
+ #print mr_old['param']
+
+ # some format corrections are necessary to compare older versions with 7.1
+ mr_old['param'] = mr_old['param'].apply(to_param_id, args=[table128])
+ mr_old['number'] = mr_old['number'].apply(convert_param_numbers)
+ if '142' in mr_old.loc[0,'param']: # if flux request
+ mr_old.loc[0,'step'] = convert_param_step(mr_old.loc[0,'step'])
+
+ print('Results: ', c)
+
+ # do tests on mr files
+ lcoleq = test_mr_column_equality(mr_old, mr_new)
+ lnoeq = test_mr_number_equality(mr_old, mr_new)
+ lcoeq = test_mr_content_equality(mr_old, mr_new)
+
+ # check results for mr file
+ lfinal = lfinal and lcoleq and lnoeq and lcoeq
+
+ # write out result to logging file
+ with open(logfile, 'a') as f:
+ if lcoleq and lnoeq and lcoeq:
+ f.write('... ' + c + ' ... OK!' + '\n')
+ else:
+ f.write('... ' + c + ' ... FAILED!' + '\n')
+ if err_msg:
+ f.write('... ' + err_msg + '\n')
+
+ exit
+
+ # exit with success or error status
+ if lfinal:
+ sys.exit(0) # 'SUCCESS'
+ else:
+ sys.exit(1) # 'FAIL'