From 53d3b2a98458bb896f69efe86b1267b7879b7612 Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Fri, 31 Jan 2020 13:32:29 +0100
Subject: [PATCH] added more tests for grib file comparison and removed
detected BUGS
---
Source/Python/Classes/EcFlexpart.py | 21 +--
.../7.1/Controls/CONTROL_CERA.compare | 3 +-
.../7.1/Controls/CONTROL_EI.compare | 2 +-
.../CONTROL_OD.OPER.FC.36hours.compare | 1 +
.../Controls/CONTROL_OD.OPER.FC.eta.compare | 1 +
.../Log/log_2020-01-31_13-17-45 | 135 ++++++++++++++++++
.../Compare_gribfiles/run_cmp_test.sh | 59 ++++++++
.../Compare_gribfiles/test_cmp_grib_file.py | 12 ++
.../Mars_request/test_cmp_mars_requests.py | 11 ++
9 files changed, 235 insertions(+), 10 deletions(-)
create mode 100644 Testing/Regression/Compare_gribfiles/Log/log_2020-01-31_13-17-45
create mode 100755 Testing/Regression/Compare_gribfiles/run_cmp_test.sh
diff --git a/Source/Python/Classes/EcFlexpart.py b/Source/Python/Classes/EcFlexpart.py
index 360cbfa..0678067 100644
--- a/Source/Python/Classes/EcFlexpart.py
+++ b/Source/Python/Classes/EcFlexpart.py
@@ -320,11 +320,16 @@ class EcFlexpart(object):
------
'''
+ if self.purefc:
+ # need to retrieve forecasts for step 000 in case of pure forecast
+ steps = '{}/to/{}/by/{}'.format(0, self.accmaxstep, self.dtime)
+ else:
+ steps = '{}/to/{}/by/{}'.format(self.dtime,
+ self.accmaxstep,
+ self.dtime)
+
self.types[str(self.acctype)] = {'times': str(self.acctime),
- 'steps': '{}/to/{}/by/{}'.format(
- self.dtime,
- self.accmaxstep,
- self.dtime)}
+ 'steps': steps}
return
def _create_params(self, gauss, eta, omega, cwc, wrf):
@@ -780,8 +785,8 @@ class EcFlexpart(object):
self._start_retrievement(request, retr_param_dict)
elif self.basetime == 0:
- retr_param_dict['date'] = \
- datetime.strftime(elimit - t24h, '%Y%m%d')
+# retr_param_dict['date'] = \
+# datetime.strftime(elimit - t24h, '%Y%m%d')
timesave = ''.join(retr_param_dict['time'])
@@ -1601,11 +1606,11 @@ class EcFlexpart(object):
'%Y%m%d%H')
# skip all temporary times
- # which are outside the retrieval period
+ # which are outside the retrieval period
if timestamp < start_period or \
timestamp > end_period:
continue
-
+
# @WRF
# THIS IS NOT YET CORRECTLY IMPLEMENTED !!!
diff --git a/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_CERA.compare b/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_CERA.compare
index da50d5e..a3f7e01 100644
--- a/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_CERA.compare
+++ b/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_CERA.compare
@@ -1,4 +1,4 @@
-START_DATE 20090809
+START_DATE 20000908
DTIME 3
TYPE AN AN AN AN AN AN AN AN
TIME 00 03 06 09 12 15 18 21
@@ -20,3 +20,4 @@ ETA 1
CWC 1
PREFIX CE
ECTRANS 1
+FORMAT GRIB2
diff --git a/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_EI.compare b/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_EI.compare
index 4aeb048..a5f9fad 100644
--- a/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_EI.compare
+++ b/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_EI.compare
@@ -1,4 +1,4 @@
-START_DATE 20090809
+START_DATE 20150809
DTIME 3
TYPE AN FC FC FC AN FC FC FC
TIME 00 00 00 00 12 12 12 12
diff --git a/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_OD.OPER.FC.36hours.compare b/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_OD.OPER.FC.36hours.compare
index 6341703..daf538e 100644
--- a/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_OD.OPER.FC.36hours.compare
+++ b/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_OD.OPER.FC.36hours.compare
@@ -20,3 +20,4 @@ ETA 1
CWC 1
PREFIX FC
ECTRANS 1
+FORMAT GRIB2
diff --git a/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_OD.OPER.FC.eta.compare b/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_OD.OPER.FC.eta.compare
index b944e8c..61659b1 100644
--- a/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_OD.OPER.FC.eta.compare
+++ b/Testing/Regression/Compare_gribfiles/7.1/Controls/CONTROL_OD.OPER.FC.eta.compare
@@ -13,6 +13,7 @@ RIGHT 15.
LEVELIST 130/TO/137
RESOL 799
ETA 1
+CWC 1
FORMAT GRIB2
PREFIX ENE
ECTRANS 1
diff --git a/Testing/Regression/Compare_gribfiles/Log/log_2020-01-31_13-17-45 b/Testing/Regression/Compare_gribfiles/Log/log_2020-01-31_13-17-45
new file mode 100644
index 0000000..bdac0af
--- /dev/null
+++ b/Testing/Regression/Compare_gribfiles/Log/log_2020-01-31_13-17-45
@@ -0,0 +1,135 @@
+Compare GRIB files between version + old_version + and version + new_version + : \n
+
+
+Welcome!
+Reference path is: 7.0.4/BASETIME/
+New path is: 7.1/BASETIME/
+Filepattern is: *
+The input files are: ['BT19063015', 'BT19063018', 'BT19063021', 'BT19070100']
+The input files are: ['BT19063015', 'BT19063018', 'BT19063021', 'BT19070100']
+
+-- GRIB #55 -- shortName=ssr paramId=176 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [totalLength]: [220] != [583]
+
+... FILES HAVE DIFFERENCES IN GRIB MESSAGES!
+===================================================================================================
+
+
+Welcome!
+Reference path is: 7.0.4/CERA/
+New path is: 7.1/CERA/
+Filepattern is: *
+The input files are: ['CE00090800', 'CE00090803', 'CE00090806', 'CE00090809', 'CE00090812', 'CE00090815', 'CE00090818', 'CE00090821']
+The input files are: ['CE00090800', 'CE00090803', 'CE00090806', 'CE00090809', 'CE00090812', 'CE00090815', 'CE00090818', 'CE00090821']
+
+-- GRIB #548 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [tablesVersion]: [23] != [5]
+string [typeOfProcessedData]: [1] != [fc]
+string [typeOfFirstFixedSurface]: [1] != [sfc]
+
+
+-- GRIB #548 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [tablesVersion]: [23] != [5]
+string [typeOfProcessedData]: [1] != [fc]
+string [typeOfFirstFixedSurface]: [1] != [sfc]
+
+
+-- GRIB #548 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [tablesVersion]: [23] != [5]
+string [typeOfProcessedData]: [1] != [fc]
+string [typeOfFirstFixedSurface]: [1] != [sfc]
+
+
+-- GRIB #548 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [tablesVersion]: [23] != [5]
+string [typeOfProcessedData]: [1] != [fc]
+string [typeOfFirstFixedSurface]: [1] != [sfc]
+
+
+-- GRIB #548 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [tablesVersion]: [23] != [5]
+string [typeOfProcessedData]: [1] != [fc]
+string [typeOfFirstFixedSurface]: [1] != [sfc]
+
+
+-- GRIB #548 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [tablesVersion]: [23] != [5]
+string [typeOfProcessedData]: [1] != [fc]
+string [typeOfFirstFixedSurface]: [1] != [sfc]
+
+
+-- GRIB #548 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [tablesVersion]: [23] != [5]
+string [typeOfProcessedData]: [1] != [fc]
+string [typeOfFirstFixedSurface]: [1] != [sfc]
+
+
+-- GRIB #548 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [tablesVersion]: [23] != [5]
+string [typeOfProcessedData]: [1] != [fc]
+string [typeOfFirstFixedSurface]: [1] != [sfc]
+
+... FILES HAVE DIFFERENCES IN GRIB MESSAGES!
+===================================================================================================
+
+
+Welcome!
+Reference path is: 7.0.4/EA5/
+New path is: 7.1/EA5/
+Filepattern is: *
+The input files are: ['EA18120100', 'EA18120103', 'EA18120106', 'EA18120109', 'EA18120112', 'EA18120115', 'EA18120118', 'EA18120121']
+The input files are: ['EA18120100', 'EA18120103', 'EA18120106', 'EA18120109', 'EA18120112', 'EA18120115', 'EA18120118', 'EA18120121']
+GRIB_COMPARISON: SUCCESSFULL!
+===================================================================================================
+
+
+Welcome!
+Reference path is: 7.0.4/EI/
+New path is: 7.1/EI/
+Filepattern is: *
+The input files are: ['EI15080900', 'EI15080903', 'EI15080906', 'EI15080909', 'EI15080912', 'EI15080915', 'EI15080918', 'EI15080921']
+The input files are: ['EI15080900', 'EI15080903', 'EI15080906', 'EI15080909', 'EI15080912', 'EI15080915', 'EI15080918', 'EI15080921']
+GRIB_COMPARISON: SUCCESSFULL!
+===================================================================================================
+
+
+Welcome!
+Reference path is: 7.0.4/ETAOD/
+New path is: 7.1/ETAOD/
+Filepattern is: *
+The input files are: ['ENE18080900', 'ENE18080901', 'ENE18080902', 'ENE18080903', 'ENE18080904', 'ENE18080905', 'ENE18080906', 'ENE18080907', 'ENE18080908', 'ENE18080909', 'ENE18080910', 'ENE18080911', 'ENE18080912', 'ENE18080913', 'ENE18080914', 'ENE18080915', 'ENE18080916', 'ENE18080917', 'ENE18080918', 'ENE18080919', 'ENE18080920', 'ENE18080921', 'ENE18080922', 'ENE18080923']
+The input files are: ['ENE18080900', 'ENE18080901', 'ENE18080902', 'ENE18080903', 'ENE18080904', 'ENE18080905', 'ENE18080906', 'ENE18080907', 'ENE18080908', 'ENE18080909', 'ENE18080910', 'ENE18080911', 'ENE18080912', 'ENE18080913', 'ENE18080914', 'ENE18080915', 'ENE18080916', 'ENE18080917', 'ENE18080918', 'ENE18080919', 'ENE18080920', 'ENE18080921', 'ENE18080922', 'ENE18080923']
+GRIB_COMPARISON: SUCCESSFULL!
+===================================================================================================
+
+
+Welcome!
+Reference path is: 7.0.4/GAUSSOD/
+New path is: 7.1/GAUSSOD/
+Filepattern is: *
+The input files are: ['ENG18080900', 'ENG18080901', 'ENG18080902', 'ENG18080903', 'ENG18080904', 'ENG18080905', 'ENG18080906', 'ENG18080907', 'ENG18080908', 'ENG18080909', 'ENG18080910', 'ENG18080911', 'ENG18080912', 'ENG18080913', 'ENG18080914', 'ENG18080915', 'ENG18080916', 'ENG18080917', 'ENG18080918', 'ENG18080919', 'ENG18080920', 'ENG18080921', 'ENG18080922', 'ENG18080923']
+The input files are: ['ENG18080900', 'ENG18080901', 'ENG18080902', 'ENG18080903', 'ENG18080904', 'ENG18080905', 'ENG18080906', 'ENG18080907', 'ENG18080908', 'ENG18080909', 'ENG18080910', 'ENG18080911', 'ENG18080912', 'ENG18080913', 'ENG18080914', 'ENG18080915', 'ENG18080916', 'ENG18080917', 'ENG18080918', 'ENG18080919', 'ENG18080920', 'ENG18080921', 'ENG18080922', 'ENG18080923']
+GRIB_COMPARISON: SUCCESSFULL!
+===================================================================================================
+
+
+Welcome!
+Reference path is: 7.0.4/PUREFC/
+New path is: 7.1/PUREFC/
+Filepattern is: *
+The input files are: ['FC180809.00.000', 'FC180809.00.003', 'FC180809.00.006', 'FC180809.00.009', 'FC180809.00.012', 'FC180809.00.015', 'FC180809.00.018', 'FC180809.00.021', 'FC180809.00.024', 'FC180809.00.027', 'FC180809.00.030', 'FC180809.00.033', 'FC180809.00.036']
+The input files are: ['FC180809.00.000', 'FC180809.00.003', 'FC180809.00.006', 'FC180809.00.009', 'FC180809.00.012', 'FC180809.00.015', 'FC180809.00.018', 'FC180809.00.021', 'FC180809.00.024', 'FC180809.00.027', 'FC180809.00.030', 'FC180809.00.033', 'FC180809.00.036']
+
+-- GRIB #50 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [day]: [10] != [9]
+long [hour]: [9] != [0]
+long [forecastTime]: [0] != [33]
+
+
+-- GRIB #50 -- shortName=lsp paramId=142 stepRange=0 levelType=1 level=0 packingType= gridType=regular_ll --
+long [day]: [10] != [9]
+long [hour]: [12] != [0]
+long [forecastTime]: [0] != [36]
+
+... FILES HAVE DIFFERENCES IN GRIB MESSAGES!
+===================================================================================================
diff --git a/Testing/Regression/Compare_gribfiles/run_cmp_test.sh b/Testing/Regression/Compare_gribfiles/run_cmp_test.sh
new file mode 100755
index 0000000..b363ab5
--- /dev/null
+++ b/Testing/Regression/Compare_gribfiles/run_cmp_test.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# @Author: Anne Philipp
+#
+# @Date: November, 20 2019
+#
+# @Description: Starts the comparison script for all cases found in the new
+# version directory. Results are written to a log file placed
+# in the Log directory.
+#
+# @Call command: ./run_cmp_test.sh <reference version> <new version>
+#
+# @ChangeHistory:
+#
+# @Licence:
+# (C) Copyright 2014-2019.
+#
+# SPDX-License-Identifier: CC-BY-4.0
+#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
+#
+# @Example:
+# ./run_cmp_test.sh 7.0.4 7.1
+#
+
+if [ $# -eq 0 ]; then
+ echo "No arguments passed"
+ exit
+fi
+
+if [ $# -eq 1 ]; then
+ echo "Second argument is missing"
+ exit
+fi
+
+old_version=$1
+new_version=$2
+
+current_time=$(date "+%Y-%m-%d_%H-%M-%S")
+testcases=`ls ${new_version}/`
+
+echo 'Test to compare GRIB files between two versions'
+echo 'Compare GRIB files between version ' + old_version + ' and version ' + new_version + ' : \n' > Log/log_$current_time
+
+for case in $testcases; do
+
+ if [[ "$case" == "Controls" ]]; then
+ continue
+ fi
+ echo "Compare $case ..."
+ python test_cmp_grib_file.py -r 7.0.4/${case}/ -n 7.1/${case}/ -p '*' >> Log/log_$current_time 2>&1
+
+ echo "===================================================================================================" >> Log/log_$current_time
+
+done
+
diff --git a/Testing/Regression/Compare_gribfiles/test_cmp_grib_file.py b/Testing/Regression/Compare_gribfiles/test_cmp_grib_file.py
index 920fe71..196131c 100644
--- a/Testing/Regression/Compare_gribfiles/test_cmp_grib_file.py
+++ b/Testing/Regression/Compare_gribfiles/test_cmp_grib_file.py
@@ -11,6 +11,17 @@ The script should be called like:
Note
----
+Licence:
+--------
+ (C) Copyright 2014-2019.
+
+ SPDX-License-Identifier: CC-BY-4.0
+
+ This work is licensed under the Creative Commons Attribution 4.0
+ International License. To view a copy of this license, visit
+ http://creativecommons.org/licenses/by/4.0/ or send a letter to
+ Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
+
Example
-------
@@ -22,6 +33,7 @@ Example
# ------------------------------------------------------------------------------
import os
import sys
+from datetime import datetime
from eccodes import GribFile, GribMessage
# ------------------------------------------------------------------------------
diff --git a/Testing/Regression/Mars_request/test_cmp_mars_requests.py b/Testing/Regression/Mars_request/test_cmp_mars_requests.py
index d5dc0de..9bfade9 100755
--- a/Testing/Regression/Mars_request/test_cmp_mars_requests.py
+++ b/Testing/Regression/Mars_request/test_cmp_mars_requests.py
@@ -23,6 +23,17 @@ with the "run_local.sh" script.
It is necessary to have a directory named after the version number of
flex_extract. For example: "7.0.3" and "7.1".
+Licence:
+--------
+ (C) Copyright 2014-2019.
+
+ SPDX-License-Identifier: CC-BY-4.0
+
+ This work is licensed under the Creative Commons Attribution 4.0
+ International License. To view a copy of this license, visit
+ http://creativecommons.org/licenses/by/4.0/ or send a letter to
+ Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
+
Example
-------
python test_cmp_mars_requests.py 7.0.4 7.1
--
GitLab