From 0a75335e5737a1a8e19c43a9d96c0b86a0bfd820 Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Fri, 10 Jul 2020 10:20:31 +0200
Subject: [PATCH] changed cds api retrieval for single levels to retrieve from
CS3 disks instead of MARS - severe performance boost
---
Source/Python/Classes/ControlFile.py | 2 +-
Source/Python/Classes/MarsRetrieval.py | 80 ++++++++++++++++++-
Source/Python/Mods/checks.py | 4 +-
Source/Python/_config.py | 3 +-
.../Api_testscripts/test_cdsapi_modellevel.py | 2 +-
.../Api_testscripts/test_ecmwfapi_public.py | 1 +
6 files changed, 85 insertions(+), 7 deletions(-)
diff --git a/Source/Python/Classes/ControlFile.py b/Source/Python/Classes/ControlFile.py
index ffd4ddf..8ffff5a 100644
--- a/Source/Python/Classes/ControlFile.py
+++ b/Source/Python/Classes/ControlFile.py
@@ -615,7 +615,7 @@ class ControlFile(object):
check_request(self.request,
os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
- check_public(self.public, self.dataset)
+ check_public(self.public, self.dataset, self.marsclass)
self.type = check_type(self.type, self.step)
diff --git a/Source/Python/Classes/MarsRetrieval.py b/Source/Python/Classes/MarsRetrieval.py
index 0073221..dd97879 100644
--- a/Source/Python/Classes/MarsRetrieval.py
+++ b/Source/Python/Classes/MarsRetrieval.py
@@ -445,6 +445,75 @@ class MarsRetrieval(object):
f.write('\n')
return
+
+ def _convert_to_cdsera5_sfc_request(self, attrs):
+ '''
+ The keywords and values for the single level download
+ with CDS API is different from MARS. This function
+ converts the old request keywords to the new ones.
+
+ Example request for single level downloads in CDS API
+
+ retrieve(
+ 'reanalysis-era5-single-levels',
+ {
+ 'product_type': 'reanalysis',
+ 'variable': 'total_precipitation',
+ 'year': '2019',
+ 'month': '01',
+ 'day': '01',
+ 'time': '00:00',
+ 'format': 'grib',
+ 'grid':[1.0, 1.0],
+ 'area': [
+ 45, 0, 43,
+ 12,
+ ],
+ },
+ 'download.grib')
+
+ Parameters
+ ----------
+ attrs : dict
+ Dictionary of the mars request parameters.
+
+ Return
+ ------
+
+ '''
+ from datetime import datetime, timedelta
+ newattrs = {}
+
+ if '/' in attrs['date']:
+ year = set()
+ month = set()
+ day = set()
+ start,end = attrs['date'].split('/')[::2]
+ sdate = datetime.strptime(start, '%Y%m%d')
+ edate = datetime.strptime(end, '%Y%m%d')
+ date = sdate
+ while date <= edate:
+ year.add(date.year)
+ month.add(date.month)
+ day.add(date.day)
+ date = date + timedelta(days=1)
+ newattrs['year'] =list(year)
+ newattrs['month'] = list(month)
+ newattrs['day'] = list(day)
+ else:
+ date = datetime.strptime(attrs['date'], '%Y%m%d')
+ newattrs['year'] = date.year
+ newattrs['month'] = date.month
+ newattrs['day'] = date.day
+
+ newattrs['product_type'] = 'reanalysis'
+ newattrs['area'] = attrs['area'].split('/')
+ newattrs['grid'] = list(map(float,attrs['grid'].split('/')))
+ newattrs['param'] = attrs['param'].split('/')
+ newattrs['time'] = list(map(str,range(0,24,3)))
+ newattrs['format'] = 'grib'
+
+ return newattrs
def data_retrieve(self):
'''Submits a MARS retrieval. Depending on the existence of
@@ -478,7 +547,7 @@ class MarsRetrieval(object):
if not int(self.public):
del attrs['target']
print('target: ' + target)
-
+
# find all keys without a value and convert all other values to strings
empty_keys = []
for key, value in attrs.items():
@@ -497,8 +566,15 @@ class MarsRetrieval(object):
if self.server:
try:
if cds_api and isinstance(self.server, cdsapi.Client):
+ # distinguish between model (ECMWF MARS access)
+ # and surface level (CS3 online access)
+ if attrs['levtype'].lower() == 'ml':
+ dataset = _config.CDS_DATASET_ML
+ else:
+ dataset = _config.CDS_DATASET_SFC
+ attrs = self._convert_to_cdsera5_sfc_request(attrs)
print('RETRIEVE ERA5 WITH CDS API!')
- self.server.retrieve(_config.CDS_DATASET,
+ self.server.retrieve(dataset,
attrs, target)
elif ec_api and isinstance(self.server, ecmwfapi.ECMWFDataServer):
print('RETRIEVE PUBLIC DATA (NOT ERA5)!')
diff --git a/Source/Python/Mods/checks.py b/Source/Python/Mods/checks.py
index 2de186e..27b4d47 100644
--- a/Source/Python/Mods/checks.py
+++ b/Source/Python/Mods/checks.py
@@ -650,7 +650,7 @@ def check_request(request, marsfile):
silent_remove(marsfile)
return
-def check_public(public, dataset):
+def check_public(public, dataset, marsclass):
'''Check wether the dataset parameter is set to a
public data set.
@@ -666,7 +666,7 @@ def check_public(public, dataset):
------
'''
- if public and not dataset:
+ if public and not dataset and not (marsclass.upper() == 'EA'):
raise ValueError('ERROR: If public MARS data are to be retrieved, '
'the "dataset"-parameter has to be set, too!')
return
diff --git a/Source/Python/_config.py b/Source/Python/_config.py
index 15b4ce0..dae1de9 100644
--- a/Source/Python/_config.py
+++ b/Source/Python/_config.py
@@ -45,7 +45,8 @@ QUEUES_LIST = ['ecgate', 'cca', 'ccb']
INSTALL_TARGETS = ['local', 'ecgate', 'cca', 'ccb']
-CDS_DATASET = 'reanalysis-era5-complete'
+CDS_DATASET_ML = 'reanalysis-era5-complete'
+CDS_DATASET_SFC = 'reanalysis-era5-single-levels'
# up-to-date available maximum level numbers at ECMWF, 05.10.2018
MAX_LEVEL_LIST = [16, 19, 31, 40, 50, 60, 62, 91, 137]
diff --git a/Testing/Installation/Api_testscripts/test_cdsapi_modellevel.py b/Testing/Installation/Api_testscripts/test_cdsapi_modellevel.py
index baba95c..591184f 100644
--- a/Testing/Installation/Api_testscripts/test_cdsapi_modellevel.py
+++ b/Testing/Installation/Api_testscripts/test_cdsapi_modellevel.py
@@ -12,7 +12,7 @@ c.retrieve('reanalysis-era5-complete',
'stream' : 'oper',
'type' : 'fc',
'step' : '3/to/12/by/3',
- 'param' : '130.128',
+ 'param' : 't',
'levtype' : 'ml',
'levelist': '135/to/137',
'date' : '2013-01-01',
diff --git a/Testing/Installation/Api_testscripts/test_ecmwfapi_public.py b/Testing/Installation/Api_testscripts/test_ecmwfapi_public.py
index 89013e0..bac172d 100644
--- a/Testing/Installation/Api_testscripts/test_ecmwfapi_public.py
+++ b/Testing/Installation/Api_testscripts/test_ecmwfapi_public.py
@@ -16,5 +16,6 @@ server.retrieve({
'date' : "2000-07-01/to/2000-07-31",
'type' : "an",
'class' : "ep",
+ 'number' : "0",
'target' : "download_cera20c_ecmwfapi.grib"
})
--
GitLab