changed cds api retrieval for single levels to retrieve from CS3 disks instead...

changed cds api retrieval for single levels to retrieve from CS3 disks instead of MARS - severe performance boost

Source/Python/Classes/ControlFile.py

@@ -615,7 +615,7 @@ class ControlFile(object):
         check_request(self.request,
                       os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
 
-        check_public(self.public, self.dataset)
+        check_public(self.public, self.dataset, self.marsclass)
 
         self.type = check_type(self.type, self.step)
 

Source/Python/Classes/MarsRetrieval.py

@@ -446,6 +446,75 @@ class MarsRetrieval(object):
 
         return
     
+    def _convert_to_cdsera5_sfc_request(self, attrs):
+        '''
+        The keywords and values for the single level download
+        with CDS API is different from MARS. This function 
+        converts the old request keywords to the new ones.
+        
+        Example request for single level downloads in CDS API
+        
+        retrieve(
+            'reanalysis-era5-single-levels',
+            {
+                'product_type': 'reanalysis',
+                'variable': 'total_precipitation',
+                'year': '2019',
+                'month': '01',
+                'day': '01',
+                'time': '00:00',
+                'format': 'grib',
+                'grid':[1.0, 1.0],
+                'area': [
+                    45, 0, 43,
+                    12,
+                ],
+            },
+            'download.grib')
+            
+        Parameters
+        ----------
+        attrs : dict
+            Dictionary of the mars request parameters.
+
+        Return
+        ------
+
+        '''
+        from datetime import datetime, timedelta
+        newattrs = {}
+
+        if '/' in attrs['date']:
+            year = set()
+            month = set()
+            day = set()
+            start,end = attrs['date'].split('/')[::2]
+            sdate = datetime.strptime(start, '%Y%m%d')
+            edate = datetime.strptime(end, '%Y%m%d')
+            date = sdate
+            while date <= edate:
+                year.add(date.year)
+                month.add(date.month)
+                day.add(date.day)      
+                date = date + timedelta(days=1)
+            newattrs['year'] =list(year)
+            newattrs['month'] = list(month)
+            newattrs['day'] =  list(day)                       
+        else:
+            date = datetime.strptime(attrs['date'], '%Y%m%d')
+            newattrs['year'] = date.year
+            newattrs['month'] = date.month
+            newattrs['day'] =  date.day                  
+        
+        newattrs['product_type'] = 'reanalysis'
+        newattrs['area'] = attrs['area'].split('/')
+        newattrs['grid'] = list(map(float,attrs['grid'].split('/')))
+        newattrs['param'] = attrs['param'].split('/')        
+        newattrs['time'] = list(map(str,range(0,24,3)))
+        newattrs['format'] = 'grib'
+                
+        return newattrs
+
     def data_retrieve(self):
         '''Submits a MARS retrieval. Depending on the existence of
         ECMWF Web-API or CDS API it is submitted via Python or a
@@ -497,8 +566,15 @@ class MarsRetrieval(object):
         if self.server:
             try:
                 if cds_api and isinstance(self.server, cdsapi.Client):
+                    # distinguish between model (ECMWF MARS access) 
+                    # and surface level (CS3 online access)
+                    if attrs['levtype'].lower() == 'ml':
+                        dataset = _config.CDS_DATASET_ML
+                    else:
+                        dataset = _config.CDS_DATASET_SFC
+                        attrs = self._convert_to_cdsera5_sfc_request(attrs)
                     print('RETRIEVE ERA5 WITH CDS API!')
-                    self.server.retrieve(_config.CDS_DATASET,
+                    self.server.retrieve(dataset,
                                          attrs, target)
                 elif ec_api and isinstance(self.server, ecmwfapi.ECMWFDataServer):
                     print('RETRIEVE PUBLIC DATA (NOT ERA5)!')

Source/Python/Mods/checks.py

@@ -650,7 +650,7 @@ def check_request(request, marsfile):
             silent_remove(marsfile)
     return
 
-def check_public(public, dataset):
+def check_public(public, dataset, marsclass):
     '''Check wether the dataset parameter is set to a
     public data set.
 
@@ -666,7 +666,7 @@ def check_public(public, dataset):
     ------
 
     '''
-    if public and not dataset:
+    if public and not dataset and not (marsclass.upper() == 'EA'):
         raise ValueError('ERROR: If public MARS data are to be retrieved, '
                          'the "dataset"-parameter has to be set, too!')
     return

Source/Python/_config.py

@@ -45,7 +45,8 @@ QUEUES_LIST = ['ecgate', 'cca', 'ccb']
 
 INSTALL_TARGETS = ['local', 'ecgate', 'cca', 'ccb']
 
-CDS_DATASET = 'reanalysis-era5-complete'
+CDS_DATASET_ML = 'reanalysis-era5-complete'
+CDS_DATASET_SFC = 'reanalysis-era5-single-levels'
 
 # up-to-date available maximum level numbers at ECMWF, 05.10.2018
 MAX_LEVEL_LIST = [16, 19, 31, 40, 50, 60, 62, 91, 137]

Testing/Installation/Api_testscripts/test_cdsapi_modellevel.py

@@ -12,7 +12,7 @@ c.retrieve('reanalysis-era5-complete',
 	'stream'  : 'oper',
 	'type'    : 'fc',
 	'step'    : '3/to/12/by/3',
-	'param'   : '130.128',
+	'param'   : 't',
 	'levtype' : 'ml',
 	'levelist': '135/to/137',
 	'date'    : '2013-01-01',

Testing/Installation/Api_testscripts/test_ecmwfapi_public.py

@@ -16,5 +16,6 @@ server.retrieve({
 	'date'      : "2000-07-01/to/2000-07-31",
 	'type'      : "an",
 	'class'     : "ep",
+    'number'    : "0",
 	'target'    : "download_cera20c_ecmwfapi.grib"
 })