added possibility to extract public datasets via an logical public parameter

run/control/CONTROL_EI.public

+DAY1 
+DAY2 
+DTIME 6
+TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
+TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
+STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
+CLASS EI
+DATASET interim
+STREAM OPER
+NUMBER OFF
+EXPVER 1
+GRID 5000 
+LEFT -15000
+LOWER 30000
+UPPER 75000
+RIGHT 45000
+LEVELIST 55/to/60
+RESOL 63
+ACCURACY 24
+GAUSS 1
+OMEGA 0
+OMEGADIFF 0
+ETA 0
+ETADIFF 0
+DPDETA 1
+SMOOTH 0
+FORMAT GRIB1
+ADDPAR 186/187/188/235/139/39
+PREFIX EIpub  
+ECSTORAGE 0
+ECTRANS 1
+ECFSDIR ectmp:/${USER}/econdemand/
+MAILFAIL ${USER} 
+MAILOPS ${USER}
+GRIB2FLEXPART 0
+EOF
+

run/run.sh

@@ -22,14 +22,15 @@ BASETIME=None
 STEP=None
 LEVELIST=None
 AREA=None
-INPUTDIR='../../run/workspace/test'
+INPUTDIR='/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/run/workspace/test'
 OUTPUTDIR=None
 FLEXPART_ROOT_SCRIPTS=None 
 PP_ID=None
 JOB_TEMPLATE='job.temp' 
-CONTROLFILE='CONTROL.temp' 
+CONTROLFILE='CONTROL_EI.public' 
 DEBUG=1 
 REQUEST=1
+PUBLIC=1
 
 # -----------------------------------------------------------------
 #
@@ -92,6 +93,9 @@ fi
 if [ -n "$REQUEST" ]; then
   parameterlist+=" --request=$REQUEST"
 fi
+if [ -n "$PUBLIC" ]; then
+  parameterlist+=" --public=$PUBLIC"
+fi
 
 # -----------------------------------------------------------------
 # CALL INSTALLATION SCRIPT WITH DETERMINED COMMANDLINE ARGUMENTS

source/python/classes/ControlFile.py

@@ -113,6 +113,7 @@ class ControlFile(object):
         self.time = None
         self.step = None
         self.marsclass = None
+        self.dataset = None
         self.stream = None
         self.number = 'OFF'
         self.expver = '1'
@@ -158,10 +159,11 @@ class ControlFile(object):
         self.install_target = None
         self.debug = 0
         self.request = 0
+        self.public = 0
 
         self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
                          'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
-                         'ectrans', 'debug', 'request']
+                         'ectrans', 'debug', 'request', 'public']
 
         self.__read_controlfile__()
 
@@ -265,7 +267,7 @@ class ControlFile(object):
         '''
         import collections
 
-        attrs = vars(self)
+        attrs = vars(self).copy()
         attrs = collections.OrderedDict(sorted(attrs.items()))
 
         return '\n'.join("%s: %s" % item for item in attrs.items())
@@ -453,6 +455,12 @@ class ControlFile(object):
             if not isinstance(getattr(self, var), int):
                 setattr(self, var, int(getattr(self, var)))
 
+        if self.public and not self.dataset:
+            print('ERROR: ')
+            print('If public mars data wants to be retrieved, '
+                  'the "dataset"-parameter has to be set in the control file!')
+            sys.exit(1)
+
         return
 
     def check_install_conditions(self):
@@ -523,7 +531,7 @@ class ControlFile(object):
 
         import collections
 
-        attrs = collections.OrderedDict(sorted(vars(self).items()))
+        attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
 
         l = list()
 

source/python/classes/EcFlexpart.py

@@ -88,7 +88,8 @@ from gribapi import grib_set, grib_index_select, grib_new_from_index, grib_get,\
 sys.path.append('../')
 import _config
 from GribTools import GribTools
-from mods.tools import init128, to_param_id, silent_remove, product, my_error
+from mods.tools import (init128, to_param_id, silent_remove, product,
+                        my_error, make_dir)
 from MarsRetrieval import MarsRetrieval
 import mods.disaggregation as disaggregation
 
@@ -142,6 +143,7 @@ class EcFlexpart(object):
                 c.time.append(c.time[0])
 
         self.inputdir = c.inputdir
+        self.dataset = c.dataset
         self.basetime = c.basetime
         self.dtime = c.dtime
         i = 0
@@ -337,9 +339,9 @@ class EcFlexpart(object):
             par_dict: dictionary
                 Contains all parameter which have to be set for creating the
                 Mars Retrievals. The parameter are:
-                marsclass, stream, type, levtype, levelist, resol, gaussian,
-                accuracy, grid, target, area, date, time, number, step, expver,
-                param
+                marsclass, dataset, stream, type, levtype, levelist, resol,
+                gaussian, accuracy, grid, target, area, date, time, number,
+                step, expver, param
 
         @Return:
             <nothing>
@@ -348,7 +350,9 @@ class EcFlexpart(object):
         self.mreq_count += 1
 
         MR = MarsRetrieval(self.server,
+                           self.public,
                            marsclass=par_dict['marsclass'],
+                           dataset=par_dict['dataset'],
                            stream=par_dict['stream'],
                            type=par_dict['type'],
                            levtype=par_dict['levtype'],
@@ -438,7 +442,7 @@ class EcFlexpart(object):
         return iid, index_vals
 
 
-    def retrieve(self, server, dates, request, inputdir='.'):
+    def retrieve(self, server, dates, public, request, inputdir='.'):
         '''
         @Description:
             Finalizing the retrieval information by setting final details
@@ -476,6 +480,7 @@ class EcFlexpart(object):
         '''
         self.dates = dates
         self.server = server
+        self.public = public
         self.inputdir = inputdir
         oro = False
 
@@ -487,6 +492,7 @@ class EcFlexpart(object):
         # entries with a "None" will change in different requests and will
         # therefore be set in each request seperately
         retr_param_dict = {'marsclass':self.marsclass,
+                           'dataset':self.dataset,
                            'stream':None,
                            'type':None,
                            'levtype':None,
@@ -1264,7 +1270,7 @@ class EcFlexpart(object):
 
         # create Options dir if necessary
         if not os.path.exists(pwd + '/Options'):
-            os.makedirs(pwd+'/Options')
+            make_dir(pwd+'/Options')
 
         # read template COMMAND file
         with open(os.path.expandvars(os.path.expanduser(

source/python/classes/MarsRetrieval.py

@@ -82,10 +82,10 @@ class MarsRetrieval(object):
 
     '''
 
-    def __init__(self, server, marsclass="ei", type="", levtype="",
-                 levelist="", repres="", date="", resol="", stream="",
-                 area="", time="", step="", expver="1", number="",
-                 accuracy="", grid="", gaussian="", target="",
+    def __init__(self, server, public, marsclass="ei", dataset="", type="",
+                 levtype="", levelist="", repres="", date="", resol="",
+                 stream="", area="", time="", step="", expver="1",
+                 number="", accuracy="", grid="", gaussian="", target="",
                  param=""):
         '''
         @Description:
@@ -105,11 +105,25 @@ class MarsRetrieval(object):
                 This is the connection to the ECMWF data servers.
                 It is needed for the pythonic access of ECMWF data.
 
+            public: integer
+                Decides which Web API version is used:
+                0: member-state users and full archive access
+                1: public access and limited access to the public server and
+                   datasets. Needs the parameter dataset.
+                Default is "0" and for member-state users.
+
             marsclass: string, optional
                 Characterisation of dataset. E.g. EI (ERA-Interim),
                 E4 (ERA40), OD (Operational archive), ea (ERA5).
                 Default is the ERA-Interim dataset "ei".
 
+            dataset: string, optional
+                For public datasets there is the specific naming and parameter
+                dataset which has to be used to characterize the type of
+                data. Usually there is less data available, either in times,
+                domain or parameter.
+                Default is an empty string.
+
             type: string, optional
                 Determines the type of fields to be retrieved.
                 Selects between observations, images or fields.
@@ -288,7 +302,9 @@ class MarsRetrieval(object):
         '''
 
         self.server = server
+        self.public = public
         self.marsclass = marsclass
+        self.dataset = dataset
         self.type = type
         self.levtype = levtype
         self.levelist = levelist
@@ -324,12 +340,12 @@ class MarsRetrieval(object):
             <nothing>
         '''
         # Get all class attributes and their values as a dictionary
-        attrs = vars(self)
+        attrs = vars(self).copy()
 
         # iterate through all attributes and print them
         # with their corresponding values
         for item in attrs.items():
-            if item[0] in 'server':
+            if item[0] in ['server', 'public']:
                 pass
             else:
                 print(item[0] + ': ' + str(item[1]))
@@ -357,7 +373,7 @@ class MarsRetrieval(object):
             <nothing>
         '''
         # Get all class attributes and their values as a dictionary
-        attrs = vars(self)
+        attrs = vars(self).copy()
 
         # open a file to store all requests to
         with open(os.path.join(inputdir,
@@ -366,7 +382,7 @@ class MarsRetrieval(object):
             # iterate through all attributes and print them
             # with their corresponding values
             for item in attrs.items():
-                if item[0] in 'server':
+                if item[0] in ['server', 'public']:
                     pass
                 else:
                     f.write(item[0] + ': ' + str(item[1]) + '\n')
@@ -395,8 +411,9 @@ class MarsRetrieval(object):
         '''
 
         # Get all class attributes and their values as a dictionary
-        attrs = vars(self)
+        attrs = vars(self).copy()
         del attrs['server']
+        del attrs['public']
 
         # open a file to store all requests to
         with open(os.path.join(inputdir,
@@ -424,50 +441,80 @@ class MarsRetrieval(object):
             <nothing>
         '''
         # Get all class attributes and their values as a dictionary
-        attrs = vars(self)
-
-        # convert the dictionary of attributes into a comma
-        # seperated list of attributes with their values
-        # needed for the retrieval call
-        s = 'ret'
-        for k, v in attrs.iteritems():
-            if k in 'server':
-                continue
-            if k == 'marsclass':
-                k = 'class'
-            if v == '':
-                continue
-            if k.lower() == 'target':
-                target = v
+        attrs = vars(self).copy()
+
+        # eliminate unnecessary attributes from the dictionary attrs
+        del attrs['server']
+        del attrs['public']
+
+        # exchange parameter name for marsclass
+        mclass = attrs.get('marsclass')
+        del attrs['marsclass']
+        attrs['class'] = mclass
+
+        # prepare target variable as needed for the Web API mode
+        # within the dictionary for full access
+        # as a single variable for public access
+        target = attrs.get('target')
+        if not int(self.public):
+            del attrs['target']
+        print('target: ' + target)
+
+        # find all keys without a value and convert all other values to strings
+        empty_keys = []
+        for key, value in attrs.itteritems():
+            if value == '':
+                empty_keys.append(str(key))
             else:
-                s = s + ',' + k + '=' + str(v)
+                attrs[key] = str(value)
+
+        # delete all empty parameter from the dictionary
+        for key in empty_keys:
+            del attrs[key]
 
         # MARS request via Python script
-        if self.server is not False:
+        if self.server:
             try:
-                self.server.execute(s, target)
+                if self.public:
+                    print('RETRIEVE PUBLIC DATA!')
+                    self.server.retrieve(attrs)
+                else:
+                    print('EXECUTE NON-PUBLIC RETRIEVAL!')
+                    self.server.execute(attrs, target)
             except:
-                print('MARS Request failed, \
-                      have you already registered at apps.ecmwf.int?')
-                raise IOError
-            if os.stat(target).st_size == 0:
+                e = sys.exc_info()[0]
+                print("ERROR: ", e)
+                print('MARS Request failed!')
+            if not self.public and os.stat(target).st_size == 0:
                 print('MARS Request returned no data - please check request')
                 raise IOError
+            elif self.public and os.stat(target).st_size == 0:
+                print('Public MARS Request returned no data - '
+                      'please check request')
+                raise IOError
+            else:
+                raise IOError
         # MARS request via extra process in shell
         else:
-            s += ',target = "' + target + '"'
-            p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
+            request_str = 'ret'
+            for key, value in attrs.iteritems():
+                request_str = request_str + ',' + key + '=' + str(value)
+            request_str += ',target="' + target + '"'
+            p = subprocess.Popen(['mars'],
+                                 stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE, bufsize=1)
-            pout = p.communicate(input=s)[0]
+                                 stderr=subprocess.PIPE,
+                                 bufsize=1)
+            pout = p.communicate(input=request_str)[0]
             print(pout.decode())
 
             if 'Some errors reported' in pout.decode():
                 print('MARS Request failed - please check request')
                 raise IOError
-
-            if os.stat(target).st_size == 0:
+            elif os.stat(target).st_size == 0:
                 print('MARS Request returned no data - please check request')
                 raise IOError
+            else:
+                raise
 
         return

source/python/mods/get_mars_data.py

@@ -53,7 +53,8 @@ from datetime import datetime, timedelta
 # software specific classes and modules from flex_extract
 sys.path.append('../')
 import _config
-from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
+from tools import (my_error, normal_exit, get_cmdline_arguments,
+                   read_ecenv, make_dir)
 from classes.EcFlexpart import EcFlexpart
 from classes.UioFiles import UioFiles
 
@@ -113,7 +114,7 @@ def get_mars_data(c):
     '''
 
     if not os.path.exists(c.inputdir):
-        os.makedirs(c.inputdir)
+        make_dir(c.inputdir)
 
     if c.request == 0 or c.request == 2:
         print("Retrieving EC data!")
@@ -124,6 +125,9 @@ def get_mars_data(c):
     print("end date %s " % (c.end_date))
 
     if ecapi:
+        if c.public:
+            server = ecmwfapi.ECMWFDataServer()
+        else:
             server = ecmwfapi.ECMWFService("mars")
     else:
         server = False
@@ -253,7 +257,7 @@ def do_retrievement(c, server, start, end, delta_t, fluxes=False):
         print("... retrieve " + dates + " in dir " + c.inputdir)
 
         try:
-            flexpart.retrieve(server, dates, c.request, c.inputdir)
+            flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
         except IOError:
             my_error(c.mailfail, 'MARS request failed')
 

source/python/mods/prepare_flexpart.py

@@ -62,7 +62,7 @@ sys.path.append('../')
 import _config
 from classes.UioFiles import UioFiles
 from classes.ControlFile import ControlFile
-from tools import clean_up, get_cmdline_arguments, read_ecenv
+from tools import clean_up, get_cmdline_arguments, read_ecenv, make_dir
 from classes.EcFlexpart import EcFlexpart
 
 ecapi = 'ecmwf' not in socket.gethostname()
@@ -152,7 +152,7 @@ def prepare_flexpart(ppid, c):
 
     # create output dir if necessary
     if not os.path.exists(c.outputdir):
-        os.makedirs(c.outputdir)
+        make_dir(c.outputdir)
 
     # get all files with flux data to be deaccumulated
     inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')

source/python/mods/tools.py

@@ -138,6 +138,9 @@ def get_cmdline_arguments():
                         type=none_or_int, default=None,
                         help="list all mars request in file mars_requests.dat \
                         and skip submission to mars")
+    parser.add_argument("--public", dest="public",
+                        type=none_or_int, default=None,
+                        help="public mode - retrieves the public datasets")
 
     # some arguments that override the default in the CONTROL file
     parser.add_argument("--basetime", dest="basetime",