outsourced all of the necessary checks on control variables into own module functions

source/python/mods/checks.py

@@ -25,11 +25,38 @@
 # MODULES
 # ------------------------------------------------------------------------------
 
+import os
 import _config
+import exceptions
+from tools import my_error, silent_remove
+from datetime import datetime
 # ------------------------------------------------------------------------------
 # FUNCTIONS
 # ------------------------------------------------------------------------------
 
+def check_logicals_type(c, logicals):
+    '''Check that the logical variables have correct type integer.
+
+    Parameters
+    ----------
+    c : :obj:`ControlFile`
+        Contains all the parameters of CONTROL file and
+        command line.
+
+    logicals : :obj:`list` of (:obj:`string` or :obj:`integer`)
+        Names of the switches that are used to control the flow of the
+        program.
+
+    Return
+    ------
+
+    '''
+
+    for var in logicals:
+        if not isinstance(getattr(c, var), int):
+            setattr(c, var, int(getattr(c, var)))
+
+    return
 
 def check_grid(grid):
     '''Convert grid into correct Lat/Lon format. E.g. '0.5/0.5'
@@ -81,7 +108,22 @@ def check_area(grid, area, upper, lower, left , right):
     Parameters
     ----------
     grid : :obj:`string`
-        Contains grid information
+        Contains grid information.
+
+    area : :obj:`string`
+        Contains area informtion.
+
+    upper : :obj:`string`
+        The northern most latitude.
+
+    lower : :obj:`string`
+        The souther most latitude.
+
+    left : :obj:`string`
+        The western most longitude.
+
+    right : :obj:`string`
+        The eastern most longiude.
 
     Return
     ------
@@ -98,19 +140,19 @@ def check_area(grid, area, upper, lower, left , right):
         upper, left, lower, right = components
 
     # determine area format
-    if (abs(float(upper) / 1000.) >= 0.05 and
-        abs(float(lower) / 1000.) >= 0.05 and
-        abs(float(left) / 1000.) >= 0.05 and
-        abs(float(right) / 1000.) >= 0.05):
+    if ((abs(float(upper) / 10000.) >= 0.01 or float(upper) / 1000. == 0. ) and
+        (abs(float(lower) / 10000.) >= 0.01 or float(lower) / 1000. == 0. ) and
+        (abs(float(left) / 10000.) >= 0.01 or float(left) / 1000. == 0. ) and
+        (abs(float(right) / 10000.) >= 0.01 or float(right) / 1000. == 0.)):
         # area is defined in 1/1000 degrees; old format
         area = '{}/{}/{}/{}'.format(float(upper) / 1000.,
                                     float(left) / 1000.,
                                     float(lower) / 1000.,
                                     float(right) / 1000.)
-    elif (abs(float(upper) / 1000.) < 0.05 and
-          abs(float(lower) / 1000.) < 0.05 and
-          abs(float(left) / 1000.) < 0.05 and
-          abs(float(right) / 1000.) < 0.05):
+    elif (abs(float(upper) / 10000.) < 0.05 and
+          abs(float(lower) / 10000.) < 0.05 and
+          abs(float(left) / 10000.) < 0.05 and
+          abs(float(right) / 10000.) < 0.05):
         # area is already in new format
         area = '{}/{}/{}/{}'.format(float(upper),
                                     float(left),
@@ -199,12 +241,12 @@ def check_ppid(c, ppid):
     return
 
 
-def check_purefc(type):
+def check_purefc(ftype):
     '''Check for a pure forecast mode.
 
     Parameters
     ----------
-    type : :obj:`list` of :obj:`string`
+    ftype : :obj:`list` of :obj:`string`
         List of field types.
 
     Return
@@ -214,23 +256,542 @@ def check_purefc(type):
         analysis fields in between.
     '''
 
-    if 'AN' not in type and '4V' not in type:
+    if 'AN' not in ftype and '4V' not in ftype:
         # pure forecast
-        return True
+        return 1
+
+    return 0
+
+
+def check_step(step, mailfail):
+    '''Checks on step format and convert into a list of steps.
+
+    If the steps were defined with "to" and "by" they are converted into
+    a list of steps. If the steps were set in a string, it is
+    converted into a list.
+
+    Parameters
+    ----------
+    step : :obj:`list` of :obj:`string` or :obj:`string`
+        Specifies the forecast time step from forecast base time.
+        Valid values are hours (HH) from forecast base time.
+
+    mailfail : :obj:`list` of :obj:``string`
+        Contains all email addresses which should be notified.
+        It might also contain just the ecmwf user name which will trigger
+        mailing to the associated email address for this user.
+
+    Return
+    ------
+    step : :obj:`list` of :obj:`string`
+        List of forecast steps in format e.g. [001, 002, ...]
+    '''
+
+    if '/' in step:
+        steps = step.split('/')
+        if 'to' in step.lower() and 'by' in step.lower():
+            ilist = np.arange(int(steps[0]),
+                              int(steps[2]) + 1,
+                              int(steps[4]))
+            step = ['{:0>3}'.format(i) for i in ilist]
+        elif 'to' in step.lower() and 'by' not in step.lower():
+            my_error(mailfail, step + ':\n' +
+                     'if "to" is used in steps parameter, '
+                     'please use "by" as well')
+        else:
+            step = steps
+
+    if not isinstance(step, list):
+        step = [step]
+
+    return step
+
+def check_type(ftype, steps):
+    '''Check if type variable is of type list and if analysis field has
+    forecast step 0.
+
+    Parameters
+    ----------
+    ftype : :obj:`list` of :obj:`string` or :obj:`string`
+        List of field types.
+
+    steps : :obj:`string`
+        Specifies the forecast time step from forecast base time.
+        Valid values are hours (HH) from forecast base time.
+
+    Return
+    ------
+    ftype : :obj:`list` of :obj:`string`
+        List of field types.
+    '''
+    if not isinstance(ftype, list):
+        ftype = [ftype]
+
+    for i, val in enumerate(ftype):
+        if ftype[i] == 'AN' and int(steps[i]) != 0:
+            print('Analysis retrievals must have STEP = 0 (now set to 0)')
+            ftype[i] = 0
+
+    return ftype
+
+def check_time(ftime):
+    '''Check if time variable is of type list. Otherwise convert to list.
+
+    Parameters
+    ----------
+    ftime : :obj:`list` of :obj:`string` or :obj:`string`
+        The time in hours of the field.
+
+    Return
+    ------
+    ftime : :obj:`list` of :obj:`string`
+        The time in hours of the field.
+    '''
+    if not isinstance(ftime, list):
+        ftime = [ftime]
+
+    return ftime
+
+def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc):
+    '''Check if
+
+    Parameters
+    ----------
+    ftype : :obj:`list` of :obj:`string`
+        List of field types.
+
+    ftime : :obj:`list` of :obj:`string` or :obj:`string`
+        The time in hours of the field.
+
+    steps : :obj:`string`
+        Specifies the forecast time step from forecast base time.
+        Valid values are hours (HH) from forecast base time.
+
+    maxstep : :obj:`integer`
+        The maximum forecast time step in hours from the forecast base time.
+        This is the maximum step for non flux (accumulated) forecast data.
+
+    purefc : :obj:`integer`
+        Switch for definition of pure forecast mode or not.
+
+    Return
+    ------
+    ftype : :obj:`list` of :obj:`string`
+        List of field types.
+
+    ftime : :obj:`list` of :obj:`string`
+        The time in hours of the field.
+
+    steps : :obj:`string`
+        Specifies the forecast time step from forecast base time.
+        Valid values are hours (HH) from forecast base time.
+    '''
+    if not (len(ftype) == len(ftime) == len(steps)):
+        raise ValueError('ERROR: The number of field types, times and steps '
+                         'are not the same! Please check the setting in the '
+                         'CONTROL file!')
+
+    # if pure forecast is selected and only one field type/time is set
+    # prepare a complete list of type/time/step combination upto maxstep
+    if len(ftype) == 1 and purefc:
+        ftype = []
+        steps = []
+        ftime = []
+        for i in range(0, maxstep + 1):
+            ftype.append(ftype[0])
+            steps.append('{:0>3}'.format(i))
+            ftime.append(ftime[0])
+
+    return ftype, ftime, steps
+
+def check_mail(mail):
+    '''Check the string of mail addresses, seperate them and convert to a list.
+
+    Parameters
+    ----------
+    mail : :obj:`list` of :obj:`string` or :obj:`string`
+        Contains email addresses for notifications.
+        It might also contain just the ecmwf user name which will trigger
+        mailing to the associated email address for this user.
+
+    Return
+    ------
+    mail : :obj:`list` of :obj:``string`
+        Contains email addresses for notifications.
+        It might also contain just the ecmwf user name which will trigger
+        mailing to the associated email address for this user.
+
+    '''
+    if not isinstance(mail, list):
+        if ',' in mail:
+            mail = mail.split(',')
+        elif ' ' in mail:
+            mail = mail.split()
+        else:
+            mail = [mail]
+
+    return mail
+
+def check_queue(queue, gateway, destination, ecuid, ecgid):
+    '''Check if the necessary ECMWF parameters are set if the queue is
+    one of the QUEUES_LIST (in _config).
+
+    Parameters
+    ----------
+    queue : :obj:`string`
+        Name of the queue if submitted to the ECMWF servers.
+        Used to check if ecuid, ecgid, gateway and destination
+        are set correctly and are not empty.
+
+    gateway : :obj:`string`
+        The address of the gateway server.
+
+    destination : :obj:`string`
+        The name of the destination of the gateway server for data
+        transfer through ectrans. E.g. name@genericSftp
+
+    ecuid : :obj:`string`
+        ECMWF user id.
+
+    ecgid : :obj:`string`
+        ECMWF group id.
+
+    Return
+    ------
+
+    '''
+    if queue in _config.QUEUES_LIST and \
+        not gateway or not destination or \
+        not ecuid or not ecgid:
+        raise ValueError('\nEnvironment variables GATEWAY, DESTINATION, ECUID '
+                         'and ECGID were not set properly! \n '
+                         'Please check for existence of file "ECMWF_ENV" '
+                         'in the run directory!')
+    return
+
+def check_pathes(idir, odir, fpdir, fedir):
+    '''Check if output and flexpart pathes are set.
+
+    Parameters
+    ----------
+    idir : :obj:`string`
+        Path to the temporary directory for MARS retrieval data.
 
-    return False
+    odir : :obj:`string`
+        Path to the final output directory where the FLEXPART input files
+        will be stored.
 
+    fpdir : :obj:`string`
+        Path to FLEXPART root directory.
+
+    fedir : :obj:`string`
+        Path to flex_extract root directory.
+
+    Return
+    ------
+    odir : :obj:`string`
+        Path to the final output directory where the FLEXPART input files
+        will be stored.
+
+    fpdir : :obj:`string`
+        Path to FLEXPART root directory.
+
+    '''
+    if not fpdir:
+        fpdir = fedir
+
+    if not odir:
+        odir = idir
+
+    return odir, fpdir
+
+def check_dates(start, end):
+    '''Checks if there is at least a start date for a one day retrieval.
+
+    Checks if end date lies after start date and end date is set.
+
+    Parameters
+    ----------
+    start : :obj:`string`
+        The start date of the retrieval job.
+
+    end : :obj:`string`
+        The end date of the retrieval job.
+
+    Return
+    ------
+    start : :obj:`string`
+        The start date of the retrieval job.
+
+    end : :obj:`string`
+        The end date of the retrieval job.
+
+    '''
+    # check for having at least a starting date
+    # otherwise program is not allowed to run
+    if not start:
+        raise ValueError('start_date was neither specified in command line nor '
+                         'in CONTROL file.\n'
+                         'Try "{} -h" to print usage information'
+                         .format(sys.argv[0].split('/')[-1]) )
+
+    # retrieve just one day if end_date isn't set
+    if not end:
+        end = start
+
+    dstart = datetime.strptime(start, '%Y%m%d')
+    dend = datetime.strptime(end, '%Y%m%d')
+    if dstart > dend:
+        raise ValueError('ERROR: Start date is after end date! \n'
+                         'Please adapt the dates in CONTROL file or '
+                         'command line! (start={}; end={})'.format(start, end))
+
+    return start, end
+
+def check_maxstep(maxstep, steps):
+    '''Convert maxstep into integer if it is already given. Otherwise, select
+    maxstep by going through the steps list.
+
+    Parameters
+    ----------
+    maxstep : :obj:`string`
+        The maximum forecast time step in hours from the forecast base time.
+        This is the maximum step for non flux (accumulated) forecast data.
+
+    steps : :obj:`string`
+        Specifies the forecast time step from forecast base time.
+        Valid values are hours (HH) from forecast base time.
+
+    Return
+    ------
+    maxstep : :obj:`integer`
+        The maximum forecast time step in hours from the forecast base time.
+        This is the maximum step for non flux (accumulated) forecast data.
+
+    '''
+    # if maxstep wasn't provided
+    # search for it in the "step" parameter
+    if not maxstep:
+        maxstep = 0
+        for s in steps:
+            if int(s) > maxstep:
+                maxstep = int(s)
+    else:
+        maxstep = int(maxstep)
+
+    return maxstep
+
+def check_basetime(basetime):
+    '''Check if basetime is set and contains one of the two
+    possible values (0, 12).
+
+    Parameters
+    ----------
+    basetime : :obj:``
+        The time for a half day retrieval. The 12 hours upfront are to be
+        retrieved.
+
+    Return
+    ------
 
-def check_():
     '''
+    if basetime:
+        if int(basetime) != 0 and int(basetime) != 12:
+            raise ValueError('ERROR: Basetime has an invalid value '
+                             '-> {}'.format(str(basetime)))
+    return
+
+def check_request(request, marsfile):
+    '''Check if there is an old mars request file and remove it.
 
     Parameters
     ----------
-    par : :obj:``
-        ...
+    request : :obj:`integer`
+        Selects the mode of retrieval.
+        0: Retrieves the data from ECMWF.
+        1: Prints the mars requests to an output file.
+        2: Retrieves the data and prints the mars request.
+
+    marsfile : :obj:`string`
+        Path to the mars request file.
 
     Return
     ------
 
     '''
+    if request != 0:
+        if os.path.isfile(marsfile):
+            silent_remove(marsfile)
     return
+
+def check_public(public, dataset):
+    '''Check wether the dataset parameter is set for a
+    public data set retrieval.
+
+    Parameters
+    ----------
+    public : :obj:`ìnteger`
+        Specifies if public data are to be retrieved or not.
+
+    dataset : :obj:`string`
+        Specific name which identifies the public dataset.
+
+    Return
+    ------
+
+    '''
+    if public and not dataset:
+        raise ValueError('ERROR: If public mars data wants to be retrieved, '
+                         'the "dataset"-parameter has to be set too!')
+    return
+
+def check_acctype(acctype, ftype):
+    '''Guarantees that the accumulation field type is set.
+
+    If not set, it is derivated as in the old method (TYPE[1]).
+
+    Parameters
+    ----------
+    acctype : :obj:`string`
+        The field type for the accumulated forecast fields.
+
+    ftype : :obj:`list` of :obj:`string`
+        List of field types.
+
+    Return
+    ------
+    acctype : :obj:`string`
+        The field type for the accumulated forecast fields.
+    '''
+    if not acctype:
+        print('... Control parameter ACCTYPE was not defined.')
+        try:
+            if len(ftype) == 1 and ftype[0] != 'AN':
+                print('Use same field type as for the non-flux fields.')
+                acctype = ftype[0]
+            elif len(ftype) > 1 and ftype[1] != 'AN':
+                print('Use old setting by using TYPE[1] for flux forecast!')
+                acctype = ftype[1]
+        except:
+            raise ValueError('ERROR: Accumulation field type could not be set!')
+    else:
+        if acctype.upper() == 'AN':
+            raise ValueError('ERROR: Accumulation forecast fields can not be '
+                             'of type "analysis"!')
+    return acctype
+
+
+def check_acctime(acctime, acctype, purefc):
+    '''Guarantees that the accumulation forecast times were set.
+
+    If it is not set, it is tried to set the value fore some of the
+    most commonly used data sets. Otherwise it raises an error.
+
+    Parameters
+    ----------
+    acctime : :obj:`string`
+        The starting time from the accumulated forecasts.
+
+    acctype : :obj:`string`
+        The field type for the accumulated forecast fields.
+
+    purefc : :obj:`integer`
+        Switch for definition of pure forecast mode or not.
+
+    Return
+    ------
+    acctime : :obj:`string`
+        The starting time from the accumulated forecasts.
+    '''
+    if not acctime:
+        print('... Control parameter ACCTIME was not defined.')
+        print('... Value will be set depending on field type: '
+              '\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t  EP=18')
+        if acctype.upper() == 'EA': # Era 5
+            acctime = '06/18'
+        elif acctype.upper() == 'EI': # Era-Interim
+            acctime = '00/12'
+        elif acctype.upper() == 'EP': # CERA
+            acctime = '18'
+        elif acctype.upper() == 'OD' and not purefc: # On-demand operational
+            acctime = '00/12'
+        else:
+            raise ValueError('ERROR: Accumulation forecast time can not '
+                             'automatically be derived!')
+    return acctime
+
+def check_accmaxstep(accmaxstep, acctype, purefc, maxstep):
+    '''Guarantees that the accumulation forecast step were set.
+
+    Parameters
+    ----------
+    accmaxstep : :obj:`string`
+        The maximum forecast step for the accumulated forecast fields.
+
+    acctype : :obj:`string`
+        The field type for the accumulated forecast fields.
+
+    purefc : :obj:`integer`
+        Switch for definition of pure forecast mode or not.
+
+    maxstep : :obj:`string`
+        The maximum forecast time step in hours from the forecast base time.
+        This is the maximum step for non flux (accumulated) forecast data.
+
+    Return
+    ------
+    accmaxstep : :obj:`string`
+        The maximum forecast step for the accumulated forecast fields.
+    '''
+    if not accmaxstep:
+        print('... Control parameter ACCMAXSTEP was not defined.')
+        print('... Value will be set depending on field type/time: '
+              '\t\t EA/EI/OD=12\n\t\t  EP=24')
+        if acctype.upper() in ['EA', 'EI', 'OD'] and not purefc:
+            # Era 5, Era-Interim, On-demand operational
+            accmaxstep = '12'
+        elif acctype.upper() == 'EP': # CERA
+            accmaxstep = '18'
+        elif purefc and accmaxstep != maxstep:
+            accmaxstep = maxstep
+            print('... For pure forecast mode, the accumulated forecast must '
+                  'have the same maxstep as the normal forecast fields!\n'
+                  '\t\t Accmaxstep was set to maxstep!')
+        else:
+            raise ValueError('ERROR: Accumulation forecast step can not '
+                             'automatically be derived!')
+    else:
+        if purefc and int(accmaxstep) != int(maxstep):
+            accmaxstep = maxstep
+            print('... For pure forecast mode, the accumulated forecast must '
+                          'have the same maxstep as the normal forecast fields!\n'
+                          '\t\t Accmaxstep was set to maxstep!')
+    return accmaxstep
+
+def check_addpar(addpar):
+    '''Check that addpar has correct format of additional parameters in
+    a single string, so that it can be easily appended to the hard coded
+    parameters that are retrieved in any case.
+
+    Parameters
+    ----------
+    addpar : :obj:`string` or :obj:'list' of :obj:'string'
+        List of additional parameters to be retrieved.
+
+    Return
+    ------
+    addpar : :obj:'string'
+        List of additional parameters to be retrieved.
+    '''
+
+    if addpar and isinstance(addpar, str):
+        if '/' in addpar:
+            parlist = addpar.split('/')
+            parlist = [p for p in parlist if p is not '']
+        else:
+            parlist = [addpar]
+
+        addpar = '/' + '/'.join(parlist)
+
+    return addpar
+