some bug corrections, minor code improvements

source/python/_config.py

@@ -27,6 +27,8 @@ import inspect
 
 _VERSION_STR = '7.1'
 
+QUEUES_LIST = ['ecgate', 'cca']
+
 # ------------------------------------------------------------------------------
 # FILENAMES
 # ------------------------------------------------------------------------------
@@ -75,11 +77,11 @@ PATH_JOBSCRIPTS = os.path.join(PATH_RUN_DIR, 'jobscripts')
 PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'fortran')
 PATH_TEST_DIR = os.path.join(PATH_SOURCES, 'pythontest')
 PATH_INPUT_DIR = os.path.join(PATH_RUN_DIR, INPUT_DIRNAME_DEFAULT)
-if os.getenv('CONTROL') and '/' in os.getenv('CONTROL'):
+if os.getenv('CONTROL'):# and '/' in os.getenv('CONTROL'):
     # this is only needed if remote version with job script is used!
     # because job is directly submitted from SCRATCH and because the
     # CONTROL file is stored there, the normal path is not valid.
-    PATH_CONTROLFILES = os.getenv('CONTROL')
+    PATH_CONTROLFILES = '.'
 else:
     PATH_CONTROLFILES = os.path.join(PATH_RUN_DIR, 'control')
 #

source/python/classes/ControlFile.py

@@ -59,7 +59,7 @@ import inspect
 # software specific classes and modules from flex_extract
 sys.path.append('../')
 import _config
-from mods.tools import my_error
+from mods.tools import my_error, silent_remove
 
 # ------------------------------------------------------------------------------
 # CLASS
@@ -187,11 +187,11 @@ class ControlFile(object):
         '''
 
         try:
-            with open(os.path.join(_config.PATH_CONTROLFILES,
-                                   self.controlfile)) as f:
+            cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
+            with open(cfile) as f:
                 fdata = f.read().split('\n')
         except IOError:
-            print('Could not read CONTROL file "' + args.controlfile + '"')
+            print('Could not read CONTROL file "' + cfile + '"')
             print('Either it does not exist or its syntax is wrong.')
             print('Try "' + sys.argv[0].split('/')[-1] + \
                       ' -h" to print usage information')
@@ -449,7 +449,7 @@ class ControlFile(object):
             else:
                 self.mailops = [self.mailops]
 
-        if queue in ['ecgate', 'cca'] and \
+        if queue in _config.QUEUES_LIST and \
            not self.gateway or not self.destination or \
            not self.ecuid or not self.ecgid:
             print('\nEnvironment variables GATEWAY, DESTINATION, ECUID and \
@@ -462,7 +462,7 @@ class ControlFile(object):
             marsfile = os.path.join(self.inputdir,
                                     _config.FILE_MARS_REQUESTS)
             if os.path.isfile(marsfile):
-                os.remove(marsfile)
+                silent_remove(marsfile)
 
         # check all logical variables for data type
         # if its a string change to integer

source/python/classes/EcFlexpart.py

@@ -84,7 +84,7 @@ import numpy as np
 from eccodes import (codes_index_select, codes_new_from_index, codes_get,
                      codes_get_values, codes_set_values, codes_set,
                      codes_write, codes_release, codes_new_from_index,
-                     codes_index_release)
+                     codes_index_release, codes_index_get)
 
 # software specific classes and modules from flex_extract
 sys.path.append('../')
@@ -135,7 +135,8 @@ class EcFlexpart(object):
         # different mars types for retrieving data for flexpart
         self.types = dict()
 
-        if c.maxstep > len(c.type):    # Pure forecast mode
+        # Pure forecast mode
+        if c.maxstep > len(c.type) and 'AN' not in c.type:
             c.type = [c.type[0]]
             c.step = ['{:0>3}'.format(int(c.step[0]))]
             c.time = [c.time[0]]
@@ -167,11 +168,9 @@ class EcFlexpart(object):
                 if c.basetime == '00':
                     btlist = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0]
 
-                if ((ty.upper() == 'AN' and
-                     int(c.time[i]) % int(c.dtime) ==0) or
-                    (ty.upper() != 'AN' and
-                     int(c.step[i]) % int(c.dtime) == 0 and
-                     int(c.step[i]) % int(c.dtime) == 0) ) and \
+                if ((ty.upper() == 'AN' and (int(c.time[i]) % int(c.dtime)) == 0) or
+                    (ty.upper() != 'AN' and (int(c.step[i]) % int(c.dtime)) == 0 and
+                     (int(c.step[i]) % int(c.dtime) == 0)) ) and \
                     (int(c.time[i]) in btlist or c.maxstep > 24):
 
                     if ty not in self.types.keys():
@@ -433,7 +432,7 @@ class EcFlexpart(object):
         for key in index_keys:
             #index_vals.append(grib_index_get(iid, key))
             #print(index_vals[-1])
-            key_vals = grib_index_get(iid, key)
+            key_vals = codes_index_get(iid, key)
             print(key_vals)
             # have to sort the steps for disaggregation,
             # therefore convert to int first
@@ -531,13 +530,14 @@ class EcFlexpart(object):
                 #     [param, levtype, levelist, grid]
                 if isinstance(pv, str):
                     continue
-                retr_param_dict['type'] = '' + ftype
+                retr_param_dict['type'] = ftype
                 retr_param_dict['time'] = self.types[ftype]['times']
                 retr_param_dict['step'] = self.types[ftype]['steps']
                 retr_param_dict['date'] = self.dates
                 retr_param_dict['stream'] = self.stream
                 retr_param_dict['target'] = \
-                    self._mk_targetname(ftype, pk,
+                    self._mk_targetname(ftype,
+                                        pk,
                                         retr_param_dict['date'].split('/')[0])
                 retr_param_dict['param'] = pv[0]
                 retr_param_dict['levtype'] = pv[1]
@@ -688,7 +688,7 @@ class EcFlexpart(object):
         from genshi.template import  TemplateLoader
 
         loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
-        compile_template = loader.load(_config.TEMPFILE_NAMELIST,
+        namelist_template = loader.load(_config.TEMPFILE_NAMELIST,
                                        cls=NewTextTemplate)
 
         self.inputdir = c.inputdir
@@ -700,7 +700,7 @@ class EcFlexpart(object):
         maxl = int((area[3] - area[1]) / grid[1]) + 1
         maxb = int((area[0] - area[2]) / grid[0]) + 1
 
-        stream = compile_template.generate(
+        stream = namelist_template.generate(
             maxl = str(maxl),
             maxb = str(maxb),
             mlevel = str(self.level),
@@ -1215,26 +1215,25 @@ class EcFlexpart(object):
         print('Output filelist: ')
         print(self.outputfilelist)
 
-        if c.format.lower() == 'grib2':
         for ofile in self.outputfilelist:
+            ofile = os.path.join(self.inputdir, ofile)
+
+            if c.format.lower() == 'grib2':
                 p = subprocess.check_call(['grib_set', '-s', 'edition=2, \
                                            productDefinitionTemplateNumber=8',
                                            ofile, ofile + '_2'])
                 p = subprocess.check_call(['mv', ofile + '_2', ofile])
 
             if c.ectrans and not c.ecapi:
-            for ofile in self.outputfilelist:
                 p = subprocess.check_call(['ectrans', '-overwrite', '-gateway',
                                            c.gateway, '-remote', c.destination,
                                            '-source', ofile])
 
             if c.ecstorage and not c.ecapi:
-            for ofile in self.outputfilelist:
                 p = subprocess.check_call(['ecp', '-o', ofile,
                                            os.path.expandvars(c.ecfsdir)])
 
             if c.outputdir != c.inputdir:
-            for ofile in self.outputfilelist:
                 p = subprocess.check_call(['mv',
                                            os.path.join(c.inputdir, ofile),
                                            c.outputdir])

source/python/mods/get_mars_data.py

@@ -51,7 +51,8 @@ import inspect
 from datetime import datetime, timedelta
 
 # software specific classes and modules from flex_extract
-sys.path.append('../')
+sys.path.append(os.path.dirname(os.path.abspath(
+    inspect.getfile(inspect.currentframe()))) + '/../')
 import _config
 from tools import (my_error, normal_exit, get_cmdline_arguments,
                    read_ecenv, make_dir)

source/python/mods/prepare_flexpart.py

@@ -58,7 +58,9 @@ import sys
 import socket
 
 # software specific classes and modules from flex_extract
-sys.path.append('../')
+
+sys.path.append(os.path.dirname(os.path.abspath(
+    inspect.getfile(inspect.currentframe()))) + '/../')
 import _config
 from classes.UioFiles import UioFiles
 from classes.ControlFile import ControlFile
@@ -155,7 +157,7 @@ def prepare_flexpart(ppid, c):
         make_dir(c.outputdir)
 
     # get all files with flux data to be deaccumulated
-    inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')
+    inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + str(c.ppid) + '.*')
 
     # deaccumulate the flux data
     flexpart = EcFlexpart(c, fluxes=True)
@@ -163,7 +165,7 @@ def prepare_flexpart(ppid, c):
     flexpart.deacc_fluxes(inputfiles, c)
 
     # get a list of all files from the root inputdir
-    inputfiles = UioFiles(c.inputdir, '????__??.*' + c.ppid + '.*')
+    inputfiles = UioFiles(c.inputdir, '????__??.*' + str(c.ppid) + '.*')
 
     # produce FLEXPART-ready GRIB files and process them -
     # copy/transfer/interpolate them or make them GRIB2