From 0f89116d0fc1de9b182758abd84208eb9c6933b5 Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Thu, 19 Dec 2019 21:07:12 +0100
Subject: [PATCH] diverse changes due to PEP8 style guide and eliminating
grib2flexpart; removed unused parameter
---
Source/Python/Classes/ControlFile.py | 27 ++--
Source/Python/Classes/EcFlexpart.py | 192 +++++++++++++------------
Source/Python/Classes/GribUtil.py | 46 +++---
Source/Python/Classes/MarsRetrieval.py | 5 +-
Source/Python/Classes/UioFiles.py | 27 ++--
Source/Python/Mods/checks.py | 51 +++----
Source/Python/Mods/disaggregation.py | 40 +++---
Source/Python/Mods/get_mars_data.py | 27 +++-
Source/Python/Mods/prepare_flexpart.py | 21 ++-
Source/Python/Mods/profiling.py | 13 +-
Source/Python/Mods/tools.py | 24 ++--
Source/Python/_config.py | 4 +-
Source/Python/install.py | 113 +++++++--------
Source/Python/submit.py | 9 +-
14 files changed, 295 insertions(+), 304 deletions(-)
diff --git a/Source/Python/Classes/ControlFile.py b/Source/Python/Classes/ControlFile.py
index 96e901e..ba795f2 100644
--- a/Source/Python/Classes/ControlFile.py
+++ b/Source/Python/Classes/ControlFile.py
@@ -37,14 +37,13 @@
from __future__ import print_function
import os
-import re
import sys
-import inspect
# software specific classes and modules from flex_extract
+#pylint: disable=wrong-import-position
sys.path.append('../')
import _config
-from Mods.tools import my_error, silent_remove
+from Mods.tools import my_error
from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
check_step, check_mail, check_queue, check_pathes,
check_dates, check_maxstep, check_type, check_request,
@@ -52,6 +51,7 @@ from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
check_acctime, check_accmaxstep, check_time,
check_logicals_type, check_len_type_time_step,
check_addpar, check_job_chunk, check_number)
+#pylint: enable=wrong-import-position
# ------------------------------------------------------------------------------
# CLASS
@@ -64,7 +64,7 @@ class ControlFile(object):
software. All necessary parameters needed to retrieve the data fields
from the MARS archive for driving FLEXPART are set in a CONTROL file.
Some specific parameters like the start and end dates can be overwritten
- by the command line parameters, but in generel all parameters needed
+ by the command line parameters, but in generall all parameters needed
for a complete set of fields for FLEXPART can be set in the CONTROL file.
Attributes
@@ -237,11 +237,6 @@ class ControlFile(object):
The email addresses should be seperated by a comma.
Default value is ['${USER}'].
- grib2flexpart : int 0
- Switch to select generation of preprocessed FLEXPART files ".fp".
- If it is selected, the program grib2flexpart will try
- to convert the flex_extract output files into ".fp" format.
-
ecstorage : int
Switch to select storage of FLEXPART ready output files
in the ECFS file system. Default value is 0.
@@ -333,7 +328,7 @@ class ControlFile(object):
logicals : list of str
List of the names of logical switches which controls the flow
of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
- 'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
+ 'etadiff', 'dpdeta', 'cwc', 'wrf', 'ecstorage',
'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
'''
@@ -399,7 +394,6 @@ class ControlFile(object):
self.ecfsdir = 'ectmp:/${USER}/econdemand/'
self.mailfail = ['${USER}']
self.mailops = ['${USER}']
- self.grib2flexpart = 0
self.ecstorage = 0
self.ectrans = 0
self.inputdir = _config.PATH_INPUT_DIR
@@ -424,7 +418,7 @@ class ControlFile(object):
self.doubleelda = 0
self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
- 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
+ 'dpdeta', 'cwc', 'wrf', 'ecstorage',
'ectrans', 'debug', 'oper', 'request', 'public',
'purefc', 'rrint', 'doubleelda']
@@ -601,7 +595,9 @@ class ControlFile(object):
self.ecuid, self.ecgid)
self.outputdir, self.installdir = check_pathes(self.inputdir,
- self.outputdir, self.installdir, self.flexextractdir)
+ self.outputdir,
+ self.installdir,
+ self.flexextractdir)
self.start_date, self.end_date = check_dates(self.start_date,
self.end_date)
@@ -610,7 +606,7 @@ class ControlFile(object):
self.levelist, self.level = check_levels(self.levelist, self.level)
- self.step = check_step(self.step, self.mailfail)
+ self.step = check_step(self.step)
self.maxstep = check_maxstep(self.maxstep, self.step)
@@ -648,7 +644,7 @@ class ControlFile(object):
self.job_chunk = check_job_chunk(self.job_chunk)
- self.number = check_number(self.number, self.mailfail)
+ self.number = check_number(self.number)
return
@@ -694,4 +690,3 @@ class ControlFile(object):
l.append("%s %s\n" % item)
return sorted(l)
-
diff --git a/Source/Python/Classes/EcFlexpart.py b/Source/Python/Classes/EcFlexpart.py
index 85c458c..360cbfa 100644
--- a/Source/Python/Classes/EcFlexpart.py
+++ b/Source/Python/Classes/EcFlexpart.py
@@ -52,6 +52,10 @@
# this is disabled because for this specific case its an error in pylint
#pylint: disable=consider-using-enumerate
# this is not useful in this case
+#pylint: disable=unsubscriptable-object
+# this error is a bug
+#pylint: disable=ungrouped-imports
+# not necessary that we group the imports
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
@@ -61,21 +65,21 @@ import os
import sys
import glob
import shutil
-import subprocess
from datetime import datetime, timedelta
# software specific classes and modules from flex_extract
+#pylint: disable=wrong-import-position
sys.path.append('../')
import _config
from Classes.GribUtil import GribUtil
from Mods.tools import (init128, to_param_id, silent_remove, product,
- my_error, make_dir, get_informations, get_dimensions,
+ my_error, get_informations, get_dimensions,
execute_subprocess, to_param_id_with_tablenumber,
generate_retrieval_period_boundary)
from Classes.MarsRetrieval import MarsRetrieval
from Classes.UioFiles import UioFiles
import Mods.disaggregation as disaggregation
-
+#pylint: enable=wrong-import-position
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
@@ -400,8 +404,8 @@ class EcFlexpart(object):
self.params['OG__ML'][0] += '/U/V'
else: # GAUSS and ETA
print('Warning: Collecting etadot and parameters for gaussian grid '
- 'is a very costly parameter combination, '
- 'use this combination only for debugging!')
+ 'is a very costly parameter combination, '
+ 'use this combination only for debugging!')
self.params['GG__SL'] = ['Q', 'ML', '1',
'{}'.format((int(self.resol) + 1) // 2)]
self.params['GG__ML'] = ['U/V/D/ETADOT', 'ML', self.glevelist,
@@ -414,7 +418,7 @@ class EcFlexpart(object):
self.params['OG__ML'][0] += '/CLWC/CIWC'
# ADDITIONAL FIELDS FOR FLEXPART-WRF MODEL (IF QUESTIONED)
- # -----------------------------------------------------------------------
+ # ----------------------------------------------------------------------
if wrf:
# @WRF
# THIS IS NOT YET CORRECTLY IMPLEMENTED !!!
@@ -709,7 +713,8 @@ class EcFlexpart(object):
retr_param_dict['step'] = '000'
retr_param_dict['date'] = self.dates.split('/')[0]
retr_param_dict['target'] = self._mk_targetname('',
- pk, retr_param_dict['date'])
+ pk,
+ retr_param_dict['date'])
elif pk == 'OG_OROLSM__SL' and oro:
continue
if pk == 'GG__SL' and pv[0] == 'Q':
@@ -751,7 +756,7 @@ class EcFlexpart(object):
# -------------- flux data ----------------------------
if 'acc' in pk:
startdate = retr_param_dict['date'].split('/')[0]
- enddate = datetime.strftime(elimit - t24h,'%Y%m%d')
+ enddate = datetime.strftime(elimit - t24h, '%Y%m%d')
retr_param_dict['date'] = '/'.join([startdate,
'to',
enddate])
@@ -780,9 +785,9 @@ class EcFlexpart(object):
timesave = ''.join(retr_param_dict['time'])
- if ('/' in retr_param_dict['time'] and
- pk != 'OG_OROLSM__SL' and
- 'acc' not in pk ) :
+ if all(['/' in retr_param_dict['time'],
+ pk != 'OG_OROLSM__SL',
+ 'acc' not in pk]):
times = retr_param_dict['time'].split('/')
steps = retr_param_dict['step'].split('/')
@@ -793,9 +798,9 @@ class EcFlexpart(object):
else:
retr_param_dict['time'] = times[0]
- if (pk != 'OG_OROLSM__SL' and
- int(retr_param_dict['step'].split('/')[0]) == 0 and
- int(timesave.split('/')[0]) == 0):
+ if all([pk != 'OG_OROLSM__SL',
+ int(retr_param_dict['step'].split('/')[0]) == 0,
+ int(timesave.split('/')[0]) == 0]):
retr_param_dict['date'] = \
datetime.strftime(elimit, '%Y%m%d')
@@ -809,7 +814,7 @@ class EcFlexpart(object):
self._start_retrievement(request, retr_param_dict)
else:
raise ValueError('ERROR: Basetime has an invalid value '
- '-> {}'.format(str(basetime)))
+ '-> {}'.format(str(self.basetime)))
if request == 0 or request == 2:
print('MARS retrieve done ... ')
@@ -859,23 +864,23 @@ class EcFlexpart(object):
maxb = int(round((area[0] - area[2]) / grid[0])) + 1
stream = namelist_template.generate(
- maxl = str(maxl),
- maxb = str(maxb),
- mlevel = str(self.level),
- mlevelist = str(self.levelist),
- mnauf = str(self.resol),
- metapar = '77',
- rlo0 = str(area[1]),
- rlo1 = str(area[3]),
- rla0 = str(area[2]),
- rla1 = str(area[0]),
- momega = str(c.omega),
- momegadiff = str(c.omegadiff),
- mgauss = str(c.gauss),
- msmooth = str(c.smooth),
- meta = str(c.eta),
- metadiff = str(c.etadiff),
- mdpdeta = str(c.dpdeta)
+ maxl=str(maxl),
+ maxb=str(maxb),
+ mlevel=str(self.level),
+ mlevelist=str(self.levelist),
+ mnauf=str(self.resol),
+ metapar='77',
+ rlo0=str(area[1]),
+ rlo1=str(area[3]),
+ rla0=str(area[2]),
+ rla1=str(area[0]),
+ momega=str(c.omega),
+ momegadiff=str(c.omegadiff),
+ mgauss=str(c.gauss),
+ msmooth=str(c.smooth),
+ meta=str(c.eta),
+ metadiff=str(c.etadiff),
+ mdpdeta=str(c.dpdeta)
)
except UndefinedError as e:
print('... ERROR ' + str(e))
@@ -927,7 +932,7 @@ class EcFlexpart(object):
'''
import numpy as np
- from eccodes import (codes_index_select, codes_new_from_index, codes_get,
+ from eccodes import (codes_index_select, codes_get,
codes_get_values, codes_set_values, codes_set,
codes_write, codes_release, codes_new_from_index,
codes_index_release)
@@ -1052,7 +1057,6 @@ class EcFlexpart(object):
time = codes_get(gid, 'time') // 100 # integer
step = codes_get(gid, 'step') # integer
ctime = '{:0>2}'.format(time)
- cstep = '{:0>3}'.format(step)
t_date = datetime.strptime(cdate + ctime, '%Y%m%d%H')
t_dt = t_date + timedelta(hours=step)
@@ -1131,7 +1135,7 @@ class EcFlexpart(object):
# do de-accumulation
deac_vals[parId].append(
(orig_vals[parId][-1] - orig_vals[parId][-2]) /
- int(c.dtime))
+ int(c.dtime))
# store precipitation if new disaggregation method is selected
# only the exact days are needed
@@ -1331,12 +1335,12 @@ class EcFlexpart(object):
if maxnum:
for inum in range(maxnum):
for ix in range(ni*nj):
- lsp_new_np[inum,ix,:] = disaggregation.IA3(lsp_np[inum,ix,:])[:-1]
- cp_new_np[inum,ix,:] = disaggregation.IA3(cp_np[inum,ix,:])[:-1]
+ lsp_new_np[inum, ix, :] = disaggregation.IA3(lsp_np[inum, ix, :])[:-1]
+ cp_new_np[inum, ix, :] = disaggregation.IA3(cp_np[inum, ix, :])[:-1]
else:
for ix in range(ni*nj):
- lsp_new_np[0,ix,:] = disaggregation.IA3(lsp_np[ix,:])[:-1]
- cp_new_np[0,ix,:] = disaggregation.IA3(cp_np[ix,:])[:-1]
+ lsp_new_np[0, ix, :] = disaggregation.IA3(lsp_np[ix, :])[:-1]
+ cp_new_np[0, ix, :] = disaggregation.IA3(cp_np[ix, :])[:-1]
# write to grib files (full/orig times to flux file and inbetween
# times with step 1 and 2, respectively)
@@ -1401,46 +1405,52 @@ class EcFlexpart(object):
# write original time step to flux file as usual
fluxfile = GribUtil(os.path.join(c.inputdir, fluxfilename))
- fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
+ fluxfile.set_keys(tmpfile, filemode='ab',
wherekeynames=['paramId'], wherekeyvalues=[142],
- keynames=['perturbationNumber','date','time','stepRange','values'],
+ keynames=['perturbationNumber', 'date', 'time',
+ 'stepRange', 'values'],
keyvalues=[inumb, int(date.strftime('%Y%m%d')),
- date.hour*100, 0, lsp_new_np[inumb,:,it]],
+ date.hour*100, 0, lsp_new_np[inumb, :, it]],
)
- fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
+ fluxfile.set_keys(tmpfile, filemode='ab',
wherekeynames=['paramId'], wherekeyvalues=[143],
- keynames=['perturbationNumber','date','time','stepRange','values'],
- keyvalues=[inumb,int(date.strftime('%Y%m%d')),
- date.hour*100, 0, cp_new_np[inumb,:,it]]
+ keynames=['perturbationNumber', 'date', 'time',
+ 'stepRange', 'values'],
+ keyvalues=[inumb, int(date.strftime('%Y%m%d')),
+ date.hour*100, 0, cp_new_np[inumb, :, it]]
)
# rr for first subgrid point is identified by step = 1
- fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
+ fluxfile.set_keys(tmpfile, filemode='ab',
wherekeynames=['paramId'], wherekeyvalues=[142],
- keynames=['perturbationNumber','date','time','stepRange','values'],
- keyvalues=[inumb,int(date.strftime('%Y%m%d')),
- date.hour*100, '1', lsp_new_np[inumb,:,it+1]]
- )
- fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
+ keynames=['perturbationNumber', 'date', 'time',
+ 'stepRange', 'values'],
+ keyvalues=[inumb, int(date.strftime('%Y%m%d')),
+ date.hour*100, '1', lsp_new_np[inumb, :, it+1]]
+ )
+ fluxfile.set_keys(tmpfile, filemode='ab',
wherekeynames=['paramId'], wherekeyvalues=[143],
- keynames=['perturbationNumber','date','time','stepRange','values'],
- keyvalues=[inumb,int(date.strftime('%Y%m%d')),
- date.hour*100, '1', cp_new_np[inumb,:,it+1]]
- )
+ keynames=['perturbationNumber', 'date', 'time',
+ 'stepRange', 'values'],
+ keyvalues=[inumb, int(date.strftime('%Y%m%d')),
+ date.hour*100, '1', cp_new_np[inumb, :, it+1]]
+ )
# rr for second subgrid point is identified by step = 2
- fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
+ fluxfile.set_keys(tmpfile, filemode='ab',
wherekeynames=['paramId'], wherekeyvalues=[142],
- keynames=['perturbationNumber','date','time','stepRange','values'],
- keyvalues=[inumb,int(date.strftime('%Y%m%d')),
- date.hour*100, '2', lsp_new_np[inumb,:,it+2]]
- )
- fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
+ keynames=['perturbationNumber', 'date', 'time',
+ 'stepRange', 'values'],
+ keyvalues=[inumb, int(date.strftime('%Y%m%d')),
+ date.hour*100, '2', lsp_new_np[inumb, :, it+2]]
+ )
+ fluxfile.set_keys(tmpfile, filemode='ab',
wherekeynames=['paramId'], wherekeyvalues=[143],
- keynames=['perturbationNumber','date','time','stepRange','values'],
- keyvalues=[inumb,int(date.strftime('%Y%m%d')),
- date.hour*100, '2', cp_new_np[inumb,:,it+2]]
- )
+ keynames=['perturbationNumber', 'date', 'time',
+ 'stepRange', 'values'],
+ keyvalues=[inumb, int(date.strftime('%Y%m%d')),
+ date.hour*100, '2', cp_new_np[inumb, :, it+2]]
+ )
it = it + 3 # jump to next original time step in rr fields
return
@@ -1462,13 +1472,13 @@ class EcFlexpart(object):
'''
- gribfile = GribUtil(os.path.join(inputdir,'rr_grib_dummy.grb'))
+ gribfile = GribUtil(os.path.join(inputdir, 'rr_grib_dummy.grb'))
gribfile.copy_dummy_msg(ifile, keynames=['paramId'],
- keyvalues=[142], filemode='wb')
+ keyvalues=[142], filemode='wb')
gribfile.copy_dummy_msg(ifile, keynames=['paramId'],
- keyvalues=[143], filemode='ab')
+ keyvalues=[143], filemode='ab')
return
@@ -1500,7 +1510,7 @@ class EcFlexpart(object):
------
'''
- from eccodes import (codes_index_select, codes_new_from_index, codes_get,
+ from eccodes import (codes_index_select, codes_get,
codes_get_values, codes_set_values, codes_set,
codes_write, codes_release, codes_new_from_index,
codes_index_release)
@@ -1582,22 +1592,20 @@ class EcFlexpart(object):
timestamp += timedelta(hours=int(cstep))
cdate_hour = datetime.strftime(timestamp, '%Y%m%d%H')
+ # if basetime is used, adapt start/end date period
+ if c.basetime is not None:
+ time_delta = timedelta(hours=12-int(c.dtime))
+ start_period = datetime.strptime(c.end_date + str(c.basetime),
+ '%Y%m%d%H') - time_delta
+ end_period = datetime.strptime(c.end_date + str(c.basetime),
+ '%Y%m%d%H')
+
# skip all temporary times
- # which are outside the retrieval period
+ # which are outside the retrieval period
if timestamp < start_period or \
timestamp > end_period:
continue
-
- # if the timestamp is out of basetime start/end date period,
- # skip this specific product
- if c.basetime is not None:
- time_delta = timedelta(hours=12-int(c.dtime))
- start_time = datetime.strptime(c.end_date + str(c.basetime),
- '%Y%m%d%H') - time_delta
- end_time = datetime.strptime(c.end_date + str(c.basetime),
- '%Y%m%d%H')
- if timestamp < start_time or timestamp > end_time:
- continue
+
# @WRF
# THIS IS NOT YET CORRECTLY IMPLEMENTED !!!
@@ -1619,7 +1627,6 @@ class EcFlexpart(object):
break
paramId = codes_get(gid, 'paramId')
gridtype = codes_get(gid, 'gridType')
- levtype = codes_get(gid, 'typeOfLevel')
if paramId == 77: # ETADOT
codes_write(gid, fdict['21'])
elif paramId == 130: # T
@@ -1703,7 +1710,7 @@ class EcFlexpart(object):
# Fortran program creates file fort.15 (with u,v,etadot,t,sp,q)
execute_subprocess([os.path.join(c.exedir,
- _config.FORTRAN_EXECUTABLE)],
+ _config.FORTRAN_EXECUTABLE)],
error_msg='FORTRAN PROGRAM FAILED!')#shell=True)
os.chdir(pwd)
@@ -1732,7 +1739,9 @@ class EcFlexpart(object):
# create outputfile and copy all data from intermediate files
# to the outputfile (final GRIB input files for FLEXPART)
orolsm = os.path.basename(glob.glob(c.inputdir +
- '/OG_OROLSM__SL.*.' + c.ppid + '*')[0])
+ '/OG_OROLSM__SL.*.' +
+ c.ppid +
+ '*')[0])
fluxfile = 'flux' + cdate[0:2] + suffix
if not c.cwc:
flist = ['fort.15', fluxfile, 'fort.16', orolsm]
@@ -1782,19 +1791,19 @@ class EcFlexpart(object):
'''
from eccodes import (codes_grib_new_from_file, codes_get_array,
- codes_set_array, codes_release, codes_set_values,
- codes_set, codes_write, codes_release)
+ codes_set_array, codes_release,
+ codes_set, codes_write)
# max number
maxnum = int(self.number.split('/')[-1])
# get a list of all prepared output files with control forecast (CF)
- CF_filelist = UioFiles(path, prefix + '*.N000')
- CF_filelist.files = sorted(CF_filelist.files)
+ cf_filelist = UioFiles(path, prefix + '*.N000')
+ cf_filelist.files = sorted(cf_filelist.files)
- for cffile in CF_filelist.files:
+ for cffile in cf_filelist.files:
with open(cffile, 'rb') as f:
- cfvalues=[]
+ cfvalues = []
while True:
fid = codes_grib_new_from_file(f)
if fid is None:
@@ -1899,4 +1908,3 @@ class EcFlexpart(object):
'TO OUTPUTDIR FAILED!')
return
-
diff --git a/Source/Python/Classes/GribUtil.py b/Source/Python/Classes/GribUtil.py
index 2a895d6..809544e 100644
--- a/Source/Python/Classes/GribUtil.py
+++ b/Source/Python/Classes/GribUtil.py
@@ -67,7 +67,7 @@ class GribUtil(object):
return
- def get_keys(self, keynames, wherekeynames=[], wherekeyvalues=[]):
+ def get_keys(self, keynames, wherekeynames, wherekeyvalues):
'''Get keyvalues for a given list of keynames a where statement
can be given (list of key and list of values)
@@ -76,11 +76,11 @@ class GribUtil(object):
keynames : :obj:`list` of :obj:`string`
List of keynames.
- wherekeynames : :obj:`list` of :obj:`string`, optional
- Default value is an empty list.
+ wherekeynames : :obj:`list` of :obj:`string`
+ List of key names for indexing grib message parameter.
- wherekeyvalues : :obj:`list` of :obj:`string`, optional
- Default value is an empty list.
+ wherekeyvalues : :obj:`list` of :obj:`string`
+ List of key values corresponding the key names.
Return
------
@@ -121,8 +121,8 @@ class GribUtil(object):
return return_list
- def set_keys(self, fromfile, keynames, keyvalues, wherekeynames=[],
- wherekeyvalues=[], strict=False, filemode='wb'):
+ def set_keys(self, fromfile, keynames, keyvalues, wherekeynames,
+ wherekeyvalues, filemode='wb'):
'''Opens the file to read the grib messages and then write
the selected messages (with wherekeys) to a new output file.
Also, the keyvalues of the passed list of keynames are set.
@@ -140,18 +140,11 @@ class GribUtil(object):
List of keyvalues to set in the selected messages.
Default is an empty list.
- wherekeynames : :obj:`list` of :obj:`string`, optional
+ wherekeynames : :obj:`list` of :obj:`string`
List of keynames to select correct message.
- Default value is an empty list.
- wherekeyvalues : :obj:`list` of :obj:`string`, optional
+ wherekeyvalues : :obj:`list` of :obj:`string`
List of keyvalues for keynames to select correct message.
- Default value is an empty list.
-
- strict : :obj:`boolean`, optional
- Decides if everything from keynames and keyvalues
- is written out the grib file (False) or only those
- meeting the where statement (True). Default is False.
filemode : :obj:`string`, optional
Sets the mode for the output file. Default is "wb".
@@ -199,8 +192,8 @@ class GribUtil(object):
return
- def copy_dummy_msg(self, filename_in, selectWhere=True,
- keynames=[], keyvalues=[], filemode='wb'):
+ def copy_dummy_msg(self, filename_in, keynames, keyvalues,
+ selectwhere=True, filemode='wb'):
'''Add the content of another input grib file to the objects file but
only messages corresponding to keys/values passed to the function.
The selectWhere switch decides if to copy the keys equal to (True) or
@@ -211,16 +204,16 @@ class GribUtil(object):
filename_in : :obj:`string`
Filename of the input file to read the grib messages from.
- selectWhere : :obj:`boolean`, optional
+ selectwhere : :obj:`boolean`, optional
Decides if to copy the keynames and values equal to (True) or
different to (False) the keynames/keyvalues list passed to the
function. Default is True.
- keynames : :obj:`list` of :obj:`string`, optional
- List of keynames. Default is an empty list.
+ keynames : :obj:`list` of :obj:`string`
+ List of keynames.
- keyvalues : :obj:`list` of :obj:`string`, optional
- List of keyvalues. Default is an empty list.
+ keyvalues : :obj:`list` of :obj:`string`
+ List of keyvalues.
filemode : :obj:`string`, optional
Sets the mode for the output file. Default is "wb".
@@ -253,7 +246,7 @@ class GribUtil(object):
if not codes_is_defined(gid, key):
raise Exception("Key was not defined")
- if selectWhere:
+ if selectwhere:
select = (select and (str(keyvalues[i]) ==
str(codes_get(gid, key))))
else:
@@ -271,16 +264,15 @@ class GribUtil(object):
return
- def index(self, index_keys=["mars"], index_file="my.idx"):
+ def index(self, index_keys, index_file="my.idx"):
'''Create index file from a list of files if it does not exist or
read an index file.
Parameters
----------
- index_keys: :obj:`list` of :obj:`string`, optional
+ index_keys: :obj:`list` of :obj:`string`
Contains the list of key parameter names from
which the index is to be created.
- Default is a list with a single entry string "mars".
index_file: :obj:`string`, optional
Filename where the indices are stored.
diff --git a/Source/Python/Classes/MarsRetrieval.py b/Source/Python/Classes/MarsRetrieval.py
index d68c183..105f9b9 100644
--- a/Source/Python/Classes/MarsRetrieval.py
+++ b/Source/Python/Classes/MarsRetrieval.py
@@ -41,8 +41,10 @@ import subprocess
import traceback
# software specific classes and modules from flex_extract
+#pylint: disable=wrong-import-position
sys.path.append('../')
import _config
+#pylint: disable=invalid-name
try:
ec_api = True
import ecmwfapi
@@ -54,6 +56,8 @@ try:
import cdsapi
except ImportError:
cds_api = False
+#pylint: enable=invalid-name
+#pylint: enable=wrong-import-position
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
@@ -509,7 +513,6 @@ class MarsRetrieval(object):
except Exception as e:
print('\n\nMARS Request failed!')
print(e)
- tb = sys.exc_info()[2]
print(traceback.format_exc())
sys.exit()
diff --git a/Source/Python/Classes/UioFiles.py b/Source/Python/Classes/UioFiles.py
index 10bcfb1..cff0ab4 100644
--- a/Source/Python/Classes/UioFiles.py
+++ b/Source/Python/Classes/UioFiles.py
@@ -41,15 +41,17 @@ import sys
import fnmatch
# software specific modules from flex_extract
+#pylint: disable=wrong-import-position
sys.path.append('../')
from Mods.tools import silent_remove, get_list_as_string
+#pylint: enable=wrong-import-position
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
class UioFiles(object):
- '''Collection of files matching a specific pattern.
+ """Collection of files matching a specific pattern.
The pattern can contain regular expressions for the files.
The files are listed and can be transformed to a single string or
@@ -61,16 +63,16 @@ class UioFiles(object):
Directory where to list the files.
pattern : str
- Regular expression pattern. For example: '\*.grb'
+ Regular expression pattern. For example: '*.grb'
files : list of str
List of files matching the pattern in the path.
- '''
+ """
# --------------------------------------------------------------------------
# CLASS METHODS
# --------------------------------------------------------------------------
def __init__(self, path, pattern):
- '''Assignes a specific pattern for these files.
+ """Assignes a specific pattern for these files.
Parameters
----------
@@ -78,12 +80,12 @@ class UioFiles(object):
Directory where to list the files.
pattern : str
- Regular expression pattern. For example: '\*.grb'
+ Regular expression pattern. For example: '*.grb'
Return
------
- '''
+ """
self.path = path
self.pattern = pattern
@@ -95,7 +97,7 @@ class UioFiles(object):
def _list_files(self, path):
- '''Lists all files in the directory with the matching
+ """Lists all files in the directory with the matching
regular expression pattern.
Parameters
@@ -106,11 +108,12 @@ class UioFiles(object):
Return
------
- '''
+ """
# Get the absolute path
path = os.path.abspath(path)
# get all files in the dir and subdir as absolut path
+ # pylint: disable=W0612
for root, dirnames, filenames in os.walk(path):
for filename in fnmatch.filter(filenames, self.pattern):
self.files.append(os.path.join(root, filename))
@@ -119,7 +122,7 @@ class UioFiles(object):
def __str__(self):
- '''Converts the list of files into a single string.
+ """Converts the list of files into a single string.
The entries are sepereated by "," sign.
Parameters
@@ -129,7 +132,7 @@ class UioFiles(object):
------
files_string : str
The content of the list as a single string.
- '''
+ """
filenames = [os.path.basename(f) for f in self.files]
files_string = get_list_as_string(filenames, concatenate_sign=', ')
@@ -138,7 +141,7 @@ class UioFiles(object):
def delete_files(self):
- '''Deletes the files.
+ """Deletes the files.
Parameters
----------
@@ -146,7 +149,7 @@ class UioFiles(object):
Return
------
- '''
+ """
for old_file in self.files:
silent_remove(old_file)
diff --git a/Source/Python/Mods/checks.py b/Source/Python/Mods/checks.py
index 74e7ba6..e9bc5e0 100644
--- a/Source/Python/Mods/checks.py
+++ b/Source/Python/Mods/checks.py
@@ -28,13 +28,16 @@
from __future__ import print_function
import os
import sys
-
-import _config
+from datetime import datetime
+# pylint: disable=unused-import
try:
import exceptions
except ImportError:
import builtins as exceptions
-from datetime import datetime
+# pylint: enable=unused-import
+
+# software specific classes and modules from flex_extract
+import _config
from Mods.tools import my_error, silent_remove
# ------------------------------------------------------------------------------
# FUNCTIONS
@@ -112,7 +115,7 @@ def check_grid(grid):
return grid
-def check_area(grid, area, upper, lower, left , right):
+def check_area(grid, area, upper, lower, left, right):
'''Defines the correct area string.
Checks on the format of the four area components. Wether it is of
@@ -157,19 +160,19 @@ def check_area(grid, area, upper, lower, left , right):
upper, left, lower, right = components
# determine area format
- if ((abs(float(upper) / 10000.) >= 0.01 or float(upper) / 1000. == 0. ) and
- (abs(float(lower) / 10000.) >= 0.01 or float(lower) / 1000. == 0. ) and
- (abs(float(left) / 10000.) >= 0.01 or float(left) / 1000. == 0. ) and
- (abs(float(right) / 10000.) >= 0.01 or float(right) / 1000. == 0.)):
+ if all([(abs(float(upper) / 10000.) >= 0.01 or float(upper) / 1000. == 0.),
+ (abs(float(lower) / 10000.) >= 0.01 or float(lower) / 1000. == 0.),
+ (abs(float(left) / 10000.) >= 0.01 or float(left) / 1000. == 0.),
+ (abs(float(right) / 10000.) >= 0.01 or float(right) / 1000. == 0.)]):
# area is defined in 1/1000 degrees; old format
area = '{}/{}/{}/{}'.format(float(upper) / 1000.,
float(left) / 1000.,
float(lower) / 1000.,
float(right) / 1000.)
- elif (abs(float(upper) / 10000.) < 0.05 and
- abs(float(lower) / 10000.) < 0.05 and
- abs(float(left) / 10000.) < 0.05 and
- abs(float(right) / 10000.) < 0.05):
+ elif all([abs(float(upper) / 10000.) < 0.05,
+ abs(float(lower) / 10000.) < 0.05,
+ abs(float(left) / 10000.) < 0.05,
+ abs(float(right) / 10000.) < 0.05]):
# area is already in new format
area = '{}/{}/{}/{}'.format(float(upper),
float(left),
@@ -179,7 +182,7 @@ def check_area(grid, area, upper, lower, left , right):
raise ValueError('The area components have different '
'formats (upper, lower, left, right): '
'{}/{}/{}/{}'.format(str(upper), str(lower),
- str(left) , str(right)))
+ str(left), str(right)))
return area
@@ -280,7 +283,7 @@ def check_purefc(ftype):
return 0
-def check_step(step, mailfail):
+def check_step(step):
'''Checks on step format and convert into a list of steps.
If the steps were defined with "to" and "by" they are converted into
@@ -293,11 +296,6 @@ def check_step(step, mailfail):
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time.
- mailfail : list of str
- Contains all email addresses which should be notified.
- It might also contain just the ecmwf user name which will trigger
- mailing to the associated email address for this user.
-
Return
------
step : list of str
@@ -404,7 +402,7 @@ def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc):
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time.
'''
- if not (len(ftype) == len(ftime) == len(steps)):
+ if not len(ftype) == len(ftime) == len(steps):
raise ValueError('ERROR: The number of field types, times and steps '
'are not the same! Please check the setting in the '
'CONTROL file!')
@@ -552,7 +550,7 @@ def check_dates(start, end):
raise ValueError('start_date was neither specified in command line nor '
'in CONTROL file.\n'
'Try "{} -h" to print usage information'
- .format(sys.argv[0].split('/')[-1]) )
+ .format(sys.argv[0].split('/')[-1]))
# retrieve just one day if end_date isn't set
if not end:
@@ -789,8 +787,8 @@ def check_accmaxstep(accmaxstep, marsclass, purefc, maxstep):
if purefc and int(accmaxstep) != int(maxstep):
accmaxstep = maxstep
print('... For pure forecast mode, the accumulated forecast must '
- 'have the same maxstep as the normal forecast fields!\n'
- '\t\t Accmaxstep was set to maxstep!')
+ 'have the same maxstep as the normal forecast fields!\n'
+ '\t\t Accmaxstep was set to maxstep!')
return accmaxstep
def check_addpar(addpar):
@@ -850,7 +848,7 @@ def check_job_chunk(job_chunk):
return job_chunk
-def check_number(number, mailfail):
+def check_number(number):
'''Check for correct string format of ensemble member numbers.
Parameters
@@ -858,11 +856,6 @@ def check_number(number, mailfail):
number : str
List of ensemble member forecast runs.
- mailfail : list of str
- Contains all email addresses which should be notified.
- It might also contain just the ecmwf user name which will trigger
- mailing to the associated email address for this user.
-
Return
------
number : str
diff --git a/Source/Python/Mods/disaggregation.py b/Source/Python/Mods/disaggregation.py
index 07c593b..5a35619 100644
--- a/Source/Python/Mods/disaggregation.py
+++ b/Source/Python/Mods/disaggregation.py
@@ -246,7 +246,7 @@ def IA3(g):
# the value at the end of the interval (fip1) is prescribed by the
# geometric mean, restricted such that non-negativity is guaranteed
# according to Eq. (25)
- fip1=min( 3.*g[i] , 3.*g[i+1] , np.sqrt(g[i+1]*g[i]) )
+ fip1 = min(3. * g[i], 3. * g[i + 1], np.sqrt(g[i + 1] * g[i]))
# the function value at the first sub-grid point (fi1) is determined
# according to the equal area condition with Eq. (19)
@@ -280,10 +280,10 @@ def IA3(g):
# the monotonicity filter corrects the value at (fim1) by
# substituting (fim1) with (fmon), see Eq. (27), (28) and (29)
- fmon = min(3.*g[i-2], \
- 3.*g[i-1], \
- np.sqrt(max(0,(18./13.*g[i-2] - 5./13.*f[-7]) *
- (18./13.*g[i-1] - 5./13.*f[-1]))))
+ fmon = min(3. * g[i - 2],
+ 3. * g[i - 1],
+ np.sqrt(max(0, (18. / 13. * g[i - 2] - 5. / 13. * f[-7]) *
+ (18. / 13. * g[i - 1] - 5. / 13. * f[-1]))))
# recomputation of the sub-grid interval values while the
# interval boundaries (fi) and (fip2) remains unchanged
@@ -294,7 +294,7 @@ def IA3(g):
f[-3] = 3./2.*g[i-1]-5./12.*f[-1]-1./12.*fmon
f[-2] = f[-3]+(f[-1]-fmon)/3.
- f.extend([0.,0.,0.])
+ f.extend([0., 0., 0.])
# otherwise the sub-grid values are calculated and added to the list
else:
@@ -305,7 +305,7 @@ def IA3(g):
# the value at the end of the interval (fip1) is prescribed by the
# geometric mean, restricted such that non-negativity is guaranteed
# according to Eq. (25)
- fip1 = min( 3.*g[i] , 3.*g[i+1] , np.sqrt(g[i+1]*g[i]) )
+ fip1 = min(3. * g[i], 3. * g[i + 1], np.sqrt(g[i + 1] * g[i]))
# the function value at the first sub-grid point (fi1) is determined
# according to the equal area condition with Eq. (19)
@@ -323,10 +323,10 @@ def IA3(g):
# the monotonicity filter corrects the value at (fim1) by
# substituting (fim1) with fmon, see Eq. (27), (28) and (29)
- fmon = min(3.*g[i-2], \
- 3.*g[i-1], \
- np.sqrt(max(0,(18./13.*g[i-2] - 5./13.*f[-7]) *
- (18./13.*g[i-1] - 5./13.*f[-1]))))
+ fmon = min(3. * g[i - 2],
+ 3. * g[i - 1],
+ np.sqrt(max(0, (18. / 13. * g[i - 2] - 5. / 13. * f[-7]) *
+ (18. / 13. * g[i - 1] - 5. / 13. * f[-1]))))
# recomputation of the sub-grid interval values while the
# interval boundaries (fi) and (fip2) remains unchanged
@@ -357,10 +357,10 @@ def IA3(g):
# the monotonicity filter corrects the value at (fim1) by
# substituting (fim1) with (fmon), see Eq. (27), (28) and (29)
- fmon = min(3.*g[-3], \
- 3.*g[-2], \
- np.sqrt(max(0,(18./13.*g[-3] - 5./13.*f[-7]) *
- (18./13.*g[-2] - 5./13.*f[-1]))))
+ fmon = min(3. * g[-3],
+ 3. * g[-2],
+ np.sqrt(max(0, (18. / 13. * g[-3] - 5. / 13. * f[-7]) *
+ (18. / 13. * g[-2] - 5. / 13. * f[-1]))))
# recomputation of the sub-grid interval values while the
# interval boundaries (fi) and (fip2) remains unchanged
@@ -371,7 +371,7 @@ def IA3(g):
f[-3] = 3./2.*g[-2]-5./12.*f[-1]-1./12.*fmon
f[-2] = f[-3]+(f[-1]-fmon)/3.
- f.extend([0.,0.,0.])
+ f.extend([0., 0., 0.])
# otherwise the sub-grid values are calculated and added to the list
# using the persistence hypothesis as boundary condition
@@ -396,10 +396,10 @@ def IA3(g):
# the monotonicity filter corrects the value at (fim1) by
# substituting (fim1) with (fmon), see Eq. (27), (28) and (29)
- fmon = min(3.*g[-3], \
- 3.*g[-2], \
- np.sqrt(max(0,(18./13.*g[-3] - 5./13.*f[-7]) *
- (18./13.*g[-2] - 5./13.*f[-1]))))
+ fmon = min(3. * g[-3],
+ 3. * g[-2],
+ np.sqrt(max(0, (18. / 13. * g[-3] - 5. / 13. * f[-7]) *
+ (18. / 13. * g[-2] - 5. / 13. * f[-1]))))
# recomputation of the sub-grid interval values while the
# interval boundaries (fi) and (fip2) remains unchanged
diff --git a/Source/Python/Mods/get_mars_data.py b/Source/Python/Mods/get_mars_data.py
index 48a1d36..ecdb311 100755
--- a/Source/Python/Mods/get_mars_data.py
+++ b/Source/Python/Mods/get_mars_data.py
@@ -69,13 +69,14 @@ from datetime import datetime, timedelta
# add path to local main python path for flex_extract to get full access
sys.path.append(os.path.dirname(os.path.abspath(
inspect.getfile(inspect.currentframe()))) + '/../')
+# pylint: disable=wrong-import-position
import _config
-from Mods.tools import (setup_controldata, my_error, normal_exit, get_cmdline_args,
- read_ecenv, make_dir)
+from Mods.tools import (setup_controldata, my_error, normal_exit, make_dir)
from Classes.EcFlexpart import EcFlexpart
from Classes.UioFiles import UioFiles
from Classes.MarsRetrieval import MarsRetrieval
-
+# pylint: enable=wrong-import-position
+# pylint: disable=invalid-name
try:
ec_api = True
import ecmwfapi
@@ -87,6 +88,7 @@ try:
import cdsapi
except ImportError:
cds_api = False
+# pylint: enable=invalid-name
# ------------------------------------------------------------------------------
# FUNCTION
# ------------------------------------------------------------------------------
@@ -222,9 +224,22 @@ def mk_server(c):
def check_dates_for_nonflux_fc_times(types, times):
+ '''Checks if the time 18UTC corresponds to forecast field.
+
+ Parameters
+ ----------
+ types : list of str
+ List of field types.
+
+ times : list of str or str
+ The time in hours of the field.
+
+ Return
+ ------
+ True or False
+
'''
- '''
- for ty, ti in zip(types,times):
+ for ty, ti in zip(types, times):
if ty.upper() == 'FC' and int(ti) == 18:
return True
return False
@@ -244,7 +259,7 @@ def mk_dates(c, fluxes):
boundary conditions)
Since for basetime the extraction contains the 12 hours upfront,
- if basetime is 0, the starting date has to be the day before and
+ if basetime is 0, the starting date has to be the day before
Parameters
----------
diff --git a/Source/Python/Mods/prepare_flexpart.py b/Source/Python/Mods/prepare_flexpart.py
index 5c1fd47..5732324 100755
--- a/Source/Python/Mods/prepare_flexpart.py
+++ b/Source/Python/Mods/prepare_flexpart.py
@@ -37,7 +37,9 @@
# International License. To view a copy of this license, visit
# http://creativecommons.org/licenses/by/4.0/ or send a letter to
# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
-#*******************************************************************************
+# *******************************************************************************
+# pylint: disable=ungrouped-imports
+# not necessary that we group the imports
'''This script prepares the final version of the grib files which are
then used by FLEXPART.
@@ -67,19 +69,19 @@ import datetime
import os
import inspect
import sys
-import socket
# software specific classes and modules from flex_extract
# add path to local main python path for flex_extract to get full access
sys.path.append(os.path.dirname(os.path.abspath(
inspect.getfile(inspect.currentframe()))) + '/../')
-import _config
+# pylint: disable=wrong-import-position
+#import _config
from Mods.checks import check_ppid
from Classes.UioFiles import UioFiles
-from Classes.ControlFile import ControlFile
-from Mods.tools import (setup_controldata, clean_up, get_cmdline_args,
- read_ecenv, make_dir, normal_exit)
+#from Classes.ControlFile import ControlFile
+from Mods.tools import (setup_controldata, clean_up, make_dir, normal_exit)
from Classes.EcFlexpart import EcFlexpart
+# pylint: enable=wrong-import-position
# ------------------------------------------------------------------------------
# FUNCTION
@@ -145,7 +147,7 @@ def prepare_flexpart(ppid, c):
start = start - datetime.timedelta(days=1)
print('Prepare ' + start.strftime("%Y%m%d") +
- "/to/" + end.strftime("%Y%m%d"))
+ '/to/' + end.strftime("%Y%m%d"))
# create output dir if necessary
if not os.path.exists(c.outputdir):
@@ -170,11 +172,6 @@ def prepare_flexpart(ppid, c):
flexpart.calc_extra_elda(c.inputdir, c.prefix)
flexpart.process_output(c)
- # make use of a possible conversion to a
- # specific flexpart binary format
- if c.grib2flexpart:
- flexpart.prepare_fp_files(c)
-
# check if in debugging mode, then store all files
# otherwise delete temporary files
if c.debug:
diff --git a/Source/Python/Mods/profiling.py b/Source/Python/Mods/profiling.py
index c95e4e0..0439377 100644
--- a/Source/Python/Mods/profiling.py
+++ b/Source/Python/Mods/profiling.py
@@ -36,12 +36,12 @@ import time
# ------------------------------------------------------------------------------
# FUNCTION
# ------------------------------------------------------------------------------
-def timefn(fn):
+def timefn(func):
'''
@Description:
Decorator function. It takes the inner function as an argument.
'''
- @wraps(fn)
+ @wraps(func)
def measure_time(*args, **kwargs):
'''
@Descripton:
@@ -64,10 +64,11 @@ def timefn(fn):
<nothing>
'''
- t1 = time.time()
- result = fn(*args, **kwargs)
- t2 = time.time()
- print("@timefn:" + fn.__name__ + " took " + str(t2 - t1) + " seconds")
+ time1 = time.time()
+ result = func(*args, **kwargs)
+ time2 = time.time()
+ print("@timefn:" + func.__name__ + " took " +
+ str(time2 - time1) + " seconds")
return result
diff --git a/Source/Python/Mods/tools.py b/Source/Python/Mods/tools.py
index 078c77e..32c865c 100644
--- a/Source/Python/Mods/tools.py
+++ b/Source/Python/Mods/tools.py
@@ -68,10 +68,12 @@ import sys
import glob
import subprocess
import traceback
+# pylint: disable=unused-import
try:
import exceptions
except ImportError:
import builtins as exceptions
+# pylint: enable=unused-import
from datetime import datetime, timedelta
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
@@ -267,7 +269,7 @@ def read_ecenv(filepath):
Contains the environment parameter ecuid, ecgid, gateway
and destination for ECMWF server environments.
'''
- envs= {}
+ envs = {}
try:
with open(filepath, 'r') as f:
for line in f:
@@ -278,7 +280,7 @@ def read_ecenv(filepath):
print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
sys.exit('\n... Error occured while trying to read ECMWF_ENV '
- 'file: ' + str(filepath))
+ 'file: ' + str(filepath))
return envs
@@ -301,8 +303,9 @@ def clean_up(c):
print("... clean inputdir!")
- cleanlist = [file for file in glob.glob(os.path.join(c.inputdir, "*"))
- if not os.path.basename(file).startswith(c.prefix)]
+ cleanlist = [filename for filename in
+ glob.glob(os.path.join(c.inputdir, "*"))
+ if not os.path.basename(filename).startswith(c.prefix)]
if cleanlist:
for element in cleanlist:
@@ -370,7 +373,7 @@ def send_mail(users, success_mode, message):
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1)
- pout = p.communicate(input=message.encode() + '\n\n')[0]
+ pout = p.communicate(input=message + '\n\n')[0]
except ValueError as e:
print('... ERROR: ' + str(e))
sys.exit('... Email could not be sent!')
@@ -530,7 +533,7 @@ def to_param_id(pars, table):
if not pars:
return []
if not isinstance(pars, str):
- pars=str(pars)
+ pars = str(pars)
cpar = pars.upper().split('/')
ipar = []
@@ -574,7 +577,7 @@ def to_param_id_with_tablenumber(pars, table):
if not pars:
return []
if not isinstance(pars, str):
- pars=str(pars)
+ pars = str(pars)
cpar = pars.upper().split('/')
spar = []
@@ -767,8 +770,7 @@ def get_informations(filename):
gid = codes_grib_new_from_file(f)
# information needed from grib message
- keys = [
- 'Ni',
+ keys = ['Ni',
'Nj',
'latitudeOfFirstGridPointInDegrees',
'longitudeOfFirstGridPointInDegrees',
@@ -782,8 +784,8 @@ def get_informations(filename):
print('\nInformations are: ')
for key in keys:
# Get the value of the key in a grib message.
- data[key] = codes_get(gid,key)
- print("%s = %s" % (key,data[key]))
+ data[key] = codes_get(gid, key)
+ print("%s = %s" % (key, data[key]))
# Free the memory for the message referred as gribid.
codes_release(gid)
diff --git a/Source/Python/_config.py b/Source/Python/_config.py
index 03126f7..82b7bc0 100644
--- a/Source/Python/_config.py
+++ b/Source/Python/_config.py
@@ -53,7 +53,7 @@ MAX_LEVEL_LIST = [16, 19, 31, 40, 50, 60, 62, 91, 137]
# ------------------------------------------------------------------------------
FILE_MARS_REQUESTS = 'mars_requests.csv'
-FORTRAN_EXECUTABLE = 'CONVERT2'
+FORTRAN_EXECUTABLE = 'calc_etadot'
TEMPFILE_USER_ENVVARS = 'ECMWF_ENV.template'
FILE_USER_ENVVARS = 'ECMWF_ENV'
TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.template'
@@ -96,7 +96,7 @@ PATH_JOBSCRIPTS = os.path.join(PATH_RUN_DIR, 'Jobscripts')
PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'Fortran')
PATH_PYTHONTEST_SRC = os.path.join(PATH_SOURCES, 'Pythontest')
PATH_INPUT_DIR = os.path.join(PATH_RUN_DIR, INPUT_DIRNAME_DEFAULT)
-PATH_TEST = os.path.join(PATH_FLEXEXTRACT_DIR, 'Test')
+PATH_TEST = os.path.join(PATH_FLEXEXTRACT_DIR, 'Testing')
if os.getenv('CONTROL'):
# this is only needed if (gateway) version with job script is used!
# because job is directly submitted from SCRATCH and because the
diff --git a/Source/Python/install.py b/Source/Python/install.py
index db8c736..9a3e17a 100755
--- a/Source/Python/install.py
+++ b/Source/Python/install.py
@@ -67,9 +67,7 @@ from __future__ import print_function
import os
import sys
-import glob
import subprocess
-import inspect
import tarfile
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
@@ -178,16 +176,13 @@ def install_via_gateway(c):
------
'''
- import tarfile
tarball_name = _config.FLEXEXTRACT_DIRNAME + '.tar'
tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
- mk_compilejob(c.makefile, c.install_target, c.ecuid, c.ecgid,
- c.installdir)
+ mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
- mk_job_template(c.ecuid, c.ecgid, c.gateway,
- c.destination, c.installdir)
+ mk_job_template(c.ecuid, c.ecgid, c.installdir)
mk_env_vars(c.ecuid, c.ecgid, c.gateway, c.destination)
@@ -204,7 +199,7 @@ def install_via_gateway(c):
print('Job compilation script has been submitted to ecgate for ' +
'installation in ' + c.installdir +
- '/' + _config.FLEXEXTRACT_DIRNAME)
+ '/' + _config.FLEXEXTRACT_DIRNAME)
print('You should get an email with subject "flexcompile" within ' +
'the next few minutes!')
@@ -223,12 +218,11 @@ def install_local(c):
------
'''
- import tarfile
tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR,
_config.FLEXEXTRACT_DIRNAME + '.tar')
- if c.installdir == _config.PATH_FLEXEXTRACT_DIR :
+ if c.installdir == _config.PATH_FLEXEXTRACT_DIR:
print('WARNING: installdir has not been specified')
print('flex_extract will be installed in here by compiling the ' +
'Fortran source in ' + _config.PATH_FORTRAN_SRC)
@@ -240,12 +234,12 @@ def install_local(c):
os.path.abspath(c.installdir):
mk_tarball(tar_file, c.install_target)
make_dir(os.path.join(c.installdir,
- _config.FLEXEXTRACT_DIRNAME))
+ _config.FLEXEXTRACT_DIRNAME))
os.chdir(os.path.join(c.installdir,
- _config.FLEXEXTRACT_DIRNAME))
+ _config.FLEXEXTRACT_DIRNAME))
un_tarball(tar_file)
os.chdir(os.path.join(c.installdir,
- _config.FLEXEXTRACT_DIRNAME,
+ _config.FLEXEXTRACT_DIRNAME,
_config.PATH_REL_FORTRAN_SRC))
# Create Fortran executable - CONVERT2
@@ -285,7 +279,7 @@ def check_install_conditions(c):
print('ERROR: unknown or missing installation target ')
print('target: ', c.install_target)
print('please specify correct installation target ' +
- str(INSTALL_TARGETS))
+ str(_config.INSTALL_TARGETS))
print('use -h or --help for help')
sys.exit(1)
@@ -333,7 +327,6 @@ def mk_tarball(tarball_path, target):
------
'''
- from glob import glob
print('Create tarball ...')
@@ -344,33 +337,33 @@ def mk_tarball(tarball_path, target):
# get lists of the files to be added to the tar file
if target == 'local':
- ECMWF_ENV_FILE = []
+ ecmwf_env_file = []
runfile = [os.path.relpath(x, ecd)
for x in UioFiles(_config.PATH_REL_RUN_DIR,
'run_local.sh').files]
else:
- ECMWF_ENV_FILE = [_config.PATH_REL_ECMWF_ENV]
+ ecmwf_env_file = [_config.PATH_REL_ECMWF_ENV]
runfile = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_REL_RUN_DIR,
- 'run.sh').files]
+ for x in UioFiles(_config.PATH_REL_RUN_DIR,
+ 'run.sh').files]
pyfiles = [os.path.relpath(x, ecd)
for x in UioFiles(_config.PATH_REL_PYTHON_SRC, '*py').files]
pytestfiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
+ for x in UioFiles(_config.PATH_REL_PYTHONTEST_SRC, '*py').files]
controlfiles = [os.path.relpath(x, ecd)
for x in UioFiles(_config.PATH_REL_CONTROLFILES,
'CONTROL*').files]
testfiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_REL_TEST , '*').files]
+ for x in UioFiles(_config.PATH_REL_TEST, '*').files]
tempfiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.temp').files]
+ for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.temp').files]
nlfiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_REL_TEMPLATES , '*.nl').files]
+ for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.nl').files]
gribtable = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_REL_TEMPLATES , '*grib*').files]
+ for x in UioFiles(_config.PATH_REL_TEMPLATES, '*grib*').files]
ffiles = [os.path.relpath(x, ecd)
- for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f*').files]
+ for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.f90').files]
hfiles = [os.path.relpath(x, ecd)
for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, '*.h').files]
makefiles = [os.path.relpath(x, ecd)
@@ -379,23 +372,25 @@ def mk_tarball(tarball_path, target):
# concatenate single lists to one for a better looping
filelist = pyfiles + pytestfiles + controlfiles + tempfiles + nlfiles + \
- ffiles + gribtable + hfiles + makefiles + ECMWF_ENV_FILE + \
+ ffiles + gribtable + hfiles + makefiles + ecmwf_env_file + \
runfile + jobdir + testfiles +\
['CODE_OF_CONDUCT.md', 'LICENSE.md', 'README.md']
# create installation tar-file
- exclude_files = [".ksh"]
+ exclude_files = [".ksh", ".tar"]
try:
with tarfile.open(tarball_path, "w:gz") as tar_handle:
- for file in filelist:
- tar_handle.add(file, recursive=False,
+ for filename in filelist:
+ tar_handle.add(filename, recursive=False,
filter=lambda tarinfo: None
- if os.path.splitext(tarinfo.name)[1]
- in exclude_files
- else tarinfo)
+ if os.path.splitext(tarinfo.name)[1]
+ in exclude_files
+ else tarinfo)
except tarfile.TarError as e:
+ print('... ERROR: ' + str(e))
+
sys.exit('\n... error occured while trying to create the tar-file ' +
- str(tarball_path))
+ str(tarball_path))
return
@@ -421,7 +416,7 @@ def un_tarball(tarball_path):
tar_handle.extractall()
except tarfile.TarError as e:
sys.exit('\n... error occured while trying to read tar-file ' +
- str(tarball_path))
+ str(tarball_path))
except OSError as e:
print('... ERROR CODE: ' + str(e.errno))
print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
@@ -464,11 +459,11 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
ecmwfvars_template = loader.load(_config.TEMPFILE_USER_ENVVARS,
cls=NewTextTemplate)
- stream = ecmwfvars_template.generate(user_name = ecuid,
- user_group = ecgid,
- gateway_name = gateway,
- destination_name = destination
- )
+ stream = ecmwfvars_template.generate(user_name=ecuid,
+ user_group=ecgid,
+ gateway_name=gateway,
+ destination_name=destination
+ )
except UndefinedError as e:
print('... ERROR ' + str(e))
@@ -493,7 +488,7 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
return
-def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
+def mk_compilejob(makefile, ecuid, ecgid, fp_root):
'''Modifies the original job template file so that it is specified
for the user and the environment were it will be applied. Result
is stored in a new file "job.temp" in the python directory.
@@ -504,9 +499,6 @@ def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
Name of the makefile which should be used to compile FORTRAN
CONVERT2 program.
- target : str
- The target where the installation should be done, e.g. the queue.
-
ecuid : str
The user id on ECMWF server.
@@ -534,12 +526,12 @@ def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
cls=NewTextTemplate)
stream = compile_template.generate(
- usergroup = ecgid,
- username = ecuid,
- version_number = _config._VERSION_STR,
- fp_root_scripts = fp_root,
- makefile = makefile,
- fortran_program = _config.FORTRAN_EXECUTABLE
+ usergroup=ecgid,
+ username=ecuid,
+ version_number=_config._VERSION_STR,
+ fp_root_scripts=fp_root,
+ makefile=makefile,
+ fortran_program=_config.FORTRAN_EXECUTABLE
)
except UndefinedError as e:
print('... ERROR ' + str(e))
@@ -568,7 +560,7 @@ def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
return
-def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
+def mk_job_template(ecuid, ecgid, fp_root):
'''Modifies the original job template file so that it is specified
for the user and the environment were it will be applied. Result
is stored in a new file.
@@ -581,13 +573,6 @@ def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
ecgid : str
The group id on ECMWF server.
- gateway : str
- The gateway server the user is using.
-
- destination : str
- The remote destination which is used to transfer files
- from ECMWF server to local gateway server.
-
fp_root : str
Path to the root directory of FLEXPART environment or flex_extract
environment.
@@ -614,10 +599,10 @@ def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
cls=NewTextTemplate)
stream = compile_template.generate(
- usergroup = ecgid,
- username = ecuid,
- version_number = _config._VERSION_STR,
- fp_root_path = fp_root_path_to_python,
+ usergroup=ecgid,
+ username=ecuid,
+ version_number=_config._VERSION_STR,
+ fp_root_path=fp_root_path_to_python,
)
except UndefinedError as e:
print('... ERROR ' + str(e))
@@ -710,9 +695,9 @@ def mk_convert_build(src_path, makefile):
print('ERROR: Makefile call failed:')
print(e)
else:
- execute_subprocess(['ls', '-l', os.path.join(src_path,
- _config.FORTRAN_EXECUTABLE)], error_msg=
- 'FORTRAN EXECUTABLE COULD NOT BE FOUND!')
+ execute_subprocess(['ls', '-l',
+ os.path.join(src_path, _config.FORTRAN_EXECUTABLE)],
+ error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')
return
diff --git a/Source/Python/submit.py b/Source/Python/submit.py
index 62a042a..885c030 100755
--- a/Source/Python/submit.py
+++ b/Source/Python/submit.py
@@ -63,18 +63,15 @@ from __future__ import print_function
import os
import sys
-import subprocess
-import inspect
-import collections
from datetime import datetime, timedelta
# software specific classes and modules from flex_extract
import _config
-from Mods.tools import (setup_controldata, normal_exit, get_cmdline_args,
- submit_job_to_ecserver, read_ecenv)
+from Mods.tools import (setup_controldata, normal_exit,
+ submit_job_to_ecserver)
from Mods.get_mars_data import get_mars_data
from Mods.prepare_flexpart import prepare_flexpart
-from Classes.ControlFile import ControlFile
+#from Classes.ControlFile import ControlFile
# ------------------------------------------------------------------------------
--
GitLab