diff --git a/ECMWF_FPparameter.xlsx b/ECMWF_FPparameter.xlsx index 21bae48a2bef0dbb1dc5d02d0482921cb980a01e..20207d5a2ffdfa77f4f6686d6fc3332e284e3988 100644 Binary files a/ECMWF_FPparameter.xlsx and b/ECMWF_FPparameter.xlsx differ diff --git a/python/CONTROL_OD b/python/CONTROL_OD new file mode 100644 index 0000000000000000000000000000000000000000..f82261e440a0f0cac1a9b44f8c9b15969d6feb58 --- /dev/null +++ b/python/CONTROL_OD @@ -0,0 +1,35 @@ +DAY1 20151007 +DAY2 20151007 +DTIME 3 +M_TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC +M_TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12 +M_STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11 +M_CLASS OD +M_STREAM OPER +M_NUMBER OFF +M_EXPVER 1 +M_GRID 1000 +M_LEFT -10000 +M_LOWER 30000 +M_UPPER 60000 +M_RIGHT 30000 +M_LEVEL 137 +M_LEVELIST 130/to/137 +M_RESOL 399 +M_GAUSS 0 +M_ACCURACY 16 +M_OMEGA 0 +M_OMEGADIFF 0 +M_ETA 1 +M_ETADIFF 0 +M_DPDETA 1 +M_SMOOTH 0 +M_FORMAT GRIB1 +M_ADDPAR 186/187/188/235/139/39 +PREFIX EO +ECSTORAGE 0 +ECTRANS 1 +ECFSDIR ectmp:/${USER}/econdemand/ +MAILOPS ${USER} +MAILFAIL ${USER} +EOF diff --git a/python/ControlFile.py b/python/ControlFile.py index 89747570fa1550890e6c3d71bed2437a489252e3..2fd438d1ef40c30e1b70ad31f0abc330662f2be9 100644 --- a/python/ControlFile.py +++ b/python/ControlFile.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP +# ToDo AP # - write a test class #************************************************************************ #******************************************************************************* @@ -36,7 +36,28 @@ # @Class Content: # - __init__ # - __str__ -# - tolist +# - to_list +# +# @Class Attributes: +# - start_date +# - end_date +# - accuracy +# - omega +# - cwc +# - omegadiff +# - etadiff +# - level +# - levelist +# - step +# - maxstep +# - prefix +# - makefile +# - basetime +# - date_chunk +# - grib2flexpart +# - exedir +# - flexpart_root_scripts +# - ecmwfdatadir # #******************************************************************************* @@ -45,13 +66,14 @@ # ------------------------------------------------------------------------------ import os import inspect + # software specific module from flex_extract -import Tools +from tools import get_list_as_string, my_error # ------------------------------------------------------------------------------ # CLASS # ------------------------------------------------------------------------------ -class ControlFile: +class ControlFile(object): ''' Class containing the information of the flex_extract CONTROL file. @@ -113,7 +135,6 @@ class ControlFile: data = [data[0]] for d in dd: data.append(d) - pass if len(data) == 2: if '$' in data[1]: setattr(self, data[0].lower(), data[1]) @@ -125,11 +146,9 @@ class ControlFile: if var is not None: data[1] = data[1][:i] + var + data[1][k+1:] else: - Tools.myerror(None, - 'Could not find variable ' + - data[1][j+1:k] + - ' while reading ' + - filename) + my_error(None, 'Could not find variable ' + + data[1][j+1:k] + ' while reading ' + + filename) setattr(self, data[0].lower() + '_expanded', data[1]) else: if data[1].lower() != 'none': @@ -159,8 +178,8 @@ class ControlFile: self.etadiff = '0' if not hasattr(self, 'levelist'): if not hasattr(self, 'level'): - print('Warning: neither levelist nor level \ - specified in CONTROL file') + print 'Warning: neither levelist nor level \ + specified in CONTROL file' else: self.levelist = '1/to/' + self.level else: @@ -223,7 +242,7 @@ class ControlFile: return ', '.join("%s: %s" % item for item in attrs.items()) - def tolist(self): + def to_list(self): ''' @Description: Just generates a list of strings containing the attributes and @@ -254,7 +273,7 @@ class ControlFile: elif 'ecmwfdatadir' in item[0]: pass else: - if type(item[1]) is list: + if isinstance(item[1], list): stot = '' for s in item[1]: stot += s + ' ' @@ -264,3 +283,24 @@ class ControlFile: l.append("%s %s" % item) return sorted(l) + + # def to_dict(self): + # ''' + + # ''' + # parameters_dict = vars(self) + + # # remove unneeded parameter + # parameters_dict.pop('_expanded', None) + # parameters_dict.pop('exedir', None) + # parameters_dict.pop('flexpart_root_scripts', None) + # parameters_dict.pop('ecmwfdatadir', None) + + # parameters_dict_str = {} + # for key, value in parameters_dict.iteritems(): + # if isinstance(value, list): + # parameters_dict_str[str(key)] = get_list_as_string(value, ' ') + # else: + # parameters_dict_str[str(key)] = str(value) + + # return parameters_dict_str diff --git a/python/ECFlexpart.py b/python/EcFlexpart.py similarity index 69% rename from python/ECFlexpart.py rename to python/EcFlexpart.py index 0537feb7c30b6e51237ac6e7d87d66ddeb50d6ed..4d53874971455cd4bda42f56ca07415a4a900b74 100644 --- a/python/ECFlexpart.py +++ b/python/EcFlexpart.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP +# ToDo AP # - specifiy file header documentation # - add class description in header information # - apply classtests @@ -20,8 +20,8 @@ # November 2015 - Leopold Haimberger (University of Vienna): # - extended with class Control # - removed functions mkdir_p, daterange, years_between, months_between -# - added functions darain, dapoly, toparamId, init128, normalexit, -# myerror, cleanup, install_args_and_control, +# - added functions darain, dapoly, to_param_id, init128, normal_exit, +# my_error, clean_up, install_args_and_control, # interpret_args_and_control, # - removed function __del__ in class EIFLexpart # - added the following functions in EIFlexpart: @@ -39,10 +39,10 @@ # February 2018 - Anne Philipp (University of Vienna): # - applied PEP8 style guide # - added documentation -# - removed function getFlexpartTime in class ECFlexpart +# - removed function getFlexpartTime in class EcFlexpart # - outsourced class ControlFile # - outsourced class MarsRetrieval -# - changed class name from EIFlexpart to ECFlexpart +# - changed class name from EIFlexpart to EcFlexpart # - applied minor code changes (style) # - removed "dead code" , e.g. retrieval of Q since it is not needed # - removed "times" parameter from retrieve-method since it is not used @@ -69,38 +69,57 @@ # - create # - deacc_fluxes # +# @Class Attributes: +# - dtime +# - basetime +# - server +# - marsclass +# - stream +# - resol +# - accuracy +# - number +# - expver +# - glevelist +# - area +# - grid +# - level +# - levelist +# - types +# - dates +# - area +# - gaussian +# - params +# - inputdir +# - outputfilelist +# #******************************************************************************* - +#pylint: disable=unsupported-assignment-operation +# this is disabled because its an error in pylint for this specific case +#pylint: disable=consider-using-enumerate +# this is not useful in this case # ------------------------------------------------------------------------------ # MODULES # ------------------------------------------------------------------------------ import subprocess import shutil import os -import sys -import inspect import glob -import datetime -from numpy import * -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -ecapi = True -try: - import ecmwfapi -except ImportError: - ecapi = False -from gribapi import * +from datetime import datetime, timedelta +import numpy as np +from gribapi import grib_set, grib_index_select, grib_new_from_index, grib_get,\ + grib_write, grib_get_values, grib_set_values, grib_release,\ + grib_index_release, grib_index_get # software specific classes and modules from flex_extract -from GribTools import GribTools -from Tools import init128, toparamId, silentremove, product -from ControlFile import ControlFile -from MARSretrieval import MARSretrieval -import Disagg +from Gribtools import Gribtools +from tools import init128, to_param_id, silent_remove, product, my_error +from MarsRetrieval import MarsRetrieval +import disaggregation # ------------------------------------------------------------------------------ # CLASS # ------------------------------------------------------------------------------ -class ECFlexpart: +class EcFlexpart(object): ''' Class to retrieve FLEXPART specific ECMWF data. ''' @@ -110,11 +129,11 @@ class ECFlexpart: def __init__(self, c, fluxes=False): ''' @Description: - Creates an object/instance of ECFlexpart with the + Creates an object/instance of EcFlexpart with the associated settings of its attributes for the retrieval. @Input: - self: instance of ECFlexpart + self: instance of EcFlexpart The current object of the class. c: instance of class ControlFile @@ -141,17 +160,15 @@ class ECFlexpart: # different mars types for retrieving data for flexpart self.types = dict() - try: - if c.maxstep > len(c.type): # Pure forecast mode - c.type = [c.type[1]] - c.step = ['{:0>3}'.format(int(c.step[0]))] - c.time = [c.time[0]] - for i in range(1, c.maxstep + 1): - c.type.append(c.type[0]) - c.step.append('{:0>3}'.format(i)) - c.time.append(c.time[0]) - except: - pass + + if c.maxstep > len(c.type): # Pure forecast mode + c.type = [c.type[1]] + c.step = ['{:0>3}'.format(int(c.step[0]))] + c.time = [c.time[0]] + for i in range(1, c.maxstep + 1): + c.type.append(c.type[0]) + c.step.append('{:0>3}'.format(i)) + c.time.append(c.time[0]) self.inputdir = c.inputdir self.basetime = c.basetime @@ -174,57 +191,51 @@ class ECFlexpart: if c.basetime == '00': btlist = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0] - if mod(i, int(c.dtime)) == 0 and \ - (c.maxstep > 24 or i in btlist): + if i % int(c.dtime) == 0 and c.maxstep > 24 or i in btlist: if ty not in self.types.keys(): self.types[ty] = {'times': '', 'steps': ''} if ti not in self.types[ty]['times']: - if len(self.types[ty]['times']) > 0: + if self.types[ty]['times']: self.types[ty]['times'] += '/' self.types[ty]['times'] += ti if st not in self.types[ty]['steps']: - if len(self.types[ty]['steps']) > 0: + if self.types[ty]['steps']: self.types[ty]['steps'] += '/' self.types[ty]['steps'] += st i += 1 - # Different grids need different retrievals - # SH = Spherical Harmonics, GG = Gaussian Grid, - # OG = Output Grid, ML = MultiLevel, SL = SingleLevel - self.params = {'SH__ML': '', 'SH__SL': '', - 'GG__ML': '', 'GG__SL': '', - 'OG__ML': '', 'OG__SL': '', - 'OG_OROLSM_SL': '', 'OG_acc_SL': ''} + self.marsclass = c.marsclass self.stream = c.stream self.number = c.number self.resol = c.resol self.accuracy = c.accuracy self.level = c.level - try: + + if c.levelist: self.levelist = c.levelist - except: + else: self.levelist = '1/to/' + c.level # for gaussian grid retrieval self.glevelist = '1/to/' + c.level - try: + if c.gaussian: self.gaussian = c.gaussian - except: + else: self.gaussian = '' - try: + if c.expver: self.expver = c.expver - except: + else: self.expver = '1' - try: + if c.number: self.number = c.number - except: + else: self.number = '0' if 'N' in c.grid: # Gaussian output grid @@ -249,12 +260,21 @@ class ECFlexpart: # 3) Calculation/Retrieval of omega # 4) Download also data for WRF + + # Different grids need different retrievals + # SH = Spherical Harmonics, GG = Gaussian Grid, + # OG = Output Grid, ML = MultiLevel, SL = SingleLevel + self.params = {'SH__ML': '', 'SH__SL': '', + 'GG__ML': '', 'GG__SL': '', + 'OG__ML': '', 'OG__SL': '', + 'OG_OROLSM_SL': '', 'OG_acc_SL': ''} + if fluxes is False: self.params['SH__SL'] = ['LNSP', 'ML', '1', 'OFF'] # "SD/MSL/TCC/10U/10V/2T/2D/129/172" self.params['OG__SL'] = ["141/151/164/165/166/167/168/129/172", \ 'SFC', '1', self.grid] - if len(c.addpar) > 0: + if c.addpar: if c.addpar[0] == '/': c.addpar = c.addpar[1:] self.params['OG__SL'][0] += '/' + '/'.join(c.addpar) @@ -276,8 +296,8 @@ class ECFlexpart: '{}'.format((int(self.resol) + 1) / 2)] self.params['SH__ML'] = ['U/V/D', 'ML', self.glevelist, 'OFF'] else: - print('Warning: This is a very costly parameter combination, \ - use only for debugging!') + print 'Warning: This is a very costly parameter combination, \ + use only for debugging!' self.params['GG__SL'] = ['Q', 'ML', '1', \ '{}'.format((int(self.resol) + 1) / 2)] self.params['GG__ML'] = ['U/V/D/77', 'ML', self.glevelist, \ @@ -286,29 +306,24 @@ class ECFlexpart: if c.omega == '1': self.params['OG__ML'][0] += '/W' - try: - # add cloud water content if necessary - if c.cwc == '1': - self.params['OG__ML'][0] += '/CLWC/CIWC' - except: - pass + # add cloud water content if necessary + if c.cwc == '1': + self.params['OG__ML'][0] += '/CLWC/CIWC' + + # add vorticity and geopotential height for WRF if necessary + if c.wrf == '1': + self.params['OG__ML'][0] += '/Z/VO' + if '/D' not in self.params['OG__ML'][0]: + self.params['OG__ML'][0] += '/D' + #wrf_sfc = 'sp/msl/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/ / + # stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4'.upper() + wrf_sfc = '134/235/167/165/166/168/129/172/34/31/141/ \ + 139/170/183/236/39/40/41/42'.upper() + lwrt_sfc = wrf_sfc.split('/') + for par in lwrt_sfc: + if par not in self.params['OG__SL'][0]: + self.params['OG__SL'][0] += '/' + par - try: - # add vorticity and geopotential height for WRF if necessary - if c.wrf == '1': - self.params['OG__ML'][0] += '/Z/VO' - if '/D' not in self.params['OG__ML'][0]: - self.params['OG__ML'][0] += '/D' - #wrf_sfc = 'sp/msl/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/ / - # stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4'.upper() - wrf_sfc = '134/235/167/165/166/168/129/172/34/31/141/ \ - 139/170/183/236/39/40/41/42'.upper() - lwrt_sfc = wrf_sfc.split('/') - for par in lwrt_sfc: - if par not in self.params['OG__SL'][0]: - self.params['OG__SL'][0] += '/' + par - except: - pass else: self.params['OG_acc_SL'] = ["LSP/CP/SSHF/EWSS/NSSS/SSR", \ 'SFC', '1', self.grid] @@ -327,7 +342,7 @@ class ECFlexpart: momega, momegadiff, mgauss, msmooth, meta, metadiff, mdpdeta @Input: - self: instance of ECFlexpart + self: instance of EcFlexpart The current object of the class. c: instance of class ControlFile @@ -351,27 +366,32 @@ class ECFlexpart: ''' self.inputdir = c.inputdir - area = asarray(self.area.split('/')).astype(float) - grid = asarray(self.grid.split('/')).astype(float) + area = np.asarray(self.area.split('/')).astype(float) + grid = np.asarray(self.grid.split('/')).astype(float) if area[1] > area[3]: area[1] -= 360 - zyk = abs((area[3] - area[1] - 360.) + grid[1]) < 1.e-6 maxl = int((area[3] - area[1]) / grid[1]) + 1 maxb = int((area[0] - area[2]) / grid[0]) + 1 with open(self.inputdir + '/' + filename, 'w') as f: f.write('&NAMGEN\n') f.write(',\n '.join(['maxl = ' + str(maxl), 'maxb = ' + str(maxb), - 'mlevel = ' + self.level, - 'mlevelist = ' + '"' + self.levelist + '"', - 'mnauf = ' + self.resol, 'metapar = ' + '77', - 'rlo0 = ' + str(area[1]), 'rlo1 = ' + str(area[3]), - 'rla0 = ' + str(area[2]), 'rla1 = ' + str(area[0]), - 'momega = ' + c.omega, 'momegadiff = ' + c.omegadiff, - 'mgauss = ' + c.gauss, 'msmooth = ' + c.smooth, - 'meta = ' + c.eta, 'metadiff = ' + c.etadiff, - 'mdpdeta = ' + c.dpdeta])) + 'mlevel = ' + self.level, + 'mlevelist = ' + '"' + self.levelist + '"', + 'mnauf = ' + self.resol, + 'metapar = ' + '77', + 'rlo0 = ' + str(area[1]), + 'rlo1 = ' + str(area[3]), + 'rla0 = ' + str(area[2]), + 'rla1 = ' + str(area[0]), + 'momega = ' + c.omega, + 'momegadiff = ' + c.omegadiff, + 'mgauss = ' + c.gauss, + 'msmooth = ' + c.smooth, + 'meta = ' + c.eta, + 'metadiff = ' + c.etadiff, + 'mdpdeta = ' + c.dpdeta])) f.write('\n/\n') @@ -385,7 +405,7 @@ class ECFlexpart: Prepares MARS retrievals per grid type and submits them. @Input: - self: instance of ECFlexpart + self: instance of EcFlexpart The current object of the class. server: instance of ECMWFService or ECMWFDataServer @@ -448,17 +468,27 @@ class ECFlexpart: # ------ on demand path -------------------------------------------------- if self.basetime is None: - MR = MARSretrieval(self.server, - marsclass=self.marsclass, stream=mfstream, - type=mftype, levtype=pv[1], levelist=pv[2], - resol=self.resol, gaussian=gaussian, - accuracy=self.accuracy, grid=pv[3], - target=mftarget, area=area, date=mfdate, - time=mftime, number=self.number, step=mfstep, - expver=self.expver, param=pv[0]) - - MR.displayInfo() - MR.dataRetrieve() + MR = MarsRetrieval(self.server, + marsclass=self.marsclass, + stream=mfstream, + type=mftype, + levtype=pv[1], + levelist=pv[2], + resol=self.resol, + gaussian=gaussian, + accuracy=self.accuracy, + grid=pv[3], + target=mftarget, + area=area, + date=mfdate, + time=mftime, + number=self.number, + step=mfstep, + expver=self.expver, + param=pv[0]) + + MR.display_info() + MR.data_retrieve() # ------ operational path ------------------------------------------------ else: # check if mars job requests fields beyond basetime. @@ -474,13 +504,14 @@ class ECFlexpart: else: tm1 = -1 - maxtime = datetime.datetime.strptime( - mfdate.split('/')[-1] + mftime.split('/')[tm1], - '%Y%m%d%H') + datetime.timedelta( - hours=int(mfstep.split('/')[sm1])) - elimit = datetime.datetime.strptime( - mfdate.split('/')[-1] + - self.basetime, '%Y%m%d%H') + maxdate = datetime.strptime(mfdate.split('/')[-1] + + mftime.split('/')[tm1], + '%Y%m%d%H') + istep = int(mfstep.split('/')[sm1]) + maxtime = maxdate + timedelta(hours=istep) + + elimit = datetime.strptime(mfdate.split('/')[-1] + + self.basetime, '%Y%m%d%H') if self.basetime == '12': # -------------- flux data ---------------------------- @@ -491,117 +522,157 @@ class ECFlexpart: # if 12h <= maxtime-elimit<12h reduce time for last date # if maxtime-elimit<12h reduce step for last time # A split of the MARS job into 2 is likely necessary. - maxtime = elimit - datetime.timedelta(hours=24) - mfdate = '/'.join(('/'.join(mfdate.split('/')[:-1]), - datetime.datetime.strftime( - maxtime, '%Y%m%d'))) - - MR = MARSretrieval(self.server, - marsclass=self.marsclass, - stream=self.stream, type=mftype, - levtype=pv[1], levelist=pv[2], - resol=self.resol, gaussian=gaussian, - accuracy=self.accuracy, grid=pv[3], - target=mftarget, area=area, - date=mfdate, time=mftime, - number=self.number, step=mfstep, - expver=self.expver, param=pv[0]) - - MR.displayInfo() - MR.dataRetrieve() - - maxtime = elimit - datetime.timedelta(hours=12) - mfdate = datetime.datetime.strftime(maxtime, - '%Y%m%d') + maxtime = elimit - timedelta(hours=24) + mfdate = '/'.join(['/'.join(mfdate.split('/')[:-1]), + datetime.strftime(maxtime, + '%Y%m%d')]) + + MR = MarsRetrieval(self.server, + marsclass=self.marsclass, + stream=self.stream, + type=mftype, + levtype=pv[1], + levelist=pv[2], + resol=self.resol, + gaussian=gaussian, + accuracy=self.accuracy, + grid=pv[3], + target=mftarget, + area=area, + date=mfdate, + time=mftime, + number=self.number, + step=mfstep, + expver=self.expver, + param=pv[0]) + + MR.display_info() + MR.data_retrieve() + + maxtime = elimit - timedelta(hours=12) + mfdate = datetime.strftime(maxtime, '%Y%m%d') mftime = '00' mftarget = self.inputdir + "/" + ftype + pk + \ '.' + mfdate + '.' + str(os.getppid()) +\ '.' + str(os.getpid()) + ".grb" - MR = MARSretrieval(self.server, - marsclass=self.marsclass, - stream=self.stream, type=mftype, - levtype=pv[1], levelist=pv[2], - resol=self.resol, gaussian=gaussian, - accuracy=self.accuracy, grid=pv[3], - target=mftarget, area=area, - date=mfdate, time=mftime, - number=self.number, step=mfstep, - expver=self.expver, param=pv[0]) - - MR.displayInfo() - MR.dataRetrieve() + MR = MarsRetrieval(self.server, + marsclass=self.marsclass, + stream=self.stream, + type=mftype, + levtype=pv[1], + levelist=pv[2], + resol=self.resol, + gaussian=gaussian, + accuracy=self.accuracy, + grid=pv[3], + target=mftarget, + area=area, + date=mfdate, + time=mftime, + number=self.number, + step=mfstep, + expver=self.expver, + param=pv[0]) + + MR.display_info() + MR.data_retrieve() # -------------- non flux data ------------------------ else: - MR = MARSretrieval(self.server, - marsclass=self.marsclass, - stream=self.stream, type=mftype, - levtype=pv[1], levelist=pv[2], - resol=self.resol, gaussian=gaussian, - accuracy=self.accuracy, grid=pv[3], - target=mftarget, area=area, - date=mfdate, time=mftime, - number=self.number, step=mfstep, - expver=self.expver, param=pv[0]) - - MR.displayInfo() - MR.dataRetrieve() + MR = MarsRetrieval(self.server, + marsclass=self.marsclass, + stream=self.stream, + type=mftype, + levtype=pv[1], + levelist=pv[2], + resol=self.resol, + gaussian=gaussian, + accuracy=self.accuracy, + grid=pv[3], + target=mftarget, + area=area, + date=mfdate, + time=mftime, + number=self.number, + step=mfstep, + expver=self.expver, + param=pv[0]) + + MR.display_info() + MR.data_retrieve() else: # basetime == 0 ??? #AP - maxtime = elimit - datetime.timedelta(hours=24) - mfdate = datetime.datetime.strftime(maxtime,'%Y%m%d') + maxtime = elimit - timedelta(hours=24) + mfdate = datetime.strftime(maxtime, '%Y%m%d') mftimesave = ''.join(mftime) if '/' in mftime: times = mftime.split('/') while ((int(times[0]) + - int(mfstep.split('/')[0]) <= 12) and - (pk != 'OG_OROLSM__SL') and 'acc' not in pk): + int(mfstep.split('/')[0]) <= 12) and + (pk != 'OG_OROLSM__SL') and 'acc' not in pk): times = times[1:] if len(times) > 1: mftime = '/'.join(times) else: mftime = times[0] - MR = MARSretrieval(self.server, - marsclass=self.marsclass, - stream=self.stream, type=mftype, - levtype=pv[1], levelist=pv[2], - resol=self.resol, gaussian=gaussian, - accuracy=self.accuracy, grid=pv[3], - target=mftarget, area=area, - date=mfdate, time=mftime, - number=self.number, step=mfstep, - expver=self.expver, param=pv[0]) - - MR.displayInfo() - MR.dataRetrieve() + MR = MarsRetrieval(self.server, + marsclass=self.marsclass, + stream=self.stream, + type=mftype, + levtype=pv[1], + levelist=pv[2], + resol=self.resol, + gaussian=gaussian, + accuracy=self.accuracy, + grid=pv[3], + target=mftarget, + area=area, + date=mfdate, + time=mftime, + number=self.number, + step=mfstep, + expver=self.expver, + param=pv[0]) + + MR.display_info() + MR.data_retrieve() if (int(mftimesave.split('/')[0]) == 0 and - int(mfstep.split('/')[0]) == 0 and - pk != 'OG_OROLSM__SL'): - mfdate = datetime.datetime.strftime(elimit,'%Y%m%d') + int(mfstep.split('/')[0]) == 0 and + pk != 'OG_OROLSM__SL'): + + mfdate = datetime.strftime(elimit, '%Y%m%d') mftime = '00' mfstep = '000' mftarget = self.inputdir + "/" + ftype + pk + \ '.' + mfdate + '.' + str(os.getppid()) +\ '.' + str(os.getpid()) + ".grb" - MR = MARSretrieval(self.server, - marsclass=self.marsclass, - stream=self.stream, type=mftype, - levtype=pv[1], levelist=pv[2], - resol=self.resol, gaussian=gaussian, - accuracy=self.accuracy, grid=pv[3], - target=mftarget, area=area, - date=mfdate, time=mftime, - number=self.number, step=mfstep, - expver=self.expver, param=pv[0]) - - MR.displayInfo() - MR.dataRetrieve() - - print("MARS retrieve done... ") + MR = MarsRetrieval(self.server, + marsclass=self.marsclass, + stream=self.stream, + type=mftype, + levtype=pv[1], + levelist=pv[2], + resol=self.resol, + gaussian=gaussian, + accuracy=self.accuracy, + grid=pv[3], + target=mftarget, + area=area, + date=mfdate, + time=mftime, + number=self.number, + step=mfstep, + expver=self.expver, + param=pv[0]) + + MR.display_info() + MR.data_retrieve() + + print "MARS retrieve done... " return @@ -620,7 +691,7 @@ class ECFlexpart: GRIB2FLEXPART - Conversion of GRIB files to FLEXPART binary format @Input: - self: instance of ECFlexpart + self: instance of EcFlexpart The current object of the class. c: instance of class ControlFile @@ -641,7 +712,7 @@ class ECFlexpart: ''' - print('\n\nPostprocessing:\n Format: {}\n'.format(c.format)) + print '\n\nPostprocessing:\n Format: {}\n'.format(c.format) if c.ecapi is False: print('ecstorage: {}\n ecfsdir: {}\n'. @@ -651,29 +722,29 @@ class ECFlexpart: if not hasattr(c, 'destination'): c.destination = os.getenv('DESTINATION') print('ectrans: {}\n gateway: {}\n destination: {}\n ' - .format(c.ectrans, c.gateway, c.destination)) + .format(c.ectrans, c.gateway, c.destination)) - print('Output filelist: \n') - print(self.outputfilelist) + print 'Output filelist: \n' + print self.outputfilelist if c.format.lower() == 'grib2': for ofile in self.outputfilelist: p = subprocess.check_call(['grib_set', '-s', 'edition=2, \ - productDefinitionTemplateNumber=8', - ofile, ofile + '_2']) + productDefinitionTemplateNumber=8', + ofile, ofile + '_2']) p = subprocess.check_call(['mv', ofile + '_2', ofile]) if int(c.ectrans) == 1 and c.ecapi is False: for ofile in self.outputfilelist: p = subprocess.check_call(['ectrans', '-overwrite', '-gateway', - c.gateway, '-remote', c.destination, - '-source', ofile]) + c.gateway, '-remote', c.destination, + '-source', ofile]) print('ectrans:', p) if int(c.ecstorage) == 1 and c.ecapi is False: for ofile in self.outputfilelist: p = subprocess.check_call(['ecp', '-o', ofile, - os.path.expandvars(c.ecfsdir)]) + os.path.expandvars(c.ecfsdir)]) if c.outputdir != c.inputdir: for ofile in self.outputfilelist: @@ -691,11 +762,11 @@ class ECFlexpart: fname = ofile.split('/') if '.' in fname[-1]: l = fname[-1].split('.') - timestamp = datetime.datetime.strptime(l[0][-6:] + l[1], - '%y%m%d%H') - timestamp += datetime.timedelta(hours=int(l[2])) - cdate = datetime.datetime.strftime(timestamp, '%Y%m%d') - chms = datetime.datetime.strftime(timestamp, '%H%M%S') + timestamp = datetime.strptime(l[0][-6:] + l[1], + '%y%m%d%H') + timestamp += timedelta(hours=int(l[2])) + cdate = datetime.strftime(timestamp, '%Y%m%d') + chms = datetime.strftime(timestamp, '%H%M%S') else: cdate = '20' + fname[-1][-8:-2] chms = fname[-1][-2:] + '0000' @@ -707,7 +778,7 @@ class ECFlexpart: # generate pathnames file pwd = os.path.abspath(c.outputdir) - with open(pwd + '/pathnames','w') as f: + with open(pwd + '/pathnames', 'w') as f: f.write(pwd + '/Options/\n') f.write(pwd + '/\n') f.write(pwd + '/\n') @@ -719,9 +790,8 @@ class ECFlexpart: os.makedirs(pwd+'/Options') # read template COMMAND file - with open(os.path.expandvars( - os.path.expanduser(c.flexpart_root_scripts)) + - '/../Options/COMMAND', 'r') as f: + with open(os.path.expandvars(os.path.expanduser( + c.flexpart_root_scripts)) + '/../Options/COMMAND', 'r') as f: lflist = f.read().split('\n') # find index of list where to put in the @@ -746,10 +816,9 @@ class ECFlexpart: # change to outputdir and start the grib2flexpart run # afterwards switch back to the working dir os.chdir(c.outputdir) - p = subprocess.check_call([os.path.expandvars( - os.path.expanduser(c.flexpart_root_scripts)) + - '/../FLEXPART_PROGRAM/grib2flexpart', - 'useAvailable', '.']) + p = subprocess.check_call([ + os.path.expandvars(os.path.expanduser(c.flexpart_root_scripts)) + + '/../FLEXPART_PROGRAM/grib2flexpart', 'useAvailable', '.']) os.chdir(pwd) return @@ -770,10 +839,10 @@ class ECFlexpart: "stepRange"). @Input: - self: instance of ECFlexpart + self: instance of EcFlexpart The current object of the class. - inputfiles: instance of UIOFiles + inputfiles: instance of UioFiles Contains a list of files. c: instance of class ControlFile @@ -795,14 +864,14 @@ class ECFlexpart: table128 = init128(c.ecmwfdatadir + '/grib_templates/ecmwf_grib1_table_128') - wrfpars = toparamId('sp/mslp/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/\ + wrfpars = to_param_id('sp/mslp/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/\ stl1/stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4', table128) index_keys = ["date", "time", "step"] indexfile = c.inputdir + "/date_time_stepRange.idx" - silentremove(indexfile) - grib = GribTools(inputfiles.files) + silent_remove(indexfile) + grib = Gribtools(inputfiles.files) # creates new index file iid = grib.index(index_keys=index_keys, index_file=indexfile) @@ -810,7 +879,7 @@ class ECFlexpart: index_vals = [] for key in index_keys: index_vals.append(grib_index_get(iid, key)) - print(index_vals[-1]) + print index_vals[-1] # index_vals looks for example like: # index_vals[0]: ('20171106', '20171107', '20171108') ; date # index_vals[1]: ('0', '1200', '1800', '600') ; time @@ -841,39 +910,33 @@ class ECFlexpart: convertFlag = True # remove old fort.* files and open new ones for k, f in fdict.iteritems(): - silentremove(c.inputdir + "/fort." + k) + silent_remove(c.inputdir + "/fort." + k) fdict[k] = open(c.inputdir + '/fort.' + k, 'w') cdate = str(grib_get(gid, 'date')) time = grib_get(gid, 'time') - type = grib_get(gid, 'type') step = grib_get(gid, 'step') # create correct timestamp from the three time informations # date, time, step - timestamp = datetime.datetime.strptime( - cdate + '{:0>2}'.format(time/100), '%Y%m%d%H') - timestamp += datetime.timedelta(hours=int(step)) + timestamp = datetime.strptime(cdate + '{:0>2}'.format(time/100), + '%Y%m%d%H') + timestamp += timedelta(hours=int(step)) - cdateH = datetime.datetime.strftime(timestamp, '%Y%m%d%H') - chms = datetime.datetime.strftime(timestamp, '%H%M%S') + cdateH = datetime.strftime(timestamp, '%Y%m%d%H') if c.basetime is not None: - slimit = datetime.datetime.strptime( - c.start_date + '00', '%Y%m%d%H') + slimit = datetime.strptime(c.start_date + '00', '%Y%m%d%H') bt = '23' if c.basetime == '00': bt = '00' - slimit = datetime.datetime.strptime( - c.end_date + bt, '%Y%m%d%H') - \ - datetime.timedelta(hours=12-int(c.dtime)) + slimit = datetime.strptime(c.end_date + bt, '%Y%m%d%H')\ + - timedelta(hours=12-int(c.dtime)) if c.basetime == '12': bt = '12' - slimit = datetime.datetime.strptime( - c.end_date + bt, '%Y%m%d%H') - \ - datetime.timedelta(hours=12-int(c.dtime)) + slimit = datetime.strptime(c.end_date + bt, '%Y%m%d%H')\ + - timedelta(hours=12-int(c.dtime)) - elimit = datetime.datetime.strptime( - c.end_date + bt, '%Y%m%d%H') + elimit = datetime.strptime(c.end_date + bt, '%Y%m%d%H') if timestamp < slimit or timestamp > elimit: continue @@ -900,7 +963,6 @@ class ECFlexpart: break paramId = grib_get(gid, 'paramId') gridtype = grib_get(gid, 'gridType') - datatype = grib_get(gid, 'dataType') levtype = grib_get(gid, 'typeOfLevel') if paramId == 133 and gridtype == 'reduced_gg': # Relative humidity (Q.grb) is used as a template only @@ -938,7 +1000,7 @@ class ECFlexpart: grib_write(gid, fdict['16']) savedfields.append(paramId) else: - print('duplicate ' + str(paramId) + ' not written') + print 'duplicate ' + str(paramId) + ' not written' try: if c.wrf == '1': @@ -962,16 +1024,17 @@ class ECFlexpart: pwd = os.getcwd() os.chdir(c.inputdir) if os.stat('fort.21').st_size == 0 and int(c.eta) == 1: - print('Parameter 77 (etadot) is missing, most likely it is \ - not available for this type or date/time\n') - print('Check parameters CLASS, TYPE, STREAM, START_DATE\n') - myerror(c, 'fort.21 is empty while parameter eta is set \ + print 'Parameter 77 (etadot) is missing, most likely it is \ + not available for this type or date/time\n' + print 'Check parameters CLASS, TYPE, STREAM, START_DATE\n' + my_error(c, 'fort.21 is empty while parameter eta is set \ to 1 in CONTROL file') # create the corresponding output file fort.15 # (generated by CONVERT2) + fort.16 (paramId 167 and 168) - p = subprocess.check_call([os.path.expandvars( - os.path.expanduser(c.exedir)) + '/CONVERT2'], shell=True) + p = subprocess.check_call( + [os.path.expandvars(os.path.expanduser(c.exedir)) + + '/CONVERT2'], shell=True) os.chdir(pwd) # create final output filename, e.g. EN13040500 (ENYYMMDDHH) @@ -982,7 +1045,7 @@ class ECFlexpart: else: suffix = cdateH[2:10] fnout += suffix - print("outputfile = " + fnout) + print "outputfile = " + fnout self.outputfilelist.append(fnout) # needed for final processing # create outputfile and copy all data from intermediate files @@ -1004,11 +1067,8 @@ class ECFlexpart: shutil.copyfileobj( open(c.inputdir + '/fort.25', 'rb'), fout) - try: - if c.wrf == '1': - fwrf.close() - except: - pass + if c.wrf == '1': + fwrf.close() grib_index_release(iid) @@ -1024,10 +1084,10 @@ class ECFlexpart: stress data (dapoly, cubic polynomial). @Input: - self: instance of ECFlexpart + self: instance of EcFlexpart The current object of the class. - inputfiles: instance of UIOFiles + inputfiles: instance of UioFiles Contains a list of files. c: instance of class ControlFile @@ -1049,19 +1109,19 @@ class ECFlexpart: table128 = init128(c.ecmwfdatadir + '/grib_templates/ecmwf_grib1_table_128') - pars = toparamId(self.params['OG_acc_SL'][0], table128) + pars = to_param_id(self.params['OG_acc_SL'][0], table128) index_keys = ["date", "time", "step"] indexfile = c.inputdir + "/date_time_stepRange.idx" - silentremove(indexfile) - grib = GribTools(inputfiles.files) + silent_remove(indexfile) + grib = Gribtools(inputfiles.files) # creates new index file iid = grib.index(index_keys=index_keys, index_file=indexfile) # read values of index keys index_vals = [] for key in index_keys: - key_vals = grib_index_get(iid,key) - print(key_vals) + key_vals = grib_index_get(iid, key) + print key_vals # have to sort the steps for disaggregation, # therefore convert to int first if key == 'step': @@ -1082,6 +1142,8 @@ class ECFlexpart: svalsdict[str(p)] = [] stepsdict[str(p)] = [] + print 'maxstep: ', c.maxstep + for prod in product(*index_vals): # e.g. prod = ('20170505', '0', '12') # ( date ,time, step) @@ -1091,23 +1153,19 @@ class ECFlexpart: grib_index_select(iid, index_keys[i], prod[i]) gid = grib_new_from_index(iid) - # do convert2 program if gid at this time is not None, - # therefore save in hid - hid = gid if gid is not None: cdate = grib_get(gid, 'date') time = grib_get(gid, 'time') - type = grib_get(gid, 'type') step = grib_get(gid, 'step') # date+time+step-2*dtime # (since interpolated value valid for step-2*dtime) - sdate = datetime.datetime(year=cdate/10000, - month=mod(cdate, 10000)/100, - day=mod(cdate, 100), - hour=time/100) - fdate = sdate + datetime.timedelta( - hours=step-2*int(c.dtime)) - sdates = sdate + datetime.timedelta(hours=step) + sdate = datetime(year=cdate/10000, + month=(cdate % 10000)/100, + day=(cdate % 100), + hour=time/100) + fdate = sdate + timedelta(hours=step-2*int(c.dtime)) + sdates = sdate + timedelta(hours=step) + elimit = None else: break @@ -1125,13 +1183,14 @@ class ECFlexpart: h = open(hnout, 'w') else: fnout = c.inputdir + '/flux' + fdate.strftime('%Y%m%d%H') - gnout = c.inputdir + '/flux' + (fdate+datetime.timedelta( - hours = int(c.dtime))).strftime('%Y%m%d%H') + gnout = c.inputdir + '/flux' + (fdate + + timedelta(hours=int(c.dtime)) + ).strftime('%Y%m%d%H') hnout = c.inputdir + '/flux' + sdates.strftime('%Y%m%d%H') g = open(gnout, 'w') h = open(hnout, 'w') - print("outputfile = " + fnout) + print "outputfile = " + fnout f = open(fnout, 'w') # read message for message and store relevant data fields @@ -1155,7 +1214,7 @@ class ECFlexpart: else: fak = 3600. - values = (reshape(values, (nj, ni))).flatten() / fak + values = (np.reshape(values, (nj, ni))).flatten() / fak vdp.append(values[:]) # save the accumulated values if step <= int(c.dtime): svdp.append(values[:] / int(c.dtime)) @@ -1163,7 +1222,7 @@ class ECFlexpart: svdp.append((vdp[-1] - vdp[-2]) / int(c.dtime)) print(cparamId, atime, step, len(values), - values[0], std(values)) + values[0], np.std(values)) # save the 1/3-hourly or specific values # svdp.append(values[:]) sd.append(step) @@ -1171,9 +1230,9 @@ class ECFlexpart: if len(svdp) >= 3: if len(svdp) > 3: if cparamId == '142' or cparamId == '143': - values = Disagg.darain(svdp) + values = disaggregation.darain(svdp) else: - values = Disagg.dapoly(svdp) + values = disaggregation.dapoly(svdp) if not (step == c.maxstep and c.maxstep > 12 \ or sdates == elimit): @@ -1196,27 +1255,26 @@ class ECFlexpart: grib_write(gid, f) if c.basetime is not None: - elimit = datetime.datetime.strptime(c.end_date + - c.basetime, - '%Y%m%d%H') + elimit = datetime.strptime(c.end_date + + c.basetime, '%Y%m%d%H') else: - elimit = sdate + datetime.timedelta(2*int(c.dtime)) + elimit = sdate + timedelta(2*int(c.dtime)) # squeeze out information of last two steps contained # in svdp # if step+int(c.dtime) == c.maxstep and c.maxstep>12 - # or sdates+datetime.timedelta(hours = int(c.dtime)) + # or sdates+timedelta(hours = int(c.dtime)) # >= elimit: # Note that svdp[0] has not been popped in this case - if (step == c.maxstep and c.maxstep > 12 - or sdates == elimit): + if step == c.maxstep and c.maxstep > 12 or \ + sdates == elimit: values = svdp[3] grib_set_values(gid, values) grib_set(gid, 'step', 0) - truedatetime = fdate + datetime.timedelta( - hours=2*int(c.dtime)) + truedatetime = fdate + timedelta(hours= + 2*int(c.dtime)) grib_set(gid, 'time', truedatetime.hour * 100) grib_set(gid, 'date', truedatetime.year * 10000 + truedatetime.month * 100 + @@ -1225,13 +1283,12 @@ class ECFlexpart: #values = (svdp[1]+svdp[2])/2. if cparamId == '142' or cparamId == '143': - values = Disagg.darain(list(reversed(svdp))) + values = disaggregation.darain(list(reversed(svdp))) else: - values = Disagg.dapoly(list(reversed(svdp))) + values = disaggregation.dapoly(list(reversed(svdp))) - grib_set(gid, 'step',0) - truedatetime = fdate + datetime.timedelta( - hours=int(c.dtime)) + grib_set(gid, 'step', 0) + truedatetime = fdate + timedelta(hours=int(c.dtime)) grib_set(gid, 'time', truedatetime.hour * 100) grib_set(gid, 'date', truedatetime.year * 10000 + truedatetime.month * 100 + @@ -1249,4 +1306,5 @@ class ECFlexpart: grib_index_release(iid) + exit() return diff --git a/python/GribTools.py b/python/GribTools.py index 2870334b2fff0495deed82bab6042b71ad92f1dd..e79e7681323209e4894c2eca2443b4023f56562b 100644 --- a/python/GribTools.py +++ b/python/GribTools.py @@ -1,11 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -#************************************************************************ -# TODO AP -# - GribTools name möglicherweise etwas verwirrend. -# - change self.filename in self.filenames!!! -# - bis auf --init-- und index wird keine Funktion verwendet!? -#************************************************************************ #******************************************************************************* # @Author: Anne Fouilloux (University of Oslo) # @@ -35,23 +29,29 @@ # # @Class Content: # - __init__ -# - getkeys -# - setkeys +# - get_keys +# - set_keys # - copy # - index # +# @Class Attributes: +# - filenames +# #******************************************************************************* # ------------------------------------------------------------------------------ # MODULES # ------------------------------------------------------------------------------ import os -from gribapi import * +from gribapi import grib_new_from_file, grib_is_defined, grib_get, \ + grib_release, grib_set, grib_write, grib_index_read, \ + grib_index_new_from_file, grib_index_add_file, \ + grib_index_write # ------------------------------------------------------------------------------ # CLASS # ------------------------------------------------------------------------------ -class GribTools: +class Gribtools(object): ''' Class for GRIB utilities (new methods) based on GRIB API ''' @@ -61,7 +61,7 @@ class GribTools: def __init__(self, filenames): ''' @Description: - Initialise an object of GribTools and assign a list + Initialise an object of Gribtools and assign a list of filenames. @Input: @@ -72,12 +72,12 @@ class GribTools: <nothing> ''' - self.filename = filenames + self.filenames = filenames return - def getkeys(self, keynames, wherekeynames=[], wherekeyvalues=[]): + def get_keys(self, keynames, wherekeynames=[], wherekeyvalues=[]): ''' @Description: get keyvalues for a given list of keynames @@ -87,10 +87,10 @@ class GribTools: keynames: list of strings List of keynames. - wherekeynames: list of ???, optional + wherekeynames: list of strings, optional Default value is an empty list. - wherekeyvalues: list of ???, optional + wherekeyvalues: list of strings, optional Default value is an empty list. @Return: @@ -98,7 +98,7 @@ class GribTools: List of keyvalues for given keynames. ''' - fileid = open(self.filename, 'r') + fileid = open(self.filenames, 'r') return_list = [] @@ -135,8 +135,8 @@ class GribTools: return return_list - def setkeys(self, fromfile, keynames, keyvalues, wherekeynames=[], - wherekeyvalues=[], strict=False, filemode='w'): + def set_keys(self, fromfile, keynames, keyvalues, wherekeynames=[], + wherekeyvalues=[], strict=False, filemode='w'): ''' @Description: Opens the file to read the grib messages and then write @@ -149,16 +149,16 @@ class GribTools: fromfile: string Filename of the input file to read the grib messages from. - keynames: list of ??? + keynames: list of strings List of keynames. Default is an empty list. - keyvalues: list of ??? + keyvalues: list of strings List of keynames. Default is an empty list. - wherekeynames: list of ???, optional + wherekeynames: list of strings, optional Default value is an empty list. - wherekeyvalues: list of ???, optional + wherekeyvalues: list of strings, optional Default value is an empty list. strict: boolean, optional @@ -173,7 +173,7 @@ class GribTools: <nothing> ''' - fout = open(self.filename, filemode) + fout = open(self.filenames, filemode) fin = open(fromfile) while 1: @@ -195,24 +195,14 @@ class GribTools: str(grib_get(gid_in, wherekey)))) i += 1 -#AP is it secured that the order of keynames is equal to keyvalues? if select: i = 0 for key in keynames: grib_set(gid_in, key, keyvalues[i]) i += 1 -#AP this is a redundant code section -# delete the if/else : -# -# grib_write(gid_in, fout) -# - if strict: - if select: - grib_write(gid_in, fout) - else: - grib_write(gid_in, fout) -#AP end + grib_write(gid_in, fout) + grib_release(gid_in) fin.close() @@ -237,10 +227,10 @@ class GribTools: different to (False) the keynames/keyvalues list passed to the function. Default is True. - keynames: list of ???, optional + keynames: list of strings, optional List of keynames. Default is an empty list. - keyvalues: list of ???, optional + keyvalues: list of strings, optional List of keynames. Default is an empty list. filemode: string, optional @@ -251,7 +241,7 @@ class GribTools: ''' fin = open(filename_in) - fout = open(self.filename, filemode) + fout = open(self.filenames, filemode) while 1: gid_in = grib_new_from_file(fin) @@ -306,29 +296,24 @@ class GribTools: iid: integer Grib index id. ''' - print("... index will be done") - self.iid = None + print "... index will be done" + iid = None - if (os.path.exists(index_file)): - self.iid = grib_index_read(index_file) - print("Use existing index file: %s " % (index_file)) + if os.path.exists(index_file): + iid = grib_index_read(index_file) + print "Use existing index file: %s " % (index_file) else: - for file in self.filename: - print("Inputfile: %s " % (file)) - if self.iid is None: - self.iid = grib_index_new_from_file(file, index_keys) + for filename in self.filenames: + print "Inputfile: %s " % (filename) + if iid is None: + iid = grib_index_new_from_file(filename, index_keys) else: print 'in else zweig' - grib_index_add_file(self.iid, file) - - if self.iid is not None: - grib_index_write(self.iid, index_file) - - print('... index done') - - return self.iid - - + grib_index_add_file(iid, filename) + if iid is not None: + grib_index_write(iid, index_file) + print '... index done' + return iid diff --git a/python/MARSretrieval.py b/python/MarsRetrieval.py similarity index 90% rename from python/MARSretrieval.py rename to python/MarsRetrieval.py index 3c1c5b3756121a41e94ad1ff6643a9254eb4ee0a..eccac65c4c38444d29beec73af375d4fe5b07774 100644 --- a/python/MARSretrieval.py +++ b/python/MarsRetrieval.py @@ -1,9 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -#************************************************************************ -# TODO AP -# - -#************************************************************************ #******************************************************************************* # @Author: Anne Fouilloux (University of Oslo) # @@ -12,8 +8,8 @@ # @Change History: # # November 2015 - Leopold Haimberger (University of Vienna): -# - optimized displayInfo -# - optimized dataRetrieve and seperate between python and shell +# - optimized display_info +# - optimized data_retrieve and seperate between python and shell # script call # # February 2018 - Anne Philipp (University of Vienna): @@ -36,8 +32,29 @@ # # @Class Content: # - __init__ -# - displayInfo -# - dataRetrieve +# - display_info +# - data_retrieve +# +# @Class Attributes: +# - server +# - marsclass +# - dtype +# - levtype +# - levelist +# - repres +# - date +# - resol +# - stream +# - area +# - time +# - step +# - expver +# - number +# - accuracy +# - grid +# - gaussian +# - target +# - param # #******************************************************************************* @@ -47,16 +64,10 @@ import subprocess import os -ecapi = True -try: - import ecmwfapi -except ImportError: - ecapi = False - # ------------------------------------------------------------------------------ # CLASS # ------------------------------------------------------------------------------ -class MARSretrieval: +class MarsRetrieval(object): ''' Class for submitting MARS retrievals. @@ -67,14 +78,14 @@ class MARSretrieval: ''' - def __init__(self, server, marsclass = "ei", type = "", levtype = "", - levelist = "", repres = "", date = "", resol = "", stream = "", - area = "", time = "", step = "", expver = "1", number = "", - accuracy = "", grid = "", gaussian = "", target = "", - param = ""): + def __init__(self, server, marsclass="ei", dtype="", levtype="", + levelist="", repres="", date="", resol="", stream="", + area="", time="", step="", expver="1", number="", + accuracy="", grid="", gaussian="", target="", + param=""): ''' @Description: - Initialises the instance of the MARSretrieval class and + Initialises the instance of the MarsRetrieval class and defines and assigns a set of the necessary retrieval parameters for the FLEXPART input data. A description of MARS keywords/arguments, their dependencies @@ -83,7 +94,7 @@ class MARSretrieval: https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords @Input: - self: instance of MARSretrieval + self: instance of MarsRetrieval For description see class documentation. server: instance of ECMWFService (from ECMWF Web-API) @@ -95,7 +106,7 @@ class MARSretrieval: E4 (ERA40), OD (Operational archive), ea (ERA5). Default is the ERA-Interim dataset "ei". - type: string, optional + dtype: string, optional Determines the type of fields to be retrieved. Selects between observations, images or fields. Examples for fields: Analysis (an), Forecast (fc), @@ -274,7 +285,7 @@ class MARSretrieval: self.server = server self.marsclass = marsclass - self.type = type + self.dtype = dtype self.levtype = levtype self.levelist = levelist self.repres = repres @@ -295,13 +306,13 @@ class MARSretrieval: return - def displayInfo(self): + def display_info(self): ''' @Description: Prints all class attributes and their values. @Input: - self: instance of MARSretrieval + self: instance of MarsRetrieval For description see class documentation. @Return: @@ -313,14 +324,14 @@ class MARSretrieval: # iterate through all attributes and print them # with their corresponding values for item in attrs.items(): - if item[0] in ('server'): + if item[0] in 'server': pass else: - print(item[0] + ': ' + str(item[1])) + print item[0] + ': ' + str(item[1]) return - def dataRetrieve(self): + def data_retrieve(self): ''' @Description: Submits a MARS retrieval. Depending on the existence of @@ -329,7 +340,7 @@ class MARSretrieval: are taken from the defined class attributes. @Input: - self: instance of MARSretrieval + self: instance of MarsRetrieval For description see class documentation. @Return: @@ -343,7 +354,7 @@ class MARSretrieval: # needed for the retrieval call s = 'ret' for k, v in attrs.iteritems(): - if k in ('server'): + if k in 'server': continue if k == 'marsclass': k = 'class' @@ -359,11 +370,11 @@ class MARSretrieval: try: self.server.execute(s, target) except: - print('MARS Request failed, \ - have you already registered at apps.ecmwf.int?') + print 'MARS Request failed, \ + have you already registered at apps.ecmwf.int?' raise IOError if os.stat(target).st_size == 0: - print('MARS Request returned no data - please check request') + print 'MARS Request returned no data - please check request' raise IOError # MARS request via extra process in shell else: @@ -372,14 +383,14 @@ class MARSretrieval: stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1) pout = p.communicate(input=s)[0] - print(pout.decode()) + print pout.decode() if 'Some errors reported' in pout.decode(): - print('MARS Request failed - please check request') + print 'MARS Request failed - please check request' raise IOError if os.stat(target).st_size == 0: - print('MARS Request returned no data - please check request') + print 'MARS Request returned no data - please check request' raise IOError return diff --git a/python/UIOFiles.py b/python/UioFiles.py similarity index 79% rename from python/UIOFiles.py rename to python/UioFiles.py index 7b0133381a1dc5ba220b9dcd825abf730bf8a1e5..dbb5409cc6112ad98cffdceb4dfea42f8899d037 100644 --- a/python/UIOFiles.py +++ b/python/UioFiles.py @@ -1,11 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -#************************************************************************ -# TODO AP -# - checken welche regelmässigen methoden auf diese Files noch angewendet werden -# und dann hier implementieren -# cleanup hier rein -#************************************************************************ #******************************************************************************* # @Author: Anne Fouilloux (University of Oslo) # @@ -14,15 +8,15 @@ # @Change History: # # November 2015 - Leopold Haimberger (University of Vienna): -# - modified method listFiles to work with glob instead of listdir -# - added pattern search in method listFiles +# - modified method list_files to work with glob instead of listdir +# - added pattern search in method list_files # # February 2018 - Anne Philipp (University of Vienna): # - applied PEP8 style guide # - added documentation -# - optimisation of method listFiles since it didn't work correctly +# - optimisation of method list_files since it didn't work correctly # for sub directories -# - additional speed up of method listFiles +# - additional speed up of method list_files # - modified the class so that it is initiated with a pattern instead # of suffixes. Gives more precision in selection of files. # @@ -39,8 +33,12 @@ # # @Class Content: # - __init__ -# - listFiles -# - deleteFiles +# - list_files +# - delete_files +# +# @Class Attributes: +# - pattern +# - files # #******************************************************************************* @@ -48,19 +46,17 @@ # MODULES # ------------------------------------------------------------------------------ import os -import glob import fnmatch -import time # software specific module from flex_extract -import profiling -from Tools import silentremove +#import profiling +from tools import silent_remove # ------------------------------------------------------------------------------ # CLASS # ------------------------------------------------------------------------------ -class UIOFiles: +class UioFiles(object): ''' Class to manipulate files. At initialisation it has the attribute pattern which stores a regular expression pattern for the files associated @@ -75,7 +71,7 @@ class UIOFiles: Assignes a specific pattern for these files. @Input: - self: instance of UIOFiles + self: instance of UioFiles Description see class documentation. pattern: string @@ -86,18 +82,19 @@ class UIOFiles: ''' self.pattern = pattern + self.files = None return #@profiling.timefn - def listFiles(self, path, callid=0): + def list_files(self, path, callid=0): ''' @Description: Lists all files in the directory with the matching regular expression pattern. @Input: - self: instance of UIOFiles + self: instance of UioFiles Description see class documentation. path: string @@ -131,24 +128,24 @@ class UIOFiles: # do recursive calls for sub-direcorties if subdirs: for subdir in subdirs: - self.listFiles(os.path.join(path, subdir), callid=1) + self.list_files(os.path.join(path, subdir), callid=1) return - def deleteFiles(self): + def delete_files(self): ''' @Description: Deletes the files. @Input: - self: instance of UIOFiles + self: instance of UioFiles Description see class documentation. @Return: <nothing> ''' - for f in self.files: - silentremove(f) + for old_file in self.files: + silent_remove(old_file) return diff --git a/python/Disagg.py b/python/disaggregation.py similarity index 93% rename from python/Disagg.py rename to python/disaggregation.py index 20aee506176ec38fb194a87c03343b0e63b9c522..9f285c13a7ab186b96b0625ae73507a9be28b284 100644 --- a/python/Disagg.py +++ b/python/disaggregation.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP +# ToDo AP # - check alist of size 4 ? # - write a test, IMPORTANT #************************************************************************ @@ -19,7 +19,7 @@ # - applied PEP8 style guide # - added structured documentation # - outsourced the disaggregation functions dapoly and darain -# to a new module named Disagg +# to a new module named disaggregation # # @License: # (C) Copyright 2015-2018. @@ -28,7 +28,7 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # @Module Description: -# Disaggregation of deaccumulated flux data from an ECMWF model FG field. +# disaggregationregation of deaccumulated flux data from an ECMWF model FG field. # Initially the flux data to be concerned are: # - large-scale precipitation # - convective precipitation @@ -69,7 +69,7 @@ def dapoly(alist): Interpolation of deaccumulated fluxes of an ECMWF model FG field using a cubic polynomial solution which conserves the integrals of the fluxes within each timespan. - Disaggregation is done for 4 accumluated timespans which generates + disaggregationregation is done for 4 accumluated timespans which generates a new, disaggregated value which is output at the central point of the 4 accumulation timespans. This new point is used for linear interpolation of the complete timeseries afterwards. @@ -110,7 +110,7 @@ def darain(alist): Interpolation of deaccumulated fluxes of an ECMWF model FG rainfall field using a modified linear solution which conserves the integrals of the fluxes within each timespan. - Disaggregation is done for 4 accumluated timespans which generates + disaggregationregation is done for 4 accumluated timespans which generates a new, disaggregated value which is output at the central point of the 4 accumulation timespans. This new point is used for linear interpolation of the complete timeseries afterwards. @@ -143,13 +143,3 @@ def darain(alist): nfield = xac + xbd return nfield - - - - - - - - - - diff --git a/python/getMARSdata.py b/python/get_mars_data.py similarity index 74% rename from python/getMARSdata.py rename to python/get_mars_data.py index 43872cbadd42e233a080fea4f2b278206805a096..14f6c03f2fbf02b20f52d3d60a20ae04bb1988fb 100755 --- a/python/getMARSdata.py +++ b/python/get_mars_data.py @@ -1,9 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -#************************************************************************ -# TODO AP -# - add function docstrings!!!! -#************************************************************************ #******************************************************************************* # @Author: Anne Fouilloux (University of Oslo) # @@ -12,10 +8,10 @@ # @Change History: # # November 2015 - Leopold Haimberger (University of Vienna): -# - moved the getEIdata program into a function "getMARSdata" +# - moved the getEIdata program into a function "get_mars_data" # - moved the AgurmentParser into a seperate function # - adatpted the function for the use in flex_extract -# - renamed file to getMARSdata +# - renamed file to get_mars_data # # February 2018 - Anne Philipp (University of Vienna): # - applied PEP8 style guide @@ -41,7 +37,7 @@ # # @Program Content: # - main -# - getMARSdata +# - get_mars_data # #******************************************************************************* @@ -53,23 +49,21 @@ import sys import datetime import inspect try: - ecapi=True + ecapi = True import ecmwfapi except ImportError: - ecapi=False + ecapi = False + +# software specific classes and modules from flex_extract +from tools import my_error, normal_exit, interpret_args_and_control +from EcFlexpart import EcFlexpart +from UioFiles import UioFiles # add path to pythonpath so that python finds its buddies -localpythonpath = os.path.dirname(os.path.abspath( +LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( inspect.getfile(inspect.currentframe()))) -if localpythonpath not in sys.path: - sys.path.append(localpythonpath) - -# software specific classes and modules from flex_extract -from ControlFile import ControlFile -from Tools import myerror, normalexit, \ - interpret_args_and_control -from ECFlexpart import ECFlexpart -from UIOFiles import UIOFiles +if LOCAL_PYTHON_PATH not in sys.path: + sys.path.append(LOCAL_PYTHON_PATH) # ------------------------------------------------------------------------------ # FUNCTION @@ -77,9 +71,9 @@ from UIOFiles import UIOFiles def main(): ''' @Description: - If getMARSdata is called from command line, this function controls + If get_mars_data is called from command line, this function controls the program flow and calls the argumentparser function and - the getMARSdata function for retrieving EC data. + the get_mars_data function for retrieving EC data. @Input: <nothing> @@ -88,12 +82,12 @@ def main(): <nothing> ''' args, c = interpret_args_and_control() - getMARSdata(args, c) - normalexit(c) + get_mars_data(c) + normal_exit(c) return -def getMARSdata(args, c): +def get_mars_data(c): ''' @Description: Retrieves the EC data needed for a FLEXPART simulation. @@ -102,9 +96,6 @@ def getMARSdata(args, c): is set. @Input: - args: instance of ArgumentParser - Contains the commandline arguments from script/program call. - c: instance of class ControlFile Contains all the parameters of CONTROL file, which are e.g.: DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME, @@ -125,9 +116,9 @@ def getMARSdata(args, c): if not os.path.exists(c.inputdir): os.makedirs(c.inputdir) - print("Retrieving EC data!") - print("start date %s " % (c.start_date)) - print("end date %s " % (c.end_date)) + print "Retrieving EC data!" + print "start date %s " % (c.start_date) + print "end date %s " % (c.end_date) if ecapi: server = ecmwfapi.ECMWFService("mars") @@ -159,9 +150,9 @@ def getMARSdata(args, c): # -------------- flux data ------------------------------------------------ print 'removing old flux content of ' + c.inputdir - tobecleaned = UIOFiles('*_acc_*.' + str(os.getppid()) + '.*.grb') - tobecleaned.listFiles(c.inputdir) - tobecleaned.deleteFiles() + tobecleaned = UioFiles('*_acc_*.' + str(os.getppid()) + '.*.grb') + tobecleaned.list_files(c.inputdir) + tobecleaned.delete_files() # if forecast for maximum one day (upto 23h) are to be retrieved, # collect accumulation data (flux data) @@ -171,7 +162,7 @@ def getMARSdata(args, c): day = startm1 while day < endp1: # retrieve MARS data for the whole period - flexpart = ECFlexpart(c, fluxes=True) + flexpart = EcFlexpart(c, fluxes=True) tmpday = day + datechunk - datetime.timedelta(days=1) if tmpday < endp1: dates = day.strftime("%Y%m%d") + "/to/" + \ @@ -185,7 +176,7 @@ def getMARSdata(args, c): try: flexpart.retrieve(server, dates, c.inputdir) except IOError: - myerror(c, 'MARS request failed') + my_error(c, 'MARS request failed') day += datechunk @@ -198,7 +189,7 @@ def getMARSdata(args, c): day = start while day <= end: # retrieve MARS data for the whole period - flexpart = ECFlexpart(c, fluxes=True) + flexpart = EcFlexpart(c, fluxes=True) tmpday = day + datechunk - datetime.timedelta(days=1) if tmpday < end: dates = day.strftime("%Y%m%d") + "/to/" + \ @@ -212,39 +203,38 @@ def getMARSdata(args, c): try: flexpart.retrieve(server, dates, c.inputdir) except IOError: - myerror(c, 'MARS request failed') + my_error(c, 'MARS request failed') day += datechunk # -------------- non flux data -------------------------------------------- print 'removing old non flux content of ' + c.inputdir - tobecleaned = UIOFiles('*__*.' + str(os.getppid()) + '.*.grb') - tobecleaned.listFiles(c.inputdir) - tobecleaned.deleteFiles() + tobecleaned = UioFiles('*__*.' + str(os.getppid()) + '.*.grb') + tobecleaned.list_files(c.inputdir) + tobecleaned.delete_files() day = start while day <= end: - # retrieve all non flux MARS data for the whole period - flexpart = ECFlexpart(c, fluxes=False) - tmpday = day + datechunk - datetime.timedelta(days=1) - if tmpday < end: - dates = day.strftime("%Y%m%d") + "/to/" + \ - tmpday.strftime("%Y%m%d") - else: - dates = day.strftime("%Y%m%d") + "/to/" + \ - end.strftime("%Y%m%d") + # retrieve all non flux MARS data for the whole period + flexpart = EcFlexpart(c, fluxes=False) + tmpday = day + datechunk - datetime.timedelta(days=1) + if tmpday < end: + dates = day.strftime("%Y%m%d") + "/to/" + \ + tmpday.strftime("%Y%m%d") + else: + dates = day.strftime("%Y%m%d") + "/to/" + \ + end.strftime("%Y%m%d") - print "retrieve " + dates + " in dir " + c.inputdir + print "retrieve " + dates + " in dir " + c.inputdir - try: - flexpart.retrieve(server, dates, c.inputdir) - except IOError: - myerror(c, 'MARS request failed') + try: + flexpart.retrieve(server, dates, c.inputdir) + except IOError: + my_error(c, 'MARS request failed') - day += datechunk + day += datechunk return if __name__ == "__main__": main() - diff --git a/python/install.py b/python/install.py index 09cc622fab4b2af46cfffa0af6e1630426a5e018..835b506faff68461ba5e06f7496b3e9227b24074 100755 --- a/python/install.py +++ b/python/install.py @@ -1,8 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP -# - localpythonpath should not be set in module load section! +# ToDo AP # - create a class Installation and divide installation in 3 subdefs for # ecgate, local and cca seperatly # - Change History ist nicht angepasst ans File! Original geben lassen @@ -17,6 +16,7 @@ # February 2018 - Anne Philipp (University of Vienna): # - applied PEP8 style guide # - added documentation +# - moved install_args_and_control in here # # @License: # (C) Copyright 2015-2018. @@ -44,23 +44,22 @@ # ------------------------------------------------------------------------------ # MODULES # ------------------------------------------------------------------------------ -import datetime import os import sys import glob import subprocess import inspect -from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter - -# add path to pythonpath so that python finds its buddies -localpythonpath = os.path.dirname(os.path.abspath( - inspect.getfile(inspect.currentframe()))) -if localpythonpath not in sys.path: - sys.path.append(localpythonpath) +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter # software specific classes and modules from flex_extract from ControlFile import ControlFile +# add path to pythonpath so that python finds its buddies +LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( + inspect.getfile(inspect.currentframe()))) +if LOCAL_PYTHON_PATH not in sys.path: + sys.path.append(LOCAL_PYTHON_PATH) + # ------------------------------------------------------------------------------ # FUNCTIONS # ------------------------------------------------------------------------------ @@ -77,14 +76,14 @@ def main(): <nothing> ''' - os.chdir(localpythonpath) + os.chdir(LOCAL_PYTHON_PATH) args, c = install_args_and_control() if args.install_target is not None: install_via_gateway(c, args.install_target) else: - print('Please specify installation target (local|ecgate|cca)') - print('use -h or --help for help') + print 'Please specify installation target (local|ecgate|cca)' + print 'use -h or --help for help' sys.exit() @@ -151,25 +150,25 @@ def install_args_and_control(): try: c = ControlFile(args.controlfile) - except: - print('Could not read CONTROL file "' + args.controlfile + '"') - print('Either it does not exist or its syntax is wrong.') - print('Try "' + sys.argv[0].split('/')[-1] + - ' -h" to print usage information') + except IOError: + print 'Could not read CONTROL file "' + args.controlfile + '"' + print 'Either it does not exist or its syntax is wrong.' + print 'Try "' + sys.argv[0].split('/')[-1] + \ + ' -h" to print usage information' exit(1) if args.install_target != 'local': - if (args.ecgid is None or args.ecuid is None or args.gateway is None - or args.destination is None): - print('Please enter your ECMWF user id and group id as well as \ + if args.ecgid is None or args.ecuid is None or args.gateway is None \ + or args.destination is None: + print 'Please enter your ECMWF user id and group id as well as \ the \nname of the local gateway and the ectrans \ - destination ') - print('with command line options --ecuid --ecgid \ - --gateway --destination') - print('Try "' + sys.argv[0].split('/')[-1] + - ' -h" to print usage information') - print('Please consult ecaccess documentation or ECMWF user support \ - for further details') + destination ' + print 'with command line options --ecuid --ecgid \ + --gateway --destination' + print 'Try "' + sys.argv[0].split('/')[-1] + \ + ' -h" to print usage information' + print 'Please consult ecaccess documentation or ECMWF user support \ + for further details' sys.exit(1) else: c.ecuid = args.ecuid @@ -177,10 +176,8 @@ def install_args_and_control(): c.gateway = args.gateway c.destination = args.destination - try: + if args.makefile: c.makefile = args.makefile - except: - pass if args.install_target == 'local': if args.flexpart_root_scripts is None: @@ -239,7 +236,7 @@ def install_via_gateway(c, target): data = 'export FLEXPART_ROOT_SCRIPTS=' + \ c.flexpart_root_scripts else: - data='export FLEXPART_ROOT_SCRIPTS=$HOME' + data = 'export FLEXPART_ROOT_SCRIPTS=$HOME' if target.lower() != 'local': if '--workdir' in data: data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \ @@ -291,16 +288,14 @@ def install_via_gateway(c, target): fo.write('DESTINATION ' + c.destination + '\n') fo.close() - - if target.lower() == 'local': # compile CONVERT2 if c.flexpart_root_scripts is None or c.flexpart_root_scripts == '../': - print('Warning: FLEXPART_ROOT_SCRIPTS has not been specified') - print('Only CONVERT2 will be compiled in ' + ecd + '/../src') + print 'Warning: FLEXPART_ROOT_SCRIPTS has not been specified' + print 'Only CONVERT2 will be compiled in ' + ecd + '/../src' else: c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser( - c.flexpart_root_scripts)) + c.flexpart_root_scripts)) if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts): os.chdir('/') p = subprocess.check_call(['tar', '-cvf', @@ -310,7 +305,7 @@ def install_via_gateway(c, target): ecd + 'src']) try: os.makedirs(c.flexpart_root_scripts + '/ECMWFDATA7.1') - except: + finally: pass os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1') p = subprocess.check_call(['tar', '-xvf', @@ -328,17 +323,18 @@ def install_via_gateway(c, target): if flist: p = subprocess.check_call(['rm'] + flist) try: - print(('Using makefile: ' + makefile)) + print 'Using makefile: ' + makefile p = subprocess.check_call(['make', '-f', makefile]) - p = subprocess.check_call(['ls', '-l','CONVERT2']) - except: - print('compile failed - please edit ' + makefile + - ' or try another Makefile in the src directory.') - print('most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB ' - 'and EMOSLIB must be adapted.') - print('Available Makefiles:') - print(glob.glob('Makefile*')) - + p = subprocess.check_call(['ls', '-l', 'CONVERT2']) + except subprocess.CalledProcessError as e: + print 'compile failed with the following error:' + print e.output + print 'please edit ' + makefile + \ + ' or try another Makefile in the src directory.' + print 'most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB \ + and EMOSLIB must be adapted.' + print 'Available Makefiles:' + print glob.glob('Makefile*') elif target.lower() == 'ecgate': os.chdir('/') p = subprocess.check_call(['tar', '-cvf', @@ -351,18 +347,24 @@ def install_via_gateway(c, target): ecd + '../ECMWFDATA7.1.tar', 'ecgate:/home/ms/' + c.ecgid + '/' + c.ecuid + '/ECMWFDATA7.1.tar']) - except: - print('ecaccess-file-put failed! Probably the eccert key has expired.') + except subprocess.CalledProcessError as e: + print 'ecaccess-file-put failed! \ + Probably the eccert key has expired.' + exit(1) + + try: + p = subprocess.check_call(['ecaccess-job-submit', + '-queueName', + target, + ecd + 'python/compilejob.ksh']) + print 'compilejob.ksh has been submitted to ecgate for \ + installation in ' + c.ec_flexpart_root_scripts + \ + '/ECMWFDATA7.1' + print 'You should get an email with subject flexcompile within \ + the next few minutes' + except subprocess.CalledProcessError as e: + print 'ecaccess-job-submit failed!' exit(1) - p = subprocess.check_call(['ecaccess-job-submit', - '-queueName', - target, - ecd + 'python/compilejob.ksh']) - print('compilejob.ksh has been submitted to ecgate for ' - 'installation in ' + c.ec_flexpart_root_scripts + - '/ECMWFDATA7.1') - print('You should get an email with subject flexcompile within ' - 'the next few minutes') elif target.lower() == 'cca': os.chdir('/') @@ -376,23 +378,27 @@ def install_via_gateway(c, target): ecd + '../ECMWFDATA7.1.tar', 'cca:/home/ms/' + c.ecgid + '/' + c.ecuid + '/ECMWFDATA7.1.tar']) - except: - print('ecaccess-file-put failed! ' - 'Probably the eccert key has expired.') + except subprocess.CalledProcessError as e: + print 'ecaccess-file-put failed! \ + Probably the eccert key has expired.' exit(1) - p=subprocess.check_call(['ecaccess-job-submit', - '-queueName', - target, - ecd + 'python/compilejob.ksh']) - print('compilejob.ksh has been submitted to cca for installation in ' + - c.ec_flexpart_root_scripts + '/ECMWFDATA7.1') - print('You should get an email with subject flexcompile ' - 'within the next few minutes') + try: + p = subprocess.check_call(['ecaccess-job-submit', + '-queueName', + target, + ecd + 'python/compilejob.ksh']) + print 'compilejob.ksh has been submitted to cca for installation in ' +\ + c.ec_flexpart_root_scripts + '/ECMWFDATA7.1' + print 'You should get an email with subject flexcompile \ + within the next few minutes' + except subprocess.CalledProcessError as e: + print 'ecaccess-job-submit failed!' + exit(1) else: - print('ERROR: unknown installation target ', target) - print('Valid targets: ecgate, cca, local') + print 'ERROR: unknown installation target ', target + print 'Valid targets: ecgate, cca, local' return diff --git a/python/plot_retrieved.py b/python/plot_retrieved.py index b5a086f54c8d09f77ff859eb480936561c572b8a..d924c357e5a21569af4bf88dcdf857de20301c53 100755 --- a/python/plot_retrieved.py +++ b/python/plot_retrieved.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP +# ToDo AP # - documentation der Funktionen # - docu der progam functionality # - apply pep8 @@ -18,7 +18,7 @@ # - added documentation # - created function main and moved the two function calls for # arguments and plotting into it -# - added function getBasics to extract the boundary conditions +# - added function get_basics to extract the boundary conditions # of the data fields from the first grib file it gets. # # @License: @@ -32,11 +32,11 @@ # # @Program Content: # - main -# - getBasics -# - plotRetrieved -# - plotTS -# - plotMap -# - getPlotArgs +# - get_basics +# - plot_retrieved +# - plot_timeseries +# - plot_map +# - get_plot_args # #******************************************************************************* @@ -48,48 +48,36 @@ import datetime import os import inspect import sys -import glob from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter - -from matplotlib.pylab import * -import matplotlib.patches as mpatches -from mpl_toolkits.basemap import Basemap, addcyclic -import matplotlib.colors as mcolors -from matplotlib.font_manager import FontProperties -from matplotlib.patches import Polygon -import matplotlib.cm as cmx -import matplotlib.colors as colors -#from rasotools.utils import stats - -font = {'family': 'monospace', 'size': 12} -matplotlib.rcParams['xtick.major.pad'] = '20' - -matplotlib.rc('font', **font) - -from eccodes import * +import matplotlib +import matplotlib.pyplot as plt +from mpl_toolkits.basemap import Basemap +from eccodes import codes_grib_new_from_file, codes_get, codes_release, \ + codes_get_values import numpy as np -# add path to pythonpath so that python finds its buddies -localpythonpath = os.path.dirname(os.path.abspath( - inspect.getfile(inspect.currentframe()))) -if localpythonpath not in sys.path: - sys.path.append(localpythonpath) - # software specific classes and modules from flex_extract -from Tools import silentremove from ControlFile import ControlFile -#from GribTools import GribTools -from UIOFiles import UIOFiles +from UioFiles import UioFiles + +# add path to pythonpath so that python finds its buddies +LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( + inspect.getfile(inspect.currentframe()))) +if LOCAL_PYTHON_PATH not in sys.path: + sys.path.append(LOCAL_PYTHON_PATH) +font = {'family': 'monospace', 'size': 12} +matplotlib.rcParams['xtick.major.pad'] = '20' +matplotlib.rc('font', **font) # ------------------------------------------------------------------------------ # FUNCTION # ------------------------------------------------------------------------------ def main(): ''' @Description: - If plotRetrieved is called from command line, this function controls + If plot_retrieved is called from command line, this function controls the program flow and calls the argumentparser function and - the plotRetrieved function for plotting the retrieved GRIB data. + the plot_retrieved function for plotting the retrieved GRIB data. @Input: <nothing> @@ -97,12 +85,12 @@ def main(): @Return: <nothing> ''' - args, c = getPlotArgs() - plotRetrieved(args, c) + args, c = get_plot_args() + plot_retrieved(c) return -def getBasics(ifile, verb=False): +def get_basics(ifile, verb=False): """ @Description: An example grib file will be opened and basic information will @@ -112,6 +100,7 @@ def getBasics(ifile, verb=False): @Input: ifile: string Contains the full absolute path to the ECMWF grib file. + verb (opt): bool Is True if there should be extra output in verbose mode. Default value is False. @@ -126,7 +115,6 @@ def getBasics(ifile, verb=False): 'jDirectionIncrementInDegrees', 'iDirectionIncrementInDegrees' """ - from eccodes import * data = {} @@ -140,37 +128,38 @@ def getBasics(ifile, verb=False): gid = codes_grib_new_from_file(f) # information needed from grib message - keys = [ - 'Ni', + keys = ['Ni', 'Nj', 'latitudeOfFirstGridPointInDegrees', 'longitudeOfFirstGridPointInDegrees', 'latitudeOfLastGridPointInDegrees', 'longitudeOfLastGridPointInDegrees', 'jDirectionIncrementInDegrees', - 'iDirectionIncrementInDegrees' - ] + 'iDirectionIncrementInDegrees'] - if verb: print('\nInformations are: ') + if verb: + print '\nInformations are: ' for key in keys: # Get the value of the key in a grib message. - data[key] = codes_get(gid,key) - if verb: print "%s = %s" % (key,data[key]) - if verb: print '\n' + data[key] = codes_get(gid, key) + if verb: + print "%s = %s" % (key, data[key]) + if verb: + print '\n' # Free the memory for the message referred as gribid. codes_release(gid) return data -def getFilesPerDate(files, datelist): +def get_files_per_date(files, datelist): ''' @Description: The filenames contain dates which are used to select a list of files for a specific time period specified in datelist. @Input: - files: instance of UIOFiles + files: instance of UioFiles For description see class documentation. It contains the attribute "files" which is a list of pathes to filenames. @@ -184,24 +173,21 @@ def getFilesPerDate(files, datelist): ''' filelist = [] - for file in files: - fdate = file[-8:] - ddate = datetime.datetime.strptime(fdate, '%y%m%d%H') + for filename in files: + filedate = filename[-8:] + ddate = datetime.datetime.strptime(filedate, '%y%m%d%H') if ddate in datelist: - filelist.append(file) + filelist.append(filename) return filelist -def plotRetrieved(args, c): +def plot_retrieved(c): ''' @Description: Reads GRIB data from a specified time period, a list of levels and a specified list of parameter. @Input: - args: instance of ArgumentParser - Contains the commandline arguments from script/program call. - c: instance of class ControlFile Contains all necessary information of a CONTROL file. The parameters are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM, @@ -228,40 +214,16 @@ def plotRetrieved(args, c): print 'datelist: ', datelist - c.paramIds = asarray(c.paramIds, dtype='int') - c.levels = asarray(c.levels, dtype='int') - c.area = asarray(c.area) - - # index_keys = ["date", "time", "step"] - # indexfile = c.inputdir + "/date_time_stepRange.idx" - # silentremove(indexfile) - # grib = GribTools(inputfiles.files) - # # creates new index file - # iid = grib.index(index_keys=index_keys, index_file=indexfile) - - # # read values of index keys - # index_vals = [] - # for key in index_keys: - # index_vals.append(grib_index_get(iid, key)) - # print(index_vals[-1]) - # # index_vals looks for example like: - # # index_vals[0]: ('20171106', '20171107', '20171108') ; date - # # index_vals[1]: ('0', '1200', '1800', '600') ; time - # # index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange - - + c.paramIds = np.asarray(c.paramIds, dtype='int') + c.levels = np.asarray(c.levels, dtype='int') + c.area = np.asarray(c.area) - - #index_keys = ["date", "time", "step", "paramId"] - #indexfile = c.inputdir + "/date_time_stepRange.idx" - #silentremove(indexfile) - - files = UIOFiles(c.prefix+'*') - files.listFiles(c.inputdir) - ifiles = getFilesPerDate(files.files, datelist) + files = UioFiles(c.prefix+'*') + files.list_files(c.inputdir) + ifiles = get_files_per_date(files.files, datelist) ifiles.sort() - gdict = getBasics(ifiles[0], verb=False) + gdict = get_basics(ifiles[0], verb=False) fdict = dict() fmeta = dict() @@ -273,14 +235,14 @@ def plotRetrieved(args, c): fmeta[key] = [] fstamp[key] = [] - for file in ifiles: - f = open(file) - print( "Opening file for reading data --- %s" % file) - fdate = datetime.datetime.strptime(file[-8:], "%y%m%d%H") + for filename in ifiles: + f = open(filename) + print "Opening file for reading data --- %s" % filename + fdate = datetime.datetime.strptime(filename[-8:], "%y%m%d%H") # Load in memory a grib message from a file. gid = codes_grib_new_from_file(f) - while(gid is not None): + while gid is not None: gtype = codes_get(gid, 'type') paramId = codes_get(gid, 'paramId') parameterName = codes_get(gid, 'parameterName') @@ -289,32 +251,32 @@ def plotRetrieved(args, c): if paramId in c.paramIds and level in c.levels: key = '{:0>3}_{:0>3}'.format(paramId, level) print 'key: ', key - if len(fstamp[key]) != 0 : + if fstamp[key]: for i in range(len(fstamp[key])): if fdate < fstamp[key][i]: fstamp[key].insert(i, fdate) fmeta[key].insert(i, [paramId, parameterName, gtype, - fdate, level]) - fdict[key].insert(i, flipud(reshape( - codes_get_values(gid), - [gdict['Nj'], gdict['Ni']]))) + fdate, level]) + fdict[key].insert(i, np.flipud(np.reshape( + codes_get_values(gid), + [gdict['Nj'], gdict['Ni']]))) break elif fdate > fstamp[key][i] and i == len(fstamp[key])-1: fstamp[key].append(fdate) fmeta[key].append([paramId, parameterName, gtype, - fdate, level]) - fdict[key].append(flipud(reshape( - codes_get_values(gid), - [gdict['Nj'], gdict['Ni']]))) + fdate, level]) + fdict[key].append(np.flipud(np.reshape( + codes_get_values(gid), + [gdict['Nj'], gdict['Ni']]))) break elif fdate > fstamp[key][i] and i != len(fstamp[key])-1 \ and fdate < fstamp[key][i+1]: fstamp[key].insert(i, fdate) fmeta[key].insert(i, [paramId, parameterName, gtype, - fdate, level]) - fdict[key].insert(i, flipud(reshape( - codes_get_values(gid), - [gdict['Nj'], gdict['Ni']]))) + fdate, level]) + fdict[key].insert(i, np.flipud(np.reshape( + codes_get_values(gid), + [gdict['Nj'], gdict['Ni']]))) break else: pass @@ -322,7 +284,7 @@ def plotRetrieved(args, c): fstamp[key].append(fdate) fmeta[key].append((paramId, parameterName, gtype, fdate, level)) - fdict[key].append(flipud(reshape( + fdict[key].append(np.flipud(np.reshape( codes_get_values(gid), [gdict['Nj'], gdict['Ni']]))) codes_release(gid) @@ -331,25 +293,21 @@ def plotRetrieved(args, c): gid = codes_grib_new_from_file(f) f.close() - #print 'fstamp: ', fstamp - #exit() - for k in fdict.keys(): - print 'fmeta: ', len(fmeta),fmeta + for k in fdict.iterkeys(): + print 'fmeta: ', len(fmeta), fmeta fml = fmeta[k] fdl = fdict[k] print 'fm1: ', len(fml), fml - #print 'fd1: ', fdl - #print zip(fdl, fml) for fd, fm in zip(fdl, fml): print fm ftitle = fm[1] + ' {} '.format(fm[-1]) + \ - datetime.datetime.strftime(fm[3], '%Y%m%d%H') #+ ' ' + stats(fd) + datetime.datetime.strftime(fm[3], '%Y%m%d%H') pname = '_'.join(fm[1].split()) + '_{}_'.format(fm[-1]) + \ datetime.datetime.strftime(fm[3], '%Y%m%d%H') - plotMap(c, fd, fm, gdict, ftitle, pname, 'png') + plot_map(c, fd, fm, gdict, ftitle, pname, 'png') - for k in fdict.keys(): + for k in fdict.iterkeys(): fml = fmeta[k] fdl = fdict[k] fsl = fstamp[k] @@ -357,25 +315,35 @@ def plotRetrieved(args, c): fm = fml[0] fd = fdl[0] ftitle = fm[1] + ' {} '.format(fm[-1]) + \ - datetime.datetime.strftime(fm[3], '%Y%m%d%H') #+ ' ' + stats(fd) + datetime.datetime.strftime(fm[3], '%Y%m%d%H') pname = '_'.join(fm[1].split()) + '_{}_'.format(fm[-1]) + \ datetime.datetime.strftime(fm[3], '%Y%m%d%H') - lat = -20 - lon = 20 - plotTS(c, fdl, fml, fsl, lat, lon, - gdict, ftitle, pname, 'png') + lat = -20. + lon = 20. + plot_timeseries(c, fdl, fml, fsl, lat, lon, gdict, + ftitle, pname, 'png') return -def plotTS(c, flist, fmetalist, ftimestamps, lat, lon, - gdict, ftitle, filename, fending, show=False): +def plot_timeseries(c, flist, fmetalist, ftimestamps, lat, lon, + gdict, ftitle, filename, fending, show=False): ''' @Description: + Creates a timeseries plot for a given lat/lon position. @Input: - c: + c: instance of class ControlFile + Contains all necessary information of a CONTROL file. The parameters + are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM, + NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST, + RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, + SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, + ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR, + OUTPUTDIR, FLEXPART_ROOT_SCRIPTS + For more information about format and content of the parameter see + documentation. - flist: + flist: numpy array, 2d The actual data values to be plotted from the grib messages. fmetalist: list of strings @@ -385,11 +353,20 @@ def plotTS(c, flist, fmetalist, ftimestamps, lat, lon, ftimestamps: list of datetime Contains the time stamps. - lat: + lat: float + The latitude for which the timeseries should be plotted. - lon: + lon: float + The longitude for which the timeseries should be plotted. - gdict: + gdict: dict + Contains basic informations of the ECMWF grib files, e.g. + 'Ni', 'Nj', 'latitudeOfFirstGridPointInDegrees', + 'longitudeOfFirstGridPointInDegrees', + 'latitudeOfLastGridPointInDegrees', + 'longitudeOfLastGridPointInDegrees', + 'jDirectionIncrementInDegrees', + 'iDirectionIncrementInDegrees' ftitle: string The title of the timeseries. @@ -410,39 +387,36 @@ def plotTS(c, flist, fmetalist, ftimestamps, lat, lon, t1 = time.time() - llx = gdict['longitudeOfFirstGridPointInDegrees'] - if llx > 180. : - llx -= 360. - lly = gdict['latitudeOfLastGridPointInDegrees'] - dxout = gdict['iDirectionIncrementInDegrees'] - dyout = gdict['jDirectionIncrementInDegrees'] - urx = gdict['longitudeOfLastGridPointInDegrees'] - ury = gdict['latitudeOfFirstGridPointInDegrees'] - numxgrid = gdict['Ni'] - numygrid = gdict['Nj'] - - farr = asarray(flist) + #llx = gdict['longitudeOfFirstGridPointInDegrees'] + #if llx > 180. : + # llx -= 360. + #lly = gdict['latitudeOfLastGridPointInDegrees'] + #dxout = gdict['iDirectionIncrementInDegrees'] + #dyout = gdict['jDirectionIncrementInDegrees'] + #urx = gdict['longitudeOfLastGridPointInDegrees'] + #ury = gdict['latitudeOfFirstGridPointInDegrees'] + #numxgrid = gdict['Ni'] + #numygrid = gdict['Nj'] + + farr = np.asarray(flist) #(time, lat, lon) - lonindex = linspace(llx, urx, numxgrid) - latindex = linspace(lly, ury, numygrid) - #print len(lonindex), len(latindex), farr.shape - - #latindex = (lat + 90) * 180 / (gdict['Nj'] - 1) - #lonindex = (lon + 179) * 360 / gdict['Ni'] - #print latindex, lonindex + #lonindex = linspace(llx, urx, numxgrid) + #latindex = linspace(lly, ury, numygrid) - - ts = farr[:, 0, 0]#latindex[0], lonindex[0]] + ts = farr[:, 0, 0] fig = plt.figure(figsize=(12, 6.7)) plt.plot(ftimestamps, ts) plt.title(ftitle) - plt.savefig(c.outputdir+'/'+filename+'_TS.'+fending, facecolor=fig.get_facecolor(), edgecolor='none',format=fending) + plt.savefig(c.outputdir + '/' + filename + '_TS.' + fending, + facecolor=fig.get_facecolor(), + edgecolor='none', + format=fending) print 'created ', c.outputdir + '/' + filename - if show == True: + if show: plt.show() fig.clf() plt.close(fig) @@ -451,9 +425,10 @@ def plotTS(c, flist, fmetalist, ftimestamps, lat, lon, return -def plotMap(c, flist, fmetalist, gdict, ftitle, filename, fending, show=False): +def plot_map(c, flist, fmetalist, gdict, ftitle, filename, fending, show=False): ''' @Description: + Creates a basemap plot with imshow for a given data field. @Input: c: instance of class ControlFile @@ -467,9 +442,12 @@ def plotMap(c, flist, fmetalist, gdict, ftitle, filename, fending, show=False): For more information about format and content of the parameter see documentation. - flist + flist: numpy array, 2d + The actual data values to be plotted from the grib messages. - fmetalist + fmetalist: list of strings + Contains some meta date for the data field to be plotted: + parameter id, parameter Name, grid type, datetime, level gdict: dict Contains basic informations of the ECMWF grib files, e.g. @@ -503,60 +481,81 @@ def plotMap(c, flist, fmetalist, gdict, ftitle, filename, fending, show=False): #mbaxes = fig.add_axes([0.05, 0.15, 0.8, 0.7]) llx = gdict['longitudeOfFirstGridPointInDegrees'] #- 360 - if llx > 180. : + if llx > 180.: llx -= 360. lly = gdict['latitudeOfLastGridPointInDegrees'] - dxout = gdict['iDirectionIncrementInDegrees'] - dyout = gdict['jDirectionIncrementInDegrees'] + #dxout = gdict['iDirectionIncrementInDegrees'] + #dyout = gdict['jDirectionIncrementInDegrees'] urx = gdict['longitudeOfLastGridPointInDegrees'] ury = gdict['latitudeOfFirstGridPointInDegrees'] - numxgrid = gdict['Ni'] - numygrid = gdict['Nj'] + #numxgrid = gdict['Ni'] + #numygrid = gdict['Nj'] m = Basemap(projection='cyl', llcrnrlon=llx, llcrnrlat=lly, - urcrnrlon=urx, urcrnrlat=ury,resolution='i') + urcrnrlon=urx, urcrnrlat=ury, resolution='i') - lw = 0.5 + #lw = 0.5 m.drawmapboundary() - x = linspace(llx, urx, numxgrid) - y = linspace(lly, ury, numygrid) + #x = linspace(llx, urx, numxgrid) + #y = linspace(lly, ury, numygrid) - xx, yy = m(*meshgrid(x, y)) + #xx, yy = m(*meshgrid(x, y)) #s = m.contourf(xx, yy, flist) s = plt.imshow(flist.T, - extent=(llx, urx, lly, ury), - alpha=1.0, - interpolation='nearest' - #vmin=vn, - #vmax=vx, - #cmap=my_cmap, - #levels=levels, - #cmap=my_cmap, - #norm=LogNorm(vn,vx) - ) - - title(ftitle, y=1.08) + extent=(llx, urx, lly, ury), + alpha=1.0, + interpolation='nearest' + #vmin=vn, + #vmax=vx, + #cmap=my_cmap, + #levels=levels, + #cmap=my_cmap, + #norm=LogNorm(vn,vx) + ) + + plt.title(ftitle, y=1.08) cb = m.colorbar(s, location="right", pad="10%") - #cb.set_label('Contribution per cell (ng m$^{-3}$)',size=14) - - thickline = np.arange(lly,ury+1,10.) - thinline = np.arange(lly,ury+1,5.) - m.drawparallels(thickline,color='gray',dashes=[1,1],linewidth=0.5,labels=[1,1,1,1], xoffset=1.) # draw parallels - m.drawparallels(np.setdiff1d(thinline,thickline),color='lightgray',dashes=[1,1],linewidth=0.5,labels=[0,0,0,0]) # draw parallels - - thickline = np.arange(llx,urx+1,10.) - thinline = np.arange(llx,urx+1,5.) - m.drawmeridians(thickline,color='gray',dashes=[1,1],linewidth=0.5,labels=[1,1,1,1],yoffset=1.) # draw meridians - m.drawmeridians(np.setdiff1d(thinline,thickline),color='lightgray',dashes=[1,1],linewidth=0.5,labels=[0,0,0,0]) # draw meridians + cb.set_label('label', size=14) + + thickline = np.arange(lly, ury+1, 10.) + thinline = np.arange(lly, ury+1, 5.) + m.drawparallels(thickline, + color='gray', + dashes=[1, 1], + linewidth=0.5, + labels=[1, 1, 1, 1], + xoffset=1.) + m.drawparallels(np.setdiff1d(thinline, thickline), + color='lightgray', + dashes=[1, 1], + linewidth=0.5, + labels=[0, 0, 0, 0]) + + thickline = np.arange(llx, urx+1, 10.) + thinline = np.arange(llx, urx+1, 5.) + m.drawmeridians(thickline, + color='gray', + dashes=[1, 1], + linewidth=0.5, + labels=[1, 1, 1, 1], + yoffset=1.) + m.drawmeridians(np.setdiff1d(thinline, thickline), + color='lightgray', + dashes=[1, 1], + linewidth=0.5, + labels=[0, 0, 0, 0]) m.drawcoastlines() m.drawcountries() - plt.savefig(c.outputdir+'/'+filename+'_MAP.'+fending, facecolor=fig.get_facecolor(), edgecolor='none',format=fending) + plt.savefig(c.outputdir + '/' + filename + '_MAP.' + fending, + facecolor=fig.get_facecolor(), + edgecolor='none', + format=fending) print 'created ', c.outputdir + '/' + filename - if show == True: + if show: plt.show() fig.clf() plt.close(fig) @@ -565,7 +564,7 @@ def plotMap(c, flist, fmetalist, gdict, ftitle, filename, fending, show=False): return -def getPlotArgs(): +def get_plot_args(): ''' @Description: Assigns the command line arguments and reads CONTROL file @@ -596,13 +595,13 @@ def getPlotArgs(): # the most important arguments parser.add_argument("--start_date", dest="start_date", help="start date YYYYMMDD") - parser.add_argument( "--end_date", dest="end_date", - help="end_date YYYYMMDD") + parser.add_argument("--end_date", dest="end_date", + help="end_date YYYYMMDD") parser.add_argument("--start_step", dest="start_step", help="start step in hours") - parser.add_argument( "--end_step", dest="end_step", - help="end step in hours") + parser.add_argument("--end_step", dest="end_step", + help="end step in hours") # some arguments that override the default in the CONTROL file parser.add_argument("--levelist", dest="levelist", @@ -634,9 +633,8 @@ def getPlotArgs(): c = ControlFile(args.controlfile) except IOError: try: - c = ControlFile(localpythonpath + args.controlfile) - - except: + c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile) + except IOError: print 'Could not read CONTROL file "' + args.controlfile + '"' print 'Either it does not exist or its syntax is wrong.' print 'Try "' + sys.argv[0].split('/')[-1] + \ @@ -681,4 +679,3 @@ def getPlotArgs(): if __name__ == "__main__": main() - diff --git a/python/prepareFLEXPART.py b/python/prepare_flexpart.py similarity index 79% rename from python/prepareFLEXPART.py rename to python/prepare_flexpart.py index b1e0b866d48439ee77bdb3c9e123931d6e67c417..2e5a160a52252fa2a45809fc506ccf7940543a2c 100755 --- a/python/prepareFLEXPART.py +++ b/python/prepare_flexpart.py @@ -1,9 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP -# - wieso cleanup in main wenn es in prepareflexpart bereits abgefragt wurde? -# doppelt gemoppelt? +# ToDo AP # - wieso start=startm1 wenn basetime = 0 ? wenn die fluxes nicht mehr # relevant sind? verstehe ich nicht #************************************************************************ @@ -28,11 +26,11 @@ # - applied PEP8 style guide # - added documentation # - minor changes in programming style for consistence -# - BUG: removed call of cleanup-Function after call of +# - BUG: removed call of clean_up-Function after call of # prepareFlexpart in main since it is already called in # prepareFlexpart at the end! # - created function main and moved the two function calls for -# arguments and prepareFLEXPART into it +# arguments and prepare_flexpart into it # # @License: # (C) Copyright 2014-2018. @@ -43,7 +41,7 @@ # @Program Functionality: # This program prepares the final version of the grib files which are # then used by FLEXPART. It converts the bunch of grib files extracted -# via getMARSdata by doing for example the necessary conversion to get +# via get_mars_data by doing for example the necessary conversion to get # consistent grids or the disaggregation of flux data. Finally, the # program combines the data fields in files per available hour with the # naming convention xxYYMMDDHH, where xx should be 2 arbitrary letters @@ -51,24 +49,25 @@ # # @Program Content: # - main -# - prepareFLEXPART +# - prepare_flexpart # #******************************************************************************* # ------------------------------------------------------------------------------ # MODULES # ------------------------------------------------------------------------------ -import shutil import datetime -#import time import os import inspect import sys import socket -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -hostname = socket.gethostname() -ecapi = 'ecmwf' not in hostname +# software specific classes and modules from flex_extract +from UioFiles import UioFiles +from tools import interpret_args_and_control, clean_up +from EcFlexpart import EcFlexpart + +ecapi = 'ecmwf' not in socket.gethostname() try: if ecapi: import ecmwfapi @@ -76,24 +75,20 @@ except ImportError: ecapi = False # add path to pythonpath so that python finds its buddies -localpythonpath = os.path.dirname(os.path.abspath( +LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( inspect.getfile(inspect.currentframe()))) -if localpythonpath not in sys.path: - sys.path.append(localpythonpath) +if LOCAL_PYTHON_PATH not in sys.path: + sys.path.append(LOCAL_PYTHON_PATH) -# software specific classes and modules from flex_extract -from UIOFiles import UIOFiles -from Tools import interpret_args_and_control, cleanup -from ECFlexpart import ECFlexpart # ------------------------------------------------------------------------------ # FUNCTION # ------------------------------------------------------------------------------ def main(): ''' @Description: - If prepareFLEXPART is called from command line, this function controls + If prepare_flexpart is called from command line, this function controls the program flow and calls the argumentparser function and - the prepareFLEXPART function for preparation of GRIB data for FLEXPART. + the prepare_flexpart function for preparation of GRIB data for FLEXPART. @Input: <nothing> @@ -102,14 +97,14 @@ def main(): <nothing> ''' args, c = interpret_args_and_control() - prepareFLEXPART(args, c) + prepare_flexpart(args, c) return -def prepareFLEXPART(args, c): +def prepare_flexpart(args, c): ''' @Description: - Lists all grib files retrieved from MARS with getMARSdata and + Lists all grib files retrieved from MARS with get_mars_data and uses prepares data for the use in FLEXPART. Specific data fields are converted to a different grid and the flux data are going to be disaggregated. The data fields are collected by hour and stored in @@ -156,27 +151,27 @@ def prepareFLEXPART(args, c): # one day ahead of the start date and # one day after the end date is needed startm1 = start - datetime.timedelta(days=1) - endp1 = end + datetime.timedelta(days=1) +# endp1 = end + datetime.timedelta(days=1) # get all files with flux data to be deaccumulated - inputfiles = UIOFiles('*OG_acc_SL*.' + c.ppid + '.*') - inputfiles.listFiles(c.inputdir) + inputfiles = UioFiles('*OG_acc_SL*.' + c.ppid + '.*') + inputfiles.list_files(c.inputdir) # create output dir if necessary if not os.path.exists(c.outputdir): os.makedirs(c.outputdir) # deaccumulate the flux data - flexpart = ECFlexpart(c, fluxes=True) + flexpart = EcFlexpart(c, fluxes=True) flexpart.write_namelist(c, 'fort.4') flexpart.deacc_fluxes(inputfiles, c) - print('Prepare ' + start.strftime("%Y%m%d") + - "/to/" + end.strftime("%Y%m%d")) + print 'Prepare ' + start.strftime("%Y%m%d") + \ + "/to/" + end.strftime("%Y%m%d") # get a list of all files from the root inputdir - inputfiles = UIOFiles('????__??.*' + c.ppid + '.*') - inputfiles.listFiles(c.inputdir) + inputfiles = UioFiles('????__??.*' + c.ppid + '.*') + inputfiles.list_files(c.inputdir) # produce FLEXPART-ready GRIB files and # process GRIB files - @@ -184,16 +179,16 @@ def prepareFLEXPART(args, c): if c.basetime == '00': start = startm1 - flexpart = ECFlexpart(c, fluxes=False) + flexpart = EcFlexpart(c, fluxes=False) flexpart.create(inputfiles, c) flexpart.process_output(c) # check if in debugging mode, then store all files # otherwise delete temporary files if int(c.debug) != 0: - print('Temporary files left intact') + print 'Temporary files left intact' else: - cleanup(c) + clean_up(c) return diff --git a/python/profiling.py b/python/profiling.py index b20e6e60419129866ed7b4682933b64158ee34c4..526c17fc5d4e0e2de413530ea7189542521f1435 100644 --- a/python/profiling.py +++ b/python/profiling.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP +# ToDo AP # - check of license of book content #************************************************************************ #******************************************************************************* @@ -65,7 +65,7 @@ def timefn(fn): t1 = time.time() result = fn(*args, **kwargs) t2 = time.time() - print("@timefn:" + fn.func_name + " took " + str(t2 - t1) + " seconds") + print "@timefn:" + fn.func_name + " took " + str(t2 - t1) + " seconds" return result diff --git a/python/submit.py b/python/submit.py index 0685103a988299d12586a5a15c222747a031ab25..fcf5735d4693082ac31a587313c8dbd411ff4f58 100755 --- a/python/submit.py +++ b/python/submit.py @@ -1,15 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -#************************************************************************ -# TODO AP -# -# - Change History ist nicht angepasst ans File! Original geben lassen -# - dead code ? what to do? -# - seperate operational and reanlysis for clarification -# - divide in two submits , ondemand und operational -# - -#************************************************************************ - #******************************************************************************* # @Author: Anne Fouilloux (University of Oslo) # @@ -36,7 +26,7 @@ # This program is the main program of flex_extract and controls the # program flow. # If it is supposed to work locally then it works through the necessary -# functions getMARSdata and prepareFlexpart. Otherwise it prepares +# functions get_mars_data and prepareFlexpart. Otherwise it prepares # a shell job script which will do the necessary work on the # ECMWF server and is submitted via ecaccess-job-submit. # @@ -51,19 +41,19 @@ # ------------------------------------------------------------------------------ import os import sys -import glob import subprocess import inspect -# add path to pythonpath so that python finds its buddies -localpythonpath = os.path.dirname(os.path.abspath( - inspect.getfile(inspect.currentframe()))) -if localpythonpath not in sys.path: - sys.path.append(localpythonpath) # software specific classes and modules from flex_extract -from Tools import interpret_args_and_control, normalexit -from getMARSdata import getMARSdata -from prepareFLEXPART import prepareFLEXPART +from tools import interpret_args_and_control, normal_exit +from get_mars_data import get_mars_data +from prepare_flexpart import prepare_flexpart + +# add path to pythonpath so that python finds its buddies +LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( + inspect.getfile(inspect.currentframe()))) +if LOCAL_PYTHON_PATH not in sys.path: + sys.path.append(LOCAL_PYTHON_PATH) # ------------------------------------------------------------------------------ # FUNCTIONS @@ -83,17 +73,19 @@ def main(): @Return: <nothing> ''' - calledfromdir = os.getcwd() + + called_from_dir = os.getcwd() args, c = interpret_args_and_control() + # on local side if args.queue is None: if c.inputdir[0] != '/': - c.inputdir = os.path.join(calledfromdir, c.inputdir) + c.inputdir = os.path.join(called_from_dir, c.inputdir) if c.outputdir[0] != '/': - c.outputdir = os.path.join(calledfromdir, c.outputdir) - getMARSdata(args, c) - prepareFLEXPART(args, c) - normalexit(c) + c.outputdir = os.path.join(called_from_dir, c.outputdir) + get_mars_data(args, c) + prepare_flexpart(args, c) + normal_exit(c) # on ECMWF server else: submit(args.job_template, c, args.queue) @@ -138,11 +130,10 @@ def submit(jtemplate, c, queue): insert_point = lftext.index('EOF') # put all parameters of ControlFile instance into a list - clist = c.tolist() + clist = c.to_list() # ondemand colist = [] # operational mt = 0 -#AP wieso 2 for loops? for elem in clist: if 'maxstep' in elem: mt = int(elem.split(' ')[1]) @@ -151,8 +142,7 @@ def submit(jtemplate, c, queue): if 'start_date' in elem: elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' if 'end_date' in elem: -#AP Fehler?! Muss end_date heissen - elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' + elem = 'end_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' if 'base_time' in elem: elem = 'base_time ' + '${MSJ_BASETIME}' if 'time' in elem and mt > 24: @@ -172,11 +162,13 @@ def submit(jtemplate, c, queue): try: p = subprocess.check_call(['ecaccess-job-submit', '-queueName', queue, 'job.ksh']) - except: - print('ecaccess-job-submit failed, probably eccert has expired') + except subprocess.CalledProcessError as e: + print 'ecaccess-job-submit failed!' + print 'Error Message: ' + print e.output exit(1) - print('You should get an email with subject flex.hostname.pid') + print 'You should get an email with subject flex.hostname.pid' return diff --git a/python/testsuite.py b/python/test_suite.py similarity index 86% rename from python/testsuite.py rename to python/test_suite.py index 199fa89adc8cca15cd6673554ba71d63ffbfe186..c20b5f8ccd7afdb90fa33e0a225f45956d72c798 100755 --- a/python/testsuite.py +++ b/python/test_suite.py @@ -1,11 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP -# +# ToDo AP # - provide more tests # - provide more documentation -# - #************************************************************************ #******************************************************************************* @@ -27,7 +25,7 @@ # # @Program Functionality: # This script triggers the ECMWFDATA test suite. Call with -# testsuite.py [test group] +# test_suite.py [test group] # # @Program Content: # @@ -45,9 +43,9 @@ import subprocess # PROGRAM # ------------------------------------------------------------------------------ try: - taskfile = open('testsuite.json') -except: - print 'could not open suite definition file testsuite.json' + taskfile = open('test_suite.json') +except IOError: + print 'could not open suite definition file test_suite.json' exit() if not os.path.isfile('../src/CONVERT2'): @@ -72,14 +70,14 @@ jobfailed = 0 for g in groups: try: tk, tv = g, tasks[g] - except: - continue + finally: + pass garglist = [] for ttk, ttv in tv.iteritems(): if isinstance(ttv, basestring): if ttk != 'script': garglist.append('--' + ttk) - if '$' == ttv[0]: + if ttv[0] == '$': garglist.append(os.path.expandvars(ttv)) else: garglist.append(ttv) @@ -88,11 +86,11 @@ for g in groups: arglist = [] for tttk, tttv in ttv.iteritems(): if isinstance(tttv, basestring): - arglist.append('--' + tttk) - if '$' in tttv[0]: - arglist.append(os.path.expandvars(tttv)) - else: - arglist.append(tttv) + arglist.append('--' + tttk) + if '$' in tttv[0]: + arglist.append(os.path.expandvars(tttv)) + else: + arglist.append(tttv) print 'Command: ', ' '.join([tv['script']] + garglist + arglist) o = '../test/' + tk + '_' + ttk + '_' + '_'.join(ttv.keys()) print 'Output will be sent to ', o @@ -100,7 +98,7 @@ for g in groups: try: p = subprocess.check_call([tv['script']] + garglist + arglist, stdout=f, stderr=f) - except: + except subprocess.CalledProcessError as e: f.write('\nFAILED\n') print 'FAILED' jobfailed += 1 @@ -110,5 +108,3 @@ for g in groups: print 'Test suite tasks completed' print str(jobcounter-jobfailed) + ' successful, ' + str(jobfailed) + ' failed' print 'If tasks have been submitted via ECACCESS please check emails' - -#print tasks diff --git a/python/Tools.py b/python/tools.py similarity index 78% rename from python/Tools.py rename to python/tools.py index 24e9e020dd8bd15b6d5f02cc2aec3a4acd28e5d2..b74fd685609c04e291849870a0c64f759a79220d 100644 --- a/python/Tools.py +++ b/python/tools.py @@ -1,10 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- #************************************************************************ -# TODO AP -# - check myerror -# - check normalexit -# - check getListAsString +# ToDo AP +# - check my_error +# - check normal_exit +# - check get_list_as_string # - seperate args and control interpretation #************************************************************************ #******************************************************************************* @@ -14,17 +14,17 @@ # # @Change History: # October 2014 - Anne Fouilloux (University of Oslo) -# - created functions silentremove and product (taken from ECMWF) +# - created functions silent_remove and product (taken from ECMWF) # # November 2015 - Leopold Haimberger (University of Vienna) -# - created functions: interpret_args_and_control, cleanup -# myerror, normalexit, init128, toparamId +# - created functions: interpret_args_and_control, clean_up +# my_error, normal_exit, init128, to_param_id # # April 2018 - Anne Philipp (University of Vienna): # - applied PEP8 style guide # - added documentation -# - moved all functions from file FlexpartTools to this file Tools -# - added function getListAsString +# - moved all functions from file Flexparttools to this file tools +# - added function get_list_as_string # # @License: # (C) Copyright 2014-2018. @@ -38,14 +38,14 @@ # # @Module Content: # - interpret_args_and_control -# - cleanup -# - myerror -# - normalexit +# - clean_up +# - my_error +# - normal_exit # - product -# - silentremove +# - silent_remove # - init128 -# - toparamId -# - getListAsString +# - to_param_id +# - get_list_as_string # #******************************************************************************* @@ -56,10 +56,11 @@ import os import errno import sys import glob +import inspect +import subprocess import traceback -from numpy import * -from gribapi import * from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +import numpy as np # software specific class from flex_extract from ControlFile import ControlFile @@ -122,13 +123,13 @@ def interpret_args_and_control(): help="root directory for storing output files") parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts", help="FLEXPART root directory (to find grib2flexpart \ - and COMMAND file)\n\ Normally ECMWFDATA resides in \ + and COMMAND file)\n Normally ECMWFDATA resides in \ the scripts directory of the FLEXPART distribution") - # this is only used by prepareFLEXPART.py to rerun a postprocessing step + # this is only used by prepare_flexpart.py to rerun a postprocessing step parser.add_argument("--ppid", dest="ppid", help="Specify parent process id for \ - rerun of prepareFLEXPART") + rerun of prepare_flexpart") # arguments for job submission to ECMWF, only needed by submit.py parser.add_argument("--job_template", dest='job_template', @@ -151,20 +152,22 @@ def interpret_args_and_control(): c = ControlFile(args.controlfile) except IOError: try: - c = ControlFile(localpythonpath + args.controlfile) - except: - print('Could not read CONTROL file "' + args.controlfile + '"') - print('Either it does not exist or its syntax is wrong.') - print('Try "' + sys.argv[0].split('/')[-1] + - ' -h" to print usage information') + LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath( + inspect.getfile(inspect.currentframe()))) + c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile) + except IOError: + print 'Could not read CONTROL file "' + args.controlfile + '"' + print 'Either it does not exist or its syntax is wrong.' + print 'Try "' + sys.argv[0].split('/')[-1] + \ + ' -h" to print usage information' exit(1) # check for having at least a starting date if args.start_date is None and getattr(c, 'start_date') is None: - print('start_date specified neither in command line nor \ - in CONTROL file ' + args.controlfile) - print('Try "' + sys.argv[0].split('/')[-1] + - ' -h" to print usage information') + print 'start_date specified neither in command line nor \ + in CONTROL file ' + args.controlfile + print 'Try "' + sys.argv[0].split('/')[-1] + \ + ' -h" to print usage information' exit(1) # save all existing command line parameter to the ControlFile instance @@ -205,8 +208,8 @@ def interpret_args_and_control(): afloat = '.' in args.area l = args.area.split('/') if afloat: - for i in range(len(l)): - l[i] = str(int(float(l[i]) * 1000)) + for i, item in enumerate(l): + item = str(int(float(item) * 1000)) c.upper, c.left, c.lower, c.right = l # NOTE: basetime activates the ''operational mode'' @@ -217,11 +220,11 @@ def interpret_args_and_control(): l = args.step.split('/') if 'to' in args.step.lower(): if 'by' in args.step.lower(): - ilist = arange(int(l[0]), int(l[2]) + 1, int(l[4])) + ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4])) c.step = ['{:0>3}'.format(i) for i in ilist] else: - myerror(None, args.step + ':\n' + - 'please use "by" as well if "to" is used') + my_error(None, args.step + ':\n' + + 'please use "by" as well if "to" is used') else: c.step = l @@ -238,7 +241,7 @@ def interpret_args_and_control(): return args, c -def cleanup(c): +def clean_up(c): ''' @Description: Remove all files from intermediate directory @@ -262,21 +265,21 @@ def cleanup(c): <nothing> ''' - print("cleanup") + print "clean_up" cleanlist = glob.glob(c.inputdir + "/*") for cl in cleanlist: if c.prefix not in cl: - silentremove(cl) + silent_remove(cl) if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'): - silentremove(cl) + silent_remove(cl) - print("Done") + print "Done" return -def myerror(c, message='ERROR'): +def my_error(c, message='ERROR'): ''' @Description: Prints a specified error message which can be passed to the function @@ -302,32 +305,34 @@ def myerror(c, message='ERROR'): @Return: <nothing> ''' - # uncomment if user wants email notification directly from python - #try: - #target = c.mailfail - #except AttributeError: - #target = os.getenv('USER') - - #if(type(target) is not list): - #target = [target] - - print(message) - - # uncomment if user wants email notification directly from python - #for t in target: - #p = subprocess.Popen(['mail','-s ECMWFDATA v7.0 ERROR', os.path.expandvars(t)], - # stdin = subprocess.PIPE, stdout = subprocess.PIPE, - # stderr = subprocess.PIPE, bufsize = 1) - #tr = '\n'.join(traceback.format_stack()) - #pout = p.communicate(input = message+'\n\n'+tr)[0] - #print 'Email sent to '+os.path.expandvars(t) # +' '+pout.decode() + + print message + + # comment if user does not want email notification directly from python + try: + target = [] + target.extend(c.mailfail) + except AttributeError: + target = [] + target.extend(os.getenv('USER')) + + for t in target: + p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 ERROR', + os.path.expandvars(t)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=1) + tr = '\n'.join(traceback.format_stack()) + pout = p.communicate(input=message + '\n\n' + tr)[0] + print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode() exit(1) return -def normalexit(c, message='Done!'): +def normal_exit(c, message='Done!'): ''' @Description: Prints a specific exit message which can be passed to the function. @@ -353,23 +358,23 @@ def normalexit(c, message='Done!'): <nothing> ''' - # Uncomment if user wants notification directly from python - #try: - #target = c.mailops - #if(type(target) is not list): - #target = [target] - #for t in target: - #p = subprocess.Popen(['mail','-s ECMWFDATA v7.0 normal exit', - # os.path.expandvars(t)], - # stdin = subprocess.PIPE, - # stdout = subprocess.PIPE, - # stderr = subprocess.PIPE, bufsize = 1) - #pout = p.communicate(input = message+'\n\n')[0] - #print pout.decode() - #except: - #pass - - print(message) + print message + + # comment if user does not want notification directly from python + try: + target = [] + target.extend(c.mailops) + for t in target: + p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 normal exit', + os.path.expandvars(t)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=1) + pout = p.communicate(input=message+'\n\n')[0] + print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode() + finally: + pass return @@ -410,7 +415,7 @@ def product(*args, **kwds): return -def silentremove(filename): +def silent_remove(filename): ''' @Description: If "filename" exists , it is removed. @@ -434,13 +439,13 @@ def silentremove(filename): return -def init128(fn): +def init128(filepath): ''' @Description: Opens and reads the grib file with table 128 information. @Input: - fn: string + filepath: string Path to file of ECMWF grib table number 128. @Return: @@ -450,7 +455,7 @@ def init128(fn): short name of the parameter. ''' table128 = dict() - with open(fn) as f: + with open(filepath) as f: fdata = f.read().split('\n') for data in fdata: if data[0] != '!': @@ -459,7 +464,7 @@ def init128(fn): return table128 -def toparamId(pars, table): +def to_param_id(pars, table): ''' @Description: Transform parameter names to parameter ids @@ -485,31 +490,33 @@ def toparamId(pars, table): cpar = pars.upper().split('/') ipar = [] for par in cpar: - found = False for k, v in table.iteritems(): if par == k or par == v: ipar.append(int(k)) - found = True break - if found is False: - print('Warning: par ' + par + ' not found in table 128') + else: + print 'Warning: par ' + par + ' not found in table 128' return ipar -def getListAsString(listObj): +def get_list_as_string(list_obj, concatenate_sign=', '): ''' @Description: Converts a list of arbitrary content into a single string. @Input: - listObj: list + list_obj: list A list with arbitrary content. + concatenate_sign: string, optional + A string which is used to concatenate the single + list elements. Default value is ", ". + @Return: - strOfList: string + str_of_list: string The content of the list as a single string. ''' - strOfList = ", ".join( str(l) for l in listObj) + str_of_list = concatenate_sign.join(str(l) for l in list_obj) - return strOfList + return str_of_list