From c77630a97116cfe48788a474eaf76b1887d1479f Mon Sep 17 00:00:00 2001 From: Anne Philipp <anne.philipp@univie.ac.at> Date: Tue, 13 Aug 2019 14:32:22 +0200 Subject: [PATCH] BUGFIX: for python3-eccodes the file openings need to be in binary mode! added binary marker in filemode --- Source/Python/Classes/EcFlexpart.py | 26 +++++++++--------- Source/Python/Classes/GribUtil.py | 37 +++++++++----------------- Source/Python/Classes/MarsRetrieval.py | 4 +-- Source/Python/Mods/tools.py | 2 +- 4 files changed, 29 insertions(+), 40 deletions(-) diff --git a/Source/Python/Classes/EcFlexpart.py b/Source/Python/Classes/EcFlexpart.py index a41442a..85c458c 100644 --- a/Source/Python/Classes/EcFlexpart.py +++ b/Source/Python/Classes/EcFlexpart.py @@ -1093,9 +1093,9 @@ class EcFlexpart(object): t_dt.strftime('%Y%m%d%H') + numbersuffix) print("outputfile = " + fnout) - f_handle = open(fnout, 'w') - h_handle = open(hnout, 'w') - g_handle = open(gnout, 'w') + f_handle = open(fnout, 'wb') + h_handle = open(hnout, 'wb') + g_handle = open(gnout, 'wb') # read message for message and store relevant data fields, where # data keywords are stored in pars @@ -1401,13 +1401,13 @@ class EcFlexpart(object): # write original time step to flux file as usual fluxfile = GribUtil(os.path.join(c.inputdir, fluxfilename)) - fluxfile.set_keys(tmpfile, filemode='a', strict=True, + fluxfile.set_keys(tmpfile, filemode='ab', strict=True, wherekeynames=['paramId'], wherekeyvalues=[142], keynames=['perturbationNumber','date','time','stepRange','values'], keyvalues=[inumb, int(date.strftime('%Y%m%d')), date.hour*100, 0, lsp_new_np[inumb,:,it]], ) - fluxfile.set_keys(tmpfile, filemode='a', strict=True, + fluxfile.set_keys(tmpfile, filemode='ab', strict=True, wherekeynames=['paramId'], wherekeyvalues=[143], keynames=['perturbationNumber','date','time','stepRange','values'], keyvalues=[inumb,int(date.strftime('%Y%m%d')), @@ -1415,13 +1415,13 @@ class EcFlexpart(object): ) # rr for first subgrid point is identified by step = 1 - fluxfile.set_keys(tmpfile, filemode='a', strict=True, + fluxfile.set_keys(tmpfile, filemode='ab', strict=True, wherekeynames=['paramId'], wherekeyvalues=[142], keynames=['perturbationNumber','date','time','stepRange','values'], keyvalues=[inumb,int(date.strftime('%Y%m%d')), date.hour*100, '1', lsp_new_np[inumb,:,it+1]] ) - fluxfile.set_keys(tmpfile, filemode='a', strict=True, + fluxfile.set_keys(tmpfile, filemode='ab', strict=True, wherekeynames=['paramId'], wherekeyvalues=[143], keynames=['perturbationNumber','date','time','stepRange','values'], keyvalues=[inumb,int(date.strftime('%Y%m%d')), @@ -1429,13 +1429,13 @@ class EcFlexpart(object): ) # rr for second subgrid point is identified by step = 2 - fluxfile.set_keys(tmpfile, filemode='a', strict=True, + fluxfile.set_keys(tmpfile, filemode='ab', strict=True, wherekeynames=['paramId'], wherekeyvalues=[142], keynames=['perturbationNumber','date','time','stepRange','values'], keyvalues=[inumb,int(date.strftime('%Y%m%d')), date.hour*100, '2', lsp_new_np[inumb,:,it+2]] ) - fluxfile.set_keys(tmpfile, filemode='a', strict=True, + fluxfile.set_keys(tmpfile, filemode='ab', strict=True, wherekeynames=['paramId'], wherekeyvalues=[143], keynames=['perturbationNumber','date','time','stepRange','values'], keyvalues=[inumb,int(date.strftime('%Y%m%d')), @@ -1465,10 +1465,10 @@ class EcFlexpart(object): gribfile = GribUtil(os.path.join(inputdir,'rr_grib_dummy.grb')) gribfile.copy_dummy_msg(ifile, keynames=['paramId'], - keyvalues=[142], filemode='w') + keyvalues=[142], filemode='wb') gribfile.copy_dummy_msg(ifile, keynames=['paramId'], - keyvalues=[143], filemode='a') + keyvalues=[143], filemode='ab') return @@ -1572,7 +1572,7 @@ class EcFlexpart(object): for k, f in fdict.items(): fortfile = os.path.join(c.inputdir, 'fort.' + k) silent_remove(fortfile) - fdict[k] = open(fortfile, 'w') + fdict[k] = open(fortfile, 'wb') #============================================================================================ # create correct timestamp from the three time informations cdate = str(codes_get(gid, 'date')) @@ -1607,7 +1607,7 @@ class EcFlexpart(object): #if c.wrf: # if 'olddate' not in locals() or cdate != olddate: # fwrf = open(os.path.join(c.outputdir, - # 'WRF' + cdate + '.' + ctime + '.000.grb2'), 'w') + # 'WRF' + cdate + '.' + ctime + '.000.grb2'), 'wb') # olddate = cdate[:] #============================================================================================ # savedfields remembers which fields were already used. diff --git a/Source/Python/Classes/GribUtil.py b/Source/Python/Classes/GribUtil.py index 68e8746..2a895d6 100644 --- a/Source/Python/Classes/GribUtil.py +++ b/Source/Python/Classes/GribUtil.py @@ -90,15 +90,11 @@ class GribUtil(object): from eccodes import (codes_new_from_file, codes_is_defined, codes_get, codes_release) - fileid = open(self.filenames, 'r') - return_list = [] - while 1: - gid = codes_new_from_file(fileid) + with open(self.filenames, 'rb') as fileid: - if gid is None: - break + gid = codes_new_from_file(fileid) if len(wherekeynames) != len(wherekeyvalues): raise Exception("Number of key values and key names must be \ @@ -122,13 +118,11 @@ class GribUtil(object): codes_release(gid) - fileid.close() - return return_list def set_keys(self, fromfile, keynames, keyvalues, wherekeynames=[], - wherekeyvalues=[], strict=False, filemode='w'): + wherekeyvalues=[], strict=False, filemode='wb'): '''Opens the file to read the grib messages and then write the selected messages (with wherekeys) to a new output file. Also, the keyvalues of the passed list of keynames are set. @@ -160,7 +154,7 @@ class GribUtil(object): meeting the where statement (True). Default is False. filemode : :obj:`string`, optional - Sets the mode for the output file. Default is "w". + Sets the mode for the output file. Default is "wb". Return ------ @@ -174,14 +168,10 @@ class GribUtil(object): raise Exception("Give a value for each keyname!") fout = open(self.filenames, filemode) - fin = open(fromfile) - while 1: + with open(fromfile, 'rb') as fin: gid = codes_grib_new_from_file(fin) - if gid is None: - break - select = True i = 0 for wherekey in wherekeynames: @@ -205,13 +195,12 @@ class GribUtil(object): codes_release(gid) - fin.close() fout.close() return def copy_dummy_msg(self, filename_in, selectWhere=True, - keynames=[], keyvalues=[], filemode='w'): + keynames=[], keyvalues=[], filemode='wb'): '''Add the content of another input grib file to the objects file but only messages corresponding to keys/values passed to the function. The selectWhere switch decides if to copy the keys equal to (True) or @@ -234,7 +223,7 @@ class GribUtil(object): List of keyvalues. Default is an empty list. filemode : :obj:`string`, optional - Sets the mode for the output file. Default is "w". + Sets the mode for the output file. Default is "wb". Return ------ @@ -246,16 +235,17 @@ class GribUtil(object): if len(keynames) != len(keyvalues): raise Exception("Give a value for each keyname!") - fin = open(filename_in, 'rb') + fout = open(self.filenames, filemode) fields = 0 - while fields < 1: - gid = codes_grib_new_from_file(fin) + with open(filename_in, 'rb') as fin: + if fields >= 1: + fout.close() + return - if gid is None: - break + gid = codes_grib_new_from_file(fin) select = True i = 0 @@ -277,7 +267,6 @@ class GribUtil(object): codes_release(gid) - fin.close() fout.close() return diff --git a/Source/Python/Classes/MarsRetrieval.py b/Source/Python/Classes/MarsRetrieval.py index 5fe2211..d68c183 100644 --- a/Source/Python/Classes/MarsRetrieval.py +++ b/Source/Python/Classes/MarsRetrieval.py @@ -487,7 +487,7 @@ class MarsRetrieval(object): for key in empty_keys: del attrs[key] - attrs['ppengine'] = 'emos' +# attrs['ppengine'] = 'emos' # MARS request via Python script if self.server: @@ -519,7 +519,7 @@ class MarsRetrieval(object): for key, value in attrs.items(): request_str = request_str + ',' + key + '=' + str(value) request_str += ',target="' + target + '"' - p = subprocess.Popen(['mars', '-e'], + p = subprocess.Popen(['mars'], #'-e'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/Source/Python/Mods/tools.py b/Source/Python/Mods/tools.py index 8a94f58..078c77e 100644 --- a/Source/Python/Mods/tools.py +++ b/Source/Python/Mods/tools.py @@ -762,7 +762,7 @@ def get_informations(filename): # --- open file --- print("Opening file for getting information data --- %s" % filename) - with open(filename) as f: + with open(filename, 'rb') as f: # load first message from file gid = codes_grib_new_from_file(f) -- GitLab