From c77630a97116cfe48788a474eaf76b1887d1479f Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Tue, 13 Aug 2019 14:32:22 +0200
Subject: [PATCH] BUGFIX: for python3-eccodes the file openings need to be in
 binary mode! added binary marker in filemode

---
 Source/Python/Classes/EcFlexpart.py    | 26 +++++++++---------
 Source/Python/Classes/GribUtil.py      | 37 +++++++++-----------------
 Source/Python/Classes/MarsRetrieval.py |  4 +--
 Source/Python/Mods/tools.py            |  2 +-
 4 files changed, 29 insertions(+), 40 deletions(-)

diff --git a/Source/Python/Classes/EcFlexpart.py b/Source/Python/Classes/EcFlexpart.py
index a41442a..85c458c 100644
--- a/Source/Python/Classes/EcFlexpart.py
+++ b/Source/Python/Classes/EcFlexpart.py
@@ -1093,9 +1093,9 @@ class EcFlexpart(object):
                                      t_dt.strftime('%Y%m%d%H') + numbersuffix)
 
             print("outputfile = " + fnout)
-            f_handle = open(fnout, 'w')
-            h_handle = open(hnout, 'w')
-            g_handle = open(gnout, 'w')
+            f_handle = open(fnout, 'wb')
+            h_handle = open(hnout, 'wb')
+            g_handle = open(gnout, 'wb')
 
             # read message for message and store relevant data fields, where
             # data keywords are stored in pars
@@ -1401,13 +1401,13 @@ class EcFlexpart(object):
 
             # write original time step to flux file as usual
             fluxfile = GribUtil(os.path.join(c.inputdir, fluxfilename))
-            fluxfile.set_keys(tmpfile, filemode='a', strict=True,
+            fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
                               wherekeynames=['paramId'], wherekeyvalues=[142],
                               keynames=['perturbationNumber','date','time','stepRange','values'],
                               keyvalues=[inumb, int(date.strftime('%Y%m%d')),
                                          date.hour*100, 0, lsp_new_np[inumb,:,it]],
                              )
-            fluxfile.set_keys(tmpfile, filemode='a', strict=True,
+            fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
                               wherekeynames=['paramId'], wherekeyvalues=[143],
                               keynames=['perturbationNumber','date','time','stepRange','values'],
                               keyvalues=[inumb,int(date.strftime('%Y%m%d')),
@@ -1415,13 +1415,13 @@ class EcFlexpart(object):
                              )
 
             # rr for first subgrid point is identified by step = 1
-            fluxfile.set_keys(tmpfile, filemode='a', strict=True,
+            fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
                               wherekeynames=['paramId'], wherekeyvalues=[142],
                               keynames=['perturbationNumber','date','time','stepRange','values'],
                               keyvalues=[inumb,int(date.strftime('%Y%m%d')),
                                          date.hour*100, '1', lsp_new_np[inumb,:,it+1]]
                               )
-            fluxfile.set_keys(tmpfile, filemode='a', strict=True,
+            fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
                               wherekeynames=['paramId'], wherekeyvalues=[143],
                               keynames=['perturbationNumber','date','time','stepRange','values'],
                               keyvalues=[inumb,int(date.strftime('%Y%m%d')),
@@ -1429,13 +1429,13 @@ class EcFlexpart(object):
                               )
 
             # rr for second subgrid point is identified by step = 2
-            fluxfile.set_keys(tmpfile, filemode='a', strict=True,
+            fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
                               wherekeynames=['paramId'], wherekeyvalues=[142],
                               keynames=['perturbationNumber','date','time','stepRange','values'],
                               keyvalues=[inumb,int(date.strftime('%Y%m%d')),
                                          date.hour*100, '2', lsp_new_np[inumb,:,it+2]]
                               )
-            fluxfile.set_keys(tmpfile, filemode='a', strict=True,
+            fluxfile.set_keys(tmpfile, filemode='ab', strict=True,
                               wherekeynames=['paramId'], wherekeyvalues=[143],
                               keynames=['perturbationNumber','date','time','stepRange','values'],
                               keyvalues=[inumb,int(date.strftime('%Y%m%d')),
@@ -1465,10 +1465,10 @@ class EcFlexpart(object):
         gribfile = GribUtil(os.path.join(inputdir,'rr_grib_dummy.grb'))
 
         gribfile.copy_dummy_msg(ifile, keynames=['paramId'],
-                      keyvalues=[142], filemode='w')
+                      keyvalues=[142], filemode='wb')
 
         gribfile.copy_dummy_msg(ifile, keynames=['paramId'],
-                      keyvalues=[143], filemode='a')
+                      keyvalues=[143], filemode='ab')
 
         return
 
@@ -1572,7 +1572,7 @@ class EcFlexpart(object):
             for k, f in fdict.items():
                 fortfile = os.path.join(c.inputdir, 'fort.' + k)
                 silent_remove(fortfile)
-                fdict[k] = open(fortfile, 'w')
+                fdict[k] = open(fortfile, 'wb')
 #============================================================================================
             # create correct timestamp from the three time informations
             cdate = str(codes_get(gid, 'date'))
@@ -1607,7 +1607,7 @@ class EcFlexpart(object):
             #if c.wrf:
             #    if 'olddate' not in locals() or cdate != olddate:
             #        fwrf = open(os.path.join(c.outputdir,
-            #                    'WRF' + cdate + '.' + ctime + '.000.grb2'), 'w')
+            #                    'WRF' + cdate + '.' + ctime + '.000.grb2'), 'wb')
             #        olddate = cdate[:]
 #============================================================================================
             # savedfields remembers which fields were already used.
diff --git a/Source/Python/Classes/GribUtil.py b/Source/Python/Classes/GribUtil.py
index 68e8746..2a895d6 100644
--- a/Source/Python/Classes/GribUtil.py
+++ b/Source/Python/Classes/GribUtil.py
@@ -90,15 +90,11 @@ class GribUtil(object):
         from eccodes import (codes_new_from_file, codes_is_defined, codes_get,
                              codes_release)
 
-        fileid = open(self.filenames, 'r')
-
         return_list = []
 
-        while 1:
-            gid = codes_new_from_file(fileid)
+        with open(self.filenames, 'rb') as fileid:
 
-            if gid is None:
-                break
+            gid = codes_new_from_file(fileid)
 
             if len(wherekeynames) != len(wherekeyvalues):
                 raise Exception("Number of key values and key names must be \
@@ -122,13 +118,11 @@ class GribUtil(object):
 
             codes_release(gid)
 
-        fileid.close()
-
         return return_list
 
 
     def set_keys(self, fromfile, keynames, keyvalues, wherekeynames=[],
-                 wherekeyvalues=[], strict=False, filemode='w'):
+                 wherekeyvalues=[], strict=False, filemode='wb'):
         '''Opens the file to read the grib messages and then write
         the selected messages (with wherekeys) to a new output file.
         Also, the keyvalues of the passed list of keynames are set.
@@ -160,7 +154,7 @@ class GribUtil(object):
             meeting the where statement (True). Default is False.
 
         filemode : :obj:`string`, optional
-            Sets the mode for the output file. Default is "w".
+            Sets the mode for the output file. Default is "wb".
 
         Return
         ------
@@ -174,14 +168,10 @@ class GribUtil(object):
             raise Exception("Give a value for each keyname!")
 
         fout = open(self.filenames, filemode)
-        fin = open(fromfile)
 
-        while 1:
+        with open(fromfile, 'rb') as fin:
             gid = codes_grib_new_from_file(fin)
 
-            if gid is None:
-                break
-
             select = True
             i = 0
             for wherekey in wherekeynames:
@@ -205,13 +195,12 @@ class GribUtil(object):
 
             codes_release(gid)
 
-        fin.close()
         fout.close()
 
         return
 
     def copy_dummy_msg(self, filename_in, selectWhere=True,
-                 keynames=[], keyvalues=[], filemode='w'):
+                 keynames=[], keyvalues=[], filemode='wb'):
         '''Add the content of another input grib file to the objects file but
         only messages corresponding to keys/values passed to the function.
         The selectWhere switch decides if to copy the keys equal to (True) or
@@ -234,7 +223,7 @@ class GribUtil(object):
             List of keyvalues. Default is an empty list.
 
         filemode : :obj:`string`, optional
-            Sets the mode for the output file. Default is "w".
+            Sets the mode for the output file. Default is "wb".
 
         Return
         ------
@@ -246,16 +235,17 @@ class GribUtil(object):
         if len(keynames) != len(keyvalues):
             raise Exception("Give a value for each keyname!")
 
-        fin = open(filename_in, 'rb')
+
         fout = open(self.filenames, filemode)
 
         fields = 0
 
-        while fields < 1:
-            gid = codes_grib_new_from_file(fin)
+        with open(filename_in, 'rb') as fin:
+            if fields >= 1:
+                fout.close()
+                return
 
-            if gid is None:
-                break
+            gid = codes_grib_new_from_file(fin)
 
             select = True
             i = 0
@@ -277,7 +267,6 @@ class GribUtil(object):
 
             codes_release(gid)
 
-        fin.close()
         fout.close()
 
         return
diff --git a/Source/Python/Classes/MarsRetrieval.py b/Source/Python/Classes/MarsRetrieval.py
index 5fe2211..d68c183 100644
--- a/Source/Python/Classes/MarsRetrieval.py
+++ b/Source/Python/Classes/MarsRetrieval.py
@@ -487,7 +487,7 @@ class MarsRetrieval(object):
         for key in empty_keys:
             del attrs[key]
 
-        attrs['ppengine'] = 'emos'
+#        attrs['ppengine'] = 'emos'
 
         # MARS request via Python script
         if self.server:
@@ -519,7 +519,7 @@ class MarsRetrieval(object):
             for key, value in attrs.items():
                 request_str = request_str + ',' + key + '=' + str(value)
             request_str += ',target="' + target + '"'
-            p = subprocess.Popen(['mars', '-e'],
+            p = subprocess.Popen(['mars'], #'-e'],
                                  stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
diff --git a/Source/Python/Mods/tools.py b/Source/Python/Mods/tools.py
index 8a94f58..078c77e 100644
--- a/Source/Python/Mods/tools.py
+++ b/Source/Python/Mods/tools.py
@@ -762,7 +762,7 @@ def get_informations(filename):
 
     # --- open file ---
     print("Opening file for getting information data --- %s" % filename)
-    with open(filename) as f:
+    with open(filename, 'rb') as f:
         # load first message from file
         gid = codes_grib_new_from_file(f)
 
-- 
GitLab