From 79729d5dfa465f4ecb605e2c76c10d614028dc21 Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Fri, 8 Mar 2019 17:07:28 +0100
Subject: [PATCH] switched from python2 to python3
---
source/python/__init__.py | 2 +-
source/python/_config.py | 2 +-
source/python/classes/ControlFile.py | 6 +++---
source/python/classes/EcFlexpart.py | 23 +++++++++++++----------
source/python/classes/GribUtil.py | 2 +-
source/python/classes/MarsRetrieval.py | 10 +++++-----
source/python/classes/UioFiles.py | 2 +-
source/python/classes/__init__.py | 2 +-
source/python/install.py | 20 ++++++--------------
source/python/mods/__init__.py | 2 +-
source/python/mods/checks.py | 4 ++--
source/python/mods/disaggregation.py | 2 +-
source/python/mods/get_mars_data.py | 4 ++--
source/python/mods/prepare_flexpart.py | 2 +-
source/python/mods/profiling.py | 4 ++--
source/python/mods/tools.py | 14 +++++++-------
source/python/submit.py | 4 ++--
templates/compilejob.template | 10 ++++++----
templates/job.temp | 10 ++++++----
templates/job.template | 10 ++++++----
20 files changed, 68 insertions(+), 67 deletions(-)
diff --git a/source/python/__init__.py b/source/python/__init__.py
index 4609bf2..c44e323 100644
--- a/source/python/__init__.py
+++ b/source/python/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
diff --git a/source/python/_config.py b/source/python/_config.py
index 9354d08..6a9aee1 100644
--- a/source/python/_config.py
+++ b/source/python/_config.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
diff --git a/source/python/classes/ControlFile.py b/source/python/classes/ControlFile.py
index f6c4353..48026ad 100644
--- a/source/python/classes/ControlFile.py
+++ b/source/python/classes/ControlFile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
@@ -552,7 +552,7 @@ class ControlFile(object):
if args_dict[k] != None}
# assign all passed command line arguments to ControlFile instance
- for k, v in arguments.iteritems():
+ for k, v in arguments.items():
setattr(self, str(k), v)
return
@@ -572,7 +572,7 @@ class ControlFile(object):
'''
- for k, v in envs.iteritems():
+ for k, v in envs.items():
setattr(self, str(k).lower(), str(v))
return
diff --git a/source/python/classes/EcFlexpart.py b/source/python/classes/EcFlexpart.py
index 8213bf2..c52aa68 100644
--- a/source/python/classes/EcFlexpart.py
+++ b/source/python/classes/EcFlexpart.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
@@ -277,7 +277,7 @@ class EcFlexpart(object):
'''
i = 0
for ty, st, ti in zip(ftype, fstep, ftime):
- btlist = range(len(ftime))
+ btlist = list(range(len(ftime)))
if self.basetime == 12:
btlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
if self.basetime == 0:
@@ -397,8 +397,8 @@ class EcFlexpart(object):
if not gauss and eta:
self.params['OG__ML'][0] += '/U/V/ETADOT'
elif gauss and not eta:
- self.params['GG__SL'] = ['Q', 'ML', '1', \
- '{}'.format((int(self.resol) + 1) / 2)]
+ self.params['GG__SL'] = ['Q', 'ML', '1',
+ '{}'.format((int(self.resol) + 1) // 2)]
self.params['SH__ML'] = ['U/V/D', 'ML', self.glevelist, 'OFF']
elif not gauss and not eta:
self.params['OG__ML'][0] += '/U/V'
@@ -407,9 +407,9 @@ class EcFlexpart(object):
'is a very costly parameter combination, '
'use this combination only for debugging!')
self.params['GG__SL'] = ['Q', 'ML', '1',
- '{}'.format((int(self.resol) + 1) / 2)]
+ '{}'.format((int(self.resol) + 1) // 2)]
self.params['GG__ML'] = ['U/V/D/ETADOT', 'ML', self.glevelist,
- '{}'.format((int(self.resol) + 1) / 2)]
+ '{}'.format((int(self.resol) + 1) // 2)]
if omega:
self.params['OG__ML'][0] += '/W'
@@ -667,7 +667,7 @@ class EcFlexpart(object):
for ftype in self.types:
# ftype contains field types such as
# [AN, FC, PF, CV]
- for pk, pv in self.params.iteritems():
+ for pk, pv in self.params.items():
# pk contains one of these keys of params
# [SH__ML, SH__SL, GG__ML, GG__SL, OG__ML, OG__SL,
# OG_OROLSM_SL, OG_acc_SL]
@@ -992,7 +992,7 @@ class EcFlexpart(object):
# create correct timestamp from the three time informations
cdate = str(codes_get(gid, 'date'))
- time = codes_get(gid, 'time')/100 # integer
+ time = codes_get(gid, 'time') // 100 # integer
step = codes_get(gid, 'step') # integer
ctime = '{:0>2}'.format(time)
cstep = '{:0>3}'.format(step)
@@ -1254,7 +1254,10 @@ class EcFlexpart(object):
# write to grib files (full/orig times to flux file and inbetween
# times into seperate end files)
print('... write disaggregated precipitation to files.')
+ # index variable of disaggregated fields
it = 0
+ # loop over times and write original time step and the two newly
+ # generated sub time steps for each loop
for date in date_list:
for step in step_list:
tmpfile = os.path.join(c.inputdir, 'rr_grib_dummy.grb')
@@ -1433,14 +1436,14 @@ class EcFlexpart(object):
#============================================================================================
# remove old fort.* files and open new ones
# they are just valid for a single product
- for k, f in fdict.iteritems():
+ for k, f in fdict.items():
fortfile = os.path.join(c.inputdir, 'fort.' + k)
silent_remove(fortfile)
fdict[k] = open(fortfile, 'w')
#============================================================================================
# create correct timestamp from the three time informations
cdate = str(codes_get(gid, 'date'))
- ctime = '{:0>2}'.format(codes_get(gid, 'time')/100)
+ ctime = '{:0>2}'.format(codes_get(gid, 'time') // 100)
cstep = '{:0>3}'.format(codes_get(gid, 'step'))
timestamp = datetime.strptime(cdate + ctime, '%Y%m%d%H')
timestamp += timedelta(hours=int(cstep))
diff --git a/source/python/classes/GribUtil.py b/source/python/classes/GribUtil.py
index dd99828..fdb25ea 100644
--- a/source/python/classes/GribUtil.py
+++ b/source/python/classes/GribUtil.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
diff --git a/source/python/classes/MarsRetrieval.py b/source/python/classes/MarsRetrieval.py
index 76f4a22..20f585f 100644
--- a/source/python/classes/MarsRetrieval.py
+++ b/source/python/classes/MarsRetrieval.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
@@ -433,7 +433,7 @@ class MarsRetrieval(object):
_config.FILE_MARS_REQUESTS), 'a') as f:
f.write(str(request_number) + ', ')
f.write(', '.join(str(attrs[key])
- for key in sorted(attrs.iterkeys())))
+ for key in sorted(attrs.keys())))
f.write('\n')
return
@@ -473,7 +473,7 @@ class MarsRetrieval(object):
# find all keys without a value and convert all other values to strings
empty_keys = []
- for key, value in attrs.iteritems():
+ for key, value in attrs.items():
if value == '':
empty_keys.append(str(key))
else:
@@ -510,7 +510,7 @@ class MarsRetrieval(object):
# MARS request via call in shell
else:
request_str = 'ret'
- for key, value in attrs.iteritems():
+ for key, value in attrs.items():
request_str = request_str + ',' + key + '=' + str(value)
request_str += ',target="' + target + '"'
p = subprocess.Popen(['mars', '-p'],
@@ -518,7 +518,7 @@ class MarsRetrieval(object):
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1)
- pout = p.communicate(input=request_str)[0]
+ pout = p.communicate(input=request_str.encode())[0]
print(pout.decode())
if 'Some errors reported' in pout.decode():
diff --git a/source/python/classes/UioFiles.py b/source/python/classes/UioFiles.py
index 3cca0df..4023d8b 100644
--- a/source/python/classes/UioFiles.py
+++ b/source/python/classes/UioFiles.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
diff --git a/source/python/classes/__init__.py b/source/python/classes/__init__.py
index 4609bf2..c44e323 100644
--- a/source/python/classes/__init__.py
+++ b/source/python/classes/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
diff --git a/source/python/install.py b/source/python/install.py
index 395d1ac..417329a 100755
--- a/source/python/install.py
+++ b/source/python/install.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
@@ -366,17 +366,9 @@ def mk_tarball(tarball_path, target):
if os.path.splitext(tarinfo.name)[1]
in exclude_files
else tarinfo)
- except subprocess.CalledProcessError as e:
- print('... ERROR CODE:\n ... ' + str(e.returncode))
- print('... ERROR MESSAGE:\n ... ' + str(e))
-
- sys.exit('\n... could not make installation tar ball!')
- except OSError as e:
- print('... ERROR CODE: ' + str(e.errno))
- print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
-
- sys.exit('\n... error occured while trying to read tar-file ' +
- str(tarball_path))
+ except tarfile.TarError as e:
+ sys.exit('\n... error occured while trying to create the tar-file ' +
+ str(tarball_path))
return
@@ -677,9 +669,9 @@ def mk_convert_build(src_path, makefile):
stderr=subprocess.PIPE,
bufsize=1)
pout, perr = p.communicate()
- print(pout)
+ print(pout.decode())
if p.returncode != 0:
- print(perr)
+ print(perr.decode())
print('Please edit ' + makefile +
' or try another Makefile in the src directory.')
print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '
diff --git a/source/python/mods/__init__.py b/source/python/mods/__init__.py
index 4609bf2..c44e323 100644
--- a/source/python/mods/__init__.py
+++ b/source/python/mods/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
diff --git a/source/python/mods/checks.py b/source/python/mods/checks.py
index 97c27b8..434a42a 100644
--- a/source/python/mods/checks.py
+++ b/source/python/mods/checks.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
@@ -30,9 +30,9 @@ try:
import exceptions
except ImportError:
import builtins as exceptions
-from .tools import my_error, silent_remove
from datetime import datetime
import numpy as np
+from .tools import my_error, silent_remove
# ------------------------------------------------------------------------------
# FUNCTIONS
# ------------------------------------------------------------------------------
diff --git a/source/python/mods/disaggregation.py b/source/python/mods/disaggregation.py
index 5407ca1..a5d36ca 100644
--- a/source/python/mods/disaggregation.py
+++ b/source/python/mods/disaggregation.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
diff --git a/source/python/mods/get_mars_data.py b/source/python/mods/get_mars_data.py
index 6dc9dd0..56e6b3c 100755
--- a/source/python/mods/get_mars_data.py
+++ b/source/python/mods/get_mars_data.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
@@ -175,7 +175,7 @@ def write_reqheader(marsfile):
del attrs['public']
with open(marsfile, 'w') as f:
f.write('request_number' + ', ')
- f.write(', '.join(str(key) for key in sorted(attrs.iterkeys())))
+ f.write(', '.join(str(key) for key in sorted(attrs.keys())))
f.write('\n')
return
diff --git a/source/python/mods/prepare_flexpart.py b/source/python/mods/prepare_flexpart.py
index b554944..0d6cfe8 100755
--- a/source/python/mods/prepare_flexpart.py
+++ b/source/python/mods/prepare_flexpart.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
diff --git a/source/python/mods/profiling.py b/source/python/mods/profiling.py
index a20cad6..c4c79d1 100644
--- a/source/python/mods/profiling.py
+++ b/source/python/mods/profiling.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#************************************************************************
# ToDo AP
@@ -65,7 +65,7 @@ def timefn(fn):
t1 = time.time()
result = fn(*args, **kwargs)
t2 = time.time()
- print("@timefn:" + fn.func_name + " took " + str(t2 - t1) + " seconds")
+ print("@timefn:" + fn.__name__ + " took " + str(t2 - t1) + " seconds")
return result
diff --git a/source/python/mods/tools.py b/source/python/mods/tools.py
index 2cd1ec9..f8eb0ab 100644
--- a/source/python/mods/tools.py
+++ b/source/python/mods/tools.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
@@ -367,7 +367,7 @@ def send_mail(users, success_mode, message):
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1)
- pout = p.communicate(input=message + '\n\n')[0]
+ pout = p.communicate(input=message.encode() + '\n\n')[0]
except ValueError as e:
print('... ERROR: ' + str(e))
sys.exit('... Email could not be sent!')
@@ -432,7 +432,7 @@ def product(*args, **kwds):
See example in description above.
'''
try:
- pools = map(tuple, args) * kwds.get('repeat', 1)
+ pools = [tuple(arg) for arg in args] * kwds.get('repeat', 1)
result = [[]]
for pool in pools:
result = [x + [y] for x in result for y in pool]
@@ -532,7 +532,7 @@ def to_param_id(pars, table):
cpar = pars.upper().split('/')
ipar = []
for par in cpar:
- for k, v in table.iteritems():
+ for k, v in table.items():
if par == k or par == v:
ipar.append(int(k))
break
@@ -573,7 +573,7 @@ def to_param_id_with_tablenumber(pars, table):
cpar = pars.upper().split('/')
spar = []
for par in cpar:
- for k, v in table.iteritems():
+ for k, v in table.items():
if par == k or par == v:
spar.append(k + '.128')
break
@@ -725,7 +725,7 @@ def submit_job_to_ecserver(target, jobname):
print('\n... Most likely the ECACCESS library is not available!')
sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
- return job_id
+ return job_id.decode()
def get_informations(filename):
@@ -826,7 +826,7 @@ def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date):
jy = info['Nj']
if not purefc:
- it = ((end_date - start_date).days + 1) * 24/int(dtime)
+ it = ((end_date - start_date).days + 1) * 24 // int(dtime)
else:
# #no of step * #no of times * #no of days
it = len(index_vals[2]) * len(index_vals[1]) * len(index_vals[0])
diff --git a/source/python/submit.py b/source/python/submit.py
index bd9f74e..488ac5e 100755
--- a/source/python/submit.py
+++ b/source/python/submit.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
@@ -67,7 +67,7 @@ from datetime import datetime, timedelta
# software specific classes and modules from flex_extract
import _config
from mods.tools import (setup_controldata, normal_exit, get_cmdline_args,
- submit_job_to_ecserver, read_ecenv)
+ submit_job_to_ecserver, read_ecenv)
from mods.get_mars_data import get_mars_data
from mods.prepare_flexpart import prepare_flexpart
from classes.ControlFile import ControlFile
diff --git a/templates/compilejob.template b/templates/compilejob.template
index f798477..a115b59 100644
--- a/templates/compilejob.template
+++ b/templates/compilejob.template
@@ -29,18 +29,20 @@ case $${HOST} in
*ecg*)
module unload grib_api
module unload eccodes
- module load eccodes
- module load python
+ module unload python
module unload emos
+ module load python3
+ module load eccodes/2.12.0
module load emos/455-r64
export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts
export MAKEFILE=$makefile
;;
*cca*)
+ module unload python
module switch PrgEnv-cray PrgEnv-intel
- module load eccodes
+ module load python3
+ module load eccodes/2.12.0
module load emos
- module load python
echo $${GROUP}
echo $${HOME}
echo $${HOME} | awk -F / '{print $1, $2, $3, $4}'
diff --git a/templates/job.temp b/templates/job.temp
index f0d8564..2ece17e 100644
--- a/templates/job.temp
+++ b/templates/job.temp
@@ -27,19 +27,21 @@ set -x
export VERSION=7.1
case $${HOST} in
*ecg*)
- module load python
module unload grib_api
module unload eccodes
- module load eccodes
+ module unload python
module unload emos
+ module load python3
+ module load eccodes/2.12.0
module load emos/455-r64
export PATH=$${PATH}:$${HOME}/flex_extract_v7.1/source/python
;;
*cca*)
+ module unload python
module switch PrgEnv-cray PrgEnv-intel
- module load eccodes
+ module load python3
+ module load eccodes/2.12.0
module load emos
- module load python
export SCRATCH=$${TMPDIR}
export PATH=$${PATH}:$${HOME}/flex_extract_v7.1/source/python
;;
diff --git a/templates/job.template b/templates/job.template
index 5f12a0b..35652c1 100644
--- a/templates/job.template
+++ b/templates/job.template
@@ -27,19 +27,21 @@ set -x
export VERSION=$version_number
case $$$${HOST} in
*ecg*)
- module load python
module unload grib_api
module unload eccodes
- module load eccodes
+ module unload python
module unload emos
+ module load python3
+ module load eccodes/2.12.0
module load emos/455-r64
export PATH=$$$${PATH}:$fp_root_path
;;
*cca*)
+ module unload python
module switch PrgEnv-cray PrgEnv-intel
- module load eccodes
+ module load python3
+ module load eccodes/2.12.0
module load emos
- module load python
export SCRATCH=$$$${TMPDIR}
export PATH=$$$${PATH}:$fp_root_path
;;
--
GitLab