From 70fee58eedd41328d9f5cdc56db77e03ec78bf71 Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Fri, 12 Oct 2018 15:17:12 +0200
Subject: [PATCH] some bug corrections, minor code improvements
---
source/python/_config.py | 6 ++--
source/python/classes/ControlFile.py | 12 ++++----
source/python/classes/EcFlexpart.py | 39 +++++++++++++-------------
source/python/classes/MarsRetrieval.py | 2 +-
source/python/mods/get_mars_data.py | 3 +-
source/python/mods/prepare_flexpart.py | 8 ++++--
6 files changed, 37 insertions(+), 33 deletions(-)
diff --git a/source/python/_config.py b/source/python/_config.py
index 05f1ecc..4c94074 100644
--- a/source/python/_config.py
+++ b/source/python/_config.py
@@ -27,6 +27,8 @@ import inspect
_VERSION_STR = '7.1'
+QUEUES_LIST = ['ecgate', 'cca']
+
# ------------------------------------------------------------------------------
# FILENAMES
# ------------------------------------------------------------------------------
@@ -75,11 +77,11 @@ PATH_JOBSCRIPTS = os.path.join(PATH_RUN_DIR, 'jobscripts')
PATH_FORTRAN_SRC = os.path.join(PATH_SOURCES, 'fortran')
PATH_TEST_DIR = os.path.join(PATH_SOURCES, 'pythontest')
PATH_INPUT_DIR = os.path.join(PATH_RUN_DIR, INPUT_DIRNAME_DEFAULT)
-if os.getenv('CONTROL') and '/' in os.getenv('CONTROL'):
+if os.getenv('CONTROL'):# and '/' in os.getenv('CONTROL'):
# this is only needed if remote version with job script is used!
# because job is directly submitted from SCRATCH and because the
# CONTROL file is stored there, the normal path is not valid.
- PATH_CONTROLFILES = os.getenv('CONTROL')
+ PATH_CONTROLFILES = '.'
else:
PATH_CONTROLFILES = os.path.join(PATH_RUN_DIR, 'control')
#
diff --git a/source/python/classes/ControlFile.py b/source/python/classes/ControlFile.py
index 5f9be59..b0c6212 100644
--- a/source/python/classes/ControlFile.py
+++ b/source/python/classes/ControlFile.py
@@ -59,7 +59,7 @@ import inspect
# software specific classes and modules from flex_extract
sys.path.append('../')
import _config
-from mods.tools import my_error
+from mods.tools import my_error, silent_remove
# ------------------------------------------------------------------------------
# CLASS
@@ -187,11 +187,11 @@ class ControlFile(object):
'''
try:
- with open(os.path.join(_config.PATH_CONTROLFILES,
- self.controlfile)) as f:
+ cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
+ with open(cfile) as f:
fdata = f.read().split('\n')
except IOError:
- print('Could not read CONTROL file "' + args.controlfile + '"')
+ print('Could not read CONTROL file "' + cfile + '"')
print('Either it does not exist or its syntax is wrong.')
print('Try "' + sys.argv[0].split('/')[-1] + \
' -h" to print usage information')
@@ -449,7 +449,7 @@ class ControlFile(object):
else:
self.mailops = [self.mailops]
- if queue in ['ecgate', 'cca'] and \
+ if queue in _config.QUEUES_LIST and \
not self.gateway or not self.destination or \
not self.ecuid or not self.ecgid:
print('\nEnvironment variables GATEWAY, DESTINATION, ECUID and \
@@ -462,7 +462,7 @@ class ControlFile(object):
marsfile = os.path.join(self.inputdir,
_config.FILE_MARS_REQUESTS)
if os.path.isfile(marsfile):
- os.remove(marsfile)
+ silent_remove(marsfile)
# check all logical variables for data type
# if its a string change to integer
diff --git a/source/python/classes/EcFlexpart.py b/source/python/classes/EcFlexpart.py
index bcac717..badc735 100644
--- a/source/python/classes/EcFlexpart.py
+++ b/source/python/classes/EcFlexpart.py
@@ -84,7 +84,7 @@ import numpy as np
from eccodes import (codes_index_select, codes_new_from_index, codes_get,
codes_get_values, codes_set_values, codes_set,
codes_write, codes_release, codes_new_from_index,
- codes_index_release)
+ codes_index_release, codes_index_get)
# software specific classes and modules from flex_extract
sys.path.append('../')
@@ -135,7 +135,8 @@ class EcFlexpart(object):
# different mars types for retrieving data for flexpart
self.types = dict()
- if c.maxstep > len(c.type): # Pure forecast mode
+ # Pure forecast mode
+ if c.maxstep > len(c.type) and 'AN' not in c.type:
c.type = [c.type[0]]
c.step = ['{:0>3}'.format(int(c.step[0]))]
c.time = [c.time[0]]
@@ -167,11 +168,9 @@ class EcFlexpart(object):
if c.basetime == '00':
btlist = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0]
- if ((ty.upper() == 'AN' and
- int(c.time[i]) % int(c.dtime) ==0) or
- (ty.upper() != 'AN' and
- int(c.step[i]) % int(c.dtime) == 0 and
- int(c.step[i]) % int(c.dtime) == 0) ) and \
+ if ((ty.upper() == 'AN' and (int(c.time[i]) % int(c.dtime)) == 0) or
+ (ty.upper() != 'AN' and (int(c.step[i]) % int(c.dtime)) == 0 and
+ (int(c.step[i]) % int(c.dtime) == 0)) ) and \
(int(c.time[i]) in btlist or c.maxstep > 24):
if ty not in self.types.keys():
@@ -433,7 +432,7 @@ class EcFlexpart(object):
for key in index_keys:
#index_vals.append(grib_index_get(iid, key))
#print(index_vals[-1])
- key_vals = grib_index_get(iid, key)
+ key_vals = codes_index_get(iid, key)
print(key_vals)
# have to sort the steps for disaggregation,
# therefore convert to int first
@@ -531,13 +530,14 @@ class EcFlexpart(object):
# [param, levtype, levelist, grid]
if isinstance(pv, str):
continue
- retr_param_dict['type'] = '' + ftype
+ retr_param_dict['type'] = ftype
retr_param_dict['time'] = self.types[ftype]['times']
retr_param_dict['step'] = self.types[ftype]['steps']
retr_param_dict['date'] = self.dates
retr_param_dict['stream'] = self.stream
retr_param_dict['target'] = \
- self._mk_targetname(ftype, pk,
+ self._mk_targetname(ftype,
+ pk,
retr_param_dict['date'].split('/')[0])
retr_param_dict['param'] = pv[0]
retr_param_dict['levtype'] = pv[1]
@@ -688,7 +688,7 @@ class EcFlexpart(object):
from genshi.template import TemplateLoader
loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False)
- compile_template = loader.load(_config.TEMPFILE_NAMELIST,
+ namelist_template = loader.load(_config.TEMPFILE_NAMELIST,
cls=NewTextTemplate)
self.inputdir = c.inputdir
@@ -700,7 +700,7 @@ class EcFlexpart(object):
maxl = int((area[3] - area[1]) / grid[1]) + 1
maxb = int((area[0] - area[2]) / grid[0]) + 1
- stream = compile_template.generate(
+ stream = namelist_template.generate(
maxl = str(maxl),
maxb = str(maxb),
mlevel = str(self.level),
@@ -1215,26 +1215,25 @@ class EcFlexpart(object):
print('Output filelist: ')
print(self.outputfilelist)
- if c.format.lower() == 'grib2':
- for ofile in self.outputfilelist:
+ for ofile in self.outputfilelist:
+ ofile = os.path.join(self.inputdir, ofile)
+
+ if c.format.lower() == 'grib2':
p = subprocess.check_call(['grib_set', '-s', 'edition=2, \
productDefinitionTemplateNumber=8',
ofile, ofile + '_2'])
p = subprocess.check_call(['mv', ofile + '_2', ofile])
- if c.ectrans and not c.ecapi:
- for ofile in self.outputfilelist:
+ if c.ectrans and not c.ecapi:
p = subprocess.check_call(['ectrans', '-overwrite', '-gateway',
c.gateway, '-remote', c.destination,
'-source', ofile])
- if c.ecstorage and not c.ecapi:
- for ofile in self.outputfilelist:
+ if c.ecstorage and not c.ecapi:
p = subprocess.check_call(['ecp', '-o', ofile,
os.path.expandvars(c.ecfsdir)])
- if c.outputdir != c.inputdir:
- for ofile in self.outputfilelist:
+ if c.outputdir != c.inputdir:
p = subprocess.check_call(['mv',
os.path.join(c.inputdir, ofile),
c.outputdir])
diff --git a/source/python/classes/MarsRetrieval.py b/source/python/classes/MarsRetrieval.py
index 5e5d55c..daf8260 100644
--- a/source/python/classes/MarsRetrieval.py
+++ b/source/python/classes/MarsRetrieval.py
@@ -417,7 +417,7 @@ class MarsRetrieval(object):
# open a file to store all requests to
with open(os.path.join(inputdir,
- _config.FILE_MARS_REQUESTS), 'a') as f:
+ _config.FILE_MARS_REQUESTS), 'a') as f:
f.write(str(request_number) + ', ')
f.write(', '.join(str(attrs[key])
for key in sorted(attrs.iterkeys())))
diff --git a/source/python/mods/get_mars_data.py b/source/python/mods/get_mars_data.py
index 79b1ec4..cf13d65 100755
--- a/source/python/mods/get_mars_data.py
+++ b/source/python/mods/get_mars_data.py
@@ -51,7 +51,8 @@ import inspect
from datetime import datetime, timedelta
# software specific classes and modules from flex_extract
-sys.path.append('../')
+sys.path.append(os.path.dirname(os.path.abspath(
+ inspect.getfile(inspect.currentframe()))) + '/../')
import _config
from tools import (my_error, normal_exit, get_cmdline_arguments,
read_ecenv, make_dir)
diff --git a/source/python/mods/prepare_flexpart.py b/source/python/mods/prepare_flexpart.py
index c45c32f..2e2d84e 100755
--- a/source/python/mods/prepare_flexpart.py
+++ b/source/python/mods/prepare_flexpart.py
@@ -58,7 +58,9 @@ import sys
import socket
# software specific classes and modules from flex_extract
-sys.path.append('../')
+
+sys.path.append(os.path.dirname(os.path.abspath(
+ inspect.getfile(inspect.currentframe()))) + '/../')
import _config
from classes.UioFiles import UioFiles
from classes.ControlFile import ControlFile
@@ -155,7 +157,7 @@ def prepare_flexpart(ppid, c):
make_dir(c.outputdir)
# get all files with flux data to be deaccumulated
- inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')
+ inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + str(c.ppid) + '.*')
# deaccumulate the flux data
flexpart = EcFlexpart(c, fluxes=True)
@@ -163,7 +165,7 @@ def prepare_flexpart(ppid, c):
flexpart.deacc_fluxes(inputfiles, c)
# get a list of all files from the root inputdir
- inputfiles = UioFiles(c.inputdir, '????__??.*' + c.ppid + '.*')
+ inputfiles = UioFiles(c.inputdir, '????__??.*' + str(c.ppid) + '.*')
# produce FLEXPART-ready GRIB files and process them -
# copy/transfer/interpolate them or make them GRIB2
--
GitLab