From 991df6ad0a703d97e1e192a0a6106e1d9ba47e8e Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Tue, 15 May 2018 00:11:29 +0200
Subject: [PATCH] finished documentation (except plot_retrieved)
---
python/Control.py | 259 -------------------
python/Disagg.py | 85 +++---
python/ECFlexpart.py | 377 ++++++++++++++-------------
python/ECMWF_ENV | 4 -
python/GribTools.py | 70 ++---
python/MARSretrieval.py | 43 +++-
python/Tools.py | 118 ++++++---
python/UIOFiles.py | 121 +++++----
python/compilejob.ksh | 6 +-
python/getMARSdata.py | 282 +++++++++++---------
python/install.py | 175 +++++++------
python/job.ksh | 4 +-
python/job.temp | 2 +-
python/joboper.ksh | 4 +-
python/plot_retrieved.py | 498 ++++++++++++++++++++++++------------
python/prepareFLEXPART.py | 159 +++++++-----
python/profiling.py | 37 +--
python/submit.py | 81 +++---
python/testsuite.py | 106 +++++---
src/Makefile.local.gfortran | 4 +-
src/Makefile.local.ifort | 16 +-
21 files changed, 1298 insertions(+), 1153 deletions(-)
delete mode 100644 python/Control.py
delete mode 100644 python/ECMWF_ENV
diff --git a/python/Control.py b/python/Control.py
deleted file mode 100644
index 8fb6a45..0000000
--- a/python/Control.py
+++ /dev/null
@@ -1,259 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#************************************************************************
-# TODO AP
-# - write a test class
-# - check documentation
-#************************************************************************
-"""
-@Author: Leopold Haimberger (University of Vienna)
-
-@Date: November 2015
-
-@ChangeHistory:
- February 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added documentation
- - applied some minor modifications in programming style/structure
- - outsource of class Control
-
-@License:
- (C) Copyright 2015-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- A standard python 2.6 or 2.7 installation
-
-@Description:
- The Control files are the steering part of the FLEXPART extraction
- software. All necessary parameters needed to retrieve the data fields
- from the MARS archive for driving FLEXPART are set in a Control file.
- Some specific parameters like the start and end dates can be overwritten
- by the command line parameters, but in generel all parameters needed
- for a complete set of fields for FLEXPART can be set in the Control files.
-
-"""
-# ------------------------------------------------------------------------------
-# MODULES
-# ------------------------------------------------------------------------------
-import os
-import inspect
-import Tools
-# ------------------------------------------------------------------------------
-# CLASS
-# ------------------------------------------------------------------------------
-class Control:
- '''
- Class containing the information of the ECMWFDATA control file.
-
- Contains all the parameters of control files, which are e.g.:
- DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
- STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
- LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
- OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
- ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
- MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR,
- BASETIME, DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
-
- For more information about format and content of the parameter
- see documentation.
-
- '''
-
- def __init__(self, filename):
- '''
- @Description:
- Initialises the instance of Control class and defines and
- assign all controlfile variables. Set default values if
- parameter was not in CONTROL file.
-
- @Input:
- self: instance of Control class
- Description see class documentation.
-
- filename: string
- Name of control file.
-
- @Return:
- <nothing>
- '''
-
- # read whole CONTROL file
- with open(filename) as f:
- fdata = f.read().split('\n')
-
- # go through every line and store parameter
- # as class variable
- for ldata in fdata:
- data = ldata.split()
- if len(data) > 1:
- if 'm_' in data[0].lower():
- data[0] = data[0][2:]
- if data[0].lower() == 'class':
- data[0] = 'marsclass'
- if data[0].lower() == 'day1':
- data[0] = 'start_date'
- if data[0].lower() == 'day2':
- data[0] = 'end_date'
- if data[0].lower() == 'addpar':
- if '/' in data[1]:
- # remove leading '/' sign from addpar content
- if data[1][0] == '/':
- data[1] = data[1][1:]
- dd = data[1].split('/')
- data = [data[0]]
- for d in dd:
- data.append(d)
- pass
- if len(data) == 2:
- if '$' in data[1]:
- setattr(self, data[0].lower(), data[1])
- while '$' in data[1]:
- i = data[1].index('$')
- j = data[1].find('{')
- k = data[1].find('}')
- var = os.getenv(data[1][j+1:k])
- if var is not None:
- data[1] = data[1][:i] + var + data[1][k+1:]
- else:
- Tools.myerror(None,
- 'Could not find variable ' +
- data[1][j+1:k] +
- ' while reading ' +
- filename)
- setattr(self, data[0].lower() + '_expanded', data[1])
- else:
- if data[1].lower() != 'none':
- setattr(self, data[0].lower(), data[1])
- else:
- setattr(self, data[0].lower(), None)
- elif len(data) > 2:
- setattr(self, data[0].lower(), (data[1:]))
- else:
- pass
-
- # check a couple of necessary attributes if they contain values
- # otherwise set default values
- if not hasattr(self, 'start_date'):
- self.start_date = None
- if not hasattr(self, 'end_date'):
- self.end_date = self.start_date
- if not hasattr(self, 'accuracy'):
- self.accuracy = 24
- if not hasattr(self, 'omega'):
- self.omega = '0'
- if not hasattr(self, 'cwc'):
- self.cwc = '0'
- if not hasattr(self, 'omegadiff'):
- self.omegadiff = '0'
- if not hasattr(self, 'etadiff'):
- self.etadiff = '0'
- if not hasattr(self, 'levelist'):
- if not hasattr(self, 'level'):
- print('Warning: neither levelist nor level \
- specified in CONTROL file')
- else:
- self.levelist = '1/to/' + self.level
- else:
- if 'to' in self.levelist:
- self.level = self.levelist.split('/')[2]
- else:
- self.level = self.levelist.split('/')[-1]
-
- if not hasattr(self, 'maxstep'):
- # find out maximum step
- self.maxstep = 0
- for s in self.step:
- if int(s) > self.maxstep:
- self.maxstep = int(s)
- else:
- self.maxstep = int(self.maxstep)
-
- if not hasattr(self, 'prefix'):
- self.prefix = 'EN'
- if not hasattr(self, 'makefile'):
- self.makefile = None
- if not hasattr(self, 'basetime'):
- self.basetime = None
- if not hasattr(self, 'date_chunk'):
- self.date_chunk = '3'
- if not hasattr(self, 'grib2flexpart'):
- self.grib2flexpart = '0'
-
- # script directory
- self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile(
- inspect.currentframe()))) + '/../'
- # Fortran source directory
- self.exedir = self.ecmwfdatadir + 'src/'
-
- # FLEXPART directory
- if not hasattr(self, 'flexpart_root_scripts'):
- self.flexpart_root_scripts = self.ecmwfdatadir
-
- return
-
- def __str__(self):
- '''
- @Description:
- Prepares a single string with all the comma seperated Control
- class attributes including their values.
-
- Example:
- {'kids': 0, 'name': 'Dog', 'color': 'Spotted',
- 'age': 10, 'legs': 2, 'smell': 'Alot'}
-
- @Input:
- self: instance of Control class
- Description see class documentation.
-
- @Return:
- string of Control class attributes with their values
- '''
-
- attrs = vars(self)
-
- return ', '.join("%s: %s" % item for item in attrs.items())
-
- def tolist(self):
- '''
- @Description:
- Just generates a list of strings containing the attributes and
- assigned values except the attributes "_expanded", "exedir",
- "ecmwfdatadir" and "flexpart_root_scripts".
-
- @Input:
- self: instance of Control class
- Description see class documentation.
-
- @Return:
- l: list
- A sorted list of the all Control class attributes with
- their values except the attributes "_expanded", "exedir",
- "ecmwfdatadir" and "flexpart_root_scripts".
- '''
-
- attrs = vars(self)
- l = list()
-
- for item in attrs.items():
- if '_expanded' in item[0]:
- pass
- elif 'exedir' in item[0]:
- pass
- elif 'flexpart_root_scripts' in item[0]:
- pass
- elif 'ecmwfdatadir' in item[0]:
- pass
- else:
- if type(item[1]) is list:
- stot = ''
- for s in item[1]:
- stot += s + ' '
-
- l.append("%s %s" % (item[0], stot))
- else:
- l.append("%s %s" % item)
-
- return sorted(l)
diff --git a/python/Disagg.py b/python/Disagg.py
index cf2f4c9..20aee50 100644
--- a/python/Disagg.py
+++ b/python/Disagg.py
@@ -2,51 +2,50 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-# - make a class out of this ???
-# - write a test class
+# - check alist of size 4 ?
+# - write a test, IMPORTANT
#************************************************************************
-"""
-@Author: Anne Philipp (University of Vienna)
+#*******************************************************************************
+# @Author: Anne Philipp (University of Vienna)
+#
+# @Date: March 2018
+#
+# @Change History:
+# November 2015 - Leopold Haimberger (University of Vienna):
+# - migration of the methods dapoly and darain from Fortran
+# (flex_extract_v6 and earlier) to Python
+#
+# April 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added structured documentation
+# - outsourced the disaggregation functions dapoly and darain
+# to a new module named Disagg
+#
+# @License:
+# (C) Copyright 2015-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Module Description:
+# Disaggregation of deaccumulated flux data from an ECMWF model FG field.
+# Initially the flux data to be concerned are:
+# - large-scale precipitation
+# - convective precipitation
+# - surface sensible heat flux
+# - surface solar radiation
+# - u stress
+# - v stress
+# Different versions of disaggregation is provided for rainfall
+# data (darain, modified linear) and the surface fluxes and
+# stress data (dapoly, cubic polynomial).
+#
+# @Module Content:
+# - dapoly
+# - darain
+#
+#*******************************************************************************
-@Date: March 2018
-
-@ChangeHistory:
- November 2015 - Leopold Haimberger (University of Vienna):
- - integrated methods dapoly and darain from Fortran to Python
-
- April 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added structured documentation
- - outsourced the disaggregation functions dapoly and darain
- to a new module named Disagg
-
-@License:
- (C) Copyright 2015-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- A standard python 2.6 or 2.7 installation
-
-@Description:
- Further documentation may be obtained from www.flexpart.eu.
-
- Functionality provided:
- Disaggregation of deaccumulated flux data from an ECMWF model FG field.
- Initially the flux data to be concerned are:
- - large-scale precipitation
- - convective precipitation
- - surface sensible heat flux
- - surface solar radiation
- - u stress
- - v stress
-
- Different versions of disaggregation is provided for rainfall
- data (darain, modified linear) and the surface fluxes and
- stress data (dapoly, cubic polynomial).
-
-"""
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
diff --git a/python/ECFlexpart.py b/python/ECFlexpart.py
index 8e1e57f..0537feb 100644
--- a/python/ECFlexpart.py
+++ b/python/ECFlexpart.py
@@ -2,102 +2,112 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-#AP
# - specifiy file header documentation
+# - add class description in header information
# - apply classtests
# - add references to ECMWF specific software packages
+# - add describtion of deacc_fluxes
+# - change name of func deacc ( weil disagg )
+# - add desc of retrieve function
#************************************************************************
-"""
-@Author: Anne Fouilloux (University of Oslo)
-
-@Date: October 2014
-
-@ChangeHistory:
- November 2015 - Leopold Haimberger (University of Vienna):
- - extended with Class Control
- - removed functions mkdir_p, daterange, years_between, months_between
- - added functions darain, dapoly, toparamId, init128, normalexit,
- myerror, cleanup, install_args_and_control,
- interpret_args_and_control,
- - removed function __del__ in class EIFLexpart
- - added the following functions in EIFlexpart:
- - create_namelist
- - process_output
- - deacc_fluxes
- - modified existing EIFlexpart - functions for the use in
- flex_extract
- - retrieve also longer term forecasts, not only analyses and
- short term forecast data
- - added conversion into GRIB2
- - added conversion into .fp format for faster execution of FLEXPART
- (see https://www.flexpart.eu/wiki/FpCtbtoWo4FpFormat)
-
- February 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added documentation
- - outsourced class Control
- - outsourced class MarsRetrieval
- - changed class name from EIFlexpart to ECFlexpart
- - applied minor code changes (style)
-
-@License:
- (C) Copyright 2014-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- - A standard python 2.6 or 2.7 installation
- - dateutils
- - matplotlib (optional, for debugging)
- - ECMWF specific packages, all available from https://software.ecmwf.int/
- ECMWF WebMARS, gribAPI with python enabled, emoslib and
- ecaccess web toolkit
-
-@Description:
- Further documentation may be obtained from www.flexpart.eu.
-
- Functionality provided:
- Prepare input 3D-wind fields in hybrid coordinates +
- surface fields for FLEXPART runs
-"""
+#*******************************************************************************
+# @Author: Anne Fouilloux (University of Oslo)
+#
+# @Date: October 2014
+#
+# @Change History:
+#
+# November 2015 - Leopold Haimberger (University of Vienna):
+# - extended with class Control
+# - removed functions mkdir_p, daterange, years_between, months_between
+# - added functions darain, dapoly, toparamId, init128, normalexit,
+# myerror, cleanup, install_args_and_control,
+# interpret_args_and_control,
+# - removed function __del__ in class EIFLexpart
+# - added the following functions in EIFlexpart:
+# - create_namelist
+# - process_output
+# - deacc_fluxes
+# - modified existing EIFlexpart - functions for the use in
+# flex_extract
+# - retrieve also longer term forecasts, not only analyses and
+# short term forecast data
+# - added conversion into GRIB2
+# - added conversion into .fp format for faster execution of FLEXPART
+# (see https://www.flexpart.eu/wiki/FpCtbtoWo4FpFormat)
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+# - removed function getFlexpartTime in class ECFlexpart
+# - outsourced class ControlFile
+# - outsourced class MarsRetrieval
+# - changed class name from EIFlexpart to ECFlexpart
+# - applied minor code changes (style)
+# - removed "dead code" , e.g. retrieval of Q since it is not needed
+# - removed "times" parameter from retrieve-method since it is not used
+#
+# @License:
+# (C) Copyright 2014-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Class Description:
+# FLEXPART needs grib files in a specifc format. All necessary data fields
+# for one time step are stored in a single file. The class represents an
+# instance with all the parameter and settings necessary for retrieving
+# MARS data and modifing them so they are fitting FLEXPART need. The class
+# is able to disaggregate the fluxes and convert grid types to the one needed
+# by FLEXPART, therefore using the FORTRAN program.
+#
+# @Class Content:
+# - __init__
+# - write_namelist
+# - retrieve
+# - process_output
+# - create
+# - deacc_fluxes
+#
+#*******************************************************************************
+
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import subprocess
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-import traceback
import shutil
import os
-import errno
import sys
import inspect
import glob
import datetime
-from string import atoi
from numpy import *
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
ecapi = True
try:
import ecmwfapi
except ImportError:
ecapi = False
from gribapi import *
+
+# software specific classes and modules from flex_extract
from GribTools import GribTools
from Tools import init128, toparamId, silentremove, product
-from Control import Control
+from ControlFile import ControlFile
from MARSretrieval import MARSretrieval
import Disagg
+
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
class ECFlexpart:
'''
- Class to retrieve ECMWF data specific for running FLEXPART.
+ Class to retrieve FLEXPART specific ECMWF data.
'''
# --------------------------------------------------------------------------
# CLASS FUNCTIONS
# --------------------------------------------------------------------------
- def __init__(self, c, fluxes=False): #done/ verstehen
+ def __init__(self, c, fluxes=False):
'''
@Description:
Creates an object/instance of ECFlexpart with the
@@ -107,8 +117,8 @@ class ECFlexpart:
self: instance of ECFlexpart
The current object of the class.
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -129,7 +139,7 @@ class ECFlexpart:
<nothing>
'''
- # different mars types for retrieving reanalysis data for flexpart
+ # different mars types for retrieving data for flexpart
self.types = dict()
try:
if c.maxstep > len(c.type): # Pure forecast mode
@@ -146,25 +156,16 @@ class ECFlexpart:
self.inputdir = c.inputdir
self.basetime = c.basetime
self.dtime = c.dtime
- #self.mars = {}
i = 0
- #j = 0
if fluxes is True and c.maxstep < 24:
# no forecast beyond one day is needed!
# Thus, prepare flux data manually as usual
- # with only FC fields with start times at 00/12
+ # with only forecast fields with start times at 00/12
# (but without 00/12 fields since these are
# the initialisation times of the flux fields
# and therefore are zero all the time)
- self.types['FC'] = {'times': '00/12',
- 'steps': '{}/to/12/by/{}'.format(c.dtime,
- c.dtime)}
- #i = 1
- #for k in [0, 12]:
- # for j in range(int(c.dtime), 13, int(c.dtime)):
- # self.mars['{:0>3}'.format(i * int(c.dtime))] = \
- # [c.type[1], '{:0>2}'.format(k), '{:0>3}'.format(j)]
- # i += 1
+ self.types[c.type[1]] = {'times': '00/12', 'steps':
+ '{}/to/12/by/{}'.format(c.dtime, c.dtime)}
else:
for ty, st, ti in zip(c.type, c.step, c.time):
btlist = range(24)
@@ -188,14 +189,8 @@ class ECFlexpart:
if len(self.types[ty]['steps']) > 0:
self.types[ty]['steps'] += '/'
self.types[ty]['steps'] += st
-
- #self.mars['{:0>3}'.format(j)] = [ty,
- # '{:0>2}'.format(int(ti)),
- # '{:0>3}'.format(int(st))]
- #j += int(c.dtime)
-
i += 1
- print 'EC init: ', self.types #AP
+
# Different grids need different retrievals
# SH = Spherical Harmonics, GG = Gaussian Grid,
# OG = Output Grid, ML = MultiLevel, SL = SingleLevel
@@ -273,7 +268,7 @@ class ECFlexpart:
# the simplest case
self.params['OG__ML'][0] += '/U/V/77'
elif c.gauss == '0' and c.eta == '0':
-#AP then remove?!?!?!? # this is not recommended (inaccurate)
+ # this is not recommended (inaccurate)
self.params['OG__ML'][0] += '/U/V'
elif c.gauss == '1' and c.eta == '0':
# this is needed for data before 2008, or for reanalysis data
@@ -323,7 +318,7 @@ class ECFlexpart:
return
- def write_namelist(self, c, filename): #done
+ def write_namelist(self, c, filename):
'''
@Description:
Creates a namelist file in the temporary directory and writes
@@ -335,8 +330,8 @@ class ECFlexpart:
self: instance of ECFlexpart
The current object of the class.
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL files, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -382,49 +377,41 @@ class ECFlexpart:
return
- def retrieve(self, server, dates, times, inputdir=''):
+ def retrieve(self, server, dates, inputdir='.'):
'''
@Description:
-
+ Finalizing the retrieval information by setting final details
+ depending on grid type.
+ Prepares MARS retrievals per grid type and submits them.
@Input:
self: instance of ECFlexpart
+ The current object of the class.
- server: instance of ECMWFService
-
- dates:
+ server: instance of ECMWFService or ECMWFDataServer
+ The connection to the ECMWF server. This is different
+ for member state users which have full access and non
+ member state users which have only access to the public
+ data sets. The decision is made from command line argument
+ "public"; for public access its True (ECMWFDataServer)
+ for member state users its False (ECMWFService)
- times:
+ dates: string
+ Contains start and end date of the retrieval in the format
+ "YYYYMMDD/to/YYYYMMDD"
inputdir: string, optional
- Default string is empty ('').
+ Path to the directory where the retrieved data is about
+ to be stored. The default is the current directory ('.').
@Return:
<nothing>
'''
self.dates = dates
self.server = server
-
- if inputdir == "":
- self.inputdir = '.'
- else:
- self.inputdir = inputdir
-
- # Retrieve Q not for using Q but as a template for a reduced gaussian
- # grid one date and time is enough
- # Take analysis at 00
- qdate = self.dates
- idx = qdate.find("/")
- if (idx > 0):
- qdate = self.dates[:idx]
-
- #QG = MARSretrieval(self.server, marsclass = self.marsclass, stream = self.stream, type = "an", levtype = "ML", levelist = "1",
- #gaussian = "reduced",grid = '{}'.format((int(self.resol)+1)/2), resol = self.resol,accuracy = self.accuracy,target = self.inputdir+"/"+"QG.grb",
- #date = qdate, time = "00",expver = self.expver, param = "133.128")
- #QG.displayInfo()
- #QG.dataRetrieve()
-
+ self.inputdir = inputdir
oro = False
+
for ftype in self.types:
for pk, pv in self.params.iteritems():
if isinstance(pv, str):
@@ -452,7 +439,6 @@ class ECFlexpart:
oro = True
else:
continue
-
if pk == 'GG__SL' and pv[0] == 'Q':
area = ""
gaussian = 'reduced'
@@ -460,6 +446,7 @@ class ECFlexpart:
area = self.area
gaussian = self.gaussian
+ # ------ on demand path --------------------------------------------------
if self.basetime is None:
MR = MARSretrieval(self.server,
marsclass=self.marsclass, stream=mfstream,
@@ -472,35 +459,39 @@ class ECFlexpart:
MR.displayInfo()
MR.dataRetrieve()
- # The whole else section is only necessary for operational scripts.
- # It could be removed
+ # ------ operational path ------------------------------------------------
else:
# check if mars job requests fields beyond basetime.
# If yes eliminate those fields since they may not
# be accessible with user's credentials
- sm1 = -1
if 'by' in mfstep:
sm1 = 2
- tm1 = -1
+ else:
+ sm1 = -1
+
if 'by' in mftime:
tm1 = 2
+ else:
+ tm1 = -1
+
maxtime = datetime.datetime.strptime(
mfdate.split('/')[-1] + mftime.split('/')[tm1],
'%Y%m%d%H') + datetime.timedelta(
hours=int(mfstep.split('/')[sm1]))
-
elimit = datetime.datetime.strptime(
mfdate.split('/')[-1] +
self.basetime, '%Y%m%d%H')
if self.basetime == '12':
+ # -------------- flux data ----------------------------
if 'acc' in pk:
- # Strategy: if maxtime-elimit> = 24h reduce date by 1,
- # if 12h< = maxtime-elimit<12h reduce time for last date
- # if maxtime-elimit<12h reduce step for last time
- # A split of the MARS job into 2 is likely necessary.
- maxtime = elimit-datetime.timedelta(hours=24)
+ # Strategy:
+ # if maxtime-elimit >= 24h reduce date by 1,
+ # if 12h <= maxtime-elimit<12h reduce time for last date
+ # if maxtime-elimit<12h reduce step for last time
+ # A split of the MARS job into 2 is likely necessary.
+ maxtime = elimit - datetime.timedelta(hours=24)
mfdate = '/'.join(('/'.join(mfdate.split('/')[:-1]),
datetime.datetime.strftime(
maxtime, '%Y%m%d')))
@@ -540,6 +531,7 @@ class ECFlexpart:
MR.displayInfo()
MR.dataRetrieve()
+ # -------------- non flux data ------------------------
else:
MR = MARSretrieval(self.server,
marsclass=self.marsclass,
@@ -554,10 +546,10 @@ class ECFlexpart:
MR.displayInfo()
MR.dataRetrieve()
- else:
+ else: # basetime == 0 ??? #AP
+
maxtime = elimit - datetime.timedelta(hours=24)
mfdate = datetime.datetime.strftime(maxtime,'%Y%m%d')
-
mftimesave = ''.join(mftime)
if '/' in mftime:
@@ -614,14 +606,14 @@ class ECFlexpart:
return
- def process_output(self, c): #done
+ def process_output(self, c):
'''
@Description:
- The grib files are postprocessed depending on selection in
- control file. The resulting files are moved to the output
+ The grib files are postprocessed depending on the selection in
+ CONTROL file. The resulting files are moved to the output
directory if its not equla to the input directory.
The following modifications might be done if
- properly switched in control file:
+ properly switched in CONTROL file:
GRIB2 - Conversion to GRIB2
ECTRANS - Transfer of files to gateway server
ECSTORAGE - Storage at ECMWF server
@@ -631,8 +623,8 @@ class ECFlexpart:
self: instance of ECFlexpart
The current object of the class.
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -649,7 +641,7 @@ class ECFlexpart:
'''
- print('Postprocessing:\n Format: {}\n'.format(c.format))
+ print('\n\nPostprocessing:\n Format: {}\n'.format(c.format))
if c.ecapi is False:
print('ecstorage: {}\n ecfsdir: {}\n'.
@@ -674,14 +666,14 @@ class ECFlexpart:
if int(c.ectrans) == 1 and c.ecapi is False:
for ofile in self.outputfilelist:
p = subprocess.check_call(['ectrans', '-overwrite', '-gateway',
- c.gateway, '-remote', c.destination,
- '-source', ofile])
+ c.gateway, '-remote', c.destination,
+ '-source', ofile])
print('ectrans:', p)
if int(c.ecstorage) == 1 and c.ecapi is False:
for ofile in self.outputfilelist:
p = subprocess.check_call(['ecp', '-o', ofile,
- os.path.expandvars(c.ecfsdir)])
+ os.path.expandvars(c.ecfsdir)])
if c.outputdir != c.inputdir:
for ofile in self.outputfilelist:
@@ -692,7 +684,7 @@ class ECFlexpart:
if c.grib2flexpart == '1':
# generate AVAILABLE file
- # Example of AVAILABLE file data
+ # Example of AVAILABLE file data:
# 20131107 000000 EN13110700 ON DISC
clist = []
for ofile in self.outputfilelist:
@@ -741,8 +733,7 @@ class ECFlexpart:
break
i += 1
-# clist.sort()
- # insert the date and time information of run star and end
+ # insert the date and time information of run start and end
# into the list of lines of COMMAND file
lflist = lflist[:i+1] + \
[clist[0][:16], clist[-1][:16]] + \
@@ -752,8 +743,7 @@ class ECFlexpart:
with open(pwd + '/Options/COMMAND', 'w') as g:
g.write('\n'.join(lflist) + '\n')
- # change to outputdir and start the
- # grib2flexpart run
+ # change to outputdir and start the grib2flexpart run
# afterwards switch back to the working dir
os.chdir(c.outputdir)
p = subprocess.check_call([os.path.expandvars(
@@ -764,7 +754,7 @@ class ECFlexpart:
return
- def create(self, inputfiles, c): #done
+ def create(self, inputfiles, c):
'''
@Description:
This method is based on the ECMWF example index.py
@@ -772,11 +762,12 @@ class ECFlexpart:
An index file will be created which depends on the combination
of "date", "time" and "stepRange" values. This is used to iterate
- over all messages in the grib files passed through the parameter
- "inputfiles" to seperate specific parameters into fort.* files.
- Afterwards the FORTRAN program Convert2 is called to convert
+ over all messages in each grib file which were passed through the
+ parameter "inputfiles" to seperate specific parameters into fort.*
+ files. Afterwards the FORTRAN program Convert2 is called to convert
the data fields all to the same grid and put them in one file
- per day.
+ per unique time step (combination of "date", "time" and
+ "stepRange").
@Input:
self: instance of ECFlexpart
@@ -785,8 +776,8 @@ class ECFlexpart:
inputfiles: instance of UIOFiles
Contains a list of files.
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL files, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -825,14 +816,12 @@ class ECFlexpart:
# index_vals[1]: ('0', '1200', '1800', '600') ; time
# index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
- # delete old fort.* files and open them newly
fdict = {'10':None, '11':None, '12':None, '13':None, '16':None,
'17':None, '19':None, '21':None, '22':None, '20':None}
- #for f in fdict.keys():
- # silentremove(c.inputdir + "/fort." + f)
-
for prod in product(*index_vals):
+ # flag for Fortran program CONVERT2, initially False
+ convertFlag = False
print 'current prod: ', prod
# e.g. prod = ('20170505', '0', '12')
# ( date ,time, step)
@@ -841,11 +830,16 @@ class ECFlexpart:
for i in range(len(index_keys)):
grib_index_select(iid, index_keys[i], prod[i])
+ # get first id from current product
gid = grib_new_from_index(iid)
- # do convert2 program if gid at this time is not None,
- # therefore save in hid
- hid = gid
+
+ # if there is data for this product combination
+ # prepare some date and time parameter before reading the data
if gid is not None:
+ # Fortran program CONVERT2 is only done if gid at this time is
+ # not None, therefore save information in convertFlag
+ convertFlag = True
+ # remove old fort.* files and open new ones
for k, f in fdict.iteritems():
silentremove(c.inputdir + "/fort." + k)
fdict[k] = open(c.inputdir + '/fort.' + k, 'w')
@@ -899,7 +893,7 @@ class ECFlexpart:
except AttributeError:
pass
-
+ # helper variable to remember which fields are already used.
savedfields = []
while 1:
if gid is None:
@@ -923,8 +917,8 @@ class ECFlexpart:
grib_write(gid, fdict['12'])
elif paramId == 155 and gridtype == 'sh':
grib_write(gid, fdict['13'])
- elif paramId in [129, 138, 155] and levtype == 'hybrid' \
- and c.wrf == '1':
+ elif paramId in [129, 138, 155] and levtype == 'hybrid' \
+ and c.wrf == '1':
pass
elif paramId == 246 or paramId == 247:
# cloud liquid water and ice
@@ -948,11 +942,10 @@ class ECFlexpart:
try:
if c.wrf == '1':
-# die if abfrage scheint ueberfluessig da eh das gleihce ausgefuehrt wird
- if levtype == 'hybrid':
+ if levtype == 'hybrid': # model layer
if paramId in [129, 130, 131, 132, 133, 138, 155]:
grib_write(gid, fwrf)
- else:
+ else: # sfc layer
if paramId in wrfpars:
grib_write(gid, fwrf)
except AttributeError:
@@ -964,10 +957,8 @@ class ECFlexpart:
for f in fdict.values():
f.close()
- # call for CONVERT2
-# AUSLAGERN IN EIGENE FUNKTION
-
- if hid is not None:
+ # call for CONVERT2 if flag is True
+ if convertFlag:
pwd = os.getcwd()
os.chdir(c.inputdir)
if os.stat('fort.21').st_size == 0 and int(c.eta) == 1:
@@ -977,37 +968,41 @@ class ECFlexpart:
myerror(c, 'fort.21 is empty while parameter eta is set \
to 1 in CONTROL file')
+ # create the corresponding output file fort.15
+ # (generated by CONVERT2) + fort.16 (paramId 167 and 168)
p = subprocess.check_call([os.path.expandvars(
os.path.expanduser(c.exedir)) + '/CONVERT2'], shell=True)
os.chdir(pwd)
- # create the corresponding output file fort.15
- # (generated by CONVERT2)
- # + fort.16 (paramId 167 and paramId 168)
+
+ # create final output filename, e.g. EN13040500 (ENYYMMDDHH)
fnout = c.inputdir + '/' + c.prefix
if c.maxstep > 12:
suffix = cdate[2:8] + '.{:0>2}'.format(time/100) + \
'.{:0>3}'.format(step)
else:
suffix = cdateH[2:10]
-
fnout += suffix
print("outputfile = " + fnout)
self.outputfilelist.append(fnout) # needed for final processing
- fout = open(fnout, 'wb')
- shutil.copyfileobj(open(c.inputdir + '/fort.15', 'rb'), fout)
- if c.cwc == '1':
- shutil.copyfileobj(open(c.inputdir + '/fort.22', 'rb'), fout)
- shutil.copyfileobj(open(c.inputdir + '/flux' + cdate[0:2] +
- suffix, 'rb'), fout)
- shutil.copyfileobj(open(c.inputdir + '/fort.16', 'rb'), fout)
- orolsm = glob.glob(c.inputdir +
- '/OG_OROLSM__SL.*.' + c.ppid + '*')[0]
- shutil.copyfileobj(open(orolsm, 'rb'), fout)
- fout.close()
+
+ # create outputfile and copy all data from intermediate files
+ # to the outputfile (final GRIB files)
+ orolsm = os.path.basename(glob.glob(
+ c.inputdir + '/OG_OROLSM__SL.*.' + c.ppid + '*')[0])
+ fluxfile = 'flux' + cdate[0:2] + suffix
+ if c.cwc != '1':
+ flist = ['fort.15', fluxfile, 'fort.16', orolsm]
+ else:
+ flist = ['fort.15', 'fort.22', fluxfile, 'fort.16', orolsm]
+
+ with open(fnout, 'wb') as fout:
+ for f in flist:
+ shutil.copyfileobj(open(c.inputdir + '/' + f, 'rb'), fout)
+
if c.omega == '1':
- fnout = c.outputdir + '/OMEGA'
- fout = open(fnout, 'wb')
- shutil.copyfileobj(open(c.inputdir + '/fort.25', 'rb'), fout)
+ with open(c.outputdir + '/OMEGA', 'wb') as fout:
+ shutil.copyfileobj(
+ open(c.inputdir + '/fort.25', 'rb'), fout)
try:
if c.wrf == '1':
@@ -1019,11 +1014,14 @@ class ECFlexpart:
return
-
def deacc_fluxes(self, inputfiles, c):
'''
@Description:
-
+ Goes through all flux fields in ordered time and de-accumulate
+ the fields. Afterwards the fields are disaggregated in time.
+ Different versions of disaggregation is provided for rainfall
+ data (darain, modified linear) and the surface fluxes and
+ stress data (dapoly, cubic polynomial).
@Input:
self: instance of ECFlexpart
@@ -1032,8 +1030,8 @@ class ECFlexpart:
inputfiles: instance of UIOFiles
Contains a list of files.
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -1132,6 +1130,7 @@ class ECFlexpart:
hnout = c.inputdir + '/flux' + sdates.strftime('%Y%m%d%H')
g = open(gnout, 'w')
h = open(hnout, 'w')
+
print("outputfile = " + fnout)
f = open(fnout, 'w')
diff --git a/python/ECMWF_ENV b/python/ECMWF_ENV
deleted file mode 100644
index 6706466..0000000
--- a/python/ECMWF_ENV
+++ /dev/null
@@ -1,4 +0,0 @@
-ECUID km4a
-ECGID at
-GATEWAY srvx8.img.univie.ac.at
-DESTINATION philipa8@genericSftp
diff --git a/python/GribTools.py b/python/GribTools.py
index 3029352..5c2a925 100644
--- a/python/GribTools.py
+++ b/python/GribTools.py
@@ -2,48 +2,52 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-#AP
# - GribTools name möglicherweise etwas verwirrend.
# - change self.filename in self.filenames!!!
-# - add file description
# - bis auf --init-- und index wird keine Funktion verwendet!?
#************************************************************************
-"""
-@Author: Anne Fouilloux (University of Oslo)
-
-@Date: July 2014
-
-@ChangeHistory:
- February 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added documentation
- - changed some naming
-
-@License:
- (C) Copyright 2014-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- - A standard python 2.6 or 2.7 installation
- - dateutils
- - ECMWF specific packages, all available from https://software.ecmwf.int/
- ECMWF WebMARS, gribAPI with python enabled, emoslib and
- ecaccess web toolkit
-
-@Description:
- Further documentation may be obtained from www.flexpart.eu.
+#*******************************************************************************
+# @Author: Anne Fouilloux (University of Oslo)
+#
+# @Date: July 2014
+#
+# @Change History:
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+# - changed some naming
+#
+# @License:
+# (C) Copyright 2014-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Class Description:
+# The GRIB API provides all necessary tools to work directly with the
+# grib files. Nevertheless, the GRIB API tools are very basic and are in
+# direct connection with the grib files. This class provides some higher
+# functions which apply a set of GRIB API tools together in the respective
+# context. So, the class initially contains a list of grib files (their
+# names) and the using program then applies the methods directly on the
+# class objects without having to think about how the actual GRIB API
+# tools have to be arranged.
+#
+# @Class Content:
+# - __init__
+# - getkeys
+# - setkeys
+# - copy
+# - index
+#
+#*******************************************************************************
- ...
-"""
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
-from gribapi import *
-import traceback
-import sys
import os
+from gribapi import *
+
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
diff --git a/python/MARSretrieval.py b/python/MARSretrieval.py
index e5b6d06..3c1c5b3 100644
--- a/python/MARSretrieval.py
+++ b/python/MARSretrieval.py
@@ -2,11 +2,46 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-#AP
-# -
# -
#************************************************************************
-## ------------------------------------------------------------------------------
+#*******************************************************************************
+# @Author: Anne Fouilloux (University of Oslo)
+#
+# @Date: October 2014
+#
+# @Change History:
+#
+# November 2015 - Leopold Haimberger (University of Vienna):
+# - optimized displayInfo
+# - optimized dataRetrieve and seperate between python and shell
+# script call
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+# - applied some minor modifications in programming style/structure
+#
+# @License:
+# (C) Copyright 2015-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Class Description:
+# A MARS revtrieval has a specific syntax with a selection of keywords and
+# their corresponding values. This class provides the necessary functions
+# by displaying the selected parameters and their values and the actual
+# retrievement of the data through a mars request or a Python web api
+# interface. The initialization already expects all the keyword values.
+#
+# @Class Content:
+# - __init__
+# - displayInfo
+# - dataRetrieve
+#
+#*******************************************************************************
+
+# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import subprocess
@@ -17,7 +52,7 @@ try:
import ecmwfapi
except ImportError:
ecapi = False
-#from gribapi import *
+
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
diff --git a/python/Tools.py b/python/Tools.py
index 9d55a6b..24e9e02 100644
--- a/python/Tools.py
+++ b/python/Tools.py
@@ -2,23 +2,67 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-#AP
-# -
+# - check myerror
+# - check normalexit
+# - check getListAsString
+# - seperate args and control interpretation
#************************************************************************
-"""
+#*******************************************************************************
+# @Author: Anne Philipp (University of Vienna)
+#
+# @Date: May 2018
+#
+# @Change History:
+# October 2014 - Anne Fouilloux (University of Oslo)
+# - created functions silentremove and product (taken from ECMWF)
+#
+# November 2015 - Leopold Haimberger (University of Vienna)
+# - created functions: interpret_args_and_control, cleanup
+# myerror, normalexit, init128, toparamId
+#
+# April 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+# - moved all functions from file FlexpartTools to this file Tools
+# - added function getListAsString
+#
+# @License:
+# (C) Copyright 2014-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Modul Description:
+# This module contains a couple of helpful functions which are
+# used in different places in flex_extract.
+#
+# @Module Content:
+# - interpret_args_and_control
+# - cleanup
+# - myerror
+# - normalexit
+# - product
+# - silentremove
+# - init128
+# - toparamId
+# - getListAsString
+#
+#*******************************************************************************
-"""
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import os
import errno
import sys
import glob
+import traceback
from numpy import *
from gribapi import *
-import Control
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+# software specific class from flex_extract
+from ControlFile import ControlFile
# ------------------------------------------------------------------------------
# FUNCTIONS
@@ -27,7 +71,7 @@ import Control
def interpret_args_and_control():
'''
@Description:
- Assigns the command line arguments and reads control file
+ Assigns the command line arguments and reads CONTROL file
content. Apply default values for non mentioned arguments.
@Input:
@@ -37,8 +81,8 @@ def interpret_args_and_control():
args: instance of ArgumentParser
Contains the commandline arguments from script/program call.
- c: instance of class Control
- Contains all necessary information of a control file. The parameters
+ c: instance of class ControlFile
+ Contains all necessary information of a CONTROL file. The parameters
are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
@@ -61,7 +105,7 @@ def interpret_args_and_control():
parser.add_argument("--date_chunk", dest="date_chunk", default=None,
help="# of days to be retrieved at once")
- # some arguments that override the default in the control file
+ # some arguments that override the default in the CONTROL file
parser.add_argument("--basetime", dest="basetime",
help="base such as 00/12 (for half day retrievals)")
parser.add_argument("--step", dest="step",
@@ -95,21 +139,21 @@ def interpret_args_and_control():
(e.g. ecgate or cca )")
parser.add_argument("--controlfile", dest="controlfile",
default='CONTROL.temp',
- help="file with control parameters")
+ help="file with CONTROL parameters")
parser.add_argument("--debug", dest="debug", default=0,
help="Debug mode - leave temporary files intact")
args = parser.parse_args()
- # create instance of Control for specified controlfile
+ # create instance of ControlFile for specified controlfile
# and assign the parameters (and default values if necessary)
try:
- c = Control.Control(args.controlfile)
+ c = ControlFile(args.controlfile)
except IOError:
try:
- c = Control.Control(localpythonpath + args.controlfile)
+ c = ControlFile(localpythonpath + args.controlfile)
except:
- print('Could not read control file "' + args.controlfile + '"')
+ print('Could not read CONTROL file "' + args.controlfile + '"')
print('Either it does not exist or its syntax is wrong.')
print('Try "' + sys.argv[0].split('/')[-1] +
' -h" to print usage information')
@@ -118,12 +162,12 @@ def interpret_args_and_control():
# check for having at least a starting date
if args.start_date is None and getattr(c, 'start_date') is None:
print('start_date specified neither in command line nor \
- in control file ' + args.controlfile)
+ in CONTROL file ' + args.controlfile)
print('Try "' + sys.argv[0].split('/')[-1] +
' -h" to print usage information')
exit(1)
- # save all existing command line parameter to the Control instance
+ # save all existing command line parameter to the ControlFile instance
# if parameter is not specified through the command line or CONTROL file
# set default values
if args.start_date is not None:
@@ -198,11 +242,11 @@ def cleanup(c):
'''
@Description:
Remove all files from intermediate directory
- (inputdir from control file).
+ (inputdir from CONTROL file).
@Input:
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -239,8 +283,8 @@ def myerror(c, message='ERROR'):
before exiting the program.
@Input:
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -289,8 +333,8 @@ def normalexit(c, message='Done!'):
Prints a specific exit message which can be passed to the function.
@Input:
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -356,7 +400,6 @@ def product(*args, **kwds):
Return will be done with "yield". A tuple of combined arguments.
See example in description above.
'''
-
pools = map(tuple, args) * kwds.get('repeat', 1)
result = [[]]
for pool in pools:
@@ -370,9 +413,8 @@ def product(*args, **kwds):
def silentremove(filename):
'''
@Description:
- Removes the file which name is passed to the function if
- it exists. The function does not fail if the file does not
- exist.
+ If "filename" exists , it is removed.
+ The function does not fail if the file does not exist.
@Input:
filename: string
@@ -425,7 +467,7 @@ def toparamId(pars, table):
@Input:
pars: string
- Addpar argument from control file in the format of
+ Addpar argument from CONTROL file in the format of
parameter names instead of ids. The parameter short
names are sepearted with "/" and they are passed as
one single string.
@@ -437,7 +479,7 @@ def toparamId(pars, table):
@Return:
ipar: list of integer
- List of addpar parameters from control file transformed to
+ List of addpar parameters from CONTROL file transformed to
parameter ids in the format of integer.
'''
cpar = pars.upper().split('/')
@@ -454,8 +496,20 @@ def toparamId(pars, table):
return ipar
-def getListAsString(listobj):
+def getListAsString(listObj):
'''
@Description:
+ Converts a list of arbitrary content into a single string.
+
+ @Input:
+ listObj: list
+ A list with arbitrary content.
+
+ @Return:
+ strOfList: string
+ The content of the list as a single string.
'''
- return ", ".join( str(l) for l in listobj)
\ No newline at end of file
+
+ strOfList = ", ".join( str(l) for l in listObj)
+
+ return strOfList
diff --git a/python/UIOFiles.py b/python/UIOFiles.py
index faeead6..7b01333 100644
--- a/python/UIOFiles.py
+++ b/python/UIOFiles.py
@@ -2,40 +2,48 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-#AP
# - checken welche regelmässigen methoden auf diese Files noch angewendet werden
# und dann hier implementieren
-# - add description of file!
+# cleanup hier rein
#************************************************************************
-"""
-@Author: Anne Fouilloux (University of Oslo)
+#*******************************************************************************
+# @Author: Anne Fouilloux (University of Oslo)
+#
+# @Date: October 2014
+#
+# @Change History:
+#
+# November 2015 - Leopold Haimberger (University of Vienna):
+# - modified method listFiles to work with glob instead of listdir
+# - added pattern search in method listFiles
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+# - optimisation of method listFiles since it didn't work correctly
+# for sub directories
+# - additional speed up of method listFiles
+# - modified the class so that it is initiated with a pattern instead
+# of suffixes. Gives more precision in selection of files.
+#
+# @License:
+# (C) Copyright 2014-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Class Decription:
+# The class is for file manipulation. It is initiated with a regular
+# expression pattern for this instance and can produce a list of Files
+# from the given file pattern. These files can be deleted.
+#
+# @Class Content:
+# - __init__
+# - listFiles
+# - deleteFiles
+#
+#*******************************************************************************
-@Date: October 2014
-
-@ChangeHistory:
- November 2015 - Leopold Haimberger (University of Vienna):
- - modified method listFiles to work with glob instead of listdir
- - added pattern search in method listFiles
-
- February 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added documentation
- - optimisation of method listFiles since it didn't work correctly
- for sub directories
- - additional speed up of method listFiles
-
-@License:
- (C) Copyright 2014-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- A standard python 2.6 or 2.7 installation
-
-@Description:
- ...
-"""
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
@@ -43,48 +51,50 @@ import os
import glob
import fnmatch
import time
+
+# software specific module from flex_extract
import profiling
+from Tools import silentremove
+
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
+
class UIOFiles:
'''
Class to manipulate files. At initialisation it has the attribute
- suffix which stores a list of suffixes of the files associated
+ pattern which stores a regular expression pattern for the files associated
with the instance of the class.
'''
# --------------------------------------------------------------------------
# CLASS FUNCTIONS
# --------------------------------------------------------------------------
- def __init__(self, suffix):
+ def __init__(self, pattern):
'''
@Description:
- Assignes the suffixes of the files which should be
- associated with the instance of the class.
+ Assignes a specific pattern for these files.
@Input:
self: instance of UIOFiles
Description see class documentation.
- suffix: list of strings
- The types of files which should be manipulated such as
- ['grib', 'grb', 'grib1', 'grib2', 'grb1', 'grb2']
+ pattern: string
+ Regular expression pattern. For example: '*.grb'
@Return:
<nothing>
'''
- self.suffix = suffix
+ self.pattern = pattern
return
#@profiling.timefn
- def listFiles(self, path, pattern, callid=0):
+ def listFiles(self, path, callid=0):
'''
@Description:
Lists all files in the directory with the matching
- regular expression pattern. The suffixes are already stored
- in a list attribute "suffix".
+ regular expression pattern.
@Input:
self: instance of UIOFiles
@@ -93,10 +103,6 @@ class UIOFiles:
path: string
Directory where to list the files.
- pattern: string
- Regular expression pattern. For example:
- '*OG_acc_SL*.'+c.ppid+'.*'
-
callid: integer
Id which tells the function if its the first call
or a recursive call. Default and first call is 0.
@@ -114,10 +120,9 @@ class UIOFiles:
path = os.path.abspath(path)
# get the file list of the path if its not a directory and
- # if it contains one of the suffixes
+ # if it contains the pattern
self.files.extend([os.path.join(path, k) for k in os.listdir(path)
- if fnmatch.fnmatch(k, pattern) and
- os.path.splitext(k)[-1] in self.suffix])
+ if fnmatch.fnmatch(k, self.pattern)])
# find possible sub-directories in the path
subdirs = [s for s in os.listdir(path)
@@ -126,6 +131,24 @@ class UIOFiles:
# do recursive calls for sub-direcorties
if subdirs:
for subdir in subdirs:
- self.listFiles(os.path.join(path, subdir), pattern, callid=1)
+ self.listFiles(os.path.join(path, subdir), callid=1)
+
+ return
+
+ def deleteFiles(self):
+ '''
+ @Description:
+ Deletes the files.
+
+ @Input:
+ self: instance of UIOFiles
+ Description see class documentation.
+
+ @Return:
+ <nothing>
+ '''
+
+ for f in self.files:
+ silentremove(f)
return
diff --git a/python/compilejob.ksh b/python/compilejob.ksh
index d276521..3afc4e4 100644
--- a/python/compilejob.ksh
+++ b/python/compilejob.ksh
@@ -4,7 +4,7 @@
# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server
# start with sbatch NAME_OF_THIS_FILE directly on machine
-#SBATCH --workdir=/scratch/ms/at/km4a
+#SBATCH --workdir=/scratch/ms/spatlh00/lh0
#SBATCH --qos=normal
#SBATCH --job-name=flex_ecmwf
#SBATCH --output=flex_ecmwf.%j.out
@@ -32,7 +32,7 @@ case $HOST in
module unload emos
module load grib_api/1.14.5
module load emos/437-r64
-export FLEXPART_ROOT_SCRIPTS=${HOME}
+export FLEXPART_ROOT_SCRIPTS=$HOME
# export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
# export PYTHONPATH=$ECMWFDATA/python
# export PATH=${PATH}:$ECMWFDATA/python
@@ -48,7 +48,7 @@ export FLEXPART_ROOT_SCRIPTS=${HOME}
echo $HOME | awk -F / '{print $1, $2, $3, $4}'
export GROUP=`echo $HOME | awk -F / '{print $4}'`
export SCRATCH=/scratch/ms/${GROUP}/${USER}
-export FLEXPART_ROOT_SCRIPTS=${HOME}
+export FLEXPART_ROOT_SCRIPTS=$HOME
# export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
# export PYTHONPATH=$ECMWFDATA/python
# export PATH=${PATH}:$ECMWFDATA/python
diff --git a/python/getMARSdata.py b/python/getMARSdata.py
index 278176d..43872cb 100755
--- a/python/getMARSdata.py
+++ b/python/getMARSdata.py
@@ -2,82 +2,130 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-# - Change History ist nicht angepasst ans File!
-# - add file description
+# - add function docstrings!!!!
#************************************************************************
-"""
-@Author: Anne Fouilloux (University of Oslo)
-
-@Date: October 2014
-
-@ChangeHistory:
- November 2015 - Leopold Haimberger (University of Vienna):
- - using the WebAPI also for general MARS retrievals
- - job submission on ecgate and cca
- - job templates suitable for twice daily operational dissemination
- - dividing retrievals of longer periods into digestable chunks
- - retrieve also longer term forecasts, not only analyses and
- short term forecast data
- - conversion into GRIB2
- - conversion into .fp format for faster execution of FLEXPART
-
- February 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added documentation
- - minor changes in programming style for consistence
-
-@License:
- (C) Copyright 2014-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- - A standard python 2.6 or 2.7 installation
- - dateutils
- - ECMWF specific packages, all available from https://software.ecmwf.int/
- ECMWF WebMARS, gribAPI with python enabled, emoslib and
- ecaccess web toolkit
-
-@Description:
- Further documentation may be obtained from www.flexpart.eu.
-
-"""
+#*******************************************************************************
+# @Author: Anne Fouilloux (University of Oslo)
+#
+# @Date: October 2014
+#
+# @Change History:
+#
+# November 2015 - Leopold Haimberger (University of Vienna):
+# - moved the getEIdata program into a function "getMARSdata"
+# - moved the AgurmentParser into a seperate function
+# - adatpted the function for the use in flex_extract
+# - renamed file to getMARSdata
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added structured documentation
+# - minor changes in programming style for consistence
+# - added function main and moved function calls vom __main__ there
+# (necessary for better documentation with docstrings for later
+# online documentation)
+# - use of UIFiles class for file selection and deletion
+
+#
+# @License:
+# (C) Copyright 2014-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Program Functionality:
+# This program can be used as a module in the whole flex_extract process
+# or can be run by itself to just extract MARS data from ECMWF. To do so,
+# a couple of necessary parameters has to be passed with the program call.
+# See documentation for more details.
+#
+# @Program Content:
+# - main
+# - getMARSdata
+#
+#*******************************************************************************
+
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
+import os
+import sys
+import datetime
+import inspect
try:
ecapi=True
import ecmwfapi
except ImportError:
ecapi=False
-import calendar
-import shutil
-import datetime
-import time
-import os
-import glob
-import sys
-import inspect
-# add path to submit.py to pythonpath so that python finds its buddies
-localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+# add path to pythonpath so that python finds its buddies
+localpythonpath = os.path.dirname(os.path.abspath(
+ inspect.getfile(inspect.currentframe())))
if localpythonpath not in sys.path:
sys.path.append(localpythonpath)
-from Control import Control
+# software specific classes and modules from flex_extract
+from ControlFile import ControlFile
from Tools import myerror, normalexit, \
interpret_args_and_control
from ECFlexpart import ECFlexpart
+from UIOFiles import UIOFiles
# ------------------------------------------------------------------------------
# FUNCTION
# ------------------------------------------------------------------------------
-def getMARSdata(args, c):
+def main():
+ '''
+ @Description:
+ If getMARSdata is called from command line, this function controls
+ the program flow and calls the argumentparser function and
+ the getMARSdata function for retrieving EC data.
+
+ @Input:
+ <nothing>
+
+ @Return:
+ <nothing>
+ '''
+ args, c = interpret_args_and_control()
+ getMARSdata(args, c)
+ normalexit(c)
+ return
+
+def getMARSdata(args, c):
+ '''
+ @Description:
+ Retrieves the EC data needed for a FLEXPART simulation.
+ Start and end dates for retrieval period is set. Retrievals
+ are divided into smaller periods if necessary and datechunk parameter
+ is set.
+
+ @Input:
+ args: instance of ArgumentParser
+ Contains the commandline arguments from script/program call.
+
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
+ DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
+ STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
+ LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
+ OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
+ ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
+ MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
+ DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
+
+ For more information about format and content of the parameter
+ see documentation.
+
+ @Return:
+ <nothing>
+ '''
if not os.path.exists(c.inputdir):
os.makedirs(c.inputdir)
+
+ print("Retrieving EC data!")
print("start date %s " % (c.start_date))
print("end date %s " % (c.end_date))
@@ -87,40 +135,34 @@ def getMARSdata(args, c):
server = False
c.ecapi = ecapi
- print 'ecapi:', c.ecapi
-
-# Retrieve EC data for running flexpart
-#AP change this variant to correct format conversion with datetime
-#AP import datetime and timedelta explicitly
- syear = int(c.start_date[:4])
- smonth = int(c.start_date[4:6])
- sday = int(c.start_date[6:])
- start = datetime.date(year=syear, month=smonth, day=sday)
+ print 'ecapi: ', c.ecapi
+
+ # set start date of retrieval period
+ start = datetime.date(year=int(c.start_date[:4]),
+ month=int(c.start_date[4:6]),
+ day=int(c.start_date[6:]))
startm1 = start - datetime.timedelta(days=1)
if c.basetime == '00':
start = startm1
- eyear = int(c.end_date[:4])
- emonth = int(c.end_date[4:6])
- eday = int(c.end_date[6:])
- end = datetime.date(year=eyear, month=emonth, day=eday)
+ # set end date of retrieval period
+ end = datetime.date(year=int(c.end_date[:4]),
+ month=int(c.end_date[4:6]),
+ day=int(c.end_date[6:]))
if c.basetime == '00' or c.basetime == '12':
endp1 = end + datetime.timedelta(days=1)
else:
endp1 = end + datetime.timedelta(days=2)
+ # set time period of one single retrieval
datechunk = datetime.timedelta(days=int(c.date_chunk))
- # retrieving of accumulated data fields (flux data), (maximum one month)
-
- # remove old files
- print 'removing content of ' + c.inputdir
- tobecleaned = glob.glob(c.inputdir + '/*_acc_*.' + \
- str(os.getppid()) + '.*.grb')
- for f in tobecleaned:
- os.remove(f)
+ # -------------- flux data ------------------------------------------------
+ print 'removing old flux content of ' + c.inputdir
+ tobecleaned = UIOFiles('*_acc_*.' + str(os.getppid()) + '.*.grb')
+ tobecleaned.listFiles(c.inputdir)
+ tobecleaned.deleteFiles()
- times = None
# if forecast for maximum one day (upto 23h) are to be retrieved,
# collect accumulation data (flux data)
# with additional days in the beginning and at the end
@@ -128,24 +170,24 @@ def getMARSdata(args, c):
if c.maxstep < 24:
day = startm1
while day < endp1:
- # retrieve MARS data for the whole period
- flexpart = ECFlexpart(c, fluxes=True)
- tmpday = day + datechunk - datetime.timedelta(days=1)
- if tmpday < endp1:
- dates = day.strftime("%Y%m%d") + "/to/" + \
- tmpday.strftime("%Y%m%d")
- else:
- dates = day.strftime("%Y%m%d") + "/to/" + \
- end.strftime("%Y%m%d")
+ # retrieve MARS data for the whole period
+ flexpart = ECFlexpart(c, fluxes=True)
+ tmpday = day + datechunk - datetime.timedelta(days=1)
+ if tmpday < endp1:
+ dates = day.strftime("%Y%m%d") + "/to/" + \
+ tmpday.strftime("%Y%m%d")
+ else:
+ dates = day.strftime("%Y%m%d") + "/to/" + \
+ end.strftime("%Y%m%d")
- print "retrieve " + dates + " in dir " + c.inputdir
+ print "retrieve " + dates + " in dir " + c.inputdir
- try:
- flexpart.retrieve(server, dates, times, c.inputdir)
- except IOError:
- myerror(c,'MARS request failed')
+ try:
+ flexpart.retrieve(server, dates, c.inputdir)
+ except IOError:
+ myerror(c, 'MARS request failed')
- day += datechunk
+ day += datechunk
# if forecast data longer than 24h are to be retrieved,
# collect accumulation data (flux data)
@@ -155,38 +197,36 @@ def getMARSdata(args, c):
else:
day = start
while day <= end:
- # retrieve MARS data for the whole period
- flexpart = ECFlexpart(c, fluxes=True)
- tmpday = day + datechunk - datetime.timedelta(days=1)
- if tmpday < end:
- dates = day.strftime("%Y%m%d") + "/to/" + \
- tmpday.trftime("%Y%m%d")
- else:
- dates = day.strftime("%Y%m%d") + "/to/" + \
- end.strftime("%Y%m%d")
-
- print "retrieve " + dates + " in dir " + c.inputdir
-
- try:
- flexpart.retrieve(server, dates, times, c.inputdir)
- except IOError:
- myerror(c, 'MARS request failed')
-
- day += datechunk
-
- # retrieving of normal data fields (non flux data), (maximum one month)
-
- # remove old *__* files
- tobecleaned = glob.glob(c.inputdir + '/*__*.' +
- str(os.getppid()) + '.*.grb')
- for f in tobecleaned:
- os.remove(f)
+ # retrieve MARS data for the whole period
+ flexpart = ECFlexpart(c, fluxes=True)
+ tmpday = day + datechunk - datetime.timedelta(days=1)
+ if tmpday < end:
+ dates = day.strftime("%Y%m%d") + "/to/" + \
+ tmpday.trftime("%Y%m%d")
+ else:
+ dates = day.strftime("%Y%m%d") + "/to/" + \
+ end.strftime("%Y%m%d")
+
+ print "retrieve " + dates + " in dir " + c.inputdir
+
+ try:
+ flexpart.retrieve(server, dates, c.inputdir)
+ except IOError:
+ myerror(c, 'MARS request failed')
+
+ day += datechunk
+
+ # -------------- non flux data --------------------------------------------
+ print 'removing old non flux content of ' + c.inputdir
+ tobecleaned = UIOFiles('*__*.' + str(os.getppid()) + '.*.grb')
+ tobecleaned.listFiles(c.inputdir)
+ tobecleaned.deleteFiles()
+
day = start
- times = None
while day <= end:
- # retrieve MARS data for the whole period
+ # retrieve all non flux MARS data for the whole period
flexpart = ECFlexpart(c, fluxes=False)
- tmpday = day+datechunk-datetime.timedelta(days=1)
+ tmpday = day + datechunk - datetime.timedelta(days=1)
if tmpday < end:
dates = day.strftime("%Y%m%d") + "/to/" + \
tmpday.strftime("%Y%m%d")
@@ -197,7 +237,7 @@ def getMARSdata(args, c):
print "retrieve " + dates + " in dir " + c.inputdir
try:
- flexpart.retrieve(server, dates, times, c.inputdir)
+ flexpart.retrieve(server, dates, c.inputdir)
except IOError:
myerror(c, 'MARS request failed')
@@ -206,7 +246,5 @@ def getMARSdata(args, c):
return
if __name__ == "__main__":
+ main()
- args, c = interpret_args_and_control()
- getMARSdata(args, c)
- normalexit(c)
diff --git a/python/install.py b/python/install.py
index d73b08c..09cc622 100755
--- a/python/install.py
+++ b/python/install.py
@@ -2,84 +2,100 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-#AP
-# - Functionality Provided is not correct for this file
# - localpythonpath should not be set in module load section!
# - create a class Installation and divide installation in 3 subdefs for
# ecgate, local and cca seperatly
# - Change History ist nicht angepasst ans File! Original geben lassen
#************************************************************************
-"""
-@Author: Anne Fouilloux (University of Oslo)
-
-@Date: October 2014
-
-@ChangeHistory:
- November 2015 - Leopold Haimberger (University of Vienna):
- - using the WebAPI also for general MARS retrievals
- - job submission on ecgate and cca
- - job templates suitable for twice daily operational dissemination
- - dividing retrievals of longer periods into digestable chunks
- - retrieve also longer term forecasts, not only analyses and
- short term forecast data
- - conversion into GRIB2
- - conversion into .fp format for faster execution of FLEXPART
-
- February 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added documentation
-
-@License:
- (C) Copyright 2014 UIO.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- - A standard python 2.6 or 2.7 installation
- - dateutils
- - matplotlib (optional, for debugging)
- - ECMWF specific packages, all available from https://software.ecmwf.int/
- ECMWF WebMARS, gribAPI with python enabled, emoslib and
- ecaccess web toolkit
-
-@Description:
- Further documentation may be obtained from www.flexpart.eu.
-
- Functionality provided:
- Prepare input 3D-wind fields in hybrid coordinates +
- surface fields for FLEXPART runs
-"""
+#*******************************************************************************
+# @Author: Leopold Haimberger (University of Vienna)
+#
+# @Date: November 2015
+#
+# @Change History:
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+#
+# @License:
+# (C) Copyright 2015-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Program Functionality:
+# Depending on the selected installation environment (locally or on the
+# ECMWF server ecgate or cca) the program extracts the commandline
+# arguments and the CONTROL file parameter and prepares the corresponding
+# environment. The necessary files are collected in a tar-ball and placed
+# at the target location. There its untared, the environment variables will
+# be set and the Fortran code will be compiled. If the ECMWF environment is
+# selected a job script is prepared and submitted for the remaining
+# configurations after putting the tar-ball to the target ECMWF server.
+#
+# @Program Content:
+# - main
+# - install_args_and_control
+# - install_via_gateway
+#
+#*******************************************************************************
+
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
-import calendar
-import shutil
import datetime
-import time
-import os,sys,glob
+import os
+import sys
+import glob
import subprocess
import inspect
-# add path to submit.py to pythonpath so that python finds its buddies
-localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
-sys.path.append(localpythonpath)
-from UIOFiles import UIOFiles
-from string import strip
from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
-from GribTools import GribTools
-from Control import Control
-from getMARSdata import getMARSdata
-from prepareFLEXPART import prepareFLEXPART
-from ECFlexpart import ECFlexpart
+
+# add path to pythonpath so that python finds its buddies
+localpythonpath = os.path.dirname(os.path.abspath(
+ inspect.getfile(inspect.currentframe())))
+if localpythonpath not in sys.path:
+ sys.path.append(localpythonpath)
+
+# software specific classes and modules from flex_extract
+from ControlFile import ControlFile
# ------------------------------------------------------------------------------
# FUNCTIONS
# ------------------------------------------------------------------------------
+def main():
+ '''
+ @Description:
+ Controls the installation process. Calls the installation function
+ if target is specified.
+
+ @Intput:
+ <nothing>
+
+ @Return:
+ <nothing>
+ '''
+
+ os.chdir(localpythonpath)
+ args, c = install_args_and_control()
+
+ if args.install_target is not None:
+ install_via_gateway(c, args.install_target)
+ else:
+ print('Please specify installation target (local|ecgate|cca)')
+ print('use -h or --help for help')
+
+ sys.exit()
+
+ return
+
+
def install_args_and_control():
'''
@Description:
Assigns the command line arguments for installation and reads
- control file content. Apply default values for non mentioned arguments.
+ CONTROL file content. Apply default values for non mentioned arguments.
@Input:
<nothing>
@@ -88,8 +104,8 @@ def install_args_and_control():
args: instance of ArgumentParser
Contains the commandline arguments from script/program call.
- c: instance of class Control
- Contains all necessary information of a control file. The parameters
+ c: instance of class ControlFile
+ Contains all necessary information of a CONTROL file. The parameters
are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
@@ -129,14 +145,14 @@ def install_args_and_control():
parser.add_argument("--controlfile", dest="controlfile",
default='CONTROL.temp',
- help="file with control parameters")
+ help="file with CONTROL parameters")
args = parser.parse_args()
try:
- c = Control(args.controlfile)
+ c = ControlFile(args.controlfile)
except:
- print('Could not read control file "' + args.controlfile + '"')
+ print('Could not read CONTROL file "' + args.controlfile + '"')
print('Either it does not exist or its syntax is wrong.')
print('Try "' + sys.argv[0].split('/')[-1] +
' -h" to print usage information')
@@ -181,20 +197,31 @@ def install_args_and_control():
return args, c
-def main():
- '''
+def install_via_gateway(c, target):
'''
- os.chdir(localpythonpath)
- args, c = install_args_and_control()
- if args.install_target is not None:
- install_via_gateway(c, args.install_target)
- else:
- print('Please specify installation target (local|ecgate|cca)')
- print('use -h or --help for help')
- sys.exit()
+ @Description:
+ Perform the actual installation on local machine or prepare data
+ transfer to remote gate and submit a job script which will
+ install everything on the remote gate.
-def install_via_gateway(c, target):
+ @Input:
+ c: instance of class ControlFile
+ Contains all necessary information of a CONTROL file. The parameters
+ are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
+ NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
+ RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
+ SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
+ ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
+ For more information about format and content of the parameter see
+ documentation.
+
+ target: string
+ The target where the installation should be processed.
+ E.g. "local", "ecgate" or "cca"
+ @Return:
+ <nothing>
+ '''
ecd = c.ecmwfdatadir
template = ecd + 'python/compilejob.temp'
job = ecd + 'python/compilejob.ksh'
diff --git a/python/job.ksh b/python/job.ksh
index 70e561c..8fba02e 100644
--- a/python/job.ksh
+++ b/python/job.ksh
@@ -63,13 +63,13 @@ export CONTROL=CONTROL
cat >$CONTROL<<EOF
GATEWAY srvx8.img.univie.ac.at
-DESTINATION philipa8@genericSftp
+DESTINATION annep@genericSftp
accuracy 16
addpar 186 187 188 235 139 39
basetime None
cwc 0
date_chunk 3
-debug 0
+debug True
dpdeta 1
dtime 3
ecfsdir ectmp:/${USER}/econdemand/
diff --git a/python/job.temp b/python/job.temp
index c9cff9f..4a20430 100644
--- a/python/job.temp
+++ b/python/job.temp
@@ -63,7 +63,7 @@ export CONTROL=CONTROL
cat >$CONTROL<<EOF
GATEWAY srvx8.img.univie.ac.at
-DESTINATION philipa8@genericSftp
+DESTINATION annep@genericSftp
EOF
cat >>$CONTROL<<EOF
EOF
diff --git a/python/joboper.ksh b/python/joboper.ksh
index 94ab09f..27621bf 100644
--- a/python/joboper.ksh
+++ b/python/joboper.ksh
@@ -63,13 +63,13 @@ export CONTROL=CONTROL
cat >$CONTROL<<EOF
GATEWAY srvx8.img.univie.ac.at
-DESTINATION philipa8@genericSftp
+DESTINATION annep@genericSftp
accuracy 16
addpar 186 187 188 235 139 39
basetime None
cwc 0
date_chunk 3
-debug 0
+debug True
dpdeta 1
dtime 3
ecfsdir ectmp:/${USER}/econdemand/
diff --git a/python/plot_retrieved.py b/python/plot_retrieved.py
index 7d4c6ad..e34bd0d 100755
--- a/python/plot_retrieved.py
+++ b/python/plot_retrieved.py
@@ -1,203 +1,367 @@
#!/usr/bin/env python
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# -*- coding: utf-8 -*-
+#************************************************************************
+# TODO AP
+# - documentation der Funktionen
+# - docu der progam functionality
+# - apply pep8
+#************************************************************************
+#*******************************************************************************
+# @Author: Leopold Haimberger (University of Vienna)
#
-# Functionality provided: Simple tool for creating maps and time series of retrieved fields.
+# @Date: November 2015
#
-# Requirements:
-# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
-# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
-# dateutils
-# matplotlib (optional, for debugging)
+# @Change History:
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+# - created function main and moved the two function calls for
+# arguments and plotting into it
+#
+# @License:
+# (C) Copyright 2015-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Program Functionality:
+# Simple tool for creating maps and time series of retrieved fields.
+#
+# @Program Content:
+# - plot_retrieved
+# - plottimeseries
+# - plotmap
+# - interpret_plotargs
+#
+#*******************************************************************************
+# ------------------------------------------------------------------------------
+# MODULES
+# ------------------------------------------------------------------------------
import datetime
import time
-import os,inspect,sys,glob
-import socket
-# add path to submit.py to pythonpath so that python finds its buddies
-localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
-if localpythonpath not in sys.path:
- sys.path.append(localpythonpath)
+import os
+import inspect
+import sys
+import glob
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from matplotlib.pylab import *
import matplotlib.patches as mpatches
-from mpl_toolkits.basemap import Basemap,addcyclic
+from mpl_toolkits.basemap import Basemap, addcyclic
import matplotlib.colors as mcolors
from matplotlib.font_manager import FontProperties
from matplotlib.patches import Polygon
import matplotlib.cm as cmx
import matplotlib.colors as colors
-from argparse import ArgumentParser,ArgumentDefaultsHelpFormatter
-
-from Tools import interpret_args_and_control, silentremove, product
-from Control import Control
-from GribTools import GribTools
+#from rasotools.utils import stats
from gribapi import *
-from rasotools.utils import stats
-def plot_retrieved(args,c):
-
- start = datetime.datetime.strptime(c.start_date,'%Y%m%d%H')
- end = datetime.datetime.strptime(c.end_date,'%Y%m%d%H')
+# add path to pythonpath so that python finds its buddies
+localpythonpath = os.path.dirname(os.path.abspath(
+ inspect.getfile(inspect.currentframe())))
+if localpythonpath not in sys.path:
+ sys.path.append(localpythonpath)
- c.paramIds=asarray(c.paramIds,dtype='int')
- c.levels=asarray(c.levels,dtype='int')
- c.area=asarray(c.area)
+# software specific classes and modules from flex_extract
+from Tools import silentremove, product
+from ControlFile import ControlFile
+from GribTools import GribTools
- index_keys=["date","time","step"]
- indexfile=c.inputdir+"/date_time_stepRange.idx"
+# ------------------------------------------------------------------------------
+# FUNCTION
+# ------------------------------------------------------------------------------
+def main():
+ '''
+ @Description:
+ If plot_retrieved is called from command line, this function controls
+ the program flow and calls the argumentparser function and
+ the plot_retrieved function for plotting the retrieved GRIB data.
+
+ @Input:
+ <nothing>
+
+ @Return:
+ <nothing>
+ '''
+ args, c = interpret_plotargs()
+ plot_retrieved(args, c)
+
+ return
+
+def plot_retrieved(args, c):
+ '''
+ @Description:
+ Reads GRIB data from a specified time period, a list of levels
+ and a specified list of parameter.
+
+ @Input:
+ args: instance of ArgumentParser
+ Contains the commandline arguments from script/program call.
+
+ c: instance of class ControlFile
+ Contains all necessary information of a CONTROL file. The parameters
+ are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
+ NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
+ RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
+ SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
+ ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
+ OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
+ For more information about format and content of the parameter see
+ documentation.
+
+ @Return:
+ <nothing>
+ '''
+ start = datetime.datetime.strptime(c.start_date, '%Y%m%d%H')
+ end = datetime.datetime.strptime(c.end_date, '%Y%m%d%H')
+
+ c.paramIds = asarray(c.paramIds, dtype='int')
+ c.levels = asarray(c.levels, dtype='int')
+ c.area = asarray(c.area)
+
+ index_keys = ["date", "time", "step"]
+ indexfile = c.inputdir + "/date_time_stepRange.idx"
silentremove(indexfile)
- files=glob.glob(c.inputdir+'/'+c.prefix+'*')
- grib=GribTools(files)
- iid=grib.index(index_keys=index_keys, index_file = indexfile)
-
- gdict=dict(Ni = 360, Nj = 181, iScansNegatively = 0, jScansPositively = 0,
- jPointsAreConsecutive = 0, alternativeRowScanning = 0,
- latitudeOfFirstGridPointInDegrees = 90,
- longitudeOfFirstGridPointInDegrees = 181,
- latitudeOfLastGridPointInDegrees = -90,
- longitudeOfLastGridPointInDegrees = 180,
- iDirectionIncrementInDegrees = 1,
- jDirectionIncrementInDegrees = 1
- )
+ files = glob.glob(c.inputdir + '/' + c.prefix + '*')
+ grib = GribTools(files)
+ iid = grib.index(index_keys=index_keys, index_file = indexfile)
+
+ gdict = dict(Ni = 360, Nj = 181,
+ iScansNegatively = 0, jScansPositively = 0,
+ jPointsAreConsecutive = 0, alternativeRowScanning = 0,
+ latitudeOfFirstGridPointInDegrees = 90,
+ longitudeOfFirstGridPointInDegrees = 181,
+ latitudeOfLastGridPointInDegrees = -90,
+ longitudeOfLastGridPointInDegrees = 180,
+ iDirectionIncrementInDegrees = 1,
+ jDirectionIncrementInDegrees = 1
+ )
index_vals = []
for key in index_keys:
- key_vals = grib_index_get(iid,key)
+ key_vals = grib_index_get(iid, key)
print key_vals
index_vals.append(key_vals)
- fdict=dict()
- fmeta=dict()
- fstamp=dict()
+ fdict = dict()
+ fmeta = dict()
+ fstamp = dict()
for p in c.paramIds:
for l in c.levels:
- key='{:0>3}_{:0>3}'.format(p,l)
- fdict[key]=[]
- fmeta[key]=[]
- fstamp[key]=[]
+ key = '{:0>3}_{:0>3}'.format(p, l)
+ fdict[key] = []
+ fmeta[key] = []
+ fstamp[key] = []
for prod in product(*index_vals):
for i in range(len(index_keys)):
- grib_index_select(iid,index_keys[i],prod[i])
+ grib_index_select(iid, index_keys[i], prod[i])
gid = grib_new_from_index(iid)
-# if gid is not None:
while(gid is not None):
date = grib_get(gid, 'date')
- fdate= datetime.datetime(date/10000,mod(date,10000)/100,mod(date,100))
+ fdate = datetime.datetime(date/10000, mod(date,10000)/100,
+ mod(date,100))
gtime = grib_get(gid, 'time')
step = grib_get(gid, 'step')
- fdatetime=fdate+datetime.timedelta(hours=gtime/100)
-# if fdatetime<start or fdatetime>end:
-# break
+ fdatetime = fdate + datetime.timedelta(hours=gtime/100)
gtype = grib_get(gid, 'type')
paramId = grib_get(gid, 'paramId')
parameterName = grib_get(gid, 'parameterName')
- level=grib_get(gid,'level')
- if step>=c.start_step and step <=c.end_step and fdatetime>=start and fdatetime<=end and paramId in c.paramIds and level in c.levels:
- key='{:0>3}_{:0>3}'.format(paramId,level)
+ level = grib_get(gid, 'level')
+ if step >= c.start_step and step <= c.end_step and \
+ fdatetime >= start and fdatetime <= end and \
+ paramId in c.paramIds and level in c.levels:
+ key = '{:0>3}_{:0>3}'.format(paramId, level)
print key
- fdatetimestep=fdatetime+datetime.timedelta(hours=step)
- if len(fstamp)==0:
+ fdatetimestep = fdatetime + datetime.timedelta(hours=step)
+ if len(fstamp) == 0:
fstamp[key].append(fdatetimestamp)
- fmeta[key].append((paramId,parameterName,gtype,fdatetime,gtime,step,level))
- fdict[key].append(flipud(reshape(grib_get_values(gid),[gdict['Nj'],gdict['Ni']])))
+ fmeta[key].append((paramId, parameterName, gtype,
+ fdatetime, gtime, step, level))
+ fdict[key].append(flipud(reshape(
+ grib_get_values(gid), [gdict['Nj'], gdict['Ni']])))
else:
- i=0
- inserted=False
+ i = 0
+ inserted = False
for i in range(len(fstamp[key])):
- if fdatetimestep<fstamp[key][i]:
- fstamp[key][i:i]=[fdatetimestep]
- fmeta[key][i:i]=[(paramId,parameterName,gtype,fdatetime,gtime,step,level)]
- fdict[key][i:i]=[flipud(reshape(grib_get_values(gid),[gdict['Nj'],gdict['Ni']]))]
- inserted=True
+ if fdatetimestep < fstamp[key][i]:
+ fstamp[key][i:i] = [fdatetimestep]
+ fmeta[key][i:i] = [(paramId, parameterName, gtype,
+ fdatetime, gtime, step, level)]
+ fdict[key][i:i] = [flipud(reshape(
+ grib_get_values(gid),
+ [gdict['Nj'], gdict['Ni']]))]
+ inserted = True
break
if not inserted:
fstamp[key].append(fdatetimestep)
- fmeta[key].append((paramId,parameterName,gtype,fdatetime,gtime,step,level))
- fdict[key].append(flipud(reshape(grib_get_values(gid),[gdict['Nj'],gdict['Ni']])))
-
+ fmeta[key].append((paramId, parameterName, gtype,
+ fdatetime, gtime, step, level))
+ fdict[key].append(flipud(reshape(
+ grib_get_values(gid), [gdict['Nj'], gdict['Ni']])))
grib_release(gid)
gid = grib_new_from_index(iid)
for k in fdict.keys():
- fml=fmeta[k]
- fdl=fdict[k]
+ fml = fmeta[k]
+ fdl = fdict[k]
- for fd,fm in zip(fdl,fml):
- ftitle=fm[1]+' {} '.format(fm[-1])+datetime.datetime.strftime(fm[3],'%Y%m%d%H')+' '+stats(fd)
- pname='_'.join(fm[1].split())+'_{}_'.format(fm[-1])+datetime.datetime.strftime(fm[3],'%Y%m%d%H')+'.{:0>3}'.format(fm[5])
- plotmap(fd, fm,gdict,ftitle,pname+'.eps')
+ for fd, fm in zip(fdl, fml):
+ ftitle = fm[1] + ' {} '.format(fm[-1]) + \
+ datetime.datetime.strftime(fm[3], '%Y%m%d%H') #+ ' ' + stats(fd)
+ pname = '_'.join(fm[1].split()) + '_{}_'.format(fm[-1]) + \
+ datetime.datetime.strftime(fm[3], '%Y%m%d%H') + \
+ '.{:0>3}'.format(fm[5])
+ plotmap(fd, fm, gdict, ftitle, pname + '.eps')
for k in fdict.keys():
- fml=fmeta[k]
- fdl=fdict[k]
- fsl=fstamp[k]
+ fml = fmeta[k]
+ fdl = fdict[k]
+ fsl = fstamp[k]
if fdl:
- fm=fml[0]
- fd=fdl[0]
- ftitle=fm[1]+' {} '.format(fm[-1])+datetime.datetime.strftime(fm[3],'%Y%m%d%H')+' '+stats(fd)
- pname='_'.join(fm[1].split())+'_{}_'.format(fm[-1])+datetime.datetime.strftime(fm[3],'%Y%m%d%H')+'.{:0>3}'.format(fm[5])
- lat=-20
- lon=20
- plottimeseries(fdl,fml,fsl,lat,lon, gdict, ftitle, pname+'.eps')
-
-def plottimeseries(flist,fmetalist,ftimestamps,lat,lon,gdict,ftitle,filename):
- t1=time.time()
- latindex=(lat+90)*180/(gdict['Nj']-1)
- lonindex=(lon+179)*360/gdict['Ni']
- farr=asarray(flist)
- ts=farr[:,latindex,lonindex]
- f=plt.figure(figsize=(12,6.7))
- plt.plot(ftimestamps,ts)
+ fm = fml[0]
+ fd = fdl[0]
+ ftitle = fm[1] + ' {} '.format(fm[-1]) + \
+ datetime.datetime.strftime(fm[3], '%Y%m%d%H') #+ ' ' + stats(fd)
+ pname = '_'.join(fm[1].split()) + '_{}_'.format(fm[-1]) + \
+ datetime.datetime.strftime(fm[3], '%Y%m%d%H') + \
+ '.{:0>3}'.format(fm[5])
+ lat = -20
+ lon = 20
+ plottimeseries(fdl, fml, fsl, lat, lon,
+ gdict, ftitle, pname + '.eps')
+
+ return
+
+def plottimeseries(flist, fmetalist, ftimestamps, lat, lon,
+ gdict, ftitle, filename):
+ '''
+ @Description:
+
+ @Input:
+ flist:
+ The actual data values to be plotted from the grib messages.
+
+ fmetalist: list of strings
+ Contains some meta date for the data field to be plotted:
+ parameter id, parameter Name, grid type, date and time,
+ time, forecast step, level
+
+ ftimestamps: list of datetime
+ Contains the time stamps in a datetime format, e.g.
+
+ lat:
+
+ lon:
+
+ gdict:
+
+ ftitle: string
+ The title of the timeseries.
+
+ filename: string
+ The time series is stored in a file with this name.
+
+ @Return:
+ <nothing>
+ '''
+ t1 = time.time()
+ latindex = (lat + 90) * 180 / (gdict['Nj'] - 1)
+ lonindex = (lon + 179) * 360 / gdict['Ni']
+ farr = asarray(flist)
+ ts = farr[:, latindex, lonindex]
+ f = plt.figure(figsize=(12,6.7))
+ plt.plot(ftimestamps, ts)
plt.title(ftitle)
- savefig(c.outputdir+'/'+filename)
- print 'created ',c.outputdir+'/'+filename
+ savefig(c.outputdir + '/' + filename)
+ print 'created ', c.outputdir + '/' + filename
plt.close(f)
- print time.time()-t1,'s'
-
-def plotmap(flist,fmetalist,gdict,ftitle,filename):
- t1=time.time()
- f=plt.figure(figsize=(12,6.7))
+ print time.time() - t1, 's'
+
+ return
+
+def plotmap(flist, fmetalist, gdict, ftitle, filename):
+ '''
+ @Description:
+
+ @Input:
+ flist
+ fmetalist
+ gdict
+ ftitle
+ filename
+
+ @Return:
+ <nothing>
+ '''
+ t1 = time.time()
+ f = plt.figure(figsize=(12, 6.7))
mbaxes = f.add_axes([0.05, 0.15, 0.8, 0.7])
- m =Basemap(llcrnrlon=-180.,llcrnrlat=-90.,urcrnrlon=180,urcrnrlat=90.)
+ m = Basemap(llcrnrlon=-180., llcrnrlat=-90., urcrnrlon=180, urcrnrlat=90.)
#if bw==0 :
#fill_color=rgb(0.6,0.8,1)
#else:
#fill_color=rgb(0.85,0.85,0.85)
- lw=0.3
+ lw = 0.3
m.drawmapboundary()
- parallels = arange(-90.,91,90.)
- # labels = [left,right,top,bottom]
- m.drawparallels(parallels,labels=[True,True,True,True],linewidth=lw)
- meridians = arange(-180.,181.,60.)
- m.drawmeridians(meridians,labels=[True,True,True,True],linewidth=lw)
+ parallels = arange(-90., 91, 90.)
+ # labels = [left, right, top, bottom]
+ m.drawparallels(parallels, labels=[True, True, True, True], linewidth=lw)
+ meridians = arange(-180., 181., 60.)
+ m.drawmeridians(meridians, labels=[True, True, True, True], linewidth=lw)
m.drawcoastlines(linewidth=lw)
- xleft=gdict['longitudeOfFirstGridPointInDegrees']
- if xleft>180.0:
- xleft-=360.
- x=linspace(xleft,gdict['longitudeOfLastGridPointInDegrees'],gdict['Ni'])
- y=linspace(gdict['latitudeOfLastGridPointInDegrees'],gdict['latitudeOfFirstGridPointInDegrees'],gdict['Nj'])
- xx, yy = m(*meshgrid(x,y))
-
- s=m.contourf(xx,yy, flist)
- title(ftitle,y=1.1)
+ xleft = gdict['longitudeOfFirstGridPointInDegrees']
+ if xleft > 180.0:
+ xleft -= 360.
+ x = linspace(xleft, gdict['longitudeOfLastGridPointInDegrees'], gdict['Ni'])
+ y = linspace(gdict['latitudeOfLastGridPointInDegrees'],
+ gdict['latitudeOfFirstGridPointInDegrees'], gdict['Nj'])
+ xx, yy = m(*meshgrid(x, y))
+
+ s = m.contourf(xx, yy, flist)
+ title(ftitle, y=1.1)
cbaxes = f.add_axes([0.9, 0.2, 0.04, 0.6])
- cb=colorbar(cax=cbaxes)
+ cb = colorbar(cax=cbaxes)
- savefig(c.outputdir+'/'+filename)
- print 'created ',c.outputdir+'/'+filename
+ savefig(c.outputdir + '/' + filename)
+ print 'created ', c.outputdir + '/' + filename
plt.close(f)
- print time.time()-t1,'s'
-
-
-def interpret_plotargs(*args,**kwargs):
-
- parser = ArgumentParser(description='Retrieve FLEXPART input from ECMWF MARS archive',
+ print time.time() - t1, 's'
+
+ return
+
+def interpret_plotargs():
+ '''
+ @Description:
+ Assigns the command line arguments and reads CONTROL file
+ content. Apply default values for non mentioned arguments.
+
+ @Input:
+ <nothing>
+
+ @Return:
+ args: instance of ArgumentParser
+ Contains the commandline arguments from script/program call.
+
+ c: instance of class ControlFile
+ Contains all necessary information of a CONTROL file. The parameters
+ are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
+ NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
+ RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
+ SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
+ ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
+ OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
+ For more information about format and content of the parameter see
+ documentation.
+ '''
+ parser = ArgumentParser(description='Retrieve FLEXPART input from ' + \
+ 'ECMWF MARS archive',
formatter_class=ArgumentDefaultsHelpFormatter)
# the most important arguments
@@ -211,68 +375,74 @@ def interpret_plotargs(*args,**kwargs):
parser.add_argument( "--end_step", dest="end_step",
help="end_step in hours")
-# some arguments that override the default in the control file
- parser.add_argument("--levelist", dest="levelist",help="Vertical levels to be retrieved, e.g. 30/to/60")
- parser.add_argument("--area", dest="area", help="area defined as north/west/south/east")
- parser.add_argument("--paramIds", dest="paramIds", help="parameter IDs")
- parser.add_argument("--prefix", dest="prefix",default='EN', help="output file name prefix")
+# some arguments that override the default in the CONTROL file
+ parser.add_argument("--levelist", dest="levelist",
+ help="Vertical levels to be retrieved, e.g. 30/to/60")
+ parser.add_argument("--area", dest="area",
+ help="area defined as north/west/south/east")
+ parser.add_argument("--paramIds", dest="paramIds",
+ help="parameter IDs")
+ parser.add_argument("--prefix", dest="prefix", default='EN',
+ help="output file name prefix")
# set the working directories
- parser.add_argument("--inputdir", dest="inputdir",default=None,
+ parser.add_argument("--inputdir", dest="inputdir", default=None,
help="root directory for storing intermediate files")
- parser.add_argument("--outputdir", dest="outputdir",default=None,
+ parser.add_argument("--outputdir", dest="outputdir", default=None,
help="root directory for storing output files")
parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
- help="FLEXPART root directory (to find grib2flexpart and COMMAND file)\n\
- Normally ECMWFDATA resides in the scripts directory of the FLEXPART distribution")
+ help="FLEXPART root directory (to find \
+ 'grib2flexpart and COMMAND file)\n \
+ Normally ECMWFDATA resides in the scripts directory \
+ of the FLEXPART distribution")
- parser.add_argument("--controlfile", dest="controlfile",default='CONTROL.temp',
- help="file with control parameters")
+ parser.add_argument("--controlfile", dest="controlfile",
+ default='CONTROL.temp', help="file with CONTROL parameters")
args = parser.parse_args()
try:
- c=Control(args.controlfile)
+ c = ControlFile(args.controlfile)
except IOError:
try:
- c=Control(localpythonpath+args.controlfile)
+ c = ControlFile(localpythonpath + args.controlfile)
except:
- print 'Could not read control file "'+args.controlfile+'"'
+ print 'Could not read CONTROL file "' + args.controlfile + '"'
print 'Either it does not exist or its syntax is wrong.'
- print 'Try "'+sys.argv[0].split('/')[-1]+' -h" to print usage information'
+ print 'Try "' + sys.argv[0].split('/')[-1] + \
+ ' -h" to print usage information'
exit(1)
if args.levelist:
- c.levels=args.levelist.split('/')
+ c.levels = args.levelist.split('/')
else:
- c.levels=[0]
+ c.levels = [0]
if args.area:
- c.area=args.area.split('/')
+ c.area = args.area.split('/')
else:
- c.area='[0,0]'
+ c.area = '[0,0]'
- c.paramIds=args.paramIds.split('/')
+ c.paramIds = args.paramIds.split('/')
if args.start_step:
- c.start_step=int(args.start_step)
+ c.start_step = int(args.start_step)
else:
- c.start_step=0
+ c.start_step = 0
if args.end_step:
- c.end_step=int(args.end_step)
+ c.end_step = int(args.end_step)
else:
- c.end_step=0
+ c.end_step = 0
- c.start_date=args.start_date
- c.end_date=args.end_date
- c.prefix=args.prefix
- c.inputdir=args.inputdir
+ c.start_date = args.start_date
+ c.end_date = args.end_date
+ c.prefix = args.prefix
+ c.inputdir = args.inputdir
if args.outputdir:
- c.outputdir=args.outputdir
+ c.outputdir = args.outputdir
else:
- c.outputdir=c.inputdir
+ c.outputdir = c.inputdir
- return args,c
+ return args, c
if __name__ == "__main__":
+ main()
- args,c=interpret_plotargs()
- plot_retrieved(args,c)
diff --git a/python/prepareFLEXPART.py b/python/prepareFLEXPART.py
index 9c5ccbe..b1e0b86 100755
--- a/python/prepareFLEXPART.py
+++ b/python/prepareFLEXPART.py
@@ -2,72 +2,70 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-#AP
-# - Change History ist nicht angepasst ans File! Original geben lassen
# - wieso cleanup in main wenn es in prepareflexpart bereits abgefragt wurde?
# doppelt gemoppelt?
# - wieso start=startm1 wenn basetime = 0 ? wenn die fluxes nicht mehr
# relevant sind? verstehe ich nicht
#************************************************************************
-"""
-@Author: Anne Fouilloux (University of Oslo)
-
-@Date: October 2014
-
-@ChangeHistory:
- November 2015 - Leopold Haimberger (University of Vienna):
- - using the WebAPI also for general MARS retrievals
- - job submission on ecgate and cca
- - job templates suitable for twice daily operational dissemination
- - dividing retrievals of longer periods into digestable chunks
- - retrieve also longer term forecasts, not only analyses and
- short term forecast data
- - conversion into GRIB2
- - conversion into .fp format for faster execution of FLEXPART
-
- February 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added documentation
- - minor changes in programming style for consistence
- - BUG: removed call of cleanup-Function after call of prepareFlexpart
- since it is already called in prepareFlexpart at the end!
-
-@License:
- (C) Copyright 2014-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- - A standard python 2.6 or 2.7 installation
- - dateutils
- - ECMWF specific packages, all available from https://software.ecmwf.int/
- ECMWF WebMARS, gribAPI with python enabled, emoslib and
- ecaccess web toolkit
-
-@Description:
- Further documentation may be obtained from www.flexpart.eu.
-
- Functionality provided:
- Prepare input 3D-wind fields in hybrid coordinates +
- surface fields for FLEXPART runs
-"""
+#*******************************************************************************
+# @Author: Anne Fouilloux (University of Oslo)
+#
+# @Date: October 2014
+#
+# @Change History:
+#
+# November 2015 - Leopold Haimberger (University of Vienna):
+# - using the WebAPI also for general MARS retrievals
+# - job submission on ecgate and cca
+# - job templates suitable for twice daily operational dissemination
+# - dividing retrievals of longer periods into digestable chunks
+# - retrieve also longer term forecasts, not only analyses and
+# short term forecast data
+# - conversion into GRIB2
+# - conversion into .fp format for faster execution of FLEXPART
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+# - minor changes in programming style for consistence
+# - BUG: removed call of cleanup-Function after call of
+# prepareFlexpart in main since it is already called in
+# prepareFlexpart at the end!
+# - created function main and moved the two function calls for
+# arguments and prepareFLEXPART into it
+#
+# @License:
+# (C) Copyright 2014-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Program Functionality:
+# This program prepares the final version of the grib files which are
+# then used by FLEXPART. It converts the bunch of grib files extracted
+# via getMARSdata by doing for example the necessary conversion to get
+# consistent grids or the disaggregation of flux data. Finally, the
+# program combines the data fields in files per available hour with the
+# naming convention xxYYMMDDHH, where xx should be 2 arbitrary letters
+# (mostly xx is chosen to be "EN").
+#
+# @Program Content:
+# - main
+# - prepareFLEXPART
+#
+#*******************************************************************************
+
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
-import calendar
import shutil
import datetime
-import time
+#import time
import os
import inspect
import sys
import socket
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-import UIOFiles
-import Control
-import Tools
-import ECFlexpart
hostname = socket.gethostname()
ecapi = 'ecmwf' not in hostname
@@ -77,24 +75,52 @@ try:
except ImportError:
ecapi = False
-# add path to submit.py to pythonpath so that python finds its buddies
-localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+# add path to pythonpath so that python finds its buddies
+localpythonpath = os.path.dirname(os.path.abspath(
+ inspect.getfile(inspect.currentframe())))
if localpythonpath not in sys.path:
sys.path.append(localpythonpath)
+
+# software specific classes and modules from flex_extract
+from UIOFiles import UIOFiles
+from Tools import interpret_args_and_control, cleanup
+from ECFlexpart import ECFlexpart
# ------------------------------------------------------------------------------
# FUNCTION
# ------------------------------------------------------------------------------
-def prepareFLEXPART(args, c):
+def main():
'''
@Description:
+ If prepareFLEXPART is called from command line, this function controls
+ the program flow and calls the argumentparser function and
+ the prepareFLEXPART function for preparation of GRIB data for FLEXPART.
+
+ @Input:
+ <nothing>
+ @Return:
+ <nothing>
+ '''
+ args, c = interpret_args_and_control()
+ prepareFLEXPART(args, c)
+
+ return
+
+def prepareFLEXPART(args, c):
+ '''
+ @Description:
+ Lists all grib files retrieved from MARS with getMARSdata and
+ uses prepares data for the use in FLEXPART. Specific data fields
+ are converted to a different grid and the flux data are going to be
+ disaggregated. The data fields are collected by hour and stored in
+ a file with a specific FLEXPART relevant naming convention.
@Input:
args: instance of ArgumentParser
Contains the commandline arguments from script/program call.
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -133,17 +159,15 @@ def prepareFLEXPART(args, c):
endp1 = end + datetime.timedelta(days=1)
# get all files with flux data to be deaccumulated
- inputfiles = UIOFiles.UIOFiles(['.grib', '.grb', '.grib1',
- '.grib2', '.grb1', '.grb2'])
-
- inputfiles.listFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')
+ inputfiles = UIOFiles('*OG_acc_SL*.' + c.ppid + '.*')
+ inputfiles.listFiles(c.inputdir)
# create output dir if necessary
if not os.path.exists(c.outputdir):
os.makedirs(c.outputdir)
# deaccumulate the flux data
- flexpart = ECFlexpart.ECFlexpart(c, fluxes=True)
+ flexpart = ECFlexpart(c, fluxes=True)
flexpart.write_namelist(c, 'fort.4')
flexpart.deacc_fluxes(inputfiles, c)
@@ -151,10 +175,8 @@ def prepareFLEXPART(args, c):
"/to/" + end.strftime("%Y%m%d"))
# get a list of all files from the root inputdir
- inputfiles = UIOFiles.UIOFiles(['.grib', '.grb', '.grib1',
- '.grib2', '.grb1', '.grb2'])
-
- inputfiles.listFiles(c.inputdir, '????__??.*' + c.ppid + '.*')
+ inputfiles = UIOFiles('????__??.*' + c.ppid + '.*')
+ inputfiles.listFiles(c.inputdir)
# produce FLEXPART-ready GRIB files and
# process GRIB files -
@@ -162,7 +184,7 @@ def prepareFLEXPART(args, c):
if c.basetime == '00':
start = startm1
- flexpart = ECFlexpart.ECFlexpart(c, fluxes=False)
+ flexpart = ECFlexpart(c, fluxes=False)
flexpart.create(inputfiles, c)
flexpart.process_output(c)
@@ -171,10 +193,9 @@ def prepareFLEXPART(args, c):
if int(c.debug) != 0:
print('Temporary files left intact')
else:
- Tools.cleanup(c)
+ cleanup(c)
return
if __name__ == "__main__":
- args, c = Tools.interpret_args_and_control()
- prepareFLEXPART(args, c)
+ main()
diff --git a/python/profiling.py b/python/profiling.py
index c03d35d..b20e6e6 100644
--- a/python/profiling.py
+++ b/python/profiling.py
@@ -2,26 +2,29 @@
# -*- coding: utf-8 -*-
#************************************************************************
# TODO AP
-# - add description of file
# - check of license of book content
#************************************************************************
-"""
-@Author: Anne Philipp (University of Vienna)
+#*******************************************************************************
+#
+# @Author: Anne Philipp (University of Vienna)
+#
+# @Date: March 2018
+#
+# @License:
+# (C) Copyright 2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Program functionality:
+# This module is not part of flex_extract. It is just used for testing and
+# performance analysis of some functions.
+#
+# @Program Content:
+# - timefn
+#
+#*******************************************************************************
-@Date: March 2018
-
-@License:
- (C) Copyright 2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- A standard python 2.6 or 2.7 installation
-
-@Description:
- ...
-"""
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
diff --git a/python/submit.py b/python/submit.py
index 3c4030c..0685103 100755
--- a/python/submit.py
+++ b/python/submit.py
@@ -6,39 +6,46 @@
# - Change History ist nicht angepasst ans File! Original geben lassen
# - dead code ? what to do?
# - seperate operational and reanlysis for clarification
-# - add correct file description
# - divide in two submits , ondemand und operational
# -
#************************************************************************
-"""
-@Author: Anne Fouilloux (University of Oslo)
-
-@Date: October 2014
-
-@ChangeHistory:
- November 2015 - Leopold Haimberger (University of Vienna):
- - job submission on ecgate and cca
- - job templates suitable for twice daily operational dissemination
-
- February 2018 - Anne Philipp (University of Vienna):
- - applied PEP8 style guide
- - added documentation
- - minor changes in programming style (for consistence)
-
-@License:
- (C) Copyright 2014-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
-@Requirements:
- - A standard python 2.6 or 2.7 installation
-
-@Description:
- Further documentation may be obtained from www.flexpart.eu.
+#*******************************************************************************
+# @Author: Anne Fouilloux (University of Oslo)
+#
+# @Date: October 2014
+#
+# @Change History:
+#
+# November 2015 - Leopold Haimberger (University of Vienna):
+# - job submission on ecgate and cca
+# - job templates suitable for twice daily operational dissemination
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
+# - minor changes in programming style (for consistence)
+#
+# @License:
+# (C) Copyright 2014-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Program Functionality:
+# This program is the main program of flex_extract and controls the
+# program flow.
+# If it is supposed to work locally then it works through the necessary
+# functions getMARSdata and prepareFlexpart. Otherwise it prepares
+# a shell job script which will do the necessary work on the
+# ECMWF server and is submitted via ecaccess-job-submit.
+#
+# @Program Content:
+# - main
+# - submit
+#
+#*******************************************************************************
-"""
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
@@ -47,21 +54,25 @@ import sys
import glob
import subprocess
import inspect
-# add the pythondir path so that python finds its buddies (from flex_extract)
-localpythonpath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
-sys.path.append(localpythonpath)
+# add path to pythonpath so that python finds its buddies
+localpythonpath = os.path.dirname(os.path.abspath(
+ inspect.getfile(inspect.currentframe())))
+if localpythonpath not in sys.path:
+ sys.path.append(localpythonpath)
# software specific classes and modules from flex_extract
from Tools import interpret_args_and_control, normalexit
from getMARSdata import getMARSdata
from prepareFLEXPART import prepareFLEXPART
+
# ------------------------------------------------------------------------------
# FUNCTIONS
# ------------------------------------------------------------------------------
+
def main():
'''
@Description:
- Get the arguments from script call and from Control file.
+ Get the arguments from script call and from CONTROL file.
Decides from the argument "queue" if the local version
is done "queue=None" or the gateway version with "queue=ecgate"
or "queue=cca".
@@ -101,8 +112,8 @@ def submit(jtemplate, c, queue):
the job call and mail report instructions.
Default is "job.temp".
- c: instance of class Control
- Contains all the parameters of control files, which are e.g.:
+ c: instance of class ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
@@ -126,7 +137,7 @@ def submit(jtemplate, c, queue):
lftext = f.read().split('\n')
insert_point = lftext.index('EOF')
- # put all parameters of control instance into a list
+ # put all parameters of ControlFile instance into a list
clist = c.tolist()
colist = [] # operational
mt = 0
diff --git a/python/testsuite.py b/python/testsuite.py
index 217a4e5..199fa89 100755
--- a/python/testsuite.py
+++ b/python/testsuite.py
@@ -1,28 +1,51 @@
#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#************************************************************************
+# TODO AP
+#
+# - provide more tests
+# - provide more documentation
+# -
+#************************************************************************
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#*******************************************************************************
+# @Author: Leopold Haimberger (University of Vienna)
+#
+# @Date: December 2015
+#
+# @Change History:
+#
+# February 2018 - Anne Philipp (University of Vienna):
+# - applied PEP8 style guide
+# - added documentation
#
-# Leopold Haimberger, Dec 2015
+# @License:
+# (C) Copyright 2015-2018.
#
-# Functionality provided: This script triggers the ECMWFDATA test suite. Call with
-# testsuite.py [test group]
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
+# @Program Functionality:
+# This script triggers the ECMWFDATA test suite. Call with
+# testsuite.py [test group]
#
-# Further documentation may be obtained from www.flexpart.eu
+# @Program Content:
#
-# Test groups are specified in testsuite.json
-# in addition to a standard python 2.6 or 2.7 installation the following packages need to be installed
-# ECMWF WebMARS, gribAPI with python enabled, emoslib, ecaccess web toolkit, all available from https://software.ecmwf.int/
-# dateutils
-# matplotlib (optional, for debugging)
+#*******************************************************************************
-import os,sys
+# ------------------------------------------------------------------------------
+# MODULES
+# ------------------------------------------------------------------------------
+import os
+import sys
import json
import subprocess
+# ------------------------------------------------------------------------------
+# PROGRAM
+# ------------------------------------------------------------------------------
try:
- taskfile=open('testsuite.json')
+ taskfile = open('testsuite.json')
except:
print 'could not open suite definition file testsuite.json'
exit()
@@ -32,59 +55,60 @@ if not os.path.isfile('../src/CONVERT2'):
print 'please run "install.py --target=local" first'
exit()
-fprs=os.getenv('FLEXPART_ROOT_SCRIPTS')
+fprs = os.getenv('FLEXPART_ROOT_SCRIPTS')
if fprs is None:
print 'FLEXPART_ROOT_SCRIPTS not set .. some test jobs may fail'
-tasks=json.load(taskfile,encoding='latin-1')
+tasks = json.load(taskfile, encoding='latin-1')
taskfile.close()
if not os.path.exists('../test'):
os.makedirs('../test')
-if len(sys.argv)>1:
- groups=sys.argv[1:]
+if len(sys.argv) > 1:
+ groups = sys.argv[1:]
else:
- groups=['xinstall','default','ops','work','cv','fc']#,'hires']
-jobcounter=0
-jobfailed=0
+ groups = ['xinstall', 'default', 'ops', 'work', 'cv', 'fc']#,'hires']
+jobcounter = 0
+jobfailed = 0
for g in groups:
try:
- tk,tv=g,tasks[g]
+ tk, tv = g, tasks[g]
except:
continue
- garglist=[]
- for ttk,ttv in tv.iteritems():
- if isinstance(ttv,basestring):
- if ttk!='script':
- garglist.append('--'+ttk)
- if '$'==ttv[0]:
+ garglist = []
+ for ttk, ttv in tv.iteritems():
+ if isinstance(ttv, basestring):
+ if ttk != 'script':
+ garglist.append('--' + ttk)
+ if '$' == ttv[0]:
garglist.append(os.path.expandvars(ttv))
else:
garglist.append(ttv)
- for ttk,ttv in tv.iteritems():
- if isinstance(ttv,dict):
- arglist=[]
- for tttk,tttv in ttv.iteritems():
- if isinstance(tttv,basestring):
- arglist.append('--'+tttk)
+ for ttk, ttv in tv.iteritems():
+ if isinstance(ttv, dict):
+ arglist = []
+ for tttk, tttv in ttv.iteritems():
+ if isinstance(tttv, basestring):
+ arglist.append('--' + tttk)
if '$' in tttv[0]:
arglist.append(os.path.expandvars(tttv))
else:
arglist.append(tttv)
- print 'Command: ',' '.join([tv['script']]+garglist+arglist)
- o='../test/'+tk+'_'+ttk+'_'+'_'.join(ttv.keys())
- print 'Output will be sent to ',o
- f=open(o,'w')
+ print 'Command: ', ' '.join([tv['script']] + garglist + arglist)
+ o = '../test/' + tk + '_' + ttk + '_' + '_'.join(ttv.keys())
+ print 'Output will be sent to ', o
+ f = open(o, 'w')
try:
- p=subprocess.check_call([tv['script']]+garglist+arglist,stdout=f,stderr=f)
+ p = subprocess.check_call([tv['script']] + garglist + arglist,
+ stdout=f, stderr=f)
except:
f.write('\nFAILED\n')
print 'FAILED'
- jobfailed+=1
- jobcounter+=1
+ jobfailed += 1
+ jobcounter += 1
f.close()
print 'Test suite tasks completed'
-print str(jobcounter-jobfailed)+' successful, '+str(jobfailed)+' failed'
+print str(jobcounter-jobfailed) + ' successful, ' + str(jobfailed) + ' failed'
print 'If tasks have been submitted via ECACCESS please check emails'
#print tasks
diff --git a/src/Makefile.local.gfortran b/src/Makefile.local.gfortran
index e7a060b..3847d57 100644
--- a/src/Makefile.local.gfortran
+++ b/src/Makefile.local.gfortran
@@ -11,8 +11,8 @@
.s .s~ .sh .sh~ .h .h~ .C .C~ .a
-GRIB_API_INCLUDE_DIR=/usr/local/gcc-4.9.3/grib1.12.3//include
-GRIB_API_LIB= -L/usr/local/gcc-4.9.3/grib1.12.3/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper
+GRIB_API_INCLUDE_DIR=/usr/local/gcc-4.9.3/grib_api-1.14.3//include
+GRIB_API_LIB= -L/usr/local/gcc-4.9.3/grib_api-1.14.3/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper
EMOSLIB=-lemosR64
OPT = -g -O3 -fopenmp
diff --git a/src/Makefile.local.ifort b/src/Makefile.local.ifort
index ef6a78b..6f58a35 100644
--- a/src/Makefile.local.ifort
+++ b/src/Makefile.local.ifort
@@ -11,15 +11,15 @@
.s .s~ .sh .sh~ .h .h~ .C .C~ .a
-GRIB_API_INCLUDE_DIR=/usr/local/ifort/grib1.12.3//include
-GRIB_API_LIB=-openmp -L/usr/local/ifort/grib1.12.3//lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -Bdynamic -lm -ljasper #-lopenjpeg
+GRIB_API_INCLUDE_DIR=/home/srvx1/tmc/TestEnv/Libraries/eccodes-2.6.0_ifort/include
+GRIB_API_LIB= -L/home/srvx1/tmc/TestEnv/Libraries/eccodes-2.6.0_ifort/lib -Bstatic -leccodes_f90 -leccodes -Bdynamic -lm -ljasper
-OPT = -g
+OPT = -g -O3 -mcmodel=medium -unroll -inline -heap-arrays 32
DEBUG = -g
LIB = $(GRIB_API_LIB) -lemosR64 -lgfortran
-FC=ifort -132 -traceback -r8
-F90C=ifort -132 -traceback -r8
+FC=/opt/intel/bin/ifort -132 -traceback -r8
+F90C=/opt/intel/bin/ifort -132 -traceback -r8
FFLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
F90FLAGS = $(OPT) -I. -I$(GRIB_API_INCLUDE_DIR)
@@ -42,13 +42,13 @@ clean:
rm *.o
phgrreal.o: phgrreal.f
- $(F90C) -c -g -O3 -fopenmp phgrreal.f
+ $(F90C) -c -g -O3 phgrreal.f
grphreal.o: grphreal.f
- $(F90C) -c -g -O3 -fopenmp grphreal.f
+ $(F90C) -c -g -O3 grphreal.f
ftrafo.o: ftrafo.f
- $(F90C) -c -g -O3 -fopenmp ftrafo.f
+ $(F90C) -c -g -O3 ftrafo.f
$(BINDIR)/CONVERT2: phgrreal.o grphreal.o ftrafo.o rwGRIB2.o posnam.o preconvert.o
$(F90C) $(DEBUG) $(OPT) -o $(BINDIR)/CONVERT2 ftrafo.o phgrreal.o grphreal.o rwGRIB2.o posnam.o preconvert.o ${LIB}
--
GitLab