diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..7aafb3cecc62724615f0342789ad819faef1a36c
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,37 @@
+#
+# gitignore lists all directories and files which should not be versionized!!!
+#
+
+# these dirs and files shouldn't be versionized at any time in this project!!!
+*.pyc
+*.o
+*.mod
+*.tar*
+/work*
+/src/CONVERT2
+/python/ECMWF_ENV
+
+
+# temporary ignorance of the following files!
+Content_Paper2.txt
+DOCUFPEXTRACT.txt
+FLEXEXTRACT_CONTENT.txt
+NewDocu.odt
+dokuECMWFDATA
+
+/python/.cache/
+/python/CONTROL.temp.backup
+
+/python/SphinxDoc/
+/python/assoc.txt
+/python/classes.dot
+/python/classes.dot.png
+
+/python/packages.dot
+/python/packages.dot.png
+/python/prepare_flexpart.py.lprof
+/python/pylintrc
+/python/testecmwfapi.py
+
+/pythontest/
+
diff --git a/doc/.~lock.ErrorDebug.ods# b/doc/.~lock.ErrorDebug.ods#
new file mode 100644
index 0000000000000000000000000000000000000000..06cf131f74fd43c201de6200ad39a547aa7a8e78
--- /dev/null
+++ b/doc/.~lock.ErrorDebug.ods#
@@ -0,0 +1 @@
+,philipa8,localhost,21.08.2018 16:15,file:///home/philipa8/.config/libreoffice/4;
\ No newline at end of file
diff --git a/doc/ECMWF_FPparameter.xlsx b/doc/ECMWF_FPparameter.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..20207d5a2ffdfa77f4f6686d6fc3332e284e3988
Binary files /dev/null and b/doc/ECMWF_FPparameter.xlsx differ
diff --git a/doc/ErrorDebug.ods b/doc/ErrorDebug.ods
new file mode 100644
index 0000000000000000000000000000000000000000..11547601ff905a06a7628fc10458e8c94cee0f16
Binary files /dev/null and b/doc/ErrorDebug.ods differ
diff --git a/doc/GUIDE.odt b/doc/GUIDE.odt
new file mode 100644
index 0000000000000000000000000000000000000000..edfa22d896f30ff66f03c4c980d8d988e8ca0278
Binary files /dev/null and b/doc/GUIDE.odt differ
diff --git a/doc/SIP.doc b/doc/SIP.doc
new file mode 100644
index 0000000000000000000000000000000000000000..9472fc7fc47a2997d2ec776483fba4b94f1ca981
Binary files /dev/null and b/doc/SIP.doc differ
diff --git a/doc/SIP.pdf b/doc/SIP.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..71e75f355e5aae9f0d9f127860ec61e1895c290c
Binary files /dev/null and b/doc/SIP.pdf differ
diff --git a/doc/SUT_ondemand.doc b/doc/SUT_ondemand.doc
new file mode 100644
index 0000000000000000000000000000000000000000..e8a7594f3227f34a19c794f5b796b63f9505340a
Binary files /dev/null and b/doc/SUT_ondemand.doc differ
diff --git a/doc/SUT_ondemand.pdf b/doc/SUT_ondemand.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..d2b75f3d69a14a0d9e4952c38b993baa6744dcdc
Binary files /dev/null and b/doc/SUT_ondemand.pdf differ
diff --git a/mk_upload_tarball.sh b/mk_upload_tarball.sh
new file mode 100644
index 0000000000000000000000000000000000000000..4266f64ddf6c27dc57e618d7432fb3b0a47bbde1
--- /dev/null
+++ b/mk_upload_tarball.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# @Author: Anne Philipp
+#
+# @Date: June, 7 2018
+#
+# @Description: Makes a tarball for uploading on flexpart.eu
+#
+
+tarname='flex_extract_v7.1.tar.gz'
+
+
+tar --exclude='./src/*.o' --exclude='./src/*.mod' --exclude='./src/CONVERT2' --exclude='./python/*.pyc' --exclude='./python/*.ksh' --exclude='./python/ECMWF_ENV' --exclude='./python/*sh' --exclude='*sh' --exclude='*tar.gz' --exclude='work' --exclude='./python/job.temp' -zcvf $tarname .
diff --git a/python/CONTROL.temp b/python/CONTROL.temp
index c3d097411cfc71a179dc87df88c0407b6f026d7c..b972567b215eeb95a15c4ad666f79263cbdd0e3b 100644
--- a/python/CONTROL.temp
+++ b/python/CONTROL.temp
@@ -1,4 +1,4 @@
-DAY1
+DAY1 20130501
DAY2
DTIME 3
TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
diff --git a/python/CONTROL.test b/python/CONTROL.test
new file mode 100644
index 0000000000000000000000000000000000000000..6a9ce870554144c1ad88b9a724cdf123f9f9e10b
--- /dev/null
+++ b/python/CONTROL.test
@@ -0,0 +1,18 @@
+DTIME 3
+TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
+TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
+STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
+CLASS EI
+STREAM OPER
+EXPVER 1
+GRID 5000
+LEFT -15000
+LOWER 30000
+UPPER 75000
+RIGHT 45000
+LEVELIST 55/to/60
+RESOL 63
+GAUSS 1
+ADDPAR 186/187/188/235/139/39
+PREFIX EItest_
+ECTRANS 1
diff --git a/python/CONTROL.worktest b/python/CONTROL.worktest
new file mode 100644
index 0000000000000000000000000000000000000000..7bd637d8b4a27f63ffc03eb0ea1b4669661afa85
--- /dev/null
+++ b/python/CONTROL.worktest
@@ -0,0 +1,37 @@
+DAY1 20100101
+DAY2
+DTIME 3
+TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
+TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
+STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
+CLASS EI
+STREAM OPER
+NUMBER OFF
+EXPVER 1
+GRID 5000
+LEFT -10000
+LOWER 30000
+UPPER 40000
+RIGHT 10000
+LEVEL 60
+LEVELIST 58/to/60
+RESOL 63
+GAUSS 1
+ACCURACY 16
+OMEGA 0
+OMEGADIFF 0
+ETA 0
+ETADIFF 0
+DPDETA 1
+SMOOTH 0
+FORMAT GRIB1
+ADDPAR 186/187/188/235/139/39
+PREFIX EI
+ECSTORAGE 0
+ECTRANS 1
+ECFSDIR ectmp:/${USER}/econdemand/
+MAILFAIL ${USER}
+MAILOPS ${USER}
+GRIB2FLEXPART 0
+EOF
+
diff --git a/python/CONTROL_EI b/python/CONTROL_EI
new file mode 100644
index 0000000000000000000000000000000000000000..c3d097411cfc71a179dc87df88c0407b6f026d7c
--- /dev/null
+++ b/python/CONTROL_EI
@@ -0,0 +1,37 @@
+DAY1
+DAY2
+DTIME 3
+TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
+TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
+STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
+CLASS EI
+STREAM OPER
+NUMBER OFF
+EXPVER 1
+GRID 5000
+LEFT -15000
+LOWER 30000
+UPPER 75000
+RIGHT 45000
+LEVEL 60
+LEVELIST 55/to/60
+RESOL 63
+GAUSS 1
+ACCURACY 16
+OMEGA 0
+OMEGADIFF 0
+ETA 0
+ETADIFF 0
+DPDETA 1
+SMOOTH 0
+FORMAT GRIB1
+ADDPAR 186/187/188/235/139/39
+PREFIX EI
+ECSTORAGE 0
+ECTRANS 1
+ECFSDIR ectmp:/${USER}/econdemand/
+MAILFAIL ${USER}
+MAILOPS ${USER}
+GRIB2FLEXPART 0
+EOF
+
diff --git a/python/CONTROL_FC b/python/CONTROL_FC
index 11efa7254276eca1aa0893a5e29120b59c171936..e05bac201300f65cb974f9aae68c59bd24b311ef 100644
--- a/python/CONTROL_FC
+++ b/python/CONTROL_FC
@@ -1,9 +1,12 @@
-DAY1 20131107
-DAY2 20131108
-DTIME 1
+DAY1
+DAY2
+DTIME 3
M_TYPE FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC FC
M_TIME 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
M_STEP 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
+ACCTYPE FC
+ACCTIME 00
+ACCMAXSTEP 36
M_MAXSTEP 36
M_LEVEL 137
M_LEVELIST 136/to/137
@@ -12,27 +15,17 @@ M_STREAM OPER
M_NUMBER OFF
M_EXPVER 1
M_GRID 1000
-M_LEFT -179000
-M_LOWER -90000
-M_UPPER 90000
-M_RIGHT 180000
+M_LEFT -25000
+M_LOWER 10000
+M_UPPER 60000
+M_RIGHT 60000
M_RESOL 159
-M_GAUSS 0
-M_ACCURACY 16
-M_OMEGA 0
-M_OMEGADIFF 0
M_ETA 1
-M_ETADIFF 0
M_DPDETA 1
M_SMOOTH 0
-M_FORMAT GRIB1
M_ADDPAR /186/187/188/235/139/39
M_WRF 1
M_CWC 1
PREFIX EE
-ECSTORAGE 1
-ECTRANS 0
-ECFSDIR ectmp:/${USER}/econdemand/
-MAILOPS ${USER}
-MAILFAIL ${USER}
+ECTRANS 1
EOF
diff --git a/python/CONTROL_PF.temp b/python/CONTROL_PF.temp
index f40bdd31acef64a76c023aab0628e3a0683fc768..8eb9cb511a84744878b370f3f9208c93080abc54 100644
--- a/python/CONTROL_PF.temp
+++ b/python/CONTROL_PF.temp
@@ -33,6 +33,6 @@ ECTRANS 1
ECFSDIR ectmp:/${USER}/econdemand/
MAILOPS ${USER}
MAILFAIL ${USER}
-FLEXPARTDIR ${HOME}/ECMWFDATA7.0/flexpart_code
+FLEXPARTDIR ${HOME}/flex_extract_v7.1/flexpart_code
GRIB2FLEXPART grib2flexpart
EOF
diff --git a/python/ControlFile.py b/python/ControlFile.py
index b4bf7121f1e8ce81da076cd9bd4ceefd2dc9cf08..257e072543b77281134b4dcf5123bd1d3dfdb03b 100644
--- a/python/ControlFile.py
+++ b/python/ControlFile.py
@@ -1,9 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#************************************************************************
-# ToDo AP
-# - write a test class
-#************************************************************************
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
#
@@ -18,6 +14,10 @@
# - changed name of class Control to ControlFile for more
# self-explanation naming
# - outsource of class ControlFile
+# - initialisation of class attributes ( to avoid high number of
+# conditional statements and set default values )
+# - divided assignment of attributes and the check of conditions
+# - outsourced the commandline argument assignments to control attributes
#
# @License:
# (C) Copyright 2015-2018.
@@ -35,29 +35,16 @@
#
# @Class Content:
# - __init__
+# - __read_controlfile__
# - __str__
+# - assign_args_to_control
+# - assign_envs_to_control
+# - check_conditions
+# - check_install_conditions
# - to_list
#
# @Class Attributes:
-# - start_date
-# - end_date
-# - accuracy
-# - omega
-# - cwc
-# - omegadiff
-# - etadiff
-# - level
-# - levelist
-# - step
-# - maxstep
-# - prefix
-# - makefile
-# - basetime
-# - date_chunk
-# - grib2flexpart
-# - exedir
-# - flexpart_root_scripts
-# - ecmwfdatadir
+#
#
#*******************************************************************************
@@ -65,6 +52,7 @@
# MODULES
# ------------------------------------------------------------------------------
import os
+import sys
import inspect
# ------------------------------------------------------------------------------
@@ -91,9 +79,10 @@ class ControlFile(object):
def __init__(self, filename):
'''
@Description:
- Initialises the instance of ControlFile class and defines and
- assign all CONTROL file variables. Set default values if
- parameter was not in CONTROL file.
+ Initialises the instance of ControlFile class and defines
+ all class attributes with default values. Afterwards calls
+ function __read_controlfile__ to read parameter from
+ Control file.
@Input:
self: instance of ControlFile class
@@ -102,17 +91,90 @@ class ControlFile(object):
filename: string
Name of CONTROL file.
+ @Return:
+ <nothing>
+ '''
+
+ # list of all possible class attributes and their default values
+ self.controlfile = filename
+ self.start_date = None
+ self.end_date = None
+ self.date_chunk = 3
+ self.dtime = None
+ self.basetime = None
+ self.maxstep = None
+ self.type = None
+ self.time = None
+ self.step = None
+ self.marsclass = None
+ self.stream = None
+ self.number = 'OFF'
+ self.expver = None
+ self.grid = None
+ self.area = ''
+ self.left = None
+ self.lower = None
+ self.upper = None
+ self.right = None
+ self.level = None
+ self.levelist = None
+ self.resol = None
+ self.gauss = 0
+ self.accuracy = 24
+ self.omega = 0
+ self.omegadiff = 0
+ self.eta = 0
+ self.etadiff = 0
+ self.etapar = 77
+ self.dpdeta = 1
+ self.smooth = 0
+ self.format = 'GRIB1'
+ self.addpar = None
+ self.prefix = 'EN'
+ self.cwc = 0
+ self.wrf = 0
+ self.ecfsdir = 'ectmp:/${USER}/econdemand/'
+ self.mailfail = ['${USER}']
+ self.mailops = ['${USER}']
+ self.grib2flexpart = 0
+ self.ecstorage = 0
+ self.ectrans = 0
+ self.inputdir = '../work'
+ self.outputdir = self.inputdir
+ self.ecmwfdatadir = None
+ self.exedir = None
+ self.flexpart_root_scripts = None
+ self.makefile = None
+ self.destination = None
+ self.gateway = None
+ self.ecuid = None
+ self.ecgid = None
+ self.install_target = None
+ self.debug = 0
+
+ self.__read_controlfile__()
+
+ return
+
+ def __read_controlfile__(self):
+ '''
+ @Description:
+ Read CONTROL file and assign all CONTROL file variables.
+
+ @Input:
+ self: instance of ControlFile class
+ Description see class documentation.
+
@Return:
<nothing>
'''
from tools import my_error
# read whole CONTROL file
- with open(filename) as f:
+ with open(self.controlfile) as f:
fdata = f.read().split('\n')
# go through every line and store parameter
- # as class variable
for ldata in fdata:
data = ldata.split()
if len(data) > 1:
@@ -144,9 +206,10 @@ class ControlFile(object):
if var is not None:
data[1] = data[1][:i] + var + data[1][k+1:]
else:
- my_error(None, 'Could not find variable ' +
- data[1][j+1:k] + ' while reading ' +
- filename)
+ my_error(self.mailfail,
+ 'Could not find variable '
+ + data[1][j+1:k] + ' while reading ' +
+ self.controlfile)
setattr(self, data[0].lower() + '_expanded', data[1])
else:
if data[1].lower() != 'none':
@@ -158,36 +221,164 @@ class ControlFile(object):
else:
pass
- # check a couple of necessary attributes if they contain values
- # otherwise set default values
- if not hasattr(self, 'start_date'):
- self.start_date = None
- if not hasattr(self, 'end_date'):
+ # script directory
+ self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile(
+ inspect.currentframe()))) + '/../'
+
+ # Fortran source directory
+ self.exedir = self.ecmwfdatadir + 'src/'
+
+ return
+
+ def __str__(self):
+ '''
+ @Description:
+ Prepares a string which have all the ControlFile
+ class attributes with its associated values.
+ Each attribute is printed in one line and in
+ alphabetical order.
+
+ Example:
+ 'age': 10
+ 'color': 'Spotted'
+ 'kids': 0
+ 'legs': 2
+ 'name': 'Dog'
+ 'smell': 'Alot'
+
+ @Input:
+ self: instance of ControlFile class
+ Description see class documentation.
+
+ @Return:
+ string of ControlFile class attributes with their values
+ '''
+ import collections
+
+ attrs = vars(self)
+ attrs = collections.OrderedDict(sorted(attrs.items()))
+
+ return '\n'.join("%s: %s" % item for item in attrs.items())
+
+ def assign_args_to_control(self, args):
+ '''
+ @Description:
+ Overwrites the existing ControlFile instance attributes with
+ the command line arguments.
+
+ @Input:
+ self: instance of ControlFile class
+ Description see class documentation.
+
+ args: instance of ArgumentParser
+ Contains the commandline arguments from script/program call.
+
+ @Return:
+ <nothing>
+ '''
+
+ # get dictionary of command line parameters and eliminate all
+ # parameters which are None (were not specified)
+ args_dict = vars(args)
+ arguments = {k : args_dict[k] for k in args_dict
+ if args_dict[k] != None}
+
+ # assign all passed command line arguments to ControlFile instance
+ for k, v in arguments.iteritems():
+ setattr(self, str(k), v)
+
+ return
+
+ def assign_envs_to_control(self, envs):
+ '''
+ @Description:
+ Assigns the ECMWF environment parameter.
+
+ @Input:
+ envs: dict of strings
+ Contains the ECMWF environment parameternames "ECUID", "ECGID",
+ "DESTINATION" and "GATEWAY" with its corresponding values.
+ They were read from the file "ECMWF_ENV".
+
+ @Return:
+ <nothing>
+ '''
+
+ for k, v in envs.iteritems():
+ setattr(self, str(k).lower(), str(v))
+
+ return
+
+ def check_conditions(self):
+ '''
+ @Description:
+ Checks a couple of necessary attributes and conditions,
+ such as if they exist and contain values.
+ Otherwise set default values.
+
+ @Input:
+ self: instance of ControlFile class
+ Description see class documentation.
+
+ @Return:
+ <nothing>
+ '''
+ from tools import my_error
+ import numpy as np
+
+ # check for having at least a starting date
+ # otherwise program is not allowed to run
+ if self.start_date is None:
+ print 'start_date specified neither in command line nor ' + \
+ 'in CONTROL file ' + self.controlfile
+ print 'Try "' + sys.argv[0].split('/')[-1] + \
+ ' -h" to print usage information'
+ sys.exit(1)
+
+ # retrieve just one day if end_date isn't set
+ if self.end_date is None:
self.end_date = self.start_date
- if not hasattr(self, 'accuracy'):
- self.accuracy = 24
- if not hasattr(self, 'omega'):
- self.omega = '0'
- if not hasattr(self, 'cwc'):
- self.cwc = '0'
- if not hasattr(self, 'omegadiff'):
- self.omegadiff = '0'
- if not hasattr(self, 'etadiff'):
- self.etadiff = '0'
- if not hasattr(self, 'levelist'):
- if not hasattr(self, 'level'):
- print 'Warning: neither levelist nor level \
- specified in CONTROL file'
+
+ # assure consistency of levelist and level
+ if self.levelist is None:
+ if self.level is None:
+ print 'Warning: neither levelist nor level ' + \
+ 'specified in CONTROL file'
+ sys.exit(1)
else:
self.levelist = '1/to/' + self.level
else:
- if 'to' in self.levelist:
+ if 'to' in self.levelist.lower():
self.level = self.levelist.split('/')[2]
else:
self.level = self.levelist.split('/')[-1]
- if not hasattr(self, 'maxstep'):
- # find out maximum step
+ # if area was provided at command line
+ # decompse area into its 4 components
+ if self.area:
+ afloat = '.' in self.area
+ l = self.area.split('/')
+ if afloat:
+ for i, item in enumerate(l):
+ item = str(int(float(item) * 1000))
+ self.upper, self.left, self.lower, self.right = l
+
+ # prepare step for correct usage
+ if '/' in self.step:
+ l = self.step.split('/')
+ if 'to' in self.step.lower():
+ if 'by' in self.step.lower():
+ ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4]))
+ self.step = ['{:0>3}'.format(i) for i in ilist]
+ else:
+ my_error(self.mailfail, self.step + ':\n' +
+ 'if "to" is used, please use "by" as well')
+ else:
+ self.step = l
+
+ # if maxstep wasn't provided
+ # search for it in the "step" parameter
+ if self.maxstep is None:
self.maxstep = 0
for s in self.step:
if int(s) > self.maxstep:
@@ -195,50 +386,86 @@ class ControlFile(object):
else:
self.maxstep = int(self.maxstep)
- if not hasattr(self, 'prefix'):
- self.prefix = 'EN'
- if not hasattr(self, 'makefile'):
- self.makefile = None
- if not hasattr(self, 'basetime'):
- self.basetime = None
- if not hasattr(self, 'date_chunk'):
- self.date_chunk = '3'
- if not hasattr(self, 'grib2flexpart'):
- self.grib2flexpart = '0'
+ # set root scripts since it is needed later on
+ if not self.flexpart_root_scripts:
+ self.flexpart_root_scripts = self.ecmwfdatadir
- # script directory
- self.ecmwfdatadir = os.path.dirname(os.path.abspath(inspect.getfile(
- inspect.currentframe()))) + '/../'
- # Fortran source directory
- self.exedir = self.ecmwfdatadir + 'src/'
+ if not isinstance(self.mailfail, list):
+ if ',' in self.mailfail:
+ self.mailfail = self.mailfail.split(',')
+ elif ' ' in self.mailfail:
+ self.mailfail = self.mailfail.split()
+ else:
+ self.mailfail = [self.mailfail]
- # FLEXPART directory
- if not hasattr(self, 'flexpart_root_scripts'):
- self.flexpart_root_scripts = self.ecmwfdatadir
+ if not isinstance(self.mailops, list):
+ if ',' in self.mailops:
+ self.mailops = self.mailops.split(',')
+ elif ' ' in self.mailops:
+ self.mailops = self.mailops.split()
+ else:
+ self.mailops = [self.mailops]
+
+ if not self.gateway or not self.destination or \
+ not self.ecuid or not self.ecgid:
+ print '\nEnvironment variables GATWAY, DESTINATION, ECUID and ' + \
+ 'ECGID were not set properly!'
+ print 'Please check for excistence of file "ECMWF_ENV" in the ' + \
+ 'python directory!'
+ sys.exit(1)
return
- def __str__(self):
+ def check_install_conditions(self):
'''
@Description:
- Prepares a single string with all the comma seperated ControlFile
- class attributes including their values.
-
- Example:
- {'kids': 0, 'name': 'Dog', 'color': 'Spotted',
- 'age': 10, 'legs': 2, 'smell': 'Alot'}
+ Checks a couple of necessary attributes and conditions
+ for the installation such as if they exist and contain values.
+ Otherwise set default values.
@Input:
self: instance of ControlFile class
Description see class documentation.
@Return:
- string of ControlFile class attributes with their values
+ <nothing>
'''
- attrs = vars(self)
+ if self.install_target and \
+ self.install_target not in ['local', 'ecgate', 'cca']:
+ print 'ERROR: unknown or missing installation target '
+ print 'target: ', self.install_target
+ print 'please specify correct installation target \
+ (local | ecgate | cca)'
+ print 'use -h or --help for help'
+ sys.exit(1)
+
+ if self.install_target and self.install_target != 'local':
+ if not self.ecgid or not self.ecuid or \
+ not self.gateway or not self.destination:
+ print 'Please enter your ECMWF user id and group id as well as \
+ the \nname of the local gateway and the ectrans \
+ destination '
+ print 'with command line options --ecuid --ecgid \
+ --gateway --destination'
+ print 'Try "' + sys.argv[0].split('/')[-1] + \
+ ' -h" to print usage information'
+ print 'Please consult ecaccess documentation or ECMWF user \
+ support for further details'
+ sys.exit(1)
+
+ if not self.flexpart_root_scripts:
+ self.flexpart_root_scripts = '${HOME}'
+ else:
+ self.flexpart_root_scripts = self.flexpart_root_scripts
+ else:
+ if not self.flexpart_root_scripts:
+ self.flexpart_root_scripts = '../'
- return ', '.join("%s: %s" % item for item in attrs.items())
+ if not self.makefile:
+ self.makefile = 'Makefile.gfortran'
+
+ return
def to_list(self):
'''
@@ -258,7 +485,10 @@ class ControlFile(object):
"ecmwfdatadir" and "flexpart_root_scripts".
'''
- attrs = vars(self)
+ import collections
+
+ attrs = collections.OrderedDict(sorted(vars(self).items()))
+
l = list()
for item in attrs.items():
@@ -282,23 +512,3 @@ class ControlFile(object):
return sorted(l)
- # def to_dict(self):
- # '''
-
- # '''
- # parameters_dict = vars(self)
-
- # # remove unneeded parameter
- # parameters_dict.pop('_expanded', None)
- # parameters_dict.pop('exedir', None)
- # parameters_dict.pop('flexpart_root_scripts', None)
- # parameters_dict.pop('ecmwfdatadir', None)
-
- # parameters_dict_str = {}
- # for key, value in parameters_dict.iteritems():
- # if isinstance(value, list):
- # parameters_dict_str[str(key)] = get_list_as_string(value, ' ')
- # else:
- # parameters_dict_str[str(key)] = str(value)
-
- # return parameters_dict_str
diff --git a/python/EcFlexpart.py b/python/EcFlexpart.py
index bca2d1cd44dfe5a025ec04bb500e3455868243ce..e030b7a43f9e7066edfb55508a183bf96d777a6b 100644
--- a/python/EcFlexpart.py
+++ b/python/EcFlexpart.py
@@ -1,15 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#************************************************************************
-# ToDo AP
-# - specifiy file header documentation
-# - add class description in header information
-# - apply classtests
-# - add references to ECMWF specific software packages
-# - add describtion of deacc_fluxes
-# - change name of func deacc ( weil disagg )
-# - add desc of retrieve function
-#************************************************************************
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
#
@@ -150,7 +140,7 @@ class EcFlexpart(object):
see documentation.
fluxes: boolean, optional
- Decides if a the flux parameter settings are stored or
+ Decides if the flux parameter settings are stored or
the rest of the parameter list.
Default value is False.
@@ -174,7 +164,7 @@ class EcFlexpart(object):
self.basetime = c.basetime
self.dtime = c.dtime
i = 0
- if fluxes is True and c.maxstep < 24:
+ if fluxes and c.maxstep <= 24:
# no forecast beyond one day is needed!
# Thus, prepare flux data manually as usual
# with only forecast fields with start times at 00/12
@@ -191,7 +181,7 @@ class EcFlexpart(object):
if c.basetime == '00':
btlist = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0]
- if i % int(c.dtime) == 0 and c.maxstep > 24 or i in btlist:
+ if i % int(c.dtime) == 0 and (i in btlist or c.maxstep > 24):
if ty not in self.types.keys():
self.types[ty] = {'times': '', 'steps': ''}
@@ -207,7 +197,6 @@ class EcFlexpart(object):
self.types[ty]['steps'] += st
i += 1
-
self.marsclass = c.marsclass
self.stream = c.stream
self.number = c.number
@@ -303,15 +292,15 @@ class EcFlexpart(object):
self.params['GG__ML'] = ['U/V/D/77', 'ML', self.glevelist, \
'{}'.format((int(self.resol) + 1) / 2)]
- if c.omega == '1':
+ if hasattr(c, 'omega') and c.omega == '1':
self.params['OG__ML'][0] += '/W'
# add cloud water content if necessary
- if c.cwc == '1':
+ if hasattr(c, 'cwc') and c.cwc == '1':
self.params['OG__ML'][0] += '/CLWC/CIWC'
# add vorticity and geopotential height for WRF if necessary
- if c.wrf == '1':
+ if hasattr(c, 'wrf') and c.wrf == '1':
self.params['OG__ML'][0] += '/Z/VO'
if '/D' not in self.params['OG__ML'][0]:
self.params['OG__ML'][0] += '/D'
@@ -739,7 +728,7 @@ class EcFlexpart(object):
p = subprocess.check_call(['ectrans', '-overwrite', '-gateway',
c.gateway, '-remote', c.destination,
'-source', ofile])
- print('ectrans:', p)
+ #print('ectrans:', p)
if int(c.ecstorage) == 1 and c.ecapi is False:
for ofile in self.outputfilelist:
@@ -889,7 +878,7 @@ class EcFlexpart(object):
'17':None, '19':None, '21':None, '22':None, '20':None}
for prod in product(*index_vals):
- # flag for Fortran program CONVERT2, initially False
+ # flag for Fortran program CONVERT2 and file merging
convertFlag = False
print 'current prod: ', prod
# e.g. prod = ('20170505', '0', '12')
@@ -905,10 +894,13 @@ class EcFlexpart(object):
# if there is data for this product combination
# prepare some date and time parameter before reading the data
if gid is not None:
- # Fortran program CONVERT2 is only done if gid at this time is
- # not None, therefore save information in convertFlag
+ # Combine all temporary data files into final grib file if
+ # gid is at least one time not None. Therefore set convertFlag
+ # to save information. The fortran program CONVERT2 is also
+ # only done if convertFlag is True
convertFlag = True
# remove old fort.* files and open new ones
+ # they are just valid for a single product
for k, f in fdict.iteritems():
silent_remove(c.inputdir + "/fort." + k)
fdict[k] = open(c.inputdir + '/fort.' + k, 'w')
@@ -921,7 +913,6 @@ class EcFlexpart(object):
timestamp = datetime.strptime(cdate + '{:0>2}'.format(time/100),
'%Y%m%d%H')
timestamp += timedelta(hours=int(step))
-
cdateH = datetime.strftime(timestamp, '%Y%m%d%H')
if c.basetime is not None:
@@ -965,7 +956,7 @@ class EcFlexpart(object):
gridtype = grib_get(gid, 'gridType')
levtype = grib_get(gid, 'typeOfLevel')
if paramId == 133 and gridtype == 'reduced_gg':
- # Relative humidity (Q.grb) is used as a template only
+ # Specific humidity (Q.grb) is used as a template only
# so we need the first we "meet"
with open(c.inputdir + '/fort.18', 'w') as fout:
grib_write(gid, fout)
@@ -1027,8 +1018,8 @@ class EcFlexpart(object):
print 'Parameter 77 (etadot) is missing, most likely it is \
not available for this type or date/time\n'
print 'Check parameters CLASS, TYPE, STREAM, START_DATE\n'
- my_error(c, 'fort.21 is empty while parameter eta is set \
- to 1 in CONTROL file')
+ my_error(c.mailfail, 'fort.21 is empty while parameter eta \
+ is set to 1 in CONTROL file')
# create the corresponding output file fort.15
# (generated by CONVERT2) + fort.16 (paramId 167 and 168)
@@ -1060,14 +1051,15 @@ class EcFlexpart(object):
with open(fnout, 'wb') as fout:
for f in flist:
- shutil.copyfileobj(open(c.inputdir + '/' + f, 'rb'), fout)
+ shutil.copyfileobj(
+ open(c.inputdir + '/' + f, 'rb'), fout)
if c.omega == '1':
with open(c.outputdir + '/OMEGA', 'wb') as fout:
shutil.copyfileobj(
open(c.inputdir + '/fort.25', 'rb'), fout)
- if c.wrf == '1':
+ if hasattr(c, 'wrf') and c.wrf == '1':
fwrf.close()
grib_index_release(iid)
@@ -1306,5 +1298,4 @@ class EcFlexpart(object):
grib_index_release(iid)
- exit()
return
diff --git a/python/GribTools.py b/python/GribTools.py
index c977d4ac28b7142c96adf91eec90808da1515f30..a68d1a5485f1b7e3039081ec7fb176d90466ab06 100644
--- a/python/GribTools.py
+++ b/python/GribTools.py
@@ -308,7 +308,6 @@ class GribTools(object):
if iid is None:
iid = grib_index_new_from_file(filename, index_keys)
else:
- print 'in else zweig'
grib_index_add_file(iid, filename)
if iid is not None:
diff --git a/python/MarsRetrieval.py b/python/MarsRetrieval.py
index eccac65c4c38444d29beec73af375d4fe5b07774..54a32b06513e9428f27cf0771b148e2c98e86893 100644
--- a/python/MarsRetrieval.py
+++ b/python/MarsRetrieval.py
@@ -78,7 +78,7 @@ class MarsRetrieval(object):
'''
- def __init__(self, server, marsclass="ei", dtype="", levtype="",
+ def __init__(self, server, marsclass="ei", type="", levtype="",
levelist="", repres="", date="", resol="", stream="",
area="", time="", step="", expver="1", number="",
accuracy="", grid="", gaussian="", target="",
@@ -106,7 +106,7 @@ class MarsRetrieval(object):
E4 (ERA40), OD (Operational archive), ea (ERA5).
Default is the ERA-Interim dataset "ei".
- dtype: string, optional
+ type: string, optional
Determines the type of fields to be retrieved.
Selects between observations, images or fields.
Examples for fields: Analysis (an), Forecast (fc),
@@ -285,7 +285,7 @@ class MarsRetrieval(object):
self.server = server
self.marsclass = marsclass
- self.dtype = dtype
+ self.type = type
self.levtype = levtype
self.levelist = levelist
self.repres = repres
diff --git a/python/UioFiles.py b/python/UioFiles.py
index dbb5409cc6112ad98cffdceb4dfea42f8899d037..fe6995320308bfb88805745ac5753ffbdc9dd799 100644
--- a/python/UioFiles.py
+++ b/python/UioFiles.py
@@ -33,7 +33,8 @@
#
# @Class Content:
# - __init__
-# - list_files
+# - __str__
+# - __list_files__
# - delete_files
#
# @Class Attributes:
@@ -50,7 +51,7 @@ import fnmatch
# software specific module from flex_extract
#import profiling
-from tools import silent_remove
+from tools import silent_remove, get_list_as_string
# ------------------------------------------------------------------------------
# CLASS
@@ -65,7 +66,7 @@ class UioFiles(object):
# --------------------------------------------------------------------------
# CLASS FUNCTIONS
# --------------------------------------------------------------------------
- def __init__(self, pattern):
+ def __init__(self, path, pattern):
'''
@Description:
Assignes a specific pattern for these files.
@@ -74,6 +75,9 @@ class UioFiles(object):
self: instance of UioFiles
Description see class documentation.
+ path: string
+ Directory where to list the files.
+
pattern: string
Regular expression pattern. For example: '*.grb'
@@ -81,13 +85,16 @@ class UioFiles(object):
<nothing>
'''
+ self.path = path
self.pattern = pattern
self.files = None
+ self.__list_files__(self.path)
+
return
#@profiling.timefn
- def list_files(self, path, callid=0):
+ def __list_files__(self, path, callid=0):
'''
@Description:
Lists all files in the directory with the matching
@@ -98,7 +105,7 @@ class UioFiles(object):
Description see class documentation.
path: string
- Directory where to list the files.
+ Path to the files.
callid: integer
Id which tells the function if its the first call
@@ -128,10 +135,30 @@ class UioFiles(object):
# do recursive calls for sub-direcorties
if subdirs:
for subdir in subdirs:
- self.list_files(os.path.join(path, subdir), callid=1)
+ self.__list_files__(os.path.join(path, subdir), callid=1)
return
+ def __str__(self):
+ '''
+ @Description:
+ Converts the list of files into a single string.
+ The entries are sepereated by "," sign.
+
+ @Input:
+ self: instance of UioFiles
+ Description see class documentation.
+
+ @Return:
+ files_string: string
+ The content of the list as a single string.
+ '''
+
+ filenames = [os.path.basename(f) for f in self.files]
+ files_string = get_list_as_string(filenames, concatenate_sign=', ')
+
+ return files_string
+
def delete_files(self):
'''
@Description:
diff --git a/python/_config.py b/python/_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..93f29081a11654963f21b5eca89aa71aa036c846
--- /dev/null
+++ b/python/_config.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#*******************************************************************************
+# @Author: Anne Philipp (University of Vienna)
+#
+# @Date: August 2018
+#
+# @Change History:
+#
+# @License:
+# (C) Copyright 2014-2018.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# @Description:
+# Contains constant value parameter for flex_extract.
+#
+#*******************************************************************************
+
+# ------------------------------------------------------------------------------
+# MODULES
+# ------------------------------------------------------------------------------
+import os
+import sys
+import inspect
+
+
+_VERSION_STR = '7.1'
+
+
+# add path to pythonpath
+LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
+ inspect.getfile(inspect.currentframe())))
+if LOCAL_PYTHON_PATH not in sys.path:
+ sys.path.append(LOCAL_PYTHON_PATH)
diff --git a/python/compilejob.ksh b/python/compilejob.ksh
index 3afc4e4e8eb0bb73005f55f7fe720cdabf3fbccc..027c087e944751812bb448e6f6dc99f73ea5640d 100644
--- a/python/compilejob.ksh
+++ b/python/compilejob.ksh
@@ -4,7 +4,7 @@
# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server
# start with sbatch NAME_OF_THIS_FILE directly on machine
-#SBATCH --workdir=/scratch/ms/spatlh00/lh0
+#SBATCH --workdir=/scratch/ms/at/km4a
#SBATCH --qos=normal
#SBATCH --job-name=flex_ecmwf
#SBATCH --output=flex_ecmwf.%j.out
@@ -32,11 +32,8 @@ case $HOST in
module unload emos
module load grib_api/1.14.5
module load emos/437-r64
-export FLEXPART_ROOT_SCRIPTS=$HOME
-# export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
-# export PATH=${PATH}:$ECMWFDATA/python
- export MAKEFILE=Makefile.gfortran
+export FLEXPART_ROOT_SCRIPTS=${HOME}
+export MAKEFILE=Makefile.gfortran
;;
*cca*)
module switch PrgEnv-cray PrgEnv-intel
@@ -48,17 +45,14 @@ export FLEXPART_ROOT_SCRIPTS=$HOME
echo $HOME | awk -F / '{print $1, $2, $3, $4}'
export GROUP=`echo $HOME | awk -F / '{print $4}'`
export SCRATCH=/scratch/ms/${GROUP}/${USER}
-export FLEXPART_ROOT_SCRIPTS=$HOME
-# export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
-# export PATH=${PATH}:$ECMWFDATA/python
- export MAKEFILE=Makefile.CRAY
+export FLEXPART_ROOT_SCRIPTS=${HOME}
+export MAKEFILE=Makefile.gfortran
;;
esac
-mkdir -p $FLEXPART_ROOT_SCRIPTS/ECMWFDATA$VERSION
-cd $FLEXPART_ROOT_SCRIPTS/ECMWFDATA$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory
-tar -xvf $HOME/ECMWFDATA$VERSION.tar
+mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
+cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory
+tar -xvf $HOME/flex_extract_v$VERSION.tar
cd src
\rm *.o *.mod CONVERT2
make -f $MAKEFILE >flexcompile 2>flexcompile
diff --git a/python/compilejob.temp b/python/compilejob.temp
index e699a01daeaa2487c8f3d446652f2d475bd219c5..715308b3d361e2b0f5c088b344ecde539d74243e 100644
--- a/python/compilejob.temp
+++ b/python/compilejob.temp
@@ -33,9 +33,6 @@ case $HOST in
module load grib_api/1.14.5
module load emos/437-r64
export FLEXPART_ROOT_SCRIPTS=
-# export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
-# export PATH=${PATH}:$ECMWFDATA/python
export MAKEFILE=Makefile.gfortran
;;
*cca*)
@@ -49,16 +46,13 @@ case $HOST in
export GROUP=`echo $HOME | awk -F / '{print $4}'`
export SCRATCH=/scratch/ms/${GROUP}/${USER}
export FLEXPART_ROOT_SCRIPTS=
-# export ECMWFDATA=$FLEXPART_ROOT/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
-# export PATH=${PATH}:$ECMWFDATA/python
export MAKEFILE=Makefile.CRAY
;;
esac
-mkdir -p $FLEXPART_ROOT_SCRIPTS/ECMWFDATA$VERSION
-cd $FLEXPART_ROOT_SCRIPTS/ECMWFDATA$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory
-tar -xvf $HOME/ECMWFDATA$VERSION.tar
+mkdir -p $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION
+cd $FLEXPART_ROOT_SCRIPTS/flex_extract_v$VERSION # if FLEXPART_ROOT is not set this means cd to the home directory
+tar -xvf $HOME/flex_extract_v$VERSION.tar
cd src
\rm *.o *.mod CONVERT2
make -f $MAKEFILE >flexcompile 2>flexcompile
diff --git a/python/disaggregation.py b/python/disaggregation.py
index 9f285c13a7ab186b96b0625ae73507a9be28b284..aa84eaf0c28110f6772ff3e7a48fd411082d52a4 100644
--- a/python/disaggregation.py
+++ b/python/disaggregation.py
@@ -1,16 +1,12 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#************************************************************************
-# ToDo AP
-# - check alist of size 4 ?
-# - write a test, IMPORTANT
-#************************************************************************
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
#
# @Date: March 2018
#
# @Change History:
+#
# November 2015 - Leopold Haimberger (University of Vienna):
# - migration of the methods dapoly and darain from Fortran
# (flex_extract_v6 and earlier) to Python
@@ -28,7 +24,7 @@
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# @Module Description:
-# disaggregationregation of deaccumulated flux data from an ECMWF model FG field.
+# disaggregation of deaccumulated flux data from an ECMWF model FG field.
# Initially the flux data to be concerned are:
# - large-scale precipitation
# - convective precipitation
diff --git a/python/get_mars_data.py b/python/get_mars_data.py
index 14f6c03f2fbf02b20f52d3d60a20ae04bb1988fb..f059ed4cb6b452b02fc30b7ff14650326a91d552 100755
--- a/python/get_mars_data.py
+++ b/python/get_mars_data.py
@@ -21,7 +21,7 @@
# (necessary for better documentation with docstrings for later
# online documentation)
# - use of UIFiles class for file selection and deletion
-
+#
#
# @License:
# (C) Copyright 2014-2018.
@@ -38,6 +38,7 @@
# @Program Content:
# - main
# - get_mars_data
+# - do_retrievement
#
#*******************************************************************************
@@ -55,7 +56,7 @@ except ImportError:
ecapi = False
# software specific classes and modules from flex_extract
-from tools import my_error, normal_exit, interpret_args_and_control
+from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
from EcFlexpart import EcFlexpart
from UioFiles import UioFiles
@@ -81,9 +82,28 @@ def main():
@Return:
<nothing>
'''
- args, c = interpret_args_and_control()
+
+ args = get_cmdline_arguments()
+
+ try:
+ c = ControlFile(args.controlfile)
+ except IOError:
+ try:
+ c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
+ except IOError:
+ print 'Could not read CONTROL file "' + args.controlfile + '"'
+ print 'Either it does not exist or its syntax is wrong.'
+ print 'Try "' + sys.argv[0].split('/')[-1] + \
+ ' -h" to print usage information'
+ sys.exit(1)
+
+ env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
+ c.assign_args_to_control(args, env_parameter)
+ c.assign_envs_to_control(env_parameter)
+ c.check_conditions()
+
get_mars_data(c)
- normal_exit(c)
+ normal_exit(c.mailfail, 'Done!')
return
@@ -128,57 +148,60 @@ def get_mars_data(c):
c.ecapi = ecapi
print 'ecapi: ', c.ecapi
+ # basetime geht rückwärts
+
+ # if basetime 00
+ # dann wird von 12 am vortag bis 00 am start tag geholt
+ # aber ohne 12 selbst sondern 12 + step
+
+ # if basetime 12
+ # dann wird von 00 + step bis 12 am start tag geholt
+
+ # purer forecast wird vorwärts bestimmt.
+ # purer forecast mode ist dann wenn größer 24 stunden
+ # wie kann das noch festgestellt werden ????
+ # nur FC und steps mehr als 24 ?
+ # die einzige problematik beim reinen forecast ist die benennung der files!
+ # also sobald es Tagesüberschneidungen gibt
+ # allerdings ist das relevant und ersichtlich an den NICHT FLUSS DATEN
+
+
# set start date of retrieval period
start = datetime.date(year=int(c.start_date[:4]),
month=int(c.start_date[4:6]),
day=int(c.start_date[6:]))
startm1 = start - datetime.timedelta(days=1)
- if c.basetime == '00':
- start = startm1
# set end date of retrieval period
end = datetime.date(year=int(c.end_date[:4]),
month=int(c.end_date[4:6]),
day=int(c.end_date[6:]))
- if c.basetime == '00' or c.basetime == '12':
- endp1 = end + datetime.timedelta(days=1)
- else:
- endp1 = end + datetime.timedelta(days=2)
- # set time period of one single retrieval
+ # set time period for one single retrieval
datechunk = datetime.timedelta(days=int(c.date_chunk))
+ if c.basetime == '00':
+ start = startm1
+
+ if c.basetime == '00' or c.basetime == '12':
+ # endp1 = end + datetime.timedelta(days=1)
+ endp1 = end
+ else:
+ # endp1 = end + datetime.timedelta(days=2)
+ endp1 = end + datetime.timedelta(days=1)
+
# -------------- flux data ------------------------------------------------
print 'removing old flux content of ' + c.inputdir
- tobecleaned = UioFiles('*_acc_*.' + str(os.getppid()) + '.*.grb')
- tobecleaned.list_files(c.inputdir)
+ tobecleaned = UioFiles(c.inputdir,
+ '*_acc_*.' + str(os.getppid()) + '.*.grb')
tobecleaned.delete_files()
- # if forecast for maximum one day (upto 23h) are to be retrieved,
+ # if forecast for maximum one day (upto 24h) are to be retrieved,
# collect accumulation data (flux data)
# with additional days in the beginning and at the end
# (used for complete disaggregation of original period)
- if c.maxstep < 24:
- day = startm1
- while day < endp1:
- # retrieve MARS data for the whole period
- flexpart = EcFlexpart(c, fluxes=True)
- tmpday = day + datechunk - datetime.timedelta(days=1)
- if tmpday < endp1:
- dates = day.strftime("%Y%m%d") + "/to/" + \
- tmpday.strftime("%Y%m%d")
- else:
- dates = day.strftime("%Y%m%d") + "/to/" + \
- end.strftime("%Y%m%d")
-
- print "retrieve " + dates + " in dir " + c.inputdir
-
- try:
- flexpart.retrieve(server, dates, c.inputdir)
- except IOError:
- my_error(c, 'MARS request failed')
-
- day += datechunk
+ if c.maxstep <= 24:
+ do_retrievement(c, server, startm1, endp1, datechunk, fluxes=True)
# if forecast data longer than 24h are to be retrieved,
# collect accumulation data (flux data)
@@ -186,38 +209,69 @@ def get_mars_data(c):
# (disaggregation will be done for the
# exact time period with boundary conditions)
else:
- day = start
- while day <= end:
- # retrieve MARS data for the whole period
- flexpart = EcFlexpart(c, fluxes=True)
- tmpday = day + datechunk - datetime.timedelta(days=1)
- if tmpday < end:
- dates = day.strftime("%Y%m%d") + "/to/" + \
- tmpday.trftime("%Y%m%d")
- else:
- dates = day.strftime("%Y%m%d") + "/to/" + \
- end.strftime("%Y%m%d")
-
- print "retrieve " + dates + " in dir " + c.inputdir
-
- try:
- flexpart.retrieve(server, dates, c.inputdir)
- except IOError:
- my_error(c, 'MARS request failed')
-
- day += datechunk
+ do_retrievement(c, server, start, end, datechunk, fluxes=True)
# -------------- non flux data --------------------------------------------
print 'removing old non flux content of ' + c.inputdir
- tobecleaned = UioFiles('*__*.' + str(os.getppid()) + '.*.grb')
- tobecleaned.list_files(c.inputdir)
+ tobecleaned = UioFiles(c.inputdir,
+ '*__*.' + str(os.getppid()) + '.*.grb')
tobecleaned.delete_files()
+ do_retrievement(c, server, start, end, datechunk, fluxes=False)
+
+ return
+
+def do_retrievement(c, server, start, end, delta_t, fluxes=False):
+ '''
+ @Description:
+ Divides the complete retrieval period in smaller chunks and
+ retrieves the data from MARS.
+
+ @Input:
+ c: instance of ControlFile
+ Contains all the parameters of CONTROL file, which are e.g.:
+ DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
+ STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
+ LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
+ OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
+ ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
+ MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
+ DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
+
+ For more information about format and content of the parameter
+ see documentation.
+
+ server: instance of ECMWFService
+ The server connection to ECMWF
+
+ start: instance of datetime
+ The start date of the retrieval.
+
+ end: instance of datetime
+ The end date of the retrieval.
+
+ delta_t: instance of datetime
+ Delta_t +1 is the maximal time period of a single
+ retrieval.
+
+ fluxes: boolean, optional
+ Decides if the flux parameters are to be retrieved or
+ the rest of the parameter list.
+ Default value is False.
+
+ @Return:
+ <nothing>
+ '''
+
+ # since actual day also counts as one day,
+ # we only need to add datechunk - 1 days to retrieval
+ # for a period
+ delta_t_m1 = delta_t - datetime.timedelta(days=1)
+
day = start
while day <= end:
- # retrieve all non flux MARS data for the whole period
- flexpart = EcFlexpart(c, fluxes=False)
- tmpday = day + datechunk - datetime.timedelta(days=1)
+ flexpart = EcFlexpart(c, fluxes)
+ tmpday = day + delta_t_m1
if tmpday < end:
dates = day.strftime("%Y%m%d") + "/to/" + \
tmpday.strftime("%Y%m%d")
@@ -230,9 +284,9 @@ def get_mars_data(c):
try:
flexpart.retrieve(server, dates, c.inputdir)
except IOError:
- my_error(c, 'MARS request failed')
+ my_error(c.mailfail, 'MARS request failed')
- day += datechunk
+ day += delta_t
return
diff --git a/python/install.py b/python/install.py
index 835b506faff68461ba5e06f7496b3e9227b24074..31074b3305e4d90e69db5ebea450f9b4e009b9e8 100755
--- a/python/install.py
+++ b/python/install.py
@@ -1,11 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#************************************************************************
-# ToDo AP
-# - create a class Installation and divide installation in 3 subdefs for
-# ecgate, local and cca seperatly
-# - Change History ist nicht angepasst ans File! Original geben lassen
-#************************************************************************
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
#
@@ -36,8 +30,14 @@
#
# @Program Content:
# - main
-# - install_args_and_control
+# - get_install_cmdline_arguments
# - install_via_gateway
+# - mk_tarball
+# - mk_env_vars
+# - mk_compilejob
+# - mk_job_template
+# - delete_convert_build
+# - make_convert_build
#
#*******************************************************************************
@@ -46,13 +46,14 @@
# ------------------------------------------------------------------------------
import os
import sys
-import glob
import subprocess
import inspect
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
# software specific classes and modules from flex_extract
from ControlFile import ControlFile
+from UioFiles import UioFiles
+from tools import make_dir, put_file_to_ecserver, submit_job_to_ecserver
# add path to pythonpath so that python finds its buddies
LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
@@ -60,6 +61,8 @@ LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
if LOCAL_PYTHON_PATH not in sys.path:
sys.path.append(LOCAL_PYTHON_PATH)
+_VERSION_STR = '7.1'
+
# ------------------------------------------------------------------------------
# FUNCTIONS
# ------------------------------------------------------------------------------
@@ -77,24 +80,29 @@ def main():
'''
os.chdir(LOCAL_PYTHON_PATH)
- args, c = install_args_and_control()
+ args = get_install_cmdline_arguments()
- if args.install_target is not None:
- install_via_gateway(c, args.install_target)
- else:
- print 'Please specify installation target (local|ecgate|cca)'
- print 'use -h or --help for help'
+ try:
+ c = ControlFile(args.controlfile)
+ except IOError:
+ print 'Could not read CONTROL file "' + args.controlfile + '"'
+ print 'Either it does not exist or its syntax is wrong.'
+ print 'Try "' + sys.argv[0].split('/')[-1] + \
+ ' -h" to print usage information'
+ exit(1)
- sys.exit()
+ c.assign_args_to_control(args)
+ c.check_install_conditions()
- return
+ install_via_gateway(c)
+ return
-def install_args_and_control():
+def get_install_cmdline_arguments():
'''
@Description:
- Assigns the command line arguments for installation and reads
- CONTROL file content. Apply default values for non mentioned arguments.
+ Decomposes the command line arguments and assigns them to variables.
+ Apply default values for non mentioned arguments.
@Input:
<nothing>
@@ -102,42 +110,32 @@ def install_args_and_control():
@Return:
args: instance of ArgumentParser
Contains the commandline arguments from script/program call.
-
- c: instance of class ControlFile
- Contains all necessary information of a CONTROL file. The parameters
- are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
- NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
- RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
- SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
- ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR
- For more information about format and content of the parameter see
- documentation.
'''
- parser = ArgumentParser(description='Install ECMWFDATA software locally or \
+ parser = ArgumentParser(description='Install flex_extract software locally or \
on ECMWF machines',
formatter_class=ArgumentDefaultsHelpFormatter)
- parser.add_argument('--target', dest='install_target',
+ parser.add_argument('--target', dest='install_target', default=None,
help="Valid targets: local | ecgate | cca , \
the latter two are at ECMWF")
- parser.add_argument("--makefile", dest="makefile",
+ parser.add_argument("--makefile", dest="makefile", default=None,
help='Name of Makefile to use for compiling CONVERT2')
- parser.add_argument("--ecuid", dest="ecuid",
+ parser.add_argument("--ecuid", dest="ecuid", default=None,
help='user id at ECMWF')
- parser.add_argument("--ecgid", dest="ecgid",
+ parser.add_argument("--ecgid", dest="ecgid", default=None,
help='group id at ECMWF')
- parser.add_argument("--gateway", dest="gateway",
+ parser.add_argument("--gateway", dest="gateway", default=None,
help='name of local gateway server')
- parser.add_argument("--destination", dest="destination",
+ parser.add_argument("--destination", dest="destination", default=None,
help='ecaccess destination, e.g. leo@genericSftp')
parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
- help="FLEXPART root directory on ECMWF servers \
- (to find grib2flexpart and COMMAND file)\n\
- Normally ECMWFDATA resides in the scripts directory \
+ default=None, help="FLEXPART root directory on ECMWF \
+ servers (to find grib2flexpart and COMMAND file)\n\
+ Normally flex_extract resides in the scripts directory \
of the FLEXPART distribution, thus the:")
-# arguments for job submission to ECMWF, only needed by submit.py
+ # arguments for job submission to ECMWF, only needed by submit.py
parser.add_argument("--job_template", dest='job_template',
default="job.temp.o",
help="job template file for submission to ECMWF")
@@ -148,53 +146,10 @@ def install_args_and_control():
args = parser.parse_args()
- try:
- c = ControlFile(args.controlfile)
- except IOError:
- print 'Could not read CONTROL file "' + args.controlfile + '"'
- print 'Either it does not exist or its syntax is wrong.'
- print 'Try "' + sys.argv[0].split('/')[-1] + \
- ' -h" to print usage information'
- exit(1)
+ return args
+
- if args.install_target != 'local':
- if args.ecgid is None or args.ecuid is None or args.gateway is None \
- or args.destination is None:
- print 'Please enter your ECMWF user id and group id as well as \
- the \nname of the local gateway and the ectrans \
- destination '
- print 'with command line options --ecuid --ecgid \
- --gateway --destination'
- print 'Try "' + sys.argv[0].split('/')[-1] + \
- ' -h" to print usage information'
- print 'Please consult ecaccess documentation or ECMWF user support \
- for further details'
- sys.exit(1)
- else:
- c.ecuid = args.ecuid
- c.ecgid = args.ecgid
- c.gateway = args.gateway
- c.destination = args.destination
-
- if args.makefile:
- c.makefile = args.makefile
-
- if args.install_target == 'local':
- if args.flexpart_root_scripts is None:
- c.flexpart_root_scripts = '../'
- else:
- c.flexpart_root_scripts = args.flexpart_root_scripts
-
- if args.install_target != 'local':
- if args.flexpart_root_scripts is None:
- c.ec_flexpart_root_scripts = '${HOME}'
- else:
- c.ec_flexpart_root_scripts = args.flexpart_root_scripts
-
- return args, c
-
-
-def install_via_gateway(c, target):
+def install_via_gateway(c):
'''
@Description:
Perform the actual installation on local machine or prepare data
@@ -212,193 +167,326 @@ def install_via_gateway(c, target):
For more information about format and content of the parameter see
documentation.
+ @Return:
+ <nothing>
+ '''
+
+ ecd = c.ecmwfdatadir
+ tarball_name = 'flex_extract_v' + _VERSION_STR + '.tar'
+ target_dir = 'flex_extract_v' + _VERSION_STR
+ fortran_executable = 'CONVERT2'
+
+ if c.install_target.lower() != 'local':
+
+ mk_compilejob(ecd + 'python/compilejob.temp', c.makefile,
+ c.install_target, c.ecuid, c.ecgid,
+ c.flexpart_root_scripts)
+
+ mk_job_template(ecd + 'python/job.temp.o', c.ecuid, c.ecgid, c.gateway,
+ c.destination, c.flexpart_root_scripts)
+
+ mk_env_vars(ecd, c.ecuid, c.ecgid, c.gateway, c.destination)
+
+ #os.chdir('/')
+
+ mk_tarball(ecd, tarball_name)
+
+ put_file_to_ecserver(ecd, tarball_name, c.install_target,
+ c.ecuid, c.ecgid)
+
+ submit_job_to_ecserver(ecd + '/python/', c.install_target,
+ 'compilejob.ksh')
+
+ print 'job compilation script has been submitted to ecgate for ' + \
+ 'installation in ' + c.flexpart_root_scripts + \
+ '/' + target_dir
+ print 'You should get an email with subject flexcompile within ' + \
+ 'the next few minutes!'
+
+ else: #local
+ if not c.flexpart_root_scripts or c.flexpart_root_scripts == '../':
+ print 'WARNING: FLEXPART_ROOT_SCRIPTS has not been specified'
+ print 'There will be only the compilation of ' + \
+ ' in ' + ecd + '/src'
+ os.chdir(ecd + '/src')
+ else: # creates the target working directory for flex_extract
+ c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
+ c.flexpart_root_scripts))
+ if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
+ os.chdir('/')
+ mk_tarball(ecd, tarball_name)
+ make_dir(c.flexpart_root_scripts + '/' + target_dir)
+ os.chdir(c.flexpart_root_scripts + '/' + target_dir)
+ print 'Untar ...'
+ subprocess.check_call(['tar', '-xvf',
+ ecd + '../' + tarball_name])
+ os.chdir(c.flexpart_root_scripts + '/' + target_dir + '/src')
+
+ # Create Fortran executable - CONVERT2
+ print 'Install ' + target_dir + ' software on ' + \
+ c.install_target + ' in directory ' + \
+ os.path.abspath(os.getcwd() + '/../') + '\n'
+
+ delete_convert_build('')
+ make_convert_build('', c.makefile, fortran_executable)
+
+ return
+
+def mk_tarball(ecd, tarname):
+ '''
+ @Description:
+ Creates a tarball from all files which need to be sent to the
+ installation directory.
+ It does not matter if this is local or remote.
+ Collects all python files, the Fortran source and makefiles,
+ the ECMWF_ENV file, the CONTROL files as well as
+ the korn shell and template files.
+
+ @Input:
+ ecd: string
+ The path were the file is to be stored.
+
+ tarname: string
+ The name of the file to send to the ECMWF server.
+
+ @Return:
+ <nothing>
+ '''
+
+ print 'Create tarball ...'
+ try:
+ subprocess.check_call(['tar -cvf '+
+ ecd + '../' + tarname + ' ' +
+ ecd + 'python/*py ' +
+ ecd + 'python/CONTROL* ' +
+ ecd + 'python/*ksh ' +
+ ecd + 'python/*temp* ' +
+ ecd + 'python/ECMWF_ENV ' +
+ ecd + 'grib_templates ' +
+ ecd + 'src/*.f ' +
+ ecd + 'src/*.f90 ' +
+ ecd + 'src/*.h ' +
+ ecd + 'src/Makefile*'], shell=True)
+ except subprocess.CalledProcessError as e:
+ print 'ERROR:'
+ print e.output
+ sys.exit('could not make installation tar ball!')
+
+ return
+
+def mk_env_vars(ecd, ecuid, ecgid, gateway, destination):
+ '''
+ @Description:
+ Creates a file named ECMWF_ENV which contains the
+ necessary environmental variables at ECMWF servers.
+
+ @Input:
+ ecd: string
+ The path were the file is to be stored.
+
+ ecuid: string
+ The user id on ECMWF server.
+
+ ecgid: string
+ The group id on ECMWF server.
+
+ gateway: string
+ The gateway server the user is using.
+
+ destination: string
+ The remote destination which is used to transfer files
+ from ECMWF server to local gateway server.
+
+ @Return:
+ <nothing>
+ '''
+
+ with open(ecd + 'python/ECMWF_ENV', 'w') as fo:
+ fo.write('ECUID ' + ecuid + '\n')
+ fo.write('ECGID ' + ecgid + '\n')
+ fo.write('GATEWAY ' + gateway + '\n')
+ fo.write('DESTINATION ' + destination + '\n')
+
+ return
+
+def mk_compilejob(template, makefile, target, ecuid, ecgid, fp_root):
+ '''
+ @Description:
+ Modifies the original job template file so that it is specified
+ for the user and the environment were it will be applied. Result
+ is stored in a new file "job.temp" in the python directory.
+
+ @Input:
+ template: string
+ File which contains the original text for the job template.
+ It must contain the complete path to the file.
+
+ makefile: string
+ Name of the makefile which should be used to compile FORTRAN
+ CONVERT2 program.
+
target: string
- The target where the installation should be processed.
- E.g. "local", "ecgate" or "cca"
+ The target where the installation should be done, e.g. the queue.
+
+ ecuid: string
+ The user id on ECMWF server.
+
+ ecgid: string
+ The group id on ECMWF server.
+
+ fp_root: string
+ Path to the root directory of FLEXPART environment or flex_extract
+ environment.
@Return:
<nothing>
'''
- ecd = c.ecmwfdatadir
- template = ecd + 'python/compilejob.temp'
- job = ecd + 'python/compilejob.ksh'
- fo = open(job, 'w')
-#AP could do with open(template) as f, open(job, 'w') as fo:
-#AP or nested with statements
+
with open(template) as f:
fdata = f.read().split('\n')
+
+ with open(template[:-4] + 'ksh', 'w') as fo:
for data in fdata:
if 'MAKEFILE=' in data:
- if c.makefile is not None:
- data = 'export MAKEFILE=' + c.makefile
- if 'FLEXPART_ROOT_SCRIPTS=' in data:
- if c.flexpart_root_scripts != '../':
- data = 'export FLEXPART_ROOT_SCRIPTS=' + \
- c.flexpart_root_scripts
+ data = 'export MAKEFILE=' + makefile
+ elif 'FLEXPART_ROOT_SCRIPTS=' in data:
+ if fp_root != '../':
+ data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
else:
data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
- if target.lower() != 'local':
+ elif target.lower() != 'local':
if '--workdir' in data:
- data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \
- '/' + c.ecuid
- if '##PBS -o' in data:
- data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + c.ecuid + \
- 'flex_ecmwf.$Jobname.$Job_ID.out'
- if 'FLEXPART_ROOT_SCRIPTS=' in data:
- if c.ec_flexpart_root_scripts != '../':
- data = 'export FLEXPART_ROOT_SCRIPTS=' + \
- c.ec_flexpart_root_scripts
+ data = '#SBATCH --workdir=/scratch/ms/' + \
+ ecgid + '/' + ecuid
+ elif '##PBS -o' in data:
+ data = '##PBS -o /scratch/ms/' + ecgid + '/' + ecuid + \
+ 'flex_ecmwf.$Jobname.$Job_ID.out'
+ elif 'FLEXPART_ROOT_SCRIPTS=' in data:
+ if fp_root != '../':
+ data = 'export FLEXPART_ROOT_SCRIPTS=' + fp_root
else:
data = 'export FLEXPART_ROOT_SCRIPTS=$HOME'
fo.write(data + '\n')
- f.close()
- fo.close()
-
- if target.lower() != 'local':
- template = ecd + 'python/job.temp.o'
-#AP hier eventuell Zeile für Zeile lesen und dann if Entscheidung
- with open(template) as f:
- fdata = f.read().split('\n')
- f.close()
- fo = open(template[:-2], 'w')
+
+ return
+
+def mk_job_template(template, ecuid, ecgid, gateway, destination, fp_root):
+ '''
+ @Description:
+ Modifies the original job template file so that it is specified
+ for the user and the environment were it will be applied. Result
+ is stored in a new file "job.temp" in the python directory.
+
+ @Input:
+ template: string
+ File which contains the original text for the job template.
+ It must contain the complete path to the file.
+
+ ecuid: string
+ The user id on ECMWF server.
+
+ ecgid: string
+ The group id on ECMWF server.
+
+ gateway: string
+ The gateway server the user is using.
+
+ destination: string
+ The remote destination which is used to transfer files
+ from ECMWF server to local gateway server.
+
+ fp_root: string
+ Path to the root directory of FLEXPART environment or flex_extract
+ environment.
+
+ @Return:
+ <nothing>
+ '''
+
+ with open(template) as f:
+ fdata = f.read().split('\n')
+
+ with open(template[:-2], 'w') as fo:
for data in fdata:
if '--workdir' in data:
- data = '#SBATCH --workdir=/scratch/ms/' + c.ecgid + \
- '/' + c.ecuid
- if '##PBS -o' in data:
- data = '##PBS -o /scratch/ms/' + c.ecgid + '/' + \
- c.ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
- if 'export PATH=${PATH}:' in data:
- data += c.ec_flexpart_root_scripts + '/ECMWFDATA7.1/python'
- if 'cat>>' in data or 'cat >>' in data:
- i = data.index('>')
- fo.write(data[:i] + data[i+1:] + '\n')
- fo.write('GATEWAY ' + c.gateway + '\n')
- fo.write('DESTINATION ' + c.destination + '\n')
- fo.write('EOF\n')
+ data = '#SBATCH --workdir=/scratch/ms/' + ecgid + \
+ '/' + ecuid
+ elif '##PBS -o' in data:
+ data = '##PBS -o /scratch/ms/' + ecgid + '/' + \
+ ecuid + 'flex_ecmwf.$Jobname.$Job_ID.out'
+ elif 'export PATH=${PATH}:' in data:
+ data += fp_root + '/flex_extract_v7.1/python'
fo.write(data + '\n')
- fo.close()
-
- job = ecd + 'python/ECMWF_ENV'
- with open(job, 'w') as fo:
- fo.write('ECUID ' + c.ecuid + '\n')
- fo.write('ECGID ' + c.ecgid + '\n')
- fo.write('GATEWAY ' + c.gateway + '\n')
- fo.write('DESTINATION ' + c.destination + '\n')
- fo.close()
-
- if target.lower() == 'local':
- # compile CONVERT2
- if c.flexpart_root_scripts is None or c.flexpart_root_scripts == '../':
- print 'Warning: FLEXPART_ROOT_SCRIPTS has not been specified'
- print 'Only CONVERT2 will be compiled in ' + ecd + '/../src'
- else:
- c.flexpart_root_scripts = os.path.expandvars(os.path.expanduser(
- c.flexpart_root_scripts))
- if os.path.abspath(ecd) != os.path.abspath(c.flexpart_root_scripts):
- os.chdir('/')
- p = subprocess.check_call(['tar', '-cvf',
- ecd + '../ECMWFDATA7.1.tar',
- ecd + 'python',
- ecd + 'grib_templates',
- ecd + 'src'])
- try:
- os.makedirs(c.flexpart_root_scripts + '/ECMWFDATA7.1')
- finally:
- pass
- os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1')
- p = subprocess.check_call(['tar', '-xvf',
- ecd + '../ECMWFDATA7.1.tar'])
- os.chdir(c.flexpart_root_scripts + '/ECMWFDATA7.1/src')
-
- os.chdir('../src')
- print(('install ECMWFDATA7.1 software on ' + target + ' in directory '
- + os.getcwd()))
- if c.makefile is None:
- makefile = 'Makefile.local.ifort'
- else:
- makefile = c.makefile
- flist = glob.glob('*.mod') + glob.glob('*.o')
- if flist:
- p = subprocess.check_call(['rm'] + flist)
- try:
- print 'Using makefile: ' + makefile
- p = subprocess.check_call(['make', '-f', makefile])
- p = subprocess.check_call(['ls', '-l', 'CONVERT2'])
- except subprocess.CalledProcessError as e:
- print 'compile failed with the following error:'
- print e.output
- print 'please edit ' + makefile + \
+ return
+
+def delete_convert_build(ecd):
+ '''
+ @Description:
+ Clean up the Fortran source directory and remove all
+ build files (e.g. *.o, *.mod and CONVERT2)
+
+ @Input:
+ ecd: string
+ The path to the Fortran program.
+
+ @Return:
+ <nothing>
+ '''
+
+ modfiles = UioFiles(ecd, '*.mod')
+ objfiles = UioFiles(ecd, '*.o')
+ exefile = UioFiles(ecd, 'CONVERT2')
+
+ modfiles.delete_files()
+ objfiles.delete_files()
+ exefile.delete_files()
+
+ return
+
+def make_convert_build(ecd, makefile, f_executable):
+ '''
+ @Description:
+ Compiles the Fortran code and generates the executable.
+
+ @Input:
+ ecd: string
+ The path were the file is to be stored.
+
+ makefile: string
+ The name of the makefile which should be used.
+
+ f_executable: string
+ The name of the executable the Fortran program generates after
+ compilation.
+
+ @Return:
+ <nothing>
+ '''
+
+ try:
+ print 'Using makefile: ' + makefile
+ p = subprocess.Popen(['make', '-f', ecd + makefile],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ bufsize=1)
+ pout, perr = p.communicate()
+ print pout
+ if p.returncode != 0:
+ print perr
+ print 'Please edit ' + makefile + \
' or try another Makefile in the src directory.'
- print 'most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB \
- and EMOSLIB must be adapted.'
+ print 'Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB ' \
+ 'and EMOSLIB must be adapted.'
print 'Available Makefiles:'
- print glob.glob('Makefile*')
- elif target.lower() == 'ecgate':
- os.chdir('/')
- p = subprocess.check_call(['tar', '-cvf',
- ecd + '../ECMWFDATA7.1.tar',
- ecd + 'python',
- ecd + 'grib_templates',
- ecd + 'src'])
- try:
- p = subprocess.check_call(['ecaccess-file-put',
- ecd + '../ECMWFDATA7.1.tar',
- 'ecgate:/home/ms/' + c.ecgid + '/' +
- c.ecuid + '/ECMWFDATA7.1.tar'])
- except subprocess.CalledProcessError as e:
- print 'ecaccess-file-put failed! \
- Probably the eccert key has expired.'
- exit(1)
-
- try:
- p = subprocess.check_call(['ecaccess-job-submit',
- '-queueName',
- target,
- ecd + 'python/compilejob.ksh'])
- print 'compilejob.ksh has been submitted to ecgate for \
- installation in ' + c.ec_flexpart_root_scripts + \
- '/ECMWFDATA7.1'
- print 'You should get an email with subject flexcompile within \
- the next few minutes'
- except subprocess.CalledProcessError as e:
- print 'ecaccess-job-submit failed!'
- exit(1)
-
- elif target.lower() == 'cca':
- os.chdir('/')
- p = subprocess.check_call(['tar', '-cvf',
- ecd + '../ECMWFDATA7.1.tar',
- ecd + 'python',
- ecd + 'grib_templates',
- ecd + 'src'])
- try:
- p = subprocess.check_call(['ecaccess-file-put',
- ecd + '../ECMWFDATA7.1.tar',
- 'cca:/home/ms/' + c.ecgid + '/' +
- c.ecuid + '/ECMWFDATA7.1.tar'])
- except subprocess.CalledProcessError as e:
- print 'ecaccess-file-put failed! \
- Probably the eccert key has expired.'
- exit(1)
-
- try:
- p = subprocess.check_call(['ecaccess-job-submit',
- '-queueName',
- target,
- ecd + 'python/compilejob.ksh'])
- print 'compilejob.ksh has been submitted to cca for installation in ' +\
- c.ec_flexpart_root_scripts + '/ECMWFDATA7.1'
- print 'You should get an email with subject flexcompile \
- within the next few minutes'
- except subprocess.CalledProcessError as e:
- print 'ecaccess-job-submit failed!'
- exit(1)
-
+ print UioFiles('.', 'Makefile*')
+ sys.exit('Compilation failed!')
+ except ValueError as e:
+ print 'ERROR: Makefile call failed:'
+ print e
else:
- print 'ERROR: unknown installation target ', target
- print 'Valid targets: ecgate, cca, local'
+ subprocess.check_call(['ls', '-l', ecd + f_executable])
return
diff --git a/python/job.ksh b/python/job.ksh
index 243d74656f3b5a76e7f3adbb5e8884c6f539297a..8cbc1d82d16626d48e938bd80ce8690705a43828 100644
--- a/python/job.ksh
+++ b/python/job.ksh
@@ -32,9 +32,7 @@ case $HOST in
module unload emos
module load grib_api/1.14.5
module load emos/437-r64
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
- export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
+ export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
;;
*cca*)
module switch PrgEnv-cray PrgEnv-intel
@@ -42,17 +40,8 @@ case $HOST in
module load emos
module load python
export SCRATCH=$TMPDIR
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
- export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
+ export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
;;
-# *)
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}
-# export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python
-# export SCRATCH=$ECMWFDATA/python
-# which python
-# ;;
esac
cd $SCRATCH
@@ -62,34 +51,41 @@ cd python$$
export CONTROL=CONTROL
cat >$CONTROL<<EOF
-GATEWAY srvx8.img.univie.ac.at
-DESTINATION annep@genericSftp
-accuracy 16
+accuracy 24
addpar 186 187 188 235 139 39
+area
basetime None
+controlfile CONTROL.test
cwc 0
date_chunk 3
-debug 1
+debug 0
+destination annep@genericSftp
dpdeta 1
dtime 3
ecfsdir ectmp:/${USER}/econdemand/
+ecgid at
ecstorage 0
ectrans 1
-end_date 20160809
+ecuid km4a
+end_date 20000101
eta 0
etadiff 0
+etapar 77
expver 1
format GRIB1
+gateway srvx8.img.univie.ac.at
gauss 1
grib2flexpart 0
grid 5000
inputdir ../work
+install_target None
+job_template job.temp
left -15000
level 60
levelist 55/to/60
lower 30000
-mailfail ${USER}
-mailops ${USER}
+mailfail ${USER}
+mailops ${USER}
makefile None
marsclass EI
maxstep 11
@@ -97,16 +93,18 @@ number OFF
omega 0
omegadiff 0
outputdir ../work
-prefix EI
+prefix EItest_
+queue ecgate
resol 63
right 45000
smooth 0
-start_date 20160809
+start_date 20000101
step 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
stream OPER
time 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
type AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
upper 75000
+wrf 0
EOF
diff --git a/python/job.temp b/python/job.temp
index 4a20430215cdcb51cca4cdf6604f719353d35676..e8ce37658f0f12745bee16445498904c464fe929 100644
--- a/python/job.temp
+++ b/python/job.temp
@@ -32,9 +32,7 @@ case $HOST in
module unload emos
module load grib_api/1.14.5
module load emos/437-r64
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
- export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
+ export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
;;
*cca*)
module switch PrgEnv-cray PrgEnv-intel
@@ -42,17 +40,8 @@ case $HOST in
module load emos
module load python
export SCRATCH=$TMPDIR
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
- export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
+ export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
;;
-# *)
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}
-# export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python
-# export SCRATCH=$ECMWFDATA/python
-# which python
-# ;;
esac
cd $SCRATCH
@@ -62,10 +51,6 @@ cd python$$
export CONTROL=CONTROL
cat >$CONTROL<<EOF
-GATEWAY srvx8.img.univie.ac.at
-DESTINATION annep@genericSftp
-EOF
-cat >>$CONTROL<<EOF
EOF
diff --git a/python/job.temp.o b/python/job.temp.o
index b3ff36c1202bc7c22f29837bb8a072a1f40844b2..fbdbd81aa1ea7100a84ac0858ed29804526e98a5 100644
--- a/python/job.temp.o
+++ b/python/job.temp.o
@@ -32,8 +32,6 @@ case $HOST in
module unload emos
module load grib_api/1.14.5
module load emos/437-r64
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
export PATH=${PATH}:
;;
*cca*)
@@ -42,17 +40,8 @@ case $HOST in
module load emos
module load python
export SCRATCH=$TMPDIR
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
export PATH=${PATH}:
;;
-# *)
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}
-# export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python
-# export SCRATCH=$ECMWFDATA/python
-# which python
-# ;;
esac
cd $SCRATCH
@@ -61,7 +50,7 @@ cd python$$
export CONTROL=CONTROL
-cat >>$CONTROL<<EOF
+cat >$CONTROL<<EOF
EOF
diff --git a/python/joboper.ksh b/python/joboper.ksh
index 8bb5f874dc3f2100d6009dc106af849e8ac3a46e..7eb5a8093c50aa26a1ba1f58eeba96f6cd669bbc 100644
--- a/python/joboper.ksh
+++ b/python/joboper.ksh
@@ -32,9 +32,7 @@ case $HOST in
module unload emos
module load grib_api/1.14.5
module load emos/437-r64
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
- export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
+ export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
;;
*cca*)
module switch PrgEnv-cray PrgEnv-intel
@@ -42,17 +40,8 @@ case $HOST in
module load emos
module load python
export SCRATCH=$TMPDIR
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PYTHONPATH=$ECMWFDATA/python
- export PATH=${PATH}:${HOME}/ECMWFDATA7.1/python
+ export PATH=${PATH}:${HOME}/flex_extract_v7.1/python
;;
-# *)
-# export ECMWFDATA=$HOME/ECMWFDATA$VERSION
-# export PATH=/opt/anaconda/bin:$ECMWFDATA/python:${PATH}
-# export PYTHONPATH=/opt/anaconda/lib/python2.7/site-packages/grib_api:$ECMWFDATA/python
-# export SCRATCH=$ECMWFDATA/python
-# which python
-# ;;
esac
cd $SCRATCH
@@ -66,30 +55,40 @@ GATEWAY srvx8.img.univie.ac.at
DESTINATION annep@genericSftp
accuracy 16
addpar 186 187 188 235 139 39
+area
+base_time ${MSJ_BASETIME}
basetime None
+controlfile CONTROL.temp
cwc 0
date_chunk 3
debug 1
+destination None
dpdeta 1
dtime 3
ecfsdir ectmp:/${USER}/econdemand/
+ecgid None
ecstorage 0
ectrans 1
-start_date ${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}
+ecuid None
+end_date ${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}
eta 0
etadiff 0
+etapar 77
expver 1
format GRIB1
+gateway None
gauss 1
grib2flexpart 0
grid 5000
inputdir ../work
+install_target None
+job_template job.temp
left -15000
level 60
levelist 55/to/60
lower 30000
-mailfail ${USER}
-mailops ${USER}
+mailfail ${USER}
+mailops ${USER}
makefile None
marsclass EI
maxstep 11
@@ -98,6 +97,7 @@ omega 0
omegadiff 0
outputdir ../work
prefix EI
+queue ecgate
resol 63
right 45000
smooth 0
@@ -107,6 +107,7 @@ stream OPER
time 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
type AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
upper 75000
+wrf 0
EOF
diff --git a/python/mk_install_tar.sh b/python/mk_install_tar.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f13f82ecc1ae6b5ff36b67d3c415418c175dbb01
--- /dev/null
+++ b/python/mk_install_tar.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# @Author: Anne Philipp
+#
+# @Date: June, 7 2018
+#
+# @Description: Makes a tarball for installation
+#
+
+tarname='flex_extract_v7.1.tar.gz'
+path='/nas/tmc/Anne/Interpolation/flexextract/flexextract/python/../'
+
+tar -cvf $tarname ${path}python/*py ${path}python/CONTROL* ${path}python/*ksh ${path}python/*temp ${path}python/ECMWF_ENV ${path}python/*json ${path}grib_templates ${path}src/*.f ${path}src/*.f90 ${path}src/*.h ${path}src/Makefile*
+
+
diff --git a/python/plot_retrieved.py b/python/plot_retrieved.py
index d924c357e5a21569af4bf88dcdf857de20301c53..45e7bb2e7783cfe0644b1fe2eb4c296d1bf75fe2 100755
--- a/python/plot_retrieved.py
+++ b/python/plot_retrieved.py
@@ -1,11 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#************************************************************************
-# ToDo AP
-# - documentation der Funktionen
-# - docu der progam functionality
-# - apply pep8
-#************************************************************************
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
#
@@ -33,6 +27,7 @@
# @Program Content:
# - main
# - get_basics
+# - get_files_per_date
# - plot_retrieved
# - plot_timeseries
# - plot_map
@@ -218,8 +213,7 @@ def plot_retrieved(c):
c.levels = np.asarray(c.levels, dtype='int')
c.area = np.asarray(c.area)
- files = UioFiles(c.prefix+'*')
- files.list_files(c.inputdir)
+ files = UioFiles(c.inputdir, c.prefix+'*')
ifiles = get_files_per_date(files.files, datelist)
ifiles.sort()
@@ -621,7 +615,7 @@ def get_plot_args():
parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
help="FLEXPART root directory (to find \
'grib2flexpart and COMMAND file)\n \
- Normally ECMWFDATA resides in the scripts directory \
+ Normally flex_extract resides in the scripts directory \
of the FLEXPART distribution")
parser.add_argument("--controlfile", dest="controlfile",
diff --git a/python/prepare_flexpart.py b/python/prepare_flexpart.py
index 2e5a160a52252fa2a45809fc506ccf7940543a2c..088c2a0fd49e2cc9452d622640b8b0c137b47f07 100755
--- a/python/prepare_flexpart.py
+++ b/python/prepare_flexpart.py
@@ -1,10 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#************************************************************************
-# ToDo AP
-# - wieso start=startm1 wenn basetime = 0 ? wenn die fluxes nicht mehr
-# relevant sind? verstehe ich nicht
-#************************************************************************
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
#
@@ -61,10 +56,11 @@ import os
import inspect
import sys
import socket
+import _config
# software specific classes and modules from flex_extract
from UioFiles import UioFiles
-from tools import interpret_args_and_control, clean_up
+from tools import clean_up, get_cmdline_arguments, read_ecenv
from EcFlexpart import EcFlexpart
ecapi = 'ecmwf' not in socket.gethostname()
@@ -80,6 +76,7 @@ LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
if LOCAL_PYTHON_PATH not in sys.path:
sys.path.append(LOCAL_PYTHON_PATH)
+
# ------------------------------------------------------------------------------
# FUNCTION
# ------------------------------------------------------------------------------
@@ -96,12 +93,30 @@ def main():
@Return:
<nothing>
'''
- args, c = interpret_args_and_control()
- prepare_flexpart(args, c)
+
+ args = get_cmdline_arguments()
+
+ try:
+ c = ControlFile(args.controlfile)
+ except IOError:
+ try:
+ c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
+ except IOError:
+ print 'Could not read CONTROL file "' + args.controlfile + '"'
+ print 'Either it does not exist or its syntax is wrong.'
+ print 'Try "' + sys.argv[0].split('/')[-1] + \
+ ' -h" to print usage information'
+ sys.exit(1)
+
+ env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
+ c.assign_args_to_control(args, env_parameter)
+ c.assign_envs_to_control(env_parameter)
+ c.check_conditions()
+ prepare_flexpart(args.ppid, c)
return
-def prepare_flexpart(args, c):
+def prepare_flexpart(ppid, c):
'''
@Description:
Lists all grib files retrieved from MARS with get_mars_data and
@@ -111,8 +126,9 @@ def prepare_flexpart(args, c):
a file with a specific FLEXPART relevant naming convention.
@Input:
- args: instance of ArgumentParser
- Contains the commandline arguments from script/program call.
+ ppid: int
+ Contains the ppid number of the current ECMWF job. If it is called
+ from this script, it is "None".
c: instance of class ControlFile
Contains all the parameters of CONTROL file, which are e.g.:
@@ -131,10 +147,10 @@ def prepare_flexpart(args, c):
<nothing>
'''
- if not args.ppid:
+ if not ppid:
c.ppid = str(os.getppid())
else:
- c.ppid = args.ppid
+ c.ppid = ppid
c.ecapi = ecapi
@@ -147,38 +163,32 @@ def prepare_flexpart(args, c):
month=int(c.end_date[4:6]),
day=int(c.end_date[6:]))
- # to deaccumulate the fluxes correctly
- # one day ahead of the start date and
- # one day after the end date is needed
- startm1 = start - datetime.timedelta(days=1)
-# endp1 = end + datetime.timedelta(days=1)
+ # assign starting date minus 1 day
+ # since for basetime 00 we need the 12 hours upfront
+ # (the day before from 12 UTC to current day 00 UTC)
+ if c.basetime == '00':
+ start = start - datetime.timedelta(days=1)
- # get all files with flux data to be deaccumulated
- inputfiles = UioFiles('*OG_acc_SL*.' + c.ppid + '.*')
- inputfiles.list_files(c.inputdir)
+ print 'Prepare ' + start.strftime("%Y%m%d") + \
+ "/to/" + end.strftime("%Y%m%d")
# create output dir if necessary
if not os.path.exists(c.outputdir):
os.makedirs(c.outputdir)
+ # get all files with flux data to be deaccumulated
+ inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')
+
# deaccumulate the flux data
flexpart = EcFlexpart(c, fluxes=True)
flexpart.write_namelist(c, 'fort.4')
flexpart.deacc_fluxes(inputfiles, c)
- print 'Prepare ' + start.strftime("%Y%m%d") + \
- "/to/" + end.strftime("%Y%m%d")
-
# get a list of all files from the root inputdir
- inputfiles = UioFiles('????__??.*' + c.ppid + '.*')
- inputfiles.list_files(c.inputdir)
+ inputfiles = UioFiles(c.inputdir, '????__??.*' + c.ppid + '.*')
- # produce FLEXPART-ready GRIB files and
- # process GRIB files -
+ # produce FLEXPART-ready GRIB files and process them -
# copy/transfer/interpolate them or make them GRIB2
- if c.basetime == '00':
- start = startm1
-
flexpart = EcFlexpart(c, fluxes=False)
flexpart.create(inputfiles, c)
flexpart.process_output(c)
@@ -186,7 +196,7 @@ def prepare_flexpart(args, c):
# check if in debugging mode, then store all files
# otherwise delete temporary files
if int(c.debug) != 0:
- print 'Temporary files left intact'
+ print '\nTemporary files left intact'
else:
clean_up(c)
diff --git a/python/profiling.py b/python/profiling.py
index 526c17fc5d4e0e2de413530ea7189542521f1435..4511af2aca3a41265a9dd035b11430e84626ac62 100644
--- a/python/profiling.py
+++ b/python/profiling.py
@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
#************************************************************************
# ToDo AP
-# - check of license of book content
+# - check license of book content
#************************************************************************
#*******************************************************************************
#
diff --git a/python/set_CTBTO_env.csh b/python/set_CTBTO_env.csh
deleted file mode 100644
index f504d866db9643c13bcee4563b938520a50fd306..0000000000000000000000000000000000000000
--- a/python/set_CTBTO_env.csh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/csh
-
-# script to prepare CTBTO environment for
-# ECMWFDATA7.0
-#
-# Leo Haimberger 1.3.2016
-
-setenv PATH /dvl/atm/klinkl/software/local/bin:$PATH
-setenv PYTHONPATH /dvl/atm/klinkl/software/local/lib/python2.7/site-packages/grib_api
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:/dvl/atm/klinkl/software/local/lib/
-setenv GRIB_API_INCLUDE_DIR /dvl/atm/klinkl/software/local/include/
-setenv GRIB_API_LIB '-L/dvl/atm/klinkl/software/local/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -lemosR64 -Bdynamic -lm -ljasper'
diff --git a/python/set_CTBTO_env.sh b/python/set_CTBTO_env.sh
deleted file mode 100644
index 078cee7cbf3124a94cfb3a9570281cc4eec91b55..0000000000000000000000000000000000000000
--- a/python/set_CTBTO_env.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# script to prepare CTBTO environment for
-# ECMWFDATA7.0
-#
-# Leo Haimberger 1.3.2016
-
-export PATH=/dvl/atm/klinkl/software/local/bin:$PATH
-export PYTHONPATH=/dvl/atm/klinkl/software/local/lib/python2.7/site-packages/grib_api
-export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/dvl/atm/klinkl/software/local/lib/
-export GRIB_API_INCLUDE_DIR=/dvl/atm/klinkl/software/local/include/
-export GRIB_API_LIB='-L/dvl/atm/klinkl/software/local/lib -Bstatic -lgrib_api_f77 -lgrib_api_f90 -lgrib_api -lemosR64 -Bdynamic -lm -ljasper'
diff --git a/python/submit.py b/python/submit.py
index fcf5735d4693082ac31a587313c8dbd411ff4f58..14cd3caaba0169cb08d5f7c42695d0d78a49601f 100755
--- a/python/submit.py
+++ b/python/submit.py
@@ -43,17 +43,15 @@ import os
import sys
import subprocess
import inspect
+import collections
# software specific classes and modules from flex_extract
-from tools import interpret_args_and_control, normal_exit
+import _config
+from tools import normal_exit, get_cmdline_arguments, submit_job_to_ecserver, \
+ read_ecenv
from get_mars_data import get_mars_data
from prepare_flexpart import prepare_flexpart
-
-# add path to pythonpath so that python finds its buddies
-LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
- inspect.getfile(inspect.currentframe())))
-if LOCAL_PYTHON_PATH not in sys.path:
- sys.path.append(LOCAL_PYTHON_PATH)
+from ControlFile import ControlFile
# ------------------------------------------------------------------------------
# FUNCTIONS
@@ -75,17 +73,36 @@ def main():
'''
called_from_dir = os.getcwd()
- args, c = interpret_args_and_control()
+
+ args = get_cmdline_arguments()
+
+ try:
+ c = ControlFile(args.controlfile)
+ except IOError:
+ try:
+ c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
+ except IOError:
+ print 'Could not read CONTROL file "' + args.controlfile + '"'
+ print 'Either it does not exist or its syntax is wrong.'
+ print 'Try "' + sys.argv[0].split('/')[-1] + \
+ ' -h" to print usage information'
+ sys.exit(1)
+
+ env_parameter = read_ecenv(c.ecmwfdatadir + 'python/ECMWF_ENV')
+ c.assign_args_to_control(args)
+ c.assign_envs_to_control(env_parameter)
+ c.check_conditions()
# on local side
+ # on ECMWF server this would be the local side
if args.queue is None:
if c.inputdir[0] != '/':
c.inputdir = os.path.join(called_from_dir, c.inputdir)
if c.outputdir[0] != '/':
c.outputdir = os.path.join(called_from_dir, c.outputdir)
- get_mars_data(args, c)
- prepare_flexpart(args, c)
- normal_exit(c)
+ get_mars_data(c)
+ prepare_flexpart(args.ppid, c)
+ normal_exit(c.mailfail, 'Done!')
# on ECMWF server
else:
submit(args.job_template, c, args.queue)
@@ -124,50 +141,54 @@ def submit(jtemplate, c, queue):
<nothing>
'''
- # read template file and split from newline signs
+ # read template file and get index for CONTROL input
with open(jtemplate) as f:
lftext = f.read().split('\n')
- insert_point = lftext.index('EOF')
-
- # put all parameters of ControlFile instance into a list
- clist = c.to_list() # ondemand
- colist = [] # operational
- mt = 0
-
- for elem in clist:
- if 'maxstep' in elem:
- mt = int(elem.split(' ')[1])
-
- for elem in clist:
- if 'start_date' in elem:
- elem = 'start_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
- if 'end_date' in elem:
- elem = 'end_date ' + '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
- if 'base_time' in elem:
- elem = 'base_time ' + '${MSJ_BASETIME}'
- if 'time' in elem and mt > 24:
- elem = 'time ' + '${MSJ_BASETIME} {MSJ_BASETIME}'
- colist.append(elem)
-
- lftextondemand = lftext[:insert_point] + clist + lftext[insert_point + 2:]
- lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
-
- with open('job.ksh', 'w') as h:
- h.write('\n'.join(lftextondemand))
-
- with open('joboper.ksh', 'w') as h:
- h.write('\n'.join(lftextoper))
-
- # submit job script to queue
- try:
- p = subprocess.check_call(['ecaccess-job-submit', '-queueName',
- queue, 'job.ksh'])
- except subprocess.CalledProcessError as e:
- print 'ecaccess-job-submit failed!'
- print 'Error Message: '
- print e.output
- exit(1)
+ insert_point = lftext.index('EOF')
+
+ if not c.basetime:
+ # --------- create on demand job script ------------------------------------
+ if c.maxstep > 24:
+ print '---- Pure forecast mode! ----'
+ else:
+ print '---- On-demand mode! ----'
+ job_file = jtemplate[:-4] + 'ksh'
+ clist = c.to_list()
+
+ lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:]
+
+ with open(job_file, 'w') as f:
+ f.write('\n'.join(lftextondemand))
+
+ submit_job_to_ecserver('', queue, job_file)
+
+ else:
+ # --------- create operational job script ----------------------------------
+ print '---- Operational mode! ----'
+ job_file = jtemplate[:-5] + 'oper.ksh'
+ #colist = []
+
+ if c.maxstep:
+ mt = int(c.maxstep)
+ else:
+ mt = 0
+
+ c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
+ c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}'
+ c.base_time = '${MSJ_BASETIME}'
+ if mt > 24:
+ c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
+
+ colist = c.to_list()
+
+ lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:]
+
+ with open(job_file, 'w') as f:
+ f.write('\n'.join(lftextoper))
+
+ submit_job_to_ecserver('', queue, job_file)
+ # --------------------------------------------------------------------------
print 'You should get an email with subject flex.hostname.pid'
return
diff --git a/python/test_suite.py b/python/test_suite.py
index c20b5f8ccd7afdb90fa33e0a225f45956d72c798..6cd9ed7cfb41cd7faf5b1f5487524535216a889d 100755
--- a/python/test_suite.py
+++ b/python/test_suite.py
@@ -1,11 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#************************************************************************
-# ToDo AP
-# - provide more tests
-# - provide more documentation
-#************************************************************************
-
#*******************************************************************************
# @Author: Leopold Haimberger (University of Vienna)
#
@@ -24,7 +18,7 @@
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# @Program Functionality:
-# This script triggers the ECMWFDATA test suite. Call with
+# This script triggers the flex_extract test suite. Call with
# test_suite.py [test group]
#
# @Program Content:
diff --git a/python/tools.py b/python/tools.py
index b74fd685609c04e291849870a0c64f759a79220d..6d9933d07e3b9b1cd901f28aace63317acbd5b14 100644
--- a/python/tools.py
+++ b/python/tools.py
@@ -1,12 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#************************************************************************
-# ToDo AP
-# - check my_error
-# - check normal_exit
-# - check get_list_as_string
-# - seperate args and control interpretation
-#************************************************************************
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
#
@@ -25,6 +18,7 @@
# - added documentation
# - moved all functions from file Flexparttools to this file tools
# - added function get_list_as_string
+# - seperated args and control interpretation
#
# @License:
# (C) Copyright 2014-2018.
@@ -37,7 +31,7 @@
# used in different places in flex_extract.
#
# @Module Content:
-# - interpret_args_and_control
+# - get_cmdline_arguments
# - clean_up
# - my_error
# - normal_exit
@@ -46,6 +40,7 @@
# - init128
# - to_param_id
# - get_list_as_string
+# - make_dir
#
#*******************************************************************************
@@ -56,24 +51,19 @@ import os
import errno
import sys
import glob
-import inspect
import subprocess
import traceback
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-import numpy as np
-
-# software specific class from flex_extract
-from ControlFile import ControlFile
# ------------------------------------------------------------------------------
# FUNCTIONS
# ------------------------------------------------------------------------------
-def interpret_args_and_control():
+def get_cmdline_arguments():
'''
@Description:
- Assigns the command line arguments and reads CONTROL file
- content. Apply default values for non mentioned arguments.
+ Decomposes the command line arguments and assigns them to variables.
+ Apply default values for non mentioned arguments.
@Input:
<nothing>
@@ -81,39 +71,28 @@ def interpret_args_and_control():
@Return:
args: instance of ArgumentParser
Contains the commandline arguments from script/program call.
-
- c: instance of class ControlFile
- Contains all necessary information of a CONTROL file. The parameters
- are: DAY1, DAY2, DTIME, MAXSTEP, TYPE, TIME, STEP, CLASS, STREAM,
- NUMBER, EXPVER, GRID, LEFT, LOWER, UPPER, RIGHT, LEVEL, LEVELIST,
- RESOL, GAUSS, ACCURACY, OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA,
- SMOOTH, FORMAT, ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS,
- ECFSDIR, MAILOPS, MAILFAIL, GRIB2FLEXPART, DEBUG, INPUTDIR,
- OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
- For more information about format and content of the parameter see
- documentation.
-
'''
+
parser = ArgumentParser(description='Retrieve FLEXPART input from \
- ECMWF MARS archive',
+ ECMWF MARS archive',
formatter_class=ArgumentDefaultsHelpFormatter)
# the most important arguments
- parser.add_argument("--start_date", dest="start_date",
+ parser.add_argument("--start_date", dest="start_date", default=None,
help="start date YYYYMMDD")
- parser.add_argument("--end_date", dest="end_date",
+ parser.add_argument("--end_date", dest="end_date", default=None,
help="end_date YYYYMMDD")
parser.add_argument("--date_chunk", dest="date_chunk", default=None,
help="# of days to be retrieved at once")
# some arguments that override the default in the CONTROL file
- parser.add_argument("--basetime", dest="basetime",
+ parser.add_argument("--basetime", dest="basetime", default=None,
help="base such as 00/12 (for half day retrievals)")
- parser.add_argument("--step", dest="step",
+ parser.add_argument("--step", dest="step", default=None,
help="steps such as 00/to/48")
- parser.add_argument("--levelist", dest="levelist",
+ parser.add_argument("--levelist", dest="levelist", default=None,
help="Vertical levels to be retrieved, e.g. 30/to/60")
- parser.add_argument("--area", dest="area",
+ parser.add_argument("--area", dest="area", default=None,
help="area defined as north/west/south/east")
# set the working directories
@@ -122,124 +101,54 @@ def interpret_args_and_control():
parser.add_argument("--outputdir", dest="outputdir", default=None,
help="root directory for storing output files")
parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
+ default=None,
help="FLEXPART root directory (to find grib2flexpart \
- and COMMAND file)\n Normally ECMWFDATA resides in \
+ and COMMAND file)\n Normally flex_extract resides in \
the scripts directory of the FLEXPART distribution")
# this is only used by prepare_flexpart.py to rerun a postprocessing step
- parser.add_argument("--ppid", dest="ppid",
- help="Specify parent process id for \
+ parser.add_argument("--ppid", dest="ppid", default=None,
+ help="specify parent process id for \
rerun of prepare_flexpart")
# arguments for job submission to ECMWF, only needed by submit.py
parser.add_argument("--job_template", dest='job_template',
default="job.temp",
help="job template file for submission to ECMWF")
- parser.add_argument("--queue", dest="queue",
+ parser.add_argument("--queue", dest="queue", default=None,
help="queue for submission to ECMWF \
(e.g. ecgate or cca )")
parser.add_argument("--controlfile", dest="controlfile",
default='CONTROL.temp',
help="file with CONTROL parameters")
- parser.add_argument("--debug", dest="debug", default=0,
- help="Debug mode - leave temporary files intact")
+ parser.add_argument("--debug", dest="debug", default=None,
+ help="debug mode - leave temporary files intact")
args = parser.parse_args()
- # create instance of ControlFile for specified controlfile
- # and assign the parameters (and default values if necessary)
- try:
- c = ControlFile(args.controlfile)
- except IOError:
- try:
- LOCAL_PYTHON_PATH = os.path.dirname(os.path.abspath(
- inspect.getfile(inspect.currentframe())))
- c = ControlFile(LOCAL_PYTHON_PATH + args.controlfile)
- except IOError:
- print 'Could not read CONTROL file "' + args.controlfile + '"'
- print 'Either it does not exist or its syntax is wrong.'
- print 'Try "' + sys.argv[0].split('/')[-1] + \
- ' -h" to print usage information'
- exit(1)
-
- # check for having at least a starting date
- if args.start_date is None and getattr(c, 'start_date') is None:
- print 'start_date specified neither in command line nor \
- in CONTROL file ' + args.controlfile
- print 'Try "' + sys.argv[0].split('/')[-1] + \
- ' -h" to print usage information'
- exit(1)
-
- # save all existing command line parameter to the ControlFile instance
- # if parameter is not specified through the command line or CONTROL file
- # set default values
- if args.start_date is not None:
- c.start_date = args.start_date
- if args.end_date is not None:
- c.end_date = args.end_date
- if c.end_date is None:
- c.end_date = c.start_date
- if args.date_chunk is not None:
- c.date_chunk = args.date_chunk
-
- if not hasattr(c, 'debug'):
- c.debug = args.debug
-
- if args.inputdir is None and args.outputdir is None:
- c.inputdir = '../work'
- c.outputdir = '../work'
- else:
- if args.inputdir is not None:
- c.inputdir = args.inputdir
- if args.outputdir is None:
- c.outputdir = args.inputdir
- if args.outputdir is not None:
- c.outputdir = args.outputdir
- if args.inputdir is None:
- c.inputdir = args.outputdir
-
- if hasattr(c, 'outputdir') is False and args.outputdir is None:
- c.outputdir = c.inputdir
- else:
- if args.outputdir is not None:
- c.outputdir = args.outputdir
-
- if args.area is not None:
- afloat = '.' in args.area
- l = args.area.split('/')
- if afloat:
- for i, item in enumerate(l):
- item = str(int(float(item) * 1000))
- c.upper, c.left, c.lower, c.right = l
-
- # NOTE: basetime activates the ''operational mode''
- if args.basetime is not None:
- c.basetime = args.basetime
-
- if args.step is not None:
- l = args.step.split('/')
- if 'to' in args.step.lower():
- if 'by' in args.step.lower():
- ilist = np.arange(int(l[0]), int(l[2]) + 1, int(l[4]))
- c.step = ['{:0>3}'.format(i) for i in ilist]
- else:
- my_error(None, args.step + ':\n' +
- 'please use "by" as well if "to" is used')
- else:
- c.step = l
+ return args
- if args.levelist is not None:
- c.levelist = args.levelist
- if 'to' in c.levelist:
- c.level = c.levelist.split('/')[2]
- else:
- c.level = c.levelist.split('/')[-1]
+def read_ecenv(filename):
+ '''
+ @Description:
+ Reads the file into a dictionary where the key values are the parameter
+ names.
- if args.flexpart_root_scripts is not None:
- c.flexpart_root_scripts = args.flexpart_root_scripts
+ @Input:
+ filename: string
+ Name of file where the ECMWV environment parameters are stored.
- return args, c
+ @Return:
+ envs: dict
+ '''
+ envs= {}
+ print filename
+ with open(filename, 'r') as f:
+ for line in f:
+ data = line.strip().split()
+ envs[str(data[0])] = str(data[1])
+ return envs
def clean_up(c):
'''
@@ -268,36 +177,28 @@ def clean_up(c):
print "clean_up"
cleanlist = glob.glob(c.inputdir + "/*")
- for cl in cleanlist:
- if c.prefix not in cl:
- silent_remove(cl)
+ for clist in cleanlist:
+ if c.prefix not in clist:
+ silent_remove(clist)
if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
- silent_remove(cl)
+ silent_remove(clist)
print "Done"
return
-def my_error(c, message='ERROR'):
+def my_error(users, message='ERROR'):
'''
@Description:
Prints a specified error message which can be passed to the function
before exiting the program.
@Input:
- c: instance of class ControlFile
- Contains all the parameters of CONTROL file, which are e.g.:
- DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
- STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
- LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
- OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
- ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
- MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
- DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
-
- For more information about format and content of the parameter
- see documentation.
+ user: list of strings
+ Contains all email addresses which should be notified.
+ It might also contain just the ecmwf user name which wil trigger
+ mailing to the associated email address for this user.
message: string, optional
Error message. Default value is "ERROR".
@@ -309,47 +210,40 @@ def my_error(c, message='ERROR'):
print message
# comment if user does not want email notification directly from python
- try:
- target = []
- target.extend(c.mailfail)
- except AttributeError:
- target = []
- target.extend(os.getenv('USER'))
-
- for t in target:
- p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 ERROR',
- os.path.expandvars(t)],
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- bufsize=1)
- tr = '\n'.join(traceback.format_stack())
- pout = p.communicate(input=message + '\n\n' + tr)[0]
- print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode()
-
- exit(1)
+ for user in users:
+ if '${USER}' in user:
+ user = os.getenv('USER')
+ try:
+ p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
+ os.path.expandvars(user)],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ bufsize=1)
+ trace = '\n'.join(traceback.format_stack())
+ pout = p.communicate(input=message + '\n\n' + trace)[0]
+ except ValueError as e:
+ print 'ERROR: ', e
+ sys.exit('Email could not be sent!')
+ else:
+ print 'Email sent to ' + os.path.expandvars(user) + ' ' + \
+ pout.decode()
+
+ sys.exit(1)
return
-def normal_exit(c, message='Done!'):
+def normal_exit(users, message='Done!'):
'''
@Description:
Prints a specific exit message which can be passed to the function.
@Input:
- c: instance of class ControlFile
- Contains all the parameters of CONTROL file, which are e.g.:
- DAY1(start_date), DAY2(end_date), DTIME, MAXSTEP, TYPE, TIME,
- STEP, CLASS(marsclass), STREAM, NUMBER, EXPVER, GRID, LEFT,
- LOWER, UPPER, RIGHT, LEVEL, LEVELIST, RESOL, GAUSS, ACCURACY,
- OMEGA, OMEGADIFF, ETA, ETADIFF, DPDETA, SMOOTH, FORMAT,
- ADDPAR, WRF, CWC, PREFIX, ECSTORAGE, ECTRANS, ECFSDIR,
- MAILOPS, MAILFAIL, GRIB2FLEXPART, FLEXPARTDIR, BASETIME
- DATE_CHUNK, DEBUG, INPUTDIR, OUTPUTDIR, FLEXPART_ROOT_SCRIPTS
-
- For more information about format and content of the parameter
- see documentation.
+ user: list of strings
+ Contains all email addresses which should be notified.
+ It might also contain just the ecmwf user name which wil trigger
+ mailing to the associated email address for this user.
message: string, optional
Message for exiting program. Default value is "Done!".
@@ -361,20 +255,23 @@ def normal_exit(c, message='Done!'):
print message
# comment if user does not want notification directly from python
- try:
- target = []
- target.extend(c.mailops)
- for t in target:
- p = subprocess.Popen(['mail', '-s ECMWFDATA v7.0 normal exit',
- os.path.expandvars(t)],
+ for user in users:
+ if '${USER}' in user:
+ user = os.getenv('USER')
+ try:
+ p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
+ os.path.expandvars(user)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1)
pout = p.communicate(input=message+'\n\n')[0]
- print 'Email sent to ' + os.path.expandvars(t) + ' ' + pout.decode()
- finally:
- pass
+ except ValueError as e:
+ print 'ERROR: ', e
+ print 'Email could not be sent!'
+ else:
+ print 'Email sent to ' + os.path.expandvars(user) + ' ' + \
+ pout.decode()
return
@@ -432,7 +329,7 @@ def silent_remove(filename):
os.remove(filename)
except OSError as e:
# this would be "except OSError, e:" before Python 2.6
- if e.errno is not errno.ENOENT:
+ if e.errno != errno.ENOENT:
# errno.ENOENT = no such file or directory
raise # re-raise exception if a different error occured
@@ -520,3 +417,97 @@ def get_list_as_string(list_obj, concatenate_sign=', '):
str_of_list = concatenate_sign.join(str(l) for l in list_obj)
return str_of_list
+
+def make_dir(directory):
+ '''
+ @Description:
+ Creates a directory and gives a warning if the directory
+ already exists. The program stops only if there is another problem.
+
+ @Input:
+ directory: string
+ The directory path which should be created.
+
+ @Return:
+ <nothing>
+ '''
+ try:
+ os.makedirs(directory)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ # errno.EEXIST = directory already exists
+ raise # re-raise exception if a different error occured
+ else:
+ print 'WARNING: Directory {0} already exists!'.format(directory)
+
+ return
+
+def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
+ '''
+ @Description:
+ Uses the ecaccess command to send a file to the ECMWF servers.
+ Catches and prints the error if it failed.
+
+ @Input:
+ ecd: string
+ The path were the file is to be stored.
+
+ filename: string
+ The name of the file to send to the ECMWF server.
+
+ target: string
+ The target where the file should be sent to, e.g. the queue.
+
+ ecuid: string
+ The user id on ECMWF server.
+
+ ecgid: string
+ The group id on ECMWF server.
+
+ @Return:
+ <nothing>
+ '''
+
+ try:
+ subprocess.check_call(['ecaccess-file-put',
+ ecd + '../' + filename,
+ target + ':/home/ms/' +
+ ecgid + '/' + ecuid +
+ '/' + filename])
+ except subprocess.CalledProcessError as e:
+ print 'ERROR:'
+ print e
+ sys.exit('ecaccess-file-put failed!\n' + \
+ 'Probably the eccert key has expired.')
+
+ return
+
+def submit_job_to_ecserver(ecd, target, jobname):
+ '''
+ @Description:
+ Uses ecaccess to submit a job to the ECMWF server.
+ Catches and prints the error if one arise.
+
+ @Input:
+ ecd: string
+ The path were the file is to be stored.
+
+ target: string
+ The target where the file should be sent to, e.g. the queue.
+
+ jobname: string
+ The name of the jobfile to be submitted to the ECMWF server.
+
+ @Return:
+ <nothing>
+ '''
+
+ try:
+ subprocess.check_call(['ecaccess-job-submit',
+ '-queueName', target,
+ jobname])
+ except subprocess.CalledProcessError as e:
+ print '... ERROR CODE: ', e.returncode
+ sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
+
+ return
diff --git a/pythontest/TestTools.py b/pythontest/TestTools.py
index 5fe4feb1532564c8a82290b343dc92d365727ebd..ee0d97034ee7f2419ba25f5080f24144d8f920a3 100644
--- a/pythontest/TestTools.py
+++ b/pythontest/TestTools.py
@@ -3,8 +3,9 @@
import unittest
import sys
+import os
sys.path.append('../python')
-from Tools import init128, toparamId
+from tools import init128, to_param_id, my_error, read_ecenv
class TestTools(unittest.TestCase):
@@ -24,14 +25,35 @@ class TestTools(unittest.TestCase):
self.assertEqual(result, True)
- def test_toparamId(self):
+ def test_to_param_id(self):
'''
'''
table128 = init128('../grib_templates/ecmwf_grib1_table_128')
- pars = toparamId("T/SP/LSP/SSHF", table128)
+ pars = to_param_id("T/SP/LSP/SSHF", table128)
for par in pars:
self.assertIn(par, [130, 134, 142, 146])
+ def test_error_notifcation(self):
+ '''
+ '''
+ with self.assertRaises(SystemExit) as re:
+ my_error(['${USER}', 'anne.philipp@univie.ac.at'], 'Failed!')
+ self.assertEqual(re.exception.code, 1)
+
+ def test_read_ecenv(self):
+
+ envs_ref = {'ECUID': 'km4a',
+ 'ECGID': 'at',
+ 'GATEWAY': 'srvx8.img.univie.ac.at',
+ 'DESTINATION': 'annep@genericSftp'
+ }
+ envs = read_ecenv(os.getcwd() + '/TestData/ECMWF_ENV')
+
+ self.assertDictEqual(envs_ref, envs)
+
+
+
+
if __name__ == "__main__":
unittest.main()
\ No newline at end of file
diff --git a/pythontest/TestUIOFiles.py b/pythontest/TestUIOFiles.py
index 3d929cb3d096fde8ce03d25c9def2bed4537c130..d60cbf040c3a7223502190568824040b95a03c53 100644
--- a/pythontest/TestUIOFiles.py
+++ b/pythontest/TestUIOFiles.py
@@ -5,10 +5,10 @@ import unittest
import os
import sys
sys.path.append('../python')
-import UIOFiles
+import UioFiles
-class TestUIOFiles(unittest.TestCase):
+class TestUioFiles(unittest.TestCase):
'''
Test class to test the UIOFiles methods.
'''
@@ -51,8 +51,8 @@ class TestUIOFiles(unittest.TestCase):
'''
# Initialise and collect filenames
- files = UIOFiles.UIOFiles(['.grb'])
- files.listFiles(self.testpath, '*')
+ files = UioFiles.UioFiles(['.grb'])
+ files.list_files(self.testpath, '*')
# get the basename to just check for equality of filenames
filelist = [os.path.basename(f) for f in files.files]
# comparison of expected filenames against the collected ones