From 5bad6ecd8c2b5597cc84f56143c36adc3ad18530 Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Fri, 5 Oct 2018 19:20:48 +0200
Subject: [PATCH] added possibility to extract public datasets via an logical
public parameter
---
run/control/CONTROL_EI.public | 37 ++++++++
run/run.sh | 8 +-
source/python/classes/ControlFile.py | 14 ++-
source/python/classes/EcFlexpart.py | 18 ++--
source/python/classes/MarsRetrieval.py | 121 +++++++++++++++++--------
source/python/mods/get_mars_data.py | 12 ++-
source/python/mods/prepare_flexpart.py | 4 +-
source/python/mods/tools.py | 3 +
8 files changed, 163 insertions(+), 54 deletions(-)
create mode 100644 run/control/CONTROL_EI.public
diff --git a/run/control/CONTROL_EI.public b/run/control/CONTROL_EI.public
new file mode 100644
index 0000000..e51e5f8
--- /dev/null
+++ b/run/control/CONTROL_EI.public
@@ -0,0 +1,37 @@
+DAY1
+DAY2
+DTIME 6
+TYPE AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC AN FC FC FC FC FC
+TIME 00 00 00 00 00 00 06 00 00 00 00 00 12 12 12 12 12 12 18 12 12 12 12 12
+STEP 00 01 02 03 04 05 00 07 08 09 10 11 00 01 02 03 04 05 00 07 08 09 10 11
+CLASS EI
+DATASET interim
+STREAM OPER
+NUMBER OFF
+EXPVER 1
+GRID 5000
+LEFT -15000
+LOWER 30000
+UPPER 75000
+RIGHT 45000
+LEVELIST 55/to/60
+RESOL 63
+ACCURACY 24
+GAUSS 1
+OMEGA 0
+OMEGADIFF 0
+ETA 0
+ETADIFF 0
+DPDETA 1
+SMOOTH 0
+FORMAT GRIB1
+ADDPAR 186/187/188/235/139/39
+PREFIX EIpub
+ECSTORAGE 0
+ECTRANS 1
+ECFSDIR ectmp:/${USER}/econdemand/
+MAILFAIL ${USER}
+MAILOPS ${USER}
+GRIB2FLEXPART 0
+EOF
+
diff --git a/run/run.sh b/run/run.sh
index f192210..91ee484 100755
--- a/run/run.sh
+++ b/run/run.sh
@@ -22,14 +22,15 @@ BASETIME=None
STEP=None
LEVELIST=None
AREA=None
-INPUTDIR='../../run/workspace/test'
+INPUTDIR='/nas/tmc/Anne/Interpolation/flexextract/flex_extract_v7.1/run/workspace/test'
OUTPUTDIR=None
FLEXPART_ROOT_SCRIPTS=None
PP_ID=None
JOB_TEMPLATE='job.temp'
-CONTROLFILE='CONTROL.temp'
+CONTROLFILE='CONTROL_EI.public'
DEBUG=1
REQUEST=1
+PUBLIC=1
# -----------------------------------------------------------------
#
@@ -92,6 +93,9 @@ fi
if [ -n "$REQUEST" ]; then
parameterlist+=" --request=$REQUEST"
fi
+if [ -n "$PUBLIC" ]; then
+ parameterlist+=" --public=$PUBLIC"
+fi
# -----------------------------------------------------------------
# CALL INSTALLATION SCRIPT WITH DETERMINED COMMANDLINE ARGUMENTS
diff --git a/source/python/classes/ControlFile.py b/source/python/classes/ControlFile.py
index 0b8a62b..25ea969 100644
--- a/source/python/classes/ControlFile.py
+++ b/source/python/classes/ControlFile.py
@@ -113,6 +113,7 @@ class ControlFile(object):
self.time = None
self.step = None
self.marsclass = None
+ self.dataset = None
self.stream = None
self.number = 'OFF'
self.expver = '1'
@@ -158,10 +159,11 @@ class ControlFile(object):
self.install_target = None
self.debug = 0
self.request = 0
+ self.public = 0
self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
- 'ectrans', 'debug', 'request']
+ 'ectrans', 'debug', 'request', 'public']
self.__read_controlfile__()
@@ -265,7 +267,7 @@ class ControlFile(object):
'''
import collections
- attrs = vars(self)
+ attrs = vars(self).copy()
attrs = collections.OrderedDict(sorted(attrs.items()))
return '\n'.join("%s: %s" % item for item in attrs.items())
@@ -453,6 +455,12 @@ class ControlFile(object):
if not isinstance(getattr(self, var), int):
setattr(self, var, int(getattr(self, var)))
+ if self.public and not self.dataset:
+ print('ERROR: ')
+ print('If public mars data wants to be retrieved, '
+ 'the "dataset"-parameter has to be set in the control file!')
+ sys.exit(1)
+
return
def check_install_conditions(self):
@@ -523,7 +531,7 @@ class ControlFile(object):
import collections
- attrs = collections.OrderedDict(sorted(vars(self).items()))
+ attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
l = list()
diff --git a/source/python/classes/EcFlexpart.py b/source/python/classes/EcFlexpart.py
index 3bec913..8ce3f35 100644
--- a/source/python/classes/EcFlexpart.py
+++ b/source/python/classes/EcFlexpart.py
@@ -88,7 +88,8 @@ from gribapi import grib_set, grib_index_select, grib_new_from_index, grib_get,\
sys.path.append('../')
import _config
from GribTools import GribTools
-from mods.tools import init128, to_param_id, silent_remove, product, my_error
+from mods.tools import (init128, to_param_id, silent_remove, product,
+ my_error, make_dir)
from MarsRetrieval import MarsRetrieval
import mods.disaggregation as disaggregation
@@ -142,6 +143,7 @@ class EcFlexpart(object):
c.time.append(c.time[0])
self.inputdir = c.inputdir
+ self.dataset = c.dataset
self.basetime = c.basetime
self.dtime = c.dtime
i = 0
@@ -337,9 +339,9 @@ class EcFlexpart(object):
par_dict: dictionary
Contains all parameter which have to be set for creating the
Mars Retrievals. The parameter are:
- marsclass, stream, type, levtype, levelist, resol, gaussian,
- accuracy, grid, target, area, date, time, number, step, expver,
- param
+ marsclass, dataset, stream, type, levtype, levelist, resol,
+ gaussian, accuracy, grid, target, area, date, time, number,
+ step, expver, param
@Return:
<nothing>
@@ -348,7 +350,9 @@ class EcFlexpart(object):
self.mreq_count += 1
MR = MarsRetrieval(self.server,
+ self.public,
marsclass=par_dict['marsclass'],
+ dataset=par_dict['dataset'],
stream=par_dict['stream'],
type=par_dict['type'],
levtype=par_dict['levtype'],
@@ -438,7 +442,7 @@ class EcFlexpart(object):
return iid, index_vals
- def retrieve(self, server, dates, request, inputdir='.'):
+ def retrieve(self, server, dates, public, request, inputdir='.'):
'''
@Description:
Finalizing the retrieval information by setting final details
@@ -476,6 +480,7 @@ class EcFlexpart(object):
'''
self.dates = dates
self.server = server
+ self.public = public
self.inputdir = inputdir
oro = False
@@ -487,6 +492,7 @@ class EcFlexpart(object):
# entries with a "None" will change in different requests and will
# therefore be set in each request seperately
retr_param_dict = {'marsclass':self.marsclass,
+ 'dataset':self.dataset,
'stream':None,
'type':None,
'levtype':None,
@@ -1264,7 +1270,7 @@ class EcFlexpart(object):
# create Options dir if necessary
if not os.path.exists(pwd + '/Options'):
- os.makedirs(pwd+'/Options')
+ make_dir(pwd+'/Options')
# read template COMMAND file
with open(os.path.expandvars(os.path.expanduser(
diff --git a/source/python/classes/MarsRetrieval.py b/source/python/classes/MarsRetrieval.py
index b507518..640ad8c 100644
--- a/source/python/classes/MarsRetrieval.py
+++ b/source/python/classes/MarsRetrieval.py
@@ -82,10 +82,10 @@ class MarsRetrieval(object):
'''
- def __init__(self, server, marsclass="ei", type="", levtype="",
- levelist="", repres="", date="", resol="", stream="",
- area="", time="", step="", expver="1", number="",
- accuracy="", grid="", gaussian="", target="",
+ def __init__(self, server, public, marsclass="ei", dataset="", type="",
+ levtype="", levelist="", repres="", date="", resol="",
+ stream="", area="", time="", step="", expver="1",
+ number="", accuracy="", grid="", gaussian="", target="",
param=""):
'''
@Description:
@@ -105,11 +105,25 @@ class MarsRetrieval(object):
This is the connection to the ECMWF data servers.
It is needed for the pythonic access of ECMWF data.
+ public: integer
+ Decides which Web API version is used:
+ 0: member-state users and full archive access
+ 1: public access and limited access to the public server and
+ datasets. Needs the parameter dataset.
+ Default is "0" and for member-state users.
+
marsclass: string, optional
Characterisation of dataset. E.g. EI (ERA-Interim),
E4 (ERA40), OD (Operational archive), ea (ERA5).
Default is the ERA-Interim dataset "ei".
+ dataset: string, optional
+ For public datasets there is the specific naming and parameter
+ dataset which has to be used to characterize the type of
+ data. Usually there is less data available, either in times,
+ domain or parameter.
+ Default is an empty string.
+
type: string, optional
Determines the type of fields to be retrieved.
Selects between observations, images or fields.
@@ -288,7 +302,9 @@ class MarsRetrieval(object):
'''
self.server = server
+ self.public = public
self.marsclass = marsclass
+ self.dataset = dataset
self.type = type
self.levtype = levtype
self.levelist = levelist
@@ -324,12 +340,12 @@ class MarsRetrieval(object):
<nothing>
'''
# Get all class attributes and their values as a dictionary
- attrs = vars(self)
+ attrs = vars(self).copy()
# iterate through all attributes and print them
# with their corresponding values
for item in attrs.items():
- if item[0] in 'server':
+ if item[0] in ['server', 'public']:
pass
else:
print(item[0] + ': ' + str(item[1]))
@@ -357,7 +373,7 @@ class MarsRetrieval(object):
<nothing>
'''
# Get all class attributes and their values as a dictionary
- attrs = vars(self)
+ attrs = vars(self).copy()
# open a file to store all requests to
with open(os.path.join(inputdir,
@@ -366,7 +382,7 @@ class MarsRetrieval(object):
# iterate through all attributes and print them
# with their corresponding values
for item in attrs.items():
- if item[0] in 'server':
+ if item[0] in ['server', 'public']:
pass
else:
f.write(item[0] + ': ' + str(item[1]) + '\n')
@@ -395,8 +411,9 @@ class MarsRetrieval(object):
'''
# Get all class attributes and their values as a dictionary
- attrs = vars(self)
+ attrs = vars(self).copy()
del attrs['server']
+ del attrs['public']
# open a file to store all requests to
with open(os.path.join(inputdir,
@@ -424,50 +441,80 @@ class MarsRetrieval(object):
<nothing>
'''
# Get all class attributes and their values as a dictionary
- attrs = vars(self)
-
- # convert the dictionary of attributes into a comma
- # seperated list of attributes with their values
- # needed for the retrieval call
- s = 'ret'
- for k, v in attrs.iteritems():
- if k in 'server':
- continue
- if k == 'marsclass':
- k = 'class'
- if v == '':
- continue
- if k.lower() == 'target':
- target = v
+ attrs = vars(self).copy()
+
+ # eliminate unnecessary attributes from the dictionary attrs
+ del attrs['server']
+ del attrs['public']
+
+ # exchange parameter name for marsclass
+ mclass = attrs.get('marsclass')
+ del attrs['marsclass']
+ attrs['class'] = mclass
+
+ # prepare target variable as needed for the Web API mode
+ # within the dictionary for full access
+ # as a single variable for public access
+ target = attrs.get('target')
+ if not int(self.public):
+ del attrs['target']
+ print('target: ' + target)
+
+ # find all keys without a value and convert all other values to strings
+ empty_keys = []
+ for key, value in attrs.itteritems():
+ if value == '':
+ empty_keys.append(str(key))
else:
- s = s + ',' + k + '=' + str(v)
+ attrs[key] = str(value)
+
+ # delete all empty parameter from the dictionary
+ for key in empty_keys:
+ del attrs[key]
# MARS request via Python script
- if self.server is not False:
+ if self.server:
try:
- self.server.execute(s, target)
+ if self.public:
+ print('RETRIEVE PUBLIC DATA!')
+ self.server.retrieve(attrs)
+ else:
+ print('EXECUTE NON-PUBLIC RETRIEVAL!')
+ self.server.execute(attrs, target)
except:
- print('MARS Request failed, \
- have you already registered at apps.ecmwf.int?')
- raise IOError
- if os.stat(target).st_size == 0:
+ e = sys.exc_info()[0]
+ print("ERROR: ", e)
+ print('MARS Request failed!')
+ if not self.public and os.stat(target).st_size == 0:
print('MARS Request returned no data - please check request')
raise IOError
+ elif self.public and os.stat(target).st_size == 0:
+ print('Public MARS Request returned no data - '
+ 'please check request')
+ raise IOError
+ else:
+ raise IOError
# MARS request via extra process in shell
else:
- s += ',target = "' + target + '"'
- p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
+ request_str = 'ret'
+ for key, value in attrs.iteritems():
+ request_str = request_str + ',' + key + '=' + str(value)
+ request_str += ',target="' + target + '"'
+ p = subprocess.Popen(['mars'],
+ stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, bufsize=1)
- pout = p.communicate(input=s)[0]
+ stderr=subprocess.PIPE,
+ bufsize=1)
+ pout = p.communicate(input=request_str)[0]
print(pout.decode())
if 'Some errors reported' in pout.decode():
print('MARS Request failed - please check request')
raise IOError
-
- if os.stat(target).st_size == 0:
+ elif os.stat(target).st_size == 0:
print('MARS Request returned no data - please check request')
raise IOError
+ else:
+ raise
return
diff --git a/source/python/mods/get_mars_data.py b/source/python/mods/get_mars_data.py
index 17ac358..79b1ec4 100755
--- a/source/python/mods/get_mars_data.py
+++ b/source/python/mods/get_mars_data.py
@@ -53,7 +53,8 @@ from datetime import datetime, timedelta
# software specific classes and modules from flex_extract
sys.path.append('../')
import _config
-from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv
+from tools import (my_error, normal_exit, get_cmdline_arguments,
+ read_ecenv, make_dir)
from classes.EcFlexpart import EcFlexpart
from classes.UioFiles import UioFiles
@@ -113,7 +114,7 @@ def get_mars_data(c):
'''
if not os.path.exists(c.inputdir):
- os.makedirs(c.inputdir)
+ make_dir(c.inputdir)
if c.request == 0 or c.request == 2:
print("Retrieving EC data!")
@@ -124,7 +125,10 @@ def get_mars_data(c):
print("end date %s " % (c.end_date))
if ecapi:
- server = ecmwfapi.ECMWFService("mars")
+ if c.public:
+ server = ecmwfapi.ECMWFDataServer()
+ else:
+ server = ecmwfapi.ECMWFService("mars")
else:
server = False
@@ -253,7 +257,7 @@ def do_retrievement(c, server, start, end, delta_t, fluxes=False):
print("... retrieve " + dates + " in dir " + c.inputdir)
try:
- flexpart.retrieve(server, dates, c.request, c.inputdir)
+ flexpart.retrieve(server, dates, c.public, c.request, c.inputdir)
except IOError:
my_error(c.mailfail, 'MARS request failed')
diff --git a/source/python/mods/prepare_flexpart.py b/source/python/mods/prepare_flexpart.py
index ade11a8..a3ca22b 100755
--- a/source/python/mods/prepare_flexpart.py
+++ b/source/python/mods/prepare_flexpart.py
@@ -62,7 +62,7 @@ sys.path.append('../')
import _config
from classes.UioFiles import UioFiles
from classes.ControlFile import ControlFile
-from tools import clean_up, get_cmdline_arguments, read_ecenv
+from tools import clean_up, get_cmdline_arguments, read_ecenv, make_dir
from classes.EcFlexpart import EcFlexpart
ecapi = 'ecmwf' not in socket.gethostname()
@@ -152,7 +152,7 @@ def prepare_flexpart(ppid, c):
# create output dir if necessary
if not os.path.exists(c.outputdir):
- os.makedirs(c.outputdir)
+ make_dir(c.outputdir)
# get all files with flux data to be deaccumulated
inputfiles = UioFiles(c.inputdir, '*OG_acc_SL*.' + c.ppid + '.*')
diff --git a/source/python/mods/tools.py b/source/python/mods/tools.py
index cda9c15..d8b6c02 100644
--- a/source/python/mods/tools.py
+++ b/source/python/mods/tools.py
@@ -138,6 +138,9 @@ def get_cmdline_arguments():
type=none_or_int, default=None,
help="list all mars request in file mars_requests.dat \
and skip submission to mars")
+ parser.add_argument("--public", dest="public",
+ type=none_or_int, default=None,
+ help="public mode - retrieves the public datasets")
# some arguments that override the default in the CONTROL file
parser.add_argument("--basetime", dest="basetime",
--
GitLab