Skip to content
Snippets Groups Projects
Commit 295ff45a authored by Anne Philipp's avatar Anne Philipp
Browse files

added request output option also for basetime requests; changed output to csv format

parent 7b4e39e7
No related branches found
No related tags found
No related merge requests found
...@@ -31,7 +31,7 @@ _VERSION_STR = '7.1' ...@@ -31,7 +31,7 @@ _VERSION_STR = '7.1'
# FILENAMES # FILENAMES
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
FILE_MARS_REQUESTS = 'mars_requests.dat' FILE_MARS_REQUESTS = 'mars_requests.csv'
FORTRAN_EXECUTABLE = 'CONVERT2' FORTRAN_EXECUTABLE = 'CONVERT2'
FILE_USER_ENVVARS = 'ECMWF_ENV' FILE_USER_ENVVARS = 'ECMWF_ENV'
TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.temp' TEMPFILE_INSTALL_COMPILEJOB = 'compilejob.temp'
......
...@@ -129,6 +129,8 @@ class EcFlexpart(object): ...@@ -129,6 +129,8 @@ class EcFlexpart(object):
@Return: @Return:
<nothing> <nothing>
''' '''
# set a counter for the number of mars requests generated
self.mreq_count = 0
# different mars types for retrieving data for flexpart # different mars types for retrieving data for flexpart
self.types = dict() self.types = dict()
...@@ -224,6 +226,8 @@ class EcFlexpart(object): ...@@ -224,6 +226,8 @@ class EcFlexpart(object):
'GG__ML': '', 'GG__SL': '', 'GG__ML': '', 'GG__SL': '',
'OG__ML': '', 'OG__SL': '', 'OG__ML': '', 'OG__SL': '',
'OG_OROLSM_SL': '', 'OG_acc_SL': ''} 'OG_OROLSM_SL': '', 'OG_acc_SL': ''}
# the self.params dictionary stores a list of
# [param, levtype, levelist, grid] per key
if fluxes is False: if fluxes is False:
self.params['SH__SL'] = ['LNSP', 'ML', '1', 'OFF'] self.params['SH__SL'] = ['LNSP', 'ML', '1', 'OFF']
...@@ -277,7 +281,7 @@ class EcFlexpart(object): ...@@ -277,7 +281,7 @@ class EcFlexpart(object):
#wrf_sfc = 'sp/msl/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/ / #wrf_sfc = 'sp/msl/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/ /
# stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4'.upper() # stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4'.upper()
wrf_sfc = '134/235/167/165/166/168/129/172/34/31/141/ \ wrf_sfc = '134/235/167/165/166/168/129/172/34/31/141/ \
139/170/183/236/39/40/41/42'.upper() 139/170/183/236/39/40/41/42'
lwrt_sfc = wrf_sfc.split('/') lwrt_sfc = wrf_sfc.split('/')
for par in lwrt_sfc: for par in lwrt_sfc:
if par not in self.params['OG__SL'][0]: if par not in self.params['OG__SL'][0]:
...@@ -292,6 +296,95 @@ class EcFlexpart(object): ...@@ -292,6 +296,95 @@ class EcFlexpart(object):
return return
def _start_retrievement(self, request, par_dict):
'''
@Description:
Creates the Mars Retrieval and prints or submits the request
depending on the status of the request variable.
@Input:
self: instance of EcFlexpart
The current object of the class.
request: integer
Selects the mode of retrieval.
0: Retrieves the data from ECMWF.
1: Prints the mars requests to an output file.
2: Retrieves the data and prints the mars request.
par_dict: dictionary
Contains all parameter which have to be set for creating the
Mars Retrievals. The parameter are:
marsclass, stream, type, levtype, levelist, resol, gaussian,
accuracy, grid, target, area, date, time, number, step, expver,
param
@Return:
<nothing>
'''
# increase number of mars requests
self.mreq_count += 1
MR = MarsRetrieval(self.server,
marsclass=par_dict['marsclass'],
stream=par_dict['stream'],
type=par_dict['type'],
levtype=par_dict['levtype'],
levelist=par_dict['levelist'],
resol=par_dict['resol'],
gaussian=par_dict['gaussian'],
accuracy=par_dict['accuracy'],
grid=par_dict['grid'],
target=par_dict['target'],
area=par_dict['area'],
date=par_dict['date'],
time=par_dict['time'],
number=par_dict['number'],
step=par_dict['step'],
expver=par_dict['expver'],
param=par_dict['param'])
if request == 0:
MR.display_info()
MR.data_retrieve()
elif request == 1:
MR.print_infodata_csv(self.inputdir, self.mreq_count)
elif request == 2:
MR.print_infodata_csv(self.inputdir, self.mreq_count)
MR.display_info()
MR.data_retrieve()
else:
print('Failure')
return
def _mk_targetname(self, ftype, param, date):
'''
@Description:
Creates the filename for the requested grib data to be stored in.
This name is passed as the "target" parameter in the request.
@Input:
ftype: string
Shortcut name of the type of the field. E.g. AN, FC, PF, ...
param: string
Shortcut of the grid type. E.g. SH__ML, SH__SL, GG__ML,
GG__SL, OG__ML, OG__SL, OG_OROLSM_SL, OG_acc_SL
date: string
The date period of the grib data to be stored in this file.
@Return:
targetname: string
The target filename for the grib data.
'''
targetname = (self.inputdir + '/' + ftype + param + '.' + date + '.' +
str(os.getppid()) + '.' + str(os.getpid()) + '.grb')
return targetname
def write_namelist(self, c, filename): def write_namelist(self, c, filename):
''' '''
@Description: @Description:
...@@ -380,6 +473,12 @@ class EcFlexpart(object): ...@@ -380,6 +473,12 @@ class EcFlexpart(object):
Contains start and end date of the retrieval in the format Contains start and end date of the retrieval in the format
"YYYYMMDD/to/YYYYMMDD" "YYYYMMDD/to/YYYYMMDD"
request: integer
Selects the mode of retrieval.
0: Retrieves the data from ECMWF.
1: Prints the mars requests to an output file.
2: Retrieves the data and prints the mars request.
inputdir: string, optional inputdir: string, optional
Path to the directory where the retrieved data is about Path to the directory where the retrieved data is about
to be stored. The default is the current directory ('.'). to be stored. The default is the current directory ('.').
...@@ -392,94 +491,101 @@ class EcFlexpart(object): ...@@ -392,94 +491,101 @@ class EcFlexpart(object):
self.inputdir = inputdir self.inputdir = inputdir
oro = False oro = False
# define times
t12h = timedelta(hours=12)
t24h = timedelta(hours=24)
# dictionary which contains all parameter for the mars request
# Entries with a "None" will change for different requests and therefore
# will be set in each request seperately
retr_param_dict = {'marsclass':self.marsclass,
'stream':None,
'type':None,
'levtype':None,
'levelist':None,
'resol':self.resol,
'gaussian':None,
'accuracy':self.accuracy,
'grid':None,
'target':None,
'area':None,
'date':None,
'time':None,
'number':self.number,
'step':None,
'expver':self.expver,
'param':None}
for ftype in self.types: for ftype in self.types:
# fk contains fields types such as
# [AN, FC, PF, CV]
# fv contains all of the items of the belonging key
# [times, steps]
for pk, pv in self.params.iteritems(): for pk, pv in self.params.iteritems():
# pk contains one of these keys of params
# [SH__ML, SH__SL, GG__ML, GG__SL, OG__ML, OG__SL,
# OG_OROLSM_SL, OG_acc_SL]
# pv contains all of the items of the belonging key
# [param, levtype, levelist, grid]
if isinstance(pv, str): if isinstance(pv, str):
continue continue
mftype = '' + ftype retr_param_dict['type'] = '' + ftype
mftime = self.types[ftype]['times'] retr_param_dict['time'] = self.types[ftype]['times']
mfstep = self.types[ftype]['steps'] retr_param_dict['step'] = self.types[ftype]['steps']
mfdate = self.dates retr_param_dict['date'] = self.dates
mfstream = self.stream retr_param_dict['stream'] = self.stream
mftarget = self.inputdir + "/" + ftype + pk + '.' + \ retr_param_dict['target'] = self.inputdir + "/" + ftype + \
self.dates.split('/')[0] + '.' + str(os.getppid()) +\ pk + '.' + self.dates.split('/')[0] + '.' + \
'.' + str(os.getpid()) + ".grb" str(os.getppid()) + '.' + str(os.getpid()) + ".grb"
retr_param_dict['param'] = pv[0]
retr_param_dict['levtype'] = pv[1]
retr_param_dict['levelist'] = pv[2]
retr_param_dict['grid'] = pv[3]
retr_param_dict['area'] = self.area
retr_param_dict['gaussian'] = self.gaussian
if pk == 'OG__SL': if pk == 'OG__SL':
pass pass
if pk == 'OG_OROLSM__SL': if pk == 'OG_OROLSM__SL' and not oro:
if not oro:
mfstream = 'OPER'
mftype = 'AN'
mftime = '00'
mfstep = '000'
mfdate = self.dates.split('/')[0]
mftarget = self.inputdir + "/" + pk + '.' + mfdate + \
'.' + str(os.getppid()) + '.' + \
str(os.getpid()) + ".grb"
oro = True oro = True
else: retr_param_dict['stream'] = 'OPER'
retr_param_dict['type'] = 'AN'
retr_param_dict['time'] = '00'
retr_param_dict['step'] = '000'
retr_param_dict['date'] = self.dates.split('/')[0]
retr_param_dict['target'] = self._mk_targetname('',
pk, retr_param_dict['date'])
elif pk == 'OG_OROLSM__SL' and oro:
continue continue
if pk == 'GG__SL' and pv[0] == 'Q': if pk == 'GG__SL' and pv[0] == 'Q':
area = "" retr_param_dict['area'] = ""
gaussian = 'reduced' retr_param_dict['gaussian'] = 'reduced'
else:
area = self.area
gaussian = self.gaussian
# ------ on demand path -------------------------------------------------- # ------ on demand path --------------------------------------------------
if not self.basetime: if not self.basetime:
MR = MarsRetrieval(self.server, self._start_retrievement(request, retr_param_dict)
marsclass=self.marsclass,
stream=mfstream,
type=mftype,
levtype=pv[1],
levelist=pv[2],
resol=self.resol,
gaussian=gaussian,
accuracy=self.accuracy,
grid=pv[3],
target=mftarget,
area=area,
date=mfdate,
time=mftime,
number=self.number,
step=mfstep,
expver=self.expver,
param=pv[0])
if request == 0:
MR.display_info()
MR.data_retrieve()
elif request == 1:
MR.print_info(self.inputdir)
elif request == 2:
MR.print_info(self.inputdir)
MR.display_info()
MR.data_retrieve()
else:
print('Failure')
# ------ operational path ------------------------------------------------ # ------ operational path ------------------------------------------------
else: else:
# check if mars job requests fields beyond basetime. # check if mars job requests fields beyond basetime.
# If yes eliminate those fields since they may not # If yes eliminate those fields since they may not
# be accessible with user's credentials # be accessible with user's credentials
if 'by' in mfstep: if 'by' in retr_param_dict['step']:
sm1 = 2 sm1 = 2
else: else:
sm1 = -1 sm1 = -1
if 'by' in mftime: if 'by' in retr_param_dict['time']:
tm1 = 2 tm1 = 2
else: else:
tm1 = -1 tm1 = -1
maxdate = datetime.strptime(mfdate.split('/')[-1] + maxdate = datetime.strptime(retr_param_dict['date'].split('/')[-1] +
mftime.split('/')[tm1], retr_param_dict['time'].split('/')[tm1],
'%Y%m%d%H') '%Y%m%d%H')
istep = int(mfstep.split('/')[sm1]) maxtime = maxdate + \
maxtime = maxdate + timedelta(hours=istep) timedelta(hours=int(retr_param_dict['step'].split('/')[sm1]))
elimit = datetime.strptime(mfdate.split('/')[-1] + elimit = datetime.strptime(retr_param_dict['date'].split('/')[-1] +
self.basetime, '%Y%m%d%H') self.basetime, '%Y%m%d%H')
if self.basetime == '12': if self.basetime == '12':
...@@ -491,156 +597,61 @@ class EcFlexpart(object): ...@@ -491,156 +597,61 @@ class EcFlexpart(object):
# if 12h <= maxtime-elimit<12h reduce time for last date # if 12h <= maxtime-elimit<12h reduce time for last date
# if maxtime-elimit<12h reduce step for last time # if maxtime-elimit<12h reduce step for last time
# A split of the MARS job into 2 is likely necessary. # A split of the MARS job into 2 is likely necessary.
maxtime = elimit - timedelta(hours=24)
mfdate = '/'.join(['/'.join(mfdate.split('/')[:-1]),
datetime.strftime(maxtime,
'%Y%m%d')])
MR = MarsRetrieval(self.server,
marsclass=self.marsclass,
stream=self.stream,
type=mftype,
levtype=pv[1],
levelist=pv[2],
resol=self.resol,
gaussian=gaussian,
accuracy=self.accuracy,
grid=pv[3],
target=mftarget,
area=area,
date=mfdate,
time=mftime,
number=self.number,
step=mfstep,
expver=self.expver,
param=pv[0])
MR.display_info() startdate = retr_param_dict['date'].split('/')[0]
MR.data_retrieve() enddate = datetime.strftime(elimit - t24h,'%Y%m%d')
retr_param_dict['date'] = '/'.join([startdate, 'to', enddate])
maxtime = elimit - timedelta(hours=12) self._start_retrievement(request, retr_param_dict)
mfdate = datetime.strftime(maxtime, '%Y%m%d')
mftime = '00'
mftarget = self.inputdir + "/" + ftype + pk + \
'.' + mfdate + '.' + str(os.getppid()) +\
'.' + str(os.getpid()) + ".grb"
MR = MarsRetrieval(self.server, retr_param_dict['date'] = \
marsclass=self.marsclass, datetime.strftime(elimit - t12h, '%Y%m%d')
stream=self.stream, retr_param_dict['time'] = '00'
type=mftype, retr_param_dict['target'] = \
levtype=pv[1], self._mk_targetname(ftype, pk,
levelist=pv[2], retr_param_dict['date'])
resol=self.resol,
gaussian=gaussian,
accuracy=self.accuracy,
grid=pv[3],
target=mftarget,
area=area,
date=mfdate,
time=mftime,
number=self.number,
step=mfstep,
expver=self.expver,
param=pv[0])
MR.display_info() self._start_retrievement(request, retr_param_dict)
MR.data_retrieve()
# -------------- non flux data ------------------------ # -------------- non flux data ------------------------
else: else:
MR = MarsRetrieval(self.server, self._start_retrievement(request, retr_param_dict)
marsclass=self.marsclass,
stream=self.stream,
type=mftype,
levtype=pv[1],
levelist=pv[2],
resol=self.resol,
gaussian=gaussian,
accuracy=self.accuracy,
grid=pv[3],
target=mftarget,
area=area,
date=mfdate,
time=mftime,
number=self.number,
step=mfstep,
expver=self.expver,
param=pv[0])
MR.display_info() else: # basetime = 0
MR.data_retrieve() retr_param_dict['date'] = \
else: # basetime == 0 ??? #AP datetime.strftime(elimit - t24h, '%Y%m%d')
maxtime = elimit - timedelta(hours=24) timesave = ''.join(retr_param_dict['time'])
mfdate = datetime.strftime(maxtime, '%Y%m%d')
mftimesave = ''.join(mftime)
if '/' in mftime: if '/' in retr_param_dict['time']:
times = mftime.split('/') times = retr_param_dict['time'].split('/')
while ((int(times[0]) + steps = retr_param_dict['step'].split('/')
int(mfstep.split('/')[0]) <= 12) and while (pk != 'OG_OROLSM__SL' and
(pk != 'OG_OROLSM__SL') and 'acc' not in pk): 'acc' not in pk and
(int(times[0]) + int(steps[0])) <= 12):
times = times[1:] times = times[1:]
if len(times) > 1: if len(times) > 1:
mftime = '/'.join(times) retr_param_dict['time'] = '/'.join(times)
else: else:
mftime = times[0] retr_param_dict['time'] = times[0]
MR = MarsRetrieval(self.server, self._start_retrievement(request, retr_param_dict)
marsclass=self.marsclass,
stream=self.stream,
type=mftype,
levtype=pv[1],
levelist=pv[2],
resol=self.resol,
gaussian=gaussian,
accuracy=self.accuracy,
grid=pv[3],
target=mftarget,
area=area,
date=mfdate,
time=mftime,
number=self.number,
step=mfstep,
expver=self.expver,
param=pv[0])
MR.display_info() if (pk != 'OG_OROLSM__SL' and
MR.data_retrieve() int(retr_param_dict['step'].split('/')[0]) == 0 and
int(timesave.split('/')[0]) == 0):
if (int(mftimesave.split('/')[0]) == 0 and
int(mfstep.split('/')[0]) == 0 and
pk != 'OG_OROLSM__SL'):
mfdate = datetime.strftime(elimit, '%Y%m%d') retr_param_dict['date'] = \
mftime = '00' datetime.strftime(elimit, '%Y%m%d')
mfstep = '000' retr_param_dict['time'] = '00'
mftarget = self.inputdir + "/" + ftype + pk + \ retr_param_dict['step'] = '000'
'.' + mfdate + '.' + str(os.getppid()) +\ retr_param_dict['target'] = \
'.' + str(os.getpid()) + ".grb" self._mk_targetname(ftype, pk,
retr_param_dict['date'])
MR = MarsRetrieval(self.server, self._start_retrievement(request, retr_param_dict)
marsclass=self.marsclass,
stream=self.stream,
type=mftype,
levtype=pv[1],
levelist=pv[2],
resol=self.resol,
gaussian=gaussian,
accuracy=self.accuracy,
grid=pv[3],
target=mftarget,
area=area,
date=mfdate,
time=mftime,
number=self.number,
step=mfstep,
expver=self.expver,
param=pv[0])
MR.display_info()
MR.data_retrieve()
if request == 0 or request == 2: if request == 0 or request == 2:
print('MARS retrieve done ... ') print('MARS retrieve done ... ')
...@@ -903,7 +914,7 @@ class EcFlexpart(object): ...@@ -903,7 +914,7 @@ class EcFlexpart(object):
timestamp += timedelta(hours=int(step)) timestamp += timedelta(hours=int(step))
cdateH = datetime.strftime(timestamp, '%Y%m%d%H') cdateH = datetime.strftime(timestamp, '%Y%m%d%H')
if c.basetime is not None: if c.basetime:
slimit = datetime.strptime(c.start_date + '00', '%Y%m%d%H') slimit = datetime.strptime(c.start_date + '00', '%Y%m%d%H')
bt = '23' bt = '23'
if c.basetime == '00': if c.basetime == '00':
...@@ -1241,7 +1252,7 @@ class EcFlexpart(object): ...@@ -1241,7 +1252,7 @@ class EcFlexpart(object):
fdate.month*100+fdate.day) fdate.month*100+fdate.day)
grib_write(gid, f_handle) grib_write(gid, f_handle)
if c.basetime is not None: if c.basetime:
elimit = datetime.strptime(c.end_date + elimit = datetime.strptime(c.end_date +
c.basetime, '%Y%m%d%H') c.basetime, '%Y%m%d%H')
else: else:
...@@ -1294,3 +1305,6 @@ class EcFlexpart(object): ...@@ -1294,3 +1305,6 @@ class EcFlexpart(object):
grib_index_release(iid) grib_index_release(iid)
return return
...@@ -334,7 +334,7 @@ class MarsRetrieval(object): ...@@ -334,7 +334,7 @@ class MarsRetrieval(object):
return return
def print_info(self, inputdir): def print_info(self, inputdir, request_number):
''' '''
@Description: @Description:
Prints all mars requests to an extra file for debugging and Prints all mars requests to an extra file for debugging and
...@@ -347,6 +347,9 @@ class MarsRetrieval(object): ...@@ -347,6 +347,9 @@ class MarsRetrieval(object):
inputdir: string inputdir: string
The path where all data from the retrievals are stored. The path where all data from the retrievals are stored.
request_number: integer
Number of mars requests for flux and non-flux data.
@Return: @Return:
<nothing> <nothing>
''' '''
...@@ -356,7 +359,7 @@ class MarsRetrieval(object): ...@@ -356,7 +359,7 @@ class MarsRetrieval(object):
# open a file to store all requests to # open a file to store all requests to
with open(os.path.join(inputdir, with open(os.path.join(inputdir,
_config.FILE_MARS_REQUESTS), 'a') as f: _config.FILE_MARS_REQUESTS), 'a') as f:
f.write('mars\n') f.write('mars_request #' + str(request_number) + '\n')
# iterate through all attributes and print them # iterate through all attributes and print them
# with their corresponding values # with their corresponding values
for item in attrs.items(): for item in attrs.items():
...@@ -368,6 +371,40 @@ class MarsRetrieval(object): ...@@ -368,6 +371,40 @@ class MarsRetrieval(object):
return return
def print_infodata_csv(self, inputdir, request_number):
'''
@Description:
Write all request parameter in alpabetical order into a "csv" file.
@Input:
self: instance of MarsRetrieval
For description see class documentation.
inputdir: string
The path where all data from the retrievals are stored.
request_number: integer
Number of mars requests for flux and non-flux data.
@Return:
<nothing>
'''
# Get all class attributes and their values as a dictionary
attrs = vars(self)
del attrs['server']
# open a file to store all requests to
with open(os.path.join(inputdir,
_config.FILE_MARS_REQUESTS), 'a') as f:
f.write(str(request_number) + ', ')
f.write(', '.join(str(attrs[key])
for key in sorted(attrs.iterkeys())))
f.write('\n')
return
def data_retrieve(self): def data_retrieve(self):
''' '''
@Description: @Description:
......
...@@ -66,7 +66,8 @@ from classes.UioFiles import UioFiles ...@@ -66,7 +66,8 @@ from classes.UioFiles import UioFiles
def main(): def main():
''' '''
@Description: @Description:
If get_mars_data is called from command line, this function controls If get_mars_data is called directly from command line,
the program flow and calls the argumentparser function and the program flow and calls the argumentparser function and
the get_mars_data function for retrieving EC data. the get_mars_data function for retrieving EC data.
...@@ -81,7 +82,7 @@ def main(): ...@@ -81,7 +82,7 @@ def main():
c = ControlFile(args.controlfile) c = ControlFile(args.controlfile)
env_parameter = read_ecenv(_config.PATH_ECMWF_ENV) env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
c.assign_args_to_control(args, env_parameter) c.assign_args_to_control(args)
c.assign_envs_to_control(env_parameter) c.assign_envs_to_control(env_parameter)
c.check_conditions(args.queue) c.check_conditions(args.queue)
......
...@@ -91,9 +91,10 @@ def main(): ...@@ -91,9 +91,10 @@ def main():
c = ControlFile(args.controlfile) c = ControlFile(args.controlfile)
env_parameter = read_ecenv(_config.PATH_ECMWF_ENV) env_parameter = read_ecenv(_config.PATH_ECMWF_ENV)
c.assign_args_to_control(args, env_parameter) c.assign_args_to_control(args)
c.assign_envs_to_control(env_parameter) c.assign_envs_to_control(env_parameter)
c.check_conditions(args.queue) c.check_conditions(args.queue)
prepare_flexpart(args.ppid, c) prepare_flexpart(args.ppid, c)
return return
......
...@@ -86,7 +86,7 @@ def main(): ...@@ -86,7 +86,7 @@ def main():
# on local side # on local side
# on ECMWF server this would also be the local side # on ECMWF server this would also be the local side
called_from_dir = os.getcwd() called_from_dir = os.getcwd()
if not args.queue: if args.queue is None:
if c.inputdir[0] != '/': if c.inputdir[0] != '/':
c.inputdir = os.path.join(called_from_dir, c.inputdir) c.inputdir = os.path.join(called_from_dir, c.inputdir)
if c.outputdir[0] != '/': if c.outputdir[0] != '/':
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment