Skip to content
Snippets Groups Projects
Commit 5e7c588b authored by Anne Philipp's avatar Anne Philipp
Browse files

removed static test install tardir

parent 3946de57
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 0 additions and 5310 deletions
test/InstallTar/flex_extract_v7.1_ecgate/source/python/classes/GribTools.py deleted 100644 → 0
+ 0
− 318
View file @ 3946de57
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
#
# @Date: July 2014
#
# @Change History:
# February 2018 - Anne Philipp (University of Vienna):
# - applied PEP8 style guide
# - added documentation
# - changed some naming
#
# @License:
# (C) Copyright 2014-2018.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# @Class Description:
# The GRIB API provides all necessary tools to work directly with the
# grib files. Nevertheless, the GRIB API tools are very basic and are in
# direct connection with the grib files. This class provides some higher
# functions which apply a set of GRIB API tools together in the respective
# context. So, the class initially contains a list of grib files (their
# names) and the using program then applies the methods directly on the
# class objects without having to think about how the actual GRIB API
# tools have to be arranged.
#
# @Class Content:
# - __init__
# - get_keys
# - set_keys
# - copy
# - index
#
# @Class Attributes:
# - filenames
#
#*******************************************************************************
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import os
from gribapi import grib_new_from_file, grib_is_defined, grib_get, \
grib_release, grib_set, grib_write, grib_index_read, \
grib_index_new_from_file, grib_index_add_file, \
grib_index_write
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
class GribTools(object):
'''
Class for GRIB utilities (new methods) based on GRIB API
'''
# --------------------------------------------------------------------------
# CLASS FUNCTIONS
# --------------------------------------------------------------------------
def __init__(self, filenames):
'''
@Description:
Initialise an object of GribTools and assign a list
of filenames.
@Input:
filenames: list of strings
A list of filenames.
@Return:
<nothing>
'''
self.filenames = filenames
return
def get_keys(self, keynames, wherekeynames=[], wherekeyvalues=[]):
'''
@Description:
get keyvalues for a given list of keynames
a where statement can be given (list of key and list of values)
@Input:
keynames: list of strings
List of keynames.
wherekeynames: list of strings, optional
Default value is an empty list.
wherekeyvalues: list of strings, optional
Default value is an empty list.
@Return:
return_list: list of strings
List of keyvalues for given keynames.
'''
fileid = open(self.filenames, 'r')
return_list = []
while 1:
gid_in = grib_new_from_file(fileid)
if gid_in is None:
break
if len(wherekeynames) != len(wherekeyvalues):
raise Exception("Number of key values and key names must be \
the same. Give a value for each keyname!")
select = True
i = 0
for wherekey in wherekeynames:
if not grib_is_defined(gid_in, wherekey):
raise Exception("where key was not defined")
select = (select and (str(wherekeyvalues[i]) ==
str(grib_get(gid_in, wherekey))))
i += 1
if select:
llist = []
for key in keynames:
llist.extend([str(grib_get(gid_in, key))])
return_list.append(llist)
grib_release(gid_in)
fileid.close()
return return_list
def set_keys(self, fromfile, keynames, keyvalues, wherekeynames=[],
wherekeyvalues=[], strict=False, filemode='w'):
'''
@Description:
Opens the file to read the grib messages and then write
them to a new output file. By default all messages are
written out. Also, the keyvalues of the passed list of
keynames are set or only those meeting the where statement.
(list of key and list of values).
@Input:
fromfile: string
Filename of the input file to read the grib messages from.
keynames: list of strings
List of keynames. Default is an empty list.
keyvalues: list of strings
List of keynames. Default is an empty list.
wherekeynames: list of strings, optional
Default value is an empty list.
wherekeyvalues: list of strings, optional
Default value is an empty list.
strict: boolean, optional
Decides if everything from keynames and keyvalues
is written out the grib file (False) or only those
meeting the where statement (True). Default is False.
filemode: string, optional
Sets the mode for the output file. Default is "w".
@Return:
<nothing>
'''
fout = open(self.filenames, filemode)
fin = open(fromfile)
while 1:
gid_in = grib_new_from_file(fin)
if gid_in is None:
break
if len(wherekeynames) != len(wherekeyvalues):
raise Exception("Give a value for each keyname!")
select = True
i = 0
for wherekey in wherekeynames:
if not grib_is_defined(gid_in, wherekey):
raise Exception("where Key was not defined")
select = (select and (str(wherekeyvalues[i]) ==
str(grib_get(gid_in, wherekey))))
i += 1
if select:
i = 0
for key in keynames:
grib_set(gid_in, key, keyvalues[i])
i += 1
grib_write(gid_in, fout)
grib_release(gid_in)
fin.close()
fout.close()
return
def copy(self, filename_in, selectWhere=True,
keynames=[], keyvalues=[], filemode='w'):
'''
Add the content of another input grib file to the objects file but
only messages corresponding to keys/values passed to the function.
The selectWhere switch decides if to copy the keys equal to (True) or
different to (False) the keynames/keyvalues list passed to the function.
@Input:
filename_in: string
Filename of the input file to read the grib messages from.
selectWhere: boolean, optional
Decides if to copy the keynames and values equal to (True) or
different to (False) the keynames/keyvalues list passed to the
function. Default is True.
keynames: list of strings, optional
List of keynames. Default is an empty list.
keyvalues: list of strings, optional
List of keynames. Default is an empty list.
filemode: string, optional
Sets the mode for the output file. Default is "w".
@Return:
<nothing>
'''
fin = open(filename_in)
fout = open(self.filenames, filemode)
while 1:
gid_in = grib_new_from_file(fin)
if gid_in is None:
break
if len(keynames) != len(keyvalues):
raise Exception("Give a value for each keyname!")
select = True
i = 0
for key in keynames:
if not grib_is_defined(gid_in, key):
raise Exception("Key was not defined")
if selectWhere:
select = (select and (str(keyvalues[i]) ==
str(grib_get(gid_in, key))))
else:
select = (select and (str(keyvalues[i]) !=
str(grib_get(gid_in, key))))
i += 1
if select:
grib_write(gid_in, fout)
grib_release(gid_in)
fin.close()
fout.close()
return
def index(self, index_keys=["mars"], index_file="my.idx"):
'''
@Description:
Create index file from a list of files if it does not exist or
read an index file.
@Input:
index_keys: list of strings, optional
Contains the list of key parameter names from
which the index is to be created.
Default is a list with a single entry string "mars".
index_file: string, optional
Filename where the indices are stored.
Default is "my.idx".
@Return:
iid: integer
Grib index id.
'''
print("... index will be done")
iid = None
if os.path.exists(index_file):
iid = grib_index_read(index_file)
print("Use existing index file: %s " % (index_file))
else:
for filename in self.filenames:
print("Inputfile: %s " % (filename))
if iid is None:
iid = grib_index_new_from_file(filename, index_keys)
else:
grib_index_add_file(iid, filename)
if iid is not None:
grib_index_write(iid, index_file)
print('... index done')
return iid
test/InstallTar/flex_extract_v7.1_ecgate/source/python/classes/MarsRetrieval.py deleted 100644 → 0
+ 0
− 433
View file @ 3946de57
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Fouilloux (University of Oslo)
#
# @Date: October 2014
#
# @Change History:
#
# November 2015 - Leopold Haimberger (University of Vienna):
# - optimized display_info
# - optimized data_retrieve and seperate between python and shell
# script call
#
# February 2018 - Anne Philipp (University of Vienna):
# - applied PEP8 style guide
# - added documentation
# - applied some minor modifications in programming style/structure
#
# @License:
# (C) Copyright 2015-2018.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# @Class Description:
# A MARS revtrieval has a specific syntax with a selection of keywords and
# their corresponding values. This class provides the necessary functions
# by displaying the selected parameters and their values and the actual
# retrievement of the data through a mars request or a Python web api
# interface. The initialization already expects all the keyword values.
#
# @Class Content:
# - __init__
# - display_info
# - data_retrieve
#
# @Class Attributes:
# - server
# - marsclass
# - dtype
# - levtype
# - levelist
# - repres
# - date
# - resol
# - stream
# - area
# - time
# - step
# - expver
# - number
# - accuracy
# - grid
# - gaussian
# - target
# - param
#
#*******************************************************************************
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import subprocess
import os
import _config
# ------------------------------------------------------------------------------
# CLASS
# ------------------------------------------------------------------------------
class MarsRetrieval(object):
'''
Class for submitting MARS retrievals.
A description of MARS keywords/arguments and examples of their
values can be found here:
https://software.ecmwf.int/wiki/display/UDOC/\
Identification+keywords#Identificationkeywords-class
'''
def __init__(self, server, marsclass="ei", type="", levtype="",
levelist="", repres="", date="", resol="", stream="",
area="", time="", step="", expver="1", number="",
accuracy="", grid="", gaussian="", target="",
param=""):
'''
@Description:
Initialises the instance of the MarsRetrieval class and
defines and assigns a set of the necessary retrieval parameters
for the FLEXPART input data.
A description of MARS keywords/arguments, their dependencies
on each other and examples of their values can be found here:
https://software.ecmwf.int/wiki/display/UDOC/MARS+keywords
@Input:
self: instance of MarsRetrieval
For description see class documentation.
server: instance of ECMWFService (from ECMWF Web-API)
This is the connection to the ECMWF data servers.
It is needed for the pythonic access of ECMWF data.
marsclass: string, optional
Characterisation of dataset. E.g. EI (ERA-Interim),
E4 (ERA40), OD (Operational archive), ea (ERA5).
Default is the ERA-Interim dataset "ei".
type: string, optional
Determines the type of fields to be retrieved.
Selects between observations, images or fields.
Examples for fields: Analysis (an), Forecast (fc),
Perturbed Forecast (pf), Control Forecast (cf) and so on.
Default is an empty string.
levtype: string, optional
Denotes type of level. Has a direct implication on valid
levelist values!
E.g. model level (ml), pressure level (pl), surface (sfc),
potential vorticity (pv), potential temperature (pt)
and depth (dp).
Default is an empty string.
levelist: string, optional
Specifies the required levels. It has to have a valid
correspondence to the selected levtype.
Examples: model level: 1/to/137, pressure levels: 500/to/1000
Default is an empty string.
repres: string, optional
Selects the representation of the archived data.
E.g. sh - spherical harmonics, gg - Gaussian grid,
ll - latitude/longitude, ...
Default is an empty string.
date: string, optional
Specifies the Analysis date, the Forecast base date or
Observations date. Valid formats are:
Absolute as YYYY-MM-DD or YYYYMMDD.
Default is an empty string.
resol: string, optional
Specifies the desired triangular truncation of retrieved data,
before carrying out any other selected post-processing.
The default is automatic truncation (auto), by which the lowest
resolution compatible with the value specified in grid is
automatically selected for the retrieval.
Users wanting to perform post-processing from full spectral
resolution should specify Archived Value (av).
The following are examples of existing resolutions found in
the archive: 63, 106, 159, 213, 255, 319, 399, 511, 799 or 1279.
This keyword has no meaning/effect if the archived data is
not in spherical harmonics representation.
The best selection can be found here:
https://software.ecmwf.int/wiki/display/UDOC/\
Retrieve#Retrieve-Truncationbeforeinterpolation
Default is an empty string.
stream: string, optional
Identifies the forecasting system used to generate the data.
E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
Default is an empty string.
area: string, optional
Specifies the desired sub-area of data to be extracted.
Areas can be defined to wrap around the globe.
Latitude values must be given as signed numbers, with:
north latitudes (i.e. north of the equator)
being positive (e.g: 40.5)
south latitutes (i.e. south of the equator)
being negative (e.g: -50.5)
Longtitude values must be given as signed numbers, with:
east longitudes (i.e. east of the 0 degree meridian)
being positive (e.g: 35.0)
west longitudes (i.e. west of the 0 degree meridian)
being negative (e.g: -20.5)
E.g.: North/West/South/East
Default is an empty string.
time: string, optional
Specifies the time of the data in hours and minutes.
Valid values depend on the type of data: Analysis time,
Forecast base time or First guess verification time
(all usually at synoptic hours: 00, 06, 12 and 18 ).
Observation time (any combination in hours and minutes is valid,
subject to data availability in the archive).
The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
Default is an empty string.
step: string, optional
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time. It also
specifies the length of the forecast which verifies at
First Guess time.
E.g. 1/3/6-hourly
Default is an empty string.
expver: string, optional
The version of the dataset. Each experiment is assigned a
unique code (version). Production data is assigned 1 or 2,
and experimental data in Operations 11, 12 ,...
Research or Member State's experiments have a four letter
experiment identifier.
Default is "1".
number: string, optional
Selects the member in ensemble forecast run. (Only then it
is necessary.) It has a different meaning depending on
the type of data.
E.g. Perturbed Forecasts: specifies the Ensemble forecast member
Default is an empty string.
accuracy: string, optional
Specifies the number of bits per value to be used in the
generated GRIB coded fields.
A positive integer may be given to specify the preferred number
of bits per packed value. This must not be greater than the
number of bits normally used for a Fortran integer on the
processor handling the request (typically 32 or 64 bit).
Within a compute request the accuracy of the original fields
can be passed to the result field by specifying accuracy=av.
Default is an empty string.
grid: string, optional
Specifies the output grid which can be either a Gaussian grid
or a Latitude/Longitude grid. MARS requests specifying
grid=av will return the archived model grid.
Lat/Lon grid: The grid spacing needs to be an integer
fraction of 90 degrees e.g. grid = 0.5/0.5
Gaussian grid: specified by a letter denoting the type of
Gaussian grid followed by an integer (the grid number)
representing the number of lines between the Pole and Equator,
e.g.
grid = F160 - full (or regular) Gaussian grid with
160 latitude lines between the pole and equator
grid = N320 - ECMWF original reduced Gaussian grid with
320 latitude lines between the pole and equator,
see Reduced Gaussian Grids for grid numbers used at ECMWF
grid = O640 - ECMWF octahedral (reduced) Gaussian grid with
640 latitude lines between the pole and equator
Default is an empty string.
gaussian: string, optional
This parameter is deprecated and should no longer be used.
Specifies the desired type of Gaussian grid for the output.
Valid Gaussian grids are quasi-regular (reduced) or regular.
Keyword gaussian can only be specified together with
keyword grid. Gaussian without grid has no effect.
Default is an empty string.
target: string, optional
Specifies a file into which data is to be written after
retrieval or manipulation. Path names should always be
enclosed in double quotes. The MARS client supports automatic
generation of multiple target files using MARS keywords
enclosed in square brackets [ ]. If the environment variable
MARS_MULTITARGET_STRICT_FORMAT is set to 1 before calling mars,
the keyword values will be used in the filename as shown by
the ecCodes GRIB tool grib_ls -m, e.g. with
MARS_MULTITARGET_STRICT_FORMAT set to 1 the keywords time,
expver and param will be formatted as 0600, 0001 and 129.128
rather than 600, 1 and 129.
Default is an empty string.
param: string, optional
Specifies the meteorological parameter.
The list of meteorological parameters in MARS is extensive.
Their availability is directly related to their meteorological
meaning and, therefore, the rest of directives specified
in the MARS request.
Meteorological parameters can be specified by their
GRIB code (param=130), their mnemonic (param=t) or
full name (param=temperature).
The list of parameter should be seperated by a "/"-sign.
E.g. 130/131/133
Default is an empty string.
@Return:
<nothing>
'''
self.server = server
self.marsclass = marsclass
self.type = type
self.levtype = levtype
self.levelist = levelist
self.repres = repres
self.date = date
self.resol = resol
self.stream = stream
self.area = area
self.time = time
self.step = step
self.expver = expver
self.number = number
self.accuracy = accuracy
self.grid = grid
self.gaussian = gaussian
self.target = target
self.param = param
return
def display_info(self):
'''
@Description:
Prints all class attributes and their values to the
standard output.
@Input:
self: instance of MarsRetrieval
For description see class documentation.
@Return:
<nothing>
'''
# Get all class attributes and their values as a dictionary
attrs = vars(self)
# iterate through all attributes and print them
# with their corresponding values
for item in attrs.items():
if item[0] in 'server':
pass
else:
print(item[0] + ': ' + str(item[1]))
return
def print_info(self, inputdir):
'''
@Description:
Prints all mars requests to an extra file for debugging and
information.
@Input:
self: instance of MarsRetrieval
For description see class documentation.
inputdir: string
The path where all data from the retrievals are stored.
@Return:
<nothing>
'''
# Get all class attributes and their values as a dictionary
attrs = vars(self)
# open a file to store all requests to
with open(os.path.join(inputdir,
_config.FILE_MARS_REQUESTS), 'a') as f:
f.write('mars\n')
# iterate through all attributes and print them
# with their corresponding values
for item in attrs.items():
if item[0] in 'server':
pass
else:
f.write(item[0] + ': ' + str(item[1]) + '\n')
f.write('\n\n')
return
def data_retrieve(self):
'''
@Description:
Submits a MARS retrieval. Depending on the existence of
ECMWF Web-API it is submitted via Python or a
subprocess in the Shell. The parameter for the mars retrieval
are taken from the defined class attributes.
@Input:
self: instance of MarsRetrieval
For description see class documentation.
@Return:
<nothing>
'''
# Get all class attributes and their values as a dictionary
attrs = vars(self)
# convert the dictionary of attributes into a comma
# seperated list of attributes with their values
# needed for the retrieval call
s = 'ret'
for k, v in attrs.iteritems():
if k in 'server':
continue
if k == 'marsclass':
k = 'class'
if v == '':
continue
if k.lower() == 'target':
target = v
else:
s = s + ',' + k + '=' + str(v)
# MARS request via Python script
if self.server is not False:
try:
self.server.execute(s, target)
except:
print('MARS Request failed, \
have you already registered at apps.ecmwf.int?')
raise IOError
if os.stat(target).st_size == 0:
print('MARS Request returned no data - please check request')
raise IOError
# MARS request via extra process in shell
else:
s += ',target = "' + target + '"'
p = subprocess.Popen(['mars'], stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
pout = p.communicate(input=s)[0]
print(pout.decode())
if 'Some errors reported' in pout.decode():
print('MARS Request failed - please check request')
raise IOError
if os.stat(target).st_size == 0:
print('MARS Request returned no data - please check request')
raise IOError
return
test/InstallTar/flex_extract_v7.1_ecgate/source/python/classes/__init__.py deleted 100644 → 0
+ 0
− 14
View file @ 3946de57
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
#
# @Date: September 2018
#
# @License:
# (C) Copyright 2015-2018.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
#*******************************************************************************
test/InstallTar/flex_extract_v7.1_ecgate/source/python/install.py deleted 100755 → 0
+ 0
− 547
View file @ 3946de57
This diff is collapsed.
test/InstallTar/flex_extract_v7.1_ecgate/source/python/submit.py deleted 100755 → 0
+ 0
− 201
View file @ 3946de57
This diff is collapsed.
test/InstallTar/flex_extract_v7.1_ecgate/templates/compilejob.temp deleted 100644 → 0
+ 0
− 77
View file @ 3946de57
This diff is collapsed.
test/InstallTar/flex_extract_v7.1_ecgate/templates/job.temp deleted 100644 → 0
+ 0
− 76
View file @ 3946de57
This diff is collapsed.
test/InstallTar/flex_extract_v7.1_ecgate/templates/job.temp.o deleted 100644 → 0
+ 0
− 75
View file @ 3946de57
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment