From 6f951ca0346aa6a2c3e11ee62df85214a290d838 Mon Sep 17 00:00:00 2001
From: Anne Philipp <anne.philipp@univie.ac.at>
Date: Tue, 15 Jan 2019 02:05:10 +0100
Subject: [PATCH] new style of docstring params and updates in docstrings
---
source/python/_config.py | 42 ++--
source/python/classes/ControlFile.py | 313 ++++++++++++++++++++++---
source/python/classes/EcFlexpart.py | 222 ++++++++++++------
source/python/classes/MarsRetrieval.py | 171 +++++++++-----
source/python/classes/UioFiles.py | 64 ++---
source/python/install.py | 109 +++++----
source/python/mods/checks.py | 178 +++++++-------
source/python/mods/disaggregation.py | 54 +++--
source/python/mods/get_mars_data.py | 83 ++++---
source/python/mods/prepare_flexpart.py | 50 ++--
source/python/mods/tools.py | 143 +++++------
source/python/submit.py | 63 +++--
12 files changed, 967 insertions(+), 525 deletions(-)
diff --git a/source/python/_config.py b/source/python/_config.py
index 908ee3c..e35a73d 100644
--- a/source/python/_config.py
+++ b/source/python/_config.py
@@ -1,25 +1,26 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+#*******************************************************************************
+# @Author: Anne Philipp (University of Vienna)
+#
+# @Date: August 2018
+#
+# @Change History:
+#
+# @License:
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
+#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
+#*******************************************************************************
+'''Configurations for flex_extract.
+
+Defines a couple of global constant parameters as well as filenames
+and pathes.
'''
-*******************************************************************************
- @Author: Anne Philipp (University of Vienna)
-
- @Date: August 2018
-
- @Change History:
-
- @License:
- (C) Copyright 2014-2018.
-
- This software is licensed under the terms of the Apache Licence Version 2.0
- which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-
- @Description:
- Contains constant value parameter for flex_extract.
-
-*******************************************************************************
-'''
-
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
@@ -94,8 +95,7 @@ else:
PATH_CONTROLFILES = os.path.join(PATH_RUN_DIR, 'control')
#
# ------------------------------------------------------------------------------
-# for making the installation tar ball, the relative pathes to the
-# flex_extract root directory are needed
+# RELATIVE PATHES FOR INSTALLATION TAR-BALL
# ------------------------------------------------------------------------------
PATH_REL_PYTHON_SRC = os.path.relpath(PATH_LOCAL_PYTHON, PATH_FLEXEXTRACT_DIR)
diff --git a/source/python/classes/ControlFile.py b/source/python/classes/ControlFile.py
index 00c316a..c3a55fa 100644
--- a/source/python/classes/ControlFile.py
+++ b/source/python/classes/ControlFile.py
@@ -20,32 +20,23 @@
# - outsourced the commandline argument assignments to control attributes
#
# @License:
-# (C) Copyright 2015-2018.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Class Description:
-# The CONTROL file is the steering part of the FLEXPART extraction
-# software. All necessary parameters needed to retrieve the data fields
-# from the MARS archive for driving FLEXPART are set in a CONTROL file.
-# Some specific parameters like the start and end dates can be overwritten
-# by the command line parameters, but in generel all parameters needed
-# for a complete set of fields for FLEXPART can be set in the CONTROL file.
-#
-# @Class Content:
-# - __init__
-# - __read_controlfile__
-# - __str__
-# - assign_args_to_control
-# - assign_envs_to_control
-# - check_conditions
-# - check_install_conditions
-# - to_list
-#
-# @Class Attributes:
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#
+# @Class Methods:
+# __init__
+# _read_controlfile
+# __str__
+# assign_args_to_control
+# assign_envs_to_control
+# check_conditions
+# check_install_conditions
+# to_list
#*******************************************************************************
# ------------------------------------------------------------------------------
@@ -75,6 +66,269 @@ from mods.checks import (check_grid, check_area, check_levels, check_purefc,
class ControlFile(object):
'''
Contains the information which are stored in the CONTROL files.
+
+ The CONTROL file is the steering part of the FLEXPART extraction
+ software. All necessary parameters needed to retrieve the data fields
+ from the MARS archive for driving FLEXPART are set in a CONTROL file.
+ Some specific parameters like the start and end dates can be overwritten
+ by the command line parameters, but in generel all parameters needed
+ for a complete set of fields for FLEXPART can be set in the CONTROL file.
+
+ Attributes
+ ----------
+ controlfile : str
+ The name of the control file to be processed. Default value is the
+ filename passed to the init function when initialised.
+
+ start_date : str
+ The first day of the retrieval period. Default value is None.
+
+ end_date :str
+ The last day of the retrieval period. Default value is None.
+
+ date_chunk : int
+ Length of period for a single mars retrieval. Default value is 3.
+
+ dtime :str
+ The time step in hours. Default value is None.
+
+ basetime : str
+ The time for a half day retrieval. The 12 hours upfront are to be
+ retrieved. Default value is None.
+
+ maxstep : int
+ The maximum forecast step for non flux data. Default value is None.
+
+ type : list of str
+ List of field type per retrieving hour. Default value is None.
+
+ time : list of str
+ List of retrieving times in hours. Default valuer is None.
+
+ step : list of str or str
+ List of forecast time steps in hours for non flux data.
+ Default value is None.
+
+ acctype : str
+ The field type for the accumulated forecast fields.
+ Default value is None.
+
+ acctime : str
+ The starting time of the accumulated forecasts. Default value is None.
+
+ accmaxstep : int
+ The maximum forecast step for the accumulated forecast fields
+ (flux data). Default value is None.
+
+ marsclass : str
+ Characterisation of dataset. Default value is None.
+
+ dataset : str
+ For public datasets there is the specific naming and parameter
+ dataset which has to be used to characterize the type of
+ data. Default value is None.
+
+ stream : str
+ Identifies the forecasting system used to generate the data.
+ Default value is None.
+
+ number : str
+ Selects the member in ensemble forecast run. Default value is 'OFF'.
+
+ expver : str
+ The version number of the dataset. Default value is '1'.
+
+ gaussian : str
+ This parameter is deprecated and should no longer be used.
+ Specifies the desired type of Gaussian grid for the output.
+ Default value is an empty string ''.
+
+ grid : str
+ Specifies the output grid which can be either a Gaussian grid
+ or a Latitude/Longitude grid. Default value is None.
+
+ area : str
+ Specifies the desired sub-area of data to be extracted.
+ Default value is None.
+
+ left : str
+ The western most longitude of the area to be extracted.
+ Default value is None.
+
+ lower : str
+ The southern most latitude of the area to be extracted.
+ Default value is None.
+
+ upper : str
+ The northern most latitued of the area to be extracted.
+ Default value is None.
+
+ right : str
+ The eastern most longitude of the area to be extracted.
+ Default value is None.
+
+ level : str
+ Specifies the maximum level. Default value is None.
+
+ levelist : str
+ Specifies the required level list. Default value is None.
+
+ resol : str
+ Specifies the desired triangular truncation of retrieved data,
+ before carrying out any other selected post-processing.
+ Default value is None.
+
+ gauss : int
+ Switch to select gaussian fields (1) or regular lat/lon (0).
+ Default value is 0.
+
+ accuracy : int
+ Specifies the number of bits per value to be used in the
+ generated GRIB coded fields. Default value is 24.
+
+ omega : int
+ Switch to select omega retrieval (1) or not (0). Default value is 0.
+
+ omegadiff : int
+ Switch to decide to calculate Omega and Dps/Dt from continuity
+ equation for diagnostic purposes (1) or not (0). Default value is 0.
+
+ eta : int
+ Switch to select direct retrieval of etadot from MARS (1) or
+ wether it has to be calculated (0). Then Default value is 0.
+
+ etadiff : int
+ Switch to select calculation of etadot and Dps/Dt from continuity
+ equation for diagnostic purposes (1) or not (0). Default value is 0.
+
+ etapar : int
+ GRIB parameter Id for etadot fields. Default value is 77.
+
+ dpdeta : int
+ Switch to select multiplication of etadot with dpdeta.
+ Default value is 1.
+
+ smooth : int
+ Spectral truncation of ETADOT after calculation on Gaussian grid.
+ Default value is 0.
+
+ format : str
+ The format of the GRIB data. Default value is 'GRIB1'.
+
+ addpar : str
+ List of additional surface level ECMWF parameter to be retrieved.
+ Default value is None.
+
+ prefix : str
+ Prefix string for the final FLEXPART/FLEXTRA ready input files.
+ Default value is 'EN'.
+
+ cwc : int
+ Switch to select wether the sum of cloud liquid water content and
+ cloud ice water content should be retrieved. Default value is 0.
+
+ wrf : int
+ Switch to select further parameters for retrievment to support
+ WRF simulations. Default value is 0.
+
+ ecfsdir : str
+ Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
+
+ mailfail : list of str
+ Email list for sending error log files from ECMWF servers.
+ The email addresses should be seperated by a comma.
+ Default value is ['${USER}'].
+
+ mailops : list of str
+ Email list for sending operational log files from ECMWF servers.
+ The email addresses should be seperated by a comma.
+ Default value is ['${USER}'].
+
+ grib2flexpart : int 0
+ Switch to select generation of preprocessed FLEXPART files ".fp".
+ If it is selected, the program grib2flexpart will try
+ to convert the flex_extract output files into ".fp" format.
+
+ ecstorage : int
+ Switch to select storage of FLEXPART ready output files
+ in the ECFS file system. Default value is 0.
+
+ ectrans : int
+ Switch to select the transfer of FLEXPART ready output files
+ to the gateway server. Default value is 0.
+
+ inputdir : str
+ Path to the temporary directory for the retrieval grib files and
+ other processing files. Default value is _config.PATH_INPUT_DIR.
+
+ outputdir : str
+ Path to the final directory where the final FLEXPART ready input
+ files are stored. Default value is None.
+
+ flexextractdir : str
+ Path to the flex_extract root directory. Default value is
+ _config.PATH_FLEXEXTRACT_DIR.
+
+ exedir : str
+ Path to the FORTRAN executable file. Default value is
+ _config.PATH_FORTRAN_SRC.
+
+ flexpartdir : str
+ Path to a FLEXPART root directory. Default value is None.
+
+ makefile : str
+ Name of the makefile to be used for the Fortran program.
+ Default value is 'Makefile.gfortran'.
+
+ destination : str
+ The remote destination which is used to transfer files
+ from ECMWF server to local gateway server. Default value is None.
+
+ gateway : str
+ The gateway server the user is using. Default value is None.
+
+ ecuid : str
+ The user id on ECMWF server. Default value is None.
+
+ ecgid : str
+ The group id on ECMWF server. Default value is None.
+
+ install_target : str
+ Defines the location where the installation is to be done.
+ Default value is None.
+
+ debug : int
+ Switch to keep temporary files at the end of postprocessing (1) or
+ to delete all temporary files except the final output files (0).
+ Default value is 0.
+
+ request : int
+ Switch to select between just retrieving the data (0), writing the mars
+ parameter values to a csv file (1) or doing both (2).
+ Default value is 0.
+
+ public : int
+ Switch to select kind of ECMWF Web Api access and the
+ possible data sets. Public data sets (1) and Memberstate data sets (0).
+ Default value is 0.
+
+ ecapi : boolean
+ Tells wether the ECMWF Web APi was able to load or not.
+ Default value is None.
+
+ purefc : int
+ Switch to decide wether the job is a pure forecast retrieval or
+ coupled with analysis data. Default value is 0.
+
+ rrint: int
+ Switch to select between old precipitation disaggregation method (0)
+ or the new IA3 disaggegration method (1). Default value is 0.
+
+ logicals : list of str
+ List of the names of logical switches which controls the flow
+ of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
+ 'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
+ 'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint']
'''
def __init__(self, filename):
@@ -84,7 +338,7 @@ class ControlFile(object):
Parameters
----------
- filename : :obj:`string`
+ filename : str
Name of CONTROL file.
Return
@@ -97,6 +351,7 @@ class ControlFile(object):
self.start_date = None
self.end_date = None
self.date_chunk = 3
+ self.job_chunk = None
self.dtime = None
self.basetime = None
self.maxstep = None
@@ -272,7 +527,7 @@ class ControlFile(object):
Parameters
----------
- args : :obj:`Namespace`
+ args : Namespace
Contains the commandline arguments from script/program call.
Return
@@ -297,7 +552,7 @@ class ControlFile(object):
Parameters
----------
- envs : :obj:`dictionary` of :obj:`strings`
+ envs : dict of str
Contains the ECMWF environment parameternames "ECUID", "ECGID",
"DESTINATION" and "GATEWAY" with its corresponding values.
They were read from the file "ECMWF_ENV".
@@ -319,7 +574,7 @@ class ControlFile(object):
Parameters
----------
- queue : :obj:`string`
+ queue : str
Name of the queue if submitted to the ECMWF servers.
Used to check if ecuid, ecgid, gateway and destination
are set correctly and are not empty.
@@ -396,7 +651,7 @@ class ControlFile(object):
Return
------
- l : :obj:`list`
+ l : list of *
A sorted list of the all ControlFile class attributes with
their values except the attributes "_expanded", "exedir",
"flexextractdir" and "flexpartdir".
diff --git a/source/python/classes/EcFlexpart.py b/source/python/classes/EcFlexpart.py
index ffdcc8e..70f4fd8 100644
--- a/source/python/classes/EcFlexpart.py
+++ b/source/python/classes/EcFlexpart.py
@@ -40,31 +40,13 @@
# duplication, easier testing)
#
# @License:
-# (C) Copyright 2014-2018.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Class Description:
-# FLEXPART needs grib files in a specifc format. All necessary data fields
-# for one time step are stored in a single file. The class represents an
-# instance with all the parameter and settings necessary for retrieving
-# MARS data and modifing them so they are fitting FLEXPART need. The class
-# is able to disaggregate the fluxes and convert grid types to the one needed
-# by FLEXPART, therefore using the FORTRAN program.
-#
-# @Class Content:
-# - __init__
-# - write_namelist
-# - retrieve
-# - process_output
-# - create
-# - deacc_fluxes
-#
-# @Class Attributes:
-#
-# TODO
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#*******************************************************************************
#pylint: disable=unsupported-assignment-operation
# this is disabled because for this specific case its an error in pylint
@@ -101,8 +83,112 @@ import mods.disaggregation as disaggregation
# ------------------------------------------------------------------------------
class EcFlexpart(object):
'''
- Class to retrieve FLEXPART specific ECMWF data.
+ Class to represent FLEXPART specific ECMWF data.
+
+ FLEXPART needs grib files in a specifc format. All necessary data fields
+ for one time step are stored in a single file. The class represents an
+ instance with all the parameter and settings necessary for retrieving
+ MARS data and modifing them so they are fitting FLEXPART needs. The class
+ is able to disaggregate the fluxes and convert grid types to the one needed
+ by FLEXPART, therefore using the FORTRAN program.
+
+ Attributes
+ ----------
+ mreq_count : int
+ Counter for the number of generated mars requests.
+
+ inputdir : str
+ Path to the directory where the retrieved data is stored.
+
+ dataset : str
+ For public datasets there is the specific naming and parameter
+ dataset which has to be used to characterize the type of
+ data.
+
+ basetime : str
+ The time for a half day retrieval. The 12 hours upfront are to be
+ retrieved.
+
+ dtime : str
+ Time step in hours.
+
+ acctype : str
+ The field type for the accumulated forecast fields.
+
+ acctime : str
+ The starting time from the accumulated forecasts.
+
+ accmaxstep : str
+ The maximum forecast step for the accumulated forecast fields.
+
+ marsclass : str
+ Characterisation of dataset.
+
+ stream : str
+ Identifies the forecasting system used to generate the data.
+
+ number : str
+ Selects the member in ensemble forecast run.
+
+ resol : str
+ Specifies the desired triangular truncation of retrieved data,
+ before carrying out any other selected post-processing.
+
+ accuracy : str
+ Specifies the number of bits per value to be used in the
+ generated GRIB coded fields.
+
+ addpar : str
+ List of additional parameters to be retrieved.
+
+ level : str
+ Specifies the maximum level.
+
+ expver : str
+ The version of the dataset.
+
+ levelist : str
+ Specifies the required levels.
+
+ glevelist : str
+ Specifies the required levels for gaussian grids.
+
+ gaussian : str
+ This parameter is deprecated and should no longer be used.
+ Specifies the desired type of Gaussian grid for the output.
+
+ grid : str
+ Specifies the output grid which can be either a Gaussian grid
+ or a Latitude/Longitude grid.
+
+ area : str
+ Specifies the desired sub-area of data to be extracted.
+
+ purefc : int
+ Switch for definition of pure forecast mode or not.
+
+ outputfilelist : list of str
+ The final list of FLEXPART ready input files.
+
+ types : dictionary
+ Determines the combination of type of fields, time and forecast step
+ to be retrieved.
+
+ params : dictionary
+ Collection of grid types and their corresponding parameters,
+ levels, level types and the grid definition.
+
+ server : ECMWFService or ECMWFDataServer
+ This is the connection to the ECMWF data servers.
+
+ public : int
+ Decides which Web API Server version is used.
+
+ dates : str
+ Contains start and end date of the retrieval in the format
+ "YYYYMMDD/to/YYYYMMDD"
'''
+
# --------------------------------------------------------------------------
# CLASS FUNCTIONS
# --------------------------------------------------------------------------
@@ -112,11 +198,11 @@ class EcFlexpart(object):
Parameters:
-----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
- fluxes : :obj:`boolean`, optional
+ fluxes : boolean, optional
Decides if the flux parameter settings are stored or
the rest of the parameter list.
Default value is False.
@@ -173,13 +259,13 @@ class EcFlexpart(object):
Parameters:
-----------
- ftype : :obj:`list` of :obj:`string`
+ ftype : list of str
List of field types.
- ftime : :obj:`list` of :obj:`string`
+ ftime : list of str
The time in hours of the field.
- fstep : :obj:`string`
+ fstep : str
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time.
@@ -266,19 +352,19 @@ class EcFlexpart(object):
Parameters:
-----------
- gauss : :obj:`integer`
+ gauss : int
Gaussian grid is retrieved.
- eta : :obj:`integer`
+ eta : int
Etadot parameter will be directly retrieved.
- omega : :obj:`integer`
+ omega : int
The omega paramterwill be retrieved.
- cwc : :obj:`integer`
+ cwc : int
The cloud liquid and ice water content will be retrieved.
- wrf : :obj:`integer`
+ wrf : int
Additional model level and surface level data will be retrieved for
WRF/FLEXPART-WRF simulations.
@@ -373,19 +459,19 @@ class EcFlexpart(object):
Parameters
----------
- ftype : :obj:`string`
+ ftype : str
Shortcut name of the type of the field. E.g. AN, FC, PF, ...
- param : :obj:`string`
+ param : str
Shortcut of the grid type. E.g. SH__ML, SH__SL, GG__ML,
GG__SL, OG__ML, OG__SL, OG_OROLSM_SL, OG_acc_SL
- date : :obj:`string`
+ date : str
The date period of the grib data to be stored in this file.
Return
------
- targetname : :obj:`string`
+ targetname : str
The target filename for the grib data.
'''
targetname = (self.inputdir + '/' + ftype + param + '.' + date + '.' +
@@ -400,13 +486,13 @@ class EcFlexpart(object):
Parameters
----------
- request : :obj:`integer`
+ request : int
Selects the mode of retrieval.
0: Retrieves the data from ECMWF.
1: Prints the mars requests to an output file.
2: Retrieves the data and prints the mars request.
- par_dict : :obj:`dictionary`
+ par_dict : dictionary
Contains all parameter which have to be set for creating the
Mars Retrievals. The parameter are:
marsclass, dataset, stream, type, levtype, levelist, resol,
@@ -462,19 +548,19 @@ class EcFlexpart(object):
Parameters
----------
- keys : :obj:`dictionary`
+ keys : dictionary
List of parameter names which serves as index.
- inputfiles : :obj:`UioFiles`
+ inputfiles : UioFiles
Contains a list of files.
Return
------
- iid : :obj:`codes_index`
+ iid : codes_index
This is a grib specific index structure to access
messages in a file.
- index_vals : :obj:`list`
+ index_vals : list of list of str
Contains the values from the keys used for a distinct selection
of grib messages in processing the grib files.
Content looks like e.g.:
@@ -516,7 +602,7 @@ class EcFlexpart(object):
Parameters
----------
- server : :obj:`ECMWFService` or :obj:`ECMWFDataServer`
+ server : ECMWFService or ECMWFDataServer
The connection to the ECMWF server. This is different
for member state users which have full access and non
member state users which have only access to the public
@@ -524,17 +610,17 @@ class EcFlexpart(object):
"public"; for public access its True (ECMWFDataServer)
for member state users its False (ECMWFService)
- dates : :obj:`string`
+ dates : str
Contains start and end date of the retrieval in the format
"YYYYMMDD/to/YYYYMMDD"
- request : :obj:`integer`
+ request : int
Selects the mode of retrieval.
0: Retrieves the data from ECMWF.
1: Prints the mars requests to an output file.
2: Retrieves the data and prints the mars request.
- inputdir : :obj:`string`, optional
+ inputdir : str, optional
Path to the directory where the retrieved data is about
to be stored. The default is the current directory ('.').
@@ -722,11 +808,11 @@ class EcFlexpart(object):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
- filename : :obj:`string`
+ filename : str
Name of the namelist file.
Return
@@ -809,10 +895,10 @@ class EcFlexpart(object):
Parameters
----------
- inputfiles : :obj:`UioFiles`
+ inputfiles : UioFiles
Contains the list of files that contain flux data.
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
@@ -836,7 +922,7 @@ class EcFlexpart(object):
# index_vals looks like e.g.:
# index_vals[0]: ('20171106', '20171107', '20171108') ; date
# index_vals[1]: ('0', '600', '1200', '1800') ; time
- # index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
+ # index_vals[2]: ('0', '3', '6', '9', '12') ; stepRange
if c.rrint:
if not c.purefc:
@@ -1095,31 +1181,31 @@ class EcFlexpart(object):
Parameters
----------
- ni : :obj:`integer`
+ ni : int
Amount of zonal grid points.
- nj : :obj:`integer`
+ nj : int
Amount of meridional grid points.
- nt : :obj:`integer`
+ nt : int
Number of time steps.
- lsp_np : :obj:`numpy array` of :obj:`float`
+ lsp_np : numpy array of float
The large scale precipitation fields for each time step.
Shape (ni * nj, nt).
- cp_np : :obj:`numpy array` of :obj:`float`
+ cp_np : numpy array of float
The convective precipitation fields for each time step.
Shape (ni * nj, nt).
- date_list : :obj:`list` of :obj:`datetime`
+ date_list : list of datetime
The list of dates for which the disaggregation is to be done.
- step_list : :obj:`list` of :obj:`integer`
+ step_list : list of int
The list of steps for a single forecast time.
Only necessary for pure forecasts.
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
@@ -1216,10 +1302,10 @@ class EcFlexpart(object):
Parameters
----------
- ifile : :obj:`string`
+ ifile : str
Filename of the input file to read the grib messages from.
- inputdir : :obj:`string`, optional
+ inputdir : str, optional
Path to the directory where the retrieved data is stored.
Return
@@ -1254,10 +1340,10 @@ class EcFlexpart(object):
Parameters
----------
- inputfiles : :obj:`UioFiles`
+ inputfiles : UioFiles
Contains a list of files.
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
@@ -1485,7 +1571,7 @@ class EcFlexpart(object):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
@@ -1544,7 +1630,7 @@ class EcFlexpart(object):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
diff --git a/source/python/classes/MarsRetrieval.py b/source/python/classes/MarsRetrieval.py
index c48598d..a8e2902 100644
--- a/source/python/classes/MarsRetrieval.py
+++ b/source/python/classes/MarsRetrieval.py
@@ -16,46 +16,16 @@
# - applied PEP8 style guide
# - added documentation
# - applied some minor modifications in programming style/structure
+# - added writing of mars request attributes to a csv file
#
# @License:
-# (C) Copyright 2015-2018.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Class Description:
-# A MARS revtrieval has a specific syntax with a selection of keywords and
-# their corresponding values. This class provides the necessary functions
-# by displaying the selected parameters and their values and the actual
-# retrievement of the data through a mars request or a Python web api
-# interface. The initialization already expects all the keyword values.
-#
-# @Class Content:
-# - __init__
-# - display_info
-# - data_retrieve
-#
-# @Class Attributes:
-# - server
-# - marsclass
-# - dtype
-# - levtype
-# - levelist
-# - repres
-# - date
-# - resol
-# - stream
-# - area
-# - time
-# - step
-# - expver
-# - number
-# - accuracy
-# - grid
-# - gaussian
-# - target
-# - param
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#*******************************************************************************
# ------------------------------------------------------------------------------
@@ -72,12 +42,91 @@ import _config
# CLASS
# ------------------------------------------------------------------------------
class MarsRetrieval(object):
- '''Class for submitting MARS retrievals.
+ '''Specific syntax and content for submission of MARS retrievals.
+
+ A MARS revtrieval has a specific syntax with a selection of keywords and
+ their corresponding values. This class provides the necessary functions
+ by displaying the selected parameters and their values and the actual
+ retrievement of the data through a mars request or a Python web api
+ interface. The initialization already expects all the keyword values.
A description of MARS keywords/arguments and examples of their
values can be found here:
https://software.ecmwf.int/wiki/display/UDOC/\
Identification+keywords#Identificationkeywords-class
+
+ Attributes
+ ----------
+ server : ECMWFService or ECMWFDataServer
+ This is the connection to the ECMWF data servers.
+
+ public : int
+ Decides which Web API Server version is used.
+
+ marsclass : str, optional
+ Characterisation of dataset.
+
+ dataset : str, optional
+ For public datasets there is the specific naming and parameter
+ dataset which has to be used to characterize the type of
+ data.
+
+ type : str, optional
+ Determines the type of fields to be retrieved.
+
+ levtype : str, optional
+ Denotes type of level.
+
+ levelist : str, optional
+ Specifies the required levels.
+
+ repres : str, optional
+ Selects the representation of the archived data.
+
+ date : str, optional
+ Specifies the Analysis date, the Forecast base date or
+ Observations date.
+
+ resol : str, optional
+ Specifies the desired triangular truncation of retrieved data,
+ before carrying out any other selected post-processing.
+
+ stream : str, optional
+ Identifies the forecasting system used to generate the data.
+
+ area : str, optional
+ Specifies the desired sub-area of data to be extracted.
+
+ time : str, optional
+ Specifies the time of the data in hours and minutes.
+
+ step : str, optional
+ Specifies the forecast time step from forecast base time.
+
+ expver : str, optional
+ The version of the dataset.
+
+ number : str, optional
+ Selects the member in ensemble forecast run.
+
+ accuracy : str, optional
+ Specifies the number of bits per value to be used in the
+ generated GRIB coded fields.
+
+ grid : str, optional
+ Specifies the output grid which can be either a Gaussian grid
+ or a Latitude/Longitude grid.
+
+ gaussian : str, optional
+ This parameter is deprecated and should no longer be used.
+ Specifies the desired type of Gaussian grid for the output.
+
+ target : str, optional
+ Specifies a file into which data is to be written after
+ retrieval or manipulation.
+
+ param : str, optional
+ Specifies the meteorological parameter.
'''
def __init__(self, server, public, marsclass="ei", dataset="", type="",
@@ -95,37 +144,37 @@ class MarsRetrieval(object):
Parameters
----------
- server : :obj:`ECMWFService`
+ server : ECMWFService or ECMWFDataServer
This is the connection to the ECMWF data servers.
It is needed for the pythonic access of ECMWF data.
- public : :obj:`integer`
+ public : int
Decides which Web API version is used:
0: member-state users and full archive access
1: public access and limited access to the public server and
datasets. Needs the parameter dataset.
Default is "0" and for member-state users.
- marsclass : :obj:`string`, optional
+ marsclass : str, optional
Characterisation of dataset. E.g. EI (ERA-Interim),
E4 (ERA40), OD (Operational archive), ea (ERA5).
Default is the ERA-Interim dataset "ei".
- dataset : :obj:`string`, optional
+ dataset : str, optional
For public datasets there is the specific naming and parameter
dataset which has to be used to characterize the type of
data. Usually there is less data available, either in times,
domain or parameter.
Default is an empty string.
- type : :obj:`string`, optional
+ type : str, optional
Determines the type of fields to be retrieved.
Selects between observations, images or fields.
Examples for fields: Analysis (an), Forecast (fc),
Perturbed Forecast (pf), Control Forecast (cf) and so on.
Default is an empty string.
- levtype : :obj:`string`, optional
+ levtype : str, optional
Denotes type of level. Has a direct implication on valid
levelist values!
E.g. model level (ml), pressure level (pl), surface (sfc),
@@ -133,25 +182,25 @@ class MarsRetrieval(object):
and depth (dp).
Default is an empty string.
- levelist : :obj:`string`, optional
+ levelist : str, optional
Specifies the required levels. It has to have a valid
correspondence to the selected levtype.
Examples: model level: 1/to/137, pressure levels: 500/to/1000
Default is an empty string.
- repres : :obj:`string`, optional
+ repres : str, optional
Selects the representation of the archived data.
E.g. sh - spherical harmonics, gg - Gaussian grid,
ll - latitude/longitude, ...
Default is an empty string.
- date : :obj:`string`, optional
+ date : str, optional
Specifies the Analysis date, the Forecast base date or
Observations date. Valid formats are:
Absolute as YYYY-MM-DD or YYYYMMDD.
Default is an empty string.
- resol : :obj:`string`, optional
+ resol : str, optional
Specifies the desired triangular truncation of retrieved data,
before carrying out any other selected post-processing.
The default is automatic truncation (auto), by which the lowest
@@ -168,12 +217,12 @@ class MarsRetrieval(object):
Retrieve#Retrieve-Truncationbeforeinterpolation
Default is an empty string.
- stream : :obj:`string`, optional
+ stream : str, optional
Identifies the forecasting system used to generate the data.
E.g. oper (Atmospheric model), enfo (Ensemble forecats), ...
Default is an empty string.
- area : :obj:`string`, optional
+ area : str, optional
Specifies the desired sub-area of data to be extracted.
Areas can be defined to wrap around the globe.
@@ -191,7 +240,7 @@ class MarsRetrieval(object):
E.g.: North/West/South/East
Default is an empty string.
- time : :obj:`string`, optional
+ time : str, optional
Specifies the time of the data in hours and minutes.
Valid values depend on the type of data: Analysis time,
Forecast base time or First guess verification time
@@ -201,7 +250,7 @@ class MarsRetrieval(object):
The syntax is HHMM or HH:MM. If MM is omitted it defaults to 00.
Default is an empty string.
- step : :obj:`string`, optional
+ step : str, optional
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time. It also
specifies the length of the forecast which verifies at
@@ -209,7 +258,7 @@ class MarsRetrieval(object):
E.g. 1/3/6-hourly
Default is an empty string.
- expver : :obj:`string`, optional
+ expver : str, optional
The version of the dataset. Each experiment is assigned a
unique code (version). Production data is assigned 1 or 2,
and experimental data in Operations 11, 12 ,...
@@ -217,14 +266,14 @@ class MarsRetrieval(object):
experiment identifier.
Default is "1".
- number : :obj:`string`, optional
+ number : str, optional
Selects the member in ensemble forecast run. (Only then it
is necessary.) It has a different meaning depending on
the type of data.
E.g. Perturbed Forecasts: specifies the Ensemble forecast member
Default is an empty string.
- accuracy : :obj:`string`, optional
+ accuracy : str, optional
Specifies the number of bits per value to be used in the
generated GRIB coded fields.
A positive integer may be given to specify the preferred number
@@ -235,7 +284,7 @@ class MarsRetrieval(object):
can be passed to the result field by specifying accuracy=av.
Default is an empty string.
- grid : :obj:`string`, optional
+ grid : str, optional
Specifies the output grid which can be either a Gaussian grid
or a Latitude/Longitude grid. MARS requests specifying
grid=av will return the archived model grid.
@@ -256,7 +305,7 @@ class MarsRetrieval(object):
640 latitude lines between the pole and equator
Default is an empty string.
- gaussian : :obj:`string`, optional
+ gaussian : str, optional
This parameter is deprecated and should no longer be used.
Specifies the desired type of Gaussian grid for the output.
Valid Gaussian grids are quasi-regular (reduced) or regular.
@@ -264,7 +313,7 @@ class MarsRetrieval(object):
keyword grid. Gaussian without grid has no effect.
Default is an empty string.
- target : :obj:`string`, optional
+ target : str, optional
Specifies a file into which data is to be written after
retrieval or manipulation. Path names should always be
enclosed in double quotes. The MARS client supports automatic
@@ -278,7 +327,7 @@ class MarsRetrieval(object):
rather than 600, 1 and 129.
Default is an empty string.
- param : :obj:`string`, optional
+ param : str, optional
Specifies the meteorological parameter.
The list of meteorological parameters in MARS is extensive.
Their availability is directly related to their meteorological
@@ -351,10 +400,10 @@ class MarsRetrieval(object):
Parameters
----------
- inputdir : :obj:`string`
+ inputdir : str
The path where all data from the retrievals are stored.
- request_number : :obj:`integer`
+ request_number : int
Number of mars requests for flux and non-flux data.
Return
diff --git a/source/python/classes/UioFiles.py b/source/python/classes/UioFiles.py
index 7a1257c..3cca0df 100644
--- a/source/python/classes/UioFiles.py
+++ b/source/python/classes/UioFiles.py
@@ -11,7 +11,7 @@
# - modified method list_files to work with glob instead of listdir
# - added pattern search in method list_files
#
-# February 2018 - Anne Philipp (University of Vienna):
+# February - December 2018 - Anne Philipp (University of Vienna):
# - applied PEP8 style guide
# - added documentation
# - optimisation of method list_files since it didn't work correctly
@@ -19,28 +19,16 @@
# - additional speed up of method list_files
# - modified the class so that it is initiated with a pattern instead
# of suffixes. Gives more precision in selection of files.
+# - added delete method
#
# @License:
-# (C) Copyright 2014-2018.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Class Decription:
-# The class is for file manipulation. It is initiated with a regular
-# expression pattern for this instance and can produce a list of Files
-# from the given file pattern. These files can be deleted.
-#
-# @Class Content:
-# - __init__
-# - __str__
-# - __list_files__
-# - delete_files
-#
-# @Class Attributes:
-# - pattern
-# - files
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#*******************************************************************************
# ------------------------------------------------------------------------------
@@ -50,9 +38,8 @@ import os
import sys
import fnmatch
-# software specific module from flex_extract
+# software specific modules from flex_extract
sys.path.append('../')
-#import profiling
from mods.tools import silent_remove, get_list_as_string
# ------------------------------------------------------------------------------
@@ -60,22 +47,35 @@ from mods.tools import silent_remove, get_list_as_string
# ------------------------------------------------------------------------------
class UioFiles(object):
- '''Class to manipulate files. At initialisation it has the pattern
- which stores a regular expression pattern for the files, the path
- to the files and the files already.
+ '''Collection of files matching a specific pattern.
+
+ The pattern can contain regular expressions for the files.
+ The files are listed and can be transformed to a single string or
+ they can be deleted.
+
+ Attributes
+ ----------
+ path : str
+ Directory where to list the files.
+
+ pattern : str
+ Regular expression pattern. For example: '\*.grb'
+
+ files : list of str
+ List of files matching the pattern in the path.
'''
# --------------------------------------------------------------------------
- # CLASS FUNCTIONS
+ # CLASS METHODS
# --------------------------------------------------------------------------
def __init__(self, path, pattern):
'''Assignes a specific pattern for these files.
Parameters
----------
- path : :obj:`string`
+ path : str
Directory where to list the files.
- pattern : :obj:`string`
+ pattern : str
Regular expression pattern. For example: '\*.grb'
Return
@@ -91,14 +91,14 @@ class UioFiles(object):
return
- #@profiling.timefn
+
def _list_files(self, path):
'''Lists all files in the directory with the matching
regular expression pattern.
Parameters
----------
- path : :obj:`string`
+ path : str
Path to the files.
Return
@@ -115,6 +115,7 @@ class UioFiles(object):
return
+
def __str__(self):
'''Converts the list of files into a single string.
The entries are sepereated by "," sign.
@@ -124,7 +125,7 @@ class UioFiles(object):
Return
------
- files_string : :obj:`string`
+ files_string : str
The content of the list as a single string.
'''
@@ -133,6 +134,7 @@ class UioFiles(object):
return files_string
+
def delete_files(self):
'''Deletes the files.
diff --git a/source/python/install.py b/source/python/install.py
index e719746..395d1ac 100755
--- a/source/python/install.py
+++ b/source/python/install.py
@@ -13,37 +13,50 @@
# - moved install_args_and_control in here
# - splitted code in smaller functions
# - delete convert build files in here instead of compile job script
-# - changed static path names to Variables from config file
+# - changed static path names to variables from config file
+# - splitted install function into several smaller pieces
+# - use of tarfile package in python
#
# @License:
-# (C) Copyright 2015-2018.
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#
-# @Program Functionality:
-# Depending on the selected installation environment (locally or on the
-# ECMWF server ecgate or cca) the program extracts the commandline
-# arguments and the CONTROL file parameter and prepares the corresponding
-# environment. The necessary files are collected in a tar-ball and placed
-# at the target location. There its untared, the environment variables will
-# be set and the Fortran code will be compiled. If the ECMWF environment is
-# selected a job script is prepared and submitted for the remaining
-# configurations after putting the tar-ball to the target ECMWF server.
-#
-# @Program Content:
-# - main
-# - get_install_cmdline_args
-# - install_via_gateway
-# - mk_tarball
-# - un_tarball
-# - mk_env_vars
-# - mk_compilejob
-# - mk_job_template
-# - del_convert_build
-# - mk_convert_build
+# @Methods:
+# main
+# get_install_cmdline_args
+# install_via_gateway
+# check_install_conditions
+# mk_tarball
+# un_tarball
+# mk_env_vars
+# mk_compilejob
+# mk_job_template
+# del_convert_build
+# mk_convert_build
#
#*******************************************************************************
+'''This script installs the flex_extract program.
+
+Depending on the selected installation environment (locally or on the
+ECMWF server ecgate or cca) the program extracts the commandline
+arguments and the CONTROL file parameter and prepares the corresponding
+environment.
+The necessary files are collected in a tar-ball and placed
+at the target location. There its untared, the environment variables will
+be set and the Fortran code will be compiled.
+If the ECMWF environment is selected a job script is prepared and submitted
+for the remaining configurations after putting the tar-ball to the
+target ECMWF server.
+
+Type: install.py --help
+to get information about command line parameters.
+Read the documentation for usage instructions.
+'''
# ------------------------------------------------------------------------------
# MODULES
@@ -95,7 +108,7 @@ def get_install_cmdline_args():
Return
------
- args : :obj:`Namespace`
+ args : Namespace
Contains the commandline arguments from script/program call.
'''
parser = ArgumentParser(description='Install flex_extract software locally or \
@@ -143,7 +156,7 @@ def install_via_gateway(c):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
@@ -228,7 +241,7 @@ def check_install_conditions(c):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
@@ -282,11 +295,11 @@ def mk_tarball(tarball_path, target):
Parameters
----------
- tarball_path : :obj:`string`
+ tarball_path : str
The complete path to the tar file which will contain all
relevant data for flex_extract.
- target : :obj:`string`
+ target : str
The queue where the job is submitted to.
Return
@@ -373,7 +386,7 @@ def un_tarball(tarball_path):
Parameters
----------
- tarball_path : :obj:`string`
+ tarball_path : str
The complete path to the tar file which will contain all
relevant data for flex_extract.
@@ -406,16 +419,16 @@ def mk_env_vars(ecuid, ecgid, gateway, destination):
Parameters
----------
- ecuid : :obj:`string`
+ ecuid : str
The user id on ECMWF server.
- ecgid : :obj:`string`
+ ecgid : str
The group id on ECMWF server.
- gateway : :obj:`string`
+ gateway : str
The gateway server the user is using.
- destination : :obj:`string`
+ destination : str
The remote destination which is used to transfer files
from ECMWF server to local gateway server.
@@ -468,20 +481,20 @@ def mk_compilejob(makefile, target, ecuid, ecgid, fp_root):
Parameters
----------
- makefile : :obj:`string`
+ makefile : str
Name of the makefile which should be used to compile FORTRAN
CONVERT2 program.
- target : :obj:`string`
+ target : str
The target where the installation should be done, e.g. the queue.
- ecuid : :obj:`string`
+ ecuid : str
The user id on ECMWF server.
- ecgid : :obj:`string`
+ ecgid : str
The group id on ECMWF server.
- fp_root : :obj:`string`
+ fp_root : str
Path to the root directory of FLEXPART environment or flex_extract
environment.
@@ -543,20 +556,20 @@ def mk_job_template(ecuid, ecgid, gateway, destination, fp_root):
Parameters
----------
- ecuid : :obj:`string`
+ ecuid : str
The user id on ECMWF server.
- ecgid : :obj:`string`
+ ecgid : str
The group id on ECMWF server.
- gateway : :obj:`string`
+ gateway : str
The gateway server the user is using.
- destination : :obj:`string`
+ destination : str
The remote destination which is used to transfer files
from ECMWF server to local gateway server.
- fp_root : :obj:`string`
+ fp_root : str
Path to the root directory of FLEXPART environment or flex_extract
environment.
@@ -621,7 +634,7 @@ def del_convert_build(src_path):
Parameters
----------
- src_path : :obj:`string`
+ src_path : str
Path to the fortran source directory.
Return
@@ -644,10 +657,10 @@ def mk_convert_build(src_path, makefile):
Parameters
----------
- src_path : :obj:`string`
+ src_path : str
Path to the fortran source directory.
- makefile : :obj:`string`
+ makefile : str
The name of the makefile which should be used.
Return
diff --git a/source/python/mods/checks.py b/source/python/mods/checks.py
index 7083339..e57259a 100644
--- a/source/python/mods/checks.py
+++ b/source/python/mods/checks.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-##*******************************************************************************
+#*******************************************************************************
# @Author: Anne Philipp (University of Vienna)
#
# @Date: November 2018
@@ -8,24 +8,23 @@
# @Change History:
#
# @License:
-# (C) Copyright 2014-2018.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Modul Description:
-#
-#
-# @Module Content:
-
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#*******************************************************************************
+'''This module contains check methods for the CONTROL paramaters.
+'''
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
import os
+import sys
import _config
import exceptions
from tools import my_error, silent_remove
@@ -39,11 +38,11 @@ def check_logicals_type(c, logicals):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
- logicals : :obj:`list` of (:obj:`string` or :obj:`integer`)
+ logicals : list of (str or int)
Names of the switches that are used to control the flow of the
program.
@@ -66,12 +65,12 @@ def check_grid(grid):
Parameters
----------
- grid : :obj:`string`
+ grid : str
Contains grid information
Return
------
- grid : :obj:``string`
+ grid : str
Contains grid in format Lat/lon. E.g. 0.1/0.1
'''
@@ -107,27 +106,27 @@ def check_area(grid, area, upper, lower, left , right):
Parameters
----------
- grid : :obj:`string`
+ grid : str
Contains grid information.
- area : :obj:`string`
+ area : str
Contains area informtion.
- upper : :obj:`string`
+ upper : str
The northern most latitude.
- lower : :obj:`string`
+ lower : str
The souther most latitude.
- left : :obj:`string`
+ left : str
The western most longitude.
- right : :obj:`string`
+ right : str
The eastern most longiude.
Return
------
- grid : :obj:``string`
+ grid : str
Contains grid in format Lat/lon. E.g. 0.1/0.1
'''
if 'N' in grid: # Gaussian output grid
@@ -172,21 +171,21 @@ def check_levels(levelist, level):
Parameters
----------
- levelist : :obj:`string`
+ levelist : str
Specifies the level list.
Examples: model level: 1/to/137, pressure levels: 500/to/1000
- level : :obj:`string`
+ level : str
Specifies the maximum level.
Return
------
- levelist : :obj:`string`
+ levelist : str
Specifies the required levels. It has to have a valid
correspondence to the selected levtype.
Examples: model level: 1/to/137, pressure levels: 500/to/1000
- level : :obj:`string`
+ level : str
Specifies the maximum level. It has to be one of the
available maximum level number as contained in the variable
MAX_LEVEL_LIST in "_config". E.g. [16, 19, 31, 40, 50, 60, 62, 91, 137]
@@ -220,11 +219,11 @@ def check_ppid(c, ppid):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
- ppid : :obj:`int` or :obj:`None`
+ ppid : int or None
Contains the ppid number provided by the command line parameter
of is None otherwise.
@@ -246,7 +245,7 @@ def check_purefc(ftype):
Parameters
----------
- ftype : :obj:`list` of :obj:`string`
+ ftype : list of str
List of field types.
Return
@@ -272,18 +271,18 @@ def check_step(step, mailfail):
Parameters
----------
- step : :obj:`list` of :obj:`string` or :obj:`string`
+ step : list of str or str
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time.
- mailfail : :obj:`list` of :obj:``string`
+ mailfail : list of str
Contains all email addresses which should be notified.
It might also contain just the ecmwf user name which will trigger
mailing to the associated email address for this user.
Return
------
- step : :obj:`list` of :obj:`string`
+ step : list of str
List of forecast steps in format e.g. [001, 002, ...]
'''
@@ -312,16 +311,16 @@ def check_type(ftype, steps):
Parameters
----------
- ftype : :obj:`list` of :obj:`string` or :obj:`string`
+ ftype : list of str or str
List of field types.
- steps : :obj:`string`
+ steps : str
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time.
Return
------
- ftype : :obj:`list` of :obj:`string`
+ ftype : list of str
List of field types.
'''
if not isinstance(ftype, list):
@@ -339,12 +338,12 @@ def check_time(ftime):
Parameters
----------
- ftime : :obj:`list` of :obj:`string` or :obj:`string`
+ ftime : list of str or str
The time in hours of the field.
Return
------
- ftime : :obj:`list` of :obj:`string`
+ ftime : list of str
The time in hours of the field.
'''
if not isinstance(ftime, list):
@@ -357,32 +356,32 @@ def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc):
Parameters
----------
- ftype : :obj:`list` of :obj:`string`
+ ftype : list of str
List of field types.
- ftime : :obj:`list` of :obj:`string` or :obj:`string`
+ ftime : list of str or str
The time in hours of the field.
- steps : :obj:`string`
+ steps : str
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time.
- maxstep : :obj:`integer`
+ maxstep : int
The maximum forecast time step in hours from the forecast base time.
This is the maximum step for non flux (accumulated) forecast data.
- purefc : :obj:`integer`
+ purefc : int
Switch for definition of pure forecast mode or not.
Return
------
- ftype : :obj:`list` of :obj:`string`
+ ftype : list of str
List of field types.
- ftime : :obj:`list` of :obj:`string`
+ ftime : list of str
The time in hours of the field.
- steps : :obj:`string`
+ steps : str
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time.
'''
@@ -409,14 +408,14 @@ def check_mail(mail):
Parameters
----------
- mail : :obj:`list` of :obj:`string` or :obj:`string`
+ mail : list of str or str
Contains email addresses for notifications.
It might also contain just the ecmwf user name which will trigger
mailing to the associated email address for this user.
Return
------
- mail : :obj:`list` of :obj:``string`
+ mail : list of str
Contains email addresses for notifications.
It might also contain just the ecmwf user name which will trigger
mailing to the associated email address for this user.
@@ -438,22 +437,22 @@ def check_queue(queue, gateway, destination, ecuid, ecgid):
Parameters
----------
- queue : :obj:`string`
+ queue : str
Name of the queue if submitted to the ECMWF servers.
Used to check if ecuid, ecgid, gateway and destination
are set correctly and are not empty.
- gateway : :obj:`string`
+ gateway : str
The address of the gateway server.
- destination : :obj:`string`
+ destination : str
The name of the destination of the gateway server for data
transfer through ectrans. E.g. name@genericSftp
- ecuid : :obj:`string`
+ ecuid : str
ECMWF user id.
- ecgid : :obj:`string`
+ ecgid : str
ECMWF group id.
Return
@@ -474,26 +473,26 @@ def check_pathes(idir, odir, fpdir, fedir):
Parameters
----------
- idir : :obj:`string`
+ idir : str
Path to the temporary directory for MARS retrieval data.
- odir : :obj:`string`
+ odir : str
Path to the final output directory where the FLEXPART input files
will be stored.
- fpdir : :obj:`string`
+ fpdir : str
Path to FLEXPART root directory.
- fedir : :obj:`string`
+ fedir : str
Path to flex_extract root directory.
Return
------
- odir : :obj:`string`
+ odir : str
Path to the final output directory where the FLEXPART input files
will be stored.
- fpdir : :obj:`string`
+ fpdir : str
Path to FLEXPART root directory.
'''
@@ -512,18 +511,18 @@ def check_dates(start, end):
Parameters
----------
- start : :obj:`string`
+ start : str
The start date of the retrieval job.
- end : :obj:`string`
+ end : str
The end date of the retrieval job.
Return
------
- start : :obj:`string`
+ start : str
The start date of the retrieval job.
- end : :obj:`string`
+ end : str
The end date of the retrieval job.
'''
@@ -554,17 +553,17 @@ def check_maxstep(maxstep, steps):
Parameters
----------
- maxstep : :obj:`string`
+ maxstep : str
The maximum forecast time step in hours from the forecast base time.
This is the maximum step for non flux (accumulated) forecast data.
- steps : :obj:`string`
+ steps : str
Specifies the forecast time step from forecast base time.
Valid values are hours (HH) from forecast base time.
Return
------
- maxstep : :obj:`integer`
+ maxstep : int
The maximum forecast time step in hours from the forecast base time.
This is the maximum step for non flux (accumulated) forecast data.
@@ -587,7 +586,7 @@ def check_basetime(basetime):
Parameters
----------
- basetime : :obj:``
+ basetime : str
The time for a half day retrieval. The 12 hours upfront are to be
retrieved.
@@ -606,13 +605,13 @@ def check_request(request, marsfile):
Parameters
----------
- request : :obj:`integer`
+ request : int
Selects the mode of retrieval.
0: Retrieves the data from ECMWF.
1: Prints the mars requests to an output file.
2: Retrieves the data and prints the mars request.
- marsfile : :obj:`string`
+ marsfile : str
Path to the mars request file.
Return
@@ -630,10 +629,10 @@ def check_public(public, dataset):
Parameters
----------
- public : :obj:`ìnteger`
+ public : int
Specifies if public data are to be retrieved or not.
- dataset : :obj:`string`
+ dataset : str
Specific name which identifies the public dataset.
Return
@@ -652,15 +651,15 @@ def check_acctype(acctype, ftype):
Parameters
----------
- acctype : :obj:`string`
+ acctype : str
The field type for the accumulated forecast fields.
- ftype : :obj:`list` of :obj:`string`
+ ftype : list of str
List of field types.
Return
------
- acctype : :obj:`string`
+ acctype : str
The field type for the accumulated forecast fields.
'''
if not acctype:
@@ -689,23 +688,23 @@ def check_acctime(acctime, acctype, purefc):
Parameters
----------
- acctime : :obj:`string`
+ acctime : str
The starting time from the accumulated forecasts.
- acctype : :obj:`string`
+ acctype : str
The field type for the accumulated forecast fields.
- purefc : :obj:`integer`
+ purefc : int
Switch for definition of pure forecast mode or not.
Return
------
- acctime : :obj:`string`
+ acctime : str
The starting time from the accumulated forecasts.
'''
if not acctime:
print('... Control parameter ACCTIME was not defined.')
- print('... Value will be set depending on field type: '
+ print('... Value will be set depending on field type:\n '
'\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t EP=18')
if acctype.upper() == 'EA': # Era 5
acctime = '06/18'
@@ -725,22 +724,22 @@ def check_accmaxstep(accmaxstep, acctype, purefc, maxstep):
Parameters
----------
- accmaxstep : :obj:`string`
+ accmaxstep : str
The maximum forecast step for the accumulated forecast fields.
- acctype : :obj:`string`
+ acctype : str
The field type for the accumulated forecast fields.
- purefc : :obj:`integer`
+ purefc : int
Switch for definition of pure forecast mode or not.
- maxstep : :obj:`string`
+ maxstep : str
The maximum forecast time step in hours from the forecast base time.
This is the maximum step for non flux (accumulated) forecast data.
Return
------
- accmaxstep : :obj:`string`
+ accmaxstep : str
The maximum forecast step for the accumulated forecast fields.
'''
if not accmaxstep:
@@ -775,12 +774,12 @@ def check_addpar(addpar):
Parameters
----------
- addpar : :obj:`string` or :obj:'list' of :obj:'string'
+ addpar : str or list of str
List of additional parameters to be retrieved.
Return
------
- addpar : :obj:'string'
+ addpar : str
List of additional parameters to be retrieved.
'''
@@ -797,18 +796,21 @@ def check_addpar(addpar):
def check_job_chunk(job_chunk):
- '''Checks that the job chunk number is positive and non zero.
+ '''Checks that if job chunk is set, the number is positive and non zero.
Parameters
----------
- job_chunk : :obj:`integer`
+ job_chunk : int
The number of days for a single job script.
Return
------
- job_chunk : :obj:`integer`
+ job_chunk : int
The number of days for a single job script.
'''
+ if not job_chunk:
+ return job_chunk
+
if job_chunk < 0:
raise ValueError('ERROR: The number of job chunk is negative!\n'
'It has to be a positive number!')
diff --git a/source/python/mods/disaggregation.py b/source/python/mods/disaggregation.py
index 337aa44..5407ca1 100644
--- a/source/python/mods/disaggregation.py
+++ b/source/python/mods/disaggregation.py
@@ -16,32 +16,36 @@
# - added structured documentation
# - outsourced the disaggregation functions dapoly and darain
# to a new module named disaggregation
+# - added the new disaggregation method for precipitation
#
# @License:
-# (C) Copyright 2015-2018.
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#
-# @Module Description:
-# disaggregation of deaccumulated flux data from an ECMWF model FG field.
-# Initially the flux data to be concerned are:
-# - large-scale precipitation
-# - convective precipitation
-# - surface sensible heat flux
-# - surface solar radiation
-# - u stress
-# - v stress
-# Different versions of disaggregation is provided for rainfall
-# data (darain, modified linear) and the surface fluxes and
-# stress data (dapoly, cubic polynomial).
-#
-# @Module Content:
+# @Methods:
# - dapoly
# - darain
# - IA3
-#
#*******************************************************************************
+'''Disaggregation of deaccumulated flux data from an ECMWF model FG field.
+
+Initially the flux data to be concerned are:
+ - large-scale precipitation
+ - convective precipitation
+ - surface sensible heat flux
+ - surface solar radiation
+ - u stress
+ - v stress
+
+Different versions of disaggregation is provided for rainfall
+data (darain, modified linear) and the surface fluxes and
+stress data (dapoly, cubic polynomial).
+'''
# ------------------------------------------------------------------------------
# MODULES
@@ -64,13 +68,13 @@ def dapoly(alist):
Parameters
----------
- alist : :obj:`list` of :obj:`array` of :obj:`float`
+ alist : list of array of float
List of 4 timespans as 2-dimensional, horizontal fields.
E.g. [[array_t1], [array_t2], [array_t3], [array_t4]]
Return
------
- nfield : :obj:`array` of :obj:`float`
+ nfield : array of float
Interpolated flux at central point of accumulation timespan.
Note
@@ -108,13 +112,13 @@ def darain(alist):
Parameters
----------
- alist : :obj:`list` of :obj:`array` of :obj:`float`
+ alist : list of array of float
List of 4 timespans as 2-dimensional, horizontal fields.
E.g. [[array_t1], [array_t2], [array_t3], [array_t4]]
Return
------
- nfield : :obj:`array` of :obj:`float`
+ nfield : array of float
Interpolated flux at central point of accumulation timespan.
Note
@@ -159,7 +163,7 @@ def IA3(g):
Note
----
- Copyright 2017
+ (C) Copyright 2017-2019
Sabine Hittmeir, Anne Philipp, Petra Seibert
This work is licensed under the Creative Commons Attribution 4.0
@@ -169,13 +173,13 @@ def IA3(g):
Parameters
----------
- g : :obj:`list` of :obj:`float`
+ g : list of float
Complete data series that will be interpolated having
the dimension of the original raw series.
Return
------
- f : :obj:`list` of :obj:`float`
+ f : list of float
The interpolated data series with additional subgrid points.
Its dimension is equal to the length of the input data series
times three.
diff --git a/source/python/mods/get_mars_data.py b/source/python/mods/get_mars_data.py
index 9805570..b577abc 100755
--- a/source/python/mods/get_mars_data.py
+++ b/source/python/mods/get_mars_data.py
@@ -21,42 +21,38 @@
# (necessary for better documentation with docstrings for later
# online documentation)
# - use of UIFiles class for file selection and deletion
-#
+# - seperated get_mars_data function into several smaller pieces:
+# write_reqheader, mk_server, mk_dates, remove_old, do_retrievment
#
# @License:
-# (C) Copyright 2014-2018.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Program Functionality:
-# This program can be used as a module in the whole flex_extract process
-# or can be run by itself to just extract MARS data from ECMWF. To do so,
-# a couple of necessary parameters has to be passed with the program call.
-# See documentation for more details.
-#
-# @Program Content:
-# - main
-# - get_mars_data
-# - do_retrievement
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#*******************************************************************************
-"""ToDo: Name of litte program
-
-ToDo: Add desccription
-
-ToDo: Add Conditions
+'''This script extracts MARS data from ECMWF servers.
-This script requires that `...` be installed within the Python
-environment you are running this script in.
+At first, the necessary parameters from command line and CONTROL files are
+extracted. They define the data set to be extracted from MARS.
This file can also be imported as a module and contains the following
functions:
- * get_mars_data -
- * do_retrievement -
* main - the main function of the script
-"""
+ * get_mars_data - overall control of ECMWF data retrievment
+ * write_reqheader - writes the header into the mars_request file
+ * mk_server - creates the server connection to ECMWF servers
+ * mk_dates - defines the start and end date
+ * remove_old - deletes old retrieved grib files
+ * do_retrievement - creates individual retrievals
+
+Type: get_mars_data.py --help
+to get information about command line parameters.
+Read the documentation for usage instructions.
+'''
# ------------------------------------------------------------------------------
# MODULES
# ------------------------------------------------------------------------------
@@ -66,6 +62,7 @@ import inspect
from datetime import datetime, timedelta
# software specific classes and modules from flex_extract
+# add path to local main python path for flex_extract to get full access
sys.path.append(os.path.dirname(os.path.abspath(
inspect.getfile(inspect.currentframe()))) + '/../')
import _config
@@ -119,7 +116,7 @@ def get_mars_data(c):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
@@ -165,7 +162,7 @@ def write_reqheader(marsfile):
Parameters
----------
- marsfile : :obj:`string`
+ marsfile : str
Path to the mars request file.
Return
@@ -188,13 +185,13 @@ def mk_server(c):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
Return
------
- server : :obj:`ECMWFDataServer` or :obj:`ECMWFService`
+ server : ECMWFDataServer or ECMWFService
Connection to ECMWF server via python interface ECMWF WebAPI.
'''
@@ -229,24 +226,24 @@ def mk_dates(c, fluxes):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
- fluxes : :obj:`boolean`, optional
+ fluxes : boolean, optional
Decides if the flux parameter settings are stored or
the rest of the parameter list.
Default value is False.
Return
------
- start : :obj:`datetime`
+ start : datetime
The start date of the retrieving data set.
- end : :obj:`datetime`
+ end : datetime
The end date of the retrieving data set.
- chunk : :obj:`datetime`
+ chunk : datetime
Time period in days for one single mars retrieval.
'''
@@ -270,10 +267,10 @@ def remove_old(pattern, inputdir):
Parameters
----------
- pattern : :obj:`string`
+ pattern : str
The sub string pattern which identifies the files to be deleted.
- inputdir : :obj:`string`, optional
+ inputdir : str, optional
Path to the directory where the retrieved data is stored.
Return
@@ -294,24 +291,24 @@ def do_retrievement(c, server, start, end, delta_t, fluxes=False):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
- server : :obj:`ECMWFService`
+ server : ECMWFService or ECMWFDataServer
The server connection to ECMWF.
- start : :obj:`datetime`
+ start : datetime
The start date of the retrieval.
- end : :obj:`datetime`
+ end : datetime
The end date of the retrieval.
- delta_t : :obj:`datetime`
+ delta_t : datetime
Delta_t + 1 is the maximal time period of a single
retrieval.
- fluxes : :obj:`boolean`, optional
+ fluxes : boolean, optional
Decides if the flux parameters are to be retrieved or
the rest of the parameter list.
Default value is False.
diff --git a/source/python/mods/prepare_flexpart.py b/source/python/mods/prepare_flexpart.py
index 1ef589c..45e3256 100755
--- a/source/python/mods/prepare_flexpart.py
+++ b/source/python/mods/prepare_flexpart.py
@@ -21,32 +21,40 @@
# - applied PEP8 style guide
# - added documentation
# - minor changes in programming style for consistence
-# - BUG: removed call of clean_up-Function after call of
+# - BUGFIX: removed call of clean_up-Function after call of
# prepareFlexpart in main since it is already called in
# prepareFlexpart at the end!
# - created function main and moved the two function calls for
# arguments and prepare_flexpart into it
#
# @License:
-# (C) Copyright 2014-2018.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Program Functionality:
-# This program prepares the final version of the grib files which are
-# then used by FLEXPART. It converts the bunch of grib files extracted
-# via get_mars_data by doing for example the necessary conversion to get
-# consistent grids or the disaggregation of flux data. Finally, the
-# program combines the data fields in files per available hour with the
-# naming convention xxYYMMDDHH, where xx should be 2 arbitrary letters
-# (mostly xx is chosen to be "EN").
-#
-# @Program Content:
-# - main
-# - prepare_flexpart
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#*******************************************************************************
+'''This script prepares the final version of the grib files which are
+then used by FLEXPART.
+
+It converts the bunch of grib files extracted via get_mars_data before,
+by doing the necessary conversion to get consistent grids or the
+disaggregation of flux data. Finally, the data fields are combined
+in files per available hour with the naming convention xxYYMMDDHH,
+where xx should be 2 arbitrary letters (mostly xx is chosen to be "EN").
+
+This file can also be imported as a module which then contains the following
+functions:
+
+ * main
+ * prepare_flexpart
+
+Type: prepare_flexpart.py --help
+to get information about command line parameters.
+Read the documentation for usage instructions.
+'''
# ------------------------------------------------------------------------------
# MODULES
@@ -58,7 +66,7 @@ import sys
import socket
# software specific classes and modules from flex_extract
-
+# add path to local main python path for flex_extract to get full access
sys.path.append(os.path.dirname(os.path.abspath(
inspect.getfile(inspect.currentframe()))) + '/../')
import _config
@@ -114,11 +122,11 @@ def prepare_flexpart(ppid, c):
Parameters
----------
- ppid : :obj:`int`
+ ppid : int
Contains the ppid number of the current ECMWF job. It will be None if
the method was called within this module.
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
diff --git a/source/python/mods/tools.py b/source/python/mods/tools.py
index 6f64903..1a755f8 100644
--- a/source/python/mods/tools.py
+++ b/source/python/mods/tools.py
@@ -13,36 +13,47 @@
# - created functions: interpret_args_and_control, clean_up
# my_error, normal_exit, init128, to_param_id
#
-# April 2018 - Anne Philipp (University of Vienna):
+# April - December 2018 - Anne Philipp (University of Vienna):
# - applied PEP8 style guide
# - added documentation
-# - moved all functions from file Flexparttools to this file tools
-# - added function get_list_as_string
+# - moved all non class methods from former file Flexparttools in here
# - seperated args and control interpretation
+# - added functions get_list_as_string, read_ecenv, send_mail, make_dir,
+# put_file_to_ecserver, submit_job_to_ecserver, get_informations,
+# get_dimensions, execute_subprocess, none_or_int, none_or_str
#
# @License:
-# (C) Copyright 2014-2018.
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Modul Description:
-# This module contains a couple of helpful functions which are
-# used in different places in flex_extract.
-#
-# @Module Content:
-# - get_cmdline_args
-# - clean_up
-# - my_error
-# - normal_exit
-# - product
-# - silent_remove
-# - init128
-# - to_param_id
-# - get_list_as_string
-# - make_dir
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#
+# @Methods:
+# none_or_str
+# none_or_int
+# get_cmdline_args
+# read_ecenv
+# clean_up
+# my_error
+# send_mail
+# normal_exit
+# product
+# silent_remove
+# init128
+# to_param_id
+# get_list_as_string
+# make_dir
+# put_file_to_ecserver
+# submit_job_to_ecserver
+# get_informations
+# get_dimensions
+# execute_subprocess
#*******************************************************************************
+'''This module contains a collection of diverse tasks within flex_extract.
+'''
# ------------------------------------------------------------------------------
# MODULES
@@ -58,7 +69,7 @@ from datetime import datetime
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
# ------------------------------------------------------------------------------
-# FUNCTIONS
+# METHODS
# ------------------------------------------------------------------------------
def none_or_str(value):
@@ -67,7 +78,7 @@ def none_or_str(value):
Parameters
----------
- value : :obj:`string`
+ value : str
String to be checked for the "None" word.
Return
@@ -86,7 +97,7 @@ def none_or_int(value):
Parameters
----------
- value : :obj:`string`
+ value : str
String to be checked for the "None" word.
Return
@@ -109,7 +120,7 @@ def get_cmdline_args():
Return
------
- args : :obj:`Namespace`
+ args : Namespace
Contains the commandline arguments from script/program call.
'''
@@ -200,12 +211,12 @@ def read_ecenv(filepath):
Parameters
----------
- filepath : :obj:`string`
+ filepath : str
Path to file where the ECMWF environment parameters are stored.
Return
------
- envs : :obj:`dictionary`
+ envs : dict
Contains the environment parameter ecuid, ecgid, gateway
and destination for ECMWF server environments.
'''
@@ -232,7 +243,7 @@ def clean_up(c):
Parameters
----------
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
@@ -264,12 +275,12 @@ def my_error(users, message='ERROR'):
Parameters
----------
- user : :obj:`list` of :obj:`string`
+ user : list of str
Contains all email addresses which should be notified.
It might also contain just the ecmwf user name which wil trigger
mailing to the associated email address for this user.
- message : :obj:`string`, optional
+ message : str, optional
Error message. Default value is "ERROR".
Return
@@ -294,16 +305,16 @@ def send_mail(users, success_mode, message):
Parameters
----------
- users : :obj:`list` of :obj:`string`
+ users : list of str
Contains all email addresses which should be notified.
It might also contain just the ecmwf user name which wil trigger
mailing to the associated email address for this user.
- success_mode : :obj:``string`
+ success_mode : str
States the exit mode of the program to put into
the mail subject line.
- message : :obj:`string`, optional
+ message : str, optional
Message for exiting program. Default value is "Done!".
Return
@@ -340,7 +351,7 @@ def normal_exit(message='Done!'):
Parameters
----------
- message : :obj:`string`, optional
+ message : str, optional
Message for exiting program. Default value is "Done!".
Return
@@ -373,10 +384,10 @@ def product(*args, **kwds):
Parameters
----------
- \*args : :obj:`list` or :obj:`string`
+ \*args : list or str
Positional arguments (arbitrary number).
- \*\*kwds : :obj:`dictionary`
+ \*\*kwds : dict
Contains all the keyword arguments from \*args.
Return
@@ -404,7 +415,7 @@ def silent_remove(filename):
Parameters
----------
- filename : :obj:`string`
+ filename : str
The name of the file to be removed without notification.
Return
@@ -428,12 +439,12 @@ def init128(filepath):
Parameters
----------
- filepath : :obj:`string`
+ filepath : str
Path to file of ECMWF grib table number 128.
Return
------
- table128 : :obj:`dictionary`
+ table128 : dict
Contains the ECMWF grib table 128 information.
The key is the parameter number and the value is the
short name of the parameter.
@@ -461,20 +472,20 @@ def to_param_id(pars, table):
Parameters
----------
- pars : :obj:`string`
+ pars : str
Addpar argument from CONTROL file in the format of
parameter names instead of ids. The parameter short
names are sepearted with "/" and they are passed as
one single string.
- table : :obj:`dictionary`
+ table : dict
Contains the ECMWF grib table 128 information.
The key is the parameter number and the value is the
short name of the parameter.
Return
------
- ipar : :obj:`list` of :obj:`integer`
+ ipar : list of int
List of addpar parameters from CONTROL file transformed to
parameter ids in the format of integer.
'''
@@ -500,16 +511,16 @@ def get_list_as_string(list_obj, concatenate_sign=', '):
Parameters
----------
- list_obj : :obj:`list`
+ list_obj : list of *
A list with arbitrary content.
- concatenate_sign : :obj:`string`, optional
+ concatenate_sign : str, optional
A string which is used to concatenate the single
list elements. Default value is ", ".
Return
------
- str_of_list : :obj:`string`
+ str_of_list : str
The content of the list as a single string.
'''
@@ -527,7 +538,7 @@ def make_dir(directory):
Parameters
----------
- directory : :obj:`string`
+ directory : str
The path to directory which should be created.
Return
@@ -556,19 +567,19 @@ def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
Parameters
----------
- ecd : :obj:`string`
+ ecd : str
The path were the file is stored.
- filename : :obj:`string`
+ filename : str
The name of the file to send to the ECMWF server.
- target : :obj:`string`
+ target : str
The target queue where the file should be sent to.
- ecuid : :obj:`string`
+ ecuid : str
The user id on ECMWF server.
- ecgid : :obj:`string`
+ ecgid : str
The group id on ECMWF server.
Return
@@ -609,15 +620,15 @@ def submit_job_to_ecserver(target, jobname):
Parameters
----------
- target : :obj:`string`
+ target : str
The target where the file should be sent to, e.g. the queue.
- jobname : :obj:`string`
+ jobname : str
The name of the jobfile to be submitted to the ECMWF server.
Return
------
- job_id : :obj:`int`
+ job_id : int
The id number of the job as a reference at the ecmwf server.
'''
@@ -649,12 +660,12 @@ def get_informations(filename):
Parameters
----------
- filename : :obj:`string`
+ filename : str
Name of the file which will be opened to extract basic information.
Return
------
- data : :obj:`dictionary`
+ data : dict
Contains basic informations of the ECMWF grib files, e.g.
'Ni', 'Nj', 'latitudeOfFirstGridPointInDegrees',
'longitudeOfFirstGridPointInDegrees', 'latitudeOfLastGridPointInDegrees',
@@ -701,20 +712,20 @@ def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date):
Parameters
----------
- info : :obj:`dictionary`
+ info : dict
Contains basic informations of the ECMWF grib files, e.g.
'Ni', 'Nj', 'latitudeOfFirstGridPointInDegrees',
'longitudeOfFirstGridPointInDegrees', 'latitudeOfLastGridPointInDegrees',
'longitudeOfLastGridPointInDegrees', 'jDirectionIncrementInDegrees',
'iDirectionIncrementInDegrees', 'missingValue'
- purefc : :obj:`integer`
+ purefc : int
Switch for definition of pure forecast mode or not.
- dtime : :obj:`string`
+ dtime : str
Time step in hours.
- index_vals : :obj:`list`
+ index_vals : list of list of str
Contains the values from the keys used for a distinct selection
of grib messages in processing the grib files.
Content looks like e.g.:
@@ -722,15 +733,15 @@ def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date):
index_vals[1]: ('0', '1200', '1800', '600') ; time
index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
- start_date : :obj:`string`
+ start_date : str
The start date of the retrieval job.
- end_date : :obj:`string`
+ end_date : str
The end date of the retrieval job.
Return
------
- (ix, jy, it) : :obj:`tuple` of :obj:`integer`
+ (ix, jy, it) : tuple of int
Dimension in x-direction, y-direction and in time.
'''
@@ -754,7 +765,7 @@ def execute_subprocess(cmd_list, error_msg='SUBPROCESS FAILED!'):
Parameters
----------
- cmd_list : :obj:`list` of `:obj:`string`
+ cmd_list : list of str
A list of the components for the command line execution. Each
list entry is a single part of the command which is seperated from
the rest by a blank space.
@@ -762,7 +773,7 @@ def execute_subprocess(cmd_list, error_msg='SUBPROCESS FAILED!'):
Return
------
- error_msg : :obj:`string`, optional
+ error_msg : str, optional
The possible error message if the subprocess failed.
By default it will just tell "SUBPROCESS FAILED!".
'''
diff --git a/source/python/submit.py b/source/python/submit.py
index 50cce9d..9427b94 100755
--- a/source/python/submit.py
+++ b/source/python/submit.py
@@ -17,27 +17,42 @@
# - minor changes in programming style (for consistence)
# - changed path names to variables from config file
# - added option for writing mars requests to extra file
-# additionally,as option without submitting the mars jobs
+# additionally, as option without submitting the mars jobs
+# - splitted submit function to use genshi templates for the
+# job script and avoid code duplication
#
# @License:
-# (C) Copyright 2014-2018.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Program Functionality:
-# This program is the main program of flex_extract and controls the
-# program flow.
-# If it is supposed to work locally then it works through the necessary
-# functions get_mars_data and prepareFlexpart. Otherwise it prepares
-# a shell job script which will do the necessary work on the
-# ECMWF server and is submitted via ecaccess-job-submit.
-#
-# @Program Content:
-# - main
-# - submit
+# (C) Copyright 2014-2019.
+# Anne Philipp, Leopold Haimberger
#
+# This work is licensed under the Creative Commons Attribution 4.0
+# International License. To view a copy of this license, visit
+# http://creativecommons.org/licenses/by/4.0/ or send a letter to
+# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
#*******************************************************************************
+'''This script allows the user to extract meteorological fields from the ECMWF.
+
+It prepares the settings for retrieving the data from ECMWF servers and
+process the resulting files to prepare them for the use with FLEXPART or
+FLEXTRA.
+
+If it is supposed to work locally then it works through the necessary
+functions get_mars_data and prepare_flexpart. Otherwise it prepares
+a job script (korn shell) which will do the necessary work on the
+ECMWF server. This script will de submitted via the ecaccess command
+ecaccess-job-submit.
+
+This file can also be imported as a module which then contains the following
+functions:
+
+ * main - the main function of the script
+ * submit - calls mk_jobscript depending on operation mode and submits its
+ * mk_jobscript - creates the job script from a template
+
+Type: submit.py --help
+to get information about command line parameters.
+Read the documentation for usage instructions.
+'''
# ------------------------------------------------------------------------------
# MODULES
@@ -58,7 +73,7 @@ from mods.prepare_flexpart import prepare_flexpart
from classes.ControlFile import ControlFile
# ------------------------------------------------------------------------------
-# FUNCTIONS
+# METHODS
# ------------------------------------------------------------------------------
def main():
@@ -111,18 +126,18 @@ def submit(jtemplate, c, queue):
Parameters
----------
- jtemplate : :obj:`string`
+ jtemplate : str
Job template file from sub-directory "_templates" for
submission to ECMWF. It contains all necessary
module and variable settings for the ECMWF environment as well as
the job call and mail report instructions.
Default is "job.temp".
- c : :obj:`ControlFile`
+ c : ControlFile
Contains all the parameters of CONTROL file and
command line.
- queue : :obj:`string`
+ queue : str
Name of queue for submission to ECMWF (e.g. ecgate or cca )
Return
@@ -200,17 +215,17 @@ def mk_jobscript(jtemplate, job_file, clist):
Parameters
----------
- jtemplate : :obj:`string`
+ jtemplate : str
Job template file from sub-directory "_templates" for
submission to ECMWF. It contains all necessary
module and variable settings for the ECMWF environment as well as
the job call and mail report instructions.
Default is "job.temp".
- job_file : :obj:`string`
+ job_file : str
Path to the job script file.
- clist : :obj:`list` of :obj:`string`
+ clist : list of str
Contains all necessary parameters for ECMWF CONTROL file.
Return
--
GitLab