diff --git a/Source/Python/Mods/checks.py b/Source/Python/Mods/checks.py index 5e0c65c3d6f13131d390ffbba6f70b49c999fd3c..181ec02f7768da1d5d6c7bd93335cec1d1cd6be2 100644 --- a/Source/Python/Mods/checks.py +++ b/Source/Python/Mods/checks.py @@ -118,9 +118,10 @@ def check_grid(grid): def check_area(grid, area, upper, lower, left, right): '''Defines the correct area string. - Checks on the format of the four area components. Wether it is of - the order of 1000 or 1. Also checks wether area was already set by command - line, then the four components are overwritten. + Checks the format of the four area components wether it is on + the order of 1000 or 1. + Also checks wether area was already set on command line, + then the four components are overwritten. Convert to correct format of the order of magnitude "1" and sets the area parameter (North/West/South/East). E.g.: -5./20./10./10. @@ -134,21 +135,21 @@ def check_area(grid, area, upper, lower, left, right): Contains area informtion. upper : str - The northern most latitude. + The northernmost latitude. lower : str - The souther most latitude. + The southernmost latitude. left : str - The western most longitude. + The westernmost longitude. right : str - The eastern most longiude. + The easternmost longitude. Return ------ grid : str - Contains grid in format Lat/lon. E.g. 0.1/0.1 + Contains grid in format lat/lon. E.g. 0.1/0.1 ''' if 'N' in grid: # Gaussian output grid area = 'G' @@ -179,7 +180,7 @@ def check_area(grid, area, upper, lower, left, right): float(lower), float(right)) else: - raise ValueError('The area components have different ' + raise ValueError('Area components have inconsisten or unrecognised ' 'formats (upper, lower, left, right): ' '{}/{}/{}/{}'.format(str(upper), str(lower), str(left), str(right))) @@ -208,8 +209,8 @@ def check_levels(levelist, level): level : str Specifies the maximum level. It has to be one of the - available maximum level number as contained in the variable - MAX_LEVEL_LIST in "_config". E.g. [16, 19, 31, 40, 50, 60, 62, 91, 137] + available maximum level numbers as contained in the variable + MAX_LEVEL_LIST in "_config": [16, 19, 31, 40, 50, 60, 62, 91, 137] ''' # assure consistency of levelist and level @@ -345,7 +346,7 @@ def check_type(ftype, steps): for i, val in enumerate(ftype): if ftype[i] == 'AN' and int(steps[i]) != 0: - print('Analysis retrievals must have STEP = 0 (now set to 0)') + print('For analysis data, STEP = 0 is needed. Setting to 0 now.)') ftype[i] = 0 return ftype @@ -385,7 +386,7 @@ def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc): maxstep : int The maximum forecast time step in hours from the forecast base time. - This is the maximum step for non flux (accumulated) forecast data. + This is the maximum step for non-flux (not accumulated) forecast data. purefc : int Switch for definition of pure forecast mode or not. @@ -404,7 +405,7 @@ def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc): ''' if not len(ftype) == len(ftime) == len(steps): raise ValueError('ERROR: The number of field types, times and steps ' - 'are not the same! Please check the setting in the ' + 'are not the same! Please check the settings in the ' 'CONTROL file!') # if pure forecast is selected and only one field type/time is set @@ -422,7 +423,7 @@ def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc): return ftype, ftime, steps def check_mail(mail): - '''Check the string of mail addresses, seperate them and convert to a list. + '''Check the string of mail addresses, separate them and convert to list. Parameters ---------- @@ -528,7 +529,7 @@ def check_pathes(idir, odir, fpdir, fedir): return odir, fpdir def check_dates(start, end): - '''Checks if there is at least a start date for a one day retrieval. + '''Checks if there is at least a start date for a one-day retrieval. Checks if end date lies after start date and end date is set. @@ -627,7 +628,7 @@ def check_basetime(basetime): return basetime def check_request(request, marsfile): - '''Check if there is an old mars request file and remove it. + '''Check if there is an old MARS request file; if so, remove it. Parameters ---------- @@ -650,8 +651,8 @@ def check_request(request, marsfile): return def check_public(public, dataset): - '''Check wether the dataset parameter is set for a - public data set retrieval. + '''Check wether the dataset parameter is set to a + public data set. Parameters ---------- @@ -666,14 +667,14 @@ def check_public(public, dataset): ''' if public and not dataset: - raise ValueError('ERROR: If public mars data wants to be retrieved, ' - 'the "dataset"-parameter has to be set too!') + raise ValueError('ERROR: If public MARS data are to be retrieved, ' + 'the "dataset"-parameter has to be set, too!') return def check_acctype(acctype, ftype): '''Guarantees that the accumulation field type is set. - If not set, it is derivated as in the old method (TYPE[1]). + If not set, it is derived with the old method (TYPE[1]). Parameters ---------- @@ -709,13 +710,13 @@ def check_acctype(acctype, ftype): def check_acctime(acctime, marsclass, purefc, time): '''Guarantees that the accumulation forecast times were set. - If it is not set, it tries to set the value for some of the - most commonly used data sets. Otherwise it raises an error. + If not set, setting the value to some of the most commonly used data sets + is attempted. Otherwise, an eror is raised. Parameters ---------- acctime : str - The starting time from the accumulated forecasts. + The starting time for the accumulated forecasts. marsclass : str ECMWF data classification identifier. @@ -726,11 +727,11 @@ def check_acctime(acctime, marsclass, purefc, time): Return ------ acctime : str - The starting time from the accumulated forecasts. + The starting time for the accumulated forecasts. ''' if not acctime: - print('... Control parameter ACCTIME was not defined.') + print('... Control parameter ACCTIME was not set.') print('... Value will be set depending on field type:\n ' '\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t EP=18') if marsclass.upper() == 'EA': # Era 5 @@ -744,12 +745,12 @@ def check_acctime(acctime, marsclass, purefc, time): elif marsclass.upper() == 'OD' and purefc: # On-demand acctime = time[0] else: - raise ValueError('ERROR: Accumulation forecast time can not ' - 'automatically be derived!') + raise ValueError('ERROR: Accumulation forecast time can not be' + 'derived automatically!') return acctime def check_accmaxstep(accmaxstep, marsclass, purefc, maxstep): - '''Guarantees that the accumulation forecast step were set. + '''Guarantees that the accumulation forecast step was set. Parameters ---------- @@ -764,7 +765,7 @@ def check_accmaxstep(accmaxstep, marsclass, purefc, maxstep): maxstep : str The maximum forecast time step in hours from the forecast base time. - This is the maximum step for non flux (accumulated) forecast data. + This is the maximum step for non-flux (accumulated) forecast data. Return ------ @@ -772,7 +773,7 @@ def check_accmaxstep(accmaxstep, marsclass, purefc, maxstep): The maximum forecast step for the accumulated forecast fields. ''' if not accmaxstep: - print('... Control parameter ACCMAXSTEP was not defined.') + print('... Control parameter ACCMAXSTEP was not set.') print('... Value will be set depending on field type/time: ' '\n\t\t EA/EI/OD=12\n\t\t EP=24') if marsclass.upper() in ['EA', 'EI', 'OD'] and not purefc: @@ -784,22 +785,22 @@ def check_accmaxstep(accmaxstep, marsclass, purefc, maxstep): accmaxstep = maxstep print('... For pure forecast mode, the accumulated forecast must ' 'have the same maxstep as the normal forecast fields!\n' - '\t\t Accmaxstep was set to maxstep!') + '\t\t ACCMAXSTEP was set to MAXSTEP!') else: - raise ValueError('ERROR: Accumulation forecast step can not ' - 'automatically be derived!') + raise ValueError('ERROR: Accumulation forecast step can not be' + 'derived automatically!') else: if purefc and int(accmaxstep) != int(maxstep): accmaxstep = maxstep print('... For pure forecast mode, the accumulated forecast must ' 'have the same maxstep as the normal forecast fields!\n' - '\t\t Accmaxstep was set to maxstep!') + '\t\t ACCMAXSTEP was set to MAXSTEP!') return accmaxstep def check_addpar(addpar): '''Check that addpar has correct format of additional parameters in - a single string, so that it can be easily appended to the hard coded - parameters that are retrieved in any case. + a single string, so that it can be easily appended to the hard-coded + parameters retrieved in any case. Parameters ---------- @@ -825,7 +826,7 @@ def check_addpar(addpar): def check_job_chunk(job_chunk): - '''Checks that if job chunk is set, the number is positive and non zero. + '''Checks that if job chunk is set, the number is positive and nonzero. Parameters ---------- @@ -843,7 +844,7 @@ def check_job_chunk(job_chunk): job_chunk = int(job_chunk) if job_chunk < 0: - raise ValueError('ERROR: The number of job chunk is negative!\n' + raise ValueError('ERROR: The number of job chunks is negative!\n' 'It has to be a positive number!') elif job_chunk == 0: job_chunk = None diff --git a/Source/Python/Mods/get_mars_data.py b/Source/Python/Mods/get_mars_data.py index ecdb311f8b4cbf3661fa0be9987e36062d135c22..515803f498a63a0d013653718d3021cc9dbb9654 100755 --- a/Source/Python/Mods/get_mars_data.py +++ b/Source/Python/Mods/get_mars_data.py @@ -9,9 +9,9 @@ # # November 2015 - Leopold Haimberger (University of Vienna): # - moved the getEIdata program into a function "get_mars_data" -# - moved the AgurmentParser into a seperate function -# - adatpted the function for the use in flex_extract -# - renamed file to get_mars_data +# - moved the AgurmentParser into a separate function +# - adapted the function for use in flex_extract +# - renamed source file to get_mars_data # # February 2018 - Anne Philipp (University of Vienna): # - applied PEP8 style guide @@ -21,7 +21,7 @@ # (necessary for better documentation with docstrings for later # online documentation) # - use of UIFiles class for file selection and deletion -# - seperated get_mars_data function into several smaller pieces: +# - separated get_mars_data function into several smaller pieces: # write_reqheader, mk_server, mk_dates, remove_old, do_retrievment # # @License: @@ -35,7 +35,7 @@ # http://creativecommons.org/licenses/by/4.0/ or send a letter to # Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. #******************************************************************************* -'''This script extracts MARS data from ECMWF servers. +'''This script extracts MARS data from ECMWF. At first, the necessary parameters from command line and CONTROL files are extracted. They define the data set to be extracted from MARS. @@ -43,15 +43,15 @@ extracted. They define the data set to be extracted from MARS. This file can also be imported as a module and contains the following functions: - * main - the main function of the script - * get_mars_data - overall control of ECMWF data retrievment + * main - the main function of the script + * get_mars_data - overall control of ECMWF data retrievment * write_reqheader - writes the header into the mars_request file - * mk_server - creates the server connection to ECMWF servers - * mk_dates - defines the start and end date - * remove_old - deletes old retrieved grib files - * do_retrievement - creates individual retrievals + * mk_server - creates the server connection to ECMWF servers + * mk_dates - defines the start and end date + * remove_old - deletes old retrieved grib files + * do_retrieval - creates individual retrievals -Type: get_mars_data.py --help +Type get_mars_data.py --help to get information about command line parameters. Read the documentation for usage instructions. ''' @@ -65,8 +65,8 @@ import sys import inspect from datetime import datetime, timedelta -# software specific classes and modules from flex_extract -# add path to local main python path for flex_extract to get full access +# software-specific classes and modules from flex_extract +# add path to local main Python path for flex_extract to get full access sys.path.append(os.path.dirname(os.path.abspath( inspect.getfile(inspect.currentframe()))) + '/../') # pylint: disable=wrong-import-position @@ -93,10 +93,10 @@ except ImportError: # FUNCTION # ------------------------------------------------------------------------------ def main(): - '''Controls the program to get data out of mars. + '''Controls the program to retrieve data from MARS. - This is done if it is called directly from command line. - Then it also takes program call arguments and control file input. + This is done if called directly from command line. + Then, arguments and control file are taken as input. Parameters ---------- @@ -113,10 +113,10 @@ def main(): return def get_mars_data(c): - '''Retrieves the EC data needed for a FLEXPART simulation. + '''Retrieves the ECMWF data required for a FLEXPART simulation. - Start and end dates for retrieval period is set. Retrievals - are divided into smaller periods if necessary and datechunk parameter + Start and end dates for retrieval period are set. Retrievals + are divided into shorter periods if necessary and if datechunk parameter is set. Parameters @@ -136,12 +136,12 @@ def get_mars_data(c): make_dir(c.inputdir) if c.request == 0: - print("Retrieving EC data!") + print("Retrieving ECMWF data!") else: if c.request == 1: - print("Printing mars requests!") + print("Printing MARS requests!") elif c.request == 2: - print("Retrieving EC data and printing mars request!") + print("Retrieving ECMWF data and printing MARS request!") write_reqheader(os.path.join(c.inputdir, _config.FILE_MARS_REQUESTS)) print("start date %s " % (c.start_date)) @@ -164,12 +164,12 @@ def get_mars_data(c): return def write_reqheader(marsfile): - '''Writes header with column names into mars request file. + '''Writes header with column names into MARS request file. Parameters ---------- marsfile : str - Path to the mars request file. + Path to the MARS request file. Return ------ @@ -187,11 +187,12 @@ def write_reqheader(marsfile): return def mk_server(c): - '''Creates a server connection with available python API. + '''Creates a server connection with available Python API. - Which API is used depends on availability and the dataset to be retrieved. - The CDS API is used for ERA5 dataset no matter if the user is a member or - a public user. ECMWF WebAPI is used for all other available datasets. + The API selected depends on availability and the data set to be retrieved. + The CDS API is used for ERA5 data, no matter whether the user is a + member-state or a public user. + ECMWF WebAPI is used for all other available datasets. Parameters ---------- @@ -246,14 +247,14 @@ def check_dates_for_nonflux_fc_times(types, times): def mk_dates(c, fluxes): - '''Prepares start and end date depending on flux or non flux data. + '''Prepares start and end date depending on flux or non-flux type of data. - If forecast for maximum one day (upto 24h) are to be retrieved, then + If forecasts for a maximum of one day (24 h) are to be retrieved, then collect accumulation data (flux data) with additional days in the - beginning and at the end (used for complete disaggregation of + beginning and at the end (needed for complete disaggregation of original period) - If forecast data longer than 24h are to be retrieved, then + If forecast data for more than +24 h are to be retrieved, then collect accumulation data (flux data) with the exact start and end date (disaggregation will be done for the exact time period with boundary conditions) @@ -313,16 +314,16 @@ def remove_old(pattern, inputdir): Parameters ---------- pattern : str - The sub string pattern which identifies the files to be deleted. + The substring pattern which identifies the files to be deleted. inputdir : str, optional - Path to the directory where the retrieved data is stored. + Path to the directory where the retrieved data are stored. Return ------ ''' - print('... removing old content of ' + inputdir) + print('... removing old files in ' + inputdir) tobecleaned = UioFiles(inputdir, pattern) tobecleaned.delete_files() @@ -331,7 +332,7 @@ def remove_old(pattern, inputdir): def do_retrievement(c, server, start, end, delta_t, fluxes=False): - '''Divides the complete retrieval period in smaller chunks and + '''Divides the total retrieval period into smaller chunks and retrieves the data from MARS. Parameters @@ -350,8 +351,7 @@ def do_retrievement(c, server, start, end, delta_t, fluxes=False): The end date of the retrieval. delta_t : datetime - Delta_t + 1 is the maximal time period of a single - retrieval. + Delta_t + 1 is the maximum time period of a single retrieval. fluxes : boolean, optional Decides if the flux parameters are to be retrieved or @@ -364,8 +364,7 @@ def do_retrievement(c, server, start, end, delta_t, fluxes=False): ''' # since actual day also counts as one day, - # we only need to add datechunk - 1 days to retrieval - # for a period + # we only need to add datechunk - 1 days to retrieval for a period delta_t_m1 = delta_t - timedelta(days=1) day = start diff --git a/Source/Python/Mods/prepare_flexpart.py b/Source/Python/Mods/prepare_flexpart.py index 57323245dfadc05b833b2090964a39c6b506dafa..79785d7b77ccbf73982765dba930ec555dbbf41d 100755 --- a/Source/Python/Mods/prepare_flexpart.py +++ b/Source/Python/Mods/prepare_flexpart.py @@ -12,10 +12,10 @@ # - job submission on ecgate and cca # - job templates suitable for twice daily operational dissemination # - dividing retrievals of longer periods into digestable chunks -# - retrieve also longer term forecasts, not only analyses and -# short term forecast data +# - retrieve also longer-term forecasts, not only analyses and +# short-term forecast data # - conversion into GRIB2 -# - conversion into .fp format for faster execution of FLEXPART +# # # February 2018 - Anne Philipp (University of Vienna): # - applied PEP8 style guide @@ -44,9 +44,9 @@ then used by FLEXPART. It converts the bunch of grib files extracted via get_mars_data before, -by doing the necessary conversion to get consistent grids or the +by doing the necessary conversion to obtain consistent grids or the disaggregation of flux data. Finally, the data fields are combined -in files per available hour with the naming convention xxYYMMDDHH, +in files per hour available with the naming convention xxYYMMDDHH, where xx should be 2 arbitrary letters (mostly xx is chosen to be "EN"). This file can also be imported as a module which then contains the following @@ -87,10 +87,10 @@ from Classes.EcFlexpart import EcFlexpart # FUNCTION # ------------------------------------------------------------------------------ def main(): - '''Controls the program to prepare flexpart input files from mars data. + '''Controls the program to prepare FLEXPART input files from MARS data. - This is done if it is called directly from command line. - Then it also takes program call arguments and control file input. + This is done if called directly from the command line. + Then, arguments and control file are taken as input. Parameters ---------- @@ -107,12 +107,11 @@ def main(): return def prepare_flexpart(ppid, c): - '''Converts the mars data into flexpart ready input files. + '''Converts the MARS data into files ready as input for FLEXPART. - Specific data fields are converted to a different grid and the flux - data are going to be disaggregated. The data fields are collected by - hour and stored in a file with a specific FLEXPART relevant naming - convention. + Certain fields are converted to a different grid and the flux + data are disaggregated. Fields are collected by hour and stored in a file + with a specific naming convention. Parameters ---------- diff --git a/Source/Python/Mods/tools.py b/Source/Python/Mods/tools.py index 7888926bc9b14abe78d494d7dcd9819590b5d5b9..b21ebc44eb3db57762d11a928b9b6327cee53067 100644 --- a/Source/Python/Mods/tools.py +++ b/Source/Python/Mods/tools.py @@ -17,7 +17,7 @@ # - applied PEP8 style guide # - added documentation # - moved all non class methods from former file Flexparttools in here -# - seperated args and control interpretation +# - separated args and control interpretation # - added functions get_list_as_string, read_ecenv, send_mail, make_dir, # put_file_to_ecserver, submit_job_to_ecserver, get_informations, # get_dimensions, execute_subprocess, none_or_int, none_or_str @@ -117,8 +117,8 @@ def setup_controldata(): return c, args.ppid, args.queue, args.job_template def none_or_str(value): - '''Converts the input string into pythons None-type if the string - contains string "None". + '''Converts the input string into Pythons None type if it + contains the string "None". Parameters ---------- @@ -129,15 +129,15 @@ def none_or_str(value): ------ None or value: Return depends on the content of the input value. If it was "None", - then the python type None is returned. Otherwise the string itself. + then the Python type None is returned, otherwise the string itself. ''' if value == 'None': return None return value def none_or_int(value): - '''Converts the input string into pythons None-type if the string - contains string "None". Otherwise it is converted to an integer value. + '''Converts the input string into Pythons None-type if it + contains string "None"; otherwise it is converted to an integer value. Parameters ---------- @@ -157,7 +157,7 @@ def none_or_int(value): def get_cmdline_args(): '''Decomposes the command line arguments and assigns them to variables. - Apply default values for non mentioned arguments. + Apply default values for arguments not present. Parameters ---------- @@ -165,7 +165,7 @@ def get_cmdline_args(): Return ------ args : Namespace - Contains the commandline arguments from script/program call. + Contains the command line arguments from the script / program call. ''' parser = ArgumentParser(description='Retrieve FLEXPART input from \ @@ -190,7 +190,7 @@ def get_cmdline_args(): help="The file with all CONTROL parameters.") parser.add_argument("--basetime", dest="basetime", type=none_or_int, default=None, - help="base such as 0 or 12 (for half day retrievals)") + help="base time such as 0 or 12 (for half day retrievals)") parser.add_argument("--step", dest="step", type=none_or_str, default=None, help="Forecast steps such as 00/to/48") @@ -199,55 +199,56 @@ def get_cmdline_args(): help="Vertical levels to be retrieved, e.g. 30/to/60") parser.add_argument("--area", dest="area", type=none_or_str, default=None, - help="area defined as north/west/south/east") + help="area, defined by north/west/south/east") # some switches parser.add_argument("--debug", dest="debug", type=none_or_int, default=None, - help="debug mode - leave temporary files intact") + help="debug mode - temporary files will be conserved") parser.add_argument("--oper", dest="oper", type=none_or_int, default=None, - help='operational mode - prepares dates with ' + help='operational mode - prepares dates from ' 'environment variables') parser.add_argument("--request", dest="request", type=none_or_int, default=None, - help="list all mars requests in file mars_requests.dat") + help="list all MARS requests in file mars_requests.dat") parser.add_argument("--public", dest="public", type=none_or_int, default=None, - help="public mode - retrieves the public datasets") + help="public mode - retrieves public datasets") parser.add_argument("--rrint", dest="rrint", type=none_or_int, default=None, - help='Selection of old or new precipitation ' - 'interpolation:\n' + help='Selection of old or new ' + 'interpolation method for precipitation:\n' ' 0 - old method\n' ' 1 - new method (additional subgrid points)') # set directories parser.add_argument("--inputdir", dest="inputdir", type=none_or_str, default=None, - help='Path to the temporary directory for the ' - 'retrieval grib files and other processing files.') + help='Path to temporary directory for ' + 'retrieved grib files and other processing files.') parser.add_argument("--outputdir", dest="outputdir", type=none_or_str, default=None, - help='Path to the final directory where the final ' - 'FLEXPART ready input files are stored.') + help='Path to final directory where ' + 'FLEXPART input files will be stored.') # this is only used by prepare_flexpart.py to rerun a postprocessing step parser.add_argument("--ppid", dest="ppid", type=none_or_str, default=None, - help='This is the specify parent process id of a ' + help='This is the specify the parent process id of a ' 'single flex_extract run to identify the files. ' 'It is the second number in the GRIB files.') # arguments for job submission to ECMWF, only needed by submit.py parser.add_argument("--job_template", dest='job_template', type=none_or_str, default="job.temp", - help='The job template file which are adapted to be ' - 'submitted to the batch system on ECMWF server.') + help='Job template file. Will be used for submission ' + 'to the batch system on the ECMWF server after ' + 'modification.') parser.add_argument("--queue", dest="queue", type=none_or_str, default=None, - help='The ECMWF server name for submission of the ' - 'job script to the batch system ' + help='The name of the ECMWF server name where the' + 'job script is to be submitted ' '(e.g. ecgate | cca | ccb)') args = parser.parse_args() @@ -288,7 +289,7 @@ def clean_up(c): '''Remove files from the intermediate directory (inputdir). It keeps the final FLEXPART input files if program runs without - ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1". + ECMWF API and keywords "ectrans" or "ecstorage" are set to "1". Parameters ---------- @@ -409,7 +410,7 @@ def product(*args, **kwds): '''Creates combinations of all passed arguments. This method combines the single characters of the passed arguments - with each other. So that each character of each argument value + with each other in a way that each character of each argument value will be combined with each character of the other arguments as a tuple. Note @@ -518,9 +519,8 @@ def to_param_id(pars, table): ---------- pars : str Addpar argument from CONTROL file in the format of - parameter names instead of ids. The parameter short - names are sepearted with "/" and they are passed as - one single string. + parameter names instead of IDs. The parameter short + names are separated by "/" and passed as one single string. table : dict Contains the ECMWF grib table 128 information. @@ -554,7 +554,7 @@ def to_param_id(pars, table): return ipar def to_param_id_with_tablenumber(pars, table): - '''Transform parameter names to parameter ids and add table id. + '''Transform parameter names to parameter IDs and add table ID. Conversion with ECMWF grib table 128. @@ -562,9 +562,8 @@ def to_param_id_with_tablenumber(pars, table): ---------- pars : str Addpar argument from CONTROL file in the format of - parameter names instead of ids. The parameter short - names are sepearted with "/" and they are passed as - one single string. + parameter names instead of ID. The parameter short + names are separated by "/" and passed as one single string. table : dict Contains the ECMWF grib table 128 information. @@ -575,7 +574,7 @@ def to_param_id_with_tablenumber(pars, table): ------ spar : str List of addpar parameters from CONTROL file transformed to - parameter ids in the format of integer. + parameter IDs in the format of integer. ''' if not pars: return [] @@ -597,7 +596,8 @@ def to_param_id_with_tablenumber(pars, table): return '/'.join(spar) def get_list_as_string(list_obj, concatenate_sign=', '): - '''Converts a list of arbitrary content into a single string. + '''Converts a list of arbitrary content into a single string using a given + concatenation character. Parameters ---------- @@ -623,8 +623,8 @@ def get_list_as_string(list_obj, concatenate_sign=', '): def make_dir(directory): '''Creates a directory. - It gives a warning if the directory already exists and skips process. - The program stops only if there is another problem. + If the directory already exists, an information is printed and the creation + skipped. The program stops only if there is another problem. Parameters ---------- @@ -640,7 +640,7 @@ def make_dir(directory): except OSError as e: # errno.EEXIST = directory already exists if e.errno == errno.EEXIST: - print('WARNING: Directory {0} already exists!'.format(directory)) + print('INFORMATION: Directory {0} already exists!'.format(directory)) else: raise # re-raise exception if a different error occured @@ -719,7 +719,7 @@ def submit_job_to_ecserver(target, jobname): Return ------ job_id : int - The id number of the job as a reference at the ecmwf server. + The id number of the job as a reference at the ECMWF server. ''' try: @@ -731,22 +731,22 @@ def submit_job_to_ecserver(target, jobname): print('... ERROR MESSAGE:\n \t ' + str(e)) print('\n... Do you have a valid ecaccess certification key?') - sys.exit('... ECACCESS-JOB-SUBMIT FAILED!') + sys.exit('... ecaccess-job-submit FAILED!') except OSError as e: print('... ERROR CODE: ' + str(e.errno)) print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) print('\n... Most likely the ECACCESS library is not available!') - sys.exit('... ECACCESS-JOB-SUBMIT FAILED!') + sys.exit('... ecaccess-job-submit FAILED!') return job_id.decode() def get_informations(filename): - '''Gets basic information from an example grib file. + '''Extracts basic information from a sample grib file. - These information are important for later use and the - initialization of numpy arrays for data storing. + This information is needed for later use and the + initialization of numpy arrays where data are stored. Parameters ---------- @@ -767,7 +767,7 @@ def get_informations(filename): data = {} # --- open file --- - print("Opening file for getting information data --- %s" % filename) + print("Opening grib file for extraction of information --- %s" % filename) with open(filename, 'rb') as f: # load first message from file gid = codes_grib_new_from_file(f) @@ -784,7 +784,7 @@ def get_informations(filename): 'missingValue', ] - print('\nInformations are: ') + print('\nInformation extracted: ') for key in keys: # Get the value of the key in a grib message. data[key] = codes_get(gid, key) @@ -797,7 +797,7 @@ def get_informations(filename): def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date): - '''This function specifies the correct dimensions for x, y and t. + '''This function specifies the correct dimensions for x, y, and t. Parameters ---------- @@ -816,7 +816,7 @@ def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date): index_vals : list of list of str Contains the values from the keys used for a distinct selection - of grib messages in processing the grib files. + of GRIB messages in processing the grib files. Content looks like e.g.: index_vals[0]: ('20171106', '20171107', '20171108') ; date index_vals[1]: ('0', '1200', '1800', '600') ; time @@ -848,23 +848,22 @@ def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date): def execute_subprocess(cmd_list, error_msg='SUBPROCESS FAILED!'): - '''Executes a command line instruction via a subprocess. + '''Executes a command via a subprocess. Error handling is done if an error occures. Parameters ---------- cmd_list : list of str - A list of the components for the command line execution. Each - list entry is a single part of the command which is seperated from - the rest by a blank space. - E.g. ['mv', file1, file2] + A list of the components for the command line execution. + They will be concatenated with blank space for the command + to be submitted, like ['mv', file1, file2] for mv file1 file2. Return ------ error_msg : str, optional - The possible error message if the subprocess failed. - By default it will just tell "SUBPROCESS FAILED!". + Error message if the subprocess fails. + By default it will just say "SUBPROCESS FAILED!". ''' try: diff --git a/Source/Python/install.py b/Source/Python/install.py index 2ed782b2636344fb930ebdc98ca9d4fa8c3c79ca..9c41599acfe8a56c42e48890d14fc9b3abd98405 100755 --- a/Source/Python/install.py +++ b/Source/Python/install.py @@ -45,14 +45,14 @@ '''This script installs the flex_extract program. Depending on the selected installation environment (locally or on the -ECMWF server ecgate or cca) the program extracts the commandline +ECMWF server ecgate or cca) the program extracts the command line arguments and the CONTROL file parameter and prepares the corresponding environment. -The necessary files are collected in a tar-ball and placed -at the target location. There its untared, the environment variables will -be set and the Fortran code will be compiled. -If the ECMWF environment is selected a job script is prepared and submitted -for the remaining configurations after putting the tar-ball to the +The necessary files are collected in a tar ball and placed +at the target location. There, is is untared, the environment variables are +set, and the Fortran code is compiled. +If the ECMWF environment is selected, a job script is prepared and submitted +for the remaining configurations after putting the tar ball on the target ECMWF server. Type: install.py --help @@ -106,7 +106,7 @@ def main(): def get_install_cmdline_args(): '''Decomposes the command line arguments and assigns them to variables. - Apply default values for non mentioned arguments. + Apply default values for arguments not present. Parameters ---------- @@ -126,36 +126,36 @@ def get_install_cmdline_args(): the latter two are at ECMWF") parser.add_argument("--makefile", dest="makefile", type=none_or_str, default=None, - help='Name of Makefile to use for compiling the ' + help='Name of makefile for compiling the ' 'Fortran program') parser.add_argument("--ecuid", dest="ecuid", type=none_or_str, default=None, - help='The user id at ECMWF.') + help='User id at ECMWF') parser.add_argument("--ecgid", dest="ecgid", type=none_or_str, default=None, - help='The group id at ECMWF.') + help='Group id at ECMWF') parser.add_argument("--gateway", dest="gateway", type=none_or_str, default=None, - help='The name of the local gateway server.') + help='Name of the local gateway server') parser.add_argument("--destination", dest="destination", type=none_or_str, default=None, - help='The ecaccess association, e.g. ' + help='ecaccess association, e.g. ' 'myUser@genericSftp') parser.add_argument("--installdir", dest="installdir", type=none_or_str, default=None, - help='Root directory where ' - 'flex_extract will be installed to.') + help='Root directory of the ' + 'flex_extract installation') # arguments for job submission to ECMWF, only needed by submit.py parser.add_argument("--job_template", dest='job_template', type=none_or_str, default="job.template", - help='The rudimentary template file to create a batch ' - 'job template for submission to ECMWF servers.') + help='Rudimentary template file to create a batch ' + 'job template for submission to ECMWF servers') parser.add_argument("--controlfile", dest="controlfile", type=none_or_str, default='CONTROL_EA5', - help="The file with all CONTROL parameters.") + help="A file that contains all CONTROL parameters.") args = parser.parse_args() @@ -163,8 +163,8 @@ def get_install_cmdline_args(): def install_via_gateway(c): - '''Prepare data transfer to remote gate and submit a job script which will - install everything on the remote gate. + '''Prepare data transfer to remote gateway and submit a job script which will + install everything on the remote gateway. Parameters ---------- @@ -258,8 +258,8 @@ def install_local(c): def check_install_conditions(c): - '''Checks a couple of necessary attributes and conditions - for the installation such as if they exist and contain values. + '''Checks necessary attributes and conditions + for the installation, e.g. whether they exist and contain values. Otherwise set default values. Parameters @@ -309,8 +309,8 @@ def check_install_conditions(c): def mk_tarball(tarball_path, target): '''Creates a tarball with all necessary files which need to be sent to the installation directory. - It does not matter if this is local or remote. - Collects all python files, the Fortran source and makefiles, + It does not matter whether this is local or remote. + Collects all Python files, the Fortran source and makefiles, the ECMWF_ENV file, the CONTROL files as well as the template files. @@ -496,8 +496,8 @@ def mk_compilejob(makefile, ecuid, ecgid, fp_root): Parameters ---------- makefile : str - Name of the makefile which should be used to compile FORTRAN - CONVERT2 program. + Name of the makefile which should be used to compile the Fortran + program. ecuid : str The user id on ECMWF server. @@ -685,17 +685,17 @@ def mk_convert_build(src_path, makefile): if p.returncode != 0: print(perr.decode()) print('Please edit ' + makefile + - ' or try another Makefile in the src directory.') + ' or try another makefile in the src directory.') print('Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB ' 'and EMOSLIB must be adapted.') - print('Available Makefiles:') - print(UioFiles(src_path, 'Makefile*')) + print('Available makefiles:') + print(UioFiles(src_path, 'makefile*')) sys.exit('Compilation failed!') except ValueError as e: - print('ERROR: Makefile call failed:') + print('ERROR: make of Fortran code failed:') print(e) else: - execute_subprocess(['ls', '-l', + execute_subprocess(['ls', '-l', os.path.join(src_path, _config.FORTRAN_EXECUTABLE)], error_msg='FORTRAN EXECUTABLE COULD NOT BE FOUND!')