From ff2a11c3d0b3be6186a11e10b8ec5e7e4b9faa77 Mon Sep 17 00:00:00 2001 From: Anne Philipp <anne.philipp@univie.ac.at> Date: Thu, 29 Nov 2018 22:05:05 +0100 Subject: [PATCH] changed generation of final job_scripts by using genshi templates --- source/python/classes/ControlFile.py | 4 +- source/python/install.py | 4 ++ source/python/submit.py | 93 ++++++++++++++++++++-------- 3 files changed, 72 insertions(+), 29 deletions(-) diff --git a/source/python/classes/ControlFile.py b/source/python/classes/ControlFile.py index 26fc932..53c673c 100644 --- a/source/python/classes/ControlFile.py +++ b/source/python/classes/ControlFile.py @@ -540,9 +540,9 @@ class ControlFile(object): for s in item[1]: stot += s + ' ' - l.append("%s %s" % (item[0], stot)) + l.append("%s %s\n" % (item[0], stot)) else: - l.append("%s %s" % item) + l.append("%s %s\n" % item) return sorted(l) diff --git a/source/python/install.py b/source/python/install.py index 4b7f74c..9b084b5 100755 --- a/source/python/install.py +++ b/source/python/install.py @@ -566,6 +566,10 @@ def mk_job_template(ecuid, ecgid, gateway, destination, fp_root): fp_root_path_to_python = os.path.join(fp_root, _config.FLEXEXTRACT_DIRNAME, _config.PATH_REL_PYTHON_SRC) + if '$' in fp_root_path_to_python: + ind = fp_root_path_to_python.index('$') + fp_root_path_to_python = fp_root_path_to_python[0:ind] + '$' + \ + fp_root_path_to_python[ind:] try: loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False) diff --git a/source/python/submit.py b/source/python/submit.py index 7c48213..fe4c990 100755 --- a/source/python/submit.py +++ b/source/python/submit.py @@ -106,7 +106,7 @@ def main(): return def submit(jtemplate, c, queue): - '''Prepares the job script and submit it to the specified queue. + '''Prepares the job script and submits it to the specified queue. Parameters ---------- @@ -129,59 +129,98 @@ def submit(jtemplate, c, queue): ''' - # read template file and get index for CONTROL input - with open(os.path.join(_config.PATH_TEMPLATES, jtemplate)) as f: - lftext = f.read().split('\n') - insert_point = lftext.index('EOF') - if not c.basetime: # --------- create on demand job script ------------------------------------ if c.maxstep > 24: print('---- Pure forecast mode! ----') else: print('---- On-demand mode! ----') - job_file = os.path.join(_config.PATH_JOBSCRIPTS, - jtemplate[:-4] + 'ksh') - clist = c.to_list() - lftextondemand = lftext[:insert_point] + clist + lftext[insert_point:] + job_file = os.path.join(_config.PATH_JOBSCRIPTS, + jtemplate[:-5] + '.ksh') - with open(job_file, 'w') as f: - f.write('\n'.join(lftextondemand)) + clist = c.to_list() - job_id = submit_job_to_ecserver(queue, job_file) + mk_jobscript(jtemplate, job_file, clist) else: # --------- create operational job script ---------------------------------- print('---- Operational mode! ----') + job_file = os.path.join(_config.PATH_JOBSCRIPTS, jtemplate[:-5] + 'oper.ksh') - if c.maxstep: - mt = int(c.maxstep) - else: - mt = 0 - c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' c.base_time = '${MSJ_BASETIME}' - if mt > 24: + if c.maxstep > 24: c.time = '${MSJ_BASETIME} {MSJ_BASETIME}' - colist = c.to_list() - - lftextoper = lftext[:insert_point] + colist + lftext[insert_point + 2:] - - with open(job_file, 'w') as f: - f.write('\n'.join(lftextoper)) + clist = c.to_list() - job_id = submit_job_to_ecserver(queue, job_file) + mk_jobscript(jtemplate, job_file, clist) - # -------------------------------------------------------------------------- + # --------- submit the job_script to the ECMWF server + job_id = submit_job_to_ecserver(queue, job_file) print('The job id is: ' + str(job_id.strip())) print('You should get an email with subject flex.hostname.pid') return +def mk_jobscript(jtemplate, job_file, clist): + '''Creates the job script from template. + + Parameters + ---------- + jtemplate : :obj:`string` + Job template file from sub-directory "_templates" for + submission to ECMWF. It contains all necessary + module and variable settings for the ECMWF environment as well as + the job call and mail report instructions. + Default is "job.temp". + + job_file : :obj:`string` + Path to the job script file. + + clist : :obj:`list` of :obj:`string` + Contains all necessary parameters for ECMWF CONTROL file. + + Return + ------ + + ''' + from genshi.template.text import NewTextTemplate + from genshi.template import TemplateLoader + from genshi.template.eval import UndefinedError + + # load template and insert control content as list + try: + loader = TemplateLoader(_config.PATH_TEMPLATES, auto_reload=False) + control_template = loader.load(jtemplate, + cls=NewTextTemplate) + + stream = control_template.generate(control_content=clist) + except UndefinedError as e: + print('... ERROR ' + str(e)) + + sys.exit('\n... error occured while trying to generate jobscript') + except OSError as e: + print('... ERROR CODE: ' + str(e.errno)) + print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) + + sys.exit('\n... error occured while trying to generate jobscript') + + # create jobscript file + try: + with open(job_file, 'w') as f: + f.write(stream.render('text')) + except OSError as e: + print('... ERROR CODE: ' + str(e.errno)) + print('... ERROR MESSAGE:\n \t ' + str(e.strerror)) + + sys.exit('\n... error occured while trying to write ' + job_file) + + return + if __name__ == "__main__": main() -- GitLab