implemented a job split with a new parameter 'job_chunk' so that huge time...

implemented a job split with a new parameter 'job_chunk' so that huge time periods can automatically be splitted

source/python/classes/ControlFile.py

@@ -67,7 +67,7 @@ from mods.checks import (check_grid, check_area, check_levels, check_purefc,
                          check_basetime, check_public, check_acctype,
                          check_acctime, check_accmaxstep, check_time,
                          check_logicals_type, check_len_type_time_step,
-                         check_addpar)
+                         check_addpar, check_job_chunk)
 
 # ------------------------------------------------------------------------------
 # CLASS
@@ -382,6 +382,7 @@ class ControlFile(object):
 
         self.addpar = check_addpar(self.addpar)
 
+        self.job_chunk = check_job_chunk(self.job_chunk)
 
         return
 

source/python/mods/checks.py

@@ -795,3 +795,26 @@ def check_addpar(addpar):
 
     return addpar
 
+
+def check_job_chunk(job_chunk):
+    '''Checks that the job chunk number is positive and non zero.
+
+    Parameters
+    ----------
+    job_chunk : :obj:`integer`
+        The number of days for a single job script.
+
+    Return
+    ------
+    job_chunk : :obj:`integer`
+        The number of days for a single job script.
+    '''
+    if job_chunk < 0:
+        raise ValueError('ERROR: The number of job chunk is negative!\n'
+                         'It has to be a positive number!')
+    elif job_chunk == 0:
+        job_chunk = None
+    else:
+        pass
+
+    return job_chunk

source/python/mods/tools.py

@@ -127,6 +127,9 @@ def get_cmdline_args():
     parser.add_argument("--date_chunk", dest="date_chunk",
                         type=none_or_int, default=None,
                         help="# of days to be retrieved at once")
+    parser.add_argument("--job_chunk", dest="job_chunk",
+                        type=none_or_int, default=None,
+                        help="# of days to be retrieved within a single job")
     parser.add_argument("--controlfile", dest="controlfile",
                         type=none_or_str, default='CONTROL.temp',
                         help="file with CONTROL parameters")

source/python/submit.py

@@ -47,6 +47,7 @@ import sys
 import subprocess
 import inspect
 import collections
+from datetime import datetime, timedelta
 
 # software specific classes and modules from flex_extract
 import _config
@@ -139,10 +140,38 @@ def submit(jtemplate, c, queue):
         job_file = os.path.join(_config.PATH_JOBSCRIPTS,
                                 jtemplate[:-5] + '.ksh')
 
+        # divide time periode into specified number of job chunks
+        # to have multiple job scripts
+        if c.job_chunk:
+            start = datetime.strptime(c.start_date, '%Y%m%d')
+            end = datetime.strptime(c.end_date, '%Y%m%d')
+            chunk = timedelta(days=c.job_chunk)
+
+            while start <= end:
+                if (start + chunk) <= end:
+                    c.end_date = (start + chunk).strftime("%Y%m%d")
+                else:
+                    c.end_date = end.strftime("%Y%m%d")
+                print c.start_date +' bis ' + c.end_date
+
                 clist = c.to_list()
 
                 mk_jobscript(jtemplate, job_file, clist)
 
+                job_id = submit_job_to_ecserver(queue, job_file)
+                print('The job id is: ' + str(job_id.strip()))
+
+                start = start + chunk
+                c.start_date = start.strftime("%Y%m%d")
+        # submit a single job script
+        else:
+            clist = c.to_list()
+
+            mk_jobscript(jtemplate, job_file, clist)
+
+            job_id = submit_job_to_ecserver(queue, job_file)
+            print('The job id is: ' + str(job_id.strip()))
+
     else:
     # --------- create operational job script ----------------------------------
         print('---- Operational mode! ----')
@@ -160,10 +189,11 @@ def submit(jtemplate, c, queue):
 
         mk_jobscript(jtemplate, job_file, clist)
 
-    # --------- submit the job_script to the ECMWF server
         job_id = submit_job_to_ecserver(queue, job_file)
         print('The job id is: ' + str(job_id.strip()))
-    print('You should get an email with subject flex.hostname.pid')
+
+
+    print('You should get an email per job with subject flex.hostname.pid')
 
     return
 
@@ -222,5 +252,6 @@ def mk_jobscript(jtemplate, job_file, clist):
 
     return
 
+
 if __name__ == "__main__":
     main()