diff --git a/Source/Python/_config.py b/Source/Python/_config.py index 53ced99367e0fe8aadcf9e3054ff50c8957c77c7..b7d3741fb361c867f6c914314a1ae82f1b4557cf 100644 --- a/Source/Python/_config.py +++ b/Source/Python/_config.py @@ -87,9 +87,9 @@ FILE_MARS_REQUESTS = 'mars_requests.csv' FORTRAN_EXECUTABLE = 'calc_etadot' TEMPFILE_USER_ENVVARS = 'ECMWF_ENV.template' FILE_USER_ENVVARS = 'ECMWF_ENV' -TEMPFILE_INSTALL_COMPILEJOB = 'installscript.template.bologna' +TEMPFILE_INSTALL_COMPILEJOB = 'installscript.template' FILE_INSTALL_COMPILEJOB = 'compilejob.ksh' -TEMPFILE_INSTALL_JOB = 'jobscript.template.bologna' +TEMPFILE_INSTALL_JOB = 'jobscript.template' TEMPFILE_JOB = 'submitscript.template' FILE_JOB_OD = 'job.ksh' FILE_JOB_OP = 'jopoper.ksh' diff --git a/Templates/installscript.template b/Templates/installscript.template deleted file mode 100644 index ac699b8d60bfb238c8678aa1dd61f94d22f5eb44..0000000000000000000000000000000000000000 --- a/Templates/installscript.template +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/ksh - -# ON ECGB: -# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server -# start with sbatch NAME_OF_THIS_FILE directly on machine - -#SBATCH --workdir=/scratch/ms/$usergroup/$username -#SBATCH --qos=normal -#SBATCH --job-name=flex_ecmwf -#SBATCH --output=flex_ecmwf.%j.out -#SBATCH --error=flex_ecmwf.%j.out -#SBATCH --mail-type=FAIL -#SBATCH --time=12:00:00 - -## CRAY specific batch requests -##PBS -N flex_ecmwf -##PBS -q ns -##PBS -S /usr/bin/ksh -##PBS -o /scratch/ms/$usergroup/$username/flex_ecmwf.$${Jobname}.$${Job_ID}.out -# job output is in .ecaccess_DO_NOT_REMOVE -##PBS -j oe -##PBS -V -##PBS -l EC_threads_per_task=1 -##PBS -l EC_memory_per_task=3200MB - -set -x -export VERSION=$version_number -case $${HOST} in - *ecg*) - module unload grib_api - module unload emos - module load python3 - module load eccodes - module load emos/455-r64 - export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts - export MAKEFILE=$makefile - ;; - *cca*) - module switch PrgEnv-cray PrgEnv-intel - module load python3 - module load eccodes - module load emos/455-r64 - echo $${GROUP} - echo $${HOME} - echo $${HOME} | awk -F / '{print $1, $2, $3, $4}' - export GROUP=`echo $${HOME} | awk -F / '{print $4}'` - export SCRATCH=/scratch/ms/$${GROUP}/$${USER} - export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts - export MAKEFILE=$makefile - ;; -esac - -mkdir -p $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION} -cd $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION} # if FLEXPART_ROOT is not set this means cd to the home directory -tar -xvf $${HOME}/flex_extract_v$${VERSION}.tar -cd Source/Fortran -\rm *.o *.mod $fortran_program -make -f $${MAKEFILE} >flexcompile 2>flexcompile - -ls -l $fortran_program >>flexcompile -if [ $$? -eq 0 ]; then - echo 'SUCCESS!' >>flexcompile - mail -s flexcompile.$${HOST}.$$$$ $${USER} <flexcompile -else - echo Environment: >>flexcompile - env >> flexcompile - mail -s "ERROR! flexcompile.$${HOST}.$$$$" $${USER} <flexcompile -fi diff --git a/Templates/installscript.template b/Templates/installscript.template new file mode 120000 index 0000000000000000000000000000000000000000..f08f1b749e77e6f513cf2045056aa681d0365903 --- /dev/null +++ b/Templates/installscript.template @@ -0,0 +1 @@ +installscript.template.bologna \ No newline at end of file diff --git a/Templates/installscript.template.bologna b/Templates/installscript.template.bologna new file mode 100644 index 0000000000000000000000000000000000000000..13c9b88b6e254b8f61170262471e5343a4edac95 --- /dev/null +++ b/Templates/installscript.template.bologna @@ -0,0 +1,67 @@ +#!/bin/ksh + +# ON ECS or HPC servers: +# start with ecaccess-job-submit -queueName <QUEUENAME> <NAME_OF_THIS_FILE> on gateway server +# start with srun <NAME_OF_THIS_FILE> directly on machine + +#SBATCH --chdir=$$SCRATCH +#SBATCH --qos=el +#SBATCH --job-name=flex_extract +#SBATCH --output=flex_extract.%j.out +#SBATCH --error=flex_error.%j.out +#SBATCH --mail-type=FAIL +#SBATCH --mail-user=$username + +## CRAY specific batch requests +##PBS -N flex_ecmwf +##PBS -q ns +##PBS -S /usr/bin/ksh +##PBS -o $$SCRATCH/flex_extract.$${Jobname}.$${Job_ID}.out +# job output is in .ecaccess_DO_NOT_REMOVE +##PBS -j oe +##PBS -V +##PBS -l EC_threads_per_task=1 +##PBS -l EC_memory_per_task=3200MB + +set -x +export VERSION=$version_number +case $${EC_CLUSTER} in + *ecs*) +# module switch prgenv/gnu + module load ecmwf-toolbox + module load python3 + module load ecaccess + export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts + export MAKEFILE=$makefile + ;; + *hpc*) +# module switch PrgEnv-cray PrgEnv-intel + module load python3 + module load ecmwf-toolbox + module load ecaccess + echo $${GROUP} + echo $${HOME} + echo $${HOME} | awk -F / '{print $1, $2, $3, $4}' + export GROUP=`echo $${HOME} | awk -F / '{print $4}'` + export SCRATCH=$${SCRATCH} + export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts + export MAKEFILE=$makefile + ;; +esac + +mkdir -p $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION} +cd $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION} # if FLEXPART_ROOT is not set this means cd to the home directory +tar -xvf $${HOME}/flex_extract_v$${VERSION}.tar +cd Source/Fortran +\rm *.o *.mod $fortran_program +make -f $${MAKEFILE} >flexcompile 2>flexcompile + +ls -l $fortran_program >>flexcompile +if [ $$? -eq 0 ]; then + echo 'SUCCESS!' >>flexcompile + mail -s flexcompile.$${HOST}.$$$$ $${USER} <flexcompile +else + echo Environment: >>flexcompile + env >> flexcompile + mail -s "ERROR! flexcompile.$${HOST}.$$$$" $${USER} <flexcompile +fi diff --git a/Templates/installscript.template.reading b/Templates/installscript.template.reading new file mode 100644 index 0000000000000000000000000000000000000000..ac699b8d60bfb238c8678aa1dd61f94d22f5eb44 --- /dev/null +++ b/Templates/installscript.template.reading @@ -0,0 +1,68 @@ +#!/bin/ksh + +# ON ECGB: +# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server +# start with sbatch NAME_OF_THIS_FILE directly on machine + +#SBATCH --workdir=/scratch/ms/$usergroup/$username +#SBATCH --qos=normal +#SBATCH --job-name=flex_ecmwf +#SBATCH --output=flex_ecmwf.%j.out +#SBATCH --error=flex_ecmwf.%j.out +#SBATCH --mail-type=FAIL +#SBATCH --time=12:00:00 + +## CRAY specific batch requests +##PBS -N flex_ecmwf +##PBS -q ns +##PBS -S /usr/bin/ksh +##PBS -o /scratch/ms/$usergroup/$username/flex_ecmwf.$${Jobname}.$${Job_ID}.out +# job output is in .ecaccess_DO_NOT_REMOVE +##PBS -j oe +##PBS -V +##PBS -l EC_threads_per_task=1 +##PBS -l EC_memory_per_task=3200MB + +set -x +export VERSION=$version_number +case $${HOST} in + *ecg*) + module unload grib_api + module unload emos + module load python3 + module load eccodes + module load emos/455-r64 + export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts + export MAKEFILE=$makefile + ;; + *cca*) + module switch PrgEnv-cray PrgEnv-intel + module load python3 + module load eccodes + module load emos/455-r64 + echo $${GROUP} + echo $${HOME} + echo $${HOME} | awk -F / '{print $1, $2, $3, $4}' + export GROUP=`echo $${HOME} | awk -F / '{print $4}'` + export SCRATCH=/scratch/ms/$${GROUP}/$${USER} + export FLEXPART_ROOT_SCRIPTS=$fp_root_scripts + export MAKEFILE=$makefile + ;; +esac + +mkdir -p $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION} +cd $${FLEXPART_ROOT_SCRIPTS}/flex_extract_v$${VERSION} # if FLEXPART_ROOT is not set this means cd to the home directory +tar -xvf $${HOME}/flex_extract_v$${VERSION}.tar +cd Source/Fortran +\rm *.o *.mod $fortran_program +make -f $${MAKEFILE} >flexcompile 2>flexcompile + +ls -l $fortran_program >>flexcompile +if [ $$? -eq 0 ]; then + echo 'SUCCESS!' >>flexcompile + mail -s flexcompile.$${HOST}.$$$$ $${USER} <flexcompile +else + echo Environment: >>flexcompile + env >> flexcompile + mail -s "ERROR! flexcompile.$${HOST}.$$$$" $${USER} <flexcompile +fi diff --git a/Templates/jobscript.template b/Templates/jobscript.template deleted file mode 100644 index 7cd4873ed9ccd2edaa963499226cdda663f342a8..0000000000000000000000000000000000000000 --- a/Templates/jobscript.template +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/ksh - -# ON ECGB: -# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server -# start with sbatch NAME_OF_THIS_FILE directly on machine - -#SBATCH --workdir=/scratch/ms/$usergroup/$username -#SBATCH --qos=normal -#SBATCH --job-name=flex_ecmwf -#SBATCH --output=flex_ecmwf.%j.out -#SBATCH --error=flex_ecmwf.%j.out -#SBATCH --mail-type=FAIL -#SBATCH --time=12:00:00 - -## CRAY specific batch requests -##PBS -N flex_ecmwf -##PBS -q np -##PBS -S /usr/bin/ksh -## -o /scratch/ms/$usergroup/$username/flex_ecmwf.$$$${PBS_JOBID}.out -## job output is in .ecaccess_DO_NOT_REMOVE -##PBS -j oe -##PBS -V -##PBS -l EC_threads_per_task=24 -##PBS -l EC_memory_per_task=32000MB - -set -x -export VERSION=$version_number -case $$$${HOST} in - *ecg*) - module unload grib_api - module unload emos - module load python3 - module load eccodes - module load emos/455-r64 - export PATH=$$$${PATH}:$fp_root_path - ;; - *cca*) - module switch PrgEnv-cray PrgEnv-intel - module load python3 - module load eccodes - module load emos/455-r64 - export SCRATCH=$$$${TMPDIR} - export PATH=$$$${PATH}:$fp_root_path - ;; -esac - -cd $$$${SCRATCH} -mkdir -p extract$$$$$$$$ -cd extract$$$$$$$$ - -export CONTROL=CONTROL - -cat >$$$${CONTROL}<<EOF -$$control_content -EOF - - -submit.py --controlfile=$$$${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1 - -if [ $? -eq 0 ] ; then - l=0 - for muser in `grep -i MAILOPS $$$${CONTROL}`; do - if [ $$$${l} -gt 0 ] ; then - mail -s flex.$$$${HOST}.$$$$$$$$ $$$${muser} <prot - fi - l=$(($$$${l}+1)) - done -else - l=0 - for muser in `grep -i MAILFAIL $$$${CONTROL}`; do - if [ $$$${l} -gt 0 ] ; then - mail -s "ERROR! flex.$$$${HOST}.$$$$$$$$" $$$${muser} <prot - fi - l=$(($$$${l}+1)) - done -fi - diff --git a/Templates/jobscript.template b/Templates/jobscript.template new file mode 120000 index 0000000000000000000000000000000000000000..d83deb3c3b9e54f5b8a2ebee4050fa544bf7237e --- /dev/null +++ b/Templates/jobscript.template @@ -0,0 +1 @@ +jobscript.template.reading \ No newline at end of file diff --git a/Templates/jobscript.template.bologna b/Templates/jobscript.template.bologna new file mode 100644 index 0000000000000000000000000000000000000000..8e613371ab5084b72f2238e64337df5b54a52833 --- /dev/null +++ b/Templates/jobscript.template.bologna @@ -0,0 +1,76 @@ +#!/bin/ksh + +# ON ECS or HPC: +# start with ecaccess-job-submit -queueName <QUEUE_NAME> <NAME_OF_THIS_FILE> on gateway server +# start with srun <NAME_OF_THIS_FILE> directly on machine + +#SBATCH --chdir=$$$$SCRATCH +#SBATCH --qos=el +#SBATCH --job-name=flex_extract +#SBATCH --output=flex_extract.%j.out +#SBATCH --error=flex_error.%j.out +#SBATCH --mail-type=FAIL +#SBATCH --mail-user=$username +#SBATCH --time=24:00:00 + +## CRAY specific batch requests +##PBS -N flex_extract +##PBS -q np +##PBS -S /usr/bin/ksh +## -o $$$${SCRATCH}/flex_ecmwf.$$$${PBS_JOBID}.out +## job output is in .ecaccess_DO_NOT_REMOVE +##PBS -j oe +##PBS -V +##PBS -l EC_threads_per_task=24 +##PBS -l EC_memory_per_task=32000MB + +set -x +export VERSION=$version_number +case $$$${EC_CLUSTER} in + *ecs*) + module load python3 + module load ecmwf-toolbox + module load ecaccess + export PATH=$$$${PATH}:$fp_root_path + ;; + *hpc*) +# module switch PrgEnv-cray PrgEnv-intel + module load python3 + module load ecmwf-toolbox + module load ecaccess + export SCRATCH=$$$${TMPDIR} + export PATH=$$$${PATH}:$fp_root_path + ;; +esac + +cd $$$${SCRATCH} +mkdir -p extract$$$$$$$$ +cd extract$$$$$$$$ + +export CONTROL=CONTROL + +cat >$$$${CONTROL}<<EOF +$$control_content +EOF + + +submit.py --controlfile=$$$${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1 + +if [ $? -eq 0 ] ; then + l=0 + for muser in `grep -i MAILOPS $$$${CONTROL}`; do + if [ $$$${l} -gt 0 ] ; then + mail -s flex.$$$${HOST}.$$$$$$$$ $$$${muser} <prot + fi + l=$(($$$${l}+1)) + done +else + l=0 + for muser in `grep -i MAILFAIL $$$${CONTROL}`; do + if [ $$$${l} -gt 0 ] ; then + mail -s "ERROR! flex.$$$${HOST}.$$$$$$$$" $$$${muser} <prot + fi + l=$(($$$${l}+1)) + done +fi + diff --git a/Templates/jobscript.template.reading b/Templates/jobscript.template.reading new file mode 100644 index 0000000000000000000000000000000000000000..7cd4873ed9ccd2edaa963499226cdda663f342a8 --- /dev/null +++ b/Templates/jobscript.template.reading @@ -0,0 +1,77 @@ +#!/bin/ksh + +# ON ECGB: +# start with ecaccess-job-submit -queueName ecgb NAME_OF_THIS_FILE on gateway server +# start with sbatch NAME_OF_THIS_FILE directly on machine + +#SBATCH --workdir=/scratch/ms/$usergroup/$username +#SBATCH --qos=normal +#SBATCH --job-name=flex_ecmwf +#SBATCH --output=flex_ecmwf.%j.out +#SBATCH --error=flex_ecmwf.%j.out +#SBATCH --mail-type=FAIL +#SBATCH --time=12:00:00 + +## CRAY specific batch requests +##PBS -N flex_ecmwf +##PBS -q np +##PBS -S /usr/bin/ksh +## -o /scratch/ms/$usergroup/$username/flex_ecmwf.$$$${PBS_JOBID}.out +## job output is in .ecaccess_DO_NOT_REMOVE +##PBS -j oe +##PBS -V +##PBS -l EC_threads_per_task=24 +##PBS -l EC_memory_per_task=32000MB + +set -x +export VERSION=$version_number +case $$$${HOST} in + *ecg*) + module unload grib_api + module unload emos + module load python3 + module load eccodes + module load emos/455-r64 + export PATH=$$$${PATH}:$fp_root_path + ;; + *cca*) + module switch PrgEnv-cray PrgEnv-intel + module load python3 + module load eccodes + module load emos/455-r64 + export SCRATCH=$$$${TMPDIR} + export PATH=$$$${PATH}:$fp_root_path + ;; +esac + +cd $$$${SCRATCH} +mkdir -p extract$$$$$$$$ +cd extract$$$$$$$$ + +export CONTROL=CONTROL + +cat >$$$${CONTROL}<<EOF +$$control_content +EOF + + +submit.py --controlfile=$$$${CONTROL} --inputdir=./work --outputdir=./work 1> prot 2>&1 + +if [ $? -eq 0 ] ; then + l=0 + for muser in `grep -i MAILOPS $$$${CONTROL}`; do + if [ $$$${l} -gt 0 ] ; then + mail -s flex.$$$${HOST}.$$$$$$$$ $$$${muser} <prot + fi + l=$(($$$${l}+1)) + done +else + l=0 + for muser in `grep -i MAILFAIL $$$${CONTROL}`; do + if [ $$$${l} -gt 0 ] ; then + mail -s "ERROR! flex.$$$${HOST}.$$$$$$$$" $$$${muser} <prot + fi + l=$(($$$${l}+1)) + done +fi +