diff --git a/Scripts_for_postprocessing/.gitkeep b/Scripts_for_postprocessing/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/Scripts_for_postprocessing/copy_ml_atm_2km.sh b/Scripts_for_postprocessing/copy_ml_atm_2km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..d0206b742f21e503cc2a2f2aa5e555f7479bfa37
--- /dev/null
+++ b/Scripts_for_postprocessing/copy_ml_atm_2km.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=copy_atm_2km
+#SBATCH --partition=compute
+#SBATCH --nodes=1
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/work/bb1152/Module_A/A6_CyclEx/b380782_Christoph/pp_logs/LOG.copy_atm_2km.%j.o
+#SBATCH --error=/work/bb1152/Module_A/A6_CyclEx/b380782_Christoph/pp_logs/LOG.copy_atm_2km.%j.o
+#SBATCH --time=02:00:00
+#=========================================================================================
+# this script is for copying data sets from scratch to work
+
+# define directories
+exp=channel_2km_0006
+inpath=/scratch/b/b380906 # directory with data that will be copied
+projpath=/work/bb1152/Module_A/A6_CyclEx # directory of project on work
+cppath=sim_data/production # directory that data will be copied to
+
+simpath=$inpath/$exp
+echo $simpath
+
+outpath=$projpath/$cppath/$exp
+echo $outpath
+
+# create folder in which the data will be stored
+mkdir -p $outpath
+cd $outpath
+
+cp $simpath/icon-extra_ml.nc $outpath
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+cp $simpath/*atm2d*${day}T${hour}0000Z.nc $outpath
+cp $simpath/*atm3d*${day}T${hour}0000Z.nc $outpath
+cp $simpath/*ddt3d*${day}T${hour}0000Z.nc $outpath
+
+done
+done
+
+day=10
+hour=00
+
+cp $simpath/*atm2d*${day}T${hour}0000Z.nc $outpath
+cp $simpath/*atm3d*${day}T${hour}0000Z.nc $outpath
+cp $simpath/*ddt3d*${day}T${hour}0000Z.nc $outpath
diff --git a/Scripts_for_postprocessing/copy_ml_atm_80km.sh b/Scripts_for_postprocessing/copy_ml_atm_80km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6a054067df884e5c8bd7118f33478090180f9ae5
--- /dev/null
+++ b/Scripts_for_postprocessing/copy_ml_atm_80km.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=copy_atm_80km
+#SBATCH --partition=compute
+#SBATCH --nodes=1
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/work/bb1152/Module_A/A6_CyclEx/b380782_Christoph/pp_logs/LOG.copy_atm_80km.%j.o
+#SBATCH --error=/work/bb1152/Module_A/A6_CyclEx/b380782_Christoph/pp_logs/LOG.copy_atm_80km.%j.o
+#SBATCH --time=01:00:00
+#=========================================================================================
+# this script is for copying data sets from scratch to work
+
+# define directories
+exp=channel_80km_0001
+inpath=/scratch/b/b380906 # directory with data that will be copied
+projpath=/work/bb1152/Module_A/A6_CyclEx # directory of project on work
+cppath=sim_data/production # directory that data will be copied to
+
+simpath=$inpath/$exp
+echo $simpath
+
+outpath=$projpath/$cppath/$exp
+echo $outpath
+
+# create folder in which the data will be stored
+mkdir -p $outpath
+cd $outpath
+
+# cp $simpath/icon-extra_ml.nc $outpath
+cp $simpath/*2d* $outpath
+# cp $simpath/*3d*.nc $outpath
diff --git a/Scripts_for_postprocessing/grid_info_025x025.txt b/Scripts_for_postprocessing/grid_info_025x025.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d31bd4ad2e1b590e015c4cb0c1556f7b0356c013
--- /dev/null
+++ b/Scripts_for_postprocessing/grid_info_025x025.txt
@@ -0,0 +1,17 @@
+#
+# gridID 1
+#
+gridtype = lonlat
+gridsize = 65484
+xsize = 204
+ysize = 321
+xname = lon
+xlongname = "longitude"
+xunits = "degrees_east"
+yname = lat
+ylongname = "latitude"
+yunits = "degrees_north"
+xfirst = 12.5
+xinc = 0.25
+yfirst = 4.5
+yinc = 0.25
diff --git a/Scripts_for_postprocessing/grid_info_0p05x0p05.txt b/Scripts_for_postprocessing/grid_info_0p05x0p05.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d7e6e4061b984cb282072428fe41bc65d8ac38ce
--- /dev/null
+++ b/Scripts_for_postprocessing/grid_info_0p05x0p05.txt
@@ -0,0 +1,17 @@
+#
+# gridID 1
+#
+gridtype = lonlat
+gridsize = 1637100
+xsize = 1020
+ysize = 1605
+xname = lon
+xlongname = "longitude"
+xunits = "degrees_east"
+yname = lat
+ylongname = "latitude"
+yunits = "degrees_north"
+xfirst = 12.5
+xinc = 0.05
+yfirst = 4.5
+yinc = 0.05
diff --git a/Scripts_for_postprocessing/grid_info_1x1.txt b/Scripts_for_postprocessing/grid_info_1x1.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8719a3920857b7fe931f583b5bae0b2e1cd6f46d
--- /dev/null
+++ b/Scripts_for_postprocessing/grid_info_1x1.txt
@@ -0,0 +1,17 @@
+#
+# gridID 1
+#
+gridtype = lonlat
+gridsize = 4131
+xsize = 51
+ysize = 81
+xname = lon
+xlongname = "longitude"
+xunits = "degrees_east"
+yname = lat
+ylongname = "latitude"
+yunits = "degrees_north"
+xfirst = 12.5
+xinc = 1
+yfirst = 4.5
+yinc = 1
diff --git a/Scripts_for_postprocessing/grid_info_2km.txt b/Scripts_for_postprocessing/grid_info_2km.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ad33c65a6ff6296a965a74e862dfdb700dda8d74
--- /dev/null
+++ b/Scripts_for_postprocessing/grid_info_2km.txt
@@ -0,0 +1,17 @@
+#
+# gridID 1
+#
+gridtype = lonlat
+gridsize = 5762016
+xsize = 2106
+ysize = 2736
+xname = lon
+xlongname = "longitude"
+xunits = "degrees_east"
+yname = lat
+ylongname = "latitude"
+yunits = "degrees_north"
+xfirst = 12.5
+xinc = 0.024216524216524215
+yfirst = 4.5
+yinc = 0.029605263157894735
diff --git a/Scripts_for_postprocessing/grid_info_2km_to_1x1.txt b/Scripts_for_postprocessing/grid_info_2km_to_1x1.txt
new file mode 100644
index 0000000000000000000000000000000000000000..fa47a2de4b997b63b3e57690c7e20bc0c9d5fb68
--- /dev/null
+++ b/Scripts_for_postprocessing/grid_info_2km_to_1x1.txt
@@ -0,0 +1,17 @@
+#
+# gridID 1
+#
+gridtype = lonlat
+gridsize = 4131
+xsize = 51
+ysize = 81
+xname = lon
+xlongname = "longitude"
+xunits = "degrees_east"
+yname = lat
+ylongname = "latitude"
+yunits = "degrees_north"
+xfirst = 12.987891737891738
+xinc = 1
+yfirst = 4.5
+yinc = 1
diff --git a/Scripts_for_postprocessing/grid_info_80km.txt b/Scripts_for_postprocessing/grid_info_80km.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8bc180854a05b2f46768a98fc1bade4416c55b05
--- /dev/null
+++ b/Scripts_for_postprocessing/grid_info_80km.txt
@@ -0,0 +1,18 @@
+#
+# gridID 1
+#
+gridtype = lonlat
+gridsize = 5676
+xsize = 66
+ysize = 86
+xname = lon
+xlongname = "longitude"
+xunits = "degrees_east"
+yname = lat
+ylongname = "latitude"
+yunits = "degrees_north"
+xfirst = 12.5
+xinc = 0.7727272727272727
+yfirst = 4.5
+yinc = 0.9418604651162791
+
diff --git a/Scripts_for_postprocessing/grid_info_80km_to_1x1.txt b/Scripts_for_postprocessing/grid_info_80km_to_1x1.txt
new file mode 100644
index 0000000000000000000000000000000000000000..602611c87133a4c90e19664c5bc677bfe7ed8d99
--- /dev/null
+++ b/Scripts_for_postprocessing/grid_info_80km_to_1x1.txt
@@ -0,0 +1,17 @@
+#
+# gridID 1
+#
+gridtype = lonlat
+gridsize = 4131
+xsize = 51
+ysize = 81
+xname = lon
+xlongname = "longitude"
+xunits = "degrees_east"
+yname = lat
+ylongname = "latitude"
+yunits = "degrees_north"
+xfirst = 12.613636363636363
+xinc = 1
+yfirst = 4.5
+yinc = 1
diff --git a/Scripts_for_postprocessing/interpolate_mlpl_qv_u_v_850hPa_2km.sh b/Scripts_for_postprocessing/interpolate_mlpl_qv_u_v_850hPa_2km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..20b82e004126206d4c885675580edc40725c64da
--- /dev/null
+++ b/Scripts_for_postprocessing/interpolate_mlpl_qv_u_v_850hPa_2km.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=interpolate_mlpl_qv_u_v_850hPa_2km
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.interpolate_mlpl_qv_u_v_850hPa_2km.o%j
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.interpolate_mlpl_qv_u_v_850hPa_2km.e%j
+#SBATCH --time=02:00:00
+#=========================================================================================
+# this script is for interpolating the variables qv, u, and v from model levels to 850 hPa (staying on the native triangular grid)
+
+# folder with data that will be remapped
+exp=channel_2km_0006
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+projpath=/work/bb1152/Module_A/A6_CyclEx
+
+simpath=$datapath/$exp
+echo $simpath
+
+# folder in which the remapped data will be stored
+mkdir -p $simpath/interpol_850hPa_qv_u_v_native
+cd $simpath/interpol_850hPa_qv_u_v_native
+
+# define pressure levels that data will be remapped to
+plevout="85000"
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 1) select variables from the 3d dataset
+
+cdo -P 38 selname,pres,u,v,qv ../icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 2) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from 2d files
+cdo -P 38 selname,pres_sfc ../icon-atm2d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-3d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc icon-qv_u_v_850hPa_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 1) select variables from the 3d dataset
+
+cdo -P 38 selname,pres,u,v,qv ../icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 2) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from 2d files
+cdo -P 38 selname,pres_sfc ../icon-atm2d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-3d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc icon-qv_u_v_850hPa_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc
+
+#
diff --git a/Scripts_for_postprocessing/interpolate_mlpl_qv_u_v_850hPa_80km.sh b/Scripts_for_postprocessing/interpolate_mlpl_qv_u_v_850hPa_80km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..427de21f2a9253b7b6fe8498895bd0f10240d245
--- /dev/null
+++ b/Scripts_for_postprocessing/interpolate_mlpl_qv_u_v_850hPa_80km.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=interpolate_mlpl_qv_u_v_850hPa_80km
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.interpolate_mlpl_qv_u_v_850hPa_80km.o%j
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.interpolate_mlpl_qv_u_v_850hPa_80km.e%j
+#SBATCH --time=00:30:00
+#=========================================================================================
+# this script is for interpolating the variables qv, u, and v from model levels to 850 hPa (staying on the native triangular grid)
+
+# folder with data that will be remapped
+exp=channel_80km_0006
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+projpath=/work/bb1152/Module_A/A6_CyclEx
+
+simpath=$datapath/$exp
+echo $simpath
+
+# folder in which the remapped data will be stored
+mkdir -p $simpath/interpol_850hPa_qv_u_v_native
+cd $simpath/interpol_850hPa_qv_u_v_native
+
+# define pressure levels that data will be remapped to
+plevout="85000"
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 ; do
+
+# 1) select variables from the 3d dataset
+
+cdo -P 38 selname,pres,u,v,qv ../raw_data/icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 2) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from 2d files
+cdo -P 38 selname,pres_sfc ../raw_data/icon-atm2d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-3d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc icon-qv_u_v_850hPa_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 1) select variables from the 3d dataset
+
+cdo -P 38 selname,pres,u,v,qv ../raw_data/icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 2) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from 2d files
+cdo -P 38 selname,pres_sfc ../raw_data/icon-atm2d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-3d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc icon-qv_u_v_850hPa_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc pres_sfc_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.psfc.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_ml_atm2d_2km.sh b/Scripts_for_postprocessing/remap_ml_atm2d_2km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..defaafe650e403240a91a80bb9f173cbcd8852c2
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_ml_atm2d_2km.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_atm2d_2km
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_atm2d_2km.o%j
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_atm2d_2km.e%j
+#SBATCH --time=1:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_2km_0005
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/scratch/b/$username/check_remapping_on_PTE/$exp/
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_atm2d_latlon
+cd $outpath/remapped_atm2d_latlon
+
+#### Remapping
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_2km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_2km.txt $simpath/icon-extra_ml.nc remapweights_2km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+# remap all variables
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-atm2d_ML_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 0.25x0.25 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_2km_to_1x1.txt icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+# remap all variables
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-atm2d_ML_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_2km_to_1x1.txt icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+
+rm remapweights_2km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_ml_atm2d_2km_025x025deg.sh b/Scripts_for_postprocessing/remap_ml_atm2d_2km_025x025deg.sh
new file mode 100644
index 0000000000000000000000000000000000000000..44299d3b870d153f662525b6a17ad77b8d17aea8
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_ml_atm2d_2km_025x025deg.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_atm2d_2km_025x025
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_atm2d_2km_025x025.o%j
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_atm2d_2km_025x025.e%j
+#SBATCH --time=1:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_2km_0005
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/scratch/b/$username/check_remapping_on_PTE/$exp
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_atm2d_latlon_025x025
+cd $outpath/remapped_atm2d_latlon_025x025
+
+#### Remapping
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_2km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_2km.txt $simpath/icon-extra_ml.nc remapweights_2km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+# remap all variables
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-atm2d_ML_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 0.25x0.25 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_025x025.txt icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+# remap all variables
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-atm2d_ML_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_025x025.txt icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_2km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_ml_atm2d_2km_0p05x0p05deg.sh b/Scripts_for_postprocessing/remap_ml_atm2d_2km_0p05x0p05deg.sh
new file mode 100644
index 0000000000000000000000000000000000000000..97f74f420da9755ccdc7d26936ad79b1397d86e3
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_ml_atm2d_2km_0p05x0p05deg.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_atm2d_2km_0p05x0p05
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_atm2d_2km_0p05x0p05.o%j
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_atm2d_2km_0p05x0p05.e%j
+#SBATCH --time=4:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_2km_0004
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/scratch/b/$username/check_remapping_on_PTE/$exp
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_atm2d_latlon_0p05x0p05
+cd $outpath/remapped_atm2d_latlon_0p05x0p05
+
+#### Remapping
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_2km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_2km.txt $simpath/icon-extra_ml.nc remapweights_2km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+# remap all variables
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-atm2d_ML_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 0.05x0.05 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_0p05x0p05.txt icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+# remap all variables
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-atm2d_ML_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 0.05x0.05 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_0p05x0p05.txt icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_2km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_ml_atm2d_80km.sh b/Scripts_for_postprocessing/remap_ml_atm2d_80km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..cdd557bf5890e8c34bd5861dfbf4a2976ebe6cc6
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_ml_atm2d_80km.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_atm2d_80km
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_atm2d_80km.o%j
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_atm2d_80km.e%j
+#SBATCH --time=1:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_80km_0005
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp/raw_data
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/scratch/b/$username/check_remapping_on_PTE/$exp/
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_atm2d_latlon
+cd $outpath/remapped_atm2d_latlon
+
+#### Remapping
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_80km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_80km.txt $simpath/icon-extra_ml.nc remapweights_80km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+# remap all variables
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_80km.txt,remapweights_80km.nc $simpath/icon-atm2d_ML_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 0.25x0.25 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_1x1.txt icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+# remap all variables
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_80km.txt,remapweights_80km.nc $simpath/icon-atm2d_ML_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_1x1.txt icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm2d_ML_reg_202101${day}T${hour}0000Z.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_80km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_mlpl_atm3d_2km.sh b/Scripts_for_postprocessing/remap_mlpl_atm3d_2km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6d2452a1f8eaef00bb37c52c74f418d592187634
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_mlpl_atm3d_2km.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_mlpl_atm3d_2km
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/work/bb1152/Module_A/A6_CyclEx/b380782_Christoph/pp_logs/LOG.remap_mlpl_atm3d_2km.%j.o
+#SBATCH --error=/work/bb1152/Module_A/A6_CyclEx/b380782_Christoph/pp_logs/LOG.remap_mlpl_atm3d_2km.%j.o
+#SBATCH --time=04:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+# folder with data that will be remapped
+exp=channel_2km_0006
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+projpath=/work/bb1152/Module_A/A6_CyclEx
+
+simpath=$datapath/$exp
+echo $simpath
+
+# folder in which the remapped data will be stored
+mkdir -p $simpath/remapped_atm3d_latlon
+cd $simpath/remapped_atm3d_latlon
+
+# define pressure levels that data will be remapped to
+plevout="1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,14000,15000,16000,17000,18000,19000,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000"
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_2km.txt
+
+cdo -P 32 gennn,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_2km.txt $simpath/icon-extra_ml.nc remapweights_2km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) select variables from the 3d dataset
+
+cdo -P 38 selname,u,v,pres,w,omega,temp,vor,div,z_ifc,geopot,qv ../icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_2km.txt,remapweights_2km.nc icon-3d_ML_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 4) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_1x1.txt icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 5) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-atm3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) select variables from the 3d dataset
+
+cdo -P 38 selname,u,v,pres,w,omega,temp,vor,div,z_ifc,geopot,qv ../icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_2km.txt,remapweights_2km.nc icon-3d_ML_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 4) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_1x1.txt icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 5) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-atm3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_2km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_mlpl_atm3d_2km_025x025deg.sh b/Scripts_for_postprocessing/remap_mlpl_atm3d_2km_025x025deg.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ebdeb3094956bc80266f4b229f517c70c2baa8bd
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_mlpl_atm3d_2km_025x025deg.sh
@@ -0,0 +1,156 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_mlpl_atm3d_2km_025x025
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_atm3d_2km_025x025.%j.o
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_atm3d_2km_025x025.%j.o
+#SBATCH --time=04:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_2km_0005
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/scratch/b/$username/check_remapping_on_PTE/$exp
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_atm3d_latlon_025x025
+cd $outpath/remapped_atm3d_latlon_025x025
+
+#### Remapping
+
+# define pressure levels that data will be remapped to
+plevout="1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,14000,15000,16000,17000,18000,19000,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000"
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_2km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_2km.txt $simpath/icon-extra_ml.nc remapweights_2km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) select variables from the 3d dataset
+
+cdo -P 38 selname,u,v,pres,w,omega,temp,vor,div,z_ifc,geopot,qv $simpath/icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc icon-3d_ML_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 4) conservative remapping to 0.25x0.25 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_025x025.txt icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 5) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon_025x025/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-atm3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) select variables from the 3d dataset
+
+cdo -P 38 selname,u,v,pres,w,omega,temp,vor,div,z_ifc,geopot,qv $simpath/icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc icon-3d_ML_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 4) conservative remapping to 0.25x0.25 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_025x025.txt icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 5) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon_025x025/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-atm3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_2km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_mlpl_atm3d_2km_0p05x0p05deg.sh b/Scripts_for_postprocessing/remap_mlpl_atm3d_2km_0p05x0p05deg.sh
new file mode 100644
index 0000000000000000000000000000000000000000..fef40effdb3a3ce2c6e219b07138e16392844aad
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_mlpl_atm3d_2km_0p05x0p05deg.sh
@@ -0,0 +1,156 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_mlpl_atm3d_2km_0p05x0p05
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_atm3d_2km_0p05x0p05.%j.o
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_atm3d_2km_0p05x0p05.%j.o
+#SBATCH --time=04:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_2km_0004
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/scratch/b/$username/check_remapping_on_PTE/$exp
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_atm3d_latlon_0p05x0p05
+cd $outpath/remapped_atm3d_latlon_0p05x0p05
+
+#### Remapping
+
+# define pressure levels that data will be remapped to
+plevout="1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,14000,15000,16000,17000,18000,19000,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000"
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_2km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_2km.txt $simpath/icon-extra_ml.nc remapweights_2km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) select variables from the 3d dataset
+
+cdo -P 38 selname,u,v,pres,w,omega,temp,vor,div,z_ifc,geopot,qv $simpath/icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc icon-3d_ML_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 4) conservative remapping to 0.05x0.05 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_0p05x0p05.txt icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 5) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon_0p05x0p05/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-atm3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) select variables from the 3d dataset
+
+cdo -P 38 selname,u,v,pres,w,omega,temp,vor,div,z_ifc,geopot,qv $simpath/icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc icon-3d_ML_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 4) conservative remapping to 0.05x0.05 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_0p05x0p05.txt icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 5) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon_0p05x0p05/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-atm3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_2km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_mlpl_atm3d_80km.sh b/Scripts_for_postprocessing/remap_mlpl_atm3d_80km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..369a6b1e9945db1f88bf6cef048fe2bad2a15676
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_mlpl_atm3d_80km.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_mlpl_atm3d_80km
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/work/bb1152/Module_A/A6_CyclEx/b380782_Christoph/pp_logs/LOG.remap_mlpl_atm3d_80km.%j.o
+#SBATCH --error=/work/bb1152/Module_A/A6_CyclEx/b380782_Christoph/pp_logs/LOG.remap_mlpl_atm3d_80km.%j.o
+#SBATCH --time=00:30:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+# folder with data that will be remapped
+exp=channel_80km_0006
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+projpath=/work/bb1152/Module_A/A6_CyclEx
+
+simpath=$datapath/$exp
+echo $simpath
+
+# folder in which the remapped data will be stored
+mkdir -p $simpath/remapped_atm3d_latlon
+cd $simpath/remapped_atm3d_latlon
+
+# define pressure levels that data will be remapped to
+plevout="1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,14000,15000,16000,17000,18000,19000,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000"
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_80km.txt
+
+cdo -P 32 gennn,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_80km.txt $simpath/icon-extra_ml.nc remapweights_80km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 ; do
+
+# 2) select variables from the 3d dataset
+
+cdo -P 38 selname,u,v,pres,w,omega,temp,vor,div,z_ifc,geopot,qv ../icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_80km.txt,remapweights_80km.nc icon-3d_ML_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 4) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_1x1.txt icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 5) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-atm3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) select variables from the 3d dataset
+
+cdo -P 38 selname,u,v,pres,w,omega,temp,vor,div,z_ifc,geopot,qv ../icon-atm3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_80km.txt,remapweights_80km.nc icon-3d_ML_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 4) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$projpath/b380782_Christoph/icon-climxtreme/postprocessing_scripts/grid_info_1x1.txt icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-atm3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 5) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-atm3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-atm3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_80km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_mlpl_ddt3d_2km.sh b/Scripts_for_postprocessing/remap_mlpl_ddt3d_2km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..2b7e5348be333f945126a4066475035cc5b9c428
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_mlpl_ddt3d_2km.sh
@@ -0,0 +1,144 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_mlpl_ddt3d_2km
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_ddt3d_2km.%j.o
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_ddt3d_2km.%j.e
+#SBATCH --time=04:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_2km_0006
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production/$exp
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_ddt3d_latlon
+cd $outpath/remapped_ddt3d_latlon
+
+#### Remapping
+
+# define pressure levels that data will be remapped to
+plevout="1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,14000,15000,16000,17000,18000,19000,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000"
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_2km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_2km.txt $simpath/icon-extra_ml.nc remapweights_2km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-ddt3d_ML_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_1x1.txt icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 4) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-ddt3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-ddt3d_ML_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_1x1.txt icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 4) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-ddt3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_2km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_mlpl_ddt3d_2km_025x025deg.sh b/Scripts_for_postprocessing/remap_mlpl_ddt3d_2km_025x025deg.sh
new file mode 100644
index 0000000000000000000000000000000000000000..e8b8a0c2c09c2af5e76ee2b649b29adfcc2aa004
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_mlpl_ddt3d_2km_025x025deg.sh
@@ -0,0 +1,164 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_mlpl_ddt3d_2km_025x025
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_ddt3d_2km_025x025.%j.o
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_ddt3d_2km_025x025.%j.e
+#SBATCH --time=04:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_2km_0005
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/scratch/b/$username/check_remapping_on_PTE/$exp
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_ddt3d_latlon_025x025
+cd $outpath/remapped_ddt3d_latlon_025x025
+
+#### Remapping
+
+# define pressure levels that data will be remapped to
+plevout="1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,14000,15000,16000,17000,18000,19000,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000"
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_2km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_2km.txt $simpath/icon-extra_ml.nc remapweights_2km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+### CB remap all vars in the ddt files
+
+# 2) select variables from the 3d dataset
+
+#cdo -P 38 selname,pres,ddt_temp_totnwpphy $simpath/icon-ddt3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-ddt3d_ML_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+### CB file not created, see comment above
+
+# rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 0.25x0.25 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_025x025.txt icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 4) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon_025x025/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-ddt3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+### CB remap all vars in the ddt files
+
+# 2) select variables from the 3d dataset
+
+#cdo -P 38 selname,pres,ddt_temp_totnwpphy $simpath/icon-ddt3d_ML_202101${day}T${hour}0000Z.nc icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_2km.txt,remapweights_2km.nc $simpath/icon-ddt3d_ML_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+### CB file not created, see comment above
+
+# rm icon-3d_ML_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 0.25x0.25 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_025x025.txt icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 4) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon_025x025/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-ddt3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_2km.nc
+
+#
diff --git a/Scripts_for_postprocessing/remap_mlpl_ddt3d_80km.sh b/Scripts_for_postprocessing/remap_mlpl_ddt3d_80km.sh
new file mode 100644
index 0000000000000000000000000000000000000000..0d4ae56c90fdaa5182b59aed0bf9b62c2dccb197
--- /dev/null
+++ b/Scripts_for_postprocessing/remap_mlpl_ddt3d_80km.sh
@@ -0,0 +1,144 @@
+#!/bin/bash
+#=============================================================================
+#SBATCH --account=bb1152
+#SBATCH --job-name=remap_mlpl_ddt3d_80km
+#SBATCH --partition=compute
+#SBATCH --nodes=10
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --exclusive
+#SBATCH --output=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_ddt3d_80km.%j.o
+#SBATCH --error=/home/b/b380782/pp_logs_cyclex/LOG.remap_mlpl_ddt3d_80km.%j.e
+#SBATCH --time=04:00:00
+#=========================================================================================
+# this script is for remapping data sets from native unstructured grid
+# to regular lat/lon grid
+# first near neighbor interpolation with the same resolution then conservative remapping
+
+
+#### INFO: If you cloned the code repository to your home directory (as suggested in the wiki)
+#### you need to change the username and experiment in the following section. The username
+#### also needs to be changed in the job header (path for log and error files).
+
+#### Set username and experiment
+###
+## Set username here and in job header
+#
+# name of user submitting this script
+username=b380782
+
+# experiment name
+exp=channel_80km_0001
+
+####
+####
+
+#### load required module
+module load cdo
+
+#### Set paths
+
+### raw data
+# folder with data that will be remapped
+datapath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production
+
+simpath=$datapath/$exp/raw_data
+echo $simpath
+
+### required scripts
+# path to clone of code repository
+repopath=/home/b/$username/icon-climxtreme
+# path to scripts WITHIN code repository
+scriptpath=CC_postprocessing_scripts
+
+### output directory
+# indicated scratch with predefined directory
+outpath=/work/bb1152/Module_A/A6_CyclEx/sim_data/production/$exp
+echo $outpath
+
+# directory, in which the remapped data will be stored
+mkdir -p $outpath/remapped_ddt3d_latlon
+cd $outpath/remapped_ddt3d_latlon
+
+#### Remapping
+
+# define pressure levels that data will be remapped to
+plevout="1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,14000,15000,16000,17000,18000,19000,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000"
+
+# 1) Creating weights from a dataset which contains grid information!
+# icon-extra_ml.nc is the dataset which contains the clon,clat or from the grid file itself
+
+# Grid description: grid_info_80km.txt
+
+cdo -P 32 gennn,$repopath/$scriptpath/grid_info_80km.txt $simpath/icon-extra_ml.nc remapweights_80km.nc
+
+#------------------------------------------------------------------------------------------
+# loop over the time steps
+for day in 01 02 03 04 05 06 07 08 09 ; do
+
+for hour in 00 06 12 18 ; do
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_80km.txt,remapweights_80km.nc $simpath/icon-ddt3d_ML_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_1x1.txt icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 4) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-ddt3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+done
+done
+
+#-----------------------------------------------------------------------------------------
+# last time step
+
+day=10
+hour=00
+
+# 2) remap to regular lat,lon grid (near neighbor)
+
+cdo -P 38 remap,$repopath/$scriptpath/grid_info_80km.txt,remapweights_80km.nc $simpath/icon-ddt3d_ML_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 3) conservative remapping to 1x1 degree
+
+cdo -P 32 remapcon,$repopath/$scriptpath/grid_info_1x1.txt icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc
+
+rm icon-ddt3d_ML_reg_202101${day}T${hour}0000Z.nc
+
+# 4) Interpolate from height to pressure levels (var 'pres' must be present in the dataset)
+
+# get sfc pressure from remapped 2d files
+cdo -P 38 selname,pres_sfc ../remapped_atm2d_latlon/icon-atm2d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc
+
+# merge sfc pressure into remapped 3d files
+cdo -P 38 merge icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.nc pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+# interpolate to pressure levels
+cdo -P 38 ap2pl,$plevout icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc icon-ddt3d_PL_reg_con_202101${day}T${hour}0000Z.nc
+
+# cleanup
+rm pres_sfc_reg_con_202101${day}T${hour}0000Z.nc icon-ddt3d_ML_reg_con_202101${day}T${hour}0000Z.psfc.nc
+
+#-----------------------------------------------------------------------------------------
+# delete grid file and remap weights
+rm remapweights_80km.nc
+
+#
diff --git a/Scripts_for_running_simulations/README_simulations.txt b/Scripts_for_running_simulations/README_simulations.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b081cdf8b0d1f70d9ebe28f59a4497127759046f
--- /dev/null
+++ b/Scripts_for_running_simulations/README_simulations.txt
@@ -0,0 +1,18 @@
+ICON-NWP aquaplanet simulations of a baroclinic lifecycle (9 days)
+in limited area mode with channel setup.
+
+# 2km horizontal resolution
+channel_2km_0001: control simulation
+channel_2km_0002: +4K atmospheric and surface temperatures, qv consistent with T
+channel_2km_0003: +4K temperatures, qv from control simulation
+channel_2km_0004: +temperature anomaly from MPI-ESM1-2-LR far future
+channel_2km_0005: +tropical temperature anomaly from MPI-ESM1-2-LR far future
+channel_2km_0006: +polar temperature anomaly from MPI-ESM1-2-LR far future
+
+# 80km horizontal resolution
+channel_80km_0001: control simulation
+channel_80km_0002: +4K atmospheric and surface temperatures, qv consistent with T
+channel_80km_0003: +4K temperatures, qv from control simulation
+channel_80km_0004: +temperature anomaly from MPI-ESM1-2-LR far future
+channel_80km_0005: +tropical temperature anomaly from MPI-ESM1-2-LR far future
+channel_80km_0006: +polar temperature anomaly from MPI-ESM1-2-LR far future
diff --git a/Scripts_for_running_simulations/exp.channel_2km_0001.run b/Scripts_for_running_simulations/exp.channel_2km_0001.run
new file mode 100644
index 0000000000000000000000000000000000000000..dee93a44fc7d9b8681471139e5c41f0896e0c482
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_2km_0001.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=2km0001
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0001
+#SBATCH --nodes=80
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0001/LOG.channel_2km_0001.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0001/LOG.channel_2km_0001.%j.o
+#SBATCH --exclusive
+#SBATCH --time=08:00:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_2km_0001.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_2km_0001"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0001/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_2500m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/lc1_ifs_CTL_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 24 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+ lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_2km_0002.run b/Scripts_for_running_simulations/exp.channel_2km_0002.run
new file mode 100644
index 0000000000000000000000000000000000000000..f4b5a99fa652bab5b9c482886f2879c12f4a1f10
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_2km_0002.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=2km0002
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0002
+#SBATCH --nodes=80
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0002/LOG.channel_2km_0002.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0002/LOG.channel_2km_0002.%j.o
+#SBATCH --exclusive
+#SBATCH --time=08:00:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_2km_0002.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_2km_0002"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0002/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_2500m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/lc1_ifs_4K_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 24 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+ lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_2km_0003.run b/Scripts_for_running_simulations/exp.channel_2km_0003.run
new file mode 100644
index 0000000000000000000000000000000000000000..9b8cbf4bf3cf1a371d893a5be531fe7c9f3c0acc
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_2km_0003.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=2km0003
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0003
+#SBATCH --nodes=80
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0003/LOG.channel_2km_0003.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0003/LOG.channel_2km_0003.%j.o
+#SBATCH --exclusive
+#SBATCH --time=08:00:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_2km_0003.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_2km_0003"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0003/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_2500m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/lc1_ifs_4K_qvCTL_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 24 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+ lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_2km_0004.run b/Scripts_for_running_simulations/exp.channel_2km_0004.run
new file mode 100644
index 0000000000000000000000000000000000000000..11e941c66d2c29cb2236b89345e047a387c39f1d
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_2km_0004.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=2km0004
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0004
+#SBATCH --nodes=80
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0004/LOG.channel_2km_0004.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0004/LOG.channel_2km_0004.%j.o
+#SBATCH --exclusive
+#SBATCH --time=08:00:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_2km_0004.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_2km_0004"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0004/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_2500m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/lc1_ifs_Tanom_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 24 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+ lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_2km_0005.run b/Scripts_for_running_simulations/exp.channel_2km_0005.run
new file mode 100644
index 0000000000000000000000000000000000000000..5de89a18951099437ebb9de8bd6693650100311c
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_2km_0005.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=2km0005
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0005
+#SBATCH --nodes=80
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0005/LOG.channel_2km_0005.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0005/LOG.channel_2km_0005.%j.o
+#SBATCH --exclusive
+#SBATCH --time=08:00:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_2km_0005.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_2km_0005"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0005/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_2500m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/lc1_ifs_Tanom_tropics_linear_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 24 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+ lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_2km_0006.run b/Scripts_for_running_simulations/exp.channel_2km_0006.run
new file mode 100644
index 0000000000000000000000000000000000000000..d41ae1efb4b4bbc0eb3d0208ceeb30f1fdd638ed
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_2km_0006.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=2km0006
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0006
+#SBATCH --nodes=80
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0006/LOG.channel_2km_0006.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0006/LOG.channel_2km_0006.%j.o
+#SBATCH --exclusive
+#SBATCH --time=08:00:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_2km_0006.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_2km_0006"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_2km_0006/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_2500m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/lc1_ifs_Tanom_polar_linear_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_2km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 24 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+ lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_80km_0001.run b/Scripts_for_running_simulations/exp.channel_80km_0001.run
new file mode 100644
index 0000000000000000000000000000000000000000..f4a80a0dc7f37b04adb61c784de5bd3e3a7dc076
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_80km_0001.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=80km0001
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0001
+#SBATCH --nodes=5
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0001/LOG.channel_80km_0001.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0001/LOG.channel_80km_0001.%j.o
+#SBATCH --exclusive
+#SBATCH --time=00:30:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_80km_0001.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_80km_0001"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0001/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_80000m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/lc1_ifs_CTL_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 360 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+! lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_80km_0002.run b/Scripts_for_running_simulations/exp.channel_80km_0002.run
new file mode 100644
index 0000000000000000000000000000000000000000..bce1e01dc41903e5a6489b8ce36ab9cea5a1c2a4
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_80km_0002.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=80km0002
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0002
+#SBATCH --nodes=5
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0002/LOG.channel_80km_0002.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0002/LOG.channel_80km_0002.%j.o
+#SBATCH --exclusive
+#SBATCH --time=00:30:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_80km_0002.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_80km_0002"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0002/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_80000m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/lc1_ifs_4K_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 360 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+! lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_80km_0003.run b/Scripts_for_running_simulations/exp.channel_80km_0003.run
new file mode 100644
index 0000000000000000000000000000000000000000..aac676fae4329dfb77701f81e68eb2d91866f2ea
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_80km_0003.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=80km0003
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0003
+#SBATCH --nodes=5
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0003/LOG.channel_80km_0003.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0003/LOG.channel_80km_0003.%j.o
+#SBATCH --exclusive
+#SBATCH --time=00:30:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_80km_0003.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_80km_0003"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0003/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_80000m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/lc1_ifs_4K_qvCTL_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 360 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+! lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_80km_0004.run b/Scripts_for_running_simulations/exp.channel_80km_0004.run
new file mode 100644
index 0000000000000000000000000000000000000000..5af7584bf21e74bc21b9216820f0a385d9bafa12
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_80km_0004.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=80km0004
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0004
+#SBATCH --nodes=5
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0004/LOG.channel_80km_0004.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0004/LOG.channel_80km_0004.%j.o
+#SBATCH --exclusive
+#SBATCH --time=00:30:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_80km_0004.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_80km_0004"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0004/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_80000m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/lc1_ifs_Tanom_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 360 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+! lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_80km_0005.run b/Scripts_for_running_simulations/exp.channel_80km_0005.run
new file mode 100644
index 0000000000000000000000000000000000000000..004719aeb55ce65ce9e7d7aa1ae7290a207e0740
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_80km_0005.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=80km0005
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0005
+#SBATCH --nodes=5
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0005/LOG.channel_80km_0005.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0005/LOG.channel_80km_0005.%j.o
+#SBATCH --exclusive
+#SBATCH --time=00:30:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_80km_0005.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_80km_0005"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0005/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_80000m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/lc1_ifs_Tanom_tropics_linear_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 360 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+! lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
diff --git a/Scripts_for_running_simulations/exp.channel_80km_0006.run b/Scripts_for_running_simulations/exp.channel_80km_0006.run
new file mode 100644
index 0000000000000000000000000000000000000000..e00ac468d26c822b2a4ebc6da3183c9db7c3f1be
--- /dev/null
+++ b/Scripts_for_running_simulations/exp.channel_80km_0006.run
@@ -0,0 +1,672 @@
+#! /bin/ksh
+#=============================================================================
+
+# levante cpu batch job parameters
+
+#SBATCH --account=bb1152
+#SBATCH --job-name=80km0006
+#SBATCH --partition=compute
+#SBATCH --chdir=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0006
+#SBATCH --nodes=5
+#SBATCH --threads-per-core=2
+# the following is needed to work around a bug that otherwise leads to
+# a too low number of ranks when using compute,compute2 as queue
+#SBATCH --mem=0
+#SBATCH --output=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0006/LOG.channel_80km_0006.%j.o
+#SBATCH --error=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0006/LOG.channel_80km_0006.%j.o
+#SBATCH --exclusive
+#SBATCH --time=00:30:00
+
+#=============================================================================
+set -x
+ulimit -s unlimited
+#=============================================================================
+# OpenMP environment variables
+# ----------------------------
+export OMP_NUM_THREADS=4
+export ICON_THREADS=$OMP_NUM_THREADS
+export OMP_SCHEDULE="guided"
+export OMP_DYNAMIC="false"
+export OMP_STACKSIZE=500M
+#
+# MPI variables
+# -------------
+no_of_nodes=${SLURM_JOB_NUM_NODES:=1}
+mpi_procs_pernode=$((128 / OMP_NUM_THREADS))
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+
+#=============================================================================
+
+# load local setting, if existing
+# -------------------------------
+if [ -a ../setting ]
+then
+ echo "Load Setting"
+ . ../setting
+fi
+
+# environment variables for the experiment and the target system
+# --------------------------------------------------------------
+
+export EXPNAME="atm_rte_rrtmgp_amip_test"
+export KMP_AFFINITY="granularity=fine,scatter"
+export KMP_LIBRARY="turnaround"
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+export OMPI_MCA_coll="^ml"
+export OMPI_MCA_coll_hcoll_enable="1"
+export HCOLL_ENABLE_MCAST_ALL="1"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export MALLOC_TRIM_THRESHOLD_="-1"
+export MKL_ENABLE_INSTRUCTIONS=AVX2
+export MKL_DEBUG_CPU_TYPE=5
+export UCX_HANDLE_ERRORS=bt
+
+# load profile
+# ------------
+if [[ -a /etc/profile ]]
+then
+ . /etc/profile
+fi
+
+#=============================================================================
+
+# directories with absolute paths
+# -------------------------------
+basedir="/home/b/b380906/icon-on-jet"
+export basedir
+
+# how to start the icon model
+mask="0xf,0xf0000,0xf00000000,0xf000000000000,0xf0000000000000000,0xf00000000000000000000,0xf000000000000000000000000,0xf0000000000000000000000000000,0xf0,0xf00000,0xf000000000,0xf0000000000000,0xf00000000000000000,0xf000000000000000000000,0xf0000000000000000000000000,0xf00000000000000000000000000000,0xf00,0xf000000,0xf0000000000,0xf00000000000000,0xf000000000000000000,0xf0000000000000000000000,0xf00000000000000000000000000,0xf000000000000000000000000000000,0xf000,0xf0000000,0xf00000000000,0xf000000000000000,0xf0000000000000000000,0xf00000000000000000000000,0xf000000000000000000000000000,0xf0000000000000000000000000000000"
+export START="srun -l --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks=$((no_of_nodes * mpi_procs_pernode)) --propagate=STACK,CORE --distribution=block:block --cpu-bind=v,mask_cpu=$mask "
+export MODEL="${basedir}/bin/icon"
+
+# how to submit the next job
+# --------------------------
+submit="sbatch"
+job_name="exp.channel_80km_0006.run"
+
+#=============================================================================
+
+ulimit -s $((4 * 1024 * 1024))
+ulimit -c 0
+
+#-----------------------------------------------------------------------------
+export EXPNAME="channel_80km_0006"
+
+# atmo namelist filename
+atmo_namelist=NAMELIST_${EXPNAME}_atm
+
+# directories definition
+RUNSCRIPTDIR=/home/b/b380906/icon-on-jet/run/runscripts/channel_80km_0006/ # run script directory
+
+#-----------------------------------------------------------------------------
+# global timing
+start_date="2021-01-01T00:00:00Z"
+end_date="2021-01-10T00:00:00Z"
+
+# restart intervals
+checkpoint_interval="P20D"
+restart_interval="P40D"
+
+# output intervals
+output_interval="PT1H"
+file_interval="PT1H"
+
+#-----------------------------------------------------------------------------
+# model parameters
+model_equations=3 # equation system
+# 1=hydrost. atm. T
+# 1=hydrost. atm. theta dp
+# 3=non-hydrost. atm.,
+# 0=shallow water model
+# -1=hydrost. ocean
+#-----------------------------------------------------------------------------
+
+# experiment directory, with plenty of space, create if new
+
+EXPDIR=/scratch/b/b380906/${EXPNAME}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ; then
+ mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+check_error $? "${EXPDIR} does not exist?"
+
+cd ${EXPDIR}
+
+#-----------------------------------------------------------------------------
+# Link experiment initial files
+inputdir='/work/bb1152/Module_A/A6_CyclEx/input_data'
+# Grid
+ln -sf $inputdir/Channel_4000x9000_80000m_with_boundary.nc grid_DOM01.nc
+
+# Extpar
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/extpar_remapped_12_months.nc extpar_DOM01.nc
+
+# Initial file
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/lc1_ifs_Tanom_polar_linear_remapped.nc ifs2icon_init.nc
+
+# Ozone
+ln -sf $inputdir/inputs_planar_channel_51x81_80km/ape_O3_remapped.nc o3_icon_DOM01.nc
+
+# Model required files
+ln -sf $basedir/data/rrtmg_lw.nc ./
+ln -sf $basedir/data/rrtmg_sw.nc ./
+ln -sf $basedir/data/ECHAM6_CldOptProps.nc ./
+
+#=============================================================================
+
+# create ICON namelist parameters
+
+cat > ${atmo_namelist} << EOF
+¶llel_nml
+ nproma = 16 ! optimal setting 8 for CRAY; use 16 or 24 for IBM
+ p_test_run = .false.
+ l_test_openmp = .false.
+ l_log_checks = .false.
+ num_io_procs = 10 ! asynchronous output for values >= 1
+ itype_comm = 1
+ iorder_sendrecv = 3 ! best value for CRAY (slightly faster than option 1)
+ num_prefetch_proc = 1
+/
+&grid_nml
+ dynamics_grid_filename = 'grid_DOM01.nc'
+ dynamics_parent_grid_id = 0
+ lredgrid_phys = .false.
+ is_plane_cylinder = .TRUE.
+ l_limited_area = .true.
+ corio_lat = 45.0
+/
+&initicon_nml
+ init_mode = 2 ! operation mode 2: IFS
+ ifs2icon_filename = 'ifs2icon_init.nc'
+ zpbl1 = 500.
+ zpbl2 = 1000.
+ l_sst_in = .true.
+/
+&limarea_nml
+ itype_latbc = 0 ! fix
+/
+&run_nml
+ num_lev = 90 ! 60
+ dtime = 360 ! 360 for R2B5 180 for R2B7
+ ldynamics = .TRUE.
+ ltransport = .true.
+ iforcing = 3 ! NWP forcing
+ ltestcase = .false. ! false: run with real data
+ msg_level = 7 ! print maximum wind speeds every 5 time steps
+ ltimer = .false. ! set .TRUE. for timer output
+ timers_level = 1 ! can be increased up to 10 for detailed timer output
+ output = "nml"
+/
+&nwp_phy_nml
+ inwp_gscp = 1
+ inwp_convection = 1
+! lshallowconv_only = .true. ! use only shallow convection scheme
+ inwp_radiation = 1
+ inwp_cldcover = 1
+ inwp_turb = 1
+ inwp_satad = 1
+ inwp_sso = 1
+ inwp_gwd = 1
+ inwp_surface = 1
+ icapdcycl = 3
+ latm_above_top = .false. ! the second entry refers to the nested domain (if present)
+ efdt_min_raylfric = 7200.
+ ldetrain_conv_prec = .false.
+ itype_z0 = 2
+ icpl_aero_conv = 0
+ icpl_aero_gscp = 0
+ ! resolution-dependent settings - please choose the appropriate one
+ dt_rad = 720
+ dt_conv = 300
+ dt_sso = 300
+ dt_gwd = 300.
+/
+&turbdiff_nml
+ tkhmin = 0.75
+ tkmmin = 0.75
+ pat_len = 100. !750.
+ c_diff = 0.2
+ rat_sea = 10
+ ltkesso = .true.
+ frcsmot = 0.2
+ imode_frcsmot = 2
+ itype_sher = 3
+ ltkeshs = .true.
+ a_hshr = 2.0
+/
+&lnd_nml
+ ntiles = 3 !!! 1 for assimilation cycle and forecast
+ nlev_snow = 3 !!! 1 for assimilation cycle and forecast
+ lmulti_snow = .true. !!! .false. for assimilation cycle and forecast
+ itype_heatcond = 2
+ idiag_snowfrac = 2
+ lsnowtile = .false. !! later on .true. if GRIB encoding issues are solved
+ lseaice = .true. !!Sophia Schäfer, 23/03/2017: reads in sea ice (e.g. zero ice), instead of setting by temperature
+ llake = .false.
+ itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere
+ itype_root = 2
+/
+&radiation_nml
+ irad_o3 = 4
+ irad_aero = 0
+ irad_cfc11 = 0
+ irad_cfc12 = 0
+ albedo_type = 2 ! Modis albedo
+ vmr_co2 = 348.0e-6 ! values representative for 2012
+ vmr_ch4 = 1650.0e-09
+ vmr_n2o = 396.0e-09
+ vmr_o2 = 0.20946
+ izenith = 3
+ vmr_cfc11 = 0
+ vmr_cfc12 = 0
+/
+&nonhydrostatic_nml
+ iadv_rhotheta = 2
+ ivctype = 2
+ itime_scheme = 4
+ exner_expol = 0.333
+ vwind_offctr = 0.2
+ damp_height = 50000.
+ rayleigh_coeff = 0.10 ! R3B7: 1.0 for forecasts, 5.0 in assimilation cycle; 0.5/2.5 for R2B6 (i.e. ensemble) runs
+ lhdiff_rcf = .true.
+ divdamp_order = 24 ! setting for forecast runs; use '2' for assimilation cycle
+ divdamp_type = 32 ! ** new setting for assimilation and forecast runs **
+ divdamp_fac = 0.004 ! use 0.032 in conjunction with divdamp_order = 2 in assimilation cycle
+ l_open_ubc = .false.
+ igradp_method = 3
+ l_zdiffu_t = .true.
+ thslp_zdiffu = 0.02
+ thhgtd_zdiffu = 125.
+ htop_moist_proc= 22500.
+ hbot_qvsubstep = 22500. ! use 22500. with R2B6
+/
+&nudging_nml
+ nudge_type = 0
+ max_nudge_coeff_vn = 0.00000
+/
+&sleve_nml
+ min_lay_thckn = 20.
+ max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit
+ htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels
+ top_height = 75000.
+ stretch_fac = 0.9
+ decay_scale_1 = 4000.
+ decay_scale_2 = 2500.
+ decay_exp = 1.2
+ flat_height = 16000.
+/
+&dynamics_nml
+ iequations = 3
+ idiv_method = 1
+ divavg_cntrwgt = 0.50
+ !lcoriolis = .TRUE.
+/
+&transport_nml
+ ctracer_list = '12345'
+ ivadv_tracer = 3,3,3,3,3
+ itype_hlimit = 3,4,4,4,4
+ ihadv_tracer = 52,2,2,2,2
+ iadv_tke = 0
+/
+&diffusion_nml
+ hdiff_order = 5
+ itype_vn_diffu = 1
+ itype_t_diffu = 2
+ hdiff_efdt_ratio = 24.0
+ hdiff_smag_fac = 0.025
+ lhdiff_vn = .TRUE.
+ lhdiff_temp = .TRUE.
+/
+&interpol_nml
+ nudge_zone_width = 0.0 ! deactivating nudging as needed for mass conservation for this setup
+ nudge_efold_width = 0.0
+ nudge_max_coeff = 0.00000
+ lsq_high_ord = 3
+ l_intp_c2l = .true.
+ l_mono_c2l = .true.
+ support_baryctr_intp = .false.
+ rbf_vec_scale_c = 0.3, 0.1, 0.03
+ rbf_vec_scale_v = 0.4, 0.25, 0.07
+ rbf_vec_scale_e = 0.45, 0.37, 0.25
+/
+&extpar_nml
+ itopo = 1
+ extpar_filename = 'extpar_DOM01.nc'
+ n_iter_smooth_topo = 1
+ heightdiff_threshold = 3000.
+/
+&io_nml
+ itype_pres_msl = 5 ! 4 New extrapolation method to circumvent Ninjo problem with surface inversions
+ itype_rh = 1 ! RH w.r.t. water
+ restart_file_type = 5 ! 4: netcdf2, 5: netcdf4
+/
+&nh_pzlev_nml
+ nplev = 18 ! number of p level output (attention: levels in [Pa] and top-down)
+ nzlev = 20 ! number of z level output (attention: levels in [m] and top-down)
+ plevels = 5000,7000,10000,15000,20000,25000,30000,40000,50000,60000,70000,
+ 80000,85000,90000,92500,95000,97500,100000
+ zlevels = 25000,20000,18000,16000,14000,12000,10000,8000,6000,5000,4000,3000,2000,
+ 1000,800,600,400,200,100,10
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres','qc','qi','qv','rho','temp','u','v','w','vn','z_ifc','vor','rh','pv','omega','geopot','div','cape','tke'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-cld3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'clc','tot_qv_dia','tot_qc_dia','tot_qi_dia','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'lwflxall', 'lwflx_dn_clr','lwflx_up_clr','swflx_dn_clr','swflx_up_clr','swflx_up','swflx_dn','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-ddt3d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'ddt_temp_totnwpphy','ddt_temp_radsw','ddt_temp_radlw','ddt_temp_turb','ddt_temp_pconv','ddt_temp_drag','ddt_temp_satad','ddt_temp_diff','ddt_temp_dyn','ddt_temp_gscp','pres'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-atm2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'pres_msl','pres_sfc','ddt_pres_sfc','u_10m','v_10m','t_g','t_2m','qv_2m','td_2m','rh_2m','group:precip_vars','rain_gsp_rate','rain_con_rate','cape_ml','cin_ml','clct','clch','clcm','clcl','tqv_dia','tqc_dia','tqi_dia','tqv','tqc','tqi','tqr','shfl_s','lhfl_s','qhfl_s','w_so'
+ remap = 0
+/
+&output_nml
+ filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2
+ dom = 1 ! write all domains
+ output_start = "${start_date}"
+ output_end = "${end_date}"
+ output_interval = "${output_interval}"
+ file_interval = "${file_interval}"
+ include_last = .FALSE.
+ output_filename = 'icon-rad2d' ! file name base
+ filename_format = "<output_filename>_ML_<datetime2>"
+ ml_varlist = 'sob_t','thb_t','sob_s','thb_s','thbclr_s','sobclr_s'
+ remap = 0
+/
+&output_nml
+ output_filename = "icon-extra"
+ filename_format = "<output_filename>_<levtype_l>"
+ filetype = 5
+ remap = 0
+ output_grid = .TRUE.
+ output_start = "${start_date}" ! output_start = output_end
+ output_end = "${start_date}" ! --> write once only irrespective of
+ output_interval = "PT1H" ! the output interval and
+ file_interval = "${file_interval}" ! the file interval
+ ml_varlist = 'z_mc','t_seasfc'
+/
+EOF
+
+#=============================================================================
+#
+# This section of the run script prepares and starts the model integration.
+#-----------------------------------------------------------------------------
+
+final_status_file=${EXPDIR}/${job_name}.final_status
+rm -f ${final_status_file}
+#-----------------------------------------------------------------------------
+
+# set up the model lists if they do not exist
+# this works for single model runs
+# for coupled runs the lists should be declared explicilty
+if [ x$namelist_list = x ]; then
+# minrank_list=( 0 )
+# maxrank_list=( 65535 )
+# incrank_list=( 1 )
+ minrank_list[0]=0
+ maxrank_list[0]=65535
+ incrank_list[0]=1
+ if [ x$atmo_namelist != x ]; then
+ # this is the atmo model
+ namelist_list[0]="$atmo_namelist"
+ modelname_list[0]="atmo"
+ modeltype_list[0]=1
+ run_atmo="true"
+ else
+ check_error 1 "No namelist is defined"
+ fi
+fi
+
+#-----------------------------------------------------------------------------
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+ restart=.true.
+ # do not delete switch-file, to enable restart after unintended abort
+ #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+#
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+ if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+ sleep 10;
+ fi
+fi
+# fill some checks
+
+run_atmo=${run_atmo="false"}
+if [ x$atmo_namelist != x ]; then
+ run_atmo="true"
+fi
+#-----------------------------------------------------------------------------
+# get restart files
+
+if [ x$restart_atmo_from != "x" ] ; then
+ rm -f restart_atm_DOM01.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} ${EXPDIR}/restart_atm_DOM01.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_atmo_from} cp_restart_atm.nc
+ ln -s cp_restart_atm.nc restart_atm_DOM01.nc
+ restart=".true."
+fi
+if [ x$restart_ocean_from != "x" ] ; then
+ rm -f restart_oce.nc
+# ln -s ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} ${EXPDIR}/restart_oce.nc
+ cp ${basedir}/experiments/${restart_from_folder}/${restart_ocean_from} cp_restart_oce_DOM01.nc
+ ln -s cp_restart_oce_DOM01.nc restart_oce_DOM01.nc
+ restart=".true."
+fi
+#-----------------------------------------------------------------------------
+
+
+read_restart_namelists=${read_restart_namelists:=".true."}
+
+#-----------------------------------------------------------------------------
+#
+# create ICON master namelist
+# ------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+
+#-----------------------------------------------------------------------------
+# create master_namelist
+master_namelist=icon_master.namelist
+if [ x$end_date = x ]; then
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+/
+&master_time_control_nml
+ experimentStartDate = "$start_date"
+ restartTimeIntval = "$restart_interval"
+ checkpointTimeIntval = "$checkpoint_interval"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+else
+if [ x$calendar = x ]; then
+ calendar='proleptic gregorian'
+ calendar_type=1
+else
+ calendar=$calendar
+ calendar_type=$calendar_type
+fi
+cat > $master_namelist << EOF
+&master_nml
+ lrestart = $restart
+ read_restart_namelists = $read_restart_namelists
+/
+&master_time_control_nml
+ calendar = "$calendar"
+ checkpointTimeIntval = "$checkpoint_interval"
+ restartTimeIntval = "$restart_interval"
+ experimentStartDate = "$start_date"
+ experimentStopDate = "$end_date"
+/
+&time_nml
+ is_relative_time = .false.
+/
+EOF
+fi
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# add model component to master_namelist
+add_component_to_master_namelist()
+{
+
+ model_namelist_filename="$1"
+ model_name=$2
+ model_type=$3
+ model_min_rank=$4
+ model_max_rank=$5
+ model_inc_rank=$6
+
+cat >> $master_namelist << EOF
+&master_model_nml
+ model_name="$model_name"
+ model_namelist_filename="$model_namelist_filename"
+ model_type=$model_type
+ model_min_rank=$model_min_rank
+ model_max_rank=$model_max_rank
+ model_inc_rank=$model_inc_rank
+/
+EOF
+
+}
+#-----------------------------------------------------------------------------
+
+
+no_of_models=${#namelist_list[*]}
+echo "no_of_models=$no_of_models"
+
+j=0
+while [ $j -lt ${no_of_models} ]
+do
+ add_component_to_master_namelist "${namelist_list[$j]}" "${modelname_list[$j]}" ${modeltype_list[$j]} ${minrank_list[$j]} ${maxrank_list[$j]} ${incrank_list[$j]}
+ j=`expr ${j} + 1`
+done
+
+#-----------------------------------------------------------------------------
+#
+# get model
+#
+ls -l ${MODEL}
+check_error $? "${MODEL} does not exist?"
+#-----------------------------------------------------------------------------
+
+# start experiment
+
+rm -f finish.status
+date
+${START} ${MODEL}
+date
+#
+if [ -r finish.status ] ; then
+ check_final_status 0 "${START} ${MODEL}"
+else
+ check_final_status -1 "${START} ${MODEL}"
+fi
+#
+
+#-----------------------------------------------------------------------------
+#
+finish_status=`cat finish.status`
+echo $finish_status
+echo "============================"
+echo "Script run successfully: $finish_status"
+echo "============================"
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+# Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+ echo "restart next experiment..."
+ this_script="${RUNSCRIPTDIR}/exp.${EXPNAME}.run"
+ echo 'this_script: ' "$this_script"
+ # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+ touch ${restartSemaphoreFilename}
+ cd ${RUNSCRIPTDIR}
+ sbatch exp.${EXPNAME}.run
+else
+ [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------