From 8df9a02e8218a23433837b09e853a52b1e917683 Mon Sep 17 00:00:00 2001
From: Voigt Aiko <avoigt_teach@l41.vsc.xcat>
Date: Tue, 30 Apr 2024 14:47:36 +0200
Subject: [PATCH] Adds run scripts and gitignore

---
 .gitignore                         |   5 +
 runs/slab4x-sun/exp.slab4x-sun.run | 574 ++++++++++++++++++++++++++++
 runs/slab4x-vap/exp.slab4x-vap.run | 575 +++++++++++++++++++++++++++++
 runs/slab4x/exp.slab4x.run         | 573 ++++++++++++++++++++++++++++
 runs/slabctr/exp.slabctr.run       | 573 ++++++++++++++++++++++++++++
 5 files changed, 2300 insertions(+)
 create mode 100644 .gitignore
 create mode 100755 runs/slab4x-sun/exp.slab4x-sun.run
 create mode 100755 runs/slab4x-vap/exp.slab4x-vap.run
 create mode 100755 runs/slab4x/exp.slab4x.run
 create mode 100755 runs/slabctr/exp.slabctr.run

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..8ede629
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+runs/amip/logfiles/
+runs/slab4x-sun/logfiles/
+runs/slab4x-vap/logfiles/
+runs/slab4x/logfiles/
+runs/slabctr/logfiles/
\ No newline at end of file
diff --git a/runs/slab4x-sun/exp.slab4x-sun.run b/runs/slab4x-sun/exp.slab4x-sun.run
new file mode 100755
index 0000000..acaf061
--- /dev/null
+++ b/runs/slab4x-sun/exp.slab4x-sun.run
@@ -0,0 +1,574 @@
+#! /bin/ksh
+#=============================================================================
+#SBATCH --account=p72044
+#SBATCH --partition=skylake_0096
+#SBATCH --qos=skylake_0096
+#SBATCH --job-name=slab4x-sun
+#SBATCH --nodes=3
+#SBATCH --ntasks-per-node=48
+#SBATCH --ntasks-per-core=1
+#SBATCH --output=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x-sun/logfiles/LOG.exp.slab4x-sun.run.%j.o
+#SBATCH --error=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x-sun/logfiles/LOG.exp.slab4x-sun.run.%j.o
+#SBATCH --exclusive
+#SBATCH --time=03:00:00
+#SBATCH --mail-user=aiko.voigt@univie.ac.at
+#SBATCH --mail-type=BEGIN,END,FAIL
+
+set -x # debugging command: enables a mode of the shell where all executed commands are printed to the terminal
+ulimit -s unlimited # unsets limits for RAM
+
+# MPI variables
+# -------------
+no_of_nodes=3
+mpi_procs_pernode=48
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+echo $mpi_total_procs
+
+# manual fix for mpi pinning with intel mpirun on vsc
+# see https://wiki.vsc.ac.at/doku.php?id=doku:vsc5quickstart#intel_mpi
+export I_MPI_PIN_RESPECT_CPUSET=0
+
+#
+# blocking length
+# ---------------
+nproma=16
+
+#=============================================================================
+# Input variables:
+
+# SIMULATION NAME
+EXP=slab4x-sun
+
+ICONFOLDER=/home/fs72044/avoigt_teach/climlab_s2024/icon-esm-univie       # DIRECTORY OF ICON MODEL CODE
+RUNSCRIPTDIR=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x-sun/
+basedir=$ICONFOLDER # icon base directory
+
+. ${ICONFOLDER}/run/add_run_routines
+
+# experiment directory, with plenty of space, create if new
+EXPDIR=/gpfs/data/fs72044/avoigt_teach/experiments/s2024/${EXP}
+if [ ! -d ${EXPDIR} ] ;  then
+  mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ;  then
+    mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+
+cd $EXPDIR
+
+
+
+
+#=================================================================================
+
+#-----------------------------------------------------------------------------
+# global timing
+initial_date="1979-01-01"
+final_date="2039-01-01"
+start_date=$initial_date
+end_date=$final_date
+y0=${start_date%%-*}
+yN=${end_date%%-*}
+
+
+# restart intervals
+restart_interval="P2Y"
+checkpoint_interval="P1Y"
+
+file_interval="P1M"
+
+############################################################
+#
+# NO FURTHER CHANGES TO THE DIRECTORIES AND SIMULATION NAME
+# SHOULD BE NEEDED BELOW THIS LINE
+#
+############################################################
+
+#-----------------------------------------------------------------------------
+# Provide input files
+# $Id: format.tmpl 9264 2021-06-21 21:24:57Z m221078 $
+#
+# [files]
+
+# [files.atmosphere]
+data_dir=/gpfs/data/fs72044/avoigt_teach/ICON-inputdata/amip-VSC4
+
+# [files.atmosphere.mapped]
+grid_dir=$data_dir/grid
+ln -sfv $grid_dir/icon_grid_0013_R02B04_G.nc icon_grid_G.nc
+
+# [files.atmosphere.mapped.initial]
+initial_dir=$data_dir/initial_condition
+ln -sfv $initial_dir/ifs2icon_1979010100_R02B04_G.nc ifs2icon.nc
+
+# [files.atmosphere.mapped.ozone]
+for((yr = y0 + -1; yr <= yN + 1; ++yr)) 
+do
+    label=${yr}
+        ((yr >= 2015)) && label=2014
+    ozone_dir=$data_dir/ozone
+    ln -sfv $ozone_dir/bc_ozone_historical_1979-2008.ymonmean.${label}.nc bc_ozone_${yr}.nc
+done # offsets
+
+# [files.atmosphere.mapped.ocean_surface]
+ocean_surface_dir=$data_dir/sst_and_seaice
+ln -sfv $ocean_surface_dir/bc_sic_1979_2008.ymonmean.1978-2079.nc bc_sic.nc
+ln -sfv $ocean_surface_dir/bc_sst_1979_2008.ymonmean.1978-2079.nc bc_sst.nc
+
+# files for slab ocean
+# sst, sic, and seb
+ln -sfv $ocean_surface_dir/bc_sic_1979_2008.ymonmean.1978-2079.nc bc_mlo_sic.nc
+ln -sfv $ocean_surface_dir/bc_sst_1979_2008.ymonmean.1978-2079.nc bc_mlo_sst.nc
+# q-flux file is taken from Phaidra archive of S2023 course
+ln -sfv /gpfs/data/fs72044/avoigt_teach/ICON-inputdata/slabocean/sstclim_seb_atm_seb_2d_ml_1980-2008.ymonmean.seb_wtr.addc_3.1970-2069.nc bc_mlo_seb.nc
+
+# [files.atmosphere.mapped.aerosols]
+aerosols_dir=$data_dir/aerosol
+ln -sfv $aerosols_dir/bc_aeropt_kinne_lw_b16_coa.nc .
+ln -sfv $aerosols_dir/bc_aeropt_kinne_sw_b14_coa.nc .
+ln -sfv $aerosols_dir/bc_aeropt_kinne_sw_b14_fin_1850.nc bc_aeropt_kinne_sw_b14_fin.nc
+
+# [files.atmosphere.model]
+model_dir=/home/fs72044/avoigt_teach/climlab_s2024/icon-esm-univie
+
+# [files.atmosphere.model.data]
+ln -sfv $model_dir/data/lsdata.nc .
+ln -sfv $model_dir/data/ECHAM6_CldOptProps.nc .
+ln -sfv $model_dir/data/MACv2.0-SP_v1.nc .
+
+# [files.atmosphere.model.run]
+run_dir=$model_dir/run
+cp -fv $run_dir/dict.iconam.mpim dict.txt
+
+# [files.atmosphere.independent.volcano_aerosols]
+for((yr = y0 + -1; yr <= yN + 1; ++yr)) 
+do
+    volcano_aerosols_dir=$data_dir/aerosol
+    ln -sfv $volcano_aerosols_dir/bc_aeropt_cmip6_volc_lw_b16_sw_b14_2000.nc \
+        bc_aeropt_cmip6_volc_lw_b16_sw_b14_${yr}.nc
+done # offsets
+
+
+# [files.land]
+land_dir=$data_dir/land
+
+# [files.land.mapped]
+ln -sfv $land_dir/ic_land_soil_1976.nc ic_land_soil.nc
+ln -sfv $land_dir/bc_land_frac_11pfts_1976.nc bc_land_frac.nc
+ln -sfv $land_dir/bc_land_phys_1976.nc bc_land_phys.nc
+ln -sfv $land_dir/bc_land_soil_1976.nc bc_land_soil.nc
+ln -sfv $land_dir/bc_land_sso_1976.nc bc_land_sso.nc
+
+# [files.land.hydro]
+hydro_dir=$land_dir
+# preliminary test version
+ln -sfv $hydro_dir/hdpara_r2b4_0013_0035_v3.nc bc_land_hd.nc
+
+# [files.land.model]
+model_dir=$basedir/externals/jsbach/data
+ln -sfv $model_dir/lctlib_nlct21.def .
+
+
+#-----------------------------------------------------------------------------
+# automatic restart setup
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+  restart=.true.
+  #  do not delete switch-file, to enable restart after unintended abort
+  #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+  if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+    sleep 10;
+  fi
+fi
+
+
+
+#
+# create ICON master, coupling and model namelists
+# ------------------------------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+#
+
+cat > icon_master.namelist << EOF
+&master_nml
+    lrestart =  ${restart}
+/
+&master_time_control_nml
+    calendar = 'proleptic gregorian'
+    checkpointtimeintval = '$checkpoint_interval'
+    restarttimeintval = '$restart_interval'
+    experimentstartdate = '1979-01-01' ! TODO: hack to reproduce result
+    experimentstopdate = '$final_date'
+/
+&master_model_nml ! 'atmo'
+    model_name = 'atmo'
+    model_namelist_filename = 'NAMELIST_atm'
+    model_type = 1
+/
+&jsb_control_nml
+    is_standalone = .false.
+/
+&jsb_model_nml
+    model_id = 1
+    model_name = 'JSBACH'
+    model_shortname = 'jsb'
+    model_description = 'JSBACH land surface model'
+    model_namelist_filename = 'NAMELIST_lnd'
+/
+
+EOF
+
+#-----------------------------------------------------------------------------
+# II. ATMOSPHERE and LAND
+#-----------------------------------------------------------------------------
+#
+# atmosphere namelist
+# -------------------
+cat > NAMELIST_atm << EOF
+&parallel_nml
+    nproma = $nproma
+    num_io_procs = 0
+    num_prefetch_proc = 0
+    pio_type = 1 !1=default, assync I/O, 2=experimental CDI, 0=nothing?
+/
+&grid_nml
+    dynamics_grid_filename = 'icon_grid_G.nc'
+/
+&run_nml
+    num_lev = 47 ! number of full levels
+    modeltimestep = 'PT15M'
+    ltestcase = .false. ! run testcase
+    ldynamics = .true. ! dynamics
+    ltransport = .true. ! transport
+    iforcing = 2 ! 0: none, 1: HS, 2: ECHAM, 3: NWP
+    output = 'nml'
+    msg_level = 8 ! level of details report during integration
+    restart_filename = '${EXP}_restart_atm_<rsttime>.nc'
+    activate_sync_timers = .true.
+/
+&extpar_nml
+    itopo = 1 ! 1: read topography from the grid file
+    itype_lwemiss = 0
+/
+&initicon_nml
+    init_mode = 2 ! 2: initialize from IFS analysis
+    ifs2icon_filename = 'ifs2icon.nc'
+/
+&nonhydrostatic_nml
+    ndyn_substeps = 8 ! dtime/dt_dyn
+    damp_height = 50000. ! [m]
+    rayleigh_coeff = 0.1000 ! set to 0.1001 for rerun with little change
+    vwind_offctr = 0.2
+    divdamp_fac = 0.004
+/
+&interpol_nml
+    rbf_scale_mode_ll = 1
+/
+&sleve_nml
+    min_lay_thckn = 40. ! [m]
+    top_height = 83000. ! [m]
+    stretch_fac = 0.9
+    decay_scale_1 = 4000. ! [m]
+    decay_scale_2 = 2500. ! [m]
+    decay_exp = 1.2
+    flat_height = 16000. ! [m]
+/
+&diffusion_nml
+!    hdiff_smag_fac = 0.015000001
+/
+&transport_nml
+    ihadv_tracer = 52, 2, 2
+    itype_hlimit = 3, 4, 4
+    ivadv_tracer = 3, 3, 3
+    tracer_names = 'hus', 'clw', 'cli'
+/
+&echam_phy_nml
+    ! domain 1
+    ! atmospheric physics ("" = never)
+    echam_phy_config(1)%dt_rad = 'PT90M'
+    echam_phy_config(1)%dt_vdf = 'PT15M'
+    echam_phy_config(1)%dt_cnv = 'PT15M'
+    echam_phy_config(1)%dt_cld = 'PT15M'
+    echam_phy_config(1)%dt_gwd = 'PT15M'
+    echam_phy_config(1)%dt_sso = 'PT15M'
+    ! atmospheric chemistry ("" = never)
+    echam_phy_config(1)%dt_mox = 'PT15M'
+    ! sea ice on mixed-layer ocean (""=never)
+    echam_phy_config(1)%dt_ice = 'PT15M'
+    ! surface (true or false)
+    echam_phy_config(1)%ljsb = .true.
+    echam_phy_config(1)%lamip = .false. 
+    echam_phy_config(1)%lice = .true.
+    echam_phy_config(1)%lmlo = .true.
+    echam_phy_config(1)%llake = .true.
+/
+&echam_rad_nml
+    ! domain 1
+    echam_rad_config(1)%isolrad = 3  ! Use insolation for AMIP type CMIP5 simulation (average from 1979-1988
+    echam_rad_config(1)%irad_h2o = 1
+    echam_rad_config(1)%irad_co2 = 2 ! constant concentration given by vmr_co2 etc.
+    echam_rad_config(1)%irad_ch4 = 2
+    echam_rad_config(1)%irad_n2o = 2
+    echam_rad_config(1)%irad_o3 = 8 ! constant annual cycle climatology
+    echam_rad_config(1)%irad_o2 = 2
+    echam_rad_config(1)%irad_cfc11 = 2
+    echam_rad_config(1)%irad_cfc12 = 2
+    echam_rad_config(1)%irad_aero = 18 ! as in AMIP
+    echam_rad_config(1)%vmr_co2 = 1436.0e-6 !359.0e-6 --> 4xCO2
+    echam_rad_config(1)%vmr_ch4 = 1693.0e-9
+    echam_rad_config(1)%vmr_n2o = 311.0e-9
+    echam_rad_config(1)%vmr_o2  = 0.20946
+    echam_rad_config(1)%vmr_cfc11 = 237.0e-12
+    echam_rad_config(1)%vmr_cfc12 = 462.0e-12
+    echam_rad_config(1)%fsolrad = 0.97  ! dim the sun by 3% to compensate for 4xco2
+/
+&echam_gwd_nml
+/
+&echam_sso_nml
+/
+&echam_vdf_nml
+/
+&echam_cnv_nml
+/
+&echam_cld_nml
+/
+&echam_cov_nml
+/
+&ccycle_nml
+/
+&sea_ice_nml
+ i_ice_therm  = 1    ! 1=0L-Semtner
+/
+&echam_seaice_mlo_nml
+ lqflux               = .true.  ! default .TRUE.
+ max_seaice_thickness = 99999.  ! default 5
+ qbot_mlo_nh          = 0.      ! default 10
+ qbot_mlo_sh          = 0.      ! default 10
+/
+! Parameters for all output files
+! -------------------------------
+&io_nml
+    output_nml_dict = 'dict.txt'
+    netcdf_dict = 'dict.txt'
+    itype_pres_msl = 4
+    ! restart_file_type = 5
+    ! restart_write_mode = 'joint procs multifile' ! not useful in r2b4 setup
+    ! lnetcdf_flt64_output = .true. ! 64 bit output in all files
+    ! lkeep_in_sync = .true. ! sync after each timestep
+    write_initial_state = .false.
+/
+&dbg_index_nml
+    idbg_mxmn = 0 ! initialize MIN/MAX debug output
+    idbg_val = 0 ! initialize one cell debug output
+    idbg_slev = 1 ! initialize start level for debug output
+    idbg_elev = 2 ! initialize end level for debug output
+    dbg_lat_in = 30.0 ! latitude location of one cell debug output
+    dbg_lon_in = -30.0 ! longitude location of one cell debug output
+    str_mod_tst = 'InterFaceOce' ! define modules to print out in debug mode
+/
+
+! Define output files
+! -------------------
+!
+! 3-dimensional files include 'ps' and 'pfull' to allow the vertical
+! interpolation to pressure levels by cdo ap2pl.
+
+/
+! Standard AMIP output...
+&output_nml ! 'atm_3d'
+    output_filename = '${EXP}_atm_3d'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1M'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'zg', 'ps', 'pfull', 'rho', 'ta', 'ua', 'va', 'wap', 'hus',
+                 'clw', 'cli', 'hur', 'cl'
+/
+! Standard AMIP output...
+&output_nml ! 'atm_2d'
+    output_filename = '${EXP}_atm_2d'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1M'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'orog', 'ps', 'psl', 'cosmu0', 'rsdt', 'rsut', 'rsutcs',
+                 'rlut', 'rlutcs', 'rsds', 'rsdscs', 'rlds', 'rldscs', 'rsus',
+                 'rsuscs', 'rlus', 'ts', 'sic', 'sit', 'albedo', 'clt', 'prlr',
+                 'prls', 'prcr', 'prcs', 'pr', 'prw', 'cllvi', 'clivi', 'hfls',
+                 'hfss', 'evspsbl', 'tauu', 'tauv', 'tauu_sso', 'tauv_sso',
+                 'diss_sso', 'sfcwind', 'uas', 'vas', 'tas', 'dew2', 'ptp'
+/
+! Standard AMIP output...for daily means
+&output_nml ! 'atm_2d'
+    output_filename = '${EXP}_atm_2d_daily'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1D'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'orog', 'ps', 'psl', 'cosmu0', 'rsdt', 'rsut', 'rsutcs',
+                 'rlut', 'rlutcs', 'rsds', 'rsdscs', 'rlds', 'rldscs', 'rsus',
+                 'rsuscs', 'rlus', 'ts', 'sic', 'sit', 'albedo', 'clt', 'prlr',
+                 'prls', 'prcr', 'prcs', 'pr', 'prw', 'cllvi', 'clivi', 'hfls',
+                 'hfss', 'evspsbl', 'tauu', 'tauv', 'tauu_sso', 'tauv_sso',
+                 'diss_sso', 'sfcwind', 'uas', 'vas', 'tas', 'dew2', 'ptp'
+/
+EOF
+
+
+# jsbach namelist
+# ---------------
+
+cat > NAMELIST_lnd << EOF
+
+&jsb_model_nml
+    usecase = 'jsbach_pfts'
+    use_lakes = .true.
+/
+&jsb_seb_nml
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_rad_nml
+    use_alb_veg_simple = .false. ! if jsbach_lite
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_turb_nml
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_sse_nml
+    l_heat_cap_map = .false.
+    l_heat_cond_map = .false.
+    l_heat_cap_dyn = .false.
+    l_heat_cond_dyn = .false.
+    l_snow = .true.
+    l_dynsnow = .true.
+    l_freeze = .true.
+    l_supercool = .true.
+    bc_filename = 'bc_land_soil.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_hydro_nml
+    l_organic = .false.
+    bc_filename = 'bc_land_soil.nc'
+    ic_filename = 'ic_land_soil.nc'
+    bc_sso_filename = 'bc_land_sso.nc'
+/
+&jsb_assimi_nml
+    active = .true. ! if jsbach_pfts
+/
+&jsb_pheno_nml
+    scheme = 'logrop' ! 'climatology' if jsbach_lite
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_carbon_nml
+    active = .true.
+    bc_filename = 'bc_land_carbon.nc'
+    ic_filename = 'ic_land_carbon.nc'
+    read_cpools = .false.
+/
+&jsb_fuel_nml
+    active = .true.
+    fuel_algorithm = 1
+/
+&jsb_disturb_nml
+    active = .false.
+    ic_filename = 'ic_land_soil.nc'
+    bc_filename = 'bc_land_phys.nc'
+    fire_algorithm = 1
+    windbreak_algorithm = 1
+    lburn_pasture = .false.
+/
+
+EOF
+
+
+## setup for status check & restart
+final_status_file=${EXPDIR}/${EXP}.final_status
+
+## Copy icon executable to working directory
+cp -p $ICONFOLDER/bin/icon ./icon.exe
+##
+
+## Start model
+date
+ulimit -s unlimited
+
+ldd icon.exe
+
+START="/gpfs/opt/sw/skylake/spack-0.19.0/opt/spack/linux-almalinux8-skylake_avx512/intel-2021.7.1/intel-oneapi-mpi-2021.7.1-fzg6q4xcj7efjmce3cuqa2b7cum5d3po/mpi/2021.7.1/bin/mpiexec -n $mpi_total_procs"
+MODEL=${EXPDIR}/icon.exe
+
+rm -f finish.status
+
+${START} ${MODEL}
+
+if [ -r finish.status ] ; then
+  check_final_status 0 "${START} ${MODEL}"
+else
+  check_final_status -1 "${START} ${MODEL}"
+fi
+
+#-----------------------------------------------------------------------------
+finish_status=`cat finish.status`
+echo $finish_status
+
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+#   Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+  echo "restart next experiment..."
+  this_script="${RUNSCRIPTDIR}/exp.${EXP}.run"
+  echo 'this_script: ' "$this_script"
+  # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+  touch ${restartSemaphoreFilename}
+  cd ${RUNSCRIPTDIR}
+  sbatch exp.${EXP}.run
+else
+  [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
+#
+
+echo "============================"
+echo "Script run successfully: ${finish_status}"
+echo "============================"
+#-----------------------------------------------------------------------------
diff --git a/runs/slab4x-vap/exp.slab4x-vap.run b/runs/slab4x-vap/exp.slab4x-vap.run
new file mode 100755
index 0000000..fddbe93
--- /dev/null
+++ b/runs/slab4x-vap/exp.slab4x-vap.run
@@ -0,0 +1,575 @@
+#! /bin/ksh
+#=============================================================================
+#SBATCH --account=p72044
+#SBATCH --partition=skylake_0096
+#SBATCH --qos=skylake_0096
+#SBATCH --job-name=slab4x-vap
+#SBATCH --nodes=3
+#SBATCH --ntasks-per-node=48
+#SBATCH --ntasks-per-core=1
+#SBATCH --output=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x-vap/logfiles/LOG.exp.slab4x-vap.run.%j.o
+#SBATCH --error=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x-vap/logfiles/LOG.exp.slab4x-vap.run.%j.o
+#SBATCH --exclusive
+#SBATCH --time=03:00:00
+#SBATCH --mail-user=aiko.voigt@univie.ac.at
+#SBATCH --mail-type=BEGIN,END,FAIL
+
+set -x # debugging command: enables a mode of the shell where all executed commands are printed to the terminal
+ulimit -s unlimited # unsets limits for RAM
+
+# MPI variables
+# -------------
+no_of_nodes=3
+mpi_procs_pernode=48
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+echo $mpi_total_procs
+
+# manual fix for mpi pinning with intel mpirun on vsc
+# see https://wiki.vsc.ac.at/doku.php?id=doku:vsc5quickstart#intel_mpi
+export I_MPI_PIN_RESPECT_CPUSET=0
+
+#
+# blocking length
+# ---------------
+nproma=16
+
+#=============================================================================
+# Input variables:
+
+# SIMULATION NAME
+EXP=slab4x-vap
+
+ICONFOLDER=/home/fs72044/avoigt_teach/climlab_s2024/icon-esm-univie       # DIRECTORY OF ICON MODEL CODE
+RUNSCRIPTDIR=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x-vap/
+basedir=$ICONFOLDER # icon base directory
+
+. ${ICONFOLDER}/run/add_run_routines
+
+# experiment directory, with plenty of space, create if new
+EXPDIR=/gpfs/data/fs72044/avoigt_teach/experiments/s2024/${EXP}
+if [ ! -d ${EXPDIR} ] ;  then
+  mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ;  then
+    mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+
+cd $EXPDIR
+
+
+
+
+#=================================================================================
+
+#-----------------------------------------------------------------------------
+# global timing
+initial_date="1979-01-01"
+final_date="2039-01-01"
+start_date=$initial_date
+end_date=$final_date
+y0=${start_date%%-*}
+yN=${end_date%%-*}
+
+
+# restart intervals
+restart_interval="P2Y"
+checkpoint_interval="P1Y"
+
+file_interval="P1M"
+
+############################################################
+#
+# NO FURTHER CHANGES TO THE DIRECTORIES AND SIMULATION NAME
+# SHOULD BE NEEDED BELOW THIS LINE
+#
+############################################################
+
+#-----------------------------------------------------------------------------
+# Provide input files
+# $Id: format.tmpl 9264 2021-06-21 21:24:57Z m221078 $
+#
+# [files]
+
+# [files.atmosphere]
+data_dir=/gpfs/data/fs72044/avoigt_teach/ICON-inputdata/amip-VSC4
+
+# [files.atmosphere.mapped]
+grid_dir=$data_dir/grid
+ln -sfv $grid_dir/icon_grid_0013_R02B04_G.nc icon_grid_G.nc
+
+# [files.atmosphere.mapped.initial]
+initial_dir=$data_dir/initial_condition
+ln -sfv $initial_dir/ifs2icon_1979010100_R02B04_G.nc ifs2icon.nc
+
+# [files.atmosphere.mapped.ozone]
+for((yr = y0 + -1; yr <= yN + 1; ++yr)) 
+do
+    label=${yr}
+        ((yr >= 2015)) && label=2014
+    ozone_dir=$data_dir/ozone
+    ln -sfv $ozone_dir/bc_ozone_historical_1979-2008.ymonmean.${label}.nc bc_ozone_${yr}.nc
+done # offsets
+
+# [files.atmosphere.mapped.ocean_surface]
+ocean_surface_dir=$data_dir/sst_and_seaice
+ln -sfv $ocean_surface_dir/bc_sic_1979_2008.ymonmean.1978-2079.nc bc_sic.nc
+ln -sfv $ocean_surface_dir/bc_sst_1979_2008.ymonmean.1978-2079.nc bc_sst.nc
+
+# files for slab ocean
+# sst, sic, and seb
+ln -sfv $ocean_surface_dir/bc_sic_1979_2008.ymonmean.1978-2079.nc bc_mlo_sic.nc
+ln -sfv $ocean_surface_dir/bc_sst_1979_2008.ymonmean.1978-2079.nc bc_mlo_sst.nc
+# q-flux file is taken from Phaidra archive of S2023 course
+ln -sfv /gpfs/data/fs72044/avoigt_teach/ICON-inputdata/slabocean/sstclim_seb_atm_seb_2d_ml_1980-2008.ymonmean.seb_wtr.addc_3.1970-2069.nc bc_mlo_seb.nc
+
+# [files.atmosphere.mapped.aerosols]
+aerosols_dir=$data_dir/aerosol
+ln -sfv $aerosols_dir/bc_aeropt_kinne_lw_b16_coa.nc .
+ln -sfv $aerosols_dir/bc_aeropt_kinne_sw_b14_coa.nc .
+ln -sfv $aerosols_dir/bc_aeropt_kinne_sw_b14_fin_1850.nc bc_aeropt_kinne_sw_b14_fin.nc
+
+# [files.atmosphere.model]
+model_dir=/home/fs72044/avoigt_teach/climlab_s2024/icon-esm-univie
+
+# [files.atmosphere.model.data]
+ln -sfv $model_dir/data/lsdata.nc .
+ln -sfv $model_dir/data/ECHAM6_CldOptProps.nc .
+ln -sfv $model_dir/data/MACv2.0-SP_v1.nc .
+
+# [files.atmosphere.model.run]
+run_dir=$model_dir/run
+cp -fv $run_dir/dict.iconam.mpim dict.txt
+
+# [files.atmosphere.independent.volcano_aerosols]
+for((yr = y0 + -1; yr <= yN + 1; ++yr)) 
+do
+    volcano_aerosols_dir=$data_dir/aerosol
+    ln -sfv $volcano_aerosols_dir/bc_aeropt_cmip6_volc_lw_b16_sw_b14_2000.nc \
+        bc_aeropt_cmip6_volc_lw_b16_sw_b14_${yr}.nc
+done # offsets
+
+
+# [files.land]
+land_dir=$data_dir/land
+
+# [files.land.mapped]
+ln -sfv $land_dir/ic_land_soil_1976.nc ic_land_soil.nc
+ln -sfv $land_dir/bc_land_frac_11pfts_1976.nc bc_land_frac.nc
+ln -sfv $land_dir/bc_land_phys_1976.nc bc_land_phys.nc
+ln -sfv $land_dir/bc_land_soil_1976.nc bc_land_soil.nc
+ln -sfv $land_dir/bc_land_sso_1976.nc bc_land_sso.nc
+
+# [files.land.hydro]
+hydro_dir=$land_dir
+# preliminary test version
+ln -sfv $hydro_dir/hdpara_r2b4_0013_0035_v3.nc bc_land_hd.nc
+
+# [files.land.model]
+model_dir=$basedir/externals/jsbach/data
+ln -sfv $model_dir/lctlib_nlct21.def .
+
+
+#-----------------------------------------------------------------------------
+# automatic restart setup
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+  restart=.true.
+  #  do not delete switch-file, to enable restart after unintended abort
+  #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+  if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+    sleep 10;
+  fi
+fi
+
+
+
+#
+# create ICON master, coupling and model namelists
+# ------------------------------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+#
+
+cat > icon_master.namelist << EOF
+&master_nml
+    lrestart =  ${restart}
+/
+&master_time_control_nml
+    calendar = 'proleptic gregorian'
+    checkpointtimeintval = '$checkpoint_interval'
+    restarttimeintval = '$restart_interval'
+    experimentstartdate = '1979-01-01' ! TODO: hack to reproduce result
+    experimentstopdate = '$final_date'
+/
+&master_model_nml ! 'atmo'
+    model_name = 'atmo'
+    model_namelist_filename = 'NAMELIST_atm'
+    model_type = 1
+/
+&jsb_control_nml
+    is_standalone = .false.
+/
+&jsb_model_nml
+    model_id = 1
+    model_name = 'JSBACH'
+    model_shortname = 'jsb'
+    model_description = 'JSBACH land surface model'
+    model_namelist_filename = 'NAMELIST_lnd'
+/
+
+EOF
+
+#-----------------------------------------------------------------------------
+# II. ATMOSPHERE and LAND
+#-----------------------------------------------------------------------------
+#
+# atmosphere namelist
+# -------------------
+cat > NAMELIST_atm << EOF
+&parallel_nml
+    nproma = $nproma
+    num_io_procs = 0
+    num_prefetch_proc = 0
+    pio_type = 1 !1=default, assync I/O, 2=experimental CDI, 0=nothing?
+/
+&grid_nml
+    dynamics_grid_filename = 'icon_grid_G.nc'
+/
+&run_nml
+    num_lev = 47 ! number of full levels
+    modeltimestep = 'PT15M'
+    ltestcase = .false. ! run testcase
+    ldynamics = .true. ! dynamics
+    ltransport = .true. ! transport
+    iforcing = 2 ! 0: none, 1: HS, 2: ECHAM, 3: NWP
+    output = 'nml'
+    msg_level = 8 ! level of details report during integration
+    restart_filename = '${EXP}_restart_atm_<rsttime>.nc'
+    activate_sync_timers = .true.
+/
+&extpar_nml
+    itopo = 1 ! 1: read topography from the grid file
+    itype_lwemiss = 0
+/
+&initicon_nml
+    init_mode = 2 ! 2: initialize from IFS analysis
+    ifs2icon_filename = 'ifs2icon.nc'
+/
+&nonhydrostatic_nml
+    ndyn_substeps = 8 ! dtime/dt_dyn
+    damp_height = 50000. ! [m]
+    rayleigh_coeff = 0.1000 ! set to 0.1001 for rerun with little change
+    vwind_offctr = 0.2
+    divdamp_fac = 0.004
+/
+&interpol_nml
+    rbf_scale_mode_ll = 1
+/
+&sleve_nml
+    min_lay_thckn = 40. ! [m]
+    top_height = 83000. ! [m]
+    stretch_fac = 0.9
+    decay_scale_1 = 4000. ! [m]
+    decay_scale_2 = 2500. ! [m]
+    decay_exp = 1.2
+    flat_height = 16000. ! [m]
+/
+&diffusion_nml
+!    hdiff_smag_fac = 0.015000001
+/
+&transport_nml
+    ihadv_tracer = 52, 2, 2
+    itype_hlimit = 3, 4, 4
+    ivadv_tracer = 3, 3, 3
+    tracer_names = 'hus', 'clw', 'cli'
+/
+&echam_phy_nml
+    ! domain 1
+    ! atmospheric physics ("" = never)
+    echam_phy_config(1)%dt_rad = 'PT90M'
+    echam_phy_config(1)%dt_vdf = 'PT15M'
+    echam_phy_config(1)%dt_cnv = 'PT15M'
+    echam_phy_config(1)%dt_cld = 'PT15M'
+    echam_phy_config(1)%dt_gwd = 'PT15M'
+    echam_phy_config(1)%dt_sso = 'PT15M'
+    ! atmospheric chemistry ("" = never)
+    echam_phy_config(1)%dt_mox = 'PT15M'
+    ! sea ice on mixed-layer ocean (""=never)
+    echam_phy_config(1)%dt_ice = 'PT15M'
+    ! surface (true or false)
+    echam_phy_config(1)%ljsb = .true.
+    echam_phy_config(1)%lamip = .false. 
+    echam_phy_config(1)%lice = .true.
+    echam_phy_config(1)%lmlo = .true.
+    echam_phy_config(1)%llake = .true.
+/
+&echam_rad_nml
+    ! domain 1
+    echam_rad_config(1)%isolrad = 3  ! Use insolation for AMIP type CMIP5 simulation (average from 1979-1988
+    echam_rad_config(1)%irad_h2o = 1
+    echam_rad_config(1)%irad_co2 = 2 ! constant concentration given by vmr_co2 etc.
+    echam_rad_config(1)%irad_ch4 = 2
+    echam_rad_config(1)%irad_n2o = 2
+    echam_rad_config(1)%irad_o3 = 8 ! constant annual cycle climatology
+    echam_rad_config(1)%irad_o2 = 2
+    echam_rad_config(1)%irad_cfc11 = 2
+    echam_rad_config(1)%irad_cfc12 = 2
+    echam_rad_config(1)%irad_aero = 18 ! as in AMIP
+    echam_rad_config(1)%vmr_co2 = 1436.0e-6 !359.0e-6 --> 4xCO2
+    echam_rad_config(1)%vmr_ch4 = 1693.0e-9
+    echam_rad_config(1)%vmr_n2o = 311.0e-9
+    echam_rad_config(1)%vmr_o2  = 0.20946
+    echam_rad_config(1)%vmr_cfc11 = 237.0e-12
+    echam_rad_config(1)%vmr_cfc12 = 462.0e-12
+    echam_rad_config(1)%l_stratvapor_zero = .true. ! transparent stratospheric water vapor
+    echam_rad_config(1)%klev_stratvapor_zero = 22  ! transparent stratospheric water vapor
+/
+&echam_gwd_nml
+/
+&echam_sso_nml
+/
+&echam_vdf_nml
+/
+&echam_cnv_nml
+/
+&echam_cld_nml
+/
+&echam_cov_nml
+/
+&ccycle_nml
+/
+&sea_ice_nml
+ i_ice_therm  = 1    ! 1=0L-Semtner
+/
+&echam_seaice_mlo_nml
+ lqflux               = .true.  ! default .TRUE.
+ max_seaice_thickness = 99999.  ! default 5
+ qbot_mlo_nh          = 0.      ! default 10
+ qbot_mlo_sh          = 0.      ! default 10
+/
+! Parameters for all output files
+! -------------------------------
+&io_nml
+    output_nml_dict = 'dict.txt'
+    netcdf_dict = 'dict.txt'
+    itype_pres_msl = 4
+    ! restart_file_type = 5
+    ! restart_write_mode = 'joint procs multifile' ! not useful in r2b4 setup
+    ! lnetcdf_flt64_output = .true. ! 64 bit output in all files
+    ! lkeep_in_sync = .true. ! sync after each timestep
+    write_initial_state = .false.
+/
+&dbg_index_nml
+    idbg_mxmn = 0 ! initialize MIN/MAX debug output
+    idbg_val = 0 ! initialize one cell debug output
+    idbg_slev = 1 ! initialize start level for debug output
+    idbg_elev = 2 ! initialize end level for debug output
+    dbg_lat_in = 30.0 ! latitude location of one cell debug output
+    dbg_lon_in = -30.0 ! longitude location of one cell debug output
+    str_mod_tst = 'InterFaceOce' ! define modules to print out in debug mode
+/
+
+! Define output files
+! -------------------
+!
+! 3-dimensional files include 'ps' and 'pfull' to allow the vertical
+! interpolation to pressure levels by cdo ap2pl.
+
+/
+! Standard AMIP output...
+&output_nml ! 'atm_3d'
+    output_filename = '${EXP}_atm_3d'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1M'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'zg', 'ps', 'pfull', 'rho', 'ta', 'ua', 'va', 'wap', 'hus',
+                 'clw', 'cli', 'hur', 'cl'
+/
+! Standard AMIP output...
+&output_nml ! 'atm_2d'
+    output_filename = '${EXP}_atm_2d'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1M'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'orog', 'ps', 'psl', 'cosmu0', 'rsdt', 'rsut', 'rsutcs',
+                 'rlut', 'rlutcs', 'rsds', 'rsdscs', 'rlds', 'rldscs', 'rsus',
+                 'rsuscs', 'rlus', 'ts', 'sic', 'sit', 'albedo', 'clt', 'prlr',
+                 'prls', 'prcr', 'prcs', 'pr', 'prw', 'cllvi', 'clivi', 'hfls',
+                 'hfss', 'evspsbl', 'tauu', 'tauv', 'tauu_sso', 'tauv_sso',
+                 'diss_sso', 'sfcwind', 'uas', 'vas', 'tas', 'dew2', 'ptp'
+/
+! Standard AMIP output...for daily means
+&output_nml ! 'atm_2d'
+    output_filename = '${EXP}_atm_2d_daily'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1D'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'orog', 'ps', 'psl', 'cosmu0', 'rsdt', 'rsut', 'rsutcs',
+                 'rlut', 'rlutcs', 'rsds', 'rsdscs', 'rlds', 'rldscs', 'rsus',
+                 'rsuscs', 'rlus', 'ts', 'sic', 'sit', 'albedo', 'clt', 'prlr',
+                 'prls', 'prcr', 'prcs', 'pr', 'prw', 'cllvi', 'clivi', 'hfls',
+                 'hfss', 'evspsbl', 'tauu', 'tauv', 'tauu_sso', 'tauv_sso',
+                 'diss_sso', 'sfcwind', 'uas', 'vas', 'tas', 'dew2', 'ptp'
+/
+EOF
+
+
+# jsbach namelist
+# ---------------
+
+cat > NAMELIST_lnd << EOF
+
+&jsb_model_nml
+    usecase = 'jsbach_pfts'
+    use_lakes = .true.
+/
+&jsb_seb_nml
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_rad_nml
+    use_alb_veg_simple = .false. ! if jsbach_lite
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_turb_nml
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_sse_nml
+    l_heat_cap_map = .false.
+    l_heat_cond_map = .false.
+    l_heat_cap_dyn = .false.
+    l_heat_cond_dyn = .false.
+    l_snow = .true.
+    l_dynsnow = .true.
+    l_freeze = .true.
+    l_supercool = .true.
+    bc_filename = 'bc_land_soil.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_hydro_nml
+    l_organic = .false.
+    bc_filename = 'bc_land_soil.nc'
+    ic_filename = 'ic_land_soil.nc'
+    bc_sso_filename = 'bc_land_sso.nc'
+/
+&jsb_assimi_nml
+    active = .true. ! if jsbach_pfts
+/
+&jsb_pheno_nml
+    scheme = 'logrop' ! 'climatology' if jsbach_lite
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_carbon_nml
+    active = .true.
+    bc_filename = 'bc_land_carbon.nc'
+    ic_filename = 'ic_land_carbon.nc'
+    read_cpools = .false.
+/
+&jsb_fuel_nml
+    active = .true.
+    fuel_algorithm = 1
+/
+&jsb_disturb_nml
+    active = .false.
+    ic_filename = 'ic_land_soil.nc'
+    bc_filename = 'bc_land_phys.nc'
+    fire_algorithm = 1
+    windbreak_algorithm = 1
+    lburn_pasture = .false.
+/
+
+EOF
+
+
+## setup for status check & restart
+final_status_file=${EXPDIR}/${EXP}.final_status
+
+## Copy icon executable to working directory
+cp -p $ICONFOLDER/bin/icon ./icon.exe
+##
+
+## Start model
+date
+ulimit -s unlimited
+
+ldd icon.exe
+
+START="/gpfs/opt/sw/skylake/spack-0.19.0/opt/spack/linux-almalinux8-skylake_avx512/intel-2021.7.1/intel-oneapi-mpi-2021.7.1-fzg6q4xcj7efjmce3cuqa2b7cum5d3po/mpi/2021.7.1/bin/mpiexec -n $mpi_total_procs"
+MODEL=${EXPDIR}/icon.exe
+
+rm -f finish.status
+
+${START} ${MODEL}
+
+if [ -r finish.status ] ; then
+  check_final_status 0 "${START} ${MODEL}"
+else
+  check_final_status -1 "${START} ${MODEL}"
+fi
+
+#-----------------------------------------------------------------------------
+finish_status=`cat finish.status`
+echo $finish_status
+
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+#   Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+  echo "restart next experiment..."
+  this_script="${RUNSCRIPTDIR}/exp.${EXP}.run"
+  echo 'this_script: ' "$this_script"
+  # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+  touch ${restartSemaphoreFilename}
+  cd ${RUNSCRIPTDIR}
+  sbatch exp.${EXP}.run
+else
+  [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
+#
+
+echo "============================"
+echo "Script run successfully: ${finish_status}"
+echo "============================"
+#-----------------------------------------------------------------------------
diff --git a/runs/slab4x/exp.slab4x.run b/runs/slab4x/exp.slab4x.run
new file mode 100755
index 0000000..1aa8818
--- /dev/null
+++ b/runs/slab4x/exp.slab4x.run
@@ -0,0 +1,573 @@
+#! /bin/ksh
+#=============================================================================
+#SBATCH --account=p72044
+#SBATCH --partition=skylake_0096
+#SBATCH --qos=skylake_0096
+#SBATCH --job-name=slab4x
+#SBATCH --nodes=3
+#SBATCH --ntasks-per-node=48
+#SBATCH --ntasks-per-core=1
+#SBATCH --output=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x/logfiles/LOG.exp.slab4x.run.%j.o
+#SBATCH --error=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x/logfiles/LOG.exp.slab4x.run.%j.o
+#SBATCH --exclusive
+#SBATCH --time=03:00:00
+#SBATCH --mail-user=aiko.voigt@univie.ac.at
+#SBATCH --mail-type=BEGIN,END,FAIL
+
+set -x # debugging command: enables a mode of the shell where all executed commands are printed to the terminal
+ulimit -s unlimited # unsets limits for RAM
+
+# MPI variables
+# -------------
+no_of_nodes=3
+mpi_procs_pernode=48
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+echo $mpi_total_procs
+
+# manual fix for mpi pinning with intel mpirun on vsc
+# see https://wiki.vsc.ac.at/doku.php?id=doku:vsc5quickstart#intel_mpi
+export I_MPI_PIN_RESPECT_CPUSET=0
+
+#
+# blocking length
+# ---------------
+nproma=16
+
+#=============================================================================
+# Input variables:
+
+# SIMULATION NAME
+EXP=slab4x
+
+ICONFOLDER=/home/fs72044/avoigt_teach/climlab_s2024/icon-esm-univie       # DIRECTORY OF ICON MODEL CODE
+RUNSCRIPTDIR=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slab4x/
+basedir=$ICONFOLDER # icon base directory
+
+. ${ICONFOLDER}/run/add_run_routines
+
+# experiment directory, with plenty of space, create if new
+EXPDIR=/gpfs/data/fs72044/avoigt_teach/experiments/s2024/${EXP}
+if [ ! -d ${EXPDIR} ] ;  then
+  mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ;  then
+    mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+
+cd $EXPDIR
+
+
+
+
+#=================================================================================
+
+#-----------------------------------------------------------------------------
+# global timing
+initial_date="1979-01-01"
+final_date="2039-01-01"
+start_date=$initial_date
+end_date=$final_date
+y0=${start_date%%-*}
+yN=${end_date%%-*}
+
+
+# restart intervals
+restart_interval="P2Y"
+checkpoint_interval="P1Y"
+
+file_interval="P1M"
+
+############################################################
+#
+# NO FURTHER CHANGES TO THE DIRECTORIES AND SIMULATION NAME
+# SHOULD BE NEEDED BELOW THIS LINE
+#
+############################################################
+
+#-----------------------------------------------------------------------------
+# Provide input files
+# $Id: format.tmpl 9264 2021-06-21 21:24:57Z m221078 $
+#
+# [files]
+
+# [files.atmosphere]
+data_dir=/gpfs/data/fs72044/avoigt_teach/ICON-inputdata/amip-VSC4
+
+# [files.atmosphere.mapped]
+grid_dir=$data_dir/grid
+ln -sfv $grid_dir/icon_grid_0013_R02B04_G.nc icon_grid_G.nc
+
+# [files.atmosphere.mapped.initial]
+initial_dir=$data_dir/initial_condition
+ln -sfv $initial_dir/ifs2icon_1979010100_R02B04_G.nc ifs2icon.nc
+
+# [files.atmosphere.mapped.ozone]
+for((yr = y0 + -1; yr <= yN + 1; ++yr)) 
+do
+    label=${yr}
+        ((yr >= 2015)) && label=2014
+    ozone_dir=$data_dir/ozone
+    ln -sfv $ozone_dir/bc_ozone_historical_1979-2008.ymonmean.${label}.nc bc_ozone_${yr}.nc
+done # offsets
+
+# [files.atmosphere.mapped.ocean_surface]
+ocean_surface_dir=$data_dir/sst_and_seaice
+ln -sfv $ocean_surface_dir/bc_sic_1979_2008.ymonmean.1978-2079.nc bc_sic.nc
+ln -sfv $ocean_surface_dir/bc_sst_1979_2008.ymonmean.1978-2079.nc bc_sst.nc
+
+# files for slab ocean
+# sst, sic, and seb
+ln -sfv $ocean_surface_dir/bc_sic_1979_2008.ymonmean.1978-2079.nc bc_mlo_sic.nc
+ln -sfv $ocean_surface_dir/bc_sst_1979_2008.ymonmean.1978-2079.nc bc_mlo_sst.nc
+# q-flux file is taken from Phaidra archive of S2023 course
+ln -sfv /gpfs/data/fs72044/avoigt_teach/ICON-inputdata/slabocean/sstclim_seb_atm_seb_2d_ml_1980-2008.ymonmean.seb_wtr.addc_3.1970-2069.nc bc_mlo_seb.nc
+
+# [files.atmosphere.mapped.aerosols]
+aerosols_dir=$data_dir/aerosol
+ln -sfv $aerosols_dir/bc_aeropt_kinne_lw_b16_coa.nc .
+ln -sfv $aerosols_dir/bc_aeropt_kinne_sw_b14_coa.nc .
+ln -sfv $aerosols_dir/bc_aeropt_kinne_sw_b14_fin_1850.nc bc_aeropt_kinne_sw_b14_fin.nc
+
+# [files.atmosphere.model]
+model_dir=/home/fs72044/avoigt_teach/climlab_s2024/icon-esm-univie
+
+# [files.atmosphere.model.data]
+ln -sfv $model_dir/data/lsdata.nc .
+ln -sfv $model_dir/data/ECHAM6_CldOptProps.nc .
+ln -sfv $model_dir/data/MACv2.0-SP_v1.nc .
+
+# [files.atmosphere.model.run]
+run_dir=$model_dir/run
+cp -fv $run_dir/dict.iconam.mpim dict.txt
+
+# [files.atmosphere.independent.volcano_aerosols]
+for((yr = y0 + -1; yr <= yN + 1; ++yr)) 
+do
+    volcano_aerosols_dir=$data_dir/aerosol
+    ln -sfv $volcano_aerosols_dir/bc_aeropt_cmip6_volc_lw_b16_sw_b14_2000.nc \
+        bc_aeropt_cmip6_volc_lw_b16_sw_b14_${yr}.nc
+done # offsets
+
+
+# [files.land]
+land_dir=$data_dir/land
+
+# [files.land.mapped]
+ln -sfv $land_dir/ic_land_soil_1976.nc ic_land_soil.nc
+ln -sfv $land_dir/bc_land_frac_11pfts_1976.nc bc_land_frac.nc
+ln -sfv $land_dir/bc_land_phys_1976.nc bc_land_phys.nc
+ln -sfv $land_dir/bc_land_soil_1976.nc bc_land_soil.nc
+ln -sfv $land_dir/bc_land_sso_1976.nc bc_land_sso.nc
+
+# [files.land.hydro]
+hydro_dir=$land_dir
+# preliminary test version
+ln -sfv $hydro_dir/hdpara_r2b4_0013_0035_v3.nc bc_land_hd.nc
+
+# [files.land.model]
+model_dir=$basedir/externals/jsbach/data
+ln -sfv $model_dir/lctlib_nlct21.def .
+
+
+#-----------------------------------------------------------------------------
+# automatic restart setup
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+  restart=.true.
+  #  do not delete switch-file, to enable restart after unintended abort
+  #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+  if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+    sleep 10;
+  fi
+fi
+
+
+
+#
+# create ICON master, coupling and model namelists
+# ------------------------------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+#
+
+cat > icon_master.namelist << EOF
+&master_nml
+    lrestart =  ${restart}
+/
+&master_time_control_nml
+    calendar = 'proleptic gregorian'
+    checkpointtimeintval = '$checkpoint_interval'
+    restarttimeintval = '$restart_interval'
+    experimentstartdate = '1979-01-01' ! TODO: hack to reproduce result
+    experimentstopdate = '$final_date'
+/
+&master_model_nml ! 'atmo'
+    model_name = 'atmo'
+    model_namelist_filename = 'NAMELIST_atm'
+    model_type = 1
+/
+&jsb_control_nml
+    is_standalone = .false.
+/
+&jsb_model_nml
+    model_id = 1
+    model_name = 'JSBACH'
+    model_shortname = 'jsb'
+    model_description = 'JSBACH land surface model'
+    model_namelist_filename = 'NAMELIST_lnd'
+/
+
+EOF
+
+#-----------------------------------------------------------------------------
+# II. ATMOSPHERE and LAND
+#-----------------------------------------------------------------------------
+#
+# atmosphere namelist
+# -------------------
+cat > NAMELIST_atm << EOF
+&parallel_nml
+    nproma = $nproma
+    num_io_procs = 0
+    num_prefetch_proc = 0
+    pio_type = 1 !1=default, assync I/O, 2=experimental CDI, 0=nothing?
+/
+&grid_nml
+    dynamics_grid_filename = 'icon_grid_G.nc'
+/
+&run_nml
+    num_lev = 47 ! number of full levels
+    modeltimestep = 'PT15M'
+    ltestcase = .false. ! run testcase
+    ldynamics = .true. ! dynamics
+    ltransport = .true. ! transport
+    iforcing = 2 ! 0: none, 1: HS, 2: ECHAM, 3: NWP
+    output = 'nml'
+    msg_level = 8 ! level of details report during integration
+    restart_filename = '${EXP}_restart_atm_<rsttime>.nc'
+    activate_sync_timers = .true.
+/
+&extpar_nml
+    itopo = 1 ! 1: read topography from the grid file
+    itype_lwemiss = 0
+/
+&initicon_nml
+    init_mode = 2 ! 2: initialize from IFS analysis
+    ifs2icon_filename = 'ifs2icon.nc'
+/
+&nonhydrostatic_nml
+    ndyn_substeps = 8 ! dtime/dt_dyn
+    damp_height = 50000. ! [m]
+    rayleigh_coeff = 0.1000 ! set to 0.1001 for rerun with little change
+    vwind_offctr = 0.2
+    divdamp_fac = 0.004
+/
+&interpol_nml
+    rbf_scale_mode_ll = 1
+/
+&sleve_nml
+    min_lay_thckn = 40. ! [m]
+    top_height = 83000. ! [m]
+    stretch_fac = 0.9
+    decay_scale_1 = 4000. ! [m]
+    decay_scale_2 = 2500. ! [m]
+    decay_exp = 1.2
+    flat_height = 16000. ! [m]
+/
+&diffusion_nml
+!    hdiff_smag_fac = 0.015000001
+/
+&transport_nml
+    ihadv_tracer = 52, 2, 2
+    itype_hlimit = 3, 4, 4
+    ivadv_tracer = 3, 3, 3
+    tracer_names = 'hus', 'clw', 'cli'
+/
+&echam_phy_nml
+    ! domain 1
+    ! atmospheric physics ("" = never)
+    echam_phy_config(1)%dt_rad = 'PT90M'
+    echam_phy_config(1)%dt_vdf = 'PT15M'
+    echam_phy_config(1)%dt_cnv = 'PT15M'
+    echam_phy_config(1)%dt_cld = 'PT15M'
+    echam_phy_config(1)%dt_gwd = 'PT15M'
+    echam_phy_config(1)%dt_sso = 'PT15M'
+    ! atmospheric chemistry ("" = never)
+    echam_phy_config(1)%dt_mox = 'PT15M'
+    ! sea ice on mixed-layer ocean (""=never)
+    echam_phy_config(1)%dt_ice = 'PT15M'
+    ! surface (true or false)
+    echam_phy_config(1)%ljsb = .true.
+    echam_phy_config(1)%lamip = .false. 
+    echam_phy_config(1)%lice = .true.
+    echam_phy_config(1)%lmlo = .true.
+    echam_phy_config(1)%llake = .true.
+/
+&echam_rad_nml
+    ! domain 1
+    echam_rad_config(1)%isolrad = 3  ! Use insolation for AMIP type CMIP5 simulation (average from 1979-1988
+    echam_rad_config(1)%irad_h2o = 1
+    echam_rad_config(1)%irad_co2 = 2 ! constant concentration given by vmr_co2 etc.
+    echam_rad_config(1)%irad_ch4 = 2
+    echam_rad_config(1)%irad_n2o = 2
+    echam_rad_config(1)%irad_o3 = 8 ! constant annual cycle climatology
+    echam_rad_config(1)%irad_o2 = 2
+    echam_rad_config(1)%irad_cfc11 = 2
+    echam_rad_config(1)%irad_cfc12 = 2
+    echam_rad_config(1)%irad_aero = 18 ! as in AMIP
+    echam_rad_config(1)%vmr_co2 = 1436.0e-6 !359.0e-6 --> 4xCO2
+    echam_rad_config(1)%vmr_ch4 = 1693.0e-9
+    echam_rad_config(1)%vmr_n2o = 311.0e-9
+    echam_rad_config(1)%vmr_o2  = 0.20946
+    echam_rad_config(1)%vmr_cfc11 = 237.0e-12
+    echam_rad_config(1)%vmr_cfc12 = 462.0e-12
+/
+&echam_gwd_nml
+/
+&echam_sso_nml
+/
+&echam_vdf_nml
+/
+&echam_cnv_nml
+/
+&echam_cld_nml
+/
+&echam_cov_nml
+/
+&ccycle_nml
+/
+&sea_ice_nml
+ i_ice_therm  = 1    ! 1=0L-Semtner
+/
+&echam_seaice_mlo_nml
+ lqflux               = .true.  ! default .TRUE.
+ max_seaice_thickness = 99999.  ! default 5
+ qbot_mlo_nh          = 0.      ! default 10
+ qbot_mlo_sh          = 0.      ! default 10
+/
+! Parameters for all output files
+! -------------------------------
+&io_nml
+    output_nml_dict = 'dict.txt'
+    netcdf_dict = 'dict.txt'
+    itype_pres_msl = 4
+    ! restart_file_type = 5
+    ! restart_write_mode = 'joint procs multifile' ! not useful in r2b4 setup
+    ! lnetcdf_flt64_output = .true. ! 64 bit output in all files
+    ! lkeep_in_sync = .true. ! sync after each timestep
+    write_initial_state = .false.
+/
+&dbg_index_nml
+    idbg_mxmn = 0 ! initialize MIN/MAX debug output
+    idbg_val = 0 ! initialize one cell debug output
+    idbg_slev = 1 ! initialize start level for debug output
+    idbg_elev = 2 ! initialize end level for debug output
+    dbg_lat_in = 30.0 ! latitude location of one cell debug output
+    dbg_lon_in = -30.0 ! longitude location of one cell debug output
+    str_mod_tst = 'InterFaceOce' ! define modules to print out in debug mode
+/
+
+! Define output files
+! -------------------
+!
+! 3-dimensional files include 'ps' and 'pfull' to allow the vertical
+! interpolation to pressure levels by cdo ap2pl.
+
+/
+! Standard AMIP output...
+&output_nml ! 'atm_3d'
+    output_filename = '${EXP}_atm_3d'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1M'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'zg', 'ps', 'pfull', 'rho', 'ta', 'ua', 'va', 'wap', 'hus',
+                 'clw', 'cli', 'hur', 'cl'
+/
+! Standard AMIP output...
+&output_nml ! 'atm_2d'
+    output_filename = '${EXP}_atm_2d'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1M'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'orog', 'ps', 'psl', 'cosmu0', 'rsdt', 'rsut', 'rsutcs',
+                 'rlut', 'rlutcs', 'rsds', 'rsdscs', 'rlds', 'rldscs', 'rsus',
+                 'rsuscs', 'rlus', 'ts', 'sic', 'sit', 'albedo', 'clt', 'prlr',
+                 'prls', 'prcr', 'prcs', 'pr', 'prw', 'cllvi', 'clivi', 'hfls',
+                 'hfss', 'evspsbl', 'tauu', 'tauv', 'tauu_sso', 'tauv_sso',
+                 'diss_sso', 'sfcwind', 'uas', 'vas', 'tas', 'dew2', 'ptp'
+/
+! Standard AMIP output...for daily means
+&output_nml ! 'atm_2d'
+    output_filename = '${EXP}_atm_2d_daily'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1D'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'orog', 'ps', 'psl', 'cosmu0', 'rsdt', 'rsut', 'rsutcs',
+                 'rlut', 'rlutcs', 'rsds', 'rsdscs', 'rlds', 'rldscs', 'rsus',
+                 'rsuscs', 'rlus', 'ts', 'sic', 'sit', 'albedo', 'clt', 'prlr',
+                 'prls', 'prcr', 'prcs', 'pr', 'prw', 'cllvi', 'clivi', 'hfls',
+                 'hfss', 'evspsbl', 'tauu', 'tauv', 'tauu_sso', 'tauv_sso',
+                 'diss_sso', 'sfcwind', 'uas', 'vas', 'tas', 'dew2', 'ptp'
+/
+EOF
+
+
+# jsbach namelist
+# ---------------
+
+cat > NAMELIST_lnd << EOF
+
+&jsb_model_nml
+    usecase = 'jsbach_pfts'
+    use_lakes = .true.
+/
+&jsb_seb_nml
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_rad_nml
+    use_alb_veg_simple = .false. ! if jsbach_lite
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_turb_nml
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_sse_nml
+    l_heat_cap_map = .false.
+    l_heat_cond_map = .false.
+    l_heat_cap_dyn = .false.
+    l_heat_cond_dyn = .false.
+    l_snow = .true.
+    l_dynsnow = .true.
+    l_freeze = .true.
+    l_supercool = .true.
+    bc_filename = 'bc_land_soil.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_hydro_nml
+    l_organic = .false.
+    bc_filename = 'bc_land_soil.nc'
+    ic_filename = 'ic_land_soil.nc'
+    bc_sso_filename = 'bc_land_sso.nc'
+/
+&jsb_assimi_nml
+    active = .true. ! if jsbach_pfts
+/
+&jsb_pheno_nml
+    scheme = 'logrop' ! 'climatology' if jsbach_lite
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_carbon_nml
+    active = .true.
+    bc_filename = 'bc_land_carbon.nc'
+    ic_filename = 'ic_land_carbon.nc'
+    read_cpools = .false.
+/
+&jsb_fuel_nml
+    active = .true.
+    fuel_algorithm = 1
+/
+&jsb_disturb_nml
+    active = .false.
+    ic_filename = 'ic_land_soil.nc'
+    bc_filename = 'bc_land_phys.nc'
+    fire_algorithm = 1
+    windbreak_algorithm = 1
+    lburn_pasture = .false.
+/
+
+EOF
+
+
+## setup for status check & restart
+final_status_file=${EXPDIR}/${EXP}.final_status
+
+## Copy icon executable to working directory
+cp -p $ICONFOLDER/bin/icon ./icon.exe
+##
+
+## Start model
+date
+ulimit -s unlimited
+
+ldd icon.exe
+
+START="/gpfs/opt/sw/skylake/spack-0.19.0/opt/spack/linux-almalinux8-skylake_avx512/intel-2021.7.1/intel-oneapi-mpi-2021.7.1-fzg6q4xcj7efjmce3cuqa2b7cum5d3po/mpi/2021.7.1/bin/mpiexec -n $mpi_total_procs"
+MODEL=${EXPDIR}/icon.exe
+
+rm -f finish.status
+
+${START} ${MODEL}
+
+if [ -r finish.status ] ; then
+  check_final_status 0 "${START} ${MODEL}"
+else
+  check_final_status -1 "${START} ${MODEL}"
+fi
+
+#-----------------------------------------------------------------------------
+finish_status=`cat finish.status`
+echo $finish_status
+
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+#   Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+  echo "restart next experiment..."
+  this_script="${RUNSCRIPTDIR}/exp.${EXP}.run"
+  echo 'this_script: ' "$this_script"
+  # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+  touch ${restartSemaphoreFilename}
+  cd ${RUNSCRIPTDIR}
+  sbatch exp.${EXP}.run
+else
+  [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
+#
+
+echo "============================"
+echo "Script run successfully: ${finish_status}"
+echo "============================"
+#-----------------------------------------------------------------------------
diff --git a/runs/slabctr/exp.slabctr.run b/runs/slabctr/exp.slabctr.run
new file mode 100755
index 0000000..6f9e3df
--- /dev/null
+++ b/runs/slabctr/exp.slabctr.run
@@ -0,0 +1,573 @@
+#! /bin/ksh
+#=============================================================================
+#SBATCH --account=p72044
+#SBATCH --partition=skylake_0096
+#SBATCH --qos=skylake_0096
+#SBATCH --job-name=slabctr
+#SBATCH --nodes=3
+#SBATCH --ntasks-per-node=48
+#SBATCH --ntasks-per-core=1
+#SBATCH --output=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slabctr/logfiles/LOG.exp.slabctr.run.%j.o
+#SBATCH --error=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slabctr/logfiles/LOG.exp.slabctr.run.%j.o
+#SBATCH --exclusive
+#SBATCH --time=03:00:00
+#SBATCH --mail-user=aiko.voigt@univie.ac.at
+#SBATCH --mail-type=BEGIN,END,FAIL
+
+set -x # debugging command: enables a mode of the shell where all executed commands are printed to the terminal
+ulimit -s unlimited # unsets limits for RAM
+
+# MPI variables
+# -------------
+no_of_nodes=3
+mpi_procs_pernode=48
+((mpi_total_procs=no_of_nodes * mpi_procs_pernode))
+echo $mpi_total_procs
+
+# manual fix for mpi pinning with intel mpirun on vsc
+# see https://wiki.vsc.ac.at/doku.php?id=doku:vsc5quickstart#intel_mpi
+export I_MPI_PIN_RESPECT_CPUSET=0
+
+#
+# blocking length
+# ---------------
+nproma=16
+
+#=============================================================================
+# Input variables:
+
+# SIMULATION NAME
+EXP=slabctr
+
+ICONFOLDER=/home/fs72044/avoigt_teach/climlab_s2024/icon-esm-univie       # DIRECTORY OF ICON MODEL CODE
+RUNSCRIPTDIR=/home/fs72044/avoigt_teach/climlab_s2024/msc-climmodlab-s2024/runs/slabctr/
+basedir=$ICONFOLDER # icon base directory
+
+. ${ICONFOLDER}/run/add_run_routines
+
+# experiment directory, with plenty of space, create if new
+EXPDIR=/gpfs/data/fs72044/avoigt_teach/experiments/s2024/${EXP}
+if [ ! -d ${EXPDIR} ] ;  then
+  mkdir -p ${EXPDIR}
+fi
+#
+ls -ld ${EXPDIR}
+if [ ! -d ${EXPDIR} ] ;  then
+    mkdir ${EXPDIR}
+fi
+ls -ld ${EXPDIR}
+
+cd $EXPDIR
+
+
+
+
+#=================================================================================
+
+#-----------------------------------------------------------------------------
+# global timing
+initial_date="1979-01-01"
+final_date="2029-01-01"
+start_date=$initial_date
+end_date=$final_date
+y0=${start_date%%-*}
+yN=${end_date%%-*}
+
+
+# restart intervals
+restart_interval="P2Y"
+checkpoint_interval="P1Y"
+
+file_interval="P1M"
+
+############################################################
+#
+# NO FURTHER CHANGES TO THE DIRECTORIES AND SIMULATION NAME
+# SHOULD BE NEEDED BELOW THIS LINE
+#
+############################################################
+
+#-----------------------------------------------------------------------------
+# Provide input files
+# $Id: format.tmpl 9264 2021-06-21 21:24:57Z m221078 $
+#
+# [files]
+
+# [files.atmosphere]
+data_dir=/gpfs/data/fs72044/avoigt_teach/ICON-inputdata/amip-VSC4
+
+# [files.atmosphere.mapped]
+grid_dir=$data_dir/grid
+ln -sfv $grid_dir/icon_grid_0013_R02B04_G.nc icon_grid_G.nc
+
+# [files.atmosphere.mapped.initial]
+initial_dir=$data_dir/initial_condition
+ln -sfv $initial_dir/ifs2icon_1979010100_R02B04_G.nc ifs2icon.nc
+
+# [files.atmosphere.mapped.ozone]
+for((yr = y0 + -1; yr <= yN + 1; ++yr)) 
+do
+    label=${yr}
+        ((yr >= 2015)) && label=2014
+    ozone_dir=$data_dir/ozone
+    ln -sfv $ozone_dir/bc_ozone_historical_1979-2008.ymonmean.${label}.nc bc_ozone_${yr}.nc
+done # offsets
+
+# [files.atmosphere.mapped.ocean_surface]
+ocean_surface_dir=$data_dir/sst_and_seaice
+ln -sfv $ocean_surface_dir/bc_sic_1979_2008.ymonmean.1978-2079.nc bc_sic.nc
+ln -sfv $ocean_surface_dir/bc_sst_1979_2008.ymonmean.1978-2079.nc bc_sst.nc
+
+# files for slab ocean
+# sst, sic, and seb
+ln -sfv $ocean_surface_dir/bc_sic_1979_2008.ymonmean.1978-2079.nc bc_mlo_sic.nc
+ln -sfv $ocean_surface_dir/bc_sst_1979_2008.ymonmean.1978-2079.nc bc_mlo_sst.nc
+# q-flux file is taken from Phaidra archive of S2023 course
+ln -sfv /gpfs/data/fs72044/avoigt_teach/ICON-inputdata/slabocean/sstclim_seb_atm_seb_2d_ml_1980-2008.ymonmean.seb_wtr.addc_3.1970-2069.nc bc_mlo_seb.nc
+
+# [files.atmosphere.mapped.aerosols]
+aerosols_dir=$data_dir/aerosol
+ln -sfv $aerosols_dir/bc_aeropt_kinne_lw_b16_coa.nc .
+ln -sfv $aerosols_dir/bc_aeropt_kinne_sw_b14_coa.nc .
+ln -sfv $aerosols_dir/bc_aeropt_kinne_sw_b14_fin_1850.nc bc_aeropt_kinne_sw_b14_fin.nc
+
+# [files.atmosphere.model]
+model_dir=/home/fs72044/avoigt_teach/climlab_s2024/icon-esm-univie
+
+# [files.atmosphere.model.data]
+ln -sfv $model_dir/data/lsdata.nc .
+ln -sfv $model_dir/data/ECHAM6_CldOptProps.nc .
+ln -sfv $model_dir/data/MACv2.0-SP_v1.nc .
+
+# [files.atmosphere.model.run]
+run_dir=$model_dir/run
+cp -fv $run_dir/dict.iconam.mpim dict.txt
+
+# [files.atmosphere.independent.volcano_aerosols]
+for((yr = y0 + -1; yr <= yN + 1; ++yr)) 
+do
+    volcano_aerosols_dir=$data_dir/aerosol
+    ln -sfv $volcano_aerosols_dir/bc_aeropt_cmip6_volc_lw_b16_sw_b14_2000.nc \
+        bc_aeropt_cmip6_volc_lw_b16_sw_b14_${yr}.nc
+done # offsets
+
+
+# [files.land]
+land_dir=$data_dir/land
+
+# [files.land.mapped]
+ln -sfv $land_dir/ic_land_soil_1976.nc ic_land_soil.nc
+ln -sfv $land_dir/bc_land_frac_11pfts_1976.nc bc_land_frac.nc
+ln -sfv $land_dir/bc_land_phys_1976.nc bc_land_phys.nc
+ln -sfv $land_dir/bc_land_soil_1976.nc bc_land_soil.nc
+ln -sfv $land_dir/bc_land_sso_1976.nc bc_land_sso.nc
+
+# [files.land.hydro]
+hydro_dir=$land_dir
+# preliminary test version
+ln -sfv $hydro_dir/hdpara_r2b4_0013_0035_v3.nc bc_land_hd.nc
+
+# [files.land.model]
+model_dir=$basedir/externals/jsbach/data
+ln -sfv $model_dir/lctlib_nlct21.def .
+
+
+#-----------------------------------------------------------------------------
+# automatic restart setup
+# set some default values and derive some run parameteres
+restart=${restart:=".false."}
+restartSemaphoreFilename='isRestartRun.sem'
+#AUTOMATIC_RESTART_SETUP:
+if [ -f ${restartSemaphoreFilename} ]; then
+  restart=.true.
+  #  do not delete switch-file, to enable restart after unintended abort
+  #[[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+#END AUTOMATIC_RESTART_SETUP
+
+# wait 5min to let GPFS finish the write operations
+if [ "x$restart" != 'x.false.' -a "x$submit" != 'x' ]; then
+  if [ x$(df -T ${EXPDIR} | cut -d ' ' -f 2) = gpfs ]; then
+    sleep 10;
+  fi
+fi
+
+
+
+#
+# create ICON master, coupling and model namelists
+# ------------------------------------------------
+# For a complete list see Namelist_overview and Namelist_overview.pdf
+#
+
+cat > icon_master.namelist << EOF
+&master_nml
+    lrestart =  ${restart}
+/
+&master_time_control_nml
+    calendar = 'proleptic gregorian'
+    checkpointtimeintval = '$checkpoint_interval'
+    restarttimeintval = '$restart_interval'
+    experimentstartdate = '1979-01-01' ! TODO: hack to reproduce result
+    experimentstopdate = '$final_date'
+/
+&master_model_nml ! 'atmo'
+    model_name = 'atmo'
+    model_namelist_filename = 'NAMELIST_atm'
+    model_type = 1
+/
+&jsb_control_nml
+    is_standalone = .false.
+/
+&jsb_model_nml
+    model_id = 1
+    model_name = 'JSBACH'
+    model_shortname = 'jsb'
+    model_description = 'JSBACH land surface model'
+    model_namelist_filename = 'NAMELIST_lnd'
+/
+
+EOF
+
+#-----------------------------------------------------------------------------
+# II. ATMOSPHERE and LAND
+#-----------------------------------------------------------------------------
+#
+# atmosphere namelist
+# -------------------
+cat > NAMELIST_atm << EOF
+&parallel_nml
+    nproma = $nproma
+    num_io_procs = 0
+    num_prefetch_proc = 0
+    pio_type = 1 !1=default, assync I/O, 2=experimental CDI, 0=nothing?
+/
+&grid_nml
+    dynamics_grid_filename = 'icon_grid_G.nc'
+/
+&run_nml
+    num_lev = 47 ! number of full levels
+    modeltimestep = 'PT15M'
+    ltestcase = .false. ! run testcase
+    ldynamics = .true. ! dynamics
+    ltransport = .true. ! transport
+    iforcing = 2 ! 0: none, 1: HS, 2: ECHAM, 3: NWP
+    output = 'nml'
+    msg_level = 8 ! level of details report during integration
+    restart_filename = '${EXP}_restart_atm_<rsttime>.nc'
+    activate_sync_timers = .true.
+/
+&extpar_nml
+    itopo = 1 ! 1: read topography from the grid file
+    itype_lwemiss = 0
+/
+&initicon_nml
+    init_mode = 2 ! 2: initialize from IFS analysis
+    ifs2icon_filename = 'ifs2icon.nc'
+/
+&nonhydrostatic_nml
+    ndyn_substeps = 8 ! dtime/dt_dyn
+    damp_height = 50000. ! [m]
+    rayleigh_coeff = 0.1000 ! set to 0.1001 for rerun with little change
+    vwind_offctr = 0.2
+    divdamp_fac = 0.004
+/
+&interpol_nml
+    rbf_scale_mode_ll = 1
+/
+&sleve_nml
+    min_lay_thckn = 40. ! [m]
+    top_height = 83000. ! [m]
+    stretch_fac = 0.9
+    decay_scale_1 = 4000. ! [m]
+    decay_scale_2 = 2500. ! [m]
+    decay_exp = 1.2
+    flat_height = 16000. ! [m]
+/
+&diffusion_nml
+!    hdiff_smag_fac = 0.015000001
+/
+&transport_nml
+    ihadv_tracer = 52, 2, 2
+    itype_hlimit = 3, 4, 4
+    ivadv_tracer = 3, 3, 3
+    tracer_names = 'hus', 'clw', 'cli'
+/
+&echam_phy_nml
+    ! domain 1
+    ! atmospheric physics ("" = never)
+    echam_phy_config(1)%dt_rad = 'PT90M'
+    echam_phy_config(1)%dt_vdf = 'PT15M'
+    echam_phy_config(1)%dt_cnv = 'PT15M'
+    echam_phy_config(1)%dt_cld = 'PT15M'
+    echam_phy_config(1)%dt_gwd = 'PT15M'
+    echam_phy_config(1)%dt_sso = 'PT15M'
+    ! atmospheric chemistry ("" = never)
+    echam_phy_config(1)%dt_mox = 'PT15M'
+    ! sea ice on mixed-layer ocean (""=never)
+    echam_phy_config(1)%dt_ice = 'PT15M'
+    ! surface (true or false)
+    echam_phy_config(1)%ljsb = .true.
+    echam_phy_config(1)%lamip = .false. 
+    echam_phy_config(1)%lice = .true.
+    echam_phy_config(1)%lmlo = .true.
+    echam_phy_config(1)%llake = .true.
+/
+&echam_rad_nml
+    ! domain 1
+    echam_rad_config(1)%isolrad = 3  ! Use insolation for AMIP type CMIP5 simulation (average from 1979-1988
+    echam_rad_config(1)%irad_h2o = 1
+    echam_rad_config(1)%irad_co2 = 2 ! constant concentration given by vmr_co2 etc.
+    echam_rad_config(1)%irad_ch4 = 2
+    echam_rad_config(1)%irad_n2o = 2
+    echam_rad_config(1)%irad_o3 = 8 ! constant annual cycle climatology
+    echam_rad_config(1)%irad_o2 = 2
+    echam_rad_config(1)%irad_cfc11 = 2
+    echam_rad_config(1)%irad_cfc12 = 2
+    echam_rad_config(1)%irad_aero = 18 ! as in AMIP
+    echam_rad_config(1)%vmr_co2 = 359.0e-6
+    echam_rad_config(1)%vmr_ch4 = 1693.0e-9
+    echam_rad_config(1)%vmr_n2o = 311.0e-9
+    echam_rad_config(1)%vmr_o2  = 0.20946
+    echam_rad_config(1)%vmr_cfc11 = 237.0e-12
+    echam_rad_config(1)%vmr_cfc12 = 462.0e-12
+/
+&echam_gwd_nml
+/
+&echam_sso_nml
+/
+&echam_vdf_nml
+/
+&echam_cnv_nml
+/
+&echam_cld_nml
+/
+&echam_cov_nml
+/
+&ccycle_nml
+/
+&sea_ice_nml
+ i_ice_therm  = 1    ! 1=0L-Semtner
+/
+&echam_seaice_mlo_nml
+ lqflux               = .true.  ! default .TRUE.
+ max_seaice_thickness = 99999.  ! default 5
+ qbot_mlo_nh          = 0.      ! default 10
+ qbot_mlo_sh          = 0.      ! default 10
+/
+! Parameters for all output files
+! -------------------------------
+&io_nml
+    output_nml_dict = 'dict.txt'
+    netcdf_dict = 'dict.txt'
+    itype_pres_msl = 4
+    ! restart_file_type = 5
+    ! restart_write_mode = 'joint procs multifile' ! not useful in r2b4 setup
+    ! lnetcdf_flt64_output = .true. ! 64 bit output in all files
+    ! lkeep_in_sync = .true. ! sync after each timestep
+    write_initial_state = .false.
+/
+&dbg_index_nml
+    idbg_mxmn = 0 ! initialize MIN/MAX debug output
+    idbg_val = 0 ! initialize one cell debug output
+    idbg_slev = 1 ! initialize start level for debug output
+    idbg_elev = 2 ! initialize end level for debug output
+    dbg_lat_in = 30.0 ! latitude location of one cell debug output
+    dbg_lon_in = -30.0 ! longitude location of one cell debug output
+    str_mod_tst = 'InterFaceOce' ! define modules to print out in debug mode
+/
+
+! Define output files
+! -------------------
+!
+! 3-dimensional files include 'ps' and 'pfull' to allow the vertical
+! interpolation to pressure levels by cdo ap2pl.
+
+/
+! Standard AMIP output...
+&output_nml ! 'atm_3d'
+    output_filename = '${EXP}_atm_3d'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1M'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'zg', 'ps', 'pfull', 'rho', 'ta', 'ua', 'va', 'wap', 'hus',
+                 'clw', 'cli', 'hur', 'cl'
+/
+! Standard AMIP output...
+&output_nml ! 'atm_2d'
+    output_filename = '${EXP}_atm_2d'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1M'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'orog', 'ps', 'psl', 'cosmu0', 'rsdt', 'rsut', 'rsutcs',
+                 'rlut', 'rlutcs', 'rsds', 'rsdscs', 'rlds', 'rldscs', 'rsus',
+                 'rsuscs', 'rlus', 'ts', 'sic', 'sit', 'albedo', 'clt', 'prlr',
+                 'prls', 'prcr', 'prcs', 'pr', 'prw', 'cllvi', 'clivi', 'hfls',
+                 'hfss', 'evspsbl', 'tauu', 'tauv', 'tauu_sso', 'tauv_sso',
+                 'diss_sso', 'sfcwind', 'uas', 'vas', 'tas', 'dew2', 'ptp'
+/
+! Standard AMIP output...for daily means
+&output_nml ! 'atm_2d'
+    output_filename = '${EXP}_atm_2d_daily'
+    filename_format = '<output_filename>_<levtype_l>_<datetime2>'
+    filetype = 5
+    remap = 0
+    operation = 'mean'
+    output_grid = .TRUE.
+    output_start = '${initial_date}'
+    output_end = '${final_date}'
+    output_interval = 'P1D'
+    file_interval = '$file_interval' !'P10YT1S'
+    include_last = .false.
+    ml_varlist = 'orog', 'ps', 'psl', 'cosmu0', 'rsdt', 'rsut', 'rsutcs',
+                 'rlut', 'rlutcs', 'rsds', 'rsdscs', 'rlds', 'rldscs', 'rsus',
+                 'rsuscs', 'rlus', 'ts', 'sic', 'sit', 'albedo', 'clt', 'prlr',
+                 'prls', 'prcr', 'prcs', 'pr', 'prw', 'cllvi', 'clivi', 'hfls',
+                 'hfss', 'evspsbl', 'tauu', 'tauv', 'tauu_sso', 'tauv_sso',
+                 'diss_sso', 'sfcwind', 'uas', 'vas', 'tas', 'dew2', 'ptp'
+/
+EOF
+
+
+# jsbach namelist
+# ---------------
+
+cat > NAMELIST_lnd << EOF
+
+&jsb_model_nml
+    usecase = 'jsbach_pfts'
+    use_lakes = .true.
+/
+&jsb_seb_nml
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_rad_nml
+    use_alb_veg_simple = .false. ! if jsbach_lite
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_turb_nml
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_sse_nml
+    l_heat_cap_map = .false.
+    l_heat_cond_map = .false.
+    l_heat_cap_dyn = .false.
+    l_heat_cond_dyn = .false.
+    l_snow = .true.
+    l_dynsnow = .true.
+    l_freeze = .true.
+    l_supercool = .true.
+    bc_filename = 'bc_land_soil.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_hydro_nml
+    l_organic = .false.
+    bc_filename = 'bc_land_soil.nc'
+    ic_filename = 'ic_land_soil.nc'
+    bc_sso_filename = 'bc_land_sso.nc'
+/
+&jsb_assimi_nml
+    active = .true. ! if jsbach_pfts
+/
+&jsb_pheno_nml
+    scheme = 'logrop' ! 'climatology' if jsbach_lite
+    bc_filename = 'bc_land_phys.nc'
+    ic_filename = 'ic_land_soil.nc'
+/
+&jsb_carbon_nml
+    active = .true.
+    bc_filename = 'bc_land_carbon.nc'
+    ic_filename = 'ic_land_carbon.nc'
+    read_cpools = .false.
+/
+&jsb_fuel_nml
+    active = .true.
+    fuel_algorithm = 1
+/
+&jsb_disturb_nml
+    active = .false.
+    ic_filename = 'ic_land_soil.nc'
+    bc_filename = 'bc_land_phys.nc'
+    fire_algorithm = 1
+    windbreak_algorithm = 1
+    lburn_pasture = .false.
+/
+
+EOF
+
+
+## setup for status check & restart
+final_status_file=${EXPDIR}/${EXP}.final_status
+
+## Copy icon executable to working directory
+cp -p $ICONFOLDER/bin/icon ./icon.exe
+##
+
+## Start model
+date
+ulimit -s unlimited
+
+ldd icon.exe
+
+START="/gpfs/opt/sw/skylake/spack-0.19.0/opt/spack/linux-almalinux8-skylake_avx512/intel-2021.7.1/intel-oneapi-mpi-2021.7.1-fzg6q4xcj7efjmce3cuqa2b7cum5d3po/mpi/2021.7.1/bin/mpiexec -n $mpi_total_procs"
+MODEL=${EXPDIR}/icon.exe
+
+rm -f finish.status
+
+${START} ${MODEL}
+
+if [ -r finish.status ] ; then
+  check_final_status 0 "${START} ${MODEL}"
+else
+  check_final_status -1 "${START} ${MODEL}"
+fi
+
+#-----------------------------------------------------------------------------
+finish_status=`cat finish.status`
+echo $finish_status
+
+#-----------------------------------------------------------------------------
+namelist_list=""
+#-----------------------------------------------------------------------------
+# check if we have to restart, ie resubmit
+#   Note: this is a different mechanism from checking the restart
+if [ $finish_status = "RESTART" ]; then
+  echo "restart next experiment..."
+  this_script="${RUNSCRIPTDIR}/exp.${EXP}.run"
+  echo 'this_script: ' "$this_script"
+  # note that if ${restartSemaphoreFilename} does not exist yet, then touch will create it
+  touch ${restartSemaphoreFilename}
+  cd ${RUNSCRIPTDIR}
+  sbatch exp.${EXP}.run
+else
+  [[ -f ${restartSemaphoreFilename} ]] && rm ${restartSemaphoreFilename}
+fi
+
+#-----------------------------------------------------------------------------
+
+cd ${RUNSCRIPTDIR}
+
+#-----------------------------------------------------------------------------
+#
+
+echo "============================"
+echo "Script run successfully: ${finish_status}"
+echo "============================"
+#-----------------------------------------------------------------------------
-- 
GitLab