diff --git a/definition-files/MPI/Singularity.miniconda3-py39-4.9.2-ubuntu-18.04-OMPI b/definition-files/MPI/Singularity.miniconda3-py39-4.9.2-ubuntu-18.04-OMPI
deleted file mode 100644
index 635e63d289e3d57654085169770a95d0f4021bdf..0000000000000000000000000000000000000000
--- a/definition-files/MPI/Singularity.miniconda3-py39-4.9.2-ubuntu-18.04-OMPI
+++ /dev/null
@@ -1,89 +0,0 @@
-Bootstrap: library
-From: mblaschek/imgw/ubuntu:18.04
-
-%labels
-
- APPLICATION_NAME miniconda3
- APPLICATION_VERSION py39-4.9.2-Linux-x86_64
- APPLICATION_URL https://docs.conda.io
-
- AUTHOR_NAME Michael Blaschek
- AUTHOR_EMAIL michael.blaschek@univie.ac.at
-
- LAST_UPDATED 20211118
-
-%setup
-
-%environment
-
- # Set the conda distribution type, its version number, the python
- # version it utilizes, the root and installation directories where
- # the distribution will be installed within the container, and the
- # root URL to the installer
- export CONDA_DISTRIBUTION='miniconda'
- export CONDA_VERSION='3'
- export CONDA_PYTHON_VERSION='py39'
- export CONDA_INSTALLER_VERSION='4.9.2'
- export CONDA_ARCH='Linux-x86_64'
- export CONDA_INSTALL_DIR="/opt/${CONDA_DISTRIBUTION}${CONDA_VERSION}"
-
- # Set PATH to conda distribution
- export PATH="${CONDA_INSTALL_DIR}/bin:${PATH}"
-
-%post -c /bin/bash
-
- # Set operating system mirror URL
- export MIRRORURL='http://at.archive.ubuntu.com/ubuntu'
-
- # Set operating system version
- export OSVERSION='bionic'
-
- # Set system locale
- export LC_ALL='C'
-
- # Set debian frontend interface
- export DEBIAN_FRONTEND='noninteractive'
-
- # Upgrade all software packages to their latest versions
- apt-get -y update && apt-get -y upgrade
-
- cd /tmp
-
- # Set the conda distribution type, its version number, the python
- # version it utilizes, the root and installation directories where
- # the distribution will be installed within the container, and the
- # root URL to the installer
- export CONDA_DISTRIBUTION='miniconda'
- export CONDA_VERSION='3'
- export CONDA_PYTHON_VERSION='py39'
- export CONDA_INSTALLER_VERSION='4.9.2'
- export CONDA_ARCH='Linux-x86_64'
- export CONDA_INSTALLER="${CONDA_DISTRIBUTION^}${CONDA_VERSION}-${CONDA_PYTHON_VERSION}_${CONDA_INSTALLER_VERSION}-${CONDA_ARCH}.sh"
- export CONDA_INSTALL_DIR="/opt/${CONDA_DISTRIBUTION}${CONDA_VERSION}"
- export CONDA_ROOT_URL='https://repo.anaconda.com'
-
- # Download and install conda distribution
- wget "${CONDA_ROOT_URL}/${CONDA_DISTRIBUTION}/${CONDA_INSTALLER}"
- chmod +x "${CONDA_INSTALLER}"
- "./${CONDA_INSTALLER}" -b -p "${CONDA_INSTALL_DIR}"
-
- # Remove conda installer
- rm "${CONDA_INSTALLER}"
-
- # Add MPI Package from conda-forge
- # ucx
- # openmpi
- $CONDA_INSTALL_DIR/bin/conda install -y -c conda-forge ucx openmpi mpi4py
-
- # Cleanup
- apt-get -y autoremove --purge
- apt-get -y clean
-
- # Update database for mlocate
- updatedb
-
-%files
-
-%runscript
-
-%test
diff --git a/definition-files/MPI/Singularity.py39.ompi b/definition-files/MPI/Singularity.py39.ompi
new file mode 100644
index 0000000000000000000000000000000000000000..96f0a87bbc9c7b4559ea73cb56c822216c4ff38f
--- /dev/null
+++ b/definition-files/MPI/Singularity.py39.ompi
@@ -0,0 +1,52 @@
+Bootstrap: docker
+From: mambaorg/micromamba:latest
+
+%labels
+ maintainer IT-IMGW <it.img-wien@univie.ac.at>
+
+%files
+ $PWD/definition-files/runscript /.singularity.d/runscript
+ $PWD/definition-files/run-help /.singularity.d/runscript.help
+
+
+%apprun mpitest
+ nproc=4
+ if [ $# -eq 1 ]; then
+ nproc=$1
+ fi
+ echo "Running MPITest : $nproc"
+ exec mpirun -np $nproc mpitest.x
+
+%appfiles mpitest
+ ./definition-files/MPI/mpitest.c ./mpitest.c
+
+%appinstall mpitest
+ export PATH=/opt/conda/bin:$PATH
+ export LIBRARY=/opt/conda/lib
+ export INCLUDE=/opt/conda/include
+ mpicc mpitest.c -o mpitest.x
+
+%environment
+ export LANG=C.UTF-8
+ export PATH=/opt/conda/bin:$PATH
+ export LIBRARY=/opt/conda/lib
+ export INCLUDE=/opt/conda/include
+
+%post
+ micromamba -q install -y -n base -c conda-forge \
+ python=3.9 \
+ ucx \
+ openmpi \
+ mpi4py \
+ && micromamba clean --all --yes
+ # command prompt name
+ CNAME=m.ompi
+ # does not work goes into /.singularity.d/env/91-environment.sh
+ echo "export PS1=\"[IMGW-$CNAME]\w\$ \"" >> /.singularity.d/env/99-zz-custom-env.sh
+ # add some labels
+ echo "libc $(ldd --version | head -n1 | cut -d' ' -f4)" >> "$SINGULARITY_LABELS"
+ echo "linux $(cat /etc/os-release | grep PRETTY_NAME | cut -d'=' -f2)" >> "$SINGULARITY_LABELS"
+
+%test
+ mpicc --version
+
diff --git a/definition-files/debian/Singularity.texlive b/definition-files/debian/Singularity.texlive
index 496770b904e7aca36e7ba74ecf8bf85b681b8268..9c767e216f3906b373d6ac091e7d990edc59cb3b 100644
--- a/definition-files/debian/Singularity.texlive
+++ b/definition-files/debian/Singularity.texlive
@@ -1,5 +1,5 @@
Bootstrap: docker
-From: texlive:latest
+From: texlive/texlive:latest
%labels
maintainer IT-IMGW <it.img-wien@univie.ac.at>
diff --git a/models/ICON/Singularity.gcc6 b/models/ICON/Singularity.gcc6
index 40e9dec08db99ee05dc3c18d0f8195ec30687491..99d6eb46d66cc2f511cf141304af4d21ba882929 100644
--- a/models/ICON/Singularity.gcc6
+++ b/models/ICON/Singularity.gcc6
@@ -11,6 +11,7 @@ apt install -y \
python3 \
gcc-6 \
gfortran-6 \
+ libopenmpi-dev \
libmpich-dev \
libhdf5-dev \
libnetcdf-dev \
diff --git a/models/WRF/README.md b/models/WRF/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..543735168fdfa2b472e8d31335379b13cd9b7d2d
--- /dev/null
+++ b/models/WRF/README.md
@@ -0,0 +1,50 @@
+# WRF in a container
+
+
+:construction:
+
+This is still experimental. No guarantees.
+
+
+Please find the following containers available for download:
+- WRF
+- GSI
+- WRFpy
+- WRF.dev
+
+
+## Design
+
+build a container with all requirements installed and make it easily available to users to further develop. We will use a two stage build to complete the development
+
+
+1. WRF sandbox / requirements
+2. Build container
+3. Final container
+
+Build some special containers, that are most requested, e.g. em_real and em_les
+
+
+## Example: Hurricane Sandy
+
+## Performance tests
+
+We ran the same container:
+- VSC
+- JET
+- SRVX1
+
+Multi-Node?
+
+# Development
+
+If you like to build your own WRF container with customized source code or a different setting/purpose, then you can find here some guidance on how to do this.
+
+Steps:
+1. Pull the WRF development container from
+2. Pull the source code of WRF / WPS ...
+3. Configure the Source code to use the libraries inside the container
+4. Test your settings / compilation
+5. Build your own container from the WRF development container and your SRC Code.
+
+
diff --git a/models/WRF/Singularity.dev b/models/WRF/Singularity.dev
new file mode 100644
index 0000000000000000000000000000000000000000..3bd8b04ebf7cea428fbf59742964df6b15d84262
--- /dev/null
+++ b/models/WRF/Singularity.dev
@@ -0,0 +1,72 @@
+Bootstrap: localimage
+From: ../../containers/alma8.base.sif
+
+%labels
+maintainer IT-IMGW <it.img-wien@univie.ac.at>
+baseimage AlmaLinux8
+
+
+%post
+# Every line will be a layer in the container
+# See https://fedoraproject.org/wiki/EPEL#Quickstart for powertools
+# yum --enablerepo epel groupinstall -y "Development Tools" \
+yum update -y \
+&& yum install -y dnf-plugins-core \
+&& dnf config-manager --set-enabled powertools \
+&& yum install -y epel-release \
+&& yum update -y \
+&& yum --enablerepo epel install -y \
+ curl wget \
+ file \
+ findutils \
+ gcc-c++ \
+ gcc \
+ gcc-gfortran \
+ glibc.i686 libgcc.i686 \
+ libpng-devel jasper-libs jasper-devel \
+ m4 make perl cmake \
+ flex flex-devel bison bison-devel \
+ libcurl-devel \
+ libxml2 libxml2-devel perl-XML-LibXML ImageMagick \
+ python3 python3-pip python3-devel \
+ tar bash tcsh time which zlib zlib-devel \
+ git scl \
+ gnupg2 \
+ hostname \
+ iproute \
+ patch \
+ openmpi-devel \
+ openmpi \
+ hdf5-openmpi-devel \
+ hdf5-openmpi-static \
+ netcdf-openmpi-devel \
+ netcdf-openmpi-static \
+ netcdf-fortran-openmpi-devel \
+ netcdf-fortran-openmpi-static \
+ openblas-devel.x86_64 \
+ openblas-openmp.x86_64 \
+&& rm -rf /var/cache/yum \
+&& yum clean all \
+&& dnf clean all \
+&& rm -rf /usr/share/doc \
+&& rm -rf /usr/share/man \
+&& ln -s /usr/include/openmpi-x86_64/ /usr/lib64/openmpi/include
+
+
+# command prompt name
+CNAME=wrf.dev
+# does not work goes into /.singularity.d/env/91-environment.sh
+echo "export PS1=\"[IMGW-$CNAME]\w\$ \"" >> /.singularity.d/env/99-zz-custom-env.sh
+# not sure why that does not happen as default
+echo "export PKG_CONFIG_PATH=/usr/lib64/openmpi/lib/pkgconfig/" >> $SINGULARITY_ENVIRONMENT
+
+%environment
+export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:/usr/lib64:/lib64:/lib
+export PATH=/usr/lib64/openmpi/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export LIBRARY=/usr/lib64/openmpi/lib:/usr/lib64:/lib64:/lib
+export INCLUDE=/usr/include/openmpi-x86_64/:/usr/lib64/gfortran/modules/openmpi:/usr/include
+export NETCDF=/usr/lib64/openmpi
+export NETCDF_ROOT=/usr/lib64/openmpi
+export HDF5_ROOT=/usr/lib64/openmpi
+export JASPERINC=/usr/include/jasper/
+export JASPERLIB=/usr/lib64/
diff --git a/models/WRF/Singularity.gsi b/models/WRF/Singularity.gsi
new file mode 100644
index 0000000000000000000000000000000000000000..03a2fc5addf46ad703e07f6066596b41ce89103f
--- /dev/null
+++ b/models/WRF/Singularity.gsi
@@ -0,0 +1,82 @@
+Bootstrap: localimage
+From: sandbox.wrf.dev
+Stage: build
+
+# Two Stage Build of GSI
+# use the sandbox development image
+# 1. Compile the target
+# 2. Copy relevant exe to target GSI container
+# Make sure the ldd / linking and inputs are ok
+
+%labels
+maintainer IT-IMGW <it.img-wien@univie.ac.at>
+
+%files
+./scripts/runscript /.singularity.d/runscript
+./scripts/run_gsi.ksh /wrf/bin
+
+%apprun init
+echo "Please modify run_gsi.ksh to your needs and settings or use set_env.ksh"
+cp -uv /wrf/bin/run_gsi.ksh .
+
+%apphelp init
+Use this app to copy the run_gsi.ksh to your current directory.
+This is a modified version from dtcenter container version.
+Please update either the run_gsi.ksh or the set_env.ksh in your local directory.
+
+%post
+# GSI requires cmake and openblas + lapack
+yum -y install cmake openblas-devel.x86_64 openblas-openmp.x86_64
+# Build a WRF release from the sandbox
+WRF_BUILD_TARGET=em_real
+LDFLAGS="-lm"
+NETCDF=/usr/lib64/openmpi/
+HDF5_ROOT=$NETCDF
+
+export GSI_VERSION=3.7
+export ENKF_VERSION=1.3
+
+mkdir -p /wrf/gsi \
+&& curl -SL https://dtcenter.org/sites/default/files/comGSIv${GSI_VERSION}_EnKFv${ENKF_VERSION}.tar.gz | tar -xzC /wrf/gsi
+#
+#
+#
+ln -s /usr/lib64/gfortran/modules/openmpi/* /usr/lib64/openmpi/include/
+#
+# prep GSI build
+#
+mkdir -p /wrf/gsi/build \
+&& cd /wrf/gsi/build \
+&& cmake /wrf/gsi/comGSIv${GSI_VERSION}_EnKFv${ENKF_VERSION}
+#
+# Fix a few GSI bugs
+#
+RUN umask 0002 \
+ && sed -i 's/wij(1)/wij/g' /wrf/gsi/comGSIv3.7_EnKFv1.3/src/setuplight.f90 \
+ && sed -i "s,\$, -L$NETCDF,g" /wrf/gsi/build/src/CMakeFiles/gsi.x.dir/link.txt \
+ && sed -i "s,\$, -L$NETCDF,g" /wrf/gsi/build/src/enkf/CMakeFiles/enkf_wrf.x.dir/link.txt
+#
+# Build GSI
+#
+cd /wrf/gsi/build \
+&& make -j 4
+#
+# Fix all wired paths
+#
+# sed -i 's,/comsoftware/gsi/,/wrf/gsi/' /wrf/bin/run_gsi.ksh \
+# && sed -i 's,/gsi_build,/build' /wrf/bin/run_gsi.ksh \
+# && sed -i 's,/data,./data,g' /wrf/bin/run_gsi.ksh \
+# && sed -i 's,/home/scripts/case,.,g' /wrf/bin/run_gsi.ksh \
+# && sed -i 's,/home/wrfprd,.,g' /wrf/bin/run_gsi.ksh \
+# && sed -i 's,/home/gsiprd,.,g' /wrf/bin/run_gsi.ksh
+#
+# Wrap it up
+#
+echo "export GSI_VERSION=$GSI_VERSION" >> $SINGULARITY_ENVIRONMENT
+echo "export ENKF_VERSION=$ENKF_VERSION" >> $SINGULARITY_ENVIRONMENT
+CNAME=alma8.gsi
+# does not work goes into /.singularity.d/env/91-environment.sh
+echo "export PS1=\"[IMGW-$CNAME]\w\$ \"" >> /.singularity.d/env/99-zz-custom-env.sh
+
+%environment
+export PATH=/wrf/gsi/build/bin:/wrf/bin:$PATH
\ No newline at end of file
diff --git a/models/WRF/Singularity.sandbox.dev b/models/WRF/Singularity.sandbox.dev
index 71e138f90066d0c397c2b1ef9cce5afbd11c518f..f8d1b1fa5d82625669920f357d91997e81aa50c4 100644
--- a/models/WRF/Singularity.sandbox.dev
+++ b/models/WRF/Singularity.sandbox.dev
@@ -72,7 +72,7 @@ echo "export PKG_CONFIG_PATH=/usr/lib64/openmpi/lib/pkgconfig/" >> $SINGULARITY_
export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:/usr/lib64:/lib64:/lib
export PATH=/usr/lib64/openmpi/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LIBRARY=/usr/lib64/openmpi/lib:/usr/lib64:/lib64:/lib
-export INCLUDE=/usr/include/openmpi-x86_64/:/usr/lib64/gfortran/modules:/usr/include
+export INCLUDE=/usr/include/openmpi-x86_64/:/usr/lib64/gfortran/modules/openmpi:/usr/include
export NETCDF=/usr/lib64/openmpi
export NETCDF_ROOT=/usr/lib64/openmpi
export JASPERINC=/usr/include/jasper/
diff --git a/models/WRF/Singularity.wrf b/models/WRF/Singularity.wrf
index ef6277ba09a7872292fa82bdfd79f67e86a7fb40..db66903f47ea9d75b1d296fb337fab20e429942a 100644
--- a/models/WRF/Singularity.wrf
+++ b/models/WRF/Singularity.wrf
@@ -48,8 +48,14 @@ Stage: final
# Copy only executables from WRF to directory
/wrf/bin /wrf/bin
/wrf/run /wrf/run
+ # WPS
/wrf/WPS/link_grib.csh /wrf/bin
/wrf/WPS/ungrib/Variable_Tables /wrf/run
+ /wrf/WPS/geogrid/GEOGRID* /wrf/run/geogrid/
+ /wrf/WPS/geogrid/gribmap.txt /wrf/run/geogrid/
+ /wrf/WPS/metgrid/METGRID* /wrf/run/metgrid/
+ /wrf/WPS/metgrid/gribmap.txt /wrf/run/metgrid/
+ # Log files
/wrf/WRF/compile_wrf_arw_opt34.1.log /wrf
/wrf/WPS/compile_wps.log /wrf
diff --git a/models/WRF/Singularity.wrf.py b/models/WRF/Singularity.wrf.py
new file mode 100644
index 0000000000000000000000000000000000000000..014af8ef1529ce043da93dcb656e75447d4716fa
--- /dev/null
+++ b/models/WRF/Singularity.wrf.py
@@ -0,0 +1,44 @@
+Bootstrap: docker
+From: mambaorg/micromamba:latest
+
+%labels
+maintainer IT-IMGW <it.img-wien@univie.ac.at>
+
+%files
+../../definition-files/runscript /.singularity.d/runscript
+../../definition-files/run-help /.singularity.d/runscript.help
+
+%post
+# install curl
+apt -y update && apt -y install curl && apt -y clean
+# install python packages
+micromamba -q install -y -n base -c conda-forge \
+ pygrib=2.1.4 \
+ cartopy=0.21 \
+ netCDF4=1.5.8 \
+ pyyaml=6.0
+# include WRF/WPS scripts
+mkdir -p /opt/wrf && cd /opt/wrf
+curl -sL https://github.com/NCAR/container-dtc-nwp/archive/refs/tags/v4.1.0.tar.gz | tar --strip-components=5 -zxC . container-dtc-nwp-4.1.0/components/scripts/common/python/
+# fix wired paths
+sed -i 's,/home/pythonprd,./,g' *.py
+sed -i 's,/home/postprd/,./,g' *.py
+sed -i 's,/home/scripts/case/,./,g' *.py
+sed -i 's,cartopy.config,#cartopy.config,g' *.py
+
+# final
+CNAME=wrfpy
+# does not work goes into /.singularity.d/env/91-environment.sh
+echo "export PS1=\"[IMGW-$CNAME]\w\$ \"" >> /.singularity.d/env/99-zz-custom-env.sh
+# add some labels
+echo "libc $(ldd --version | head -n1 | cut -d' ' -f4)" >> "$SINGULARITY_LABELS"
+echo "linux $(cat /etc/os-release | grep PRETTY_NAME | cut -d'=' -f2)" >> "$SINGULARITY_LABELS"
+
+%environment
+export MPLCONFIGDIR=.
+export LANG=C.UTF-8
+export PATH=/opt/conda/bin:$PATH
+export LIBRARY=/opt/conda/lib
+export INCLUDE=/opt/conda/include
+export PYTHONDONTWRITEBYTECODE=1
+export PYTHONPATH=/opt/wrf:$PYTHONPATH
diff --git a/models/WRF/dtc_nwp_flow_3.png b/models/WRF/dtc_nwp_flow_3.png
new file mode 100644
index 0000000000000000000000000000000000000000..5bcfea5826de26d3e5c43da12d660fde3ec112bf
Binary files /dev/null and b/models/WRF/dtc_nwp_flow_3.png differ
diff --git a/models/WRF/hurricane-sandy-example.sh b/models/WRF/hurricane-sandy-example.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c5e5e94c3a936b09861bc57700fc6579f811c0e7
--- /dev/null
+++ b/models/WRF/hurricane-sandy-example.sh
@@ -0,0 +1,277 @@
+#!/bin/bash
+# By MB
+# Based on the Example from DTCenter, Boulder NCAR.
+# https://dtcenter.org/nwp-containers-online-tutorial
+# https://dtcenter.org/nwp-containers-online-tutorial/hurricane-sandy-case-27-oct-2012
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+pause() {
+ if $DEBUG; then
+ read -n1 -rsp $'Press any key to continue or Ctrl+C to exit...\n'
+ fi
+}
+
+warning() {
+ printf "%-68s[$YELLOW%10s$NC]\n" "$@" "SKIPPED"
+ return 0
+}
+
+report() {
+ if [ $? -eq 0 ]; then
+ printf "%-68s[$GREEN%10s$NC]\n" "$@" "OK"
+ return 0
+ else
+ printf "%-68s[$RED%10s$NC]\n" "$@" "FAILED"
+ return 1
+ fi
+}
+
+question() {
+ read -p "$@ (y/n)" yn
+ case $yn in
+ [Yy]*) return 0 ;;
+ *)
+ warning "skipping"
+ return 1
+ ;;
+ esac
+}
+
+INAME="SANDY"
+CPATH=$(dirname $0)
+case_name=sandy
+WRFCONTAINER=$CPATH/WRF-4.4.1-em_real.sif
+PYCONTAINER=$CPATH/WRFpy.sif
+GISCONTAINER=$CPATH/GSI-3.7.sif
+NPROC=1
+if [ $NPROC -gt 1 ]; then
+ warning "[$INAME] Trying to run with n=$NPROC processes."
+ echo "[$INAME] Make sure mpirun can be accessed!"
+ command -v mpirun
+ report "[$INAME] mpirun command works" || exit 1
+ # Check if Versions are similar ?
+ # MPI Version outside needs to be newer!
+
+fi
+
+# Check if we have the container?
+
+# test -f Makefile
+# report "[$INAME] Makefile found"
+# if [ $? -eq 0 ]; then
+# echo "[$INAME] Building WRF ..."
+# make em_real
+# fi
+
+# Do a pull request from the IMGW library to retrieve the container
+test -f $WRFCONTAINER
+report "[$INAME] WRF em_real [$WRFCONTAINER]"
+if [ $? -ne 0 ]; then
+ # Try to pull
+ echo "Manual: https://cloud.sylabs.io/library/mblaschek/models/wrf"
+ singularity pull $WRFCONTAINER library://mblaschek//models/wrf-emreal:4.4.1
+ report "[$INAME] WRF em_real pulled from library" || exit 1
+fi
+
+test -f $PYCONTAINER
+report "[$INAME] WRF python [$PYCONTAINER]" || exit 1
+
+test -f $GISCONTAINER
+report "[$INAME] GIS [$GISCONTAINER]" || exit 1
+
+PROJ_PATH=
+read -p "[$INAME] Where do you want to place all files? [$PWD] :" PROJ_PATH
+if [ -z "$PROJ_PATH" ]; then
+ PROJ_PATH=.
+fi
+
+INPUT_DIR=$PROJ_PATH/data
+
+mkdir -vp $PROJ_PATH/sandy/data/
+report "[$INAME] using $PROJ_PATH"
+cd $PROJ_PATH/sandy/data/
+# curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/container-dtc-nwp-derechodata_20120629.tar.gz | tar zxC .
+# curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/container-dtc-nwp-snowdata_20160123.tar.gz | tar zxC .
+
+test -d model_data
+report "[$INAME] Sandy input data"
+if [ $? -ne 0 ]; then
+ curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/container-dtc-nwp-sandydata_20121027.tar.gz | tar zxC .
+ report "[$INAME] using Sandy input data" || exit 1
+fi
+test -d obs_data
+report "[$INAME] Sandy obs data"
+if [ $? -ne 0 ]; then
+ curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/CRTM_v2.3.0.tar.gz | tar zxC .
+ report "[$INAME] using Sandy obs data" || exit 1
+fi
+test -d WPS_GEOG
+report "[$INAME] Sandy geogrid data"
+if [ $? -ne 0 ]; then
+ curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/wps_geog.tar.gz | tar zxC .
+ report "[$INAME] using Sandy geogrid data" || exit 1
+fi
+test -d shapefiles
+report "[$INAME] Sandy natural earth data"
+if [ $? -ne 0 ]; then
+ curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/shapefiles.tar.gz | tar zxC .
+ report "[$INAME] using Sandy natural earth data" || exit 1
+fi
+# return back
+cd $PROJ_PATH/sandy
+
+#
+# dtcenter
+# https://github.com/NCAR/container-dtc-nwp
+#
+
+#
+# Setup Case files for SANDY
+#
+# Get sandy case files
+curl -sL https://github.com/NCAR/container-dtc-nwp/archive/refs/tags/v4.1.0.tar.gz | tar --strip-components=4 -zxC . container-dtc-nwp-4.1.0/components/scripts/sandy_20121027
+# fix paths
+sed -i 's,/data/,./data/,g' set_env.ksh
+sed -i 's,/data/,./data/,g' namelist.wps
+sed -i 's,/comsoftware/wrf/WPS-4.3,./g' namelist.wps
+
+# Optional run plot
+# needs cartopy and python
+# /home/scripts/common/run_python_domain.ksh
+
+#
+# Get default em_real configuration files from /wrf/run
+# WRF/run and WPS/geogrid. WPS/metgrid, WPS/ungrib
+#
+echo "[$INAME] CP em_real files to current directory [no overwriting]"
+./$WRFCONTAINER init
+report "[$INAME] WRF init complete" || exit 1
+
+# Check if Variable Tables are present / depends on input files
+source set_env.ksh
+# input_data must be sourced
+test -d Vtable.${input_data}
+report "[$INAME] Varible Table Vtable.${input_data} found" || {echo "Please provide Variable Table!"
+exit 1}
+# Link Grib files to here
+link_grib.csh $INPUT_DIR/model_data/${case_name}/*
+report "[$INAME] Model input data from: $INPUT_DIR/model_data/${case_name}/"
+
+#
+# geogrid data
+#
+# Remove old files
+if [ -e geo_em.d*.nc ]; then
+ rm -rf geo_em.d*.nc
+fi
+
+# Command for geogrid
+echo "[$INAME] Running geogrib ..."
+singularity exec $WRFCONTAINER geogrid.exe >run_geogrid.log 2>&1
+
+test -f geo_em.d01.nc
+report "[$INAME] WPS geogrid ready." || exit 1
+
+#
+# ungrib data
+#
+file_date=$(cat namelist.wps | grep -i start_date | cut -d"'" -f2 | cut -d":" -f1)
+# remove old files
+if [ -e PFILE:${file_date} ]; then
+ rm -rf PFILE*
+fi
+if [ -e FILE:${file_date} ]; then
+ rm -rf FILE*
+fi
+
+# Command for ungrib
+echo "[$INAME] Running ungrib ..."
+singularity exec $WRFCONTAINER ungrib.exe >run_ungrib.log 2>&1
+
+ls -ls FILE:*
+report "[$INAME] WPS ungrib ready." || exit 1
+
+#
+# metgrid data
+#
+
+# Remove old files
+if [ -e met_em.d*.${file_date}:00:00.nc ]; then
+ rm -rf met_em.d*
+fi
+
+# Command for metgrid
+echo "[$INAME] Running metgrid ..."
+singularity exec $WRFCONTAINER metgrid.exe >run_metgrid.log 2>&1
+
+ls -ls met_em.d01.*
+report "[$INAME] WPS metgrid ready." || exit 1
+
+echo "[$INAME] WPS setup complete. Ready for WRF"
+
+#
+# WRF
+#
+
+sed -e '/nocolons/d' namelist.input >nml
+cp namelist.input namelist.nocolons
+mv nml namelist.input
+
+# Remove old files
+if [ -e wrfinput_d* ]; then
+ rm -rf wrfi* wrfb*
+fi
+
+# Command for real
+echo "[$INAME] Running real ..."
+# This can be run with MPI or serial?
+if [ $NPROC -gt 1 ]; then
+ # MPI
+ module load openmpi/
+ mpirun -np $NPROC singularity exec $WRFCONTAINER real.exe >run_real.log 2>&1
+else
+ # Serial run
+ singularity exec $WRFCONTAINER real.exe >run_real.log 2>&1
+fi
+test -f wrfinput_d01 && test -f wrfbdy_d01
+report "[$INAME] WRF run finished"
+
+#
+# GSI Data Assimilation
+#
+if false; then
+ echo "[$INAME] Running gsi ..."
+ singularity run --app init $GISCONTAINER
+ test -f run_gsi.ksh
+ report "[$INAME] GSI run script ready" || exit 1
+ # add the correct run Command
+ if [ $NPROC -gt 1 ]; then
+ echo "export RUN_COMMAND=\"mpirun -np $NPROC singularity exec $GISCONTAINER \$GIS_EXE\"" >>set_env.ksh
+ else
+ echo "export RUN_COMMAND=\"singularity exec $GISCONTAINER \$GIS_EXE\"" >>set_env.ksh
+ fi
+ # Execute GSI
+ ./run_gsi.ksh
+fi
+#
+# run WRF
+#
+# If wrfinput_d01.orig exists, rename it to wrfinput_d01 to reset the state
+if [[ -e wrfinput_d01.orig ]]; then
+ mv wrfinput_d01.orig wrfinput_d01
+fi
+
+# If GSI was run, update the wrfinput file
+if [[ -e ./wrf_inout ]]; then
+ mv wrfinput_d01 wrfinput_d01.orig
+ cp ./wrf_inout wrfinput_d01
+fi
+if [ $NPROC -gt 1 ]; then
+ mpirun -np $NPROC singularity exec $WRFCONTAINER wrf.exe >run_wrf.log 2>&1
+else
+ singularity exec $WRFCONTAINER wrf.exe >run_wrf.log 2>&1
+fi
diff --git a/models/WRF/hurricane-sany-example.sh b/models/WRF/hurricane-sany-example.sh
deleted file mode 100755
index f5b73716a2780d23c42e26e45b521dc65c41012c..0000000000000000000000000000000000000000
--- a/models/WRF/hurricane-sany-example.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/bin/bash
-# By MB
-# Based on the Example from DTCenter, Boulder NCAR.
-# https://dtcenter.org/nwp-containers-online-tutorial
-# https://dtcenter.org/nwp-containers-online-tutorial/hurricane-sandy-case-27-oct-2012
-
-CPATH=$(dirname $0)
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m'
-
-pause() {
- if $DEBUG; then
- read -n1 -rsp $'Press any key to continue or Ctrl+C to exit...\n'
- fi
-}
-
-warning() {
- printf "%-68s[$YELLOW%10s$NC]\n" "$@" "SKIPPED"
- return 0
-}
-
-report() {
- if [ $? -eq 0 ]; then
- printf "%-68s[$GREEN%10s$NC]\n" "$@" "OK"
- return 0
- else
- printf "%-68s[$RED%10s$NC]\n" "$@" "FAILED"
- return 1
- fi
-}
-
-question(){
- read -p "$@ (y/n)" yn
- case $yn in
- [Yy]*) return 0;;
- * ) warning "skipping"; return 1;;
- esac
-}
-
-INAME="SANDY"
-# Check if we have the container?
-
-test -f Makefile
-report "[$INAME] Makefile found"
-if [ $? -eq 0 ]; then
- echo "[$INAME] Building WRF ..."
- make em_real
-fi
-
-test -f WRF-4.4.1-em_real.sif
-report "[$INAME] WRF em_real container available" || exit 1
-
-PROJ_PATH=
-read -p "[$INAME] Where do you want to place all files? [$PWD] :" PROJ_PATH
-if [ -z "$PROJ_PATH" ]; then
- PROJ_PATH=.
-fi
-
-mkdir -vp $PROJ_PATH/sandy/data/
-report "[$INAME] using $PROJ_PATH"
-cd $PROJ_PATH/sandy/data/
-# curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/container-dtc-nwp-derechodata_20120629.tar.gz | tar zxC .
-# curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/container-dtc-nwp-snowdata_20160123.tar.gz | tar zxC .
-
-test -d model_data
-report "[$INAME] Sandy input data"
-if [ $? -ne 0 ]; then
- curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/container-dtc-nwp-sandydata_20121027.tar.gz | tar zxC .
- report "[$INAME] using Sandy input data" || exit 1
-fi
-test -d obs_data
-report "[$INAME] Sandy obs data"
-if [ $? -ne 0 ]; then
- curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/CRTM_v2.3.0.tar.gz | tar zxC .
- report "[$INAME] using Sandy obs data" || exit 1
-fi
-test -d WPS_GEOG
-report "[$INAME] Sandy geogrid data"
-if [ $? -ne 0 ]; then
- curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/wps_geog.tar.gz | tar zxC .
- report "[$INAME] using Sandy geogrid data" || exit 1
-fi
-test -d shapefiles
-report "[$INAME] Sandy trajectory data"
-if [ $? -ne 0 ]; then
- curl -SL https://dtcenter.ucar.edu/dfiles/container_nwp_tutorial/tar_files/shapefiles.tar.gz | tar zxC .
- report "[$INAME] using Sandy trajectory data" || exit 1
-fi
-
-exit 0
-/home/scripts/common/run_python_domain.ksh
-
-WPS_GEOG
-/home/scripts/common/run_wps.ksh
-
-
-/home/scripts/common/run_real.ksh
\ No newline at end of file
diff --git a/models/WRF/scripts/run-help b/models/WRF/scripts/run-help
index a0351f7527dc606864b7290552d474ce61eba37b..ef55af8e76ed7a6503989a4c2f2aeaa934a7816f 100644
--- a/models/WRF/scripts/run-help
+++ b/models/WRF/scripts/run-help
@@ -16,16 +16,19 @@ bind special directories
development using includes and libraries from inside the container
> ./image.sif gfortran -I\$INCLUDE -L\$LIBRARY -o test.x test.f90
-
WRF Simulations - Example
default run files are located in /wrf/run
executables are located in /wrf/bin
-Default simulation of the container (still requires input)
-1. copies /wrf/run files to the local directory
-2. No local files will be overwritten (e.g. namelists )
-3. Execute real.exe
+Default simulation of the container (still requires input), will do
+1. It will copy /wrf/run files to the local directory (.)
+2. No local files will be overwritten (e.g. namelists), just missing files.
+3. Execute real.exe, run:
> ./image.sif real.exe
+> ./image.sif wrf.exe
+
+Using OpenMPI (requires an openmpi version on the host system)
+> mpirun -np 8 ./image.sif real.exe
Manually copy run files to current directory
> ./image.sif init
diff --git a/models/WRF/scripts/run_gsi.ksh b/models/WRF/scripts/run_gsi.ksh
new file mode 100755
index 0000000000000000000000000000000000000000..a94f27b17a01d254e88ca52d67dcf2145953cd9c
--- /dev/null
+++ b/models/WRF/scripts/run_gsi.ksh
@@ -0,0 +1,656 @@
+#!/bin/ksh
+#####################################################
+# machine set up (users should change this part)
+#####################################################
+set -x
+#
+# Constants
+INPUT_DIR="./data/gsi"
+CASE_DIR="."
+WRFPRD_DIR="."
+#####################################################
+# case set up (users should change this part)
+#####################################################
+#
+# ANAL_TIME= analysis time (YYYYMMDDHH)
+# WORK_ROOT= working directory, where GSI runs
+# PREPBUFR = path of PreBUFR conventional obs
+# BK_FILE = path and name of background file
+# OBS_ROOT = path of observations files
+# FIX_ROOT = path of fix files
+# GSI_EXE = path and name of the gsi executable
+# ENS_ROOT = path where ensemble background files exist
+WORK_ROOT=.
+BK_ROOT=${WRFPRD_DIR}
+CRTM_ROOT=./data/gsi/CRTM_v2.3.0
+GSI_ROOT=/wrf/gsi/comGSIv3.7_EnKFv1.3
+GSI_BUILD=/wrf/gsi/build
+FIX_ROOT=${GSI_ROOT}/fix/
+GSI_EXE=${GSI_BUILD}/bin/gsi.x
+RUN_COMMAND=$GSI_EXE
+GSI_NAMELIST=${GSI_ROOT}/ush/comgsi_namelist.sh
+#####################################################
+#
+# Include case-specific settings
+#
+#
+# This can overwrite all above directory/settings
+#
+#####################################################
+. $CASE_DIR/set_env.ksh
+#####################################################
+#
+# Extract START_DATE
+#
+BK_FILE=${BK_ROOT}/wrfinput_d01
+YYYY=$(ncdump -h ${BK_FILE} | grep ":START_DATE" | cut -f2 -d"=" | cut -c3-6)
+MM=$(ncdump -h ${BK_FILE} | grep ":START_DATE" | cut -f2 -d"=" | cut -c8-9)
+DD=$(ncdump -h ${BK_FILE} | grep ":START_DATE" | cut -f2 -d"=" | cut -c11-12)
+HH=$(ncdump -h ${BK_FILE} | grep ":START_DATE" | cut -f2 -d"=" | cut -c14-15)
+ANAL_TIME=${YYYY}${MM}${DD}${HH}
+# using tm06 NDAS prepbufr here, so need have proper date for the observation file, which is ANAL_TIME + 6 hour
+OBSTIME=$(${GSI_BUILD}/bin/ndate.x +6 ${ANAL_TIME})
+HHH=$(echo $OBSTIME | cut -c9-10)
+PREPBUFR=${PREPBUFR:-./data/obs_data/prepbufr/$OBSTIME/ndas.t${HHH}z.prepbufr.tm06.nr}
+#
+#------------------------------------------------
+# bk_core= which WRF core is used as background (NMM or ARW or NMMB)
+# bkcv_option= which background error covariance and parameter will be used
+# (GLOBAL or NAM)
+# if_clean = clean : delete temperal files in working directory (default)
+# no : leave running directory as is (this is for debug only)
+# if_observer = Yes : only used as observation operater for enkf
+# if_hybrid = Yes : Run GSI as 3D/4D EnVar
+# if_4DEnVar = Yes : Run GSI as 4D EnVar
+# if_nemsio = Yes : The GFS background files are in NEMSIO format
+# if_oneob = Yes : Do single observation test
+if_hybrid=No # Yes, or, No -- case sensitive !
+if_4DEnVar=No # Yes, or, No -- case sensitive (set if_hybrid=Yes first)!
+if_observer=No # Yes, or, No -- case sensitive !
+if_nemsio=No # Yes, or, No -- case sensitive !
+if_oneob=No # Yes, or, No -- case sensitive !
+
+bk_core=ARW
+bkcv_option=NAM
+if_clean=clean
+
+# Check if background file exists in the right place and is non-zero size
+
+if [[ ! -s $BK_FILE ]]; then
+ echo
+ echo ERROR: The background file $BK_FILE does not exist!
+ echo
+ exit 1
+fi
+
+#
+# setup whether to do single obs test
+if [ ${if_oneob} = Yes ]; then
+ if_oneobtest='.true.'
+else
+ if_oneobtest='.false.'
+fi
+#
+# setup for GSI 3D/4D EnVar hybrid
+if [ ${if_hybrid} = Yes ]; then
+ PDYa=$(echo $ANAL_TIME | cut -c1-8)
+ cyca=$(echo $ANAL_TIME | cut -c9-10)
+ gdate=$(date -u -d "$PDYa $cyca -6 hour" +%Y%m%d%H) #guess date is 6hr ago
+ gHH=$(echo $gdate | cut -c9-10)
+ datem1=$(date -u -d "$PDYa $cyca -1 hour" +%Y-%m-%d_%H:%M:%S) #1hr ago
+ datep1=$(date -u -d "$PDYa $cyca 1 hour" +%Y-%m-%d_%H:%M:%S) #1hr later
+ if [ ${if_nemsio} = Yes ]; then
+ if_gfs_nemsio='.true.'
+ ENSEMBLE_FILE_mem=${ENS_ROOT}/gdas.t${gHH}z.atmf006s.mem
+ else
+ if_gfs_nemsio='.false.'
+ ENSEMBLE_FILE_mem=${ENS_ROOT}/sfg_${gdate}_fhr06s_mem
+ fi
+
+ if [ ${if_4DEnVar} = Yes ]; then
+ BK_FILE_P1=${BK_ROOT}/wrfout_d01_${datep1}
+ BK_FILE_M1=${BK_ROOT}/wrfout_d01_${datem1}
+
+ if [ ${if_nemsio} = Yes ]; then
+ ENSEMBLE_FILE_mem_p1=${ENS_ROOT}/gdas.t${gHH}z.atmf009s.mem
+ ENSEMBLE_FILE_mem_m1=${ENS_ROOT}/gdas.t${gHH}z.atmf003s.mem
+ else
+ ENSEMBLE_FILE_mem_p1=${ENS_ROOT}/sfg_${gdate}_fhr09s_mem
+ ENSEMBLE_FILE_mem_m1=${ENS_ROOT}/sfg_${gdate}_fhr03s_mem
+ fi
+ fi
+fi
+
+# The following two only apply when if_observer = Yes, i.e. run observation operator for EnKF
+# no_member number of ensemble members
+# BK_FILE_mem path and base for ensemble members
+no_member=20
+BK_FILE_mem=${BK_ROOT}/wrfarw.mem
+#
+#
+#####################################################
+# Users should NOT make changes after this point
+#####################################################
+#
+BYTE_ORDER=Big_Endian
+# BYTE_ORDER=Little_Endian
+
+##################################################################################
+# Check GSI needed environment variables are defined and exist
+#
+
+# Make sure ANAL_TIME is defined and in the correct format
+if [ ! "${ANAL_TIME}" ]; then
+ echo "ERROR: \$ANAL_TIME is not defined!"
+ exit 1
+fi
+
+# Make sure WORK_ROOT is defined and exists
+if [ ! "${WORK_ROOT}" ]; then
+ echo "ERROR: \$WORK_ROOT is not defined!"
+ exit 1
+fi
+
+# Make sure the background file exists
+if [ ! -r "${BK_FILE}" ]; then
+ echo "ERROR: ${BK_FILE} does not exist!"
+ exit 1
+fi
+
+# Make sure OBS_ROOT is defined and exists
+if [ ! "${OBS_ROOT}" ]; then
+ echo "ERROR: \$OBS_ROOT is not defined!"
+ exit 1
+fi
+if [ ! -d "${OBS_ROOT}" ]; then
+ echo "ERROR: OBS_ROOT directory '${OBS_ROOT}' does not exist!"
+ exit 1
+fi
+
+# Set the path to the GSI static files
+if [ ! "${FIX_ROOT}" ]; then
+ echo "ERROR: \$FIX_ROOT is not defined!"
+ exit 1
+fi
+if [ ! -d "${FIX_ROOT}" ]; then
+ echo "ERROR: fix directory '${FIX_ROOT}' does not exist!"
+ exit 1
+fi
+
+# Set the path to the CRTM coefficients
+if [ ! "${CRTM_ROOT}" ]; then
+ echo "ERROR: \$CRTM_ROOT is not defined!"
+ exit 1
+fi
+if [ ! -d "${CRTM_ROOT}" ]; then
+ echo "ERROR: fix directory '${CRTM_ROOT}' does not exist!"
+ exit 1
+fi
+
+# Make sure the GSI executable exists
+if [ ! -x "${GSI_EXE}" ]; then
+ echo "ERROR: ${GSI_EXE} does not exist!"
+ exit 1
+fi
+
+# Check to make sure the number of processors for running GSI was specified
+if [ -z "${GSIPROC}" ]; then
+ echo "ERROR: The variable $GSIPROC must be set to contain the number of processors to run GSI"
+ exit 1
+fi
+
+#
+##################################################################################
+# Create the ram work directory and cd into it
+
+workdir=${WORK_ROOT}
+echo " Create working directory:" ${workdir}
+
+if [ -d "${workdir}" ]; then
+ rm -rf ${workdir}/*
+fi
+mkdir -p ${workdir}
+cd ${workdir}
+
+#
+##################################################################################
+
+echo " Copy GSI executable, background file, and link observation bufr to working directory"
+
+# Save a copy of the GSI executable in the workdir
+# cp ${GSI_EXE} gsi.x
+
+# Bring over background field (it's modified by GSI so we can't link to it)
+cp ${BK_FILE} ./wrf_inout
+if [ ${if_4DEnVar} = Yes ]; then
+ cp ${BK_FILE_P1} ./wrf_inou3
+ cp ${BK_FILE_M1} ./wrf_inou1
+fi
+
+# Link to the prepbufr data
+ln -s ${PREPBUFR} ./prepbufr
+
+# ln -s ${OBS_ROOT}/gdas1.t${HH}z.sptrmm.tm00.bufr_d tmirrbufr
+# Link to the radiance data
+srcobsfile[1]=${OBS_ROOT}/gdas1.t${HH}z.satwnd.tm00.bufr_d
+gsiobsfile[1]=satwnd
+srcobsfile[2]=${OBS_ROOT}/gdas1.t${HH}z.1bamua.tm00.bufr_d
+gsiobsfile[2]=amsuabufr
+srcobsfile[3]=${OBS_ROOT}/gdas1.t${HH}z.1bhrs4.tm00.bufr_d
+gsiobsfile[3]=hirs4bufr
+srcobsfile[4]=${OBS_ROOT}/gdas1.t${HH}z.1bmhs.tm00.bufr_d
+gsiobsfile[4]=mhsbufr
+srcobsfile[5]=${OBS_ROOT}/gdas1.t${HH}z.1bamub.tm00.bufr_d
+gsiobsfile[5]=amsubbufr
+srcobsfile[6]=${OBS_ROOT}/gdas1.t${HH}z.ssmisu.tm00.bufr_d
+gsiobsfile[6]=ssmirrbufr
+# srcobsfile[7]=${OBS_ROOT}/gdas1.t${HH}z.airsev.tm00.bufr_d
+gsiobsfile[7]=airsbufr
+srcobsfile[8]=${OBS_ROOT}/gdas1.t${HH}z.sevcsr.tm00.bufr_d
+gsiobsfile[8]=seviribufr
+srcobsfile[9]=${OBS_ROOT}/gdas1.t${HH}z.iasidb.tm00.bufr_d
+gsiobsfile[9]=iasibufr
+srcobsfile[10]=${OBS_ROOT}/gdas1.t${HH}z.gpsro.tm00.bufr_d
+gsiobsfile[10]=gpsrobufr
+srcobsfile[11]=${OBS_ROOT}/gdas1.t${HH}z.amsr2.tm00.bufr_d
+gsiobsfile[11]=amsrebufr
+srcobsfile[12]=${OBS_ROOT}/gdas1.t${HH}z.atms.tm00.bufr_d
+gsiobsfile[12]=atmsbufr
+srcobsfile[13]=${OBS_ROOT}/gdas1.t${HH}z.geoimr.tm00.bufr_d
+gsiobsfile[13]=gimgrbufr
+srcobsfile[14]=${OBS_ROOT}/gdas1.t${HH}z.gome.tm00.bufr_d
+gsiobsfile[14]=gomebufr
+srcobsfile[15]=${OBS_ROOT}/gdas1.t${HH}z.omi.tm00.bufr_d
+gsiobsfile[15]=omibufr
+srcobsfile[16]=${OBS_ROOT}/gdas1.t${HH}z.osbuv8.tm00.bufr_d
+gsiobsfile[16]=sbuvbufr
+srcobsfile[17]=${OBS_ROOT}/gdas1.t${HH}z.eshrs3.tm00.bufr_d
+gsiobsfile[17]=hirs3bufrears
+srcobsfile[18]=${OBS_ROOT}/gdas1.t${HH}z.esamua.tm00.bufr_d
+gsiobsfile[18]=amsuabufrears
+srcobsfile[19]=${OBS_ROOT}/gdas1.t${HH}z.esmhs.tm00.bufr_d
+gsiobsfile[19]=mhsbufrears
+srcobsfile[20]=${OBS_ROOT}/rap.t${HH}z.nexrad.tm00.bufr_d
+gsiobsfile[20]=l2rwbufr
+srcobsfile[21]=${OBS_ROOT}/rap.t${HH}z.lgycld.tm00.bufr_d
+gsiobsfile[21]=larcglb
+srcobsfile[22]=${OBS_ROOT}/gdas1.t${HH}z.glm.tm00.bufr_d
+gsiobsfile[22]=
+# ii=1
+# while [[ $ii -le 21 ]]; do
+# if [ -r "${srcobsfile[$ii]}" ]; then
+# # ln -s ${srcobsfile[$ii]} ${gsiobsfile[$ii]}
+# echo "link source obs file ${srcobsfile[$ii]}"
+# fi
+# ((ii = $ii + 1))
+# done
+
+#
+##################################################################################
+
+ifhyb=.false.
+if [ ${if_hybrid} = Yes ]; then
+ ls ${ENSEMBLE_FILE_mem}* >filelist02
+ if [ ${if_4DEnVar} = Yes ]; then
+ ls ${ENSEMBLE_FILE_mem_p1}* >filelist03
+ ls ${ENSEMBLE_FILE_mem_m1}* >filelist01
+ fi
+
+ nummem=$(more filelist02 | wc -l)
+ nummem=$((nummem - 3))
+
+ if [[ ${nummem} -ge 5 ]]; then
+ ifhyb=.true.
+ ${ECHO} " GSI hybrid uses ${ENSEMBLE_FILE_mem} with n_ens=${nummem}"
+ fi
+fi
+if4d=.false.
+if [[ ${ifhyb} = .true. && ${if_4DEnVar} = Yes ]]; then
+ if4d=.true.
+fi
+#
+##################################################################################
+
+echo " Copy fixed files and link CRTM coefficient files to working directory"
+
+# Set fixed files
+# berror = forecast model background error statistics
+# specoef = CRTM spectral coefficients
+# trncoef = CRTM transmittance coefficients
+# emiscoef = CRTM coefficients for IR sea surface emissivity model
+# aerocoef = CRTM coefficients for aerosol effects
+# cldcoef = CRTM coefficients for cloud effects
+# satinfo = text file with information about assimilation of brightness temperatures
+# satangl = angle dependent bias correction file (fixed in time)
+# pcpinfo = text file with information about assimilation of prepcipitation rates
+# ozinfo = text file with information about assimilation of ozone data
+# errtable = text file with obs error for conventional data (regional only)
+# convinfo = text file with information about assimilation of conventional data
+# lightinfo= text file with information about assimilation of GLM lightning data
+# bufrtable= text file ONLY needed for single obs test (oneobstest=.true.)
+# bftab_sst= bufr table for sst ONLY needed for sst retrieval (retrieval=.true.)
+
+if [ ${bkcv_option} = GLOBAL ]; then
+ echo ' Use global background error covariance'
+ BERROR=${FIX_ROOT}/${BYTE_ORDER}/nam_glb_berror.f77.gcv
+ OBERROR=${FIX_ROOT}/prepobs_errtable.global
+ if [ ${bk_core} = NMM ]; then
+ ANAVINFO=${FIX_ROOT}/anavinfo_ndas_netcdf_glbe
+ fi
+ if [ ${bk_core} = ARW ]; then
+ ANAVINFO=${FIX_ROOT}/anavinfo_arw_netcdf_glbe
+ fi
+ if [ ${bk_core} = NMMB ]; then
+ ANAVINFO=${FIX_ROOT}/anavinfo_nems_nmmb_glb
+ fi
+else
+ echo ' Use NAM background error covariance'
+ BERROR=${FIX_ROOT}/${BYTE_ORDER}/nam_nmmstat_na.gcv
+ OBERROR=${FIX_ROOT}/nam_errtable.r3dv
+ if [ ${bk_core} = NMM ]; then
+ ANAVINFO=${FIX_ROOT}/anavinfo_ndas_netcdf
+ fi
+ if [ ${bk_core} = ARW ]; then
+ ANAVINFO=${FIX_ROOT}/anavinfo_arw_netcdf
+ fi
+ if [ ${bk_core} = NMMB ]; then
+ ANAVINFO=${FIX_ROOT}/anavinfo_nems_nmmb
+ fi
+fi
+
+SATANGL=${FIX_ROOT}/global_satangbias.txt
+SATINFO=${FIX_ROOT}/global_satinfo.txt
+CONVINFO=${FIX_ROOT}/global_convinfo.txt
+OZINFO=${FIX_ROOT}/global_ozinfo.txt
+PCPINFO=${FIX_ROOT}/global_pcpinfo.txt
+LIGHTINFO=${FIX_ROOT}/global_lightinfo.txt
+
+# copy Fixed fields to working directory
+cp $ANAVINFO anavinfo
+cp $BERROR berror_stats
+cp $SATANGL satbias_angle
+cp $SATINFO satinfo
+cp $CONVINFO convinfo
+cp $OZINFO ozinfo
+cp $PCPINFO pcpinfo
+cp $LIGHTINFO lightinfo
+cp $OBERROR errtable
+#
+# # CRTM Spectral and Transmittance coefficients
+CRTM_ROOT_ORDER=${CRTM_ROOT}/${BYTE_ORDER}
+emiscoef_IRwater=${CRTM_ROOT_ORDER}/Nalli.IRwater.EmisCoeff.bin
+emiscoef_IRice=${CRTM_ROOT_ORDER}/NPOESS.IRice.EmisCoeff.bin
+emiscoef_IRland=${CRTM_ROOT_ORDER}/NPOESS.IRland.EmisCoeff.bin
+emiscoef_IRsnow=${CRTM_ROOT_ORDER}/NPOESS.IRsnow.EmisCoeff.bin
+emiscoef_VISice=${CRTM_ROOT_ORDER}/NPOESS.VISice.EmisCoeff.bin
+emiscoef_VISland=${CRTM_ROOT_ORDER}/NPOESS.VISland.EmisCoeff.bin
+emiscoef_VISsnow=${CRTM_ROOT_ORDER}/NPOESS.VISsnow.EmisCoeff.bin
+emiscoef_VISwater=${CRTM_ROOT_ORDER}/NPOESS.VISwater.EmisCoeff.bin
+emiscoef_MWwater=${CRTM_ROOT_ORDER}/FASTEM6.MWwater.EmisCoeff.bin
+aercoef=${CRTM_ROOT_ORDER}/AerosolCoeff.bin
+cldcoef=${CRTM_ROOT_ORDER}/CloudCoeff.bin
+
+ln -s $emiscoef_IRwater ./Nalli.IRwater.EmisCoeff.bin
+ln -s $emiscoef_IRice ./NPOESS.IRice.EmisCoeff.bin
+ln -s $emiscoef_IRsnow ./NPOESS.IRsnow.EmisCoeff.bin
+ln -s $emiscoef_IRland ./NPOESS.IRland.EmisCoeff.bin
+ln -s $emiscoef_VISice ./NPOESS.VISice.EmisCoeff.bin
+ln -s $emiscoef_VISland ./NPOESS.VISland.EmisCoeff.bin
+ln -s $emiscoef_VISsnow ./NPOESS.VISsnow.EmisCoeff.bin
+ln -s $emiscoef_VISwater ./NPOESS.VISwater.EmisCoeff.bin
+ln -s $emiscoef_MWwater ./FASTEM6.MWwater.EmisCoeff.bin
+ln -s $aercoef ./AerosolCoeff.bin
+ln -s $cldcoef ./CloudCoeff.bin
+# Copy CRTM coefficient files based on entries in satinfo file
+for file in $(awk '{if($1!~"!"){print $1}}' ./satinfo | sort | uniq); do
+ ln -s ${CRTM_ROOT_ORDER}/${file}.SpcCoeff.bin ./
+ ln -s ${CRTM_ROOT_ORDER}/${file}.TauCoeff.bin ./
+done
+
+# Only need this file for single obs test
+bufrtable=${FIX_ROOT}/prepobs_prep.bufrtable
+cp $bufrtable ./prepobs_prep.bufrtable
+
+# for satellite bias correction
+# Users may need to use their own satbias files for correct bias correction
+cp ${GSI_ROOT}/fix/comgsi_satbias_in ./satbias_in
+cp ${GSI_ROOT}/fix/comgsi_satbias_pc_in ./satbias_pc_in
+
+#
+##################################################################################
+# Set some parameters for use by the GSI executable and to build the namelist
+echo " Build the namelist "
+
+# default is NAM
+# as_op='1.0,1.0,0.5 ,0.7,0.7,0.5,1.0,1.0,'
+vs_op='1.0,'
+hzscl_op='0.373,0.746,1.50,'
+if [ ${bkcv_option} = GLOBAL ]; then
+ # as_op='0.6,0.6,0.75,0.75,0.75,0.75,1.0,1.0'
+ vs_op='0.7,'
+ hzscl_op='1.7,0.8,0.5,'
+fi
+if [ ${bk_core} = NMMB ]; then
+ vs_op='0.6,'
+fi
+
+# default is NMM
+bk_core_arw='.false.'
+bk_core_nmm='.true.'
+bk_core_nmmb='.false.'
+bk_if_netcdf='.true.'
+if [ ${bk_core} = ARW ]; then
+ bk_core_arw='.true.'
+ bk_core_nmm='.false.'
+ bk_core_nmmb='.false.'
+ bk_if_netcdf='.true.'
+fi
+if [ ${bk_core} = NMMB ]; then
+ bk_core_arw='.false.'
+ bk_core_nmm='.false.'
+ bk_core_nmmb='.true.'
+ bk_if_netcdf='.false.'
+fi
+
+if [ ${if_observer} = Yes ]; then
+ nummiter=0
+ if_read_obs_save='.true.'
+ if_read_obs_skip='.false.'
+else
+ nummiter=2
+ if_read_obs_save='.false.'
+ if_read_obs_skip='.false.'
+fi
+
+# Build the GSI namelist on-the-fly
+. $GSI_NAMELIST
+
+# modify the anavinfo vertical levels based on wrf_inout for WRF ARW and NMM
+if [ ${bk_core} = ARW ] || [ ${bk_core} = NMM ]; then
+ bklevels=$(ncdump -h wrf_inout | grep "bottom_top =" | awk '{print $3}')
+ bklevels_stag=$(ncdump -h wrf_inout | grep "bottom_top_stag =" | awk '{print $3}')
+ anavlevels=$(cat anavinfo | grep ' sf ' | tail -1 | awk '{print $2}') # levels of sf, vp, u, v, t, etc
+ anavlevels_stag=$(cat anavinfo | grep ' prse ' | tail -1 | awk '{print $2}') # levels of prse
+ sed -i 's/ '$anavlevels'/ '$bklevels'/g' anavinfo
+ sed -i 's/ '$anavlevels_stag'/ '$bklevels_stag'/g' anavinfo
+fi
+
+#
+###################################################
+# run GSI
+###################################################
+echo ' Run GSI with' ${bk_core} 'background'
+
+# case $ARCH in
+# 'IBM_LSF')
+# ${RUN_COMMAND} ./gsi.x <gsiparm.anl >stdout 2>&1
+# ;;
+
+# *)
+# ${RUN_COMMAND} ./gsi.x >stdout 2>&1
+# ;;
+# esac
+$RUN_COMMAND >stdout 2>&1
+
+##################################################################
+# run time error check
+##################################################################
+error=$?
+
+if [ ${error} -ne 0 ]; then
+ echo "ERROR: ${GSI} crashed Exit status=${error}"
+ exit ${error}
+fi
+
+#
+##################################################################
+#
+# GSI updating satbias_in
+#
+# GSI updating satbias_in (only for cycling assimilation)
+
+# Copy the output to more understandable names
+ln -s stdout stdout.anl.${ANAL_TIME}
+ln -s wrf_inout wrfanl.${ANAL_TIME}
+ln -s fort.201 fit_p1.${ANAL_TIME}
+ln -s fort.202 fit_w1.${ANAL_TIME}
+ln -s fort.203 fit_t1.${ANAL_TIME}
+ln -s fort.204 fit_q1.${ANAL_TIME}
+ln -s fort.207 fit_rad1.${ANAL_TIME}
+
+# Loop over first and last outer loops to generate innovation
+# diagnostic files for indicated observation types (groups)
+#
+# NOTE: Since we set miter=2 in GSI namelist SETUP, outer
+# loop 03 will contain innovations with respect to
+# the analysis. Creation of o-a innovation files
+# is triggered by write_diag(3)=.true. The setting
+# write_diag(1)=.true. turns on creation of o-g
+# innovation files.
+#
+
+loops="01 03"
+for loop in $loops; do
+
+ case $loop in
+ 01) string=ges ;;
+ 03) string=anl ;;
+ *) string=$loop ;;
+ esac
+
+ # Collect diagnostic files for obs types (groups) below
+ # listall="conv amsua_metop-a mhs_metop-a hirs4_metop-a hirs2_n14 msu_n14 \
+ # sndr_g08 sndr_g10 sndr_g12 sndr_g08_prep sndr_g10_prep sndr_g12_prep \
+ # sndrd1_g08 sndrd2_g08 sndrd3_g08 sndrd4_g08 sndrd1_g10 sndrd2_g10 \
+ # sndrd3_g10 sndrd4_g10 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 \
+ # hirs3_n15 hirs3_n16 hirs3_n17 amsua_n15 amsua_n16 amsua_n17 \
+ # amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua \
+ # goes_img_g08 goes_img_g10 goes_img_g11 goes_img_g12 \
+ # pcp_ssmi_dmsp pcp_tmi_trmm sbuv2_n16 sbuv2_n17 sbuv2_n18 \
+ # omi_aura ssmi_f13 ssmi_f14 ssmi_f15 hirs4_n18 amsua_n18 mhs_n18 \
+ # amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_las_f16 \
+ # ssmis_uas_f16 ssmis_img_f16 ssmis_env_f16 mhs_metop_b \
+ # hirs4_metop_b hirs4_n19 amusa_n19 mhs_n19 goes_glm_16"
+ listall=$(ls pe* | cut -f2 -d"." | awk '{print substr($0, 0, length($0)-3)}' | sort | uniq)
+
+ for type in $listall; do
+ count=$(ls pe*${type}_${loop}* | wc -l)
+ if [[ $count -gt 0 ]]; then
+ cat pe*${type}_${loop}* >diag_${type}_${string}.${ANAL_TIME}
+ fi
+ done
+done
+
+# Clean working directory to save only important files
+ls -l * >list_run_directory
+if [[ ${if_clean} = clean && ${if_observer} != Yes ]]; then
+ echo ' Clean working directory after GSI run'
+ rm -f *Coeff.bin # all CRTM coefficient files
+ rm -f pe0* # diag files on each processor
+ rm -f obs_input.* # observation middle files
+ rm -f siganl sigf0? # background middle files
+ rm -f fsize_* # delete temperal file for bufr size
+fi
+#
+#
+#################################################
+# start to calculate diag files for each member
+#################################################
+#
+if [ ${if_observer} = Yes ]; then
+ string=ges
+ for type in $listall; do
+ count=0
+ if [[ -f diag_${type}_${string}.${ANAL_TIME} ]]; then
+ mv diag_${type}_${string}.${ANAL_TIME} diag_${type}_${string}.ensmean
+ fi
+ done
+ mv wrf_inout wrf_inout_ensmean
+
+ # Build the GSI namelist on-the-fly for each member
+ nummiter=0
+ if_read_obs_save='.false.'
+ if_read_obs_skip='.true.'
+ . $GSI_NAMELIST
+
+ # Loop through each member
+ loop="01"
+ ensmem=1
+ while [[ $ensmem -le $no_member ]]; do
+
+ rm pe0*
+
+ print "\$ensmem is $ensmem"
+ ensmemid=$(printf %3.3i $ensmem)
+
+ # get new background for each member
+ if [[ -f wrf_inout ]]; then
+ rm wrf_inout
+ fi
+
+ BK_FILE=${BK_FILE_mem}${ensmemid}
+ echo $BK_FILE
+ ln -s $BK_FILE wrf_inout
+
+ # run GSI
+ echo ' Run GSI with' ${bk_core} 'for member ', ${ensmemid}
+
+ # case $ARCH in
+ # 'IBM_LSF')
+ # ${RUN_COMMAND} ./gsi.x <gsiparm.anl >stdout_mem${ensmemid} 2>&1
+ # ;;
+
+ # *)
+ # ${RUN_COMMAND} ./gsi.x >stdout_mem${ensmemid} 2>&1
+ # ;;
+ # esac
+
+ $RUN_COMMAND >stdout_mem${ensmemid} 2>&1
+
+ # run time error check and save run time file status
+ error=$?
+
+ if [ ${error} -ne 0 ]; then
+ echo "ERROR: ${GSI} crashed for member ${ensmemid} Exit status=${error}"
+ exit ${error}
+ fi
+
+ ls -l * >list_run_directory_mem${ensmemid}
+
+ # generate diag files
+
+ for type in $listall; do
+ count=$(ls pe*${type}_${loop}* | wc -l)
+ if [[ $count -gt 0 ]]; then
+ cat pe*${type}_${loop}* >diag_${type}_${string}.mem${ensmemid}
+ fi
+ done
+
+ # next member
+ ((ensmem += 1))
+
+ done
+
+fi
+
+exit 0
diff --git a/models/WRF/scripts/runscript b/models/WRF/scripts/runscript
index f02abe0c386132a59ab2f7b9f8a1a54197da5b8b..0f733b6de047601e74ee107dba0fffdc505ae832 100644
--- a/models/WRF/scripts/runscript
+++ b/models/WRF/scripts/runscript
@@ -14,16 +14,17 @@ if [ $# -gt 0 ]; then
echo "[WRF] no automatic copying"
fi
# run cmd line options
- echo "Executing"
+ echo "[WRF] Executing"
exec "$@"
- echo "Finished inside the container."
+ echo "[WRF] Finished inside the container."
else
# interactive
- echo "Welcome inside an IMGW container!"
- echo "Get help: singularity run-help $SINGULARITY_NAME"
- echo "WRF ($WRF_VERSION, $WRF_BUILD_TARGET)"
- echo "WRF $WRF_BUILD_TARGET -> /wrf/run"
- echo "WRF exe -> /wrf/bin"
- cd /wrf/run
+ echo "[WRF] Welcome inside an IMGW container!"
+ echo "[WRF] Get help: singularity run-help $SINGULARITY_NAME"
+ echo "[WRF] $WRF_VERSION, $WRF_BUILD_TARGET"
+ echo "[WRF] $WRF_BUILD_TARGET -> /wrf/run"
+ echo "[WRF] exe -> /wrf/bin"
+ echo "[WRF] Automatic copy of /wrf/run to current directory: ./$(basename $SINGULARITY_CONTAINER) init"
+ echo "[WRF] e.g. run real.exe from outside: ./$(basename $SINGULARITY_CONTAINER) real.exe"
exec /.singularity.d/actions/shell "$@"
fi
\ No newline at end of file