diff --git a/ICON.md b/ICON.md index 6f1edf21e03a4fc1576bcec29e81c687f0f72725..2e42b1686e873ee5ee022c2d1cf6d3f22d9e20da 100644 --- a/ICON.md +++ b/ICON.md @@ -1,133 +1,137 @@ -(mypy36) [serafin@jet01 icon-2.6.6]$ module list -Currently Loaded Modulefiles: - 1) intel-parallel-studio/composer.2020.2-intel-20.0.2-zuot22y - 2) zlib/1.2.11-intel-20.0.2-3h374ov - 3) openmpi/4.0.5-intel-20.0.2-4wfaaz4 - 4) hdf5/1.12.0-intel-20.0.2-ezeotzr - 5) parallel-netcdf/1.12.1-intel-20.0.2-sgz3yqs - 6) netcdf-c/4.7.4-intel-20.0.2-337uqtc - 7) netcdf-fortran/4.5.3-intel-20.0.2-irdm5gq - 8) netlib-lapack/3.9.1-gcc-8.5.0-ipqdnxj - 9) openblas/0.3.18-gcc-8.5.0-zv6qss4 - - - -ICON can be downloaded and used under a personal non-commercial research license (https://code.mpimet.mpg.de/projects/iconpublic/wiki/Instructions_to_obtain_the_ICON_model_code_with_a_personal_non-commercial_research_license). An account at <code.mpimet.mpg.de> is required. Instructions are available on the [public ICON page](https://code.mpimet.mpg.de/projects/iconpublic). - -ICON [Tutorial](https://www.dwd.de/EN/ourservices/nwv_icon_tutorial/pdf_volume/icon_tutorial2023_en.pdf?__blob=publicationFile&v=3) - +## What is ICON + +ICON can be downloaded and used under a [personal non-commercial research license](https://code.mpimet.mpg.de/projects/iconpublic/wiki/Instructions_to_obtain_the_ICON_model_code_with_a_personal_non-commercial_research_license). An account at <code.mpimet.mpg.de> is required. Instructions are available on the [public ICON page](https://code.mpimet.mpg.de/projects/iconpublic). Useful information can be found in the ICON [Tutorial](https://www.dwd.de/EN/ourservices/nwv_icon_tutorial/pdf_volume/icon_tutorial2023_en.pdf?__blob=publicationFile&v=3). + +## How to set up and run ICON on jet + +1. Load the necessary modules. + ``` + Currently Loaded Modulefiles: + 1) intel-parallel-studio/composer.2020.2-intel-20.0.2-zuot22y + 2) zlib/1.2.11-intel-20.0.2-3h374ov + 3) openmpi/4.0.5-intel-20.0.2-4wfaaz4 + 4) hdf5/1.12.0-intel-20.0.2-ezeotzr + 5) parallel-netcdf/1.12.1-intel-20.0.2-sgz3yqs + 6) netcdf-c/4.7.4-intel-20.0.2-337uqtc + 7) netcdf-fortran/4.5.3-intel-20.0.2-irdm5gq + 8) netlib-lapack/3.9.1-gcc-8.5.0-ipqdnxj + 9) openblas/0.3.18-gcc-8.5.0-zv6qss4 + 10) libxml2/2.9.12-intel-20.0.2-anavgpx + ``` 1. Unpack the tarball of the source code. -```sh -tar xvf icon-2.6.6.tar.xz -``` -The unpacked source code should look like this: -```sh -(mypy36) [serafin@jet01 test]$ ls icon-2.6.6 -total 684K -drwxr-xr-x 5 serafin users 4,0K 20 mar 17.05 adify -drwxr-xr-x 14 serafin users 4,0K 20 mar 17.05 config -drwxr-xr-x 5 serafin users 4,0K 20 mar 17.05 data -drwxr-xr-x 10 serafin users 4,0K 20 mar 17.05 doc -drwxr-xr-x 3 serafin users 4,0K 20 mar 17.05 etc -drwxr-xr-x 18 serafin users 4,0K 20 mar 17.05 externals -drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 grids -drwxr-xr-x 3 serafin users 4,0K 20 mar 17.05 license -drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 m4 -drwxr-xr-x 18 serafin users 16K 20 mar 17.05 run -drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 run_art -drwxr-xr-x 3 serafin users 4,0K 20 mar 17.05 schedulers -drwxr-xr-x 20 serafin users 4,0K 20 mar 17.05 scripts -drwxr-xr-x 38 serafin users 4,0K 20 mar 17.05 src -drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 support -drwxr-xr-x 8 serafin users 4,0K 20 mar 17.05 utils -drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 vertical_coord_tables --rw-r--r-- 1 serafin users 13K 9 feb 14.48 claw.mk.in --rw-r--r-- 1 serafin users 2,6K 7 feb 11.50 config.h.in --rwxr-xr-x 1 serafin users 371K 9 feb 14.48 configure --rw-r--r-- 1 serafin users 113K 9 feb 14.48 configure.ac --rw-r--r-- 1 serafin users 641 7 feb 11.50 COPYING --rw-r--r-- 1 serafin users 47 7 feb 11.50 depgen.c.config.in --rw-r--r-- 1 serafin users 1,7K 9 feb 14.48 depgen.f90.config.in --rw-r--r-- 1 serafin users 389 9 feb 14.48 deplist.config.in --rw-r--r-- 1 serafin users 27K 9 feb 14.48 icon.mk.in --rw-r--r-- 1 serafin users 2,9K 9 feb 14.48 inlib.mk.in --rw-r--r-- 1 serafin users 3,8K 9 feb 14.48 Makefile.in --rwxr-xr-x 1 serafin users 8,8K 9 feb 14.48 make_runscripts --rw-r--r-- 1 serafin users 2,4K 9 feb 14.48 README.art --rw-r--r-- 1 serafin users 73K 9 feb 14.48 README.md --rw-r--r-- 1 serafin users 11K 9 feb 14.48 RELEASE_NOTES.md -``` - -1. Generate a configuration script for the machine where the code has to run. In practice, go to `config` and create a directory `univie` in there. In the `univie` directory, place this file: -```jet.ifort.sh -#!/bin/bash -set -eu - -module purge -module load intel-parallel-studio/composer.2020.2-intel-20.0.2-zuot22y \ -zlib/1.2.11-intel-20.0.2-3h374ov \ -openmpi/4.0.5-intel-20.0.2-4wfaaz4 \ -hdf5/1.12.0-intel-20.0.2-ezeotzr \ -parallel-netcdf/1.12.1-intel-20.0.2-sgz3yqs \ -netcdf-c/4.7.4-intel-20.0.2-337uqtc \ -netcdf-fortran/4.5.3-intel-20.0.2-irdm5gq \ -netlib-lapack/3.9.1-gcc-8.5.0-ipqdnxj \ -openblas/0.3.18-gcc-8.5.0-zv6qss4 \ -libxml2/2.9.12-intel-20.0.2-anavgpx - -./configure \ -CC='/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/intel-parallel-studio-composer.2020.2-zuot22yfoe7jl67ttimvkzghwluvyaas/compilers_and_libraries_2020.2.254/linux/bin/intel64/icc' \ -FC='/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/openmpi-4.0.5-4wfaaz4grfqvlblibkdrpjlwd4xnq3ud/bin/mpif90' \ -FCFLAGS="-I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/openmpi-4.0.5-4wfaaz4grfqvlblibkdrpjlwd4xnq3ud/include \ - -I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/netcdf-fortran-4.5.3-irdm5gqccsig2om7jqu376h6tlebh4bc/include \ - -O3 -g" \ -CFLAGS="-I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/netcdf-c-4.7.4-337uqtcryw6nabweovh6kgsxozdwvylb/include" \ -CPPFLAGS="-I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-2021.4.0/eccodes-2.21.0-xnc5g2flgzmvm72lbnp4mrxior4txqlo/include \ - -I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/libxml2-2.9.12-anavgpxavrdktex4xecu6646yxhvqcul/include/libxml2" \ -LDFLAGS="-L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/openmpi-4.0.5-4wfaaz4grfqvlblibkdrpjlwd4xnq3ud/lib \ - -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/netcdf-c-4.7.4-337uqtcryw6nabweovh6kgsxozdwvylb/lib \ - -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/netcdf-fortran-4.5.3-irdm5gqccsig2om7jqu376h6tlebh4bc/lib \ - -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/openblas-0.3.18-zv6qss4vqhaqt4a3x4zeh7sbi6buuai3/lib \ - -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/netlib-lapack-3.9.1-ipqdnxjet2tlzossnlxlz33v7f5x6ckf/lib64 \ - -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/libxml2-2.9.12-anavgpxavrdktex4xecu6646yxhvqcul/lib \ - -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-2021.4.0/eccodes-2.21.0-xnc5g2flgzmvm72lbnp4mrxior4txqlo/lib64" \ -LIBS='-leccodes -lxml2 -llapack -lblas -lnetcdff -lnetcdf -lmpi' \ ---enable-mpi --enable-grib2 --without-external-cdi -``` -This configure script does two things: i) loads a selection of environment modules; ii) runs the ICON configure scripts passing all the required compiler flags. - -1. Prepare for an out-of-source build. Create a build directory wherever you want and move into it. From the build directory, execute `jet.ifort.sh` (the configuration script above): - + ```sh + tar xvf icon-2.6.6.tar.xz + ``` + The unpacked source code should look like this: + ```sh + (mypy36) [serafin@jet01 test]$ ls icon-2.6.6 + total 684K + drwxr-xr-x 5 serafin users 4,0K 20 mar 17.05 adify + drwxr-xr-x 14 serafin users 4,0K 20 mar 17.05 config + drwxr-xr-x 5 serafin users 4,0K 20 mar 17.05 data + drwxr-xr-x 10 serafin users 4,0K 20 mar 17.05 doc + drwxr-xr-x 3 serafin users 4,0K 20 mar 17.05 etc + drwxr-xr-x 18 serafin users 4,0K 20 mar 17.05 externals + drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 grids + drwxr-xr-x 3 serafin users 4,0K 20 mar 17.05 license + drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 m4 + drwxr-xr-x 18 serafin users 16K 20 mar 17.05 run + drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 run_art + drwxr-xr-x 3 serafin users 4,0K 20 mar 17.05 schedulers + drwxr-xr-x 20 serafin users 4,0K 20 mar 17.05 scripts + drwxr-xr-x 38 serafin users 4,0K 20 mar 17.05 src + drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 support + drwxr-xr-x 8 serafin users 4,0K 20 mar 17.05 utils + drwxr-xr-x 2 serafin users 4,0K 20 mar 17.05 vertical_coord_tables + -rw-r--r-- 1 serafin users 13K 9 feb 14.48 claw.mk.in + -rw-r--r-- 1 serafin users 2,6K 7 feb 11.50 config.h.in + -rwxr-xr-x 1 serafin users 371K 9 feb 14.48 configure + -rw-r--r-- 1 serafin users 113K 9 feb 14.48 configure.ac + -rw-r--r-- 1 serafin users 641 7 feb 11.50 COPYING + -rw-r--r-- 1 serafin users 47 7 feb 11.50 depgen.c.config.in + -rw-r--r-- 1 serafin users 1,7K 9 feb 14.48 depgen.f90.config.in + -rw-r--r-- 1 serafin users 389 9 feb 14.48 deplist.config.in + -rw-r--r-- 1 serafin users 27K 9 feb 14.48 icon.mk.in + -rw-r--r-- 1 serafin users 2,9K 9 feb 14.48 inlib.mk.in + -rw-r--r-- 1 serafin users 3,8K 9 feb 14.48 Makefile.in + -rwxr-xr-x 1 serafin users 8,8K 9 feb 14.48 make_runscripts + -rw-r--r-- 1 serafin users 2,4K 9 feb 14.48 README.art + -rw-r--r-- 1 serafin users 73K 9 feb 14.48 README.md + -rw-r--r-- 1 serafin users 11K 9 feb 14.48 RELEASE_NOTES.md + ``` + +1. Generate a configuration script for the machine where the code has to run. In practice, go to `config` and create a directory `univie` in there. In the `univie` directory, place the following contents in a file called `jet.ifort.sh`, then make the file executable (`chmod u+x jet.ifort.sh`): + ```jet.ifort.sh + #!/bin/bash + set -eu + + module purge + module load intel-parallel-studio/composer.2020.2-intel-20.0.2-zuot22y \ + zlib/1.2.11-intel-20.0.2-3h374ov \ + openmpi/4.0.5-intel-20.0.2-4wfaaz4 \ + hdf5/1.12.0-intel-20.0.2-ezeotzr \ + parallel-netcdf/1.12.1-intel-20.0.2-sgz3yqs \ + netcdf-c/4.7.4-intel-20.0.2-337uqtc \ + netcdf-fortran/4.5.3-intel-20.0.2-irdm5gq \ + netlib-lapack/3.9.1-gcc-8.5.0-ipqdnxj \ + openblas/0.3.18-gcc-8.5.0-zv6qss4 \ + libxml2/2.9.12-intel-20.0.2-anavgpx + + ./configure \ + CC='/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/intel-parallel-studio-composer.2020.2-zuot22yfoe7jl67ttimvkzghwluvyaas/compilers_and_libraries_2020.2.254/linux/bin/intel64/icc' \ + FC='/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/openmpi-4.0.5-4wfaaz4grfqvlblibkdrpjlwd4xnq3ud/bin/mpif90' \ + FCFLAGS="-I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/openmpi-4.0.5-4wfaaz4grfqvlblibkdrpjlwd4xnq3ud/include \ + -I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/netcdf-fortran-4.5.3-irdm5gqccsig2om7jqu376h6tlebh4bc/include \ + -O3 -g" \ + CFLAGS="-I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/netcdf-c-4.7.4-337uqtcryw6nabweovh6kgsxozdwvylb/include" \ + CPPFLAGS="-I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-2021.4.0/eccodes-2.21.0-xnc5g2flgzmvm72lbnp4mrxior4txqlo/include \ + -I/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/libxml2-2.9.12-anavgpxavrdktex4xecu6646yxhvqcul/include/libxml2" \ + LDFLAGS="-L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/openmpi-4.0.5-4wfaaz4grfqvlblibkdrpjlwd4xnq3ud/lib \ + -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/netcdf-c-4.7.4-337uqtcryw6nabweovh6kgsxozdwvylb/lib \ + -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/netcdf-fortran-4.5.3-irdm5gqccsig2om7jqu376h6tlebh4bc/lib \ + -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/openblas-0.3.18-zv6qss4vqhaqt4a3x4zeh7sbi6buuai3/lib \ + -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/gcc-8.5.0/netlib-lapack-3.9.1-ipqdnxjet2tlzossnlxlz33v7f5x6ckf/lib64 \ + -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/libxml2-2.9.12-anavgpxavrdktex4xecu6646yxhvqcul/lib \ + -L/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-2021.4.0/eccodes-2.21.0-xnc5g2flgzmvm72lbnp4mrxior4txqlo/lib64" \ + LIBS='-leccodes -lxml2 -llapack -lblas -lnetcdff -lnetcdf -lmpi' \ + --enable-mpi --enable-grib2 --without-external-cdi + ``` + This configure script does two things: i) loads a selection of environment modules; ii) runs the ICON configure scripts passing all the required compiler flags. + +1. Prepare for an out-of-source build. Create a build directory wherever you want and move into it. In the build directory, execute `jet.ifort.sh` (the configuration script above) + ```sh + /jetfs/home/serafin/ICON/icon-2.6.6/config/univie/jet.ifort.sh + ``` 1. Set the `LD_LIBRARY_PATH` environmental variable so that the Intel Math Library is accessible during compilation: -```sh -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/intel-parallel-studio-composer.2020.2-zuot22yfoe7jl67ttimvkzghwluvyaas/lib/intel64_lin -``` -The code will not build otherwise. Setting this variable may have to be necessary even at runtime (not tested yet!). + ```sh + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/jetfs/spack/opt/spack/linux-rhel8-skylake_avx512/intel-20.0.2/intel-parallel-studio-composer.2020.2-zuot22yfoe7jl67ttimvkzghwluvyaas/lib/intel64_lin + ``` + The code will not build otherwise. Setting this variable may have to be necessary even at runtime (not tested yet!). -1. Compile the code by running in the build directory: -```sh -make -j8 -``` +1. Compile the code by running the following in the build directory (`-j8` is optional, it's only about running multiple compile jobs in parallel): + ```sh + make -j8 + ``` 1. Copy in the build directory all the files needed to generate the ICON run scripts. The easiest way to do so is to use the following script (after editing the paths `ICON_SRC_DIR` and `ICON_BUILD_DIR`): -```sh -#!/bin/bash -set -eu - -ICON_SRC_DIR=/jetfs/home/serafin/ICON/icon-2.6.6 -ICON_BUILD_DIR=/jetfs/home/serafin/ICON/icon-2.6.6/jet - -cd $ICON_BUILD_DIR -rsync -uavz $ICON_SRC_DIR/run . --exclude='*in' --exclude='.*' -rsync -uavz $ICON_SRC_DIR/run . --exclude='*in' --exclude='.*' -rsync -uavz $ICON_SRC_DIR/externals . --exclude='.git' \ - --exclude='*.f90' --exclude='*.F90' --exclude='*.c' --exclude='*.h' \ - --exclude='*.Po' --exclude='tests' --exclude='rrtmgp*.nc' --exclude='*.mod' \ - --exclude='*.o' -rsync -uavz $ICON_SRC_DIR/make_runscripts . -ln -sf $ICON_SRC_DIR/data -ln -sf $ICON_SRC_DIR/vertical_coord_tables -``` + ```sh + #!/bin/bash + set -eu + + ICON_SRC_DIR=/jetfs/home/serafin/ICON/icon-2.6.6 + ICON_BUILD_DIR=/jetfs/home/serafin/ICON/icon-2.6.6/jet + + cd $ICON_BUILD_DIR + rsync -uavz $ICON_SRC_DIR/run . --exclude='*in' --exclude='.*' + rsync -uavz $ICON_SRC_DIR/run . --exclude='*in' --exclude='.*' + rsync -uavz $ICON_SRC_DIR/externals . --exclude='.git' \ + --exclude='*.f90' --exclude='*.F90' --exclude='*.c' --exclude='*.h' \ + --exclude='*.Po' --exclude='tests' --exclude='rrtmgp*.nc' --exclude='*.mod' \ + --exclude='*.o' + rsync -uavz $ICON_SRC_DIR/make_runscripts . + ln -sf $ICON_SRC_DIR/data + ln -sf $ICON_SRC_DIR/vertical_coord_tables + ```