From 96265eb6aaa64f4ad1ad61a12c3e3c7fb05d566e Mon Sep 17 00:00:00 2001
From: MB <michael.blaschek@univie.ac.at>
Date: Tue, 14 Jun 2022 14:06:30 +0200
Subject: [PATCH] containerized development
---
development/README.md | 143 ++++++++++++++++++++++++++++
development/Singularity.hdf5.1.10.7 | 56 +++++++++++
development/spack.yaml | 22 +++++
3 files changed, 221 insertions(+)
create mode 100644 development/README.md
create mode 100644 development/Singularity.hdf5.1.10.7
create mode 100644 development/spack.yaml
diff --git a/development/README.md b/development/README.md
new file mode 100644
index 0000000..22166f1
--- /dev/null
+++ b/development/README.md
@@ -0,0 +1,143 @@
+# Development Containers
+
+We are going to use spack to build a container with different libraries and headers inside and will produce different stages.
+
+
+## Define a spack configuration
+We can choose some libraries and version that we need and build it into a concise image.
+
+This needs to be in a file called spack.yaml
+```yaml
+spack:
+ specs:
+ - hdf5@1.10.7+mpi target=skylake_avx512
+ - openmpi@3.1.6 target=skylake_avx512
+
+ container:
+ images:
+ os: "centos:8"
+ spack: 0.17.2
+
+ format: singularity
+
+ strip: true
+
+ os_packages:
+ final:
+ - gcc
+ - gcc-gfortran
+
+ labels:
+ app: "hdf5@1.10.7+mpi"
+ mpi: "openmpi@3.1.6"
+```
+
+micro architectures known to spack:
+
+```
+Generic architectures (families)
+ aarch64 arm ppc ppc64 ppc64le ppcle riscv64 sparc sparc64 x86 x86_64 x86_64_v2 x86_64_v3 x86_64_v4
+
+GenuineIntel - x86
+ i686 pentium2 pentium3 pentium4 prescott
+
+GenuineIntel - x86_64
+ nocona nehalem sandybridge haswell skylake cannonlake cascadelake
+ core2 westmere ivybridge broadwell mic_knl skylake_avx512 icelake
+
+AuthenticAMD - x86_64
+ k10 bulldozer piledriver zen steamroller zen2 zen3 excavator
+
+IBM - ppc64
+ power7 power8 power9
+
+IBM - ppc64le
+ power8le power9le
+
+Cavium - aarch64
+ thunderx2
+
+Fujitsu - aarch64
+ a64fx
+
+ARM - aarch64
+ graviton graviton2
+
+Apple - aarch64
+ m1
+
+SiFive - riscv64
+ u74mc
+```
+
+convert this configuration into a singularity (apptainer) recipes
+
+```bash
+spack containerize > Singularity.hdf5.1.10.7
+```
+
+which has the following content
+
+```singularity
+Bootstrap: docker
+From: spack/centos7:0.16.3
+Stage: build
+
+%post
+ # Create the manifest file for the installation in /opt/spack-environment
+ mkdir /opt/spack-environment && cd /opt/spack-environment
+ cat << EOF > spack.yaml
+spack:
+ specs:
+ - hdf5@1.10.7+mpi
+ - openmpi@3.6.1
+ concretization: together
+ config:
+ install_tree: /opt/software
+ view: /opt/view
+EOF
+
+ # Install all the required software
+ . /opt/spack/share/spack/setup-env.sh
+ spack env activate .
+ spack install --fail-fast
+ spack gc -y
+ spack env deactivate
+ spack env activate --sh -d . >> /opt/spack-environment/environment_modifications.sh
+
+ # Strip the binaries to reduce the size of the image
+ find -L /opt/view/* -type f -exec readlink -f '{}' \; | \
+ xargs file -i | \
+ grep 'charset=binary' | \
+ grep 'x-executable\|x-archive\|x-sharedlib' | \
+ awk -F: '{print $1}' | xargs strip -s
+
+
+Bootstrap: docker
+From: centos:7
+Stage: final
+
+%files from build
+ /opt/spack-environment /opt
+ /opt/software /opt
+ /opt/view /opt
+ /opt/spack-environment/environment_modifications.sh /opt/spack-environment/environment_modifications.sh
+
+%post
+ # Update, install and cleanup of system packages needed at run-time
+ yum update -y && yum install -y epel-release && yum update -y
+ yum install -y gcc gcc-gfortran
+ rm -rf /var/cache/yum && yum clean all
+ # Modify the environment without relying on sourcing shell specific files at startup
+ cat /opt/spack-environment/environment_modifications.sh >> $SINGULARITY_ENVIRONMENT
+
+%labels
+ app hdf5@1.10.7+mpi
+ mpi openmpi@3.6.1
+```
+
+Then we can use that recipes to build a singularity container:
+
+```bash
+singularity build hdf5.1.10.7.sif Singularity.hdf5.1.10.7
+```
\ No newline at end of file
diff --git a/development/Singularity.hdf5.1.10.7 b/development/Singularity.hdf5.1.10.7
new file mode 100644
index 0000000..e3993fe
--- /dev/null
+++ b/development/Singularity.hdf5.1.10.7
@@ -0,0 +1,56 @@
+Bootstrap: docker
+From: spack/ubuntu-bionic:0.16.3
+Stage: build
+
+%post
+ # Create the manifest file for the installation in /opt/spack-environment
+ mkdir /opt/spack-environment && cd /opt/spack-environment
+ cat << EOF > spack.yaml
+spack:
+ specs:
+ - hdf5@1.10.7+mpi
+ - openmpi@3.1.6
+ concretization: together
+ config:
+ install_tree: /opt/software
+ view: /opt/view
+EOF
+
+ # Install all the required software
+ . /opt/spack/share/spack/setup-env.sh
+ spack env activate .
+ spack install --fail-fast
+ spack gc -y
+ spack env deactivate
+ spack env activate --sh -d . >> /opt/spack-environment/environment_modifications.sh
+
+ # Strip the binaries to reduce the size of the image
+ find -L /opt/view/* -type f -exec readlink -f '{}' \; | \
+ xargs file -i | \
+ grep 'charset=binary' | \
+ grep 'x-executable\|x-archive\|x-sharedlib' | \
+ awk -F: '{print $1}' | xargs strip -s
+
+
+Bootstrap: docker
+From: ubuntu:18.04
+Stage: final
+
+%files from build
+ /opt/spack-environment /opt
+ /opt/software /opt
+ /opt/view /opt
+ /opt/spack-environment/environment_modifications.sh /opt/spack-environment/environment_modifications.sh
+
+%post
+ # Update, install and cleanup of system packages needed at run-time
+ apt-get -yqq update && apt-get -yqq upgrade
+ apt-get -yqq install gcc gfortran
+ rm -rf /var/lib/apt/lists/*
+ # Modify the environment without relying on sourcing shell specific files at startup
+ cat /opt/spack-environment/environment_modifications.sh >> $SINGULARITY_ENVIRONMENT
+
+%labels
+ app hdf5@1.10.7+mpi
+ mpi openmpi@3.1.6
+
diff --git a/development/spack.yaml b/development/spack.yaml
new file mode 100644
index 0000000..c952c54
--- /dev/null
+++ b/development/spack.yaml
@@ -0,0 +1,22 @@
+spack:
+ specs:
+ - hdf5@1.10.7+mpi target=skylake
+ - openmpi@3.1.6 target=skylake
+
+ container:
+ images:
+ os: "ubuntu:18.04"
+ spack: 0.16.3
+
+ format: singularity
+
+ strip: true
+
+ os_packages:
+ final:
+ - gcc
+ - gcc-gfortran
+
+ labels:
+ app: "hdf5@1.10.7+mpi"
+ mpi: "openmpi@3.1.6"
\ No newline at end of file
--
GitLab