Skip to content
Snippets Groups Projects
Commit 5465d2a1 authored by Marty Kandes's avatar Marty Kandes
Browse files

Add OMB container layer dependency chain for multi-GPU node tests

These containers were built to assist with testing CUDA-enabled OpenMPI
on Comet to help investigate a problem with multi-GPU node support in
Expanse's software deployment. In addition, the base container layers
here have also allowed us to use Comet's GPU-accelerated compute nodes
to prepare acceptance tests for the upcoming deployment of Voyager.
parent 1d5b8fee
No related branches found
No related tags found
No related merge requests found
...@@ -60,8 +60,8 @@ University of California, San Diego ...@@ -60,8 +60,8 @@ University of California, San Diego
## Version ## Version
1.9.1 1.9.3
## Last Updated ## Last Updated
Friday, September 3rd, 2021 Saturday, September 4th, 2021
Bootstrap: oras
From: ghcr.io/mkandes/naked-singularity:ubuntu-18.04-cuda-10.2-mlnx-ofed-4.6-1.0.1.1-openmpi-3.1.4
%labels
APPLICATION_NAME OSU Micro-Benchmarks
APPLICATION_VERSION 5.8
APPLICATION_URL http://mvapich.cse.ohio-state.edu/benchmarks
AUTHOR_NAME Marty Kandes
AUTHOR_EMAIL mkandes@sdsc.edu
LAST_UPDATED 20210909
%setup
%environment
# Set OMB major and minor version numbers, root and
# installation directories
export OMB_MAJOR='5'
export OMB_MINOR='8'
export OMB_VERSION="${OMB_MAJOR}.${OMB_MINOR}"
export OMB_ROOT_DIR='/opt/omb'
export OMB_INSTALL_DIR="${OMB_ROOT_DIR}/${OMB_VERSION}"
# Set paths to OSU Microbenchmarks binaries
export PATH="${OMB_INSTALL_DIR}/collective:${PATH}"
export PATH="${OMB_INSTALL_DIR}/one-sided:${PATH}"
export PATH="${OMB_INSTALL_DIR}/pt2pt:${PATH}"
export PATH="${OMB_INSTALL_DIR}/startup:${PATH}"
%post -c /bin/bash
# Set operating system mirror URL
export MIRRORURL='http://us.archive.ubuntu.com/ubuntu'
# Set operating system version
export OSVERSION='bionic'
# Set system locale
export LC_ALL='C'
# Set debian frontend interface
export DEBIAN_FRONTEND='noninteractive'
# Set NVIDIA CUDA version
export CUDA_MAJOR='10'
export CUDA_MINOR='2'
export CUDA_VERSION="${CUDA_MAJOR}.${CUDA_MINOR}"
# Set paths to CUDA binaries and libraries
export CUDA_HOME="/usr/local/cuda-${CUDA_VERSION}"
export PATH="${CUDA_HOME}/bin${PATH:+:${PATH}}"
export LD_LIBRARY_PATH="${CUDA_HOME}/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
# Set OpenMPI major, minor, and revision numbers, root and
# installation directories
export OMPI_MAJOR='3'
export OMPI_MINOR='1'
export OMPI_REVISION='4'
export OMPI_VERSION="${OMPI_MAJOR}.${OMPI_MINOR}.${OMPI_REVISION}"
export OMPI_ROOT_DIR='/opt/openmpi'
export OMPI_INSTALL_DIR="${OMPI_ROOT_DIR}/${OMPI_VERSION}"
# Set paths to OpenMPI binaries and libraries
export PATH="${OMPI_INSTALL_DIR}/bin:${PATH}"
export LD_LIBRARY_PATH="${OMPI_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}"
# Upgrade all software packages to their latest versions
apt-get -y update && apt-get -y upgrade
cd /tmp
# Set OMB major and minor version numbers, root and
# installation directories
export OMB_ROOT_URL='http://mvapich.cse.ohio-state.edu/download/mvapich'
export OMB_MAJOR='5'
export OMB_MINOR='8'
export OMB_VERSION="${OMB_MAJOR}.${OMB_MINOR}"
export OMB_ROOT_DIR='/opt/omb'
export OMB_INSTALL_DIR="${OMB_ROOT_DIR}/${OMB_VERSION}"
# Download, build, and install OSU Microbenchmarks
wget "${OMB_ROOT_URL}/osu-micro-benchmarks-${OMB_VERSION}.tgz"
tar -xf "osu-micro-benchmarks-${OMB_VERSION}.tgz"
cd "osu-micro-benchmarks-${OMB_VERSION}"
./configure --prefix="${OMB_INSTALL_DIR}" CC='mpicc' CXX='mpicxx' --enable-cuda --with-cuda-include="${CUDA_HOME}/include" --with-cuda-libpath="${CUDA_HOME}/lib64"
make
make install
# Reorganize OSU Microbenchmarks directory structure
cd "${OMB_INSTALL_DIR}"
mv libexec/osu-micro-benchmarks/mpi/collective ./
mv libexec/osu-micro-benchmarks/mpi/one-sided ./
mv libexec/osu-micro-benchmarks/mpi/pt2pt ./
mv libexec/osu-micro-benchmarks/mpi/startup ./
rm -rf libexec
cd /tmp
# Remove OMB build directory and source tarball
rm -rf "osu-micro-benchmarks-${OMB_VERSION}"
rm "osu-micro-benchmarks-${OMB_VERSION}.tgz"
# Cleanup
apt-get -y autoremove --purge
apt-get -y clean
# Update database for mlocate
updatedb
%files
%runscript
%test
Bootstrap: oras
From: ghcr.io/mkandes/naked-singularity:ubuntu-18.04-mlnx-ofed-4.6-1.0.1.1-openmpi-3.1.4
%labels
APPLICATION_NAME OSU Micro-Benchmarks
APPLICATION_VERSION 5.8
APPLICATION_URL http://mvapich.cse.ohio-state.edu/benchmarks
AUTHOR_NAME Marty Kandes
AUTHOR_EMAIL mkandes@sdsc.edu
LAST_UPDATED 20210908
%setup
%environment
# Set OMB major and minor version numbers, root and
# installation directories
export OMB_MAJOR='5'
export OMB_MINOR='8'
export OMB_VERSION="${OMB_MAJOR}.${OMB_MINOR}"
export OMB_ROOT_DIR='/opt/omb'
export OMB_INSTALL_DIR="${OMB_ROOT_DIR}/${OMB_VERSION}"
# Set paths to OSU Microbenchmarks binaries
export PATH="${OMB_INSTALL_DIR}/collective:${PATH}"
export PATH="${OMB_INSTALL_DIR}/one-sided:${PATH}"
export PATH="${OMB_INSTALL_DIR}/pt2pt:${PATH}"
export PATH="${OMB_INSTALL_DIR}/startup:${PATH}"
%post -c /bin/bash
# Set operating system mirror URL
export MIRRORURL='http://us.archive.ubuntu.com/ubuntu'
# Set operating system version
export OSVERSION='bionic'
# Set system locale
export LC_ALL='C'
# Set debian frontend interface
export DEBIAN_FRONTEND='noninteractive'
# Set OpenMPI major, minor, and revision numbers, root and
# installation directories
export OMPI_MAJOR='3'
export OMPI_MINOR='1'
export OMPI_REVISION='4'
export OMPI_VERSION="${OMPI_MAJOR}.${OMPI_MINOR}.${OMPI_REVISION}"
export OMPI_ROOT_DIR='/opt/openmpi'
export OMPI_INSTALL_DIR="${OMPI_ROOT_DIR}/${OMPI_VERSION}"
# Set paths to OpenMPI binaries and libraries
export PATH="${OMPI_INSTALL_DIR}/bin:${PATH}"
export LD_LIBRARY_PATH="${OMPI_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}"
# Upgrade all software packages to their latest versions
apt-get -y update && apt-get -y upgrade
cd /tmp
# Set OMB major and minor version numbers, root and
# installation directories
export OMB_ROOT_URL='http://mvapich.cse.ohio-state.edu/download/mvapich'
export OMB_MAJOR='5'
export OMB_MINOR='8'
export OMB_VERSION="${OMB_MAJOR}.${OMB_MINOR}"
export OMB_ROOT_DIR='/opt/omb'
export OMB_INSTALL_DIR="${OMB_ROOT_DIR}/${OMB_VERSION}"
# Download, build, and install OSU Microbenchmarks
wget "${OMB_ROOT_URL}/osu-micro-benchmarks-${OMB_VERSION}.tgz"
tar -xf "osu-micro-benchmarks-${OMB_VERSION}.tgz"
cd "osu-micro-benchmarks-${OMB_VERSION}"
./configure --prefix="${OMB_INSTALL_DIR}" CC='mpicc' CXX='mpicxx'
make
make install
# Reorganize OSU Microbenchmarks directory structure
cd "${OMB_INSTALL_DIR}"
mv libexec/osu-micro-benchmarks/mpi/collective ./
mv libexec/osu-micro-benchmarks/mpi/one-sided ./
mv libexec/osu-micro-benchmarks/mpi/pt2pt ./
mv libexec/osu-micro-benchmarks/mpi/startup ./
rm -rf libexec
cd /tmp
# Remove OMB build directory and source tarball
rm -rf "osu-micro-benchmarks-${OMB_VERSION}"
rm "osu-micro-benchmarks-${OMB_VERSION}.tgz"
# Cleanup
apt-get -y autoremove --purge
apt-get -y clean
# Update database for mlocate
updatedb
%files
%runscript
%test
Bootstrap: oras
From: ghcr.io/mkandes/naked-singularity:ubuntu-18.04-cuda-10.2
%labels
APPLICATION_NAME ubuntu + cuda mlnx-ofed
APPLICATION_VERSION 18.04 + 10.2. + 4.6-1.0.1.1
APPLICATION_URL https://www.mellanox.com/products/infiniband-drivers/linux/mlnx_ofed
AUTHOR_NAME Marty Kandes
AUTHOR_EMAIL mkandes@sdsc.edu
LAST_UPDATED 20210908
%setup
%environment
# Set Mellanox OFED version, operating system, and hardware platform
export MLNX_OFED_VERSION='4.6-1.0.1.1'
export MLNX_OS_VERSION='ubuntu18.04'
export MLNX_PLATFORM='x86_64'
%post -c /bin/bash
# Set operating system mirror URL
export MIRRORURL='http://us.archive.ubuntu.com/ubuntu'
# Set operating system version
export OSVERSION='bionic'
# Set system locale
export LC_ALL='C'
# Set debian frontend interface
export DEBIAN_FRONTEND='noninteractive'
# Upgrade all software packages to their latest versions
apt-get -y update && apt-get -y upgrade
# Install all dependencies and/or prerequisites for Mellanox OFED
apt-get -y install bison
apt-get -y install chrpath
apt-get -y install debhelper
apt-get -y install dpatch
apt-get -y install flex
apt-get -y install graphviz
apt-get -y install libnl-3-dev
apt-get -y install libnl-route-3-200
apt-get -y install tcl-dev
apt-get -y install tk-dev
apt-get -y install swig
cd /tmp
# Set Mellanox OFED version, operating system, and hardware platform
export MLNX_ROOT_URL='http://www.mellanox.com/downloads/ofed'
export MLNX_OFED_VERSION='4.6-1.0.1.1'
export MLNX_OS_VERSION='ubuntu18.04'
export MLNX_PLATFORM='x86_64'
# Download and install Mellanox OFED drivers and supporting
# libraries for user space access to Ethernet, RDMA, and Infiniband.
# https://docs.mellanox.com/pages/releaseview.action?pageId=15049785
wget "${MLNX_ROOT_URL}/MLNX_OFED-${MLNX_OFED_VERSION}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}.tgz"
tar -xf "MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}.tgz"
cd "MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}"
./mlnxofedinstall --user-space-only --without-fw-update --force
cd /tmp
# Remove Mellanox OFED archive directory and tarball
rm -rf "MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}"
rm "MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}.tgz"
# Install additional tools
apt-get -y install numactl
apt-get -y install libnuma-dev
# Cleanup
apt-get -y autoremove --purge
apt-get -y clean
# Update database for mlocate
updatedb
%files
%runscript
%test
Bootstrap: oras
From: ghcr.io/mkandes/naked-singularity:ubuntu-18.04-cuda-10.2-mlnx-ofed-4.6-1.0.1.1
%labels
APPLICATION_NAME ubuntu + cuda + mlnx-ofed + openmpi
APPLICATION_VERSION 18.04 + 10.2. + 4.6-1.0.1.1 + 3.1.4
APPLICATION_URL https://www.open-mpi.org
AUTHOR_NAME Marty Kandes
AUTHOR_EMAIL mkandes@sdsc.edu
LAST_UPDATED 20210909
%setup
%environment
# Set OpenMPI major, minor, and revision numbers, root and
# installation directories
export OMPI_MAJOR='3'
export OMPI_MINOR='1'
export OMPI_REVISION='4'
export OMPI_VERSION="${OMPI_MAJOR}.${OMPI_MINOR}.${OMPI_REVISION}"
export OMPI_ROOT_DIR='/opt/openmpi'
export OMPI_INSTALL_DIR="${OMPI_ROOT_DIR}/${OMPI_VERSION}"
# Set paths to OpenMPI binaries and libraries
export PATH="${OMPI_INSTALL_DIR}/bin:${PATH}"
export LD_LIBRARY_PATH="${OMPI_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}"
%post -c /bin/bash
# Set operating system mirror URL
export MIRRORURL='http://us.archive.ubuntu.com/ubuntu'
# Set operating system version
export OSVERSION='bionic'
# Set system locale
export LC_ALL='C'
# Set debian frontend interface
export DEBIAN_FRONTEND='noninteractive'
# Set NVIDIA CUDA version
export CUDA_MAJOR='10'
export CUDA_MINOR='2'
export CUDA_VERSION="${CUDA_MAJOR}.${CUDA_MINOR}"
# Set paths to CUDA binaries and libraries
export CUDA_HOME="/usr/local/cuda-${CUDA_VERSION}"
export PATH="${CUDA_HOME}/bin${PATH:+:${PATH}}"
export LD_LIBRARY_PATH="${CUDA_HOME}/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
# Upgrade all software packages to their latest versions
apt-get -y update && apt-get -y upgrade
# Install OpenMPI dependencies
apt-get -y install zlib1g-dev
# Set OpenMPI major, minor, and revision numbers, root and
# installation directories
export OMPI_ROOT_URL='https://download.open-mpi.org/release/open-mpi'
export OMPI_MAJOR='3'
export OMPI_MINOR='1'
export OMPI_REVISION='4'
export OMPI_VERSION="${OMPI_MAJOR}.${OMPI_MINOR}.${OMPI_REVISION}"
export OMPI_ROOT_DIR='/opt/openmpi'
export OMPI_INSTALL_DIR="${OMPI_ROOT_DIR}/${OMPI_VERSION}"
cd /tmp
# Download, build, and install OpenMPI
wget "${OMPI_ROOT_URL}/v${OMPI_MAJOR}.${OMPI_MINOR}/openmpi-${OMPI_VERSION}.tar.gz"
tar -xf "openmpi-${OMPI_VERSION}.tar.gz"
cd "openmpi-${OMPI_VERSION}"
./configure --prefix="${OMPI_INSTALL_DIR}" --enable-shared \
--disable-silent-rules --disable-builtin-atomics --enable-static \
--without-hcoll --without-psm --without-knem --without-psm2 \
--with-verbs=/usr --without-ofi --without-mxm --without-fca \
--without-xpmem --without-ucx --without-cma --without-slurm \
--without-tm --without-loadleveler --without-alps --without-lsf \
--without-sge --disable-memchecker --disable-java \
--disable-mpi-java --with-cuda --enable-mpi-cxx \
--enable-cxx-exceptions
make all install
cd /tmp
# Remove OpenMPI build directory and source tarball
rm -rf "openmpi-${OMPI_VERSION}"
rm "openmpi-${OMPI_VERSION}.tar.gz"
# Cleanup
apt-get -y autoremove --purge
apt-get -y clean
# Update database for mlocate
updatedb
%files
%runscript
%test
Bootstrap: oras
From: ghcr.io/mkandes/naked-singularity:ubuntu-18.04
%labels
APPLICATION_NAME ubuntu + mlnx-ofed
APPLICATION_VERSION 18.04 + 4.6-1.0.1.1
APPLICATION_URL https://www.mellanox.com/products/infiniband-drivers/linux/mlnx_ofed
AUTHOR_NAME Marty Kandes
AUTHOR_EMAIL mkandes@sdsc.edu
LAST_UPDATED 20210908
%setup
%environment
# Set Mellanox OFED version, operating system, and hardware platform
export MLNX_OFED_VERSION='4.6-1.0.1.1'
export MLNX_OS_VERSION='ubuntu18.04'
export MLNX_PLATFORM='x86_64'
%post -c /bin/bash
# Set operating system mirror URL
export MIRRORURL='http://us.archive.ubuntu.com/ubuntu'
# Set operating system version
export OSVERSION='bionic'
# Set system locale
export LC_ALL='C'
# Set debian frontend interface
export DEBIAN_FRONTEND='noninteractive'
# Upgrade all software packages to their latest versions
apt-get -y update && apt-get -y upgrade
# Install all dependencies and/or prerequisites for Mellanox OFED
apt-get -y install bison
apt-get -y install chrpath
apt-get -y install debhelper
apt-get -y install dpatch
apt-get -y install flex
apt-get -y install graphviz
apt-get -y install libnl-3-dev
apt-get -y install libnl-route-3-200
apt-get -y install tcl-dev
apt-get -y install tk-dev
apt-get -y install swig
cd /tmp
# Set Mellanox OFED version, operating system, and hardware platform
export MLNX_ROOT_URL='http://www.mellanox.com/downloads/ofed'
export MLNX_OFED_VERSION='4.6-1.0.1.1'
export MLNX_OS_VERSION='ubuntu18.04'
export MLNX_PLATFORM='x86_64'
# Download and install Mellanox OFED drivers and supporting
# libraries for user space access to Ethernet, RDMA, and Infiniband.
# https://docs.mellanox.com/pages/releaseview.action?pageId=15049785
wget "${MLNX_ROOT_URL}/MLNX_OFED-${MLNX_OFED_VERSION}/MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}.tgz"
tar -xf "MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}.tgz"
cd "MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}"
./mlnxofedinstall --user-space-only --without-fw-update --force
cd /tmp
# Remove Mellanox OFED archive directory and tarball
rm -rf "MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}"
rm "MLNX_OFED_LINUX-${MLNX_OFED_VERSION}-${MLNX_OS_VERSION}-${MLNX_PLATFORM}.tgz"
# Install additional tools
apt-get -y install numactl
apt-get -y install libnuma-dev
# Cleanup
apt-get -y autoremove --purge
apt-get -y clean
# Update database for mlocate
updatedb
%files
%runscript
%test
Bootstrap: oras
From: ghcr.io/mkandes/naked-singularity:ubuntu-18.04-mlnx-ofed-4.6-1.0.1.1
%labels
APPLICATION_NAME ubuntu + mlnx-ofed + openmpi
APPLICATION_VERSION 18.04 + 4.6-1.0.1.1 + 3.1.4
APPLICATION_URL https://www.open-mpi.org
AUTHOR_NAME Marty Kandes
AUTHOR_EMAIL mkandes@sdsc.edu
LAST_UPDATED 20210909
%setup
%environment
# Set OpenMPI major, minor, and revision numbers, root and
# installation directories
export OMPI_MAJOR='3'
export OMPI_MINOR='1'
export OMPI_REVISION='4'
export OMPI_VERSION="${OMPI_MAJOR}.${OMPI_MINOR}.${OMPI_REVISION}"
export OMPI_ROOT_DIR='/opt/openmpi'
export OMPI_INSTALL_DIR="${OMPI_ROOT_DIR}/${OMPI_VERSION}"
# Set paths to OpenMPI binaries and libraries
export PATH="${OMPI_INSTALL_DIR}/bin:${PATH}"
export LD_LIBRARY_PATH="${OMPI_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}"
%post -c /bin/bash
# Set operating system mirror URL
export MIRRORURL='http://us.archive.ubuntu.com/ubuntu'
# Set operating system version
export OSVERSION='bionic'
# Set system locale
export LC_ALL='C'
# Set debian frontend interface
export DEBIAN_FRONTEND='noninteractive'
# Upgrade all software packages to their latest versions
apt-get -y update && apt-get -y upgrade
# Install OpenMPI dependencies
apt-get -y install zlib1g-dev
# Set OpenMPI major, minor, and revision numbers, root and
# installation directories
export OMPI_ROOT_URL='https://download.open-mpi.org/release/open-mpi'
export OMPI_MAJOR='3'
export OMPI_MINOR='1'
export OMPI_REVISION='4'
export OMPI_VERSION="${OMPI_MAJOR}.${OMPI_MINOR}.${OMPI_REVISION}"
export OMPI_ROOT_DIR='/opt/openmpi'
export OMPI_INSTALL_DIR="${OMPI_ROOT_DIR}/${OMPI_VERSION}"
cd /tmp
# Download, build, and install OpenMPI
wget "${OMPI_ROOT_URL}/v${OMPI_MAJOR}.${OMPI_MINOR}/openmpi-${OMPI_VERSION}.tar.gz"
tar -xf "openmpi-${OMPI_VERSION}.tar.gz"
cd "openmpi-${OMPI_VERSION}"
./configure --prefix="${OMPI_INSTALL_DIR}" --enable-shared \
--disable-silent-rules --disable-builtin-atomics --enable-static \
--without-hcoll --without-psm --without-knem --without-psm2 \
--with-verbs=/usr --without-ofi --without-mxm --without-fca \
--without-xpmem --without-ucx --without-cma --without-slurm \
--without-tm --without-loadleveler --without-alps --without-lsf \
--without-sge --disable-memchecker --disable-java \
--disable-mpi-java --without-cuda --enable-mpi-cxx \
--enable-cxx-exceptions
make all install
cd /tmp
# Remove OpenMPI build directory and source tarball
rm -rf "openmpi-${OMPI_VERSION}"
rm "openmpi-${OMPI_VERSION}.tar.gz"
# Cleanup
apt-get -y autoremove --purge
apt-get -y clean
# Update database for mlocate
updatedb
%files
%runscript
%test
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment