Skip to content
Snippets Groups Projects
Commit f409b67d authored by Marty Kandes's avatar Marty Kandes
Browse files

naked-singularity-v0.2.2 :: added ubuntu-cuda-openmpi.def for running

CUDA-aware OpenMPI-based applications on comet
parent be489ab8
No related branches found
No related tags found
No related merge requests found
...@@ -29,8 +29,8 @@ AUTHOR ...@@ -29,8 +29,8 @@ AUTHOR
VERSION VERSION
0.2.1 0.2.2
LAST UPDATED LAST UPDATED
Friday, March 16th, 2018 Tuesday, March 20th, 2018
Bootstrap: debootstrap
MirrorURL: http://us.archive.ubuntu.com/ubuntu
OSVersion: xenial
%labels
AUTHOR_NAME Marty Kandes
AUTHOR_EMAIL mkandes@sdsc.edu
APPLICATION_NAME cuda + openmpi
APPLICATION_VERSION 8.0.44 + 1.8.4
APPLICATION_URL https://www.open-mpi.org/faq/?category=runcuda
SYSTEM_NAME comet
SYSTEM_SINGULARITY_VERSION 2.3.2
SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html
LAST_UPDATED 20180316
%setup
%environment
%post -c /bin/bash
# Set system locale
export LC_ALL=C
# Install system metapackages
apt-get -y install ubuntu-standard
apt-get -y install ubuntu-server
# Add repositories
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} main"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} universe"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} multiverse"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} restricted"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates main"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates universe"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates multiverse"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates restricted"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports main"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports universe"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports multiverse"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports restricted"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security main"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security universe"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security multiverse"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security restricted"
# Upgrade all packages to their latest versions
apt-get -y update && apt-get -y upgrade
# Install common packages from 'main'
apt-get -y install autoconf
apt-get -y install automake
apt-get -y install build-essential
apt-get -y install libtool
apt-get -y install pkg-config
apt-get -y install gfortran
apt-get -y install zip
# Install expect to automate responses for interactive build questions
apt-get -y install expect
# Make filesystem mount points
mkdir /cvmfs /oasis /projects /scratch
# Use /tmp to store temporary files within the container during the
# bootstraping process
cd /tmp
# Install miniconda2
wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
chmod +x Miniconda2-latest-Linux-x86_64.sh
echo '#!/usr/bin/expect -f' > install-miniconda2.exp
echo 'spawn ./Miniconda2-latest-Linux-x86_64.sh' >> install-miniconda2.exp
echo 'expect "Please, press ENTER to continue\r' >> install-miniconda2.exp
echo '>>> "' >> install-miniconda2.exp
echo 'send "\r "' >> install-miniconda2.exp
echo 'expect "Do you accept the license terms? \[yes|no\]\r' >> install-miniconda2.exp
echo '\[no\] >>> "' >> install-miniconda2.exp
echo 'send "yes\r"' >> install-miniconda2.exp
echo 'expect "Miniconda2 will now be installed into this location:\r' >> install-miniconda2.exp
echo '/root/miniconda2\r' >> install-miniconda2.exp
echo '\r' >> install-miniconda2.exp
echo ' - Press ENTER to confirm the location\r' >> install-miniconda2.exp
echo ' - Press CTRL-C to abort the installation\r' >> install-miniconda2.exp
echo ' - Or specify a different location below\r' >> install-miniconda2.exp
echo '\r' >> install-miniconda2.exp
echo '\[/root/miniconda2\] >>> "' >> install-miniconda2.exp
echo 'send "/opt/miniconda2\r"' >> install-miniconda2.exp
echo 'expect "Do you wish the installer to prepend the Miniconda2 install location\r' >> install-miniconda2.exp
echo 'to PATH in your /root/.bashrc ? \[yes|no\]\r' >> install-miniconda2.exp
echo '\[no\] >>> "' >> install-miniconda2.exp
echo 'send "no\r"' >> install-miniconda2.exp
echo 'expect "Thank you for installing Miniconda2!\r"' >> install-miniconda2.exp
chmod +x install-miniconda2.exp
./install-miniconda2.exp
# Install miniconda3
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
chmod +x Miniconda3-latest-Linux-x86_64.sh
echo '#!/usr/bin/expect -f' > install-miniconda3.exp
echo 'spawn ./Miniconda3-latest-Linux-x86_64.sh' >> install-miniconda3.exp
echo 'expect "Please, press ENTER to continue\r' >> install-miniconda3.exp
echo '>>> "' >> install-miniconda3.exp
echo 'send "\r "' >> install-miniconda3.exp
echo 'expect "Do you accept the license terms? \[yes|no\]\r' >> install-miniconda3.exp
echo '\[no\] >>> "' >> install-miniconda3.exp
echo 'send "yes\r"' >> install-miniconda3.exp
echo 'expect "Miniconda3 will now be installed into this location:\r' >> install-miniconda3.exp
echo '/root/miniconda3\r' >> install-miniconda3.exp
echo '\r' >> install-miniconda3.exp
echo ' - Press ENTER to confirm the location\r' >> install-miniconda3.exp
echo ' - Press CTRL-C to abort the installation\r' >> install-miniconda3.exp
echo ' - Or specify a different location below\r' >> install-miniconda3.exp
echo '\r' >> install-miniconda3.exp
echo '\[/root/miniconda3\] >>> "' >> install-miniconda3.exp
echo 'send "/opt/miniconda3\r"' >> install-miniconda3.exp
echo 'expect "Do you wish the installer to prepend the Miniconda3 install location\r' >> install-miniconda3.exp
echo 'to PATH in your /root/.bashrc ? \[yes|no\]\r' >> install-miniconda3.exp
echo '\[no\] >>> "' >> install-miniconda3.exp
echo 'send "no\r"' >> install-miniconda3.exp
echo 'expect "Thank you for installing Miniconda2!\r"' >> install-miniconda3.exp
chmod +x install-miniconda3.exp
./install-miniconda3.exp
# Location of NVIDIA repositories
declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute'
declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/x86_64"
declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/x86_64/"
# Install NVIDIA package dependencies
apt-get -y install x11-common
apt-get -y install xserver-xorg-core
apt-get -y install cmake
apt-get -y install make
apt-get -y install dkms
apt-get -y install linux-libc-dev
apt-get -y install libc6-dev
apt-get -y install lib32gcc1
apt-get -y install libc6-i386
apt-get -y install libgl1
apt-get -y install libwayland-client0
apt-get -y install libwayland-server0
apt-get -y install pkg-config
apt-get -y install screen-resolution-extra
apt-get -y install libvdpau1
apt-get -y install libatk1.0-0
apt-get -y install libcairo-gobject2
apt-get -y install libcairo2
apt-get -y install libfontconfig1
apt-get -y install libgdk-pixbuf2.0-0
apt-get -y install libgtk-3-0
apt-get -y install libgtk2.0-0
apt-get -y install libjansson4
apt-get -y install freeglut3-dev
apt-get -y install libx11-dev
apt-get -y install libxmu-dev
apt-get -y install libxi-dev
apt-get -y install libglu1-mesa
apt-get -y install libglu1-mesa-dev
apt-get -y install default-jre
# Download NVIDIA drivers, libraries, and packages
wget --wait=10 "${CUDA_REPO_URL}/nvidia-367_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-367-dev_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/libcuda1-367_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_367.48-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-license-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-core-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${ML_REPO_URL}/libcudnn6_6.0.21-1+cuda8.0_amd64.deb"
wget --wait=10 "${ML_REPO_URL}/libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb"
wget --wait=10 "${ML_REPO_URL}/libnccl2_2.1.2-1+cuda8.0_amd64.deb"
wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.1.2-1+cuda8.0_amd64.deb"
# Install NVIDIA drivers, libraries, and packages
dpkg -i nvidia-367_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-367-dev_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-modprobe_367.48-0ubuntu1_amd64.deb
dpkg -i libcuda1-367_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb
dpkg -i libxnvctrl0_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-settings_367.48-0ubuntu1_amd64.deb
dpkg -i cuda-drivers_367.48-1_amd64.deb
dpkg -i cuda-license-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-misc-headers-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-core-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cudart-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-driver-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cudart-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-command-line-tools-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvrtc-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvml-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvgraph-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cusolver-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cublas-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cublas-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cufft-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cufft-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-curand-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-curand-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cusparse-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-npp-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-npp-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-samples-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-documentation-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-visual-tools-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-toolkit-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-runtime-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-demo-suite-8-0_8.0.44-1_amd64.deb
dpkg -i libcudnn6_6.0.21-1+cuda8.0_amd64.deb
dpkg -i libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb
dpkg -i libnccl2_2.1.2-1+cuda8.0_amd64.deb
dpkg -i libnccl-dev_2.1.2-1+cuda8.0_amd64.deb
# Set paths to CUDA binaries and libraries
export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
# Install basic drivers for user space access to Ethernet, RDMA,
# and Infiniband. See https://community.mellanox.com/docs/DOC-2431
apt-get -y install dkms
apt-get -y install infiniband-diags
apt-get -y install libibverbs-dev
apt-get -y install ibacm
apt-get -y install librdmacm-dev
apt-get -y install libmlx4-dev
apt-get -y install libmlx5-dev
apt-get -y install mstflint
apt-get -y install libibcm-dev
apt-get -y install libibmad-dev
apt-get -y install libibumad-dev
apt-get -y install opensm
apt-get -y install srptools
# Install additional tools
apt-get -y install ibutils
apt-get -y install ibverbs-utils
apt-get -y install rdmacm-utils
apt-get -y install perftest
apt-get -y install numactl
apt-get -y install libnuma-dev
# Install libnl
apt-get -y install libnl-3-200
apt-get -y install libnl-route-3-200
apt-get -y install libnl-route-3-dev
apt-get -y install libnl-utils
# Download, build, and install CUDA-aware OpenMPI
wget https://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.4.tar.gz
tar -xzvf openmpi-1.8.4.tar.gz
cd openmpi-1.8.4
./configure --prefix=/opt/openmpi-1.8.4 --with-cuda
make all install
# Set paths to OpenMPI binaries and libraries
export PATH="/opt/openmpi-1.8.4/bin:${PATH}"
export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"
# Set container environment variables
cd /.singularity.d/env
echo 'export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"' >> 90-environment.sh
echo 'export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' >> 90-environment.sh
echo 'export PATH="/opt/openmpi-1.8.4/bin:${PATH}"' >> 90-environment.sh
echo 'export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"' >> 90-environment.sh
# Update database for mlocate
updatedb
%files
%runscript
%test
export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
export PATH="/opt/openmpi-1.8.4/bin:${PATH}"
export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"
nvcc --version
ompi_info --parsable --all | grep mpi_built_with_cuda_support:value
ompi_info --all | grep btl_openib_have_cuda_gdr
ompi_info --all | grep btl_openib_have_driver_gdr
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment