Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
Singularity
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Deploy
Container registry
Monitor
Service Desk
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
IMGW
Singularity
Commits
f409b67d
Commit
f409b67d
authored
7 years ago
by
Marty Kandes
Browse files
Options
Downloads
Patches
Plain Diff
naked-singularity-v0.2.2 :: added ubuntu-cuda-openmpi.def for running
CUDA-aware OpenMPI-based applications on comet
parent
be489ab8
No related branches found
No related tags found
No related merge requests found
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
README
+2
-2
2 additions, 2 deletions
README
definition-files/us/ucsd/sdsc/comet/ubuntu/ubuntu-cuda-openmpi.def
+335
-0
335 additions, 0 deletions
...n-files/us/ucsd/sdsc/comet/ubuntu/ubuntu-cuda-openmpi.def
with
337 additions
and
2 deletions
README
+
2
−
2
View file @
f409b67d
...
@@ -29,8 +29,8 @@ AUTHOR
...
@@ -29,8 +29,8 @@ AUTHOR
VERSION
VERSION
0.2.
1
0.2.
2
LAST UPDATED
LAST UPDATED
Fri
day, March
16
th, 2018
Tues
day, March
20
th, 2018
This diff is collapsed.
Click to expand it.
definition-files/us/ucsd/sdsc/comet/ubuntu/ubuntu-cuda-openmpi.def
0 → 100644
+
335
−
0
View file @
f409b67d
Bootstrap: debootstrap
MirrorURL: http://us.archive.ubuntu.com/ubuntu
OSVersion: xenial
%labels
AUTHOR_NAME Marty Kandes
AUTHOR_EMAIL mkandes@sdsc.edu
APPLICATION_NAME cuda + openmpi
APPLICATION_VERSION 8.0.44 + 1.8.4
APPLICATION_URL https://www.open-mpi.org/faq/?category=runcuda
SYSTEM_NAME comet
SYSTEM_SINGULARITY_VERSION 2.3.2
SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html
LAST_UPDATED 20180316
%setup
%environment
%post -c /bin/bash
# Set system locale
export LC_ALL=C
# Install system metapackages
apt-get -y install ubuntu-standard
apt-get -y install ubuntu-server
# Add repositories
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} main"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} universe"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} multiverse"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} restricted"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates main"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates universe"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates multiverse"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates restricted"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports main"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports universe"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports multiverse"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports restricted"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security main"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security universe"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security multiverse"
add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security restricted"
# Upgrade all packages to their latest versions
apt-get -y update && apt-get -y upgrade
# Install common packages from 'main'
apt-get -y install autoconf
apt-get -y install automake
apt-get -y install build-essential
apt-get -y install libtool
apt-get -y install pkg-config
apt-get -y install gfortran
apt-get -y install zip
# Install expect to automate responses for interactive build questions
apt-get -y install expect
# Make filesystem mount points
mkdir /cvmfs /oasis /projects /scratch
# Use /tmp to store temporary files within the container during the
# bootstraping process
cd /tmp
# Install miniconda2
wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
chmod +x Miniconda2-latest-Linux-x86_64.sh
echo '#!/usr/bin/expect -f' > install-miniconda2.exp
echo 'spawn ./Miniconda2-latest-Linux-x86_64.sh' >> install-miniconda2.exp
echo 'expect "Please, press ENTER to continue\r' >> install-miniconda2.exp
echo '>>> "' >> install-miniconda2.exp
echo 'send "\r "' >> install-miniconda2.exp
echo 'expect "Do you accept the license terms? \[yes|no\]\r' >> install-miniconda2.exp
echo '\[no\] >>> "' >> install-miniconda2.exp
echo 'send "yes\r"' >> install-miniconda2.exp
echo 'expect "Miniconda2 will now be installed into this location:\r' >> install-miniconda2.exp
echo '/root/miniconda2\r' >> install-miniconda2.exp
echo '\r' >> install-miniconda2.exp
echo ' - Press ENTER to confirm the location\r' >> install-miniconda2.exp
echo ' - Press CTRL-C to abort the installation\r' >> install-miniconda2.exp
echo ' - Or specify a different location below\r' >> install-miniconda2.exp
echo '\r' >> install-miniconda2.exp
echo '\[/root/miniconda2\] >>> "' >> install-miniconda2.exp
echo 'send "/opt/miniconda2\r"' >> install-miniconda2.exp
echo 'expect "Do you wish the installer to prepend the Miniconda2 install location\r' >> install-miniconda2.exp
echo 'to PATH in your /root/.bashrc ? \[yes|no\]\r' >> install-miniconda2.exp
echo '\[no\] >>> "' >> install-miniconda2.exp
echo 'send "no\r"' >> install-miniconda2.exp
echo 'expect "Thank you for installing Miniconda2!\r"' >> install-miniconda2.exp
chmod +x install-miniconda2.exp
./install-miniconda2.exp
# Install miniconda3
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
chmod +x Miniconda3-latest-Linux-x86_64.sh
echo '#!/usr/bin/expect -f' > install-miniconda3.exp
echo 'spawn ./Miniconda3-latest-Linux-x86_64.sh' >> install-miniconda3.exp
echo 'expect "Please, press ENTER to continue\r' >> install-miniconda3.exp
echo '>>> "' >> install-miniconda3.exp
echo 'send "\r "' >> install-miniconda3.exp
echo 'expect "Do you accept the license terms? \[yes|no\]\r' >> install-miniconda3.exp
echo '\[no\] >>> "' >> install-miniconda3.exp
echo 'send "yes\r"' >> install-miniconda3.exp
echo 'expect "Miniconda3 will now be installed into this location:\r' >> install-miniconda3.exp
echo '/root/miniconda3\r' >> install-miniconda3.exp
echo '\r' >> install-miniconda3.exp
echo ' - Press ENTER to confirm the location\r' >> install-miniconda3.exp
echo ' - Press CTRL-C to abort the installation\r' >> install-miniconda3.exp
echo ' - Or specify a different location below\r' >> install-miniconda3.exp
echo '\r' >> install-miniconda3.exp
echo '\[/root/miniconda3\] >>> "' >> install-miniconda3.exp
echo 'send "/opt/miniconda3\r"' >> install-miniconda3.exp
echo 'expect "Do you wish the installer to prepend the Miniconda3 install location\r' >> install-miniconda3.exp
echo 'to PATH in your /root/.bashrc ? \[yes|no\]\r' >> install-miniconda3.exp
echo '\[no\] >>> "' >> install-miniconda3.exp
echo 'send "no\r"' >> install-miniconda3.exp
echo 'expect "Thank you for installing Miniconda2!\r"' >> install-miniconda3.exp
chmod +x install-miniconda3.exp
./install-miniconda3.exp
# Location of NVIDIA repositories
declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute'
declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/x86_64"
declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/x86_64/"
# Install NVIDIA package dependencies
apt-get -y install x11-common
apt-get -y install xserver-xorg-core
apt-get -y install cmake
apt-get -y install make
apt-get -y install dkms
apt-get -y install linux-libc-dev
apt-get -y install libc6-dev
apt-get -y install lib32gcc1
apt-get -y install libc6-i386
apt-get -y install libgl1
apt-get -y install libwayland-client0
apt-get -y install libwayland-server0
apt-get -y install pkg-config
apt-get -y install screen-resolution-extra
apt-get -y install libvdpau1
apt-get -y install libatk1.0-0
apt-get -y install libcairo-gobject2
apt-get -y install libcairo2
apt-get -y install libfontconfig1
apt-get -y install libgdk-pixbuf2.0-0
apt-get -y install libgtk-3-0
apt-get -y install libgtk2.0-0
apt-get -y install libjansson4
apt-get -y install freeglut3-dev
apt-get -y install libx11-dev
apt-get -y install libxmu-dev
apt-get -y install libxi-dev
apt-get -y install libglu1-mesa
apt-get -y install libglu1-mesa-dev
apt-get -y install default-jre
# Download NVIDIA drivers, libraries, and packages
wget --wait=10 "${CUDA_REPO_URL}/nvidia-367_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-367-dev_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/libcuda1-367_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_367.48-0ubuntu1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_367.48-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-license-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-core-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-8-0_8.0.44-1_amd64.deb"
wget --wait=10 "${ML_REPO_URL}/libcudnn6_6.0.21-1+cuda8.0_amd64.deb"
wget --wait=10 "${ML_REPO_URL}/libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb"
wget --wait=10 "${ML_REPO_URL}/libnccl2_2.1.2-1+cuda8.0_amd64.deb"
wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.1.2-1+cuda8.0_amd64.deb"
# Install NVIDIA drivers, libraries, and packages
dpkg -i nvidia-367_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-367-dev_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-modprobe_367.48-0ubuntu1_amd64.deb
dpkg -i libcuda1-367_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb
dpkg -i libxnvctrl0_367.48-0ubuntu1_amd64.deb
dpkg -i nvidia-settings_367.48-0ubuntu1_amd64.deb
dpkg -i cuda-drivers_367.48-1_amd64.deb
dpkg -i cuda-license-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-misc-headers-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-core-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cudart-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-driver-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cudart-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-command-line-tools-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvrtc-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvml-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvgraph-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cusolver-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cublas-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cublas-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cufft-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cufft-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-curand-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-curand-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cusparse-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-npp-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-npp-dev-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-samples-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-documentation-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-visual-tools-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-toolkit-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-runtime-8-0_8.0.44-1_amd64.deb
dpkg -i cuda-demo-suite-8-0_8.0.44-1_amd64.deb
dpkg -i libcudnn6_6.0.21-1+cuda8.0_amd64.deb
dpkg -i libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb
dpkg -i libnccl2_2.1.2-1+cuda8.0_amd64.deb
dpkg -i libnccl-dev_2.1.2-1+cuda8.0_amd64.deb
# Set paths to CUDA binaries and libraries
export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
# Install basic drivers for user space access to Ethernet, RDMA,
# and Infiniband. See https://community.mellanox.com/docs/DOC-2431
apt-get -y install dkms
apt-get -y install infiniband-diags
apt-get -y install libibverbs-dev
apt-get -y install ibacm
apt-get -y install librdmacm-dev
apt-get -y install libmlx4-dev
apt-get -y install libmlx5-dev
apt-get -y install mstflint
apt-get -y install libibcm-dev
apt-get -y install libibmad-dev
apt-get -y install libibumad-dev
apt-get -y install opensm
apt-get -y install srptools
# Install additional tools
apt-get -y install ibutils
apt-get -y install ibverbs-utils
apt-get -y install rdmacm-utils
apt-get -y install perftest
apt-get -y install numactl
apt-get -y install libnuma-dev
# Install libnl
apt-get -y install libnl-3-200
apt-get -y install libnl-route-3-200
apt-get -y install libnl-route-3-dev
apt-get -y install libnl-utils
# Download, build, and install CUDA-aware OpenMPI
wget https://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.4.tar.gz
tar -xzvf openmpi-1.8.4.tar.gz
cd openmpi-1.8.4
./configure --prefix=/opt/openmpi-1.8.4 --with-cuda
make all install
# Set paths to OpenMPI binaries and libraries
export PATH="/opt/openmpi-1.8.4/bin:${PATH}"
export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"
# Set container environment variables
cd /.singularity.d/env
echo 'export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"' >> 90-environment.sh
echo 'export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' >> 90-environment.sh
echo 'export PATH="/opt/openmpi-1.8.4/bin:${PATH}"' >> 90-environment.sh
echo 'export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"' >> 90-environment.sh
# Update database for mlocate
updatedb
%files
%runscript
%test
export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
export PATH="/opt/openmpi-1.8.4/bin:${PATH}"
export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"
nvcc --version
ompi_info --parsable --all | grep mpi_built_with_cuda_support:value
ompi_info --all | grep btl_openib_have_cuda_gdr
ompi_info --all | grep btl_openib_have_driver_gdr
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment