diff --git a/README b/README index d176f60ac8fd61808593a466102f0d3372bdb499..fa461341c6660c7a1b434007bbed053e8f16ebee 100644 --- a/README +++ b/README @@ -29,8 +29,8 @@ AUTHOR VERSION - 0.2.1 + 0.2.2 LAST UPDATED - Friday, March 16th, 2018 + Tuesday, March 20th, 2018 diff --git a/definition-files/us/ucsd/sdsc/comet/ubuntu/ubuntu-cuda-openmpi.def b/definition-files/us/ucsd/sdsc/comet/ubuntu/ubuntu-cuda-openmpi.def new file mode 100644 index 0000000000000000000000000000000000000000..02e967ec983ae244c389e2372d6ffc0a3f6cd07f --- /dev/null +++ b/definition-files/us/ucsd/sdsc/comet/ubuntu/ubuntu-cuda-openmpi.def @@ -0,0 +1,335 @@ +Bootstrap: debootstrap +MirrorURL: http://us.archive.ubuntu.com/ubuntu +OSVersion: xenial + +%labels + + AUTHOR_NAME Marty Kandes + AUTHOR_EMAIL mkandes@sdsc.edu + APPLICATION_NAME cuda + openmpi + APPLICATION_VERSION 8.0.44 + 1.8.4 + APPLICATION_URL https://www.open-mpi.org/faq/?category=runcuda + SYSTEM_NAME comet + SYSTEM_SINGULARITY_VERSION 2.3.2 + SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html + LAST_UPDATED 20180316 + +%setup + +%environment + +%post -c /bin/bash + + # Set system locale + export LC_ALL=C + + # Install system metapackages + apt-get -y install ubuntu-standard + apt-get -y install ubuntu-server + + # Add repositories + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} main" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} universe" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} multiverse" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} restricted" + + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates main" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates universe" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates multiverse" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates restricted" + + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports main" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports universe" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports multiverse" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports restricted" + + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security main" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security universe" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security multiverse" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security restricted" + + # Upgrade all packages to their latest versions + apt-get -y update && apt-get -y upgrade + + # Install common packages from 'main' + apt-get -y install autoconf + apt-get -y install automake + apt-get -y install build-essential + apt-get -y install libtool + apt-get -y install pkg-config + apt-get -y install gfortran + apt-get -y install zip + + # Install expect to automate responses for interactive build questions + apt-get -y install expect + + # Make filesystem mount points + mkdir /cvmfs /oasis /projects /scratch + + # Use /tmp to store temporary files within the container during the + # bootstraping process + cd /tmp + + # Install miniconda2 + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh + chmod +x Miniconda2-latest-Linux-x86_64.sh + + echo '#!/usr/bin/expect -f' > install-miniconda2.exp + echo 'spawn ./Miniconda2-latest-Linux-x86_64.sh' >> install-miniconda2.exp + echo 'expect "Please, press ENTER to continue\r' >> install-miniconda2.exp + echo '>>> "' >> install-miniconda2.exp + echo 'send "\r "' >> install-miniconda2.exp + echo 'expect "Do you accept the license terms? \[yes|no\]\r' >> install-miniconda2.exp + echo '\[no\] >>> "' >> install-miniconda2.exp + echo 'send "yes\r"' >> install-miniconda2.exp + echo 'expect "Miniconda2 will now be installed into this location:\r' >> install-miniconda2.exp + echo '/root/miniconda2\r' >> install-miniconda2.exp + echo '\r' >> install-miniconda2.exp + echo ' - Press ENTER to confirm the location\r' >> install-miniconda2.exp + echo ' - Press CTRL-C to abort the installation\r' >> install-miniconda2.exp + echo ' - Or specify a different location below\r' >> install-miniconda2.exp + echo '\r' >> install-miniconda2.exp + echo '\[/root/miniconda2\] >>> "' >> install-miniconda2.exp + echo 'send "/opt/miniconda2\r"' >> install-miniconda2.exp + echo 'expect "Do you wish the installer to prepend the Miniconda2 install location\r' >> install-miniconda2.exp + echo 'to PATH in your /root/.bashrc ? \[yes|no\]\r' >> install-miniconda2.exp + echo '\[no\] >>> "' >> install-miniconda2.exp + echo 'send "no\r"' >> install-miniconda2.exp + echo 'expect "Thank you for installing Miniconda2!\r"' >> install-miniconda2.exp + + chmod +x install-miniconda2.exp + ./install-miniconda2.exp + + # Install miniconda3 + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + chmod +x Miniconda3-latest-Linux-x86_64.sh + + echo '#!/usr/bin/expect -f' > install-miniconda3.exp + echo 'spawn ./Miniconda3-latest-Linux-x86_64.sh' >> install-miniconda3.exp + echo 'expect "Please, press ENTER to continue\r' >> install-miniconda3.exp + echo '>>> "' >> install-miniconda3.exp + echo 'send "\r "' >> install-miniconda3.exp + echo 'expect "Do you accept the license terms? \[yes|no\]\r' >> install-miniconda3.exp + echo '\[no\] >>> "' >> install-miniconda3.exp + echo 'send "yes\r"' >> install-miniconda3.exp + echo 'expect "Miniconda3 will now be installed into this location:\r' >> install-miniconda3.exp + echo '/root/miniconda3\r' >> install-miniconda3.exp + echo '\r' >> install-miniconda3.exp + echo ' - Press ENTER to confirm the location\r' >> install-miniconda3.exp + echo ' - Press CTRL-C to abort the installation\r' >> install-miniconda3.exp + echo ' - Or specify a different location below\r' >> install-miniconda3.exp + echo '\r' >> install-miniconda3.exp + echo '\[/root/miniconda3\] >>> "' >> install-miniconda3.exp + echo 'send "/opt/miniconda3\r"' >> install-miniconda3.exp + echo 'expect "Do you wish the installer to prepend the Miniconda3 install location\r' >> install-miniconda3.exp + echo 'to PATH in your /root/.bashrc ? \[yes|no\]\r' >> install-miniconda3.exp + echo '\[no\] >>> "' >> install-miniconda3.exp + echo 'send "no\r"' >> install-miniconda3.exp + echo 'expect "Thank you for installing Miniconda2!\r"' >> install-miniconda3.exp + + chmod +x install-miniconda3.exp + ./install-miniconda3.exp + + # Location of NVIDIA repositories + declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute' + declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/x86_64" + declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/x86_64/" + + # Install NVIDIA package dependencies + apt-get -y install x11-common + apt-get -y install xserver-xorg-core + apt-get -y install cmake + apt-get -y install make + apt-get -y install dkms + apt-get -y install linux-libc-dev + apt-get -y install libc6-dev + apt-get -y install lib32gcc1 + apt-get -y install libc6-i386 + apt-get -y install libgl1 + apt-get -y install libwayland-client0 + apt-get -y install libwayland-server0 + apt-get -y install pkg-config + apt-get -y install screen-resolution-extra + apt-get -y install libvdpau1 + apt-get -y install libatk1.0-0 + apt-get -y install libcairo-gobject2 + apt-get -y install libcairo2 + apt-get -y install libfontconfig1 + apt-get -y install libgdk-pixbuf2.0-0 + apt-get -y install libgtk-3-0 + apt-get -y install libgtk2.0-0 + apt-get -y install libjansson4 + apt-get -y install freeglut3-dev + apt-get -y install libx11-dev + apt-get -y install libxmu-dev + apt-get -y install libxi-dev + apt-get -y install libglu1-mesa + apt-get -y install libglu1-mesa-dev + apt-get -y install default-jre + + # Download NVIDIA drivers, libraries, and packages + wget --wait=10 "${CUDA_REPO_URL}/nvidia-367_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-367-dev_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libcuda1-367_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_367.48-1_amd64.deb" + + wget --wait=10 "${CUDA_REPO_URL}/cuda-license-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-core-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-8-0_8.0.44-1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libcudnn6_6.0.21-1+cuda8.0_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libnccl2_2.1.2-1+cuda8.0_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.1.2-1+cuda8.0_amd64.deb" + + # Install NVIDIA drivers, libraries, and packages + dpkg -i nvidia-367_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-367-dev_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-modprobe_367.48-0ubuntu1_amd64.deb + dpkg -i libcuda1-367_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb + dpkg -i libxnvctrl0_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-settings_367.48-0ubuntu1_amd64.deb + dpkg -i cuda-drivers_367.48-1_amd64.deb + + dpkg -i cuda-license-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-misc-headers-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-core-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cudart-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-driver-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cudart-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-command-line-tools-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvrtc-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvml-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvgraph-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cusolver-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cublas-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cublas-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cufft-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cufft-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-curand-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-curand-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cusparse-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-npp-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-npp-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-samples-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-documentation-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-visual-tools-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-toolkit-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-runtime-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-demo-suite-8-0_8.0.44-1_amd64.deb + + dpkg -i libcudnn6_6.0.21-1+cuda8.0_amd64.deb + dpkg -i libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb + + dpkg -i libnccl2_2.1.2-1+cuda8.0_amd64.deb + dpkg -i libnccl-dev_2.1.2-1+cuda8.0_amd64.deb + + # Set paths to CUDA binaries and libraries + export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}" + export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + + # Install basic drivers for user space access to Ethernet, RDMA, + # and Infiniband. See https://community.mellanox.com/docs/DOC-2431 + apt-get -y install dkms + apt-get -y install infiniband-diags + apt-get -y install libibverbs-dev + apt-get -y install ibacm + apt-get -y install librdmacm-dev + apt-get -y install libmlx4-dev + apt-get -y install libmlx5-dev + apt-get -y install mstflint + apt-get -y install libibcm-dev + apt-get -y install libibmad-dev + apt-get -y install libibumad-dev + apt-get -y install opensm + apt-get -y install srptools + + # Install additional tools + apt-get -y install ibutils + apt-get -y install ibverbs-utils + apt-get -y install rdmacm-utils + apt-get -y install perftest + apt-get -y install numactl + apt-get -y install libnuma-dev + + # Install libnl + apt-get -y install libnl-3-200 + apt-get -y install libnl-route-3-200 + apt-get -y install libnl-route-3-dev + apt-get -y install libnl-utils + + # Download, build, and install CUDA-aware OpenMPI + wget https://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.4.tar.gz + tar -xzvf openmpi-1.8.4.tar.gz + cd openmpi-1.8.4 + ./configure --prefix=/opt/openmpi-1.8.4 --with-cuda + make all install + + # Set paths to OpenMPI binaries and libraries + export PATH="/opt/openmpi-1.8.4/bin:${PATH}" + export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}" + + # Set container environment variables + cd /.singularity.d/env + echo 'export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"' >> 90-environment.sh + echo 'export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' >> 90-environment.sh + echo 'export PATH="/opt/openmpi-1.8.4/bin:${PATH}"' >> 90-environment.sh + echo 'export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"' >> 90-environment.sh + + # Update database for mlocate + updatedb + +%files + +%runscript + +%test + + export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}" + export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + export PATH="/opt/openmpi-1.8.4/bin:${PATH}" + export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}" + + nvcc --version + ompi_info --parsable --all | grep mpi_built_with_cuda_support:value + ompi_info --all | grep btl_openib_have_cuda_gdr + ompi_info --all | grep btl_openib_have_driver_gdr