naked-singularity-v0.3.4 :: fixed typo in ubuntu-cuda-opencl.def; also

added ubuntu-cuda-opencl-openmpi.def to meet user request on comet

README

@@ -46,8 +46,8 @@ AUTHOR
 
 VERSION
 
-    0.3.3
+    0.3.4
 
 LAST UPDATED
 
-    Tuesday, June 26th, 2018
+    Friday, July 6th, 2018

definition-files/us/ucsd/sdsc/comet/ubuntu/ubuntu-cuda-opencl-openmpi.def

+Bootstrap: debootstrap
+MirrorURL: http://us.archive.ubuntu.com/ubuntu
+OSVersion: xenial
+
+%labels
+
+    APPLICATION_NAME ubuntu + cuda + opencl + openmpi
+    APPLICATION_VERSION ubuntu + 8.0.44 + 16.1.2 + 1.8.4
+    APPLICATION_URL https://software.intel.com/en-us/articles/opencl-drivers#latest_CPU_runtime
+
+    SYSTEM_NAME comet
+    SYSTEM_SINGULARITY_VERSION 2.5.1
+    SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html
+
+    SINGULARITY_IMAGE_SIZE 8192
+
+    AUTHOR_NAME Marty Kandes
+    AUTHOR_EMAIL mkandes@sdsc.edu
+
+    LAST_UPDATED 20180628
+
+%setup
+
+%environment
+
+    # Set system locale
+    export LC_ALL=C
+
+%post -c /bin/bash
+
+    # Set system locale
+    export LC_ALL=C
+
+    # Install system metapackages
+    apt-get -y install ubuntu-standard
+    apt-get -y install ubuntu-server
+
+    # Add repositories
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} main"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} universe"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} multiverse"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} restricted"
+
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates main"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates universe"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates multiverse"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates restricted"
+
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports main"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports universe"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports multiverse"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports restricted"
+
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security main"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security universe"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security multiverse"
+    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security restricted"
+
+    # Upgrade all packages to their latest versions
+    apt-get -y update && apt-get -y upgrade
+
+    # Install common packages from 'main'
+    apt-get -y install autoconf
+    apt-get -y install automake
+    apt-get -y install build-essential
+    apt-get -y install cmake
+    apt-get -y install libtool
+    apt-get -y install pkg-config
+    apt-get -y install gfortran
+    apt-get -y install zip
+
+    # Install expect to automate responses for interactive build questions
+    apt-get -y install expect
+
+    # Make filesystem mount points
+    mkdir /cvmfs /oasis /projects /scratch
+
+    # Use /tmp to store temporary files within the container during the
+    # bootstraping process
+    cd /tmp
+
+    # Install miniconda2
+    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
+    chmod +x Miniconda2-latest-Linux-x86_64.sh
+    ./Miniconda2-latest-Linux-x86_64.sh -b -p /opt/miniconda2
+
+    # Install miniconda3
+    wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    chmod +x Miniconda3-latest-Linux-x86_64.sh
+    ./Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3
+
+    # Location of NVIDIA repositories
+    declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute'
+    declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/x86_64"
+    declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/x86_64/"
+
+    # Install NVIDIA package dependencies
+    apt-get -y install x11-common
+    apt-get -y install xserver-xorg-core
+    apt-get -y install xserver-xorg-legacy
+    apt-get -y install cmake
+    apt-get -y install make
+    apt-get -y install dkms
+    apt-get -y install linux-libc-dev
+    apt-get -y install libc6-dev
+    apt-get -y install lib32gcc1
+    apt-get -y install libc6-i386
+    apt-get -y install libgl1
+    apt-get -y install libwayland-client0
+    apt-get -y install libwayland-server0
+    apt-get -y install pkg-config
+    apt-get -y install screen-resolution-extra
+    apt-get -y install libvdpau1
+    apt-get -y install libatk1.0-0
+    apt-get -y install libcairo-gobject2
+    apt-get -y install libcairo2
+    apt-get -y install libfontconfig1
+    apt-get -y install libgdk-pixbuf2.0-0
+    apt-get -y install libgtk-3-0
+    apt-get -y install libgtk2.0-0
+    apt-get -y install libjansson4
+    apt-get -y install freeglut3-dev
+    apt-get -y install libx11-dev
+    apt-get -y install libxmu-dev
+    apt-get -y install libxi-dev
+    apt-get -y install libglu1-mesa
+    apt-get -y install libglu1-mesa-dev
+    apt-get -y install default-jre
+
+    # Download NVIDIA drivers, libraries, and packages
+    wget --wait=10 "${CUDA_REPO_URL}/nvidia-396_396.26-0ubuntu1_amd64.deb" 
+    wget --wait=10 "${CUDA_REPO_URL}/nvidia-396-dev_396.26-0ubuntu1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_396.26-0ubuntu1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/libcuda1-396_396.26-0ubuntu1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/nvidia-libopencl1-396_396.26-0ubuntu1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/nvidia-opencl-icd-396_396.26-0ubuntu1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_396.26-0ubuntu1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_396.26-0ubuntu1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_396.26-1_amd64.deb"
+
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-license-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-core-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-8-0_8.0.44-1_amd64.deb"
+    wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-8-0_8.0.44-1_amd64.deb"
+
+    wget --wait=10 "${ML_REPO_URL}/libcudnn7_7.1.4.18-1+cuda8.0_amd64.deb"
+    wget --wait=10 "${ML_REPO_URL}/libcudnn7-dev_7.1.4.18-1+cuda8.0_amd64.deb"
+
+    wget --wait=10 "${ML_REPO_URL}/libnccl2_2.2.12-1+cuda8.0_amd64.deb"
+    wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.2.12-1+cuda8.0_amd64.deb"
+
+    # Fix for "update-alternatives: error: error creating symbolic link '/usr/lib/nvidia/alternate-install-present.dpkg-tmp': No such file or directory"
+    mkdir -p /usr/lib/nvidia
+
+    # Install NVIDIA drivers, libraries, and packages
+    dpkg -i nvidia-396_396.26-0ubuntu1_amd64.deb
+    dpkg -i nvidia-396-dev_396.26-0ubuntu1_amd64.deb
+    dpkg -i nvidia-modprobe_396.26-0ubuntu1_amd64.deb
+    dpkg -i libcuda1-396_396.26-0ubuntu1_amd64.deb
+    dpkg -i nvidia-libopencl1-396_396.26-0ubuntu1_amd64.deb
+    dpkg -i nvidia-opencl-icd-396_396.26-0ubuntu1_amd64.deb
+    dpkg -i libxnvctrl0_396.26-0ubuntu1_amd64.deb
+    dpkg -i nvidia-settings_396.26-0ubuntu1_amd64.deb
+    dpkg -i cuda-drivers_396.26-1_amd64.deb
+
+    dpkg -i cuda-license-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-misc-headers-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-core-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cudart-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-driver-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cudart-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-command-line-tools-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-nvrtc-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-nvml-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-nvgraph-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cusolver-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cublas-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cublas-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cufft-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cufft-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-curand-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-curand-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cusparse-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-npp-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-npp-dev-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-samples-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-documentation-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-visual-tools-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-toolkit-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-runtime-8-0_8.0.44-1_amd64.deb
+    dpkg -i cuda-demo-suite-8-0_8.0.44-1_amd64.deb
+
+    dpkg -i libcudnn7_7.1.4.18-1+cuda8.0_amd64.deb
+    dpkg -i libcudnn7-dev_7.1.4.18-1+cuda8.0_amd64.deb
+
+    dpkg -i libnccl2_2.2.12-1+cuda8.0_amd64.deb
+    dpkg -i libnccl-dev_2.2.12-1+cuda8.0_amd64.deb
+
+    # Set paths to CUDA binaries and libraries
+    export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
+    export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+
+    # Location of Intel repository
+    declare -r INTEL_ROOT_URL='http://registrationcenter-download.intel.com'
+
+    # Install generic OpenCL packages
+    apt-get -y install ocl-icd-libopencl1
+    apt-get -y install opencl-headers
+    apt-get -y install clinfo
+    apt-get -y install ocl-icd-opencl-dev
+
+    # Install packages for turing rpm into deb
+    apt-get -y install rpm
+    apt-get -y install alien
+    apt-get -y install libnuma1
+
+    # Download OpenCL runtime rpm, convert to deb, then install
+    wget "${INTEL_ROOT_URL}/akdlm/irc_nas/12556/opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz"
+    tar -xzf opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz
+
+    cd opencl_runtime_16.1.2_x64_rh_6.4.0.37/rpm
+
+    fakeroot alien --to-deb opencl-1.2-base-6.4.0.37-1.x86_64.rpm
+    fakeroot alien --to-deb opencl-1.2-intel-cpu-6.4.0.37-1.x86_64.rpm
+
+    dpkg -i opencl-1.2-base_6.4.0.37-2_amd64.deb
+    dpkg -i opencl-1.2-intel-cpu_6.4.0.37-2_amd64.deb
+
+    # Add path to clinfo
+    echo '/opt/intel/opencl-1.2-6.4.0.37/lib64/clinfo' > /etc/ld.so.conf.d/intelOpenCL.conf
+
+    # Create a vendors dir and add intel.icd 
+    mkdir -p /etc/OpenCL/vendors
+    ln /opt/intel/opencl-1.2-6.4.0.37/etc/intel64.icd /etc/OpenCL/vendors/intel64.icd
+    ldconfig
+
+    # Install basic drivers for user space access to Ethernet, RDMA,
+    # and Infiniband. See https://community.mellanox.com/docs/DOC-2431
+    apt-get -y install dkms
+    apt-get -y install infiniband-diags
+    apt-get -y install libibverbs-dev
+    apt-get -y install ibacm
+    apt-get -y install librdmacm-dev
+    apt-get -y install libmlx4-dev
+    apt-get -y install libmlx5-dev
+    apt-get -y install mstflint
+    apt-get -y install libibcm-dev
+    apt-get -y install libibmad-dev
+    apt-get -y install libibumad-dev
+    apt-get -y install opensm
+    apt-get -y install srptools
+
+    # Install additional tools
+    apt-get -y install ibutils
+    apt-get -y install ibverbs-utils
+    apt-get -y install rdmacm-utils
+    apt-get -y install perftest
+    apt-get -y install numactl
+    apt-get -y install libnuma-dev
+
+    # Install libnl
+    apt-get -y install libnl-3-200
+    apt-get -y install libnl-route-3-200
+    apt-get -y install libnl-route-3-dev
+    apt-get -y install libnl-utils
+
+    # Install OpenMPI dependencies
+    apt-get -y install zlib1g-dev
+
+    # Download, build, and install CUDA-aware OpenMPI
+    wget https://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.4.tar.gz
+    tar -xzvf openmpi-1.8.4.tar.gz
+    cd openmpi-1.8.4
+    ./configure --prefix=/opt/openmpi-1.8.4 --with-cuda
+     make all install
+
+    # Set paths to OpenMPI binaries and libraries
+    export PATH="/opt/openmpi-1.8.4/bin:${PATH}"
+    export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"
+ 
+    # Set container environment variables
+    cd /.singularity.d/env
+    echo 'export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"' >> 90-environment.sh
+    echo 'export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' >> 90-environment.sh
+    echo 'export PATH="/opt/openmpi-1.8.4/bin:${PATH}"' >> 90-environment.sh
+    echo 'export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"' >> 90-environment.sh
+
+    # Update database for mlocate
+    updatedb
+
+%files
+
+%runscript
+
+%test
+
+    export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
+    export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+    nvcc --version
+
+    # Test OpenCL installation
+    clinfo
+
+    #export PATH="/opt/openmpi-1.8.4/bin:${PATH}"
+    #export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"
+    #mpirun --version

definition-files/us/ucsd/sdsc/comet/ubuntu/ubuntu-cuda-opencl.def

@@ -17,7 +17,7 @@ OSVersion: xenial
     AUTHOR_NAME Marty Kandes
     AUTHOR_EMAIL mkandes@sdsc.edu
 
-    LAST_UPDATED 20180619
+    LAST_UPDATED 20180628
 
 %setup
 
@@ -245,7 +245,7 @@ OSVersion: xenial
     apt-get -y install libnuma1
 
     # Download OpenCL runtime rpm, convert to deb, then install
-    wget "${INTEL_ROOT_URL}/akdlm/irc_nas/12556/opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz
+    wget "${INTEL_ROOT_URL}/akdlm/irc_nas/12556/opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz"
     tar -xzf opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz
 
     cd opencl_runtime_16.1.2_x64_rh_6.4.0.37/rpm
@@ -264,6 +264,9 @@ OSVersion: xenial
     ln /opt/intel/opencl-1.2-6.4.0.37/etc/intel64.icd /etc/OpenCL/vendors/intel64.icd
     ldconfig
 
+    # Test OpenCL installation
+    clinfo
+ 
     # Set container environment variables
     cd /.singularity.d/env
     echo 'export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"' >> 90-environment.sh
@@ -277,10 +280,3 @@ OSVersion: xenial
 %runscript
 
 %test
-
-    export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
-    export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
-    nvcc --version
-
-    # Test OpenCL installation
-    clinfo