Skip to content
Snippets Groups Projects
Select Git revision
  • 0336efae6039a19da3fa79a67242011e96e89465
  • master default protected
  • development
  • v10.4.1
  • lcm_nudge
  • m_lcm
  • flexpart_lcm
  • fix_GFS
  • optimise
  • fix_GFS_23
  • feature_gfs_fix
  • wetdepo_bugfix
  • feature/newWetDepo
  • tests
  • christine
  • split_mods
  • openmp
  • anne
  • release-10.4.1
  • dev
  • 10.4.1_pesei
  • v11_beta
  • v10.4
  • v10.4_beta
  • v10.4_alpha
  • v9.2.0.3
  • v9.2
  • FPv9.3.2e
  • FPv9.3.2d
  • FPv9.3.2c
  • FPv9.3.2b
  • v9.2.0.2_Yosemite
  • ICR-1008
  • v10.2beta
  • FPv9.3.2a
  • fp9.3.1-20170412-nc4-coded
  • fp9.3.1-20170408-nc4
  • devlan-20161106
  • FPv9.3.1f
  • FPv9.3.1e
  • FPv9.3.1d
41 results

pathnames

Blame
    • Forked from Flexpart / Flexpart
    Source project has a limited visibility.
    ubuntu-cuda-openmpi.def 12.28 KiB 
    Bootstrap: debootstrap
MirrorURL: http://us.archive.ubuntu.com/ubuntu
OSVersion: xenial

%labels

    AUTHOR_NAME Marty Kandes
    AUTHOR_EMAIL mkandes@sdsc.edu
    APPLICATION_NAME cuda + openmpi
    APPLICATION_VERSION 8.0.44 + 1.8.4
    APPLICATION_URL https://www.open-mpi.org/faq/?category=runcuda
    SYSTEM_NAME comet
    SYSTEM_SINGULARITY_VERSION 2.3.2
    SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html
    LAST_UPDATED 20180320

%setup

%environment

%post -c /bin/bash

    # Set system locale
    export LC_ALL=C

    # Install system metapackages
    apt-get -y install ubuntu-standard
    apt-get -y install ubuntu-server

    # Add repositories
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} main"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} universe"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} multiverse"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} restricted"

    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates main"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates universe"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates multiverse"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates restricted"

    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports main"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports universe"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports multiverse"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports restricted"

    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security main"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security universe"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security multiverse"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security restricted"

    # Upgrade all packages to their latest versions
    apt-get -y update && apt-get -y upgrade

    # Install common packages from 'main'
    apt-get -y install autoconf
    apt-get -y install automake
    apt-get -y install build-essential
    apt-get -y install cmake
    apt-get -y install libtool
    apt-get -y install pkg-config
    apt-get -y install gfortran
    apt-get -y install zip

    # Install expect to automate responses for interactive build questions
    apt-get -y install expect

    # Make filesystem mount points
    mkdir /cvmfs /oasis /projects /scratch

    # Use /tmp to store temporary files within the container during the
        # bootstraping process
    cd /tmp

    # Install miniconda2
    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
    chmod +x Miniconda2-latest-Linux-x86_64.sh
    ./Miniconda2-latest-Linux-x86_64.sh -b -p /opt/miniconda2

    # Install miniconda3
    wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
    chmod +x Miniconda3-latest-Linux-x86_64.sh
    ./Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3

    # Location of NVIDIA repositories
    declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute'
    declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/x86_64"
    declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/x86_64/"

    # Install NVIDIA package dependencies
    apt-get -y install x11-common
    apt-get -y install xserver-xorg-core
    apt-get -y install cmake
    apt-get -y install make
    apt-get -y install dkms
    apt-get -y install linux-libc-dev
    apt-get -y install libc6-dev
    apt-get -y install lib32gcc1
    apt-get -y install libc6-i386
    apt-get -y install libgl1
    apt-get -y install libwayland-client0
    apt-get -y install libwayland-server0
    apt-get -y install pkg-config
    apt-get -y install screen-resolution-extra
    apt-get -y install libvdpau1
    apt-get -y install libatk1.0-0
    apt-get -y install libcairo-gobject2
    apt-get -y install libcairo2
    apt-get -y install libfontconfig1
    apt-get -y install libgdk-pixbuf2.0-0
    apt-get -y install libgtk-3-0
    apt-get -y install libgtk2.0-0
    apt-get -y install libjansson4
    apt-get -y install freeglut3-dev
    apt-get -y install libx11-dev
    apt-get -y install libxmu-dev
    apt-get -y install libxi-dev
    apt-get -y install libglu1-mesa
    apt-get -y install libglu1-mesa-dev
    apt-get -y install default-jre

    # Download NVIDIA drivers, libraries, and packages
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-367_367.48-0ubuntu1_amd64.deb" 
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-367-dev_367.48-0ubuntu1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_367.48-0ubuntu1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/libcuda1-367_367.48-0ubuntu1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_367.48-0ubuntu1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_367.48-0ubuntu1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_367.48-1_amd64.deb"

    wget --wait=10 "${CUDA_REPO_URL}/cuda-license-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-core-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb"
        wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-8-0_8.0.44-1_amd64.deb"

    wget --wait=10 "${ML_REPO_URL}/libcudnn6_6.0.21-1+cuda8.0_amd64.deb"
    wget --wait=10 "${ML_REPO_URL}/libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb"

    wget --wait=10 "${ML_REPO_URL}/libnccl2_2.1.2-1+cuda8.0_amd64.deb"
    wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.1.2-1+cuda8.0_amd64.deb"

    # Install NVIDIA drivers, libraries, and packages
    dpkg -i nvidia-367_367.48-0ubuntu1_amd64.deb
    dpkg -i nvidia-367-dev_367.48-0ubuntu1_amd64.deb
    dpkg -i nvidia-modprobe_367.48-0ubuntu1_amd64.deb
    dpkg -i libcuda1-367_367.48-0ubuntu1_amd64.deb
    dpkg -i nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb
    dpkg -i nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb
    dpkg -i libxnvctrl0_367.48-0ubuntu1_amd64.deb
    dpkg -i nvidia-settings_367.48-0ubuntu1_amd64.deb
    dpkg -i cuda-drivers_367.48-1_amd64.deb

    dpkg -i cuda-license-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-misc-headers-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-core-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cudart-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-driver-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cudart-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-command-line-tools-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-nvrtc-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-nvml-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-nvgraph-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cusolver-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cublas-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cublas-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cufft-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cufft-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-curand-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-curand-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cusparse-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-npp-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-npp-dev-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-samples-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-documentation-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-visual-tools-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-toolkit-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-runtime-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-demo-suite-8-0_8.0.44-1_amd64.deb

        dpkg -i libcudnn6_6.0.21-1+cuda8.0_amd64.deb
    dpkg -i libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb

    dpkg -i libnccl2_2.1.2-1+cuda8.0_amd64.deb
    dpkg -i libnccl-dev_2.1.2-1+cuda8.0_amd64.deb

    # Set paths to CUDA binaries and libraries
    export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
    export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"

    # Install basic drivers for user space access to Ethernet, RDMA,
    # and Infiniband. See https://community.mellanox.com/docs/DOC-2431
    apt-get -y install dkms
    apt-get -y install infiniband-diags
    apt-get -y install libibverbs-dev
    apt-get -y install ibacm
    apt-get -y install librdmacm-dev
    apt-get -y install libmlx4-dev
    apt-get -y install libmlx5-dev
    apt-get -y install mstflint
    apt-get -y install libibcm-dev
    apt-get -y install libibmad-dev
    apt-get -y install libibumad-dev
    apt-get -y install opensm
    apt-get -y install srptools

    # Install additional tools
    apt-get -y install ibutils
    apt-get -y install ibverbs-utils
    apt-get -y install rdmacm-utils
    apt-get -y install perftest
    apt-get -y install numactl
    apt-get -y install libnuma-dev

    # Install libnl
    apt-get -y install libnl-3-200
    apt-get -y install libnl-route-3-200
    apt-get -y install libnl-route-3-dev
    apt-get -y install libnl-utils

    # Download, build, and install CUDA-aware OpenMPI
    wget https://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.4.tar.gz
    tar -xzvf openmpi-1.8.4.tar.gz
    cd openmpi-1.8.4
    ./configure --prefix=/opt/openmpi-1.8.4 --with-cuda
     make all install

    # Set paths to OpenMPI binaries and libraries
    export PATH="/opt/openmpi-1.8.4/bin:${PATH}"
    export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"

    # Set container environment variables
    cd /.singularity.d/env
    echo 'export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"' >> 90-environment.sh
    echo 'export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' >> 90-environment.sh
    echo 'export PATH="/opt/openmpi-1.8.4/bin:${PATH}"' >> 90-environment.sh
    echo 'export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"' >> 90-environment.sh

    # Update database for mlocate
    updatedb

%files

%runscript

%test

    export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
    export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
    export PATH="/opt/openmpi-1.8.4/bin:${PATH}"
        export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}"

    nvcc --version
    ompi_info --parsable --all | grep mpi_built_with_cuda_support:value
    ompi_info --all | grep btl_openib_have_cuda_gdr
    ompi_info --all | grep btl_openib_have_driver_gdr