naked-singularity-v0.2.7 :: updated mxnet-cpu.def and mxnet-gpu.def for

comet to MXNet v1.1.0

naked-singularity-v0.2.7 :: updated mxnet-cpu.def and mxnet-gpu.def for
b807a6dd · Marty Kandes · 53d09a7f · b807a6dd · b807a6dd · b807a6dd
Commit b807a6dd authored Mar 30, 2018 by Marty Kandes
--- a/README
+++ b/README
@@ -29,8 +29,8 @@ AUTHOR

 VERSION

-    0.2.6
+    0.2.7

 LAST UPDATED

-    Tuesday, March 27th, 2018
+    Friday, March 30th, 2018
--- a/definition-files/us/ucsd/sdsc/comet/mxnet/mxnet-cpu.def
+++ b/definition-files/us/ucsd/sdsc/comet/mxnet/mxnet-cpu.def
@@ -7,13 +7,12 @@ OSVersion: xenial
    AUTHOR_NAME Marty Kandes
    AUTHOR_EMAIL mkandes@sdsc.edu
    APPLICATION_NAME mxnet-cpu
-    APPLICATION_VERSION unknown
-    APPLICATION_URL http://mxnet.io/
+    APPLICATION_VERSION 1.1.0
+    APPLICATION_URL http://mxnet.io
    SYSTEM_NAME comet
    SYSTEM_SINGULARITY_VERSION 2.3.2
    SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html
-    VERSION 0.0.4
-    LAST_UPDATED 20171105
+    LAST_UPDATED 20180329

 %setup

@@ -21,11 +20,14 @@ OSVersion: xenial

 %post -c /bin/bash

+    # Set system locale
    export LC_ALL=C

+    # Install system metapackages
    apt-get -y install ubuntu-standard
    apt-get -y install ubuntu-server

+    # Add repositories
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} main"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} universe"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} multiverse"
@@ -46,34 +48,104 @@ OSVersion: xenial
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security multiverse"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security restricted"

+    # Upgrade all packages to their latest versions
    apt-get -y update && apt-get -y upgrade

+    # Install common packages from 'main'
+    apt-get -y install autoconf
+    apt-get -y install automake
+    apt-get -y install build-essential
+    apt-get -y install libtool
+    apt-get -y install pkg-config
+    apt-get -y install gfortran
+    apt-get -y install zip
+
+    # Install expect to automate responses for interactive build questions
+    apt-get -y install expect
+
+    # Make filesystem mount points
    mkdir /cvmfs /oasis /projects /scratch

-    cd /opt
+    # Use /tmp to store temporary files within the container during the
+    # bootstraping process
+    cd /tmp

-    apt-get -y install build-essential
+    # Install miniconda2
+    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
+    chmod +x Miniconda2-latest-Linux-x86_64.sh
+    ./Miniconda2-latest-Linux-x86_64.sh -b -p /opt/miniconda2
+
+    # Install miniconda3
+    wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    chmod +x Miniconda3-latest-Linux-x86_64.sh
+    ./Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3
+
+    # Install MXNet dependencies
    apt-get -y install libopenblas-dev
    apt-get -y install liblapack-dev
    apt-get -y install libopencv-dev

-    git clone --recursive https://github.com/dmlc/mxnet
-    cd mxnet
+    cd /opt
+
+    # Download, build and install MXNet core shared library
+    wget https://github.com/apache/incubator-mxnet/releases/download/1.1.0/apache-mxnet-src-1.1.0-incubating.tar.gz
+    tar -xzvf apache-mxnet-src-1.1.0-incubating.tar.gz
+    cd /opt/apache-mxnet-src-1.1.0-incubating
    make USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1

+    # Install MXNet python(2) bindings
+    apt-get -y install python-dev
+    apt-get -y install python-setuptools
+    apt-get -y install python-pip
+    apt-get -y install libgfortran3
+
+    cd /opt/apache-mxnet-src-1.1.0-incubating/python
+    pip install --upgrade pip
+    pip install -e ./
+
+    # Install MXNet python3 bindings
    apt-get -y install python3-dev
    apt-get -y install python3-setuptools
-    apt-get -y install python3-numpy
    apt-get -y install python3-pip
-    apt-get -y install python3-tk
-    apt-get -y install python3-matplotlib

-    cd python
-    pip3 install --upgrade pip3
+    pip3 install --upgrade pip
    pip3 install -e ./

+    # Install graphviz (needed for graph visualization using mxnet.viz).
    apt-get -y install graphviz
    
+    # Install common python packages for data science and machine learning applications
+    apt-get -y install python-scipy
+    apt-get -y install python-pandas
+    apt-get -y install python-matplotlib
+    apt-get -y install ipython
+    apt-get -y install ipython-notebook
+    apt-get -y install python-sympy
+    apt-get -y install python-nose
+    apt-get -y install python-sklearn
+    apt-get -y install python-mlpy
+    apt-get -y install python-nltk
+    apt-get -y install python-statsmodels
+    apt-get -y install libopencv-dev
+    apt-get -y install python-opencv
+
+    # Install common python3 packages for data science and machine learning applications
+    apt-get -y install python3-scipy
+    apt-get -y install python3-pandas
+    apt-get -y install python3-matplotlib
+    apt-get -y install ipython3
+    apt-get -y install ipython3-notebook
+    apt-get -y install python3-sympy
+    apt-get -y install python3-nose
+    apt-get -y install python3-sklearn
+    apt-get -y install python3-nltk
+    
+    # Set container environment variables
+    cd /.singularity.d/env
+
+    # Update database for mlocate
+    updatedb
+
 %files

 %runscript

--- a/definition-files/us/ucsd/sdsc/comet/mxnet/mxnet-gpu.def
+++ b/definition-files/us/ucsd/sdsc/comet/mxnet/mxnet-gpu.def
@@ -6,14 +6,13 @@ OSVersion: xenial

    AUTHOR_NAME Marty Kandes
    AUTHOR_EMAIL mkandes@sdsc.edu
-    APPLICATION_NAME mxnet
-    APPLICATION_VERSION 1.0.0
-    APPLICATION_URL https://mxnet.incubator.apache.org/
+    APPLICATION_NAME mxnet-gpu
+    APPLICATION_VERSION 1.1.0
+    APPLICATION_URL http://mxnet.io
    SYSTEM_NAME comet
    SYSTEM_SINGULARITY_VERSION 2.3.2
    SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html
-    VERSION 0.0.1
-    LAST_UPDATED 20180108
+    LAST_UPDATED 20180329

 %setup

@@ -21,11 +20,14 @@ OSVersion: xenial

 %post -c /bin/bash

+    # Set system locale
    export LC_ALL=C

+    # Install system metapackages
    apt-get -y install ubuntu-standard
    apt-get -y install ubuntu-server

+    # Add repositories
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} main"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} universe"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} multiverse"
@@ -46,16 +48,44 @@ OSVersion: xenial
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security multiverse"
    add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security restricted"

+    # Upgrade all packages to their latest versions
    apt-get -y update && apt-get -y upgrade

+    # Install common packages from 'main'
+    apt-get -y install autoconf
+    apt-get -y install automake
+    apt-get -y install build-essential
+    apt-get -y install libtool
+    apt-get -y install pkg-config
+    apt-get -y install gfortran
+    apt-get -y install zip
+
+    # Install expect to automate responses for interactive build questions
+    apt-get -y install expect
+
+    # Make filesystem mount points
    mkdir /cvmfs /oasis /projects /scratch

-    cd /opt
+    # Use /tmp to store temporary files within the container during the
+    # bootstraping process
+    cd /tmp
+
+    # Install miniconda2
+    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
+    chmod +x Miniconda2-latest-Linux-x86_64.sh
+    ./Miniconda2-latest-Linux-x86_64.sh -b -p /opt/miniconda2

+    # Install miniconda3
+    wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    chmod +x Miniconda3-latest-Linux-x86_64.sh
+    ./Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3
+
+    # Location of NVIDIA repositories
    declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute'
    declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/x86_64"
    declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/x86_64/"

+    # Install NVIDIA package dependencies
    apt-get -y install x11-common
    apt-get -y install xserver-xorg-core
    apt-get -y install cmake
@@ -87,6 +117,7 @@ OSVersion: xenial
    apt-get -y install libglu1-mesa-dev
    apt-get -y install default-jre

+    # Download NVIDIA drivers, libraries, and packages
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-367_367.48-0ubuntu1_amd64.deb" 
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-367-dev_367.48-0ubuntu1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_367.48-0ubuntu1_amd64.deb"
@@ -128,12 +159,13 @@ OSVersion: xenial
    wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-8-0_8.0.44-1_amd64.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-8-0_8.0.44-1_amd64.deb"

-    wget --wait=10 "${ML_REPO_URL}/libcudnn5_5.1.10-1+cuda8.0_amd64.deb"
-    wget --wait=10 "${ML_REPO_URL}/libcudnn5-dev_5.1.10-1+cuda8.0_amd64.deb"
+    wget --wait=10 "${ML_REPO_URL}/libcudnn6_6.0.21-1+cuda8.0_amd64.deb"
+    wget --wait=10 "${ML_REPO_URL}/libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb"

-    wget --wait=10 "${ML_REPO_URL}/libnccl1_1.2.3-1+cuda8.0_amd64.deb"
-    wget --wait=10 "${ML_REPO_URL}/libnccl-dev_1.2.3-1+cuda8.0_amd64.deb"
+    wget --wait=10 "${ML_REPO_URL}/libnccl2_2.1.2-1+cuda8.0_amd64.deb"
+    wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.1.2-1+cuda8.0_amd64.deb"

+    # Install NVIDIA drivers, libraries, and packages
    dpkg -i nvidia-367_367.48-0ubuntu1_amd64.deb
    dpkg -i nvidia-367-dev_367.48-0ubuntu1_amd64.deb
    dpkg -i nvidia-modprobe_367.48-0ubuntu1_amd64.deb
@@ -175,37 +207,84 @@ OSVersion: xenial
    dpkg -i cuda-runtime-8-0_8.0.44-1_amd64.deb
    dpkg -i cuda-demo-suite-8-0_8.0.44-1_amd64.deb

-    dpkg -i libcudnn5_5.1.10-1+cuda8.0_amd64.deb
-    dpkg -i libcudnn5-dev_5.1.10-1+cuda8.0_amd64.deb 
+    dpkg -i libcudnn6_6.0.21-1+cuda8.0_amd64.deb
+    dpkg -i libcudnn6-dev_6.0.21-1+cuda8.0_amd64.deb

-    dpkg -i libnccl1_1.2.3-1+cuda8.0_amd64.deb
-    dpkg -i libnccl-dev_1.2.3-1+cuda8.0_amd64.deb 
+    dpkg -i libnccl2_2.1.2-1+cuda8.0_amd64.deb
+    dpkg -i libnccl-dev_2.1.2-1+cuda8.0_amd64.deb

+    # Set paths to CUDA binaries and libraries
    export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
    export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"

-    apt-get -y install build-essential
+    # Install MXNet dependencies
    apt-get -y install libopenblas-dev
    apt-get -y install liblapack-dev
    apt-get -y install libopencv-dev

-    git clone --recursive https://github.com/dmlc/mxnet
-    cd mxnet
+    cd /opt
+
+    # Download, build and install MXNet core shared library
+    wget https://github.com/apache/incubator-mxnet/releases/download/1.1.0/apache-mxnet-src-1.1.0-incubating.tar.gz
+    tar -xzvf apache-mxnet-src-1.1.0-incubating.tar.gz
+    cd /opt/apache-mxnet-src-1.1.0-incubating
    make USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1

+    # Install MXNet python(2) bindings
+    apt-get -y install python-dev
+    apt-get -y install python-setuptools
+    apt-get -y install python-pip
+    apt-get -y install libgfortran3
+
+    cd /opt/apache-mxnet-src-1.1.0-incubating/python
+    pip install --upgrade pip
+    pip install -e ./
+
+    # Install MXNet python3 bindings
    apt-get -y install python3-dev
    apt-get -y install python3-setuptools
-    apt-get -y install python3-numpy
    apt-get -y install python3-pip
-    apt-get -y install python3-tk
-    apt-get -y install python3-matplotlib

-    cd python
-    pip3 install --upgrade pip3
-    pip3 install -e .
+    pip3 install --upgrade pip
+    pip3 install -e ./

+    # Install graphviz (needed for graph visualization using mxnet.viz).
    apt-get -y install graphviz

+    # Install common python packages for data science and machine learning applications
+    apt-get -y install python-scipy
+    apt-get -y install python-pandas
+    apt-get -y install python-matplotlib
+    apt-get -y install ipython
+    apt-get -y install ipython-notebook
+    apt-get -y install python-sympy
+    apt-get -y install python-nose
+    apt-get -y install python-sklearn
+    apt-get -y install python-mlpy
+    apt-get -y install python-nltk
+    apt-get -y install python-statsmodels
+    apt-get -y install libopencv-dev
+    apt-get -y install python-opencv
+
+    # Install common python3 packages for data science and machine learning applications
+    apt-get -y install python3-scipy
+    apt-get -y install python3-pandas
+    apt-get -y install python3-matplotlib
+    apt-get -y install ipython3
+    apt-get -y install ipython3-notebook
+    apt-get -y install python3-sympy
+    apt-get -y install python3-nose
+    apt-get -y install python3-sklearn
+    apt-get -y install python3-nltk
+
+    # Set container environment variables
+    cd /.singularity.d/env
+    echo 'export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"' >> 90-environment.sh
+    echo 'export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' >> 90-environment.sh
+
+    # Update database for mlocate
+    updatedb
+
 %files

 %runscript
@@ -214,4 +293,6 @@ OSVersion: xenial

    export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}"
    export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+
    nvcc --version
+