diff --git a/definition-files/us/ucsd/sdsc/comet/mxnet/mxnet-gpu.def b/definition-files/us/ucsd/sdsc/comet/mxnet/mxnet-gpu.def new file mode 100644 index 0000000000000000000000000000000000000000..254c2adf2405011ec3cd1ef530e57d8284451849 --- /dev/null +++ b/definition-files/us/ucsd/sdsc/comet/mxnet/mxnet-gpu.def @@ -0,0 +1,217 @@ +Bootstrap: debootstrap +MirrorURL: http://us.archive.ubuntu.com/ubuntu +OSVersion: xenial + +%labels + + AUTHOR_NAME Marty Kandes + AUTHOR_EMAIL mkandes@sdsc.edu + APPLICATION_NAME mxnet + APPLICATION_VERSION 1.0.0 + APPLICATION_URL https://mxnet.incubator.apache.org/ + SYSTEM_NAME comet + SYSTEM_SINGULARITY_VERSION 2.3.2 + SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html + VERSION 0.0.1 + LAST_UPDATED 20180108 + +%setup + +%environment + +%post -c /bin/bash + + export LC_ALL=C + + apt-get -y install ubuntu-standard + apt-get -y install ubuntu-server + + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} main" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} universe" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} multiverse" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION} restricted" + + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates main" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates universe" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates multiverse" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-updates restricted" + + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports main" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports universe" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports multiverse" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-backports restricted" + + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security main" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security universe" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security multiverse" + add-apt-repository -y "deb ${MIRRORURL} ${OSVERSION}-security restricted" + + apt-get -y update && apt-get -y upgrade + + mkdir /cvmfs /oasis /projects /scratch + + cd /opt + + declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute' + declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/x86_64" + declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/x86_64/" + + apt-get -y install x11-common + apt-get -y install xserver-xorg-core + apt-get -y install cmake + apt-get -y install make + apt-get -y install dkms + apt-get -y install linux-libc-dev + apt-get -y install libc6-dev + apt-get -y install lib32gcc1 + apt-get -y install libc6-i386 + apt-get -y install libgl1 + apt-get -y install libwayland-client0 + apt-get -y install libwayland-server0 + apt-get -y install pkg-config + apt-get -y install screen-resolution-extra + apt-get -y install libvdpau1 + apt-get -y install libatk1.0-0 + apt-get -y install libcairo-gobject2 + apt-get -y install libcairo2 + apt-get -y install libfontconfig1 + apt-get -y install libgdk-pixbuf2.0-0 + apt-get -y install libgtk-3-0 + apt-get -y install libgtk2.0-0 + apt-get -y install libjansson4 + apt-get -y install freeglut3-dev + apt-get -y install libx11-dev + apt-get -y install libxmu-dev + apt-get -y install libxi-dev + apt-get -y install libglu1-mesa + apt-get -y install libglu1-mesa-dev + apt-get -y install default-jre + + wget --wait=10 "${CUDA_REPO_URL}/nvidia-367_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-367-dev_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libcuda1-367_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_367.48-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_367.48-1_amd64.deb" + + wget --wait=10 "${CUDA_REPO_URL}/cuda-license-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-core-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-8-0_8.0.44-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-8-0_8.0.44-1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libcudnn5_5.1.10-1+cuda8.0_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libcudnn5-dev_5.1.10-1+cuda8.0_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libnccl1_1.2.3-1+cuda8.0_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libnccl-dev_1.2.3-1+cuda8.0_amd64.deb" + + dpkg -i nvidia-367_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-367-dev_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-modprobe_367.48-0ubuntu1_amd64.deb + dpkg -i libcuda1-367_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-libopencl1-367_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-opencl-icd-367_367.48-0ubuntu1_amd64.deb + dpkg -i libxnvctrl0_367.48-0ubuntu1_amd64.deb + dpkg -i nvidia-settings_367.48-0ubuntu1_amd64.deb + dpkg -i cuda-drivers_367.48-1_amd64.deb + + dpkg -i cuda-license-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-misc-headers-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-core-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cudart-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-driver-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cudart-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-command-line-tools-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvrtc-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvrtc-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvml-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvgraph-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-nvgraph-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cusolver-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cusolver-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cublas-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cublas-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cufft-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cufft-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-curand-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-curand-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cusparse-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-cusparse-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-npp-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-npp-dev-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-samples-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-documentation-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-visual-tools-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-toolkit-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-runtime-8-0_8.0.44-1_amd64.deb + dpkg -i cuda-demo-suite-8-0_8.0.44-1_amd64.deb + + dpkg -i libcudnn5_5.1.10-1+cuda8.0_amd64.deb + dpkg -i libcudnn5-dev_5.1.10-1+cuda8.0_amd64.deb + + dpkg -i libnccl1_1.2.3-1+cuda8.0_amd64.deb + dpkg -i libnccl-dev_1.2.3-1+cuda8.0_amd64.deb + + export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}" + export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + + apt-get -y install build-essential + apt-get -y install libopenblas-dev + apt-get -y install liblapack-dev + apt-get -y install libopencv-dev + + git clone --recursive https://github.com/dmlc/mxnet + cd mxnet + make USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 + + apt-get -y install python3-dev + apt-get -y install python3-setuptools + apt-get -y install python3-numpy + apt-get -y install python3-pip + apt-get -y install python3-tk + apt-get -y install python3-matplotlib + + cd python + pip3 install --upgrade pip3 + pip3 install -e . + + apt-get -y install graphviz + +%files + +%runscript + +%test + + export PATH="/usr/local/cuda-8.0/bin${PATH:+:${PATH}}" + export LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + nvcc --version diff --git a/definition-files/us/ucsd/sdsc/comet/tensorflow/tensorflow-cpu.def b/definition-files/us/ucsd/sdsc/comet/tensorflow/tensorflow-cpu.def index fc2a6a6b9fb4d97e5cf9f837f66ab7760b69ad65..71161c0fa9c501153f01ef6ea82d103cb94cb68e 100644 --- a/definition-files/us/ucsd/sdsc/comet/tensorflow/tensorflow-cpu.def +++ b/definition-files/us/ucsd/sdsc/comet/tensorflow/tensorflow-cpu.def @@ -12,8 +12,8 @@ OSVersion: xenial SYSTEM_NAME comet SYSTEM_SINGULARITY_VERSION 2.3.2 SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html - VERSION 0.0.5 - LAST_UPDATED 20171220 + VERSION 0.0.6 + LAST_UPDATED 20180108 %setup @@ -108,6 +108,8 @@ OSVersion: xenial bazel-bin/tensorflow/tools/pip_package/build_pip_package /opt/tensorflow_pkg pip install /opt/tensorflow_pkg/tensorflow-1.4.1-cp27-cp27mu-linux_x86_64.whl + + apt-get -y install python-scipy %files diff --git a/definition-files/us/ucsd/sdsc/comet/tensorflow/tensorflow-gpu.def b/definition-files/us/ucsd/sdsc/comet/tensorflow/tensorflow-gpu.def index 3406f03df20317409799a613d32a43910f43e08b..172d61bb537e620b11981a42d34cf2c80e56f4d9 100644 --- a/definition-files/us/ucsd/sdsc/comet/tensorflow/tensorflow-gpu.def +++ b/definition-files/us/ucsd/sdsc/comet/tensorflow/tensorflow-gpu.def @@ -12,8 +12,8 @@ OSVersion: xenial SYSTEM_NAME comet SYSTEM_SINGULARITY_VERSION 2.3.2 SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html - VERSION 0.0.4 - LAST_UPDATED 20171222 + VERSION 0.0.5 + LAST_UPDATED 20180108 %setup @@ -259,6 +259,8 @@ OSVersion: xenial bazel-bin/tensorflow/tools/pip_package/build_pip_package /opt/tensorflow_pkg pip3 install /opt/tensorflow_pkg/tensorflow-1.4.1-cp35-cp35m-linux_x86_64.whl + + apt-get -y install python3-scipy %files