diff --git a/definition-files/comet/pytorch/pytorch-cpu.def b/definition-files/comet/pytorch/pytorch-cpu.def index c2e19f908f2137a74103decd467aa36d8467e0ce..7c92cfad8c67d8796a37c2f77610653dfa593501 100644 --- a/definition-files/comet/pytorch/pytorch-cpu.def +++ b/definition-files/comet/pytorch/pytorch-cpu.def @@ -1,23 +1,20 @@ Bootstrap: debootstrap MirrorURL: http://us.archive.ubuntu.com/ubuntu -OSVersion: xenial +OSVersion: bionic %labels APPLICATION_NAME pytorch - APPLICATION_VERSION 1.2.0 - APPLICATION_URL http://pytorch.org/ - - SYSTEM_NAME comet - SYSTEM_SINGULARITY_VERSION 2.6.1 - SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html + APPLICATION_VERSION 1.4.0 + APPLICATION_URL http://pytorch.org + SINGULARITY_VERSION 2.6.1 SINGULARITY_IMAGE_SIZE 16384 AUTHOR_NAME Marty Kandes AUTHOR_EMAIL mkandes@sdsc.edu - LAST_UPDATED 20191015 + LAST_UPDATED 20200225 %setup @@ -27,8 +24,8 @@ OSVersion: xenial export LC_ALL=C # Set paths to OpenMPI binaries and libraries - export PATH="/opt/openmpi-1.8.4/bin:${PATH}" - export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}" + export PATH="/opt/openmpi-3.1.4/bin:${PATH}" + export LD_LIBRARY_PATH="/opt/openmpi-3.1.4/lib:${LD_LIBRARY_PATH}" # Set default miniconda environment export PATH="/opt/miniconda3/bin:${PATH}" @@ -72,26 +69,18 @@ OSVersion: xenial apt-get -y install build-essential apt-get -y install cmake apt-get -y install libtool + apt-get -y install gfortran apt-get -y install pkg-config apt-get -y install wget - apt-get -y install gfortran apt-get -y install zip # Install expect to automate responses for interactive build questions apt-get -y install expect - # Make filesystem mount points - mkdir /cvmfs /oasis /projects /scratch - # Use /tmp to store temporary files within the container during the # bootstraping process cd /tmp - # Install miniconda2 - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh - chmod +x Miniconda2-latest-Linux-x86_64.sh - ./Miniconda2-latest-Linux-x86_64.sh -b -p /opt/miniconda2 - # Install miniconda3 wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh chmod +x Miniconda3-latest-Linux-x86_64.sh @@ -131,19 +120,44 @@ OSVersion: xenial apt-get -y install zlib1g-dev # Download, build, and install OpenMPI - wget https://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.4.tar.gz - tar -xzvf openmpi-1.8.4.tar.gz - cd openmpi-1.8.4 - ./configure --prefix=/opt/openmpi-1.8.4 - make all install + wget https://download.open-mpi.org/release/open-mpi/v3.1/openmpi-3.1.4.tar.gz + tar -xzvf openmpi-3.1.4.tar.gz + cd openmpi-3.1.4 + ./configure --prefix=/opt/openmpi-3.1.4 + make all install # Set paths to OpenMPI binaries and libraries - export PATH="/opt/openmpi-1.8.4/bin:${PATH}" - export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}" + export PATH="/opt/openmpi-3.1.4/bin:${PATH}" + export LD_LIBRARY_PATH="/opt/openmpi-3.1.4/lib:${LD_LIBRARY_PATH}" - # Install PyTorch dependencies for python(2) - export PATH="/opt/miniconda2/bin:${PATH}" + # Setup conda environment + export PATH="/opt/miniconda3/bin:${PATH}" export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} + + # Install common python packages + conda install -y jupyterlab + conda install -y numpy + conda install -y matplotlib + conda install -y pandas + conda install -y plotly + conda install -y scipy + conda install -y scikit-learn + conda install -y scikit-image + conda install -y seaborn + conda install -y statsmodels + conda install -y opencv + + # Install GPU-accelerated python packages + conda install -y numba + + # Install additional python packages + conda install -y astropy + conda install -y biopython + conda install -y h5py + conda install -y nltk + conda install -y sympy + + # Install PyTorch dependencies conda install -y numpy conda install -y ninja conda install -y pyyaml @@ -152,115 +166,97 @@ OSVersion: xenial conda install -y setuptools conda install -y cmake conda install -y cffi - conda install -y typing - conda install -y mkldnn -c mingfeima - conda install -y future - - # Install common python packages for data science and machine - # learning applications - conda install -y pandas - conda install -y matplotlib - conda install -y ipython - conda install -y jupyter - conda install -y sympy - conda install -y nose - conda install -y scikit-learn - conda install -y scikit-image - conda install -y nltk - conda install -y statsmodels - conda install -y opencv cd /opt - # Download, build, and install PyTorch for python(2) + # Download, build, and install PyTorch + git clone --recursive https://github.com/pytorch/pytorch + cd pytorch + git checkout v1.4.0 + git submodule sync + git submodule update --init --recursive + export CC='mpicc' export CXX='mpicxx' - export CFLAGS='-march=haswell -mmmx -mno-3dnow -msse -msse2 -msse3 -mssse3 -mno-sse4a -mcx16 -msahf -mmovbe -maes -mno-sha -mpclmul -mpopcnt -mabm -mno-lwp -mfma -mno-fma4 -mno-xop -mbmi -mbmi2 -mno-tbm -mavx -mavx2 -msse4.2 -msse4.1 -mlzcnt -mno-rtm -mno-hle -mrdrnd -mf16c -mfsgsbase -mno-rdseed -mno-prfchw -mno-adx -mfxsr -mxsave -mxsaveopt -mno-avx512f -mno-avx512er -mno-avx512cd -mno-avx512pf -mno-prefetchwt1 -mtune=haswell' - export USE_OPENMP=1 + export CFLAGS='-mtune=generic' + + export BLAS='MKL' + export USE_FBGEMM=1 + export USE_MKLDNN=1 export USE_NNPACK=1 + export USE_NUMPY=1 export USE_QNNPACK=1 - export USE_MKLDNN=1 + export USE_DISTRIBUTED=1 - export USE_GLOO=1 - export USE_GLOO_IBVERBS=1 export USE_IBVERBS=1 + export USE_GLOO=1 export USE_MPI=1 export USE_C10D_MPI=1 + export USE_NCCL=0 + export USE_SYSTEM_NCCL=0 + export NCCL_INCLUDE_DIR='/usr/include' + + export USE_OPENMP=1 + export ATEN_THREADING='OMP' + export MKL_THREADING='OMP' + + export TORCH_CUDA_ARCH_LIST='NONE' export USE_CUDA=0 export USE_CUDNN=0 - export USE_NCCL=0 + + export USE_FFMPEG=0 export USE_OPENCV=1 - export USE_FBGEMM=0 - export TORCH_CUDA_ARCH_LIST="NONE" - git clone --recursive https://github.com/pytorch/pytorch - cd /opt/pytorch - git checkout v1.2.0 - git submodule sync - git submodule update --init + python setup.py install - # Download and install PyTorch Vision package for python(2) + # Download and install vision package for PyTorch git clone https://github.com/pytorch/vision.git - cd /opt/pytorch/vision - git checkout v0.3.0 + cd vision + git checkout v0.5.0 python setup.py install - # Install PyTorch dependencies for python3 - export PATH="/opt/miniconda3/bin:${PATH}" - export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} - conda install -y numpy - conda install -y ninja - conda install -y pyyaml - conda install -y mkl - conda install -y mkl-include - conda install -y setuptools - conda install -y cmake - conda install -y cffi - conda install -y typing - conda install -y mkldnn -c mingfeima - conda install -y future + cd ../ - # Install common python3 packages for data science and machine - # learning applications - conda install -y pandas - conda install -y matplotlib - conda install -y ipython - conda install -y jupyter - conda install -y sympy - conda install -y nose - conda install -y scikit-learn - conda install -y scikit-image - conda install -y nltk - conda install -y statsmodels - conda install -y opencv - - # Download, build, and install PyTorch for python3 - cd /opt/pytorch - python setup.py clean --all + # Download and install text package for PyTorch + git clone https://github.com/pytorch/text.git + cd text + git checkout 0.5.0 python setup.py install - # Download and install PyTorch Vision package for python3 - cd /opt/pytorch/vision - python setup.py clean --all + cd ../ + + # Download and install audio package for PyTorch + git clone https://github.com/pytorch/audio.git + cd audio + git checkout v0.4.0 python setup.py install - # Download the PyTorch examples repository - cd /opt/pytorch + cd ../ + + # Download examples for PyTorch git clone https://github.com/pytorch/examples.git + cd /opt + + # Install fastai for PyTorch + git clone https://github.com/fastai/fastai + cd fastai + git checkout 1.0.60 + tools/run-after-git-clone + pip install -e ".[dev]" + + # Clean up /tmp directory + cd /tmp + rm Miniconda3-latest-Linux-x86_64.sh + rm *.deb + rm -rf openmpi-3.1.4* + # Update database for mlocate updatedb + %files %runscript - echo " " - echo " __________________ __ _______________ " - echo " -----/ ____/ __ \/ |/ / ____/_ __/ " - echo " --/ / / / / / /|_/ / __/ / / " - echo " / /___/ /_/ / / / / /___ / / " - echo " \____/\____/_/ /_/_____/ /_/ " - echo " " - %test diff --git a/definition-files/comet/pytorch/pytorch-gpu.def b/definition-files/comet/pytorch/pytorch-gpu.def index 3b493b295a331be56a7c0d9bfdbf9b6c73aea99e..4b131b94432768edbc9087d61524e4a59a204d18 100644 --- a/definition-files/comet/pytorch/pytorch-gpu.def +++ b/definition-files/comet/pytorch/pytorch-gpu.def @@ -1,23 +1,20 @@ Bootstrap: debootstrap MirrorURL: http://us.archive.ubuntu.com/ubuntu -OSVersion: xenial +OSVersion: bionic %labels APPLICATION_NAME pytorch - APPLICATION_VERSION 1.2.0 - APPLICATION_URL http://pytorch.org/ - - SYSTEM_NAME comet - SYSTEM_SINGULARITY_VERSION 2.6.1 - SYSTEM_URL http://www.sdsc.edu/support/user_guides/comet.html + APPLICATION_VERSION 1.4.0 + APPLICATION_URL http://pytorch.org + SINGULARITY_VERSION 2.6.1 SINGULARITY_IMAGE_SIZE 32768 AUTHOR_NAME Marty Kandes AUTHOR_EMAIL mkandes@sdsc.edu - LAST_UPDATED 20191014 + LAST_UPDATED 20200225 %setup @@ -27,13 +24,12 @@ OSVersion: xenial export LC_ALL=C # Set paths to CUDA binaries and libraries - export PATH="/usr/local/cuda-9.2/bin${PATH:+:${PATH}}" - export LD_LIBRARY_PATH="/usr/local/cuda-9.2/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/extras/CUPTI/lib64" + export PATH="/usr/local/cuda-10.1/bin${PATH:+:${PATH}}" + export LD_LIBRARY_PATH="/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" # Set paths to OpenMPI binaries and libraries - export PATH="/opt/openmpi-1.8.4/bin:${PATH}" - export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}" + export PATH="/opt/openmpi-3.1.4/bin:${PATH}" + export LD_LIBRARY_PATH="/opt/openmpi-3.1.4/lib:${LD_LIBRARY_PATH}" # Set default miniconda environment export PATH="/opt/miniconda3/bin:${PATH}" @@ -85,18 +81,10 @@ OSVersion: xenial # Install expect to automate responses for interactive build questions apt-get -y install expect - # Make filesystem mount points - mkdir /cvmfs /oasis /projects /scratch - # Use /tmp to store temporary files within the container during the # bootstraping process cd /tmp - # Install miniconda2 - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh - chmod +x Miniconda2-latest-Linux-x86_64.sh - ./Miniconda2-latest-Linux-x86_64.sh -b -p /opt/miniconda2 - # Install miniconda3 wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh chmod +x Miniconda3-latest-Linux-x86_64.sh @@ -104,8 +92,8 @@ OSVersion: xenial # Location of NVIDIA repositories declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute' - declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/x86_64" - declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/x86_64/" + declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1804/x86_64" + declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1804/x86_64" # Install NVIDIA package dependencies apt-get -y install x11-common @@ -141,154 +129,216 @@ OSVersion: xenial apt-get -y install default-jre # Download NVIDIA drivers, libraries, and packages - wget --wait=10 "${CUDA_REPO_URL}/nvidia-396_396.26-0ubuntu1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/nvidia-396-dev_396.26-0ubuntu1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_396.26-0ubuntu1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/libcuda1-396_396.26-0ubuntu1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/nvidia-libopencl1-396_396.26-0ubuntu1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/nvidia-opencl-icd-396_396.26-0ubuntu1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_396.26-0ubuntu1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_396.26-0ubuntu1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_396.26-1_amd64.deb" - - wget --wait=10 "${CUDA_REPO_URL}/cuda-license-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvcc-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cuobjdump-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvprune-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-compiler-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-core-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvdisasm-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-gdb-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvprof-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-memcheck-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cupti-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-gpu-library-advisor-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvtx-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nsight-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-nvvp-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-tools-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-libraries-dev-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-libraries-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-9-2_9.2.88-1_amd64.deb" - wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-9-2_9.2.88-1_amd64.deb" - - wget --wait=10 "${ML_REPO_URL}/libcudnn7_7.6.2.24-1+cuda9.2_amd64.deb" - wget --wait=10 "${ML_REPO_URL}/libcudnn7-dev_7.6.2.24-1+cuda9.2_amd64.deb" - - wget --wait=10 "${ML_REPO_URL}/libnccl2_2.4.8-1+cuda9.2_amd64.deb" - wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.4.8-1+cuda9.2_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-common-418_418.67-0ubuntu1_all.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-compute-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-cfg1-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-decode-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-encode-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-fbc1-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-gl-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-ifr1-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl-dev_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-kernel-source-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-kernel-common-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-dkms-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-compute-utils-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-utils-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/xserver-xorg-video-nvidia-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-driver-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-diagnostic-418_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_418.67-0ubuntu1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libnvidia-diagnostic-418_418.67-0ubuntu1_amd64.deb" + + wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_418.67-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers-diagnostic_418.67-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-compat-10-1_418.67-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-license-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvcc-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cuobjdump-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvprune-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-compiler-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvdisasm-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-gdb-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvprof-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-memcheck-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cupti-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-gpu-library-advisor-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvtx-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-sanitizer-api-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvjpeg-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvjpeg-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libcublas10_10.2.0.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libcublas-dev_10.2.0.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nsight-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nsight-compute-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nsight-systems-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-nvvp-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-tools-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-libraries-dev-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-libraries-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-10-1_10.1.168-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/cuda-10-1_10.1.168-1_amd64.deb" + + wget --wait=10 "${CUDA_REPO_URL}/libcutensor1_1.0.0-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libcutensor-dev_1.0.0-1_amd64.deb" + wget --wait=10 "${CUDA_REPO_URL}/libcutensor-doc_1.0.0-1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libcudnn7_7.6.5.32-1+cuda10.1_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libcudnn7-dev_7.6.5.32-1+cuda10.1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libnccl2_2.5.6-1+cuda10.1_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.5.6-1+cuda10.1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libnvinfer6_6.0.1-1+cuda10.1_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libnvinfer-dev_6.0.1-1+cuda10.1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libnvinfer-plugin6_6.0.1-1+cuda10.1_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libnvinfer-plugin-dev_6.0.1-1+cuda10.1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libnvonnxparsers6_6.0.1-1+cuda10.1_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libnvonnxparsers-dev_6.0.1-1+cuda10.1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/libnvparsers6_6.0.1-1+cuda10.1_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/libnvparsers-dev_6.0.1-1+cuda10.1_amd64.deb" + + wget --wait=10 "${ML_REPO_URL}/python3-libnvinfer_6.0.1-1+cuda10.1_amd64.deb" + wget --wait=10 "${ML_REPO_URL}/python3-libnvinfer-dev_6.0.1-1+cuda10.1_amd64.deb" # Fix for "update-alternatives: error: error creating symbolic link '/usr/lib/nvidia/alternate-install-present.dpkg-tmp': No such file or directory" mkdir -p /usr/lib/nvidia # Install NVIDIA drivers, libraries, and packages - dpkg -i nvidia-396_396.26-0ubuntu1_amd64.deb - dpkg -i nvidia-396-dev_396.26-0ubuntu1_amd64.deb - dpkg -i nvidia-modprobe_396.26-0ubuntu1_amd64.deb - dpkg -i libcuda1-396_396.26-0ubuntu1_amd64.deb - dpkg -i nvidia-libopencl1-396_396.26-0ubuntu1_amd64.deb - dpkg -i nvidia-opencl-icd-396_396.26-0ubuntu1_amd64.deb - dpkg -i libxnvctrl0_396.26-0ubuntu1_amd64.deb - dpkg -i nvidia-settings_396.26-0ubuntu1_amd64.deb - dpkg -i cuda-drivers_396.26-1_amd64.deb - - dpkg -i cuda-license-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-misc-headers-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvcc-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cuobjdump-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvprune-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-compiler-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-core-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cudart-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-driver-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cudart-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvdisasm-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-gdb-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvprof-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-memcheck-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cupti-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-gpu-library-advisor-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvtx-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-command-line-tools-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvrtc-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvrtc-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvml-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvgraph-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvgraph-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cusolver-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cusolver-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cublas-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cublas-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cufft-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cufft-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-curand-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-curand-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cusparse-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-cusparse-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-npp-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-npp-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-samples-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-documentation-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nsight-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-nvvp-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-visual-tools-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-tools-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-libraries-dev-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-toolkit-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-libraries-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-runtime-9-2_9.2.88-1_amd64.deb - dpkg -i cuda-demo-suite-9-2_9.2.88-1_amd64.deb - - dpkg -i libcudnn7_7.6.2.24-1+cuda9.2_amd64.deb - dpkg -i libcudnn7-dev_7.6.2.24-1+cuda9.2_amd64.deb - - dpkg -i libnccl2_2.4.8-1+cuda9.2_amd64.deb - dpkg -i libnccl-dev_2.4.8-1+cuda9.2_amd64.deb - - # Fixing some path-related issues associated with deb-based install of libcudnn and libnccl ... - mv /usr/lib/x86_64-linux-gnu/libcudnn_static.a /usr/local/cuda-9.2/lib64/libcudnn_static.a - mv /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a /usr/local/cuda-9.2/lib64/libcudnn_static_v7.a - mv /usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.2 /usr/local/cuda-9.2/lib64/libcudnn.so.7.6.2 - ln -s /usr/local/cuda-9.2/lib64/libcudnn.so.7.6.2 /usr/local/cuda-9.2/lib64/libcudnn.so.7 - ln -s /usr/local/cuda-9.2/lib64/libcudnn.so.7 /usr/local/cuda-9.2/lib64/libcudnn.so - mv /usr/lib/x86_64-linux-gnu/libnccl_static.a /usr/local/cuda-9.2/lib64/libnccl_static.a - mv /usr/lib/x86_64-linux-gnu/libnccl.so.2.4.8 /usr/local/cuda-9.2/lib64/libnccl.so.2.4.8 - ln -s /usr/local/cuda-9.2/lib64/libnccl.so.2.4.8 /usr/local/cuda-9.2/lib64/libnccl.so.2 - ln -s /usr/local/cuda-9.2/lib64/libnccl.so.2 /usr/local/cuda-9.2/lib64/libnccl.so - mv /usr/include/nccl.h /usr/local/cuda-9.2/include/nccl.h - ln -s /usr/local/cuda-9.2/lib64 /usr/local/cuda-9.2/lib - mv /usr/share/doc/libnccl2/NCCL-SLA.txt.gz /usr/local/cuda-9.2/NCCL-SLA.txt + dpkg -i libnvidia-common-418_418.67-0ubuntu1_all.deb + dpkg -i libnvidia-compute-418_418.67-0ubuntu1_amd64.deb + dpkg -i libnvidia-cfg1-418_418.67-0ubuntu1_amd64.deb + dpkg -i libnvidia-decode-418_418.67-0ubuntu1_amd64.deb + dpkg -i libnvidia-encode-418_418.67-0ubuntu1_amd64.deb + dpkg -i libnvidia-fbc1-418_418.67-0ubuntu1_amd64.deb + dpkg -i libnvidia-gl-418_418.67-0ubuntu1_amd64.deb + dpkg -i libnvidia-ifr1-418_418.67-0ubuntu1_amd64.deb + dpkg -i libxnvctrl0_418.67-0ubuntu1_amd64.deb + dpkg -i libxnvctrl-dev_418.67-0ubuntu1_amd64.deb + dpkg -i nvidia-kernel-source-418_418.67-0ubuntu1_amd64.deb + dpkg -i nvidia-kernel-common-418_418.67-0ubuntu1_amd64.deb + dpkg -i nvidia-dkms-418_418.67-0ubuntu1_amd64.deb + dpkg -i nvidia-compute-utils-418_418.67-0ubuntu1_amd64.deb + dpkg -i nvidia-utils-418_418.67-0ubuntu1_amd64.deb + dpkg -i xserver-xorg-video-nvidia-418_418.67-0ubuntu1_amd64.deb + dpkg -i nvidia-driver-418_418.67-0ubuntu1_amd64.deb + dpkg -i libnvidia-diagnostic-418_418.67-0ubuntu1_amd64.deb + dpkg -i nvidia-modprobe_418.67-0ubuntu1_amd64.deb + dpkg -i nvidia-settings_418.67-0ubuntu1_amd64.deb + + dpkg -i cuda-drivers_418.67-1_amd64.deb + dpkg -i cuda-drivers-diagnostic_418.67-1_amd64.deb + dpkg -i cuda-compat-10-1_418.67-1_amd64.deb + dpkg -i cuda-license-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-misc-headers-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvcc-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cuobjdump-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvprune-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-compiler-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cudart-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-driver-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cudart-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvdisasm-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-gdb-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvprof-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-memcheck-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cupti-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-gpu-library-advisor-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvtx-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-sanitizer-api-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-command-line-tools-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvrtc-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvrtc-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvml-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvgraph-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvgraph-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvjpeg-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvjpeg-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cusolver-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cusolver-dev-10-1_10.1.168-1_amd64.deb + dpkg -i libcublas10_10.2.0.168-1_amd64.deb + dpkg -i libcublas-dev_10.2.0.168-1_amd64.deb + dpkg -i cuda-cufft-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cufft-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-curand-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-curand-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cusparse-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-cusparse-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-npp-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-npp-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-samples-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-documentation-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nsight-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nsight-compute-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nsight-systems-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-nvvp-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-visual-tools-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-tools-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-libraries-dev-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-toolkit-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-libraries-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-runtime-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-demo-suite-10-1_10.1.168-1_amd64.deb + dpkg -i cuda-10-1_10.1.168-1_amd64.deb + + dpkg -i libcutensor1_1.0.0-1_amd64.deb + dpkg -i libcutensor-dev_1.0.0-1_amd64.deb + dpkg -i libcutensor-doc_1.0.0-1_amd64.deb + + dpkg -i libcudnn7_7.6.5.32-1+cuda10.1_amd64.deb + dpkg -i libcudnn7-dev_7.6.5.32-1+cuda10.1_amd64.deb + + dpkg -i libnccl2_2.5.6-1+cuda10.1_amd64.deb + dpkg -i libnccl-dev_2.5.6-1+cuda10.1_amd64.deb + + dpkg -i libnvinfer6_6.0.1-1+cuda10.1_amd64.deb + dpkg -i libnvinfer-dev_6.0.1-1+cuda10.1_amd64.deb + + dpkg -i libnvinfer-plugin6_6.0.1-1+cuda10.1_amd64.deb + dpkg -i libnvinfer-plugin-dev_6.0.1-1+cuda10.1_amd64.deb + + dpkg -i libnvonnxparsers6_6.0.1-1+cuda10.1_amd64.deb + dpkg -i libnvonnxparsers-dev_6.0.1-1+cuda10.1_amd64.deb + + dpkg -i libnvparsers6_6.0.1-1+cuda10.1_amd64.deb + dpkg -i libnvparsers-dev_6.0.1-1+cuda10.1_amd64.deb + + dpkg -i python3-libnvinfer_6.0.1-1+cuda10.1_amd64.deb + dpkg -i python3-libnvinfer-dev_6.0.1-1+cuda10.1_amd64.deb # Set paths to CUDA binaries and libraries - export PATH="/usr/local/cuda-9.2/bin${PATH:+:${PATH}}" - export LD_LIBRARY_PATH="/usr/local/cuda-9.2/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/extras/CUPTI/lib64" + export PATH="/usr/local/cuda-10.1/bin${PATH:+:${PATH}}" + export LD_LIBRARY_PATH="/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" # Install basic drivers for user space access to Ethernet, RDMA, # and Infiniband. See https://community.mellanox.com/docs/DOC-2431 @@ -323,20 +373,45 @@ OSVersion: xenial # Install OpenMPI dependencies apt-get -y install zlib1g-dev - # Download, build, and install CUDA-aware OpenMPI - wget https://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.4.tar.gz - tar -xzvf openmpi-1.8.4.tar.gz - cd openmpi-1.8.4 - ./configure --prefix=/opt/openmpi-1.8.4 --with-cuda + # Download, build, and install OpenMPI + wget https://download.open-mpi.org/release/open-mpi/v3.1/openmpi-3.1.4.tar.gz + tar -xzvf openmpi-3.1.4.tar.gz + cd openmpi-3.1.4 + ./configure --prefix=/opt/openmpi-3.1.4 --with-cuda make all install # Set paths to OpenMPI binaries and libraries - export PATH="/opt/openmpi-1.8.4/bin:${PATH}" - export LD_LIBRARY_PATH="/opt/openmpi-1.8.4/lib:${LD_LIBRARY_PATH}" + export PATH="/opt/openmpi-3.1.4/bin:${PATH}" + export LD_LIBRARY_PATH="/opt/openmpi-3.1.4/lib:${LD_LIBRARY_PATH}" - # Install PyTorch dependencies for python(2) - export PATH="/opt/miniconda2/bin:${PATH}" + # Setup conda environment + export PATH="/opt/miniconda3/bin:${PATH}" export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} + + # Install common python packages + conda install -y jupyterlab + conda install -y numpy + conda install -y matplotlib + conda install -y pandas + conda install -y plotly + conda install -y scipy + conda install -y scikit-learn + conda install -y scikit-image + conda install -y seaborn + conda install -y statsmodels + conda install -y opencv + + # Install GPU-accelerated python packages + conda install -y numba + + # Install additional python packages + conda install -y astropy + conda install -y biopython + conda install -y h5py + conda install -y nltk + conda install -y sympy + + # Install PyTorch dependencies conda install -y numpy conda install -y ninja conda install -y pyyaml @@ -345,121 +420,100 @@ OSVersion: xenial conda install -y setuptools conda install -y cmake conda install -y cffi - conda install -y typing - conda install -y mkldnn -c mingfeima - conda install -y future # Add LAPACK support for the GPU - conda install -y magma-cuda92 -c pytorch - - # Install common python packages for data science and machine - # learning applications - conda install -y pandas - conda install -y matplotlib - conda install -y ipython - conda install -y jupyter - conda install -y sympy - conda install -y nose - conda install -y scikit-learn - conda install -y scikit-image - conda install -y nltk - conda install -y statsmodels - conda install -y opencv + conda install -y magma-cuda101 -c pytorch cd /opt - # Download, build, and install PyTorch for python(2) + # Download, build, and install PyTorch + git clone --recursive https://github.com/pytorch/pytorch + cd pytorch + git checkout v1.4.0 + git submodule sync + git submodule update --init --recursive + export CC='mpicc' export CXX='mpicxx' - export CFLAGS='-march=haswell -mmmx -mno-3dnow -msse -msse2 -msse3 -mssse3 -mno-sse4a -mcx16 -msahf -mmovbe -maes -mno-sha -mpclmul -mpopcnt -mabm -mno-lwp -mfma -mno-fma4 -mno-xop -mbmi -mbmi2 -mno-tbm -mavx -mavx2 -msse4.2 -msse4.1 -mlzcnt -mno-rtm -mno-hle -mrdrnd -mf16c -mfsgsbase -mno-rdseed -mno-prfchw -mno-adx -mfxsr -mxsave -mxsaveopt -mno-avx512f -mno-avx512er -mno-avx512cd -mno-avx512pf -mno-prefetchwt1 -mtune=haswell' - export USE_OPENMP=1 + export CFLAGS='-mtune=generic' + + export BLAS='MKL' + export USE_FBGEMM=1 + export USE_MKLDNN=1 export USE_NNPACK=1 + export USE_NUMPY=1 export USE_QNNPACK=1 - export USE_MKLDNN=1 + export USE_DISTRIBUTED=1 - export USE_GLOO=1 - export USE_GLOO_IBVERBS=1 export USE_IBVERBS=1 + export USE_GLOO=1 export USE_MPI=1 export USE_C10D_MPI=1 + export USE_NCCL=1 + export USE_SYSTEM_NCCL=1 + export NCCL_INCLUDE_DIR='/usr/include' + + export USE_OPENMP=1 + export ATEN_THREADING='OMP' + export MKL_THREADING='OMP' + + export TORCH_CUDA_ARCH_LIST='3.0;3.5;3.7;5.2;6.0;6.1;7.0;7.5' export USE_CUDA=1 export USE_CUDNN=1 - export USE_NCCL=1 + + export USE_FFMPEG=0 export USE_OPENCV=1 - export USE_FBGEMM=0 - export TORCH_CUDA_ARCH_LIST='3.7;6.0;7.0' - git clone --recursive https://github.com/pytorch/pytorch - cd /opt/pytorch - git checkout v1.2.0 - git submodule sync - git submodule update --init + python setup.py install - # Download and install PyTorch Vision package for python(2) + # Download and install vision package for PyTorch git clone https://github.com/pytorch/vision.git - cd /opt/pytorch/vision - git checkout v0.3.0 + cd vision + git checkout v0.5.0 python setup.py install - # Install PyTorch dependencies for python3 - export PATH="/opt/miniconda3/bin:${PATH}" - export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} - conda install -y numpy - conda install -y ninja - conda install -y pyyaml - conda install -y mkl - conda install -y mkl-include - conda install -y setuptools - conda install -y cmake - conda install -y cffi - conda install -y typing - conda install -y mkldnn -c mingfeima - conda install -y future + cd ../ - # Add LAPACK support for the GPU - conda install -y magma-cuda92 -c pytorch + # Download and install text package for PyTorch + git clone https://github.com/pytorch/text.git + cd text + git checkout 0.5.0 + python setup.py install - # Install common python3 packages for data science and machine - # learning applications - conda install -y pandas - conda install -y matplotlib - conda install -y ipython - conda install -y jupyter - conda install -y sympy - conda install -y nose - conda install -y scikit-learn - conda install -y scikit-image - conda install -y nltk - conda install -y statsmodels - conda install -y opencv + cd ../ - # Download, build, and install PyTorch for python3 - cd /opt/pytorch - python setup.py clean --all + # Download and install audio package for PyTorch + git clone https://github.com/pytorch/audio.git + cd audio + git checkout v0.4.0 python setup.py install - # Download and install PyTorch Vision package for python3 - cd /opt/pytorch/vision - python setup.py clean --all - python setup.py install + cd ../ - # Download the PyTorch examples repository - cd /opt/pytorch + # Download examples for PyTorch git clone https://github.com/pytorch/examples.git + cd /opt + + # Install fastai for PyTorch + git clone https://github.com/fastai/fastai + cd fastai + git checkout 1.0.60 + tools/run-after-git-clone + pip install -e ".[dev]" + + # Clean up /tmp directory + cd /tmp + rm Miniconda3-latest-Linux-x86_64.sh + rm *.deb + rm -rf openmpi-3.1.4* + # Update database for mlocate updatedb + %files %runscript - echo " " - echo " __________________ __ _______________ " - echo " -----/ ____/ __ \/ |/ / ____/_ __/ " - echo " --/ / / / / / /|_/ / __/ / / " - echo " / /___/ /_/ / / / / /___ / / " - echo " \____/\____/_/ /_/_____/ /_/ " - echo " " - %test diff --git a/definition-files/comet/ubuntu/ubuntu-cuda-openmpi.def b/definition-files/comet/ubuntu/ubuntu-cuda-openmpi.def index ce24fc160a90739a2a3dc34528f8dcf6f5f9bc29..28a00fb6a8a7f9dbfc71188c5dfb420802ced21e 100644 --- a/definition-files/comet/ubuntu/ubuntu-cuda-openmpi.def +++ b/definition-files/comet/ubuntu/ubuntu-cuda-openmpi.def @@ -14,7 +14,7 @@ OSVersion: bionic AUTHOR_NAME Marty Kandes AUTHOR_EMAIL mkandes@sdsc.edu - LAST_UPDATED 20200211 + LAST_UPDATED 20200225 %setup @@ -26,7 +26,6 @@ OSVersion: bionic # Set paths to CUDA binaries and libraries export PATH="/usr/local/cuda-10.1/bin${PATH:+:${PATH}}" export LD_LIBRARY_PATH="/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/extras/CUPTI/lib64" # Set paths to OpenMPI binaries and libraries export PATH="/opt/openmpi-3.1.4/bin:${PATH}" @@ -337,24 +336,9 @@ OSVersion: bionic dpkg -i python3-libnvinfer_6.0.1-1+cuda10.1_amd64.deb dpkg -i python3-libnvinfer-dev_6.0.1-1+cuda10.1_amd64.deb - # Fixing some path-related issues associated with deb-based install of libcudnn and libnccl ... - mv /usr/lib/x86_64-linux-gnu/libcudnn_static.a /usr/local/cuda-10.1/lib64/libcudnn_static.a - mv /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a /usr/local/cuda-10.1/lib64/libcudnn_static_v7.a - mv /usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.5 /usr/local/cuda-10.1/lib64/libcudnn.so.7.6.5 - ln -s /usr/local/cuda-10.1/lib64/libcudnn.so.7.6.5 /usr/local/cuda-10.1/lib64/libcudnn.so.7 - ln -s /usr/local/cuda-10.1/lib64/libcudnn.so.7 /usr/local/cuda-10.1/lib64/libcudnn.so - mv /usr/lib/x86_64-linux-gnu/libnccl_static.a /usr/local/cuda-10.1/lib64/libnccl_static.a - mv /usr/lib/x86_64-linux-gnu/libnccl.so.2.5.6 /usr/local/cuda-10.1/lib64/libnccl.so.2.5.6 - ln -s /usr/local/cuda-10.1/lib64/libnccl.so.2.5.6 /usr/local/cuda-10.1/lib64/libnccl.so.2 - ln -s /usr/local/cuda-10.1/lib64/libnccl.so.2 /usr/local/cuda-10.1/lib64/libnccl.so - mv /usr/include/nccl.h /usr/local/cuda-10.1/include/nccl.h - ln -s /usr/local/cuda-10.1/lib64 /usr/local/cuda-10.1/lib - mv /usr/share/doc/libnccl2/NCCL-SLA.txt.gz /usr/local/cuda-10.1/NCCL-SLA.txt - # Set paths to CUDA binaries and libraries export PATH="/usr/local/cuda-10.1/bin${PATH:+:${PATH}}" export LD_LIBRARY_PATH="/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/extras/CUPTI/lib64" # Install basic drivers for user space access to Ethernet, RDMA, # and Infiniband. See https://community.mellanox.com/docs/DOC-2431 diff --git a/definition-files/comet/ubuntu/ubuntu-cuda.def b/definition-files/comet/ubuntu/ubuntu-cuda.def index bd8d299b0ab2640c5dd163609f9f31e66ecf9ecd..7fe1cfa342ea29459a31be0a873c4015f8590945 100644 --- a/definition-files/comet/ubuntu/ubuntu-cuda.def +++ b/definition-files/comet/ubuntu/ubuntu-cuda.def @@ -14,7 +14,7 @@ OSVersion: bionic AUTHOR_NAME Marty Kandes AUTHOR_EMAIL mkandes@sdsc.edu - LAST_UPDATED 20200210 + LAST_UPDATED 20200225 %setup @@ -26,7 +26,6 @@ OSVersion: bionic # Set paths to CUDA binaries and libraries export PATH="/usr/local/cuda-10.1/bin${PATH:+:${PATH}}" export LD_LIBRARY_PATH="/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/extras/CUPTI/lib64" # Set default miniconda environment export PATH="/opt/miniconda3/bin:${PATH}" @@ -333,24 +332,14 @@ OSVersion: bionic dpkg -i python3-libnvinfer_6.0.1-1+cuda10.1_amd64.deb dpkg -i python3-libnvinfer-dev_6.0.1-1+cuda10.1_amd64.deb - # Fixing some path-related issues associated with deb-based install of libcudnn and libnccl ... - mv /usr/lib/x86_64-linux-gnu/libcudnn_static.a /usr/local/cuda-10.1/lib64/libcudnn_static.a - mv /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a /usr/local/cuda-10.1/lib64/libcudnn_static_v7.a - mv /usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.5 /usr/local/cuda-10.1/lib64/libcudnn.so.7.6.5 - ln -s /usr/local/cuda-10.1/lib64/libcudnn.so.7.6.5 /usr/local/cuda-10.1/lib64/libcudnn.so.7 - ln -s /usr/local/cuda-10.1/lib64/libcudnn.so.7 /usr/local/cuda-10.1/lib64/libcudnn.so - mv /usr/lib/x86_64-linux-gnu/libnccl_static.a /usr/local/cuda-10.1/lib64/libnccl_static.a - mv /usr/lib/x86_64-linux-gnu/libnccl.so.2.5.6 /usr/local/cuda-10.1/lib64/libnccl.so.2.5.6 - ln -s /usr/local/cuda-10.1/lib64/libnccl.so.2.5.6 /usr/local/cuda-10.1/lib64/libnccl.so.2 - ln -s /usr/local/cuda-10.1/lib64/libnccl.so.2 /usr/local/cuda-10.1/lib64/libnccl.so - mv /usr/include/nccl.h /usr/local/cuda-10.1/include/nccl.h - ln -s /usr/local/cuda-10.1/lib64 /usr/local/cuda-10.1/lib - mv /usr/share/doc/libnccl2/NCCL-SLA.txt.gz /usr/local/cuda-10.1/NCCL-SLA.txt - # Set paths to CUDA binaries and libraries export PATH="/usr/local/cuda-10.1/bin${PATH:+:${PATH}}" export LD_LIBRARY_PATH="/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/extras/CUPTI/lib64" + + # Clean up /tmp directory + cd /tmp + rm Miniconda3-latest-Linux-x86_64.sh + rm *.deb # Update database for mlocate updatedb diff --git a/definition-files/comet/ubuntu/ubuntu-openmpi.def b/definition-files/comet/ubuntu/ubuntu-openmpi.def index c2e02bbc0245a85c74d2886d8cf76428be0eb014..ea56e316259b26e510fc63ca1ab8bfaa201e743b 100644 --- a/definition-files/comet/ubuntu/ubuntu-openmpi.def +++ b/definition-files/comet/ubuntu/ubuntu-openmpi.def @@ -14,7 +14,7 @@ OSVersion: bionic AUTHOR_NAME Marty Kandes AUTHOR_EMAIL mkandes@sdsc.edu - LAST_UPDATED 20200213 + LAST_UPDATED 20200225 %setup @@ -130,6 +130,12 @@ OSVersion: bionic export PATH="/opt/openmpi-3.1.4/bin:${PATH}" export LD_LIBRARY_PATH="/opt/openmpi-3.1.4/lib:${LD_LIBRARY_PATH}" + # Clean up /tmp directory + cd /tmp + rm Miniconda3-latest-Linux-x86_64.sh + rm *.deb + rm -rf openmpi-3.1.4* + # Update database for mlocate updatedb diff --git a/definition-files/comet/ubuntu/ubuntu.def b/definition-files/comet/ubuntu/ubuntu.def index 4572ecf774f6a7e1771fc6dcb591a883d6b6a9a9..3efc3925cef8a6968ba35ecb8d805f389337a6b4 100644 --- a/definition-files/comet/ubuntu/ubuntu.def +++ b/definition-files/comet/ubuntu/ubuntu.def @@ -14,7 +14,7 @@ OSVersion: bionic AUTHOR_NAME Marty Kandes AUTHOR_EMAIL mkandes@sdsc.edu - LAST_UPDATED 20200210 + LAST_UPDATED 20200225 %setup @@ -81,6 +81,10 @@ OSVersion: bionic wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh chmod +x Miniconda3-latest-Linux-x86_64.sh ./Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 + + # Clean up /tmp directory + cd /tmp + rm Miniconda3-latest-Linux-x86_64.sh # Update database for mlocate updatedb