PYTHONPATH=/usr/local/hugectr/lib:

PATH=/usr/local/hugectr/bin:/usr/local/openmpi/bin:/usr/local/hwloc/bin:/usr/local/ucx/bin:/usr/local/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

CMAKE_BUILD_TYPE=Release ENABLE_MULTINODES=ON NCCL_A2A=ON SM=60;61;70;75;80 VAL_MODE=OFF git clone --branch v2.3 https://github.com/NVIDIA/HugeCTR.git HugeCTR &&
  cd HugeCTR &&
  git submodule update --init --recursive &&
  mkdir build &&
  cd build &&
  cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DSM=$SM -DVAL_MODE=$VAL_MODE -DENABLE_MULTINODES=$ENABLE_MULTINODES -DNCCL_A2A=$NCCL_A2A .. &&
  make -j$(nproc) &&
  mkdir /usr/local/hugectr &&
  make install &&
  chmod +x /usr/local/hugectr/bin/* &&
  chmod +x /usr/local/hugectr/lib/* &&
  rm -rf HugeCTR

CMAKE_BUILD_TYPE=Release ENABLE_MULTINODES=ON NCCL_A2A=ON SM=60;61;70;75;80 VAL_MODE=OFF mkdir -p /var/tmp &&
  cd /var/tmp &&
  git clone --depth=1 --branch branch-0.16 https://github.com/rapidsai/cudf.git cudf &&
  cd - &&
  git clone --depth=1 --branch main https://github.com/dmlc/dlpack.git /var/tmp/dlpack &&
  cd /var/tmp/cudf/cpp &&
  mkdir -p build &&
  cd build &&
  cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DGPU_ARCHS=$SM -DARROW_INCLUDE_DIR=/usr/include/arrow/ -DDLPACK_INCLUDE=/var/tmp/dlpack/include -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDISABLE_DEPRECATION_WARNING=ON &&
  make -j$(nproc) &&
  make -j$(nproc) install &&
  rm -rf /var/tmp/dlpack /var/tmp/cudf

CMAKE_BUILD_TYPE=Release ENABLE_MULTINODES=ON NCCL_A2A=ON SM=60;61;70;75;80 VAL_MODE=OFF mkdir -p /var/tmp &&
  cd /var/tmp &&
  git clone --depth=1 --branch branch-0.16 https://github.com/rapidsai/rmm.git rmm &&
  cd - &&
  cd /var/tmp/rmm &&
  mkdir -p build &&
  cd build &&
  cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX &&
  make -j$(nproc) &&
  make -j$(nproc) install &&
  rm -rf /var/tmp/rmm

CMAKE_BUILD_TYPE=Release ENABLE_MULTINODES=ON NCCL_A2A=ON SM=60;61;70;75;80 VAL_MODE=OFF if [ $(lsb_release --codename --short) = "stretch" ]; then echo "deb http://deb.debian.org/debian $(lsb_release --codename --short)-backports main" >> /etc/apt/sources.list.d/backports.list; fi &&
  wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb &&
  apt install -y -V ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb &&
  apt update &&
  apt install -y libarrow-dev libarrow-cuda-dev &&
  dpkg -r --force-depends libnvidia-compute-450-server

CONDA_PREFIX=/opt/conda NCCL_LAUNCH_MODE=PARALLEL

CMAKE_BUILD_TYPE=Release ENABLE_MULTINODES=ON NCCL_A2A=ON SM=60;61;70;75;80 VAL_MODE=OFF mkdir -p /opt/conda

CMAKE_BUILD_TYPE=Release ENABLE_MULTINODES=ON NCCL_A2A=ON SM=60;61;70;75;80 VAL_MODE=OFF env MPICC=/usr/local/openmpi/bin pip install mpi4py

LD_LIBRARY_PATH=/usr/local/openmpi/lib:/usr/local/hwloc/lib:/usr/local/ucx/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 PATH=/usr/local/openmpi/bin:/usr/local/hwloc/bin:/usr/local/ucx/bin:/usr/local/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin