NVIDIA
NVIDIA
Merlin Training
Container
NVIDIA
NVIDIA
Merlin Training

This container allows users to do preprocessing and feature engineering with NVTabular, and then train a deep-learning based recommender system model with HugeCTR.

LayerLabelCreated
sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4ENTRYPOINT
["/bin/bash" "-c" "/opt/nvidia/nvidia_entrypoint.sh"]
05/06/2022 7:54 AM UTC
sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4CMD
/bin/bash
05/06/2022 7:54 AM UTC
sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4HEALTHCHECK
&{["NONE"] "0s" "0s" "0s" '\x00'}
05/06/2022 7:54 AM UTC
sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4RUN
BUILD_HADOOP=false CORE_VER=main HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=master HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=main NVTAB_VER=main PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=main TF4REC_VER=main _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git rm -rf /usr/local/share/jupyter/lab/staging/node_modules/node-fetch
05/06/2022 7:54 AM UTC
sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4RUN
BUILD_HADOOP=false CORE_VER=main HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=master HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=main NVTAB_VER=main PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=main TF4REC_VER=main _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git rm -rf /usr/local/share/jupyter/lab/staging/node_modules/marked
05/06/2022 7:54 AM UTC
sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4RUN
BUILD_HADOOP=false CORE_VER=main HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=master HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=main NVTAB_VER=main PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=main TF4REC_VER=main _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git rm -rf /repos
05/06/2022 7:54 AM UTC
sha256:9af764e9d1ed5fe6aadcf5fbd0eeda427ef45cc0239c50a135c51d9d7bc56446RUN
BUILD_HADOOP=false CORE_VER=main HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=master HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=main NVTAB_VER=main PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=main TF4REC_VER=main _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git rm /usr/local/cuda/lib64/stubs/libcuda.so.1
05/06/2022 7:54 AM UTC
sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4ENV
PATH=/usr/local/cuda/lib64/:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin:/usr/local/cuda/bin:/usr/lib/x86_64-linux-gnu/:/opt/hadoop/bin:/opt/hadoop/sbin:/usr/local/hugectr/bin LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib:/usr/lib/jvm/default-java/lib:/usr/lib/jvm/default-java/lib/server:/usr/local/hugectr/lib PYTHONPATH=:/core:/systems:/nvtabular:/transformers4rec:/models:/usr/local/hugectr/lib
05/06/2022 7:54 AM UTC
sha256:ef94c11fd715d698bc40007b107c7812c98ba23258a5c22ceee1ff59a235ad92RUN
BUILD_HADOOP=false CORE_VER=main HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=master HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=main NVTAB_VER=main PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=main TF4REC_VER=main _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git if [[ "${HUGECTR_DEV_MODE}" == "false" ]]; then git clone --branch ${HUGECTR_VER} --depth 1 https://${_CI_JOB_TOKEN}${_HUGECTR_REPO} /hugectr &&
  cd /hugectr &&
  git submodule update --init --recursive &&
  mkdir build &&
  cd build &&
  LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs/:$LD_LIBRARY_PATH &&
  export PATH=$PATH:/usr/local/cuda-${CUDA_SHORT_VERSION}/compat &&
  if [[ -f "/usr/local/lib/libhdfs.so" ]]; then cmake -DCMAKE_BUILD_TYPE=Release -DSM="60;61;70;75;80" -DENABLE_MULTINODES=ON -DENABLE_HDFS=ON .. ; else cmake -DCMAKE_BUILD_TYPE=Release -DSM="60;61;70;75;80" -DENABLE_MULTINODES=ON .. ; fi &&
  make -j$(nproc) &&
  make install &&
  rm -rf ./* &&
  chmod +x ${HUGECTR_HOME}/bin/* ${HUGECTR_HOME}/lib/*.so &&
  cd ../onnx_converter &&
  python setup.py install ; fi
05/06/2022 7:54 AM UTC
sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4ARG
HUGECTR_HOME=/usr/local/hugectr
05/06/2022 7:32 AM UTC
...