NVIDIA
NVIDIA
Merlin Training
Container
NVIDIA
NVIDIA
Merlin Training

This container allows users to do preprocessing and feature engineering with NVTabular, and then train a deep-learning based recommender system model with HugeCTR.

  • LayerLabelCreated
    sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4ENTRYPOINT
    ["/bin/bash" "-c" "/opt/nvidia/nvidia_entrypoint.sh"]
    05/11/2022 10:51 PM UTC
    sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4CMD
    /bin/bash
    05/11/2022 10:51 PM UTC
    sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4HEALTHCHECK
    &{["NONE"] "0s" "0s" "0s" '\x00'}
    05/11/2022 10:50 PM UTC
    sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4RUN
    BUILD_HADOOP=false CORE_VER=v0.3.0 HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=v3.6 HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=v0.4.0 NVTAB_VER=v1.1.1 PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=v0.2.0 TF4REC_VER=v0.1.8 _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git rm -rf /usr/local/share/jupyter/lab/staging/node_modules/node-fetch
    05/11/2022 10:50 PM UTC
    sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4RUN
    BUILD_HADOOP=false CORE_VER=v0.3.0 HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=v3.6 HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=v0.4.0 NVTAB_VER=v1.1.1 PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=v0.2.0 TF4REC_VER=v0.1.8 _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git rm -rf /usr/local/share/jupyter/lab/staging/node_modules/marked
    05/11/2022 10:50 PM UTC
    sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4RUN
    BUILD_HADOOP=false CORE_VER=v0.3.0 HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=v3.6 HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=v0.4.0 NVTAB_VER=v1.1.1 PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=v0.2.0 TF4REC_VER=v0.1.8 _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git rm -rf /repos
    05/11/2022 10:50 PM UTC
    sha256:c34bdb9206e83dd3ea9818d8dcf28e4b8c6618daea92a63841cf6992869ebdccRUN
    BUILD_HADOOP=false CORE_VER=v0.3.0 HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=v3.6 HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=v0.4.0 NVTAB_VER=v1.1.1 PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=v0.2.0 TF4REC_VER=v0.1.8 _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git rm /usr/local/cuda/lib64/stubs/libcuda.so.1
    05/11/2022 10:50 PM UTC
    sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4ENV
    PATH=/usr/local/cuda/lib64/:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin:/usr/local/cuda/bin:/usr/lib/x86_64-linux-gnu/:/opt/hadoop/bin:/opt/hadoop/sbin:/usr/local/hugectr/bin LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib:/usr/lib/jvm/default-java/lib:/usr/lib/jvm/default-java/lib/server:/usr/local/hugectr/lib PYTHONPATH=:/usr/local/hugectr/lib
    05/11/2022 10:50 PM UTC
    sha256:5385d0a70899d1597dd170b557fe4bc24e1bb56a1a0f8b1be1ed640e4be1fb34RUN
    BUILD_HADOOP=false CORE_VER=v0.3.0 HADOOP_VER=3.3.2 HIREDIS_VER=1.0.2 HUGECTR_DEV_MODE=false HUGECTR_HOME=/usr/local/hugectr HUGECTR_VER=v3.6 HWLOC_VER=2.4.1 INSTALL_HDFS=false MODELS_VER=v0.4.0 NVTAB_VER=v1.1.1 PROTOC_VER=3.19.4 RDKAFKA_VER=1.8.2 REDIS_PP_VER=1.3.3 ROCKSDB_VER=6.29.3 SYSTEMS_VER=v0.2.0 TF4REC_VER=v0.1.8 _CI_JOB_TOKEN= _HUGECTR_REPO=github.com/NVIDIA-Merlin/HugeCTR.git if [[ "${HUGECTR_DEV_MODE}" == "false" ]]; then git clone --branch ${HUGECTR_VER} --depth 1 https://${_CI_JOB_TOKEN}${_HUGECTR_REPO} /hugectr &&
      cd /hugectr &&
      git submodule update --init --recursive &&
      mkdir build &&
      cd build &&
      LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs/:$LD_LIBRARY_PATH &&
      export PATH=$PATH:/usr/local/cuda-${CUDA_SHORT_VERSION}/compat &&
      if [[ -f "/usr/local/lib/libhdfs.so" ]]; then cmake -DCMAKE_BUILD_TYPE=Release -DSM="60;61;70;75;80" -DENABLE_MULTINODES=ON -DENABLE_HDFS=ON .. ; else cmake -DCMAKE_BUILD_TYPE=Release -DSM="60;61;70;75;80" -DENABLE_MULTINODES=ON .. ; fi &&
      make -j$(nproc) &&
      make install &&
      rm -rf ./* &&
      chmod +x ${HUGECTR_HOME}/bin/* ${HUGECTR_HOME}/lib/*.so &&
      cd ../onnx_converter &&
      python setup.py install ; fi
    05/11/2022 10:50 PM UTC
    sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4ARG
    HUGECTR_HOME=/usr/local/hugectr
    05/11/2022 10:29 PM UTC
    ...