Entrypoint /bin/bash /opt/nim/start_server.sh; WorkingDir /opt/nim

--chmod=755 ./entrypoint.sh /opt/nim/start_server.sh

SERVER_START_SCRIPT_PATH=/opt/nim/start_server.sh

model_manifest.yaml /opt/nim/etc/default/model_manifest.yaml

nvs:1000

TELEMETRY_MODE=0 TELEMETRY_ENDPOINT=0 PIP_INDEX_URL=https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi/simple mkdir -p /usr/local/lib/python3.12/dist-packages/flashinfer/data/csrc/nv_internal/tensorrt_llm/cutlass_instantiations &&
  chown -R nvs:nvs /usr/local/lib/python3.12/dist-packages/flashinfer/data/csrc/nv_internal/tensorrt_llm &&
  chmod -R 755 /usr/local/lib/python3.12/dist-packages/flashinfer/data/csrc/nv_internal/tensorrt_llm

TELEMETRY_MODE=0 TELEMETRY_ENDPOINT=0 PIP_INDEX_URL=https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi/simple [ -d /etc/ld.so.conf.d ] ||
  mkdir -p /etc/ld.so.conf.d &&
  [ -f /etc/ld.so.conf.d/nvidia.conf ] ||
  touch /etc/ld.so.conf.d/nvidia.conf &&
  echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf &&
  echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf &&
  echo "/usr/local/cuda/lib64" >> /etc/ld.so.conf.d/nvidia.conf &&
  if command -v ldconfig >/dev/null 2>&1; then ldconfig; fi

TELEMETRY_MODE=0 TELEMETRY_ENDPOINT=0 PIP_INDEX_URL=https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi/simple groupadd --gid 1000 --non-unique nvs &&
  useradd --create-home --shell /usr/sbin/nologin --uid 1000 --non-unique --gid 1000 nvs &&
  chown 1000.1000 $NIM_DIR_PATH

TELEMETRY_MODE=0 TELEMETRY_ENDPOINT=0 PIP_INDEX_URL=https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi/simple patch /usr/local/lib/python3.12/dist-packages/nimlib/nvml.py < /tmp/patches/nimlib/nvml.patch &&
  patch /usr/local/lib/python3.12/dist-packages/nimlib/profiles.py < /tmp/patches/nimlib/profiles.patch &&
  patch /usr/local/lib/python3.12/dist-packages/nimlib/profile_report.py < /tmp/patches/nimlib/profile_report.patch &&
  patch /usr/local/lib/python3.12/dist-packages/nimlib/llm/profile_selection.py < /tmp/patches/nimlib/profile_selection.patch &&
  patch /usr/local/lib/python3.12/dist-packages/nimlib/download.py < /tmp/patches/nimlib/download.patch &&
  patch /usr/local/lib/python3.12/dist-packages/nimlib/model_serve.py < /tmp/patches/nimlib/model_serve_s3.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/nimlib/custom_error_models.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/nimlib/start_server_banner.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/whitespace_protocol.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/whitespace_backend_types.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/whitespace_backend_guidance.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/whitespace_backend_lm_format_enforcer.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/whitespace_backend_outlines.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/whitespace_backend_xgrammar.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/whitespace_request.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/responses_prev_reasoning.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/protocol_validation.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/serving_responses.patch &&
  patch -p0 -d /usr/local/lib/python3.12/dist-packages/nim_inference_sdk < /tmp/patches/nim_inference_sdk/backends/vllm_backend.patch &&
  patch -p0 -d /usr/local/lib/python3.12/dist-packages/nim_inference_sdk < /tmp/patches/nim_inference_sdk/api/top_k.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/lora/01_lora_model.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/lora/02_worker_manager.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/lora/03_interfaces.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/lora/04_nemotron_h.patch &&
  for p in /tmp/patches/vllm/super_v3_custom/*.patch; do patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < "$p"; done &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/mamba_cache_rs_fp16.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/modelopt_mixed_precision.patch &&
  patch -p1 -d /usr/local/lib/python3.12/dist-packages/ < /tmp/patches/vllm/fused_moe.patch &&
  rm -rf /tmp/patches

./patches /tmp/patches