1000

NIM_MODEL_NAME=nvidia/llama-3.1-nemotron-ultra-253b-v1 NIM_VERSION=1.8.4 NIM_CONTAINER_NSPECT_ID=NSPECT-KNVB-G50E NIM_MODEL_NSPECT_ID=NSPECT-GQ1G-FQ7S NIM_TYPE=llm NIM_USER_ID=1000 rm /opt/nvidia/entrypoint.d/30-container-license.txt &&
  mkdir /.triton &&
  chown ${NIM_USER_ID}:${NIM_USER_ID} /.triton &&
  mkdir -p /etc/nim/config/ &&
  ln -s /opt/nim/etc/default/model_manifest.yaml /etc/nim/config/model_manifest.yaml &&
  mkdir /llama-nemotron-ultra &&
  python3 -c 'from nim_sdk.hub import FileSet, WorkspaceComponent, Workspace; fs = FileSet("ngc://nvstaging/nim/llama-3.1-nemotron-ultra-253b-v1:hf-e155ab2-nim-bf16-tool-use", globs=["*.json", "*.py"]); wsc = WorkspaceComponent(fs, ""); ws = Workspace([wsc]); ws = ws.make_fully_defined(); repo = ws.repo(); files = repo.get_all(); files.copy_all("/llama-nemotron-ultra")' &&
  mkdir /opt/nim/llm/nim_llm_sdk/model_specific_modules/llama-nemotron-ultra &&
  cp /llama-nemotron-ultra/*.json /opt/nim/llm/nim_llm_sdk/model_specific_modules/llama-nemotron-ultra &&
  cp /llama-nemotron-ultra/*.py /opt/nim/llm/nim_llm_sdk/model_specific_modules/llama-nemotron-ultra &&
  rm -rf /llama-nemotron-ultra &&
  chown -R ${NIM_USER_ID}:${NIM_USER_ID} /opt/nim/llm/nim_llm_sdk/model_specific_modules/llama-nemotron-ultra &&
  if [ -f /opt/nim/start-server.sh ]; then ln -s /opt/nim/start-server.sh /opt/nim/start_server.sh; fi &&
  sed -i 's/python3 -m nim_llm_sdk.entrypoints.launch/python3 -m nim_llm_sdk.entrypoints.launch --enable-auto-tool-choice --tool-call-parser llama3_json/g' /opt/nim/start_server.sh &&
  MISTRAL_FILE=$(find /opt/nim -name "mistralv3.py" 2>/dev/null | grep -m 1 "guided_decoding/models") &&
  if [ -n "$MISTRAL_FILE" ]; then echo "Found mistralv3.py at $MISTRAL_FILE" &&
  sed -i '/def check_tools_state/,/return ToolUseState.UNUSED/{/def check_tools_state/!{/return ToolUseState.UNUSED/!d}}' "$MISTRAL_FILE" &&
  sed -i '/def check_tools_state/a\        model_output = model_output.strip()\n        if model_output.startswith(self.bot_token):\n            return ToolUseState.USED\n        elif model_output.startswith("<"):\n            return ToolUseState.POSSIBLE\n        else:\n            return ToolUseState.UNUSED' "$MISTRAL_FILE" &&
  echo "Successfully patched $MISTRAL_FILE"; else echo "WARNING: Could not find mistralv3.py file" &&
  find /opt -name "mistralv3.py" 2>/dev/null ||
  echo "No mistralv3.py files found in /opt"; fi

0

CONTAINER_LICENSE /opt/nim/LICENSE

./notices/* /opt/nim/

banner.txt /opt/nvidia/entrypoint.d/31-model-license.txt

MODEL_LICENSE /opt/nim/MODEL_LICENSE

model_manifest.yaml /opt/nim/etc/default/model_manifest.yaml

NIM_HTTP_API_PORT=8000

INFERENCE_MICROSERVICE_LLM_NIM_VERSION=1.8.4