org.opencontainers.image.licenses=3rd_party_licenses.txt

README.md README.md

3rd_party_licenses.txt .

README_FILE=README.md

THIRD_PARTY_LICENSES_FILE=3rd_party_licenses.txt

pip uninstall -y nemo-evaluator-internal

tests/* /workspace/tests/

CI_COMMIT_SHA=83c6fed28a059dbdf449c085006e78b25e7e11b2 EVAL_WHEEL_VERSION=0.0.dev0+83c6fed2.19333 PKG_URL=gitlab-master.nvidia.com/api/v4/projects/153108/packages/pypi/simple NEMO_GYM_GIT_URL=https://github.com/NVIDIA-NeMo/Gym NEMO_GYM_GIT_REF=ff6f8d20ce26ddffbe6e64624487a1c69b652ec0 python -B -c "from nemo_evaluator.core.input import _copy_fdfs; _copy_fdfs('/opt/metadata/')"

CI_COMMIT_SHA=83c6fed28a059dbdf449c085006e78b25e7e11b2 EVAL_WHEEL_VERSION=0.0.dev0+83c6fed2.19333 PKG_URL=gitlab-master.nvidia.com/api/v4/projects/153108/packages/pypi/simple NEMO_GYM_GIT_URL=https://github.com/NVIDIA-NeMo/Gym NEMO_GYM_GIT_REF=ff6f8d20ce26ddffbe6e64624487a1c69b652ec0 export NEMO_GYM_HF_TOKEN="$(cat /run/secrets/nemo_gym_hf_token)" &&
  cd /opt &&
  git clone ${NEMO_GYM_GIT_URL} &&
  cd Gym &&
  git checkout ${NEMO_GYM_GIT_REF} &&
  sed -i 's|^  - benchmarks/ruler/config_nemotron_3_256k.yaml$|  # - benchmarks/ruler/config_nemotron_3_256k.yaml  # disabled in container: curtis.ml.cmu.edu hotpot download times out|' benchmarks/nemotron_3_ultra/benchmarks_no_external.yaml &&
  VIRTUAL_ENV=/opt/venv uv pip install --no-cache -e /opt/Gym &&
  export UV_CACHE_DIR=/opt/cache/uv &&
  UV_PYTHON_INSTALL_DIR=/opt/Gym/uv_python uv venv --python 3.12.12 --seed .venv &&
  . .venv/bin/activate &&
  uv sync --active --extra dev --group docs &&
  uv pip install "ray[default]==2.49.2" &&
  uv pip install --upgrade --no-cache 'aiohttp>=3.13.4,<4' 'orjson>=3.11.6,<4' 'protobuf>=6.33.5,<7' 'pyasn1>=0.6.3,<0.7' 'python-multipart>=0.0.27,<0.1' 'starlette>=0.49.1,<1' &&
  VIRTUAL_ENV=/opt/venv uv pip install --upgrade --no-cache 'aiohttp>=3.13.4,<4' 'orjson>=3.11.6,<4' 'protobuf>=6.33.5,<7' 'pyasn1>=0.6.3,<0.7' 'python-multipart>=0.0.27,<0.1' 'starlette>=0.49.1,<1' &&
  ng_prepare_benchmark hydra.output_subdir=null hydra.run.dir=/tmp/hydra_prep "+config_paths=[benchmarks/nemotron_3_ultra/ultra_remote_endpoint.yaml,benchmarks/nemotron_3_ultra/judge_remote_endpoints.yaml,benchmarks/nemotron_3_ultra/benchmarks_long.yaml,benchmarks/gdpval/config.yaml]" ++policy_base_url="dummy" ++policy_api_key="dummy" ++policy_model_name="dummy" ++Qwen3-235B-A22B-Instruct-2507-FP8.responses_api_models.vllm_model.base_url="dummy" ++Qwen3-235B-A22B-Instruct-2507-FP8.responses_api_models.vllm_model.model="dummy" ++Qwen3-235B-A22B-Instruct-2507-FP8.responses_api_models.vllm_model.api_key="dummy" ++gpt-5_2-2025-12-11.responses_api_models.openai_model.openai_base_url="dummy" ++gpt-5_2-2025-12-11.responses_api_models.openai_model.openai_model="dummy" ++gpt-5_2-2025-12-11.responses_api_models.openai_model.openai_api_key="dummy" ++allenai_wildguard_model_server.responses_api_models.vllm_model.base_url="dummy" ++allenai_wildguard_model_server.responses_api_models.vllm_model.model="dummy" ++allenai_wildguard_model_server.responses_api_models.vllm_model.api_key="dummy" ++browsecomp_tavily_search_resources_server.resources_servers.tavily_search.tavily_api_key="dummy" ++gdpval_judge_model.responses_api_models.openai_model.openai_base_url="dummy" ++gdpval_judge_model.responses_api_models.openai_model.openai_model="dummy" ++gdpval_judge_model.responses_api_models.openai_model.openai_api_key="dummy" ++hf_token=$NEMO_GYM_HF_TOKEN ++num_prepare_benchmark_processes=4 &&
  ng_run hydra.output_subdir=null hydra.run.dir=/tmp/hydra_run "+config_paths=[benchmarks/nemotron_3_ultra/ultra_remote_endpoint.yaml,benchmarks/nemotron_3_ultra/judge_remote_endpoints.yaml,benchmarks/nemotron_3_ultra/benchmarks_long.yaml,benchmarks/gdpval/config.yaml]" ++policy_base_url="dummy" ++policy_api_key="dummy" ++policy_model_name="dummy" ++Qwen3-235B-A22B-Instruct-2507-FP8.responses_api_models.vllm_model.base_url="dummy" ++Qwen3-235B-A22B-Instruct-2507-FP8.responses_api_models.vllm_model.model="dummy" ++Qwen3-235B-A22B-Instruct-2507-FP8.responses_api_models.vllm_model.api_key="dummy" ++gpt-5_2-2025-12-11.responses_api_models.openai_model.openai_base_url="dummy" ++gpt-5_2-2025-12-11.responses_api_models.openai_model.openai_model="dummy" ++gpt-5_2-2025-12-11.responses_api_models.openai_model.openai_api_key="dummy" ++allenai_wildguard_model_server.responses_api_models.vllm_model.base_url="dummy" ++allenai_wildguard_model_server.responses_api_models.vllm_model.model="dummy" ++allenai_wildguard_model_server.responses_api_models.vllm_model.api_key="dummy" ++browsecomp_tavily_search_resources_server.resources_servers.tavily_search.tavily_api_key="dummy" ++gdpval_judge_model.responses_api_models.openai_model.openai_base_url="dummy" ++gdpval_judge_model.responses_api_models.openai_model.openai_model="dummy" ++gdpval_judge_model.responses_api_models.openai_model.openai_api_key="dummy" ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.persist_deliverables_dir=/tmp/gdpval_bake_deliverables ++uv_cache_dir=$UV_CACHE_DIR ++dry_run=true &&
  for venv in /opt/venv /opt/Gym/.venv $(find /opt/Gym/responses_api_agents /opt/Gym/resources_servers -maxdepth 2 -name .venv -type d 2>/dev/null); do echo "==> CVE upgrades: $venv"; for spec in 'litellm>=1.83.7,<2' 'aiohttp>=3.13.4,<4' 'orjson>=3.11.6,<4' 'protobuf>=6.33.5,<7' 'pyasn1>=0.6.3,<0.7' 'python-multipart>=0.0.27,<0.1' 'starlette>=0.49.1,<1'; do name="${spec%%>=*}"; if "$venv/bin/python" -c "import importlib.metadata as m; m.version('$name')" 2>/dev/null; then VIRTUAL_ENV="$venv" uv pip install --upgrade --no-cache "$spec" 2>&1 | tail -2 ||
  true; fi; done; done &&
  rm -rf /tmp/hydra_prep /tmp/hydra_run /root/.cache/huggingface &&
  rm -rf /opt/cache/uv /opt/Gym/cache/uv /root/.cache/uv &&
  find /opt/Gym /opt/cache /opt/venv -type d -name .git -prune -exec rm -rf {} + 2>/dev/null ||
  true

NEMO_GYM_GIT_REF=ff6f8d20ce26ddffbe6e64624487a1c69b652ec0