—

echo "/usr/local/gdrcopy/lib" >> /etc/ld.so.conf.d/hpccm.conf &&
  ldconfig

/usr/local/gdrcopy /usr/local/gdrcopy

apt-get update -y &&
  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends -t focal dapl2-utils ibutils ibverbs-providers ibverbs-utils infiniband-diags libdapl-dev libdapl2 libibmad-dev libibmad5 libibverbs-dev libibverbs1 librdmacm-dev librdmacm1 rdmacm-utils &&
  rm -rf /var/lib/apt/lists/*

NVIDIA_DRIVER_CAPABILITIES=compute,utility NVIDIA_REQUIRE_CUDA=cuda>=11.8 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511 brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511 brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511 brand=titanrtx,driver>=510,driver<511 brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=515,driver<516 brand=geforcertx,driver>=515,driver<516 brand=quadro,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516 brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=515,driver<516 NVIDIA_VISIBLE_DEVICES=all

apt-get update -y &&
  apt-get upgrade -y &&
  rm -rf /var/lib/apt/lists/*

ln -s /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/targets/x86_64-linux/lib /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/lib64 &&
  ln -s /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/lib64

HPCX_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14 HPCX_HCOLL_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/hcoll HPCX_MPI_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi HPCX_NCCL_RDMA_SHARP_PLUGIN_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/nccl_rdma_sharp_plugin HPCX_OSHMEM_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi HPCX_SHARP_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/sharp HPCX_UCX_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ucx/mt LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/hcoll/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/nccl_rdma_sharp_plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/sharp/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ucx/mt/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ucx/mt/lib/ucx:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nccl/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/lib64:: MPI_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi OMPI_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi OPAL_PREFIX=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi OSHMEM_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/hcoll/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ucx/mt/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin SHMEM_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/ompi

ln -s libnccl.so.2.18.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nccl/lib/libnccl.so &&
  ln -s libnccl.so.2.18.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nccl/lib/libnccl.so.2 &&
  ln -s libnvshmem_host.so.3.2.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/libnvshmem_host.so &&
  ln -s libnvshmem_host.so.3.2.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/libnvshmem_host.so.3 &&
  ln -s nvshmem_bootstrap_mpi.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_mpi.so &&
  ln -s nvshmem_bootstrap_mpi.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_mpi.so.3 &&
  ln -s nvshmem_bootstrap_pmi.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_pmi.so &&
  ln -s nvshmem_bootstrap_pmi.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_pmi.so.3 &&
  ln -s nvshmem_bootstrap_pmi2.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_pmi2.so &&
  ln -s nvshmem_bootstrap_pmi2.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_pmi2.so.3 &&
  ln -s nvshmem_bootstrap_pmix.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_pmix.so &&
  ln -s nvshmem_bootstrap_pmix.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_pmix.so.3 &&
  ln -s nvshmem_bootstrap_shmem.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_shmem.so &&
  ln -s nvshmem_bootstrap_shmem.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_shmem.so.3 &&
  ln -s nvshmem_bootstrap_uid.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_uid.so &&
  ln -s nvshmem_bootstrap_uid.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_bootstrap_uid.so.3 &&
  ln -s nvshmem_transport_ibdevx.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_ibdevx.so &&
  ln -s nvshmem_transport_ibdevx.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_ibdevx.so.3 &&
  ln -s nvshmem_transport_ibgda.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_ibgda.so &&
  ln -s nvshmem_transport_ibgda.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_ibgda.so.3 &&
  ln -s nvshmem_transport_ibrc.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_ibrc.so &&
  ln -s nvshmem_transport_ibrc.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_ibrc.so.3 &&
  ln -s nvshmem_transport_libfabric.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_libfabric.so &&
  ln -s nvshmem_transport_libfabric.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_libfabric.so.3 &&
  ln -s nvshmem_transport_ucx.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_ucx.so &&
  ln -s nvshmem_transport_ucx.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/nvshmem/lib/nvshmem_transport_ucx.so.3 &&
  ln -s libqd.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqd.so &&
  ln -s libqd.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqd.so.0 &&
  ln -s libqd_f_main.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqd_f_main.so &&
  ln -s libqd_f_main.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqd_f_main.so.0 &&
  ln -s libqdmod.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqdmod.so &&
  ln -s libqdmod.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqdmod.so.0 &&
  ln -s libblas_ilp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libblas_ilp64.so &&
  ln -s libblas_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libblas.so &&
  ln -s libblas_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libblas.so.0 &&
  ln -s libblas_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libblas_lp64.so &&
  ln -s liblapack_ilp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/liblapack_ilp64.so &&
  ln -s liblapack_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/liblapack.so &&
  ln -s liblapack_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/liblapack.so.0 &&
  ln -s liblapack_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/liblapack_lp64.so &&
  ln -s libnvc.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libpgc.so &&
  ln -s libnvcpumath.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libpgm.so &&
  ln -s libnvcpumath.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libpgmath.so &&
  ln -s libnvomp.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libgomp.so &&
  ln -s libnvomp.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libgomp.so.1 &&
  ln -s libnvomp.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libomp.so &&
  ln -s libcupti.so.2022.3.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/extras/CUPTI/lib64/libcupti.so &&
  ln -s libcupti.so.2022.3.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/extras/CUPTI/lib64/libcupti.so.11.8 &&
  ln -s libnvvm.so.4.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/nvvm/lib64/libnvvm.so &&
  ln -s libnvvm.so.4.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/nvvm/lib64/libnvvm.so.4 &&
  ln -s libOpenCL.so.1.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/targets/x86_64-linux/lib/libOpenCL.so &&
  ln -s libOpenCL.so.1.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/targets/x86_64-linux/lib/libOpenCL.so.1 &&
  ln -s libOpenCL.so.1.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/targets/x86_64-linux/lib/libOpenCL.so.1.0 &&
  ln -s libcudart.so.11.8.89 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/targets/x86_64-linux/lib/libcudart.so &&
  ln -s libcudart.so.11.8.89 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/targets/x86_64-linux/lib/libcudart.so.11.0 &&
  ln -s libnvToolsExt.so.1.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/targets/x86_64-linux/lib/libnvToolsExt.so &&
  ln -s libnvToolsExt.so.1.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/11.8/targets/x86_64-linux/lib/libnvToolsExt.so.1 &&
  ln -s libcal.so.0.4.3.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcal.so &&
  ln -s libcal.so.0.4.3.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcal.so.0 &&
  ln -s libcublas.so.11.11.4.17 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcublas.so &&
  ln -s libcublas.so.11.11.4.17 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcublas.so.11 &&
  ln -s libcublasLt.so.11.11.4.17 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcublasLt.so &&
  ln -s libcublasLt.so.11.11.4.17 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcublasLt.so.11 &&
  ln -s libcublasmp.so.0.2.1 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcublasmp.so &&
  ln -s libcublasmp.so.0.2.1 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcublasmp.so.0 &&
  ln -s libcufft.so.10.9.0.58 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcufft.so &&
  ln -s libcufft.so.10.9.0.58 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcufft.so.10 &&
  ln -s libcufftw.so.10.9.0.58 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcufftw.so &&
  ln -s libcufftw.so.10.9.0.58 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcufftw.so.10 &&
  ln -s libcurand.so.10.3.0.86 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcurand.so &&
  ln -s libcurand.so.10.3.0.86 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcurand.so.10 &&
  ln -s libcusolver.so.11.4.1.48 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcusolver.so &&
  ln -s libcusolver.so.11.4.1.48 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcusolver.so.11 &&
  ln -s libcusolverMg.so.11.4.1.48 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcusolverMg.so &&
  ln -s libcusolverMg.so.11.4.1.48 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcusolverMg.so.11 &&
  ln -s libcusolverMp.so.0.5.1.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcusolverMp.so &&
  ln -s libcusolverMp.so.0.5.1.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcusolverMp.so.0 &&
  ln -s libcusparse.so.11.7.5.86 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcusparse.so &&
  ln -s libcusparse.so.11.7.5.86 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcusparse.so.11 &&
  ln -s libcutensor.so.2.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcutensor.so &&
  ln -s libcutensor.so.2.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcutensor.so.2 &&
  ln -s libcutensorMg.so.2.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcutensorMg.so &&
  ln -s libcutensorMg.so.2.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libcutensorMg.so.2 &&
  ln -s libnvblas.so.11.11.4.17 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libnvblas.so &&
  ln -s libnvblas.so.11.11.4.17 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/11.8/targets/x86_64-linux/lib/libnvblas.so.11

/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/11.8/hpcx/hpcx-2.14/