—

echo "/usr/local/gdrcopy/lib" >> /etc/ld.so.conf.d/hpccm.conf &&
  ldconfig

/usr/local/gdrcopy /usr/local/gdrcopy

apt-get update -y &&
  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends -t jammy dapl2-utils ibutils ibverbs-providers ibverbs-utils infiniband-diags libdapl-dev libdapl2 libibmad-dev libibmad5 libibverbs-dev libibverbs1 librdmacm-dev librdmacm1 rdmacm-utils &&
  rm -rf /var/lib/apt/lists/*

NVIDIA_DRIVER_CAPABILITIES=compute,utility NVIDIA_REQUIRE_CUDA=cuda>=12.9 brand=unknown,driver>=535,driver<536 brand=grid,driver>=535,driver<536 brand=tesla,driver>=535,driver<536 brand=nvidia,driver>=535,driver<536 brand=quadro,driver>=535,driver<536 brand=quadrortx,driver>=535,driver<536 brand=nvidiartx,driver>=535,driver<536 brand=vapps,driver>=535,driver<536 brand=vpc,driver>=535,driver<536 brand=vcs,driver>=535,driver<536 brand=vws,driver>=535,driver<536 brand=cloudgaming,driver>=535,driver<536 brand=unknown,driver>=550,driver<551 brand=grid,driver>=550,driver<551 brand=tesla,driver>=550,driver<551 brand=nvidia,driver>=550,driver<551 brand=quadro,driver>=550,driver<551 brand=quadrortx,driver>=550,driver<551 brand=nvidiartx,driver>=550,driver<551 brand=vapps,driver>=550,driver<551 brand=vpc,driver>=550,driver<551 brand=vcs,driver>=550,driver<551 brand=vws,driver>=550,driver<551 brand=cloudgaming,driver>=550,driver<551 brand=unknown,driver>=560,driver<561 brand=grid,driver>=560,driver<561 brand=tesla,driver>=560,driver<561 brand=nvidia,driver>=560,driver<561 brand=quadro,driver>=560,driver<561 brand=quadrortx,driver>=560,driver<561 brand=nvidiartx,driver>=560,driver<561 brand=vapps,driver>=560,driver<561 brand=vpc,driver>=560,driver<561 brand=vcs,driver>=560,driver<561 brand=vws,driver>=560,driver<561 brand=cloudgaming,driver>=560,driver<561 brand=unknown,driver>=565,driver<566 brand=grid,driver>=565,driver<566 brand=tesla,driver>=565,driver<566 brand=nvidia,driver>=565,driver<566 brand=quadro,driver>=565,driver<566 brand=quadrortx,driver>=565,driver<566 brand=nvidiartx,driver>=565,driver<566 brand=vapps,driver>=565,driver<566 brand=vpc,driver>=565,driver<566 brand=vcs,driver>=565,driver<566 brand=vws,driver>=565,driver<566 brand=cloudgaming,driver>=565,driver<566 brand=unknown,driver>=570,driver<571 brand=grid,driver>=570,driver<571 brand=tesla,driver>=570,driver<571 brand=nvidia,driver>=570,driver<571 brand=quadro,driver>=570,driver<571 brand=quadrortx,driver>=570,driver<571 brand=nvidiartx,driver>=570,driver<571 brand=vapps,driver>=570,driver<571 brand=vpc,driver>=570,driver<571 brand=vcs,driver>=570,driver<571 brand=vws,driver>=570,driver<571 brand=cloudgaming,driver>=570,driver<571 NVIDIA_VISIBLE_DEVICES=all

apt-get update -y &&
  apt-get upgrade -y &&
  rm -rf /var/lib/apt/lists/*

ln -s /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/targets/x86_64-linux/lib /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/lib64 &&
  ln -s /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/lib64

HPCX_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1 HPCX_HCOLL_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/hcoll HPCX_MPI_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi HPCX_NCCL_RDMA_SHARP_PLUGIN_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/nccl_rdma_sharp_plugin HPCX_OSHMEM_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi HPCX_SHARP_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/sharp HPCX_UCX_DIR=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ucx/mt LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/hcoll/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/nccl_rdma_sharp_plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/sharp/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ucx/mt/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ucx/mt/lib/ucx:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nccl/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/lib64:: MPI_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi OMPI_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi OPAL_PREFIX=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi OSHMEM_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/hcoll/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ucx/mt/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin SHMEM_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/ompi

ln -s libnccl.so.2.18.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nccl-2.18/lib/libnccl.so &&
  ln -s libnccl.so.2.18.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nccl-2.18/lib/libnccl.so.2 &&
  ln -s libnccl.so.2.26.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nccl-2.26/lib/libnccl.so &&
  ln -s libnccl.so.2.26.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nccl-2.26/lib/libnccl.so.2 &&
  ln -s libnvshmem_host.so.3.2.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/libnvshmem_host.so &&
  ln -s libnvshmem_host.so.3.2.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/libnvshmem_host.so.3 &&
  ln -s nvshmem_bootstrap_mpi.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_mpi.so &&
  ln -s nvshmem_bootstrap_mpi.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_mpi.so.3 &&
  ln -s nvshmem_bootstrap_pmi.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_pmi.so &&
  ln -s nvshmem_bootstrap_pmi.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_pmi.so.3 &&
  ln -s nvshmem_bootstrap_pmi2.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_pmi2.so &&
  ln -s nvshmem_bootstrap_pmi2.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_pmi2.so.3 &&
  ln -s nvshmem_bootstrap_pmix.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_pmix.so &&
  ln -s nvshmem_bootstrap_pmix.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_pmix.so.3 &&
  ln -s nvshmem_bootstrap_shmem.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_shmem.so &&
  ln -s nvshmem_bootstrap_shmem.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_shmem.so.3 &&
  ln -s nvshmem_bootstrap_uid.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_uid.so &&
  ln -s nvshmem_bootstrap_uid.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_bootstrap_uid.so.3 &&
  ln -s nvshmem_transport_ibdevx.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_ibdevx.so &&
  ln -s nvshmem_transport_ibdevx.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_ibdevx.so.3 &&
  ln -s nvshmem_transport_ibgda.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_ibgda.so &&
  ln -s nvshmem_transport_ibgda.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_ibgda.so.3 &&
  ln -s nvshmem_transport_ibrc.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_ibrc.so &&
  ln -s nvshmem_transport_ibrc.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_ibrc.so.3 &&
  ln -s nvshmem_transport_libfabric.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_libfabric.so &&
  ln -s nvshmem_transport_libfabric.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_libfabric.so.3 &&
  ln -s nvshmem_transport_ucx.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_ucx.so &&
  ln -s nvshmem_transport_ucx.so.3.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/nvshmem/lib/nvshmem_transport_ucx.so.3 &&
  ln -s libqd.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqd.so &&
  ln -s libqd.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqd.so.0 &&
  ln -s libqd_f_main.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqd_f_main.so &&
  ln -s libqd_f_main.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqd_f_main.so.0 &&
  ln -s libqdmod.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqdmod.so &&
  ln -s libqdmod.so.0.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/extras/qd/lib/libqdmod.so.0 &&
  ln -s libblas_ilp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libblas_ilp64.so &&
  ln -s libblas_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libblas.so &&
  ln -s libblas_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libblas.so.0 &&
  ln -s libblas_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libblas_lp64.so &&
  ln -s liblapack_ilp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/liblapack_ilp64.so &&
  ln -s liblapack_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/liblapack.so &&
  ln -s liblapack_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/liblapack.so.0 &&
  ln -s liblapack_lp64.so.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/liblapack_lp64.so &&
  ln -s libnvc.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libpgc.so &&
  ln -s libnvcpumath.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libpgm.so &&
  ln -s libnvcpumath.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libpgmath.so &&
  ln -s libnvomp.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libgomp.so &&
  ln -s libnvomp.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libgomp.so.1 &&
  ln -s libnvomp.so /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/compilers/lib/libomp.so &&
  ln -s libcuda.so.575.51.03 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/compat/libcuda.so &&
  ln -s libcuda.so.575.51.03 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/compat/libcuda.so.1 &&
  ln -s libcudadebugger.so.575.51.03 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/compat/libcudadebugger.so.1 &&
  ln -s libnvidia-nvvm.so.575.51.03 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/compat/libnvidia-nvvm.so.4 &&
  ln -s libnvidia-ptxjitcompiler.so.575.51.03 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/compat/libnvidia-ptxjitcompiler.so.1 &&
  ln -s libcupti.so.2025.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/extras/CUPTI/lib64/libcupti.so &&
  ln -s libcupti.so.2025.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/extras/CUPTI/lib64/libcupti.so.12 &&
  ln -s libnvvm.so.4.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/nvvm/lib64/libnvvm.so &&
  ln -s libnvvm.so.4.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/nvvm/lib64/libnvvm.so.4 &&
  ln -s libcudart.so.12.9.37 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/targets/x86_64-linux/lib/libcudart.so &&
  ln -s libcudart.so.12.9.37 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/targets/x86_64-linux/lib/libcudart.so.12 &&
  ln -s libnvJitLink.so.12.9.41 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/targets/x86_64-linux/lib/libnvJitLink.so &&
  ln -s libnvJitLink.so.12.9.41 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/targets/x86_64-linux/lib/libnvJitLink.so.12 &&
  ln -s libnvtx3interop.so.1.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/targets/x86_64-linux/lib/libnvtx3interop.so &&
  ln -s libnvtx3interop.so.1.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/cuda/12.9/targets/x86_64-linux/lib/libnvtx3interop.so.1 &&
  ln -s libcal.so.0.4.4.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcal.so &&
  ln -s libcal.so.0.4.4.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcal.so.0 &&
  ln -s libcublas.so.12.9.0.13 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcublas.so &&
  ln -s libcublas.so.12.9.0.13 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcublas.so.12 &&
  ln -s libcublasLt.so.12.9.0.13 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcublasLt.so &&
  ln -s libcublasLt.so.12.9.0.13 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcublasLt.so.12 &&
  ln -s libcublasmp.so.0.4.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcublasmp.so &&
  ln -s libcublasmp.so.0.4.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcublasmp.so.0 &&
  ln -s libcufft.so.11.4.0.6 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcufft.so &&
  ln -s libcufft.so.11.4.0.6 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcufft.so.11 &&
  ln -s libcufftMp.so.11.4.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcufftMp.so &&
  ln -s libcufftMp.so.11.4.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcufftMp.so.11 &&
  ln -s libcufftw.so.11.4.0.6 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcufftw.so &&
  ln -s libcufftw.so.11.4.0.6 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcufftw.so.11 &&
  ln -s libcurand.so.10.3.10.19 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcurand.so &&
  ln -s libcurand.so.10.3.10.19 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcurand.so.10 &&
  ln -s libcusolver.so.11.7.4.40 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcusolver.so &&
  ln -s libcusolver.so.11.7.4.40 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcusolver.so.11 &&
  ln -s libcusolverMg.so.11.7.4.40 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcusolverMg.so &&
  ln -s libcusolverMg.so.11.7.4.40 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcusolverMg.so.11 &&
  ln -s libcusolverMp.so.0.6.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcusolverMp.so &&
  ln -s libcusolverMp.so.0.6.0.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcusolverMp.so.0 &&
  ln -s libcusparse.so.12.5.9.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcusparse.so &&
  ln -s libcusparse.so.12.5.9.5 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcusparse.so.12 &&
  ln -s libcutensor.so.2.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcutensor.so &&
  ln -s libcutensor.so.2.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcutensor.so.2 &&
  ln -s libcutensorMg.so.2.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcutensorMg.so &&
  ln -s libcutensorMg.so.2.2.0 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libcutensorMg.so.2 &&
  ln -s libnvblas.so.12.9.0.13 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libnvblas.so &&
  ln -s libnvblas.so.12.9.0.13 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/math_libs/12.9/targets/x86_64-linux/lib/libnvblas.so.12

/opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1 /opt/nvidia/hpc_sdk/Linux_x86_64/25.5/comm_libs/12.9/hpcx/hpcx-2.22.1/