—

BUILD_MPI_CPU=false BUILD_MPI_VERSION=4.1.1 BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=x86_64 CUDA_CC=80 DISABLE_DFTI=true GCC_VERSION= HPCX_VERSION=2.19 REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA_NCCL=false USE_CPU_NATIVE=true USE_MKL=true USE_NVPL=false USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true /bin/bash -c echo "Benchmarks: in /opt/cp2k/benchmarks/" &&
  echo &&
  echo " Run 'mysub run-cp2k.sh <jobname> 1 X*Y MPI_PER_GPU=X TESTCASE=<testcase>'" &&
  echo "        with: X=number of MPI ranks per GPU, 2-4 typically performs best" &&
  echo "              Y=number of GPUs on your system" &&
  echo "              testcase to choose in the list below." &&
  echo &&
  echo "Test cases:" &&
  echo &&
  echo "  1) Linear Scaling SCF, a benchmark that is CPU and H2D bound:" &&
  echo "       in ./QS_DM_LS/ :" &&
  echo "             - H2O-dft-ls.NREP2.inp     (small, 16 GB total, eg. run on 1xV100) " &&
  echo "             - H2O-dft-ls.NREP4.inp     (medium, 160 GB total, eg. run on 4xA100) " &&
  echo "             - you can also adjust NREP in the test case header to change the size. in general size= NREP^3 * 2.5 GB" &&
  echo "example: cd /opt/cp2k/benchmarks/QS_DM_LS && mysub run-cp2k.sh test 1 2 MPI_PER_GPU=2 TESTCASE=H2O-dft-ls.NREP2.inp" &&
  echo &&
  echo "  2) H2O Random Phase Approximation (H2O-RPA), a benchmark that is FLOP-bound and uses the COSMA library:" &&
  echo "       in QS_mp2_rpa/ :" &&
  echo "             - 32-H2O/H2O-32-RI-dRPA-TZ.inp    (1-8 GPUs)" &&
  echo "             - 64-H2O/H2O-64-RI-dRPA-TZ.inp    (4-128 GPUs)" &&
  echo "             - 128-H2O/H2O-128-RI-dRPA-TZ.inp    (8-1024 GPUs)" &&
  echo "example: cd /opt/cp2k/benchmarks/QS_mp2_rpa/64-H2O/ && mysub run-cp2k.sh test 1 64 MPI_PER_GPU=8 TESTCASE=H2O-64-RI-dRPA-TZ.inp" &&
  echo &&
  echo "  3) Hartree-Fock Exchange LiH  (HFX-LiH), a benchmark that is entirely CPU-bound (no GPU):" &&
  echo "             - ./QS_LiH_HFX/input_bulk_HFX_3.inp (1 hour on 1 node)" &&
  echo "             - NOT AN OFFICIAL BENCHMARK: ./QS_LiH_HFX_mini/input_bulk_HFX_3.inp (10 minutes)" &&
  echo 'example: cd /opt/cp2k/benchmarks/QS_LiH_HFX/ && mysub run-cp2k.sh test 1 64 MPI_PER_GPU=8 TESTCASE=input_bulk_HFX_3.inp'

/bin/bash -c #(nop) ENV CAL_UCC_TLS=^sharp,nccl

/bin/bash -c #(nop) ENV LD_LIBRARY_PATH=/opt/pdsyevd_gpu/:/opt/cusolvermp/build/:/opt/cusolvermp/build/libcal/:/opt/mypdgemm/:/opt/pdsyevd_gpu:/opt/cusolvermp/build/:/opt/cusolvermp/build/libcal:/opt/cusolvermp/lib:/opt/cp2k/tools/toolchain/install/openblas/lib/:/opt/elpa/lib:/opt/fftw3/lib/:/opt/ScaLAPACK/lib/:/opt/libmklfake/:/opt/intel/compilers_and_libraries_2024.1/linux/tbb/lib/gcc4.8:/opt/intel/compilers_and_libraries_2024.1/linux/compiler/lib/:/opt/intel/compilers_and_libraries_2024.1/linux/mkl/lib/:/opt/intel/compilers_and_libraries_2024.1/linux/mkl/latest/lib/:/opt/hpcx/hpcx-v2.19-gcc-MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-x86_64/hcoll/lib/:/opt/ucc/lib:/opt/ucc/lib/ucc:/opt/ompi/lib/:/usr/local/cuda/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/comm_libs/nvshmem/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/comm_libs/nccl/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/math_libs/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/cuda/lib64: LIBRARY_PATH=/opt/pdsyevd_gpu/:/opt/cusolvermp/build/:/opt/cusolvermp/build/libcal/:/opt/mypdgemm/:/opt/OpenBLAS/lib/:/opt/pdsyevd_gpu:/opt/cusolvermp/build/:/opt/cusolvermp/build/libcal:/opt/cusolvermp/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7//cuda/lib64/stubs:/opt/cp2k/tools/toolchain/install/openblas/lib/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7//cuda/lib64/stubs:/opt/elpa/lib:/opt/fftw3/lib/:/opt/ScaLAPACK/lib/:/opt/libmklfake/:/opt/intel/compilers_and_libraries_2024.1/linux/tbb/lib/gcc4.8:/opt/intel/compilers_and_libraries_2024.1/linux/compiler/lib/:/opt/intel/compilers_and_libraries_2024.1/linux/mkl/lib/:/opt/intel/compilers_and_libraries_2024.1/linux/mkl/latest/lib/:/opt/hpcx/hpcx-v2.19-gcc-MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-x86_64/hcoll/lib/:/opt/ucc/lib:/opt/ucc/lib/ucc:/usr/local/gdrcopy/lib64:/opt/ompi/lib/:/usr/local/cuda/lib64:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/lib64/stubs:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/comm_libs/nvshmem/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/comm_libs/nccl/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/math_libs/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/cuda/lib64: CPATH=/opt/cusolvermp/src/:/opt/cusolvermp/Thirdparty/libcal/src/:/opt/cusolvermp/build/../src/://opt/cusolvermp/build/../Thirdparty/libcal/src/:/opt/cusolvermp/include/:/opt/cp2k/tools/toolchain/install/openblas/include/:/opt/intel/compilers_and_libraries_2024.1/linux/mkl/include:/opt/intel/compilers_and_libraries_2024.1/linux/mkl/latest/include:/opt/ucc/:/usr/local/gdrcopy/include:/usr/local/knem/include:/opt/ompi/include/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/cuda/12.5/targets/x86_64-linux/include/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/math_libs/12.5/targets/x86_64-linux/include/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.7/comm_libs/12.5/targets/x86_64-linux/include/:

/bin/bash -c #(nop) WORKDIR /opt/cp2k/benchmarks/QS_mp2_rpa/64-H2O

BUILD_MPI_CPU=false BUILD_MPI_VERSION=4.1.1 BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=x86_64 CUDA_CC=80 DISABLE_DFTI=true GCC_VERSION= HPCX_VERSION=2.19 REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA_NCCL=false USE_CPU_NATIVE=true USE_MKL=true USE_NVPL=false USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true /bin/bash -c cp -rp /opt/cp2k/benchmarks/QS_LiH_HFX /opt/cp2k/benchmarks/QS_LiH_HFX_mini

/bin/bash -c #(nop) WORKDIR /opt/cp2k/benchmarks/QS_DM_LS

/bin/bash -c #(nop) ENV CP2K_DATA_DIR=/opt/cp2k/data/

BUILD_MPI_CPU=false BUILD_MPI_VERSION=4.1.1 BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=x86_64 CUDA_CC=80 DISABLE_DFTI=true GCC_VERSION= HPCX_VERSION=2.19 REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA_NCCL=false USE_CPU_NATIVE=true USE_MKL=true USE_NVPL=false USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true /bin/bash -c chmod +x /opt/cp2k/arch/local_cuda.psmp

BUILD_MPI_CPU=false BUILD_MPI_VERSION=4.1.1 BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=x86_64 CUDA_CC=80 DISABLE_DFTI=true GCC_VERSION= HPCX_VERSION=2.19 REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA_NCCL=false USE_CPU_NATIVE=true USE_MKL=true USE_NVPL=false USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true /bin/bash -c if nm /opt/cosma/lib/libcosma_prefixed_pxgemm.a | grep cosma_pdgem -q; then bash -c 'cd /opt/cp2k &&   . /opt/cp2k/tools/toolchain/install/setup &&   rm exe/local_cuda/cp2k.psmp && for file in $(find -name cp_fm_basic_linalg.o); do objcopy --redefine-sym pdgemm_=cosma_pdgemm_ $file; done && make ARCH=local_cuda VERSION=psmp'; fi