/bin/bash -c #(nop) ENV BUILD_MPI=false BUILD_MPI_CPU=false BUILD_MPI_HCOLL=false BUILD_MPI_VERSION=4.1.1 BUILD_UCC=false BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=aarch64 CUDA_CC=80 DISABLE_DFTI=true GCC_FIX=false GCC_VERSION= HPCX_VERSION=2.19 MOFED_VERSION=24.01-0.3.3.1 MPI_TYPE=openmpi PATCH_DBCSR_SYNC=false PATCH_PERF=false REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA=true USE_COSMA_NCCL=false USE_COSMA_TOOLCHAIN=false USE_CPU_NATIVE=true USE_CUSOLVERMP=false USE_ELPA=true USE_GDRCOPY=false USE_HIP=false USE_HPCX=true USE_MKL=false USE_MULTI=false USE_MYPDGEMM=false USE_NVPL=true USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_INTEGRATE=false USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true __dummy=

BUILD_MPI_CPU=false BUILD_MPI_VERSION=4.1.1 BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=aarch64 CUDA_CC=80 DISABLE_DFTI=true GCC_VERSION= HPCX_VERSION=2.19 REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA_NCCL=false USE_CPU_NATIVE=true USE_MKL=false USE_NVPL=true USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true /bin/bash -c echo "Benchmarks: in /opt/cp2k/benchmarks/" &&
  echo &&
  echo " Run 'mysub run-cp2k.sh <jobname> 1 X*Y MPI_PER_GPU=X TESTCASE=<testcase>'" &&
  echo "        with: X=number of MPI ranks per GPU, 2-4 typically performs best" &&
  echo "              Y=number of GPUs on your system" &&
  echo "              testcase to choose in the list below." &&
  echo &&
  echo "Test cases:" &&
  echo &&
  echo "  1) Linear Scaling SCF, a benchmark that is CPU and H2D bound:" &&
  echo "       in ./QS_DM_LS/ :" &&
  echo "             - H2O-dft-ls.NREP2.inp     (small, 16 GB total, eg. run on 1xV100) " &&
  echo "             - H2O-dft-ls.NREP4.inp     (medium, 160 GB total, eg. run on 4xA100) " &&
  echo "             - you can also adjust NREP in the test case header to change the size. in general size= NREP^3 * 2.5 GB" &&
  echo "example: cd /opt/cp2k/benchmarks/QS_DM_LS && mysub run-cp2k.sh test 1 2 MPI_PER_GPU=2 TESTCASE=H2O-dft-ls.NREP2.inp" &&
  echo &&
  echo "  2) H2O Random Phase Approximation (H2O-RPA), a benchmark that is FLOP-bound and uses the COSMA library:" &&
  echo "       in QS_mp2_rpa/ :" &&
  echo "             - 32-H2O/H2O-32-RI-dRPA-TZ.inp    (1-8 GPUs)" &&
  echo "             - 64-H2O/H2O-64-RI-dRPA-TZ.inp    (4-128 GPUs)" &&
  echo "             - 128-H2O/H2O-128-RI-dRPA-TZ.inp    (8-1024 GPUs)" &&
  echo "example: cd /opt/cp2k/benchmarks/QS_mp2_rpa/64-H2O/ && mysub run-cp2k.sh test 1 64 MPI_PER_GPU=8 TESTCASE=H2O-64-RI-dRPA-TZ.inp" &&
  echo &&
  echo "  3) Hartree-Fock Exchange LiH  (HFX-LiH), a benchmark that is entirely CPU-bound (no GPU):" &&
  echo "             - ./QS_LiH_HFX/input_bulk_HFX_3.inp (1 hour on 1 node)" &&
  echo "             - NOT AN OFFICIAL BENCHMARK: ./QS_LiH_HFX_mini/input_bulk_HFX_3.inp (10 minutes)" &&
  echo 'example: cd /opt/cp2k/benchmarks/QS_LiH_HFX/ && mysub run-cp2k.sh test 1 64 MPI_PER_GPU=8 TESTCASE=input_bulk_HFX_3.inp'

/bin/bash -c #(nop) ENV CAL_UCC_TLS=^sharp,nccl

/bin/bash -c #(nop) ENV LD_LIBRARY_PATH=/opt/pdsyevd_gpu/:/opt/cusolvermp/build/:/opt/cusolvermp/build/libcal/:/opt/mypdgemm/:/opt/pdsyevd_gpu:/opt/cusolvermp/build/:/opt/cusolvermp/build/libcal:/opt/cusolvermp/lib:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/comm_libs/12.5/nccl/lib:/opt/cp2k/tools/toolchain/install/openblas/lib/:/opt/elpa/lib:/opt/fftw3/lib/:/opt/ScaLAPACK/lib/:/opt/hpcx/hpcx-v2.19-gcc-MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-aarch64/hcoll/lib/:/opt/ucc/lib:/opt/ucc/lib/ucc:/opt/ompi/lib/:/usr/local/cuda/lib64:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/comm_libs/nvshmem/lib:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/comm_libs/nccl/lib:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/math_libs/lib64:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/lib64: LIBRARY_PATH=/opt/pdsyevd_gpu/:/opt/cusolvermp/build/:/opt/cusolvermp/build/libcal/:/opt/mypdgemm/:/opt/OpenBLAS/lib/:/opt/pdsyevd_gpu:/opt/cusolvermp/build/:/opt/cusolvermp/build/libcal:/opt/cusolvermp/lib:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/comm_libs/12.5/nccl/lib:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7//cuda/lib64/stubs:/opt/cp2k/tools/toolchain/install/openblas/lib/:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7//cuda/lib64/stubs:/opt/elpa/lib:/opt/fftw3/lib/:/opt/ScaLAPACK/lib/:/opt/hpcx/hpcx-v2.19-gcc-MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-aarch64/hcoll/lib/:/opt/ucc/lib:/opt/ucc/lib/ucc:/usr/local/gdrcopy/lib64:/opt/ompi/lib/:/usr/local/cuda/lib64:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/lib64/stubs:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/comm_libs/nvshmem/lib:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/comm_libs/nccl/lib:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/math_libs/lib64:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/lib64: CPATH=/opt/cusolvermp/src/:/opt/cusolvermp/Thirdparty/libcal/src/:/opt/cusolvermp/build/../src/://opt/cusolvermp/build/../Thirdparty/libcal/src/:/opt/cusolvermp/include/:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/comm_libs/12.5/nccl/include:/opt/cp2k/tools/toolchain/install/openblas/include/:/opt/ucc/:/usr/local/gdrcopy/include:/usr/local/knem/include:/opt/ompi/include/:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/12.5/targets/aarch64-linux/include/:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/math_libs/12.5/targets/aarch64-linux/include/:/opt/nvidia/hpc_sdk/Linux_aarch64/24.7/comm_libs/12.5/targets/aarch64-linux/include/:

/bin/bash -c #(nop) WORKDIR /opt/cp2k/benchmarks/QS_mp2_rpa/64-H2O

BUILD_MPI_CPU=false BUILD_MPI_VERSION=4.1.1 BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=aarch64 CUDA_CC=80 DISABLE_DFTI=true GCC_VERSION= HPCX_VERSION=2.19 REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA_NCCL=false USE_CPU_NATIVE=true USE_MKL=false USE_NVPL=true USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true /bin/bash -c cp -rp /opt/cp2k/benchmarks/QS_LiH_HFX /opt/cp2k/benchmarks/QS_LiH_HFX_mini

/bin/bash -c #(nop) WORKDIR /opt/cp2k/benchmarks/QS_DM_LS

/bin/bash -c #(nop) ENV CP2K_DATA_DIR=/opt/cp2k/data/

BUILD_MPI_CPU=false BUILD_MPI_VERSION=4.1.1 BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=aarch64 CUDA_CC=80 DISABLE_DFTI=true GCC_VERSION= HPCX_VERSION=2.19 REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA_NCCL=false USE_CPU_NATIVE=true USE_MKL=false USE_NVPL=true USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true /bin/bash -c chmod +x /opt/cp2k/arch/local_cuda.psmp

BUILD_MPI_CPU=false BUILD_MPI_VERSION=4.1.1 BUILD_UCX_VERSION=1.17.x CMAKE_VERSION=apt-dev COSMA_WITH_GPU_AWARE_MPI=true CP2K_USE_PTX=true CPUARCH=aarch64 CUDA_CC=80 DISABLE_DFTI=true GCC_VERSION= HPCX_VERSION=2.19 REBUILD_UCX=false UBU_VERSION=22.04 USE_ACC_MANAGED=false USE_AOCL=false USE_ARMPL=false USE_COSMA_NCCL=false USE_CPU_NATIVE=true USE_MKL=false USE_NVPL=true USE_OPENBLAS=false USE_OPTIMIZED_SCALAPACK=true USE_PATCHED_PWALLOC=false USE_PLUMED=false USE_PWFFT=true USE_SCALAPACK_THREADED=true /bin/bash -c if nm /opt/cosma/lib/libcosma_prefixed_pxgemm.a | grep cosma_pdgem -q; then bash -c 'cd /opt/cp2k &&   . /opt/cp2k/tools/toolchain/install/setup &&   rm exe/local_cuda/cp2k.psmp && for file in $(find -name cp_fm_basic_linalg.o); do objcopy --redefine-sym pdgemm_=cosma_pdgemm_ $file; done && make ARCH=local_cuda VERSION=psmp'; fi