Modifying NVHPC/22.7 module for openmpi4 in VEGA

User side modulefiles with lmod and NVHPC module with openmpi4 instead of openmpi3

lmod is slight different than environment-modules. In order to modify the module search path use:

$ module use /path/to/personal/modulefiles

Add this to your startup script if you want this to be permanent.

I just copied the system-wide NVHPC/22.7 and modified for openmpi4 (by replacing default mpi symlink directories to proper openmpi4 variants)

#%Module1.0

# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
#
# NVIDIA CORPORATION and its licensors retain all intellectual property
# and proprietary rights in and to this software, related documentation
# and any modifications thereto.  Any use, reproduction, disclosure or
# distribution of this software and related documentation without an express
# license agreement from NVIDIA CORPORATION is strictly prohibited.

conflict nvhpc
conflict nvhpc-nompi
conflict nvhpc-byo-compiler

set nvhome /ceph/hpc/software/nvidia/hpc_sdk
set target Linux_x86_64
set version 22.7

set nvcudadir $nvhome/$target/$version/cuda
set nvcompdir $nvhome/$target/$version/compilers
set nvmathdir $nvhome/$target/$version/math_libs
set nvcommdir $nvhome/$target/$version/comm_libs

setenv NVHPC $nvhome
setenv NVHPC_ROOT $nvhome/$target/$version
setenv CC $nvcompdir/bin/nvc
setenv CXX $nvcompdir/bin/nvc++
setenv FC $nvcompdir/bin/nvfortran
setenv F90 $nvcompdir/bin/nvfortran
setenv F77 $nvcompdir/bin/nvfortran
setenv CPP cpp

prepend-path PATH $nvcudadir/bin
prepend-path PATH $nvcompdir/bin
prepend-path PATH $nvcommdir/openmpi4/openmpi-4.0.5/bin/
prepend-path PATH $nvcompdir/extras/qd/bin

prepend-path LD_LIBRARY_PATH $nvcudadir/lib64
prepend-path LD_LIBRARY_PATH $nvcudadir/extras/CUPTI/lib64
prepend-path LD_LIBRARY_PATH $nvcompdir/extras/qd/lib
prepend-path LD_LIBRARY_PATH $nvcompdir/lib
prepend-path LD_LIBRARY_PATH $nvmathdir/lib64
prepend-path LD_LIBRARY_PATH $nvcommdir/openmpi4/openmpi-4.0.5/lib/
prepend-path LD_LIBRARY_PATH $nvcommdir/nccl/lib
prepend-path LD_LIBRARY_PATH $nvcommdir/nvshmem/lib

prepend-path CPATH $nvmathdir/include
prepend-path CPATH $nvcommdir/openmpi4/openmpi-4.0.5/include/
prepend-path CPATH $nvcommdir/nccl/include
prepend-path CPATH $nvcommdir/nvshmem/include
prepend-path CPATH $nvcompdir/extras/qd/include/qd

prepend-path MANPATH $nvcompdir/man

setenv OPAL_PREFIX $nvcommdir/openmpi4/openmpi-4.0.5