Add fixes for Delta PrgEnv-nvidia

Author: wilfonba

CMakeLists.txt

@@ -238,6 +238,9 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
     if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
         if (MFC_Unified)
             message(STATUS "LTO/IPO is not available with NVHPC using Unified Memory")
+        elseif (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER "24.11" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "25.9")
+            message(STATUS "LTO/IPO is not supported in NVHPC Version 24.11 to 25.9. Use >=25.9 or (<=24.11 && > 23.11) Performance will be degraded.")
+            set(NVHPC_USE_TWO_PASS_IPO FALSE)
         elseif(CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "23.11")
             message(STATUS "LTO/IPO is not supported in NVHPC Version < 23.11. Use a newer version of NVHPC for best performance.")
         else()

toolchain/modules

@@ -57,9 +57,9 @@ tuo-gpu craype-accel-amd-gfx942
 tuo-gpu HSA_XNACK=0
 
 d     NCSA Delta
-d-all python/3.11.6
+d-all python
 d-cpu gcc/11.4.0 openmpi
-d-gpu nvhpc/24.1 cuda/12.3.0 openmpi/4.1.5+cuda cmake
+d-gpu PrgEnv-nvidia/8.6.0
 d-gpu CC=nvc CXX=nvc++ FC=nvfortran
 d-gpu MFC_CUDA_CC=80,86
 

toolchain/templates/delta.mako

@@ -36,8 +36,9 @@ cd "${MFC_ROOT_DIR}"
 cd - > /dev/null
 echo
 
-# Fixes Delta not being able to find core library file
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/spack/deltas11-2023-03/apps/linux-rhel8-zen3/nvhpc-24.1/openmpi-4.1.5-zkiklxi/lib/
+% if gpu_enabled:
+    export MPICH_GPU_SUPPORT_ENABLED=0 # Disable GPU-Direct MPI
+% endif
 
 % for target in targets:
     ${helpers.run_prologue(target)}