Add analytical Sedov shock solution

CMakeLists.txt

@@ -18,6 +18,11 @@ find_package(HYPRE REQUIRED)
 find_package(MFEM REQUIRED)
 find_package(MPI REQUIRED)
 
+option(LAGHOS_USE_CALIPER "Use Caliper" OFF)
+if (LAGHOS_USE_CALIPER)
+  find_package(caliper REQUIRED)
+endif()
+
 set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to use.")
 set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL
   "Force the use of the chosen C++ standard.")
@@ -53,7 +58,7 @@ endif()
 
 
 list(APPEND SOURCES
-  laghos_assembly.cpp laghos.cpp laghos_solver.cpp
+  laghos_assembly.cpp laghos.cpp laghos_solver.cpp sedov_sol.cpp
 )
 
 if (MFEM_USE_CUDA)
@@ -69,3 +74,24 @@ target_link_libraries(laghos
   PUBLIC MPI::MPI_CXX
   PUBLIC HYPRE::HYPRE
 )
+if(LAGHOS_USE_CALIPER)
+  target_link_libraries(laghos
+    PUBLIC caliper
+  )
+  target_compile_definitions(laghos
+    PUBLIC LAGHOS_USE_CALIPER)
+endif()
+
+if (MFEM_USE_CUDA)
+  set_source_files_properties(sedov_sol.cpp sedov/sedov.cpp PROPERTIES LANGUAGE CUDA)
+endif()
+
+add_executable(sedov sedov_sol.cpp sedov/sedov.cpp)
+
+target_include_directories(sedov PUBLIC "${MFEM_DIR}/../../../include/mfem")
+
+target_link_libraries(sedov
+  PUBLIC mfem
+  PUBLIC MPI::MPI_CXX
+  PUBLIC HYPRE::HYPRE
+)

README.md

@@ -128,15 +128,24 @@ Other computational motives in Laghos include the following:
 
 Laghos has the following external dependencies:
 
-- *hypre*, used for parallel linear algebra, we recommend version 2.11.2<br>
-  https://github.com/hypre-space/hypre/releases/tag/v2.11.2 
+- *hypre*, used for parallel linear algebra, we recommend version 2.31.0 or new<br>
+  https://github.com/hypre-space/hypre/releases/tag/v2.31.0
 
-- METIS, used for parallel domain decomposition (optional), we recommend [version 4.0.3](https://github.com/mfem/tpls/blob/gh-pages/metis-4.0.3.tar.gz) <br>
-  https://github.com/mfem/tpls
+- METIS, used for parallel domain decomposition (optional)
+  https://github.com/KarypisLab/METIS.git
 
 - MFEM, used for (high-order) finite element discretization, its GitHub master branch <br>
   https://github.com/mfem/mfem
 
+- Umpire, used for device memory pools in hypre and MFEM. This is only recommended for GPU-accelerated builds. (optional)<br>
+  https://github.com/LLNL/Umpire.git
+
+- CMake 3.24.0+ or GNU Make
+- C and C++17 compiler
+- MPI
+
+### Makefile
+
 To build the miniapp, first download *hypre* and METIS from the links above
 and put everything on the same level as the `Laghos` directory:
 ```sh
@@ -222,6 +231,172 @@ build respectively. See `make help` for additional options.
 See also the `make setup` target that can be used to automated the
 download and building of hypre, METIS and MFEM.
 
+### CMake
+
+This installs built dependencies to `INSTALLDIR` using the `CC` C-compiler and `CXX` C++17-compiler.
+
+Build METIS (optional):
+```sh
+git clone https://github.com/KarypisLab/METIS.git
+cd METIS
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=$CC -DCMAKE_INSTALL_PREFIX=$INSTALLDIR
+make -j install
+```
+For large runs (problem size above 2 billion unknowns), add `-DMETIS_USE_LONGINDEX=ON` option to the above `cmake` line. If building without METIS only Cartesian partitioning is supported.
+
+Build Umpire (CUDA, optional):
+```sh
+git clone https://github.com/LLNL/Umpire.git
+cd Umpire
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=native -DENABLE_CUDA=ON -DUMPIRE_ENABLE_C=ON -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_CUDA_COMPILER=$CUDACC
+make -j install
+```
+
+Build Umpire (HIP, optional):
+```sh
+git clone https://github.com/LLNL/Umpire.git
+cd Umpire
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_HIP_ARCHITECTURES=native -DENABLE_HIP=ON -DUMPIRE_ENABLE_C=ON -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_HIP_COMPILER=$HIPCC
+make -j install
+```
+
+Build *hypre* (CPU-only):
+
+```sh
+git clone https://github.com/hypre-space/hypre.git
+cd hypre/build
+cmake ../src -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX
+make -j install
+```
+
+Build *hypre* (CUDA):
+
+```sh
+git clone https://github.com/hypre-space/hypre.git
+cd hypre/build
+cmake ../src -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DHYPRE_ENABLE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_CUDA_COMPILER=$CUDACC -DHYPRE_ENABLE_GPU_AWARE_MPI=ON -DHYPRE_ENABLE_UMPIRE=ON
+make -j install
+```
+``HYPRE_ENABLE_GPU_AWARE_MPI`` and ``HYPRE_ENABLE_UMPIRE`` may be optionally turned off.
+
+Build *hypre* (HIP):
+
+```sh
+git clone https://github.com/hypre-space/hypre.git
+cd hypre/build
+cmake ../src -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DHYPRE_ENABLE_HIP=ON -DCMAKE_HIP_ARCHITECTURES=native -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_HIP_COMPILER=$HIPCC -DHYPRE_ENABLE_GPU_AWARE_MPI=ON -DHYPRE_ENABLE_UMPIRE=ON
+make -j install
+```
+``HYPRE_ENABLE_GPU_AWARE_MPI`` and ``HYPRE_ENABLE_UMPIRE`` may be optionally turned off.
+For large runs (problem size above 2 billion unknowns), enable the `HYPRE_ENABLE_MIXEDINT` option.
+
+Build MFEM (CPU-only):
+```sh
+git clone https://github.com/mfem/mfem.git
+cd mfem
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DHYPRE_DIR=$INSTALLDIR -DMETIS_DIR=$INSTALLDIR -DMFEM_USE_MPI=ON -DMFEM_USE_METIS=ON -DCMAKE_CXX_COMPILER=$CXX
+make -j install
+```
+`MFEM_USE_METIS` may be optionally disabled.
+See the [MFEM building page](http://mfem.org/building/) for additional details.
+
+Build MFEM (CUDA):
+```sh
+git clone https://github.com/mfem/mfem.git
+cd mfem
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DHYPRE_DIR=$INSTALLDIR -DMETIS_DIR=$INSTALLDIR -DMFEM_USE_MPI=ON -DMFEM_USE_METIS=ON -DMFEM_USE_CUDA=ON -DMFEM_USE_UMPIRE=ON -DCMAKE_CUDA_ARCHITECTURES=native -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_CUDA_COMPILER=$CUDACC -DUMPIRE_DIR=$INSTALLDIR
+make -j install
+```
+`MFEM_USE_METIS` and `MFEM_USE_UMPIRE may be optionally disabled.
+See the [MFEM building page](http://mfem.org/building/) for additional details.
+
+Build MFEM (HIP):
+```sh
+git clone https://github.com/mfem/mfem.git
+cd mfem
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DHYPRE_DIR=$INSTALLDIR -DMETIS_DIR=$INSTALLDIR -DMFEM_USE_MPI=ON -DMFEM_USE_METIS=ON -DMFEM_USE_HIP=ON -DMFEM_USE_UMPIRE=ON -DCMAKE_HIP_ARCHITECTURES=native -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_HIP_COMPILER=$HIPCC -DUMPIRE_DIR=$INSTALLDIR
+make -j install
+```
+`MFEM_USE_METIS` and `MFEM_USE_UMPIRE` may be optionally disabled.
+See the [MFEM building page](http://mfem.org/building/) for additional details.
+
+GLVis (optional):
+```sh
+git clone https://github.com/GLVis/glvis.git
+cd glvis
+mkdir build
+cd build
+cmake .. -DMFEM_DIR=$INSTALLDIR -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DCMAKE_CXX_COMPILER=$CXX
+```
+The easiest way to visualize Laghos results is to have GLVis running in a
+separate terminal. Then the `-vis` option in Laghos will stream results directly
+to the GLVis socket.
+
+Caliper (Optional):
+1. Clone and build Adiak:
+```sh
+git clone --recursive https://github.com/LLNL/Adiak.git
+cd Adiak
+mkdir build && cd build
+cmake -DBUILD_SHARED_LIBS=On -DENABLE_MPI=On \
+      -DCMAKE_INSTALL_PREFIX=$INSTALLDIR ..
+make -j install
+```
+2. Clone and build Caliper:
+```sh
+git clone https://github.com/LLNL/Caliper.git
+cd Caliper
+mkdir build && cd build
+cmake -DWITH_MPI=True -DWITH_ADIAK=True -Dadiak_ROOT=$INSTALLDIR \
+      -DCMAKE_INSTALL_PREFIX=$INSTALLDIR ..
+make -j install
+```
+
+Laghos (CPU-only):
+```sh
+git clone https://github.com/CEED/Laghos.git
+cd Laghos
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DCMAKE_CXX_COMPILER=$CXX -Dcaliper_ROOT=$INSTALLDIR -DLAGHOS_USE_CALIPER=ON
+make -j
+```
+`LAGHOS_USE_CALIPER` may be optionally disabled.
+
+Laghos (CUDA):
+```sh
+git clone https://github.com/CEED/Laghos.git
+cd Laghos
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_CUDA_COMPILER=$CUDACC -DCMAKE_CUDA_ARCHITECTURES=native -Dcaliper_ROOT=$INSTALLDIR -DLAGHOS_USE_CALIPER=ON
+make -j
+```
+`LAGHOS_USE_CALIPER` may be optionally disabled.
+
+Laghos (HIP):
+```sh
+git clone https://github.com/CEED/Laghos.git
+cd Laghos
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INSTALLDIR -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_HIP_COMPILER=$HIPCC -DCMAKE_HIP_ARCHITECTURES=native -Dcaliper_ROOT=$INSTALLDIR -DLAGHOS_USE_CALIPER=ON
+make -j
+```
+`LAGHOS_USE_CALIPER` may be optionally disabled.
+
 ## Running
 
 #### Sedov blast
@@ -232,13 +407,15 @@ partial assembly option (`-pa`).
 Some sample runs in 2D and 3D respectively are:
 ```sh
 mpirun -np 8 ./laghos -p 1 -dim 2 -rs 3 -tf 0.8 -pa
-mpirun -np 8 ./laghos -p 1 -dim 3 -rs 2 -tf 0.6 -pa -vis
+mpirun -np 8 ./laghos -p 1 -dim 3 -E0 2 -rs 2 -tf 0.6 -pa -vis
 ```
 
 The latter produces the following density plot (notice the `-vis` option)
 
 [![Sedov blast image](data/sedov.png)](https://glvis.org/live/?stream=../data/laghos.saved)
 
+To compare against the analytical soluton the `-err` option can be used to compute $\int_{\Omega} \|\rho_{sim} - \rho_{exact}\|_2 dV$ of the final solution.
+
 #### Taylor-Green and Gresho vortices
 
 Laghos includes also smooth test problems that expose all the principal
@@ -283,7 +460,7 @@ To make sure the results are correct, we tabulate reference final iterations
 1. `mpirun -np 8 ./laghos -p 0 -dim 2 -rs 3 -tf 0.75 -pa`
 2. `mpirun -np 8 ./laghos -p 0 -dim 3 -rs 1 -tf 0.75 -pa`
 3. `mpirun -np 8 ./laghos -p 1 -dim 2 -rs 3 -tf 0.8 -pa`
-4. `mpirun -np 8 ./laghos -p 1 -dim 3 -rs 2 -tf 0.6 -pa`
+4. `mpirun -np 8 ./laghos -p 1 -dim 3 -E0 2 -rs 2 -tf 0.6 -pa`
 5. `mpirun -np 8 ./laghos -p 2 -dim 1 -rs 5 -tf 0.2 -fa`
 6. `mpirun -np 8 ./laghos -p 3 -m data/rectangle01_quad.mesh -rs 2 -tf 3.0 -pa`
 7. `mpirun -np 8 ./laghos -p 3 -m data/box01_hex.mesh -rs 1 -tf 5.0 -pa`