Merge pull request #42 from AFD-Illinois/dev

Release KHARMA 2023.12

.gitignore

@@ -1,21 +1,29 @@
+# Memory dumps
+core*
+*.swp
+
 # Various script results/logs
 out-*.txt
 *.json
 convergence.txt
 *.h5
 *.png
 *.mp4
-core.*
+*.webm
 frames_*/
+logs/
+*.log
 
 # KHARMA/Parthenon outputs
 *.phdf
 *.rhdf
 *.xdmf
 *.hst
-# Archival files
+# Archival/test files
 kharma_parsed_*.par
 log_*.txt
+bondi_analytic_*.txt
+atmosphere_soln_*.txt
 
 # Editor documents
 .project
@@ -63,7 +71,7 @@ make_args
 # Executables
 *.host*
 *.cuda*
-*.rocm*
+*.hip*
 *.sycl*
 *.exe
 *.out

.gitlab-ci.yml

@@ -1,76 +1,23 @@
-# Continuous Integration testing for KHARMA
-# a.k.a did we break the basics?
-# This version run on LANL Darwin
-# See .gitlab-ci-docker.yml for a generic version,
-# which can be run on any Docker runner w/GPUs
-
-variables:
-  GIT_SUBMODULE_STRATEGY: recursive
-  SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p volta-x86"
-  HOST_ARCH: HSW
-  NPROC: ""
-  OMP_NUM_THREADS: 28
-  OMP_PROC_BIND: "false"
-  MPI_EXE: mpirun
-  MPI_NUM_PROCS: 2
-  HTTP_PROXY: http://proxyout.lanl.gov:8080
-  http_proxy: http://proxyout.lanl.gov:8080
-  HTTPS_PROXY: http://proxyout.lanl.gov:8080
-  https_proxy: http://proxyout.lanl.gov:8080
-  NO_PROXY: lanl.gov,localhost,127.0.0.1,0.0.0.0,::1
-  no_proxy: lanl.gov,localhost,127.0.0.1,0.0.0.0,::1
-
-### DEFAULT TEST BEHAVIOR ###
 default:
-  tags:
-    - darwin-slurm-shared
-  # Load Python
-  before_script:
-    - module load miniconda3
-
   # Always keep logs and plots.  Results should be printed to console!
   artifacts:
     when: always
     paths:
       - tests/*/*.png
       - tests/*/*.txt
 
-# Tests can be executed in parallel,
-# but be careful about GPU arch
-stages:
-  - build
-  - tests
+trigger_darwin:
+  trigger:
+    include: scripts/ci/darwin.yml
 
-# Default rules
-.default-rules:
-  rules:
-    - if: $CI_COMMIT_BRANCH == "dev"
-      when: always
-    - when: manual
-  allow_failure: false
-
-# Build, obviously overrides script/artifacts
-build:
-  extends: .default-rules
-  stage: build
-  before_script:
-    - echo "Skipping pyharm install in build."
-  script:
-    - export PREFIX_PATH=$PWD/external/hdf5
-    - ./make.sh clean cuda hdf5 volta
-  artifacts:
-    paths:
-      - kharma.*
-      - make_args
+trigger_nvhpc:
+  trigger:
+    include: scripts/ci/nvhpc.yml
 
-# Run all tests in parallel
-tests:
-  extends: .default-rules
-  stage: tests
-  script:
-    - cd tests/$TEST
-    - ./run.sh
-  parallel:
-    matrix:
-      - TEST: [bondi, bz_monopole, emhdmodes, mhdmodes, noh, reinit, restart, tilt_init, torus_sanity]
+trigger_cpu:
+  trigger:
+    include: scripts/ci/cpu.yml
+    strategy: depend
 
+# TODO trigger_cpu w/intel or similar container
+# TODO build containers here

.gitmodules

@@ -1,10 +1,7 @@
 [submodule "external/parthenon"]
 	path = external/parthenon
 	url = https://github.com/AFD-Illinois/parthenon.git
-	branch = kharma-cpp14
+	branch = kharma
 [submodule "external/variant"]
 	path = external/variant
 	url = https://github.com/mpark/variant.git
-[submodule "external/kokkos-kernels"]
-	path = external/kokkos-kernels
-	url = https://github.com/kokkos/kokkos-kernels

CMakeLists.txt

@@ -7,11 +7,13 @@ cmake_minimum_required(VERSION 3.10)
 project(kharma LANGUAGES C CXX)
 
 # We follow Parthenon in requiring C++17 going forward
-#set(CMAKE_CXX_STANDARD 17)
-#set(CMAKE_CXX_STANDARD_REQUIRED ON)
-#set(CMAKE_CXX17_STANDARD_COMPILE_OPTION "-std=c++17")
-#set(PARTHENON_ENABLE_CPP17 ON CACHE BOOL "KHARMA Override")
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX17_STANDARD_COMPILE_OPTION "-std=c++17")
+set(PARTHENON_ENABLE_CPP17 ON CACHE BOOL "KHARMA Override")
+
+# Set the path to include cmake/ dir
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
 # Parthenon options
 set(PARTHENON_DISABLE_EXAMPLES ON CACHE BOOL "KHARMA Override")
@@ -20,26 +22,25 @@ set(PARTHENON_LINT_DEFAULT OFF CACHE BOOL "KHARMA Override")
 set(PARTHENON_DISABLE_HDF5_COMPRESSION OFF CACHE BOOL "KHARMA Override")
 # Don't build sparse (selectively-allocated) variable support
 set(PARTHENON_DISABLE_SPARSE ON CACHE BOOL "KHARMA Override")
+# Set to move MPI buffers to host; slower but less crashy
+# Favor setting this per-machine in machines/
+set(PARTHENON_ENABLE_HOST_COMM_BUFFERS OFF CACHE BOOL "KHARMA Override")
 
 # Parthenon internal build options
 set(BUILD_TESTING OFF CACHE BOOL "KHARMA Override")
 set(ENABLE_COMPILER_WARNINGS OFF CACHE BOOL "KHARMA Override")
-# TODO upstream Parthenon needs support before we do this
-#set(COORDINATE_TYPE GRCoordinates)
 # Always use static HDF5
 set(HDF5_USE_STATIC_LIBRARIES ON CACHE BOOL "KHARMA Override")
 
 # Kokkos options
-set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "KHARMA Override")
 set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "KHARMA Override")
 set(Kokkos_ENABLE_CUDA_CONSTEXPR ON CACHE BOOL "KHARMA Override")
 set(Kokkos_ENABLE_HWLOC OFF CACHE BOOL "KHARMA Override") # Possible speed improvement?
 set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "KHARMA Override")
 
-# If we build KokkosKernels at all, disable the extras
-set(KokkosKernels_ENABLE_TPL_CUSPARSE OFF CACHE BOOL "KHARMA Override")
-set(KokkosKernels_ENABLE_TPL_CUBLAS OFF CACHE BOOL "KHARMA Override")
-
+include(GetGitRevisionDescription)
+get_git_head_revision(GIT_REFSPEC GIT_SHA1)
+git_describe_working_tree(GIT_VERSION --tags)
 
 # Offer a KHARMA option to disable the MPI requirement
 # The only difference from setting PARTHENON_DISABLE_MPI is that
@@ -52,25 +53,17 @@ else()
   include_directories(SYSTEM ${MPI_INCLUDE_PATH})
 endif()
 
-# OpenMP is strictly required
-find_package(OpenMP REQUIRED)
+# OpenMP is usually used host-side.  We're letting Parthenon/Kokkos
+# find it though, as sometimes we require disabling it fully
+#find_package(OpenMP REQUIRED)
 
 # Build Parthenon
 add_subdirectory(external/parthenon)
 include_directories(external/parthenon/src)
 # mpark::variant is header only, don't build anything
 include_directories(external/variant/include)
-# Kokkos kernels: don't build them (very slow), just import all headers
-# Requires KokkosKernels_config.h shipped with KHARMA, YMMV
-# In case of issues, uncomment the following line to build them
-#add_subdirectory(external/kokkos-kernels)
-include_directories(external/kokkos-kernels/src)
-include_directories(external/kokkos-kernels/src/batched)
-include_directories(external/kokkos-kernels/src/common)
-include_directories(external/kokkos-kernels/src/batched/dense)
-include_directories(external/kokkos-kernels/src/batched/dense/impl)
-include_directories(external/kokkos-kernels/src/blas)
-include_directories(external/kokkos-kernels/src/blas/impl)
+# Our hacked-up version of the Kokkos kernels
+include_directories(external/kokkos-kernels)
 
 # Finally, build KHARMA
 add_subdirectory(kharma)

README.md

@@ -44,3 +44,10 @@ Except for performance tuning, KHARMA has no compile time parameters: all of the
 
 ## Hacking
 KHARMA has some preliminary documentation for developers, hosted in its GitHub [wiki](https://github.com/AFD-Illinois/kharma/wiki).
+
+## Licenses
+KHARMA is made available under the BSD 3-clause license included in each file and in the file LICENSE at the root of this repository.
+
+This repository also carries a substantial portion of the [Kokkos Kernels](https://github.com/kokkos/kokkos-kernels), in the directory `kharma/implicit/kokkos-kernels-pivoted`, which is provided under the license included in that directory.
+
+Submodules of this repository, [Parthenon](https://github.com/parthenon-hpc-lab/parthenon) and [mpark::variant](https://github.com/mpark/variant) are made available under their own licenses.