From 7302a2bccf9f237ac73d919f2d57e0de6bce50ef Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Thu, 23 Oct 2025 14:23:05 -0600 Subject: [PATCH 01/12] Make FunctionBase class #30655 --- framework/include/functions/Function.h | 19 +------- framework/include/functions/FunctionBase.h | 48 +++++++++++++++++++ framework/src/functions/Function.C | 16 +------ framework/src/functions/FunctionBase.C | 55 ++++++++++++++++++++++ 4 files changed, 107 insertions(+), 31 deletions(-) create mode 100644 framework/include/functions/FunctionBase.h create mode 100644 framework/src/functions/FunctionBase.C diff --git a/framework/include/functions/Function.h b/framework/include/functions/Function.h index d73e2e83ebca..86466ec27537 100644 --- a/framework/include/functions/Function.h +++ b/framework/include/functions/Function.h @@ -9,14 +9,7 @@ #pragma once -#include "MooseObject.h" -#include "SetupInterface.h" -#include "TransientInterface.h" -#include "PostprocessorInterface.h" -#include "UserObjectInterface.h" -#include "Restartable.h" -#include "MeshChangedInterface.h" -#include "ScalarCoupleable.h" +#include "FunctionBase.h" #include "MooseFunctor.h" #include "ChainedReal.h" @@ -33,15 +26,7 @@ class Point; * Base class for function objects. Functions override value to supply a * value at a point. */ -class Function : public MooseObject, - public SetupInterface, - public TransientInterface, - public PostprocessorInterface, - public UserObjectInterface, - public Restartable, - public MeshChangedInterface, - public ScalarCoupleable, - public Moose::FunctorBase +class Function : public Moose::FunctionBase, public Moose::FunctorBase { public: /** diff --git a/framework/include/functions/FunctionBase.h b/framework/include/functions/FunctionBase.h new file mode 100644 index 000000000000..83ccb43532ac --- /dev/null +++ b/framework/include/functions/FunctionBase.h @@ -0,0 +1,48 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "MooseObject.h" +#include "SetupInterface.h" +#include "TransientInterface.h" +#include "PostprocessorInterface.h" +#include "UserObjectInterface.h" +#include "Restartable.h" +#include "MeshChangedInterface.h" +#include "ScalarCoupleable.h" + +namespace Moose +{ + +class FunctionBase : public MooseObject, + public SetupInterface, + public TransientInterface, + public PostprocessorInterface, + public UserObjectInterface, + public Restartable, + public MeshChangedInterface, + public ScalarCoupleable +{ +public: + static InputParameters validParams(); + + FunctionBase(const InputParameters & parameters); + +#ifdef MOOSE_KOKKOS_ENABLED + /** + * Special constructor used for Kokkos functor copy during parallel dispatch + */ + FunctionBase(const FunctionBase & object, const Moose::Kokkos::FunctorCopy & key); +#endif + + virtual ~FunctionBase(); +}; + +} diff --git a/framework/src/functions/Function.C b/framework/src/functions/Function.C index 056af80fad7e..fd46cfc4a3c3 100644 --- a/framework/src/functions/Function.C +++ b/framework/src/functions/Function.C @@ -15,26 +15,14 @@ using namespace Moose; InputParameters Function::validParams() { - InputParameters params = MooseObject::validParams(); - params += SetupInterface::validParams(); - - // Functions should be executed on the fly - params.suppressParameter("execute_on"); + InputParameters params = FunctionBase::validParams(); params.registerBase("Function"); return params; } Function::Function(const InputParameters & parameters) - : MooseObject(parameters), - SetupInterface(this), - TransientInterface(this), - PostprocessorInterface(this), - UserObjectInterface(this), - Restartable(this, "Functions"), - MeshChangedInterface(parameters), - ScalarCoupleable(this), - Moose::FunctorBase(name()) + : FunctionBase(parameters), Moose::FunctorBase(name()) { } diff --git a/framework/src/functions/FunctionBase.C b/framework/src/functions/FunctionBase.C new file mode 100644 index 000000000000..e03c7b1b10c7 --- /dev/null +++ b/framework/src/functions/FunctionBase.C @@ -0,0 +1,55 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "FunctionBase.h" + +namespace Moose +{ + +InputParameters +FunctionBase::validParams() +{ + InputParameters params = MooseObject::validParams(); + params += SetupInterface::validParams(); + + // Functions should be executed on the fly + params.suppressParameter("execute_on"); + + return params; +} + +FunctionBase::FunctionBase(const InputParameters & parameters) + : MooseObject(parameters), + SetupInterface(this), + TransientInterface(this), + PostprocessorInterface(this), + UserObjectInterface(this), + Restartable(this, "Functions"), + MeshChangedInterface(parameters), + ScalarCoupleable(this) +{ +} + +#ifdef MOOSE_KOKKOS_ENABLED +FunctionBase::FunctionBase(const FunctionBase & object, const Moose::Kokkos::FunctorCopy & key) + : MooseObject(object, key), + SetupInterface(object, key), + TransientInterface(object, key), + PostprocessorInterface(object, key), + UserObjectInterface(object, key), + Restartable(object, key), + MeshChangedInterface(object, key), + ScalarCoupleable(object, key) +{ +} +#endif + +FunctionBase::~FunctionBase() {} + +} // namespace Moose From b1c9f5a0fac6023992523877b46083deded30292 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Thu, 23 Oct 2025 14:22:24 -0600 Subject: [PATCH 02/12] Kokkos functor and function base classes #30655 --- framework/include/kokkos/base/KokkosFunctor.h | 67 ++++++ .../kokkos/base/KokkosFunctorRegistry.h | 169 ++++++++++++++ .../kokkos/base/KokkosFunctorWrapper.h | 176 ++++++++++++++ .../include/kokkos/functions/KokkosFunction.h | 197 ++++++++++++++++ .../kokkos/functions/KokkosFunctionWrapper.h | 219 ++++++++++++++++++ framework/include/utils/MooseTypes.h | 1 + framework/src/kokkos/base/KokkosFunctor.K | 48 ++++ .../src/kokkos/base/KokkosFunctorRegistry.K | 27 +++ .../src/kokkos/functions/KokkosFunction.K | 54 +++++ 9 files changed, 958 insertions(+) create mode 100644 framework/include/kokkos/base/KokkosFunctor.h create mode 100644 framework/include/kokkos/base/KokkosFunctorRegistry.h create mode 100644 framework/include/kokkos/base/KokkosFunctorWrapper.h create mode 100644 framework/include/kokkos/functions/KokkosFunction.h create mode 100644 framework/include/kokkos/functions/KokkosFunctionWrapper.h create mode 100644 framework/src/kokkos/base/KokkosFunctor.K create mode 100644 framework/src/kokkos/base/KokkosFunctorRegistry.K create mode 100644 framework/src/kokkos/functions/KokkosFunction.K diff --git a/framework/include/kokkos/base/KokkosFunctor.h b/framework/include/kokkos/base/KokkosFunctor.h new file mode 100644 index 000000000000..44c70307172c --- /dev/null +++ b/framework/include/kokkos/base/KokkosFunctor.h @@ -0,0 +1,67 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosTypes.h" +#include "KokkosFunctorWrapper.h" +#include "KokkosFunctorRegistry.h" + +class FEProblemBase; + +namespace Moose +{ +namespace Kokkos +{ + +/** + * The abstract class that provides polymorphic interfaces for a functor + */ +class Functor final +{ +public: + /** + * Constructor + * @param problem The MOOSE problem + * @param wrapper The host functor wrapper + */ + Functor(FEProblemBase & problem, std::shared_ptr wrapper); + /** + * Copy constructor for parallel dispatch + */ + Functor(const Functor & functor); + /** + * Destructor + */ + ~Functor(); + +private: + /** + * Pointer to the host functor wrapper + */ + std::shared_ptr _wrapper_host; + /** + * Pointer to the device functor wrapper + */ + FunctorWrapperDeviceBase * _wrapper_device = nullptr; + /** + * Reference of the FE problem + */ + FEProblemBase & _problem; + /** + * Current and old time + */ + ///@{ + Scalar _t; + Scalar _t_old; + ///@} +}; + +} // namespace Kokkos +} // namespace Moose diff --git a/framework/include/kokkos/base/KokkosFunctorRegistry.h b/framework/include/kokkos/base/KokkosFunctorRegistry.h new file mode 100644 index 000000000000..7bbaea51e70f --- /dev/null +++ b/framework/include/kokkos/base/KokkosFunctorRegistry.h @@ -0,0 +1,169 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosFunctorWrapper.h" +#include "KokkosFunctionWrapper.h" + +namespace Moose +{ +namespace Kokkos +{ + +class FunctorRegistryEntryBase +{ +public: + virtual ~FunctorRegistryEntryBase() {} + /** + * Build a host wrapper for this functor + * @param object The pointer to the functor + */ + virtual std::unique_ptr build(const void * object) const = 0; +}; + +template +class FunctorRegistryEntry : public FunctorRegistryEntryBase +{ +public: + std::unique_ptr build(const void * object) const override final + { + return std::make_unique>(object); + } +}; + +class FunctionRegistryEntryBase +{ +public: + virtual ~FunctionRegistryEntryBase() {} + /** + * Build a host wrapper for this function + * @param object The pointer to the function + */ + virtual std::unique_ptr build(const void * object) const = 0; +}; + +template +class FunctionRegistryEntry : public FunctionRegistryEntryBase +{ +public: + std::unique_ptr build(const void * object) const override final + { + return std::make_unique>(object); + } +}; + +class FunctorRegistry +{ +public: + FunctorRegistry() = default; + + FunctorRegistry(FunctorRegistry const &) = delete; + FunctorRegistry & operator=(FunctorRegistry const &) = delete; + + FunctorRegistry(FunctorRegistry &&) = delete; + FunctorRegistry & operator=(FunctorRegistry &&) = delete; + + /** + * Register a functor + * @tparam Object The functor class type + * @param name The registered functor type name + */ + template + static char addFunctor(const std::string & name) + { + getRegistry()._functors[name] = std::make_unique>(); + + return 0; + } + + /** + * Register a function + * @tparam Object The function class type + * @param name The registered function type name + */ + template + static char addFunction(const std::string & name) + { + getRegistry()._functions[name] = std::make_unique>(); + + return 0; + } + + /** + * Build and get a host wrapper of a functor + * @param object The pointer to the functor + * @param name The registered functor type name + * @returns The host wrapper + */ + static std::unique_ptr buildFunctor(const void * object, + const std::string & name) + { + auto it = getRegistry()._functors.find(name); + if (it == getRegistry()._functors.end()) + mooseError("Kokkos functor not registered for type '", + name, + "'. Double check that you used Kokkos-specific registration macro."); + + return it->second->build(object); + } + + /** + * Build and get a host wrapper of a function + * @param object The pointer to the function + * @param name The registered function type name + * @returns The host wrapper + */ + static std::unique_ptr buildFunction(const void * object, + const std::string & name) + { + auto it = getRegistry()._functions.find(name); + if (it == getRegistry()._functions.end()) + mooseError("Kokkos function not registered for type '", + name, + "'. Double check that you used Kokkos-specific registration macro."); + + return it->second->build(object); + } + +private: + /** + * Get the registry singleton + * @returns The registry singleton + */ + static FunctorRegistry & getRegistry(); + + /** + * Map containing the host wrapper shells of functors with the key being the registered object + * type name + */ + std::map> _functors; + /** + * Map containing the host wrapper shells of functions with the key being the registered object + * type name + */ + std::map> _functions; +}; + +} // namespace Kokkos +} // namespace Moose + +#define registerKokkosFunction(app, classname) \ + registerMooseObject(app, classname); \ + static char combineNames(kokkos_functor_##classname, __COUNTER__) = \ + Moose::Kokkos::FunctorRegistry::addFunctor(#classname); \ + static char combineNames(kokkos_function_##classname, __COUNTER__) = \ + Moose::Kokkos::FunctorRegistry::addFunction(#classname) + +#define registerKokkosFunctionAliased(app, classname, alias) \ + registerMooseObjectAliased(app, classname, alias); \ + static char combineNames(kokkos_functor_##classname, __COUNTER__) = \ + Moose::Kokkos::FunctorRegistry::addFunctor(alias); \ + static char combineNames(kokkos_function_##classname, __COUNTER__) = \ + Moose::Kokkos::FunctorRegistry::addFunction(alias) diff --git a/framework/include/kokkos/base/KokkosFunctorWrapper.h b/framework/include/kokkos/base/KokkosFunctorWrapper.h new file mode 100644 index 000000000000..1c2a197e0f91 --- /dev/null +++ b/framework/include/kokkos/base/KokkosFunctorWrapper.h @@ -0,0 +1,176 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosTypes.h" + +namespace Moose +{ +namespace Kokkos +{ + +template +class FunctorWrapperHost; + +/** + * Base class for device functor wrapper + */ +class FunctorWrapperDeviceBase +{ +public: + /** + * Constructor + */ + KOKKOS_FUNCTION FunctorWrapperDeviceBase() {} + /** + * Virtual destructor + */ + KOKKOS_FUNCTION virtual ~FunctorWrapperDeviceBase() {} +}; + +/** + * Device functor wrapper class that provides polymorphic interfaces for a functor. The functor + * itself is a static object and does not have any virtual method. Instead, the device wrapper + * defines the virtual shims and forwards the calls to the static methods of the stored functor. + * @tparam Object The functor class type + */ +template +class FunctorWrapperDevice : public FunctorWrapperDeviceBase +{ + friend class FunctorWrapperHost; + +public: + /** + * Constructor + */ + KOKKOS_FUNCTION FunctorWrapperDevice() {} + +protected: + /** + * Pointer to the functor on device + */ + Object * _functor = nullptr; +}; + +/** + * Base class for host functor wrapper + */ +class FunctorWrapperHostBase +{ +public: + /** + * Virtual destructor + */ + virtual ~FunctorWrapperHostBase() {} + + /** + * Allocate device functor and wrapper + * @returns The pointer to the device wrapper + */ + virtual FunctorWrapperDeviceBase * allocate() = 0; + /** + * Copy functor to device + */ + virtual void copyFunctor() = 0; + /** + * Free host and device copies of functor + */ + virtual void freeFunctor() = 0; +}; + +/** + * Host functor wrapper class that allocates a functor on device and creates its device wrapper. + * This class holds the actual device instance of the functor and manages its allocation and + * deallocation, and the device wrapper simply keeps a pointer to it. + * @tparam Object The functor class type + */ +template +class FunctorWrapperHost : public FunctorWrapperHostBase +{ +public: + /** + * Constructor + * @param functor Pointer to the functor + */ + FunctorWrapperHost(const void * functor) : _functor_host(*static_cast(functor)) {} + /** + * Desturctor + */ + ~FunctorWrapperHost(); + + FunctorWrapperDeviceBase * allocate() override final; + void copyFunctor() override final; + void freeFunctor() override final; + +private: + /** + * Reference of the functor on host + */ + const Object & _functor_host; + /** + * Copy of the functor on host + */ + std::unique_ptr _functor_copy; + /** + * Copy of the functor on device + */ + Object * _functor_device = nullptr; +}; + +template +FunctorWrapperDeviceBase * +FunctorWrapperHost::allocate() +{ + // Allocate storage for device wrapper on device + auto wrapper_device = static_cast *>( + ::Kokkos::kokkos_malloc(sizeof(FunctorWrapperDevice))); + + // Allocate device wrapper on device using placement new to populate vtable with device pointers + ::Kokkos::parallel_for( + 1, KOKKOS_LAMBDA(const int) { new (wrapper_device) FunctorWrapperDevice(); }); + + // Allocate storage for functor on device + _functor_device = + static_cast(::Kokkos::kokkos_malloc(sizeof(Object))); + + // Let device wrapper point to the copy + ::Kokkos::Impl::DeepCopy( + &(wrapper_device->_functor), &_functor_device, sizeof(Object *)); + + return wrapper_device; +} + +template +void +FunctorWrapperHost::copyFunctor() +{ + // Make a copy of functor on host to trigger copy constructor + _functor_copy = std::make_unique(_functor_host); + + // Copy functor to device + ::Kokkos::Impl::DeepCopy( + _functor_device, _functor_copy.get(), sizeof(Object)); +} + +template +void +FunctorWrapperHost::freeFunctor() +{ + _functor_copy.reset(); +} + +template +FunctorWrapperHost::~FunctorWrapperHost() +{ + ::Kokkos::kokkos_free(_functor_device); +} + +} // namespace Kokkos +} // namespace Moose diff --git a/framework/include/kokkos/functions/KokkosFunction.h b/framework/include/kokkos/functions/KokkosFunction.h new file mode 100644 index 000000000000..394017407986 --- /dev/null +++ b/framework/include/kokkos/functions/KokkosFunction.h @@ -0,0 +1,197 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosTypes.h" +#include "KokkosFunctionWrapper.h" +#include "KokkosFunctorRegistry.h" + +#include "FunctionBase.h" + +namespace Moose +{ +namespace Kokkos +{ + +/** + * The base class for a user to derive their own Kokkos functions. + * + * The user should define the hook methods in their derived class as inlined public methods (not + * virtual override) with the same signature. If they are defined in the derived class, they will + * hide the default definitions in the base class. However, the default definitions are not to be + * actually called. If a hook method was not defined in the derived class, it should not be called. + */ +class FunctionBase : public Moose::FunctionBase +{ +public: + static InputParameters validParams(); + + /** + * Constructor + */ + FunctionBase(const InputParameters & parameters); + /** + * Copy constructor for parallel dispatch + */ + FunctionBase(const FunctionBase & object); + + /** + * Evaluate a scalar value at point (t,x,y,z) + * @param t The time + * @param p The location in space (x,y,z) + * @returns The scalar value evaluated at the time and location + */ + KOKKOS_FUNCTION Real value(Real /* t */, Real3 /* p */) const + { + KOKKOS_ASSERT(false); + return 0; + } + /** + * Evaluate a vector value at point (t,x,y,z) + * @param t The time + * @param p The location in space (x,y,z) + * @returns The vector value evaluated at the time and location + */ + KOKKOS_FUNCTION Real3 vectorValue(Real /* t */, Real3 /* p */) const + { + KOKKOS_ASSERT(false); + return Real3(0); + } + /** + * Evaluate a gradient at point (t,x,y,z) + * @param t The time + * @param p The location in space (x,y,z) + * @returns The gradient evaluated at the time and location + */ + KOKKOS_FUNCTION Real3 gradient(Real /* t */, Real3 /* p */) const + { + KOKKOS_ASSERT(false); + return Real3(0); + } + /** + * Evaluate a curl at point (t,x,y,z) + * @param t The time + * @param p The location in space (x,y,z) + * @returns The curl evaluated at the time and location + */ + KOKKOS_FUNCTION Real3 curl(Real /* t */, Real3 /* p */) const + { + KOKKOS_ASSERT(false); + return Real3(0); + } + /** + * Evaluate a divergence at point (t,x,y,z) + * @param t The time + * @param p The location in space (x,y,z) + * @returns The divergence evaluated at the time and location + */ + KOKKOS_FUNCTION Real div(Real /* t */, Real3 /* p */) const + { + KOKKOS_ASSERT(false); + return 0; + } + /** + * Evaluate a time derivative at point (t,x,y,z) + * @param t The time + * @param p The location in space (x,y,z) + * @returns The time derivative evaluated at the time and location + */ + KOKKOS_FUNCTION Real timeDerivative(Real /* t */, Real3 /* p */) const + { + KOKKOS_ASSERT(false); + return 0; + } + /** + * Evaluate a time integral at point (x,y,z) between time \p t1 and \p t2 + * @param t1 The beginning time + * @param t2 The end time + * @param p The location in space (x,y,z) + * @returns The time integral evaluated at the location between the times + */ + KOKKOS_FUNCTION Real timeIntegral(Real /* t1 */, Real /* t2 */, Real3 /* p */) const + { + KOKKOS_ASSERT(false); + return 0; + } + /** + * Evaluate the integral over the domain + * @returns The integral over the domain + */ + KOKKOS_FUNCTION Real integral() const + { + KOKKOS_ASSERT(false); + return 0; + } + /** + * Evaluate the average over the domain + * @returns The average over the domain + */ + KOKKOS_FUNCTION Real average() const + { + KOKKOS_ASSERT(false); + return 0; + } +}; + +/** + * The abstract class that provides polymorphic interfaces for a function. + * + * NOTE: This class is not the base class for a Kokkos function derivation. The user should derive + * their own function from Moose::Kokkos::FunctionBase. + */ +class Function final +{ +public: + /** + * Constructor + * @param wrapper The host function wrapper + */ + Function(std::shared_ptr wrapper); + /** + * Copy constructor for parallel dispatch + */ + Function(const Function & function); + /** + * Destructor + */ + ~Function(); + + KOKKOS_FUNCTION Real value(Real t, Real3 p) const { return _wrapper_device->value(t, p); } + KOKKOS_FUNCTION Real3 vectorValue(Real t, Real3 p) const + { + return _wrapper_device->vectorValue(t, p); + } + KOKKOS_FUNCTION Real3 gradient(Real t, Real3 p) const { return _wrapper_device->gradient(t, p); } + KOKKOS_FUNCTION Real3 curl(Real t, Real3 p) const { return _wrapper_device->curl(t, p); } + KOKKOS_FUNCTION Real div(Real t, Real3 p) const { return _wrapper_device->div(t, p); } + KOKKOS_FUNCTION Real timeDerivative(Real t, Real3 p) const + { + return _wrapper_device->timeDerivative(t, p); + } + KOKKOS_FUNCTION Real timeIntegral(Real t1, Real t2, Real3 p) const + { + return _wrapper_device->timeIntegral(t1, t2, p); + } + KOKKOS_FUNCTION Real integral() const { return _wrapper_device->integral(); } + KOKKOS_FUNCTION Real average() const { return _wrapper_device->average(); } + +private: + /** + * Pointer to the host function wrapper + */ + std::shared_ptr _wrapper_host; + /** + * Pointer to the device function wrapper + */ + FunctionWrapperDeviceBase * _wrapper_device = nullptr; +}; + +} +} diff --git a/framework/include/kokkos/functions/KokkosFunctionWrapper.h b/framework/include/kokkos/functions/KokkosFunctionWrapper.h new file mode 100644 index 000000000000..5c2d1cf6c751 --- /dev/null +++ b/framework/include/kokkos/functions/KokkosFunctionWrapper.h @@ -0,0 +1,219 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosTypes.h" + +namespace Moose +{ +namespace Kokkos +{ + +template +class FunctionWrapperHost; + +/** + * Base class for device function wrapper + */ +class FunctionWrapperDeviceBase +{ +public: + /** + * Constructor + */ + KOKKOS_FUNCTION FunctionWrapperDeviceBase() {} + /** + * Virtual destructor + */ + KOKKOS_FUNCTION virtual ~FunctionWrapperDeviceBase() {} + + /** + * Virtual shims that calls the corresponding methods of the actual stored function + */ + ///{@ + KOKKOS_FUNCTION virtual Real value(Real t, Real3 p) const = 0; + KOKKOS_FUNCTION virtual Real3 vectorValue(Real t, Real3 p) const = 0; + KOKKOS_FUNCTION virtual Real3 gradient(Real t, Real3 p) const = 0; + KOKKOS_FUNCTION virtual Real3 curl(Real t, Real3 p) const = 0; + KOKKOS_FUNCTION virtual Real div(Real t, Real3 p) const = 0; + KOKKOS_FUNCTION virtual Real timeDerivative(Real t, Real3 p) const = 0; + KOKKOS_FUNCTION virtual Real timeIntegral(Real t1, Real t2, Real3 p) const = 0; + KOKKOS_FUNCTION virtual Real integral() const = 0; + KOKKOS_FUNCTION virtual Real average() const = 0; + ///@} +}; + +/** + * Device function wrapper class that provides polymorphic interfaces for a function. The function + * itself is a static object and does not have any virtual method. Instead, the device wrapper + * defines the virtual shims and forwards the calls to the static methods of the stored function. + * @tparam Object The function class type + */ +template +class FunctionWrapperDevice : public FunctionWrapperDeviceBase +{ + friend class FunctionWrapperHost; + +public: + /** + * Constructor + */ + KOKKOS_FUNCTION FunctionWrapperDevice() {} + + KOKKOS_FUNCTION Real value(Real t, Real3 p) const override final + { + return _function->value(t, p); + } + KOKKOS_FUNCTION Real3 vectorValue(Real t, Real3 p) const override final + { + return _function->vectorValue(t, p); + } + KOKKOS_FUNCTION Real3 gradient(Real t, Real3 p) const override final + { + return _function->gradient(t, p); + } + KOKKOS_FUNCTION Real3 curl(Real t, Real3 p) const override final { return _function->curl(t, p); } + KOKKOS_FUNCTION Real div(Real t, Real3 p) const override final { return _function->div(t, p); } + KOKKOS_FUNCTION Real timeDerivative(Real t, Real3 p) const override final + { + return _function->timeDerivative(t, p); + } + KOKKOS_FUNCTION Real timeIntegral(Real t1, Real t2, Real3 p) const override final + { + return _function->timeIntegral(t1, t2, p); + } + KOKKOS_FUNCTION Real integral() const override final { return _function->integral(); } + KOKKOS_FUNCTION Real average() const override final { return _function->average(); } + +protected: + /** + * Pointer to the function on device + */ + Object * _function = nullptr; +}; + +/** + * Base class for host function wrapper + */ +class FunctionWrapperHostBase +{ +public: + /** + * Virtual destructor + */ + virtual ~FunctionWrapperHostBase() {} + + /** + * Allocate device function and wrapper + * @returns The pointer to the device wrapper + */ + virtual FunctionWrapperDeviceBase * allocate() = 0; + /** + * Copy function to device + */ + virtual void copyFunction() = 0; + /** + * Free host and device copies of function + */ + virtual void freeFunction() = 0; +}; + +/** + * Host function wrapper class that allocates a function on device and creates its device wrapper. + * This class holds the actual device instance of the function and manages its allocation and + * deallocation, and the device wrapper simply keeps a pointer to it. + * @tparam Object The function class type + */ +template +class FunctionWrapperHost : public FunctionWrapperHostBase +{ +public: + /** + * Constructor + * @param function Pointer to the function + */ + FunctionWrapperHost(const void * function) + : _function_host(*static_cast(function)) + { + } + /** + * Destructor + */ + ~FunctionWrapperHost(); + + FunctionWrapperDeviceBase * allocate() override final; + void copyFunction() override final; + void freeFunction() override final; + +private: + /** + * Reference of the function on host + */ + const Object & _function_host; + /** + * Copy of the function on host + */ + std::unique_ptr _function_copy; + /** + * Copy of the function on device + */ + Object * _function_device = nullptr; +}; + +template +FunctionWrapperDeviceBase * +FunctionWrapperHost::allocate() +{ + // Allocate storage for device wrapper on device + auto wrapper_device = static_cast *>( + ::Kokkos::kokkos_malloc(sizeof(FunctionWrapperDevice))); + + // Allocate device wrapper on device using placement new to populate vtable with device pointers + ::Kokkos::parallel_for( + 1, KOKKOS_LAMBDA(const int) { new (wrapper_device) FunctionWrapperDevice(); }); + + // Allocate storage for function on device + _function_device = + static_cast(::Kokkos::kokkos_malloc(sizeof(Object))); + + // Let device wrapper point to the copy + ::Kokkos::Impl::DeepCopy( + &(wrapper_device->_function), &_function_device, sizeof(Object *)); + + return wrapper_device; +} + +template +void +FunctionWrapperHost::copyFunction() +{ + // Make a copy of function on host to trigger copy constructor + _function_copy = std::make_unique(_function_host); + + // Copy function to device + ::Kokkos::Impl::DeepCopy( + _function_device, _function_copy.get(), sizeof(Object)); +} + +template +void +FunctionWrapperHost::freeFunction() +{ + _function_copy.reset(); +} + +template +FunctionWrapperHost::~FunctionWrapperHost() +{ + ::Kokkos::kokkos_free(_function_device); +} + +} // namespace Kokkos +} // namespace Moose diff --git a/framework/include/utils/MooseTypes.h b/framework/include/utils/MooseTypes.h index a5ec8b7c3aa8..2b8b0084726e 100644 --- a/framework/include/utils/MooseTypes.h +++ b/framework/include/utils/MooseTypes.h @@ -1249,6 +1249,7 @@ class FunctorCopy friend class MaterialBase; friend class Material; friend class AuxKernel; + friend class FunctionBase; FunctorCopy() = default; FunctorCopy(const FunctorCopy &) = delete; diff --git a/framework/src/kokkos/base/KokkosFunctor.K b/framework/src/kokkos/base/KokkosFunctor.K new file mode 100644 index 000000000000..c7d2bc26f806 --- /dev/null +++ b/framework/src/kokkos/base/KokkosFunctor.K @@ -0,0 +1,48 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosFunctor.h" + +#include "FEProblemBase.h" + +namespace Moose +{ +namespace Kokkos +{ + +Functor::Functor(FEProblemBase & problem, std::shared_ptr wrapper) + : _wrapper_host(wrapper), _problem(problem), _t(problem.time()), _t_old(problem.timeOld()) +{ + // Allocate device wrapper + _wrapper_device = _wrapper_host->allocate(); +} + +Functor::Functor(const Functor & functor) + : _wrapper_host(functor._wrapper_host), + _wrapper_device(functor._wrapper_device), + _problem(functor._problem), + _t(functor._t), + _t_old(functor._t_old) +{ + // Copy functor to device + _wrapper_host->copyFunctor(); +} + +Functor::~Functor() +{ + // Free device wrapper + if (_wrapper_host.use_count() == 1) + ::Kokkos::kokkos_free(_wrapper_device); + + // Free host copy of functor + _wrapper_host->freeFunctor(); +} + +} // namespace Kokkos +} // namespace Moose diff --git a/framework/src/kokkos/base/KokkosFunctorRegistry.K b/framework/src/kokkos/base/KokkosFunctorRegistry.K new file mode 100644 index 000000000000..04f1ba1f769b --- /dev/null +++ b/framework/src/kokkos/base/KokkosFunctorRegistry.K @@ -0,0 +1,27 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosFunctorRegistry.h" + +namespace Moose +{ +namespace Kokkos +{ + +FunctorRegistry & +FunctorRegistry::getRegistry() +{ + static FunctorRegistry * registry = nullptr; + if (!registry) + registry = new FunctorRegistry(); + return *registry; +} + +} // namespace Kokkos +} // namespace Moose diff --git a/framework/src/kokkos/functions/KokkosFunction.K b/framework/src/kokkos/functions/KokkosFunction.K new file mode 100644 index 000000000000..91dbe2d8714d --- /dev/null +++ b/framework/src/kokkos/functions/KokkosFunction.K @@ -0,0 +1,54 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosFunction.h" + +namespace Moose +{ +namespace Kokkos +{ + +InputParameters +FunctionBase::validParams() +{ + InputParameters params = Moose::FunctionBase::validParams(); + params.registerBase("KokkosFunction"); + + return params; +} + +FunctionBase::FunctionBase(const InputParameters & parameters) : Moose::FunctionBase(parameters) {} + +FunctionBase::FunctionBase(const FunctionBase & object) : Moose::FunctionBase(object, {}) {} + +Function::Function(std::shared_ptr wrapper) : _wrapper_host(wrapper) +{ + // Allocate device wrapper + _wrapper_device = _wrapper_host->allocate(); +} + +Function::Function(const Function & function) + : _wrapper_host(function._wrapper_host), _wrapper_device(function._wrapper_device) +{ + // Copy function to device + _wrapper_host->copyFunction(); +} + +Function::~Function() +{ + // Free device wrapper + if (_wrapper_host.use_count() == 1) + ::Kokkos::kokkos_free(_wrapper_device); + + // Free host copy of function + _wrapper_host->freeFunction(); +} + +} // namespace Kokkos +} // namespace Moose From 052abc04f3099dda2dfeb273a899090cd64e1b68 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Thu, 23 Oct 2025 14:25:12 -0600 Subject: [PATCH 03/12] Refactor PiecewiseTabularBase #30655 --- .../include/functions/PiecewiseTabularBase.h | 29 +-- .../functions/PiecewiseTabularInterface.h | 61 +++++ framework/src/functions/PiecewiseConstant.C | 2 +- .../src/functions/PiecewiseTabularBase.C | 210 +-------------- .../src/functions/PiecewiseTabularInterface.C | 239 ++++++++++++++++++ 5 files changed, 312 insertions(+), 229 deletions(-) create mode 100644 framework/include/functions/PiecewiseTabularInterface.h create mode 100644 framework/src/functions/PiecewiseTabularInterface.C diff --git a/framework/include/functions/PiecewiseTabularBase.h b/framework/include/functions/PiecewiseTabularBase.h index 829f4cfe101a..7592939ef025 100644 --- a/framework/include/functions/PiecewiseTabularBase.h +++ b/framework/include/functions/PiecewiseTabularBase.h @@ -10,6 +10,7 @@ #pragma once #include "PiecewiseBase.h" +#include "PiecewiseTabularInterface.h" #include "LinearInterpolation.h" /** @@ -17,7 +18,7 @@ * input parameter specifications. Derived classes, which control the order (constant, linear) of * the approximation and how the (x,y) data set is generated, should be used directly. */ -class PiecewiseTabularBase : public PiecewiseBase +class PiecewiseTabularBase : public PiecewiseBase, public PiecewiseTabularInterface { public: static InputParameters validParams(); @@ -31,27 +32,9 @@ class PiecewiseTabularBase : public PiecewiseBase /// function value scale factor const Real & _scale_factor; - ///@{ if _has_axis is true point component to use as function argument, otherwise use t - int _axis; - const bool _has_axis; - ///@} - - /// Returns whether the raw data has been loaded already - bool isRawDataLoaded() const { return _raw_data_loaded; }; - private: - /// Reads data from supplied CSV file. - void buildFromFile(); - - /// Reads data from supplied JSON reader. - void buildFromJSON(); - - /// Builds data from 'x' and 'y' parameters. - void buildFromXandY(); - - /// Builds data from 'xy_data' parameter. - void buildFromXY(); - - /// Boolean to keep track of whether the data has been loaded - bool _raw_data_loaded; + using PiecewiseTabularInterface::buildFromFile; + using PiecewiseTabularInterface::buildFromJSON; + using PiecewiseTabularInterface::buildFromXandY; + using PiecewiseTabularInterface::buildFromXY; }; diff --git a/framework/include/functions/PiecewiseTabularInterface.h b/framework/include/functions/PiecewiseTabularInterface.h new file mode 100644 index 000000000000..916068686e53 --- /dev/null +++ b/framework/include/functions/PiecewiseTabularInterface.h @@ -0,0 +1,61 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "MooseObject.h" +#include "InputParameters.h" + +class JSONFileReader; + +class PiecewiseTabularInterface +{ +public: + static InputParameters validParams(); + + PiecewiseTabularInterface(const MooseObject & object, + std::vector & data_x, + std::vector & data_y); + +protected: + /// Returns whether the raw data has been loaded already + bool isRawDataLoaded() const { return _raw_data_loaded; } + + /// Reads data from supplied CSV file. + void buildFromFile(const libMesh::Parallel::Communicator & comm); + + /// Reads data from supplied JSON reader. + void buildFromJSON(const JSONFileReader & json_uo); + + /// Builds data from 'x' and 'y' parameters. + void buildFromXandY(); + + /// Builds data from 'xy_data' parameter. + void buildFromXY(); + + ///@{ if _has_axis is true point component to use as function argument, otherwise use t + unsigned int _axis; + const bool _has_axis; + ///@} + +private: + /// The object + const MooseObject & _object; + + /// Parameters supplied to the object + const InputParameters & _parameters; + + /// Boolean to keep track of whether the data has been loaded + bool _raw_data_loaded = false; + + ///@{ raw function data as read + std::vector & _data_x; + std::vector & _data_y; + ///@} +}; diff --git a/framework/src/functions/PiecewiseConstant.C b/framework/src/functions/PiecewiseConstant.C index dac1553ff2a5..a0795a4330b9 100644 --- a/framework/src/functions/PiecewiseConstant.C +++ b/framework/src/functions/PiecewiseConstant.C @@ -19,7 +19,7 @@ PiecewiseConstant::validParams() MooseEnum direction("LEFT RIGHT LEFT_INCLUSIVE RIGHT_INCLUSIVE", "LEFT"); params.addParam( "direction", direction, "Direction to look to find value: " + direction.getRawNames()); - params.addClassDescription("Defines data using a set of x-y data pairs"); + params.addClassDescription("Defines piece-wise constant data using a set of x-y data pairs"); return params; } diff --git a/framework/src/functions/PiecewiseTabularBase.C b/framework/src/functions/PiecewiseTabularBase.C index 46d144aa1db3..dab0c2aa134a 100644 --- a/framework/src/functions/PiecewiseTabularBase.C +++ b/framework/src/functions/PiecewiseTabularBase.C @@ -8,92 +8,28 @@ //* https://www.gnu.org/licenses/lgpl-2.1.html #include "PiecewiseTabularBase.h" -#include "DelimitedFileReader.h" #include "JSONFileReader.h" -#include "libmesh/int_range.h" InputParameters PiecewiseTabularBase::validParams() { InputParameters params = PiecewiseBase::validParams(); + params += PiecewiseTabularInterface::validParams(); - // Parameters shared across all data input methods - MooseEnum axis("x=0 y=1 z=2"); - params.addParam( - "axis", axis, "The axis used (x, y, or z) if this is to be a function of position"); params.addParam("scale_factor", 1.0, "Scale factor to be applied to the output values"); params.declareControllable("scale_factor"); - // Data from input file parameters - params.addParam>("xy_data", - "All function data, supplied in abscissa, ordinate pairs"); - params.addParam>("x", "The abscissa values"); - params.addParam>("y", "The ordinate values"); - - // Data from CSV file parameters - params.addParam("data_file", "File holding CSV data"); - params.addParam("x_index_in_file", 0, "The abscissa index in the data file"); - params.addParam("y_index_in_file", 1, "The ordinate index in the data file"); - params.addParam( - "x_title", "The title of the column/row containing the x data in the data file"); - params.addParam( - "y_title", "The title of the column/row containing the y data in the data file"); - params.addParam( - "xy_in_file_only", true, "If the data file only contains abscissa and ordinate data"); - MooseEnum format("columns=0 rows=1", "rows"); - params.addParam( - "format", format, "Format of csv data file that is in either in columns or rows"); - - // Data from JSON parameters - params.addParam("json_uo", "JSONFileReader holding the data"); - params.addParam>( - "x_keys", "Ordered vector of keys in the JSON tree to obtain the abscissa"); - params.addParam>( - "y_keys", "Ordered vector of keys in the JSON tree to obtain the ordinate"); - - params.addParamNamesToGroup("xy_data x y", "Data from input file"); - params.addParamNamesToGroup( - "data_file x_index_in_file y_index_in_file x_title y_title xy_in_file_only format", - "Data from CSV file"); - params.addParamNamesToGroup("json_uo x_keys y_keys", "Data from JSON"); return params; } PiecewiseTabularBase::PiecewiseTabularBase(const InputParameters & parameters) : PiecewiseBase(parameters), - _scale_factor(this->template getParam("scale_factor")), - _has_axis(isParamValid("axis")), - _raw_data_loaded(false) + PiecewiseTabularInterface(*this, _raw_x, _raw_y), + _scale_factor(getParam("scale_factor")) { - // determine function argument - if (_has_axis) - _axis = this->template getParam("axis"); - - // Check all parameters for inconsistencies - if (isParamValid("data_file") + isParamValid("json_uo") + - (isParamValid("x") || isParamValid("y")) + isParamValid("xy_data") > - 1) - mooseError("Either 'data_file' or 'json_uo' or 'x' and 'y' or 'xy_data' must be specified " - "exclusively."); - if ((isParamValid("x") + isParamValid("y")) % 2 != 0) - mooseError( - "Both 'x' and 'y' parameters or neither (for another input method) must be specified"); - if (!isParamValid("data_file") && - (isParamSetByUser("x_index_in_file") || isParamSetByUser("y_index_in_file") || - isParamValid("x_title") || isParamValid("y_title") || isParamSetByUser("xy_in_file_only") || - isParamSetByUser("format"))) - mooseError( - "A parameter was passed for an option using data from a CSV file but the " - "'data_file' parameter has not been set. This is not allowed. Please check the parameter " - "groups in the documentation for the list of parameters for each data input method."); - if (!isParamValid("json_uo") && (isParamValid("x_keys") || isParamValid("y_keys"))) - mooseError("A parameter was passed for a JSON input option but the 'json_uo' parameter has not " - "been set. This is not allowed. Please check the parameter groups in the " - "documentation for the list of parameters for each data input method."); - // load the data if (isParamValid("data_file")) - buildFromFile(); + buildFromFile(_communicator); else if (isParamValid("x") && isParamValid("y")) buildFromXandY(); else if (isParamValid("xy_data")) @@ -110,141 +46,5 @@ PiecewiseTabularBase::initialSetup() if (!isParamValid("json_uo")) return; else - buildFromJSON(); -} - -void -PiecewiseTabularBase::buildFromFile() -{ - // Input parameters - const auto & data_file_name = this->template getParam("data_file"); - const MooseEnum format = this->template getParam("format"); - - // xy_in_file_only does not make sense here as it restricts the file to exactly - // two cows/columns, which is not a likely scenario when looking up data by name. - // A wrong 'format' parameter would be caught by the failing name resolution anyways. - bool xy_only = this->template getParam("xy_in_file_only"); - if (isParamValid("x_title") || isParamValid("y_title")) - { - if (this->_pars.isParamSetByUser("xy_in_file_only") && xy_only) - paramError( - "xy_in_file_only", - "When accessing data through 'x_title' or 'y_title' this parameter should not be used"); - else - xy_only = false; - } - - // Read the data from CSV file - MooseUtils::DelimitedFileReader reader(data_file_name, &_communicator); - reader.setFormatFlag(format.getEnum()); - reader.setComment("#"); - reader.read(); - - const auto & columns = reader.getNames(); - const auto & data = reader.getData(); - - if (data.size() > 2 && xy_only) - mooseError("In ", - _name, - ": Read more than two ", - format, - " of data from file '", - data_file_name, - "'. Did you mean to use \"format = ", - format == "columns" ? "rows" : "columns", - "\" or set \"xy_in_file_only\" to false?"); - - unsigned int x_index = libMesh::invalid_uint; - unsigned int y_index = libMesh::invalid_uint; - const auto setIndex = [&](unsigned int & index, const std::string & prefix) - { - if (isParamValid(prefix + "_title")) - { - const auto name = this->template getParam(prefix + "_title"); - for (const auto i : index_range(columns)) - if (columns[i] == name) - index = i; - - if (index == libMesh::invalid_uint) - paramError(prefix + "_title", - "None of the ", - format, - " in the data file has the title '", - name, - "'."); - } - else - index = this->template getParam(prefix + "_index_in_file"); - - if (index >= data.size()) - paramError(prefix + "_index_in_file", - "Index out-of-range of the available data in '", - data_file_name, - "', which contains ", - data.size(), - " ", - format, - " of data."); - }; - - setIndex(x_index, "x"); - setIndex(y_index, "y"); - - if (x_index == y_index) - mooseError( - "In ", _name, ": 'x_index_in_file' and 'y_index_in_file' are set to the same value."); - - // Update the input vectors to contained the desired data - _raw_x = reader.getData(x_index); - _raw_y = reader.getData(y_index); - - // Size mismatch error - if (_raw_x.size() != _raw_y.size()) - mooseError("In ", _name, ": Lengths of x and y data do not match."); - - _raw_data_loaded = true; -} - -void -PiecewiseTabularBase::buildFromJSON() -{ - auto & json_uo = getUserObject("json_uo"); - if (!isParamValid("x_keys")) - mooseError("Missing 'x_keys' parameters for loading data from JSON"); - if (!isParamValid("y_keys")) - mooseError("Missing 'y_keys' parameters for loading data from JSON"); - json_uo.getVector(getParam>("x_keys"), _raw_x); - json_uo.getVector(getParam>("y_keys"), _raw_y); - _raw_data_loaded = true; - - // Size mismatch error - if (_raw_x.size() != _raw_y.size()) - mooseError("Lengths of x (", _raw_x.size(), ") and y (", _raw_y.size(), ") data do not match."); -} - -void -PiecewiseTabularBase::buildFromXandY() -{ - _raw_x = this->template getParam>("x"); - _raw_y = this->template getParam>("y"); - _raw_data_loaded = true; -} - -void -PiecewiseTabularBase::buildFromXY() -{ - const auto & xy = this->template getParam>("xy_data"); - const auto xy_size = xy.size(); - if (xy_size % 2 != 0) - mooseError("In ", _name, ": Length of data provided in 'xy_data' must be a multiple of 2."); - - const auto data_size = xy_size / 2; - _raw_x.resize(data_size); - _raw_y.resize(data_size); - for (const auto i : make_range(data_size)) - { - _raw_x[i] = xy[2 * i]; - _raw_y[i] = xy[2 * i + 1]; - } - _raw_data_loaded = true; + buildFromJSON(getUserObject("json_uo")); } diff --git a/framework/src/functions/PiecewiseTabularInterface.C b/framework/src/functions/PiecewiseTabularInterface.C new file mode 100644 index 000000000000..b66c042bda54 --- /dev/null +++ b/framework/src/functions/PiecewiseTabularInterface.C @@ -0,0 +1,239 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "PiecewiseTabularInterface.h" +#include "DelimitedFileReader.h" +#include "JSONFileReader.h" + +#include "libmesh/int_range.h" + +InputParameters +PiecewiseTabularInterface::validParams() +{ + InputParameters params = emptyInputParameters(); + + // Parameters shared across all data input methods + MooseEnum axis("x=0 y=1 z=2"); + params.addParam( + "axis", axis, "The axis used (x, y, or z) if this is to be a function of position"); + + // Data from input file parameters + params.addParam>("xy_data", + "All function data, supplied in abscissa, ordinate pairs"); + params.addParam>("x", "The abscissa values"); + params.addParam>("y", "The ordinate values"); + + // Data from CSV file parameters + params.addParam("data_file", "File holding CSV data"); + params.addParam("x_index_in_file", 0, "The abscissa index in the data file"); + params.addParam("y_index_in_file", 1, "The ordinate index in the data file"); + params.addParam( + "x_title", "The title of the column/row containing the x data in the data file"); + params.addParam( + "y_title", "The title of the column/row containing the y data in the data file"); + params.addParam( + "xy_in_file_only", true, "If the data file only contains abscissa and ordinate data"); + MooseEnum format("columns=0 rows=1", "rows"); + params.addParam( + "format", format, "Format of csv data file that is in either in columns or rows"); + + // Data from JSON parameters + params.addParam("json_uo", "JSONFileReader holding the data"); + params.addParam>( + "x_keys", "Ordered vector of keys in the JSON tree to obtain the abscissa"); + params.addParam>( + "y_keys", "Ordered vector of keys in the JSON tree to obtain the ordinate"); + + params.addParamNamesToGroup("xy_data x y", "Data from input file"); + params.addParamNamesToGroup( + "data_file x_index_in_file y_index_in_file x_title y_title xy_in_file_only format", + "Data from CSV file"); + params.addParamNamesToGroup("json_uo x_keys y_keys", "Data from JSON"); + + return params; +} + +PiecewiseTabularInterface::PiecewiseTabularInterface(const MooseObject & object, + std::vector & data_x, + std::vector & data_y) + : _has_axis(object.isParamValid("axis")), + _object(object), + _parameters(object.parameters()), + _data_x(data_x), + _data_y(data_y) +{ + // determine function argument + if (_has_axis) + _axis = _parameters.get("axis"); + + // Check all parameters for inconsistencies + if (_parameters.isParamValid("data_file") + _parameters.isParamValid("json_uo") + + (_parameters.isParamValid("x") || _parameters.isParamValid("y")) + + _parameters.isParamValid("xy_data") > + 1) + _object.mooseError( + "Either 'data_file' or 'json_uo' or 'x' and 'y' or 'xy_data' must be specified " + "exclusively."); + if ((_parameters.isParamValid("x") + _parameters.isParamValid("y")) % 2 != 0) + _object.mooseError( + "Both 'x' and 'y' parameters or neither (for another input method) must be specified"); + if (!_parameters.isParamValid("data_file") && + (_parameters.isParamSetByUser("x_index_in_file") || + _parameters.isParamSetByUser("y_index_in_file") || _parameters.isParamValid("x_title") || + _parameters.isParamValid("y_title") || _parameters.isParamSetByUser("xy_in_file_only") || + _parameters.isParamSetByUser("format"))) + _object.mooseError( + "A parameter was passed for an option using data from a CSV file but the " + "'data_file' parameter has not been set. This is not allowed. Please check the parameter " + "groups in the documentation for the list of parameters for each data input method."); + if (!_parameters.isParamValid("json_uo") && + (_parameters.isParamValid("x_keys") || _parameters.isParamValid("y_keys"))) + _object.mooseError( + "A parameter was passed for a JSON input option but the 'json_uo' parameter has not " + "been set. This is not allowed. Please check the parameter groups in the " + "documentation for the list of parameters for each data input method."); +} + +void +PiecewiseTabularInterface::buildFromFile(const libMesh::Parallel::Communicator & comm) +{ + // Input parameters + const auto & data_file_name = _parameters.get("data_file"); + const MooseEnum format = _parameters.get("format"); + + // xy_in_file_only does not make sense here as it restricts the file to exactly + // two cows/columns, which is not a likely scenario when looking up data by name. + // A wrong 'format' parameter would be caught by the failing name resolution anyways. + bool xy_only = _parameters.get("xy_in_file_only"); + if (_parameters.isParamValid("x_title") || _parameters.isParamValid("y_title")) + { + if (_parameters.isParamSetByUser("xy_in_file_only") && xy_only) + _parameters.paramError( + "xy_in_file_only", + "When accessing data through 'x_title' or 'y_title' this parameter should not be used"); + else + xy_only = false; + } + + // Read the data from CSV file + MooseUtils::DelimitedFileReader reader(data_file_name, &comm); + reader.setFormatFlag(format.getEnum()); + reader.setComment("#"); + reader.read(); + + const auto & columns = reader.getNames(); + const auto & data = reader.getData(); + + if (data.size() > 2 && xy_only) + _object.mooseError("In ", + _object.name(), + ": Read more than two ", + format, + " of data from file '", + data_file_name, + "'. Did you mean to use \"format = ", + format == "columns" ? "rows" : "columns", + "\" or set \"xy_in_file_only\" to false?"); + + unsigned int x_index = libMesh::invalid_uint; + unsigned int y_index = libMesh::invalid_uint; + const auto setIndex = [&](unsigned int & index, const std::string & prefix) + { + if (_parameters.isParamValid(prefix + "_title")) + { + const auto name = _parameters.get(prefix + "_title"); + for (const auto i : index_range(columns)) + if (columns[i] == name) + index = i; + + if (index == libMesh::invalid_uint) + _parameters.paramError(prefix + "_title", + "None of the ", + format, + " in the data file has the title '", + name, + "'."); + } + else + index = _parameters.get(prefix + "_index_in_file"); + + if (index >= data.size()) + _parameters.paramError(prefix + "_index_in_file", + "Index out-of-range of the available data in '", + data_file_name, + "', which contains ", + data.size(), + " ", + format, + " of data."); + }; + + setIndex(x_index, "x"); + setIndex(y_index, "y"); + + if (x_index == y_index) + _object.mooseError("In ", + _object.name(), + ": 'x_index_in_file' and 'y_index_in_file' are set to the same value."); + + // Update the input vectors to contained the desired data + _data_x = reader.getData(x_index); + _data_y = reader.getData(y_index); + + // Size mismatch error + if (_data_x.size() != _data_y.size()) + _object.mooseError("In ", _object.name(), ": Lengths of x and y data do not match."); + + _raw_data_loaded = true; +} + +void +PiecewiseTabularInterface::buildFromJSON(const JSONFileReader & json_uo) +{ + if (!_parameters.isParamValid("x_keys")) + _object.mooseError("Missing 'x_keys' parameters for loading data from JSON"); + if (!_parameters.isParamValid("y_keys")) + _object.mooseError("Missing 'y_keys' parameters for loading data from JSON"); + json_uo.getVector(_parameters.get>("x_keys"), _data_x); + json_uo.getVector(_parameters.get>("y_keys"), _data_y); + _raw_data_loaded = true; + + // Size mismatch error + if (_data_x.size() != _data_y.size()) + _object.mooseError( + "Lengths of x (", _data_x.size(), ") and y (", _data_y.size(), ") data do not match."); +} + +void +PiecewiseTabularInterface::buildFromXandY() +{ + _data_x = _parameters.get>("x"); + _data_y = _parameters.get>("y"); + _raw_data_loaded = true; +} + +void +PiecewiseTabularInterface::buildFromXY() +{ + const auto & xy = _parameters.get>("xy_data"); + const auto xy_size = xy.size(); + if (xy_size % 2 != 0) + _object.mooseError( + "In ", _object.name(), ": Length of data provided in 'xy_data' must be a multiple of 2."); + + const auto data_size = xy_size / 2; + _data_x.resize(data_size); + _data_y.resize(data_size); + for (const auto i : make_range(data_size)) + { + _data_x[i] = xy[2 * i]; + _data_y[i] = xy[2 * i + 1]; + } + _raw_data_loaded = true; +} From 20807f692b4eb2384269e73fa7fddec311e212bb Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Thu, 23 Oct 2025 17:34:04 -0600 Subject: [PATCH 04/12] Add Kokkos functions to the problem #30655 --- framework/include/base/MooseObject.h | 15 +-- .../include/functions/FunctionInterface.h | 99 ++++++++++++++++++- .../kokkos/actions/AddKokkosFunctionAction.h | 26 +++++ .../materials/MaterialPropertyInterface.h | 6 +- framework/include/problems/FEProblemBase.h | 62 ++++++++++++ framework/src/base/Moose.C | 9 ++ framework/src/functions/FunctionInterface.C | 15 +-- framework/src/interfaces/BlockRestrictable.C | 4 +- .../src/interfaces/BoundaryRestrictable.C | 6 +- .../kokkos/actions/AddKokkosFunctionAction.K | 36 +++++++ .../functions/KokkosFunctionInterface.K | 61 ++++++++++++ .../src/kokkos/problems/KokkosFEProblemBase.K | 63 +++++++++--- .../src/materials/MaterialPropertyInterface.C | 2 +- framework/src/problems/FEProblemBase.C | 28 ++++++ 14 files changed, 380 insertions(+), 52 deletions(-) create mode 100644 framework/include/kokkos/actions/AddKokkosFunctionAction.h create mode 100644 framework/src/kokkos/actions/AddKokkosFunctionAction.K create mode 100644 framework/src/kokkos/functions/KokkosFunctionInterface.K diff --git a/framework/include/base/MooseObject.h b/framework/include/base/MooseObject.h index d9f43f7fa411..21b66b42b6d1 100644 --- a/framework/include/base/MooseObject.h +++ b/framework/include/base/MooseObject.h @@ -56,26 +56,13 @@ class MooseObject : public ParallelParamObject, std::shared_ptr getSharedPtr() const; #ifdef MOOSE_KOKKOS_ENABLED - class IsKokkosObjectKey - { - friend class BlockRestrictable; - friend class BoundaryRestrictable; - friend class MaterialPropertyInterface; - IsKokkosObjectKey() = default; - IsKokkosObjectKey(const IsKokkosObjectKey &) = delete; - IsKokkosObjectKey(IsKokkosObjectKey &&) = delete; - }; - /** * Get whether this object is a Kokkos functor * The parameter is set by the Kokkos base classes: * - Moose::Kokkos::ResidualObject in KokkosResidualObject.K * - Moose::Kokkos::MaterialBase in KokkosMaterialBase.K */ - bool isKokkosObject(IsKokkosObjectKey &&) const - { - return parameters().isParamValid(MooseBase::kokkos_object_param); - } + bool isKokkosObject() const { return parameters().isParamValid(MooseBase::kokkos_object_param); } #endif // To get warnings tracked in the SolutionInvalidityOutput diff --git a/framework/include/functions/FunctionInterface.h b/framework/include/functions/FunctionInterface.h index 3d0c003290a9..85d38e4cb6da 100644 --- a/framework/include/functions/FunctionInterface.h +++ b/framework/include/functions/FunctionInterface.h @@ -10,6 +10,7 @@ #pragma once #include "MooseTypes.h" +#include "MooseObject.h" #include "InputParameters.h" #define usingFunctionInterfaceMembers \ @@ -17,11 +18,21 @@ using FunctionInterface::getFunctionByName // Forward declarations -class Function; class FEProblemBase; class Function; class InputParameters; -class MooseObject; + +namespace Moose +{ +class FunctionBase; +} + +#ifdef MOOSE_KOKKOS_ENABLED +namespace Moose::Kokkos +{ +class Function; +} +#endif template InputParameters validParams(); @@ -62,7 +73,6 @@ class FunctionInterface /** * Determine if the function exists - * * @param param_name The name of the function parameter * @param index The index of the function * @return True if the function exists @@ -71,13 +81,72 @@ class FunctionInterface /** * Determine if the function exists - * * @param name The name of the function * @return True if the function exists */ bool hasFunctionByName(const FunctionName & name) const; +#ifdef MOOSE_KOKKOS_ENABLED + /** + * Get a Kokkos function of an abstract type with a given name + * @param name The name of the parameter key of the Kokkos function to retrieve + * @return The copy of the Kokkos function of the abstract type with name associated with the + * parameter 'name' + */ + Moose::Kokkos::Function getKokkosFunction(const std::string & name) const; + + /** + * Get a Kokkos function of an abstract type with a given name + * @param name The name of the Kokkos function to retrieve + * @return The copy of the Kokkos function of the abstract type with name 'name' + */ + Moose::Kokkos::Function getKokkosFunctionByName(const FunctionName & name) const; + + /** + * Get a Kokkos function of a concrete type with a given name + * @tparam T The Kokkos function type + * @param name The name of the parameter key of the Kokkos function to retrieve + * @return The reference of the Kokkos function of the concrete type with name associated with the + * parameter 'name'. Always store it in a reference wrapper if to be used on GPU. + */ + template + const T & getKokkosFunction(const std::string & name) const; + + /** + * Get a Kokkos function of a concrete type with a given name + * @tparam T The Kokkos function type + * @param name The name of the Kokkos function to retrieve + * @return The reference of the Kokkos function of the concrete type with name 'name'. Always + * store it in a reference wrapper if to be used on GPU. + */ + template + const T & getKokkosFunctionByName(const FunctionName & name) const; + + /** + * Determine if the Kokkos function exists + * @param param_name The name of the Kokkos function parameter + * @param index The index of the Kokkos function + * @return True if the Kokkos function exists + */ + bool hasKokkosFunction(const std::string & param_name) const; + + /** + * Determine if the Kokkos function exists + * @param name The name of the Kokkos function + * @return True if the Kokkos function exists + */ + bool hasKokkosFunctionByName(const FunctionName & name) const; +#endif + private: +#ifdef MOOSE_KOKKOS_ENABLED + /// Helper function to retrieve a Kokkos function of concrete type + const Moose::FunctionBase * getKokkosFunctionByNameHelper(const FunctionName & name) const; +#endif + + /// The object + const MooseObject & _fni_object; + /// Parameters of the object with this interface const InputParameters & _fni_params; @@ -87,3 +156,25 @@ class FunctionInterface /// Thread ID const THREAD_ID _fni_tid; }; + +#ifdef MOOSE_KOKKOS_ENABLED +template +const T & +FunctionInterface::getKokkosFunction(const std::string & name) const +{ + return getKokkosFunctionByName(_fni_params.get(name)); +} + +template +const T & +FunctionInterface::getKokkosFunctionByName(const FunctionName & name) const +{ + auto function = dynamic_cast(getKokkosFunctionByNameHelper(name)); + + if (!function) + _fni_object.mooseError( + "Kokkos function '", name, "' is not of type '", MooseUtils::prettyCppType(), "'"); + + return *function; +} +#endif diff --git a/framework/include/kokkos/actions/AddKokkosFunctionAction.h b/framework/include/kokkos/actions/AddKokkosFunctionAction.h new file mode 100644 index 000000000000..7e7a266a45f1 --- /dev/null +++ b/framework/include/kokkos/actions/AddKokkosFunctionAction.h @@ -0,0 +1,26 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosObjectAction.h" + +/** + * Adds a Kokkos function for GPU computation + * Associated with the [KokkosFunctions] syntax + */ +class AddKokkosFunctionAction : public KokkosObjectAction +{ +public: + static InputParameters validParams(); + + AddKokkosFunctionAction(const InputParameters & params); + + virtual void act() override; +}; diff --git a/framework/include/materials/MaterialPropertyInterface.h b/framework/include/materials/MaterialPropertyInterface.h index db9d786247a0..b44919e13841 100644 --- a/framework/include/materials/MaterialPropertyInterface.h +++ b/framework/include/materials/MaterialPropertyInterface.h @@ -917,7 +917,8 @@ MaterialPropertyInterface::getGenericMaterialPropertyByName(const MaterialProper const unsigned int state) { if (_is_kokkos_object) - mooseError("Attempted to retrieve a standard MOOSE material property from a Kokkos object."); + _mi_moose_object.mooseError( + "Attempted to retrieve a standard MOOSE material property from a Kokkos object."); if (_use_interpolated_state) { @@ -979,7 +980,8 @@ Moose::Kokkos::MaterialProperty MaterialPropertyInterface::getKokkosMaterialPropertyByName(const std::string & prop_name_in) { if (!_is_kokkos_object) - mooseError("Attempted to retrieve a Kokkos material property from a standard MOOSE object."); + _mi_moose_object.mooseError( + "Attempted to retrieve a Kokkos material property from a standard MOOSE object."); if constexpr (std::is_same_v) { diff --git a/framework/include/problems/FEProblemBase.h b/framework/include/problems/FEProblemBase.h index 3625b7734c0c..f2c3dbeb1b30 100644 --- a/framework/include/problems/FEProblemBase.h +++ b/framework/include/problems/FEProblemBase.h @@ -98,10 +98,16 @@ class MooseAppCoordTransform; class MortarUserObject; class SolutionInvalidity; +namespace Moose +{ +class FunctionBase; +} + #ifdef MOOSE_KOKKOS_ENABLED namespace Moose::Kokkos { class MaterialPropertyStorage; +class Function; } #endif @@ -644,6 +650,26 @@ class FEProblemBase : public SubProblem, public Restartable virtual bool hasFunction(const std::string & name, const THREAD_ID tid = 0); virtual Function & getFunction(const std::string & name, const THREAD_ID tid = 0); +#ifdef MOOSE_KOKKOS_ENABLED + virtual void addKokkosFunction(const std::string & type, + const std::string & name, + InputParameters & parameters); + virtual bool hasKokkosFunction(const std::string & name); + /** + * Get a Kokkos function in an abstract type + * @param name The Kokkos function name + * @returns The copy of the Kokkos function in the abstract type + */ + virtual Moose::Kokkos::Function getKokkosFunction(const std::string & name); + /** + * Get a Kokkos function in a concrete type + * @param name The Kokkos function name + * @returns The reference of the Kokkos function in the concrete type + */ + template + T & getKokkosFunction(const std::string & name); +#endif + /// Add a MeshDivision virtual void addMeshDivision(const std::string & type, const std::string & name, InputParameters & params); @@ -2901,6 +2927,10 @@ class FEProblemBase : public SubProblem, public Restartable /// functions MooseObjectWarehouse _functions; +#ifdef MOOSE_KOKKOS_ENABLED + MooseObjectWarehouse _kokkos_functions; +#endif + /// convergence warehouse MooseObjectWarehouse _convergences; @@ -3554,3 +3584,35 @@ FEProblemBase::clearCurrentResidualVectorTags() { _current_residual_vector_tags.clear(); } + +#ifdef MOOSE_KOKKOS_ENABLED +template +T & +FEProblemBase::getKokkosFunction(const std::string & name) +{ + if (!hasKokkosFunction(name)) + { + // If we didn't find a function, it might be a default function, attempt to construct one now + std::istringstream ss(name); + Real real_value; + + // First see if it's just a constant. If it is, build a ConstantFunction + if (ss >> real_value && ss.eof()) + { + InputParameters params = _factory.getValidParams("KokkosConstantFunction"); + params.set("value") = real_value; + addKokkosFunction("KokkosConstantFunction", ss.str(), params); + } + + // Try once more + if (!hasKokkosFunction(name)) + mooseError("Unable to find Kokkos function '" + name, "'"); + } + + auto * const ret = dynamic_cast(_kokkos_functions.getActiveObject(name).get()); + if (!ret) + mooseError("No Kokkos function named '", name, "' of appropriate type"); + + return *ret; +} +#endif diff --git a/framework/src/base/Moose.C b/framework/src/base/Moose.C index 5238b70bb193..6daf245d25d7 100644 --- a/framework/src/base/Moose.C +++ b/framework/src/base/Moose.C @@ -152,6 +152,10 @@ addActionTypes(Syntax & syntax) #endif registerMooseObjectTask("add_function", Function, false); +#ifdef MOOSE_KOKKOS_ENABLED + appendMooseObjectTask ("add_function", KokkosFunction); +#endif + registerMooseObjectTask("add_distribution", Distribution, false); registerMooseObjectTask("add_sampler", Sampler, false); @@ -583,6 +587,11 @@ associateSyntaxInner(Syntax & syntax, ActionFactory & /*action_factory*/) registerSyntax("AddFunctionAction", "Functions/*"); syntax.registerSyntaxType("Functions/*", "FunctionName"); +#ifdef MOOSE_KOKKOS_ENABLED + registerSyntax("AddKokkosFunctionAction", "KokkosFunctions/*"); + syntax.registerSyntaxType("KokkosFunctions/*", "FunctionName"); +#endif + registerSyntax("AddMeshDivisionAction", "MeshDivisions/*"); syntax.registerSyntaxType("MeshDivisions/*", "MeshDivisionName"); registerSyntax("AddConvergenceAction", "Convergence/*"); diff --git a/framework/src/functions/FunctionInterface.C b/framework/src/functions/FunctionInterface.C index 6ded32437143..42e4f3beff4a 100644 --- a/framework/src/functions/FunctionInterface.C +++ b/framework/src/functions/FunctionInterface.C @@ -20,26 +20,17 @@ FunctionInterface::validParams() } FunctionInterface::FunctionInterface(const MooseObject * moose_object) - : _fni_params(moose_object->parameters()), + : _fni_object(*moose_object), + _fni_params(moose_object->parameters()), _fni_feproblem(*_fni_params.getCheckedPointerParam("_fe_problem_base")), _fni_tid(_fni_params.have_parameter("_tid") ? _fni_params.get("_tid") : 0) { } -#ifdef MOOSE_KOKKOS_ENABLED -FunctionInterface::FunctionInterface(const FunctionInterface & object, - const Moose::Kokkos::FunctorCopy &) - : _fni_params(object._fni_params), - _fni_feproblem(object._fni_feproblem), - _fni_tid(object._fni_tid) -{ -} -#endif - const Function & FunctionInterface::getFunction(const std::string & name) const { - return _fni_feproblem.getFunction(_fni_params.get(name), _fni_tid); + return getFunctionByName(_fni_params.get(name)); } const Function & diff --git a/framework/src/interfaces/BlockRestrictable.C b/framework/src/interfaces/BlockRestrictable.C index fd885c5df640..27285f1908a2 100644 --- a/framework/src/interfaces/BlockRestrictable.C +++ b/framework/src/interfaces/BlockRestrictable.C @@ -88,7 +88,7 @@ BlockRestrictable::initializeBlockRestrictable(const MooseObject * moose_object) if (_blk_feproblem != NULL) { #ifdef MOOSE_KOKKOS_ENABLED - if (_moose_object->isKokkosObject({})) + if (_moose_object->isKokkosObject()) _blk_material_data = &_blk_feproblem->getKokkosMaterialData(Moose::BLOCK_MATERIAL_DATA); else #endif @@ -187,7 +187,7 @@ BlockRestrictable::initializeBlockRestrictable(const MooseObject * moose_object) _blk_dim = _blk_mesh->dimension(); #ifdef MOOSE_KOKKOS_ENABLED - if (moose_object->isKokkosObject({})) + if (moose_object->isKokkosObject()) initializeKokkosBlockRestrictable(_blk_mesh->getKokkosMesh()); #endif } diff --git a/framework/src/interfaces/BoundaryRestrictable.C b/framework/src/interfaces/BoundaryRestrictable.C index 3a64a0f2e54d..f3c389f55ca8 100644 --- a/framework/src/interfaces/BoundaryRestrictable.C +++ b/framework/src/interfaces/BoundaryRestrictable.C @@ -44,7 +44,7 @@ BoundaryRestrictable::BoundaryRestrictable(const MooseObject * moose_object, boo _bnd_tid(moose_object->isParamValid("_tid") ? moose_object->getParam("_tid") : 0), _bnd_material_data( #ifdef MOOSE_KOKKOS_ENABLED - moose_object->isKokkosObject({}) + moose_object->isKokkosObject() ? _bnd_feproblem->getKokkosMaterialData(Moose::BOUNDARY_MATERIAL_DATA) : #endif @@ -69,7 +69,7 @@ BoundaryRestrictable::BoundaryRestrictable(const MooseObject * moose_object, _bnd_tid(moose_object->isParamValid("_tid") ? moose_object->getParam("_tid") : 0), _bnd_material_data( #ifdef MOOSE_KOKKOS_ENABLED - moose_object->isKokkosObject({}) + moose_object->isKokkosObject() ? _bnd_feproblem->getKokkosMaterialData(Moose::BOUNDARY_MATERIAL_DATA) : #endif @@ -186,7 +186,7 @@ BoundaryRestrictable::initializeBoundaryRestrictable() } #ifdef MOOSE_KOKKOS_ENABLED - if (_moose_object.isKokkosObject({})) + if (_moose_object.isKokkosObject()) initializeKokkosBoundaryRestrictable(_bnd_mesh); #endif } diff --git a/framework/src/kokkos/actions/AddKokkosFunctionAction.K b/framework/src/kokkos/actions/AddKokkosFunctionAction.K new file mode 100644 index 000000000000..b64266ffd7fb --- /dev/null +++ b/framework/src/kokkos/actions/AddKokkosFunctionAction.K @@ -0,0 +1,36 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "AddKokkosFunctionAction.h" +#include "FEProblem.h" + +registerMooseAction("MooseApp", AddKokkosFunctionAction, "add_function"); + +InputParameters +AddKokkosFunctionAction::validParams() +{ + InputParameters params = KokkosObjectAction::validParams(); + params.addClassDescription("Add a Kokkos Function object to the simulation."); + return params; +} + +AddKokkosFunctionAction::AddKokkosFunctionAction(const InputParameters & params) + : KokkosObjectAction(params, "Function") +{ +} + +void +AddKokkosFunctionAction::act() +{ + FunctionParserBase fp; + std::string vars = "x,y,z,t,NaN,pi,e"; + if (fp.Parse(_name, vars) == -1) // -1 for success + mooseWarning("Function name '" + _name + "' could evaluate as a ParsedFunction"); + _problem->addKokkosFunction(_type, _name, _moose_object_pars); +} diff --git a/framework/src/kokkos/functions/KokkosFunctionInterface.K b/framework/src/kokkos/functions/KokkosFunctionInterface.K new file mode 100644 index 000000000000..726d5938aaf1 --- /dev/null +++ b/framework/src/kokkos/functions/KokkosFunctionInterface.K @@ -0,0 +1,61 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosFunction.h" + +#include "FunctionInterface.h" +#include "FEProblemBase.h" + +FunctionInterface::FunctionInterface(const FunctionInterface & object, + const Moose::Kokkos::FunctorCopy &) + : _fni_object(object._fni_object), + _fni_params(object._fni_params), + _fni_feproblem(object._fni_feproblem), + _fni_tid(object._fni_tid) +{ +} + +Moose::Kokkos::Function +FunctionInterface::getKokkosFunction(const std::string & name) const +{ + return getKokkosFunctionByName(_fni_params.get(name)); +} + +Moose::Kokkos::Function +FunctionInterface::getKokkosFunctionByName(const FunctionName & name) const +{ + if (!_fni_object.isKokkosObject()) + _fni_object.mooseError("Attempted to retrieve a Kokkos function from a non-Kokkos object."); + + return _fni_feproblem.getKokkosFunction(name); +} + +bool +FunctionInterface::hasKokkosFunction(const std::string & param_name) const +{ + return hasKokkosFunctionByName(_fni_params.get(param_name)); +} + +bool +FunctionInterface::hasKokkosFunctionByName(const FunctionName & name) const +{ + if (!_fni_object.isKokkosObject()) + _fni_object.mooseError("Attempted to retrieve a Kokkos function from a non-Kokkos object."); + + return _fni_feproblem.hasKokkosFunction(name); +} + +const Moose::FunctionBase * +FunctionInterface::getKokkosFunctionByNameHelper(const FunctionName & name) const +{ + if (!_fni_object.isKokkosObject()) + _fni_object.mooseError("Attempted to retrieve a Kokkos function from a non-Kokkos object."); + + return &_fni_feproblem.getKokkosFunction(name); +} diff --git a/framework/src/kokkos/problems/KokkosFEProblemBase.K b/framework/src/kokkos/problems/KokkosFEProblemBase.K index d79f799e48c8..99cd81eebb51 100644 --- a/framework/src/kokkos/problems/KokkosFEProblemBase.K +++ b/framework/src/kokkos/problems/KokkosFEProblemBase.K @@ -8,6 +8,7 @@ //* https://www.gnu.org/licenses/lgpl-2.1.html #include "KokkosTypes.h" +#include "KokkosFunction.h" #include "FEProblemBase.h" #include "DisplacedProblem.h" @@ -88,6 +89,54 @@ FEProblemBase::addKokkosBoundaryCondition(const std::string & bc_name, _has_kokkos_residual_objects = true; } +void +FEProblemBase::addKokkosAuxKernel(const std::string & kernel_name, + const std::string & name, + InputParameters & parameters) +{ + parallel_object_only(); + + setAuxKernelParamsAndLog(kernel_name, name, parameters, "KokkosAuxKernel"); + + _aux->addKokkosKernel(kernel_name, name, parameters); + + _has_kokkos_objects = true; +} + +void +FEProblemBase::addKokkosFunction(const std::string & type, + const std::string & name, + InputParameters & parameters) +{ + parallel_object_only(); + + parameters.set("_subproblem") = this; + + std::shared_ptr func = + _factory.create(type, name, parameters); + logAdd("Function", name, type, parameters); + _kokkos_functions.addObject(func); + + _has_kokkos_objects = true; +} + +bool +FEProblemBase::hasKokkosFunction(const std::string & name) +{ + return _kokkos_functions.hasActiveObject(name); +} + +Moose::Kokkos::Function +FEProblemBase::getKokkosFunction(const std::string & name) +{ + const auto & function = getKokkosFunction(name); + + std::shared_ptr wrapper = + Moose::Kokkos::FunctorRegistry::buildFunction(&function, function.type()); + + return Moose::Kokkos::Function(wrapper); +} + void FEProblemBase::addKokkosMaterial(const std::string & mat_name, const std::string & name, @@ -122,20 +171,6 @@ FEProblemBase::getKokkosMaterialData(Moose::MaterialDataType type, const MooseOb mooseError("FEProblemBase::getKokkosMaterialData(): Invalid MaterialDataType ", type); } -void -FEProblemBase::addKokkosAuxKernel(const std::string & kernel_name, - const std::string & name, - InputParameters & parameters) -{ - parallel_object_only(); - - setAuxKernelParamsAndLog(kernel_name, name, parameters, "KokkosAuxKernel"); - - _aux->addKokkosKernel(kernel_name, name, parameters); - - _has_kokkos_objects = true; -} - const std::set & FEProblemBase::getKokkosMaterialPropertyStorageConsumers(Moose::MaterialDataType type) const { diff --git a/framework/src/materials/MaterialPropertyInterface.C b/framework/src/materials/MaterialPropertyInterface.C index a1c9e5fbf1bf..c9da46cb4451 100644 --- a/framework/src/materials/MaterialPropertyInterface.C +++ b/framework/src/materials/MaterialPropertyInterface.C @@ -61,7 +61,7 @@ MaterialPropertyInterface::MaterialPropertyInterface(const MooseObject * moose_o _mi_subproblem(*_mi_params.getCheckedPointerParam("_subproblem")), _mi_tid(_mi_params.get("_tid")), #ifdef MOOSE_KOKKOS_ENABLED - _is_kokkos_object(_mi_moose_object.isKokkosObject({})), + _is_kokkos_object(_mi_moose_object.isKokkosObject()), #else _is_kokkos_object(false), #endif diff --git a/framework/src/problems/FEProblemBase.C b/framework/src/problems/FEProblemBase.C index b306f94bb98f..828bcab587e0 100644 --- a/framework/src/problems/FEProblemBase.C +++ b/framework/src/problems/FEProblemBase.C @@ -1110,6 +1110,10 @@ FEProblemBase::initialSetup() // ParsedFunctions _functions.initialSetup(tid); } + +#ifdef MOOSE_KOKKOS_ENABLED + _kokkos_functions.initialSetup(); +#endif } { @@ -1607,6 +1611,10 @@ FEProblemBase::timestepSetup() _functions.timestepSetup(tid); } +#ifdef MOOSE_KOKKOS_ENABLED + _kokkos_functions.timestepSetup(); +#endif + _aux->timestepSetup(); for (auto & sys : _solver_systems) sys->timestepSetup(); @@ -4806,6 +4814,10 @@ FEProblemBase::customSetup(const ExecFlagType & exec_type) _functions.customSetup(exec_type, tid); } +#ifdef MOOSE_KOKKOS_ENABLED + _kokkos_functions.customSetup(exec_type); +#endif + _aux->customSetup(exec_type); for (auto & nl : _nl) nl->customSetup(exec_type); @@ -7232,6 +7244,10 @@ FEProblemBase::computeResidualAndJacobian(const NumericVector & soln, _functions.residualSetup(tid); } +#ifdef MOOSE_KOKKOS_ENABLED + _kokkos_functions.residualSetup(); +#endif + computeSystems(EXEC_LINEAR); computeUserObjects(EXEC_LINEAR, Moose::POST_AUX); @@ -7464,6 +7480,10 @@ FEProblemBase::computeResidualTags(const std::set & tags) _functions.residualSetup(tid); } +#ifdef MOOSE_KOKKOS_ENABLED + _kokkos_functions.residualSetup(); +#endif + computeSystems(EXEC_LINEAR); computeUserObjects(EXEC_LINEAR, Moose::POST_AUX); @@ -7610,6 +7630,10 @@ FEProblemBase::computeJacobianTags(const std::set & tags) _functions.jacobianSetup(tid); } +#ifdef MOOSE_KOKKOS_ENABLED + _kokkos_functions.jacobianSetup(); +#endif + computeSystems(EXEC_NONLINEAR); computeUserObjects(EXEC_NONLINEAR, Moose::POST_AUX); @@ -7798,6 +7822,10 @@ FEProblemBase::computeLinearSystemTags(const NumericVector & soln, _functions.jacobianSetup(tid); } +#ifdef MOOSE_KOKKOS_ENABLED + _kokkos_functions.jacobianSetup(); +#endif + try { computeSystems(EXEC_NONLINEAR); From 645e5c9c92b367a5daaa1528991d5a7beedbdc73 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Thu, 23 Oct 2025 14:25:43 -0600 Subject: [PATCH 05/12] Kokkos function examples #30655 --- .../kokkos/auxkernels/KokkosFunctionAux.h | 29 +++++ framework/include/kokkos/base/KokkosUtils.h | 12 ++ .../kokkos/functions/KokkosConstantFunction.h | 34 ++++++ .../kokkos/functions/KokkosPiecewiseBase.h | 41 +++++++ .../functions/KokkosPiecewiseConstant.h | 103 ++++++++++++++++++ .../functions/KokkosPiecewiseTabularBase.h | 43 ++++++++ .../src/kokkos/auxkernels/KokkosFunctionAux.K | 27 +++++ .../kokkos/functions/KokkosConstantFunction.K | 28 +++++ .../kokkos/functions/KokkosPiecewiseBase.K | 31 ++++++ .../functions/KokkosPiecewiseConstant.K | 29 +++++ .../functions/KokkosPiecewiseTabularBase.K | 53 +++++++++ 11 files changed, 430 insertions(+) create mode 100644 framework/include/kokkos/auxkernels/KokkosFunctionAux.h create mode 100644 framework/include/kokkos/functions/KokkosConstantFunction.h create mode 100644 framework/include/kokkos/functions/KokkosPiecewiseBase.h create mode 100644 framework/include/kokkos/functions/KokkosPiecewiseConstant.h create mode 100644 framework/include/kokkos/functions/KokkosPiecewiseTabularBase.h create mode 100644 framework/src/kokkos/auxkernels/KokkosFunctionAux.K create mode 100644 framework/src/kokkos/functions/KokkosConstantFunction.K create mode 100644 framework/src/kokkos/functions/KokkosPiecewiseBase.K create mode 100644 framework/src/kokkos/functions/KokkosPiecewiseConstant.K create mode 100644 framework/src/kokkos/functions/KokkosPiecewiseTabularBase.K diff --git a/framework/include/kokkos/auxkernels/KokkosFunctionAux.h b/framework/include/kokkos/auxkernels/KokkosFunctionAux.h new file mode 100644 index 000000000000..b4dca77cb65e --- /dev/null +++ b/framework/include/kokkos/auxkernels/KokkosFunctionAux.h @@ -0,0 +1,29 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosAuxKernel.h" +#include "KokkosFunction.h" + +class KokkosFunctionAux : public Moose::Kokkos::AuxKernel +{ +public: + static InputParameters validParams(); + + KokkosFunctionAux(const InputParameters & parameters); + + KOKKOS_FUNCTION Real computeValue(const unsigned int qp, AssemblyDatum & datum) const + { + return _func.value(_t, datum.q_point(qp)); + } + +protected: + const Moose::Kokkos::Function _func; +}; diff --git a/framework/include/kokkos/base/KokkosUtils.h b/framework/include/kokkos/base/KokkosUtils.h index 889ec15ff0de..76e42e9c4d03 100644 --- a/framework/include/kokkos/base/KokkosUtils.h +++ b/framework/include/kokkos/base/KokkosUtils.h @@ -18,6 +18,18 @@ namespace Kokkos namespace Utils { +/** + * Returns the sign of a value + * @param x The value + * @returns The sign of the value + */ +template +KOKKOS_INLINE_FUNCTION T +sign(T x) +{ + return x >= 0.0 ? 1.0 : -1.0; +} + /** * Find a value in an array * @param target The target value to find diff --git a/framework/include/kokkos/functions/KokkosConstantFunction.h b/framework/include/kokkos/functions/KokkosConstantFunction.h new file mode 100644 index 000000000000..c0758060396a --- /dev/null +++ b/framework/include/kokkos/functions/KokkosConstantFunction.h @@ -0,0 +1,34 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosFunction.h" + +/** + * Class that represents constant function + */ +class KokkosConstantFunction : public Moose::Kokkos::FunctionBase +{ + using Real3 = Moose::Kokkos::Real3; + +public: + static InputParameters validParams(); + + KokkosConstantFunction(const InputParameters & parameters); + + KOKKOS_FUNCTION Real value(Real /* t */, Real3 /* p */) const { return _value; } + KOKKOS_FUNCTION Real timeIntegral(Real t1, Real t2, Real3 /* p */) const + { + return _value * (t2 - t1); + } + +protected: + Moose::Kokkos::Scalar _value; +}; diff --git a/framework/include/kokkos/functions/KokkosPiecewiseBase.h b/framework/include/kokkos/functions/KokkosPiecewiseBase.h new file mode 100644 index 000000000000..6eb769d92e90 --- /dev/null +++ b/framework/include/kokkos/functions/KokkosPiecewiseBase.h @@ -0,0 +1,41 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosFunction.h" + +/** + * Function base which provides a piecewise approximation to a specified (x,y) point data set. + * Derived classes can either directly implement the x/y data, or provide input parameter mechanisms + * for such data formulation. + */ +class KokkosPiecewiseBase : public Moose::Kokkos::FunctionBase +{ +public: + static InputParameters validParams(); + + KokkosPiecewiseBase(const InputParameters & parameters); + + KOKKOS_FUNCTION auto functionSize() const { return _raw_x.size(); } + KOKKOS_FUNCTION Real domain(const unsigned int i) const { return _raw_x[i]; } + KOKKOS_FUNCTION Real range(const unsigned int i) const { return _raw_y[i]; } + + /** + * Provides a means for explicitly setting the x and y data. This must + * be called in the constructor of inherited classes. + */ + virtual void setData(const std::vector & x, const std::vector & y); + +protected: + ///@{ raw function data as read + Moose::Kokkos::Array _raw_x; + Moose::Kokkos::Array _raw_y; + ///@} +}; diff --git a/framework/include/kokkos/functions/KokkosPiecewiseConstant.h b/framework/include/kokkos/functions/KokkosPiecewiseConstant.h new file mode 100644 index 000000000000..3c93ed17ed6c --- /dev/null +++ b/framework/include/kokkos/functions/KokkosPiecewiseConstant.h @@ -0,0 +1,103 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosPiecewiseTabularBase.h" +#include "KokkosUtils.h" + +using Moose::Kokkos::Real3; + +/** + * Function which provides a piecewise constant interpolation of a provided (x,y) point data set. + */ +class KokkosPiecewiseConstant : public KokkosPiecewiseTabularBase +{ +public: + static InputParameters validParams(); + + KokkosPiecewiseConstant(const InputParameters & parameters); + + KOKKOS_FUNCTION Real value(Real t, Real3 p) const; + KOKKOS_FUNCTION Real integral() const; + KOKKOS_FUNCTION Real average() const; + +private: + /// Enum for which direction to apply values + const enum class Direction { LEFT, RIGHT, LEFT_INCLUSIVE, RIGHT_INCLUSIVE } _direction; +}; + +KOKKOS_FUNCTION inline Real +KokkosPiecewiseConstant::value(Real t, Real3 p) const +{ + using Moose::Kokkos::Utils::sign; + + const Real x = _has_axis ? p(_axis) : t; + + const auto len = functionSize(); + const Real tolerance = 1.0e-14; + + // endpoint cases + if ((_direction == Direction::LEFT && x < (1 + tolerance * sign(domain(0))) * domain(0)) || + (_direction == Direction::RIGHT && x < (1 - tolerance * sign(domain(0))) * domain(0)) || + (_direction == Direction::LEFT_INCLUSIVE && + x < (1 - tolerance * sign(domain(0))) * domain(0)) || + (_direction == Direction::RIGHT_INCLUSIVE && + x < (1 + tolerance * sign(domain(0))) * domain(0))) + return _scale_factor * range(0); + else if ((_direction == Direction::LEFT && + x > (1 + tolerance * sign(domain(len - 1))) * domain(len - 1)) || + (_direction == Direction::RIGHT && + x > (1 - tolerance * sign(domain(len - 1))) * domain(len - 1)) || + (_direction == Direction::LEFT_INCLUSIVE && + x > (1 - tolerance * sign(domain(len - 1))) * domain(len - 1)) || + (_direction == Direction::RIGHT_INCLUSIVE && + x > (1 + tolerance * sign(domain(len - 1))) * domain(len - 1))) + return _scale_factor * range(len - 1); + + for (unsigned int i = 1; i < len; ++i) + { + if (_direction == Direction::LEFT && x < (1 + tolerance * sign(domain(i))) * domain(i)) + return _scale_factor * range(i - 1); + else if (_direction == Direction::LEFT_INCLUSIVE && + x < (1 - tolerance * sign(domain(i))) * domain(i)) + return _scale_factor * range(i - 1); + else if ((_direction == Direction::RIGHT && x < (1 - tolerance * sign(domain(i))) * domain(i))) + return _scale_factor * range(i); + else if ((_direction == Direction::RIGHT_INCLUSIVE && + x < (1 + tolerance * sign(domain(i))) * domain(i))) + return _scale_factor * range(i); + } + + return 0.0; +} + +KOKKOS_FUNCTION inline Real +KokkosPiecewiseConstant::integral() const +{ + const auto len = functionSize(); + + unsigned int offset = 0; + + if (_direction == Direction::RIGHT || _direction == Direction::RIGHT_INCLUSIVE) + offset = 1; + + Real sum = 0; + + for (unsigned int i = 0; i < len - 1; ++i) + sum += range(i + offset) * (domain(i + 1) - domain(i)); + + return _scale_factor * sum; +} + +KOKKOS_FUNCTION inline Real +KokkosPiecewiseConstant::average() const +{ + return integral() / (domain(functionSize() - 1) - domain(0)); +} diff --git a/framework/include/kokkos/functions/KokkosPiecewiseTabularBase.h b/framework/include/kokkos/functions/KokkosPiecewiseTabularBase.h new file mode 100644 index 000000000000..650dc09a06e5 --- /dev/null +++ b/framework/include/kokkos/functions/KokkosPiecewiseTabularBase.h @@ -0,0 +1,43 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosPiecewiseBase.h" + +#include "PiecewiseTabularInterface.h" + +/** + * Function base which provides a piecewise approximation to a provided (x,y) point data set via + * input parameter specifications. Derived classes, which control the order (constant, linear) of + * the approximation and how the (x,y) data set is generated, should be used directly. + */ +class KokkosPiecewiseTabularBase : public KokkosPiecewiseBase, public PiecewiseTabularInterface +{ +public: + static InputParameters validParams(); + + KokkosPiecewiseTabularBase(const InputParameters & parameters); + + /// Needed to load data from user objects that are not available at construction + void initialSetup() override; + +protected: + /// function value scale factor + Moose::Kokkos::Scalar _scale_factor; + +private: + using PiecewiseTabularInterface::buildFromFile; + using PiecewiseTabularInterface::buildFromJSON; + using PiecewiseTabularInterface::buildFromXandY; + using PiecewiseTabularInterface::buildFromXY; + + std::vector _data_x; + std::vector _data_y; +}; diff --git a/framework/src/kokkos/auxkernels/KokkosFunctionAux.K b/framework/src/kokkos/auxkernels/KokkosFunctionAux.K new file mode 100644 index 000000000000..3a329a26be27 --- /dev/null +++ b/framework/src/kokkos/auxkernels/KokkosFunctionAux.K @@ -0,0 +1,27 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosFunctionAux.h" + +registerKokkosAuxKernel("MooseApp", KokkosFunctionAux); + +InputParameters +KokkosFunctionAux::validParams() +{ + InputParameters params = AuxKernel::validParams(); + params.addClassDescription("Auxiliary Kernel that creates and updates a field variable by " + "sampling a function through space and time."); + params.addRequiredParam("function", "The function to use as the value"); + return params; +} + +KokkosFunctionAux::KokkosFunctionAux(const InputParameters & parameters) + : AuxKernel(parameters), _func(getKokkosFunction("function")) +{ +} diff --git a/framework/src/kokkos/functions/KokkosConstantFunction.K b/framework/src/kokkos/functions/KokkosConstantFunction.K new file mode 100644 index 000000000000..357e5c6e5075 --- /dev/null +++ b/framework/src/kokkos/functions/KokkosConstantFunction.K @@ -0,0 +1,28 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosConstantFunction.h" + +registerKokkosFunction("MooseApp", KokkosConstantFunction); + +InputParameters +KokkosConstantFunction::validParams() +{ + InputParameters params = FunctionBase::validParams(); + params.addClassDescription( + "A function that returns a constant value as defined by an input parameter."); + params.addParam("value", 0.0, "The constant value"); + params.declareControllable("value"); + return params; +} + +KokkosConstantFunction::KokkosConstantFunction(const InputParameters & parameters) + : FunctionBase(parameters), _value(getParam("value")) +{ +} diff --git a/framework/src/kokkos/functions/KokkosPiecewiseBase.K b/framework/src/kokkos/functions/KokkosPiecewiseBase.K new file mode 100644 index 000000000000..2f0c5e58eab8 --- /dev/null +++ b/framework/src/kokkos/functions/KokkosPiecewiseBase.K @@ -0,0 +1,31 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosPiecewiseBase.h" + +InputParameters +KokkosPiecewiseBase::validParams() +{ + return FunctionBase::validParams(); +} + +KokkosPiecewiseBase::KokkosPiecewiseBase(const InputParameters & parameters) + : FunctionBase(parameters) +{ +} + +void +KokkosPiecewiseBase::setData(const std::vector & x, const std::vector & y) +{ + _raw_x = x; + _raw_y = y; + + if (_raw_x.size() != _raw_y.size()) + mooseError("In KokkosPiecewiseBase ", _name, ": Lengths of x and y data do not match."); +} diff --git a/framework/src/kokkos/functions/KokkosPiecewiseConstant.K b/framework/src/kokkos/functions/KokkosPiecewiseConstant.K new file mode 100644 index 000000000000..c5a1a19a013e --- /dev/null +++ b/framework/src/kokkos/functions/KokkosPiecewiseConstant.K @@ -0,0 +1,29 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosPiecewiseConstant.h" + +registerKokkosFunction("MooseApp", KokkosPiecewiseConstant); + +InputParameters +KokkosPiecewiseConstant::validParams() +{ + InputParameters params = KokkosPiecewiseTabularBase::validParams(); + MooseEnum direction("LEFT RIGHT LEFT_INCLUSIVE RIGHT_INCLUSIVE", "LEFT"); + params.addParam( + "direction", direction, "Direction to look to find value: " + direction.getRawNames()); + params.addClassDescription("Defines piece-wise constant data using a set of x-y data pairs"); + return params; +} + +KokkosPiecewiseConstant::KokkosPiecewiseConstant(const InputParameters & parameters) + : KokkosPiecewiseTabularBase(parameters), + _direction(getParam("direction").getEnum()) +{ +} diff --git a/framework/src/kokkos/functions/KokkosPiecewiseTabularBase.K b/framework/src/kokkos/functions/KokkosPiecewiseTabularBase.K new file mode 100644 index 000000000000..19eeeefd4c5f --- /dev/null +++ b/framework/src/kokkos/functions/KokkosPiecewiseTabularBase.K @@ -0,0 +1,53 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosPiecewiseTabularBase.h" + +#include "JSONFileReader.h" + +InputParameters +KokkosPiecewiseTabularBase::validParams() +{ + InputParameters params = KokkosPiecewiseBase::validParams(); + params += PiecewiseTabularInterface::validParams(); + + params.addParam("scale_factor", 1.0, "Scale factor to be applied to the output values"); + params.declareControllable("scale_factor"); + + return params; +} + +KokkosPiecewiseTabularBase::KokkosPiecewiseTabularBase(const InputParameters & parameters) + : KokkosPiecewiseBase(parameters), + PiecewiseTabularInterface(*this, _data_x, _data_y), + _scale_factor(getParam("scale_factor")) +{ + // load the data + if (isParamValid("data_file")) + buildFromFile(_communicator); + else if (isParamValid("x") && isParamValid("y")) + buildFromXandY(); + else if (isParamValid("xy_data")) + buildFromXY(); + else if (!isParamValid("json_uo")) + mooseError("Unknown X-Y data source. Are you missing a parameter? Did you misspell one?"); + // JSON data is not available at construction as we rely on a user object +} + +void +KokkosPiecewiseTabularBase::initialSetup() +{ + // For JSON UO input, we need to wait for initialSetup to load the data + if (isParamValid("json_uo")) + buildFromJSON(getUserObject("json_uo")); + + // Copy data to GPU + _raw_x = _data_x; + _raw_y = _data_y; +} From 31ae89b00d3988c5d9c38f0c15a4daa0d9b501c4 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Thu, 23 Oct 2025 17:34:18 -0600 Subject: [PATCH 06/12] Update documentation #30655 --- .../kokkos/actions/AddKokkosFunctionAction.md | 16 +++++ .../kokkos/actions/AddKokkosKernelAction.md | 2 +- .../kokkos/auxkernels/KokkosFunctionAux.md | 18 +++++ .../functions/KokkosConstantFunction.md | 18 +++++ .../functions/KokkosPiecewiseConstant.md | 18 +++++ framework/doc/content/syntax/Kokkos/index.md | 1 + .../content/syntax/KokkosFunctions/index.md | 68 +++++++++++++++++++ framework/doc/unregister.yml | 4 +- 8 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 framework/doc/content/source/kokkos/actions/AddKokkosFunctionAction.md create mode 100644 framework/doc/content/source/kokkos/auxkernels/KokkosFunctionAux.md create mode 100644 framework/doc/content/source/kokkos/functions/KokkosConstantFunction.md create mode 100644 framework/doc/content/source/kokkos/functions/KokkosPiecewiseConstant.md create mode 100644 framework/doc/content/syntax/KokkosFunctions/index.md diff --git a/framework/doc/content/source/kokkos/actions/AddKokkosFunctionAction.md b/framework/doc/content/source/kokkos/actions/AddKokkosFunctionAction.md new file mode 100644 index 000000000000..bb48f8218cf7 --- /dev/null +++ b/framework/doc/content/source/kokkos/actions/AddKokkosFunctionAction.md @@ -0,0 +1,16 @@ +# AddKokkosFunctionAction + +!if! function=hasCapability('kokkos') + +!syntax description /KokkosFunctions/AddKokkosFunctionAction + +Kokkos `Functions` are specified as an object inside the `[KokkosFunctions]` block. This action adds them to the [Problem](syntax/Problem/index.md). + +More information about Kokkos `Functions` can be found on the [Kokkos Functions syntax documentation](syntax/KokkosFunctions/index.md). + +!syntax parameters /KokkosFunctions/AddKokkosFunctionAction + +!if-end! + +!else +!include kokkos/kokkos_warning.md diff --git a/framework/doc/content/source/kokkos/actions/AddKokkosKernelAction.md b/framework/doc/content/source/kokkos/actions/AddKokkosKernelAction.md index 319003635495..f66b9aaf10f3 100644 --- a/framework/doc/content/source/kokkos/actions/AddKokkosKernelAction.md +++ b/framework/doc/content/source/kokkos/actions/AddKokkosKernelAction.md @@ -1,4 +1,4 @@ -# AddKokkosKokkosKernelAction +# AddKokkosKernelAction !if! function=hasCapability('kokkos') diff --git a/framework/doc/content/source/kokkos/auxkernels/KokkosFunctionAux.md b/framework/doc/content/source/kokkos/auxkernels/KokkosFunctionAux.md new file mode 100644 index 000000000000..6ae5ada15699 --- /dev/null +++ b/framework/doc/content/source/kokkos/auxkernels/KokkosFunctionAux.md @@ -0,0 +1,18 @@ +# KokkosFunctionAux + +!if! function=hasCapability('kokkos') + +This is the Kokkos version of [FunctionAux](FunctionAux.md). See the original document for details. + +## Example Syntax + +!listing test/tests/kokkos/functions/piecewise_constant/kokkos_piecewise_constant.i block=KokkosAuxKernels + +!syntax parameters /KokkosAuxKernels/KokkosFunctionAux + +!syntax inputs /KokkosAuxKernels/KokkosFunctionAux + +!if-end! + +!else +!include kokkos/kokkos_warning.md diff --git a/framework/doc/content/source/kokkos/functions/KokkosConstantFunction.md b/framework/doc/content/source/kokkos/functions/KokkosConstantFunction.md new file mode 100644 index 000000000000..f6950d1ba9d9 --- /dev/null +++ b/framework/doc/content/source/kokkos/functions/KokkosConstantFunction.md @@ -0,0 +1,18 @@ +# KokkosConstantFunction + +!if! function=hasCapability('kokkos') + +This is the Kokkos version of [ConstantFunction](ConstantFunction.md). See the original document for details. + +## Example Syntax + +!listing test/tests/kokkos/functions/constant_function/kokkos_constant_function.i block=KokkosFunctions + +!syntax parameters /KokkosFunctions/KokkosConstantFunction + +!syntax inputs /KokkosFunctions/KokkosConstantFunction + +!if-end! + +!else +!include kokkos/kokkos_warning.md diff --git a/framework/doc/content/source/kokkos/functions/KokkosPiecewiseConstant.md b/framework/doc/content/source/kokkos/functions/KokkosPiecewiseConstant.md new file mode 100644 index 000000000000..247921fefc6a --- /dev/null +++ b/framework/doc/content/source/kokkos/functions/KokkosPiecewiseConstant.md @@ -0,0 +1,18 @@ +# KokkosPiecewiseConstant + +!if! function=hasCapability('kokkos') + +This is the Kokkos version of [PiecewiseConstant](PiecewiseConstant.md). See the original document for details. + +## Example Syntax + +!listing test/tests/kokkos/functions/piecewise_constant/kokkos_piecewise_constant.i block=KokkosFunctions + +!syntax parameters /KokkosFunctions/KokkosPiecewiseConstant + +!syntax inputs /KokkosFunctions/KokkosPiecewiseConstant + +!if-end! + +!else +!include kokkos/kokkos_warning.md diff --git a/framework/doc/content/syntax/Kokkos/index.md b/framework/doc/content/syntax/Kokkos/index.md index b9b36137de46..d35280ae251e 100644 --- a/framework/doc/content/syntax/Kokkos/index.md +++ b/framework/doc/content/syntax/Kokkos/index.md @@ -433,6 +433,7 @@ The following objects are currently available in Kokkos-MOOSE: - [BCs](syntax/KokkosBCs/index.md) - [Materials](syntax/KokkosMaterials/index.md) - [AuxKernels](syntax/KokkosAuxKernels/index.md) +- [Functions](syntax/KokkosFunctions/index.md) !if-end! diff --git a/framework/doc/content/syntax/KokkosFunctions/index.md b/framework/doc/content/syntax/KokkosFunctions/index.md new file mode 100644 index 000000000000..59b27670d41a --- /dev/null +++ b/framework/doc/content/syntax/KokkosFunctions/index.md @@ -0,0 +1,68 @@ +# Kokkos Functions System + +!if! function=hasCapability('kokkos') + +Before reading this documentation, consider reading the following materials first for a better understanding of this documentation: + +- [Functions System](syntax/Functions/index.md) to understand the MOOSE function system, +- [Getting Started with Kokkos-MOOSE](syntax/Kokkos/index.md) to understand the programming practices for Kokkos-MOOSE, +- [Kokkos Kernels System](syntax/KokkosKernels/index.md) to understand the common design pattern of objects in Kokkos-MOOSE. + +!alert note +Kokkos-MOOSE functions do not support automatic differention yet. + +A Kokkos-MOOSE function can be created by subclassing `Moose::Kokkos::FunctionBase` (not `Moose::Kokkos::Function`), which should be registered with `registerKokkosFunction()` instead of `registerMooseObject()`. +The signatures of the hook methods are defined as follows: + +```cpp +KOKKOS_FUNCTION Real value(Real t, Real3 p) const; +KOKKOS_FUNCTION Real3 vectorValue(Real t, Real3 p) const; +KOKKOS_FUNCTION Real3 gradient(Real t, Real3 p) const; +KOKKOS_FUNCTION Real3 curl(Real t, Real3 p) const; +KOKKOS_FUNCTION Real div(Real t, Real3 p) const; +KOKKOS_FUNCTION Real timeDerivative(Real t, Real3 p) const; +KOKKOS_FUNCTION Real timeIntegral(Real t1, Real t2, Real3 p) const; +KOKKOS_FUNCTION Real integral() const; +KOKKOS_FUNCTION Real average() const; +``` + +As in other Kokkos-MOOSE objects, they should be defined in the derived class as +*inlined public*+ methods instead of virtual override. +It is not mandatory to define each hook method in the derived class, but any hook method that was not defined in the derived class should not be called. + +See the following source codes of `KokkosPiecewiseConstant` for an example of a function: + +!listing framework/include/kokkos/functions/KokkosPiecewiseConstant.h id=kokkos-piecewise-constant-header + caption=The `KokkosPiecewiseConstant` header file. + +!listing framework/src/kokkos/functions/KokkosPiecewiseConstant.K id=kokkos-piecewise-constant-source language=cpp + caption=The `KokkosPiecewiseConstant` source file. + +Functions can be acquired in your object by calling `getKokkosFunction()`, where `T` should be your function type. +Namely, the actual type of the function should be known in advance. +This is the limitation of the current Kokkos-MOOSE function implementation which is being addressed, and more discussions can be found below. +It is recommended to store the acquired function in a `ReferenceWrapper` instance. +Otherwise, it should be stored as a copy, and it becomes your responsibility to maintain synchronization between the original function and the copy. + +## Use of Dynamic Polymorphism + +Unlike most of other Kokkos-MOOSE objects, functions are pluggable objects that are to be used in other objects. +Therefore, it is desired to use functions type-agnostically in your object so that any type of function can be plugged in. +Such design requires dynamic polymorphism, which is represented by virtual functions. +Although using virtual functions on GPU can sacrifice performance due to the inability of code inlining and vtable lookup and is still advised to avoid if possible, it can be useful for avoiding code duplications and improving code maintainability. + +In fact, Kokkos-MOOSE already has the code path for acquiring functions as a type-agnostic abstract type and using virtual dispatch to evaluate actual functions. +There is another non-template API `getKokkosFunction()` which returns an abstract type `Moose::Kokkos::Function`. +It is not the base class of your function but a wrapper class that contains your function and provides shims to call the function methods. + +However, that code path is currently blocked for GPU backends, and you will hit a runtime error when you attempt to use it. +While our target GPU backends (CUDA, HIP, and Intel SYCL) support virtual functions on GPU, it requires the relocatable device code (RDC) option to be enabled during compilation, which the current MOOSE build system is lacking. +Kokkos imposes a restriction to have a consistent RDC configuration across the whole software stack, but PETSc, from which MOOSE is currently acquring Kokkos, requires significant changes in its build system to enable RDC. +The non-templated `getKokkosFunction()` code path is hypothetically functional under the assumption that the MOOSE software stack can be built with the RDC option, but this hypothetical will only become a reality after a PETSc build system rework. +The PETSc and MOOSE team are looking into this issue, and it is expected to be resolved in the foreseeable future. + +!syntax list /KokkosFunctions objects=True actions=False subsystems=False + +!if-end! + +!else +!include kokkos/kokkos_warning.md diff --git a/framework/doc/unregister.yml b/framework/doc/unregister.yml index 74b0ef5f878a..f1515f5e02af 100644 --- a/framework/doc/unregister.yml +++ b/framework/doc/unregister.yml @@ -6,6 +6,7 @@ KokkosAuxKernel: AuxKernels/* KokkosNodalKernel: NodalKernels/* KokkosBoundaryCondition: BCs/* KokkosMaterial: Materials/* +KokkosFunction: Functions/* Kernel: KokkosKernels/* VectorKernel: KokkosKernels/* ArrayKernel: KokkosKernels/* @@ -16,4 +17,5 @@ ArrayAuxKernel: KokkosAuxKernels/* NodalKernel: KokkosNodalKernels/* BoundaryCondition: KokkosBCs/* MaterialBase: KokkosMaterials/* -FunctorMaterial: KokkosMaterials/* \ No newline at end of file +FunctorMaterial: KokkosMaterials/* +Function: KokkosFunctions/* \ No newline at end of file From b6f88ea824c33a3e80ef1a1ff6a9c8054692bbb8 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Fri, 24 Oct 2025 11:56:11 -0600 Subject: [PATCH 07/12] Add tests #30655 --- .../kernels/KokkosConstantFuncCoefDiffusion.h | 53 +++++++++ .../kokkos/kernels/KokkosFuncCoefDiffusion.h | 51 +++++++++ .../kernels/KokkosConstantFuncCoefDiffusion.K | 25 +++++ .../kokkos/kernels/KokkosFuncCoefDiffusion.K | 25 +++++ .../gold/kokkos_concrete_function_out.e | Bin 0 -> 58820 bytes .../concrete_type/kokkos_concrete_function.i | 63 +++++++++++ .../kokkos/functions/concrete_type/tests | 21 ++++ .../gold/kokkos_constant_function_out.e | Bin 0 -> 58820 bytes .../kokkos_constant_function.i | 58 ++++++++++ .../kokkos/functions/constant_function/tests | 12 +++ .../gold/kokkos_default_function_out.e | Bin 0 -> 58820 bytes .../kokkos_default_function.i | 51 +++++++++ .../kokkos/functions/default_function/tests | 12 +++ .../gold/kokkos_piecewise_constant_out.e | Bin 0 -> 78132 bytes .../kokkos_piecewise_constant.i | 101 ++++++++++++++++++ .../kokkos/functions/piecewise_constant/tests | 42 ++++++++ .../functions/piecewise_constant/xy.csv | 2 + .../functions/piecewise_constant/xy.json | 16 +++ 18 files changed, 532 insertions(+) create mode 100644 test/include/kokkos/kernels/KokkosConstantFuncCoefDiffusion.h create mode 100644 test/include/kokkos/kernels/KokkosFuncCoefDiffusion.h create mode 100644 test/src/kokkos/kernels/KokkosConstantFuncCoefDiffusion.K create mode 100644 test/src/kokkos/kernels/KokkosFuncCoefDiffusion.K create mode 100644 test/tests/kokkos/functions/concrete_type/gold/kokkos_concrete_function_out.e create mode 100644 test/tests/kokkos/functions/concrete_type/kokkos_concrete_function.i create mode 100644 test/tests/kokkos/functions/concrete_type/tests create mode 100644 test/tests/kokkos/functions/constant_function/gold/kokkos_constant_function_out.e create mode 100644 test/tests/kokkos/functions/constant_function/kokkos_constant_function.i create mode 100644 test/tests/kokkos/functions/constant_function/tests create mode 100644 test/tests/kokkos/functions/default_function/gold/kokkos_default_function_out.e create mode 100644 test/tests/kokkos/functions/default_function/kokkos_default_function.i create mode 100644 test/tests/kokkos/functions/default_function/tests create mode 100644 test/tests/kokkos/functions/piecewise_constant/gold/kokkos_piecewise_constant_out.e create mode 100644 test/tests/kokkos/functions/piecewise_constant/kokkos_piecewise_constant.i create mode 100644 test/tests/kokkos/functions/piecewise_constant/tests create mode 100644 test/tests/kokkos/functions/piecewise_constant/xy.csv create mode 100644 test/tests/kokkos/functions/piecewise_constant/xy.json diff --git a/test/include/kokkos/kernels/KokkosConstantFuncCoefDiffusion.h b/test/include/kokkos/kernels/KokkosConstantFuncCoefDiffusion.h new file mode 100644 index 000000000000..a6ed90d3aa06 --- /dev/null +++ b/test/include/kokkos/kernels/KokkosConstantFuncCoefDiffusion.h @@ -0,0 +1,53 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosKernel.h" +#include "KokkosConstantFunction.h" + +class KokkosConstantFuncCoefDiffusion : public Moose::Kokkos::Kernel +{ +public: + static InputParameters validParams(); + + KokkosConstantFuncCoefDiffusion(const InputParameters & parameters); + + KOKKOS_FUNCTION Real computeQpResidual(const unsigned int i, + const unsigned int qp, + AssemblyDatum & datum) const; + KOKKOS_FUNCTION Real computeQpJacobian(const unsigned int i, + const unsigned int j, + const unsigned int qp, + AssemblyDatum & datum) const; + +private: + Moose::Kokkos::ReferenceWrapper _function; +}; + +KOKKOS_FUNCTION inline Real +KokkosConstantFuncCoefDiffusion::computeQpResidual(const unsigned int i, + const unsigned int qp, + AssemblyDatum & datum) const +{ + const auto & func = static_cast(_function); + Real k = func.value(_t, datum.q_point(qp)); + return k * _grad_u(datum, qp) * _grad_test(datum, i, qp); +} + +KOKKOS_FUNCTION inline Real +KokkosConstantFuncCoefDiffusion::computeQpJacobian(const unsigned int i, + const unsigned int j, + const unsigned int qp, + AssemblyDatum & datum) const +{ + const auto & func = static_cast(_function); + Real k = func.value(_t, datum.q_point(qp)); + return k * _grad_phi(datum, j, qp) * _grad_test(datum, i, qp); +} diff --git a/test/include/kokkos/kernels/KokkosFuncCoefDiffusion.h b/test/include/kokkos/kernels/KokkosFuncCoefDiffusion.h new file mode 100644 index 000000000000..ff2e530a65a2 --- /dev/null +++ b/test/include/kokkos/kernels/KokkosFuncCoefDiffusion.h @@ -0,0 +1,51 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosKernel.h" +#include "KokkosFunction.h" + +class KokkosFuncCoefDiffusion : public Moose::Kokkos::Kernel +{ +public: + static InputParameters validParams(); + + KokkosFuncCoefDiffusion(const InputParameters & parameters); + + KOKKOS_FUNCTION Real computeQpResidual(const unsigned int i, + const unsigned int qp, + AssemblyDatum & datum) const; + KOKKOS_FUNCTION Real computeQpJacobian(const unsigned int i, + const unsigned int j, + const unsigned int qp, + AssemblyDatum & datum) const; + +private: + const Moose::Kokkos::Function _function; +}; + +KOKKOS_FUNCTION inline Real +KokkosFuncCoefDiffusion::computeQpResidual(const unsigned int i, + const unsigned int qp, + AssemblyDatum & datum) const +{ + Real k = _function.value(_t, datum.q_point(qp)); + return k * _grad_u(datum, qp) * _grad_test(datum, i, qp); +} + +KOKKOS_FUNCTION inline Real +KokkosFuncCoefDiffusion::computeQpJacobian(const unsigned int i, + const unsigned int j, + const unsigned int qp, + AssemblyDatum & datum) const +{ + Real k = _function.value(_t, datum.q_point(qp)); + return k * _grad_phi(datum, j, qp) * _grad_test(datum, i, qp); +} diff --git a/test/src/kokkos/kernels/KokkosConstantFuncCoefDiffusion.K b/test/src/kokkos/kernels/KokkosConstantFuncCoefDiffusion.K new file mode 100644 index 000000000000..69de38e6ec99 --- /dev/null +++ b/test/src/kokkos/kernels/KokkosConstantFuncCoefDiffusion.K @@ -0,0 +1,25 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosConstantFuncCoefDiffusion.h" + +registerKokkosResidualObject("MooseTestApp", KokkosConstantFuncCoefDiffusion); + +InputParameters +KokkosConstantFuncCoefDiffusion::validParams() +{ + InputParameters params = Kernel::validParams(); + params.addParam("coef", "1", "The function for conductivity"); + return params; +} + +KokkosConstantFuncCoefDiffusion::KokkosConstantFuncCoefDiffusion(const InputParameters & parameters) + : Kernel(parameters), _function(getKokkosFunction("coef")) +{ +} diff --git a/test/src/kokkos/kernels/KokkosFuncCoefDiffusion.K b/test/src/kokkos/kernels/KokkosFuncCoefDiffusion.K new file mode 100644 index 000000000000..d4a3b42f7856 --- /dev/null +++ b/test/src/kokkos/kernels/KokkosFuncCoefDiffusion.K @@ -0,0 +1,25 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosFuncCoefDiffusion.h" + +registerKokkosResidualObject("MooseTestApp", KokkosFuncCoefDiffusion); + +InputParameters +KokkosFuncCoefDiffusion::validParams() +{ + InputParameters params = Kernel::validParams(); + params.addParam("coef", "1", "The function for conductivity"); + return params; +} + +KokkosFuncCoefDiffusion::KokkosFuncCoefDiffusion(const InputParameters & parameters) + : Kernel(parameters), _function(getKokkosFunction("coef")) +{ +} diff --git a/test/tests/kokkos/functions/concrete_type/gold/kokkos_concrete_function_out.e b/test/tests/kokkos/functions/concrete_type/gold/kokkos_concrete_function_out.e new file mode 100644 index 0000000000000000000000000000000000000000..9871fcf89e35a56707a36809f704c49131155f4b GIT binary patch literal 58820 zcmeI52bdeh*~bqXj0*-k)X=iV7#ocHxdF$zosDsgZ5+UWth>|Q3F{;w>CP<#NN9;o zLiqxi5+DRaXraasFvSK!4Vdae5)x7lBp-sgF5Msus* zzBBVqpB+t{G`Uk%RaFm2cgUWQuCYiw6c4vVs;YLt$Et3bXiFrN&P0;(Nnbn1TU$a6 z(H8k6-|r@$=0;hK|{K#5~+*zh7@BZ`DoGpvVsPF&$sy5)ok z&L1W?f0*F%!`i|r36nHY=@yMQCh&2Ce3tx(MdQeis;@$Jf$R+F5l%)!ZIM(unuz0L z)lQHd#_zE1dvJFy5x!4jED_E`<4vJtA{x(x7AH~-NS|fGmh@83huf_&8bjLaf8_j6 z{=a)mcuBNH@*6f1f^tonW(0fRci`Uz@KMz;nsBC7vemHRt=cIQ&BWw^xWsl2T$EU} zD3J~|L>j}bu}rA3HC~@VoI;7#%wXOp_W2b^cV&x~Nr9X5#3T{{2pZh$yx1;Bo?@!0Q-O`266WcIX?(t`Z6ObeX=};aOYZM3Y3I>&JS84-mB}-= z?<@E3BK2)ZDx{XMRL%bUKty)7yx2;8nYb#{CAz9sjPE0wR@D|Lmra=3@|LZIPxbgz_U!<7nLxROrh%H=LZ zpy76O@A+~s^_655Zcm80a=CXP&~Q7tcca{^?ZHdmT@4hja1Ym?`-@}lHR-UM{-64r zd#`-Q+-u?{>1eLpdyhw;x!361+uR5@_ujMdc^4^I(rF0k#!u|+OT6FMWSQl@zob>q zwr~ukRrAEiaS2n>xJ2Fa!0Y%Q+N#DxC>5zk&1UMK-4VVA#9ZK9aJuBC9u5mvg&$##N`*moJetv!Y&-Xvi&)?2fNm?@hx2LZd-W3IQ|2eTU zQyz?K^3k~d@c-}61@(%lU$)jYc3Nq7lVm^gztZj-Iw!pPvQnQ-eOqbQ)T5PlP5obK z*VOZsc1?R!Vb_$)ZPl$yWHJdgrL%3OKq}hQoY^*`u40`@B)8$%S5?I#joWheS8Xr< z+sps9TmfwB<$qhw`|WX8DIjxHW^PL6>tr5Y=Fer`+sqfsysgZ!${d}{UCKP8%pdlG z$b6s7;hFg}Sx?*xvNvQO$d@7eLiU5~4>nQ!X{IRqk;mNGwl zDC986K#0su4uTAZ41o-V$h4o#Y06xs%t4NXjDn1YjDd`W9055JaulQnax}z+jDw7a zOn^*;OoG%xCPSt`j)6>t91EESnGTr&nF*N%nGN|WWDaC5WFF)=$b87xAjd;afSd?f z067VAGUOCU2(l0ohSWjoAq^0jif@E8L7E{^$f=M;kQhXI>v2c|BAvI>ASp;1B7K=w zNE>7^WC=u8V|IXaf^>#R9*f`PvxJG?Lk(L-!d!o)9nI1mXFzu0gehyxKj7CjDv^rc_)IGADL7d`qhO#Gt9Aq*40=pi;O zVd57(20$cC{Gx~0x`c^e^f(M6Vd8fn10fP7e$nl4h=hqwqMIoj;uqbdj7XUHMYo|4 z2@}8QA!SFx#4mabhe(+CMGq-c5+;7pqZ%S%;uk%ntVx*oMGq-=5+;7pZ8Su}217&- zQy#@HdK|$p@rxcuGEDrU$59LuzvxlJF!75XM>9=-2qIzP7d>JS2@}8Q(E^b$@r!P8h=hqwqFVxzq+fJ9jbY*!-BJt_ zzvz}`nD|Av48z1PdbBc3{Gvx2!^AIoEM}PaMUN#66Tj%Tlwmu-?F8uz*%7i6qzhzc zNLNTVNOwpN$S#nckX<3WL3%-YL%syr9kK^xPsm=7y&?NRz6{wHvL9rB$N`W8Azy(w zkUo%uAblYRL;67uf%JzAfE)@r3^EXMIAjoHFk}d1C}bF9IAjE*8Zr_x3Njip1~L|6 z%J@<6*FcVjxR7y>@sJ6SiI7Qc zRBTgh)Yzf1H)B^}{I3ooHY0Yj1EdqAqx_Kj$`Q#s$rnj;Nl!^DNe79e=r6j-bLE-k zNV)M4{mT)FgV>hXjo5_bx8$YdljMQaC-Tf4%M_D-V#i{`V!y%{yA_)idlg%i=NVh= z1Hago*vtfo*{dmdhP6r0w=bnQjvJZ znTNfxxk>0uPDPzr^%-Y)wKHt!5!J(w7(QGjUq=epE{W8)X2Nx`h+<;rWJ^+k^MNDA z9x-faS-_4SX#M}G@nmbp!9Kjmfb^kTV}It!jx*%AG^}t)eKR(r4#B>}Q?WfQJ)|iW zZEQq9OCphu49P^&nIR>%hz^b#)_=yG*c_=}q;|?yBEHbkalRIg#Z(TXG-o0y>{+7u z)MCr-QfI`dp+g;MDayPYN0QLVQPeBWlcH%cCuyoJF2_NQ=tVp2Op7El)KEGC_Lt*m z(~dZ^;&~;-app~(Q9E~D?d(wP%t?+txl-Sl9Zn%rBC%+?r83#FeI%Vu zq|!Ojmd>aBrGfLB(NZXjD_nXE$BiNWOI-xN8+mI zP@tR&sIjahNW6^qaRf}CgqRhx9;;|x@ zQ{&k6Iz_1xsi*a=ar&p1;-#txr_EM`BpL&!Rt`^`vrf4M7fkrD|isYPDjG2 z`sPqeB-5N|@T*wk#PM%*=hT~GA-;|fkr$A>7sqlFLB5}kU}RS63X&^1Pe z+jEFxu!_!WHdgKWD&BKgP@(p{U9Gq3p9}QV7YsU*s)(`}Ew^S8c!e8pE5;k5vJ!xo z`6cgp*^dI+?b$deaJF*}?anv|(00zD-5Cb~&P#>k=_tn1Wo4GkI3!Y;5Kb7t!~+KP zjp1m_q@TFVMP@`hbNf>I=1CV?$*wha1|$@p@Ck|XGrM@+dTnS+UER@A$6A@94 z3>sPfyaP1Uz+^HD$@K-+Fcra!q!-aFQfW9;HcgawNtYc}S0e;?5O8e~124i^{ zEaw$EGor;jL#idDEVbO?2MtS2uy`5{wrQCKh)Tg9(d=hQ!+|4gav8uP^N=1NHDO^;C#Z?8cJbU3?j+0xZ7AY<&%2p*M4;X7oq@tPT zmTa_ZoMf~^C z+z24fwTg!H#6oUMMOE!9hf=`2R4u=>P{S~q<=ko~H;c-LNzyou?h0ds$m~v{&zVek z{Z1IRMLE$x?1@QbT9X)eN0MH}$|NkeefRPW8tU4iA=*~73`)@(Xw{|=DC!j>+UGKX zrD_QK>I=Y9(Uo*mQ66SfeF3N%y9R%2qO&DXl+_RlXTTL?vDVPCf`x`fT2HN>0$OCT zHtZPA3W_NstXM7=X+3i^#3;b38O;{D+ya~fv>oe7PX>d=WoTGrrC#Re8n8u6cdzS~ zqL(M8;}xxrJSciNv<$nv8j5AH%9Qe9@-Pi7S}o{UL%Z^=S^*P?o-8jU;&P}IdP{2F z#q>i0)HXQG5X0LN5f0Mosb)8nNJ@4T!P?c=l7@LeL>aqIb?TWk>^~@3ubV~jG%v+0 zl=@hF@!Ah8nC-+0xMDbUi9`!#Nm`nUgN!*jSgDMKm#Enk%-P5;kzD;$^=0lY%Ow!g z0bY;O544@CQVaA_9*SS~^`RQdhT77hQ^VfW4Z2}SE%k(eK&>6kGv6E-YD>UU@h_bT6lG7LgqMX5Q%cXjiC$C?v z(=k9U=K>od8BByBbG%&HN(~F+zm-6AJ5ej=4$oAF)2b`NA&)vj)r)A>W1ohSzjDZP z6vsSb+RESXZ5>c0VLo!JfcdDPW+PEHV+vn}2vTK=)pQ?r_QlikCW%_5NN2Eo&5g=R zMf(~%7g$gm25AEHKwb~(!x=R-4hEF|&)gZaP$}vS zR62v1hF1N}D%qr!s>ACJIgC87t%hHKwb5yjR6G*<9B=Yg8_8s#Q<0@xv}2N^?Jl?4 z*EawC4c-$Ik;X|#lvZ=@e7g;_M7i=Cykm*_MM5ri1xdqt#Hkodg_XHJ=p}=EY5UH! z?K{&dzhm5fXBwu0r0XiLl*9`56bC6O+qb&q*AuqwvV2=QBNg3(F2D-!Oxvb5l6h5k zr97(VApQ33JqNG2V4D^aWDu}egeDU+&bJ*U{Zwn25thGhfFFjk%^8{%E zz6kG{ZQs+Lo6SLTd3!^jn}RZLBW+^&jqJ9#V_XhWsZ_5)Qo4f~haj_artezx?PUSl zt}j=7$9TI1Pc*b$t%%0!W3BkLElfOM=?u%kP0fk4{EnVBaQn(ZF9-WD+0tSUtdlvB zP_)#`vi6yxf&NusPsIR0zLsA?N&9JM^Pa!@O*Kz3&tnh7NI9KJwPJs=diMwif?!!Q zg&o@Jm$l@h)zq`&C@aA%J2j0)@>c}xH7o?e7#s0Al3qx%&y=)!ktBqlzRFuOa(cDfFK$Quw=_p;~am&i5<2i*&Ad< zD(CfYZZ zSYwSN$8eW_Y4UJx$UI}D=X64YPXaJVQ|C<`KRxfz z@scl`$pN^DhSpLkC>oY&(X4*tfQX{+M;UeP?Xan(!kN+(Z%xJ$`NhMi7bsJjG5?wa z@O$lN+D}cF$X|`NHyiXsqAhqCRo12T7COA9nAOuBSn=U47Yd_|>a8oanG@&Wd4_3k zHh6ia5px&eWbVPYTNC9hFjOpm?9R}ch;lB_yz%0hVSxHjYbdkA zcs(&3Z^AMumNFA2qXU?Y(y;L5cQ4jjk_3Q4S^Q#2 z77aIH@aX-DPhor!VxNYKWl2vNV^Ord9b%1MZ4JSes$%_j`-Yxly#%I$)A;=?|Cq`? z%Gl~Aug^r|t%+iT0ehgVnqf6l&Ri`m|=jWtKrPb?LrR;2e}euHM%Xcc97w%{M?)zQV0_) z4z&BFp?qLo`bqqQM>&a&mf1JdJynC}GWMk5_+Qiz9!q6VaG)Fqf&=aRaSk434fN?3 zsOT}5c?^T}ulf6tjI!+{hRJ3Aq)2|=1j}Wt<(ZN!o1kzMO-PSQ6B3Jy6 zeJ*?RHibv~2h)&)GBcXMaRgZOYK!8B;*`oT8=E5}tGojhJ)p0`wU4r2L*LON_CQRt zU;_aTU{KS+1+y*o&1m7EEH4@D)369IHyQ1tTm+b#jP^iI1YsXf1c#mHwRqYNZP(}t z)W7t_wlrrOqJ5?isHQEjSuoUyL;P~~`2~)$Hb+%B^8N(Y5NiO~&WLtT-e!(qe=^5@ zfq`CT=jkW)q3r;iY3Q$54cER+k-qR^oNipAQnc4FDZO_8R#2m#M7j8H1efZWV45Y7 zR-=F{BhP1VHu;Gs9c{147qWen^%jQ`ue$^Rs=(;UWt^dJjyNkhI13Qvj?PT8PD7bYEB*_LI?A^7VRm+o6E0aB)IeJe)%-=_(Pjn< zJB$n(R$Z#LDhT(T=3p>c*WPSo6BQaJ{HQ?-I)(+Uy?rk0&U$;bqLq@4lbr}EH6dwf z*s6gFHRC7evk8~uvr-RP^aa=+GppizMPzc$3_&rh&-v{Em6En;=MNL&m%*d#{L(Wo%(mi=XoJ`-w5lw7pN9PG^bQ|HW@Iiq%FIh$)`XjZ3I zxQZ?JAQ|mjJ=u?)N-QqER-u4aD}}Nj8*liaCH2mu#1b=(J1{HE_OEOkV2L_bSFkp} zPJ%9REQ%LC)cJkrIr#P{Baep;9^(uzxtGINXFHAomMAUunZizcsi_D1l0sk_8>k{h zHniAlsMZ{tah|C!@mQ+u+mw=vm_aKC11z!&Ly-1qn68&!1dBG}C$FTB4sn{+8$@%eFvjD@XRJfEG z2IwpiC<90sP78jh34JHk$5A7VAZb|gWoS`92+-53*x;w^s+w6PN?@)}{d8oK2W`j% zuY{{Bh=!3Ue$LDQ{H}dhM1GjD;Lz4AHb|5k+uJf8L_N%*}~)Le%ZZu`5TYc?y-XIUG4*yV*k*`x6!@Tee^q*+;ro=ccnYjed6%r z&Us?(V|1T(pZU|YF1LU57TsS`zdrf;ZJt4WzE3?4;Pc1uxexJq&+$3K$^X6P{&bP%`z+@r}omE5DrJ(t{T$(>8?$>fgB=Wg6g?pks~qPaxe06FVfszN08e}?z!X+CHFvb z7i#X>Pstsxxfl5DY#h1Al6w`o4diwwcV}{^ko!l?{pcfd@6XqFbBx@V$-SQ3Rpjoc zxgWnp?g(-pr@ni+Px;F8rE6~KG^cuBSG)7Q=iS)(vZrF_WnArE<=%PS16S5w{vO@G zx%X99ot(Pk0=k#E51;b>s-s>!f$jo#{g2$+KHRc}?z`@j-Pex1@b`bE`)~Kzzja#I ze9_)e%`|^ZKpWdta1iBIGca~4T8?`>Gx>Jwq`TQ0>_dWf*KlJ2t zrjp-_+*8TTmJc1@pOL#axi^u!jNH%2J%HRsat|PPDY@VD>H8SDJ;=R>+(nxE`p)F` zCHHi44-0w3Tb|tr(+{NS`Lhjk*X4@r|Z|9#u?o!RYvj@5N zkvo`Nm)swadlxkK7lB47~re zgVS`ExUcN}&ieO`Ow(<1HxBRlmouK5M0Xzb`@K)Udg^l^^?07okMg-!>*p=*#OEw0 z|8#QallzX3+f43Z&bn=$32zY1IT@q+(qR6gxr0}y@=fJ zksBv>laCwqrNb*e?vKgcgWL|puOxR9xnsz^j@&EAT}tlT zKJEr`e?o3=a&IAbGr8l)y~@XBIzDWI_R{DlT<2gQm7N0wq&)bR5d7S)n$i17~Y(GfH_gZov z^>L3QcMQ2b$bFXFmE=B1?wvk;f9TWqMd~|;+@s0ejog>Xy@cGy$$h}by`J0$sP79t z?vdo~O72VKUQF&fa@YB|x03rXxi6DDoBE<(CHZ<|PjX)^q3>Hh?s{_HA@?`rM#w#u z+`Y)%MD8l;`*W>tuU&oI-)e4KcXH7mRCbvp_e65{Blit*uO#gmUgWLr^eGT_2a-VVEFhuRTH`G-B@7!>x;n)9TV=ubB-PgZ$%JVxdTSIp( z^~?4vw0=)hpQY5}M|}QJK6eA3H<-^^PX1oxK1}XWzIZ=R?&;*NCbyd0SI8Yk?wRE7 zOYRfoj`ne1BzHNvH<3Gn+}FshCifh24F*$-RbK_a zbpPSL7QXJC_fI^T?kVo(Z~SHR1BVRp++W`OOrON)J6-Lrp?*tz`X#B)D4!lT^SML# zynpgJ3&|f&?tSDg_ig|T_hWMF$sI-RT5_{?q0`ZDtEum&KJEx|KP9)3 z+%e=nQbOO1Pv1|-y`B0R?k7I(XmTGW_e@_pEF*Ue^~Jo0%IDk2t){+)+f+i|OMG0+ z%PD)Bpt;v{A$JYAqc!)wD(V|0_Xz5{j{0WDxjKC>A$JhicZ-mqfH zkAMG=tKCPu@HMyGz2Dl0UUjv*(S7vEhI7AlYS#Vv`d@XrZsxaL?S997?&-I>UGdCj z&)x9n_ul#J;{#poI_@i-PCoOx+Mm&V(A{{zMbY2QeT(ip?v@FCcDi-U3c4%ZUnQGw zyJ44^bmvgN?D$^m^&0isL_JpU`CEL?eTdI_m;CRMJGTVa#B;Nc`#QM?F&+<-`#!lB zll!$2>AA_L?@H?H=RQL22ju<_xhIhOxG$f!kZbfUmwO4h$CLZCkNXC>TbK^t_37J} z>0tEzkox+$ull%uB=@)EUQF(rOy7Rwu4g(Ju0I`q?c;tx?jOm$oZLT>+n?OW$o(tz z{k|`Kf8*o+mE3p8y^`E_$sIuM<38@C{}|B$QQhuwAK;+x04`7Yf*xle|sE`6vw<~Qm7 z;+)^iTDfvZSGzm8FP`|jX)6}q;kmCY8JJi;TlPsc>7=PyW0JrgnmXZqt9o)=ikZaE#z~qAs_jt_#cwH!WZvH7*EWPtN0mi z7slrbu&Uk=u=2%(JTU^V1Hw ze|lD39`~mE@44^Rb9%bk&CV~}_h4J-{Qj$5?cV6FyKvor3#K&EZE~Od`PAL7zVKwv zeZJGDKdq}f$JOq+?u$QcTQz9S&*(njzVgIpcU6zUJT~2pe@7B2a(dQIC|2*Gwm-0E)xDk^2bsEzSM&zt!b&f86=06?;6s z(beuIFZ}+Uu3NKZ$t|vSZ*|u_an_w{f3S}3diSYaetrAq>(BGt=jY$^@PL*5T;sm@ z;)%wd8K`#oNz4-xn`GcMA2c_i_E{SuS@k>U)fjJGq3u zC?6_+OuqQ_y_)IX5%-tAbU^t~+{c*D-)1@-!h9{4`-YEu7r9T6`yF!ollueeyEoHe zD$}8X>1)y<+b`DkV7NagcLlivEV*y`^u34r8tzJR4|Q3F{;w>CP<#NN9;o zLiqxi5+DRaXraasFvSK!4Vdae5)x7lBp-sgF5Msus* zzBBVqpB+t{G`Uk%RaFm2cgUWQuCYiw6c4vVs;YLt$Et3bXiFrN&P0;(Nnbn1TU$a6 z(H8k6-|r@$=0;hK|{K#5~+*zh7@BZ`DoGpvVsPF&$sy5)ok z&L1W?f0*F%!`i|r36nHY=@yMQCh&2Ce3tx(MdQeis;@$Jf$R+F5l%)!ZIM(unuz0L z)lQHd#_zE1dvJFy5x!4jED_E`<4vJtA{x(x7AH~-NS|fGmh@83huf_&8bjLaf8_j6 z{=a)mcuBNH@*6f1f^tonW(0fRci`Uz@KMz;nsBC7vemHRt=cIQ&BWw^xWsl2T$EU} zD3J~|L>j}bu}rA3HC~@VoI;7#%wXOp_W2b^cV&x~Nr9X5#3T{{2pZh$yx1;Bo?@!0Q-O`266WcIX?(t`Z6ObeX=};aOYZM3Y3I>&JS84-mB}-= z?<@E3BK2)ZDx{XMRL%bUKty)7yx2;8nYb#{CAz9sjPE0wR@D|Lmra=3@|LZIPxbgz_U!<7nLxROrh%H=LZ zpy76O@A+~s^_655Zcm80a=CXP&~Q7tcca{^?ZHdmT@4hja1Ym?`-@}lHR-UM{-64r zd#`-Q+-u?{>1eLpdyhw;x!361+uR5@_ujMdc^4^I(rF0k#!u|+OT6FMWSQl@zob>q zwr~ukRrAEiaS2n>xJ2Fa!0Y%Q+N#DxC>5zk&1UMK-4VVA#9ZK9aJuBC9u5mvg&$##N`*moJetv!Y&-Xvi&)?2fNm?@hx2LZd-W3IQ|2eTU zQyz?K^3k~d@c-}61@(%lU$)jYc3Nq7lVm^gztZj-Iw!pPvQnQ-eOqbQ)T5PlP5obK z*VOZsc1?R!Vb_$)ZPl$yWHJdgrL%3OKq}hQoY^*`u40`@B)8$%S5?I#joWheS8Xr< z+sps9TmfwB<$qhw`|WX8DIjxHW^PL6>tr5Y=Fer`+sqfsysgZ!${d}{UCKP8%pdlG z$b6s7;hFg}Sx?*xvNvQO$d@7eLiU5~4>nQ!X{IRqk;mNGwl zDC986K#0su4uTAZ41o-V$h4o#Y06xs%t4NXjDn1YjDd`W9055JaulQnax}z+jDw7a zOn^*;OoG%xCPSt`j)6>t91EESnGTr&nF*N%nGN|WWDaC5WFF)=$b87xAjd;afSd?f z067VAGUOCU2(l0ohSWjoAq^0jif@E8L7E{^$f=M;kQhXI>v2c|BAvI>ASp;1B7K=w zNE>7^WC=u8V|IXaf^>#R9*f`PvxJG?Lk(L-!d!o)9nI1mXFzu0gehyxKj7CjDv^rc_)IGADL7d`qhO#Gt9Aq*40=pi;O zVd57(20$cC{Gx~0x`c^e^f(M6Vd8fn10fP7e$nl4h=hqwqMIoj;uqbdj7XUHMYo|4 z2@}8QA!SFx#4mabhe(+CMGq-c5+;7pqZ%S%;uk%ntVx*oMGq-=5+;7pZ8Su}217&- zQy#@HdK|$p@rxcuGEDrU$59LuzvxlJF!75XM>9=-2qIzP7d>JS2@}8Q(E^b$@r!P8h=hqwqFVxzq+fJ9jbY*!-BJt_ zzvz}`nD|Av48z1PdbBc3{Gvx2!^AIoEM}PaMUN#66Tj%Tlwmu-?F8uz*%7i6qzhzc zNLNTVNOwpN$S#nckX<3WL3%-YL%syr9kK^xPsm=7y&?NRz6{wHvL9rB$N`W8Azy(w zkUo%uAblYRL;67uf%JzAfE)@r3^EXMIAjoHFk}d1C}bF9IAjE*8Zr_x3Njip1~L|6 z%J@<6*FcVjxR7y>@sJ6SiI7Qc zRBTgh)Yzf1H)B^}{I3ooHY0Yj1EdqAqx_Kj$`Q#s$rnj;Nl!^DNe79e=r6j-bLE-k zNV)M4{mT)FgV>hXjo5_bx8$YdljMQaC-Tf4%M_D-V#i{`V!y%{yA_)idlg%i=NVh= z1Hago*vtfo*{dmdhP6r0w=bnQjvJZ znTNfxxk>0uPDPzr^%-Y)wKHt!5!J(w7(QGjUq=epE{W8)X2Nx`h+<;rWJ^+k^MNDA z9x-faS-_4SX#M}G@nmbp!9Kjmfb^kTV}It!jx*%AG^}t)eKR(r4#B>}Q?WfQJ)|iW zZEQq9OCphu49P^&nIR>%hz^b#)_=yG*c_=}q;|?yBEHbkalRIg#Z(TXG-o0y>{+7u z)MCr-QfI`dp+g;MDayPYN0QLVQPeBWlcH%cCuyoJF2_NQ=tVp2Op7El)KEGC_Lt*m z(~dZ^;&~;-app~(Q9E~D?d(wP%t?+txl-Sl9Zn%rBC%+?r83#FeI%Vu zq|!Ojmd>aBrGfLB(NZXjD_nXE$BiNWOI-xN8+mI zP@tR&sIjahNW6^qaRf}CgqRhx9;;|x@ zQ{&k6Iz_1xsi*a=ar&p1;-#txr_EM`BpL&!Rt`^`vrf4M7fkrD|isYPDjG2 z`sPqeB-5N|@T*wk#PM%*=hT~GA-;|fkr$A>7sqlFLB5}kU}RS63X&^1Pe z+jEFxu!_!WHdgKWD&BKgP@(p{U9Gq3p9}QV7YsU*s)(`}Ew^S8c!e8pE5;k5vJ!xo z`6cgp*^dI+?b$deaJF*}?anv|(00zD-5Cb~&P#>k=_tn1Wo4GkI3!Y;5Kb7t!~+KP zjp1m_q@TFVMP@`hbNf>I=1CV?$*wha1|$@p@Ck|XGrM@+dTnS+UER@A$6A@94 z3>sPfyaP1Uz+^HD$@K-+Fcra!q!-aFQfW9;HcgawNtYc}S0e;?5O8e~124i^{ zEaw$EGor;jL#idDEVbO?2MtS2uy`5{wrQCKh)Tg9(d=hQ!+|4gav8uP^N=1NHDO^;C#Z?8cJbU3?j+0xZ7AY<&%2p*M4;X7oq@tPT zmTa_ZoMf~^C z+z24fwTg!H#6oUMMOE!9hf=`2R4u=>P{S~q<=ko~H;c-LNzyou?h0ds$m~v{&zVek z{Z1IRMLE$x?1@QbT9X)eN0MH}$|NkeefRPW8tU4iA=*~73`)@(Xw{|=DC!j>+UGKX zrD_QK>I=Y9(Uo*mQ66SfeF3N%y9R%2qO&DXl+_RlXTTL?vDVPCf`x`fT2HN>0$OCT zHtZPA3W_NstXM7=X+3i^#3;b38O;{D+ya~fv>oe7PX>d=WoTGrrC#Re8n8u6cdzS~ zqL(M8;}xxrJSciNv<$nv8j5AH%9Qe9@-Pi7S}o{UL%Z^=S^*P?o-8jU;&P}IdP{2F z#q>i0)HXQG5X0LN5f0Mosb)8nNJ@4T!P?c=l7@LeL>aqIb?TWk>^~@3ubV~jG%v+0 zl=@hF@!Ah8nC-+0xMDbUi9`!#Nm`nUgN!*jSgDMKm#Enk%-P5;kzD;$^=0lY%Ow!g z0bY;O544@CQVaA_9*SS~^`RQdhT77hQ^VfW4Z2}SE%k(eK&>6kGv6E-YD>UU@h_bT6lG7LgqMX5Q%cXjiC$C?v z(=k9U=K>od8BByBbG%&HN(~F+zm-6AJ5ej=4$oAF)2b`NA&)vj)r)A>W1ohSzjDZP z6vsSb+RESXZ5>c0VLo!JfcdDPW+PEHV+vn}2vTK=)pQ?r_QlikCW%_5NN2Eo&5g=R zMf(~%7g$gm25AEHKwb~(!x=R-4hEF|&)gZaP$}vS zR62v1hF1N}D%qr!s>ACJIgC87t%hHKwb5yjR6G*<9B=Yg8_8s#Q<0@xv}2N^?Jl?4 z*EawC4c-$Ik;X|#lvZ=@e7g;_M7i=Cykm*_MM5ri1xdqt#Hkodg_XHJ=p}=EY5UH! z?K{&dzhm5fXBwu0r0XiLl*9`56bC6O+qb&q*AuqwvV2=QBNg3(F2D-!Oxvb5l6h5k zr97(VApQ33JqNG2V4D^aWDu}egeDU+&bJ*U{Zwn25thGhfFFjk%^8{%E zz6kG{ZQs+Lo6SLTd3!^jn}RZLBW+^&jqJ9#V_XhWsZ_5)Qo4f~haj_artezx?PUSl zt}j=7$9TI1Pc*b$t%%0!W3BkLElfOM=?u%kP0fk4{EnVBaQn(ZF9-WD+0tSUtdlvB zP_)#`vi6yxf&NusPsIR0zLsA?N&9JM^Pa!@O*Kz3&tnh7NI9KJwPJs=diMwif?!!Q zg&o@Jm$l@h)zq`&C@aA%J2j0)@>c}xH7o?e7#s0Al3qx%&y=)!ktBqlzRFuOa(cDfFK$Quw=_p;~am&i5<2i*&Ad< zD(CfYZZ zSYwSN$8eW_Y4UJx$UI}D=X64YPXaJVQ|C<`KRxfz z@scl`$pN^DhSpLkC>oY&(X4*tfQX{+M;UeP?Xan(!kN+(Z%xJ$`NhMi7bsJjG5?wa z@O$lN+D}cF$X|`NHyiXsqAhqCRo12T7COA9nAOuBSn=U47Yd_|>a8oanG@&Wd4_3k zHh6ia5px&eWbVPYTNC9hFjOpm?9R}ch;lB_yz%0hVSxHjYbdkA zcs(&3Z^AMumNFA2qXU?Y(y;L5cQ4jjk_3Q4S^Q#2 z77aIH@aX-DPhor!VxNYKWl2vNV^Ord9b%1MZ4JSes$%_j`-Yxly#%I$)A;=?|Cq`? z%Gl~Aug^r|t%+iT0ehgVnqf6l&Ri`m|=jWtKrPb?LrR;2e}euHM%Xcc97w%{M?)zQV0_) z4z&BFp?qLo`bqqQM>&a&mf1JdJynC}GWMk5_+Qiz9!q6VaG)Fqf&=aRaSk434fN?3 zsOT}5c?^T}ulf6tjI!+{hRJ3Aq)2|=1j}Wt<(ZN!o1kzMO-PSQ6B3Jy6 zeJ*?RHibv~2h)&)GBcXMaRgZOYK!8B;*`oT8=E5}tGojhJ)p0`wU4r2L*LON_CQRt zU;_aTU{KS+1+y*o&1m7EEH4@D)369IHyQ1tTm+b#jP^iI1YsXf1c#mHwRqYNZP(}t z)W7t_wlrrOqJ5?isHQEjSuoUyL;P~~`2~)$Hb+%B^8N(Y5NiO~&WLtT-e!(qe=^5@ zfq`CT=jkW)q3r;iY3Q$54cER+k-qR^oNipAQnc4FDZO_8R#2m#M7j8H1efZWV45Y7 zR-=F{BhP1VHu;Gs9c{147qWen^%jQ`ue$^Rs=(;UWt^dJjyNkhI13Qvj?PT8PD7bYEB*_LI?A^7VRm+o6E0aB)IeJe)%-=_(Pjn< zJB$n(R$Z#LDhT(T=3p>c*WPSo6BQaJ{HQ?-I)(+Uy?rk0&U$;bqLq@4lbr}EH6dwf z*s6gFHRC7evk8~uvr-RP^aa=+GppizMPzc$3_&rh&-v{Em6En;=MNL&m%*d#{L(Wo%(mi=XoJ`-w5lw7pN9PG^bQ|HW@Iiq%FIh$)`XjZ3I zxQZ?JAQ|mjJ=u?)N-QqER-u4aD}}Nj8*liaCH2mu#1b=(J1{HE_OEOkV2L_bSFkp} zPJ%9REQ%LC)cJkrIr#P{Baep;9^(uzxtGINXFHAomMAUunZizcsi_D1l0sk_8>k{h zHniAlsMZ{tah|C!@mQ+u+mw=vm_aKC11z!&Ly-1qn68&!1dBG}C$FTB4sn{+8$@%eFvjD@XRJfEG z2IwpiC<90sP78jh34JHk$5A7VAZb|gWoS`92+-53*x;w^s+w6PN?@)}{d8oK2W`j% zuY{{Bh=!3Ue$LDQ{H}dhM1GjD;Lz4AHb|5k+uJf8L_N%*}~)Le%ZZu`5TYc?y-XIUG4*yV*k*`x6!@Tee^q*+;ro=ccnYjed6%r z&Us?(V|1T(pZU|YF1LU57TsS`zdrf;ZJt4WzE3?4;Pc1uxexJq&+$3K$^X6P{&bP%`z+@r}omE5DrJ(t{T$(>8?$>fgB=Wg6g?pks~qPaxe06FVfszN08e}?z!X+CHFvb z7i#X>Pstsxxfl5DY#h1Al6w`o4diwwcV}{^ko!l?{pcfd@6XqFbBx@V$-SQ3Rpjoc zxgWnp?g(-pr@ni+Px;F8rE6~KG^cuBSG)7Q=iS)(vZrF_WnArE<=%PS16S5w{vO@G zx%X99ot(Pk0=k#E51;b>s-s>!f$jo#{g2$+KHRc}?z`@j-Pex1@b`bE`)~Kzzja#I ze9_)e%`|^ZKpWdta1iBIGca~4T8?`>Gx>Jwq`TQ0>_dWf*KlJ2t zrjp-_+*8TTmJc1@pOL#axi^u!jNH%2J%HRsat|PPDY@VD>H8SDJ;=R>+(nxE`p)F` zCHHi44-0w3Tb|tr(+{NS`Lhjk*X4@r|Z|9#u?o!RYvj@5N zkvo`Nm)swadlxkK7lB47~re zgVS`ExUcN}&ieO`Ow(<1HxBRlmouK5M0Xzb`@K)Udg^l^^?07okMg-!>*p=*#OEw0 z|8#QallzX3+f43Z&bn=$32zY1IT@q+(qR6gxr0}y@=fJ zksBv>laCwqrNb*e?vKgcgWL|puOxR9xnsz^j@&EAT}tlT zKJEr`e?o3=a&IAbGr8l)y~@XBIzDWI_R{DlT<2gQm7N0wq&)bR5d7S)n$i17~Y(GfH_gZov z^>L3QcMQ2b$bFXFmE=B1?wvk;f9TWqMd~|;+@s0ejog>Xy@cGy$$h}by`J0$sP79t z?vdo~O72VKUQF&fa@YB|x03rXxi6DDoBE<(CHZ<|PjX)^q3>Hh?s{_HA@?`rM#w#u z+`Y)%MD8l;`*W>tuU&oI-)e4KcXH7mRCbvp_e65{Blit*uO#gmUgWLr^eGT_2a-VVEFhuRTH`G-B@7!>x;n)9TV=ubB-PgZ$%JVxdTSIp( z^~?4vw0=)hpQY5}M|}QJK6eA3H<-^^PX1oxK1}XWzIZ=R?&;*NCbyd0SI8Yk?wRE7 zOYRfoj`ne1BzHNvH<3Gn+}FshCifh24F*$-RbK_a zbpPSL7QXJC_fI^T?kVo(Z~SHR1BVRp++W`OOrON)J6-Lrp?*tz`X#B)D4!lT^SML# zynpgJ3&|f&?tSDg_ig|T_hWMF$sI-RT5_{?q0`ZDtEum&KJEx|KP9)3 z+%e=nQbOO1Pv1|-y`B0R?k7I(XmTGW_e@_pEF*Ue^~Jo0%IDk2t){+)+f+i|OMG0+ z%PD)Bpt;v{A$JYAqc!)wD(V|0_Xz5{j{0WDxjKC>A$JhicZ-mqfH zkAMG=tKCPu@HMyGz2Dl0UUjv*(S7vEhI7AlYS#Vv`d@XrZsxaL?S997?&-I>UGdCj z&)x9n_ul#J;{#poI_@i-PCoOx+Mm&V(A{{zMbY2QeT(ip?v@FCcDi-U3c4%ZUnQGw zyJ44^bmvgN?D$^m^&0isL_JpU`CEL?eTdI_m;CRMJGTVa#B;Nc`#QM?F&+<-`#!lB zll!$2>AA_L?@H?H=RQL22ju<_xhIhOxG$f!kZbfUmwO4h$CLZCkNXC>TbK^t_37J} z>0tEzkox+$ull%uB=@)EUQF(rOy7Rwu4g(Ju0I`q?c;tx?jOm$oZLT>+n?OW$o(tz z{k|`Kf8*o+mE3p8y^`E_$sIuM<38@C{}|B$QQhuwAK;+x04`7Yf*xle|sE`6vw<~Qm7 z;+)^iTDfvZSGzm8FP`|jX)6}q;kmCY8JJi;TlPsc>7=PyW0JrgnmXZqt9o)=ikZaE#z~qAs_jt_#cwH!WZvH7*EWPtN0mi z7slrbu&Uk=u=2%(JTU^V1Hw ze|lD39`~mE@44^Rb9%bk&CV~}_h4J-{Qj$5?cV6FyKvor3#K&EZE~Od`PAL7zVKwv zeZJGDKdq}f$JOq+?u$QcTQz9S&*(njzVgIpcU6zUJT~2pe@7B2a(dQIC|2*Gwm-0E)xDk^2bsEzSM&zt!b&f86=06?;6s z(beuIFZ}+Uu3NKZ$t|vSZ*|u_an_w{f3S}3diSYaetrAq>(BGt=jY$^@PL*5T;sm@ z;)%wd8K`#oNz4-xn`GcMA2c_i_E{SuS@k>U)fjJGq3u zC?6_+OuqQ_y_)IX5%-tAbU^t~+{c*D-)1@-!h9{4`-YEu7r9T6`yF!ollueeyEoHe zD$}8X>1)y<+b`DkV7NagcLlivEV*y`^u34r8tzJR4Qd;CvAqO`|XpRI3ghL1latxD6CmAvqb0mRP)K%B@!u3Yo z1r!B!y>&r%L6JoiL_`4<5kUkLL^)hJ1^@5Wdp$EfM>5sS?q5IX?|Z45O3$xfRlT~a zYX)Zzsh*OOQXixqNK=ry4$Wn8S)E!+N;UYHlIpiRHH*)$dGHfmbzA|brO@ugPx$>* z{OocUYCih^OmOOAnC8&%rb3W<^x0D2C>1v$yrKR*i%;|OcWdh5K6@cN*AU)_8Fm+9 zG6e_O=P}3P(Y)|X@ekxQz;NRDFc9My9@aHRShpBq-8rnj{v4OUiF6AKLtJDS;v&Nk zXACQ|dNB;sM5LSDRpf?`ALD1t4~N|a`H^xpNCS{MAoZ;tyQNI?`s{8Od`zhYQf*+h zmG^>MCr)_tB8S`Rx4Vih9=F})x0JiRg^)h8kd5gjo)2znk=+4lPyYz}AO8PBr?tZF z#QX*uv4C>L{t^fdeqRB91AzDJ4n|}32QXW8Hh5EN`R#rOJ`gUnol8pHrKN74rBExf z1{{7%QNU&ML!2z`fIp4zL;Gw4Qcu`od5N0=s1J&M)do*p`tYw!;HGK?Atm55-izs^ zU#J%fT&b8Tb>V)G*X==D(eK6dMt@Bz6GG=2x!Djx-_^h08$PGHd{NIso%9RuJ)b_L z_5-&r+#mitytk_7>EF+TdsBVU&qLeLukhzhhQPXTfB5t8-bA10fO~7(++Z;k0*`4@ zL-^aVbp*&D@E&|V zUf|cg7vTMvrr|6-{S#zycBV8IEJ&|#mY%kftY&zBJxseGjpB*%z$-zXetS#2zX8^_ z7Oaq*R;-#M^8+GMZ5P5*Qd;Bvb!~2!OSAdWwh@HoBo#z|b}H7@7XM5SmpQzr_SNGD zW{cWN*m~^&5C+%92jNcxali5Tdg9?t`24@1Cncp_2K=cj?)Qm`6LrP9SikUlo#;dR z72wtltFtS+_dC3oxmfw)y;xW1R~Wa$`4FgctGXA?MmmyH+R(0>^crV#QFnt^8P;imknLc+!N55B3hx6!v;&1r9_#ORT zJ#Lte`W1ff1PIjc)%CqVzY*N(B8@c(sp2Ypl3-Ea7Q-Q80pE>(B; z^7nOj|2n)rRCV_t^K|!3x}QSB^o!Id^7)a^i{ziYnO+`r zSI6#W_}-qrUS0mN-9r zImi_tSAyW&WEYS$kaUm?5S;eIIZd3a#5u?wAU#2Pf%FFH1JW1dDv*94Ss?vERFDB6 z13?CX3ZV>Fe%>eO& z_&~5P696d#DF>+l!PS^*Ak{%?fM6b@U(aU@L%*KidfwwY0p`1I1A6{r82WWP&}~Au z2i-PwyU=Y!w-4P`bUSGREYu10&}|0&s0Z2(hM^zzKs&-P^rIeVQy7MR)C27c!_bd< zpsisT`cV(GI}AfV>VY%VFq8J+9&~^rIgAI1K%$M;3>nANAM;NW z!_bd<3NbqS(2u(1av1ti zkKr7K{sAB(Krjsbs7D?MhM^yI8wrA8s1xco3S>0Nb|c(2u%} z<1qB2ZuuOBe$-<;hoK+!xQ4^fk9u6oVdzKQCU6+)gt|=xxsLs)+awM{Kk7D_!_beq zP2n)~qi$0<4E?B^g~QN~x=jPYF!ZBtR*-4zN8JiQFbw^un+*iRP$$%_5JY1?>Q=;I z=ttd(ISl=%TM37uA9b^H82V9<=^TdsX&|K_7>0h-!vTU}=tn)AAQ*;z)XfEgVW<=8 z<_7VwA9b6-VdzKQyc~vp)Xm3X=ttfB9EN_>Bfw$kM?K0o4E?A_IftPi^{C)5^rLPw zIjkDE)j?{2)C8#oQX8ZWNL`Rrka{5XK^lNG1Zf0v9?1D17l1Sdxe%lYNK=qzAQyo& z2WbJ)5~LMKYmkdUE&*u+qJUfq(iY@0kai&LK{|kR1nC5FImi_tSAui~=>n1lk`9sq zk_plkq#H$Z&ctJ^8sCfcZOhq}G#c4dYC6@Z}4pq*3$sSZ+Ae&BtH z2<9E;3#K`yC#Dsq1I7{cN8Rwb_{>Bk(RiT#i3r94Z42!NZ36Qf^Aht3^8o7;d}hrA zMNdDpW3*wkU*x0RqRpbcqOIcdbX&a?{AgQfGlM`dkM%l6FDq^c^MK3%@q+k3{2&1k zEPLf3SjMnip$@1o+W%!BXs>98*&t{)nC2RY2futaUzf$W&L}-DQwDoAt6wWr3T7%J zM~@ztqhJP@zhWqOX22m{*e3xS^-7go*r?!j!u~zC3o9tI7sc`#+Bw))30=i@zhZMc zo#6EBU0Bp3vwPRfUW(hLbj|46qf2H+m#$dOqOUxA!AQ+lqC_F1Z%r&@OiundTQp8l zx}|mPo`#jU(jS;rbxc3 z6way8YyrQuz@Z5y?3{FZ1UL$qnUN5%ss~E{KiB06_!ZcPr*-mOekS&(KUq=I$NRtv z(`_ZN88sdDB~FL!X}w2bXBJQn6Du8Sw=j z73Es1!y$4QO0!?{!k#5IpB&h-J5%Y_Gb2O6mLkDRQ7{RWFhzsKIos|-bHb)t;!+f- z5rfh8DI+wGpEbmefc&K>WZIfC+GQvyijto@GG|G>6^i zOinf{%BX--8RK=^G@sAy^@T+nJD>6w0rN|sr78@kL0+2!LL&wCr}udXm@%X zb{lk6V_jyV%#n4w95E~>OOfk!qEc~EkMzw_I{Ie9OI1zrnLl zsJ~{S9M7f&+>BxSG^^KEVsUEz5_e&widl*a{tewZ@urxCUq{gJ1*G7MW8sN_-}h-S zGK+Va$&`RkgBRDR_!WYpFl-53W`LsB3L%U>b3m^>ZfVX^?^fD}eB7m}m?wg>I)6df$mPsF+SkZ|D&SuQh94kK2J0#^Oa^eJGTe zULm7G4Xfxm$c^j?_`mHrxL~jfo!3yTD)lA3=P)Tr?R%wKpV2=j(bN|hwB}8UvM^c> z_}%adH@vOrDzxKD0KCkPdC!;pD4^1wjRFNucFv*F83zQ)&N);%|DkrE0~YL1Ux)=887k+p zwXn?UvgtL9QA62GL9>B&&u+8$Y*si-BHU&~lx2Z7oFlea1K~|wn2Pdg2n$az9|?y} zkbaPuhNzn}Y?%8JM`yq>B4S_)^SthI$Q*~&Sx{)Tz>13xGA5ykZ5jha0v&$0ZaeZN z#TM{Eu2^6K%o4(c5)q=Tb?K4#yaQ-RK#wN`3HO=S(5u0WWH6#3q|h(}H%%mVVV51M zF2BX&q1nC=ihS~B17mp{EE@`)KB9$r2Cvg1EH%;M2O7qkUZAAj0U8*HH6%VG0+j!`P|e?rv zgoFu#q9UjeTdD)lPU8 zB@V-+Q55P5!w8YH+XH=0k2~1!1jCjnCj^8&F`^(6HKP$G-TupMw#kV6<@(sDUQ>lvmYi~^*Z(NLiaFTi0y*|8q> zWMI%Z3mO(&smJ-bLfE23-Rpvx(91*92^OuYJW%xD&@$NNRTwLaMW!SU!-o+tX0<@E z2JI5JY5{aZ^l*8>?ZQK)ptmIEUG#p4iP{DZGlb!7oCup~^+dC4aeFX3VqoR!YfM7} z5TXpbPAT>DH0;>LW20sfJtZ??7D{}Ky?7l7jGFC)6>z~&3fyid%#t{ZV}o>aQedUh zVXY9eDKKY)yF|kE6V;b~Z$d5uVLBk#iD*uW>usS<0BKghWnqPJr!h=K*$=xdXVNN~s0>O%kz6;q$}tHE&cFD$3W`;lQZc&`c9R19=+MTm52gQlvnER*W;B z@`jDImKZbR5I4GcHV4H1&$yAJp;Dv`RFuJ-hEn~_6x^icEr8b@!WagxtcJgXwb2op z*QGgr!<)RtMlu}eBxUIq+A+-0N|#&ZYn%W52JgXcttcB3C7_=>Uugr4QBM8_Z-?7f zisV?AnKZOPoMN#=n9R@?(*nPAZfDxLooUJ6F)qI|4W@#y>x!?GIHL9xn<**hwz`Qr zA^1nOUB)kBXC$dx&=HvAooQ#)MzTsrgRlE4rXTO^uBA%x0ewpyS|+C9pjZ2Jk(HjwZiVQIRfx&TQKneOJ}eg zTwLPz;qT~?fy-A8v>Y6P36&OkU;)mFSnTm$mX*&G0!FR^2URoy!Jg%G;q>c`SsVI1>wxUJcf&;rAYK5z<l#FK!6zy5g4~+$x1g{*y!f7NA(!=Xv?ReD}HT_P!k%) zudPi2K?IJg~8r4=jvRKfobuIYS1H&%?u|6S)MCI_WV70g#5WFAc@EXN;}5 z;FTOnvmqew3xYu!^rnl$@(GWYf*mx8#8%t`TYqQ4!YJ0U@MHEdtT9W$W4IH)!hoU` zh(gL2y`YA{md*e@10cNZ2>YlEG{_ctpcw1XfX-50qPZ-_afW=24ZCTBk!4c}cjPs! zfch((5aE~Cu%gmeZy^n(>^enJW+q$1naS30RqTCy zyyYv{Fd)9G2x=M{0r@t?=5|EBc@$2RZ&PqI3VZFtA>n>gK-}>a*}&^i`82fo%JeVC za^-=A?jol(&M77E1bHBx9HzmU6R{KoV8`51x%s&R^9+ZM$9>@p55RR72I8fl7*ob>;1_*qP)^DRu=s4!5y*=<1ouloFVK4Fg1aBro7p}%QHnVcVYE}AAEZzQ4Rqu zvGPYg4I_Z!&BBm^P?lvUb;E#;7)PPYSFjO4nA>#1F;n=vDdGa4Vmc!wMBxQ=oYlk~}H<_6zyNf}qP&VkPN3J~3SK=-= zG7O*OYwX~G0VS|{?)HXF*$A)+4fPWU>@KLP?QjO-OiLJBK2v-%UGP>r)c>(lQ9jD> zUT1QfQBfI)_us@OZ$k?OsVE=iGJo8+XE0afGo_-?DzY=qA2Y!4SH7dLCVV~h+*j)j4 ztigah5LeA$H51QY@Zk=M@ZP4WOiB8x8IHcOvWaOlXg0FFuPRR$lm?1ZqBTn>Ee)dr z6B{nC&TXh=Vd zzxgP8V54Q|4fUX^=5rbLq`~pOP(uV+B7@8W@i-7T&@M90=A%r&2>nbIJr0rL<X^fbERX?%~_a8thNzv7c#RFtZK% zVSOmO9L#ALS+VL|`8Ean!sT$fahys~UPBM|+9S7u>iS`nWB*2Qyq*cninx7Z6cAz< zeDY>vOg!vp2b+8(%SV~EIK+9~#SBmkPOyk1Yp)1*L1K<6OhwtJ%?Lyt%^S)W`AmUd zsf7|J&XB~BS)>KY8#Yc=!6|OBOEPAPY|laL{fs~0lQ)}6T0FAD%(Py8diCss$G<0V zhQ5BpS=_;yi6~cfW}0*w;$&LvUr?kd%hrdX**TtY3E3b4Wi=G@7tu$X=~%EstxIP2 zc(qkjc*JQAIws`Gn+HC?K9{MpURkXurKF;SCW7KkNJ<*EXrL@5 z@RReQgbVXYsRvo~JJ=pGI_Y~waB@x`g2J#q?6(I*O3J2Ta3=-~E3lRb%MS9&1NFgU z^k`I!#L=+LO=83xjUrQS*&o;F{T8P??xHo!!H&8nckJjXNrUH6!V)tacc8B@%fGU%1LM@ORKd#pItg@% z9d>x(L!93SJ%@-r%8c3~h=J`H^~{6#Q(5&Yy8w)q^>pko3kI-K}R z4tR)U)Qiz3fa2H3;#3W$da&XLr47sI!T5?a>{tW`M{DJ7Z>fUS68xB4g%TRO@;F;m z(`jjGDL8gdv{XeDve+MCkqdHyE0}Q2bice|3m|M3jI;hIpD6+uW1k?a2tRk?fc-bI z;};{8ql~9n+hfeyn91d{@1)StN$)HLwM~pPgq+#5bp;3J^z@T2S20sZ`)tptR1kCk`pN{nK zK^r*1i{aujqoHPppEJ_|k*<7K1b>(@>d@8@)=ZR(D%&z{Mnf8@#L5v+)Jz(R--Upm zQJ)!$39orNWM&XCvBX(W@Pj9zgbVY@M|o0OT6z*2so#Fjq0+kT+&|Byz_}FoFQffG!{ZOw3V}yIvfF|mq zMjy4=IN=-NZW_~8ed)GauWFX{v2ZWCb&k5ExwGR7SCjkV{FCZiFN~|L{M z+*UuUg8TBV*@D|P%`Uk0o9`q2A1x;C{qvFgUdn5^2JYDP>4H1t1%+rAZo+s4*b za9{a+wBWWInkl#qzW+*aFI;n<;QA-75c-}xdf3RVSSPrn_U96}w4LB8n=OL->hqNT z?fcdy?x6bwx5?nm#O=CJa8LVN8M)mk9mZ^D?t)IF@4n}Wdr2ngJMRwSuG&W2W>bhe zex2a{G;*qeJEHUBLf@Q%J1Bi?4iVheZ|$e_J=#}rFH^6g^u1vYrNh*&l)gLG3+}#k zKO49?X%`D_zdP0l?&jN<3+_d2t{2>wn#~Yg<@Dpky|%63HopB);_mD&xVsB?8MtcA z4+OXSmkR}V!^-`F+vF9i;J#36li;?w?S$Z_-gvLzUf|y@xZbZH5!`K8-(lePZB<8b zJ74l=!Ts=|rv$f=k|wy%EodXSt!w{TaO?iHTyW1@QAXVBE>zb=e|4I-?wE4 zxw|^3?{!#q=Yhq)ko(fF>W7D~m_Pf#404bDsBUaA=g9FT>&boNsQP*1_wSz7rx&^X zuTZ!BGWhGQBesy6=NJ05YxoHN8Fyxh+FMr;tp@k z`aVWnyO+2#72@Wn8@Qzp-9p@Sg}BweC2oU4O5ZJ$iQ9G=aX(HW?zLZV`fe877B5&0 z-08nA5ZqnsR|xI{bL$B1eV?`>?!wiAJNt(v#9i8exLIq7JMs{5)211@clIOh?32Xp z&vw4}AaMgHiTiec;?D0)+zw9=x91$eZGQApBX`1Z;#y`Aw{SZ4X95Jsyk+K$|-Vd>yJxOU;V3h)@w%}Aoth-b=kV@xudST zk6cTcx~jJC!2yF-lRG$1T|237;fRO3kUQ6@Zg@Yp+1{_-X7@*R)3-a0uY0mNxeKSO zTjs5N@%~5dVt1SR)ybtRns^tH>sz7jyj30f@U2#IZ<$B>)!M5oNP3-`Mf%tlkRC%P z(DNT(PS5@NQhMG~o9Q`^{*(Cr*O|YGxQ(tca9^+1gt*i95jXdF;?55ccjv{#U33d` zR}(kz1aqGuZk<^M?rXo^PTb-R#2ubX+<6tO?-t@deVDo1S>M^rZA07!UmLiuSNkJz z?Z2|V?=d%r^-X1cn-TZ1iL7rA<}M>{jXxW>uboIEt~H0aIWG}+?h)c{&miu?r-(bZ zh`8RjhaUbkPT=#t9-h4=KCmlLz zRZm+|{z4JrjepZ*bkDns$yu>qP)SkV`?Qv4w zu%%(j;#GH(`))0DbLvMqod(@Y?!eXRmc5GT*}7Zp7X6 zh>_c}Imh=W;+xepO{z^BBW@fmTSeT=xLza{RFiNyWjkdfQ;IpX%2OWe%IiM#Sa z;;K`K>jrzk^jSQYxTl^ZZuWlSuD{91Z8n#=U}u;wnNTh;KUQ)+^uM3Emopda0n_`` zOycI$V|{Nna+^#dZtr`Do4$d#?^sCRte=VNvJiLCR^tA0iuKJQ?nkSP+-CJy-}{*R z33Fdy?osC2nX9qBHJLk(xa%VFt;w^j?`h`FV($ISeSo>`i2F2@H%y0Lwh(vlaZcYa zjrulS!TLT)+>EQ3`x9~dXEXOX*7qsqo;K?H;j{?u>&!h)+^!cA_uqAhtBxbCdkb+F zFXnWZ#N5^D4xLD@`oili4%M17oZP>>t^ZeDek8E{$>QDY-l(qrpiQ9v@yE$Mu|!>e za_h7oKbk`B`cvx0&4b@6%xuE0NBz9wpQnDdG$VKF5_Owz+36R1^kuh=`gQS#cUGj{ z#BOJGSMK~tnYHS%+eH1=>R<8hH5ufNf0p#SCwR^$=~eU%>-7rj@c}>oa(?atdS3T3 zdQRPA#IN}dbN|TkFE+-z@CV`^J51ble z#v$U?{+0DTkGT&M_fH`^7WqF-Wqmn)*T6Ul^EoY_xnr1nko8Sreb+Fzg1C46WK0K} zLfoUXNZ$`S5;uJeaVIzAblAb%bk_GW=58VGozEJ%1)nl^5$9W9=I&-Lj04aPyZV^x zWA4?=y^FZ_Y%+4SRm44>N8As`5I5sy;!a^Z>o$Y9bN`)M0mp`tkKAW>4DlF}pXYpJolMQT?YL>`qX(_WiN6*~fd?^{6}4 z!2Q!ICbO%l-_#v2vBf1{v%5syed3mPs}DOyZf5_t zZYpv6{zTm8UN)xZ(Z3M)^`D8m>ptQh+)Ui{8;D!>GjrP#cW5?qFD35Z$3<|zAnreU zGS|u6CCuH=+yl%_VQwGhwqku(L~xh0zNeV`By-h%Mx8i)_UNVxn!zK~;sx0O%GU|IGpSW*!Anvyc zbH8NnTIT+lxht7_h`C49Jw@UQa$kLZL+c0MUabmu#hzgJ%9Z!~lrAS!a*wI&CN68V za?Lh!*KScaHgE8pGA*0k^VQEgckXf5xWVk+q;8w>(8F5?o=5J0m({P!mz;ep~p~u z?tA>a=lMC%?u+L<%iL0qcU}ZnVeV4mT80yM?B~Q)pJZ-6ah*2io=@CXi;4SI2P5~& zQ^bAmE#gkSgSc0}K-~Th5I65B;yOMg?w6&^O=a#@Be(NN;=VtKx$~I&EOS}kyr-G_ z6>}??+mN|UBDf8ho5$R*n0tV^%UR#G%ss~3+n9SLao-Nj>ykY+Dv0t-AdeXnan-T+}oHthq&7=W^OrgSNzS$O{>e? z!^8!|i>RUiDa?eyqn-*5fEWckvN^-a>wkpZHUb zFt?1kD~#MuHxu`%W5k`hj=4*jo6p=5=GJ5G1H|3Ci@4S+jNFb-5O<-Exg&_Hd`H|x zoIYK?C+^bu%x%it3g*5V!QI2$FPLj%eQPs!IqBQwFmo3&w*_-K9juchxL+`LD|0KD z+mgAfI31w9$NXISEORer?o3YKhmG7$w{iNOV(vEPKF8_XkGcJsyMwutnY*01q54wT zL#GAAeLA1GmLbG#+myM_G531nzR`xcbD8@+ac#AY`d*$z+(m7fyM(!S5%-xq=5A!} zADFwJxcf%2zH1}6H!ycPai@(X?q$`P`z~?QFJ$g2;!f?r+%k3lI&lTL|7bt|Ztc#y zRpH*H2dAj-c3ZT2@<)r@#-XQ@=R6%voS_vpY}S zF}BZx$L~?uO;^9^`%m?;Vkf)h>YiS`uK#q%7It4#_vY4{Gsog$w^-djqjqa|#cFnM zCH?NaL06FVx}EjOVm;RI^Y7;8_TlGQ_&Lze7SHL$+zRG~>SKy`Yv#6KuARA86Zh`p z%0LSVD8_U`wDaSFt;6ZpJnbA=7#EXkq+x`VQwa;Loep8V(v2L9%SxS%>4&*Kj8F@ zNQaM_bNUWu?jJdQCo%UBr|%Wa&1ddT=ALA3V`Dme)RVcT%-zb|zY_PJM$EmIxwjG5 zQiHe^4TyVJ-3ac#iQA(oaUXBV+=I*=&D?vKyN0>B%$>pdhSq09`ff;N?r7%TLEOTD z%)Ohr?-O@u7v|p0+%@WfU&R&FcS)W5e;B`ff=cek!SL0MH}>B;_i2^fC)Evqt?zjK z=||aJuYRhg9JuHeH@ll5yU$_#d)JnuZF zXnWeRTVLIK(*DGZebd?fgZe}Jd9TcVsWZ7R{>1u?iqP*~*5_f?<2ru+$NXHVN5pfX zJd5YmWxmGTgUp>1!Odgt-OR1U+2z_@l_ZQ|q#oUR^oyOcdiMyu}bDhlH k#@v-gF05~faUbH7*}j#AO?h1JG(1eD+!hLE+uqG;?Qp( zrX;igmtaar0;Yp80b`1>0h<;Gfdm36kmn}3q1^v_&i{<0k+iEBi+uRA=6T*dM`xwq zymQWbdYK6mCv93+SGOgk4`fG3ub2}L#lvk*UEL=5Sl2rfZF55DjFXg4`sx|awuPFa zZSqNe-&;P%6HQK9|GyJXF9~yEj(n*J(no&|HO3aGFA02W^Se;m$?&(EnecS93HL1s z?@LBBE|{T;BkXO+2_>Br?y3GEoUJ5WbJ7wh&zK)JtVG!G5@914wxzjGT;Wu_<%J2( z8zwk!nBWS-+QTUcQ)wahXo)3443l_LaT$#_C-4=yiAtkbG>$Z?I|i~9q$gy{a55Td zcT(wSB94!Bn?p7kx5<;2!tGfm{9DbjL^uvO3;wMDA2kAo6wYKNsSO+6y3I4uOiXTw zOX9#ld1mk6#t;`YM#lc_{f?8kgp;#>Ti>3pJe?lE~HLiBIV@8!O| z^g)Am#-FAwF_{wLQxb?}}QCNK>?p&IF zPI(TwD&(23?<~Bnqzn#80n!$h;@X=Yc#utgFYvm$UDdN7Tg9D7Mr>O^%3k4`d-s;| zJCx~6>L^y<+2WXE$4yivn6mXP#^U?9c7Ggywo>0OyuXP&`I6lKGV#>a?IVABsqd#% zVJf;x`D`xv-TwN+zVe%1dF6Ol-+e&7ORkia@?9w_%$3i52Z4s$&3E^cxG`KQ)P*bY zWUf5!enSvwxZQpCc=;~%m1q_2)(~^$bI(Sg;db}k+vU639^CltYoKt2dyxLT-%D`K zcTGHOqyMM==70C|yziQ4>sR`_`%C&6uF-efGT)tp@U0|cNkf8_pV;Jf{JyctZt}g! z5Br~u-)-Fdo)|eU`GUl8nX>1t6Y;---${hv>gIRv?v3wg{d1MK{APRkF|G>r z7uDoC@NE3qO6gz7lP?%1ymeBRiEZNg{m1x&dbT+cN;wgf%4vKS9r{CdfS3zm_pZ?& z2A&Pq+*I6rk*~;SVMu)C|4;mK<^Nv=*h?47b&YXVApY3TGTgYUhpWpr?mGV8xbNs+ z+Qxm4-#6|D&FZp^`yqdC+>ckv^}KD|zmjL%rTY6iG*3UTKHmF#@8{*$xGGA^rhhGd z#qh2vviom{otgY#T$7H*^@jhy{#;b9nDS+PU1O)!cGpSrBmJxGzN1sZEibF}*_5}{ zc1<~2ZP%3l)pkueUv1aaM^$!BzT8;d#zZEQP(xH3XAGpGEv=c2^U>8)XA;Sc`04BF zVovkMocwjQ{9nue8*>4$vFHDdIqhrDu3{j6Y)Zy3We!2c%Vf-6#>{#{WXxH{hh=_JhdS=mC%eAp;=?K@Nrtf((WXfyj`h3_lKsjDU=UjDn1Y90C~wITUgjC3h4onG#0-}X9*L(NpF+(Cf$t z8^$h-jTrkdwjyJ#VkcrVqLb)hY)1T|huDsUiC^>(JCZQ*iymTA5+;7pL+ne!#4mb? ztx1^pMGvt%2@}8QAvP#s;uk$Eh=hq>bQ=J%AY#X&$DWYA=odZqW|;Uzk9`;>e$iuJ zhKXPF5Sx}T@rxe&LnKW6qKDYJgo$7DI1nOX;C+WPNLg%$PD^Lx0wtRzvy-h!^AJT&0?7NMYq`u6Tj#- zhhgFuJ&t9V_(hMo3=_ZTaU8?MFS;GiFwseLI|1??`bD>S3=_ZTb|S;XFS?z?F!76S zCo@d^qFabz;uqacfk>G6MYk~I6#7NCMu>!oUv!H=BusP?-I^c{{i0hl!^AJTwJ=Ql zqFXDox<876+wV?M*ge+py)M8d=`dc+_SCVtVQ4I*LU7v16z2@{<}w*(|fzvy-< z!^AJTr5GlD(Jjp|@r!O5hKXPF$TCd)qDMQ!#4ma*WSICxj}C^3Uv%qa*d}l{h4g@I z2H7041*9jW7o<0&4`fToR*y7W*}JDz+&$YV6S1o3X1f{?`Z*n-M$N1hOflyY!InRU(phk}eYG5}y)R5)bl> zqQB@S_mz8AB9%Ue=wFG*Gl*@8-H1&{dP`bLI!PKxc_R1RtU@vICw440EcPpWv0JfO zu~)HGxu3Ds0q~1$iOr0MNE(|m#^jX*!jh0vAt^{2l7VC)lJ^!uB#%kH5*VA?1WKPLtKxX-%6kV|Ifj z88G-&Lc#R|PD){o4;Gm&uqI=nN~#TOPZM#epn_idDd*}{uZLQ>t0kJTB8j#(I3q@n z3XeKuc%-S>O2n;ULxzn!c<7LWhe>@|d`*ouPIJ<&RuQuJtCfY!Y?w1UQjD{#;e&>a z7$lXr&ujM7hM83bPROQGPCR4H!CLS9D6}S}qSlN^#u_%l8an*YAwv!wc8H3;?i6n5 za3a}ExH0A^Cf4q@B^5XyIPB1oLk}G`x*}kAH}w7g$?;@1V_^-cvw!-4^|3#5XUiIV zY#LTLIMRxR)Pu36c0QI?rU$pAqRq_+XiFs0&ftub&I~TQym?U6(7q0LLaP&5pjLTT z<9VT>WgQoe#Z(F-H)os_)~nHc8nE2F(;7Z%$Pi0f3Kd?KB~fVQDe7kDiP5x}lQh+Q zT$Y6r(S7W+HN{D0sG&3j{a==)P1~_%#0zqYWzCs9tzq_@hMA#;=@Tvg7$ zulE=8#$`LEIH|Z3OG^jn8+cjPyg`GdJytFKB@e+wQ<2tCo0DlxsD#MlLBlxyjdW6*>|`^P?AmZ-pO(Al zE?;r^ZQ9WZn$L0l^_IU@r~ELBZCFjwbTSr>;Lg~X)9T_R4ub))Da=VF6EW$ssBKav zGX?=-$6U0mCDx-b@-x^V<>8>g)hY~;I`gW?%1`X-`K6MY%Q*aIR#O{BCDj~mi^fWq z*OF7FjyrPJxamjAKx+k;WU{K)deL|^6Aj0tBx!=Lu$&E2rx;*!D$y2FJ4KNhP5BT6 zh}|pb2$Z={FHfE|TSl8Jy9AStgt23`r8F^!hTXIrrb&0tM_LY@jWo>VEJB-%;n#XD^>Ok%DekL*NSoKnpi#SBvGRlW7q+-{z)gd0Rd?Ab=& ztW=8N20`L5orqz0A(ZYc#YAH%=BF_t52@NV*>J4H1B-@!aMBCd_*xF*Cwwy-3~r&Q zS`M-#vRbV$F_jIzdpj|To?mOs$&8=i##DKaJaV`ZnJ=4hPzzw^Jr0|QWUz&u{rez! z+4MsNUUb!4{#LOHpeY(|38lmBPAFPj&Iak4nNE)s{Y3>%kVme1iS2Y`Qeo`*M?b#9 zq=g%ofizUW!o*!KV6-h6i$*XLQ|by*FY{p&@lp+K|0T9%m5DvAFUA~D*<_qHOPXZm z#~MuDkzXazVRuSDaImIGIh$501~Kf7aLhrKS~^XO{{}G|G={qxbMQC8FDjMEbXwbb z3q}=fLQBP1TGwvXn<8D`F2H|Py{xvXRjFc@h}z)m<>^kgEgX+mR4eOY=l^uQ>~D9W zBucGbE}R3Y)yv%$7h*Qu)XTbB?v}(a2(bD?_hs6eax-)u{m)5#^XujGf@m`2s972e z9?GF?>U1Vk8HSn?ZDF*&E2~f?nl^;V1@w1B)lyfG7OaV^Qar?5)msj6%*CSdT47eH zddp!0nEuc!J7CaGsw#~U^p>-k1Wwn(VN4unFC2mPZ#G@FY=R7?bvd;Vr1cmO8K8@gIjW&Mu=s z)DMwa@aVw|8%sMva;hm#eN~g;SjLoCh{TD7+Zvm~AuK3NBVj69x25|4USKSfFt!sQ z4zp>bN(c?@P!5xeL_D%{@W{$XilLzfCS`i3#1&Y>l!FB~?t|vYO2Z+Csa<4(SqAq71FbrdirJ@urYeX0@#hA6#g5lZSsoL z64^yPvag&{0F1RHQqfFnTkf&ztzUrK22@>Oi4sC3im$>U+JZQfc>p~CmJT5zLsD~@I?G=HW?8YaWvfbR|7H_Y6}%@Y z^37)G5L|`U+r=l*uvEF~5_KGo>;h5vF9373olD8{)LUH4SopiLp9+AXWH_=Q+)_|Z zAgAC!^+;KO_a7Rl=KJHAamCDaK{X$w!6~_;qZ!8vXOm4>^_BOu%3E)Z7&d0am{FsL zjj1RiF+kAVi~_OjGR}HyVbc3pU3~@+t4E_Uxs37t{N^E5^uq@paMh{U863LxJKLzVM%xW1JwvT+Gf!Okq%4CyR)aNAKg4Lw9|2T~6 z(xEBZUNR3#)*I-nP9ad#D@6>fVYw2*fAIwk%RR2dqk80_eu^&uRb|)Ytxa^Y1bSrE zg~ICEq9oQD`Yd3fVTsyPtEYfIl2{vd4C{7c$_T5L$|Y*gJPpwc@KufGGF^TK&I9^g zBrh!)EOk2#-6)x_F1Z>&8yB^2BtyDgq1shr&T#NRf&*l}cijD3!y6tbrwq z1^v{}u5znZzyuyorWX=%Iqw^-B{lA1+98$pCfvuitS(n91*!E^wHrz#B{@pOW95av zRuZGwQjZ%T9vPb|U7JyfvfshUh^`jJ)7ptqDD|=QO4k$nAB@nybSkxL+k1Kbv;7wC7ST}{wSekgtUFAr5$Hq@RD z%@4anH)w_-w$vR00x{bg3Zv5pXreJ1j-w#M&VyDgj|~KrdRZ^gZN#e7!bkT%Taj{P z=stglvtZe75Dklgs#0LnM4Iy$1gfZU2*Yw0{-Ov=pgB@xJpQv@8en*d` z_VMrV|Rn}9(;^$vaAex;h zmGir2s=?{2DZ(y~db^GLpgE8KIF$62cXCDX1`Z5cc`Ls41F9%2M6MUG5EayH}PmD>`8R#V$Mu}`wg*%3>w*WmoaCW z3W!c^lY^O*cnRb2Tfl1I0;lALA^B&JIK)zWd2OZqy!Z+P#NZ4vOEFAB0s|;+<@r9F5WQepH ztp-ybZtyJ!Y4AO)Yi1u*0lTH z9b8+}?#+L-^KCrEY4JyiGBaA^jXw4H|GyRE#gVX{A!>*~2Iy)Py=y{PkBtbKh%wrB z%+TGFEq%&@n;J?Jy>%KggT(#mKs17&_MT<`K zB-8((t_FISd$}qG0P=X*GD`ZlLvx^yIvLJY%=P#OVkiwK)Ml{}No_L5!PZ#6BF{$+VEY`=DhWv}T_`6*`qkF0>@XEOr=Pb-YLyiWmuT}^%j zY0fofqp_wC_EfVP#n$RnUiZg;@fE=y{!}80&GV)*K}Djs3`@P|Kb3P8y<4;Eir%gF z0j<1^C~a)iprNvd%m&d=fMsizdh7TTzSC_@lKX<}u+H1ZnrAQcR=C^UC1Nd044c-< zDi|P9x;(5fNTaEJ(6TEf|M8hwDqa0aJs$WE1li1csRrWXOj&frY`H1^Wu z6a8x#bK)(T)({#HB^L$x2g)X?m=kU;(oO4$T;gj*BevvVMSHsZ!Y}`E7%yL1i;e*_ zEMHm&1ePzY33RhnB_8ac;fr_5WWRcAV#B0y$4;$uahd|^MtZ@P3QWV%7HSFl}j19!0FT2JgcUD`&&T4CTTD3Jit-2a+0BhQ-J*ZoIue;|vbGbq&*h@r7No%^14~C-dK`w?2=Y z1BObakN-sR0KL0%*5{EiiC6qePycy2PeTFX9ffTm%lp681)$GUjJ!Z)k2zw!Rp}`N zy0}4)Ue(Jy$pA*9PJe2`SbkV#p#YDZ4>Vg_T{8?&9%>CS7}Sb$2*U9eOp{_N)7^O% zz-*L;#i!i3PwnraO&sQco~QPQlkTm4WR=#jMDc2}Adj4AfmXR>$mL`4`3I(36AKF& z1)uzv*zUoroC5D2eg~_LyLwDszq864PQd%(R~|z@$&|H zA+JDx=9KBi^f+ba@+VB1=diM1asO{2Wya>wE`t+a5Sn4J%u4p`)DAgJ8 z50qIm%x20#OKDj_k>A=3lqgl7HIv@A6gDM|M(ogUKc(Aagna|~((YVb7W4`NXjlX^ z8(>tO;NP%~HAmw(ySz9e*As{>)*hxjvg+MdSUO1e*1;*~avCvAurzS)xT%vT1_Tzw zpC}vrk&{?xnLCQcRW*1jV@(>)`9}%ivQ%sa2g(r#ILg%f%)uX713mf$%6d#?p0_0J zYZ(C3Rn+<<3f~=8m^Q(Hn9B<>C(9xzoH-|Nd11cspD0r8dU8kp$6!F+_fBVv$Zz)ucof;P(V?0`)6B^Ku{!+<&4F zsD>@DSTH1KB)MfxkeuQI<)~;+zEfGx?_?{1C<3vZ5%r$Dq07PgWcK|6KeC(H9wh?< zy#=e``ma-@ExZt~F(^|g`qwZit# zZMVu7vj0cUy^y`+7=eGF>YPXxsn%K%HbF}EDFQum+Br2_bOwJeUHm6XTID!(ShV!a zycXo&uw|$U$F-Ku$w5jRwf3`ohc7@Xm#^`(9S$8ddd%oiW8^(A6}&9RoMl?Ja~2>` zx;q@zZ`Vu>rj_2+svp^JiJcprLh+nl-8!u2R!IVEV81WIeW=>R498 zp}uAcWj!`__@E|r51z&pGmbkjGtBksJ@ge9hjlq%nKD)vuzp@chDLiViVYv?pkA~b zJZqHY$j3p4Si>qk4p6;wK>^F;7XOLDZbiAF2md*Rz%&+6Io`3pKxU)n99wa4xhL{y zM*i!RvXhva=>L%ou*511LHY-#Bl7CcXfxhzDRn-d)O7*?RfAJr*(9e$7j29V08GTN zcA!kr5U2&K-z1ekjqX0Lio^YyFf@2Ur;^w992L4)d~?_O!H zs;Gp(B$uqS(FIJtX2$P7>nUJYXPNqKa4KMlb%In8?=_2I{Y^!WD_`Ox%WEs5B}Q$6 zq%u168`+WI)o(1h%M$Wd5lG4x&azi*jvGLOb7sLbO4ZgVSDX>L{h~TEP0r1%mTr<` zW%3p)VM8U)Me<+k{pXKWZ_x9_ld&`rYxZJu`3}s2)XYV#JhFP1ai|GPIq|Ae%<$<} zH;QsJh*7y>s10+@l~z^)EW4{!{uv|=yPUGFIM4B^MG4cfC>j*k8qS$Ktzq_@hMA#; z=@rap=71{BQfM;iH~Ppw*LJ~g(NoCzAF4OY~Vuv#NAHoZ+FoYN?n%ncg?hE5{j#IcJIRDv$#GwoY6`p~Z zM+^!y$2a9|$=B4NKs5-IOki>kYdB%y?p#9ceCgYpgeVV7I4f|NF_%VkS| zUPkAt2Tzdl&;ZquS8DkpGNXY80hT*eN-^uLNt0*Iu4EWZ$FclKrn$T`M@GRwk6ixz z3SjH4X$ic0$z-VUc!^<`$t{``$D!#~DB#p_bI=s&l9KmN0ftkluy@t8gsUq^0jq#A zQVhds!>gAuNTLQaxYW(>e9G6*K6LdeRo4e;IiSW1V>%_2K z@OXkGO4Xah&0Xyc3!-7wTVLu;k7#(6d{Yk2bBOsFxsfE0#!?erVd_Z^{n7s;Z*=cZ z$t!qj-dYUQVxSfSwHT#(=$av0T@TC7|Z&h^WOwEgmEWHZ(iXsa98KU$FGh zx;2+QrQJ(Db;GY}i2ZHF8@Jlp9bwF+$z(0>u#U)T>_h5bAKUPAXy`-vZ1dHZc&^`$$+e)^zeFL?Tqr|7P* zpZj>q7I*#Qk96Onegg{idwm-9`4RQljrTu<_kEQ2d!F|hM*jPn`_JB*`@=*bcijt` zyUicS9Zv3EEymoZWFl+$Q?-T(d3>_?f~jLhTO|N+{-n$-=XAY$-R)=A>{5(?kSr4$QR^} z)7*=_b~cvWqshIF+$M7SklT~oBgy@P=KkX!9IJsn>}PBBvqS~sqvmS6sKvtu8V^*v&c5pYpKm&A#Ljf*y5YC=xcb@HMHySW z*V*^p_|UZtKmLU7-|YuR)SZ;N=VH27+mD_6=f#J=d;;Bh_Oe^-C4YP44Z44_pXu|+ z$jjdUjP6(V%D->=Wb5Ul=#H~jjh_2qrnxuWUiO;tSFPBw^#r;O^*hI--)&l-#eJy9 z&Afjb@B4|q-v?XsK9kAsNA7%bbM>8m-UrFuiQL=CJ&oKi$=!|IW^#8Ux0Bp+J^DUH z?v~{KlH3KF`({sa_agU9a`z;65xHl3xOX#Mz9jbsavQ1d=j86<(RUAW&ms3m9`3W` zeopSs$Za6^3v#z1H$i>BMecc;J94K&?)SE3JWL>W6}dlRJoF`Z1i1^z-Iv_+$-TzI zJ&W8<&AoR^avvaf5V){GVw501AEx3hs!mBpKIkZBf1tJIwETEuwli6))xKepT08m z?)@`&(0#%Uzo+NDBV%vRv9)`W{lM(iU;Oy(3J zx=r@8FZI~@^w<@2ueM+K({U|twEvmzKkSzW4_tcc-f6lW_A5JmwCt0^(sbMHwZpdl z^sMJ6(w#&7-uLJip+37)j~99WDDQi{zTd)4d7snCKafK1J>ja<3xy zDsq$LzUJYsAa}9m-qoAjYsp_+@F*CI=N%Xz0SjB zJhWRL?wjN?AKkShxl71>jodNhUQ6zk)Hg9!1AEx(YW-)43dTpx)pk*fhYd6ysCJ!w z^G@Rje6h{-bnkG(?;Cjbkw3fZQd_%M+Yf&>|IGf`pS$j|)A#AUdi2w_cAv4Ix#^A3 zf1Py{-6{6-|9tY+C(hf4?m_mdC)RHH_N`xN_qI*!HMjiugptWd=ss@0cI(~8pS${W zx@Xz1KO6et!PnnPcM0|Tl}EqD)MqL6c%Jv4!TS#4{Wj-)mXm)0x%ZKq8+X*tdjq*o zc(})sdkDE(lDm@JACmhpx%YbXy~(5R%hY!kxkr$@4Y_N`y^`GJg=D@kNDr&-`|VwL2mf3j{acD{o|I|+I`x7y!U|P|Fv_5Zq{CY_I>Bv z{>e}2-e^C!-Ifm=@X|oKgY1{Wqef=-f06D>_Ub7EPgrv1{&WwtUp?yAC)VB6PPfxu zH}9aE|7UGKy4%`so_F$#o1b<+-AAb32#1 zxQF{9xo47lE4d@ceTCeia-SzR?a}vha<3-$F>_R;}0 zw-yf)12sF?FzrBVjq&YUT8Dg;S!UDCmCwIyTGaUXrE6{NuCyP$c=tceiPX{E+aae)hQAE?IKKG`h#wFHGIZxpK#qbRV}@{b=#B8&`s3TA#)bsK;}>|8(AWIq&xd@3SrW z50bk*x%+##%gOy0xxXQICb>_N`#QNjJ=`tG&DEn?-_?x2zU00^?#bjnLhf2}Hz9Ya z);IPQxkq{QT}AEya$h9(6mlOV_ic~9CZ2neyQhcC_>J`NaE~MRSDyIoL46-*{BEkb zPk!RzE@u2bN$wrgcL=$Q$$dua`|p2}dk5p;pX6TU;hs!xirg#7?MrSuxxXg&_vBtj z?(fLG)5D!gZiw8Z=GOn4+(vSjkozvVmy!D(xw&+jv|6@G&y~}eS!#CX9JiatNy~V?A zBKNoCo=xrya?y`g@pLD-*O2=xxv!9WlZV?#?pq%2RB}gp^u3PU7397~?$sXdJaS(p z_YCT5xI;Z$qp#r_eJ}EGr;__9xhZmuz3fBoEsTdBQQu|M*VyX<5BE@VZzs2f+~JIe z9m)L(xr@ksh}>rw5ALCMwExlM-au|6xd)QlpWMad{*c_I~T>GZU1Ma)L?hd*)nE%yP@u8oI0P=LYZjS!ZvI*D3cY z`^D>f&%6JOJL%qGulnk<`$nG~qdVVTGw{@(blehg-B)KmXl=IjY+JiCso!Fcep^zX z=c&i;y#FfRcNy<@o98}Z4>z|@M?d%G5gWw^`8-I?6y%W#{?J(=98PY;wVae2QP_g{M--Fw{dd!Ky!bb@+E3JJRiMzwq|=-$?#7ME7KS z)iwQ>_Py~8x~JKzpE&o8e?8L6bzgZu_STA}Sih{?OQ~OO+)?Ybi2BT+9-s35?|ANu z^*ic*=001HdkMM!?RoC+lRJgnzmfYkx!dwNu--^dU6+$d$UL1^T<7t`u>^vzQuUJxUovta{3zX^&akb z$+gM-h}^a0_91sR;~~p<*p%_`ed>FKhuco>D01H?cQv_NkUN9i6uEWeUPSJHdAJM7 z9pT}=Oz!67P9ry6hU?YWaEF!Q8hxji;eOAppZ}u9xxd;_%cGj!+Tv>QP>Tms_K0iW z*m3^SRkt5T*L}5|eeXj1)Mu`^iSCW|gM*j8uz1ZmbkDY*NR3-QcjY~&l%>T>0i(L1GQxEy<(C0p~jk_xJ>qq;ah<*&+)#tzS)ro&jvs`CiS?x=>GbCf8>2GC%?tR&8<7r&;3(!Pbc>-a>w#{F649k zp$zv?>g&F{i27bf?rG#MA=f7N0uT2x>f20xA0cWlMIRC?d((bvnpw2Zz3J^KEb+%&m&kb5}e;auwb9`(J1T!-=SFymow5BExP+sM6* z+%e>Sm)v*Ay_no4avvnueU%O4p&a*+ZnzP0A0l^OkG}s!ePbSdE9E|H>gWFhVkYCF literal 0 HcmV?d00001 diff --git a/test/tests/kokkos/functions/piecewise_constant/kokkos_piecewise_constant.i b/test/tests/kokkos/functions/piecewise_constant/kokkos_piecewise_constant.i new file mode 100644 index 000000000000..b6eb1a09efa1 --- /dev/null +++ b/test/tests/kokkos/functions/piecewise_constant/kokkos_piecewise_constant.i @@ -0,0 +1,101 @@ +[Mesh] + type = GeneratedMesh + dim = 2 + nx = 10 + ny = 10 +[] + +[Variables] + [u] + [] +[] + +[AuxVariables] + [coef] + [] +[] + +[UserObjects] + [json] + type = JSONFileReader + filename = 'xy.json' + [] +[] + +[KokkosFunctions] + [func_x_y] + type = KokkosPiecewiseConstant + x = '0.0 0.5' + y = '2.0 3.0' + [] + [func_xy_data] + type = KokkosPiecewiseConstant + xy_data = '0.0 2.0 + 0.5 3.0' + [] + [func_csv] + type = KokkosPiecewiseConstant + data_file = xy.csv + [] + [func_json] + type = KokkosPiecewiseConstant + json_uo = json + x_keys = 'data x value' + y_keys = 'data y value' + [] +[] + +[KokkosKernels] + [diff] + type = KokkosFuncCoefDiffusion + variable = u + coef = func_x_y + [] + [time] + type = KokkosTimeDerivative + variable = u + [] +[] + +[KokkosAuxKernels] + [coef_aux] + type = KokkosFunctionAux + variable = coef + function = func_x_y + [] +[] + +[KokkosBCs] + [left] + type = KokkosDirichletBC + variable = u + boundary = left + value = 0 + [] + [right] + type = KokkosNeumannBC + variable = u + boundary = right + value = 1 + [] +[] + +[Postprocessors] + [coef] + type = ElementIntegralVariablePostprocessor + variable = coef + [] +[] + +[Executioner] + type = Transient + num_steps = 10 + dt = 0.1 + solve_type = PJFNK + petsc_options_iname = '-pc_type -pc_hypre_type' + petsc_options_value = 'hypre boomeramg' +[] + +[Outputs] + exodus = true +[] diff --git a/test/tests/kokkos/functions/piecewise_constant/tests b/test/tests/kokkos/functions/piecewise_constant/tests new file mode 100644 index 000000000000..50598de8c1db --- /dev/null +++ b/test/tests/kokkos/functions/piecewise_constant/tests @@ -0,0 +1,42 @@ +[Tests] + issues = '#30655' + design = 'KokkosPiecewiseConstant.md' + [piecewise_constant] + requirement = 'The Kokkos function system shall include a piece-wise constant function based on x- and y-data' + [x_y] + type = 'Exodiff' + input = 'kokkos_piecewise_constant.i' + exodiff = 'kokkos_piecewise_constant_out.e' + capabilities = 'kokkos' + compute_devices = 'cpu cuda' + detail = 'provided separately as two vectors in the input,' + [] + [xy_data] + type = 'Exodiff' + input = 'kokkos_piecewise_constant.i' + cli_args = 'KokkosKernels/diff/coef=func_xy_data KokkosAuxKernels/coef_aux/function=func_xy_data' + exodiff = 'kokkos_piecewise_constant_out.e' + capabilities = 'kokkos' + compute_devices = 'cpu cuda' + detail = 'provided as a table in the input,' + [] + [csv] + type = 'Exodiff' + input = 'kokkos_piecewise_constant.i' + cli_args = 'KokkosKernels/diff/coef=func_csv KokkosAuxKernels/coef_aux/function=func_csv' + exodiff = 'kokkos_piecewise_constant_out.e' + capabilities = 'kokkos' + compute_devices = 'cpu cuda' + detail = 'provided as a CSV file,' + [] + [json] + type = 'Exodiff' + input = 'kokkos_piecewise_constant.i' + cli_args = 'KokkosKernels/diff/coef=func_json KokkosAuxKernels/coef_aux/function=func_json' + exodiff = 'kokkos_piecewise_constant_out.e' + capabilities = 'kokkos' + compute_devices = 'cpu cuda' + detail = 'provided as a JSON file.' + [] + [] +[] diff --git a/test/tests/kokkos/functions/piecewise_constant/xy.csv b/test/tests/kokkos/functions/piecewise_constant/xy.csv new file mode 100644 index 000000000000..2f3f6c709c9b --- /dev/null +++ b/test/tests/kokkos/functions/piecewise_constant/xy.csv @@ -0,0 +1,2 @@ +0.0 0.5 +2.0 3.0 diff --git a/test/tests/kokkos/functions/piecewise_constant/xy.json b/test/tests/kokkos/functions/piecewise_constant/xy.json new file mode 100644 index 000000000000..c4f8962ebef2 --- /dev/null +++ b/test/tests/kokkos/functions/piecewise_constant/xy.json @@ -0,0 +1,16 @@ +{ + "data": { + "x": { + "value": [ + 0.0, + 0.5 + ] + }, + "y": { + "value": [ + 2.0, + 3.0 + ] + } + } +} From 33fca5530106e39d4ad3e2b453b07d9c69c699f0 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Fri, 31 Oct 2025 20:20:47 -0600 Subject: [PATCH 08/12] Temporarily disable retrieving abstract Kokkos function for GPU #30655 --- framework/src/kokkos/functions/KokkosFunctionInterface.K | 5 +++++ test/tests/kokkos/functions/constant_function/tests | 2 +- test/tests/kokkos/functions/default_function/tests | 2 +- test/tests/kokkos/functions/piecewise_constant/tests | 8 ++++---- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/framework/src/kokkos/functions/KokkosFunctionInterface.K b/framework/src/kokkos/functions/KokkosFunctionInterface.K index 726d5938aaf1..469474b77194 100644 --- a/framework/src/kokkos/functions/KokkosFunctionInterface.K +++ b/framework/src/kokkos/functions/KokkosFunctionInterface.K @@ -30,6 +30,11 @@ FunctionInterface::getKokkosFunction(const std::string & name) const Moose::Kokkos::Function FunctionInterface::getKokkosFunctionByName(const FunctionName & name) const { +#ifdef MOOSE_ENABLE_KOKKOS_GPU + _fni_object.mooseError( + "Retrieving a Kokkos function as abstract type is currently not supported for GPU."); +#endif + if (!_fni_object.isKokkosObject()) _fni_object.mooseError("Attempted to retrieve a Kokkos function from a non-Kokkos object."); diff --git a/test/tests/kokkos/functions/constant_function/tests b/test/tests/kokkos/functions/constant_function/tests index 50afa8f9716d..77ac03a3c859 100644 --- a/test/tests/kokkos/functions/constant_function/tests +++ b/test/tests/kokkos/functions/constant_function/tests @@ -7,6 +7,6 @@ exodiff = 'kokkos_constant_function_out.e' requirement = 'The Kokkos function system shall include a constant function.' capabilities = 'kokkos' - compute_devices = 'cpu cuda' + compute_devices = 'cpu' [] [] diff --git a/test/tests/kokkos/functions/default_function/tests b/test/tests/kokkos/functions/default_function/tests index 423690c61b8d..4302f5a467df 100644 --- a/test/tests/kokkos/functions/default_function/tests +++ b/test/tests/kokkos/functions/default_function/tests @@ -7,6 +7,6 @@ exodiff = 'kokkos_default_function_out.e' requirement = 'The Kokkos function system shall include the ability to set default values for input parameters expecting a function name.' capabilities = 'kokkos' - compute_devices = 'cpu cuda' + compute_devices = 'cpu' [] [] diff --git a/test/tests/kokkos/functions/piecewise_constant/tests b/test/tests/kokkos/functions/piecewise_constant/tests index 50598de8c1db..b695778b4bba 100644 --- a/test/tests/kokkos/functions/piecewise_constant/tests +++ b/test/tests/kokkos/functions/piecewise_constant/tests @@ -8,7 +8,7 @@ input = 'kokkos_piecewise_constant.i' exodiff = 'kokkos_piecewise_constant_out.e' capabilities = 'kokkos' - compute_devices = 'cpu cuda' + compute_devices = 'cpu' detail = 'provided separately as two vectors in the input,' [] [xy_data] @@ -17,7 +17,7 @@ cli_args = 'KokkosKernels/diff/coef=func_xy_data KokkosAuxKernels/coef_aux/function=func_xy_data' exodiff = 'kokkos_piecewise_constant_out.e' capabilities = 'kokkos' - compute_devices = 'cpu cuda' + compute_devices = 'cpu' detail = 'provided as a table in the input,' [] [csv] @@ -26,7 +26,7 @@ cli_args = 'KokkosKernels/diff/coef=func_csv KokkosAuxKernels/coef_aux/function=func_csv' exodiff = 'kokkos_piecewise_constant_out.e' capabilities = 'kokkos' - compute_devices = 'cpu cuda' + compute_devices = 'cpu' detail = 'provided as a CSV file,' [] [json] @@ -35,7 +35,7 @@ cli_args = 'KokkosKernels/diff/coef=func_json KokkosAuxKernels/coef_aux/function=func_json' exodiff = 'kokkos_piecewise_constant_out.e' capabilities = 'kokkos' - compute_devices = 'cpu cuda' + compute_devices = 'cpu' detail = 'provided as a JSON file.' [] [] From e961bd0c820f87dac064e06edecf477d19d5f164 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Tue, 4 Nov 2025 13:20:54 -0700 Subject: [PATCH 09/12] Add a device linking step to be prepared for RDC #30655 --- framework/app.mk | 30 ++++++++++++++++++++++++++---- framework/moose.mk | 12 ++++++------ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/framework/app.mk b/framework/app.mk index 6dd8669bb487..39c2cac2b817 100644 --- a/framework/app.mk +++ b/framework/app.mk @@ -377,8 +377,14 @@ app_KOKKOS_DEPS := $(patsubst %.$(KOKKOS_OBJ_SUFFIX), %.$(KOKKOS_OBJ_SUFFIX app_KOKKOS_LIB := ifneq ($(app_KOKKOS_OBJECTS),) - app_KOKKOS_LIB := $(APPLICATION_DIR)/lib/lib$(APPLICATION_NAME)$(KOKKOS_LIB_SUFFIX) - app_KOKKOS_LIBS += $(app_KOKKOS_LIB) + app_KOKKOS_LIB := $(APPLICATION_DIR)/lib/lib$(APPLICATION_NAME)$(KOKKOS_LIB_SUFFIX) + app_KOKKOS_LIBS += $(app_KOKKOS_LIB) +endif + +ifeq ($(KOKKOS_COMPILER),CPU) + app_KOKKOS_LIB_COMBINED := $(MOOSE_KOKKOS_LIB) $(app_KOKKOS_LIBS) +else + app_KOKKOS_LIB_COMBINED := $(APPLICATION_DIR)/lib/lib$(APPLICATION_NAME)_combined$(KOKKOS_LIB_SUFFIX) endif KOKKOS_OBJECTS += $(app_KOKKOS_OBJECTS) @@ -407,6 +413,7 @@ $(app_KOKKOS_LIB): $(app_KOKKOS_OBJECTS) else # libtool ignores nvcc and just uses mpicxx to link, so cannot be used + $(app_KOKKOS_LIB): curr_dir := $(APPLICATION_DIR) $(app_KOKKOS_LIB): curr_objs := $(app_KOKKOS_OBJECTS) $(app_KOKKOS_LIB): $(app_KOKKOS_OBJECTS) @@ -418,6 +425,21 @@ endif endif +ifeq ($(KOKKOS_COMPILER),GPU) + +# Making a dummy object file for triggering device link is only required for NVCC + +$(app_KOKKOS_LIB_COMBINED): curr_dir := $(APPLICATION_DIR) +$(app_KOKKOS_LIB_COMBINED): $(MOOSE_KOKKOS_LIB) $(app_KOKKOS_LIBS) + @mkdir -p $(curr_dir)/lib + @echo "Device Linking Kokkos Libraries "$@"..." + @echo > dlink.K + @$(KOKKOS_CXX) $(KOKKOS_CXXFLAGS) -c dlink.K -o dlink.o + @$(KOKKOS_CXX) --shared -o $@ dlink.o $(KOKKOS_LDFLAGS) $(KOKKOS_LIBS) $(MOOSE_KOKKOS_LIB) $(app_KOKKOS_LIBS) + @rm dlink.K dlink.o + +endif + endif # Target-specific Variable Values (See GNU-make manual) @@ -516,10 +538,10 @@ ifneq (,$(findstring darwin,$(libmesh_HOST))) endif endif -$(app_EXEC): $(app_LIBS) $(mesh_library) $(main_object) $(app_test_LIB) $(depend_test_libs) $(app_resource) $(MOOSE_KOKKOS_LIB) $(app_KOKKOS_LIBS) +$(app_EXEC): $(app_LIBS) $(mesh_library) $(main_object) $(app_test_LIB) $(depend_test_libs) $(app_resource) $(app_KOKKOS_LIB_COMBINED) @echo "Linking Executable "$@"..." @bash -c '$(libmesh_LIBTOOL) --tag=CXX $(LIBTOOLFLAGS) --mode=link --quiet \ - $(libmesh_CXX) $(libmesh_CXXFLAGS) -o $@ $(main_object) $(depend_test_libs_flags) $(applibs) $(ADDITIONAL_LIBS) $(LDFLAGS) $(libmesh_LDFLAGS) $(libmesh_LIBS) $(EXTERNAL_FLAGS) $(MOOSE_KOKKOS_LIB) $(app_KOKKOS_LIBS) ${SILENCE_SOME_WARNINGS}' + $(libmesh_CXX) $(libmesh_CXXFLAGS) -o $@ $(main_object) $(depend_test_libs_flags) $(applibs) $(ADDITIONAL_LIBS) $(LDFLAGS) $(libmesh_LDFLAGS) $(libmesh_LIBS) $(EXTERNAL_FLAGS) $(app_KOKKOS_LIB_COMBINED) ${SILENCE_SOME_WARNINGS}' @$(codesign) ###### install stuff ############# diff --git a/framework/moose.mk b/framework/moose.mk index 7fe747dd38c4..8184418c88d1 100644 --- a/framework/moose.mk +++ b/framework/moose.mk @@ -504,16 +504,16 @@ MOOSE_KOKKOS_SRC_FILES := $(app_KOKKOS_UNITY_SRC_FILES) else -MOOSE_KOKKOS_SRC_FILES := $(shell find $(FRAMEWORK_DIR) -name "*.K") +MOOSE_KOKKOS_SRC_FILES := $(shell find $(FRAMEWORK_DIR) -name "*.K") endif -MOOSE_KOKKOS_OBJECTS := $(patsubst %.K, %.$(KOKKOS_OBJ_SUFFIX), $(MOOSE_KOKKOS_SRC_FILES)) -MOOSE_KOKKOS_DEPS := $(patsubst %.$(KOKKOS_OBJ_SUFFIX), %.$(KOKKOS_OBJ_SUFFIX).d, $(MOOSE_KOKKOS_OBJECTS)) -MOOSE_KOKKOS_LIB := $(FRAMEWORK_DIR)/libmoose$(KOKKOS_LIB_SUFFIX) +MOOSE_KOKKOS_OBJECTS := $(patsubst %.K, %.$(KOKKOS_OBJ_SUFFIX), $(MOOSE_KOKKOS_SRC_FILES)) +MOOSE_KOKKOS_DEPS := $(patsubst %.$(KOKKOS_OBJ_SUFFIX), %.$(KOKKOS_OBJ_SUFFIX).d, $(MOOSE_KOKKOS_OBJECTS)) +MOOSE_KOKKOS_LIB := $(FRAMEWORK_DIR)/libmoose$(KOKKOS_LIB_SUFFIX) -KOKKOS_OBJECTS := $(MOOSE_KOKKOS_OBJECTS) -KOKKOS_DEPS := $(MOOSE_KOKKOS_DEPS) +KOKKOS_OBJECTS := $(MOOSE_KOKKOS_OBJECTS) +KOKKOS_DEPS := $(MOOSE_KOKKOS_DEPS) -include $(MOOSE_KOKKOS_DEPS) From 44ef5bf269637a8ea7d29926e960d3953d04d572 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Tue, 4 Nov 2025 13:59:08 -0700 Subject: [PATCH 10/12] Add a unit test that shows virtual function call works on GPU #30655 --- unit/include/KokkosFunctionTest.h | 51 +++++++++++++++++++++++++++++++ unit/src/KokkosFunctionTest.K | 34 +++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 unit/include/KokkosFunctionTest.h create mode 100644 unit/src/KokkosFunctionTest.K diff --git a/unit/include/KokkosFunctionTest.h b/unit/include/KokkosFunctionTest.h new file mode 100644 index 000000000000..ceb7b92c85d1 --- /dev/null +++ b/unit/include/KokkosFunctionTest.h @@ -0,0 +1,51 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#pragma once + +#include "KokkosFunction.h" + +#include "gtest_include.h" + +using Moose::Kokkos::Real3; + +class KokkosDummyFunction +{ +public: + KOKKOS_FUNCTION Real value(Real t, Real3) const { return t; } + KOKKOS_FUNCTION Real3 vectorValue(Real, Real3) const { return Real3(0); } + KOKKOS_FUNCTION Real3 gradient(Real, Real3) const { return Real3(0); } + KOKKOS_FUNCTION Real3 curl(Real, Real3) const { return Real3(0); } + KOKKOS_FUNCTION Real div(Real, Real3) const { return 0; } + KOKKOS_FUNCTION Real timeDerivative(Real, Real3) const { return 0; } + KOKKOS_FUNCTION Real timeIntegral(Real, Real, Real3) const { return 0; } + KOKKOS_FUNCTION Real integral() const { return 0; } + KOKKOS_FUNCTION Real average() const { return 0; } +}; + +class KokkosFunctionTestObject +{ +public: + KokkosFunctionTestObject(Moose::Kokkos::Array array, Moose::Kokkos::Function function) + : _array(array), _function(function) + { + } + + KOKKOS_FUNCTION void operator()(const int i) const { _array[i] = _function.value(i, Real3(0)); } + +private: + Moose::Kokkos::Array _array; + Moose::Kokkos::Function _function; +}; + +class KokkosFunctionTest : public ::testing::Test +{ +public: + virtual void SetUp() override; +}; diff --git a/unit/src/KokkosFunctionTest.K b/unit/src/KokkosFunctionTest.K new file mode 100644 index 000000000000..b005b4c1b02a --- /dev/null +++ b/unit/src/KokkosFunctionTest.K @@ -0,0 +1,34 @@ +//* This file is part of the MOOSE framework +//* https://mooseframework.inl.gov +//* +//* All rights reserved, see COPYRIGHT for full restrictions +//* https://github.com/idaholab/moose/blob/master/COPYRIGHT +//* +//* Licensed under LGPL 2.1, please see LICENSE for details +//* https://www.gnu.org/licenses/lgpl-2.1.html + +#include "KokkosFunctionTest.h" + +void +KokkosFunctionTest::SetUp() +{ + Moose::Kokkos::FunctorRegistry::addFunction("KokkosDummyFunction"); +} + +TEST_F(KokkosFunctionTest, virtualFunctionCall) +{ + Moose::Kokkos::Array array(3); + KokkosDummyFunction dummy_function; + Moose::Kokkos::Function function( + Moose::Kokkos::FunctorRegistry::buildFunction(&dummy_function, "KokkosDummyFunction")); + KokkosFunctionTestObject object(array, function); + + Kokkos::parallel_for(3, object); + Kokkos::fence(); + + array.copyToHost(); + + EXPECT_EQ(array[0], 0); + EXPECT_EQ(array[1], 1); + EXPECT_EQ(array[2], 2); +} From 51802dc2cf3b0c5d115920484e047a1f224a28f7 Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Tue, 4 Nov 2025 13:59:14 -0700 Subject: [PATCH 11/12] Address comments #30655 --- .../doc/content/syntax/Functions/index.md | 2 +- .../include/functions/FunctionInterface.h | 4 +- .../kokkos/actions/AddKokkosBCAction.h | 2 +- .../kokkos/actions/AddKokkosFunctionAction.h | 2 +- .../kokkos/actions/AddKokkosKernelAction.h | 2 +- .../kokkos/actions/AddKokkosMaterialAction.h | 2 +- .../actions/AddKokkosNodalKernelAction.h | 2 +- framework/include/kokkos/base/KokkosFunctor.h | 67 ------------------- .../kokkos/functions/KokkosConstantFunction.h | 4 +- .../kokkos/functions/KokkosPiecewiseBase.h | 2 +- .../functions/KokkosPiecewiseConstant.h | 6 +- .../functions/KokkosPiecewiseTabularBase.h | 6 +- framework/include/problems/FEProblemBase.h | 12 ++++ .../src/functions/PiecewiseTabularInterface.C | 2 +- framework/src/kokkos/base/KokkosFunctor.K | 48 ------------- 15 files changed, 31 insertions(+), 132 deletions(-) delete mode 100644 framework/include/kokkos/base/KokkosFunctor.h delete mode 100644 framework/src/kokkos/base/KokkosFunctor.K diff --git a/framework/doc/content/syntax/Functions/index.md b/framework/doc/content/syntax/Functions/index.md index 82fadd7deb0b..2662928cc03a 100644 --- a/framework/doc/content/syntax/Functions/index.md +++ b/framework/doc/content/syntax/Functions/index.md @@ -25,7 +25,7 @@ Moose `Function`s should override the following member functions: and may optionally override the following member functions, which is only needed for some particular functionality: -- `Real timeIntegral(Real t1, Real t1, const Point & p)`, which computes the +- `Real timeIntegral(Real t1, Real t2, const Point & p)`, which computes the time integral of the function at the spatial point `p` between the time values `t1` and `t2`. diff --git a/framework/include/functions/FunctionInterface.h b/framework/include/functions/FunctionInterface.h index 85d38e4cb6da..419808841537 100644 --- a/framework/include/functions/FunctionInterface.h +++ b/framework/include/functions/FunctionInterface.h @@ -89,6 +89,7 @@ class FunctionInterface #ifdef MOOSE_KOKKOS_ENABLED /** * Get a Kokkos function of an abstract type with a given name + * Calling this function will error out currently if Kokkos was configured with GPU * @param name The name of the parameter key of the Kokkos function to retrieve * @return The copy of the Kokkos function of the abstract type with name associated with the * parameter 'name' @@ -97,6 +98,7 @@ class FunctionInterface /** * Get a Kokkos function of an abstract type with a given name + * Calling this function will error out currently if Kokkos was configured with GPU * @param name The name of the Kokkos function to retrieve * @return The copy of the Kokkos function of the abstract type with name 'name' */ @@ -140,7 +142,7 @@ class FunctionInterface private: #ifdef MOOSE_KOKKOS_ENABLED - /// Helper function to retrieve a Kokkos function of concrete type + /// Helper function to retrieve a Kokkos function const Moose::FunctionBase * getKokkosFunctionByNameHelper(const FunctionName & name) const; #endif diff --git a/framework/include/kokkos/actions/AddKokkosBCAction.h b/framework/include/kokkos/actions/AddKokkosBCAction.h index 00b51fe136c3..97602335d55f 100644 --- a/framework/include/kokkos/actions/AddKokkosBCAction.h +++ b/framework/include/kokkos/actions/AddKokkosBCAction.h @@ -12,7 +12,7 @@ #include "KokkosObjectAction.h" /** - * Adds a Kokkos boundary condition for GPU computation + * Adds a Kokkos boundary condition * Associated with the [KokkosBCs] syntax */ class AddKokkosBCAction : public KokkosObjectAction diff --git a/framework/include/kokkos/actions/AddKokkosFunctionAction.h b/framework/include/kokkos/actions/AddKokkosFunctionAction.h index 7e7a266a45f1..280f99a41aa7 100644 --- a/framework/include/kokkos/actions/AddKokkosFunctionAction.h +++ b/framework/include/kokkos/actions/AddKokkosFunctionAction.h @@ -12,7 +12,7 @@ #include "KokkosObjectAction.h" /** - * Adds a Kokkos function for GPU computation + * Adds a Kokkos function * Associated with the [KokkosFunctions] syntax */ class AddKokkosFunctionAction : public KokkosObjectAction diff --git a/framework/include/kokkos/actions/AddKokkosKernelAction.h b/framework/include/kokkos/actions/AddKokkosKernelAction.h index 8240306fa8b0..a80d1f1793aa 100644 --- a/framework/include/kokkos/actions/AddKokkosKernelAction.h +++ b/framework/include/kokkos/actions/AddKokkosKernelAction.h @@ -12,7 +12,7 @@ #include "KokkosObjectAction.h" /** - * Adds a Kokkos kernel for GPU computation + * Adds a Kokkos kernel * Associated with the [KokkosKernels] syntax */ class AddKokkosKernelAction : public KokkosObjectAction diff --git a/framework/include/kokkos/actions/AddKokkosMaterialAction.h b/framework/include/kokkos/actions/AddKokkosMaterialAction.h index ea004dbd242a..5c10f3a97176 100644 --- a/framework/include/kokkos/actions/AddKokkosMaterialAction.h +++ b/framework/include/kokkos/actions/AddKokkosMaterialAction.h @@ -12,7 +12,7 @@ #include "KokkosObjectAction.h" /** - * Adds a Kokkos material for GPU computation + * Adds a Kokkos material * Associated with the [KokkosMaterials] syntax */ class AddKokkosMaterialAction : public KokkosObjectAction diff --git a/framework/include/kokkos/actions/AddKokkosNodalKernelAction.h b/framework/include/kokkos/actions/AddKokkosNodalKernelAction.h index bdfcd7ee5186..0b7665378a21 100644 --- a/framework/include/kokkos/actions/AddKokkosNodalKernelAction.h +++ b/framework/include/kokkos/actions/AddKokkosNodalKernelAction.h @@ -12,7 +12,7 @@ #include "KokkosObjectAction.h" /** - * Adds a Kokkos nodal kernel for GPU computation + * Adds a Kokkos nodal kernel * Associated with the [KokkosNodalKernels] syntax */ class AddKokkosNodalKernelAction : public KokkosObjectAction diff --git a/framework/include/kokkos/base/KokkosFunctor.h b/framework/include/kokkos/base/KokkosFunctor.h deleted file mode 100644 index 44c70307172c..000000000000 --- a/framework/include/kokkos/base/KokkosFunctor.h +++ /dev/null @@ -1,67 +0,0 @@ -//* This file is part of the MOOSE framework -//* https://mooseframework.inl.gov -//* -//* All rights reserved, see COPYRIGHT for full restrictions -//* https://github.com/idaholab/moose/blob/master/COPYRIGHT -//* -//* Licensed under LGPL 2.1, please see LICENSE for details -//* https://www.gnu.org/licenses/lgpl-2.1.html - -#pragma once - -#include "KokkosTypes.h" -#include "KokkosFunctorWrapper.h" -#include "KokkosFunctorRegistry.h" - -class FEProblemBase; - -namespace Moose -{ -namespace Kokkos -{ - -/** - * The abstract class that provides polymorphic interfaces for a functor - */ -class Functor final -{ -public: - /** - * Constructor - * @param problem The MOOSE problem - * @param wrapper The host functor wrapper - */ - Functor(FEProblemBase & problem, std::shared_ptr wrapper); - /** - * Copy constructor for parallel dispatch - */ - Functor(const Functor & functor); - /** - * Destructor - */ - ~Functor(); - -private: - /** - * Pointer to the host functor wrapper - */ - std::shared_ptr _wrapper_host; - /** - * Pointer to the device functor wrapper - */ - FunctorWrapperDeviceBase * _wrapper_device = nullptr; - /** - * Reference of the FE problem - */ - FEProblemBase & _problem; - /** - * Current and old time - */ - ///@{ - Scalar _t; - Scalar _t_old; - ///@} -}; - -} // namespace Kokkos -} // namespace Moose diff --git a/framework/include/kokkos/functions/KokkosConstantFunction.h b/framework/include/kokkos/functions/KokkosConstantFunction.h index c0758060396a..a75e96206a3a 100644 --- a/framework/include/kokkos/functions/KokkosConstantFunction.h +++ b/framework/include/kokkos/functions/KokkosConstantFunction.h @@ -16,13 +16,13 @@ */ class KokkosConstantFunction : public Moose::Kokkos::FunctionBase { - using Real3 = Moose::Kokkos::Real3; - public: static InputParameters validParams(); KokkosConstantFunction(const InputParameters & parameters); + using Real3 = Moose::Kokkos::Real3; + KOKKOS_FUNCTION Real value(Real /* t */, Real3 /* p */) const { return _value; } KOKKOS_FUNCTION Real timeIntegral(Real t1, Real t2, Real3 /* p */) const { diff --git a/framework/include/kokkos/functions/KokkosPiecewiseBase.h b/framework/include/kokkos/functions/KokkosPiecewiseBase.h index 6eb769d92e90..f82cae40873b 100644 --- a/framework/include/kokkos/functions/KokkosPiecewiseBase.h +++ b/framework/include/kokkos/functions/KokkosPiecewiseBase.h @@ -23,7 +23,7 @@ class KokkosPiecewiseBase : public Moose::Kokkos::FunctionBase KokkosPiecewiseBase(const InputParameters & parameters); - KOKKOS_FUNCTION auto functionSize() const { return _raw_x.size(); } + KOKKOS_FUNCTION dof_id_type functionSize() const { return _raw_x.size(); } KOKKOS_FUNCTION Real domain(const unsigned int i) const { return _raw_x[i]; } KOKKOS_FUNCTION Real range(const unsigned int i) const { return _raw_y[i]; } diff --git a/framework/include/kokkos/functions/KokkosPiecewiseConstant.h b/framework/include/kokkos/functions/KokkosPiecewiseConstant.h index 3c93ed17ed6c..e6b9039e77c6 100644 --- a/framework/include/kokkos/functions/KokkosPiecewiseConstant.h +++ b/framework/include/kokkos/functions/KokkosPiecewiseConstant.h @@ -12,8 +12,6 @@ #include "KokkosPiecewiseTabularBase.h" #include "KokkosUtils.h" -using Moose::Kokkos::Real3; - /** * Function which provides a piecewise constant interpolation of a provided (x,y) point data set. */ @@ -24,6 +22,8 @@ class KokkosPiecewiseConstant : public KokkosPiecewiseTabularBase KokkosPiecewiseConstant(const InputParameters & parameters); + using Real3 = Moose::Kokkos::Real3; + KOKKOS_FUNCTION Real value(Real t, Real3 p) const; KOKKOS_FUNCTION Real integral() const; KOKKOS_FUNCTION Real average() const; @@ -41,7 +41,7 @@ KokkosPiecewiseConstant::value(Real t, Real3 p) const const Real x = _has_axis ? p(_axis) : t; const auto len = functionSize(); - const Real tolerance = 1.0e-14; + constexpr Real tolerance = 1.0e-14; // endpoint cases if ((_direction == Direction::LEFT && x < (1 + tolerance * sign(domain(0))) * domain(0)) || diff --git a/framework/include/kokkos/functions/KokkosPiecewiseTabularBase.h b/framework/include/kokkos/functions/KokkosPiecewiseTabularBase.h index 650dc09a06e5..a125c8f1f264 100644 --- a/framework/include/kokkos/functions/KokkosPiecewiseTabularBase.h +++ b/framework/include/kokkos/functions/KokkosPiecewiseTabularBase.h @@ -14,9 +14,9 @@ #include "PiecewiseTabularInterface.h" /** - * Function base which provides a piecewise approximation to a provided (x,y) point data set via - * input parameter specifications. Derived classes, which control the order (constant, linear) of - * the approximation and how the (x,y) data set is generated, should be used directly. + * Piecewise tabular base class which provides a piecewise approximation to a provided (x,y) point + * data set via input parameter specifications. Derived classes, which control the order (constant, + * linear) of the approximation and how the (x,y) data set is generated, should be used directly. */ class KokkosPiecewiseTabularBase : public KokkosPiecewiseBase, public PiecewiseTabularInterface { diff --git a/framework/include/problems/FEProblemBase.h b/framework/include/problems/FEProblemBase.h index f2c3dbeb1b30..b16e8a22c36a 100644 --- a/framework/include/problems/FEProblemBase.h +++ b/framework/include/problems/FEProblemBase.h @@ -651,9 +651,20 @@ class FEProblemBase : public SubProblem, public Restartable virtual Function & getFunction(const std::string & name, const THREAD_ID tid = 0); #ifdef MOOSE_KOKKOS_ENABLED + /** + * Add a Kokkos function to the problem + * @param type The Kokkos function type + * @param name The Kokkos function name + * @param parameters The Kokkos function input parameters + */ virtual void addKokkosFunction(const std::string & type, const std::string & name, InputParameters & parameters); + /** + * Get whether a Kokkos function exists + * @param name The Kokkos function name + * @returns Whether a Kokkos function exists + */ virtual bool hasKokkosFunction(const std::string & name); /** * Get a Kokkos function in an abstract type @@ -663,6 +674,7 @@ class FEProblemBase : public SubProblem, public Restartable virtual Moose::Kokkos::Function getKokkosFunction(const std::string & name); /** * Get a Kokkos function in a concrete type + * @tparam T the function type * @param name The Kokkos function name * @returns The reference of the Kokkos function in the concrete type */ diff --git a/framework/src/functions/PiecewiseTabularInterface.C b/framework/src/functions/PiecewiseTabularInterface.C index b66c042bda54..7b2fab7be894 100644 --- a/framework/src/functions/PiecewiseTabularInterface.C +++ b/framework/src/functions/PiecewiseTabularInterface.C @@ -105,7 +105,7 @@ PiecewiseTabularInterface::buildFromFile(const libMesh::Parallel::Communicator & { // Input parameters const auto & data_file_name = _parameters.get("data_file"); - const MooseEnum format = _parameters.get("format"); + const MooseEnum & format = _parameters.get("format"); // xy_in_file_only does not make sense here as it restricts the file to exactly // two cows/columns, which is not a likely scenario when looking up data by name. diff --git a/framework/src/kokkos/base/KokkosFunctor.K b/framework/src/kokkos/base/KokkosFunctor.K deleted file mode 100644 index c7d2bc26f806..000000000000 --- a/framework/src/kokkos/base/KokkosFunctor.K +++ /dev/null @@ -1,48 +0,0 @@ -//* This file is part of the MOOSE framework -//* https://mooseframework.inl.gov -//* -//* All rights reserved, see COPYRIGHT for full restrictions -//* https://github.com/idaholab/moose/blob/master/COPYRIGHT -//* -//* Licensed under LGPL 2.1, please see LICENSE for details -//* https://www.gnu.org/licenses/lgpl-2.1.html - -#include "KokkosFunctor.h" - -#include "FEProblemBase.h" - -namespace Moose -{ -namespace Kokkos -{ - -Functor::Functor(FEProblemBase & problem, std::shared_ptr wrapper) - : _wrapper_host(wrapper), _problem(problem), _t(problem.time()), _t_old(problem.timeOld()) -{ - // Allocate device wrapper - _wrapper_device = _wrapper_host->allocate(); -} - -Functor::Functor(const Functor & functor) - : _wrapper_host(functor._wrapper_host), - _wrapper_device(functor._wrapper_device), - _problem(functor._problem), - _t(functor._t), - _t_old(functor._t_old) -{ - // Copy functor to device - _wrapper_host->copyFunctor(); -} - -Functor::~Functor() -{ - // Free device wrapper - if (_wrapper_host.use_count() == 1) - ::Kokkos::kokkos_free(_wrapper_device); - - // Free host copy of functor - _wrapper_host->freeFunctor(); -} - -} // namespace Kokkos -} // namespace Moose From 13b9bb2cf03be95c99df9b2846848aa5b04f2a0e Mon Sep 17 00:00:00 2001 From: Namjae Choi Date: Tue, 4 Nov 2025 17:48:48 -0700 Subject: [PATCH 12/12] Address comments 2 #30655 --- .../include/kokkos/functions/KokkosPiecewiseBase.h | 6 ------ framework/include/problems/FEProblemBase.h | 2 +- framework/src/actions/AddFunctionAction.C | 10 +++++++++- framework/src/kokkos/actions/AddKokkosFunctionAction.K | 10 +++++++++- framework/src/kokkos/functions/KokkosPiecewiseBase.K | 10 ---------- test/tests/kokkos/functions/constant_function/tests | 2 +- test/tests/misc/check_error/tests | 2 +- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/framework/include/kokkos/functions/KokkosPiecewiseBase.h b/framework/include/kokkos/functions/KokkosPiecewiseBase.h index f82cae40873b..5b744f1ec436 100644 --- a/framework/include/kokkos/functions/KokkosPiecewiseBase.h +++ b/framework/include/kokkos/functions/KokkosPiecewiseBase.h @@ -27,12 +27,6 @@ class KokkosPiecewiseBase : public Moose::Kokkos::FunctionBase KOKKOS_FUNCTION Real domain(const unsigned int i) const { return _raw_x[i]; } KOKKOS_FUNCTION Real range(const unsigned int i) const { return _raw_y[i]; } - /** - * Provides a means for explicitly setting the x and y data. This must - * be called in the constructor of inherited classes. - */ - virtual void setData(const std::vector & x, const std::vector & y); - protected: ///@{ raw function data as read Moose::Kokkos::Array _raw_x; diff --git a/framework/include/problems/FEProblemBase.h b/framework/include/problems/FEProblemBase.h index b16e8a22c36a..3d7dab2d00f6 100644 --- a/framework/include/problems/FEProblemBase.h +++ b/framework/include/problems/FEProblemBase.h @@ -674,7 +674,7 @@ class FEProblemBase : public SubProblem, public Restartable virtual Moose::Kokkos::Function getKokkosFunction(const std::string & name); /** * Get a Kokkos function in a concrete type - * @tparam T the function type + * @tparam T The Kokkos function type * @param name The Kokkos function name * @returns The reference of the Kokkos function in the concrete type */ diff --git a/framework/src/actions/AddFunctionAction.C b/framework/src/actions/AddFunctionAction.C index 9c0f29d5f92b..cdc4921b87fa 100644 --- a/framework/src/actions/AddFunctionAction.C +++ b/framework/src/actions/AddFunctionAction.C @@ -28,6 +28,14 @@ AddFunctionAction::act() FunctionParserBase fp; std::string vars = "x,y,z,t,NaN,pi,e"; if (fp.Parse(_name, vars) == -1) // -1 for success - mooseWarning("Function name '" + _name + "' could evaluate as a ParsedFunction"); + mooseWarning( + "Function name '", + _name, + "' can be interpreted as a parsed function expression. This can be problematic. As an " + "example you may name a function 'x' whose functional form is 'xy'. You probably wouldn't " + "do this, but let's assume for the sake of argument. You might also write somewhere in a " + "consumer object 'function = x'. Well, MOOSE supports direct construction of parsed " + "functions from 'FunctionName' parameters, so is this consumer going to end up using the " + "functional form 'x' or 'xy'? It is undefined behavior."); _problem->addFunction(_type, _name, _moose_object_pars); } diff --git a/framework/src/kokkos/actions/AddKokkosFunctionAction.K b/framework/src/kokkos/actions/AddKokkosFunctionAction.K index b64266ffd7fb..2319710b3d11 100644 --- a/framework/src/kokkos/actions/AddKokkosFunctionAction.K +++ b/framework/src/kokkos/actions/AddKokkosFunctionAction.K @@ -31,6 +31,14 @@ AddKokkosFunctionAction::act() FunctionParserBase fp; std::string vars = "x,y,z,t,NaN,pi,e"; if (fp.Parse(_name, vars) == -1) // -1 for success - mooseWarning("Function name '" + _name + "' could evaluate as a ParsedFunction"); + mooseError( + "Function name '", + _name, + "' can be interpreted as a parsed function expression. This can be problematic. As an " + "example you may name a function 'x' whose functional form is 'xy'. You probably wouldn't " + "do this, but let's assume for the sake of argument. You might also write somewhere in a " + "consumer object 'function = x'. Well, MOOSE supports direct construction of parsed " + "functions from 'FunctionName' parameters, so is this consumer going to end up using the " + "functional form 'x' or 'xy'? It is undefined behavior."); _problem->addKokkosFunction(_type, _name, _moose_object_pars); } diff --git a/framework/src/kokkos/functions/KokkosPiecewiseBase.K b/framework/src/kokkos/functions/KokkosPiecewiseBase.K index 2f0c5e58eab8..22fad6ba4100 100644 --- a/framework/src/kokkos/functions/KokkosPiecewiseBase.K +++ b/framework/src/kokkos/functions/KokkosPiecewiseBase.K @@ -19,13 +19,3 @@ KokkosPiecewiseBase::KokkosPiecewiseBase(const InputParameters & parameters) : FunctionBase(parameters) { } - -void -KokkosPiecewiseBase::setData(const std::vector & x, const std::vector & y) -{ - _raw_x = x; - _raw_y = y; - - if (_raw_x.size() != _raw_y.size()) - mooseError("In KokkosPiecewiseBase ", _name, ": Lengths of x and y data do not match."); -} diff --git a/test/tests/kokkos/functions/constant_function/tests b/test/tests/kokkos/functions/constant_function/tests index 77ac03a3c859..f917819b18d7 100644 --- a/test/tests/kokkos/functions/constant_function/tests +++ b/test/tests/kokkos/functions/constant_function/tests @@ -5,7 +5,7 @@ type = 'Exodiff' input = 'kokkos_constant_function.i' exodiff = 'kokkos_constant_function_out.e' - requirement = 'The Kokkos function system shall include a constant function.' + requirement = 'The Kokkos function system shall include a constant function and the ability to retrieve it as an abstract type.' capabilities = 'kokkos' compute_devices = 'cpu' [] diff --git a/test/tests/misc/check_error/tests b/test/tests/misc/check_error/tests index 9d5ccc6b0c04..00690fe84453 100644 --- a/test/tests/misc/check_error/tests +++ b/test/tests/misc/check_error/tests @@ -899,7 +899,7 @@ # Check to see if a warning is produced when a function name could evaluate to a ParsedFunction type = 'RunApp' input = 'function_conflict.i' - expect_out = "Function name 'x' could evaluate as a ParsedFunction" + expect_out = "Function name 'x' can be interpreted as a parsed function expression. This can be problematic." allow_warnings = true requirement = 'The system shall report an error when the name of a function could be misinterpreted because it could also be evaluated (e.g. a function name of "x").'