Use functor for gridHalo and make PeriodicHalo child of Halo;

try to clean up vector use

core/src/Cabana_CommunicationPlan.hpp

@@ -30,6 +30,58 @@ namespace Cabana
 {
 namespace Impl
 {
+// Put this rank first.
+void selfFirstTopology( std::vector<int>& topology )
+{
+    int my_rank = -1;
+    MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
+    for ( auto& n : topology )
+    {
+        if ( n == my_rank )
+        {
+            std::swap( n, topology[0] );
+            break;
+        }
+    }
+}
+
+std::vector<int> getUniqueTopology( std::vector<int> topology )
+{
+    auto remove_end = std::remove( topology.begin(), topology.end(), -1 );
+    std::sort( topology.begin(), remove_end );
+    auto unique_end = std::unique( topology.begin(), remove_end );
+    topology.resize( std::distance( topology.begin(), unique_end ) );
+
+    selfFirstTopology( topology );
+    return topology;
+}
+
+// Sort and make neighbor topology unique, with this rank first.
+std::vector<int>
+getUniqueTopology( Kokkos::View<int*, Kokkos::HostSpace> topology )
+{
+    std::vector<int> unique_topology( topology.extent( 0 ) );
+    for ( std::size_t n = 0; n < topology.extent( 0 ); ++n )
+        unique_topology[n] = topology( n );
+
+    unique_topology = getUniqueTopology( unique_topology );
+    selfFirstTopology( unique_topology );
+    return unique_topology;
+}
+
+auto getUniqueTopologyView( Kokkos::View<int*, Kokkos::HostSpace> topology )
+{
+    std::vector<int> unique_topology;
+    unique_topology = getUniqueTopology( topology );
+
+    Kokkos::View<int*, Kokkos::HostSpace> unique_view( "unique_neighbors",
+                                                       unique_topology.size() );
+    for ( std::size_t n = 0; n < unique_topology.size(); ++n )
+        unique_view( n ) = unique_topology[n];
+
+    return unique_view;
+}
+
 //---------------------------------------------------------------------------//
 // Count sends and create steering algorithm tags.
 struct CountSendsAndCreateSteeringDuplicated
@@ -456,19 +508,16 @@ class CommunicationPlan
       this rank, just use this rank as the export destination and the data
       will be efficiently migrated.
     */
-    template <class ViewType>
+    template <class ViewType, class NeighType>
     Kokkos::View<size_type*, device_type>
     createFromExportsAndTopology( const ViewType& element_export_ranks,
-                                  const std::vector<int>& neighbor_ranks )
+                                  const NeighType& neighbor_ranks )
     {
         // Store the number of export elements.
         _num_export_element = element_export_ranks.size();
 
-        // Store the unique neighbors.
-        _neighbors = neighbor_ranks;
-        std::sort( _neighbors.begin(), _neighbors.end() );
-        auto unique_end = std::unique( _neighbors.begin(), _neighbors.end() );
-        _neighbors.resize( std::distance( _neighbors.begin(), unique_end ) );
+        // Store the unique neighbors (this rank first).
+        _neighbors = Impl::getUniqueTopology( neighbor_ranks );
         int num_n = _neighbors.size();
 
         // Get the size of this communicator.
@@ -483,15 +532,6 @@ class CommunicationPlan
         // communication space so any mpi tag will do.
         const int mpi_tag = 1221;
 
-        // If we are sending to ourself put that one first in the neighbor
-        // list.
-        for ( auto& n : _neighbors )
-            if ( n == my_rank )
-            {
-                std::swap( n, _neighbors[0] );
-                break;
-            }
-
         // Initialize import/export sizes.
         _num_export.assign( num_n, 0 );
         _num_import.assign( num_n, 0 );

core/src/Cabana_Distributor.hpp

@@ -70,6 +70,9 @@ class Distributor : public CommunicationPlan<DeviceType>
       \tparam ViewType The container type for the export element ranks. This
       container type can be either a Kokkos View or a Cabana Slice.
 
+      \tparam NeighType The container type for the neighbor ranks. This
+      container type can be a Kokkos View or a std::vector.
+
       \param comm The MPI communicator over which the distributor is defined.
 
       \param element_export_ranks The destination rank in the target
@@ -93,9 +96,9 @@ class Distributor : public CommunicationPlan<DeviceType>
       this rank, just use this rank as the export destination and the data
       will be efficiently migrated.
     */
-    template <class ViewType>
+    template <class ViewType, class NeighType>
     Distributor( MPI_Comm comm, const ViewType& element_export_ranks,
-                 const std::vector<int>& neighbor_ranks )
+                 const NeighType& neighbor_ranks )
         : CommunicationPlan<DeviceType>( comm )
     {
         auto neighbor_ids = this->createFromExportsAndTopology(

core/src/Cabana_Halo.hpp

@@ -21,6 +21,7 @@
 #include <mpi.h>
 
 #include <exception>
+#include <type_traits>
 #include <vector>
 
 namespace Cabana
@@ -72,6 +73,9 @@ class Halo : public CommunicationPlan<DeviceType>
       ranks. This container type can be either a Kokkos View or a Cabana
       Slice.
 
+      \tparam NeighType The container type for the neighbor ranks. This
+      container type can be a Kokkos View or a std::vector.
+
       \param comm The MPI communicator over which the halo is defined.
 
       \param num_local The number of locally-owned elements on this rank.
@@ -97,11 +101,11 @@ class Halo : public CommunicationPlan<DeviceType>
       \note Calling this function completely updates the state of this object
       and invalidates the previous state.
     */
-    template <class IdViewType, class RankViewType>
+    template <class IdViewType, class RankViewType, class NeighType>
     Halo( MPI_Comm comm, const std::size_t num_local,
           const IdViewType& element_export_ids,
           const RankViewType& element_export_ranks,
-          const std::vector<int>& neighbor_ranks )
+          const NeighType& neighbor_ranks )
         : CommunicationPlan<DeviceType>( comm )
         , _num_local( num_local )
     {
@@ -199,6 +203,19 @@ struct is_halo : public is_halo_impl<typename std::remove_cv<T>::type>::type
 {
 };
 
+template <typename, typename = void>
+struct is_callable : public std::false_type
+{
+};
+
+template <class T>
+struct is_callable<
+    T, typename std::enable_if<
+           std::is_same<decltype( void( &T::operator() ) ), void>::value>::type>
+    : std::true_type
+{
+};
+
 namespace Impl
 {
 
@@ -229,9 +246,9 @@ void sendBuffer( const Halo_t& halo, AoSoA_t& aosoa, View_t& send_buffer )
     Kokkos::fence();
 }
 
-template <class Halo_t, class AoSoA_t, class View_t, class Modify_t>
-void sendBuffer( const Halo_t& halo, AoSoA_t& aosoa, View_t& send_buffer,
-                 const Modify_t& modify_functor )
+template <class Halo_t, class AoSoA_t, class View_t>
+void sendModifiedBuffer( const Halo_t& halo, AoSoA_t& aosoa,
+                         View_t& send_buffer )
 {
     // Get the steering vector for the sends.
     auto steering = halo.getExportSteering();
@@ -241,7 +258,7 @@ void sendBuffer( const Halo_t& halo, AoSoA_t& aosoa, View_t& send_buffer,
     auto gather_send_buffer_func = KOKKOS_LAMBDA( const std::size_t i )
     {
         send_buffer( i ) = aosoa.getTuple( steering( i ) );
-        modify_functor( send_buffer, i );
+        halo( send_buffer, i );
     };
     Kokkos::RangePolicy<typename Halo_t::execution_space>
         gather_send_buffer_policy( 0, halo.totalNumExport() );
@@ -335,28 +352,28 @@ void recvBuffer( const Halo_t& halo, AoSoA_t& aosoa, const View_t& send_buffer )
   element in the locally owned decomposition will be the value assigned to the
   element in the ghosted decomposition.
 
-  \tparam Halo_t Halo type - must be a Halo.
+  \tparam Halo_t Halo type - must be derived from Halo with a functor defined to
+  modify the send buffer.
 
   \tparam AoSoA_t AoSoA type - must be an AoSoA.
 
-  \tparam Modify_t Buffer modification type.
-
-  \param halo The halo to use for the gather.
+  \param halo The halo to use for the gather and send buffer modification.
 
   \param aosoa The AoSoA on which to perform the gather. The AoSoA should have
   a size equivalent to halo.numGhost() + halo.numLocal(). The locally owned
   elements are expected to appear first (i.e. in the first halo.numLocal()
   elements) and the ghosted elements are expected to appear second (i.e. in
   the next halo.numGhost() elements()).
-
-  \param modify_functor Class containing functor to modify the send buffer
-  before it's sent (e.g. for periodic coordinate update).
 */
-template <class Halo_t, class AoSoA_t, class Modify_t>
-void gather( const Halo_t& halo, AoSoA_t& aosoa, const Modify_t& modify_functor,
-             typename std::enable_if<( is_halo<Halo_t>::value &&
-                                       is_aosoa<AoSoA_t>::value ),
-                                     int>::type* = 0 )
+template <class Halo_t, class AoSoA_t>
+void gather(
+    const Halo_t& halo, AoSoA_t& aosoa,
+    typename std::enable_if<
+        ( !is_halo<Halo_t>::value && // TODO: Should not be needed
+          std::is_base_of<Halo<typename Halo_t::device_type>, Halo_t>::value &&
+          // is_callable<Halo_t>::value &&
+          is_aosoa<AoSoA_t>::value ),
+        int>::type* = 0 )
 {
     Impl::checkSize( halo, aosoa );
 
@@ -366,7 +383,7 @@ void gather( const Halo_t& halo, AoSoA_t& aosoa, const Modify_t& modify_functor,
             Kokkos::ViewAllocateWithoutInitializing( "halo_send_buffer" ),
             halo.totalNumExport() );
 
-    Impl::sendBuffer( halo, aosoa, send_buffer, modify_functor );
+    Impl::sendModifiedBuffer( halo, aosoa, send_buffer );
     Impl::recvBuffer( halo, aosoa, send_buffer );
 }