Adding fixes when testing with PaRSEC and MPI

CMakeLists.txt

@@ -90,7 +90,7 @@ IF("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
   endif(NOT DEFINED APEX_BUILD_EXAMPLES)
   add_definitions(-DDEBUG)
   add_definitions(-DCMAKE_BUILD_TYPE=3)
-  set(APEX_ERROR_HANDLING TRUE)
+  #set(APEX_ERROR_HANDLING TRUE)
 endif()
 
 IF("${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo")

src/apex/taskstubs_implementation.cpp

@@ -45,6 +45,9 @@ std::shared_ptr<apex::task_wrapper> safeLookup(
         mtx.lock();
         task = getCommonMap().find(guid);
         mtx.unlock();
+        if (task == getCommonMap().end()) {
+            return nullptr;
+        }
         getMyMap()[guid] = task->second;
     }
     return task->second;
@@ -129,41 +132,53 @@ extern "C" {
         tasktimer_execution_space_p) {
         // TODO: capture the execution space, somehow...a new task?
         MAP_TASK(timer, apex_timer);
+        if (apex_timer != nullptr) {
         apex::start(apex_timer);
+        }
     }
     void tasktimer_yield_impl(
         tasktimer_timer_t timer) {
         static bool& over = apex::get_program_over();
         if (over) return;
         MAP_TASK(timer, apex_timer);
+        if (apex_timer != nullptr) {
         apex::yield(apex_timer);
+        }
     }
     void tasktimer_resume_impl(
         tasktimer_timer_t timer,
         tasktimer_execution_space_p) {
         // TODO: capture the execution space, somehow...a new task?
         MAP_TASK(timer, apex_timer);
         // TODO: why no resume function for task_wrapper objects?
+        if (apex_timer != nullptr) {
         apex::start(apex_timer);
+        }
     }
     void tasktimer_stop_impl(
         tasktimer_timer_t timer) {
         MAP_TASK(timer, apex_timer);
+        if (apex_timer != nullptr) {
         apex::stop(apex_timer);
+        }
     }
     void tasktimer_destroy_impl(
         tasktimer_timer_t timer) {
         MAP_TASK(timer, apex_timer);
+        if (apex_timer != nullptr) {
         // TODO: need to handle the destroy event somehow.
         // definitely need to remove it from the local map.
         safeErase(apex_timer->guid);
+        }
     }
     void tasktimer_add_parents_impl (
         tasktimer_timer_t timer,
         const tasktimer_guid_t* parents, const uint64_t parent_count) {
         // TODO: need to handle the add parents event
         MAP_TASK(timer, apex_timer);
+        if (apex_timer != nullptr) {
         APEX_UNUSED(apex_timer);
+        }
         APEX_UNUSED(parents);
         APEX_UNUSED(parent_count);
     }
@@ -172,7 +187,9 @@ extern "C" {
         const tasktimer_guid_t* children, const uint64_t child_count) {
         // TODO: need to handle the add children event
         MAP_TASK(timer, apex_timer);
+        if (apex_timer != nullptr) {
         APEX_UNUSED(apex_timer);
+        }
         APEX_UNUSED(children);
         APEX_UNUSED(child_count);
     }

src/apex/tree.cpp

@@ -5,6 +5,7 @@
 #include<unordered_map>
 #include<utility>
 #include<iostream>
+#include<sstream>
 #include "apex_assert.h"
 
 namespace apex {
@@ -15,9 +16,13 @@ std::atomic<size_t> node::count{0};
 node * node::buildTree(std::vector<std::vector<std::string>>& rows,
     node * root) {
     if(rows.size() == 0) return root;
+    // process (mpi rank)
     //size_t rank = stol(rows[0][0]);
+    // node (tree node)
     size_t index = stol(rows[0][1]);
-    size_t pindex = stol(rows[0][2]);
+    // parent (parent tree node)
+    //size_t pindex = stol(rows[0][2]);
+    // depth
     //size_t depth = stol(rows[0][3]);
     std::string name = rows[0][4];
     std::unordered_map<size_t, node*> node_map;
@@ -30,20 +35,27 @@ node * node::buildTree(std::vector<std::vector<std::string>>& rows,
     for(size_t i = 1 ; i < rows.size() ; i++) {
         //rank = stol(rows[i][0]);
         index = stol(rows[i][1]);
-        pindex = stol(rows[i][2]);
+        std::string tmpstr(rows[i][2]);
+        std::stringstream parents(tmpstr);
+        size_t pindex;
         //depth = stol(rows[i][3]);
         name = rows[i][4];
+        while(true) {
+            parents >> pindex;
+            if (!parents) break;
+            std::cout << index << " Found parent: " << pindex << std::endl;
         /*
         std::cout << rank << " " << index << " " << pindex << " "
                   << depth << " " << name << std::endl;
                   */
-        APEX_ASSERT(node_map.count(pindex) > 0);
-        auto parent = node_map.at(pindex);
-        node * newChild = parent->addChild(name);
-        node_map.insert(std::pair<size_t,node*>(index,newChild));
-        // update the row data with the tree data
-        rows[i][1] = std::to_string(newChild->_index);
-        rows[i][2] = std::to_string(parent->_index);
+            APEX_ASSERT(node_map.count(pindex) > 0);
+            auto parent = node_map.at(pindex);
+            node * newChild = parent->addChild(name);
+            node_map.insert(std::pair<size_t,node*>(index,newChild));
+            // update the row data with the tree data
+            rows[i][1] = std::to_string(newChild->_index);
+            rows[i][2] = std::to_string(parent->_index);
+        }
     }
     std::mutex foo;
     std::scoped_lock{foo};

src/scripts/apex-treesummary.py

@@ -356,13 +356,12 @@ def main():
         print('Reading tasktree...')
     df = pd.read_csv(args.filename) #, index_col=[0,1])
     df = df.fillna(0)
-    print(df)
+    #print(df)
     # Convert the string representation of the list of parents to a list
     df.loc[:, "parent index"] = df["parent index"].apply(ast.literal_eval)
     df = df.explode('parent index')
     df = df.fillna(-1)
-    print(df)
-    print(df)
+    #print(df)
 
     if args.verbose:
         print('Read', len(df.index), 'rows')
@@ -415,6 +414,7 @@ def main():
     # FIRST, build a master graph with all nodes from all ranks.
     print('building common tree...')
     root = TreeNode('apex tree base', pd.DataFrame())
+    root.index = -1
     #unique = df.drop_duplicates(subset=["node index", "parent index", "name"], keep='first')
     graphRank2(0, df, root, droplist, args)
 

src/scripts/apex_exec

@@ -818,9 +818,9 @@ else
     retval=$?
     unset LD_PRELOAD
     unset DYLD_INSERT_LIBRARIES
-    if [ "${myrank}" == "0" ] ; then
-        rm -f ${gdbcmds}
-    fi
+    #if [ "${myrank}" == "0" ] ; then
+        #rm -f ${gdbcmds}
+    #fi
     if [ ${retval} != 0 ] ; then
         echo "Error ${retval}!"
         count=`ldd ${MPI_LIB2} | grep -c tcmalloc`

src/unit_tests/C++/CMakeLists.txt

@@ -131,16 +131,25 @@ include_directories (. ${APEX_SOURCE_DIR}/src/apex ${MPI_CXX_INCLUDE_PATH})
 # Make sure the linker can find the Apex library once it is built.
 link_directories (${APEX_BINARY_DIR}/src/apex)
 
-# Add executable called "apex_hpx_annotated_functions_mpi" that is built from the source file
-# "apex_hpx_annotated_functions.cpp". The extensions are automatically found.
-add_executable (apex_hpx_annotated_functions_mpi apex_hpx_annotated_functions.cpp)
-add_dependencies (apex_hpx_annotated_functions_mpi apex)
-add_dependencies (examples apex_hpx_annotated_functions_mpi)
-
-# Link the executable to the Apex library.
-target_link_libraries (apex_hpx_annotated_functions_mpi apex ${MPI_CXX_LINK_FLAGS} ${MPI_CXX_LIBRARIES} ${LIBS} ${APEX_STDCXX_LIB} m)
-if (BUILD_STATIC_EXECUTABLES)
-    set_target_properties(apex_hpx_annotated_functions_mpi PROPERTIES LINK_SEARCH_START_STATIC 1 LINK_SEARCH_END_STATIC 1)
+if(APEX_WITH_MPI)
+    # Add executable called "apex_hpx_annotated_functions_mpi" that is built from the source file
+    # "apex_hpx_annotated_functions.cpp". The extensions are automatically found.
+    add_executable (apex_hpx_annotated_functions_mpi apex_hpx_annotated_functions.cpp)
+    target_compile_definitions(apex_hpx_annotated_functions_mpi PUBLIC -DAPEX_ENABLE_MPI)
+    add_dependencies (apex_hpx_annotated_functions_mpi apex)
+    add_dependencies (examples apex_hpx_annotated_functions_mpi)
+    add_executable (apex_multiple_parents_mpi apex_multiple_parents.cpp)
+    target_compile_definitions(apex_multiple_parents_mpi PUBLIC -DAPEX_ENABLE_MPI)
+    add_dependencies (apex_multiple_parents_mpi apex)
+    add_dependencies (examples apex_multiple_parents_mpi)
+
+    # Link the executable to the Apex library.
+    target_link_libraries (apex_hpx_annotated_functions_mpi apex ${MPI_CXX_LINK_FLAGS} ${MPI_CXX_LIBRARIES} ${LIBS} ${APEX_STDCXX_LIB} m)
+    target_link_libraries (apex_multiple_parents_mpi apex ${MPI_CXX_LINK_FLAGS} ${MPI_CXX_LIBRARIES} ${LIBS} ${APEX_STDCXX_LIB} m)
+    if (BUILD_STATIC_EXECUTABLES)
+        set_target_properties(apex_hpx_annotated_functions_mpi PROPERTIES LINK_SEARCH_START_STATIC 1 LINK_SEARCH_END_STATIC 1)
+        set_target_properties(apex_multiple_parents_mpi PROPERTIES LINK_SEARCH_START_STATIC 1 LINK_SEARCH_END_STATIC 1)
+    endif()
 endif()
 
 INSTALL(TARGETS apex_hpx_annotated_functions_mpi

src/unit_tests/C++/apex_hpx_annotated_functions.cpp

@@ -5,7 +5,7 @@
 #include <time.h>
 #include <stdint.h>
 #include "apex_api.hpp"
-#if defined(APEX_WITH_MPI)
+#if defined(APEX_ENABLE_MPI)
 #include <mpi.h>
 #endif
 
@@ -98,7 +98,7 @@ void* someThread(void* tmp)
 
 int main (int argc, char** argv) {
     /* initialize APEX */
-    #if defined(APEX_WITH_MPI)
+    #if defined(APEX_ENABLE_MPI)
     MPI_Init(&argc, &argv);
     MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank);
     MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
@@ -153,7 +153,7 @@ int main (int argc, char** argv) {
     	}
   	}
   	apex::cleanup();
-    #if defined(APEX_WITH_MPI)
+    #if defined(APEX_ENABLE_MPI)
     MPI_Finalize();
     #endif
     return 0;

src/unit_tests/C++/apex_multiple_parents.cpp

@@ -4,6 +4,9 @@
 #include <stdlib.h>
 #include "apex_api.hpp"
 #include <atomic>
+#ifdef APEX_ENABLE_MPI
+#include "mpi.h"
+#endif
 
 int parent (int in, std::shared_ptr< apex::task_wrapper > this_task) {
     apex::start(this_task);
@@ -22,7 +25,15 @@ int child (int in, std::shared_ptr< apex::task_wrapper > this_task) {
 }
 
 int main(int argc, char *argv[]) {
-    apex::init("apex_multiple_parents_cpp unit test", 0, 1);
+    int comm_rank = 0;
+    int comm_size = 1;
+#ifdef APEX_ENABLE_MPI
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
+    std::cout << "APP: rank " << comm_rank << " of " << comm_size << std::endl;
+#endif
+    apex::init("apex_multiple_parents_cpp unit test", comm_rank, comm_size);
     apex::scoped_timer foo(__func__);
 
     int task_id{0};
@@ -51,6 +62,10 @@ int main(int argc, char *argv[]) {
     std::cout << "sum is " << result << " (valid value: 6)" << std::endl;
 	foo.stop();
     //apex::finalize();
+#ifdef APEX_ENABLE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Finalize();
+#endif
     return 0;
 }