Integrate Adiak in PFA: CMake changes, Adiak smoketest

src/c/CMakeLists.txt

@@ -6,6 +6,7 @@ project(PerfFlowAspect VERSION "0.1.0")
 option(PERFFLOWASPECT_WITH_CUDA "Build CUDA smoketest" ON)
 option(PERFFLOWASPECT_WITH_MPI "Build MPI smoketest" ON)
 option(PERFFLOWASPECT_WITH_MULTITHREADS "Build multi-threaded smoketest" ON)
+option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" ON)
 
 # Fail if using Clang < 9.0
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")

src/c/cmake/Setup3rdParty.cmake

@@ -19,3 +19,7 @@ include(cmake/thirdparty/FindOpenSSL.cmake)
 if(PERFFLOWASPECT_WITH_MULTITHREADS)
     include(cmake/thirdparty/FindThreads.cmake)
 endif()
+
+if(PERFFLOWASPECT_WITH_ADIAK)
+    include(cmake/thirdparty/FindAdiak.cmake)
+endif()

src/c/cmake/thirdparty/FindAdiak.cmake

@@ -0,0 +1,17 @@
+set(adiak_DIR "/g/g92/lisan1/PFA/Adiak/install/lib/cmake/adiak")
+
+find_package(adiak REQUIRED)
+
+message(STATUS "Building Adiak smoketest (PERFFLOWASPECT_WITH_ADIAK == ON)")
+
+if(adiak_FOUND)
+    message(STATUS "Adiak found: ${adiak_FOUND}")
+    message(STATUS "Adiak include directories: ${adiak_INCLUDE_DIRS}")
+    message(STATUS "Adiak library directories: ${adiak_LIBRARY_DIRS}")
+    message(STATUS "Adiak libraries: ${adiak_LIBRARIES}")
+else()
+    message(FATAL_ERROR "Adiak not found.")
+endif()
+
+include_directories(${adiak_INCLUDE_DIRS})
+link_directories(${adiak_LIBRARY_DIRS})

src/c/test/CMakeLists.txt

@@ -3,9 +3,10 @@ set(SMOKETESTS
     smoketest2
     smoketest3
     smoketest_class
+    # smoketest_adiak
 )
 
-set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto)
+set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto adiak::adiak)
 
 message(STATUS "Adding CXX unit tests")
 foreach(TEST ${SMOKETESTS})
@@ -38,6 +39,15 @@ if(PERFFLOWASPECT_WITH_CUDA)
     target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES})
 endif()
 
+if(PERFFLOWASPECT_WITH_ADIAK)
+    message(STATUS " [*] Adding test: smoketest_adiak")
+    add_executable(smoketest_adiak smoketest_adiak.cpp)
+    set_source_files_properties(smoketest_adiak.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fPIC")
+    include_directories(${MPI_INCLUDE_PATH} ${adiak_INCLUDE_DIRS})
+    link_directories(${adiak_LIBRARY_DIRS})
+    target_link_libraries(smoketest_adiak ${perfflow_deps} ${MPI_LIBRARIES} adiak::adiak)
+endif()
+
 configure_file(t0001-cbinding-basic.t.in
     ${CMAKE_CURRENT_BINARY_DIR}/t0001-cbinding-basic.t
     @ONLY)
@@ -60,5 +70,10 @@ if(PERFFLOWASPECT_WITH_CUDA)
             DESTINATION test)
 endif()
 
+if(PERFFLOWASPECT_WITH_ADIAK)
+    install(TARGETS smoketest_adiak
+            DESTINATION test)
+endif()
+
 install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/t0001-cbinding-basic.t
         DESTINATION test)

src/c/test/smoketest_adiak.cpp

@@ -0,0 +1,63 @@
+#include <adiak.hpp>
+#include <mpi.h>
+#include <iostream>
+#include <math.h>
+
+using namespace std;
+
+#define MASTER  0       //main process
+
+__attribute__((annotate("@critical_path()")))
+int foo(const string &str)
+{
+    printf("foo\n");
+    int temp = 0;
+    int temp1[10000] = {25};
+    int temp2[10000] = {22};
+    int temp3[10000] = {0};
+    for (int i = 0; i < 10000; i++)
+    {
+        temp = temp + 1;
+        temp3[i] = temp1[i] * temp2[i];
+        for (int j = i + 1; j < 10000; j++)
+        {
+            temp3[j] = temp3[j] / 3;
+        }
+    }
+    if (str == "hello")
+    {
+        return 1;
+    }
+    return 0;
+}
+
+__attribute__((annotate("@critical_path()")))
+int main(int argc, char* argv[])
+{
+    int taskid, numProcs;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm adk_comm = MPI_COMM_WORLD;
+    MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
+    MPI_Comm_rank(MPI_COMM_WORLD, &taskid);
+
+    adiak_init(&adk_comm);
+    adiak_collect_all();
+
+    //main process code
+    if (taskid == MASTER)
+    {
+        cout << "taskid: " << taskid << '\n';
+        foo("hello");
+    }
+
+    //worker process code
+    if (taskid != MASTER)
+    {
+        cout << "taskid: " << taskid << '\n';
+        foo("hello");
+    }
+
+    adiak_fini();
+    MPI_Finalize();
+}

src/c/weaver/weave/CMakeLists.txt

@@ -12,7 +12,7 @@ set_target_properties(WeavePass PROPERTIES
     COMPILE_FLAGS "-fno-rtti"
 )
 
-target_link_libraries(WeavePass perfflow_parser ${JANSSON_LIB})
+target_link_libraries(WeavePass perfflow_parser ${JANSSON_LIB} adiak::adiak)
 
 add_library(WeavePassPlugin INTERFACE)
 target_compile_options(WeavePassPlugin INTERFACE

src/c/weaver/weave/perfflow_weave.cpp

@@ -26,6 +26,12 @@
 #include "../../parser/perfflow_parser.hpp"
 #include "perfflow_weave.hpp"
 
+#include <stdio.h>
+#include <sys/time.h>
+#include <time.h>
+#include <adiak_tool.h>
+#define UNUSED(x) (void)(x)
+
 using namespace llvm;
 
 
@@ -142,8 +148,135 @@ bool WeavingPass::insertBefore(Module &m, Function &f, StringRef &a,
  *                                                                            *
  ******************************************************************************/
 
+// static void print_value(adiak_value_t *val, adiak_datatype_t *t)
+// {
+//    if (!t)
+//       printf("ERROR");
+//    switch (t->dtype) {
+//       case adiak_type_unset:
+//          printf("UNSET");
+//          break;
+//       case adiak_long:
+//          printf("%ld", val->v_long);
+//          break;
+//       case adiak_ulong:
+//          printf("%lu", (unsigned long) val->v_long);
+//          break;
+//       case adiak_longlong:
+//          printf("%lld", val->v_longlong);
+//          break;
+//       case adiak_ulonglong:
+//          printf("%llu", (unsigned long long) val->v_longlong);
+//          break;
+//       case adiak_int:
+//          printf("%d", val->v_int);
+//          break;
+//       case adiak_uint:
+//          printf("%u", (unsigned int) val->v_int);
+//          break;
+//       case adiak_double:
+//          printf("%f", val->v_double);
+//          break;
+//       case adiak_date: {
+//          char datestr[512];
+//          signed long seconds_since_epoch = (signed long) val->v_long;
+//          struct tm *loc = localtime(&seconds_since_epoch);
+//          strftime(datestr, sizeof(datestr), "%a, %d %b %Y %T %z", loc);
+//          printf("%s", datestr);
+//          break;
+//       }
+//       case adiak_timeval: {
+//          struct timeval *tval = (struct timeval *) val->v_ptr;
+//          double duration = tval->tv_sec + (tval->tv_usec / 1000000.0);
+//          printf("%fs (timeval)", duration);
+//          break;
+//       }
+//       case adiak_version: {
+//          char *s = (char *) val->v_ptr;
+//          printf("%s (version)", s);
+//          break;
+//       }
+//       case adiak_string: {
+//          char *s = (char *) val->v_ptr;
+//          printf("%s (string)", s);
+//          break;
+//       }
+//       case adiak_catstring: {
+//          char *s = (char *) val->v_ptr;
+//          printf("%s (catstring)", s);
+//          break;
+//       }
+//       case adiak_path: {
+//          char *s = (char *) val->v_ptr;
+//          printf("%s (path)", s);
+//          break;
+//       }
+//       case adiak_range: {
+//          adiak_value_t subvals[2];
+//          adiak_datatype_t* subtypes[2];
+
+//          adiak_get_subval(t, val, 0, subtypes+0, subvals+0);
+//          adiak_get_subval(t, val, 1, subtypes+1, subvals+1);
+
+//          print_value(subvals+0, *(subtypes+0));
+//          printf(" - ");
+//          print_value(subvals+1, *(subtypes+1));
+//          break;
+//       }
+//       case adiak_set: {
+//          printf("[");
+//          for (int i = 0; i < adiak_num_subvals(t); i++) {
+//             adiak_value_t subval;
+//             adiak_datatype_t* subtype;
+//             adiak_get_subval(t, val, i, &subtype, &subval);
+//             print_value(&subval, subtype);
+//             if (i+1 != t->num_elements)
+//                printf(", ");
+//          }
+//          printf("]");
+//          break;
+//       }
+//       case adiak_list: {
+//          printf("{");
+//          for (int i = 0; i < adiak_num_subvals(t); i++) {
+//             adiak_value_t subval;
+//             adiak_datatype_t* subtype;
+//             adiak_get_subval(t, val, i, &subtype, &subval);
+//             print_value(&subval, subtype);
+//             if (i+1 != t->num_elements)
+//                printf(", ");
+//          }
+//          printf("}");
+//          break;
+//       }
+//       case adiak_tuple: {
+//          printf("(");
+//          for (int i = 0; i < adiak_num_subvals(t); i++) {
+//             adiak_value_t subval;
+//             adiak_datatype_t* subtype;
+//             adiak_get_subval(t, val, i, &subtype, &subval);
+//             print_value(&subval, subtype);
+//             if (i+1 != t->num_elements)
+//                printf(", ");
+//          }
+//          printf(")");
+//          break;
+//       }
+//    }
+// }
+
+static void print_nameval(const char *name, int category, const char *subcategory, adiak_value_t *value, adiak_datatype_t *t, void *opaque_value)
+{
+    // printf("%s: ", name);
+    printf("%s: ", "In the callback print");
+    outs() << "In the callback \n";
+    // print_value(value, t);
+    // printf("\n");
+}
+
 bool WeavingPass::doInitialization(Module &m)
 {
+    adiak_register_cb(1, adiak_category_all, print_nameval, 0, NULL);
     outs() << "WeavePass loaded successfully. \n";
 
     auto annotations = m.getNamedGlobal("llvm.global.annotations");