add support for sessions

Signed-off-by: Nathan Hjelm <[email protected]>
Signed-off-by: Howard Pritchard <[email protected]>

Author: hjelmn

config/opal_configure_options.m4

@@ -21,6 +21,7 @@ dnl Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
+dnl Copyright (c) 2019-2021 Triad National Security, LLC. All rights
 dnl                         reserved.
 dnl
 dnl $COPYRIGHT$
@@ -495,6 +496,10 @@ OPAL_WITH_OPTION_MIN_MAX_VALUE(port_name,      1024, 255, 2048)
 # Min length accroding to MPI-2.1, p. 418
 OPAL_WITH_OPTION_MIN_MAX_VALUE(datarep_string,  128,  64,  256)
 
+OPAL_WITH_OPTION_MIN_MAX_VALUE(pset_name_len,  512,  512, 4096)
+
+OPAL_WITH_OPTION_MIN_MAX_VALUE(stringtag_len,     1024, 256, 2048)
+
 # some systems don't want/like getpwuid
 AC_MSG_CHECKING([if want getpwuid support])
 AC_ARG_ENABLE([getpwuid],

ompi/Makefile.am

@@ -193,6 +193,7 @@ include patterns/net/Makefile.am
 include patterns/comm/Makefile.am
 include mca/Makefile.am
 include util/Makefile.am
+include instance/Makefile.am
 
 distclean-local:
 	rm -f mpiext/static-components.h

ompi/attribute/attribute.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -14,6 +15,8 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2021 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -240,8 +243,10 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/communicator/communicator.h"  /* ompi_communicator_t generated in [COPY|DELETE]_ATTR_CALLBACKS */
 #include "ompi/win/win.h"                    /* ompi_win_t generated in [COPY|DELETE]_ATTR_CALLBACKS */
+#include "ompi/instance/instance.h"
 #include "ompi/mpi/fortran/base/fint_2_int.h"
 
+
 /*
  * Macros
  */
@@ -256,9 +261,9 @@
 #define attr_datatype_f d_f_to_c_index
 #define attr_win_f w_f_to_c_index
 
-#define CREATE_KEY(key) opal_bitmap_find_and_set_first_unset_bit(key_bitmap, (key))
+#define CREATE_KEY(key) opal_bitmap_find_and_set_first_unset_bit(attr_subsys->key_bitmap, (key))
 
-#define FREE_KEY(key) opal_bitmap_clear_bit(key_bitmap, (key))
+#define FREE_KEY(key) opal_bitmap_clear_bit(attr_subsys->key_bitmap, (key))
 
 
 /* Not checking for NULL_DELETE_FN here, since according to the
@@ -406,13 +411,24 @@ typedef struct attribute_value_t {
     int av_sequence;
 } attribute_value_t;
 
+/*
+ * struct to hold state of attr subsys
+ */
+
+typedef  struct attr_subsys_t {
+    opal_object_t            super;
+    opal_hash_table_t *keyval_hash;
+    opal_bitmap_t *key_bitmap;
+ } attr_subsys_t;
 
 /*
  * Local functions
  */
 static void attribute_value_construct(attribute_value_t *item);
 static void ompi_attribute_keyval_construct(ompi_attribute_keyval_t *keyval);
 static void ompi_attribute_keyval_destruct(ompi_attribute_keyval_t *keyval);
+static void attr_subsys_construct(attr_subsys_t *subsys);
+static void attr_subsys_destruct(attr_subsys_t *subsys);
 static int set_value(ompi_attribute_type_t type, void *object,
                      opal_hash_table_t **attr_hash, int key,
                      attribute_value_t *new_attr,
@@ -425,6 +441,13 @@ static MPI_Aint translate_to_aint(attribute_value_t *val);
 
 static int compare_attr_sequence(const void *attr1, const void *attr2);
 
+/*
+ * attribute_subsys_t class
+ */
+static OBJ_CLASS_INSTANCE(attr_subsys_t,
+                          opal_object_t,
+                          attr_subsys_construct,
+                          attr_subsys_destruct);
 
 /*
  * attribute_value_t class
@@ -448,19 +471,17 @@ static OBJ_CLASS_INSTANCE(ompi_attribute_keyval_t,
  * Static variables
  */
 
-static opal_hash_table_t *keyval_hash;
-static opal_bitmap_t *key_bitmap;
-static int attr_sequence;
+static attr_subsys_t *attr_subsys = NULL;
 static unsigned int int_pos = 12345;
 static unsigned int integer_pos = 12345;
+static int attr_sequence;
 
 /*
  * MPI attributes are *not* high performance, so just use a One Big Lock
  * approach. However, this lock is released before a user provided callback is
  * triggered and acquired right after, allowing for recursive behaviors.
  */
-static opal_mutex_t attribute_lock;
-
+static opal_mutex_t attribute_lock = OPAL_MUTEX_STATIC_INIT; 
 
 /*
  * attribute_value_t constructor function
@@ -507,33 +528,68 @@ ompi_attribute_keyval_destruct(ompi_attribute_keyval_t *keyval)
             free(keyval->bindings_extra_state);
         }
 
-        opal_hash_table_remove_value_uint32(keyval_hash, keyval->key);
+        opal_hash_table_remove_value_uint32(attr_subsys->keyval_hash, keyval->key);
         FREE_KEY(keyval->key);
     }
 }
 
 
-/*
- * This will initialize the main list to store key- attribute
- * items. This will be called one time, during MPI_INIT().
- */
-int ompi_attr_init(void)
+int ompi_attr_get_ref(void)
+{
+    int ret = OMPI_SUCCESS;
+
+    OPAL_THREAD_LOCK(&attribute_lock);
+
+    if (NULL == attr_subsys) {
+        attr_subsys = OBJ_NEW(attr_subsys_t);
+        if (NULL == attr_subsys) {
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto fn_exit;
+        }
+        if ((NULL == attr_subsys->keyval_hash) || (NULL == attr_subsys->key_bitmap)) {
+            OBJ_RELEASE(attr_subsys);
+            attr_subsys = NULL;
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto fn_exit;
+        }
+    } else {
+        OBJ_RETAIN(attr_subsys);
+    }
+
+fn_exit:
+    OPAL_THREAD_UNLOCK(&attribute_lock);
+
+    return ret;
+}
+
+int ompi_attr_put_ref(void)
+{
+    if (NULL != attr_subsys) {
+        OBJ_RELEASE(attr_subsys);
+    }
+    return OMPI_SUCCESS;
+}
+
+static void attr_subsys_construct(attr_subsys_t *subsys)
 {
     int ret;
     void *bogus = (void*) 1;
     int *p = (int *) &bogus;
 
-    keyval_hash = OBJ_NEW(opal_hash_table_t);
-    if (NULL == keyval_hash) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    key_bitmap = OBJ_NEW(opal_bitmap_t);
+    subsys->keyval_hash = OBJ_NEW(opal_hash_table_t);
+
+    subsys->key_bitmap = OBJ_NEW(opal_bitmap_t);
+
     /*
      * Set the max size to OMPI_FORTRAN_HANDLE_MAX to enforce bound
      */
-    opal_bitmap_set_max_size (key_bitmap, OMPI_FORTRAN_HANDLE_MAX);
-    if (0 != opal_bitmap_init(key_bitmap, 32)) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
+    opal_bitmap_set_max_size (subsys->key_bitmap, 
+                              OMPI_FORTRAN_HANDLE_MAX);
+    ret = opal_bitmap_init(subsys->key_bitmap, 32);
+    assert(OPAL_SUCCESS == ret);
+
+    for (int i = 0; i < MPI_ATTR_KEY_LAST; i++) {
+        opal_bitmap_set_bit(subsys->key_bitmap, i);
     }
 
     for (int_pos = 0; int_pos < (sizeof(void*) / sizeof(int));
@@ -550,31 +606,21 @@ int ompi_attr_init(void)
         }
     }
 
-    OBJ_CONSTRUCT(&attribute_lock, opal_mutex_t);
-
-    if (OMPI_SUCCESS != (ret = opal_hash_table_init(keyval_hash,
-                                                    ATTR_TABLE_SIZE))) {
-        return ret;
-    }
-    if (OMPI_SUCCESS != (ret = ompi_attr_create_predefined())) {
-        return ret;
-    }
+    ret = opal_hash_table_init(subsys->keyval_hash, ATTR_TABLE_SIZE);
+    assert (OPAL_SUCCESS == ret);
 
-    return OMPI_SUCCESS;
+    attr_sequence = 0;    
 }
 
 
 /*
- * Cleanup everything during MPI_Finalize().
+ * Cleanup everything when no more refs to the attr subsys
  */
-int ompi_attr_finalize(void)
+static void attr_subsys_destruct(attr_subsys_t *subsys)
 {
     ompi_attr_free_predefined();
-    OBJ_DESTRUCT(&attribute_lock);
-    OBJ_RELEASE(keyval_hash);
-    OBJ_RELEASE(key_bitmap);
-
-    return OMPI_SUCCESS;
+    OBJ_RELEASE(subsys->keyval_hash);
+    OBJ_RELEASE(subsys->key_bitmap);
 }
 
 /*****************************************************************************/
@@ -609,10 +655,15 @@ static int ompi_attr_create_keyval_impl(ompi_attribute_type_t type,
 
     /* Create a new unique key and fill the hash */
     OPAL_THREAD_LOCK(&attribute_lock);
-    ret = CREATE_KEY(key);
+    ret = MPI_SUCCESS;
+    if (!(flags & OMPI_KEYVAL_PREDEFINED)) {
+        ret = CREATE_KEY(key);
+    }
+    
     if (OMPI_SUCCESS == ret) {
         keyval->key = *key;
-        ret = opal_hash_table_set_value_uint32(keyval_hash, *key, keyval);
+        ret = opal_hash_table_set_value_uint32(attr_subsys->keyval_hash,
+                                               *key, keyval);
     }
 
     if (OMPI_SUCCESS != ret) {
@@ -635,11 +686,22 @@ int ompi_attr_create_keyval(ompi_attribute_type_t type,
                             void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 
     es_tmp.c_ptr = extra_state;
-    return ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
-                                        key, &es_tmp, flags,
-                                        bindings_extra_state);
+    rc = ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
+                                      key, &es_tmp, flags,
+                                      bindings_extra_state);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        ompi_mpi_instance_release ();
+    }
+
+    return rc;
 }
 
 int ompi_attr_create_keyval_fint(ompi_attribute_type_t type,
@@ -651,6 +713,12 @@ int ompi_attr_create_keyval_fint(ompi_attribute_type_t type,
                                  void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 
     es_tmp.f_integer = extra_state;
 #if SIZEOF_INT == OMPI_SIZEOF_FORTRAN_INTEGER
@@ -670,6 +738,12 @@ int ompi_attr_create_keyval_aint(ompi_attribute_type_t type,
                                  void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 
     es_tmp.f_address = extra_state;
     return ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
@@ -687,7 +761,7 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
 
     /* Find the key-value pair */
     OPAL_THREAD_LOCK(&attribute_lock);
-    ret = opal_hash_table_get_value_uint32(keyval_hash, *key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, *key,
                                            (void **) &keyval);
     if ((OMPI_SUCCESS != ret) || (NULL == keyval) ||
         (keyval->attr_type != type) ||
@@ -707,6 +781,9 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
     opal_atomic_wmb();
     OPAL_THREAD_UNLOCK(&attribute_lock);
 
+    /* balance out retain in keyval_create */
+    ompi_mpi_instance_release ();
+
     return MPI_SUCCESS;
 }
 
@@ -720,7 +797,7 @@ int ompi_attr_set_c(ompi_attribute_type_t type, void *object,
                     opal_hash_table_t **attr_hash,
                     int key, void *attribute, bool predefined)
 {
-    int ret;
+    int ret = MPI_SUCCESS;
     attribute_value_t *new_attr = OBJ_NEW(attribute_value_t);
     if (NULL == new_attr) {
         return OMPI_ERR_OUT_OF_RESOURCE;
@@ -942,7 +1019,7 @@ int ompi_attr_copy_all(ompi_attribute_type_t type, void *old_object,
 
         /* Get the keyval in the main keyval hash - so that we know
            what the copy_attr_fn is */
-        err = opal_hash_table_get_value_uint32(keyval_hash, key,
+        err = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                                (void **) &hash_value);
         if (OMPI_SUCCESS != err) {
             /* This should not happen! */
@@ -1037,7 +1114,7 @@ static int ompi_attr_delete_impl(ompi_attribute_type_t type, void *object,
     attribute_value_t *attr;
 
     /* Check if the key is valid in the master keyval hash */
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
 
     if ((OMPI_SUCCESS != ret) || (NULL == keyval) ||
@@ -1053,7 +1130,7 @@ static int ompi_attr_delete_impl(ompi_attribute_type_t type, void *object,
         goto exit;
     }
 
-    /* Check if the key is valid for the communicator/window/dtype. If
+    /* Check if the key is valid for the communicator/window/dtype/instance. If
        yes, then delete the attribute and key entry from the object's
        hash */
     ret = opal_hash_table_get_value_uint32(attr_hash, key, (void**) &attr);
@@ -1198,7 +1275,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
     /* Note that this function can be invoked by ompi_attr_copy_all()
        to set attributes on the new object (in addition to the
        top-level MPI_* functions that set attributes). */
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
 
     /* If key not found */
@@ -1242,7 +1319,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
         had_old = true;
     }
 
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
     if ((OMPI_SUCCESS != ret ) || (NULL == keyval)) {
         /* Keyval has disappeared underneath us -- this shouldn't
@@ -1288,7 +1365,7 @@ static int get_value(opal_hash_table_t *attr_hash, int key,
        with the key, then the call is valid and returns FALSE in the
        flag argument */
     *flag = 0;
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void**) &keyval);
     if (OMPI_ERR_NOT_FOUND == ret) {
         return MPI_KEYVAL_INVALID;