Add support for encoded refs in external terms

Also add partial support for external refs. Also fix list representation of local refs as a ref with two words, matching external term encoding. Signed-off-by: Paul Guyot <[email protected]>
atomvm · Nov 24, 2024 · 598f8c3 · 598f8c3
1 parent 00a48e0
commit 598f8c3
Show file tree

Hide file tree

Showing 13 changed files with 433 additions and 43 deletions.
diff --git a/src/libAtomVM/context.c b/src/libAtomVM/context.c
@@ -231,7 +231,7 @@ void context_process_flush_monitor_signal(Context *ctx, uint64_t ref_ticks, bool
         if (term_is_tuple(msg)
             && term_get_tuple_arity(msg) == 5
             && term_get_tuple_element(msg, 0) == DOWN_ATOM
-            && term_is_reference(term_get_tuple_element(msg, 1))
+            && term_is_local_reference(term_get_tuple_element(msg, 1))
             && term_to_ref_ticks(term_get_tuple_element(msg, 1)) == ref_ticks) {
             mailbox_remove_message(&ctx->mailbox, &ctx->heap);
             // If option info is combined with option flush, false is returned if a flush was needed, otherwise true.

diff --git a/src/libAtomVM/ets_hashtable.c b/src/libAtomVM/ets_hashtable.c
@@ -287,7 +287,7 @@ static uint32_t hash_term_incr(term t, int32_t h, GlobalContext *global)
         return hash_float(t, h, global);
     } else if (term_is_local_pid(t)) {
         return hash_pid(t, h, global);
-    } else if (term_is_reference(t)) {
+    } else if (term_is_local_reference(t)) {
         return hash_reference(t, h, global);
     } else if (term_is_binary(t)) {
         return hash_binary(t, h, global);

diff --git a/src/libAtomVM/externalterm.c b/src/libAtomVM/externalterm.c
@@ -36,6 +36,7 @@
 
 #define NEW_FLOAT_EXT 70
 #define NEW_PID_EXT 88
+#define NEWER_REFERENCE_EXT 90
 #define SMALL_INTEGER_EXT 97
 #define INTEGER_EXT 98
 #define ATOM_EXT 100
@@ -421,6 +422,48 @@ static int serialize_term(uint8_t *buf, term t, GlobalContext *glb)
             WRITE_32_UNALIGNED(buf + k + 8, term_to_external_node_creation(t));
         }
         return k + 12;
+    } else if (term_is_local_reference(t)) {
+        if (!IS_NULL_PTR(buf)) {
+            buf[0] = NEWER_REFERENCE_EXT;
+        }
+        size_t k = 1;
+        if (!IS_NULL_PTR(buf)) {
+            WRITE_16_UNALIGNED(buf + k, 2); // len
+        }
+        k += 2;
+        term node_name = glb->node_name;
+        uint32_t creation = node_name == NONODE_AT_NOHOST_ATOM ? 0 : glb->creation;
+        k += serialize_term(IS_NULL_PTR(buf) ? NULL : buf + k, node_name, glb);
+        if (!IS_NULL_PTR(buf)) {
+            uint64_t ticks = term_to_ref_ticks(t);
+            WRITE_32_UNALIGNED(buf + k, creation);
+            WRITE_64_UNALIGNED(buf + k + 4, ticks);
+        }
+        return k + 12;
+    } else if (term_is_external_reference(t)) {
+        if (!IS_NULL_PTR(buf)) {
+            buf[0] = NEWER_REFERENCE_EXT;
+        }
+        size_t k = 1;
+        if (!IS_NULL_PTR(buf)) {
+            WRITE_16_UNALIGNED(buf + k, 2); // len
+        }
+        k += 2;
+        term node = term_to_external_node(t);
+        k += serialize_term(IS_NULL_PTR(buf) ? NULL : buf + k, node, glb);
+        uint32_t len = term_to_external_reference_len(t);
+        if (!IS_NULL_PTR(buf)) {
+            WRITE_32_UNALIGNED(buf + k, term_to_external_node_creation(t));
+        }
+        k += 4;
+        if (!IS_NULL_PTR(buf)) {
+            uint32_t data[len];
+            term_to_external_reference_words(t, data);
+            for (uint32_t i = 0; i < len; i++) {
+                WRITE_64_UNALIGNED(buf + k + (i * 4), data[i]);
+            }
+        }
+        return k + (4 * len);
     } else {
         fprintf(stderr, "Unknown external term type: %" TERM_U_FMT "\n", t);
         AVM_ABORT();
@@ -716,6 +759,37 @@ static term parse_external_terms(const uint8_t *external_term_buf, size_t *eterm
             }
         }
 
+        case NEWER_REFERENCE_EXT: {
+            uint16_t len = READ_16_UNALIGNED(external_term_buf + 1);
+            if (UNLIKELY(len > 5)) {
+                return term_invalid_term();
+            }
+            size_t node_size;
+            term node = parse_external_terms(external_term_buf + 3, &node_size, copy, heap, glb);
+            if (UNLIKELY(!term_is_atom(node))) {
+                return term_invalid_term();
+            }
+            uint32_t creation = READ_32_UNALIGNED(external_term_buf + node_size + 3);
+            uint32_t data[len];
+            for (uint16_t i = 0; i < len; i++) {
+                data[i] = READ_32_UNALIGNED(external_term_buf + node_size + 7 + (i * 4));
+            }
+            *eterm_size = node_size + 7 + (len * 4);
+            if (node != NONODE_AT_NOHOST_ATOM || len != 2 || creation != 0) {
+                term this_node = glb->node_name;
+                uint32_t this_creation = this_node == NONODE_AT_NOHOST_ATOM ? 0 : glb->creation;
+                if (len == 2 && node == this_node && creation == this_creation) {
+                    uint64_t ticks = ((uint64_t) data[0]) << 32 | data[1];
+                    return term_from_ref_ticks(ticks, heap);
+                } else {
+                    return term_from_external_reference(node, len, data, creation, heap);
+                }
+            } else {
+                uint64_t ticks = ((uint64_t) data[0]) << 32 | data[1];
+                return term_from_ref_ticks(ticks, heap);
+            }
+        }
+
         default:
             return term_invalid_term();
     }
@@ -1032,6 +1106,33 @@ static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaini
             return heap_size + u;
         }
 
+        case NEWER_REFERENCE_EXT: {
+            if (UNLIKELY(remaining < 3)) {
+                return INVALID_TERM_SIZE;
+            }
+            remaining -= 3;
+            int buf_pos = 3;
+            size_t heap_size = EXTERNAL_REF_SIZE;
+            uint16_t len = READ_16_UNALIGNED(external_term_buf + 1);
+            size_t node_size = 0;
+            int u = calculate_heap_usage(external_term_buf + buf_pos, remaining, &node_size, copy);
+            if (UNLIKELY(u == INVALID_TERM_SIZE)) {
+                return INVALID_TERM_SIZE;
+            }
+            if (external_term_buf[3] == SMALL_ATOM_UTF8_EXT) {
+                // Check if it's non-distributed node, in which case it's always a local ref
+                if (len == 2 && external_term_buf[4] == strlen("nonode@nohost") && memcmp(external_term_buf + 5, "nonode@nohost", strlen("nonode@nohost")) == 0) {
+                    heap_size = REF_SIZE;
+                }
+                // See above for pids
+            } else if (UNLIKELY(external_term_buf[3] != ATOM_EXT)) {
+                return INVALID_TERM_SIZE;
+            }
+            buf_pos += node_size;
+            *eterm_size = buf_pos + 4 + (len * 4);
+            return heap_size + u;
+        }
+
         default:
             return INVALID_TERM_SIZE;
     }

diff --git a/src/libAtomVM/memory.c b/src/libAtomVM/memory.c
@@ -628,6 +628,10 @@ static void memory_scan_and_copy(HeapFragment *old_fragment, term *mem_start, co
                     TRACE("- Found external pid.\n");
                     break;
 
+                case TERM_BOXED_EXTERNAL_REF:
+                    TRACE("- Found external ref.\n");
+                    break;
+
                 case TERM_BOXED_FUN: {
                     int fun_size = term_get_size_from_boxed_header(t);
                     TRACE("- Found fun, size: %i.\n", fun_size);
@@ -752,6 +756,10 @@ static void memory_scan_and_rewrite(size_t count, term *terms, const term *old_s
                     ptr += EXTERNAL_PID_SIZE - 1;
                     break;
 
+                case TERM_BOXED_EXTERNAL_REF:
+                    ptr += EXTERNAL_REF_SIZE - 1;
+                    break;
+
                 case TERM_BOXED_FUN:
                     // Skip header and module and process next terms
                     ptr++;

diff --git a/src/libAtomVM/nifs.c b/src/libAtomVM/nifs.c
@@ -3309,7 +3309,7 @@ static term nif_ets_new(Context *ctx, int argc, term argv[])
 
 static inline bool is_ets_table_id(term t)
 {
-    return term_is_reference(t) || term_is_atom(t);
+    return term_is_local_reference(t) || term_is_atom(t);
 }
 
 static term nif_ets_insert(Context *ctx, int argc, term argv[])
@@ -3460,11 +3460,11 @@ static term nif_erlang_ref_to_list(Context *ctx, int argc, term argv[])
 
     term t = argv[0];
     VALIDATE_VALUE(t, term_is_reference);
+    size_t max_len = term_is_external(t) ? EXTERNAL_REF_AS_CSTRING_LEN : LOCAL_REF_AS_CSTRING_LEN;
 
-    char buf[REF_AS_CSTRING_LEN];
-    int str_len = term_snprint(buf, REF_AS_CSTRING_LEN, t, ctx->global);
+    char buf[max_len];
+    int str_len = term_snprint(buf, max_len, t, ctx->global);
     if (UNLIKELY(str_len < 0)) {
-        // TODO: change to internal error or something like that
         RAISE_ERROR(OUT_OF_MEMORY_ATOM);
     }
 
@@ -3764,7 +3764,7 @@ static term nif_erlang_demonitor(Context *ctx, int argc, term argv[])
         info = interop_proplist_get_value_default(options, INFO_ATOM, FALSE_ATOM) == TRUE_ATOM;
     }
 
-    VALIDATE_VALUE(ref, term_is_reference);
+    VALIDATE_VALUE(ref, term_is_local_reference);
     uint64_t ref_ticks = term_to_ref_ticks(ref);
 
     bool result = globalcontext_demonitor(ctx->global, ref_ticks);

diff --git a/src/libAtomVM/otp_socket.c b/src/libAtomVM/otp_socket.c
@@ -593,7 +593,7 @@ bool term_is_otp_socket(term socket_term)
     bool ret = term_is_tuple(socket_term)
         && term_get_tuple_arity(socket_term) == 2
         && term_is_binary(term_get_tuple_element(socket_term, 0))
-        && term_is_reference(term_get_tuple_element(socket_term, 1));
+        && term_is_local_reference(term_get_tuple_element(socket_term, 1));
 
     TRACE("term is a socket: %i\n", ret);
 
@@ -898,7 +898,7 @@ static term nif_socket_select_read(Context *ctx, int argc, term argv[])
 
     term select_ref_term = argv[1];
     if (select_ref_term != UNDEFINED_ATOM) {
-        VALIDATE_VALUE(select_ref_term, term_is_reference);
+        VALIDATE_VALUE(select_ref_term, term_is_local_reference);
     }
     struct SocketResource *rsrc_obj;
     if (UNLIKELY(!term_to_otp_socket(argv[0], &rsrc_obj, ctx))) {

diff --git a/src/libAtomVM/posix_nifs.c b/src/libAtomVM/posix_nifs.c
@@ -431,7 +431,7 @@ static term nif_atomvm_posix_select(Context *ctx, term argv[], enum ErlNifSelect
     int32_t process_pid = term_to_local_process_id(process_pid_term);
     term select_ref_term = argv[2];
     if (select_ref_term != UNDEFINED_ATOM) {
-        VALIDATE_VALUE(select_ref_term, term_is_reference);
+        VALIDATE_VALUE(select_ref_term, term_is_local_reference);
     }
     void *fd_obj_ptr;
     if (UNLIKELY(!enif_get_resource(erl_nif_env_from_context(ctx), argv[0], ctx->global->posix_fd_resource_type, &fd_obj_ptr))) {

diff --git a/src/libAtomVM/resources.c b/src/libAtomVM/resources.c
@@ -121,7 +121,7 @@ int enif_select(ErlNifEnv *env, ErlNifEvent event, enum ErlNifSelectFlags mode,
     if (!(mode & (ERL_NIF_SELECT_STOP | ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE))) {
         return ERL_NIF_SELECT_BADARG;
     }
-    if (UNLIKELY(mode & (ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE) && !term_is_reference(ref) && ref != UNDEFINED_ATOM)) {
+    if (UNLIKELY(mode & (ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE) && !term_is_local_reference(ref) && ref != UNDEFINED_ATOM)) {
         return ERL_NIF_SELECT_BADARG;
     }
 

diff --git a/src/libAtomVM/term.c b/src/libAtomVM/term.c
@@ -384,11 +384,25 @@ int term_funprint(PrinterFun *fun, term t, const GlobalContext *global)
         ret += printed;
         return ret;
 
-    } else if (term_is_reference(t)) {
+    } else if (term_is_local_reference(t)) {
         uint64_t ref_ticks = term_to_ref_ticks(t);
 
-        // Update also REF_AS_CSTRING_LEN when changing this format string
-        return fun->print(fun, "#Ref<0.0.0.%" PRIu64 ">", ref_ticks);
+        // Update also LOCAL_REF_AS_CSTRING_LEN when changing this format string
+        return fun->print(fun, "#Ref<0.%" PRIu32 ".%" PRIu32 ">", (uint32_t) (ref_ticks >> 32), (uint32_t) ref_ticks);
+
+    } else if (term_is_external_reference(t)) {
+        // Update also EXTERNAL_REF_AS_CSTRING_LEN when changing this format string
+        uint32_t node_atom_index = term_to_atom_index(term_to_external_node(t));
+        uint32_t len = term_to_external_reference_len(t);
+        uint32_t data[len];
+        term_to_external_reference_words(t, data);
+        // creation is not printed
+        int ret = fun->print(fun, "#Ref<%" PRIu32, node_atom_index);
+        for (uint32_t i = 0; i < len; i++) {
+            ret += fun->print(fun, ".%" PRIu32, data[i]);
+        }
+        ret += fun->print(fun, ">");
+        return ret;
 
     } else if (term_is_boxed_integer(t)) {
         int size = term_boxed_size(t);
@@ -499,14 +513,52 @@ TermCompareResult term_compare(term t, term other, TermCompareOpts opts, GlobalC
             break;
 
         } else if (term_is_reference(t) && term_is_reference(other)) {
-            int64_t t_ticks = term_to_ref_ticks(t);
-            int64_t other_ticks = term_to_ref_ticks(other);
-            if (t_ticks == other_ticks) {
-                CMP_POP_AND_CONTINUE();
+            if (!term_is_external(t) && !term_is_external(other)) {
+                int64_t t_ticks = term_to_ref_ticks(t);
+                int64_t other_ticks = term_to_ref_ticks(other);
+                if (t_ticks == other_ticks) {
+                    CMP_POP_AND_CONTINUE();
+                } else {
+                    result = (t_ticks > other_ticks) ? TermGreaterThan : TermLessThan;
+                    break;
+                }
+            } else if (term_is_external(t) && term_is_external(other)) {
+                term node = term_to_external_node(t);
+                term other_node = term_to_external_node(other);
+                if (node == other_node) {
+                    uint32_t creation = term_to_external_node_creation(t);
+                    uint32_t other_creation = term_to_external_node_creation(other);
+                    if (creation == other_creation) {
+                        uint32_t len = term_to_external_reference_len(t);
+                        uint32_t other_len = term_to_external_reference_len(other);
+                        if (len == other_len) {
+                            uint32_t data[len];
+                            uint32_t other_data[len];
+                            term_to_external_reference_words(t, data);
+                            term_to_external_reference_words(other, other_data);
+                            int cmp = memcmp(data, other_data, len * sizeof(uint32_t));
+                            if (cmp == 0) {
+                                CMP_POP_AND_CONTINUE();
+                            } else {
+                                result = (cmp > 0) ? TermGreaterThan : TermLessThan;
+                                break;
+                            }
+                        } else {
+                            result = (len > other_len) ? TermGreaterThan : TermLessThan;
+                            break;
+                        }
+                    } else {
+                        result = (creation > other_creation) ? TermGreaterThan : TermLessThan;
+                        break;
+                    }
+                } else {
+                    result = (node > other_node) ? TermGreaterThan : TermLessThan;
+                    break;
+                }
             } else {
-                result = (t_ticks > other_ticks) ? TermGreaterThan : TermLessThan;
-                break;
+                result = term_is_external(t) ? TermGreaterThan : TermLessThan;
             }
+            break;
 
         } else if (term_is_nonempty_list(t) && term_is_nonempty_list(other)) {
             term t_tail = term_get_list_tail(t);