gh-111968: Use per-thread freelists for dict in free-threading

Include/internal/pycore_dict.h

@@ -9,6 +9,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_freelist.h"      // _PyFreeListState
 #include "pycore_identifier.h"    // _Py_Identifier
 #include "pycore_object.h"        // PyDictOrValues
 
@@ -69,7 +70,7 @@ extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other);
 
 /* runtime lifecycle */
 
-extern void _PyDict_Fini(PyInterpreterState *interp);
+extern void _PyDict_Fini(_PyFreeListState *state);
 
 
 /* other API */

Include/internal/pycore_freelist.h

@@ -17,13 +17,15 @@ extern "C" {
 #  define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE
 #  define PyTuple_MAXFREELIST 2000
 #  define PyList_MAXFREELIST 80
+#  define PyDict_MAXFREELIST 80
 #  define PyFloat_MAXFREELIST 100
 #  define PyContext_MAXFREELIST 255
 # define _PyAsyncGen_MAXFREELIST 80
 #else
 #  define PyTuple_NFREELISTS 0
 #  define PyTuple_MAXFREELIST 0
 #  define PyList_MAXFREELIST 0
+#  define PyDict_MAXFREELIST 0
 #  define PyFloat_MAXFREELIST 0
 #  define PyContext_MAXFREELIST 0
 #  define _PyAsyncGen_MAXFREELIST 0
@@ -63,6 +65,31 @@ struct _Py_float_state {
 #endif
 };
 
+#define DICT_MAX_WATCHERS 8
+
+struct _Py_dict_state {
+    /*Global counter used to set ma_version_tag field of dictionary.
+     * It is incremented each time that a dictionary is created and each
+     * time that a dictionary is modified. */
+    uint64_t global_version;
+    uint32_t next_keys_version;
+
+#ifdef WITH_FREELISTS
+    /* Dictionary reuse scheme to save calls to malloc and free */
+    PyDictObject *free_list[PyDict_MAXFREELIST];
+    PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
+    int numfree;
+    int keys_numfree;
+#endif
+
+    PyDict_WatchCallback watchers[DICT_MAX_WATCHERS];
+};
+
+#define _dict_state_INIT \
+    { \
+        .next_keys_version = 2, \
+    }
+
 struct _Py_slice_state {
 #ifdef WITH_FREELISTS
     /* Using a cache is very effective since typically only a single slice is
@@ -97,6 +124,7 @@ typedef struct _Py_freelist_state {
     struct _Py_float_state float_state;
     struct _Py_tuple_state tuple_state;
     struct _Py_list_state list_state;
+    struct _Py_dict_state dict_state;
     struct _Py_slice_state slice_state;
     struct _Py_context_state context_state;
     struct _Py_async_gen_state async_gen_state;

Include/internal/pycore_gc.h

@@ -250,7 +250,7 @@ extern void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization)
 extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization);
 extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization);
 extern void _PySlice_ClearCache(_PyFreeListState *state);
-extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
+extern void _PyDict_ClearFreeList(_PyFreeListState *state, int is_finalization);
 extern void _PyAsyncGen_ClearFreeLists(_PyFreeListState *state, int is_finalization);
 extern void _PyContext_ClearFreeList(_PyFreeListState *state, int is_finalization);
 extern void _Py_ScheduleGC(PyInterpreterState *interp);

Include/internal/pycore_interp.h

@@ -16,10 +16,10 @@ extern "C" {
 #include "pycore_code.h"          // struct callable_cache
 #include "pycore_context.h"       // struct _Py_context_state
 #include "pycore_crossinterp.h"   // struct _xidregistry
-#include "pycore_dict_state.h"    // struct _Py_dict_state
 #include "pycore_dtoa.h"          // struct _dtoa_state
 #include "pycore_exceptions.h"    // struct _Py_exc_state
 #include "pycore_floatobject.h"   // struct _Py_float_state
+#include "pycore_freelist.h"      // struct _Py_freelist_state
 #include "pycore_function.h"      // FUNC_MAX_WATCHERS
 #include "pycore_gc.h"            // struct _gc_runtime_state
 #include "pycore_genobject.h"     // struct _Py_async_gen_state

Makefile.pre.in

@@ -1821,7 +1821,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_critical_section.h \
 		$(srcdir)/Include/internal/pycore_crossinterp.h \
 		$(srcdir)/Include/internal/pycore_dict.h \
-		$(srcdir)/Include/internal/pycore_dict_state.h \
 		$(srcdir)/Include/internal/pycore_descrobject.h \
 		$(srcdir)/Include/internal/pycore_dtoa.h \
 		$(srcdir)/Include/internal/pycore_exceptions.h \

Objects/dictobject.c

@@ -118,6 +118,7 @@ As a consequence of this, split keys have a maximum size of 16.
 #include "pycore_ceval.h"         // _PyEval_GetBuiltin()
 #include "pycore_code.h"          // stats
 #include "pycore_dict.h"          // export _PyDict_SizeOf()
+#include "pycore_freelist.h"      // _PyFreeListState_GET()
 #include "pycore_gc.h"            // _PyObject_GC_IS_TRACKED()
 #include "pycore_object.h"        // _PyObject_GC_TRACK(), _PyDebugAllocatorStats()
 #include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
@@ -242,41 +243,40 @@ static PyObject* dict_iter(PyObject *dict);
 #include "clinic/dictobject.c.h"
 
 
-#if PyDict_MAXFREELIST > 0
+#ifdef WITH_FREELISTS
 static struct _Py_dict_state *
-get_dict_state(PyInterpreterState *interp)
+get_dict_state(void)
 {
-    return &interp->dict_state;
+    _PyFreeListState *state = _PyFreeListState_GET();
+    return &state->dict_state;
 }
 #endif
 
 
 void
-_PyDict_ClearFreeList(PyInterpreterState *interp)
+_PyDict_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
 {
-#if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = &interp->dict_state;
-    while (state->numfree) {
+#ifdef WITH_FREELISTS
+    struct _Py_dict_state *state = &freelist_state->dict_state;
+    while (state->numfree > 0) {
         PyDictObject *op = state->free_list[--state->numfree];
         assert(PyDict_CheckExact(op));
         PyObject_GC_Del(op);
     }
-    while (state->keys_numfree) {
+    while (state->keys_numfree > 0) {
         PyObject_Free(state->keys_free_list[--state->keys_numfree]);
     }
+    if (is_finalization) {
+        state->numfree = -1;
+        state->keys_numfree = -1;
+    }
 #endif
 }
 
-
 void
-_PyDict_Fini(PyInterpreterState *interp)
+_PyDict_Fini(_PyFreeListState *freelist_state)
 {
-    _PyDict_ClearFreeList(interp);
-#if defined(Py_DEBUG) && PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = &interp->dict_state;
-    state->numfree = -1;
-    state->keys_numfree = -1;
-#endif
+    _PyDict_ClearFreeList(freelist_state, 1);
 }
 
 static inline Py_hash_t
@@ -290,9 +290,8 @@ unicode_get_hash(PyObject *o)
 void
 _PyDict_DebugMallocStats(FILE *out)
 {
-#if PyDict_MAXFREELIST > 0
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    struct _Py_dict_state *state = get_dict_state(interp);
+#ifdef WITH_FREELISTS
+    struct _Py_dict_state *state = get_dict_state();
     _PyDebugAllocatorStats(out, "free PyDictObject",
                            state->numfree, sizeof(PyDictObject));
 #endif
@@ -627,12 +626,8 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
         log2_bytes = log2_size + 2;
     }
 
-#if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = get_dict_state(interp);
-#ifdef Py_DEBUG
-    // new_keys_object() must not be called after _PyDict_Fini()
-    assert(state->keys_numfree != -1);
-#endif
+#ifdef WITH_FREELISTS
+    struct _Py_dict_state *state = get_dict_state();
     if (log2_size == PyDict_LOG_MINSIZE && unicode && state->keys_numfree > 0) {
         dk = state->keys_free_list[--state->keys_numfree];
         OBJECT_STAT_INC(from_freelist);
@@ -683,14 +678,11 @@ free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys)
             Py_XDECREF(entries[i].me_value);
         }
     }
-#if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = get_dict_state(interp);
-#ifdef Py_DEBUG
-    // free_keys_object() must not be called after _PyDict_Fini()
-    assert(state->keys_numfree != -1);
-#endif
+#ifdef WITH_FREELISTS
+    struct _Py_dict_state *state = get_dict_state();
     if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE
             && state->keys_numfree < PyDict_MAXFREELIST
+            && state->keys_numfree >= 0
             && DK_IS_UNICODE(keys)) {
         state->keys_free_list[state->keys_numfree++] = keys;
         OBJECT_STAT_INC(to_freelist);
@@ -731,13 +723,9 @@ new_dict(PyInterpreterState *interp,
 {
     PyDictObject *mp;
     assert(keys != NULL);
-#if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = get_dict_state(interp);
-#ifdef Py_DEBUG
-    // new_dict() must not be called after _PyDict_Fini()
-    assert(state->numfree != -1);
-#endif
-    if (state->numfree) {
+#ifdef WITH_FREELISTS
+    struct _Py_dict_state *state = get_dict_state();
+    if (state->numfree > 0) {
         mp = state->free_list[--state->numfree];
         assert (mp != NULL);
         assert (Py_IS_TYPE(mp, &PyDict_Type));
@@ -1552,15 +1540,12 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
 #endif
             assert(oldkeys->dk_kind != DICT_KEYS_SPLIT);
             assert(oldkeys->dk_refcnt == 1);
-#if PyDict_MAXFREELIST > 0
-            struct _Py_dict_state *state = get_dict_state(interp);
-#ifdef Py_DEBUG
-            // dictresize() must not be called after _PyDict_Fini()
-            assert(state->keys_numfree != -1);
-#endif
+#ifdef WITH_FREELISTS
+            struct _Py_dict_state *state = get_dict_state();
             if (DK_LOG_SIZE(oldkeys) == PyDict_LOG_MINSIZE &&
                     DK_IS_UNICODE(oldkeys) &&
-                    state->keys_numfree < PyDict_MAXFREELIST)
+                    state->keys_numfree < PyDict_MAXFREELIST &&
+                    state->keys_numfree >= 0)
             {
                 state->keys_free_list[state->keys_numfree++] = oldkeys;
                 OBJECT_STAT_INC(to_freelist);
@@ -2480,13 +2465,9 @@ dict_dealloc(PyObject *self)
         assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS);
         dictkeys_decref(interp, keys);
     }
-#if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = get_dict_state(interp);
-#ifdef Py_DEBUG
-    // new_dict() must not be called after _PyDict_Fini()
-    assert(state->numfree != -1);
-#endif
-    if (state->numfree < PyDict_MAXFREELIST && Py_IS_TYPE(mp, &PyDict_Type)) {
+#ifdef WITH_FREELISTS
+    struct _Py_dict_state *state = get_dict_state();
+    if (state->numfree < PyDict_MAXFREELIST && state->numfree >=0 && Py_IS_TYPE(mp, &PyDict_Type)) {
         state->free_list[state->numfree++] = mp;
         OBJECT_STAT_INC(to_freelist);
     }

PCbuild/pythoncore.vcxproj

@@ -222,7 +222,6 @@
     <ClInclude Include="..\Include\internal\pycore_crossinterp.h" />
     <ClInclude Include="..\Include\internal\pycore_descrobject.h" />
     <ClInclude Include="..\Include\internal\pycore_dict.h" />
-    <ClInclude Include="..\Include\internal\pycore_dict_state.h" />
     <ClInclude Include="..\Include\internal\pycore_dtoa.h" />
     <ClInclude Include="..\Include\internal\pycore_exceptions.h" />
     <ClInclude Include="..\Include\internal\pycore_faulthandler.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -594,9 +594,6 @@
     <ClInclude Include="..\Include\internal\pycore_dict.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
-    <ClInclude Include="..\Include\internal\pycore_dict_state.h">
-      <Filter>Include\internal</Filter>
-    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_dtoa.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

Python/gc_free_threading.c

@@ -14,8 +14,6 @@
 void
 _PyGC_ClearAllFreeLists(PyInterpreterState *interp)
 {
-    _PyDict_ClearFreeList(interp);
-
     HEAD_LOCK(&_PyRuntime);
     _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head;
     while (tstate != NULL) {

Python/gc_gil.c

@@ -11,8 +11,6 @@
 void
 _PyGC_ClearAllFreeLists(PyInterpreterState *interp)
 {
-    _PyDict_ClearFreeList(interp);
-
     _Py_ClearFreeLists(&interp->freelist_state, 0);
 }
 

Python/pylifecycle.c

@@ -1749,12 +1749,12 @@ finalize_interp_types(PyInterpreterState *interp)
     // a dict internally.
     _PyUnicode_ClearInterned(interp);
 
-    _PyDict_Fini(interp);
     _PyUnicode_Fini(interp);
 
     _PyFreeListState *state = _PyFreeListState_GET();
     _PyTuple_Fini(state);
     _PyList_Fini(state);
+    _PyDict_Fini(state);
     _PyFloat_Fini(state);
     _PySlice_Fini(state);
     _PyContext_Fini(state);

Python/pystate.c

@@ -1461,6 +1461,7 @@ _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization)
     _PyFloat_ClearFreeList(state, is_finalization);
     _PyTuple_ClearFreeList(state, is_finalization);
     _PyList_ClearFreeList(state, is_finalization);
+    _PyDict_ClearFreeList(state, is_finalization);
     _PyContext_ClearFreeList(state, is_finalization);
     _PyAsyncGen_ClearFreeLists(state, is_finalization);
 }