Python 3.12 Unicode API

cbits/hscpython-shim.c

@@ -150,13 +150,19 @@ PyObject *hscpython_Py_False()
 
 /* Unicode */
 Py_ssize_t hscpython_PyUnicode_GetSize(PyObject *o)
-{ return PyUnicode_GetSize(o); }
-
-Py_UNICODE *hscpython_PyUnicode_AsUnicode(PyObject *o)
-{ return PyUnicode_AsUnicode(o); }
+{ return PyUnicode_GET_LENGTH(o); }
+
+wchar_t *hscpython_PyUnicode_AsUnicode(PyObject *o)
+{ wchar_t *wstr;
+	Py_ssize_t actual_size;
+	actual_size = PyUnicode_AsWideChar(o, NULL, 0);
+  wstr = malloc(actual_size);
+  PyUnicode_AsWideChar(o, wstr, actual_size);
+	return wstr;
+}
 
-PyObject *hscpython_PyUnicode_FromUnicode(Py_UNICODE *u, Py_ssize_t size)
-{ return PyUnicode_FromUnicode(u, size); }
+PyObject *hscpython_PyUnicode_FromUnicode(const wchar_t *u, Py_ssize_t size)
+{ return PyUnicode_FromWideChar(u, size); }
 
 PyObject *hscpython_PyUnicode_FromEncodedObject(PyObject *o, const char *enc, const char *err)
 { return PyUnicode_FromEncodedObject(o, enc, err); }

cbits/hscpython-shim.h

@@ -54,8 +54,8 @@ PyObject *hscpython_Py_False();
 
 /* Unicode */
 Py_ssize_t hscpython_PyUnicode_GetSize(PyObject *);
-Py_UNICODE *hscpython_PyUnicode_AsUnicode(PyObject *);
-PyObject *hscpython_PyUnicode_FromUnicode(Py_UNICODE *, Py_ssize_t);
+wchar_t *hscpython_PyUnicode_AsUnicode(PyObject *);
+PyObject *hscpython_PyUnicode_FromUnicode(const wchar_t *, Py_ssize_t);
 PyObject *hscpython_PyUnicode_FromEncodedObject(PyObject *, const char *, const char *);
 PyObject *hscpython_PyUnicode_AsEncodedString(PyObject *, const char *, const char *);
 PyObject *hscpython_PyUnicode_FromObject(PyObject *);

lib/CPython/Types/Unicode.chs

@@ -51,13 +51,8 @@ module CPython.Types.Unicode
 import           Prelude hiding (length)
 import           Control.Exception (ErrorCall (..), throwIO)
 import qualified Data.Text as T
-
-#ifdef Py_UNICODE_WIDE
-import           Data.Char (chr, ord)
-#else
-import qualified Data.Text.Foreign as TF
-#endif
-
+import           Foreign.C.String
+import           Foreign.C.Types
 import           CPython.Internal
 import           CPython.Types.Bytes (Bytes)
 
@@ -86,34 +81,24 @@ withErrors errors = withCString $ case errors of
 {# fun pure unsafe hscpython_PyUnicode_Type as unicodeType
   {} -> `Type' peekStaticObject* #}
 
+
 toUnicode :: T.Text -> IO Unicode
-toUnicode str = withBuffer toPython >>= stealObject where
-  toPython ptr len = let
-    len' = fromIntegral len
-    ptr' = castPtr ptr
-    in {# call hscpython_PyUnicode_FromUnicode #} ptr' len'
-#ifdef Py_UNICODE_WIDE
-  ords = map (fromIntegral . ord) (T.unpack str) :: [CUInt]
-  withBuffer = withArrayLen ords . flip
-#else
-  withBuffer = TF.useAsPtr str
-#endif
+toUnicode txt = withCWStringLen (T.unpack txt) $ \(wstr, sz) -> do
+  obj <- {# call hscpython_PyUnicode_FromUnicode #} (castPtr wstr) (fromIntegral sz)
+  stealObject obj
+
 
 fromUnicode :: Unicode -> IO T.Text
 fromUnicode obj = withObject obj $ \ptr -> do
-  buffer <- {# call hscpython_PyUnicode_AsUnicode #} ptr
-  size <- {# call hscpython_PyUnicode_GetSize #} ptr
-#ifdef Py_UNICODE_WIDE
-  raw <- peekArray (fromIntegral size) buffer
-  return . T.pack $ map (chr . fromIntegral) raw
-#else
-  TF.fromPtr (castPtr buffer) (fromIntegral size)
-#endif
+  wstrPtr <- {# call hscpython_PyUnicode_AsUnicode #} ptr
+  wstr <- peekCWString . castPtr $ wstrPtr
+  return . T.pack $ wstr
 
 {# fun hscpython_PyUnicode_GetSize as length
   { withObject* `Unicode'
   } -> `Integer' checkIntReturn* #}
 
+
 -- | Coerce an encoded object /obj/ to an Unicode object.
 --
 -- 'Bytes' and other char buffer compatible objects are decoded according to