Skip to content

Commit 5ccbbe5

Browse files
vstinnervstinner
authored
gh-106320: Move _PyUnicodeWriter to the internal C API (#106342)
Move also _PyUnicode_FormatAdvancedWriter(). CJK codecs and multibytecodec.c now define the Py_BUILD_CORE_MODULE macro.
1 parent d65b783 commit 5ccbbe5

File tree

8 files changed

+166
-144
lines changed

8 files changed

+166
-144
lines changed

‎Include/cpython/unicodeobject.h‎

Lines changed: 0 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -480,131 +480,6 @@ PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar (
480480
Py_ssize_t start,
481481
Py_ssize_t end);
482482

483-
/* --- _PyUnicodeWriter API ----------------------------------------------- */
484-
485-
typedef struct {
486-
PyObject *buffer;
487-
void *data;
488-
int kind;
489-
Py_UCS4 maxchar;
490-
Py_ssize_t size;
491-
Py_ssize_t pos;
492-
493-
/* minimum number of allocated characters (default: 0) */
494-
Py_ssize_t min_length;
495-
496-
/* minimum character (default: 127, ASCII) */
497-
Py_UCS4 min_char;
498-
499-
/* If non-zero, overallocate the buffer (default: 0). */
500-
unsigned char overallocate;
501-
502-
/* If readonly is 1, buffer is a shared string (cannot be modified)
503-
and size is set to 0. */
504-
unsigned char readonly;
505-
} _PyUnicodeWriter ;
506-
507-
/* Initialize a Unicode writer.
508-
*
509-
* By default, the minimum buffer size is 0 character and overallocation is
510-
* disabled. Set min_length, min_char and overallocate attributes to control
511-
* the allocation of the buffer. */
512-
PyAPI_FUNC(void)
513-
_PyUnicodeWriter_Init(_PyUnicodeWriter *writer);
514-
515-
/* Prepare the buffer to write 'length' characters
516-
with the specified maximum character.
517-
518-
Return 0 on success, raise an exception and return -1 on error. */
519-
#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \
520-
(((MAXCHAR) <= (WRITER)->maxchar \
521-
&& (LENGTH) <= (WRITER)->size - (WRITER)->pos) \
522-
? 0 \
523-
: (((LENGTH) == 0) \
524-
? 0 \
525-
: _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR))))
526-
527-
/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro
528-
instead. */
529-
PyAPI_FUNC(int)
530-
_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer,
531-
Py_ssize_t length, Py_UCS4 maxchar);
532-
533-
/* Prepare the buffer to have at least the kind KIND.
534-
For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will
535-
support characters in range U+000-U+FFFF.
536-
537-
Return 0 on success, raise an exception and return -1 on error. */
538-
#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \
539-
((KIND) <= (WRITER)->kind \
540-
? 0 \
541-
: _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND)))
542-
543-
/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind()
544-
macro instead. */
545-
PyAPI_FUNC(int)
546-
_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer,
547-
int kind);
548-
549-
/* Append a Unicode character.
550-
Return 0 on success, raise an exception and return -1 on error. */
551-
PyAPI_FUNC(int)
552-
_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer,
553-
Py_UCS4 ch
554-
);
555-
556-
/* Append a Unicode string.
557-
Return 0 on success, raise an exception and return -1 on error. */
558-
PyAPI_FUNC(int)
559-
_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer,
560-
PyObject *str /* Unicode string */
561-
);
562-
563-
/* Append a substring of a Unicode string.
564-
Return 0 on success, raise an exception and return -1 on error. */
565-
PyAPI_FUNC(int)
566-
_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer,
567-
PyObject *str, /* Unicode string */
568-
Py_ssize_t start,
569-
Py_ssize_t end
570-
);
571-
572-
/* Append an ASCII-encoded byte string.
573-
Return 0 on success, raise an exception and return -1 on error. */
574-
PyAPI_FUNC(int)
575-
_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer,
576-
const char *str, /* ASCII-encoded byte string */
577-
Py_ssize_t len /* number of bytes, or -1 if unknown */
578-
);
579-
580-
/* Append a latin1-encoded byte string.
581-
Return 0 on success, raise an exception and return -1 on error. */
582-
PyAPI_FUNC(int)
583-
_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer,
584-
const char *str, /* latin1-encoded byte string */
585-
Py_ssize_t len /* length in bytes */
586-
);
587-
588-
/* Get the value of the writer as a Unicode string. Clear the
589-
buffer of the writer. Raise an exception and return NULL
590-
on error. */
591-
PyAPI_FUNC(PyObject *)
592-
_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer);
593-
594-
/* Deallocate memory of a writer (clear its internal buffer). */
595-
PyAPI_FUNC(void)
596-
_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer);
597-
598-
599-
/* Format the object based on the format_spec, as defined in PEP 3101
600-
(Advanced String Formatting). */
601-
PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter(
602-
_PyUnicodeWriter *writer,
603-
PyObject *obj,
604-
PyObject *format_spec,
605-
Py_ssize_t start,
606-
Py_ssize_t end);
607-
608483
/* --- Manage the default encoding ---------------------------------------- */
609484

610485
/* Returns a pointer to the default encoding (UTF-8) of the
@@ -774,20 +649,6 @@ PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
774649
PyObject *sepobj
775650
);
776651

777-
/* Using explicit passed-in values, insert the thousands grouping
778-
into the string pointed to by buffer. For the argument descriptions,
779-
see Objects/stringlib/localeutil.h */
780-
PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(
781-
_PyUnicodeWriter *writer,
782-
Py_ssize_t n_buffer,
783-
PyObject *digits,
784-
Py_ssize_t d_pos,
785-
Py_ssize_t n_digits,
786-
Py_ssize_t min_width,
787-
const char *grouping,
788-
PyObject *thousands_sep,
789-
Py_UCS4 *maxchar);
790-
791652
/* === Characters Type APIs =============================================== */
792653

793654
/* These should not be used directly. Use the Py_UNICODE_IS* and

‎Include/internal/pycore_complexobject.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
#include "pycore_unicodeobject.h" // _PyUnicodeWriter
12+
1113
/* Operations on complex numbers from complexmodule.c */
1214

1315
PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex);

‎Include/internal/pycore_floatobject.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ extern "C" {
99
#endif
1010

1111

12+
#include "pycore_unicodeobject.h" // _PyUnicodeWriter
13+
1214
/* runtime lifecycle */
1315

1416
extern void _PyFloat_InitState(PyInterpreterState *);

‎Include/internal/pycore_unicodeobject.h‎

Lines changed: 143 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,148 @@ extern "C" {
1414
void _PyUnicode_ExactDealloc(PyObject *op);
1515
Py_ssize_t _PyUnicode_InternedSize(void);
1616

17-
/* runtime lifecycle */
17+
/* --- _PyUnicodeWriter API ----------------------------------------------- */
18+
19+
typedef struct {
20+
PyObject *buffer;
21+
void *data;
22+
int kind;
23+
Py_UCS4 maxchar;
24+
Py_ssize_t size;
25+
Py_ssize_t pos;
26+
27+
/* minimum number of allocated characters (default: 0) */
28+
Py_ssize_t min_length;
29+
30+
/* minimum character (default: 127, ASCII) */
31+
Py_UCS4 min_char;
32+
33+
/* If non-zero, overallocate the buffer (default: 0). */
34+
unsigned char overallocate;
35+
36+
/* If readonly is 1, buffer is a shared string (cannot be modified)
37+
and size is set to 0. */
38+
unsigned char readonly;
39+
} _PyUnicodeWriter ;
40+
41+
/* Initialize a Unicode writer.
42+
*
43+
* By default, the minimum buffer size is 0 character and overallocation is
44+
* disabled. Set min_length, min_char and overallocate attributes to control
45+
* the allocation of the buffer. */
46+
PyAPI_FUNC(void)
47+
_PyUnicodeWriter_Init(_PyUnicodeWriter *writer);
48+
49+
/* Prepare the buffer to write 'length' characters
50+
with the specified maximum character.
51+
52+
Return 0 on success, raise an exception and return -1 on error. */
53+
#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \
54+
(((MAXCHAR) <= (WRITER)->maxchar \
55+
&& (LENGTH) <= (WRITER)->size - (WRITER)->pos) \
56+
? 0 \
57+
: (((LENGTH) == 0) \
58+
? 0 \
59+
: _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR))))
60+
61+
/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro
62+
instead. */
63+
PyAPI_FUNC(int)
64+
_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer,
65+
Py_ssize_t length, Py_UCS4 maxchar);
66+
67+
/* Prepare the buffer to have at least the kind KIND.
68+
For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will
69+
support characters in range U+000-U+FFFF.
70+
71+
Return 0 on success, raise an exception and return -1 on error. */
72+
#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \
73+
((KIND) <= (WRITER)->kind \
74+
? 0 \
75+
: _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND)))
76+
77+
/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind()
78+
macro instead. */
79+
PyAPI_FUNC(int)
80+
_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer,
81+
int kind);
82+
83+
/* Append a Unicode character.
84+
Return 0 on success, raise an exception and return -1 on error. */
85+
PyAPI_FUNC(int)
86+
_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer,
87+
Py_UCS4 ch
88+
);
89+
90+
/* Append a Unicode string.
91+
Return 0 on success, raise an exception and return -1 on error. */
92+
PyAPI_FUNC(int)
93+
_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer,
94+
PyObject *str /* Unicode string */
95+
);
96+
97+
/* Append a substring of a Unicode string.
98+
Return 0 on success, raise an exception and return -1 on error. */
99+
PyAPI_FUNC(int)
100+
_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer,
101+
PyObject *str, /* Unicode string */
102+
Py_ssize_t start,
103+
Py_ssize_t end
104+
);
105+
106+
/* Append an ASCII-encoded byte string.
107+
Return 0 on success, raise an exception and return -1 on error. */
108+
PyAPI_FUNC(int)
109+
_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer,
110+
const char *str, /* ASCII-encoded byte string */
111+
Py_ssize_t len /* number of bytes, or -1 if unknown */
112+
);
113+
114+
/* Append a latin1-encoded byte string.
115+
Return 0 on success, raise an exception and return -1 on error. */
116+
PyAPI_FUNC(int)
117+
_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer,
118+
const char *str, /* latin1-encoded byte string */
119+
Py_ssize_t len /* length in bytes */
120+
);
121+
122+
/* Get the value of the writer as a Unicode string. Clear the
123+
buffer of the writer. Raise an exception and return NULL
124+
on error. */
125+
PyAPI_FUNC(PyObject *)
126+
_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer);
127+
128+
/* Deallocate memory of a writer (clear its internal buffer). */
129+
PyAPI_FUNC(void)
130+
_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer);
131+
132+
133+
/* Format the object based on the format_spec, as defined in PEP 3101
134+
(Advanced String Formatting). */
135+
PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter(
136+
_PyUnicodeWriter *writer,
137+
PyObject *obj,
138+
PyObject *format_spec,
139+
Py_ssize_t start,
140+
Py_ssize_t end);
141+
142+
/* --- Methods & Slots ---------------------------------------------------- */
143+
144+
/* Using explicit passed-in values, insert the thousands grouping
145+
into the string pointed to by buffer. For the argument descriptions,
146+
see Objects/stringlib/localeutil.h */
147+
PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(
148+
_PyUnicodeWriter *writer,
149+
Py_ssize_t n_buffer,
150+
PyObject *digits,
151+
Py_ssize_t d_pos,
152+
Py_ssize_t n_digits,
153+
Py_ssize_t min_width,
154+
const char *grouping,
155+
PyObject *thousands_sep,
156+
Py_UCS4 *maxchar);
157+
158+
/* --- Runtime lifecycle -------------------------------------------------- */
18159

19160
extern void _PyUnicode_InitState(PyInterpreterState *);
20161
extern PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *);
@@ -24,7 +165,7 @@ extern void _PyUnicode_FiniTypes(PyInterpreterState *);
24165

25166
extern PyTypeObject _PyUnicodeASCIIIter_Type;
26167

27-
/* other API */
168+
/* --- Other API ---------------------------------------------------------- */
28169

29170
struct _Py_unicode_runtime_ids {
30171
PyThread_type_lock lock;

‎Modules/cjkcodecs/cjkcodecs.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
#ifndef _CJKCODECS_H_
88
#define _CJKCODECS_H_
99

10+
#ifndef Py_BUILD_CORE_BUILTIN
11+
# define Py_BUILD_CORE_MODULE 1
12+
#endif
13+
1014
#include "Python.h"
1115
#include "multibytecodec.h"
1216

‎Modules/cjkcodecs/multibytecodec.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44
* Written by Hye-Shik Chang <[email protected]>
55
*/
66

7+
#ifndef Py_BUILD_CORE_BUILTIN
8+
# define Py_BUILD_CORE_MODULE 1
9+
#endif
10+
711
#include "Python.h"
812
#include "structmember.h" // PyMemberDef
913
#include "multibytecodec.h"

‎Modules/cjkcodecs/multibytecodec.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
extern "C" {
1111
#endif
1212

13+
#include "pycore_unicodeobject.h" // _PyUnicodeWriter
14+
1315
#ifdef uint16_t
1416
typedef uint16_t ucs2_t, DBCHAR;
1517
#else

‎Tools/c-analyzer/c_parser/preprocessor/gcc.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,14 @@
33

44
from . import common as _common
55

6+
# Modules/socketmodule.h uses pycore_time.h which needs the Py_BUILD_CORE
7+
# macro. Usually it's defined by the C file which includes it.
8+
# Other header files have a similar issue.
9+
NEED_BUILD_CORE = {
10+
'cjkcodecs.h',
11+
'multibytecodec.h',
12+
'socketmodule.h',
13+
}
614

715
TOOL = 'gcc'
816

@@ -62,9 +70,7 @@ def preprocess(filename,
6270
filename = _normpath(filename, cwd)
6371

6472
postargs = POST_ARGS
65-
if os.path.basename(filename) == 'socketmodule.h':
66-
# Modules/socketmodule.h uses pycore_time.h which needs Py_BUILD_CORE.
67-
# Usually it's defined by the C file which includes it.
73+
if os.path.basename(filename) in NEED_BUILD_CORE:
6874
postargs += ('-DPy_BUILD_CORE=1',)
6975

7076
text = _common.preprocess(

0 commit comments

Comments
 (0)