gh-158660: Add PyListWriter C API

Doc/c-api/list.rst

@@ -227,3 +227,96 @@ List Objects
 
    Return a new tuple object containing the contents of *list*; equivalent to
    ``tuple(list)``.
+
+
+.. _pylistwriter:
+
+PyListWriter
+------------
+
+The :c:type:`PyListWriter` API can be used to create a Python :class:`list`
+object.
+
+On Free Threading, it's more efficient than the ``PyList`` C API to create a
+:class:`list`, because it can avoid the critical section on the created list.
+
+.. versionadded:: next
+
+
+.. c:type:: PyListWriter
+
+   A list writer object.
+
+   The API is **not thread safe**. To share a writer with multiple threads, a
+   critical section or a lock is needed.
+
+   The instance must be destroyed by :c:func:`PyListWriter_Finish` on
+   success, or :c:func:`PyListWriter_Discard` on error.
+
+
+.. c:function:: PyListWriter* PyListWriter_Create(Py_ssize_t size)
+
+   Create a :c:type:`PyListWriter` and preallocate *size* items.
+
+   On error, set an exception and return ``NULL``.
+
+   *size* must be positive or zero.
+
+.. c:function:: PyObject* PyListWriter_Finish(PyListWriter *writer)
+
+   Finish a :c:type:`PyListWriter` created by
+   :c:func:`PyListWriter_Create`.
+
+   On success, return a Python :class:`list` object.
+   On error, set an exception and return ``NULL``.
+
+   The writer instance is invalid after the call in any case.
+   No API can be called on the writer after :c:func:`PyListWriter_Finish`.
+
+.. c:function:: void PyListWriter_Discard(PyListWriter *writer)
+
+   Discard a :c:type:`PyListWriter` created by :c:func:`PyListWriter_Create`.
+
+   Do nothing if *writer* is ``NULL``.
+
+   The writer instance is invalid after the call.
+   No API can be called on the writer after :c:func:`PyListWriter_Discard`.
+
+.. c:function:: int PyListWriter_Append(PyListWriter *writer, PyObject *item)
+
+   Append an item to the list writer.
+   Similar to :c:func:`PyList_Append`.
+
+   On success, return ``0``.
+   On error, set an exception and return ``-1``.
+
+.. c:function:: int PyListWriter_AppendSteal(PyListWriter *writer, PyObject *item)
+
+   Similar to :c:func:`PyListWriter_Append`, but :term:`steals <steal>` the
+   reference to *item*, instead of creating a new :term:`strong reference`.
+
+.. c:function:: int PyListWriter_Extend(PyListWriter *writer, PyObject *iterable)
+
+   Extend the list writer with the contents of *iterable*.
+   Similar to :c:func:`PyList_Extend`.
+
+   On success, return ``0``.
+   On error, set an exception and return ``-1``.
+
+.. c:function:: int PyListWriter_ExtendArray(PyListWriter *writer, PyObject *const *items, Py_ssize_t nitem)
+
+   Extend the list writer with the contents of *items*: copy references from
+   *items* to the list writer.
+
+   On success, return ``0``.
+   On error, set an exception and return ``-1``.
+
+.. c:function:: int PyListWriter_ExtendArraySteal(PyListWriter *writer, PyObject *const *items, Py_ssize_t nitem)
+
+   Similar to :c:func:`PyListWriter_ExtendArray`, but :term:`steal` references
+   instead of copying references.
+
+   On success, return ``0``.
+   On error, set an exception and return ``-1``.
+
+   On error, all *items* references are also cleared.

Doc/whatsnew/3.16.rst

@@ -1310,6 +1310,22 @@ New features
 * Add :c:func:`Py_HashBuffer` to the limited C API.
   (Contributed by Victor Stinner in :gh:`148233`.)
 
+* Add a new :ref:`PyListWriter <pylistwriter>` C API to create a :class:`list`
+  object. On Free Threading, it's more efficient than the ``PyList`` C API to
+  create a :class:`list`, because it can avoid the critical section on the
+  created list. Add functions:
+
+  * :c:func:`PyListWriter_Create`
+  * :c:func:`PyListWriter_Finish`
+  * :c:func:`PyListWriter_Discard`
+  * :c:func:`PyListWriter_Append`
+  * :c:func:`PyListWriter_AppendSteal`
+  * :c:func:`PyListWriter_Extend`
+  * :c:func:`PyListWriter_ExtendArray`
+  * :c:func:`PyListWriter_ExtendArraySteal`
+
+  (Contributed by Victor Stinner in :gh:`158660`.)
+
 * Add :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros to the limited C
   API.
   (Contributed by Nathan Goldbaum and Victor Stinner in :gh:`158842`.)

Include/cpython/listobject.h

@@ -51,3 +51,29 @@ PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) {
 
 PyAPI_FUNC(int) PyList_Extend(PyObject *self, PyObject *iterable);
 PyAPI_FUNC(int) PyList_Clear(PyObject *self);
+
+
+/* --- PyListWriter API --------------------------------------------------- */
+
+typedef struct PyListWriter PyListWriter;
+
+PyAPI_FUNC(PyListWriter*) PyListWriter_Create(Py_ssize_t size);
+PyAPI_FUNC(PyObject*) PyListWriter_Finish(PyListWriter *writer);
+PyAPI_FUNC(void) PyListWriter_Discard(PyListWriter *writer);
+PyAPI_FUNC(int) PyListWriter_Append(
+    PyListWriter *writer,
+    PyObject *item);
+PyAPI_FUNC(int) PyListWriter_AppendSteal(
+    PyListWriter *writer,
+    PyObject *item);
+PyAPI_FUNC(int) PyListWriter_Extend(
+    PyListWriter *writer,
+    PyObject *iterable);
+PyAPI_FUNC(int) PyListWriter_ExtendArray(
+    PyListWriter *writer,
+    PyObject *const *items,
+    Py_ssize_t nitem);
+PyAPI_FUNC(int) PyListWriter_ExtendArraySteal(
+    PyListWriter *writer,
+    PyObject *const *items,
+    Py_ssize_t nitem);

Include/internal/pycore_list.h

@@ -12,6 +12,8 @@ extern "C" {
 #include "pycore_stackref.h"
 #endif
 
+#define _Py_LIST_SMALL_ALLOCATED 32
+
 PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
 PyAPI_FUNC(PyObject) *_PyList_SliceSubscript(PyObject*, PyObject*);
 PyAPI_FUNC(PyObject *) _PyList_BinarySlice(PyObject *, PyObject *, PyObject *);

Lib/test/test_capi/test_list.py

@@ -1,12 +1,18 @@
+import threading
 import unittest
-from test.support import import_helper
 from collections import UserList
+from test import support
+from test.support import import_helper
+from test.support import threading_helper
+
 _testcapi = import_helper.import_module('_testcapi')
 _testlimitedcapi = import_helper.import_module('_testlimitedcapi')
 
+
 NULL = None
 PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN
 PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX
+LIST_SMALL_ALLOCATED = _testcapi.LIST_SMALL_ALLOCATED
 
 class ListSubclass(list):
     pass
@@ -355,5 +361,170 @@ def test_uninitialized_list_repr(self):
         self.assertEqual(repr(lst), '[<NULL>, <NULL>, <NULL>]')
 
 
+class ListWriterTests(unittest.TestCase):
+    def create_writer(self, size):
+        # Test PyListWriter_Create()
+        return _testcapi.PyListWriter(size)
+
+    def list_equal(self, result, expected):
+        if not isinstance(expected, list):
+            expected = list(expected)
+        self.assertEqual(result, expected)
+        self.assertEqual(type(result), list)
+
+    def create_items(self, nitem):
+        # Call object() to create unique objects. Don't use small integers
+        # which are immortal (and so would not check for reference leaks).
+        return tuple(object() for i in range(nitem))
+
+    def check_append(self, method):
+        items = self.create_items(3)
+        for size in (0, len(items), len(items) * 10):
+            with self.subTest(size=size):
+                writer = self.create_writer(size)
+                append = getattr(writer, method)
+                for item in items:
+                    append(item)
+                self.list_equal(writer.finish(), items)
+
+        # test MemoryError
+        writer = self.create_writer(0)
+        append = getattr(writer, method)
+        with self.assertRaises(MemoryError):
+            with support.inject_memory_error_cm():
+                append("alloc failed")
+        for item in items:
+            append(item)
+        self.list_equal(writer.finish(), items)
+
+    def test_append(self):
+        # Test PyListWriter_Append()
+        self.check_append('append')
+
+    def test_append_steal(self):
+        # Test PyListWriter_AppendSteal()
+        self.check_append('append_steal')
+
+    def check_extend(self, method):
+        items = self.create_items(5)
+        for size in (0, len(items), len(items) * 10):
+            with self.subTest(size=size):
+                writer = self.create_writer(size)
+                extend = getattr(writer, method)
+                extend(items)
+                self.list_equal(writer.finish(), items)
+
+        # test MemoryError
+        writer = self.create_writer(0)
+        extend = getattr(writer, method)
+        items_error = ("alloc failed",)
+        with self.assertRaises(MemoryError):
+            with support.inject_memory_error_cm():
+                extend(items_error)
+        extend(items)
+        self.list_equal(writer.finish(), items)
+
+    def test_extend(self):
+        # Test PyListWriter_Extend()
+        self.check_extend('extend')
+
+    def test_extend_array(self):
+        # Test PyListWriter_ExtendArray()
+        self.check_extend('extend_array')
+
+    def test_extend_array_steal(self):
+        # Test PyListWriter_ExtendArraySteal()
+        self.check_extend('extend_array_steal')
+
+    def test_empty(self):
+        list_get_allocated = _testcapi._list_get_allocated
+        empty_tuple = ()
+
+        for size in (0, LIST_SMALL_ALLOCATED, 1024):
+            with self.subTest(size=size):
+                writer = self.create_writer(size)
+                writer.extend(empty_tuple)
+                writer.extend_array(empty_tuple)
+                writer.extend_array_steal(empty_tuple)
+                result = writer.finish()
+                self.list_equal(result, [])
+
+                if size == 0 or size > LIST_SMALL_ALLOCATED:
+                    expected = 0
+                else:
+                    expected = size
+                self.assertEqual(list_get_allocated(result), expected)
+
+    def test_finish_shrink(self):
+        # PyListWriter_Finish() shrinks the list if its allocated size
+        # is way too big
+        list_get_allocated = _testcapi._list_get_allocated
+
+        def compute_allocated(size):
+            return (size + (size >> 3) + 6) & ~3
+
+        def check_allocated(items, alloc, expected):
+            writer = self.create_writer(alloc)
+            # Don't use PyListWriter_Extend() since it can shrink the list
+            for item in items:
+                writer.append(item)
+            result = writer.finish()
+            self.list_equal(result, list(items))
+            self.assertEqual(list_get_allocated(result), expected)
+
+        # small list
+        items = [1, 2, 3]
+        check_allocated(items,
+                        1024,
+                        compute_allocated(len(items)))
+
+        alloc = LIST_SMALL_ALLOCATED
+        check_allocated(items, alloc, alloc)
+
+        # larger list
+        items = tuple(range(100))
+        check_allocated(items,
+                        (len(items) + 1) * 2,
+                        compute_allocated(len(items)))
+
+        alloc = len(items) * 2
+        check_allocated(items, alloc, alloc)
+
+        # do not shrink on MemoryError
+        alloc = 1024
+        writer = self.create_writer(alloc)
+        writer.append(1)
+        with support.inject_memory_error_cm():
+            # list_resize() never fails when shrinking
+            result = writer.finish()
+        self.list_equal(result, [1])
+        self.assertEqual(list_get_allocated(result), alloc)
+
+    def test_discard(self):
+        # Test PyListWriter_Discard(): must not leak memory
+        writer = self.create_writer(0)
+        writer.extend([2, 5, 7])
+        writer.discard()
+
+    @threading_helper.requires_working_threading()
+    def test_thread(self):
+        # PyListWriter can be used by multiple threads: it's up to the caller
+        # to implement a lock to prevent concurrent accesses.
+        writer = self.create_writer(0)
+
+        def thread_func(writer):
+            writer.append('a')
+            writer.append_steal('b')
+            writer.extend(('c',))
+            writer.extend_array(('d',))
+            writer.extend_array_steal(('e',))
+
+        thread = threading.Thread(target=thread_func, args=(writer,))
+        thread.start()
+        threading_helper.join_thread(thread)
+
+        self.list_equal(writer.finish(), list('abcde'))
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst

@@ -0,0 +1,15 @@
+Add a new :ref:`PyListWriter <pylistwriter>` C API to create a :class:`list`
+object. On Free Threading, it's more efficient than the ``PyList`` C API to
+create a :class:`list`, because it can avoid the critical section on the
+created list. Add functions:
+
+* :c:func:`PyListWriter_Create`
+* :c:func:`PyListWriter_Finish`
+* :c:func:`PyListWriter_Discard`
+* :c:func:`PyListWriter_Append`
+* :c:func:`PyListWriter_AppendSteal`
+* :c:func:`PyListWriter_Extend`
+* :c:func:`PyListWriter_ExtendArray`
+* :c:func:`PyListWriter_ExtendArraySteal`
+
+Patch by Victor Stinner.