[pull] main from python:main

Doc/c-api/module.rst

@@ -685,6 +685,12 @@ remove it.
    Usually, there is only one variable of this type for each extension module
    defined this way.
 
+   The struct, including all members, is part of the
+   :ref:`Stable ABI <stable-abi>` for non-free-threaded builds (``abi3``).
+   In the Stable ABI for free-threaded builds (``abi3t``),
+   this struct is opaque, and unusable in practice; see :ref:`pymoduledef_slot`
+   for a replacement.
+
    .. c:member:: PyModuleDef_Base m_base
 
       Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`:
@@ -695,6 +701,11 @@ remove it.
 
          The type of :c:member:`!PyModuleDef.m_base`.
 
+         The struct is part of the :ref:`Stable ABI <stable-abi>` for
+         non-free-threaded builds (``abi3``).
+         In the Stable ABI for Free-Threaded Builds
+         (``abi3t``), this struct is opaque, and unusable in practice.
+
       .. c:macro:: PyModuleDef_HEAD_INIT
 
          The required initial value for :c:member:`!PyModuleDef.m_base`.

Doc/c-api/structures.rst

@@ -33,6 +33,13 @@ under :ref:`reference counting <countingrefs>`.
    The members must not be accessed directly; instead use macros such as
    :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`.
 
+   In the :ref:`Stable ABI <stable-abi>` for Free-Threaded Builds (``abi3t``),
+   this struct is opaque; its size and layout may change between
+   Python versions.
+   In Stable ABI for non-free-threaded builds (``abi3``), the
+   :c:member:`!ob_refcnt` and :c:member:`!ob_type` fields are available,
+   but using them directly is discouraged.
+
    .. c:member:: Py_ssize_t ob_refcnt
 
       The object's reference count, as returned by :c:macro:`Py_REFCNT`.
@@ -72,6 +79,19 @@ under :ref:`reference counting <countingrefs>`.
    instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and
    :c:macro:`Py_TYPE`.
 
+   In the :ref:`Stable ABI <stable-abi>` for Free-Threaded Builds (``abi3t``),
+   this struct is opaque; its size and layout may change between
+   Python versions.
+   In Stable ABI for non-free-threaded builds (``abi3``), the
+   :c:member:`!ob_base` and :c:member:`!ob_size` fields are available,
+   but using them directly is discouraged.
+
+   .. c:member:: PyObject ob_base
+
+      Common object header.
+      Typically, this field is not accessed directly; instead
+      :c:type:`!PyVarObject` can be cast to :c:type:`PyObject`.
+
    .. c:member:: Py_ssize_t ob_size
 
       A size field, whose contents should be considered an object's internal

Doc/c-api/subinterpreters.rst

@@ -399,6 +399,27 @@ High-level APIs
 
    .. versionadded:: 3.9
 
+.. c:function:: void _PyInterpreterState_SetEvalFrameAllowSpecialization(PyInterpreterState *interp, int allow_specialization)
+
+   Enables or disables specialization why a custom frame evaluator is in place.
+
+   If *allow_specialization* is non-zero, the adaptive specializer will
+   continue to specialize bytecodes even though a custom eval frame function
+   is set. When *allow_specialization* is zero, setting a custom eval frame
+   disables specialization. The standard interpreter loop will continue to deopt
+   while a frame evaluation API is in place - the frame evaluation function needs
+   to handle the specialized opcodes to take advantage of this.
+
+   .. versionadded:: 3.15
+
+.. c:function:: int _PyInterpreterState_IsSpecializationEnabled(PyInterpreterState *interp)
+
+   Return non-zero if adaptive specialization is enabled for the interpreter.
+   Specialization is enabled when no custom eval frame function is set, or
+   when one is set with *allow_specialization* enabled.
+
+   .. versionadded:: 3.15
+
 
 Low-level APIs
 --------------

Doc/tools/extensions/c_annotations.py

@@ -249,18 +249,17 @@ def _stable_abi_annotation(
         reftype="ref",
         refexplicit="False",
     )
-    struct_abi_kind = record.struct_abi_kind
-    if struct_abi_kind in {"opaque", "members"}:
-        ref_node += nodes.Text(sphinx_gettext("Limited API"))
-    else:
-        ref_node += nodes.Text(sphinx_gettext("Stable ABI"))
+    ref_node += nodes.Text(sphinx_gettext("Stable ABI"))
     emph_node += ref_node
+    struct_abi_kind = record.struct_abi_kind
     if struct_abi_kind == "opaque":
         emph_node += nodes.Text(" " + sphinx_gettext("(as an opaque struct)"))
     elif struct_abi_kind == "full-abi":
         emph_node += nodes.Text(
             " " + sphinx_gettext("(including all members)")
         )
+    elif struct_abi_kind in {"members", "abi3t-opaque"}:
+        emph_node += nodes.Text(" " + sphinx_gettext("(see below)"))
     if record.ifdef_note:
         emph_node += nodes.Text(f" {record.ifdef_note}")
     if stable_added == "3.2":
@@ -271,11 +270,7 @@ def _stable_abi_annotation(
             " " + sphinx_gettext("since version %s") % stable_added
         )
     emph_node += nodes.Text(".")
-    if struct_abi_kind == "members":
-        msg = " " + sphinx_gettext(
-            "(Only some members are part of the stable ABI.)"
-        )
-        emph_node += nodes.Text(msg)
+
     return emph_node
 
 
@@ -378,6 +373,33 @@ def run(self) -> list[nodes.Node]:
         return [node]
 
 
+class VersionHexCheatsheet(SphinxDirective):
+    """Show results of Py_PACK_VERSION(3, x) for a few relevant Python versions
+
+    This is useful for defining version before Python.h is included.
+    It should auto-update with the version being documented, so it must be an
+    extension.
+    """
+
+    has_content = False
+    required_arguments = 0
+    optional_arguments = 0
+    final_argument_whitespace = True
+
+    def run(self) -> list[nodes.Node]:
+        content = [
+            ".. code-block:: c",
+            "",
+        ]
+        current_minor = int(self.config.version.removeprefix('3.'))
+        for minor in range(current_minor - 5, current_minor + 1):
+            value = (3 << 24) | (minor << 16)
+            content.append(f'    {value:#x}  /* Py_PACK_VERSION(3.{minor}) */')
+        node = nodes.paragraph()
+        self.state.nested_parse(StringList(content), 0, node)
+        return [node]
+
+
 class CorrespondingTypeSlot(SphinxDirective):
     """Type slot annotations
 
@@ -443,6 +465,7 @@ def setup(app: Sphinx) -> ExtensionMetadata:
     app.add_config_value("stable_abi_file", "", "env", types={str})
     app.add_config_value("threadsafety_file", "", "env", types={str})
     app.add_directive("limited-api-list", LimitedAPIList)
+    app.add_directive("version-hex-cheatsheet", VersionHexCheatsheet)
     app.add_directive("corresponding-type-slot", CorrespondingTypeSlot)
     app.connect("builder-inited", init_annotations)
     app.connect("doctree-read", add_annotations)

Doc/whatsnew/3.15.rst

@@ -82,6 +82,7 @@ Summary -- Release highlights
   <whatsnew315-typeform>`
 * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object
   <whatsnew315-pybyteswriter>`
+* :pep:`803`: :ref:`Stable ABI for Free-Threaded Builds <whatsnew315-abi3t>`
 * :ref:`The JIT compiler has been significantly upgraded <whatsnew315-jit>`
 * :ref:`Improved error messages <whatsnew315-improved-error-messages>`
 * :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter
@@ -381,6 +382,41 @@ agen() for x in a)``.
 
 (Contributed by Adam Hartz in :gh:`143055`.)
 
+.. _whatsnew315-abi3t:
+
+:pep:`803`: ``abi3t`` -- Stable ABI for Free-Threaded Builds
+------------------------------------------------------------
+
+C extensions that target the :ref:`Stable ABI <stable-abi>` can now be
+compiled for the new *Stable ABI for Free-Threaded Builds* (also known
+as ``abi3t``), which makes them compatible with
+:term:`free-threaded builds <free-threaded build>` of CPython.
+This usually requires some non-trivial changes to the source code;
+specifically:
+
+- Switching to API introduced in :pep:`697` (Python 3.12), such as
+  negative :c:member:`~PyType_Spec.basicsize` and
+  :c:func:`PyObject_GetTypeData`, rather than making :c:type:`PyObject`
+  part of the instance struct; and
+- Switching from a ``PyInit_`` function to a new export hook,
+  :c:func:`PyModExport_* <PyModExport_modulename>`, introduced for this
+  purpose in :pep:`793`.
+
+Note that Stable ABI does not offer all the functionality that CPython
+has to offer.
+Extensions that cannot switch to ``abi3t`` should continue to build for
+the existing Stable ABI (``abi3``) and the version-specific ABI for
+free-threading (``cp315t``) separately.
+
+Stable ABI for Free-Threaded Builds should typically
+be selected in a build tool (such as, for example, Setuptools, meson-python,
+scikit-build-core, or Maturin).
+At the time of writing, these tools did **not** support ``abi3t``.
+If this is the case for your tool, compile for ``cp315t`` separately.
+If not using a build tool -- or when writing such a tool -- you can select
+``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed
+in :ref:`abi3-compiling`.
+
 
 .. _whatsnew315-improved-error-messages:
 

Include/cpython/object.h

@@ -230,6 +230,8 @@ struct _typeobject {
     destructor tp_finalize;
     vectorcallfunc tp_vectorcall;
 
+    /* Below here all fields are internal to the VM */
+
     /* bitset of which type-watchers care about this type */
     unsigned char tp_watched;
 
@@ -239,6 +241,7 @@ struct _typeobject {
      * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere).
      */
     uint16_t tp_versions_used;
+    _Py_iteritemfunc _tp_iteritem; /* Virtual iterator next function */
 };
 
 #define _Py_ATTR_CACHE_UNUSED (30000)  // (see tp_versions_used)

Include/cpython/pystate.h

@@ -319,3 +319,8 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
 PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc(
     PyInterpreterState *interp,
     _PyFrameEvalFunction eval_frame);
+PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameAllowSpecialization(
+    PyInterpreterState *interp,
+    int allow_specialization);
+PyAPI_FUNC(int) _PyInterpreterState_IsSpecializationEnabled(
+    PyInterpreterState *interp);

Include/internal/pycore_code.h

@@ -141,6 +141,12 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
 
+typedef struct {
+    _Py_BackoffCounter counter;
+} _PyGetIterCache;
+
+#define INLINE_CACHE_ENTRIES_GET_ITER CACHE_ENTRIES(_PyGetIterCache)
+
 typedef struct {
     _Py_BackoffCounter counter;
 } _PySendCache;
@@ -324,6 +330,7 @@ PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *inst
 PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable);
 PyAPI_FUNC(void) _Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr);
 PyAPI_FUNC(void) _Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyInterpreterFrame *frame);
+PyAPI_FUNC(void) _Py_Specialize_GetIter(_PyStackRef iterable, _Py_CODEUNIT *instr);
 
 // Utility functions for reading/writing 32/64-bit values in the inline caches.
 // Great care should be taken to ensure that these functions remain correct and

Include/internal/pycore_interp_structs.h

@@ -927,6 +927,7 @@ struct _is {
     PyObject *builtins_copy;
     // Initialized to _PyEval_EvalFrameDefault().
     _PyFrameEvalFunction eval_frame;
+    int eval_frame_allow_specialization;
 
     PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS];
     // One bit is set for each non-NULL entry in func_watchers

Include/internal/pycore_magic_number.h

@@ -295,6 +295,7 @@ Known values:
     Python 3.15a8 3662 (Add counter to RESUME)
     Python 3.15a8 3663 (Merge GET_ITER and GET_YIELD_FROM_ITER. Modify SEND to make it a bit more like FOR_ITER)
     Python 3.15a8 3664 (Fix __qualname__ for __annotate__ functions)
+    Python 3.15a8 3665 (Add FOR_ITER_VIRTUAL and GET_ITER specializations)
 
 
     Python 3.16 will start with 3700
@@ -308,7 +309,7 @@ PC/launcher.c must also be updated.
 
 */
 
-#define PYC_MAGIC_NUMBER 3664
+#define PYC_MAGIC_NUMBER 3665
 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
    (little-endian) and then appending b'\r\n'. */
 #define PYC_MAGIC_NUMBER_TOKEN \