diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59044d2d88cc16..09c9ed3ca54cf6 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -153,10 +153,12 @@ Allocating Objects on the Heap To allocate and create extension modules. -Deprecated aliases -^^^^^^^^^^^^^^^^^^ +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ -These are :term:`soft deprecated` aliases to existing functions and macros. +.. soft-deprecated:: 3.15 + +These are aliases to existing functions and macros. They exist solely for backwards compatibility. @@ -164,7 +166,7 @@ They exist solely for backwards compatibility. :widths: auto :header-rows: 1 - * * Deprecated alias + * * Soft-deprecated alias * Function * * .. c:macro:: PyObject_NEW(type, typeobj) * :c:macro:`PyObject_New` diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d1fde1baf71a45..f56bcd6333a37d 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,9 +47,9 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. - .. deprecated:: 3.15 - ``PyBytes_FromStringAndSize(NULL, len)`` is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead of + ``PyBytes_FromStringAndSize(NULL, len)``. .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -238,9 +238,8 @@ called with a non-bytes parameter. *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. - .. deprecated:: 3.15 - The function is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead. .. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 92b531665e135d..7bc04970b19503 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -191,10 +191,10 @@ the :c:data:`Py_mod_multiple_interpreters` slot. ``PyInit`` function ................... -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - This functionality is :term:`soft deprecated`. - It will not get new features, but there are no plans to remove it. + This functionality will not get new features, + but there are no plans to remove it. Instead of :c:func:`PyModExport_modulename`, an extension module can define an older-style :dfn:`initialization function` with the signature: @@ -272,10 +272,9 @@ For example, a module called ``spam`` would be defined like this:: Legacy single-phase initialization ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - Single-phase initialization is :term:`soft deprecated`. - It is a legacy mechanism to initialize extension + Single-phase initialization is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index d89072ab24e241..dcafefdc045872 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -136,11 +136,12 @@ the :mod:`io` APIs instead. failure; the appropriate exception will be set. -Deprecated API -^^^^^^^^^^^^^^ +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ +.. soft-deprecated:: 3.15 -These are :term:`soft deprecated` APIs that were included in Python's C API +These are APIs that were included in Python's C API by mistake. They are documented solely for completeness; use other ``PyFile*`` APIs instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 929b56bd8e8203..a12ad11abb107d 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -86,8 +86,7 @@ Floating-Point Objects It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard ```` header. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_NAN @@ -103,8 +102,7 @@ Floating-Point Objects Equivalent to :c:macro:`!INFINITY`. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 .. c:macro:: Py_MATH_E @@ -161,8 +159,8 @@ Floating-Point Objects that is, it is normal, subnormal or zero, but not infinite or NaN. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isfinite` instead. .. c:macro:: Py_IS_INFINITY(X) @@ -170,8 +168,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is positive or negative infinity. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isinf` instead. .. c:macro:: Py_IS_NAN(X) @@ -179,8 +177,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) value. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isnan` instead. Pack and Unpack functions diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 967cfc727655ec..4159ff6e5965fb 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -147,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -169,7 +169,7 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Legacy Local Variable APIs +Legacy local variable APIs ^^^^^^^^^^^^^^^^^^^^^^^^^^ These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. @@ -178,40 +178,34 @@ They exist solely for backwards compatibility. .. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` attribute of *f* to the internal "fast" array of local variables, allowing changes in frame objects to be visible to the interpreter. If *clear* was true, this function would process variables that were unset in the locals dictionary. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the internal "fast" array of local variables (which is used by the interpreter) to the :attr:`~frame.f_locals` attribute of *f*, allowing changes in local variables to be visible to frame objects. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function was similar to :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and ``-1`` with an exception set on failure. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. @@ -219,7 +213,7 @@ They exist solely for backwards compatibility. :pep:`667` -Internal Frames +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -249,5 +243,3 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 - - diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 2a22a023bdafb4..0e6fd3421f2b3e 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -536,16 +536,14 @@ have been standardized in C11 (or previous standards). Use the standard ``alignas`` specifier rather than this macro. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_FORMAT_SIZE_T The :c:func:`printf` formatting modifier for :c:type:`size_t`. Use ``"z"`` directly instead. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_LL(number) Py_ULL(number) @@ -558,8 +556,7 @@ have been standardized in C11 (or previous standards). Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LONG_LONG PY_INT32_T @@ -572,8 +569,7 @@ have been standardized in C11 (or previous standards). respectively. Historically, these types needed compiler-specific extensions. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LLONG_MIN PY_LLONG_MAX @@ -587,16 +583,14 @@ have been standardized in C11 (or previous standards). The required header, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_MEMCPY(dest, src, n) This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. Use :c:func:`!memcpy` directly instead. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 .. c:macro:: Py_UNICODE_SIZE @@ -606,16 +600,14 @@ have been standardized in C11 (or previous standards). The required header for the latter, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_UNICODE_WIDE Defined if ``wchar_t`` can hold a Unicode character (UCS-4). Use ``sizeof(wchar_t) >= 4`` instead - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_VA_COPY @@ -627,8 +619,7 @@ have been standardized in C11 (or previous standards). .. versionchanged:: 3.6 This is now an alias to ``va_copy``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. _api-objects: diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 790ec8da109ba8..60e3ae4a064e72 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -197,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index a66a1bfd7f8489..b67ca671a2a118 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -965,9 +965,7 @@ or code that creates modules dynamically. // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index b0227c2f4faf15..4bfcb86abf58ed 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -205,6 +205,4 @@ would typically correspond to a Python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a93a0..6bae8f25ad75d1 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 571975d4674397..ff09bb8d884ab6 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1756,11 +1756,10 @@ as a default or fallback. (or by) Python. It is recommended to only use this function as a default or fallback, - .. deprecated:: 3.15 + .. soft-deprecated:: 3.15 - This function is :term:`soft deprecated`. - It is kept for use in cases where it works, but not expected to be - updated for additional platforms and configurations. + This function is kept for use in cases where it works, but not expected to + be updated for additional platforms and configurations. On Linux, :func:`!find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 4a11aec15dfb73..9cc8c5d6886324 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -781,9 +781,8 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: 3.15 - These aliases are :term:`soft deprecated` in favor of the - :mod:`math.integer` functions. +.. soft-deprecated:: 3.15 + Use the :mod:`math.integer` functions instead of these aliases. Constants diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 1e599bde8bcddd..af9098c4970805 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -54,7 +54,7 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 + .. soft-deprecated:: 3.13 Passing a file path instead of URL is :term:`soft deprecated`. Use :func:`guess_file_type` for this. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7547967c6b32f0..d2534b3e974f36 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5110,9 +5110,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -5340,9 +5339,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 7e0a00cba2f63f..6ed285c4b11213 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -931,7 +931,6 @@ Functions .. function:: prefixmatch(pattern, string, flags=0) -.. function:: match(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular expression *pattern*, return a corresponding :class:`~re.Match`. Return @@ -954,7 +953,11 @@ Functions :func:`~re.match`. Use that name when you need to retain compatibility with older Python versions. - .. deprecated:: 3.15 + .. versionadded:: 3.15 + +.. function:: match(pattern, string, flags=0) + + .. soft-deprecated:: 3.15 :func:`~re.match` has been :term:`soft deprecated` in favor of the alternate :func:`~re.prefixmatch` name of this API which is more explicitly descriptive. Use it to better @@ -1285,7 +1288,6 @@ Regular expression objects .. method:: Pattern.prefixmatch(string[, pos[, endpos]]) -.. method:: Pattern.match(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular expression, return a corresponding :class:`~re.Match`. Return ``None`` if the @@ -1310,7 +1312,11 @@ Regular expression objects :meth:`~Pattern.match`. Use that name when you need to retain compatibility with older Python versions. - .. deprecated:: 3.15 + .. versionadded:: 3.15 + +.. method:: Pattern.match(string[, pos[, endpos]]) + + .. soft-deprecated:: 3.15 :meth:`~Pattern.match` has been :term:`soft deprecated` in favor of the alternate :meth:`~Pattern.prefixmatch` name of this API which is more explicitly descriptive. Use it to @@ -1794,8 +1800,8 @@ while new code should prefer :func:`!prefixmatch`. .. versionadded:: 3.15 :func:`!prefixmatch` -.. deprecated:: 3.15 - :func:`!match` is :term:`soft deprecated` +.. soft-deprecated:: 3.15 + :func:`!match` Making a phonebook ^^^^^^^^^^^^^^^^^^ diff --git a/Doc/tools/extensions/changes.py b/Doc/tools/extensions/changes.py index 8de5e7f78c6627..02dc51b3a76943 100644 --- a/Doc/tools/extensions/changes.py +++ b/Doc/tools/extensions/changes.py @@ -2,8 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import re +from docutils import nodes +from sphinx import addnodes from sphinx.domains.changeset import ( VersionChange, versionlabel_classes, @@ -11,6 +13,7 @@ ) from sphinx.locale import _ as sphinx_gettext +TYPE_CHECKING = False if TYPE_CHECKING: from docutils.nodes import Node from sphinx.application import Sphinx @@ -73,6 +76,76 @@ def run(self) -> list[Node]: versionlabel_classes[self.name] = "" +class SoftDeprecated(PyVersionChange): + """Directive for soft deprecations that auto-links to the glossary term. + + Usage:: + + .. soft-deprecated:: 3.15 + + Use :func:`new_thing` instead. + + Renders as: "Soft deprecated since version 3.15: Use new_thing() instead." + with "Soft deprecated" linking to the glossary definition. + """ + + _TERM_RE = re.compile(r":term:`([^`]+)`") + + def run(self) -> list[Node]: + versionlabels[self.name] = sphinx_gettext( + ":term:`Soft deprecated` since version %s" + ) + versionlabel_classes[self.name] = "soft-deprecated" + try: + result = super().run() + finally: + versionlabels[self.name] = "" + versionlabel_classes[self.name] = "" + + for node in result: + # Add "versionchanged" class so existing theme CSS applies + node["classes"] = node.get("classes", []) + ["versionchanged"] + # Replace the plain-text "Soft deprecated" with a glossary reference + for inline in node.findall(nodes.inline): + if "versionmodified" in inline.get("classes", []): + self._add_glossary_link(inline) + + return result + + @classmethod + def _add_glossary_link(cls, inline: nodes.inline) -> None: + """Replace :term:`soft deprecated` text with a cross-reference to the + 'Soft deprecated' glossary entry.""" + for child in inline.children: + if not isinstance(child, nodes.Text): + continue + + text = str(child) + match = cls._TERM_RE.search(text) + if match is None: + continue + + ref = addnodes.pending_xref( + "", + nodes.Text(match.group(1)), + refdomain="std", + reftype="term", + reftarget="soft deprecated", + refwarn=True, + ) + + start, end = match.span() + new_nodes: list[nodes.Node] = [] + if start > 0: + new_nodes.append(nodes.Text(text[:start])) + new_nodes.append(ref) + if end < len(text): + new_nodes.append(nodes.Text(text[end:])) + + child.parent.replace(child, new_nodes) + break + + def setup(app: Sphinx) -> ExtensionMetadata: # Override Sphinx's directives with support for 'next' app.add_directive("versionadded", PyVersionChange, override=True) @@ -83,6 +156,9 @@ def setup(app: Sphinx) -> ExtensionMetadata: # Register the ``.. deprecated-removed::`` directive app.add_directive("deprecated-removed", DeprecatedRemoved) + # Register the ``.. soft-deprecated::`` directive + app.add_directive("soft-deprecated", SoftDeprecated) + return { "version": "1.0", "parallel_read_safe": True, diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index f3cd8bf0ef5bb9..7bffbb8d86197d 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -1 +1,5 @@ # HTML IDs excluded from the check-html-ids.py check. + +# Remove from here in 3.16 +c-api/allocation.html: deprecated-aliases +c-api/file.html: deprecated-api diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 75f6607d8f3698..699e518801cbcd 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -29,6 +29,7 @@ {% trans %}Deprecated since version %s, will be removed in version %s{% endtrans %} {% trans %}Deprecated since version %s, removed in version %s{% endtrans %} +{% trans %}:term:`Soft deprecated` since version %s{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 016e7a18665859..322c1e93344ba3 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -242,9 +242,12 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { + PyObject *obj; assert(buffer->writer != NULL); - return PyBytesWriter_FinishWithSize(buffer->writer, - buffer->allocated - avail_out); + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; } /* Clean up the buffer when an error occurred. */ diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 78db4219e2997c..9ec37d27dfa39f 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -317,6 +317,133 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) + def test_reference_loop_list(self): + a = [] + a.append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, list) + self.assertIs(b[0], b) + + def test_reference_loop_dict(self): + a = {} + a[None] = a + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, dict) + self.assertIs(b[None], b) + + def test_reference_loop_tuple(self): + a = ([],) + a[0].append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, tuple) + self.assertIsInstance(b[0], list) + self.assertIs(b[0][0], b) + + def test_reference_loop_code(self): + def f(): + return 1234.5 + code = f.__code__ + a = [] + code = code.replace(co_consts=code.co_consts + (a,)) + a.append(code) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, code, v) + + def test_reference_loop_slice(self): + a = slice([], None) + a.start.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, []) + a.stop.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, None, []) + a.step.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_reference_loop_frozendict(self): + a = frozendict({None: []}) + a[None].append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_loads_reference_loop_list(self): + data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [] + a = marshal.loads(data) + self.assertIsInstance(a, list) + self.assertIs(a[0], a) + + def test_loads_reference_loop_dict(self): + data = b'\xfbNr\x00\x00\x00\x000' # {None: } + a = marshal.loads(data) + self.assertIsInstance(a, dict) + self.assertIs(a[None], a) + + def test_loads_abnormal_reference_loops(self): + # Indirect self-references of tuples. + data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([],) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], list) + self.assertIs(a[0][0], a) + + data = b'\xa8\x01\x00\x00\x00{Nr\x00\x00\x00\x000' # ({None: },) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], dict) + self.assertIs(a[0][None], a) + + # Direct self-reference which cannot be created in Python. + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) + + # Direct self-references which cannot be created in Python + # because of unhashability. + data = b'\xfbr\x00\x00\x00\x00N0' # {: None} + self.assertRaises(TypeError, marshal.loads, data) + data = b'\xbc\x01\x00\x00\x00r\x00\x00\x00\x00' # {} + self.assertRaises(TypeError, marshal.loads, data) + + for data in [ + # Indirect self-references of immutable objects. + b'\xba[\x01\x00\x00\x00r\x00\x00\x00\x00NN', # slice([], None) + b'\xbaN[\x01\x00\x00\x00r\x00\x00\x00\x00N', # slice(None, []) + b'\xbaNN[\x01\x00\x00\x00r\x00\x00\x00\x00', # slice(None, None, []) + b'\xba{Nr\x00\x00\x00\x000NN', # slice({None: }, None) + b'\xbaN{Nr\x00\x00\x00\x000N', # slice(None, {None: }) + b'\xbaNN{Nr\x00\x00\x00\x000', # slice(None, None, {None: }) + b'\xfdN[\x01\x00\x00\x00r\x00\x00\x00\x000', # frozendict({None: []}) + b'\xfdN{Nr\x00\x00\x00\x0000', # frozendict({None: {None: }) + + # Direct self-references which cannot be created in Python. + b'\xbe\x01\x00\x00\x00r\x00\x00\x00\x00', # frozenset({}) + b'\xfdNr\x00\x00\x00\x000', # frozendict({None: }) + b'\xfdr\x00\x00\x00\x00N0', # frozendict({: None}) + b'\xbar\x00\x00\x00\x00NN', # slice(, None) + b'\xbaNr\x00\x00\x00\x00N', # slice(None, ) + b'\xbaNNr\x00\x00\x00\x00', # slice(None, None, ) + ]: + with self.subTest(data=data): + self.assertRaises(ValueError, marshal.loads, data) + def test_exact_type_match(self): # Former bug: # >>> class Int(int): pass diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst new file mode 100644 index 00000000000000..d3242235c6059b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst @@ -0,0 +1,2 @@ +Forbid :mod:`marshalling ` recursive code objects, :class:`slice` +and :class:`frozendict` objects which cannot be correctly unmarshalled. diff --git a/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst new file mode 100644 index 00000000000000..1e367716e5a0a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst @@ -0,0 +1,2 @@ +:mod:`bz2`, :mod:`compression.zstd`, :mod:`lzma`, :mod:`zlib`: Fix a double +free on memory allocation failure. Patch by Victor Stinner. diff --git a/Python/marshal.c b/Python/marshal.c index b60a36e128cd9f..dace22da0d4a94 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -382,7 +382,6 @@ static int w_ref(PyObject *v, char *flag, WFILE *p) { _Py_hashtable_entry_t *entry; - int w; if (p->version < 3 || p->hashtable == NULL) return 0; /* not writing object references */ @@ -399,20 +398,28 @@ w_ref(PyObject *v, char *flag, WFILE *p) entry = _Py_hashtable_get_entry(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - w = (int)(uintptr_t)entry->value; + uintptr_t w = (uintptr_t)entry->value; + if (w & 0x80000000LU) { + PyErr_Format(PyExc_ValueError, "cannot marshal recursion %T objects", v); + goto err; + } /* we don't store "long" indices in the dict */ - assert(0 <= w && w <= 0x7fffffff); + assert(w <= 0x7fffffff); w_byte(TYPE_REF, p); - w_long(w, p); + w_long((int)w, p); return 1; } else { - size_t s = p->hashtable->nentries; + size_t w = p->hashtable->nentries; /* we don't support long indices */ - if (s >= 0x7fffffff) { + if (w >= 0x7fffffff) { PyErr_SetString(PyExc_ValueError, "too many objects"); goto err; } - w = (int)s; + // Corresponding code should call w_complete() after + // writing the object. + if (PyCode_Check(v) || PySlice_Check(v) || PyFrozenDict_CheckExact(v)) { + w |= 0x80000000LU; + } if (_Py_hashtable_set(p->hashtable, Py_NewRef(v), (void *)(uintptr_t)w) < 0) { Py_DECREF(v); @@ -426,6 +433,27 @@ w_ref(PyObject *v, char *flag, WFILE *p) return 1; } +static void +w_complete(PyObject *v, WFILE *p) +{ + if (p->version < 3 || p->hashtable == NULL) { + return; + } + if (_PyObject_IsUniquelyReferenced(v)) { + return; + } + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(p->hashtable, v); + if (entry == NULL) { + return; + } + assert(entry != NULL); + uintptr_t w = (uintptr_t)entry->value; + assert(w & 0x80000000LU); + w &= ~0x80000000LU; + entry->value = (void *)(uintptr_t)w; +} + static void w_complex_object(PyObject *v, char flag, WFILE *p); @@ -599,6 +627,9 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(value, p); } w_object((PyObject *)NULL, p); + if (PyFrozenDict_CheckExact(v)) { + w_complete(v, p); + } } else if (PyAnySet_CheckExact(v)) { PyObject *value; @@ -684,6 +715,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_linetable, p); w_object(co->co_exceptiontable, p); Py_DECREF(co_code); + w_complete(v, p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -709,6 +741,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(slice->start, p); w_object(slice->stop, p); w_object(slice->step, p); + w_complete(v, p); } else { W_TYPE(TYPE_UNKNOWN, p); @@ -1433,9 +1466,19 @@ r_object(RFILE *p) case TYPE_DICT: case TYPE_FROZENDICT: v = PyDict_New(); - R_REF(v); - if (v == NULL) + if (v == NULL) { break; + } + if (type == TYPE_DICT) { + R_REF(v); + } + else { + idx = r_ref_reserve(flag, p); + if (idx < 0) { + Py_CLEAR(v); + break; + } + } for (;;) { PyObject *key, *val; key = r_object(p); @@ -1458,13 +1501,7 @@ r_object(RFILE *p) Py_CLEAR(v); } if (type == TYPE_FROZENDICT && v != NULL) { - PyObject *frozendict = PyFrozenDict_New(v); - if (frozendict != NULL) { - Py_SETREF(v, frozendict); - } - else { - Py_CLEAR(v); - } + Py_SETREF(v, PyFrozenDict_New(v)); } retval = v; break;