diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 96e3aa056992ab..129144d56f4ef5 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -582,6 +582,44 @@ def test_unmarshallable(self): with self.assertRaisesRegex(ValueError, "unmarshallable object"): marshal.dumps((arg, memoryview(b''))) + def test_unorderable_set_items(self): + # Set items are sorted by their marshalled representation, they + # are not compared, even if NaNs which are distinct objects have + # the same marshalled representation. + nan = float('nan') + for kind in set, frozenset: + for version in range(marshal.version + 1): + with self.subTest(kind=kind, version=version): + sample = kind([complex(nan, 0), complex(nan, 0), 1j, + (complex(nan, 0),), (complex(nan, 0),), + float('nan'), float('nan')]) + self.assertEqual(len(sample), 7) + new = marshal.loads(marshal.dumps(sample, version)) + self.assertIs(type(new), kind) + self.assertEqual(sorted(map(repr, new)), + sorted(map(repr, sample))) + + def test_unorderable_set_items_in_code(self): + # A constant set containing NaNs which are distinct objects. + # Run in a subprocess: constants of code objects are immortal + # in the free-threaded build. + script = textwrap.dedent(""" + import marshal + source = 'x in {(1e999-1e999)*1j, (1e999-1e999)*2j, 1.5}' + code = compile(source, '', 'eval') + def consts(code): + return [sorted(map(repr, x)) if isinstance(x, frozenset) else x + for x in code.co_consts] + assert ['(nan+nanj)', '(nan+nanj)', '1.5'] in consts(code) + for version in range(marshal.version + 1): + new = marshal.loads(marshal.dumps(code, version)) + assert new.co_code == code.co_code + assert consts(new) == consts(code) + assert eval(new, {'x': 1.5}) is True + assert eval(new, {'x': complex('nan')}) is False + """) + assert_python_ok('-c', script) + def test_set_changed_size(self): # A set is modified while its items are being marshalled. class Item: diff --git a/Misc/NEWS.d/next/Library/2026-10-11-08-41-00.gh-issue-159184.Nn4tOr.rst b/Misc/NEWS.d/next/Library/2026-10-11-08-41-00.gh-issue-159184.Nn4tOr.rst new file mode 100644 index 00000000000000..7980a1e40eb5d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-11-08-41-00.gh-issue-159184.Nn4tOr.rst @@ -0,0 +1,3 @@ +Fix :func:`marshal.dumps` raising :exc:`MemoryError` for a set or frozenset +containing several complex NaNs, which also made it impossible to write a +:file:`.pyc` file for source code containing such a constant. diff --git a/Python/marshal.c b/Python/marshal.c index d2d25b5d012918..d9b4ae1932481e 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -646,9 +646,13 @@ w_complex_object(PyObject *v, char flag, WFILE *p) // bpo-37596: To support reproducible builds, sets and frozensets need // to have their elements serialized in a consistent order (even when // they have been scrambled by hash randomization). To ensure this, we - // use an order equivalent to sorted(v, key=marshal.dumps): - PyObject *pairs = PyList_New(n); - if (pairs == NULL) { + // use an order equivalent to sorted(v, key=marshal.dumps). + // The list has a (dump, index, element) tuple for each element. + // The indexes are all different, so that sorting never compares + // two elements: elements with the same dump (NaNs which are + // distinct objects) may not be orderable. + PyObject *items = PyList_New(n); + if (items == NULL) { p->error = WFERR_NOMEMORY; return; } @@ -672,34 +676,41 @@ w_complex_object(PyObject *v, char flag, WFILE *p) Py_DECREF(value); break; } - PyObject *pair = _PyTuple_FromPairSteal(dump, value); - if (pair == NULL) { + PyObject *index = PyLong_FromSsize_t(i); + PyObject *item = NULL; + if (index != NULL) { + item = PyTuple_Pack(3, dump, index, value); + Py_DECREF(index); + } + Py_DECREF(dump); + Py_DECREF(value); + if (item == NULL) { p->error = WFERR_NOMEMORY; break; } - PyList_SET_ITEM(pairs, i++, pair); + PyList_SET_ITEM(items, i++, item); } Py_END_CRITICAL_SECTION(); if (p->error == WFERR_UNMARSHALLABLE || p->error == WFERR_NOMEMORY) { - Py_DECREF(pairs); + Py_DECREF(items); return; } if (changed || i != n) { p->error = WFERR_SET_CHANGED_SIZE; - Py_DECREF(pairs); + Py_DECREF(items); return; } - if (PyList_Sort(pairs)) { + if (PyList_Sort(items)) { p->error = WFERR_NOMEMORY; - Py_DECREF(pairs); + Py_DECREF(items); return; } for (Py_ssize_t i = 0; i < n; i++) { - PyObject *pair = PyList_GET_ITEM(pairs, i); - value = PyTuple_GET_ITEM(pair, 1); + PyObject *item = PyList_GET_ITEM(items, i); + value = PyTuple_GET_ITEM(item, 2); w_object(value, p); } - Py_DECREF(pairs); + Py_DECREF(items); } else if (PyCode_Check(v)) { if (!p->allow_code) {