[3.15] gh-159184: Do not compare set elements when marshalling a set (GH-159196)

Lib/test/test_marshal.py

@@ -582,6 +582,44 @@ def test_unmarshallable(self):
                 with self.assertRaisesRegex(ValueError, "unmarshallable object"):
                     marshal.dumps((arg, memoryview(b'')))
 
+    def test_unorderable_set_items(self):
+        # Set items are sorted by their marshalled representation, they
+        # are not compared, even if NaNs which are distinct objects have
+        # the same marshalled representation.
+        nan = float('nan')
+        for kind in set, frozenset:
+            for version in range(marshal.version + 1):
+                with self.subTest(kind=kind, version=version):
+                    sample = kind([complex(nan, 0), complex(nan, 0), 1j,
+                                   (complex(nan, 0),), (complex(nan, 0),),
+                                   float('nan'), float('nan')])
+                    self.assertEqual(len(sample), 7)
+                    new = marshal.loads(marshal.dumps(sample, version))
+                    self.assertIs(type(new), kind)
+                    self.assertEqual(sorted(map(repr, new)),
+                                     sorted(map(repr, sample)))
+
+    def test_unorderable_set_items_in_code(self):
+        # A constant set containing NaNs which are distinct objects.
+        # Run in a subprocess: constants of code objects are immortal
+        # in the free-threaded build.
+        script = textwrap.dedent("""
+            import marshal
+            source = 'x in {(1e999-1e999)*1j, (1e999-1e999)*2j, 1.5}'
+            code = compile(source, '<marshal test>', 'eval')
+            def consts(code):
+                return [sorted(map(repr, x)) if isinstance(x, frozenset) else x
+                        for x in code.co_consts]
+            assert ['(nan+nanj)', '(nan+nanj)', '1.5'] in consts(code)
+            for version in range(marshal.version + 1):
+                new = marshal.loads(marshal.dumps(code, version))
+                assert new.co_code == code.co_code
+                assert consts(new) == consts(code)
+                assert eval(new, {'x': 1.5}) is True
+                assert eval(new, {'x': complex('nan')}) is False
+            """)
+        assert_python_ok('-c', script)
+
     def test_set_changed_size(self):
         # A set is modified while its items are being marshalled.
         class Item:

Misc/NEWS.d/next/Library/2026-10-11-08-41-00.gh-issue-159184.Nn4tOr.rst

@@ -0,0 +1,3 @@
+Fix :func:`marshal.dumps` raising :exc:`MemoryError` for a set or frozenset
+containing several complex NaNs, which also made it impossible to write a
+:file:`.pyc` file for source code containing such a constant.

Python/marshal.c

@@ -646,9 +646,13 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
         // bpo-37596: To support reproducible builds, sets and frozensets need
         // to have their elements serialized in a consistent order (even when
         // they have been scrambled by hash randomization). To ensure this, we
-        // use an order equivalent to sorted(v, key=marshal.dumps):
-        PyObject *pairs = PyList_New(n);
-        if (pairs == NULL) {
+        // use an order equivalent to sorted(v, key=marshal.dumps).
+        // The list has a (dump, index, element) tuple for each element.
+        // The indexes are all different, so that sorting never compares
+        // two elements: elements with the same dump (NaNs which are
+        // distinct objects) may not be orderable.
+        PyObject *items = PyList_New(n);
+        if (items == NULL) {
             p->error = WFERR_NOMEMORY;
             return;
         }
@@ -672,34 +676,41 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
                 Py_DECREF(value);
                 break;
             }
-            PyObject *pair = _PyTuple_FromPairSteal(dump, value);
-            if (pair == NULL) {
+            PyObject *index = PyLong_FromSsize_t(i);
+            PyObject *item = NULL;
+            if (index != NULL) {
+                item = PyTuple_Pack(3, dump, index, value);
+                Py_DECREF(index);
+            }
+            Py_DECREF(dump);
+            Py_DECREF(value);
+            if (item == NULL) {
                 p->error = WFERR_NOMEMORY;
                 break;
             }
-            PyList_SET_ITEM(pairs, i++, pair);
+            PyList_SET_ITEM(items, i++, item);
         }
         Py_END_CRITICAL_SECTION();
         if (p->error == WFERR_UNMARSHALLABLE || p->error == WFERR_NOMEMORY) {
-            Py_DECREF(pairs);
+            Py_DECREF(items);
             return;
         }
         if (changed || i != n) {
             p->error = WFERR_SET_CHANGED_SIZE;
-            Py_DECREF(pairs);
+            Py_DECREF(items);
             return;
         }
-        if (PyList_Sort(pairs)) {
+        if (PyList_Sort(items)) {
             p->error = WFERR_NOMEMORY;
-            Py_DECREF(pairs);
+            Py_DECREF(items);
             return;
         }
         for (Py_ssize_t i = 0; i < n; i++) {
-            PyObject *pair = PyList_GET_ITEM(pairs, i);
-            value = PyTuple_GET_ITEM(pair, 1);
+            PyObject *item = PyList_GET_ITEM(items, i);
+            value = PyTuple_GET_ITEM(item, 2);
             w_object(value, p);
         }
-        Py_DECREF(pairs);
+        Py_DECREF(items);
     }
     else if (PyCode_Check(v)) {
         if (!p->allow_code) {