gh-157384: Document and test PyImport_CreateModuleFromInitfunc() semantics

Doc/c-api/extension-modules.rst

@@ -313,16 +313,32 @@ This is typically done using :c:func:`PyModule_Create` and functions like
 Single-phase initialization differs from the :ref:`default <multi-phase-initialization>`
 in the following ways:
 
-* Single-phase modules are, or rather *contain*, “singletons”.
+* Single-phase modules with :c:member:`PyModuleDef.m_size` set to -1 are,
+  or rather *contain*, “singletons”.
 
-  When the module is first initialized, Python saves the contents of
+  When such a module is first initialized, Python saves the contents of
   the module's ``__dict__`` (that is, typically, the module's functions and
   types).
 
   For subsequent imports, Python does not call the initialization function
   again.
   Instead, it creates a new module object with a new ``__dict__``, and copies
   the saved contents to it.
+  Modules are matched to the saved contents by a (*module_name*, *file_name*)
+  tuple where:
+
+  - *module_name* is the module's full dotted name, taken from
+    :py:attr:`~importlib.machinery.ModuleSpec`
+    (or similar argument to importing function).
+    Note that names that the initialization function uses to create the module
+    (for example, :c:member:`PyModuleDef.m_name`) are not taken into account.
+  - *file_name* is the extension's file name.
+    It is unset for modules not loaded from a file, such as built-ins and
+    modules added with :c:func:`PyImport_CreateModuleFromInitfunc` or
+    :c:func:`PyImport_AppendInittab`.
+
+  As part of the first initialization, Python also adds the module to
+  :data:`sys.modules` under its name.
   For example, given a single-phase module ``_testsinglephase``
   [#testsinglephase]_ that defines a function ``sum`` and an exception class
   ``error``:
@@ -344,6 +360,16 @@ in the following ways:
 
   The exact behavior should be considered a CPython implementation detail.
 
+* For single-phase modules with non-negative :c:member:`PyModuleDef.m_size`,
+  Python may cache a pointer to the initialization function under the
+  (*module_name*, *file_name*) key as for ``m_size == -1``.
+
+  Subsequent imports that specify a different initialization function
+  (for example, using :c:func:`PyImport_CreateModuleFromInitfunc`)
+  may call either the cached function or the provided "new" one.
+
+  The exact behavior should again be considered a CPython implementation detail.
+
 * To work around the fact that ``PyInit_modulename`` does not take a *spec*
   argument, some state of the import machinery is saved and applied to the
   first suitable module created during the ``PyInit_modulename`` call.

Doc/c-api/import.rst

@@ -408,7 +408,7 @@ Importing Modules
 
    *spec* must be a :class:`~importlib.machinery.ModuleSpec` object.
 
-   *initfunc* must be an :ref:`initialization function <extension-export-hook>`,
+   *initfunc* must be an :ref:`initialization function <extension-pyinit>`,
    the same as for :c:func:`PyImport_AppendInittab`.
 
    On success, create and return a module object.
@@ -417,10 +417,26 @@ Importing Modules
    (Custom importers should do this in their
    :py:meth:`~importlib.abc.Loader.exec_module` method.)
 
-   If the spec's name is dotted and *initfunc* creates the module using only
-   the last component of the name, the module's :attr:`~module.__name__`
-   is set to the full name from the spec.
-
    On error, return NULL with an exception set.
 
+   For modules using :ref:`legacy single-phase initialization <single-phase-initialization>`,
+   most of the import process is done in this function, rather than in
+   extra steps that importers typically take after
+   :py:meth:`~importlib.abc.Loader.create_module`.
+   Specifically:
+
+   - The module is added to :data:`sys.modules` under the *spec*'s
+     :py:attr:`~importlib.machinery.ModuleSpec.name` as part of initialization.
+   - If a single-phase module was previously imported under the
+     same :py:attr:`~importlib.machinery.ModuleSpec.name` by this function
+     or as a built-in module (for example, one registered with
+     :c:func:`PyImport_AppendInittab`), Python may reuse the previous module's
+     contents instead of calling *initfunc* again.
+   - The module will be fully initialized by *initfunc*.
+     Calling :c:func:`PyModule_Exec` on it will have no effect, though it is
+     recommended (in case the module's implementation will change).
+   - Also, if the *spec*'s :py:attr:`!name` contains dots, this name is used
+     for prepending a parent package to the new module's
+     :attr:`~module.__name__`.
+
    .. versionadded:: 3.15

Lib/test/test_embed.py

@@ -273,16 +273,27 @@ def test_create_module_from_initfunc(self):
         out, err = self.run_embedded_interpreter("test_create_module_from_initfunc")
         self.assertEqual(self._nogil_filtered_err(err, "embedded_ext"), "")
         self.assertEqual(out,
-                         "<module 'my_test_extension' (static-extension)>\n"
-                         "my_test_extension.executed='yes'\n"
-                         "my_test_extension.exec_slot_ran='yes'\n"
-                         "<module 'embedded_ext' (static-extension)>\n"
-                         "embedded_ext.executed='yes'\n"
-                         "<module 'sp_pkg.sp_submod' (static-extension)>\n"
-                         "sp_pkg.sp_submod.__name__='sp_pkg.sp_submod'\n"
-                         "sys.modules[\"sp_pkg.sp_submod\"] is sp_pkg.sp_submod=True\n"
-                         "\"sp_submod\" in sys.modules=False\n"
-                         )
+            # multi-phase init: not in sys.modules until importlib adds it
+            "created my_test_extension: in sys.modules=False\n"
+            "<module 'my_test_extension' (static-extension)>\n"
+            "my_test_extension.executed='yes'\n"
+            "my_test_extension.exec_slot_ran='yes'\n"
+            # single-phase init: added to sys.modules by the init function
+            "created embedded_ext: in sys.modules=True\n"
+            "<module 'embedded_ext' (static-extension)>\n"
+            "embedded_ext.executed='yes'\n"
+            # same name, different initfuncs: the cached module is
+            # returned and the second initfunc is never called
+            "a.which='A' b.which='A' a is b=True\n"
+            "create_static_module.initfunc_calls()=(1, 0)\n"
+            # Static extension
+            "created sp_pkg: in sys.modules=False\n"
+            "created sp_pkg.sp_submod: in sys.modules=True\n"
+            "<module 'sp_pkg.sp_submod' (static-extension)>\n"
+            "sp_pkg.sp_submod.__name__='sp_pkg.sp_submod'\n"
+            "sys.modules[\"sp_pkg.sp_submod\"] is sp_pkg.sp_submod=True\n"
+            "\"sp_submod\" in sys.modules=False\n"
+        )
 
     def test_inittab_submodule_multiphase(self):
         out, err = self.run_embedded_interpreter("test_inittab_submodule_multiphase")

Programs/_testembed.c

@@ -2357,6 +2357,58 @@ static int test_repeated_init_and_inittab(void)
     return 0;
 }
 
+// Two different single-phase init functions for the same module name.
+static int cmfi_initfunc_a_calls = 0;
+static int cmfi_initfunc_b_calls = 0;
+
+static PyModuleDef cmfi_same_name_a_def = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "same_name",
+    .m_size = -1,
+};
+
+static PyModuleDef cmfi_same_name_b_def = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "same_name",
+    .m_size = -1,
+};
+
+static PyObject*
+PyInit_cmfi_same_name_a(void)
+{
+    cmfi_initfunc_a_calls++;
+    PyObject *mod = PyModule_Create(&cmfi_same_name_a_def);
+    if (mod == NULL || PyModule_AddStringConstant(mod, "which", "A") < 0) {
+        Py_XDECREF(mod);
+        return NULL;
+    }
+    return mod;
+}
+
+static PyObject*
+PyInit_cmfi_same_name_b(void)
+{
+    cmfi_initfunc_b_calls++;
+    PyObject *mod = PyModule_Create(&cmfi_same_name_b_def);
+    if (mod == NULL || PyModule_AddStringConstant(mod, "which", "B") < 0) {
+        Py_XDECREF(mod);
+        return NULL;
+    }
+    return mod;
+}
+
+static PyObject*
+create_same_name_b(PyObject* self, PyObject* spec)
+{
+    return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_same_name_b);
+}
+
+static PyObject*
+initfunc_calls(PyObject* self, PyObject* Py_UNUSED(args))
+{
+    return Py_BuildValue("(ii)", cmfi_initfunc_a_calls, cmfi_initfunc_b_calls);
+}
+
 // A single-phase init submodule that only knows its short name,
 // like pybind11's PYBIND11_MODULE(sp_submod, m) would.
 static PyModuleDef cmfi_sp_submod_def = {
@@ -2406,6 +2458,10 @@ create_module(PyObject* self, PyObject* spec)
         Py_DECREF(name);
         return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext);
     }
+    if (PyUnicode_EqualToUTF8(name, "same_name")) {
+        Py_DECREF(name);
+        return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_same_name_a);
+    }
     if (PyUnicode_EqualToUTF8(name, "sp_pkg")) {
         Py_DECREF(name);
         return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_sp_pkg);
@@ -2430,6 +2486,8 @@ exec_module(PyObject* self, PyObject* mod)
 
 static PyMethodDef create_static_module_methods[] = {
     {"create_module", create_module, METH_O, NULL},
+    {"create_same_name_b", create_same_name_b, METH_O, NULL},
+    {"initfunc_calls", initfunc_calls, METH_NOARGS, NULL},
     {"exec_module", exec_module, METH_O, NULL},
     {NULL}
 };
@@ -2461,6 +2519,12 @@ test_create_module_from_initfunc(void)
         L"import embedded_ext;"
         L"print(embedded_ext);"
         L"print(f'{embedded_ext.executed=}');"
+        // Same name, different initfuncs: the first one wins
+        L"spec = spec_from_loader('same_name', StaticExtensionImporter);"
+        L"a = create_static_module.create_module(spec);"
+        L"b = create_static_module.create_same_name_b(spec);"
+        L"print(f'{a.which=} {b.which=} {a is b=}');"
+        L"print(f'{create_static_module.initfunc_calls()=}');"
         // Single-phase submodule whose init only knows its short name
         L"import sp_pkg.sp_submod;"
         L"print(sp_pkg.sp_submod);"
@@ -2493,7 +2557,10 @@ test_create_module_from_initfunc(void)
         "       return None\n"
         "   @staticmethod\n"
         "   def create_module(spec):\n"
-        "       return create_static_module.create_module(spec)\n"
+        "       mod = create_static_module.create_module(spec)\n"
+        "       print(f'created {spec.name}: '\n"
+        "             f'in sys.modules={spec.name in sys.modules}')\n"
+        "       return mod\n"
         "   @staticmethod\n"
         "   def exec_module(module):\n"
         "       create_static_module.exec_module(module)\n"