diff --git a/src/QorePythonProgram.cpp b/src/QorePythonProgram.cpp index 0e35da1..01684c7 100644 --- a/src/QorePythonProgram.cpp +++ b/src/QorePythonProgram.cpp @@ -28,6 +28,7 @@ #include "QoreLoader.h" #include "PythonQoreClass.h" #include "QoreMetaPathFinder.h" + #include "PythonCallableCallReferenceNode.h" #include "PythonQoreCallable.h" #include "ModuleNamespace.h" @@ -455,8 +456,8 @@ void QorePythonProgram::deleteIntern(ExceptionSink* xsink) { valid = false; } if (interpreter && owns_interpreter) { -#if PY_VERSION_HEX >= 0x030E0000 - // Python 3.14+ requires that when calling PyInterpreterState_Clear on a sub-interpreter, +#if PY_VERSION_HEX >= 0x030D0000 + // Python 3.13+ requires that when calling PyInterpreterState_Clear on a sub-interpreter, // the current thread must have a thread state from that interpreter attached. // Create a temporary thread state for the sub-interpreter. { @@ -531,6 +532,12 @@ void QorePythonProgram::deleteIntern(ExceptionSink* xsink) { // attached to this interpreter. If we don't clear our cache, we'll have // dangling pointers to freed memory. py_thr_map.erase(this); + // Clear bound_gilstate for all thread states to avoid TSS assertions in 3.12+ + PyThreadState* tstate = PyInterpreterState_ThreadHead(interpreter); + while (tstate) { + tstate->_status.bound_gilstate = 0; + tstate = PyThreadState_Next(tstate); + } assert(_qore_PyRuntimeGILState_GetThreadState()); PyInterpreterState_Clear(interpreter); } @@ -752,11 +759,74 @@ int QorePythonProgram::createInterpreter(QorePythonGilHelper& qpgh, ExceptionSin _QORE_PYTHON_REENABLE_GIL_CHECK qpgh.set(python); + // Ensure the sub-interpreter has a usable sys.path (stdlib + extension modules). + const char* pyhome = getenv("PYTHONHOME"); + const char* pypath = getenv("PYTHONPATH"); + if ((pyhome && *pyhome) || (pypath && *pypath)) { +#ifdef _Q_WINDOWS + const char path_sep = ';'; +#else + const char path_sep = ':'; +#endif + std::string path; + if (pypath) { + path += pypath; + } + if (pyhome && *pyhome) { + if (!path.empty()) { + path += path_sep; + } + path += pyhome; + path += "/Lib"; + path += path_sep; + path += pyhome; + path += "/Modules"; + } + PyObject* sys_path = PySys_GetObject("path"); // borrowed + if (!sys_path || !PyList_Check(sys_path)) { + sys_path = PyList_New(0); + if (sys_path) { + PySys_SetObject("path", sys_path); + Py_DECREF(sys_path); + } + } + if (sys_path && PyList_Check(sys_path)) { + size_t start = 0; + // Preserve empty entries to keep CWD semantics (ex: leading/trailing separators). + while (start <= path.size()) { + size_t end = path.find(path_sep, start); + if (end == std::string::npos) { + end = path.size(); + } + std::string entry = path.substr(start, end - start); + PyObject* py_entry = PyUnicode_DecodeFSDefault(entry.c_str()); + if (py_entry) { + PyList_Append(sys_path, py_entry); + Py_DECREF(py_entry); + } + start = end + 1; + } + } + } + // Ensure PyDateTimeAPI is initialized for this interpreter; the macro only runs if the + // global pointer is null, so clear it to avoid stale main-interpreter state. + PyDateTimeAPI = nullptr; + PyDateTime_IMPORT; + if (!PyDateTimeAPI) { + PyErr_Clear(); + } } interpreter = python->interp; owns_interpreter = true; printd(5, "QorePythonProgram::createInterpreter() interpreter: %p\n", interpreter); + if (!PyDateTimeAPI) { + PyDateTime_IMPORT; + if (!PyDateTimeAPI) { + PyErr_Clear(); + PyDateTimeAPI = nullptr; + } + } // save thread state // NOTE: Acquire py_thr_lck here for thread map updates. @@ -902,40 +972,39 @@ QorePythonThreadInfo QorePythonProgram::setContext(bool check_interrupt) const { // The thread state's interp pointer can become stale if: // 1. PyInterpreterState_Clear was called (sets tstate->interp = NULL) // 2. The thread state was somehow detached from the interpreter - // If there's a mismatch, discard the cached state and create a new one. - if (python && python->interp != interpreter) { - printd(5, "QorePythonProgram::setContext() cached thread state %p has wrong interpreter " - "(has %p, expected %p) - will create new\n", python, python->interp, interpreter); - - // CRITICAL: The cached thread state has a different/deleted interpreter. - // If the interpreter was deleted, the thread state was also freed by zapthreads(). - // We MUST NOT access python->_status because the memory might be freed. - // Check if the interpreter is still valid before accessing the thread state. - bool interp_valid = false; - PyInterpreterState* check_interp = PyInterpreterState_Head(); - while (check_interp) { - if (check_interp == python->interp) { - interp_valid = true; + // If the cached thread state isn't in the interpreter's thread list, it's stale/invalid. + if (python) { + bool found = false; + PyThreadState* ts = PyInterpreterState_ThreadHead(interpreter); + while (ts) { + if (ts == python) { + found = true; break; } - check_interp = PyInterpreterState_Next(check_interp); - } - if (interp_valid && python->interp != nullptr) { - // Interpreter is still valid - safe to clear bound_gilstate - python->_status.bound_gilstate = 0; + ts = PyThreadState_Next(ts); } - // If interpreter is invalid/deleted, DON'T touch the thread state (memory is freed) - - // Remove this thread's entry from the cache - it's invalid - { - AutoLocker al(py_thr_lck); - py_thr_map_t::iterator i = py_thr_map.find(this); - if (i != py_thr_map.end()) { - i->second.erase(q_gettid()); + if (!found) { + printd(5, "QorePythonProgram::setContext() cached thread state %p not found in interpreter %p " + "thread list - will create new\n", python, interpreter); + // Remove this thread's entry from the cache - it's invalid + { + int tid = q_gettid(); + AutoLocker al(py_thr_lck); + py_thr_map_t::iterator i = py_thr_map.find(this); + if (i != py_thr_map.end()) { + i->second.erase(tid); + } + py_global_tid_map_t::iterator gi = py_global_tid_map.find(tid); + if (gi != py_global_tid_map.end()) { + gi->second.erase(python); + if (gi->second.empty()) { + py_global_tid_map.erase(gi); + } + } + // Don't decrement pgm_thr_cnt here - getAcquireThreadState already incremented it } - // Don't decrement pgm_thr_cnt here - getAcquireThreadState already incremented it + python = nullptr; // Force creation of new thread state below } - python = nullptr; // Force creation of new thread state below } // CRITICAL: Also check if TSS points to a stale thread state (from a deleted interpreter). @@ -1242,8 +1311,8 @@ QorePythonThreadInfo QorePythonProgram::setContext(bool check_interrupt) const { // Verify that the currently attached thread state is actually the current one PyThreadState* fast_current = PyThreadState_GetUnchecked(); if (fast_current != currently_attached) { - // Just swap to nullptr to clear, don't try to save - PyThreadState_Swap(nullptr); + // Avoid PyThreadState_Swap without the GIL; clear bound_gilstate instead + currently_attached->_status.bound_gilstate = 0; } else { // Save the current thread state to restore later released_other_interp_tstate = PyEval_SaveThread(); // Release GIL and detach @@ -1251,7 +1320,6 @@ QorePythonThreadInfo QorePythonProgram::setContext(bool check_interrupt) const { } else { // The interpreter is gone - just clear the thread state currently_attached->_status.bound_gilstate = 0; - PyThreadState_Swap(nullptr); } } PyEval_RestoreThread(python); @@ -1275,6 +1343,10 @@ QorePythonThreadInfo QorePythonProgram::setContext(bool check_interrupt) const { _qore_PyCeval_GetThreadState(), _qore_tss_initialized, q_gettid()); printd(5, "QorePythonProgram::setContext() final have_gil: %d\n", have_gil); if (have_gil) { + // If bound_gilstate is stale, clear it to avoid tstate_activate assertions. + if (python->_status.bound_gilstate && PyGILState_GetThisThreadState() != python) { + python->_status.bound_gilstate = 0; + } // We already have the GIL - swap thread states ceval_state = _qore_PyCeval_SwapThreadState(python); g_state = PyGILState_LOCKED; @@ -1286,6 +1358,55 @@ QorePythonThreadInfo QorePythonProgram::setContext(bool check_interrupt) const { } } else { // We don't have the GIL - acquire it + // In Python 3.12, TSS state can be stale; clear it before activating the new + // thread state so bind_gilstate_tstate can safely rebind. + PyThreadState* tss_before_acquire = PyGILState_GetThisThreadState(); + if (tss_before_acquire && tss_before_acquire != python) { + // Only clear the old bound_gilstate if the thread state is still valid. + bool found = false; + PyInterpreterState* interp = PyInterpreterState_Head(); + while (interp && !found) { + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + while (ts) { + if (ts == tss_before_acquire) { + found = true; + break; + } + ts = PyThreadState_Next(ts); + } + interp = PyInterpreterState_Next(interp); + } + if (found) { + tss_before_acquire->_status.bound_gilstate = 0; + } + } + // Bind our thread state to TSS directly to avoid tstate_activate assertions. + _qore_PyGILState_SetTSS(python); + PyThreadState* tss_after_set = PyGILState_GetThisThreadState(); + if (tss_after_set == python) { + python->_status.bound_gilstate = 1; + } else { + // If TSS didn't update, clear it to avoid dereferencing stale thread states. + if (tss_after_set) { + bool tss_valid = false; + PyInterpreterState* interp = PyInterpreterState_Head(); + while (interp && !tss_valid) { + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + while (ts) { + if (ts == tss_after_set) { + tss_valid = true; + break; + } + ts = PyThreadState_Next(ts); + } + interp = PyInterpreterState_Next(interp); + } + if (!tss_valid) { + _qore_PyGILState_ClearTSS(); + } + } + python->_status.bound_gilstate = 0; + } ceval_state = nullptr; PyEval_RestoreThread(python); // Set our tracking now that we have the GIL @@ -2358,12 +2479,26 @@ QoreValue QorePythonProgram::getQoreValue(ExceptionSink* xsink, PyObject* val) { } QoreValue QorePythonProgram::getQoreValue(ExceptionSink* xsink, PyObject* val, pyobj_set_t& rset) { - printd(5, "QorePythonBase::getQoreValue() this: %p PyDateTimeAPI: %p val: %p '%s'\n", this, PyDateTimeAPI, val, - Py_TYPE(val)->tp_name); - assert(PyDateTimeAPI); if (!val || val == Py_None) { return QoreValue(); } + PyTypeObject* type = Py_TYPE(val); + if (!type) { + if (xsink) { + xsink->raiseException("PYTHON-IMPORT-ERROR", "Python object has null type"); + } + return QoreValue(); + } + printd(5, "QorePythonBase::getQoreValue() this: %p PyDateTimeAPI: %p val: %p '%s'\n", this, PyDateTimeAPI, val, + type->tp_name); + if (!PyDateTimeAPI) { + PyDateTime_IMPORT; + if (!PyDateTimeAPI) { + PyErr_Clear(); + PyDateTimeAPI = nullptr; + } + } + bool have_datetime_api = (PyDateTimeAPI != nullptr); // if this is already a Qore object, then return it if (PyQoreObject_Check(val)) { @@ -2371,7 +2506,6 @@ QoreValue QorePythonProgram::getQoreValue(ExceptionSink* xsink, PyObject* val, p return pyobj->qobj ? pyobj->qobj->refSelf() : QoreValue(); } - PyTypeObject* type = Py_TYPE(val); if (type == &PyBool_Type) { return QoreValue(val == Py_True); } @@ -2419,19 +2553,19 @@ QoreValue QorePythonProgram::getQoreValue(ExceptionSink* xsink, PyObject* val, p return getQoreBinaryFromByteArray(val); } - if (type == PyDateTimeAPI->DateType) { + if (have_datetime_api && type == PyDateTimeAPI->DateType) { return getQoreDateTimeFromDate(val); } - if (type == PyDateTimeAPI->TimeType) { + if (have_datetime_api && type == PyDateTimeAPI->TimeType) { return getQoreDateTimeFromTime(val); } - if (type == PyDateTimeAPI->DateTimeType) { + if (have_datetime_api && type == PyDateTimeAPI->DateTimeType) { return getQoreDateTimeFromDateTime(xsink, val); } - if (type == PyDateTimeAPI->DeltaType) { + if (have_datetime_api && type == PyDateTimeAPI->DeltaType) { return getQoreDateTimeFromDelta(val); } @@ -2560,15 +2694,109 @@ PyObject* QorePythonProgram::getPythonDelta(ExceptionSink* xsink, const DateTime assert(dt->isRelative()); // WARNING: years are converted to 365 days; months are converted to 30 days - return PyDelta_FromDSU(dt->getYear() * 365 + dt->getMonth() * 30 + dt->getDay(), + if (PyDateTimeAPI) { + return PyDelta_FromDSU(dt->getYear() * 365 + dt->getMonth() * 30 + dt->getDay(), + dt->getHour() * 3600 + dt->getMinute() * 60 + dt->getSecond(), dt->getMicrosecond()); + } + + // Fallback: create datetime.timedelta via Python when PyDateTimeAPI is unavailable. + QorePythonReferenceHolder datetime_mod(PyImport_ImportModule("datetime")); + if (!datetime_mod) { + if (xsink) { + QorePythonProgram* ctx = QorePythonProgram::getContext(); + if (!ctx || !ctx->checkPythonException(xsink)) { + xsink->raiseException("PYTHON-DATETIME-ERROR", "failed to import Python datetime module"); + } + } + return nullptr; + } + QorePythonReferenceHolder timedelta_cls(PyObject_GetAttrString(*datetime_mod, "timedelta")); + if (!timedelta_cls) { + if (xsink) { + QorePythonProgram* ctx = QorePythonProgram::getContext(); + if (!ctx || !ctx->checkPythonException(xsink)) { + xsink->raiseException("PYTHON-DATETIME-ERROR", "failed to resolve datetime.timedelta"); + } + } + return nullptr; + } + PyObject* args = Py_BuildValue("(iii)", dt->getYear() * 365 + dt->getMonth() * 30 + dt->getDay(), dt->getHour() * 3600 + dt->getMinute() * 60 + dt->getSecond(), dt->getMicrosecond()); + if (!args) { + if (xsink) { + QorePythonProgram* ctx = QorePythonProgram::getContext(); + if (!ctx || !ctx->checkPythonException(xsink)) { + xsink->raiseException("PYTHON-DATETIME-ERROR", "failed to build timedelta arguments"); + } + } + return nullptr; + } + PyObject* rv = PyObject_CallObject(*timedelta_cls, args); + Py_DECREF(args); + if (!rv) { + if (xsink) { + QorePythonProgram* ctx = QorePythonProgram::getContext(); + if (!ctx || !ctx->checkPythonException(xsink)) { + xsink->raiseException("PYTHON-DATETIME-ERROR", "failed to create datetime.timedelta"); + } + } + return nullptr; + } + return rv; } PyObject* QorePythonProgram::getPythonDateTime(ExceptionSink* xsink, const DateTime* dt) { assert(dt->isAbsolute()); - return PyDateTime_FromDateAndTime(dt->getYear(), dt->getMonth(), dt->getDay(), dt->getHour(), dt->getMinute(), + if (PyDateTimeAPI) { + return PyDateTime_FromDateAndTime(dt->getYear(), dt->getMonth(), dt->getDay(), dt->getHour(), dt->getMinute(), dt->getSecond(), dt->getMicrosecond()); + } + + // Fallback: create datetime.datetime via Python when PyDateTimeAPI is unavailable. + QorePythonReferenceHolder datetime_mod(PyImport_ImportModule("datetime")); + if (!datetime_mod) { + if (xsink) { + QorePythonProgram* ctx = QorePythonProgram::getContext(); + if (!ctx || !ctx->checkPythonException(xsink)) { + xsink->raiseException("PYTHON-DATETIME-ERROR", "failed to import Python datetime module"); + } + } + return nullptr; + } + QorePythonReferenceHolder datetime_cls(PyObject_GetAttrString(*datetime_mod, "datetime")); + if (!datetime_cls) { + if (xsink) { + QorePythonProgram* ctx = QorePythonProgram::getContext(); + if (!ctx || !ctx->checkPythonException(xsink)) { + xsink->raiseException("PYTHON-DATETIME-ERROR", "failed to resolve datetime.datetime"); + } + } + return nullptr; + } + PyObject* args = Py_BuildValue("(iiiiiii)", dt->getYear(), dt->getMonth(), dt->getDay(), dt->getHour(), + dt->getMinute(), dt->getSecond(), dt->getMicrosecond()); + if (!args) { + if (xsink) { + QorePythonProgram* ctx = QorePythonProgram::getContext(); + if (!ctx || !ctx->checkPythonException(xsink)) { + xsink->raiseException("PYTHON-DATETIME-ERROR", "failed to build datetime arguments"); + } + } + return nullptr; + } + PyObject* rv = PyObject_CallObject(*datetime_cls, args); + Py_DECREF(args); + if (!rv) { + if (xsink) { + QorePythonProgram* ctx = QorePythonProgram::getContext(); + if (!ctx || !ctx->checkPythonException(xsink)) { + xsink->raiseException("PYTHON-DATETIME-ERROR", "failed to create datetime.datetime"); + } + } + return nullptr; + } + return rv; } PyObject* QorePythonProgram::getPythonCallable(ExceptionSink* xsink, const ResolvedCallReferenceNode* call) { @@ -3766,7 +3994,13 @@ int QorePythonProgram::checkImportSymbol(ExceptionSink* xsink, const char* modul symbol); } QorePythonReferenceHolder value(PyObject_GetAttrString(mod, symbol)); - assert(value); + if (!value) { + if (!checkPythonException(xsink)) { + throw QoreStandardException("PYTHON-IMPORT-ERROR", "module '%s' failed to get symbol '%s'", module, + symbol); + } + return -1; + } bool is_class = PyType_Check(*value); if (is_class) { @@ -3801,7 +4035,18 @@ int QorePythonProgram::findCreateQoreFunction(PyObject* value, const char* symbo int QorePythonProgram::importSymbol(ExceptionSink* xsink, PyObject* value, const char* module, const char* symbol, int filter) { - printd(5, "QorePythonProgram::importSymbol() %s.%s (type %s)\n", module, symbol, Py_TYPE(value)->tp_name); + if (!value) { + if (!checkPythonException(xsink)) { + xsink->raiseException("PYTHON-IMPORT-ERROR", "module '%s' returned null for symbol '%s'", module, symbol); + } + return -1; + } + PyTypeObject* value_type = Py_TYPE(value); + if (!value_type) { + xsink->raiseException("PYTHON-IMPORT-ERROR", "module '%s' symbol '%s' has null type", module, symbol); + return -1; + } + printd(5, "QorePythonProgram::importSymbol() %s.%s (type %s)\n", module, symbol, value_type->tp_name); // check for builtin functions -> static method if (PyCFunction_Check(value)) { // ignore errors diff --git a/src/python-module.cpp b/src/python-module.cpp index fd5af91..ebc6908 100644 --- a/src/python-module.cpp +++ b/src/python-module.cpp @@ -24,6 +24,8 @@ #include "QorePythonProgram.h" #include "QorePythonStackLocationHelper.h" +#include + static QoreStringNode* python_module_init(); static void python_module_ns_init(QoreNamespace* rns, QoreNamespace* qns); static void python_module_delete(); @@ -299,6 +301,56 @@ static QoreStringNode* python_module_init_intern(bool repeat) { // ensure that runtime version matches compiled version check_python_version(); + // Ensure sys.path is initialized when embedding (esp. for debug builds). + const char* pyhome = getenv("PYTHONHOME"); + const char* pypath = getenv("PYTHONPATH"); + if ((pyhome && *pyhome) || (pypath && *pypath)) { +#ifdef _Q_WINDOWS + const char path_sep = ';'; +#else + const char path_sep = ':'; +#endif + std::string path; + if (pypath) { + path += pypath; + } + if (pyhome && *pyhome) { + if (!path.empty()) { + path += path_sep; + } + path += pyhome; + path += "/Lib"; + path += path_sep; + path += pyhome; + path += "/Modules"; + } + PyObject* sys_path = PySys_GetObject("path"); // borrowed + if (!sys_path || !PyList_Check(sys_path)) { + sys_path = PyList_New(0); + if (sys_path) { + PySys_SetObject("path", sys_path); + Py_DECREF(sys_path); + } + } + if (sys_path && PyList_Check(sys_path)) { + size_t start = 0; + // Preserve empty entries to keep CWD semantics (ex: leading/trailing separators). + while (start <= path.size()) { + size_t end = path.find(path_sep, start); + if (end == std::string::npos) { + end = path.size(); + } + std::string entry = path.substr(start, end - start); + PyObject* py_entry = PyUnicode_DecodeFSDefault(entry.c_str()); + if (py_entry) { + PyList_Append(sys_path, py_entry); + Py_DECREF(py_entry); + } + start = end + 1; + } + } + } + // Initialize thread-local state tracking to match Python's state // This must be done before creating any QorePythonProgram instances #ifdef Py_GIL_DISABLED @@ -712,6 +764,11 @@ QorePythonGilHelper::QorePythonGilHelper(PyThreadState* new_thread_state) // Python 3.13+ GIL mode - use PyEval_AcquireThread/ReleaseThread which properly // handle TSS and fast TLS synchronization if (release_gil) { + // If bound_gilstate is stale, clear it to avoid tstate_activate assertions. + if (new_thread_state->_status.bound_gilstate + && PyGILState_GetThisThreadState() != new_thread_state) { + new_thread_state->_status.bound_gilstate = 0; + } // Need to acquire the GIL with our specific thread state // CRITICAL: If there's a stale TSS from a previous interpreter, we must clear it first. // PyEval_AcquireThread -> _PyThreadState_Attach tries to detach any existing thread state, @@ -731,6 +788,20 @@ QorePythonGilHelper::QorePythonGilHelper(PyThreadState* new_thread_state) _qore_PyGILState_SetThisThreadState(new_thread_state); #else if (release_gil) { +#if PY_VERSION_HEX >= 0x030C0000 + // Ensure bound_gilstate and TSS are consistent before acquiring the GIL in Python 3.12. + PyThreadState* tss_before = PyGILState_GetThisThreadState(); + if (new_thread_state->_status.bound_gilstate && tss_before != new_thread_state) { + _qore_PyGILState_SetTSS(new_thread_state); + PyThreadState* tss_after = PyGILState_GetThisThreadState(); + if (tss_after != new_thread_state) { + _qore_PyGILState_ClearTSS(); + new_thread_state->_status.bound_gilstate = 0; + } else { + new_thread_state->_status.bound_gilstate = 1; + } + } +#endif _qore_acquire_thread_state(new_thread_state); // Use _qore_tss_tstate for assertion since Python's TSS (PyGILState_GetThisThreadState) // might be stale from a deleted interpreter in Python 3.12 diff --git a/src/python-module.h b/src/python-module.h index 3508654..da1e8fe 100644 --- a/src/python-module.h +++ b/src/python-module.h @@ -196,13 +196,8 @@ DLLLOCAL static inline PyThreadState* _qore_check_python_created_thread_gil() { #else #if PY_MAJOR_VERSION >= 3 #if PY_MINOR_VERSION >= 14 -#ifdef Py_GIL_DISABLED -// Free-threading Python 3.14+ +// Python 3.14+ (GIL or free-threading) #include "python314_internals.h" -#else -// GIL-enabled Python 3.14+ uses 3.12 internals -#include "python312_internals.h" -#endif #elif PY_MINOR_VERSION >= 13 #include "python313_internals.h" #elif PY_MINOR_VERSION == 12 @@ -309,8 +304,7 @@ inline void PyThreadState_UpdateRecursionLimit(PyThreadState* state, int new_lim #endif // PY_VERSION_HEX >= 0x030D0000 // Python 3.14+ specific compatibility APIs -#if PY_VERSION_HEX >= 0x030E0000 - +#if PY_VERSION_HEX >= 0x030E0000 && !defined(QORE_PY_GILSTATE_UNSAFE_DEFINED) #ifndef Py_GIL_DISABLED // _PyGILState_GetInterpreterStateUnsafe was removed in Python 3.14 // Use PyInterpreterState_Main() as a replacement when there's no thread state @@ -318,8 +312,7 @@ inline PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe() { return PyInterpreterState_Main(); } #endif // !Py_GIL_DISABLED - -#endif // PY_VERSION_HEX >= 0x030E0000 +#endif // PY_VERSION_HEX >= 0x030E0000 && !QORE_PY_GILSTATE_UNSAFE_DEFINED /** Thread State Locations: - _PyRuntime.gilstate.tstate_current - must only be modified while holding the GIL diff --git a/src/python311_internals.h b/src/python311_internals.h index bf39540..0530af3 100644 --- a/src/python311_internals.h +++ b/src/python311_internals.h @@ -520,15 +520,38 @@ DLLLOCAL static inline bool _qore_PyCeval_GetGilLockedStatus() { // Check if this might be a Python-created thread that already has the GIL. // In Python 3.11, PyGILState_* functions are reliable so this is straightforward. DLLLOCAL static inline PyThreadState* _qore_check_python_created_thread_gil() { - // In Python 3.11, we can trust PyGILState_Check() and PyGILState_GetThisThreadState() - if (PyGILState_Check()) { - PyThreadState* tss_state = PyGILState_GetThisThreadState(); - if (tss_state) { - _qore_tss_tstate = tss_state; - _qore_tss_initialized = true; - _qore_gil_held = true; // Track that we hold the GIL - return tss_state; + // If our tracking is already initialized, this is a Qore-managed thread. + if (_qore_tss_initialized) { + return nullptr; + } + + // In Python 3.11, PyGILState_Check() can be unreliable with sub-interpreters, so + // also consult our GIL-locked check which uses thread IDs. + PyThreadState* tss_state = PyGILState_GetThisThreadState(); + bool gil_check = PyGILState_Check() || _qore_PyCeval_GetGilLockedStatus(); + if (tss_state != nullptr && gil_check) { + // Verify tss_state is still valid by checking interpreter thread lists. + bool found = false; + PyInterpreterState* interp = PyInterpreterState_Head(); + while (interp != nullptr && !found) { + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + while (ts != nullptr) { + if (ts == tss_state) { + found = true; + break; + } + ts = PyThreadState_Next(ts); + } + interp = PyInterpreterState_Next(interp); } + if (!found) { + return nullptr; + } + + _qore_tss_tstate = tss_state; + _qore_tss_initialized = true; + _qore_gil_held = true; // Track that we hold the GIL + return tss_state; } return nullptr; } diff --git a/src/python312_internals.h b/src/python312_internals.h index 2005687..15b1d38 100644 --- a/src/python312_internals.h +++ b/src/python312_internals.h @@ -248,21 +248,36 @@ struct _time_runtime_state { #endif }; +#if defined(HAVE_PTHREAD_STUBS) +# define QORE_PY_USE_PTHREADS 1 +#elif defined(HAVE_PTHREAD_H) +# define QORE_PY_USE_PTHREADS 1 +#else +# define QORE_PY_USE_PTHREADS 0 +#endif + +#if QORE_PY_USE_PTHREADS && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) \ + && defined(CLOCK_MONOTONIC) +# define QORE_PY_CONDATTR_MONOTONIC 1 +#else +# define QORE_PY_CONDATTR_MONOTONIC 0 +#endif + struct _pythread_runtime_state { int initialized; -#ifdef _USE_PTHREADS +#if QORE_PY_USE_PTHREADS // This matches when thread_pthread.h is used. struct { /* NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported. */ pthread_condattr_t *ptr; -# ifdef CONDATTR_MONOTONIC +# if QORE_PY_CONDATTR_MONOTONIC /* The value to which condattr_monotonic is set. */ pthread_condattr_t val; # endif } _condattr_monotonic; -#endif // USE_PTHREADS +#endif // QORE_PY_USE_PTHREADS #if defined(HAVE_PTHREAD_STUBS) struct { @@ -271,6 +286,40 @@ struct _pythread_runtime_state { #endif }; +struct pyhash_runtime_state { + struct { +#ifndef MS_WINDOWS + int fd; + dev_t st_dev; + ino_t st_ino; +#else + int _not_used; +#endif + } urandom_cache; +}; + +struct _xidregitem; + +struct _xidregitem { + struct _xidregitem *prev; + struct _xidregitem *next; + PyTypeObject *cls; + PyObject *weakref; + size_t refcount; + crossinterpdatafunc getdata; +}; + +struct _xidregistry { + PyThread_type_lock mutex; + struct _xidregitem *head; +}; + +struct _getargs_runtime_state { + PyThread_type_lock mutex; + struct _PyArg_Parser *static_parsers; +}; + + #ifdef _SIG_MAXSIG // gh-91145: On FreeBSD, defines NSIG as 32: it doesn't include // realtime signals: [SIGRTMIN,SIGRTMAX]. Use _SIG_MAXSIG instead. For @@ -332,6 +381,40 @@ struct _signals_runtime_state { int unhandled_keyboard_interrupt; }; +// Prefix of _PyRuntimeState up to autoTSSkey; matches Python 3.12 layout. +// This lets us access autoTSSkey without including internal headers. +struct _qore_runtime_state_prefix { + int _initialized; + int preinitializing; + int preinitialized; + int core_initialized; + int initialized; + _Py_atomic_address _finalizing; + struct { + PyThread_type_lock mutex; + PyInterpreterState *head; + PyInterpreterState *main; + int64_t next_id; + } interpreters; + unsigned long main_thread; + struct _xidregistry xidregistry; + struct _pymem_allocators allocators; + struct _obmalloc_global_state obmalloc; + struct pyhash_runtime_state pyhash_state; + struct _time_runtime_state time; + struct _pythread_runtime_state threads; + struct _signals_runtime_state signals; + Py_tss_t autoTSSkey; +}; + +#ifdef __cplusplus +extern "C" { +#endif +extern struct _qore_runtime_state_prefix _PyRuntime; +#ifdef __cplusplus +} +#endif + typedef void (*atexit_callbackfunc)(void); struct _atexit_runtime_state { @@ -619,6 +702,15 @@ DLLLOCAL static inline void _qore_PyGILState_SetThisThreadState(PyThreadState* s } } +// Clear or set Python's internal autoTSSkey (thread state TSS) for Python 3.12. +DLLLOCAL static inline void _qore_PyGILState_ClearTSS() { + PyThread_tss_set(&_PyRuntime.autoTSSkey, nullptr); +} + +DLLLOCAL static inline void _qore_PyGILState_SetTSS(PyThreadState* state) { + PyThread_tss_set(&_PyRuntime.autoTSSkey, state); +} + // GIL status check - use our own tracking since PyGILState_Check() in Python 3.12 // returns true even after PyEval_ReleaseThread() because the TSS isn't cleared. DLLLOCAL static inline bool _qore_PyCeval_GetGilLockedStatus() { diff --git a/src/python314_internals.h b/src/python314_internals.h index 8429ae6..863f098 100644 --- a/src/python314_internals.h +++ b/src/python314_internals.h @@ -52,6 +52,7 @@ inline int PyThreadState_GetRecursionLimit(PyThreadState* state) { inline PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe() { return PyInterpreterState_Main(); } +#define QORE_PY_GILSTATE_UNSAFE_DEFINED 1 inline void PyThreadState_UpdateRecursionLimit(PyThreadState* state, int new_limit) { (void)state; // unused in Python 3.14+