From 6834f3f139cb5f89fb1d4ae68392ffaf2aa5d5eb Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Tue, 1 Oct 2024 16:36:08 -0600 Subject: [PATCH] WIP: declare free-threaded support in pymodule macro --- noxfile.py | 8 ------- pyo3-ffi/build.rs | 8 ------- pyo3-ffi/src/moduleobject.rs | 22 +++++++++++++---- pyo3-macros-backend/src/attributes.rs | 4 +++- pyo3-macros-backend/src/module.rs | 34 +++++++++++++++++++++++---- pytests/src/free_threaded_mod.rs | 12 ++++++++++ pytests/src/lib.rs | 2 ++ pytests/tests/test_free_threaded.py | 1 + src/impl_/pymodule.rs | 26 ++++++++++++++++---- src/macros.rs | 3 ++- src/types/module.rs | 1 + 11 files changed, 89 insertions(+), 32 deletions(-) create mode 100644 pytests/src/free_threaded_mod.rs create mode 100644 pytests/tests/test_free_threaded.py diff --git a/noxfile.py b/noxfile.py index b526c71f2f3..553379253b6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -654,14 +654,6 @@ def test_version_limits(session: nox.Session): config_file.set("PyPy", "3.11") _run_cargo(session, "check", env=env, expect_error=True) - # Python build with GIL disabled should fail building - config_file.set("CPython", "3.13", build_flags=["Py_GIL_DISABLED"]) - _run_cargo(session, "check", env=env, expect_error=True) - - # Python build with GIL disabled should pass with env flag on - env["UNSAFE_PYO3_BUILD_FREE_THREADED"] = "1" - _run_cargo(session, "check", env=env) - @nox.session(name="check-feature-powerset", venv_backend="none") def check_feature_powerset(session: nox.Session): diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 622c2707110..64f951723ba 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -127,14 +127,6 @@ fn ensure_gil_enabled(interpreter_config: &InterpreterConfig) -> Result<()> { .0 .contains(&BuildFlag::Py_GIL_DISABLED) .not(); - ensure!( - gil_enabled || std::env::var("UNSAFE_PYO3_BUILD_FREE_THREADED").map_or(false, |os_str| os_str == "1"), - "the Python interpreter was built with the GIL disabled, which is not yet supported by PyO3\n\ - = help: see https://github.com/PyO3/pyo3/issues/4265 for more information\n\ - = help: please check if an updated version of PyO3 is available. Current version: {}\n\ - = help: set UNSAFE_PYO3_BUILD_FREE_THREADED=1 to suppress this check and build anyway for free-threaded Python", - std::env::var("CARGO_PKG_VERSION").unwrap() - ); if !gil_enabled && interpreter_config.abi3 { warn!( "The free-threaded build of CPython does not yet support abi3 so the build artifacts will be version-specific." diff --git a/pyo3-ffi/src/moduleobject.rs b/pyo3-ffi/src/moduleobject.rs index ff6458f4b15..e5d54631c2e 100644 --- a/pyo3-ffi/src/moduleobject.rs +++ b/pyo3-ffi/src/moduleobject.rs @@ -86,17 +86,29 @@ impl Default for PyModuleDef_Slot { pub const Py_mod_create: c_int = 1; pub const Py_mod_exec: c_int = 2; -#[cfg(Py_3_12)] +#[cfg(all(not(Py_LIMITED_API), Py_3_12))] pub const Py_mod_multiple_interpreters: c_int = 3; +#[cfg(all(not(Py_LIMITED_API), Py_3_13))] +pub const Py_mod_gil: c_int = 4; -#[cfg(Py_3_12)] +// skipped private _Py_mod_LAST_SLOT + +#[cfg(all(not(Py_LIMITED_API), Py_3_12))] pub const Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED: *mut c_void = 0 as *mut c_void; -#[cfg(Py_3_12)] +#[cfg(all(not(Py_LIMITED_API), Py_3_12))] pub const Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED: *mut c_void = 1 as *mut c_void; -#[cfg(Py_3_12)] +#[cfg(all(not(Py_LIMITED_API), Py_3_12))] pub const Py_MOD_PER_INTERPRETER_GIL_SUPPORTED: *mut c_void = 2 as *mut c_void; -// skipped non-limited _Py_mod_LAST_SLOT +#[cfg(all(not(Py_LIMITED_API), Py_3_13))] +pub const Py_MOD_GIL_USED: *mut c_void = 0 as *mut c_void; +#[cfg(all(not(Py_LIMITED_API), Py_3_13))] +pub const Py_MOD_GIL_NOT_USED: *mut c_void = 1 as *mut c_void; + +#[cfg(all(not(Py_LIMITED_API), Py_GIL_DISABLED))] +extern "C" { + pub fn PyUnstable_Module_SetGIL(module: *mut PyObject, gil: *mut c_void); +} #[repr(C)] pub struct PyModuleDef { diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index 94526e7dafc..1c9c5b1ec53 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -7,7 +7,7 @@ use syn::{ punctuated::Punctuated, spanned::Spanned, token::Comma, - Attribute, Expr, ExprPath, Ident, Index, LitStr, Member, Path, Result, Token, + Attribute, Expr, ExprPath, Ident, Index, LitBool, LitStr, Member, Path, Result, Token, }; pub mod kw { @@ -44,6 +44,7 @@ pub mod kw { syn::custom_keyword!(transparent); syn::custom_keyword!(unsendable); syn::custom_keyword!(weakref); + syn::custom_keyword!(supports_free_threaded); } fn take_int(read: &mut &str, tracker: &mut usize) -> String { @@ -308,6 +309,7 @@ pub type RenameAllAttribute = KeywordAttribute; pub type TextSignatureAttribute = KeywordAttribute; pub type SubmoduleAttribute = kw::submodule; +pub type FreeThreadedAttribute = KeywordAttribute; impl Parse for KeywordAttribute { fn parse(input: ParseStream<'_>) -> Result { diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index e0025fda6dd..041c77ff174 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -2,8 +2,8 @@ use crate::{ attributes::{ - self, kw, take_attributes, take_pyo3_options, CrateAttribute, ModuleAttribute, - NameAttribute, SubmoduleAttribute, + self, kw, take_attributes, take_pyo3_options, CrateAttribute, FreeThreadedAttribute, + ModuleAttribute, NameAttribute, SubmoduleAttribute, }, get_doc, pyclass::PyClassPyO3Option, @@ -29,6 +29,7 @@ pub struct PyModuleOptions { name: Option, module: Option, submodule: Option, + supports_free_threaded: Option, } impl Parse for PyModuleOptions { @@ -72,6 +73,9 @@ impl PyModuleOptions { submodule, " (it is implicitly always specified for nested modules)" ), + PyModulePyO3Option::SupportsFreeThreaded(supports_free_threaded) => { + set_option!(supports_free_threaded) + } } } Ok(()) @@ -344,7 +348,15 @@ pub fn pymodule_module_impl( ) } }}; - let initialization = module_initialization(&name, ctx, module_def, options.submodule.is_some()); + let initialization = module_initialization( + &name, + ctx, + module_def, + options.submodule.is_some(), + options + .supports_free_threaded + .is_some_and(|op| op.value.value), + ); Ok(quote!( #(#attrs)* @@ -383,7 +395,15 @@ pub fn pymodule_function_impl( let vis = &function.vis; let doc = get_doc(&function.attrs, None, ctx); - let initialization = module_initialization(&name, ctx, quote! { MakeDef::make_def() }, false); + let initialization = module_initialization( + &name, + ctx, + quote! { MakeDef::make_def() }, + false, + options + .supports_free_threaded + .is_some_and(|op| op.value.value), + ); // Module function called with optional Python<'_> marker as first arg, followed by the module. let mut module_args = Vec::new(); @@ -428,6 +448,7 @@ fn module_initialization( ctx: &Ctx, module_def: TokenStream, is_submodule: bool, + supports_free_threaded: bool, ) -> TokenStream { let Ctx { pyo3_path, .. } = ctx; let pyinit_symbol = format!("PyInit_{}", name); @@ -449,7 +470,7 @@ fn module_initialization( #[doc(hidden)] #[export_name = #pyinit_symbol] pub unsafe extern "C" fn __pyo3_init() -> *mut #pyo3_path::ffi::PyObject { - #pyo3_path::impl_::trampoline::module_init(|py| _PYO3_DEF.make_module(py)) + #pyo3_path::impl_::trampoline::module_init(|py| _PYO3_DEF.make_module(py, #supports_free_threaded)) } }); } @@ -596,6 +617,7 @@ enum PyModulePyO3Option { Crate(CrateAttribute), Name(NameAttribute), Module(ModuleAttribute), + SupportsFreeThreaded(FreeThreadedAttribute), } impl Parse for PyModulePyO3Option { @@ -609,6 +631,8 @@ impl Parse for PyModulePyO3Option { input.parse().map(PyModulePyO3Option::Module) } else if lookahead.peek(attributes::kw::submodule) { input.parse().map(PyModulePyO3Option::Submodule) + } else if lookahead.peek(attributes::kw::supports_free_threaded) { + input.parse().map(PyModulePyO3Option::SupportsFreeThreaded) } else { Err(lookahead.error()) } diff --git a/pytests/src/free_threaded_mod.rs b/pytests/src/free_threaded_mod.rs new file mode 100644 index 00000000000..7bf9763db14 --- /dev/null +++ b/pytests/src/free_threaded_mod.rs @@ -0,0 +1,12 @@ +use pyo3::prelude::*; + +#[pyfunction] +fn add_two(x: usize) -> usize { + x + 2 +} + +#[pymodule] +pub fn free_threaded_mod(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(add_two, m)?)?; + Ok(()) +} diff --git a/pytests/src/lib.rs b/pytests/src/lib.rs index 72f5feaa0f4..32c4b329d75 100644 --- a/pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -8,6 +8,7 @@ pub mod comparisons; pub mod datetime; pub mod dict_iter; pub mod enums; +pub mod free_threaded_mod; pub mod misc; pub mod objstore; pub mod othermod; @@ -35,6 +36,7 @@ fn pyo3_pytests(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pymodule!(pyfunctions::pyfunctions))?; m.add_wrapped(wrap_pymodule!(sequence::sequence))?; m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?; + m.add_wrapped(wrap_pymodule!(free_threaded_mod::free_threaded_mod))?; // Inserting to sys.modules allows importing submodules nicely from Python // e.g. import pyo3_pytests.buf_and_str as bas diff --git a/pytests/tests/test_free_threaded.py b/pytests/tests/test_free_threaded.py new file mode 100644 index 00000000000..584a739a98c --- /dev/null +++ b/pytests/tests/test_free_threaded.py @@ -0,0 +1 @@ +from pyo3_pytests import free_threaded_mod diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 97eb2103dfe..835b6c32f1d 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -15,7 +15,7 @@ use portable_atomic::{AtomicI64, Ordering}; not(all(windows, Py_LIMITED_API, not(Py_3_10))), target_has_atomic = "64", ))] -use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; #[cfg(not(any(PyPy, GraalPy)))] use crate::exceptions::PyImportError; @@ -41,6 +41,8 @@ pub struct ModuleDef { interpreter: AtomicI64, /// Initialized module object, cached to avoid reinitialization. module: GILOnceCell>, + /// Whether or not the module supports running without the GIL + supports_free_threaded: AtomicBool, } /// Wrapper to enable initializer to be used in const fns. @@ -85,10 +87,15 @@ impl ModuleDef { ))] interpreter: AtomicI64::new(-1), module: GILOnceCell::new(), + supports_free_threaded: AtomicBool::new(false), } } /// Builds a module using user given initializer. Used for [`#[pymodule]`][crate::pymodule]. - pub fn make_module(&'static self, py: Python<'_>) -> PyResult> { + pub fn make_module( + &'static self, + py: Python<'_>, + supports_free_threaded: bool, + ) -> PyResult> { // Check the interpreter ID has not changed, since we currently have no way to guarantee // that static data is not reused across interpreters. // @@ -134,6 +141,11 @@ impl ModuleDef { ffi::PyModule_Create(self.ffi_def.get()), )? }; + if supports_free_threaded { + unsafe { + ffi::PyUnstable_Module_SetGIL(module.as_ptr(), ffi::Py_MOD_GIL_NOT_USED) + }; + } self.initializer.0(module.bind(py))?; Ok(module) }) @@ -190,7 +202,13 @@ impl PyAddToModule for PyMethodDef { /// For adding a module to a module. impl PyAddToModule for ModuleDef { fn add_to_module(&'static self, module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_submodule(self.make_module(module.py())?.bind(module.py())) + module.add_submodule( + self.make_module( + module.py(), + self.supports_free_threaded.load(Ordering::Relaxed), + )? + .bind(module.py()), + ) } } @@ -223,7 +241,7 @@ mod tests { ) }; Python::with_gil(|py| { - let module = MODULE_DEF.make_module(py).unwrap().into_bound(py); + let module = MODULE_DEF.make_module(py, false).unwrap().into_bound(py); assert_eq!( module .getattr("__name__") diff --git a/src/macros.rs b/src/macros.rs index 6148d9662c5..31e2f14d795 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -172,7 +172,8 @@ macro_rules! wrap_pymodule { &|py| { use $module as wrapped_pymodule; wrapped_pymodule::_PYO3_DEF - .make_module(py) + // corrected in add_wrapped later based on the parent module's settings + .make_module(py, false) .expect("failed to wrap pymodule") } }; diff --git a/src/types/module.rs b/src/types/module.rs index aec3ea0c179..9408c7aa76e 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -488,6 +488,7 @@ impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { T: IntoPyCallbackOutput, { fn inner(module: &Bound<'_, PyModule>, object: Bound<'_, PyAny>) -> PyResult<()> { + if object.is_instance_of::() {} let name = object.getattr(__name__(module.py()))?; module.add(name.downcast_into::()?, object) }