Skip to content

Add codecs support to c-api #8267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bschoenmaeckers wants to merge 2 commits into
base: main
Choose a base branch
from
+295 −122
Open

Add codecs support to c-api #8267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4f9b2ef
Add codecs support to c-apo
bschoenmaeckers Jul 13, 2026
ecb2046
Reduce code duplication
bschoenmaeckers Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions crates/capi/src/abstract_.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use crate::util::CStrExt;
use crate::{PyObject, pystate::with_vm};
use alloc::slice;
use core::ffi::{CStr, c_char, c_int};
use core::ffi::{c_char, c_int};
pub use iter::*;
pub use mapping::*;
pub use number::*;
Expand Down Expand Up @@ -208,9 +209,7 @@ pub unsafe extern "C" fn PyObject_DelItem(obj: *mut PyObject, key: *mut PyObject
pub unsafe extern "C" fn PyObject_DelItemString(obj: *mut PyObject, key: *const c_char) -> c_int {
with_vm(|vm| {
let obj = unsafe { &*obj };
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_value_error("mapping key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();
obj.del_item(key, vm)
})
}
Expand Down
21 changes: 7 additions & 14 deletions crates/capi/src/abstract_/mapping.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::util::CStrExt;
use crate::{PyObject, pystate::with_vm};
use core::ffi::{CStr, c_char, c_int};
use core::ffi::{c_char, c_int};
use rustpython_vm::AsObject;

#[unsafe(no_mangle)]
Expand Down Expand Up @@ -60,9 +61,7 @@ pub unsafe extern "C" fn PyMapping_GetItemString(
) -> *mut PyObject {
with_vm(|vm| {
let obj = unsafe { &*obj };
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_value_error("mapping key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();
obj.get_item(key, vm)
})
}
Expand Down Expand Up @@ -104,9 +103,7 @@ pub unsafe extern "C" fn PyMapping_GetOptionalItemString(
*result = core::ptr::null_mut();
}
let obj = unsafe { &*obj };
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_value_error("mapping key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();

match obj.get_item(key, vm) {
Ok(value) => {
Expand Down Expand Up @@ -134,7 +131,7 @@ pub unsafe extern "C" fn PyMapping_HasKey(obj: *mut PyObject, key: *mut PyObject
pub unsafe extern "C" fn PyMapping_HasKeyString(obj: *mut PyObject, key: *const c_char) -> c_int {
with_vm(|vm| {
let obj = unsafe { &*obj };
if let Ok(key) = unsafe { CStr::from_ptr(key) }.to_str() {
if let Ok(Some(key)) = key.try_as_str(vm) {
obj.get_item(key, vm).is_ok()
} else {
false
Expand Down Expand Up @@ -166,9 +163,7 @@ pub unsafe extern "C" fn PyMapping_HasKeyStringWithError(
) -> c_int {
with_vm(|vm| {
let obj = unsafe { &*obj };
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_value_error("mapping key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();

match obj.get_item(key, vm) {
Ok(_) => Ok(true),
Expand All @@ -186,9 +181,7 @@ pub unsafe extern "C" fn PyMapping_SetItemString(
) -> c_int {
with_vm(|vm| {
let obj = unsafe { &*obj };
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_value_error("mapping key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();
let value = unsafe { &*value }.to_owned();
obj.set_item(key, value, vm)
})
Expand Down
212 changes: 212 additions & 0 deletions crates/capi/src/codecs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
use crate::util::CStrExt;
use crate::{PyObject, pystate::with_vm};
use core::ffi::{c_char, c_int};
use rustpython_vm::{AsObject, VirtualMachine};

fn call_codec_error_handler(
vm: &VirtualMachine,
handler_name: &str,
exc: *mut PyObject,
) -> rustpython_vm::PyResult {
vm.state
.codec_registry
.lookup_error(handler_name, vm)?
.call((unsafe { &*exc }.to_owned(),), vm)
}

fn codec_stream(
vm: &VirtualMachine,
encoding: *const c_char,
stream: *mut PyObject,
errors: *const c_char,
method: &str,
) -> rustpython_vm::PyResult {
let encoding = encoding.try_as_str(vm)?.unwrap_or("utf-8");
let errors = errors.try_as_str(vm)?.map(|errors| vm.ctx.new_str(errors));
let stream = unsafe { &*stream }.to_owned();
let codec = vm.state.codec_registry.lookup(encoding, vm)?;
let args = match errors {
Some(errors) => vec![stream, errors.into()],
None => vec![stream],
};
vm.call_method(codec.as_tuple().as_object(), method, args)
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_Register(search_function: *mut PyObject) -> c_int {
with_vm(|vm| {
let search_function = unsafe { &*search_function }.to_owned();
vm.state.codec_registry.register(search_function, vm)
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_Unregister(search_function: *mut PyObject) -> c_int {
with_vm(|vm| {
let search_function = unsafe { &*search_function }.to_owned();
vm.state.codec_registry.unregister(search_function);
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_KnownEncoding(encoding: *const c_char) -> c_int {
with_vm(|vm| {
let encoding = encoding.try_as_str(vm)?.unwrap();
match vm.state.codec_registry.lookup(encoding, vm) {
Ok(_) => Ok(true),
Err(err) if err.fast_isinstance(vm.ctx.exceptions.lookup_error) => Ok(false),
Err(err) => Err(err),
}
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_Encode(
object: *mut PyObject,
encoding: *const c_char,
errors: *const c_char,
) -> *mut PyObject {
with_vm(|vm| {
let object = unsafe { &*object }.to_owned();
let encoding = encoding.try_as_str(vm)?.unwrap_or("utf-8");
let errors = errors
.try_as_str(vm)?
.map(|errors| vm.ctx.new_utf8_str(errors));
vm.state.codec_registry.encode(object, encoding, errors, vm)
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_Decode(
object: *mut PyObject,
encoding: *const c_char,
errors: *const c_char,
) -> *mut PyObject {
with_vm(|vm| {
let object = unsafe { &*object }.to_owned();
let encoding = encoding.try_as_str(vm)?.unwrap_or("utf-8");
let errors = errors
.try_as_str(vm)?
.map(|errors| vm.ctx.new_utf8_str(errors));
vm.state.codec_registry.decode(object, encoding, errors, vm)
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_Encoder(encoding: *const c_char) -> *mut PyObject {
with_vm(|vm| {
let encoding = encoding.try_as_str(vm)?.unwrap();
vm.state
.codec_registry
.lookup(encoding, vm)
.map(|codec| codec.get_encode_func().to_owned())
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_Decoder(encoding: *const c_char) -> *mut PyObject {
with_vm(|vm| {
let encoding = encoding.try_as_str(vm)?.unwrap();
vm.state
.codec_registry
.lookup(encoding, vm)
.map(|codec| codec.get_decode_func().to_owned())
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_IncrementalEncoder(
encoding: *const c_char,
errors: *const c_char,
) -> *mut PyObject {
with_vm(|vm| {
let encoding = encoding.try_as_str(vm)?.unwrap();
let errors = errors.try_as_str(vm)?.map(|s| vm.ctx.new_str(s));
let codec = vm.state.codec_registry.lookup(encoding, vm)?;
codec.get_incremental_encoder(errors, vm)
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_IncrementalDecoder(
encoding: *const c_char,
errors: *const c_char,
) -> *mut PyObject {
with_vm(|vm| {
let encoding = encoding.try_as_str(vm)?.unwrap();
let errors = errors.try_as_str(vm)?.map(|s| vm.ctx.new_str(s));
let codec = vm.state.codec_registry.lookup(encoding, vm)?;
codec.get_incremental_decoder(errors, vm)
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_StreamReader(
encoding: *const c_char,
stream: *mut PyObject,
errors: *const c_char,
) -> *mut PyObject {
with_vm(|vm| codec_stream(vm, encoding, stream, errors, "streamreader"))
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_StreamWriter(
encoding: *const c_char,
stream: *mut PyObject,
errors: *const c_char,
) -> *mut PyObject {
with_vm(|vm| codec_stream(vm, encoding, stream, errors, "streamwriter"))
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_RegisterError(name: *const c_char, error: *mut PyObject) -> c_int {
with_vm(|vm| {
let name = name.try_as_str(vm)?.unwrap();
let error = unsafe { &*error }.to_owned();
if !error.is_callable() {
return Err(vm.new_type_error("handler must be callable"));
}
vm.state
.codec_registry
.register_error(name.to_owned(), error);
Ok(())
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_LookupError(name: *const c_char) -> *mut PyObject {
with_vm(|vm| {
let name = name.try_as_str(vm)?.unwrap();
vm.state.codec_registry.lookup_error(name, vm)
})
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_StrictErrors(exc: *mut PyObject) -> *mut PyObject {
with_vm(|vm| call_codec_error_handler(vm, "strict", exc))
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_IgnoreErrors(exc: *mut PyObject) -> *mut PyObject {
with_vm(|vm| call_codec_error_handler(vm, "ignore", exc))
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_ReplaceErrors(exc: *mut PyObject) -> *mut PyObject {
with_vm(|vm| call_codec_error_handler(vm, "replace", exc))
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_XMLCharRefReplaceErrors(exc: *mut PyObject) -> *mut PyObject {
with_vm(|vm| call_codec_error_handler(vm, "xmlcharrefreplace", exc))
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_BackslashReplaceErrors(exc: *mut PyObject) -> *mut PyObject {
with_vm(|vm| call_codec_error_handler(vm, "backslashreplace", exc))
}

#[unsafe(no_mangle)]
pub unsafe extern "C" fn PyCodec_NameReplaceErrors(exc: *mut PyObject) -> *mut PyObject {
with_vm(|vm| call_codec_error_handler(vm, "namereplace", exc))
}
20 changes: 5 additions & 15 deletions crates/capi/src/descrobject.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ use crate::PyObject;
use crate::methodobject::{PyMethodDef, build_method_def};
use crate::object::PyTypeObject;
use crate::pystate::with_vm;
use core::ffi::{CStr, c_char, c_int, c_void};
use crate::util::CStrExt;
use core::ffi::{c_char, c_int, c_void};
use core::ptr::NonNull;
use rustpython_vm::builtins::{
DescriptorMemberDef, MemberGetter, MemberKind, MemberSetter, PyDescriptorOwned, PyGetSet,
Expand Down Expand Up @@ -34,9 +35,7 @@ impl PyGetSetDef {
ty: &'static Py<PyType>,
vm: &VirtualMachine,
) -> PyResult<PyRef<PyGetSet>> {
let name = unsafe { CStr::from_ptr(self.name) }
.to_str()
.map_err(|_| vm.new_system_error("PyGetSetDef name was not valid UTF-8"))?;
let name = self.name.try_as_str(vm)?.unwrap();
let closure = self.closure as usize;

let descriptor = match (self.get, self.set) {
Expand Down Expand Up @@ -142,9 +141,7 @@ impl PyMemberDef {
ty: &Py<PyType>,
vm: &VirtualMachine,
) -> PyResult<PyRef<PyMemberDescriptor>> {
let name = unsafe { CStr::from_ptr(self.name) }
.to_str()
.map_err(|_| vm.new_system_error("PyMemberDef name was not valid UTF-8"))?;
let name = self.name.try_as_str(vm)?.unwrap();
let kind = match self.type_code {
6 => MemberKind::Object,
16 => MemberKind::ObjectEx,
Expand All @@ -165,14 +162,7 @@ impl PyMemberDef {
);
}

let doc = NonNull::new(self.doc.cast_mut())
.map(|doc| {
unsafe { CStr::from_ptr(doc.as_ptr()) }
.to_str()
.map(|s| s.to_owned())
.map_err(|_| vm.new_system_error("PyMemberDef doc was not valid UTF-8"))
})
.transpose()?;
let doc = self.doc.try_as_str(vm)?.map(str::to_owned);

let descriptor = PyMemberDescriptor {
common: PyDescriptorOwned {
Expand Down
19 changes: 6 additions & 13 deletions crates/capi/src/dictobject.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use crate::PyObject;
use crate::object::define_py_check;
use crate::pystate::with_vm;
use core::ffi::{CStr, c_char, c_int};
use crate::util::CStrExt;
use core::ffi::{c_char, c_int};
use core::ptr::NonNull;
use rustpython_vm::AsObject;
use rustpython_vm::PyPayload;
Expand Down Expand Up @@ -49,9 +50,7 @@ pub unsafe extern "C" fn PyDict_SetItemString(
) -> c_int {
with_vm(|vm| {
let dict = unsafe { &*dict }.try_downcast_ref::<PyDict>(vm)?;
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_value_error("dictionary key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();
let value = unsafe { &*val }.to_owned();
dict.inner_setitem(key, value, vm)
})
Expand Down Expand Up @@ -94,9 +93,7 @@ pub unsafe extern "C" fn PyDict_GetItemString(
) -> *mut PyObject {
with_vm(|vm| {
let dict = unsafe { &*dict }.try_downcast_ref::<PyDict>(vm)?;
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_unicode_decode_error("dictionary key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();

match dict.inner_getitem_opt(key, vm)? {
Some(value) => Ok(value.as_object().as_raw().cast_mut()),
Expand All @@ -116,9 +113,7 @@ pub unsafe extern "C" fn PyDict_GetItemStringRef(
*result = core::ptr::null_mut();
}
let dict = unsafe { &*dict }.try_downcast_ref::<PyDict>(vm)?;
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_value_error("dictionary key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();

if let Some(value) = dict.inner_getitem_opt(key, vm)? {
unsafe {
Expand Down Expand Up @@ -231,9 +226,7 @@ pub unsafe extern "C" fn PyDict_DelItem(dict: *mut PyObject, key: *mut PyObject)
pub unsafe extern "C" fn PyDict_DelItemString(dict: *mut PyObject, key: *const c_char) -> c_int {
with_vm(|vm| {
let dict = unsafe { &*dict }.try_downcast_ref::<PyDict>(vm)?;
let key = unsafe { CStr::from_ptr(key) }
.to_str()
.map_err(|_| vm.new_value_error("dictionary key must be valid UTF-8"))?;
let key = key.try_as_str(vm)?.unwrap();
dict.del_item(key, vm)
})
}
Expand Down
Loading
Loading