blob: 84b91da7a9c751f4dc6aa89bdf9d0329d44f8a7f [file] [log] [blame]
// Copyright 2022 The Fuchsia Authors
//
// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT
// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option.
// This file may not be copied, modified, or distributed except according to
// those terms.
//! Utilities used by macros and by `zerocopy-derive`.
//!
//! These are defined here `zerocopy` rather than in code generated by macros or
//! by `zerocopy-derive` so that they can be compiled once rather than
//! recompiled for every invocation (e.g., if they were defined in generated
//! code, then deriving `IntoBytes` and `FromBytes` on three different types
//! would result in the code in question being emitted and compiled six
//! different times).
#![allow(missing_debug_implementations)]
use core::mem::{self, ManuallyDrop};
// TODO(#29), TODO(https://github.com/rust-lang/rust/issues/69835): Remove this
// `cfg` when `size_of_val_raw` is stabilized.
#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)]
use core::ptr::{self, NonNull};
use crate::{
pointer::{
invariant::{self, AtLeast, Invariants},
AliasingSafe, AliasingSafeReason, BecauseExclusive, BecauseImmutable,
},
Immutable, IntoBytes, Ptr, TryFromBytes, Unalign, ValidityError,
};
/// Projects the type of the field at `Index` in `Self`.
///
/// The `Index` parameter is any sort of handle that identifies the field; its
/// definition is the obligation of the implementer.
///
/// # Safety
///
/// Unsafe code may assume that this accurately reflects the definition of
/// `Self`.
pub unsafe trait Field<Index> {
/// The type of the field at `Index`.
type Type: ?Sized;
}
#[cfg_attr(
zerocopy_diagnostic_on_unimplemented_1_78_0,
diagnostic::on_unimplemented(
message = "`{T}` has inter-field padding",
label = "types with padding cannot implement `IntoBytes`",
note = "consider using `zerocopy::Unalign` to lower the alignment of individual fields",
note = "consider adding explicit fields where padding would be",
note = "consider using `#[repr(packed)]` to remove inter-field padding"
)
)]
pub trait PaddingFree<T: ?Sized, const HAS_PADDING: bool> {}
impl<T: ?Sized> PaddingFree<T, false> for () {}
/// A type whose size is equal to `align_of::<T>()`.
#[repr(C)]
pub struct AlignOf<T> {
// This field ensures that:
// - The size is always at least 1 (the minimum possible alignment).
// - If the alignment is greater than 1, Rust has to round up to the next
// multiple of it in order to make sure that `Align`'s size is a multiple
// of that alignment. Without this field, its size could be 0, which is a
// valid multiple of any alignment.
_u: u8,
_a: [T; 0],
}
impl<T> AlignOf<T> {
#[inline(never)] // Make `missing_inline_in_public_items` happy.
#[cfg_attr(coverage_nightly, coverage(off))]
pub fn into_t(self) -> T {
unreachable!()
}
}
/// A type whose size is equal to `max(align_of::<T>(), align_of::<U>())`.
#[repr(C)]
pub union MaxAlignsOf<T, U> {
_t: ManuallyDrop<AlignOf<T>>,
_u: ManuallyDrop<AlignOf<U>>,
}
impl<T, U> MaxAlignsOf<T, U> {
#[inline(never)] // Make `missing_inline_in_public_items` happy.
#[cfg_attr(coverage_nightly, coverage(off))]
pub fn new(_t: T, _u: U) -> MaxAlignsOf<T, U> {
unreachable!()
}
}
const _64K: usize = 1 << 16;
// TODO(#29), TODO(https://github.com/rust-lang/rust/issues/69835): Remove this
// `cfg` when `size_of_val_raw` is stabilized.
#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)]
#[repr(C, align(65536))]
struct Aligned64kAllocation([u8; _64K]);
/// A pointer to an aligned allocation of size 2^16.
///
/// # Safety
///
/// `ALIGNED_64K_ALLOCATION` is guaranteed to point to the entirety of an
/// allocation with size and alignment 2^16, and to have valid provenance.
// TODO(#29), TODO(https://github.com/rust-lang/rust/issues/69835): Remove this
// `cfg` when `size_of_val_raw` is stabilized.
#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)]
pub const ALIGNED_64K_ALLOCATION: NonNull<[u8]> = {
const REF: &Aligned64kAllocation = &Aligned64kAllocation([0; _64K]);
let ptr: *const Aligned64kAllocation = REF;
let ptr: *const [u8] = ptr::slice_from_raw_parts(ptr.cast(), _64K);
// SAFETY:
// - `ptr` is derived from a Rust reference, which is guaranteed to be
// non-null.
// - `ptr` is derived from an `&Aligned64kAllocation`, which has size and
// alignment `_64K` as promised. Its length is initialized to `_64K`,
// which means that it refers to the entire allocation.
// - `ptr` is derived from a Rust reference, which is guaranteed to have
// valid provenance.
//
// TODO(#429): Once `NonNull::new_unchecked` docs document that it preserves
// provenance, cite those docs.
// TODO: Replace this `as` with `ptr.cast_mut()` once our MSRV >= 1.65
#[allow(clippy::as_conversions)]
unsafe {
NonNull::new_unchecked(ptr as *mut _)
}
};
/// Computes the offset of the base of the field `$trailing_field_name` within
/// the type `$ty`.
///
/// `trailing_field_offset!` produces code which is valid in a `const` context.
// TODO(#29), TODO(https://github.com/rust-lang/rust/issues/69835): Remove this
// `cfg` when `size_of_val_raw` is stabilized.
#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)]
#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`.
#[macro_export]
macro_rules! trailing_field_offset {
($ty:ty, $trailing_field_name:tt) => {{
let min_size = {
let zero_elems: *const [()] =
$crate::util::macro_util::core_reexport::ptr::slice_from_raw_parts(
// Work around https://github.com/rust-lang/rust-clippy/issues/12280
#[allow(clippy::incompatible_msrv)]
$crate::util::macro_util::core_reexport::ptr::NonNull::<()>::dangling()
.as_ptr()
.cast_const(),
0,
);
// SAFETY:
// - If `$ty` is `Sized`, `size_of_val_raw` is always safe to call.
// - Otherwise:
// - If `$ty` is not a slice DST, this pointer conversion will
// fail due to "mismatched vtable kinds", and compilation will
// fail.
// - If `$ty` is a slice DST, we have constructed `zero_elems` to
// have zero trailing slice elements. Per the `size_of_val_raw`
// docs, "For the special case where the dynamic tail length is
// 0, this function is safe to call." [1]
//
// [1] https://doc.rust-lang.org/nightly/std/mem/fn.size_of_val_raw.html
unsafe {
#[allow(clippy::as_conversions)]
$crate::util::macro_util::core_reexport::mem::size_of_val_raw(
zero_elems as *const $ty,
)
}
};
assert!(min_size <= _64K);
#[allow(clippy::as_conversions)]
let ptr = ALIGNED_64K_ALLOCATION.as_ptr() as *const $ty;
// SAFETY:
// - Thanks to the preceding `assert!`, we know that the value with zero
// elements fits in `_64K` bytes, and thus in the allocation addressed
// by `ALIGNED_64K_ALLOCATION`. The offset of the trailing field is
// guaranteed to be no larger than this size, so this field projection
// is guaranteed to remain in-bounds of its allocation.
// - Because the minimum size is no larger than `_64K` bytes, and
// because an object's size must always be a multiple of its alignment
// [1], we know that `$ty`'s alignment is no larger than `_64K`. The
// allocation addressed by `ALIGNED_64K_ALLOCATION` is guaranteed to
// be aligned to `_64K`, so `ptr` is guaranteed to satisfy `$ty`'s
// alignment.
// - As required by `addr_of!`, we do not write through `field`.
//
// Note that, as of [2], this requirement is technically unnecessary
// for Rust versions >= 1.75.0, but no harm in guaranteeing it anyway
// until we bump our MSRV.
//
// [1] Per https://doc.rust-lang.org/reference/type-layout.html:
//
// The size of a value is always a multiple of its alignment.
//
// [2] https://github.com/rust-lang/reference/pull/1387
let field = unsafe {
$crate::util::macro_util::core_reexport::ptr::addr_of!((*ptr).$trailing_field_name)
};
// SAFETY:
// - Both `ptr` and `field` are derived from the same allocated object.
// - By the preceding safety comment, `field` is in bounds of that
// allocated object.
// - The distance, in bytes, between `ptr` and `field` is required to be
// a multiple of the size of `u8`, which is trivially true because
// `u8`'s size is 1.
// - The distance, in bytes, cannot overflow `isize`. This is guaranteed
// because no allocated object can have a size larger than can fit in
// `isize`. [1]
// - The distance being in-bounds cannot rely on wrapping around the
// address space. This is guaranteed because the same is guaranteed of
// allocated objects. [1]
//
// [1] TODO(#429), TODO(https://github.com/rust-lang/rust/pull/116675):
// Once these are guaranteed in the Reference, cite it.
let offset = unsafe { field.cast::<u8>().offset_from(ptr.cast::<u8>()) };
// Guaranteed not to be lossy: `field` comes after `ptr`, so the offset
// from `ptr` to `field` is guaranteed to be positive.
assert!(offset >= 0);
Some(
#[allow(clippy::as_conversions)]
{
offset as usize
},
)
}};
}
/// Computes alignment of `$ty: ?Sized`.
///
/// `align_of!` produces code which is valid in a `const` context.
// TODO(#29), TODO(https://github.com/rust-lang/rust/issues/69835): Remove this
// `cfg` when `size_of_val_raw` is stabilized.
#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)]
#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`.
#[macro_export]
macro_rules! align_of {
($ty:ty) => {{
// SAFETY: `OffsetOfTrailingIsAlignment` is `repr(C)`, and its layout is
// guaranteed [1] to begin with the single-byte layout for `_byte`,
// followed by the padding needed to align `_trailing`, then the layout
// for `_trailing`, and finally any trailing padding bytes needed to
// correctly-align the entire struct.
//
// This macro computes the alignment of `$ty` by counting the number of
// bytes preceeding `_trailing`. For instance, if the alignment of `$ty`
// is `1`, then no padding is required align `_trailing` and it will be
// located immediately after `_byte` at offset 1. If the alignment of
// `$ty` is 2, then a single padding byte is required before
// `_trailing`, and `_trailing` will be located at offset 2.
// This correspondence between offset and alignment holds for all valid
// Rust alignments, and we confirm this exhaustively (or, at least up to
// the maximum alignment supported by `trailing_field_offset!`) in
// `test_align_of_dst`.
//
// [1]: https://doc.rust-lang.org/nomicon/other-reprs.html#reprc
#[repr(C)]
struct OffsetOfTrailingIsAlignment {
_byte: u8,
_trailing: $ty,
}
trailing_field_offset!(OffsetOfTrailingIsAlignment, _trailing)
}};
}
mod size_to_tag {
pub trait SizeToTag<const SIZE: usize> {
type Tag;
}
impl SizeToTag<1> for () {
type Tag = u8;
}
impl SizeToTag<2> for () {
type Tag = u16;
}
impl SizeToTag<4> for () {
type Tag = u32;
}
impl SizeToTag<8> for () {
type Tag = u64;
}
impl SizeToTag<16> for () {
type Tag = u128;
}
}
/// An alias for the unsigned integer of the given size in bytes.
#[doc(hidden)]
pub type SizeToTag<const SIZE: usize> = <() as size_to_tag::SizeToTag<SIZE>>::Tag;
// We put `Sized` in its own module so it can have the same name as the standard
// library `Sized` without shadowing it in the parent module.
#[cfg(zerocopy_diagnostic_on_unimplemented_1_78_0)]
mod __size_of {
#[diagnostic::on_unimplemented(
message = "`{Self}` is unsized",
label = "`IntoBytes` needs all field types to be `Sized` in order to determine whether there is inter-field padding",
note = "consider using `#[repr(packed)]` to remove inter-field padding",
note = "`IntoBytes` does not require the fields of `#[repr(packed)]` types to be `Sized`"
)]
pub trait Sized: core::marker::Sized {}
impl<T: core::marker::Sized> Sized for T {}
#[inline(always)]
#[must_use]
#[allow(clippy::needless_maybe_sized)]
pub const fn size_of<T: Sized + ?core::marker::Sized>() -> usize {
core::mem::size_of::<T>()
}
}
#[cfg(zerocopy_diagnostic_on_unimplemented_1_78_0)]
pub use __size_of::size_of;
#[cfg(not(zerocopy_diagnostic_on_unimplemented_1_78_0))]
pub use core::mem::size_of;
/// Does the struct type `$t` have padding?
///
/// `$ts` is the list of the type of every field in `$t`. `$t` must be a
/// struct type, or else `struct_has_padding!`'s result may be meaningless.
///
/// Note that `struct_has_padding!`'s results are independent of `repcr` since
/// they only consider the size of the type and the sizes of the fields.
/// Whatever the repr, the size of the type already takes into account any
/// padding that the compiler has decided to add. Structs with well-defined
/// representations (such as `repr(C)`) can use this macro to check for padding.
/// Note that while this may yield some consistent value for some `repr(Rust)`
/// structs, it is not guaranteed across platforms or compilations.
#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`.
#[macro_export]
macro_rules! struct_has_padding {
($t:ty, [$($ts:ty),*]) => {
::zerocopy::util::macro_util::size_of::<$t>() > 0 $(+ ::zerocopy::util::macro_util::size_of::<$ts>())*
};
}
/// Does the union type `$t` have padding?
///
/// `$ts` is the list of the type of every field in `$t`. `$t` must be a
/// union type, or else `union_has_padding!`'s result may be meaningless.
///
/// Note that `union_has_padding!`'s results are independent of `repr` since
/// they only consider the size of the type and the sizes of the fields.
/// Whatever the repr, the size of the type already takes into account any
/// padding that the compiler has decided to add. Unions with well-defined
/// representations (such as `repr(C)`) can use this macro to check for padding.
/// Note that while this may yield some consistent value for some `repr(Rust)`
/// unions, it is not guaranteed across platforms or compilations.
#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`.
#[macro_export]
macro_rules! union_has_padding {
($t:ty, [$($ts:ty),*]) => {
false $(|| ::zerocopy::util::macro_util::size_of::<$t>() != ::zerocopy::util::macro_util::size_of::<$ts>())*
};
}
/// Does the enum type `$t` have padding?
///
/// `$disc` is the type of the enum tag, and `$ts` is a list of fields in each
/// square-bracket-delimited variant. `$t` must be an enum, or else
/// `enum_has_padding!`'s result may be meaningless. An enum has padding if any
/// of its variant structs [1][2] contain padding, and so all of the variants of
/// an enum must be "full" in order for the enum to not have padding.
///
/// The results of `enum_has_padding!` require that the enum is not
/// `repr(Rust)`, as `repr(Rust)` enums may niche the enum's tag and reduce the
/// total number of bytes required to represent the enum as a result. As long as
/// the enum is `repr(C)`, `repr(int)`, or `repr(C, int)`, this will
/// consistently return whether the enum contains any padding bytes.
///
/// [1]: https://doc.rust-lang.org/1.81.0/reference/type-layout.html#reprc-enums-with-fields
/// [2]: https://doc.rust-lang.org/1.81.0/reference/type-layout.html#primitive-representation-of-enums-with-fields
#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`.
#[macro_export]
macro_rules! enum_has_padding {
($t:ty, $disc:ty, $([$($ts:ty),*]),*) => {
false $(
|| ::zerocopy::util::macro_util::size_of::<$t>()
!= (
::zerocopy::util::macro_util::size_of::<$disc>()
$(+ ::zerocopy::util::macro_util::size_of::<$ts>())*
)
)*
}
}
/// Does `t` have alignment greater than or equal to `u`? If not, this macro
/// produces a compile error. It must be invoked in a dead codepath. This is
/// used in `transmute_ref!` and `transmute_mut!`.
#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`.
#[macro_export]
macro_rules! assert_align_gt_eq {
($t:ident, $u: ident) => {{
// The comments here should be read in the context of this macro's
// invocations in `transmute_ref!` and `transmute_mut!`.
if false {
// The type wildcard in this bound is inferred to be `T` because
// `align_of.into_t()` is assigned to `t` (which has type `T`).
let align_of: $crate::util::macro_util::AlignOf<_> = unreachable!();
$t = align_of.into_t();
// `max_aligns` is inferred to have type `MaxAlignsOf<T, U>` because
// of the inferred types of `t` and `u`.
let mut max_aligns = $crate::util::macro_util::MaxAlignsOf::new($t, $u);
// This transmute will only compile successfully if
// `align_of::<T>() == max(align_of::<T>(), align_of::<U>())` - in
// other words, if `align_of::<T>() >= align_of::<U>()`.
//
// SAFETY: This code is never run.
max_aligns = unsafe {
// Clippy: We can't annotate the types; this macro is designed
// to infer the types from the calling context.
#[allow(clippy::missing_transmute_annotations)]
$crate::util::macro_util::core_reexport::mem::transmute(align_of)
};
} else {
loop {}
}
}};
}
/// Do `t` and `u` have the same size? If not, this macro produces a compile
/// error. It must be invoked in a dead codepath. This is used in
/// `transmute_ref!` and `transmute_mut!`.
#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`.
#[macro_export]
macro_rules! assert_size_eq {
($t:ident, $u: ident) => {{
// The comments here should be read in the context of this macro's
// invocations in `transmute_ref!` and `transmute_mut!`.
if false {
// SAFETY: This code is never run.
$u = unsafe {
// Clippy:
// - It's okay to transmute a type to itself.
// - We can't annotate the types; this macro is designed to
// infer the types from the calling context.
#[allow(clippy::useless_transmute, clippy::missing_transmute_annotations)]
$crate::util::macro_util::core_reexport::mem::transmute($t)
};
} else {
loop {}
}
}};
}
/// Transmutes a reference of one type to a reference of another type.
///
/// # Safety
///
/// The caller must guarantee that:
/// - `Src: IntoBytes + Immutable`
/// - `Dst: FromBytes + Immutable`
/// - `size_of::<Src>() == size_of::<Dst>()`
/// - `align_of::<Src>() >= align_of::<Dst>()`
#[inline(always)]
pub const unsafe fn transmute_ref<'dst, 'src: 'dst, Src: 'src, Dst: 'dst>(
src: &'src Src,
) -> &'dst Dst {
let src: *const Src = src;
let dst = src.cast::<Dst>();
// SAFETY:
// - We know that it is sound to view the target type of the input reference
// (`Src`) as the target type of the output reference (`Dst`) because the
// caller has guaranteed that `Src: IntoBytes`, `Dst: FromBytes`, and
// `size_of::<Src>() == size_of::<Dst>()`.
// - We know that there are no `UnsafeCell`s, and thus we don't have to
// worry about `UnsafeCell` overlap, because `Src: Immutable` and `Dst:
// Immutable`.
// - The caller has guaranteed that alignment is not increased.
// - We know that the returned lifetime will not outlive the input lifetime
// thanks to the lifetime bounds on this function.
//
// TODO(#67): Once our MSRV is 1.58, replace this `transmute` with `&*dst`.
#[allow(clippy::transmute_ptr_to_ref)]
unsafe {
mem::transmute(dst)
}
}
/// Transmutes a mutable reference of one type to a mutable reference of another
/// type.
///
/// # Safety
///
/// The caller must guarantee that:
/// - `Src: FromBytes + IntoBytes`
/// - `Dst: FromBytes + IntoBytes`
/// - `size_of::<Src>() == size_of::<Dst>()`
/// - `align_of::<Src>() >= align_of::<Dst>()`
// TODO(#686): Consider removing the `Immutable` requirement.
#[inline(always)]
pub unsafe fn transmute_mut<'dst, 'src: 'dst, Src: 'src, Dst: 'dst>(
src: &'src mut Src,
) -> &'dst mut Dst {
let src: *mut Src = src;
let dst = src.cast::<Dst>();
// SAFETY:
// - We know that it is sound to view the target type of the input reference
// (`Src`) as the target type of the output reference (`Dst`) and
// vice-versa because the caller has guaranteed that `Src: FromBytes +
// IntoBytes`, `Dst: FromBytes + IntoBytes`, and `size_of::<Src>() ==
// size_of::<Dst>()`.
// - The caller has guaranteed that alignment is not increased.
// - We know that the returned lifetime will not outlive the input lifetime
// thanks to the lifetime bounds on this function.
unsafe { &mut *dst }
}
/// Is a given source a valid instance of `Dst`?
///
/// If so, returns `src` casted to a `Ptr<Dst, _>`. Otherwise returns `None`.
///
/// # Safety
///
/// Unsafe code may assume that, if `try_cast_or_pme(src)` returns `Some`,
/// `*src` is a bit-valid instance of `Dst`, and that the size of `Src` is
/// greater than or equal to the size of `Dst`.
///
/// # Panics
///
/// `try_cast_or_pme` may either produce a post-monomorphization error or a
/// panic if `Dst` not the same size as `Src`. Otherwise, `try_cast_or_pme`
/// panics under the same circumstances as [`is_bit_valid`].
///
/// [`is_bit_valid`]: TryFromBytes::is_bit_valid
#[doc(hidden)]
#[inline]
fn try_cast_or_pme<Src, Dst, I, R>(
src: Ptr<'_, Src, I>,
) -> Result<
Ptr<'_, Dst, (I::Aliasing, invariant::Any, invariant::Valid)>,
ValidityError<Ptr<'_, Src, I>, Dst>,
>
where
Src: IntoBytes,
Dst: TryFromBytes + AliasingSafe<Src, I::Aliasing, R>,
I: Invariants<Validity = invariant::Valid>,
I::Aliasing: AtLeast<invariant::Shared>,
R: AliasingSafeReason,
{
static_assert!(Src, Dst => mem::size_of::<Dst>() == mem::size_of::<Src>());
// SAFETY: This is a pointer cast, satisfying the following properties:
// - `p as *mut Dst` addresses a subset of the `bytes` addressed by `src`,
// because we assert above that the size of `Dst` equal to the size of
// `Src`.
// - `p as *mut Dst` is a provenance-preserving cast
// - Because `Dst: AliasingSafe<Src, I::Aliasing, _>`, either:
// - `I::Aliasing` is `Exclusive`
// - `Src` and `Dst` are both `Immutable`, in which case they
// trivially contain `UnsafeCell`s at identical locations
#[allow(clippy::as_conversions)]
let c_ptr = unsafe { src.cast_unsized(|p| p as *mut Dst) };
// SAFETY: `c_ptr` is derived from `src` which is `IntoBytes`. By
// invariant on `IntoByte`s, `c_ptr`'s referent consists entirely of
// initialized bytes.
let c_ptr = unsafe { c_ptr.assume_initialized() };
match c_ptr.try_into_valid() {
Ok(ptr) => Ok(ptr),
Err(err) => {
// Re-cast `Ptr<Dst>` to `Ptr<Src>`.
let ptr = err.into_src();
// SAFETY: This is a pointer cast, satisfying the following
// properties:
// - `p as *mut Src` addresses a subset of the `bytes` addressed by
// `ptr`, because we assert above that the size of `Dst` is equal
// to the size of `Src`.
// - `p as *mut Src` is a provenance-preserving cast
// - Because `Dst: AliasingSafe<Src, I::Aliasing, _>`, either:
// - `I::Aliasing` is `Exclusive`
// - `Src` and `Dst` are both `Immutable`, in which case they
// trivially contain `UnsafeCell`s at identical locations
#[allow(clippy::as_conversions)]
let ptr = unsafe { ptr.cast_unsized(|p| p as *mut Src) };
// SAFETY: `ptr` is `src`, and has the same alignment invariant.
let ptr = unsafe { ptr.assume_alignment::<I::Alignment>() };
// SAFETY: `ptr` is `src` and has the same validity invariant.
let ptr = unsafe { ptr.assume_validity::<I::Validity>() };
Err(ValidityError::new(ptr.unify_invariants()))
}
}
}
/// Attempts to transmute `Src` into `Dst`.
///
/// A helper for `try_transmute!`.
///
/// # Panics
///
/// `try_transmute` may either produce a post-monomorphization error or a panic
/// if `Dst` is bigger than `Src`. Otherwise, `try_transmute` panics under the
/// same circumstances as [`is_bit_valid`].
///
/// [`is_bit_valid`]: TryFromBytes::is_bit_valid
#[inline(always)]
pub fn try_transmute<Src, Dst>(src: Src) -> Result<Dst, ValidityError<Src, Dst>>
where
Src: IntoBytes,
Dst: TryFromBytes,
{
let mut src = ManuallyDrop::new(src);
let ptr = Ptr::from_mut(&mut src);
// Wrapping `Dst` in `Unalign` ensures that this cast does not fail due to
// alignment requirements.
match try_cast_or_pme::<_, ManuallyDrop<Unalign<Dst>>, _, BecauseExclusive>(ptr) {
Ok(ptr) => {
let dst = ptr.bikeshed_recall_aligned().as_mut();
// SAFETY: By shadowing `dst`, we ensure that `dst` is not re-used
// after taking its inner value.
let dst = unsafe { ManuallyDrop::take(dst) };
Ok(dst.into_inner())
}
Err(_) => Err(ValidityError::new(ManuallyDrop::into_inner(src))),
}
}
/// Attempts to transmute `&Src` into `&Dst`.
///
/// A helper for `try_transmute_ref!`.
///
/// # Panics
///
/// `try_transmute_ref` may either produce a post-monomorphization error or a
/// panic if `Dst` is bigger or has a stricter alignment requirement than `Src`.
/// Otherwise, `try_transmute_ref` panics under the same circumstances as
/// [`is_bit_valid`].
///
/// [`is_bit_valid`]: TryFromBytes::is_bit_valid
#[inline(always)]
pub fn try_transmute_ref<Src, Dst>(src: &Src) -> Result<&Dst, ValidityError<&Src, Dst>>
where
Src: IntoBytes + Immutable,
Dst: TryFromBytes + Immutable,
{
match try_cast_or_pme::<Src, Dst, _, BecauseImmutable>(Ptr::from_ref(src)) {
Ok(ptr) => {
static_assert!(Src, Dst => mem::align_of::<Dst>() <= mem::align_of::<Src>());
// SAFETY: We have checked that `Dst` does not have a stricter
// alignment requirement than `Src`.
let ptr = unsafe { ptr.assume_alignment::<invariant::Aligned>() };
Ok(ptr.as_ref())
}
Err(err) => Err(err.map_src(Ptr::as_ref)),
}
}
/// Attempts to transmute `&mut Src` into `&mut Dst`.
///
/// A helper for `try_transmute_mut!`.
///
/// # Panics
///
/// `try_transmute_mut` may either produce a post-monomorphization error or a
/// panic if `Dst` is bigger or has a stricter alignment requirement than `Src`.
/// Otherwise, `try_transmute_mut` panics under the same circumstances as
/// [`is_bit_valid`].
///
/// [`is_bit_valid`]: TryFromBytes::is_bit_valid
#[inline(always)]
pub fn try_transmute_mut<Src, Dst>(src: &mut Src) -> Result<&mut Dst, ValidityError<&mut Src, Dst>>
where
Src: IntoBytes,
Dst: TryFromBytes,
{
match try_cast_or_pme::<Src, Dst, _, BecauseExclusive>(Ptr::from_mut(src)) {
Ok(ptr) => {
static_assert!(Src, Dst => mem::align_of::<Dst>() <= mem::align_of::<Src>());
// SAFETY: We have checked that `Dst` does not have a stricter
// alignment requirement than `Src`.
let ptr = unsafe { ptr.assume_alignment::<invariant::Aligned>() };
Ok(ptr.as_mut())
}
Err(err) => Err(err.map_src(Ptr::as_mut)),
}
}
/// A function which emits a warning if its return value is not used.
#[must_use]
#[inline(always)]
pub const fn must_use<T>(t: T) -> T {
t
}
// NOTE: We can't change this to a `pub use core as core_reexport` until [1] is
// fixed or we update to a semver-breaking version (as of this writing, 0.8.0)
// on the `main` branch.
//
// [1] https://github.com/obi1kenobi/cargo-semver-checks/issues/573
pub mod core_reexport {
pub use core::*;
pub mod mem {
pub use core::mem::*;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::util::testutil::*;
#[test]
fn test_align_of() {
macro_rules! test {
($ty:ty) => {
assert_eq!(mem::size_of::<AlignOf<$ty>>(), mem::align_of::<$ty>());
};
}
test!(());
test!(u8);
test!(AU64);
test!([AU64; 2]);
}
#[test]
fn test_max_aligns_of() {
macro_rules! test {
($t:ty, $u:ty) => {
assert_eq!(
mem::size_of::<MaxAlignsOf<$t, $u>>(),
core::cmp::max(mem::align_of::<$t>(), mem::align_of::<$u>())
);
};
}
test!(u8, u8);
test!(u8, AU64);
test!(AU64, u8);
}
#[test]
fn test_typed_align_check() {
// Test that the type-based alignment check used in
// `assert_align_gt_eq!` behaves as expected.
macro_rules! assert_t_align_gteq_u_align {
($t:ty, $u:ty, $gteq:expr) => {
assert_eq!(
mem::size_of::<MaxAlignsOf<$t, $u>>() == mem::size_of::<AlignOf<$t>>(),
$gteq
);
};
}
assert_t_align_gteq_u_align!(u8, u8, true);
assert_t_align_gteq_u_align!(AU64, AU64, true);
assert_t_align_gteq_u_align!(AU64, u8, true);
assert_t_align_gteq_u_align!(u8, AU64, false);
}
// TODO(#29), TODO(https://github.com/rust-lang/rust/issues/69835): Remove
// this `cfg` when `size_of_val_raw` is stabilized.
#[allow(clippy::decimal_literal_representation)]
#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)]
#[test]
fn test_trailing_field_offset() {
assert_eq!(mem::align_of::<Aligned64kAllocation>(), _64K);
macro_rules! test {
(#[$cfg:meta] ($($ts:ty),* ; $trailing_field_ty:ty) => $expect:expr) => {{
#[$cfg]
struct Test($(#[allow(dead_code)] $ts,)* #[allow(dead_code)] $trailing_field_ty);
assert_eq!(test!(@offset $($ts),* ; $trailing_field_ty), $expect);
}};
(#[$cfg:meta] $(#[$cfgs:meta])* ($($ts:ty),* ; $trailing_field_ty:ty) => $expect:expr) => {
test!(#[$cfg] ($($ts),* ; $trailing_field_ty) => $expect);
test!($(#[$cfgs])* ($($ts),* ; $trailing_field_ty) => $expect);
};
(@offset ; $_trailing:ty) => { trailing_field_offset!(Test, 0) };
(@offset $_t:ty ; $_trailing:ty) => { trailing_field_offset!(Test, 1) };
}
test!(#[repr(C)] #[repr(transparent)] #[repr(packed)](; u8) => Some(0));
test!(#[repr(C)] #[repr(transparent)] #[repr(packed)](; [u8]) => Some(0));
test!(#[repr(C)] #[repr(C, packed)] (u8; u8) => Some(1));
test!(#[repr(C)] (; AU64) => Some(0));
test!(#[repr(C)] (; [AU64]) => Some(0));
test!(#[repr(C)] (u8; AU64) => Some(8));
test!(#[repr(C)] (u8; [AU64]) => Some(8));
test!(#[repr(C)] (; Nested<u8, AU64>) => Some(0));
test!(#[repr(C)] (; Nested<u8, [AU64]>) => Some(0));
test!(#[repr(C)] (u8; Nested<u8, AU64>) => Some(8));
test!(#[repr(C)] (u8; Nested<u8, [AU64]>) => Some(8));
// Test that `packed(N)` limits the offset of the trailing field.
test!(#[repr(C, packed( 1))] (u8; elain::Align< 2>) => Some( 1));
test!(#[repr(C, packed( 2))] (u8; elain::Align< 4>) => Some( 2));
test!(#[repr(C, packed( 4))] (u8; elain::Align< 8>) => Some( 4));
test!(#[repr(C, packed( 8))] (u8; elain::Align< 16>) => Some( 8));
test!(#[repr(C, packed( 16))] (u8; elain::Align< 32>) => Some( 16));
test!(#[repr(C, packed( 32))] (u8; elain::Align< 64>) => Some( 32));
test!(#[repr(C, packed( 64))] (u8; elain::Align< 128>) => Some( 64));
test!(#[repr(C, packed( 128))] (u8; elain::Align< 256>) => Some( 128));
test!(#[repr(C, packed( 256))] (u8; elain::Align< 512>) => Some( 256));
test!(#[repr(C, packed( 512))] (u8; elain::Align< 1024>) => Some( 512));
test!(#[repr(C, packed( 1024))] (u8; elain::Align< 2048>) => Some( 1024));
test!(#[repr(C, packed( 2048))] (u8; elain::Align< 4096>) => Some( 2048));
test!(#[repr(C, packed( 4096))] (u8; elain::Align< 8192>) => Some( 4096));
test!(#[repr(C, packed( 8192))] (u8; elain::Align< 16384>) => Some( 8192));
test!(#[repr(C, packed( 16384))] (u8; elain::Align< 32768>) => Some( 16384));
test!(#[repr(C, packed( 32768))] (u8; elain::Align< 65536>) => Some( 32768));
test!(#[repr(C, packed( 65536))] (u8; elain::Align< 131072>) => Some( 65536));
/* Alignments above 65536 are not yet supported.
test!(#[repr(C, packed( 131072))] (u8; elain::Align< 262144>) => Some( 131072));
test!(#[repr(C, packed( 262144))] (u8; elain::Align< 524288>) => Some( 262144));
test!(#[repr(C, packed( 524288))] (u8; elain::Align< 1048576>) => Some( 524288));
test!(#[repr(C, packed( 1048576))] (u8; elain::Align< 2097152>) => Some( 1048576));
test!(#[repr(C, packed( 2097152))] (u8; elain::Align< 4194304>) => Some( 2097152));
test!(#[repr(C, packed( 4194304))] (u8; elain::Align< 8388608>) => Some( 4194304));
test!(#[repr(C, packed( 8388608))] (u8; elain::Align< 16777216>) => Some( 8388608));
test!(#[repr(C, packed( 16777216))] (u8; elain::Align< 33554432>) => Some( 16777216));
test!(#[repr(C, packed( 33554432))] (u8; elain::Align< 67108864>) => Some( 33554432));
test!(#[repr(C, packed( 67108864))] (u8; elain::Align< 33554432>) => Some( 67108864));
test!(#[repr(C, packed( 33554432))] (u8; elain::Align<134217728>) => Some( 33554432));
test!(#[repr(C, packed(134217728))] (u8; elain::Align<268435456>) => Some(134217728));
test!(#[repr(C, packed(268435456))] (u8; elain::Align<268435456>) => Some(268435456));
*/
// Test that `align(N)` does not limit the offset of the trailing field.
test!(#[repr(C, align( 1))] (u8; elain::Align< 2>) => Some( 2));
test!(#[repr(C, align( 2))] (u8; elain::Align< 4>) => Some( 4));
test!(#[repr(C, align( 4))] (u8; elain::Align< 8>) => Some( 8));
test!(#[repr(C, align( 8))] (u8; elain::Align< 16>) => Some( 16));
test!(#[repr(C, align( 16))] (u8; elain::Align< 32>) => Some( 32));
test!(#[repr(C, align( 32))] (u8; elain::Align< 64>) => Some( 64));
test!(#[repr(C, align( 64))] (u8; elain::Align< 128>) => Some( 128));
test!(#[repr(C, align( 128))] (u8; elain::Align< 256>) => Some( 256));
test!(#[repr(C, align( 256))] (u8; elain::Align< 512>) => Some( 512));
test!(#[repr(C, align( 512))] (u8; elain::Align< 1024>) => Some( 1024));
test!(#[repr(C, align( 1024))] (u8; elain::Align< 2048>) => Some( 2048));
test!(#[repr(C, align( 2048))] (u8; elain::Align< 4096>) => Some( 4096));
test!(#[repr(C, align( 4096))] (u8; elain::Align< 8192>) => Some( 8192));
test!(#[repr(C, align( 8192))] (u8; elain::Align< 16384>) => Some( 16384));
test!(#[repr(C, align( 16384))] (u8; elain::Align< 32768>) => Some( 32768));
test!(#[repr(C, align( 32768))] (u8; elain::Align< 65536>) => Some( 65536));
/* Alignments above 65536 are not yet supported.
test!(#[repr(C, align( 65536))] (u8; elain::Align< 131072>) => Some( 131072));
test!(#[repr(C, align( 131072))] (u8; elain::Align< 262144>) => Some( 262144));
test!(#[repr(C, align( 262144))] (u8; elain::Align< 524288>) => Some( 524288));
test!(#[repr(C, align( 524288))] (u8; elain::Align< 1048576>) => Some( 1048576));
test!(#[repr(C, align( 1048576))] (u8; elain::Align< 2097152>) => Some( 2097152));
test!(#[repr(C, align( 2097152))] (u8; elain::Align< 4194304>) => Some( 4194304));
test!(#[repr(C, align( 4194304))] (u8; elain::Align< 8388608>) => Some( 8388608));
test!(#[repr(C, align( 8388608))] (u8; elain::Align< 16777216>) => Some( 16777216));
test!(#[repr(C, align( 16777216))] (u8; elain::Align< 33554432>) => Some( 33554432));
test!(#[repr(C, align( 33554432))] (u8; elain::Align< 67108864>) => Some( 67108864));
test!(#[repr(C, align( 67108864))] (u8; elain::Align< 33554432>) => Some( 33554432));
test!(#[repr(C, align( 33554432))] (u8; elain::Align<134217728>) => Some(134217728));
test!(#[repr(C, align(134217728))] (u8; elain::Align<268435456>) => Some(268435456));
*/
}
// TODO(#29), TODO(https://github.com/rust-lang/rust/issues/69835): Remove
// this `cfg` when `size_of_val_raw` is stabilized.
#[allow(clippy::decimal_literal_representation)]
#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)]
#[test]
fn test_align_of_dst() {
// Test that `align_of!` correctly computes the alignment of DSTs.
assert_eq!(align_of!([elain::Align<1>]), Some(1));
assert_eq!(align_of!([elain::Align<2>]), Some(2));
assert_eq!(align_of!([elain::Align<4>]), Some(4));
assert_eq!(align_of!([elain::Align<8>]), Some(8));
assert_eq!(align_of!([elain::Align<16>]), Some(16));
assert_eq!(align_of!([elain::Align<32>]), Some(32));
assert_eq!(align_of!([elain::Align<64>]), Some(64));
assert_eq!(align_of!([elain::Align<128>]), Some(128));
assert_eq!(align_of!([elain::Align<256>]), Some(256));
assert_eq!(align_of!([elain::Align<512>]), Some(512));
assert_eq!(align_of!([elain::Align<1024>]), Some(1024));
assert_eq!(align_of!([elain::Align<2048>]), Some(2048));
assert_eq!(align_of!([elain::Align<4096>]), Some(4096));
assert_eq!(align_of!([elain::Align<8192>]), Some(8192));
assert_eq!(align_of!([elain::Align<16384>]), Some(16384));
assert_eq!(align_of!([elain::Align<32768>]), Some(32768));
assert_eq!(align_of!([elain::Align<65536>]), Some(65536));
/* Alignments above 65536 are not yet supported.
assert_eq!(align_of!([elain::Align<131072>]), Some(131072));
assert_eq!(align_of!([elain::Align<262144>]), Some(262144));
assert_eq!(align_of!([elain::Align<524288>]), Some(524288));
assert_eq!(align_of!([elain::Align<1048576>]), Some(1048576));
assert_eq!(align_of!([elain::Align<2097152>]), Some(2097152));
assert_eq!(align_of!([elain::Align<4194304>]), Some(4194304));
assert_eq!(align_of!([elain::Align<8388608>]), Some(8388608));
assert_eq!(align_of!([elain::Align<16777216>]), Some(16777216));
assert_eq!(align_of!([elain::Align<33554432>]), Some(33554432));
assert_eq!(align_of!([elain::Align<67108864>]), Some(67108864));
assert_eq!(align_of!([elain::Align<33554432>]), Some(33554432));
assert_eq!(align_of!([elain::Align<134217728>]), Some(134217728));
assert_eq!(align_of!([elain::Align<268435456>]), Some(268435456));
*/
}
#[test]
fn test_enum_casts() {
// Test that casting the variants of enums with signed integer reprs to
// unsigned integers obeys expected signed -> unsigned casting rules.
#[repr(i8)]
enum ReprI8 {
MinusOne = -1,
Zero = 0,
Min = i8::MIN,
Max = i8::MAX,
}
#[allow(clippy::as_conversions)]
let x = ReprI8::MinusOne as u8;
assert_eq!(x, u8::MAX);
#[allow(clippy::as_conversions)]
let x = ReprI8::Zero as u8;
assert_eq!(x, 0);
#[allow(clippy::as_conversions)]
let x = ReprI8::Min as u8;
assert_eq!(x, 128);
#[allow(clippy::as_conversions)]
let x = ReprI8::Max as u8;
assert_eq!(x, 127);
}
#[test]
fn test_struct_has_padding() {
// Test that, for each provided repr, `struct_has_padding!` reports the
// expected value.
macro_rules! test {
(#[$cfg:meta] ($($ts:ty),*) => $expect:expr) => {{
#[$cfg]
struct Test($(#[allow(dead_code)] $ts),*);
assert_eq!(struct_has_padding!(Test, [$($ts),*]), $expect);
}};
(#[$cfg:meta] $(#[$cfgs:meta])* ($($ts:ty),*) => $expect:expr) => {
test!(#[$cfg] ($($ts),*) => $expect);
test!($(#[$cfgs])* ($($ts),*) => $expect);
};
}
test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] () => false);
test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] (u8) => false);
test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] (u8, ()) => false);
test!(#[repr(C)] #[repr(packed)] (u8, u8) => false);
test!(#[repr(C)] (u8, AU64) => true);
// Rust won't let you put `#[repr(packed)]` on a type which contains a
// `#[repr(align(n > 1))]` type (`AU64`), so we have to use `u64` here.
// It's not ideal, but it definitely has align > 1 on /some/ of our CI
// targets, and this isn't a particularly complex macro we're testing
// anyway.
test!(#[repr(packed)] (u8, u64) => false);
}
#[test]
fn test_union_has_padding() {
// Test that, for each provided repr, `union_has_padding!` reports the
// expected value.
macro_rules! test {
(#[$cfg:meta] {$($fs:ident: $ts:ty),*} => $expect:expr) => {{
#[$cfg]
#[allow(unused)] // fields are never read
union Test{ $($fs: $ts),* }
assert_eq!(union_has_padding!(Test, [$($ts),*]), $expect);
}};
(#[$cfg:meta] $(#[$cfgs:meta])* {$($fs:ident: $ts:ty),*} => $expect:expr) => {
test!(#[$cfg] {$($fs: $ts),*} => $expect);
test!($(#[$cfgs])* {$($fs: $ts),*} => $expect);
};
}
test!(#[repr(C)] #[repr(packed)] {a: u8} => false);
test!(#[repr(C)] #[repr(packed)] {a: u8, b: u8} => false);
// Rust won't let you put `#[repr(packed)]` on a type which contains a
// `#[repr(align(n > 1))]` type (`AU64`), so we have to use `u64` here.
// It's not ideal, but it definitely has align > 1 on /some/ of our CI
// targets, and this isn't a particularly complex macro we're testing
// anyway.
test!(#[repr(C)] #[repr(packed)] {a: u8, b: u64} => true);
}
#[test]
fn test_enum_has_padding() {
// Test that, for each provided repr, `enum_has_padding!` reports the
// expected value.
macro_rules! test {
(#[repr($disc:ident $(, $c:ident)?)] { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => {
test!(@case #[repr($disc $(, $c)?)] { $($vs ($($ts),*),)* } => $expect);
};
(#[repr($disc:ident $(, $c:ident)?)] #[$cfg:meta] $(#[$cfgs:meta])* { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => {
test!(@case #[repr($disc $(, $c)?)] #[$cfg] { $($vs ($($ts),*),)* } => $expect);
test!(#[repr($disc $(, $c)?)] $(#[$cfgs])* { $($vs ($($ts),*),)* } => $expect);
};
(@case #[repr($disc:ident $(, $c:ident)?)] $(#[$cfg:meta])? { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => {{
#[repr($disc $(, $c)?)]
$(#[$cfg])?
#[allow(unused)] // variants and fields are never used
enum Test {
$($vs ($($ts),*),)*
}
assert_eq!(
enum_has_padding!(Test, $disc, $([$($ts),*]),*),
$expect
);
}};
}
#[allow(unused)]
#[repr(align(2))]
struct U16(u16);
#[allow(unused)]
#[repr(align(4))]
struct U32(u32);
test!(#[repr(u8)] #[repr(C)] {
A(u8),
} => false);
test!(#[repr(u16)] #[repr(C)] {
A(u8, u8),
B(U16),
} => false);
test!(#[repr(u32)] #[repr(C)] {
A(u8, u8, u8, u8),
B(U16, u8, u8),
C(u8, u8, U16),
D(U16, U16),
E(U32),
} => false);
// `repr(int)` can pack the discriminant more efficiently
test!(#[repr(u8)] {
A(u8, U16),
} => false);
test!(#[repr(u8)] {
A(u8, U16, U32),
} => false);
// `repr(C)` cannot
test!(#[repr(u8, C)] {
A(u8, U16),
} => true);
test!(#[repr(u8, C)] {
A(u8, u8, u8, U32),
} => true);
// And field ordering can always cause problems
test!(#[repr(u8)] #[repr(C)] {
A(U16, u8),
} => true);
test!(#[repr(u8)] #[repr(C)] {
A(U32, u8, u8, u8),
} => true);
}
}