diff --git a/library/core/src/intrinsics/simd.rs b/library/core/src/intrinsics/simd.rs index 9d5dfb1725862..60694ae20d07f 100644 --- a/library/core/src/intrinsics/simd.rs +++ b/library/core/src/intrinsics/simd.rs @@ -13,6 +13,7 @@ use crate::marker::ConstParamTy; /// `idx` must be in-bounds of the vector. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_insert(x: T, idx: u32, val: U) -> T; /// Extracts an element from a vector. @@ -24,6 +25,7 @@ pub const unsafe fn simd_insert(x: T, idx: u32, val: U) -> T; /// `idx` must be const and in-bounds of the vector. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_extract(x: T, idx: u32) -> U; /// Inserts an element into a vector, returning the updated vector. @@ -37,6 +39,7 @@ pub const unsafe fn simd_extract(x: T, idx: u32) -> U; /// `idx` must be in-bounds of the vector. #[rustc_nounwind] #[rustc_intrinsic] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_insert_dyn(x: T, idx: u32, val: U) -> T; /// Extracts an element from a vector. @@ -50,6 +53,7 @@ pub const unsafe fn simd_insert_dyn(x: T, idx: u32, val: U) -> T; /// `idx` must be in-bounds of the vector. #[rustc_nounwind] #[rustc_intrinsic] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_extract_dyn(x: T, idx: u32) -> U; /// Adds two simd vectors elementwise. @@ -57,6 +61,7 @@ pub const unsafe fn simd_extract_dyn(x: T, idx: u32) -> U; /// `T` must be a vector of integers or floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_add(x: T, y: T) -> T; /// Subtracts `rhs` from `lhs` elementwise. @@ -64,6 +69,7 @@ pub const unsafe fn simd_add(x: T, y: T) -> T; /// `T` must be a vector of integers or floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_sub(lhs: T, rhs: T) -> T; /// Multiplies two simd vectors elementwise. @@ -71,6 +77,7 @@ pub const unsafe fn simd_sub(lhs: T, rhs: T) -> T; /// `T` must be a vector of integers or floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_mul(x: T, y: T) -> T; /// Divides `lhs` by `rhs` elementwise. @@ -82,6 +89,7 @@ pub const unsafe fn simd_mul(x: T, y: T) -> T; /// Additionally for signed integers, `::MIN / -1` is undefined behavior. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_div(lhs: T, rhs: T) -> T; /// Returns remainder of two vectors elementwise. @@ -93,6 +101,7 @@ pub const unsafe fn simd_div(lhs: T, rhs: T) -> T; /// Additionally for signed integers, `::MIN / -1` is undefined behavior. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_rem(lhs: T, rhs: T) -> T; /// Shifts vector left elementwise, with UB on overflow. @@ -106,6 +115,7 @@ pub const unsafe fn simd_rem(lhs: T, rhs: T) -> T; /// Each element of `rhs` must be less than `::BITS`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_shl(lhs: T, rhs: T) -> T; /// Shifts vector right elementwise, with UB on overflow. @@ -119,6 +129,7 @@ pub const unsafe fn simd_shl(lhs: T, rhs: T) -> T; /// Each element of `rhs` must be less than `::BITS`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_shr(lhs: T, rhs: T) -> T; /// Funnel Shifts vector left elementwise, with UB on overflow. @@ -136,6 +147,7 @@ pub const unsafe fn simd_shr(lhs: T, rhs: T) -> T; /// Each element of `shift` must be less than `::BITS`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_funnel_shl(a: T, b: T, shift: T) -> T; /// Funnel Shifts vector right elementwise, with UB on overflow. @@ -153,6 +165,7 @@ pub const unsafe fn simd_funnel_shl(a: T, b: T, shift: T) -> T; /// Each element of `shift` must be less than `::BITS`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_funnel_shr(a: T, b: T, shift: T) -> T; /// "And"s vectors elementwise. @@ -160,6 +173,7 @@ pub const unsafe fn simd_funnel_shr(a: T, b: T, shift: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_and(x: T, y: T) -> T; /// "Ors" vectors elementwise. @@ -167,6 +181,7 @@ pub const unsafe fn simd_and(x: T, y: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_or(x: T, y: T) -> T; /// "Exclusive ors" vectors elementwise. @@ -174,6 +189,7 @@ pub const unsafe fn simd_or(x: T, y: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_xor(x: T, y: T) -> T; /// Numerically casts a vector, elementwise. @@ -195,6 +211,7 @@ pub const unsafe fn simd_xor(x: T, y: T) -> T; /// * Be representable in the return type, after truncating off its fractional part #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_cast(x: T) -> U; /// Numerically casts a vector, elementwise. @@ -209,6 +226,7 @@ pub const unsafe fn simd_cast(x: T) -> U; /// Otherwise, truncates or extends the value, maintaining the sign for signed integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_as(x: T) -> U; /// Negates a vector elementwise. @@ -218,6 +236,7 @@ pub const unsafe fn simd_as(x: T) -> U; /// Rust panics for `-::Min` due to overflow, but it is not UB with this intrinsic. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_neg(x: T) -> T; /// Returns absolute value of a vector, elementwise. @@ -225,6 +244,7 @@ pub const unsafe fn simd_neg(x: T) -> T; /// `T` must be a vector of floating-point primitive types. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_fabs(x: T) -> T; /// Returns the minimum of two vectors, elementwise. @@ -254,6 +274,7 @@ pub const unsafe fn simd_fmax(x: T, y: T) -> T; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_eq(x: T, y: T) -> U; /// Tests elementwise inequality equality of two vectors. @@ -265,6 +286,7 @@ pub const unsafe fn simd_eq(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ne(x: T, y: T) -> U; /// Tests if `x` is less than `y`, elementwise. @@ -276,6 +298,7 @@ pub const unsafe fn simd_ne(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_lt(x: T, y: T) -> U; /// Tests if `x` is less than or equal to `y`, elementwise. @@ -287,6 +310,7 @@ pub const unsafe fn simd_lt(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_le(x: T, y: T) -> U; /// Tests if `x` is greater than `y`, elementwise. @@ -298,6 +322,7 @@ pub const unsafe fn simd_le(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_gt(x: T, y: T) -> U; /// Tests if `x` is greater than or equal to `y`, elementwise. @@ -309,6 +334,7 @@ pub const unsafe fn simd_gt(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ge(x: T, y: T) -> U; /// Shuffles two vectors by const indices. @@ -325,6 +351,7 @@ pub const unsafe fn simd_ge(x: T, y: T) -> U; /// of `xy`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_shuffle(x: T, y: T, idx: U) -> V; /// Reads a vector of pointers. @@ -346,6 +373,7 @@ pub const unsafe fn simd_shuffle(x: T, y: T, idx: U) -> V; /// `mask` must only contain `0` or `!0` values. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_gather(val: T, ptr: U, mask: V) -> T; /// Writes to a vector of pointers. @@ -370,6 +398,7 @@ pub const unsafe fn simd_gather(val: T, ptr: U, mask: V) -> T; /// `mask` must only contain `0` or `!0` values. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_scatter(val: T, ptr: U, mask: V); /// A type for alignment options for SIMD masked load/store intrinsics. @@ -405,6 +434,7 @@ pub enum SimdAlign { /// `mask` must only contain `0` or `!0` values. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_masked_load(mask: V, ptr: U, val: T) -> T; @@ -427,6 +457,7 @@ pub const unsafe fn simd_masked_load(mask: V, p /// `mask` must only contain `0` or `!0` values. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_masked_store(mask: V, ptr: U, val: T); /// Adds two simd vectors elementwise, with saturation. @@ -434,6 +465,7 @@ pub const unsafe fn simd_masked_store(mask: V, /// `T` must be a vector of integer primitive types. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_saturating_add(x: T, y: T) -> T; /// Subtracts two simd vectors elementwise, with saturation. @@ -443,6 +475,7 @@ pub const unsafe fn simd_saturating_add(x: T, y: T) -> T; /// Subtract `rhs` from `lhs`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_saturating_sub(lhs: T, rhs: T) -> T; /// Adds elements within a vector from left to right. @@ -454,6 +487,7 @@ pub const unsafe fn simd_saturating_sub(lhs: T, rhs: T) -> T; /// Starting with the value `y`, add the elements of `x` and accumulate. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_add_ordered(x: T, y: U) -> U; /// Adds elements within a vector in arbitrary order. May also be re-associated with @@ -475,6 +509,7 @@ pub unsafe fn simd_reduce_add_unordered(x: T) -> U; /// Starting with the value `y`, multiply the elements of `x` and accumulate. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_mul_ordered(x: T, y: U) -> U; /// Multiplies elements within a vector in arbitrary order. May also be re-associated with @@ -495,6 +530,7 @@ pub unsafe fn simd_reduce_mul_unordered(x: T) -> U; /// `x` must contain only `0` or `!0`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_all(x: T) -> bool; /// Checks if any mask value is true. @@ -505,6 +541,7 @@ pub const unsafe fn simd_reduce_all(x: T) -> bool; /// `x` must contain only `0` or `!0`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_any(x: T) -> bool; /// Returns the maximum element of a vector. @@ -536,6 +573,7 @@ pub const unsafe fn simd_reduce_min(x: T) -> U; /// `U` must be the element type of `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_and(x: T) -> U; /// Logical "ors" all elements together. @@ -545,6 +583,7 @@ pub const unsafe fn simd_reduce_and(x: T) -> U; /// `U` must be the element type of `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_or(x: T) -> U; /// Logical "exclusive ors" all elements together. @@ -554,6 +593,7 @@ pub const unsafe fn simd_reduce_or(x: T) -> U; /// `U` must be the element type of `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_xor(x: T) -> U; /// Truncates an integer vector to a bitmask. @@ -591,6 +631,7 @@ pub const unsafe fn simd_reduce_xor(x: T) -> U; /// `x` must contain only `0` and `!0`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_bitmask(x: T) -> U; /// Selects elements from a mask. @@ -607,6 +648,7 @@ pub const unsafe fn simd_bitmask(x: T) -> U; /// `mask` must only contain `0` and `!0`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_select(mask: M, if_true: T, if_false: T) -> T; /// Selects elements from a bitmask. @@ -623,6 +665,7 @@ pub const unsafe fn simd_select(mask: M, if_true: T, if_false: T) -> T; /// The bitmask bit order matches `simd_bitmask`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_select_bitmask(m: M, yes: T, no: T) -> T; /// Calculates the offset from a pointer vector elementwise, potentially @@ -635,6 +678,7 @@ pub const unsafe fn simd_select_bitmask(m: M, yes: T, no: T) -> T; /// Operates as if by `::wrapping_offset`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_arith_offset(ptr: T, offset: U) -> T; /// Casts a vector of pointers. @@ -642,6 +686,7 @@ pub const unsafe fn simd_arith_offset(ptr: T, offset: U) -> T; /// `T` and `U` must be vectors of pointers with the same number of elements. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_cast_ptr(ptr: T) -> U; /// Exposes a vector of pointers as a vector of addresses. @@ -660,6 +705,7 @@ pub unsafe fn simd_expose_provenance(ptr: T) -> U; /// `U` must be a vector of pointers, with the same length as `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_with_exposed_provenance(addr: T) -> U; /// Swaps bytes of each element. @@ -667,6 +713,7 @@ pub const unsafe fn simd_with_exposed_provenance(addr: T) -> U; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_bswap(x: T) -> T; /// Reverses bits of each element. @@ -674,6 +721,7 @@ pub const unsafe fn simd_bswap(x: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_bitreverse(x: T) -> T; /// Counts the leading zeros of each element. @@ -681,6 +729,7 @@ pub const unsafe fn simd_bitreverse(x: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ctlz(x: T) -> T; /// Counts the number of ones in each element. @@ -688,6 +737,7 @@ pub const unsafe fn simd_ctlz(x: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ctpop(x: T) -> T; /// Counts the trailing zeros of each element. @@ -695,6 +745,7 @@ pub const unsafe fn simd_ctpop(x: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_cttz(x: T) -> T; /// Rounds up each element to the next highest integer-valued float. @@ -702,6 +753,7 @@ pub const unsafe fn simd_cttz(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ceil(x: T) -> T; /// Rounds down each element to the next lowest integer-valued float. @@ -709,6 +761,7 @@ pub const unsafe fn simd_ceil(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_floor(x: T) -> T; /// Rounds each element to the closest integer-valued float. @@ -717,6 +770,7 @@ pub const unsafe fn simd_floor(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_round(x: T) -> T; /// Rounds each element to the closest integer-valued float. @@ -725,6 +779,7 @@ pub const unsafe fn simd_round(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_round_ties_even(x: T) -> T; /// Returns the integer part of each element as an integer-valued float. @@ -733,6 +788,7 @@ pub const unsafe fn simd_round_ties_even(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_trunc(x: T) -> T; /// Takes the square root of each element. @@ -747,6 +803,7 @@ pub unsafe fn simd_fsqrt(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_fma(x: T, y: T, z: T) -> T; /// Computes `(x*y) + z` for each element, non-deterministically executing either