Skip to main content

portable_atomic/
utils.rs

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3#![cfg_attr(not(all(test, feature = "float")), allow(dead_code, unused_macros))]
4
5#[allow(unused_imports)]
6pub(crate) use self::generated::{RegISize, RegSize};
7#[macro_use]
8#[path = "gen/utils.rs"]
9mod generated;
10
11use core::sync::atomic::Ordering;
12
13macro_rules! static_assert {
14    ($cond:expr $(,)?) => {{
15        let [()] = [(); (true /* type check */ & $cond) as usize];
16    }};
17}
18
19macro_rules! static_assert_layout {
20    ($atomic_type:ty, $value_type:ty) => {
21        static_assert!(
22            core::mem::align_of::<$atomic_type>() == core::mem::size_of::<$atomic_type>()
23        );
24        static_assert!(core::mem::size_of::<$atomic_type>() == core::mem::size_of::<$value_type>());
25    };
26}
27
28// #[doc = concat!(...)] requires Rust 1.54
29macro_rules! doc_comment {
30    ($doc:expr, $($tt:tt)*) => {
31        #[doc = $doc]
32        $($tt)*
33    };
34}
35
36// Adapted from https://github.com/BurntSushi/memchr/blob/2.4.1/src/memchr/x86/mod.rs#L9-L71.
37/// # Safety
38///
39/// - the caller must uphold the safety contract for the function returned by $detect_body.
40/// - the memory pointed by the function pointer returned by $detect_body must be visible from any threads.
41///
42/// The second requirement is always met if the function pointer is to the function definition.
43/// (Currently, all uses of this macro in our code are in this case.)
44#[allow(unused_macros)]
45#[cfg(not(portable_atomic_no_outline_atomics))]
46#[cfg(any(
47    target_arch = "aarch64",
48    target_arch = "arm",
49    target_arch = "arm64ec",
50    target_arch = "powerpc64",
51    target_arch = "riscv32",
52    target_arch = "riscv64",
53    all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
54))]
55macro_rules! ifunc {
56    (unsafe fn($($arg_pat:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)? { $($init_body:tt)* }) => {{
57        type FnTy = unsafe fn($($arg_ty),*) $(-> $ret_ty)?;
58        static FUNC: core::sync::atomic::AtomicPtr<()>
59            = core::sync::atomic::AtomicPtr::new(init as *mut ());
60        #[cold]
61        unsafe fn init($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
62            let func: FnTy = { $($init_body)* };
63            FUNC.store(func as *mut (), core::sync::atomic::Ordering::Relaxed);
64            // SAFETY: the caller must uphold the safety contract for the function returned by $init_body.
65            unsafe { func($($arg_pat),*) }
66        }
67        // SAFETY: `FnTy` is a function pointer, which is always safe to transmute with a `*mut ()`.
68        // (To force the caller to use unsafe block for this macro, do not use
69        // unsafe block here.)
70        let func = {
71            core::mem::transmute::<*mut (), FnTy>(FUNC.load(core::sync::atomic::Ordering::Relaxed))
72        };
73        // SAFETY: the caller must uphold the safety contract for the function returned by $init_body.
74        // (To force the caller to use unsafe block for this macro, do not use
75        // unsafe block here.)
76        func($($arg_pat),*)
77    }};
78}
79
80#[cfg(not(portable_atomic_no_asm))]
81#[allow(unused_macros)]
82macro_rules! __asm {
83    ($($tt:tt)*) => {
84        core::arch::asm!($($tt)*)
85    };
86}
87#[cfg(portable_atomic_no_asm)]
88#[allow(unused_macros)]
89macro_rules! __asm {
90    ($($tt:tt)*) => {
91        asm!($($tt)*)
92    };
93}
94
95#[allow(unused_macros)]
96#[cfg(not(portable_atomic_no_outline_atomics))]
97#[cfg(any(
98    target_arch = "aarch64",
99    target_arch = "arm",
100    target_arch = "arm64ec",
101    target_arch = "powerpc64",
102    target_arch = "riscv32",
103    target_arch = "riscv64",
104    all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
105))]
106macro_rules! fn_alias {
107    (
108        $(#[$($fn_attr:tt)*])*
109        $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
110        $(#[$($alias_attr:tt)*])*
111        $new:ident = $from:ident($($last_args:tt)*);
112        $($rest:tt)*
113    ) => {
114        $(#[$($fn_attr)*])*
115        $(#[$($alias_attr)*])*
116        $vis unsafe fn $new($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
117            // SAFETY: the caller must uphold the safety contract.
118            unsafe { $from($($arg_pat,)* $($last_args)*) }
119        }
120        fn_alias! {
121            $(#[$($fn_attr)*])*
122            $vis unsafe fn($($arg_pat: $arg_ty),*) $(-> $ret_ty)?;
123            $($rest)*
124        }
125    };
126    (
127        $(#[$($attr:tt)*])*
128        $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
129    ) => {}
130}
131
132/// Make the given function const if the given condition is true.
133macro_rules! const_fn {
134    (
135        const_if: #[cfg($($cfg:tt)+)];
136        $(#[$($attr:tt)*])*
137        $vis:vis const $($rest:tt)*
138    ) => {
139        #[cfg($($cfg)+)]
140        $(#[$($attr)*])*
141        $vis const $($rest)*
142        #[cfg(not($($cfg)+))]
143        $(#[$($attr)*])*
144        $vis $($rest)*
145    };
146}
147
148/// Implements `core::fmt::Debug` and `serde::{Serialize, Deserialize}` (when serde
149/// feature is enabled) for atomic bool, integer, or float.
150macro_rules! impl_debug_and_serde {
151    // TODO(f16_and_f128): Implement serde traits for f16 & f128 once stabilized.
152    (AtomicF16) => {
153        impl_debug!(AtomicF16);
154    };
155    (AtomicF128) => {
156        impl_debug!(AtomicF128);
157    };
158    ($atomic_type:ident) => {
159        impl_debug!($atomic_type);
160        #[cfg(feature = "serde")]
161        #[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
162        impl serde::ser::Serialize for $atomic_type {
163            #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl
164            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
165            where
166                S: serde::ser::Serializer,
167            {
168                // https://github.com/serde-rs/serde/blob/v1.0.152/serde/src/ser/impls.rs#L958-L959
169                self.load(Ordering::Relaxed).serialize(serializer)
170            }
171        }
172        #[cfg(feature = "serde")]
173        #[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
174        impl<'de> serde::de::Deserialize<'de> for $atomic_type {
175            #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl
176            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
177            where
178                D: serde::de::Deserializer<'de>,
179            {
180                serde::de::Deserialize::deserialize(deserializer).map(Self::new)
181            }
182        }
183    };
184}
185macro_rules! impl_debug {
186    ($atomic_type:ident) => {
187        impl fmt::Debug for $atomic_type {
188            #[inline] // fmt is not hot path, but #[inline] on fmt seems to still be useful: https://github.com/rust-lang/rust/pull/117727
189            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
190                // std atomic types use Relaxed in Debug::fmt: https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L2188
191                fmt::Debug::fmt(&self.load(Ordering::Relaxed), f)
192            }
193        }
194    };
195}
196
197// We do not provide `nand` because it cannot be optimized on neither x86 nor MSP430.
198// https://godbolt.org/z/ahWejchbT
199macro_rules! impl_default_no_fetch_ops {
200    ($atomic_type:ident, bool) => {
201        impl $atomic_type {
202            #[inline]
203            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
204            pub(crate) fn and(&self, val: bool, order: Ordering) {
205                self.fetch_and(val, order);
206            }
207            #[inline]
208            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
209            pub(crate) fn or(&self, val: bool, order: Ordering) {
210                self.fetch_or(val, order);
211            }
212            #[inline]
213            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
214            pub(crate) fn xor(&self, val: bool, order: Ordering) {
215                self.fetch_xor(val, order);
216            }
217        }
218    };
219    ($atomic_type:ident, $int_type:ty) => {
220        impl $atomic_type {
221            #[inline]
222            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
223            pub(crate) fn add(&self, val: $int_type, order: Ordering) {
224                self.fetch_add(val, order);
225            }
226            #[inline]
227            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
228            pub(crate) fn sub(&self, val: $int_type, order: Ordering) {
229                self.fetch_sub(val, order);
230            }
231            #[inline]
232            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
233            pub(crate) fn and(&self, val: $int_type, order: Ordering) {
234                self.fetch_and(val, order);
235            }
236            #[inline]
237            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
238            pub(crate) fn or(&self, val: $int_type, order: Ordering) {
239                self.fetch_or(val, order);
240            }
241            #[inline]
242            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
243            pub(crate) fn xor(&self, val: $int_type, order: Ordering) {
244                self.fetch_xor(val, order);
245            }
246        }
247    };
248}
249macro_rules! impl_default_bit_opts {
250    (AtomicPtr, $int_type:ty) => {
251        impl<T> AtomicPtr<T> {
252            #[inline]
253            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
254            pub(crate) fn bit_set(&self, bit: u32, order: Ordering) -> bool {
255                #[cfg(portable_atomic_no_strict_provenance)]
256                use crate::utils::ptr::PtrExt as _;
257                let mask = <$int_type>::wrapping_shl(1, bit);
258                self.fetch_or(mask, order).addr() & mask != 0
259            }
260            #[inline]
261            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
262            pub(crate) fn bit_clear(&self, bit: u32, order: Ordering) -> bool {
263                #[cfg(portable_atomic_no_strict_provenance)]
264                use crate::utils::ptr::PtrExt as _;
265                let mask = <$int_type>::wrapping_shl(1, bit);
266                self.fetch_and(!mask, order).addr() & mask != 0
267            }
268            #[inline]
269            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
270            pub(crate) fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {
271                #[cfg(portable_atomic_no_strict_provenance)]
272                use crate::utils::ptr::PtrExt as _;
273                let mask = <$int_type>::wrapping_shl(1, bit);
274                self.fetch_xor(mask, order).addr() & mask != 0
275            }
276        }
277    };
278    ($atomic_type:ident, $int_type:ty) => {
279        impl $atomic_type {
280            #[inline]
281            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
282            pub(crate) fn bit_set(&self, bit: u32, order: Ordering) -> bool {
283                let mask = <$int_type>::wrapping_shl(1, bit);
284                self.fetch_or(mask, order) & mask != 0
285            }
286            #[inline]
287            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
288            pub(crate) fn bit_clear(&self, bit: u32, order: Ordering) -> bool {
289                let mask = <$int_type>::wrapping_shl(1, bit);
290                self.fetch_and(!mask, order) & mask != 0
291            }
292            #[inline]
293            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
294            pub(crate) fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {
295                let mask = <$int_type>::wrapping_shl(1, bit);
296                self.fetch_xor(mask, order) & mask != 0
297            }
298        }
299    };
300}
301
302// This just outputs the input as is, but can be used like an item-level block by using it with cfg.
303// Note: This macro is items!({ }), not items! { }.
304// An extra brace is used in input to make contents rustfmt-able.
305macro_rules! items {
306    ({$($tt:tt)*}) => {
307        $($tt)*
308    };
309}
310
311// rustfmt-compatible cfg_select/cfg_if alternative
312// Note: This macro is cfg_sel!({ }), not cfg_sel! { }.
313// An extra brace is used in input to make contents rustfmt-able.
314macro_rules! cfg_sel {
315    ({#[cfg(else)] { $($output:tt)* }}) => {
316        $($output)*
317    };
318    ({
319        #[cfg($cfg:meta)]
320        { $($output:tt)* }
321        $($( $rest:tt )+)?
322    }) => {
323        #[cfg($cfg)]
324        cfg_sel! {{#[cfg(else)] { $($output)* }}}
325        $(
326            #[cfg(not($cfg))]
327            cfg_sel! {{ $($rest)+ }}
328        )?
329    };
330    ({
331        #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg($cfg1:meta))]
332        #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg($cfg2:meta))]
333        { $($output:tt)* }
334        $($( $rest:tt )+)?
335    }) => {
336        #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg($cfg1))]
337        #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg($cfg2))]
338        cfg_sel! {{#[cfg(else)] { $($output)* }}}
339        $(
340            #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not($cfg1)))]
341            #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not($cfg2)))]
342            cfg_sel! {{ $($rest)+ }}
343        )?
344    };
345}
346
347// Equivalent to core::hint::cold_path, but compatible with pre-1.95 rustc.
348#[allow(dead_code)]
349#[inline(always)]
350#[cold]
351fn cold_path() {}
352// Stable equivalent of core::hint::{likely, unlikely}.
353#[allow(dead_code)]
354#[inline(always)]
355pub(crate) fn likely(b: bool) -> bool {
356    if b {
357        true
358    } else {
359        cold_path();
360        false
361    }
362}
363#[allow(dead_code)]
364#[inline(always)]
365pub(crate) fn unlikely(b: bool) -> bool {
366    if b {
367        cold_path();
368        true
369    } else {
370        false
371    }
372}
373
374// Equivalent to core::hint::assert_unchecked, but compatible with pre-1.81 rustc.
375#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
376#[allow(dead_code)]
377#[inline(always)]
378#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
379pub(crate) unsafe fn assert_unchecked(cond: bool) {
380    if !cond {
381        #[cfg(debug_assertions)]
382        unreachable!();
383        #[cfg(not(debug_assertions))]
384        // SAFETY: the caller promised `cond` is true.
385        unsafe {
386            core::hint::unreachable_unchecked()
387        }
388    }
389}
390
391// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3338
392#[inline]
393#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
394pub(crate) fn assert_load_ordering(order: Ordering) {
395    match order {
396        Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
397        Ordering::Release => panic!("there is no such thing as a release load"),
398        Ordering::AcqRel => panic!("there is no such thing as an acquire-release load"),
399        _ => unreachable!(),
400    }
401}
402// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3323
403#[inline]
404#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
405pub(crate) fn assert_store_ordering(order: Ordering) {
406    match order {
407        Ordering::Release | Ordering::Relaxed | Ordering::SeqCst => {}
408        Ordering::Acquire => panic!("there is no such thing as an acquire store"),
409        Ordering::AcqRel => panic!("there is no such thing as an acquire-release store"),
410        _ => unreachable!(),
411    }
412}
413// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3404
414#[inline]
415#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
416pub(crate) fn assert_compare_exchange_ordering(success: Ordering, failure: Ordering) {
417    match success {
418        Ordering::AcqRel
419        | Ordering::Acquire
420        | Ordering::Relaxed
421        | Ordering::Release
422        | Ordering::SeqCst => {}
423        _ => unreachable!(),
424    }
425    match failure {
426        Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
427        Ordering::Release => panic!("there is no such thing as a release failure ordering"),
428        Ordering::AcqRel => panic!("there is no such thing as an acquire-release failure ordering"),
429        _ => unreachable!(),
430    }
431}
432
433// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0418r2.html
434// https://github.com/rust-lang/rust/pull/98383
435#[allow(dead_code)]
436#[inline]
437pub(crate) fn upgrade_success_ordering(success: Ordering, failure: Ordering) -> Ordering {
438    match (success, failure) {
439        (Ordering::Relaxed, Ordering::Acquire) => Ordering::Acquire,
440        (Ordering::Release, Ordering::Acquire) => Ordering::AcqRel,
441        (_, Ordering::SeqCst) => Ordering::SeqCst,
442        _ => success,
443    }
444}
445
446#[cfg(not(portable_atomic_no_asm_maybe_uninit))]
447#[cfg(target_pointer_width = "32")]
448// SAFETY: MaybeUninit returned by zero_extend64_ptr is always initialized.
449const _: () = assert!(unsafe {
450    zero_extend64_ptr(ptr::without_provenance_mut(!0)).assume_init() == !0_u32 as u64
451});
452/// Zero-extends the given 32-bit pointer to `MaybeUninit<u64>`.
453/// This is used for 64-bit architecture's 32-bit ABI (e.g., AArch64 ILP32 ABI).
454/// See ptr_reg! macro in src/gen/utils.rs for details.
455#[cfg(not(portable_atomic_no_asm_maybe_uninit))]
456#[cfg(target_pointer_width = "32")]
457#[allow(dead_code)]
458#[inline]
459pub(crate) const fn zero_extend64_ptr(v: *mut ()) -> core::mem::MaybeUninit<u64> {
460    #[repr(C)]
461    struct ZeroExtended {
462        #[cfg(target_endian = "big")]
463        pad: *mut (),
464        v: *mut (),
465        #[cfg(target_endian = "little")]
466        pad: *mut (),
467    }
468    // SAFETY: we can safely transmute any 64-bit value to MaybeUninit<u64>.
469    unsafe { core::mem::transmute(ZeroExtended { v, pad: core::ptr::null_mut() }) }
470}
471
472#[allow(dead_code)]
473#[cfg(any(
474    target_arch = "aarch64",
475    target_arch = "arm64ec",
476    target_arch = "powerpc64",
477    target_arch = "riscv64",
478    target_arch = "s390x",
479    target_arch = "x86_64",
480))]
481/// A 128-bit value represented as a pair of 64-bit values.
482///
483/// This type is `#[repr(C)]`, both fields have the same in-memory representation
484/// and are plain old data types, so access to the fields is always safe.
485#[derive(Clone, Copy)]
486#[repr(C)]
487pub(crate) union U128 {
488    pub(crate) whole: u128,
489    pub(crate) pair: Pair<u64>,
490}
491#[allow(dead_code)]
492#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
493/// A 64-bit value represented as a pair of 32-bit values.
494///
495/// This type is `#[repr(C)]`, both fields have the same in-memory representation
496/// and are plain old data types, so access to the fields is always safe.
497#[derive(Clone, Copy)]
498#[repr(C)]
499pub(crate) union U64 {
500    pub(crate) whole: u64,
501    pub(crate) pair: Pair<u32>,
502}
503#[allow(dead_code)]
504#[derive(Clone, Copy)]
505#[repr(C)]
506pub(crate) struct Pair<T: Copy> {
507    // little endian order
508    #[cfg(any(
509        target_endian = "little",
510        target_arch = "aarch64",
511        target_arch = "arm",
512        target_arch = "arm64ec",
513    ))]
514    pub(crate) lo: T,
515    pub(crate) hi: T,
516    // big endian order
517    #[cfg(not(any(
518        target_endian = "little",
519        target_arch = "aarch64",
520        target_arch = "arm",
521        target_arch = "arm64ec",
522    )))]
523    pub(crate) lo: T,
524}
525
526#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
527type MinWord = u32;
528#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
529type RetInt = u32;
530// Adapted from https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.6/src/utils.rs#L255.
531// Helper for implementing sub-word atomic operations using word-sized LL/SC loop or CAS loop.
532//
533// Refs: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/CodeGen/AtomicExpandPass.cpp#L811
534// (aligned_ptr, shift, mask)
535#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
536#[allow(dead_code)]
537#[inline]
538pub(crate) fn create_sub_word_mask_values<T>(ptr: *mut T) -> (*mut MinWord, RetInt, RetInt) {
539    use core::mem;
540
541    #[cfg(portable_atomic_no_strict_provenance)]
542    use self::ptr::PtrExt as _;
543
544    // RISC-V, MIPS, SPARC, LoongArch, Xtensa, BPF: shift amount of 32-bit shift instructions is 5 bits unsigned (0-31).
545    // PowerPC, C-SKY: shift amount of 32-bit shift instructions is 6 bits unsigned (0-63) and shift amount 32-63 means "clear".
546    // Arm: shift amount of 32-bit shift instructions is 8 bits unsigned (0-255).
547    // Hexagon: shift amount of 32-bit shift instructions is 7 bits signed (-64-63) and negative shift amount means "reverse the direction of the shift".
548    // (On s390x, we don't use the mask returned from this function.)
549    // (See also https://devblogs.microsoft.com/oldnewthing/20230904-00/?p=108704 for others)
550    const SHIFT_MASK: bool = !cfg!(any(
551        target_arch = "bpf",
552        target_arch = "loongarch32",
553        target_arch = "loongarch64",
554        target_arch = "mips",
555        target_arch = "mips32r6",
556        target_arch = "mips64",
557        target_arch = "mips64r6",
558        target_arch = "riscv32",
559        target_arch = "riscv64",
560        target_arch = "s390x",
561        target_arch = "sparc",
562        target_arch = "sparc64",
563        target_arch = "xtensa",
564    ));
565    let ptr_mask = mem::size_of::<MinWord>() - 1;
566    let aligned_ptr = ptr.with_addr(ptr.addr() & !ptr_mask) as *mut MinWord;
567    let ptr_lsb = if SHIFT_MASK {
568        ptr.addr() & ptr_mask
569    } else {
570        // We use 32-bit wrapping shift instructions in asm on these platforms.
571        ptr.addr()
572    };
573    let shift = if cfg!(any(target_endian = "little", target_arch = "s390x")) {
574        ptr_lsb.wrapping_mul(8)
575    } else {
576        (ptr_lsb ^ (mem::size_of::<MinWord>() - mem::size_of::<T>())).wrapping_mul(8)
577    };
578    let mut mask: RetInt = (1 << (mem::size_of::<T>() * 8)) - 1; // !(0 as T) as RetInt
579    if SHIFT_MASK {
580        mask <<= shift;
581    }
582    #[allow(clippy::cast_possible_truncation)]
583    {
584        (aligned_ptr, shift as RetInt, mask)
585    }
586}
587
588// This module provides core::ptr strict_provenance/exposed_provenance polyfill for pre-1.84 rustc.
589#[allow(dead_code)]
590pub(crate) mod ptr {
591    cfg_sel!({
592        #[cfg(not(portable_atomic_no_strict_provenance))]
593        {
594            #[allow(unused_imports)]
595            pub(crate) use core::ptr::{
596                with_exposed_provenance, with_exposed_provenance_mut, without_provenance_mut,
597            };
598        }
599        #[cfg(else)]
600        {
601            #[inline(always)]
602            #[must_use]
603            pub(crate) const fn without_provenance_mut<T>(addr: usize) -> *mut T {
604                // An int-to-pointer transmute currently has exactly the intended semantics: it creates a
605                // pointer without provenance. Note that this is *not* a stable guarantee about transmute
606                // semantics, it relies on sysroot crates having special status.
607                // SAFETY: every valid integer is also a valid pointer (as long as you don't dereference that
608                // pointer).
609                #[cfg(miri)]
610                unsafe {
611                    core::mem::transmute(addr)
612                }
613                // const transmute requires Rust 1.56.
614                // Using transmute doesn't work with CHERI: https://github.com/kent-weak-memory/rust/blob/0c0ca909de877f889629057e1ddf139527446d75/library/core/src/ptr/mod.rs#L607
615                #[cfg(not(miri))]
616                {
617                    addr as *mut T
618                }
619            }
620            #[inline(always)]
621            #[must_use]
622            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
623            pub(crate) fn with_exposed_provenance<T>(addr: usize) -> *const T {
624                addr as *const T
625            }
626            #[inline(always)]
627            #[must_use]
628            #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
629            pub(crate) fn with_exposed_provenance_mut<T>(addr: usize) -> *mut T {
630                addr as *mut T
631            }
632
633            pub(crate) trait PtrExt<T: ?Sized>: Copy {
634                #[must_use]
635                fn addr(self) -> usize;
636                #[must_use]
637                fn with_addr(self, addr: usize) -> Self
638                where
639                    T: Sized;
640            }
641            impl<T: ?Sized> PtrExt<T> for *mut T {
642                #[inline(always)]
643                #[must_use]
644                fn addr(self) -> usize {
645                    // A pointer-to-integer transmute currently has exactly the right semantics: it returns the
646                    // address without exposing the provenance. Note that this is *not* a stable guarantee about
647                    // transmute semantics, it relies on sysroot crates having special status.
648                    // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the
649                    // provenance).
650                    #[cfg(miri)]
651                    unsafe {
652                        core::mem::transmute(self as *mut ())
653                    }
654                    // Using transmute doesn't work with CHERI: https://github.com/kent-weak-memory/rust/blob/0c0ca909de877f889629057e1ddf139527446d75/library/core/src/ptr/mut_ptr.rs#L210
655                    #[cfg(not(miri))]
656                    {
657                        self as *mut () as usize
658                    }
659                }
660                #[inline]
661                #[must_use]
662                fn with_addr(self, addr: usize) -> Self
663                where
664                    T: Sized,
665                {
666                    // This should probably be an intrinsic to avoid doing any sort of arithmetic, but
667                    // meanwhile, we can implement it with `wrapping_offset`, which preserves the pointer's
668                    // provenance.
669                    let self_addr = self.addr() as isize;
670                    let dest_addr = addr as isize;
671                    let offset = dest_addr.wrapping_sub(self_addr);
672                    (self as *mut u8).wrapping_offset(offset) as *mut T
673                }
674            }
675        }
676    });
677}
678
679// This module provides:
680// - core::ffi polyfill (c_* type aliases and CStr) for pre-1.64 rustc compatibility.
681//   (core::ffi::* (except c_void) requires Rust 1.64)
682// - Safe abstraction (c! macro) for creating static C strings without runtime checks.
683//   (c"..." requires Rust 1.77)
684// - Helper macros for defining FFI bindings with static signature/type/value assertions.
685// - Helper macros for defining asm-based syscalls on Linux.
686#[cfg(any(
687    test,
688    portable_atomic_test_no_std_static_assert_ffi,
689    not(any(target_arch = "x86", target_arch = "x86_64"))
690))]
691#[cfg(any(not(portable_atomic_no_asm), portable_atomic_unstable_asm))]
692#[allow(dead_code, non_camel_case_types, unused_macros)]
693#[macro_use]
694pub(crate) mod ffi {
695    // -------------------------------------------------------------------------
696    // core::ffi polyfill and c"..." equivalent
697
698    pub(crate) type c_void = core::ffi::c_void;
699    // c_{,u}int is {i,u}16 on 16-bit targets, otherwise {i,u}32.
700    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L156
701    cfg_sel!({
702        #[cfg(target_pointer_width = "16")]
703        {
704            pub(crate) type c_int = i16;
705            pub(crate) type c_uint = u16;
706        }
707        #[cfg(else)]
708        {
709            pub(crate) type c_int = i32;
710            pub(crate) type c_uint = u32;
711        }
712    });
713    // c_{,u}long is {i,u}64 on non-Windows 64-bit targets, otherwise {i,u}32.
714    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L168
715    cfg_sel!({
716        #[cfg(all(target_pointer_width = "64", not(windows)))]
717        {
718            pub(crate) type c_long = i64;
719            pub(crate) type c_ulong = u64;
720        }
721        #[cfg(else)]
722        {
723            pub(crate) type c_long = i32;
724            pub(crate) type c_ulong = u32;
725        }
726    });
727    // c_size_t is currently always usize.
728    // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L76
729    pub(crate) type c_size_t = usize;
730    // c_char is u8 by default on non-Apple/non-Windows/non-Vita Arm/C-SKY/Hexagon/MSP430/PowerPC/RISC-V/s390x/Xtensa targets, otherwise i8 by default.
731    // See references in https://github.com/rust-lang/rust/issues/129945 for details.
732    cfg_sel!({
733        #[cfg(all(
734            not(any(target_vendor = "apple", windows, target_os = "vita")),
735            any(
736                target_arch = "aarch64",
737                target_arch = "arm",
738                target_arch = "csky",
739                target_arch = "hexagon",
740                target_arch = "msp430",
741                target_arch = "powerpc",
742                target_arch = "powerpc64",
743                target_arch = "riscv32",
744                target_arch = "riscv64",
745                target_arch = "s390x",
746                target_arch = "xtensa",
747            ),
748        ))]
749        {
750            pub(crate) type c_char = u8;
751        }
752        #[cfg(else)]
753        {
754            pub(crate) type c_char = i8;
755        }
756    });
757
758    // Static assertions for C type definitions.
759    #[cfg(test)]
760    const _: fn() = || {
761        let _: c_int = 0 as std::os::raw::c_int;
762        let _: c_uint = 0 as std::os::raw::c_uint;
763        let _: c_long = 0 as std::os::raw::c_long;
764        let _: c_ulong = 0 as std::os::raw::c_ulong;
765        #[cfg(unix)]
766        let _: c_size_t = 0 as libc::size_t; // std::os::raw::c_size_t is unstable
767        let _: c_char = 0 as std::os::raw::c_char;
768    };
769
770    #[repr(transparent)]
771    pub(crate) struct CStr([c_char]);
772    impl CStr {
773        #[inline]
774        #[must_use]
775        pub(crate) const fn as_ptr(&self) -> *const c_char {
776            self.0.as_ptr()
777        }
778        /// # Safety
779        ///
780        /// The provided slice **must** be nul-terminated and not contain any interior
781        /// nul bytes.
782        #[inline]
783        #[must_use]
784        pub(crate) unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr {
785            // SAFETY: Casting to CStr is safe because *our* CStr is #[repr(transparent)]
786            // and its internal representation is a [u8] too. (Note that std's CStr
787            // is not #[repr(transparent)].)
788            // Dereferencing the obtained pointer is safe because it comes from a
789            // reference. Making a reference is then safe because its lifetime
790            // is bound by the lifetime of the given `bytes`.
791            unsafe { &*(bytes as *const [u8] as *const CStr) }
792        }
793        #[cfg(test)]
794        #[inline]
795        #[must_use]
796        pub(crate) fn to_bytes_with_nul(&self) -> &[u8] {
797            #[allow(clippy::unnecessary_cast)] // triggered for targets that c_char is u8
798            // SAFETY: Transmuting a slice of `c_char`s to a slice of `u8`s
799            // is safe on all supported targets.
800            unsafe {
801                &*(&self.0 as *const [c_char] as *const [u8])
802            }
803        }
804    }
805
806    macro_rules! c {
807        ($s:expr) => {{
808            const BYTES: &[u8] = concat!($s, "\0").as_bytes();
809            const _: () = static_assert!(crate::utils::ffi::_const_is_c_str(BYTES));
810            #[allow(unused_unsafe)]
811            // SAFETY: we've checked `BYTES` is a valid C string
812            unsafe {
813                crate::utils::ffi::CStr::from_bytes_with_nul_unchecked(BYTES)
814            }
815        }};
816    }
817
818    #[must_use]
819    pub(crate) const fn _const_is_c_str(bytes: &[u8]) -> bool {
820        #[cfg(portable_atomic_no_track_caller)]
821        {
822            // const_if_match/const_loop was stabilized (nightly-2020-06-30) 2 days before
823            // track_caller was stabilized (nightly-2020-07-02), so we reuse the cfg for
824            // track_caller here instead of emitting a cfg for const_if_match/const_loop.
825            // https://github.com/rust-lang/rust/pull/72437
826            // track_caller was stabilized 11 days after the oldest nightly version
827            // that uses this module, and is included in the same 1.46 stable release.
828            // The check here is insufficient in this case, but this is fine because this function
829            // is internal code that is not used to process input from the user and our CI checks
830            // all builtin targets and some custom targets with some versions of newer compilers.
831            !bytes.is_empty()
832        }
833        #[cfg(not(portable_atomic_no_track_caller))]
834        {
835            // Based on https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/c_str.rs#L417
836            // - bytes must be nul-terminated.
837            // - bytes must not contain any interior nul bytes.
838            if bytes.is_empty() {
839                return false;
840            }
841            let mut i = bytes.len() - 1;
842            if bytes[i] != 0 {
843                return false;
844            }
845            // Ending null byte exists, skip to the rest.
846            while i != 0 {
847                i -= 1;
848                if bytes[i] == 0 {
849                    return false;
850                }
851            }
852            true
853        }
854    }
855
856    // -------------------------------------------------------------------------
857    // Helper macros for defining FFI bindings with static signature/type/value assertions.
858
859    /// Defines types with #[cfg(test)] static assertions which checks
860    /// types are the same as the platform's latest header files' ones.
861    // Note: This macro is sys_ty!({ }), not sys_ty! { }.
862    // An extra brace is used in input to make contents rustfmt-able.
863    macro_rules! sys_type {
864        ({$(
865            $(#[$attr:meta])*
866            $vis:vis type $([$($windows_path:ident)::+])? $name:ident = $ty:ty;
867        )*}) => {
868            $(
869                $(#[$attr])*
870                $vis type $name = $ty;
871            )*
872            #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
873            test_helper::static_assert_sys_type!($(
874                $(#[$attr])*
875                type $([$($windows_path)::+])? $name;
876            )*);
877        };
878    }
879    /// Defines #[repr(C)] structs with #[cfg(test)] static assertions which checks
880    /// fields are the same as the platform's latest header files' ones.
881    // Note: This macro is sys_struct!({ }), not sys_struct! { }.
882    // An extra brace is used in input to make contents rustfmt-able.
883    macro_rules! sys_struct {
884        ({$(
885            $(#[$attr:meta])*
886            $vis:vis struct $([$($windows_path:ident)::+])? $name:ident {$(
887                $(#[$field_attr:meta])*
888                $field_vis:vis $field_name:ident: $field_ty:ty,
889            )*}
890        )*}) => {
891            $(
892                $(#[$attr])*
893                #[derive(Clone, Copy)]
894                #[cfg_attr(
895                    any(test, portable_atomic_test_no_std_static_assert_ffi),
896                    derive(Debug, PartialEq)
897                )]
898                #[repr(C)]
899                $vis struct $name {$(
900                    $(#[$field_attr])*
901                    $field_vis $field_name: $field_ty,
902                )*}
903            )*
904            #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
905            test_helper::static_assert_sys_struct!($(
906                $(#[$attr])*
907                struct $([$($windows_path)::+])? $name {$(
908                    $(#[$field_attr])*
909                    $field_name: $field_ty,
910                )*}
911            )*);
912        };
913    }
914    /// Defines constants with #[cfg(test)] static assertions which checks
915    /// values are the same as the platform's latest header files' ones.
916    // Note: This macro is sys_const!({ }), not sys_const! { }.
917    // An extra brace is used in input to make contents rustfmt-able.
918    macro_rules! sys_const {
919        ({$(
920            $(#[$attr:meta])*
921            $vis:vis const $([$($windows_path:ident)::+])? $name:ident: $ty:ty = $val:expr;
922        )*}) => {
923            $(
924                $(#[$attr])*
925                $vis const $name: $ty = $val;
926            )*
927            #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
928            test_helper::static_assert_sys_const!($(
929                $(#[$attr])*
930                const $([$($windows_path)::+])? $name: $ty;
931            )*);
932        };
933    }
934    /// Defines functions with #[cfg(test)] static assertions which checks
935    /// signatures are the same as the platform's latest header files' ones.
936    // Note: This macro is sys_fn!({ }), not sys_fn! { }.
937    // An extra brace is used in input to make contents rustfmt-able.
938    macro_rules! sys_fn {
939        ({
940            $(#[$extern_attr:meta])*
941            extern $abi:tt {$(
942                $(#[$fn_attr:meta])*
943                $vis:vis fn $([$($windows_path:ident)::+])? $name:ident(
944                    $($args:tt)*
945                ) $(-> $ret_ty:ty)?;
946            )*}
947        }) => {
948            $(#[$extern_attr])*
949            extern $abi {$(
950                $(#[$fn_attr])*
951                $vis fn $name($($args)*) $(-> $ret_ty)?;
952            )*}
953            #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]
954            test_helper::static_assert_sys_fn!(
955                $(#[$extern_attr])*
956                extern $abi {$(
957                    $(#[$fn_attr])*
958                    fn $([$($windows_path)::+])? $name($($args)*) $(-> $ret_ty)?;
959                )*}
960            );
961        };
962    }
963
964    // -----------------------------------------------------------------------------
965    // Helper macros for defining asm-based syscalls on Linux.
966    //
967    // Use asm-based syscall on Linux for compatibility with non-libc targets if possible.
968    //
969    // In non-Linux environments (including Android), syscalls should be called via libc,
970    // mainly for the following reasons.
971    // - asm-based syscalls are not permitted in the first place. e.g.,
972    //   - OpenBSD https://github.com/golang/go/issues/36435
973    //   - CheriBSD https://www.cheribsd.org/release-notes/25.03/index.html
974    // - Stability of asm-based syscalls is not guaranteed. e.g.,
975    //   - macOS https://go-review.googlesource.com/c/go/+/25495
976    // - Syscalls via libc (or also libsys on FreeBSD) provide additional protections. e.g.,
977    //   - OpenBSD https://lwn.net/Articles/806863/
978    //   - CheriBSD https://www.cheribsd.org/release-notes/25.03/index.html
979    //   - FreeBSD https://www.freebsd.org/status/report-2024-01-2024-03/libsys/
980    //   - Android https://android.googlesource.com/platform/bionic/+/HEAD/docs/fdtrack.md
981    //
982    // Miri and Sanitizer do not support inline assembly.
983    #[cfg(all(
984        target_os = "linux",
985        not(any(miri, portable_atomic_sanitize_thread)),
986        not(portable_atomic_no_asm_syscall),
987    ))]
988    #[macro_use]
989    mod syscall_helper {
990        // Note:
991        // - The syscall number and arguments must be extended to the register size by the caller of macros.
992        //   The kernel extends the low bits on some architectures, but it does not on many architectures. e.g.,
993        //   x86_64 (pre-5.14): https://github.com/torvalds/linux/commit/0595494891723a1dcca5eaa8eeca8ab54ad953b9
994        //   powerpc64: https://github.com/torvalds/linux/blob/v7.1/arch/powerpc/kernel/syscall.c#L16
995        //   riscv64: https://github.com/torvalds/linux/blob/v7.1/arch/riscv/kernel/traps.c#L328
996        //   loongarch64: https://github.com/torvalds/linux/blob/v7.1/arch/loongarch/kernel/syscall.c#L61
997        //
998        // Refs:
999        // - https://man7.org/linux/man-pages/man2/syscall.2.html
1000        // - aarch64 (test-only)
1001        //   https://git.musl-libc.org/cgit/musl/tree/arch/aarch64/syscall_arch.h?h=v1.2.6
1002        // - arm (test-only)
1003        //   https://git.musl-libc.org/cgit/musl/tree/arch/arm/syscall_arch.h?h=v1.2.6
1004        // - powerpc64 (test-only)
1005        //   https://github.com/torvalds/linux/blob/v7.1/Documentation/arch/powerpc/syscall64-abi.rst
1006        //   https://git.musl-libc.org/cgit/musl/tree/arch/powerpc64/syscall_arch.h?h=v1.2.6
1007        // - riscv32/riscv64
1008        //   https://git.musl-libc.org/cgit/musl/tree/arch/riscv32/syscall_arch.h?h=v1.2.6
1009        //   https://git.musl-libc.org/cgit/musl/tree/arch/riscv64/syscall_arch.h?h=v1.2.6
1010
1011        #[cfg(test)] // test-only
1012        #[cfg(all(target_arch = "aarch64", target_pointer_width = "64"))]
1013        macro_rules! asm_syscall {
1014            (
1015                $number:ident, $r:ident,
1016                $($arg1:ident $(, $arg2:ident $(, $arg3:ident
1017                    $(, $arg4:ident $(, $arg5:ident $(, $arg6:ident )?)?)?
1018                )?)?)?
1019            ) => {
1020                __asm!(
1021                    "svc 0",
1022                    in("x8") $number,
1023                    lateout("x0") $r,
1024                    $(in("x0") $arg1,
1025                        $(in("x1") $arg2,
1026                            $(in("x2") $arg3,
1027                                $(in("x3") $arg4,
1028                                    $(in("x4") $arg5,
1029                                        $(in("x5") $arg6, )?
1030                                    )?
1031                                )?
1032                            )?
1033                        )?
1034                    )?
1035                    // Clobber SVE registers and do not use `preserves_flags` because
1036                    // AArch64 Linux syscalls clears non-v[0-31] bits of z[0-31], and all of p[0-15] and ffr,
1037                    // and calls SMSTOP SM which clears z[0-31], p[0-15], ffr, and modifies FPSR
1038                    // when CPU is in streaming SVE mode.
1039                    // https://github.com/torvalds/linux/blob/v7.1/Documentation/arch/arm64/sve.rst#3--system-call-behaviour
1040                    // https://github.com/torvalds/linux/blob/v7.1/Documentation/arch/arm64/sme.rst#3--system-call-behaviour
1041                    // https://developer.arm.com/documentation/109246/0101/SME-Overview/Streaming-SVE-mode
1042                    out("z0") _,
1043                    out("z1") _,
1044                    out("z2") _,
1045                    out("z3") _,
1046                    out("z4") _,
1047                    out("z5") _,
1048                    out("z6") _,
1049                    out("z7") _,
1050                    out("z8") _,
1051                    out("z9") _,
1052                    out("z10") _,
1053                    out("z11") _,
1054                    out("z12") _,
1055                    out("z13") _,
1056                    out("z14") _,
1057                    out("z15") _,
1058                    out("z16") _,
1059                    out("z17") _,
1060                    out("z18") _,
1061                    out("z19") _,
1062                    out("z20") _,
1063                    out("z21") _,
1064                    out("z22") _,
1065                    out("z23") _,
1066                    out("z24") _,
1067                    out("z25") _,
1068                    out("z26") _,
1069                    out("z27") _,
1070                    out("z28") _,
1071                    out("z29") _,
1072                    out("z30") _,
1073                    out("z31") _,
1074                    out("p0") _,
1075                    out("p1") _,
1076                    out("p2") _,
1077                    out("p3") _,
1078                    out("p4") _,
1079                    out("p5") _,
1080                    out("p6") _,
1081                    out("p7") _,
1082                    out("p8") _,
1083                    out("p9") _,
1084                    out("p10") _,
1085                    out("p11") _,
1086                    out("p12") _,
1087                    out("p13") _,
1088                    out("p14") _,
1089                    out("p15") _,
1090                    out("ffr") _,
1091                    options(nostack),
1092                )
1093            };
1094        }
1095        #[cfg(test)] // test-only
1096        #[cfg(target_arch = "arm")]
1097        macro_rules! asm_syscall {
1098            (
1099                $number_const:path, $number:literal, $r:ident,
1100                $($arg1:ident $(, $arg2:ident $(, $arg3:ident
1101                    $(, $arg4:ident $(, $arg5:ident $(, $arg6:ident )?)?)?
1102                )?)?)?
1103            ) => {{
1104                static_assert!($number_const == $number && 0 <= $number && $number <= 255);
1105                __asm!(
1106                    // r7 is reserved on thumb and MOV requires Thumb-2 or Arm mode.
1107                    // cfg(target_feature = "thumb-mode")/cfg(target_feature = "thumb2")
1108                    // doesn't work on stable and register swapping is much cheaper than syscall,
1109                    // so we always swap register and use MOVS instead of MOV.
1110                    // Note: r6 is reserved by LLVM, so this assembly may not work for syscall6
1111                    //       with Thumb-1 (pre-v7 Arm in Thumb mode) because the allocation of
1112                    //       `tmp` may fail.
1113                    //       (syscall6 is needless in the current our use cases.)
1114                    "movs {tmp}, r7",
1115                    concat!("movs r7, ", $number),
1116                    "svc 0",
1117                    "movs r7, {tmp}",
1118                    tmp = out(reg) _,
1119                    lateout("r0") $r,
1120                    $(in("r0") $arg1,
1121                        $(in("r1") $arg2,
1122                            $(in("r2") $arg3,
1123                                $(in("r3") $arg4,
1124                                    $(in("r4") $arg5,
1125                                        $(in("r5") $arg6, )?
1126                                    )?
1127                                )?
1128                            )?
1129                        )?
1130                    )?
1131                    // Do not use `preserves_flags` because MOVS modifies the flags.
1132                    // Do not use `nostack` because SVC pushes to stack on M-profile architectures.
1133                    // https://github.com/torvalds/linux/blob/v7.1/arch/arm/kernel/entry-header.S#L61
1134                )
1135            }};
1136        }
1137        // POWER9+ has fast syscall using SCV, but it is needless in the current our use cases.
1138        #[cfg(test)] // test-only
1139        #[cfg(all(target_arch = "powerpc64", target_pointer_width = "64"))]
1140        macro_rules! asm_syscall {
1141            (
1142                $number:ident, $r:ident,
1143                $($arg1:ident $(, $arg2:ident $(, $arg3:ident
1144                    $(, $arg4:ident $(, $arg5:ident $(, $arg6:ident )?)?)?
1145                )?)?)?
1146            ) => {
1147                __asm!(
1148                    "sc",
1149                    "bns+ 2f",
1150                    "neg %r3, %r3",
1151                    "2:",
1152                    inout("r0") $number => _,
1153                    lateout("r3") $r,
1154                    $(in("r3") $arg1,
1155                        $(in("r4") $arg2,
1156                            $(in("r5") $arg3,
1157                                $(in("r6") $arg4,
1158                                    $(in("r7") $arg5,
1159                                        $(in("r8") $arg6, )?
1160                                    )?
1161                                )?
1162                            )?
1163                        )?
1164                    )?
1165                    lateout("r4") _,
1166                    lateout("r5") _,
1167                    lateout("r6") _,
1168                    lateout("r7") _,
1169                    lateout("r8") _,
1170                    out("r9") _,
1171                    out("r10") _,
1172                    out("r11") _,
1173                    out("r12") _,
1174                    out("cr0") _,
1175                    out("ctr") _,
1176                    out("xer") _,
1177                    options(nostack, preserves_flags),
1178                )
1179            };
1180        }
1181        #[cfg(any(
1182            target_arch = "riscv32",
1183            all(target_arch = "riscv64", target_pointer_width = "64"),
1184        ))]
1185        macro_rules! asm_syscall {
1186            (
1187                $number:ident, $r:ident,
1188                $($arg1:ident $(, $arg2:ident $(, $arg3:ident
1189                    $(, $arg4:ident $(, $arg5:ident $(, $arg6:ident )?)?)?
1190                )?)?)?
1191            ) => {
1192                __asm!(
1193                    "ecall",
1194                    in("a7") $number,
1195                    lateout("a0") $r,
1196                    $(in("a0") $arg1,
1197                        $(in("a1") $arg2,
1198                            $(in("a2") $arg3,
1199                                $(in("a3") $arg4,
1200                                    $(in("a4") $arg5,
1201                                        $(in("a5") $arg6, )?
1202                                    )?
1203                                )?
1204                            )?
1205                        )?
1206                    )?
1207                    // Clobber vector registers and do not use `preserves_flags` because RISC-V Linux syscalls don't preserve them.
1208                    // https://github.com/torvalds/linux/blob/v7.1/Documentation/arch/riscv/vector.rst#3--vector-register-state-across-system-calls
1209                    out("v0") _,
1210                    out("v1") _,
1211                    out("v2") _,
1212                    out("v3") _,
1213                    out("v4") _,
1214                    out("v5") _,
1215                    out("v6") _,
1216                    out("v7") _,
1217                    out("v8") _,
1218                    out("v9") _,
1219                    out("v10") _,
1220                    out("v11") _,
1221                    out("v12") _,
1222                    out("v13") _,
1223                    out("v14") _,
1224                    out("v15") _,
1225                    out("v16") _,
1226                    out("v17") _,
1227                    out("v18") _,
1228                    out("v19") _,
1229                    out("v20") _,
1230                    out("v21") _,
1231                    out("v22") _,
1232                    out("v23") _,
1233                    out("v24") _,
1234                    out("v25") _,
1235                    out("v26") _,
1236                    out("v27") _,
1237                    out("v28") _,
1238                    out("v29") _,
1239                    out("v30") _,
1240                    out("v31") _,
1241                    options(nostack),
1242                )
1243            };
1244        }
1245    }
1246
1247    #[allow(
1248        clippy::alloc_instead_of_core,
1249        clippy::std_instead_of_alloc,
1250        clippy::std_instead_of_core,
1251        clippy::undocumented_unsafe_blocks,
1252        clippy::wildcard_imports
1253    )]
1254    #[cfg(test)]
1255    mod tests {
1256        #[test]
1257        fn test_c_macro() {
1258            #[track_caller]
1259            fn t(s: &crate::utils::ffi::CStr, raw: &[u8]) {
1260                assert_eq!(s.to_bytes_with_nul(), raw);
1261            }
1262            t(c!(""), b"\0");
1263            t(c!("a"), b"a\0");
1264            t(c!("abc"), b"abc\0");
1265            t(c!(concat!("abc", "d")), b"abcd\0");
1266        }
1267
1268        #[test]
1269        fn test_is_c_str() {
1270            #[track_caller]
1271            fn t(bytes: &[u8]) {
1272                assert_eq!(
1273                    super::_const_is_c_str(bytes),
1274                    std::ffi::CStr::from_bytes_with_nul(bytes).is_ok()
1275                );
1276            }
1277            t(b"\0");
1278            t(b"a\0");
1279            t(b"abc\0");
1280            t(b"");
1281            t(b"a");
1282            t(b"abc");
1283            t(b"\0a");
1284            t(b"\0a\0");
1285            t(b"ab\0c\0");
1286            t(b"\0\0");
1287        }
1288    }
1289}