1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

//
// Copyright 2023, Colias Group, LLC
//
// SPDX-License-Identifier: BSD-2-Clause
//

use core::mem;

#[cfg(feature = "unstable")]
use core::intrinsics;

use cfg_if::cfg_if;
use volatile::ops::{BulkOps, Ops};

#[cfg(feature = "unstable")]
use volatile::ops::UnitaryOps;

#[derive(Default, Copy, Clone)]
pub struct UnorderedAtomicOps(());

impl Ops for UnorderedAtomicOps {}

#[cfg(feature = "unstable")]
impl<T: UnsignedPrimitiveWithUnorderedAtomics> UnitaryOps<T> for UnorderedAtomicOps {
    unsafe fn read(src: *const T) -> T {
        unsafe { intrinsics::atomic_load_unordered(src) }
    }

    unsafe fn write(dst: *mut T, src: T) {
        unsafe { intrinsics::atomic_store_unordered(dst, src) }
    }
}

#[allow(clippy::missing_safety_doc)]
pub unsafe trait UnsignedPrimitiveWithUnorderedAtomics: Copy {}

macro_rules! impl_unsigned_primitive_with_unordered_atomics {
    (
        $target_has_atomic_key:literal,
        $prim:path,
        $memmove:ident,
        $memcpy:ident,
        $memset:ident,
    ) => {
        cfg_if! {
            if #[cfg(target_has_atomic = $target_has_atomic_key)] {
                unsafe impl UnsignedPrimitiveWithUnorderedAtomics for $prim {}

                impl BulkOps<$prim> for UnorderedAtomicOps {
                    unsafe fn memmove(dst: *mut $prim, src: *const $prim, count: usize) {
                        unsafe { $memmove(dst, src, count * mem::size_of::<$prim>()) }
                    }

                    unsafe fn memcpy(dst: *mut $prim, src: *const $prim, count: usize) {
                        unsafe { $memcpy(dst, src, count * mem::size_of::<$prim>()) }
                    }

                    unsafe fn memset(dst: *mut $prim, val: u8, count: usize) {
                        unsafe { $memset(dst, val, count * mem::size_of::<$prim>()) }
                    }
                }

                extern "C" {
                    fn $memmove(dest: *mut $prim, src: *const $prim, bytes: usize);
                    fn $memcpy(dest: *mut $prim, src: *const $prim, bytes: usize);
                    fn $memset(s: *mut $prim, c: u8, bytes: usize);
                }
            }
        }
    };
}

impl_unsigned_primitive_with_unordered_atomics! {
    "8",
    u8,
    __llvm_memmove_element_unordered_atomic_1,
    __llvm_memcpy_element_unordered_atomic_1,
    __llvm_memset_element_unordered_atomic_1,
}

impl_unsigned_primitive_with_unordered_atomics! {
    "16",
    u16,
    __llvm_memmove_element_unordered_atomic_2,
    __llvm_memcpy_element_unordered_atomic_2,
    __llvm_memset_element_unordered_atomic_2,
}

impl_unsigned_primitive_with_unordered_atomics! {
    "32",
    u32,
    __llvm_memmove_element_unordered_atomic_4,
    __llvm_memcpy_element_unordered_atomic_4,
    __llvm_memset_element_unordered_atomic_4,
}

impl_unsigned_primitive_with_unordered_atomics! {
    "64",
    u64,
    __llvm_memmove_element_unordered_atomic_8,
    __llvm_memcpy_element_unordered_atomic_8,
    __llvm_memset_element_unordered_atomic_8,
}

#[cfg(target_pointer_width = "32")]
type UsizeCastTarget = u32;

#[cfg(target_pointer_width = "64")]
type UsizeCastTarget = u64;

#[cfg(any(
    all(target_pointer_width = "32", target_has_atomic = "32"),
    all(target_pointer_width = "64", target_has_atomic = "64"),
))]
impl BulkOps<usize> for UnorderedAtomicOps {
    unsafe fn memmove(dst: *mut usize, src: *const usize, count: usize) {
        unsafe {
            <UnorderedAtomicOps as BulkOps<UsizeCastTarget>>::memmove(dst.cast(), src.cast(), count)
        }
    }

    unsafe fn memcpy(dst: *mut usize, src: *const usize, count: usize) {
        unsafe {
            <UnorderedAtomicOps as BulkOps<UsizeCastTarget>>::memcpy(dst.cast(), src.cast(), count)
        }
    }

    unsafe fn memset(dst: *mut usize, val: u8, count: usize) {
        unsafe { <UnorderedAtomicOps as BulkOps<UsizeCastTarget>>::memset(dst.cast(), val, count) }
    }
}