sel4_shared_memory/atomic_ops/
generic.rs

Help
1//
2// Copyright 2023, Colias Group, LLC
3// Copyright 2023, Rust project contributors
4//
5// SPDX-License-Identifier: MIT OR Apache-2.0
6//
7
8use core::intrinsics;
9
10use super::ordering::OrderingExhaustive as Ordering;
11
12use Ordering::*;
13
14#[inline]
15pub(crate) unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
16    // SAFETY: the caller must uphold the safety contract for `atomic_store`.
17    unsafe {
18        match order {
19            Relaxed => intrinsics::atomic_store_relaxed(dst, val),
20            Release => intrinsics::atomic_store_release(dst, val),
21            SeqCst => intrinsics::atomic_store_seqcst(dst, val),
22            Acquire => panic!("there is no such thing as an acquire store"),
23            AcqRel => panic!("there is no such thing as an acquire-release store"),
24        }
25    }
26}
27
28#[inline]
29pub(crate) unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
30    // SAFETY: the caller must uphold the safety contract for `atomic_load`.
31    unsafe {
32        match order {
33            Relaxed => intrinsics::atomic_load_relaxed(dst),
34            Acquire => intrinsics::atomic_load_acquire(dst),
35            SeqCst => intrinsics::atomic_load_seqcst(dst),
36            Release => panic!("there is no such thing as a release load"),
37            AcqRel => panic!("there is no such thing as an acquire-release load"),
38        }
39    }
40}
41
42#[inline]
43pub(crate) unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
44    // SAFETY: the caller must uphold the safety contract for `atomic_swap`.
45    unsafe {
46        match order {
47            Relaxed => intrinsics::atomic_xchg_relaxed(dst, val),
48            Acquire => intrinsics::atomic_xchg_acquire(dst, val),
49            Release => intrinsics::atomic_xchg_release(dst, val),
50            AcqRel => intrinsics::atomic_xchg_acqrel(dst, val),
51            SeqCst => intrinsics::atomic_xchg_seqcst(dst, val),
52        }
53    }
54}
55
56#[inline]
57pub(crate) unsafe fn atomic_add<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
58    // SAFETY: the caller must uphold the safety contract for `atomic_add`.
59    unsafe {
60        match order {
61            Relaxed => intrinsics::atomic_xadd_relaxed(dst, val),
62            Acquire => intrinsics::atomic_xadd_acquire(dst, val),
63            Release => intrinsics::atomic_xadd_release(dst, val),
64            AcqRel => intrinsics::atomic_xadd_acqrel(dst, val),
65            SeqCst => intrinsics::atomic_xadd_seqcst(dst, val),
66        }
67    }
68}
69
70#[inline]
71pub(crate) unsafe fn atomic_sub<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
72    // SAFETY: the caller must uphold the safety contract for `atomic_sub`.
73    unsafe {
74        match order {
75            Relaxed => intrinsics::atomic_xsub_relaxed(dst, val),
76            Acquire => intrinsics::atomic_xsub_acquire(dst, val),
77            Release => intrinsics::atomic_xsub_release(dst, val),
78            AcqRel => intrinsics::atomic_xsub_acqrel(dst, val),
79            SeqCst => intrinsics::atomic_xsub_seqcst(dst, val),
80        }
81    }
82}
83
84#[inline]
85pub(crate) unsafe fn atomic_compare_exchange<T: Copy>(
86    dst: *mut T,
87    old: T,
88    new: T,
89    success: Ordering,
90    failure: Ordering,
91) -> Result<T, T> {
92    // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
93    let (val, ok) = unsafe {
94        match (success, failure) {
95            (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new),
96            (Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new),
97            (Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new),
98            (Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new),
99            (Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new),
100            (Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new),
101            (Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new),
102            (Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new),
103            (Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new),
104            (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new),
105            (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new),
106            (AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new),
107            (SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new),
108            (SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new),
109            (SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new),
110            (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
111            (_, Release) => panic!("there is no such thing as a release failure ordering"),
112        }
113    };
114    if ok {
115        Ok(val)
116    } else {
117        Err(val)
118    }
119}
120
121#[inline]
122pub(crate) unsafe fn atomic_compare_exchange_weak<T: Copy>(
123    dst: *mut T,
124    old: T,
125    new: T,
126    success: Ordering,
127    failure: Ordering,
128) -> Result<T, T> {
129    // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`.
130    let (val, ok) = unsafe {
131        match (success, failure) {
132            (Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new),
133            (Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new),
134            (Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new),
135            (Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new),
136            (Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new),
137            (Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new),
138            (Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new),
139            (Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new),
140            (Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new),
141            (AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new),
142            (AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new),
143            (AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new),
144            (SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new),
145            (SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new),
146            (SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new),
147            (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
148            (_, Release) => panic!("there is no such thing as a release failure ordering"),
149        }
150    };
151    if ok {
152        Ok(val)
153    } else {
154        Err(val)
155    }
156}
157
158#[inline]
159pub(crate) unsafe fn atomic_and<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
160    // SAFETY: the caller must uphold the safety contract for `atomic_and`
161    unsafe {
162        match order {
163            Relaxed => intrinsics::atomic_and_relaxed(dst, val),
164            Acquire => intrinsics::atomic_and_acquire(dst, val),
165            Release => intrinsics::atomic_and_release(dst, val),
166            AcqRel => intrinsics::atomic_and_acqrel(dst, val),
167            SeqCst => intrinsics::atomic_and_seqcst(dst, val),
168        }
169    }
170}
171
172#[inline]
173pub(crate) unsafe fn atomic_nand<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
174    // SAFETY: the caller must uphold the safety contract for `atomic_nand`
175    unsafe {
176        match order {
177            Relaxed => intrinsics::atomic_nand_relaxed(dst, val),
178            Acquire => intrinsics::atomic_nand_acquire(dst, val),
179            Release => intrinsics::atomic_nand_release(dst, val),
180            AcqRel => intrinsics::atomic_nand_acqrel(dst, val),
181            SeqCst => intrinsics::atomic_nand_seqcst(dst, val),
182        }
183    }
184}
185
186#[inline]
187pub(crate) unsafe fn atomic_or<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
188    // SAFETY: the caller must uphold the safety contract for `atomic_or`
189    unsafe {
190        match order {
191            SeqCst => intrinsics::atomic_or_seqcst(dst, val),
192            Acquire => intrinsics::atomic_or_acquire(dst, val),
193            Release => intrinsics::atomic_or_release(dst, val),
194            AcqRel => intrinsics::atomic_or_acqrel(dst, val),
195            Relaxed => intrinsics::atomic_or_relaxed(dst, val),
196        }
197    }
198}
199
200#[inline]
201pub(crate) unsafe fn atomic_xor<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
202    // SAFETY: the caller must uphold the safety contract for `atomic_xor`
203    unsafe {
204        match order {
205            SeqCst => intrinsics::atomic_xor_seqcst(dst, val),
206            Acquire => intrinsics::atomic_xor_acquire(dst, val),
207            Release => intrinsics::atomic_xor_release(dst, val),
208            AcqRel => intrinsics::atomic_xor_acqrel(dst, val),
209            Relaxed => intrinsics::atomic_xor_relaxed(dst, val),
210        }
211    }
212}
213
214#[inline]
215pub(crate) unsafe fn atomic_max<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
216    // SAFETY: the caller must uphold the safety contract for `atomic_max`
217    unsafe {
218        match order {
219            Relaxed => intrinsics::atomic_max_relaxed(dst, val),
220            Acquire => intrinsics::atomic_max_acquire(dst, val),
221            Release => intrinsics::atomic_max_release(dst, val),
222            AcqRel => intrinsics::atomic_max_acqrel(dst, val),
223            SeqCst => intrinsics::atomic_max_seqcst(dst, val),
224        }
225    }
226}
227
228#[inline]
229pub(crate) unsafe fn atomic_min<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
230    // SAFETY: the caller must uphold the safety contract for `atomic_min`
231    unsafe {
232        match order {
233            Relaxed => intrinsics::atomic_min_relaxed(dst, val),
234            Acquire => intrinsics::atomic_min_acquire(dst, val),
235            Release => intrinsics::atomic_min_release(dst, val),
236            AcqRel => intrinsics::atomic_min_acqrel(dst, val),
237            SeqCst => intrinsics::atomic_min_seqcst(dst, val),
238        }
239    }
240}
241
242#[inline]
243pub(crate) unsafe fn atomic_umax<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
244    // SAFETY: the caller must uphold the safety contract for `atomic_umax`
245    unsafe {
246        match order {
247            Relaxed => intrinsics::atomic_umax_relaxed(dst, val),
248            Acquire => intrinsics::atomic_umax_acquire(dst, val),
249            Release => intrinsics::atomic_umax_release(dst, val),
250            AcqRel => intrinsics::atomic_umax_acqrel(dst, val),
251            SeqCst => intrinsics::atomic_umax_seqcst(dst, val),
252        }
253    }
254}
255
256#[inline]
257pub(crate) unsafe fn atomic_umin<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
258    // SAFETY: the caller must uphold the safety contract for `atomic_umin`
259    unsafe {
260        match order {
261            Relaxed => intrinsics::atomic_umin_relaxed(dst, val),
262            Acquire => intrinsics::atomic_umin_acquire(dst, val),
263            Release => intrinsics::atomic_umin_release(dst, val),
264            AcqRel => intrinsics::atomic_umin_acqrel(dst, val),
265            SeqCst => intrinsics::atomic_umin_seqcst(dst, val),
266        }
267    }
268}
sel4_shared_memory/atomic_ops/generic.rs

sel4_shared_memory/atomic_ops/
generic.rs
