Skip to content

Commit 799b84c

Browse files
taiki-etaiki-e
committed
AtomicCell: Use atomic-maybe-uninit
1 parent 8b0fdfa commit 799b84c

5 files changed

Lines changed: 191 additions & 75 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ jobs:
6262
- rust: nightly
6363
os: ubuntu-latest
6464
target: armv5te-unknown-linux-gnueabi
65+
# Test target without stable inline asm support.
66+
- rust: stable
67+
os: ubuntu-latest
68+
target: sparc64-unknown-linux-gnu
6569
runs-on: ${{ matrix.os }}
6670
timeout-minutes: 60
6771
steps:

‎crossbeam-utils/Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,10 @@ default = ["std"]
2525
std = []
2626

2727
# Enable `atomic` module.
28-
atomic = []
28+
atomic = ["atomic-maybe-uninit"]
2929

3030
[dependencies]
31+
atomic-maybe-uninit = { version = "0.3.4", optional = true }
3132

3233
# Enable the use of loom for concurrency testing.
3334
#

‎crossbeam-utils/build.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ include!("build-common.rs");
1717

1818
fn main() {
1919
println!("cargo:rerun-if-changed=no_atomic.rs");
20-
println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread)");
20+
println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread,crossbeam_atomic_cell_force_fallback)");
2121

2222
let target = match env::var("TARGET") {
2323
Ok(target) => convert_custom_linux_target(target),
@@ -39,8 +39,10 @@ fn main() {
3939
}
4040

4141
// `cfg(sanitize = "..")` is not stabilized.
42-
let sanitize = env::var("CARGO_CFG_SANITIZE").unwrap_or_default();
43-
if sanitize.contains("thread") {
44-
println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
42+
if let Ok(sanitize) = env::var("CARGO_CFG_SANITIZE") {
43+
if sanitize.contains("thread") {
44+
println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
45+
}
46+
println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback");
4547
}
4648
}

‎crossbeam-utils/src/atomic/atomic_cell.rs‎

Lines changed: 141 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ impl<T> AtomicCell<T> {
9999
/// # Examples
100100
///
101101
/// ```
102+
/// # // Always use fallback for now on environments that do not support inline assembly.
103+
/// # if cfg!(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)) { return; }
102104
/// use crossbeam_utils::atomic::AtomicCell;
103105
///
104106
/// // This type is internally represented as `AtomicUsize` so we can just use atomic
@@ -307,21 +309,37 @@ macro_rules! atomic {
307309
loop {
308310
atomic!(@check, $t, AtomicUnit, $a, $atomic_op);
309311

310-
atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op);
311-
atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op);
312-
atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op);
313-
#[cfg(target_has_atomic = "64")]
314-
atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op);
315-
// TODO: AtomicU128 is unstable
316-
// atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op);
312+
// Always use fallback for now on environments that do not support inline assembly.
313+
#[cfg(not(any(
314+
miri,
315+
crossbeam_loom,
316+
crossbeam_atomic_cell_force_fallback,
317+
)))]
318+
atomic_maybe_uninit::cfg_has_atomic_cas! {
319+
atomic_maybe_uninit::cfg_has_atomic_8! {
320+
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u8>, $a, $atomic_op);
321+
}
322+
atomic_maybe_uninit::cfg_has_atomic_16! {
323+
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u16>, $a, $atomic_op);
324+
}
325+
atomic_maybe_uninit::cfg_has_atomic_32! {
326+
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u32>, $a, $atomic_op);
327+
}
328+
atomic_maybe_uninit::cfg_has_atomic_64! {
329+
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u64>, $a, $atomic_op);
330+
}
331+
atomic_maybe_uninit::cfg_has_atomic_128! {
332+
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u128>, $a, $atomic_op);
333+
}
334+
}
317335

318336
break $fallback_op;
319337
}
320338
};
321339
}
322340

323341
macro_rules! impl_arithmetic {
324-
($t:ty, fallback, $example:tt) => {
342+
($t:ty, fetch_update, $example:tt) => {
325343
impl AtomicCell<$t> {
326344
/// Increments the current value by `val` and returns the previous value.
327345
///
@@ -339,11 +357,19 @@ macro_rules! impl_arithmetic {
339357
/// ```
340358
#[inline]
341359
pub fn fetch_add(&self, val: $t) -> $t {
342-
let _guard = lock(self.as_ptr() as usize).write();
343-
let value = unsafe { &mut *(self.as_ptr()) };
344-
let old = *value;
345-
*value = value.wrapping_add(val);
346-
old
360+
atomic! {
361+
$t, _a,
362+
{
363+
self.fetch_update(|old| Some(old.wrapping_add(val))).unwrap()
364+
},
365+
{
366+
let _guard = lock(self.as_ptr() as usize).write();
367+
let value = unsafe { &mut *(self.as_ptr()) };
368+
let old = *value;
369+
*value = value.wrapping_add(val);
370+
old
371+
}
372+
}
347373
}
348374

349375
/// Decrements the current value by `val` and returns the previous value.
@@ -362,11 +388,19 @@ macro_rules! impl_arithmetic {
362388
/// ```
363389
#[inline]
364390
pub fn fetch_sub(&self, val: $t) -> $t {
365-
let _guard = lock(self.as_ptr() as usize).write();
366-
let value = unsafe { &mut *(self.as_ptr()) };
367-
let old = *value;
368-
*value = value.wrapping_sub(val);
369-
old
391+
atomic! {
392+
$t, _a,
393+
{
394+
self.fetch_update(|old| Some(old.wrapping_sub(val))).unwrap()
395+
},
396+
{
397+
let _guard = lock(self.as_ptr() as usize).write();
398+
let value = unsafe { &mut *(self.as_ptr()) };
399+
let old = *value;
400+
*value = value.wrapping_sub(val);
401+
old
402+
}
403+
}
370404
}
371405

372406
/// Applies bitwise "and" to the current value and returns the previous value.
@@ -383,11 +417,19 @@ macro_rules! impl_arithmetic {
383417
/// ```
384418
#[inline]
385419
pub fn fetch_and(&self, val: $t) -> $t {
386-
let _guard = lock(self.as_ptr() as usize).write();
387-
let value = unsafe { &mut *(self.as_ptr()) };
388-
let old = *value;
389-
*value &= val;
390-
old
420+
atomic! {
421+
$t, _a,
422+
{
423+
self.fetch_update(|old| Some(old & val)).unwrap()
424+
},
425+
{
426+
let _guard = lock(self.as_ptr() as usize).write();
427+
let value = unsafe { &mut *(self.as_ptr()) };
428+
let old = *value;
429+
*value &= val;
430+
old
431+
}
432+
}
391433
}
392434

393435
/// Applies bitwise "nand" to the current value and returns the previous value.
@@ -404,11 +446,19 @@ macro_rules! impl_arithmetic {
404446
/// ```
405447
#[inline]
406448
pub fn fetch_nand(&self, val: $t) -> $t {
407-
let _guard = lock(self.as_ptr() as usize).write();
408-
let value = unsafe { &mut *(self.as_ptr()) };
409-
let old = *value;
410-
*value = !(old & val);
411-
old
449+
atomic! {
450+
$t, _a,
451+
{
452+
self.fetch_update(|old| Some(!(old & val))).unwrap()
453+
},
454+
{
455+
let _guard = lock(self.as_ptr() as usize).write();
456+
let value = unsafe { &mut *(self.as_ptr()) };
457+
let old = *value;
458+
*value = !(old & val);
459+
old
460+
}
461+
}
412462
}
413463

414464
/// Applies bitwise "or" to the current value and returns the previous value.
@@ -425,11 +475,19 @@ macro_rules! impl_arithmetic {
425475
/// ```
426476
#[inline]
427477
pub fn fetch_or(&self, val: $t) -> $t {
428-
let _guard = lock(self.as_ptr() as usize).write();
429-
let value = unsafe { &mut *(self.as_ptr()) };
430-
let old = *value;
431-
*value |= val;
432-
old
478+
atomic! {
479+
$t, _a,
480+
{
481+
self.fetch_update(|old| Some(old | val)).unwrap()
482+
},
483+
{
484+
let _guard = lock(self.as_ptr() as usize).write();
485+
let value = unsafe { &mut *(self.as_ptr()) };
486+
let old = *value;
487+
*value |= val;
488+
old
489+
}
490+
}
433491
}
434492

435493
/// Applies bitwise "xor" to the current value and returns the previous value.
@@ -446,11 +504,19 @@ macro_rules! impl_arithmetic {
446504
/// ```
447505
#[inline]
448506
pub fn fetch_xor(&self, val: $t) -> $t {
449-
let _guard = lock(self.as_ptr() as usize).write();
450-
let value = unsafe { &mut *(self.as_ptr()) };
451-
let old = *value;
452-
*value ^= val;
453-
old
507+
atomic! {
508+
$t, _a,
509+
{
510+
self.fetch_update(|old| Some(old ^ val)).unwrap()
511+
},
512+
{
513+
let _guard = lock(self.as_ptr() as usize).write();
514+
let value = unsafe { &mut *(self.as_ptr()) };
515+
let old = *value;
516+
*value ^= val;
517+
old
518+
}
519+
}
454520
}
455521

456522
/// Compares and sets the maximum of the current value and `val`,
@@ -468,11 +534,19 @@ macro_rules! impl_arithmetic {
468534
/// ```
469535
#[inline]
470536
pub fn fetch_max(&self, val: $t) -> $t {
471-
let _guard = lock(self.as_ptr() as usize).write();
472-
let value = unsafe { &mut *(self.as_ptr()) };
473-
let old = *value;
474-
*value = cmp::max(old, val);
475-
old
537+
atomic! {
538+
$t, _a,
539+
{
540+
self.fetch_update(|old| Some(cmp::max(old, val))).unwrap()
541+
},
542+
{
543+
let _guard = lock(self.as_ptr() as usize).write();
544+
let value = unsafe { &mut *(self.as_ptr()) };
545+
let old = *value;
546+
*value = cmp::max(old, val);
547+
old
548+
}
549+
}
476550
}
477551

478552
/// Compares and sets the minimum of the current value and `val`,
@@ -490,11 +564,19 @@ macro_rules! impl_arithmetic {
490564
/// ```
491565
#[inline]
492566
pub fn fetch_min(&self, val: $t) -> $t {
493-
let _guard = lock(self.as_ptr() as usize).write();
494-
let value = unsafe { &mut *(self.as_ptr()) };
495-
let old = *value;
496-
*value = cmp::min(old, val);
497-
old
567+
atomic! {
568+
$t, _a,
569+
{
570+
self.fetch_update(|old| Some(cmp::min(old, val))).unwrap()
571+
},
572+
{
573+
let _guard = lock(self.as_ptr() as usize).write();
574+
let value = unsafe { &mut *(self.as_ptr()) };
575+
let old = *value;
576+
*value = cmp::min(old, val);
577+
old
578+
}
579+
}
498580
}
499581
}
500582
};
@@ -754,23 +836,29 @@ impl_arithmetic!(i8, AtomicI8, "let a = AtomicCell::new(7i8);");
754836
impl_arithmetic!(u16, AtomicU16, "let a = AtomicCell::new(7u16);");
755837
impl_arithmetic!(i16, AtomicI16, "let a = AtomicCell::new(7i16);");
756838

839+
#[cfg(target_has_atomic = "32")]
757840
impl_arithmetic!(u32, AtomicU32, "let a = AtomicCell::new(7u32);");
841+
#[cfg(target_has_atomic = "32")]
758842
impl_arithmetic!(i32, AtomicI32, "let a = AtomicCell::new(7i32);");
843+
#[cfg(not(target_has_atomic = "32"))]
844+
impl_arithmetic!(u32, fetch_update, "let a = AtomicCell::new(7u32);");
845+
#[cfg(not(target_has_atomic = "32"))]
846+
impl_arithmetic!(i32, fetch_update, "let a = AtomicCell::new(7i32);");
759847

760848
#[cfg(target_has_atomic = "64")]
761849
impl_arithmetic!(u64, AtomicU64, "let a = AtomicCell::new(7u64);");
762850
#[cfg(target_has_atomic = "64")]
763851
impl_arithmetic!(i64, AtomicI64, "let a = AtomicCell::new(7i64);");
764852
#[cfg(not(target_has_atomic = "64"))]
765-
impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);");
853+
impl_arithmetic!(u64, fetch_update, "let a = AtomicCell::new(7u64);");
766854
#[cfg(not(target_has_atomic = "64"))]
767-
impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);");
855+
impl_arithmetic!(i64, fetch_update, "let a = AtomicCell::new(7i64);");
768856

769-
// TODO: AtomicU128 is unstable
857+
// TODO: core::sync::atomic::AtomicU128 is unstable
770858
// impl_arithmetic!(u128, AtomicU128, "let a = AtomicCell::new(7u128);");
771859
// impl_arithmetic!(i128, AtomicI128, "let a = AtomicCell::new(7i128);");
772-
impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);");
773-
impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);");
860+
impl_arithmetic!(u128, fetch_update, "let a = AtomicCell::new(7u128);");
861+
impl_arithmetic!(i128, fetch_update, "let a = AtomicCell::new(7i128);");
774862

775863
impl_arithmetic!(usize, AtomicUsize, "let a = AtomicCell::new(7usize);");
776864
impl_arithmetic!(isize, AtomicIsize, "let a = AtomicCell::new(7isize);");

0 commit comments

Comments
 (0)