1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

// Copyright 2016 Amanieu d'Antras
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

#[cfg(windows)]
use kernel32;
#[cfg(unix)]
use libc;
#[cfg(not(any(windows, unix)))]
use std::thread;
#[cfg(not(feature = "nightly"))]
use std::sync::atomic::{Ordering, fence};

// Yields the rest of the current timeslice to the OS
#[cfg(windows)]
#[inline]
fn thread_yield() {
    unsafe {
        // We don't use SwitchToThread here because it doesn't consider all
        // threads in the system and the thread we are waiting for may not get
        // selected.
        kernel32::Sleep(0);
    }
}
#[cfg(unix)]
#[inline]
fn thread_yield() {
    unsafe {
        libc::sched_yield();
    }
}
#[cfg(not(any(windows, unix)))]
#[inline]
fn thread_yield() {
    thread::yield_now();
}

// Wastes some CPU time for the given number of iterations, preferably also
// using a hint to indicate to the CPU that we are spinning.
#[cfg(all(feature = "nightly", any(target_arch = "x86", target_arch = "x86_64")))]
#[inline]
fn cpu_relax(iterations: u32) {
    for _ in 0..iterations {
        unsafe {
            asm!("pause" ::: "memory" : "volatile");
        }
    }
}
#[cfg(all(feature = "nightly", target_arch = "aarch64"))]
#[inline]
fn cpu_relax(iterations: u32) {
    for _ in 0..iterations {
        unsafe {
            asm!("yield" ::: "memory" : "volatile");
        }
    }
}
#[cfg(all(feature = "nightly", not(any(target_arch = "x86",
                                       target_arch = "x86_64",
                                       target_arch = "aarch64"))))]
#[inline]
fn cpu_relax(iterations: u32) {
    for _ in 0..iterations {
        unsafe {
            asm!("" ::: "memory" : "volatile");
        }
    }
}
#[cfg(not(feature = "nightly"))]
#[inline]
fn cpu_relax(iterations: u32) {
    // This is a bit tricky: we rely on the fact that LLVM doesn't optimize
    // atomic operations and effectively treats them as volatile.
    for _ in 0..iterations {
        fence(Ordering::SeqCst);
    }
}

/// A counter used to perform exponential backoff in spin loops.
pub struct SpinWait {
    counter: u32,
}

impl SpinWait {
    /// Creates a new `SpinWait`.
    #[cfg(feature = "nightly")]
    #[inline]
    pub const fn new() -> SpinWait {
        SpinWait { counter: 0 }
    }

    /// Creates a new `SpinWait`.
    #[cfg(not(feature = "nightly"))]
    #[inline]
    pub fn new() -> SpinWait {
        SpinWait { counter: 0 }
    }

    /// Resets a `SpinWait` to its initial state.
    #[inline]
    pub fn reset(&mut self) {
        self.counter = 0;
    }

    /// Spins until the sleep threshold has been reached.
    ///
    /// This function returns whether the sleep threshold has been reached, at
    /// which point further spinning has diminishing returns and the thread
    /// should be parked instead.
    ///
    /// The spin strategy will initially use a CPU-bound loop but will fall back
    /// to yielding the CPU to the OS after a few iterations.
    #[inline]
    pub fn spin(&mut self) -> bool {
        if self.counter >= 20 {
            return false;
        }
        self.counter += 1;
        if self.counter <= 10 {
            cpu_relax(4 << self.counter);
        } else {
            thread_yield();
        }
        true
    }

    /// Spins without yielding the thread to the OS.
    ///
    /// Instead, the backoff is simply capped at a maximum value. This can be
    /// used to improve throughput in `compare_exchange` loops that have high
    /// contention.
    #[inline]
    pub fn spin_no_yield(&mut self) {
        self.counter += 1;
        if self.counter > 10 {
            self.counter = 10;
        }
        cpu_relax(4 << self.counter);
    }
}

impl Default for SpinWait {
    #[inline]
    fn default() -> SpinWait {
        SpinWait::new()
    }
}