t2t_clock/
counter.rs

1//! The CPU's counter: one instruction to read.
2
3use core::ops::RangeInclusive;
4
5use arch::{discover_rate, read};
6use t2t_core::{TickRate, Tickstamp};
7
8use crate::{Clock, CounterError};
9
10/// The rates a counter is believed at: below a tick a microsecond it measures nothing worth
11/// measuring, and none runs past 10 GHz.
12const PLAUSIBLE_RATES: RangeInclusive<u64> = 1_000_000..=10_000_000_000;
13
14/// The CPU's free-running counter: `cntvct_el0` on `aarch64`, the time-stamp counter on `x86_64`.
15///
16/// A reading is one instruction that touches no memory. Every core and process on a machine reads
17/// the same counter, so a [`Tickstamp`] one process took, another may subtract from. Only the
18/// difference of two readings means anything, and the counter's [`rate`](Self::rate) turns it into
19/// a span.
20///
21/// The read is not ordered against the instructions around it: right for stamping, and for timing
22/// anything much longer than the read itself.
23///
24/// # Examples
25/// ```
26/// use t2t_clock::{Clock, Counter};
27/// use t2t_core::Timedelta;
28///
29/// # // The CI's Intel macOS virtual machines promise no invariant counter, so have none to read.
30/// # if Counter::discover() == Err(t2t_clock::CounterError::NotInvariant) {
31/// #     return Ok(());
32/// # }
33/// let counter = Counter::discover()?;
34/// let start = counter.now();
35/// let took = (counter.now() - start).to_timedelta(counter.rate());
36///
37/// assert!(took < Timedelta::from_millis(1), "two reads back to back");
38/// # Ok::<(), t2t_clock::CounterError>(())
39/// ```
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
41pub struct Counter {
42    /// Ticks a second.
43    rate: TickRate,
44}
45
46impl Counter {
47    /// The counter, at the rate the CPU reports for it; where it reports none, on `x86_64` with
48    /// `std` on 64-bit Linux or macOS, one measured against the OS clocks over 10 ms.
49    ///
50    /// # Errors
51    /// - [`CounterError::NotInvariant`], the time-stamp counter's rate follows the core's.
52    /// - [`CounterError::NoRate`], the CPU reports no rate, and none could be measured.
53    /// - [`CounterError::ImplausibleRate`], the rate is outside 1 MHz to 10 GHz.
54    pub fn discover() -> Result<Self, CounterError> {
55        discover_rate().map(Self::new)
56    }
57
58    /// The counter, at `rate`: one discovered elsewhere, or known.
59    #[inline]
60    #[must_use]
61    pub const fn new(rate: TickRate) -> Self {
62        Self { rate }
63    }
64
65    /// The counter's rate.
66    #[inline]
67    #[must_use]
68    pub const fn rate(self) -> TickRate {
69        self.rate
70    }
71}
72
73impl Clock for Counter {
74    type Reading = Tickstamp;
75
76    #[inline(always)]
77    #[expect(clippy::inline_always, reason = "a call around the read would move it")]
78    fn now(&self) -> Tickstamp {
79        Tickstamp::from_ticks(read())
80    }
81}
82
83/// `hertz` as a rate, refused outside what any counter runs at.
84///
85/// # Errors
86/// [`CounterError::ImplausibleRate`], `hertz` is outside 1 MHz to 10 GHz.
87fn plausible_rate(hertz: u64) -> Result<TickRate, CounterError> {
88    TickRate::from_hertz(hertz)
89        .filter(|_| PLAUSIBLE_RATES.contains(&hertz))
90        .ok_or(CounterError::ImplausibleRate { hertz })
91}
92
93/// The Arm generic timer's virtual counter, read from user space.
94///
95/// Linux and macOS let user space read both registers here; an OS that does not traps the read,
96/// which stops the process but is no unsoundness.
97#[cfg(target_arch = "aarch64")]
98mod arch {
99    use core::arch::asm;
100
101    use t2t_core::TickRate;
102
103    use super::plausible_rate;
104    use crate::CounterError;
105
106    /// The virtual counter, `cntvct_el0`.
107    #[inline(always)]
108    #[expect(clippy::inline_always, reason = "a call around the read would move it")]
109    pub(super) fn read() -> i64 {
110        let ticks: u64;
111        // SAFETY: `mrs` writes `ticks` alone and touches no memory, stack or flags, as the options
112        // say.
113        #[expect(unsafe_code, reason = "a system register is read only through inline assembly")]
114        unsafe {
115            asm!(
116                "mrs {ticks}, cntvct_el0",
117                ticks = out(reg) ticks,
118                options(nomem, nostack, preserves_flags),
119            );
120        }
121        ticks.cast_signed()
122    }
123
124    /// The counter's rate, `cntfrq_el0`, which firmware writes and so is checked.
125    ///
126    /// # Errors
127    /// [`CounterError::ImplausibleRate`], the register holds a rate outside 1 MHz to 10 GHz.
128    pub(super) fn discover_rate() -> Result<TickRate, CounterError> {
129        let hertz: u64;
130        // SAFETY: `mrs` writes `hertz` alone and touches no memory, stack or flags, as the options
131        // say.
132        #[expect(unsafe_code, reason = "a system register is read only through inline assembly")]
133        unsafe {
134            asm!(
135                "mrs {hertz}, cntfrq_el0",
136                hertz = out(reg) hertz,
137                options(nomem, nostack, preserves_flags),
138            );
139        }
140        plausible_rate(hertz)
141    }
142}
143
144/// The time-stamp counter, read with `rdtsc`, at the rate CPUID reports or one measured.
145#[cfg(target_arch = "x86_64")]
146mod arch {
147    use core::arch::x86_64::_rdtsc;
148    #[cfg(os_clocks)]
149    use core::hint;
150
151    use raw_cpuid::CpuId;
152    use t2t_core::TickRate;
153    #[cfg(os_clocks)]
154    use t2t_core::Timedelta;
155
156    use super::{plausible_rate, tsc};
157    use crate::CounterError;
158    #[cfg(os_clocks)]
159    use crate::{Clock, RawMonotonicClock};
160
161    /// The time-stamp counter.
162    #[inline(always)]
163    #[expect(clippy::inline_always, reason = "a call around the read would move it")]
164    pub(super) fn read() -> i64 {
165        // SAFETY: `_rdtsc` has no precondition: every `x86_64` CPU has `rdtsc`, and it needs no
166        // target feature.
167        #[expect(unsafe_code, reason = "`_rdtsc` is an `unsafe` intrinsic")]
168        let ticks = unsafe { _rdtsc() };
169        ticks.cast_signed()
170    }
171
172    /// The counter's rate, from CPUID, or measured where CPUID reports none.
173    ///
174    /// # Errors
175    /// What [`tsc::hertz`] refuses, and a rate outside 1 MHz to 10 GHz.
176    pub(super) fn discover_rate() -> Result<TickRate, CounterError> {
177        plausible_rate(tsc::hertz(&CpuId::new(), measure)?)
178    }
179
180    /// The counter's rate, measured over 10 ms against the monotonic clock at the hardware's rate.
181    #[cfg(os_clocks)]
182    fn measure() -> Option<u64> {
183        /// How long the measurement runs.
184        const WINDOW: Timedelta = Timedelta::from_millis(10);
185
186        let start_ticks = read();
187        let start = RawMonotonicClock.now();
188        let mut span = Timedelta::ZERO;
189        while span < WINDOW {
190            hint::spin_loop();
191            span = RawMonotonicClock.now() - start;
192        }
193        let ticks = i128::from(read().saturating_sub(start_ticks));
194        let second = i128::from(Timedelta::SECOND.as_nanos());
195        let hertz = ticks.checked_mul(second)?.checked_div(i128::from(span.as_nanos()))?;
196        u64::try_from(hertz).ok()
197    }
198
199    /// No clock to measure against.
200    #[cfg(not(os_clocks))]
201    const fn measure() -> Option<u64> {
202        None
203    }
204}
205
206/// What CPUID says of the time-stamp counter: whether it is invariant, and its rate.
207#[cfg(any(target_arch = "x86_64", test))]
208mod tsc {
209    use raw_cpuid::{CpuId, CpuIdReader};
210
211    use crate::CounterError;
212
213    /// Hertz in a kilohertz, the unit a hypervisor reports the counter's rate in.
214    const HERTZ_PER_KILOHERTZ: u64 = 1_000;
215
216    /// The counter's ticks a second, refused where the counter is not invariant.
217    ///
218    /// The rate is the CPU's, from leaf `0x15`; else the hypervisor's, from leaf `0x4000_0010`;
219    /// else the one `measure` takes.
220    ///
221    /// # Errors
222    /// - [`CounterError::NotInvariant`], the counter's rate follows the core's.
223    /// - [`CounterError::NoRate`], neither CPUID nor `measure` gives a rate.
224    pub(super) fn hertz<R, M>(cpuid: &CpuId<R>, measure: M) -> Result<u64, CounterError>
225    where
226        R: CpuIdReader,
227        M: FnOnce() -> Option<u64>,
228    {
229        let is_invariant =
230            cpuid.get_advanced_power_mgmt_info().is_some_and(|power| power.has_invariant_tsc());
231        if !is_invariant {
232            return Err(CounterError::NotInvariant);
233        }
234        let hypervisor_rate = || {
235            let kilohertz = cpuid.get_hypervisor_info()?.tsc_frequency()?;
236            u64::from(kilohertz).checked_mul(HERTZ_PER_KILOHERTZ)
237        };
238        cpuid
239            .get_tsc_info()
240            .and_then(|leaf| leaf.tsc_frequency())
241            .or_else(hypervisor_rate)
242            .filter(|&hertz| hertz > 0)
243            .or_else(measure)
244            .ok_or(CounterError::NoRate)
245    }
246
247    #[cfg(test)]
248    mod tests {
249        use raw_cpuid::{CpuId, CpuIdReader, CpuIdResult};
250
251        use super::hertz;
252        use crate::CounterError;
253
254        /// The highest basic and extended leaves, and the power leaf saying the counter is
255        /// invariant.
256        const INVARIANT: [(u32, [u32; 4]); 3] = [
257            (0, [0x15, 0, 0, 0]),
258            (0x8000_0000, [0x8000_0007, 0, 0, 0]),
259            (0x8000_0007, [0, 0, 0, 1 << 8]),
260        ];
261
262        /// A rate leaf reporting a 38.4 MHz crystal, and a ratio of 176 to 2 to it.
263        const RATE_LEAF: (u32, [u32; 4]) = (0x15, [2, 176, 38_400_000, 0]);
264
265        /// A CPU answering each of `leaves` with its EAX, EBX, ECX and EDX, and every other with
266        /// zeros.
267        fn cpu<const COUNT: usize>(leaves: [(u32, [u32; 4]); COUNT]) -> CpuId<impl CpuIdReader> {
268            CpuId::with_cpuid_reader(move |leaf, _subleaf| {
269                let answer = leaves.iter().find(|&&(number, _)| number == leaf);
270                let [eax, ebx, ecx, edx] = answer.map_or([0; 4], |&(_, registers)| registers);
271                CpuIdResult { eax, ebx, ecx, edx }
272            })
273        }
274
275        /// No measurement, as without `std`.
276        const fn no_measurement() -> Option<u64> {
277            None
278        }
279
280        #[test]
281        fn an_invariant_counter_runs_at_the_rate_its_leaf_reports() {
282            let [highest, extended, power] = INVARIANT;
283            let cpuid = cpu([highest, extended, power, RATE_LEAF]);
284            assert_eq!(hertz(&cpuid, no_measurement), Ok(3_379_200_000));
285        }
286
287        #[test]
288        fn a_counter_that_is_not_invariant_is_refused() {
289            let [highest, extended, _] = INVARIANT;
290            let variable_rate = cpu([highest, extended, RATE_LEAF]);
291            assert_eq!(hertz(&variable_rate, no_measurement), Err(CounterError::NotInvariant));
292
293            let [highest, _, power] = INVARIANT;
294            let old_cpu = cpu([highest, (0x8000_0000, [0x8000_0001, 0, 0, 0]), power, RATE_LEAF]);
295            assert_eq!(hertz(&old_cpu, no_measurement), Err(CounterError::NotInvariant));
296        }
297
298        #[test]
299        fn a_virtual_counter_runs_at_the_rate_its_hypervisor_reports() {
300            let [_, extended, power] = INVARIANT;
301            let cpuid = cpu([
302                (0, [0x0D, 0, 0, 0]),
303                (1, [0, 0, 1 << 31, 0]),
304                extended,
305                power,
306                (0x4000_0000, [0x4000_0010, 0, 0, 0]),
307                (0x4000_0010, [2_100_000, 0, 0, 0]),
308            ]);
309            assert_eq!(hertz(&cpuid, no_measurement), Ok(2_100_000_000));
310        }
311
312        #[test]
313        fn a_counter_whose_cpu_reports_no_rate_is_measured_or_refused() {
314            let silent_leaf = cpu(INVARIANT);
315            assert_eq!(hertz(&silent_leaf, || Some(2_100_000_000)), Ok(2_100_000_000));
316            assert_eq!(hertz(&silent_leaf, no_measurement), Err(CounterError::NoRate));
317
318            let [highest, extended, power] = INVARIANT;
319            let crystal_unknown = cpu([highest, extended, power, (0x15, [2, 168, 0, 0])]);
320            assert_eq!(hertz(&crystal_unknown, no_measurement), Err(CounterError::NoRate));
321        }
322    }
323}
324
325#[cfg(test)]
326mod tests {
327    use core::hint::black_box;
328
329    use t2t_core::{TickRate, Tickdelta, Timedelta};
330
331    use super::{PLAUSIBLE_RATES, plausible_rate};
332    use crate::{Clock, Counter, CounterError};
333
334    #[test]
335    fn a_rate_outside_the_plausible_range_is_refused() {
336        let refused = |hertz| Err(CounterError::ImplausibleRate { hertz });
337        assert_eq!(plausible_rate(0), refused(0), "a counter that never ticks");
338        assert_eq!(plausible_rate(999_999), refused(999_999), "one too slow to measure by");
339    }
340
341    #[test]
342    fn a_rate_a_counter_runs_at_is_taken() {
343        let apple_silicon = TickRate::from_hertz(24_000_000).expect("a nonzero rate");
344        assert_eq!(plausible_rate(24_000_000), Ok(apple_silicon), "Apple silicon's");
345        assert_eq!(plausible_rate(1_000_000_000), Ok(TickRate::GIGAHERTZ), "and Armv8.6's");
346    }
347
348    #[test]
349    #[cfg_attr(miri, ignore = "Miri cannot execute the counter's read")]
350    fn the_counter_never_runs_backwards() {
351        // A reading needs no rate.
352        let counter = Counter::new(TickRate::GIGAHERTZ);
353        let start = counter.now();
354        for value in 0_u64..10_000 {
355            black_box(value);
356        }
357        let span = counter.now() - start;
358        assert!(span >= Tickdelta::ZERO, "the later reading is no earlier: {span}");
359    }
360
361    #[test]
362    #[cfg_attr(miri, ignore = "Miri cannot execute the counter's read")]
363    fn the_discovered_counter_runs_at_a_plausible_rate() {
364        let discovered = Counter::discover();
365        // An `x86_64` CPU, or a virtual machine's, may not promise an invariant time-stamp counter,
366        // and may report no rate, which only the OS clocks can then measure: refusals `discover`
367        // promises, which the CI's virtual machines give.
368        let refused_by_the_cpu = match discovered {
369            Err(CounterError::NotInvariant) => cfg!(target_arch = "x86_64"),
370            Err(CounterError::NoRate) => cfg!(all(target_arch = "x86_64", not(os_clocks))),
371            Ok(_) | Err(CounterError::ImplausibleRate { .. }) => false,
372        };
373        if refused_by_the_cpu {
374            return;
375        }
376        let counter = discovered.expect("a counter with a rate");
377        let rate = counter.rate();
378        assert!(PLAUSIBLE_RATES.contains(&rate.as_hertz()), "a plausible rate: {rate}");
379
380        let start = counter.now();
381        for value in 0_u64..10_000 {
382            black_box(value);
383        }
384        let took = (counter.now() - start).to_timedelta(rate);
385        assert!(took < Timedelta::from_millis(10), "ten thousand loops in under 10 ms: {took}");
386    }
387}
t2t_clock/counter.rs

t2t_clock/
counter.rs
