Support hardware counters (instructions, cache misses, ...) (#983)

* Add hardware counters to JSON exports

* Clarify Linux hardware counter setup

src/timer/linux_counters.rs

@@ -0,0 +1,113 @@
+use std::fs::File;
+use std::io::Read;
+
+use crate::benchmark::measurement::HardwareCounters;
+
+/// Disabled events on the spawning thread, inherited and enabled by the child's exec.
+/// The parent never enables its own events, so its work is not counted.
+pub struct CounterGroup {
+    events: [Option<File>; 5],
+}
+
+impl CounterGroup {
+    #[cfg(all(
+        target_pointer_width = "64",
+        any(
+            target_arch = "aarch64",
+            target_arch = "x86_64",
+            target_arch = "riscv64",
+            target_arch = "powerpc64",
+            target_arch = "loongarch64"
+        )
+    ))]
+    pub fn new() -> Self {
+        use perf_event_open_sys::{bindings, perf_event_open};
+        use std::os::fd::{AsRawFd, FromRawFd};
+
+        let mut group_leader = -1;
+        let events = [
+            bindings::PERF_COUNT_HW_CPU_CYCLES,
+            bindings::PERF_COUNT_HW_INSTRUCTIONS,
+            bindings::PERF_COUNT_HW_CACHE_REFERENCES,
+            bindings::PERF_COUNT_HW_CACHE_MISSES,
+            bindings::PERF_COUNT_HW_BRANCH_MISSES,
+        ]
+        .map(|config| {
+            let mut attr = bindings::perf_event_attr {
+                type_: bindings::PERF_TYPE_HARDWARE,
+                size: std::mem::size_of::<bindings::perf_event_attr>() as u32,
+                config: u64::from(config),
+                ..Default::default()
+            };
+            // Don't start counting immediately.
+            attr.set_disabled(1);
+            // Start counting when the child executes the benchmark program.
+            attr.set_enable_on_exec(1);
+            // New child processes and threads inherit the counters.
+            attr.set_inherit(1);
+            // Exclude kernel execution.
+            attr.set_exclude_kernel(1);
+            // Exclude hypervisor execution.
+            attr.set_exclude_hv(1);
+
+            // Attach counters to the calling thread.
+            const CURRENT_THREAD: libc::pid_t = 0;
+            // Count on whichever CPU the task runs on.
+            const ANY_CPU: libc::c_int = -1;
+
+            // SAFETY: attr is initialized and sized for the kernel API.
+            let fd = unsafe {
+                perf_event_open(
+                    &mut attr,
+                    CURRENT_THREAD,
+                    ANY_CPU,
+                    group_leader,
+                    bindings::PERF_FLAG_FD_CLOEXEC.into(),
+                )
+            };
+            if fd < 0 {
+                return None;
+            }
+            // SAFETY: perf_event_open returned a new, valid descriptor.
+            let event = unsafe { File::from_raw_fd(fd) };
+            if group_leader == -1 {
+                group_leader = event.as_raw_fd();
+            }
+            Some(event)
+        });
+        Self { events }
+    }
+
+    #[cfg(not(all(
+        target_pointer_width = "64",
+        any(
+            target_arch = "aarch64",
+            target_arch = "x86_64",
+            target_arch = "riscv64",
+            target_arch = "powerpc64",
+            target_arch = "loongarch64"
+        )
+    )))]
+    pub fn new() -> Self {
+        Self {
+            events: Default::default(),
+        }
+    }
+
+    /// Read raw counts after the child has exited. Like poop, do not scale for multiplexing.
+    pub fn read(mut self) -> HardwareCounters {
+        let [cpu_cycles, instructions, cache_references, cache_misses, branch_misses] =
+            self.events.each_mut().map(|event| {
+                let mut bytes = [0; 8];
+                event.as_mut()?.read_exact(&mut bytes).ok()?;
+                Some(u64::from_ne_bytes(bytes))
+            });
+        HardwareCounters {
+            cpu_cycles,
+            instructions,
+            cache_references,
+            cache_misses,
+            branch_misses,
+        }
+    }
+}

src/timer/macos_counters.rs

@@ -0,0 +1,55 @@
+use std::io;
+use std::mem::MaybeUninit;
+use std::process::Child;
+
+use crate::benchmark::measurement::HardwareCounters;
+
+/// Wait for exit without reaping: proc_pid_rusage can still inspect the zombie,
+/// and the existing wait4 call can subsequently collect CPU times and peak RSS.
+pub fn wait(child: &mut Child) -> io::Result<()> {
+    drop(child.stdin.take());
+    let mut info = MaybeUninit::uninit();
+    loop {
+        // SAFETY: info points to writable siginfo_t storage. WNOWAIT preserves the child.
+        let result = unsafe {
+            libc::waitid(
+                libc::P_PID,
+                child.id(),
+                info.as_mut_ptr(),
+                libc::WEXITED | libc::WNOWAIT,
+            )
+        };
+        if result == 0 {
+            return Ok(());
+        }
+        let error = io::Error::last_os_error();
+        if error.kind() != io::ErrorKind::Interrupted {
+            return Err(error);
+        }
+    }
+}
+
+pub fn read(pid: u32) -> HardwareCounters {
+    let mut usage = MaybeUninit::<libc::rusage_info_v4>::zeroed();
+    // SAFETY: the buffer has the layout and size requested by RUSAGE_INFO_V4.
+    // Despite the pointer-to-pointer signature, this API writes directly into the buffer.
+    let result = unsafe {
+        libc::proc_pid_rusage(
+            pid as libc::pid_t,
+            libc::RUSAGE_INFO_V4,
+            usage.as_mut_ptr().cast(),
+        )
+    };
+    if result != 0 {
+        return HardwareCounters::default();
+    }
+    // SAFETY: the call succeeded and initialized the buffer.
+    let usage = unsafe { usage.assume_init() };
+    HardwareCounters {
+        // Like macOS time(1), omit zero counters: the API can succeed on systems
+        // where instruction/cycle accounting is unavailable.
+        cpu_cycles: (usage.ri_cycles != 0).then_some(usage.ri_cycles),
+        instructions: (usage.ri_instructions != 0).then_some(usage.ri_instructions),
+        ..Default::default()
+    }
+}

src/timer/mod.rs

@@ -1,5 +1,10 @@
 mod wall_clock_timer;
 
+#[cfg(target_os = "linux")]
+mod linux_counters;
+#[cfg(target_os = "macos")]
+mod macos_counters;
+
 #[cfg(windows)]
 mod windows_timer;
 
@@ -56,7 +61,12 @@ fn discard(output: ChildStdout) {
 }
 
 /// Execute the given command and return a timing summary
-pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
+pub fn execute_and_measure(
+    mut command: Command,
+    _collect_hardware_counters: bool,
+) -> Result<Measurement> {
+    #[cfg(target_os = "linux")]
+    let counters = _collect_hardware_counters.then(linux_counters::CounterGroup::new);
     #[cfg(not(windows))]
     let cpu_timer = self::unix_timer::CPUTimer::start();
 
@@ -82,14 +92,36 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
         discard(output);
     }
 
+    #[cfg(target_os = "macos")]
+    let (time_wall_clock, hardware_counters) =
+        if _collect_hardware_counters && macos_counters::wait(&mut child).is_ok() {
+            // Stop timing before querying counters, but leave the child available for wait4.
+            (
+                Some(wallclock_timer.stop()),
+                macos_counters::read(child.id()),
+            )
+        } else {
+            (None, Default::default())
+        };
     let (time_user, time_system, memory_peak_resident, exit_status) = cpu_timer.stop(child)?;
+    #[cfg(target_os = "macos")]
+    let time_wall_clock = time_wall_clock.unwrap_or_else(|| wallclock_timer.stop());
+    #[cfg(not(target_os = "macos"))]
     let time_wall_clock = wallclock_timer.stop();
 
+    #[cfg(target_os = "linux")]
+    let hardware_counters = counters
+        .map(linux_counters::CounterGroup::read)
+        .unwrap_or_default();
+    #[cfg(not(any(target_os = "linux", target_os = "macos")))]
+    let hardware_counters = Default::default();
+
     Ok(Measurement {
         time_wall_clock,
         time_user,
         time_system,
         memory_peak_resident,
+        hardware_counters,
         exit_status,
     })
 }