diff --git a/.github/workflows/cpufeatures.yml b/.github/workflows/cpufeatures.yml index 4f4dafc4..ce81ce2e 100644 --- a/.github/workflows/cpufeatures.yml +++ b/.github/workflows/cpufeatures.yml @@ -75,7 +75,6 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.toolchain }} - targets: x86_64-apple-darwin - run: cargo test # Windows tests @@ -98,7 +97,7 @@ jobs: targets: ${{ matrix.target }} - run: cargo test --target ${{ matrix.target }} - # Cross-compiled tests + # Linux ARM64 tests cross: strategy: matrix: @@ -108,7 +107,7 @@ jobs: rust: 1.85.0 # MSRV - target: aarch64-unknown-linux-gnu rust: stable - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v7 - uses: RustCrypto/actions/cargo-cache@master @@ -116,8 +115,7 @@ jobs: with: toolchain: ${{ matrix.rust }} targets: ${{ matrix.target }} - - uses: RustCrypto/actions/cross-install@master - - run: cross test --target ${{ matrix.target }} + - run: cargo test --target ${{ matrix.target }} # Build-only tests build-only: diff --git a/cpufeatures/CHANGELOG.md b/cpufeatures/CHANGELOG.md index ffb525af..8b646fea 100644 --- a/cpufeatures/CHANGELOG.md +++ b/cpufeatures/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Added +- Detect ARM64 NEON, CRC, CSSC, BF16, SVE, LSE2, MOPS and floating-point extensions. +- Detect x86 SIMD prerequisites, AVX2/AVX-512 extensions, XSAVE variants and ERMSB/PREFETCHW memory hints. + +### Fixed +- Return false for unavailable Apple capability keys. +- Check all required ARM64 capability bits and x86 CPU and OS state. +- Check CPUID limits before reading leaves and subleaves. +- Require an `unsafe` context to check register state from supplied CPUID data. + ## 0.3.1 (2026-08-26) ### Changed - Use compile-time target feature detection under Miri ([#1513]) diff --git a/cpufeatures/README.md b/cpufeatures/README.md index ed240f05..52305844 100644 --- a/cpufeatures/README.md +++ b/cpufeatures/README.md @@ -61,15 +61,41 @@ calls, thus runtime overhead for them is minimal. *NOTE: target features with an asterisk are unstable (nightly-only) and subject to change to match upstream name changes in the Rust standard library.* +Detection accepts these names even if enabling a feature needs a newer compiler. + ### `aarch64` -Linux, iOS, and macOS/ARM only (ARM64 does not support OS-independent feature detection) +Runtime detection is available on Linux, Android, iOS and macOS. Target features: - `aes`* +- `bf16` +- `crc` +- `cssc`* +- `f32mm` +- `f64mm` +- `faminmax`* +- `fcma` +- `fhm` +- `frintts` +- `i8mm` +- `jsconv` +- `lse2`* +- `lut`* +- `mops`* +- `neon` +- `rand` - `sha2`* - `sha3`* +- `sve` +- `sve-b16b16`* +- `sve2` +- `sve2-aes` +- `sve2-bitperm` +- `sve2-sha3` +- `sve2-sm4` +- `sve2p1`* ### `loongarch64` @@ -102,22 +128,39 @@ Target features: - `aes` - `avx` - `avx2` +- `avxifma` +- `avxneconvert` +- `avxvnni` +- `avxvnniint8` +- `avxvnniint16` +- `avx512bf16` - `avx512bw`* - `avx512cd`* - `avx512dq`* - `avx512er`* - `avx512f`* +- `avx512fp16` - `avx512ifma`* - `avx512pf`* - `avx512vl`* - `avx512vbmi`* - `avx512vbmi2`* +- `avx512vnni` +- `avx512vp2intersect` - `bmi1` - `bmi2` +- `cmpxchg16b` +- `ermsb`* +- `f16c` - `fma` +- `fxsr` +- `lahfsahf`* +- `lzcnt` - `mmx` +- `movbe` - `pclmulqdq` - `popcnt` +- `prfchw`* - `rdrand` - `rdseed` - `sgx` @@ -131,6 +174,10 @@ Target features: - `sha512` - `sm3` - `sm4` +- `xsave` +- `xsavec` +- `xsaveopt` +- `xsaves` If you would like detection support for a target feature which is not on this list, please [open a GitHub issue][github-issue]. diff --git a/cpufeatures/src/aarch64.rs b/cpufeatures/src/aarch64.rs index 24386621..174e52a7 100644 --- a/cpufeatures/src/aarch64.rs +++ b/cpufeatures/src/aarch64.rs @@ -6,6 +6,7 @@ // Evaluate the given `$body` expression any of the supplied target features // are not enabled. Otherwise returns true. +#[cfg(not(miri))] #[macro_export] #[doc(hidden)] macro_rules! __unless_target_features { @@ -22,23 +23,42 @@ macro_rules! __unless_target_features { // Linux runtime detection of target CPU features using `getauxval`. #[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(not(miri))] #[macro_export] #[doc(hidden)] macro_rules! __detect_target_features { ($($tf:tt),+) => {{ - let hwcaps = $crate::aarch64::getauxval_hwcap(); - $($crate::check!(hwcaps, $tf) & )+ true + let hwcap2 = if false $(|| $crate::check!(@hwcap2 $tf))+ { + $crate::aarch64::getauxval_hwcap2() + } else { 0 }; + let hwcaps = [$crate::aarch64::getauxval_hwcap(), hwcap2]; + $($crate::check!(@words hwcaps, $tf) & )+ true }}; } /// Linux helper function for calling `getauxval` to get `AT_HWCAP`. #[cfg(any(target_os = "linux", target_os = "android"))] +#[must_use] pub fn getauxval_hwcap() -> u64 { - unsafe { libc::getauxval(libc::AT_HWCAP) } + getauxval(libc::AT_HWCAP) +} + +/// Read `AT_HWCAP2` with `getauxval`. +#[cfg(any(target_os = "linux", target_os = "android"))] +#[must_use] +pub fn getauxval_hwcap2() -> u64 { + getauxval(libc::AT_HWCAP2) +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn getauxval(kind: libc::c_ulong) -> u64 { + // SAFETY: getauxval returns an integer and requires no pointers. + unsafe { libc::getauxval(kind) } } // Apple platform's runtime detection of target CPU features using `sysctlbyname`. #[cfg(target_vendor = "apple")] +#[cfg(not(miri))] #[macro_export] #[doc(hidden)] macro_rules! __detect_target_features { @@ -50,14 +70,28 @@ macro_rules! __detect_target_features { // Linux `expand_check_macro` #[cfg(any(target_os = "linux", target_os = "android"))] macro_rules! __expand_check_macro { - ($(($name:tt, $hwcap:ident)),* $(,)?) => { + ($(($name:tt, $hwcap:ident $(, $hwcap2:ident)?)),* $(,)?) => { #[macro_export] #[doc(hidden)] macro_rules! check { $( - ($hwcaps:expr, $name) => { - (($hwcaps & $crate::aarch64::hwcaps::$hwcap) != 0) - }; + (@hwcap2 $name) => { false $(|| $crate::aarch64::hwcaps::$hwcap2 != 0)? }; + ($hwcaps:expr, $name) => { $crate::check!(@words [$hwcaps, 0], $name) }; + (@words $hwcaps:expr, $name) => {{ + use $crate::aarch64::hwcaps as caps; + let [hwcap, hwcap2] = $hwcaps; + let mask = caps::$hwcap; + hwcap & mask == mask + $( && hwcap2 & caps::$hwcap2 == caps::$hwcap2)? + // When scalar FP16 is present, Rust also requires vector FP16. + && (mask & 3 != 3 || hwcap & (1 << 9) == 0 || hwcap & (1 << 10) != 0) + // When SVE is present, also check its BF16 and I8MM flags. + && match $name { + "bf16" | "sve-b16b16" if hwcap & caps::SVE == caps::SVE => hwcap2 & caps::SVE_BF16 != 0, + "i8mm" if hwcap & caps::SVE == caps::SVE => hwcap2 & caps::SVE_I8MM != 0, + _ => true, + } + }}; )* } }; @@ -66,6 +100,30 @@ macro_rules! __expand_check_macro { // Linux `expand_check_macro` #[cfg(any(target_os = "linux", target_os = "android"))] __expand_check_macro! { + ("neon", NEON), + ("crc", CRC), + ("cssc", NONE, CSSC), + ("bf16", NEON, BF16), + ("i8mm", NEON, I8MM), + ("fcma", FCMA), + ("fhm", FHM), + ("jsconv", JSCVT), + ("lse2", LSE2), + ("mops", NONE, MOPS), + ("sve-b16b16", NEON, SVE_B16B16), + ("frintts", NEON, FRINTTS), + ("faminmax", NEON, FAMINMAX), + ("lut", NEON, LUT), + ("rand", NONE, RAND), + ("sve", SVE), + ("sve2", SVE, SVE2), + ("sve2p1", SVE, SVE2P1), + ("sve2-aes", SVE_AES, SVE2_AES), + ("sve2-bitperm", SVE, SVE2_BITPERM), + ("sve2-sha3", SVE_SHA3, SVE2_SHA3), + ("sve2-sm4", SVE_SM4, SVE2_SM4), + ("f32mm", SVE, F32MM), + ("f64mm", SVE, F64MM), ("aes", AES), // Enable AES support. ("dit", DIT), // Enable Data-Independent Timing support. ("sha2", SHA2), // Enable SHA1 and SHA256 support. @@ -85,12 +143,45 @@ __expand_check_macro! { pub mod hwcaps { use libc::c_ulong; - pub const AES: c_ulong = libc::HWCAP_AES | libc::HWCAP_PMULL; + pub const NONE: c_ulong = 0; + pub const NEON: c_ulong = libc::HWCAP_FP | libc::HWCAP_ASIMD; + pub const AES: c_ulong = NEON | libc::HWCAP_AES | libc::HWCAP_PMULL; pub const DIT: c_ulong = libc::HWCAP_DIT; - pub const SHA2: c_ulong = libc::HWCAP_SHA2; - pub const SHA3: c_ulong = libc::HWCAP_SHA3 | libc::HWCAP_SHA512; - pub const SM4: c_ulong = libc::HWCAP_SM3 | libc::HWCAP_SM4; + pub const SHA2: c_ulong = NEON | libc::HWCAP_SHA1 | libc::HWCAP_SHA2; + pub const SHA3: c_ulong = SHA2 | libc::HWCAP_SHA3 | libc::HWCAP_SHA512; + pub const SM4: c_ulong = NEON | libc::HWCAP_SM3 | libc::HWCAP_SM4; pub const SB: c_ulong = libc::HWCAP_SB; + pub const CRC: c_ulong = libc::HWCAP_CRC32; + pub const LSE2: c_ulong = 1 << 25; + pub const FP16: c_ulong = NEON | libc::HWCAP_FPHP | libc::HWCAP_ASIMDHP; + pub const SVE: c_ulong = FP16 | libc::HWCAP_SVE; + pub const FCMA: c_ulong = NEON | (1 << 14); + pub const FHM: c_ulong = FP16 | (1 << 23); + pub const JSCVT: c_ulong = NEON | (1 << 13); + pub const SVE_AES: c_ulong = SVE | AES; + pub const SVE_SHA3: c_ulong = SVE | SHA3; + pub const SVE_SM4: c_ulong = SVE | SM4; + + // AT_HWCAP2 masks from asm/hwcap.h; not all libc targets expose them. + pub const CSSC: c_ulong = 1 << 34; + pub const MOPS: c_ulong = 1 << 43; + pub const BF16: c_ulong = 1 << 14; + pub const I8MM: c_ulong = 1 << 13; + pub const SVE_BF16: c_ulong = 1 << 12; + pub const SVE_B16B16: c_ulong = BF16 | (1 << 45); + pub const SVE_I8MM: c_ulong = 1 << 9; + pub const SVE2: c_ulong = 1 << 1; + pub const SVE2_AES: c_ulong = SVE2 | (1 << 2) | (1 << 3); + pub const SVE2_BITPERM: c_ulong = SVE2 | (1 << 4); + pub const SVE2_SHA3: c_ulong = SVE2 | (1 << 5); + pub const SVE2_SM4: c_ulong = SVE2 | (1 << 6); + pub const F32MM: c_ulong = 1 << 10; + pub const F64MM: c_ulong = 1 << 11; + pub const FRINTTS: c_ulong = 1 << 8; + pub const FAMINMAX: c_ulong = 1 << 50; + pub const LUT: c_ulong = 1 << 49; + pub const RAND: c_ulong = 1 << 16; + pub const SVE2P1: c_ulong = SVE2 | (1 << 36); } // Apple OS (macOS, iOS, watchOS, and tvOS) `check!` macro. @@ -108,33 +199,77 @@ pub mod hwcaps { #[macro_export] #[doc(hidden)] macro_rules! check { + ("neon") => { + true + }; + ("crc") => { + $crate::aarch64::sysctlbyname(b"hw.optional.armv8_crc32\0") + || $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_CRC32\0") + }; + ("cssc") => { + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_CSSC\0") + }; + ("bf16") => { + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_BF16\0") + }; + ("i8mm") => { + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_I8MM\0") + }; + ("fcma") => { + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_FCMA\0") + }; + ("fhm") => { + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_FP16\0") + && $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_FHM\0") + }; + ("jsconv") => { + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_JSCVT\0") + }; + ("sve-b16b16") => { + $crate::check!("bf16") + && $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_SVE_B16B16\0") + }; + ("frintts") => { + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_FRINTTS\0") + }; + ("lse2") => { + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_LSE2\0") + }; + // No public Apple capability key is documented for MOPS. + ("mops") => { false }; + ("faminmax") => { false }; + ("lut") => { false }; + ("rand") => { false }; + ("sve2p1") => { false }; + ("sve") => { false }; + ("sve2") => { false }; + ("sve2-aes") => { false }; + ("sve2-bitperm") => { false }; + ("sve2-sha3") => { false }; + ("sve2-sm4") => { false }; + ("f32mm") => { false }; + ("f64mm") => { false }; ("aes") => { true }; ("dit") => { // https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Enable-DIT-for-constant-time-cryptographic-operations - unsafe { - $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_DIT\0") - } + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_DIT\0") }; ("sha2") => { true }; ("sha3") => { - unsafe { - // `sha3` target feature implies SHA-512 as well - $crate::aarch64::sysctlbyname(b"hw.optional.armv8_2_sha512\0") - && $crate::aarch64::sysctlbyname(b"hw.optional.armv8_2_sha3\0") - } + // `sha3` target feature implies SHA-512 as well + $crate::aarch64::sysctlbyname(b"hw.optional.armv8_2_sha512\0") + && $crate::aarch64::sysctlbyname(b"hw.optional.armv8_2_sha3\0") }; ("sm4") => { false }; ("sb") => { // https://support.arm.com/documentation/ddi0487/mb/-Part-B-The-AArch64-Application-Level-Architecture/-Chapter-B2-The-AArch64-Application-Level-Memory-Model/-B2-6-Memory-barriers/-B2-6-3-Speculation-Barrier - unsafe { - $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_SB\0") - } + $crate::aarch64::sysctlbyname(b"hw.optional.arm.FEAT_SB\0") }; } @@ -146,42 +281,79 @@ macro_rules! check { /// If `name` is not NUL terminated #[cfg(target_vendor = "apple")] #[must_use] -pub unsafe fn sysctlbyname(name: &[u8]) -> bool { +pub fn sysctlbyname(name: &[u8]) -> bool { + sysctlbyname_with(name, |name, value, size| { + // SAFETY: The helper checks the name is NUL terminated and supplies an + // aligned, initialized u32 with its size. Null newp and zero newlen make + // this a read-only query. + unsafe { + libc::sysctlbyname( + name.as_ptr().cast::(), + core::ptr::from_mut(value).cast::(), + size, + core::ptr::null_mut(), + 0, + ) + } + }) +} + +#[cfg(target_vendor = "apple")] +fn sysctlbyname_with( + name: &[u8], + read: impl FnOnce(&[u8], &mut u32, &mut usize) -> libc::c_int, +) -> bool { assert_eq!( name.last().cloned(), Some(0), "name is not NUL terminated: {:?}", name ); - - let mut value: u32 = 0; + let mut value = 0; let mut size = size_of::(); + let rc = read(name, &mut value, &mut size); + rc == 0 && size == size_of::() && value != 0 +} + +#[cfg(all(test, target_vendor = "apple"))] +mod tests { + extern crate std; - // SAFETY: - // - `name` is being cast from a valid byte slice we asserted was NUL terminated above. - // - `value` is a properly-aligned, writable integer. - // - `size` is initialized to the size of `value` (4-bytes). - // - The last two arguments,`newp` and `newlen`, are for setting system parameters, which we - // aren't doing here (and requires root privileges). The docs say the following: - // - `newp`: "Specify NULL if you don’t want to set the attribute’s value" - // - `newlen`: "Specify 0 if you don’t want to set the attribute’s value" - let rc = unsafe { - libc::sysctlbyname( - name.as_ptr().cast::(), - (&raw mut value).cast::(), - &raw mut size, - core::ptr::null_mut(), - 0, - ) - }; - - assert_eq!(size, 4, "unexpected sysctlbyname(3) result size"); - assert_eq!(rc, 0, "sysctlbyname returned error code: {}", rc); - value != 0 + #[test] + fn sysctl_results() { + for (rc, size, value, expected) in [ + (0, 4, 1, true), + (0, 4, u32::MAX, true), + (0, 4, 0, false), + (-1, 4, 1, false), + (1, 4, 1, false), + (0, 0, 1, false), + (0, 3, 1, false), + (0, 5, 1, false), + (0, usize::MAX, 1, false), + ] { + assert_eq!( + super::sysctlbyname_with(b"test\0", |name, out, len| { + assert_eq!(name, b"test\0"); + assert_eq!((*out, *len), (0, 4)); + *out = value; + *len = size; + rc + }), + expected + ); + } + for name in [b"".as_slice(), b"not terminated"] { + assert!( + std::panic::catch_unwind(|| super::sysctlbyname_with(name, |_, _, _| 0)).is_err() + ); + } + } } // On other targets, runtime CPU feature detection is unavailable #[cfg(not(any(target_vendor = "apple", target_os = "linux", target_os = "android",)))] +#[cfg(not(miri))] #[macro_export] #[doc(hidden)] macro_rules! __detect_target_features { diff --git a/cpufeatures/src/lib.rs b/cpufeatures/src/lib.rs index a82add86..adf9a7a1 100644 --- a/cpufeatures/src/lib.rs +++ b/cpufeatures/src/lib.rs @@ -5,7 +5,6 @@ html_favicon_url = "https://raw.githubusercontent.com/RustCrypto/media/6ee8e381/logo.svg" )] -#[cfg(not(miri))] #[cfg(target_arch = "aarch64")] #[doc(hidden)] pub mod aarch64; @@ -15,7 +14,6 @@ pub mod aarch64; #[doc(hidden)] pub mod loongarch64; -#[cfg(not(miri))] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] mod x86; diff --git a/cpufeatures/src/miri.rs b/cpufeatures/src/miri.rs index 4d0c94da..ea715e3b 100644 --- a/cpufeatures/src/miri.rs +++ b/cpufeatures/src/miri.rs @@ -1,7 +1,6 @@ -//! Minimal miri support. +//! CPU detection fallbacks for Miri. //! -//! Miri is an interpreter, and though it tries to emulate the target CPU -//! it does not support any target features. +//! Runtime CPU detection stays disabled. Pure ARM64 and x86 guards remain available for tests with supplied CPU and OS data. #[macro_export] #[doc(hidden)] @@ -11,6 +10,7 @@ macro_rules! __unless_target_features { }; } +#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] #[macro_export] #[doc(hidden)] macro_rules! __detect_target_features { diff --git a/cpufeatures/src/x86.rs b/cpufeatures/src/x86.rs index bf857729..bde53de9 100644 --- a/cpufeatures/src/x86.rs +++ b/cpufeatures/src/x86.rs @@ -8,6 +8,7 @@ /// /// The `$body` expression is not evaluated on SGX targets, and returns false /// on these targets unless *all* supplied target features are enabled. +#[cfg(not(miri))] #[macro_export] #[doc(hidden)] macro_rules! __unless_target_features { @@ -32,29 +33,62 @@ macro_rules! __unless_target_features { #[macro_export] #[doc(hidden)] macro_rules! __detect_target_features { - ($($tf:tt),+) => {{ + (@cpuid $cpuid:expr, $extended:expr, $xstate:expr) => {{ #[cfg(target_arch = "x86")] - use core::arch::x86::{__cpuid, __cpuid_count, CpuidResult}; + use core::arch::x86::CpuidResult; #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__cpuid, __cpuid_count, CpuidResult}; - - unsafe fn cpuid(leaf: u32) -> CpuidResult { - __cpuid(leaf) - } - - unsafe fn cpuid_count(leaf: u32, sub_leaf: u32) -> CpuidResult { - __cpuid_count(leaf, sub_leaf) + use core::arch::x86_64::CpuidResult; + let cpuid = $cpuid; + let zero = CpuidResult { eax: 0, ebx: 0, ecx: 0, edx: 0 }; + let max = cpuid(0, 0).eax; + let basic = if max >= 1 { cpuid(1, 0) } else { zero }; + let leaf7 = if max >= 7 { cpuid(7, 0) } else { zero }; + let subleaf = if max >= 7 && leaf7.eax >= 1 { cpuid(7, 1) } else { zero }; + let extended = if $extended && cpuid(0x8000_0000, 0).eax >= 0x8000_0001 { cpuid(0x8000_0001, 0) } else { zero }; + let xstate = if $xstate && max >= 0xd && basic.ecx & (3 << 26) == 3 << 26 { cpuid(0xd, 1) } else { zero }; + [basic, leaf7, subleaf, extended, xstate] + }}; + ($($tf:tt),+) => {{ + #[cfg(miri)] + { false } + #[cfg(not(miri))] + { + #[cfg(target_arch = "x86")] + use core::arch::x86 as arch; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64 as arch; + let extended = false $(|| matches!($tf, "lzcnt" | "lahfsahf" | "prfchw"))+; + let xstate = false $(|| matches!($tf, "xsavec" | "xsaveopt" | "xsaves"))+; + // SAFETY: CPUID provides the real CPU flags used to guard XGETBV below. + unsafe { + let cr = $crate::__detect_target_features!(@cpuid arch::__cpuid_count, extended, xstate); + $($crate::check!(cr, $tf) & )+ true + } } - - let cr = unsafe { - [cpuid(1), cpuid_count(7, 0), cpuid_count(7, 1)] - }; - - $($crate::check!(cr, $tf) & )+ true }}; } -/// Check that OS supports required SIMD registers +/// Check that the OS supports the required SIMD registers. +/// +/// # Safety +/// The supplied CPUID flags must describe the current CPU and OS. +/// Safe callers cannot supply flags to the hardware query: +/// +/// ```compile_fail,E0133 +/// # #[cfg(target_arch = "x86")] use core::arch::x86::CpuidResult; +/// # #[cfg(target_arch = "x86_64")] use core::arch::x86_64::CpuidResult; +/// let cr = [CpuidResult { eax: 0, ebx: 0, ecx: 0, edx: 0 }; 5]; +/// cpufeatures::__xgetbv!(cr, 6); +/// ``` +/// +/// The feature check also requires an unsafe context when given flags: +/// +/// ```compile_fail,E0133 +/// # #[cfg(target_arch = "x86")] use core::arch::x86::CpuidResult; +/// # #[cfg(target_arch = "x86_64")] use core::arch::x86_64::CpuidResult; +/// let cr = [CpuidResult { eax: 0, ebx: 0, ecx: 0, edx: 0 }; 5]; +/// cpufeatures::check!(cr, "avx"); +/// ``` #[macro_export] #[doc(hidden)] macro_rules! __xgetbv { @@ -64,11 +98,14 @@ macro_rules! __xgetbv { #[cfg(target_arch = "x86_64")] use core::arch::x86_64 as arch; - // Check bits 26 and 27 + $crate::__xgetbv!(@read $cr, $mask, arch::_xgetbv(arch::_XCR_XFEATURE_ENABLED_MASK)) + }}; + (@read $cr:expr, $mask:expr, $xcr0:expr) => {{ + // Check XSAVE and OSXSAVE before reading XCR0. let xmask = 0b11 << 26; let xsave = $cr[0].ecx & xmask == xmask; if xsave { - let xcr0 = unsafe { arch::_xgetbv(arch::_XCR_XFEATURE_ENABLED_MASK) }; + let xcr0 = $xcr0; (xcr0 & $mask) == $mask } else { false @@ -83,21 +120,27 @@ macro_rules! __expand_check_macro { macro_rules! check { $( ($cr:expr, $name) => {{ - // Register bits are listed here: - // https://wiki.osdev.org/CPU_Registers_x86#Extended_Control_Registers - let reg_cap = match $reg_cap { - // Bit 1 - "xmm" => $crate::__xgetbv!($cr, 0b10), - // Bits 1 and 2 - "ymm" => $crate::__xgetbv!($cr, 0b110), - // Bits 1, 2, 5, 6, and 7 - "zmm" => $crate::__xgetbv!($cr, 0b1110_0110), + let cr = $cr; + $crate::check!(cr, $name, |mask| $crate::__xgetbv!(cr, mask)) + }}; + ($cr:expr, $name, $os:expr) => {{ + let cr = $cr; + // Check Rust's requirements for AVX and AVX-512. + let avx = (1 << 0) | (1 << 9) | (1 << 19) | (1 << 20) + | (1 << 26) | (1 << 27) | (1 << 28); + let sse = (1 << 25) | (1 << 26); + let registers = match $reg_cap { + "xsave" => cr[0].ecx & (3 << 26) == 3 << 26, + "sse2" => cr[0].edx & sse == sse, + "ymm" | "zmm" => cr[0].ecx & avx == avx && cr[0].edx & sse == sse, _ => true, }; - reg_cap - $( - & ($cr[$i].$reg & (1 << $offset) != 0) - )* + let mask = match $reg_cap { "ymm" => 0b110, "zmm" => 0b1110_0110, _ => 0 }; + registers + $(& (cr[$i].$reg & (1 << $offset) != 0))* + && ($reg_cap != "zmm" || (cr[0].ecx & ((1 << 12) | (1 << 29)) == ((1 << 12) | (1 << 29)) + && cr[1].ebx & ((1 << 5) | (1 << 16)) == ((1 << 5) | (1 << 16)))) + && (mask == 0 || ($os)(mask)) }}; )* } @@ -109,20 +152,26 @@ __expand_check_macro! { ("pclmulqdq", "", 0, ecx, 1), ("ssse3", "", 0, ecx, 9), ("fma", "ymm", 0, ecx, 12, 0, ecx, 28), + ("cmpxchg16b", "", 0, ecx, 13), ("sse4.1", "", 0, ecx, 19), ("sse4.2", "", 0, ecx, 20), + ("movbe", "", 0, ecx, 22), ("popcnt", "", 0, ecx, 23), ("aes", "", 0, ecx, 25), + ("xsave", "xsave"), ("avx", "ymm", 0, ecx, 28), + ("f16c", "ymm", 0, ecx, 29), ("rdrand", "", 0, ecx, 30), ("mmx", "", 0, edx, 23), + ("fxsr", "", 0, edx, 24), ("sse", "", 0, edx, 25), ("sse2", "", 0, edx, 26), ("sgx", "", 1, ebx, 2), ("bmi1", "", 1, ebx, 3), ("bmi2", "", 1, ebx, 8), + ("ermsb", "", 1, ebx, 9), ("avx2", "ymm", 1, ebx, 5, 0, ecx, 28), ("avx512f", "zmm", 1, ebx, 16), ("avx512dq", "zmm", 1, ebx, 17), @@ -135,15 +184,32 @@ __expand_check_macro! { ("sha", "", 1, ebx, 29), ("avx512bw", "zmm", 1, ebx, 30), ("avx512vl", "zmm", 1, ebx, 31), - ("avx512vbmi", "zmm", 1, ecx, 1), - ("avx512vbmi2", "zmm", 1, ecx, 6), - ("gfni", "zmm", 1, ecx, 8), - ("vaes", "zmm", 1, ecx, 9), - ("vpclmulqdq", "zmm", 1, ecx, 10), - ("avx512bitalg", "zmm", 1, ecx, 12), + ("avx512vnni", "zmm", 1, ecx, 11), + ("avx512vp2intersect", "zmm", 1, edx, 8), + ("avx512fp16", "zmm", 1, edx, 23, 1, ebx, 30), + ("avx512vbmi", "zmm", 1, ecx, 1, 1, ebx, 30), + ("avx512vbmi2", "zmm", 1, ecx, 6, 1, ebx, 30), + ("gfni", "sse2", 1, ecx, 8), + ("vaes", "ymm", 1, ecx, 9, 0, ecx, 25, 1, ebx, 5), + ("vpclmulqdq", "ymm", 1, ecx, 10, 0, ecx, 1), + ("avx512bitalg", "zmm", 1, ecx, 12, 1, ebx, 30), ("avx512vpopcntdq", "zmm", 1, ecx, 14), - ("sha512", "ymm", 2, eax, 0), - ("sm3", "xmm", 2, eax, 1), - ("sm4", "ymm", 2, eax, 2), + ("sha512", "ymm", 2, eax, 0, 1, ebx, 5), + ("sm3", "ymm", 2, eax, 1), + ("sm4", "ymm", 2, eax, 2, 1, ebx, 5), + ("avxvnni", "ymm", 2, eax, 4, 1, ebx, 5), + ("avx512bf16", "zmm", 2, eax, 5, 1, ebx, 30), + ("avxifma", "ymm", 2, eax, 23, 1, ebx, 5), + ("avxvnniint8", "ymm", 2, edx, 4, 1, ebx, 5), + ("avxneconvert", "ymm", 2, edx, 5, 1, ebx, 5), + ("avxvnniint16", "ymm", 2, edx, 10, 1, ebx, 5), + + ("lahfsahf", "", 3, ecx, 0), + ("lzcnt", "", 3, ecx, 5), + ("prfchw", "", 3, ecx, 8), + + ("xsaveopt", "xsave", 4, eax, 0), + ("xsavec", "xsave", 4, eax, 1), + ("xsaves", "xsave", 4, eax, 3), } diff --git a/cpufeatures/tests/aarch64.rs b/cpufeatures/tests/aarch64.rs index 41a61233..33a03ca4 100644 --- a/cpufeatures/tests/aarch64.rs +++ b/cpufeatures/tests/aarch64.rs @@ -14,4 +14,152 @@ fn init() { fn init_get() { let (token, val) = armcaps::init_get(); assert_eq!(val, token.get()); + #[cfg(miri)] + const { + assert!(!cpufeatures::__detect_target_features!("neon")); + }; +} + +#[test] +fn neon_matches_std() { + cpufeatures::new!(neon, "neon"); + assert_eq!(neon::get(), std::arch::is_aarch64_feature_detected!("neon")); +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +#[test] +fn capability_masks() { + macro_rules! check { + ($feature:tt, $mask:expr $(, $mask2:expr)?) => {{ + let full: [u64; 2] = [$mask, 0 $(| $mask2)?]; + assert_eq!(cpufeatures::check!(@hwcap2 $feature), full[1] != 0, $feature); + assert_eq!(cpufeatures::check!(full[0], $feature), full[1] == 0, $feature); + assert!(cpufeatures::check!(@words full, $feature), $feature); + assert!(cpufeatures::check!(@words [u64::MAX; 2], $feature), $feature); + assert!(!cpufeatures::check!(@words full.map(|word| !word), $feature), $feature); + for word in 0..2 { + for bit in 0..64 { + if full[word] & (1 << bit) != 0 { + let mut missing = full; + missing[word] &= !(1 << bit); + assert!(!cpufeatures::check!(@words missing, $feature), $feature); + } + } + } + if full[0] & 3 == 3 && full[0] & (1 << 9) == 0 { + assert!(!cpufeatures::check!(@words [full[0] | (1 << 9), full[1]], $feature), $feature); + } + }}; + } + check!("neon", 3); + check!("fcma", 3 | (1 << 14)); + check!("fhm", 3 | (1 << 9) | (1 << 10) | (1 << 23)); + check!("jsconv", 3 | (1 << 13)); + check!("sve-b16b16", 3, (1 << 14) | (1 << 45)); + const SVE: u64 = 3 | (1 << 9) | (1 << 10) | (1 << 22); + check!("crc", 1 << 7); + check!("cssc", 0, 1 << 34); + check!("lse2", 1 << 25); + check!("mops", 0, 1 << 43); + check!("bf16", 3, 1 << 14); + check!("i8mm", 3, 1 << 13); + check!("sve", SVE); + check!("sve2", SVE, 1 << 1); + check!( + "sve2-aes", + SVE | (1 << 3) | (1 << 4), + (1 << 1) | (1 << 2) | (1 << 3) + ); + check!("sve2-bitperm", SVE, (1 << 1) | (1 << 4)); + check!( + "sve2-sha3", + SVE | (1 << 5) | (1 << 6) | (1 << 17) | (1 << 21), + (1 << 1) | (1 << 5) + ); + check!("sve2-sm4", SVE | (1 << 18) | (1 << 19), (1 << 1) | (1 << 6)); + check!("f32mm", SVE, 1 << 10); + check!("f64mm", SVE, 1 << 11); + check!("frintts", 3, 1 << 8); + check!("faminmax", 3, 1 << 50); + check!("lut", 3, 1 << 49); + check!("rand", 0, 1 << 16); + check!("sve2p1", SVE, (1 << 1) | (1 << 36)); + check!("aes", 3 | (1 << 3) | (1 << 4)); + check!("sha2", 3 | (1 << 5) | (1 << 6)); + check!("sha3", 3 | (1 << 5) | (1 << 6) | (1 << 17) | (1 << 21)); + check!("sm4", 3 | (1 << 18) | (1 << 19)); +} + +#[cfg(not(miri))] +#[test] +fn extended_detection() { + let runtime = cfg!(any( + target_vendor = "apple", + target_os = "linux", + target_os = "android" + )); + macro_rules! check { + ($($feature:tt),+) => {$({ + cpufeatures::new!(detected, $feature); + assert_eq!(detected::get(), cfg!(target_feature = $feature) || (runtime && std::arch::is_aarch64_feature_detected!($feature)), $feature); + })+}; + } + check!( + "crc", + "bf16", + "i8mm", + "sve", + "sve2", + "sve2-aes", + "sve2-bitperm", + "sve2-sha3", + "sve2-sm4", + "f32mm", + "f64mm", + "fcma", + "fhm", + "jsconv", + "lse2", + "frintts", + "rand" + ); +} + +#[cfg(not(miri))] +#[test] +fn recent_detection() { + macro_rules! check { + ($($feature:tt),+) => {{ + cpufeatures::new!(combined, $($feature),+); + assert_eq!(combined::get(), true $(& (cfg!(target_feature = $feature) || cpufeatures::__detect_target_features!($feature)))+); + } $({ + cpufeatures::new!(detected, $feature); + assert_eq!(detected::get(), cfg!(target_feature = $feature) || cpufeatures::__detect_target_features!($feature), $feature); + })+}; + } + check!("cssc", "faminmax", "lut", "mops", "sve2p1", "sve-b16b16"); + #[cfg(target_vendor = "apple")] + assert!(!cpufeatures::aarch64::sysctlbyname( + b"hw.optional.cpufeatures_missing\0" + )); +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +#[test] +fn optional_sve_components() { + macro_rules! check { + ($feature:tt, $bit:expr, $sve_bit:expr $(, $extra:expr)?) => { + for sve in [false, true] { + let hwcap = if sve { 3 | (1 << 9) | (1 << 10) | (1 << 22) } else { 3 }; + for bits in [0, 1 << $bit, 1 << $sve_bit, (1 << $bit) | (1 << $sve_bit)] { + let hwcap2 = bits $(| $extra)?; + assert_eq!(cpufeatures::check!(@words [hwcap, hwcap2], $feature), + hwcap2 & (1 << $bit) != 0 && (!sve || hwcap2 & (1 << $sve_bit) != 0), $feature); + } + } + }; + } + check!("bf16", 14, 12); + check!("sve-b16b16", 14, 12, 1 << 45); + check!("i8mm", 13, 9); } diff --git a/cpufeatures/tests/x86.rs b/cpufeatures/tests/x86.rs index 8b1692b2..16c46d4e 100644 --- a/cpufeatures/tests/x86.rs +++ b/cpufeatures/tests/x86.rs @@ -14,4 +14,264 @@ fn init() { fn init_get() { let (token, val) = cpuid::init_get(); assert_eq!(val, token.get()); + #[cfg(miri)] + const { + assert!(!cpufeatures::__detect_target_features!("sse2")); + }; +} + +mod detection { + #[cfg(target_arch = "x86")] + use core::arch::x86::CpuidResult; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::CpuidResult; + + const fn registers(bits: u32) -> [CpuidResult; 5] { + [CpuidResult { + eax: bits, + ebx: bits, + ecx: bits, + edx: bits, + }; 5] + } + + #[test] + fn cpuid_bounds() { + for (max, subleaf, expected) in [(0, 0, 1), (1, 0, 2), (7, 0, 3), (7, 1, 4)] { + let calls = core::cell::RefCell::new(Vec::new()); + let cr = cpufeatures::__detect_target_features!(@cpuid |leaf: u32, sub: u32| { + calls.borrow_mut().push((leaf, sub)); + let eax = match (leaf, sub) { (0, 0) => max, (7, 0) => subleaf, _ => u32::MAX }; + CpuidResult { eax, ebx: u32::MAX, ecx: u32::MAX, edx: u32::MAX } + }, false, false); + assert_eq!( + calls.into_inner(), + [(0, 0), (1, 0), (7, 0), (7, 1)][..expected] + ); + for (i, available) in [max >= 1, max >= 7, max >= 7 && subleaf >= 1] + .into_iter() + .enumerate() + { + assert_eq!(cr[i].ebx, if available { u32::MAX } else { 0 }); + } + } + } + + #[test] + fn extended_cpuid_bounds() { + for max in [0, 0x8000_0000, 0x8000_0001] { + let calls = core::cell::Cell::new(0); + let cr = cpufeatures::__detect_target_features!(@cpuid |leaf: u32, _: u32| { + calls.set(calls.get() + 1); + let mut result = registers(0)[0]; + match leaf { + 0 => (), + 0x8000_0000 => result.eax = max, + 0x8000_0001 => { assert!(max >= leaf); result.ecx = u32::MAX; }, + _ => panic!("unexpected CPUID leaf"), + } + result + }, true, false); + let available = max >= 0x8000_0001; + assert_eq!(cr[3].ecx, if available { u32::MAX } else { 0 }); + assert_eq!(calls.get(), if available { 3 } else { 2 }); + } + } + + #[test] + fn xsave_cpuid_bounds() { + for max in [0, 1, 7, 0xc, 0xd] { + for requested in [false, true] { + for flags in [0, 1 << 26, 1 << 27, 3 << 26] { + let calls = core::cell::Cell::new(0); + let cr = cpufeatures::__detect_target_features!(@cpuid |leaf: u32, sub: u32| { + let mut result = registers(0)[0]; + match leaf { + 0 => result.eax = max, + 1 => result.ecx = flags, + 7 => (), + 0xd => { assert_eq!(sub, 1); calls.set(calls.get() + 1); result.eax = 0xb; }, + _ => panic!("unexpected CPUID leaf"), + } + result + }, false, requested); + let available = requested && max >= 0xd && flags == 3 << 26; + assert_eq!(cr[4].eax, if available { 0xb } else { 0 }); + assert_eq!(calls.get(), u32::from(available)); + } + } + } + } + + #[test] + fn xgetbv_guard() { + for (flags, enabled) in [ + (0, false), + (1 << 26, false), + (1 << 27, false), + (3 << 26, true), + ] { + let mut cr = registers(0); + cr[0].ecx = flags; + for (mask, state, supported) in [ + (0, 0, true), + (6, 0, false), + (6, 2, false), + (6, 4, false), + (6, 6, true), + (0xe6, 6, false), + (0xe6, 0xe2, false), + (0xe6, 0xe4, false), + (0xe6, 0xc6, false), + (0xe6, 0xa6, false), + (0xe6, 0x66, false), + (0xe6, 0xe6, true), + (0xe6, u64::MAX, true), + ] { + let reads = core::cell::Cell::new(0); + assert_eq!( + cpufeatures::__xgetbv!(@read cr, mask, { + reads.set(reads.get() + 1); + state + }), + enabled && supported + ); + assert_eq!(reads.get(), u32::from(enabled)); + } + } + } + + #[test] + fn required_bits_and_state() { + macro_rules! check { + ($feature:tt, $state:expr; $($i:expr, $reg:ident, $bit:expr);+ $(;)?) => {{ + let cr = registers(u32::MAX); + let mut sparse = registers(0); + if $state != 0 { + sparse[0].ecx = (1 << 0) | (1 << 9) | (1 << 19) | (1 << 20) | (1 << 26) | (1 << 27) | (1 << 28); + sparse[0].edx = (1 << 25) | (1 << 26); + } + if $state == 0xe6 { + sparse[0].ecx |= (1 << 12) | (1 << 29); + sparse[1].ebx = (1 << 5) | (1 << 16); + } + $(sparse[$i].$reg |= 1 << $bit;)+ + assert!(cpufeatures::check!(sparse, $feature, |_| true), $feature); + $(let mut missing = cr; + missing[$i].$reg &= !(1 << $bit); + assert!(!cpufeatures::check!(missing, $feature, |_| panic!("XGETBV queried")), $feature);)+ + for state in [0, 2, 4, 6, 0x66, 0xa6, 0xc6, 0xe2, 0xe4, 0xe6] { + assert_eq!(cpufeatures::check!(cr, $feature, |mask| state & mask == mask), state & $state == $state, $feature); + } + if $state == 0 { + assert!(cpufeatures::check!(sparse, $feature, |_| panic!("XGETBV queried")), $feature); + } + if $state != 0 { + for bit in [0, 9, 12, 19, 20, 26, 27, 28, 29] { + if $state != 0xe6 && matches!(bit, 12 | 29) { continue; } + let mut missing = cr; + missing[0].ecx &= !(1 << bit); + assert!(!cpufeatures::check!(missing, $feature, |_| panic!("XGETBV queried")), $feature); + } + for bit in [25, 26] { + let mut missing = cr; + missing[0].edx &= !(1 << bit); + assert!(!cpufeatures::check!(missing, $feature, |_| panic!("XGETBV queried")), $feature); + } + if $state == 0xe6 { + for bit in [5, 16] { + let mut missing = cr; + missing[1].ebx &= !(1 << bit); + assert!(!cpufeatures::check!(missing, $feature, |_| panic!("XGETBV queried")), $feature); + } + } + } + }}; + } + check!("ermsb", 0; 1, ebx, 9); + check!("prfchw", 0; 3, ecx, 8); + check!("fxsr", 0; 0, edx, 24); + check!("cmpxchg16b", 0; 0, ecx, 13); + check!("movbe", 0; 0, ecx, 22); + check!("lahfsahf", 0; 3, ecx, 0); + check!("lzcnt", 0; 3, ecx, 5); + check!("xsave", 0; 0, ecx, 26; 0, ecx, 27); + check!("xsaveopt", 0; 0, ecx, 26; 0, ecx, 27; 4, eax, 0); + check!("xsavec", 0; 0, ecx, 26; 0, ecx, 27; 4, eax, 1); + check!("xsaves", 0; 0, ecx, 26; 0, ecx, 27; 4, eax, 3); + check!("f16c", 6; 0, ecx, 29); + check!("avxvnni", 6; 2, eax, 4; 1, ebx, 5); + check!("avxifma", 6; 2, eax, 23; 1, ebx, 5); + check!("avxneconvert", 6; 2, edx, 5; 1, ebx, 5); + check!("avxvnniint8", 6; 2, edx, 4; 1, ebx, 5); + check!("avxvnniint16", 6; 2, edx, 10; 1, ebx, 5); + check!("avx512vnni", 0xe6; 1, ecx, 11); + check!("avx512vp2intersect", 0xe6; 1, edx, 8); + check!("avx512bf16", 0xe6; 2, eax, 5; 1, ebx, 30); + check!("avx512fp16", 0xe6; 1, edx, 23; 1, ebx, 30); + check!("gfni", 0; 1, ecx, 8; 0, edx, 25; 0, edx, 26); + check!("vaes", 6; 1, ecx, 9; 0, ecx, 25; 1, ebx, 5); + check!("vpclmulqdq", 6; 1, ecx, 10; 0, ecx, 1); + check!("sha512", 6; 2, eax, 0; 1, ebx, 5); + check!("sm3", 6; 2, eax, 1); + check!("sm4", 6; 2, eax, 2; 1, ebx, 5); + check!("avx512f", 0xe6; 1, ebx, 16); + check!("avx512dq", 0xe6; 1, ebx, 17); + check!("avx512ifma", 0xe6; 1, ebx, 21); + check!("avx512pf", 0xe6; 1, ebx, 26); + check!("avx512er", 0xe6; 1, ebx, 27); + check!("avx512cd", 0xe6; 1, ebx, 28); + check!("avx512bw", 0xe6; 1, ebx, 30); + check!("avx512vl", 0xe6; 1, ebx, 31); + check!("avx512vbmi", 0xe6; 1, ecx, 1; 1, ebx, 30); + check!("avx512vbmi2", 0xe6; 1, ecx, 6; 1, ebx, 30); + check!("avx512bitalg", 0xe6; 1, ecx, 12; 1, ebx, 30); + check!("avx512vpopcntdq", 0xe6; 1, ecx, 14); + } + + #[cfg(all( + not(miri), + not(any(target_env = "sgx", target_os = "none", target_os = "uefi")) + ))] + #[test] + fn cached_detection() { + macro_rules! check { + ($($feature:tt),+) => {{ + cpufeatures::new!(combined, $($feature),+); + assert_eq!(combined::get(), true $(& (cfg!(target_feature = $feature) || cpufeatures::__detect_target_features!($feature)))+); + } $({ + cpufeatures::new!(detected, $feature); + assert_eq!(detected::get(), cfg!(target_feature = $feature) || cpufeatures::__detect_target_features!($feature), $feature); + })+}; + } + check!("lahfsahf", "ermsb", "prfchw"); + check!("xsave", "xsavec", "xsaveopt", "xsaves"); + check!( + "avxvnni", + "avxifma", + "avxneconvert", + "avxvnniint8", + "avxvnniint16" + ); + macro_rules! standard { + ($($feature:tt),+) => {$({ + cpufeatures::new!(detected, $feature); + assert_eq!(detected::get(), std::arch::is_x86_feature_detected!($feature), $feature); + })+}; + } + // Std requires AVX register state for XSAVE. The instructions do not. + if std::arch::is_x86_feature_detected!("avx") { + standard!("xsave", "xsavec", "xsaveopt", "xsaves"); + } + standard!("fxsr", "cmpxchg16b", "f16c", "movbe", "lzcnt"); + standard!( + "gfni", + "vaes", + "vpclmulqdq", + "avx512vnni", + "avx512vp2intersect", + "avx512bf16", + "avx512fp16" + ); + } }