std_detect/detect/os/linux/
riscv.rs

1//! Run-time feature detection for RISC-V on Linux.
2//!
3//! On RISC-V, detection using auxv only supports single-letter extensions.
4//! So, we use riscv_hwprobe that supports multi-letter extensions if available.
5//! <https://www.kernel.org/doc/html/latest/arch/riscv/hwprobe.html>
6
7use core::ptr;
8
9use super::super::riscv::imply_features;
10use super::auxvec;
11use crate::detect::{Feature, bit, cache};
12
13// See <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/prctl.h?h=v6.16>
14// for runtime status query constants.
15const PR_RISCV_V_GET_CONTROL: libc::c_int = 70;
16const PR_RISCV_V_VSTATE_CTRL_ON: libc::c_int = 2;
17const PR_RISCV_V_VSTATE_CTRL_CUR_MASK: libc::c_int = 3;
18
19// See <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/include/uapi/asm/hwprobe.h?h=v6.16>
20// for riscv_hwprobe struct and hardware probing constants.
21
22#[repr(C)]
23struct riscv_hwprobe {
24    key: i64,
25    value: u64,
26}
27
28impl riscv_hwprobe {
29    // key is overwritten to -1 if not supported by riscv_hwprobe syscall.
30    pub fn get(&self) -> Option<u64> {
31        (self.key != -1).then_some(self.value)
32    }
33}
34
35#[allow(non_upper_case_globals)]
36const __NR_riscv_hwprobe: libc::c_long = 258;
37
38const RISCV_HWPROBE_KEY_BASE_BEHAVIOR: i64 = 3;
39const RISCV_HWPROBE_BASE_BEHAVIOR_IMA: u64 = 1 << 0;
40
41const RISCV_HWPROBE_KEY_IMA_EXT_0: i64 = 4;
42const RISCV_HWPROBE_IMA_FD: u64 = 1 << 0;
43const RISCV_HWPROBE_IMA_C: u64 = 1 << 1;
44const RISCV_HWPROBE_IMA_V: u64 = 1 << 2;
45const RISCV_HWPROBE_EXT_ZBA: u64 = 1 << 3;
46const RISCV_HWPROBE_EXT_ZBB: u64 = 1 << 4;
47const RISCV_HWPROBE_EXT_ZBS: u64 = 1 << 5;
48const RISCV_HWPROBE_EXT_ZICBOZ: u64 = 1 << 6;
49const RISCV_HWPROBE_EXT_ZBC: u64 = 1 << 7;
50const RISCV_HWPROBE_EXT_ZBKB: u64 = 1 << 8;
51const RISCV_HWPROBE_EXT_ZBKC: u64 = 1 << 9;
52const RISCV_HWPROBE_EXT_ZBKX: u64 = 1 << 10;
53const RISCV_HWPROBE_EXT_ZKND: u64 = 1 << 11;
54const RISCV_HWPROBE_EXT_ZKNE: u64 = 1 << 12;
55const RISCV_HWPROBE_EXT_ZKNH: u64 = 1 << 13;
56const RISCV_HWPROBE_EXT_ZKSED: u64 = 1 << 14;
57const RISCV_HWPROBE_EXT_ZKSH: u64 = 1 << 15;
58const RISCV_HWPROBE_EXT_ZKT: u64 = 1 << 16;
59const RISCV_HWPROBE_EXT_ZVBB: u64 = 1 << 17;
60const RISCV_HWPROBE_EXT_ZVBC: u64 = 1 << 18;
61const RISCV_HWPROBE_EXT_ZVKB: u64 = 1 << 19;
62const RISCV_HWPROBE_EXT_ZVKG: u64 = 1 << 20;
63const RISCV_HWPROBE_EXT_ZVKNED: u64 = 1 << 21;
64const RISCV_HWPROBE_EXT_ZVKNHA: u64 = 1 << 22;
65const RISCV_HWPROBE_EXT_ZVKNHB: u64 = 1 << 23;
66const RISCV_HWPROBE_EXT_ZVKSED: u64 = 1 << 24;
67const RISCV_HWPROBE_EXT_ZVKSH: u64 = 1 << 25;
68const RISCV_HWPROBE_EXT_ZVKT: u64 = 1 << 26;
69const RISCV_HWPROBE_EXT_ZFH: u64 = 1 << 27;
70const RISCV_HWPROBE_EXT_ZFHMIN: u64 = 1 << 28;
71const RISCV_HWPROBE_EXT_ZIHINTNTL: u64 = 1 << 29;
72const RISCV_HWPROBE_EXT_ZVFH: u64 = 1 << 30;
73const RISCV_HWPROBE_EXT_ZVFHMIN: u64 = 1 << 31;
74const RISCV_HWPROBE_EXT_ZFA: u64 = 1 << 32;
75const RISCV_HWPROBE_EXT_ZTSO: u64 = 1 << 33;
76const RISCV_HWPROBE_EXT_ZACAS: u64 = 1 << 34;
77const RISCV_HWPROBE_EXT_ZICOND: u64 = 1 << 35;
78const RISCV_HWPROBE_EXT_ZIHINTPAUSE: u64 = 1 << 36;
79const RISCV_HWPROBE_EXT_ZVE32X: u64 = 1 << 37;
80const RISCV_HWPROBE_EXT_ZVE32F: u64 = 1 << 38;
81const RISCV_HWPROBE_EXT_ZVE64X: u64 = 1 << 39;
82const RISCV_HWPROBE_EXT_ZVE64F: u64 = 1 << 40;
83const RISCV_HWPROBE_EXT_ZVE64D: u64 = 1 << 41;
84const RISCV_HWPROBE_EXT_ZIMOP: u64 = 1 << 42;
85const RISCV_HWPROBE_EXT_ZCA: u64 = 1 << 43;
86const RISCV_HWPROBE_EXT_ZCB: u64 = 1 << 44;
87const RISCV_HWPROBE_EXT_ZCD: u64 = 1 << 45;
88const RISCV_HWPROBE_EXT_ZCF: u64 = 1 << 46;
89const RISCV_HWPROBE_EXT_ZCMOP: u64 = 1 << 47;
90const RISCV_HWPROBE_EXT_ZAWRS: u64 = 1 << 48;
91// Excluded because it only reports the existence of `prctl`-based pointer masking control.
92// const RISCV_HWPROBE_EXT_SUPM: u64 = 1 << 49;
93const RISCV_HWPROBE_EXT_ZICNTR: u64 = 1 << 50;
94const RISCV_HWPROBE_EXT_ZIHPM: u64 = 1 << 51;
95const RISCV_HWPROBE_EXT_ZFBFMIN: u64 = 1 << 52;
96const RISCV_HWPROBE_EXT_ZVFBFMIN: u64 = 1 << 53;
97const RISCV_HWPROBE_EXT_ZVFBFWMA: u64 = 1 << 54;
98const RISCV_HWPROBE_EXT_ZICBOM: u64 = 1 << 55;
99const RISCV_HWPROBE_EXT_ZAAMO: u64 = 1 << 56;
100const RISCV_HWPROBE_EXT_ZALRSC: u64 = 1 << 57;
101const RISCV_HWPROBE_EXT_ZABHA: u64 = 1 << 58;
102
103const RISCV_HWPROBE_KEY_CPUPERF_0: i64 = 5;
104const RISCV_HWPROBE_MISALIGNED_FAST: u64 = 3;
105const RISCV_HWPROBE_MISALIGNED_MASK: u64 = 7;
106
107const RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: i64 = 9;
108const RISCV_HWPROBE_MISALIGNED_SCALAR_FAST: u64 = 3;
109
110const RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF: i64 = 10;
111const RISCV_HWPROBE_MISALIGNED_VECTOR_FAST: u64 = 3;
112
113// syscall returns an unsupported error if riscv_hwprobe is not supported,
114// so we can safely use this function on older versions of Linux.
115fn _riscv_hwprobe(out: &mut [riscv_hwprobe]) -> bool {
116    unsafe fn __riscv_hwprobe(
117        pairs: *mut riscv_hwprobe,
118        pair_count: libc::size_t,
119        cpu_set_size: libc::size_t,
120        cpus: *mut libc::c_ulong,
121        flags: libc::c_uint,
122    ) -> libc::c_long {
123        unsafe { libc::syscall(__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags) }
124    }
125
126    unsafe { __riscv_hwprobe(out.as_mut_ptr(), out.len(), 0, ptr::null_mut(), 0) == 0 }
127}
128
129/// Read list of supported features from (1) the auxiliary vector
130/// and (2) the results of `riscv_hwprobe` and `prctl` system calls.
131pub(crate) fn detect_features() -> cache::Initializer {
132    let mut value = cache::Initializer::default();
133    let mut enable_feature = |feature, enable| {
134        if enable {
135            value.set(feature as u32);
136        }
137    };
138
139    // Use auxiliary vector to enable single-letter ISA extensions.
140    // The values are part of the platform-specific [asm/hwcap.h][hwcap]
141    //
142    // [hwcap]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/include/uapi/asm/hwcap.h?h=v6.16
143    let auxv = auxvec::auxv().expect("read auxvec"); // should not fail on RISC-V platform
144    let mut has_i = bit::test(auxv.hwcap, (b'i' - b'a').into());
145    #[allow(clippy::eq_op)]
146    enable_feature(Feature::a, bit::test(auxv.hwcap, (b'a' - b'a').into()));
147    enable_feature(Feature::c, bit::test(auxv.hwcap, (b'c' - b'a').into()));
148    enable_feature(Feature::d, bit::test(auxv.hwcap, (b'd' - b'a').into()));
149    enable_feature(Feature::f, bit::test(auxv.hwcap, (b'f' - b'a').into()));
150    enable_feature(Feature::m, bit::test(auxv.hwcap, (b'm' - b'a').into()));
151    let has_v = bit::test(auxv.hwcap, (b'v' - b'a').into());
152    let mut is_v_set = false;
153
154    // Use riscv_hwprobe syscall to query more extensions and
155    // performance-related capabilities.
156    'hwprobe: {
157        macro_rules! init {
158            { $($name: ident : $key: expr),* $(,)? } => {
159                #[repr(usize)]
160                enum Indices { $($name),* }
161                let mut t = [$(riscv_hwprobe { key: $key, value: 0 }),*];
162                macro_rules! data_mut { () => { &mut t } }
163                macro_rules! query { [$idx: ident] => { t[Indices::$idx as usize].get() } }
164            }
165        }
166        init! {
167            BaseBehavior: RISCV_HWPROBE_KEY_BASE_BEHAVIOR,
168            Extensions:   RISCV_HWPROBE_KEY_IMA_EXT_0,
169            MisalignedScalarPerf: RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF,
170            MisalignedVectorPerf: RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF,
171            MisalignedScalarPerfFallback: RISCV_HWPROBE_KEY_CPUPERF_0,
172        };
173        if !_riscv_hwprobe(data_mut!()) {
174            break 'hwprobe;
175        }
176
177        // Query scalar misaligned behavior.
178        if let Some(value) = query![MisalignedScalarPerf] {
179            enable_feature(
180                Feature::unaligned_scalar_mem,
181                value == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST,
182            );
183        } else if let Some(value) = query![MisalignedScalarPerfFallback] {
184            // Deprecated method for fallback
185            enable_feature(
186                Feature::unaligned_scalar_mem,
187                value & RISCV_HWPROBE_MISALIGNED_MASK == RISCV_HWPROBE_MISALIGNED_FAST,
188            );
189        }
190
191        // Query vector misaligned behavior.
192        if let Some(value) = query![MisalignedVectorPerf] {
193            enable_feature(
194                Feature::unaligned_vector_mem,
195                value == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST,
196            );
197        }
198
199        // Query whether "I" base and extensions "M" and "A" (as in the ISA
200        // manual version 2.2) are enabled.  "I" base at that time corresponds
201        // to "I", "Zicsr", "Zicntr" and "Zifencei" (as in the ISA manual version
202        // 20240411).
203        // This is a current requirement of
204        // `RISCV_HWPROBE_KEY_IMA_EXT_0`-based tests.
205        if query![BaseBehavior].is_none_or(|value| value & RISCV_HWPROBE_BASE_BEHAVIOR_IMA == 0) {
206            break 'hwprobe;
207        }
208        has_i = true;
209        enable_feature(Feature::zicsr, true);
210        enable_feature(Feature::zicntr, true);
211        enable_feature(Feature::zifencei, true);
212        enable_feature(Feature::m, true);
213        enable_feature(Feature::a, true);
214
215        // Enable features based on `RISCV_HWPROBE_KEY_IMA_EXT_0`.
216        let Some(ima_ext_0) = query![Extensions] else {
217            break 'hwprobe;
218        };
219        let test = |mask| (ima_ext_0 & mask) != 0;
220
221        enable_feature(Feature::d, test(RISCV_HWPROBE_IMA_FD)); // F is implied.
222        enable_feature(Feature::c, test(RISCV_HWPROBE_IMA_C));
223
224        enable_feature(Feature::zicntr, test(RISCV_HWPROBE_EXT_ZICNTR));
225        enable_feature(Feature::zihpm, test(RISCV_HWPROBE_EXT_ZIHPM));
226
227        enable_feature(Feature::zihintntl, test(RISCV_HWPROBE_EXT_ZIHINTNTL));
228        enable_feature(Feature::zihintpause, test(RISCV_HWPROBE_EXT_ZIHINTPAUSE));
229        enable_feature(Feature::zimop, test(RISCV_HWPROBE_EXT_ZIMOP));
230        enable_feature(Feature::zicbom, test(RISCV_HWPROBE_EXT_ZICBOM));
231        enable_feature(Feature::zicboz, test(RISCV_HWPROBE_EXT_ZICBOZ));
232        enable_feature(Feature::zicond, test(RISCV_HWPROBE_EXT_ZICOND));
233
234        enable_feature(Feature::zalrsc, test(RISCV_HWPROBE_EXT_ZALRSC));
235        enable_feature(Feature::zaamo, test(RISCV_HWPROBE_EXT_ZAAMO));
236        enable_feature(Feature::zawrs, test(RISCV_HWPROBE_EXT_ZAWRS));
237        enable_feature(Feature::zabha, test(RISCV_HWPROBE_EXT_ZABHA));
238        enable_feature(Feature::zacas, test(RISCV_HWPROBE_EXT_ZACAS));
239        enable_feature(Feature::ztso, test(RISCV_HWPROBE_EXT_ZTSO));
240
241        enable_feature(Feature::zba, test(RISCV_HWPROBE_EXT_ZBA));
242        enable_feature(Feature::zbb, test(RISCV_HWPROBE_EXT_ZBB));
243        enable_feature(Feature::zbs, test(RISCV_HWPROBE_EXT_ZBS));
244        enable_feature(Feature::zbc, test(RISCV_HWPROBE_EXT_ZBC));
245
246        enable_feature(Feature::zbkb, test(RISCV_HWPROBE_EXT_ZBKB));
247        enable_feature(Feature::zbkc, test(RISCV_HWPROBE_EXT_ZBKC));
248        enable_feature(Feature::zbkx, test(RISCV_HWPROBE_EXT_ZBKX));
249        enable_feature(Feature::zknd, test(RISCV_HWPROBE_EXT_ZKND));
250        enable_feature(Feature::zkne, test(RISCV_HWPROBE_EXT_ZKNE));
251        enable_feature(Feature::zknh, test(RISCV_HWPROBE_EXT_ZKNH));
252        enable_feature(Feature::zksed, test(RISCV_HWPROBE_EXT_ZKSED));
253        enable_feature(Feature::zksh, test(RISCV_HWPROBE_EXT_ZKSH));
254        enable_feature(Feature::zkt, test(RISCV_HWPROBE_EXT_ZKT));
255
256        enable_feature(Feature::zcmop, test(RISCV_HWPROBE_EXT_ZCMOP));
257        enable_feature(Feature::zca, test(RISCV_HWPROBE_EXT_ZCA));
258        enable_feature(Feature::zcf, test(RISCV_HWPROBE_EXT_ZCF));
259        enable_feature(Feature::zcd, test(RISCV_HWPROBE_EXT_ZCD));
260        enable_feature(Feature::zcb, test(RISCV_HWPROBE_EXT_ZCB));
261
262        enable_feature(Feature::zfh, test(RISCV_HWPROBE_EXT_ZFH));
263        enable_feature(Feature::zfhmin, test(RISCV_HWPROBE_EXT_ZFHMIN));
264        enable_feature(Feature::zfa, test(RISCV_HWPROBE_EXT_ZFA));
265        enable_feature(Feature::zfbfmin, test(RISCV_HWPROBE_EXT_ZFBFMIN));
266
267        // Use prctl (if any) to determine whether the vector extension
268        // is enabled on the current thread (assuming the entire process
269        // share the same status).  If prctl fails (e.g. QEMU userland emulator
270        // as of version 9.2.3), use auxiliary vector to retrieve the default
271        // vector status on the process startup.
272        let has_vectors = {
273            let v_status = unsafe { libc::prctl(PR_RISCV_V_GET_CONTROL) };
274            if v_status >= 0 {
275                (v_status & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) == PR_RISCV_V_VSTATE_CTRL_ON
276            } else {
277                has_v
278            }
279        };
280        if has_vectors {
281            enable_feature(Feature::v, test(RISCV_HWPROBE_IMA_V));
282            enable_feature(Feature::zve32x, test(RISCV_HWPROBE_EXT_ZVE32X));
283            enable_feature(Feature::zve32f, test(RISCV_HWPROBE_EXT_ZVE32F));
284            enable_feature(Feature::zve64x, test(RISCV_HWPROBE_EXT_ZVE64X));
285            enable_feature(Feature::zve64f, test(RISCV_HWPROBE_EXT_ZVE64F));
286            enable_feature(Feature::zve64d, test(RISCV_HWPROBE_EXT_ZVE64D));
287
288            enable_feature(Feature::zvbb, test(RISCV_HWPROBE_EXT_ZVBB));
289            enable_feature(Feature::zvbc, test(RISCV_HWPROBE_EXT_ZVBC));
290            enable_feature(Feature::zvkb, test(RISCV_HWPROBE_EXT_ZVKB));
291            enable_feature(Feature::zvkg, test(RISCV_HWPROBE_EXT_ZVKG));
292            enable_feature(Feature::zvkned, test(RISCV_HWPROBE_EXT_ZVKNED));
293            enable_feature(Feature::zvknha, test(RISCV_HWPROBE_EXT_ZVKNHA));
294            enable_feature(Feature::zvknhb, test(RISCV_HWPROBE_EXT_ZVKNHB));
295            enable_feature(Feature::zvksed, test(RISCV_HWPROBE_EXT_ZVKSED));
296            enable_feature(Feature::zvksh, test(RISCV_HWPROBE_EXT_ZVKSH));
297            enable_feature(Feature::zvkt, test(RISCV_HWPROBE_EXT_ZVKT));
298
299            enable_feature(Feature::zvfh, test(RISCV_HWPROBE_EXT_ZVFH));
300            enable_feature(Feature::zvfhmin, test(RISCV_HWPROBE_EXT_ZVFHMIN));
301            enable_feature(Feature::zvfbfmin, test(RISCV_HWPROBE_EXT_ZVFBFMIN));
302            enable_feature(Feature::zvfbfwma, test(RISCV_HWPROBE_EXT_ZVFBFWMA));
303        }
304        is_v_set = true;
305    };
306
307    // Set V purely depending on the auxiliary vector
308    // only if no fine-grained vector extension detection is available.
309    if !is_v_set {
310        enable_feature(Feature::v, has_v);
311    }
312
313    // Handle base ISA.
314    // If future RV128I is supported, implement with `enable_feature` here.
315    // Note that we should use `target_arch` instead of `target_pointer_width`
316    // to avoid misdetection caused by experimental ABIs such as RV64ILP32.
317    #[cfg(target_arch = "riscv64")]
318    enable_feature(Feature::rv64i, has_i);
319    #[cfg(target_arch = "riscv32")]
320    enable_feature(Feature::rv32i, has_i);
321
322    imply_features(value)
323}
std_detect/detect/os/linux/riscv.rs

std_detect/detect/os/linux/
riscv.rs
