Add a CpuGuard type that disables preemption for its lifetime. Add a
PerCpuVariable type used for declaring per-CPU variables and a
declare_per_cpu! macro for its use. Add a PerCpuRef that allows actual
use of the per-CPU variable and an usafe_get_per_cpu_ref! helper macro
to convert PerCpuVariable to PerCpuRef.
Signed-off-by: Mitchell Levy <levymitchell0@xxxxxxxxx>
---
rust/helpers/helpers.c | 1 +
rust/helpers/preempt.c | 14 ++++
rust/kernel/lib.rs | 3 +
rust/kernel/percpu.rs | 161 ++++++++++++++++++++++++++++++++++++++++
rust/kernel/percpu/cpu_guard.rs | 29 ++++++++
5 files changed, 208 insertions(+)
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index dcf827a61b52..1db0dcd8f11f 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -19,6 +19,7 @@
#include "mutex.c"
#include "page.c"
#include "pid_namespace.c"
+#include "preempt.c"
#include "rbtree.c"
#include "refcount.c"
#include "security.c"
diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c
new file mode 100644
index 000000000000..2c7529528ddd
--- /dev/null
+++ b/rust/helpers/preempt.c
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/preempt.h>
+
+void rust_helper_preempt_disable(void)
+{
+ preempt_disable();
+}
+
+void rust_helper_preempt_enable(void)
+{
+ preempt_enable();
+}
+
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index e1065a7551a3..3634e1412d60 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -49,6 +49,9 @@
#[cfg(CONFIG_NET)]
pub mod net;
pub mod page;
+// Only x86_64 is supported by percpu for now
+#[cfg(CONFIG_X86_64)]
+pub mod percpu;
pub mod pid_namespace;
pub mod prelude;
pub mod print;
diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
new file mode 100644
index 000000000000..08fcc8797078
--- /dev/null
+++ b/rust/kernel/percpu.rs
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: GPL-2.0
+//! This module contains abstractions for creating and using per-CPU variables from Rust. In
+//! particular, see the define_per_cpu! and unsafe_get_per_cpu_ref! macros.
+pub mod cpu_guard;
+
+use crate::percpu::cpu_guard::CpuGuard;
+use crate::unsafe_get_per_cpu_ref;
+
+use core::arch::asm;
+use core::marker::PhantomData;
+use core::ops::{Deref, DerefMut};
+
+/// A PerCpuRef is obtained by the unsafe_get_per_cpu_ref! macro used on a PerCpuVariable defined
+/// via the define_per_cpu! macro.
+///
+/// This type will transparently deref(mut) into a &(mut) T referencing this CPU's instance of the
+/// underlying variable.
+pub struct PerCpuRef<T> {
+ offset: usize,
+ deref_type: PhantomData<T>,
+ _guard: CpuGuard,
+}
+
+/// A wrapper used for declaring static per-cpu variables. These symbols are "virtual" in that the
+/// linker uses them to generate offsets into each cpu's per-cpu area, but shouldn't be read
+/// from/written to directly. The fact that the statics are immutable prevents them being written
+/// to (generally), this struct having _val be non-public prevents reading from them.
+///
+/// The end-user of the per-CPU API should make use of the define_per_cpu! macro instead of
+/// declaring variables of this type directly.
+#[repr(transparent)]
+pub struct PerCpuVariable<T> {
+ _val: T, // generate a correctly sized type
+}
+
+impl<T> PerCpuRef<T> {
+ /// You should be using the unsafe_get_per_cpu! macro instead
+ ///
+ /// # Safety
+ /// offset must be a valid offset into the per cpu area
+ pub unsafe fn new(offset: usize, guard: CpuGuard) -> Self {
+ PerCpuRef {
+ offset,
+ deref_type: PhantomData,
+ _guard: guard,
+ }
+ }
+
+ /// Computes this_cpu_ptr as a usize, ignoring issues of ownership and borrowing
+ fn this_cpu_ptr_usize(&self) -> usize {
+ // SAFETY: this_cpu_off is read only as soon as the per-CPU subsystem is initialized
+ let off: PerCpuRef<u64> = unsafe { unsafe_get_per_cpu_ref!(this_cpu_off, CpuGuard::new()) };
+ let mut this_cpu_area: usize;
+ // SAFETY: gs + off_val is guaranteed to be a valid pointer by the per-CPU subsystem and
+ // the invariants guaranteed by PerCpuRef (i.e., off.offset is valid)
+ unsafe {
+ asm!(
+ // For some reason, the asm! parser doesn't like
+ // mov {out}, [gs:{off_val}]
+ // so we use the less intuitive prefix version instead
+ "gs mov {out}, [{off_val}]",
+ off_val = in(reg) off.offset,
+ out = out(reg) this_cpu_area,
+ )
+ };
+ this_cpu_area + self.offset
+ }
+
+ /// Returns a pointer to self's associated per-CPU variable. Logically equivalent to C's
+ /// this_cpu_ptr
+ pub fn this_cpu_ptr(&self) -> *const T {
+ self.this_cpu_ptr_usize() as *const T
+ }
+
+ /// Returns a mut pointer to self's associated per-CPU variable. Logically equivalent to C's
+ /// this_cpu_ptr
+ pub fn this_cpu_ptr_mut(&mut self) -> *mut T {
+ self.this_cpu_ptr_usize() as *mut T
+ }
+}
+
+impl<T> Deref for PerCpuRef<T> {
+ type Target = T;
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: By the contract of unsafe_get_per_cpu_ref!, we know that self is the only
+ // PerCpuRef associated with the underlying per-CPU variable and that the underlying
+ // variable is not mutated outside of rust.
+ unsafe { &*(self.this_cpu_ptr()) }
+ }
+}
+
+impl<T> DerefMut for PerCpuRef<T> {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ // SAFETY: By the contract of unsafe_get_per_cpu_ref!, we know that self is the only
+ // PerCpuRef associated with the underlying per-CPU variable and that the underlying
+ // variable is not mutated outside of rust.
+ unsafe { &mut *(self.this_cpu_ptr_mut()) }
+ }
+}
+
+/// define_per_cpu! is analogous to the C DEFINE_PER_CPU macro in that it lets you create a
+/// statically allocated per-CPU variable.
+///
+/// # Example
+/// ```
+/// use kernel::define_per_cpu;
+/// use kernel::percpu::PerCpuVariable;
+///
+/// define_per_cpu!(pub MY_PERCPU: u64 = 0);
+/// ```
+#[macro_export]
+macro_rules! define_per_cpu {
+ ($vis:vis $id:ident: $ty:ty = $expr:expr) => {
+ $crate::macros::paste! {
+ // Expand $expr outside of the unsafe block to avoid silently allowing unsafe code to be
+ // used without a user-facing unsafe block
+ static [<__init_ $id>]: $ty = $expr;
+
+ // SAFETY: PerCpuVariable<T> is #[repr(transparent)], so we can freely convert from T
+ #[link_section = ".data..percpu"]
+ $vis static $id: PerCpuVariable<$ty> = unsafe {
+ core::mem::transmute::<$ty, PerCpuVariable<$ty>>([<__init_ $id>])
+ };
+ }
+ };
+}
+
+/// Goes from a PerCpuVariable to a usable PerCpuRef. $id is the identifier of the PerCpuVariable
+/// and $guard is an expression that evaluates to a CpuGuard.
+///
+/// # Safety
+/// Don't create two PerCpuRef that point at the same per-cpu variable, as this would allow you to
+/// accidentally break aliasing rules. Unless T is Sync, the returned PerCpuRef should not be used
+/// from interrupt contexts.
+///
+/// If $id is `extern "C"` (i.e., declared via declare_extern_per_cpu!) then the underlying per-CPU
+/// variable must not be written from C code while a PerCpuRef exists in Rust. That is, the
+/// underlying per-CPU variable must not be written in any IRQ context (unless the user ensures
+/// IRQs are disabled) and no FFI calls can be made to C functions that may write the per-CPU
+/// variable. The underlying PerCpuVariable created via declare_extern_per_cpu must also have the
+/// correct type.
+#[macro_export]
+macro_rules! unsafe_get_per_cpu_ref {
+ ($id:ident, $guard:expr) => {{
+ let off = core::ptr::addr_of!($id);
+ PerCpuRef::new(off as usize, $guard)
+ }};
+}
+
+/// Declares a PerCpuVariable corresponding to a per-CPU variable defined in C. Be sure to read the
+/// safety requirements of unsafe_get_per_cpu_ref!.
+#[macro_export]
+macro_rules! declare_extern_per_cpu {
+ ($id:ident: $ty:ty) => {
+ extern "C" {
+ static $id: PerCpuVariable<$ty>;
+ }
+ };
+}
+
+declare_extern_per_cpu!(this_cpu_off: u64);
diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard.rs
new file mode 100644
index 000000000000..6f7d320a5c9a
--- /dev/null
+++ b/rust/kernel/percpu/cpu_guard.rs
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Contains abstractions for disabling CPU preemption. See CpuGuard.
+
+/// A RAII guard for bindings::preempt_disable and bindings::preempt_enable. Guarantees preemption
+/// is disabled for as long as this object exists.
+pub struct CpuGuard {
+ // Don't make one without using new()
+ _phantom: (),
+}
+
+impl CpuGuard {
+ /// Create a new CpuGuard. Disables preemption for its lifetime.
+ pub fn new() -> Self {
+ // SAFETY: There are no preconditions required to call preempt_disable
+ unsafe {
+ bindings::preempt_disable();
+ }
+ CpuGuard { _phantom: () }
+ }
+}
+
+impl Drop for CpuGuard {
+ fn drop(&mut self) {
+ // SAFETY: There are no preconditions required to call preempt_enable
+ unsafe {
+ bindings::preempt_enable();
+ }
+ }
+}
--
2.34.1
