Merge #621

621: Rework Core Local Storage r=stlankes a=mkroening



Co-authored-by: Martin Kröning <[email protected]>

Author: bors[bot]

src/arch/aarch64/kernel/percore.rs src/arch/aarch64/kernel/core_local.rs

@@ -3,44 +3,44 @@ use core::ptr;
 use crate::scheduler::{CoreId, PerCoreScheduler};
 
 #[no_mangle]
-pub static mut PERCORE: PerCoreVariables = PerCoreVariables::new(0);
+pub static mut CORE_LOCAL: CoreLocal = CoreLocal::new(0);
 
-pub struct PerCoreVariables {
+pub struct CoreLocal {
 	/// ID of the current Core.
-	core_id: PerCoreVariable<CoreId>,
+	core_id: CoreLocalVariable<CoreId>,
 	/// Scheduler of the current Core.
-	scheduler: PerCoreVariable<*mut PerCoreScheduler>,
+	scheduler: CoreLocalVariable<*mut PerCoreScheduler>,
 }
 
-impl PerCoreVariables {
+impl CoreLocal {
 	pub const fn new(core_id: CoreId) -> Self {
 		Self {
-			core_id: PerCoreVariable::new(core_id),
-			scheduler: PerCoreVariable::new(0 as *mut PerCoreScheduler),
+			core_id: CoreLocalVariable::new(core_id),
+			scheduler: CoreLocalVariable::new(0 as *mut PerCoreScheduler),
 		}
 	}
 }
 
 #[repr(C)]
-pub struct PerCoreVariable<T> {
+pub struct CoreLocalVariable<T> {
 	data: T,
 }
 
-pub trait PerCoreVariableMethods<T: Clone> {
+pub trait CoreLocalVariableMethods<T: Clone> {
 	unsafe fn get(&self) -> T;
 	unsafe fn set(&mut self, value: T);
 }
 
-impl<T> PerCoreVariable<T> {
+impl<T> CoreLocalVariable<T> {
 	const fn new(value: T) -> Self {
 		Self { data: value }
 	}
 }
 
 // Treat all per-core variables as 64-bit variables by default. This is true for u64, usize, pointers.
-// Implement the PerCoreVariableMethods trait functions using 64-bit memory moves.
+// Implement the CoreLocalVariableMethods trait functions using 64-bit memory moves.
 // The functions are implemented as default functions, which can be overridden in specialized implementations of the trait.
-impl<T> PerCoreVariableMethods<T> for PerCoreVariable<T>
+impl<T> CoreLocalVariableMethods<T> for CoreLocalVariable<T>
 where
 	T: Clone,
 {
@@ -57,18 +57,18 @@ where
 
 #[inline]
 pub fn core_id() -> CoreId {
-	unsafe { PERCORE.core_id.get() }
+	unsafe { CORE_LOCAL.core_id.get() }
 }
 
 #[inline]
 pub fn core_scheduler() -> &'static mut PerCoreScheduler {
-	unsafe { &mut *PERCORE.scheduler.get() }
+	unsafe { &mut *CORE_LOCAL.scheduler.get() }
 }
 
 #[inline]
 pub fn set_core_scheduler(scheduler: *mut PerCoreScheduler) {
 	unsafe {
-		PERCORE.scheduler.set(scheduler);
+		CORE_LOCAL.scheduler.set(scheduler);
 	}
 }
 

src/arch/aarch64/kernel/mod.rs

@@ -1,6 +1,6 @@
+pub mod core_local;
 pub mod interrupts;
 pub mod pci;
-pub mod percore;
 pub mod processor;
 pub mod scheduler;
 pub mod serial;
@@ -15,7 +15,7 @@ use core::ptr;
 use hermit_entry::boot_info::{BootInfo, PlatformInfo, RawBootInfo};
 use hermit_sync::TicketMutex;
 
-use crate::arch::aarch64::kernel::percore::*;
+use crate::arch::aarch64::kernel::core_local::*;
 use crate::arch::aarch64::kernel::serial::SerialPort;
 pub use crate::arch::aarch64::kernel::stubs::*;
 pub use crate::arch::aarch64::kernel::systemtime::get_boot_time;
@@ -136,7 +136,7 @@ pub fn get_cmdline() -> VirtAddr {
 
 /// Earliest initialization function called by the Boot Processor.
 pub fn message_output_init() {
-	percore::init();
+	core_local::init();
 
 	unsafe {
 		COM1.port_address = boot_info()
@@ -226,7 +226,7 @@ pub fn boot_application_processors() {
 
 /// Application Processor initialization
 pub fn application_processor_init() {
-	percore::init();
+	core_local::init();
 	/*processor::configure();
 	gdt::add_current_core();
 	idt::install();

src/arch/aarch64/kernel/scheduler.rs

@@ -4,7 +4,7 @@ use alloc::rc::Rc;
 use core::cell::RefCell;
 use core::{mem, ptr};
 
-use crate::arch::aarch64::kernel::percore::*;
+use crate::arch::aarch64::kernel::core_local::*;
 use crate::arch::aarch64::kernel::processor;
 use crate::arch::aarch64::mm::paging::{BasePageSize, PageSize, PageTableEntryFlags};
 use crate::arch::aarch64::mm::{PhysAddr, VirtAddr};
@@ -109,7 +109,7 @@ impl TaskStacks {
 	}
 
 	pub fn from_boot_stacks() -> TaskStacks {
-		//let tss = unsafe { &(*PERCORE.tss.get()) };
+		//let tss = unsafe { &(*CORE_LOCAL.tss.get()) };
 		/*let stack = VirtAddr::from_usize(tss.rsp[0] as usize + 0x10 - KERNEL_STACK_SIZE);
 		debug!("Using boot stack {:#X}", stack);
 		let ist0 = VirtAddr::from_usize(tss.ist[0] as usize + 0x10 - KERNEL_STACK_SIZE);

src/arch/aarch64/mm/paging.rs

@@ -2,7 +2,7 @@ use core::arch::asm;
 use core::marker::PhantomData;
 use core::{fmt, mem, ptr, usize};
 
-use crate::arch::aarch64::kernel::percore::*;
+use crate::arch::aarch64::kernel::core_local::*;
 use crate::arch::aarch64::kernel::{
 	get_base_address, get_boot_info_address, get_image_size, get_ram_address, is_uhyve, processor,
 };

src/arch/mod.rs

@@ -7,11 +7,11 @@ pub mod x86_64;
 
 // Export our platform-specific modules.
 #[cfg(target_arch = "aarch64")]
-pub use crate::arch::aarch64::kernel::interrupts;
+pub use crate::arch::aarch64::kernel::core_local;
 #[cfg(target_arch = "aarch64")]
-pub use crate::arch::aarch64::kernel::percore;
+use crate::arch::aarch64::kernel::core_local::core_scheduler;
 #[cfg(target_arch = "aarch64")]
-use crate::arch::aarch64::kernel::percore::core_scheduler;
+pub use crate::arch::aarch64::kernel::interrupts;
 #[cfg(target_arch = "aarch64")]
 pub use crate::arch::aarch64::kernel::processor;
 #[cfg(target_arch = "aarch64")]
@@ -34,12 +34,12 @@ pub use crate::arch::x86_64::kernel::apic::{set_oneshot_timer, wakeup_core};
 #[cfg(all(target_arch = "x86_64", target_os = "none", feature = "smp"))]
 pub use crate::arch::x86_64::kernel::application_processor_init;
 #[cfg(target_arch = "x86_64")]
+pub use crate::arch::x86_64::kernel::core_local;
+#[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::kernel::gdt::set_current_kernel_stack;
 #[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::kernel::interrupts;
 #[cfg(target_arch = "x86_64")]
-pub use crate::arch::x86_64::kernel::percore;
-#[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::kernel::processor;
 #[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::kernel::scheduler;

src/arch/x86_64/kernel/apic.rs

@@ -1,4 +1,3 @@
-use alloc::boxed::Box;
 use alloc::vec::Vec;
 #[cfg(any(feature = "pci", feature = "smp"))]
 use core::arch::x86_64::_mm_mfence;
@@ -8,19 +7,18 @@ use core::ptr;
 use core::sync::atomic::Ordering;
 use core::{cmp, fmt, mem, u32};
 
-use arch::x86_64::kernel::percore::*;
+#[cfg(feature = "smp")]
+use arch::x86_64::kernel::core_local::*;
 use arch::x86_64::kernel::{interrupts, processor};
-use crossbeam_utils::CachePadded;
 use hermit_sync::without_interrupts;
 #[cfg(feature = "smp")]
 use x86::controlregs::*;
 use x86::msr::*;
 use x86_64::structures::idt::InterruptDescriptorTable;
 
-use super::interrupts::{IDT, IRQ_COUNTERS};
+use super::interrupts::IDT;
 #[cfg(feature = "acpi")]
 use crate::arch::x86_64::kernel::acpi;
-use crate::arch::x86_64::kernel::interrupts::IrqStatistics;
 use crate::arch::x86_64::kernel::CURRENT_STACK_ADDRESS;
 use crate::arch::x86_64::mm::paging::{
 	BasePageSize, PageSize, PageTableEntryFlags, PageTableEntryFlagsExt,
@@ -512,13 +510,6 @@ pub extern "C" fn eoi() {
 }
 
 pub fn init() {
-	let boxed_irq = Box::new(IrqStatistics::new());
-	let boxed_irq_raw = Box::into_raw(boxed_irq);
-	unsafe {
-		IRQ_COUNTERS.insert(0, &(*boxed_irq_raw));
-		PERCORE.irq_statistics.set(boxed_irq_raw);
-	}
-
 	// Initialize an empty vector for the Local APIC IDs of all CPUs.
 	unsafe {
 		CPU_LOCAL_APIC_IDS = Some(Vec::new());
@@ -736,30 +727,11 @@ pub fn init_x2apic() {
 }
 
 /// Initialize the required _start variables for the next CPU to be booted.
-pub fn init_next_processor_variables(core_id: CoreId) {
-	// Allocate stack and PerCoreVariables structure for the CPU and pass the addresses.
+pub fn init_next_processor_variables() {
+	// Allocate stack for the CPU and pass the addresses.
 	// Keep the stack executable to possibly support dynamically generated code on the stack (see https://security.stackexchange.com/a/47825).
 	let stack = mm::allocate(KERNEL_STACK_SIZE, true);
-	let mut boxed_percore = Box::new(CachePadded::new(PerCoreInnerVariables::new(core_id)));
-	let boxed_irq = Box::new(IrqStatistics::new());
-	let boxed_irq_raw = Box::into_raw(boxed_irq);
-
-	unsafe {
-		IRQ_COUNTERS.insert(core_id, &(*boxed_irq_raw));
-		boxed_percore.irq_statistics = PerCoreVariable::new(boxed_irq_raw);
-	}
-
 	CURRENT_STACK_ADDRESS.store(stack.as_u64(), Ordering::Relaxed);
-
-	let current_percore = Box::leak(boxed_percore);
-
-	trace!(
-		"Initialize per core data at {:p} (size {} bytes)",
-		current_percore,
-		mem::size_of_val(current_percore)
-	);
-
-	CURRENT_PERCORE_ADDRESS.store(current_percore as *mut _ as u64, Ordering::Release);
 }
 
 /// Boot all Application Processors
@@ -836,7 +808,7 @@ pub fn boot_application_processors() {
 				"Waking up CPU {} with Local APIC ID {}",
 				core_id_to_boot, apic_id
 			);
-			init_next_processor_variables(core_id_to_boot);
+			init_next_processor_variables();
 
 			// Save the current number of initialized CPUs.
 			let current_processor_count = arch::get_processor_count();

src/arch/x86_64/kernel/core_local.rs

@@ -0,0 +1,80 @@
+use alloc::boxed::Box;
+use core::arch::asm;
+use core::cell::Cell;
+use core::ptr;
+use core::sync::atomic::Ordering;
+
+use x86_64::registers::model_specific::GsBase;
+use x86_64::structures::tss::TaskStateSegment;
+use x86_64::VirtAddr;
+
+use super::interrupts::{IrqStatistics, IRQ_COUNTERS};
+use super::CPU_ONLINE;
+use crate::scheduler::{CoreId, PerCoreScheduler};
+
+pub struct CoreLocal {
+	this: *const Self,
+	/// Sequential ID of this CPU Core.
+	core_id: CoreId,
+	/// Scheduler for this CPU Core.
+	scheduler: Cell<*mut PerCoreScheduler>,
+	/// Task State Segment (TSS) allocated for this CPU Core.
+	pub tss: Cell<*mut TaskStateSegment>,
+	/// start address of the kernel stack
+	pub kernel_stack: Cell<u64>,
+	/// Interface to the interrupt counters
+	irq_statistics: &'static IrqStatistics,
+}
+
+impl CoreLocal {
+	pub fn install() {
+		assert_eq!(VirtAddr::zero(), GsBase::read());
+
+		let core_id = CPU_ONLINE.load(Ordering::Relaxed);
+
+		let irq_statistics = &*Box::leak(Box::new(IrqStatistics::new()));
+		IRQ_COUNTERS.lock().insert(core_id, irq_statistics);
+
+		let this = Self {
+			this: ptr::null_mut(),
+			core_id,
+			scheduler: Cell::new(ptr::null_mut()),
+			tss: Cell::new(ptr::null_mut()),
+			kernel_stack: Cell::new(0),
+			irq_statistics,
+		};
+		let mut this = Box::leak(Box::new(this));
+		this.this = &*this;
+
+		GsBase::write(VirtAddr::from_ptr(this));
+	}
+
+	pub fn get() -> &'static Self {
+		debug_assert_ne!(VirtAddr::zero(), GsBase::read());
+		unsafe {
+			let raw: *const Self;
+			asm!("mov {}, gs:0", out(reg) raw, options(nomem, nostack, preserves_flags));
+			&*raw
+		}
+	}
+}
+
+pub fn core_id() -> CoreId {
+	if cfg!(target_os = "none") {
+		CoreLocal::get().core_id
+	} else {
+		0
+	}
+}
+
+pub fn core_scheduler() -> &'static mut PerCoreScheduler {
+	unsafe { &mut *CoreLocal::get().scheduler.get() }
+}
+
+pub fn set_core_scheduler(scheduler: *mut PerCoreScheduler) {
+	CoreLocal::get().scheduler.set(scheduler);
+}
+
+pub fn increment_irq_counter(irq_no: usize) {
+	CoreLocal::get().irq_statistics.inc(irq_no);
+}

src/arch/x86_64/kernel/gdt.rs

@@ -10,7 +10,7 @@ use x86_64::VirtAddr;
 use super::interrupts::{IST_ENTRIES, IST_SIZE};
 use super::scheduler::TaskStacks;
 use super::CURRENT_STACK_ADDRESS;
-use crate::arch::x86_64::kernel::percore::{core_scheduler, set_kernel_stack, PERCORE};
+use crate::arch::x86_64::kernel::core_local::{core_scheduler, CoreLocal};
 use crate::config::KERNEL_STACK_SIZE;
 
 pub fn add_current_core() {
@@ -26,7 +26,7 @@ pub fn add_current_core() {
 	let rsp = CURRENT_STACK_ADDRESS.load(Ordering::Relaxed) + KERNEL_STACK_SIZE as u64
 		- TaskStacks::MARKER_SIZE as u64;
 	tss.privilege_stack_table[0] = VirtAddr::new(rsp);
-	set_kernel_stack(rsp);
+	CoreLocal::get().kernel_stack.set(rsp);
 
 	// Allocate all ISTs for this core.
 	// Every task later gets its own IST1, so the IST1 allocated here is only used by the Idle task.
@@ -36,9 +36,7 @@ pub fn add_current_core() {
 		tss.interrupt_stack_table[i] = VirtAddr::new(ist_start);
 	}
 
-	unsafe {
-		PERCORE.tss.set(tss);
-	}
+	CoreLocal::get().tss.set(tss);
 	let tss_selector = gdt.add_entry(Descriptor::tss_segment(tss));
 
 	// Load the GDT for the current core.