aarch64: support FIQs and use them for TLB shootdown IPIs

kernel/ap_start/src/lib.rs

@@ -52,6 +52,8 @@ pub fn kstart_ap(
     nmi_flags: u16,
 ) -> ! {
     irq_safety::disable_interrupts();
+    #[cfg(target_arch = "aarch64")]
+    irq_safety::disable_fast_interrupts();
 
     info!("Booted CPU {}, proc: {}, stack: {:#X} to {:#X}, nmi_lint: {}, nmi_flags: {:#X}",
         cpu_id, processor_id, _stack_start, _stack_end, nmi_lint, nmi_flags
@@ -100,6 +102,7 @@ pub fn kstart_ap(
 
     #[cfg(target_arch = "aarch64")] {
         interrupts::init_ap();
+        irq_safety::enable_fast_interrupts();
 
         // Register this CPU as online in the system
         // This is the equivalent of `LocalApic::init` on aarch64

kernel/captain/src/lib.rs

@@ -123,6 +123,7 @@ pub fn init(
         interrupt_controller::init()?;
 
         interrupts::init()?;
+        irq_safety::enable_fast_interrupts();
 
         // register BSP CpuId
         cpu::register_cpu(true)?;

kernel/context_switch_regular/src/aarch64.rs

@@ -85,21 +85,21 @@ macro_rules! save_registers_regular {
         // Save all general purpose registers into the previous task.
         r#"
             // Make room on the stack for the registers.
-            sub sp,  sp,  #8 * 2 * 6
+            sub sp,  sp,  #8 * 13
 
             // Push registers on the stack, two at a time.
-            stp x19, x20, [sp, #8 * 2 * 0]
-            stp x21, x22, [sp, #8 * 2 * 1]
-            stp x23, x24, [sp, #8 * 2 * 2]
-            stp x25, x26, [sp, #8 * 2 * 3]
-            stp x27, x28, [sp, #8 * 2 * 4]
-            stp x29, x30, [sp, #8 * 2 * 5]
+            stp x19, x20, [sp, #8 * 0]
+            stp x21, x22, [sp, #8 * 2]
+            stp x23, x24, [sp, #8 * 4]
+            stp x25, x26, [sp, #8 * 6]
+            stp x27, x28, [sp, #8 * 8]
+            stp x29, x30, [sp, #8 * 10]
 
             // Push an OR of DAIF and NZCV flags of PSTATE
             mrs x29, DAIF
             mrs x30, NZCV
             orr x29, x29, x30
-            str x29, [sp, #8 * 2 * 6]
+            str x29, [sp, #8 * 12]
         "#
     );
 }
@@ -133,20 +133,20 @@ macro_rules! restore_registers_regular {
         r#"
             // Pop DAIF and NZCV flags of PSTATE
             // These MSRs discard irrelevant bits; no AND is required.
-            ldr x29, [sp, #8 * 2 * 6]
+            ldr x29, [sp, #8 * 12]
             msr DAIF, x29
             msr NZCV, x29
 
             // Pop registers from the stack, two at a time.
-            ldp x29, x30, [sp, #8 * 2 * 5]
-            ldp x27, x28, [sp, #8 * 2 * 4]
-            ldp x25, x26, [sp, #8 * 2 * 3]
-            ldp x23, x24, [sp, #8 * 2 * 2]
-            ldp x21, x22, [sp, #8 * 2 * 1]
-            ldp x19, x20, [sp, #8 * 2 * 0]
+            ldp x29, x30, [sp, #8 * 10]
+            ldp x27, x28, [sp, #8 * 8]
+            ldp x25, x26, [sp, #8 * 6]
+            ldp x23, x24, [sp, #8 * 4]
+            ldp x21, x22, [sp, #8 * 2]
+            ldp x19, x20, [sp, #8 * 0]
 
             // Move the stack pointer back up.
-            add sp,  sp,  #8 * 2 * 6
+            add sp,  sp,  #8 * 13
         "#
     );
 }

kernel/gic/src/gic/cpu_interface_gicv2.rs

@@ -8,6 +8,7 @@
 
 use super::Priority;
 use super::InterruptNumber;
+use super::SPURIOUS_INTERRUPT_NUM;
 
 use volatile::{Volatile, ReadOnly, WriteOnly};
 use zerocopy::FromBytes;
@@ -42,7 +43,7 @@ pub struct CpuRegsP1 {            // base offset
 }
 
 // enable group 0
-// const CTLR_ENGRP0: u32 = 0b01;
+const CTLR_ENGRP0: u32 = 0b01;
 
 // enable group 1
 const CTLR_ENGRP1: u32 = 0b10;
@@ -51,6 +52,7 @@ impl CpuRegsP1 {
     /// Enables routing of group 1 interrupts for the current CPU.
     pub fn init(&mut self) {
         let mut reg = self.ctlr.read();
+        reg |= CTLR_ENGRP0;
         reg |= CTLR_ENGRP1;
         self.ctlr.write(reg);
     }
@@ -87,12 +89,17 @@ impl CpuRegsP1 {
     ///
     /// This tells the GIC that the requested interrupt is being
     /// handled by this CPU.
-    pub fn acknowledge_interrupt(&mut self) -> (InterruptNumber, Priority) {
+    ///
+    /// Returns None if a spurious interrupt is detected.
+    pub fn acknowledge_interrupt(&mut self) -> Option<(InterruptNumber, Priority)> {
         // Reading the interrupt number has the side effect
         // of acknowledging the interrupt.
         let int_num = self.acknowledge.read() as InterruptNumber;
         let priority = self.running_prio.read() as u8;
 
-        (int_num, priority)
+        match int_num {
+            SPURIOUS_INTERRUPT_NUM => None,
+            _ => Some((int_num, priority))
+        }
     }
 }

kernel/gic/src/gic/cpu_interface_gicv3.rs

@@ -11,6 +11,8 @@ use core::arch::asm;
 use super::IpiTargetCpu;
 use super::Priority;
 use super::InterruptNumber;
+use super::InterruptGroup;
+use super::SPURIOUS_INTERRUPT_NUM;
 
 const SGIR_TARGET_ALL_OTHER_PE: u64 = 1 << 40;
 const IGRPEN_ENABLED: u64 = 1;
@@ -28,7 +30,7 @@ pub fn init() {
 
     // Enable Group 0
     // bit 0 = group 0 enable
-    // unsafe { asm!("msr ICC_IGRPEN0_EL1, {}", in(reg) IGRPEN_ENABLED) };
+    unsafe { asm!("msr ICC_IGRPEN0_EL1, {}", in(reg) IGRPEN_ENABLED) };
 
     // Enable Groupe 1 (non-secure)
     // bit 0 = group 1 (non-secure) enable
@@ -61,34 +63,47 @@ pub fn set_minimum_priority(priority: Priority) {
 /// the current CPU.
 ///
 /// This implies that the CPU is ready to process interrupts again.
-pub fn end_of_interrupt(int: InterruptNumber) {
+pub fn end_of_interrupt(int: InterruptNumber, group: InterruptGroup) {
     let reg_value = int as u64;
-    unsafe { asm!("msr ICC_EOIR1_EL1, {}", in(reg) reg_value) };
+
+    match group {
+        InterruptGroup::Group0 => unsafe { asm!("msr ICC_EOIR0_EL1, {}", in(reg) reg_value) },
+        InterruptGroup::Group1 => unsafe { asm!("msr ICC_EOIR1_EL1, {}", in(reg) reg_value) },
+    }
 }
 
 /// Acknowledge the currently serviced interrupt and fetches its
 /// number.
 ///
 /// This tells the GIC that the requested interrupt is being
 /// handled by this CPU.
-pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) {
+///
+/// Returns None if a spurious interrupt is detected.
+pub fn acknowledge_interrupt(group: InterruptGroup) -> Option<(InterruptNumber, Priority)> {
     let int_num: u64;
     let priority: u64;
 
     // Reading the interrupt number has the side effect
     // of acknowledging the interrupt.
+    match group {
+        InterruptGroup::Group0 => unsafe { asm!("mrs {}, ICC_IAR0_EL1", out(reg) int_num) },
+        InterruptGroup::Group1 => unsafe { asm!("mrs {}, ICC_IAR1_EL1", out(reg) int_num) },
+    }
+
     unsafe {
-        asm!("mrs {}, ICC_IAR1_EL1", out(reg) int_num);
         asm!("mrs {}, ICC_RPR_EL1", out(reg) priority);
     }
 
     let int_num = int_num & 0xffffff;
     let priority = priority & 0xff;
-    (int_num as InterruptNumber, priority as u8)
+    match int_num as InterruptNumber {
+        SPURIOUS_INTERRUPT_NUM => None,
+        n => Some((n, priority as u8)),
+    }
 }
 
 /// Generates an interrupt in CPU interfaces of the system
-pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu) {
+pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu, group: InterruptGroup) {
     let mut value = match target {
         IpiTargetCpu::Specific(cpu) => {
             let mpidr: cpu::MpidrValue = cpu.into();
@@ -118,5 +133,8 @@ pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu) {
     };
 
     value |= (int_num as u64) << 24;
-    unsafe { asm!("msr ICC_SGI1R_EL1, {}", in(reg) value) };
+    match group {
+        InterruptGroup::Group0 => unsafe { asm!("msr ICC_SGI0R_EL1, {}", in(reg) value) },
+        InterruptGroup::Group1 => unsafe { asm!("msr ICC_SGI1R_EL1, {}", in(reg) value) },
+    }
 }

kernel/gic/src/gic/dist_interface.rs

@@ -16,6 +16,7 @@
 use super::IpiTargetCpu;
 use super::SpiDestination;
 use super::InterruptNumber;
+use super::InterruptGroup;
 use super::Enabled;
 use super::Priority;
 use super::TargetList;
@@ -75,7 +76,7 @@ pub struct DistRegsP6 {     // base offset
 }
 
 // enable group 0
-// const CTLR_ENGRP0: u32 = 0b01;
+const CTLR_ENGRP0: u32 = 0b01;
 
 // enable group 1
 const CTLR_ENGRP1: u32 = 0b10;
@@ -96,9 +97,6 @@ const SGIR_TARGET_ALL_OTHER_PE: u32 = 1 << 24;
 //   0 = route to specific PE
 const P6IROUTER_ANY_AVAILABLE_PE: u64 = 1 << 31;
 
-// const GROUP_0: u32 = 0;
-const GROUP_1: u32 = 1;
-
 // bit 15: which interrupt group to target
 const SGIR_NSATT_GRP1: u32 = 1 << 15;
 
@@ -112,6 +110,7 @@ impl DistRegsP1 {
     /// states.
     pub fn init(&mut self) -> Enabled {
         let mut reg = self.ctlr.read();
+        reg |= CTLR_ENGRP0;
         reg |= CTLR_ENGRP1;
         reg |= CTLR_E1NWF;
         self.ctlr.write(reg);
@@ -123,26 +122,26 @@ impl DistRegsP1 {
     }
 
     /// Returns whether the given SPI (shared peripheral interrupt) will be
-    /// forwarded by the distributor
-    pub fn is_spi_enabled(&self, int: InterruptNumber) -> Enabled {
-        // enabled?
-        read_array_volatile::<32>(&self.set_enable, int) > 0
-        &&
-        // part of group 1?
-        read_array_volatile::<32>(&self.group, int) == GROUP_1
+    /// forwarded by the distributor.
+    pub fn get_spi_state(&self, int: InterruptNumber) -> Option<InterruptGroup> {
+        if read_array_volatile::<32>(&self.set_enable, int) == 1 {
+            match read_array_volatile::<32>(&self.group, int) {
+                0 => return Some(InterruptGroup::Group0),
+                1 => return Some(InterruptGroup::Group1),
+                _ => { }
+            }
+        }
+        None
     }
 
-    /// Enables or disables the forwarding of a particular SPI (shared peripheral interrupt)
-    pub fn enable_spi(&mut self, int: InterruptNumber, enabled: Enabled) {
-        let reg_base = match enabled {
-            true  => &mut self.set_enable,
-            false => &mut self.clear_enable,
-        };
-        write_array_volatile::<32>(reg_base, int, 1);
-
-        // whether we're enabling or disabling,
-        // set as part of group 1
-        write_array_volatile::<32>(&mut self.group, int, GROUP_1);
+    /// Enables or disables the forwarding of a particular SPI (shared peripheral interrupt).
+    pub fn set_spi_state(&mut self, int: InterruptNumber, state: Option<InterruptGroup>) {
+        if let Some(group) = state {
+            write_array_volatile::<32>(&mut self.group, int, group as u32);
+            write_array_volatile::<32>(&mut self.set_enable, int, 1);
+        } else {
+            write_array_volatile::<32>(&mut self.clear_enable, int, 1);
+        }
     }
 
     /// Returns the priority of an SPI.