Early support for the SBI HSM extension

Signed-off-by: Wojciech Ozga <woz@zurich.ibm.com>

confidential-vms/baremetal/src/uart.rs

@@ -46,6 +46,7 @@ impl Uart {
     }
 
     pub fn println(&mut self, out: &str) {
+        use alloc::format;
         crate::sync::acquire(crate::sync::UART_SYNC_ADDRESS);
         for c in out.bytes() {
             self.put(c);

confidential-vms/baremetal/src/worker.rs

@@ -28,6 +28,10 @@ extern "C" fn worker_init(hart_id: usize) {
         riscv::register::sstatus::set_sie();
     }
 
+    loop {
+        uart.println(&format!("Hello from hart id: {}", hart_id));
+    }
+
     sbi::system_reset::system_reset(
         sbi::system_reset::ResetType::Shutdown,
         sbi::system_reset::ResetReason::NoReason,

security-monitor/src/confidential_flow/control_flow/mod.rs

@@ -1,9 +1,11 @@
 // SPDX-FileCopyrightText: 2023 IBM Corporation
 // SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
 // SPDX-License-Identifier: Apache-2.0
+use crate::core::architecture::HartLifecycleStateTransition;
 use crate::core::architecture::SbiExtension::*;
 use crate::core::control_data::{ConfidentialVmId, ControlData, HardwareHart};
 use crate::core::transformations::{ExposeToConfidentialVm, PendingRequest};
+use crate::error::Error;
 use crate::non_confidential_flow::NonConfidentialFlow;
 
 extern "C" {
@@ -60,9 +62,11 @@ impl<'a> ConfidentialFlow<'a> {
             }
             VsEcall(Rfence(_)) => invalid_call::handle(self),
             VsEcall(Hsm(HartStart)) => sbi_hsm_hart_start::handle(confidential_hart.sbi_hsm_hart_start(), self),
+            VsEcall(Hsm(HartStop)) => sbi_hsm_hart_stop::handle(self),
+            VsEcall(Hsm(HartSuspend)) => sbi_hsm_hart_suspend::handle(confidential_hart.sbi_hsm_hart_suspend(), self),
             VsEcall(Hsm(HartGetStatus)) => hypercall::handle(confidential_hart.hypercall_request(), self),
             VsEcall(Ipi(SendIpi)) => inter_hart_request::handle(confidential_hart.sbi_ipi(), self),
-            VsEcall(Srst(SystemReset)) => hypercall::handle(confidential_hart.hypercall_request(), self),
+            VsEcall(Srst(SystemReset)) => sbi_srst::handle(confidential_hart.hypercall_request(), self),
             VsEcall(_) => invalid_call::handle(self),
             GuestLoadPageFault => {
                 guest_load_page_fault::handle(confidential_hart.guest_load_page_fault_request(), self)
@@ -117,6 +121,10 @@ impl<'a> ConfidentialFlow<'a> {
             }
             Some(GuestStorePageFault(request)) => guest_store_page_fault_result::handle(self, request),
             Some(SharePage(request)) => share_page_result::handle(self.hart.share_page_result(), self, request),
+            Some(SbiHsmHartStart()) => self.exit_to_confidential_vm(ExposeToConfidentialVm::SbiHsmHartStart()),
+            Some(SbiHsmHartStartPending()) => {
+                self.exit_to_confidential_vm(ExposeToConfidentialVm::SbiHsmHartStartPending())
+            }
             None => self.exit_to_confidential_vm(ExposeToConfidentialVm::Resume()),
         }
     }
@@ -151,6 +159,22 @@ impl<'a> ConfidentialFlow<'a> {
         self.hart.confidential_hart().confidential_hart_id()
     }
 
+    pub fn transit_hart_lifecycle(&mut self, state_transition: HartLifecycleStateTransition) -> Result<(), Error> {
+        use crate::core::architecture::HartLifecycleStateTransition::*;
+        match state_transition {
+            StoppedToStartPending(request) => {
+                ControlData::try_confidential_vm_mut(self.confidential_vm_id(), |ref mut confidential_vm| {
+                    confidential_vm.transit_confidential_hart_to_start_pending(request)
+                })
+            }
+            StartedToSuspended(request) => {
+                self.hart.confidential_hart_mut().transition_from_started_to_suspended(request)
+            }
+            SuspendedToStarted() => self.hart.confidential_hart_mut().transition_from_suspended_to_started(),
+            StartedToStopped() => self.hart.confidential_hart_mut().transition_from_started_to_stopped(),
+        }
+    }
+
     pub fn set_pending_request(self, request: PendingRequest) -> Self {
         if let Err(error) = self.hart.confidential_hart_mut().set_pending_request(request) {
             self.exit_to_confidential_vm(error.into_confidential_transformation());

security-monitor/src/confidential_flow/handlers/mod.rs

@@ -11,5 +11,8 @@ pub mod inter_hart_request;
 pub mod interrupt;
 pub mod invalid_call;
 pub mod sbi_hsm_hart_start;
+pub mod sbi_hsm_hart_stop;
+pub mod sbi_hsm_hart_suspend;
+pub mod sbi_srst;
 pub mod share_page;
 pub mod share_page_result;

security-monitor/src/confidential_flow/handlers/sbi_hsm_hart_start.rs

@@ -2,21 +2,27 @@
 // SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
 // SPDX-License-Identifier: Apache-2.0
 use crate::confidential_flow::ConfidentialFlow;
-use crate::core::control_data::ControlData;
-use crate::core::transformations::{ExposeToHypervisor, InterHartRequest, PendingRequest, SbiHsmHartStart, SbiRequest};
+use crate::core::architecture::HartLifecycleStateTransition;
+use crate::core::transformations::{ExposeToHypervisor, PendingRequest, SbiHsmHartStart, SbiRequest};
 
-/// Handles HartStart function from the HSM extension of SBI.
-pub fn handle(request: SbiHsmHartStart, confidential_flow: ConfidentialFlow) -> ! {
+/// Starts a confidential hart. This is an implementation of the HartStart function from the HSM extension of SBI.
+///
+/// This call is triggered by a confidential hart that wants to start another confidential hart. Error is returned to
+/// the caller if the targetted confidential hart is not in the stopped state or it does not exist. The security monitor
+/// moves targetted confidential harts into `StartPending` state and informs then the hypervisor that these harts are
+/// runnable. Once the hypervisor schedules such a confidential hart for execution, the confidential hart will change
+/// the state to `Started`.
+pub fn handle(request: SbiHsmHartStart, mut confidential_flow: ConfidentialFlow) -> ! {
     let confidential_hart_id = request.confidential_hart_id;
-    // TODO: we must check the state of the hart and only allow starting a HART that is in the shutdown state
-    match ControlData::try_confidential_vm_mut(confidential_flow.confidential_vm_id(), |ref mut confidential_vm| {
-        Ok(confidential_vm.add_inter_hart_request(InterHartRequest::SbiHsmHartStart(request)))
-    }) {
+    match confidential_flow.transit_hart_lifecycle(HartLifecycleStateTransition::StoppedToStartPending(request)) {
         Ok(_) => confidential_flow
-            .set_pending_request(PendingRequest::SbiRequest())
+            .set_pending_request(PendingRequest::SbiHsmHartStart())
             .into_non_confidential_flow()
             .exit_to_hypervisor(ExposeToHypervisor::SbiRequest(SbiRequest::kvm_hsm_hart_start(confidential_hart_id))),
         Err(error) => {
+            // starting a confidential hart might fail if the incoming request is invalid. For example, the confidential
+            // hart id does not exist or is the same as the one currently assigned to the hardware hart. In such cases,
+            // return an error to the calling confidential hart.
             confidential_flow.exit_to_confidential_vm(error.into_confidential_transformation());
         }
     }

security-monitor/src/confidential_flow/handlers/sbi_hsm_hart_stop.rs

@@ -0,0 +1,21 @@
+// SPDX-FileCopyrightText: 2023 IBM Corporation
+// SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
+// SPDX-License-Identifier: Apache-2.0
+use crate::confidential_flow::ConfidentialFlow;
+use crate::core::architecture::HartLifecycleStateTransition;
+use crate::core::transformations::{ExposeToHypervisor, SbiRequest};
+
+/// Stops the confidential hart as defined in the HSM extension of SBI. Error is returned to the confidential hart if
+/// the security monitor cannot stop it, for example, because it is not in the started state.
+///
+/// The request to stop the confidential hart comes from the confidential hart itself. The security monitor stops the
+/// hart and informs the hypervisor that the hart has been stopped. The hypervisor should not resume execution of a
+/// stopped confidential hart. Only another confidential hart of the confidential VM can start the confidential hart.
+pub fn handle(mut confidential_flow: ConfidentialFlow) -> ! {
+    match confidential_flow.transit_hart_lifecycle(HartLifecycleStateTransition::StartedToStopped()) {
+        Ok(_) => confidential_flow
+            .into_non_confidential_flow()
+            .exit_to_hypervisor(ExposeToHypervisor::SbiRequest(SbiRequest::kvm_hsm_hart_stop())),
+        Err(error) => confidential_flow.exit_to_confidential_vm(error.into_confidential_transformation()),
+    }
+}

security-monitor/src/confidential_flow/handlers/sbi_hsm_hart_suspend.rs

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: 2023 IBM Corporation
+// SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
+// SPDX-License-Identifier: Apache-2.0
+use crate::confidential_flow::ConfidentialFlow;
+use crate::core::architecture::HartLifecycleStateTransition;
+use crate::core::transformations::{ExposeToHypervisor, SbiHsmHartSuspend, SbiRequest};
+
+/// Suspends a confidential hart that made this request. This is an implementation of the HartSuspend function from the
+/// HSM extension of SBI.
+///
+/// The request to suspend the confidential hart comes frmo the confidential hart itself. The security monitor suspends
+/// the confidential hart and informs about it the hypervisor. This functions returns an error to the calling
+/// confidential hart if this confidential hart cannot be suspended, for example, because it is not in the started
+/// state.
+pub fn handle(request: SbiHsmHartSuspend, mut confidential_flow: ConfidentialFlow) -> ! {
+    match confidential_flow.transit_hart_lifecycle(HartLifecycleStateTransition::StartedToSuspended(request)) {
+        Ok(_) => confidential_flow
+            .into_non_confidential_flow()
+            .exit_to_hypervisor(ExposeToHypervisor::SbiRequest(SbiRequest::kvm_hsm_hart_suspend())),
+        Err(error) => confidential_flow.exit_to_confidential_vm(error.into_confidential_transformation()),
+    }
+}

security-monitor/src/confidential_flow/handlers/sbi_srst.rs

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: 2023 IBM Corporation
+// SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
+// SPDX-License-Identifier: Apache-2.0
+use crate::confidential_flow::ConfidentialFlow;
+use crate::core::control_data::ControlData;
+use crate::core::transformations::{ExposeToHypervisor, SbiRequest};
+
+pub fn handle(request: SbiRequest, confidential_flow: ConfidentialFlow) -> ! {
+    let confidential_vm_id = confidential_flow.confidential_vm_id();
+    match ControlData::try_write(|control_data| control_data.remove_confidential_vm(confidential_vm_id)) {
+        Ok(_) => {
+            confidential_flow.into_non_confidential_flow().exit_to_hypervisor(ExposeToHypervisor::SbiRequest(request))
+        }
+        Err(error) => confidential_flow.exit_to_confidential_vm(error.into_confidential_transformation()),
+    }
+}

security-monitor/src/confidential_flow/handlers/share_page.rs

@@ -14,7 +14,7 @@ pub fn handle(request: Result<(SharePageRequest, SbiRequest), Error>, confidenti
     match request {
         Ok((share_page_request, sbi_request)) => {
             debug!(
-                "Confidential VM[confidential_vm_id={:?}] requested a shared page mapped to address [guest_physical_address={:?}]",
+                "Confidential VM[{:?}] requested a shared page mapped to address [{:?}]",
                 confidential_flow.confidential_vm_id(),
                 share_page_request.confidential_vm_virtual_address()
             );

security-monitor/src/confidential_flow/handlers/share_page_result.rs

@@ -25,8 +25,9 @@ pub fn handle(share_page_result: SharePageResult, confidential_flow: Confidentia
     };
 
     debug!(
-        "Hypervisor shared a page with the confidential VM [confidential_vm_id={:?}] at address [physical address=0x{:x}]",
-        confidential_vm_id, share_page_result.hypervisor_page_address()
+        "Hypervisor shared a page with the confidential VM [{:?}] at address [physical address=0x{:x}]",
+        confidential_vm_id,
+        share_page_result.hypervisor_page_address()
     );
 
     let transformation = ControlData::try_confidential_vm_mut(confidential_vm_id, |mut confidential_vm| {

security-monitor/src/core/architecture/mod.rs

@@ -2,8 +2,9 @@
 // SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
 // SPDX-License-Identifier: Apache-2.0
 pub use riscv::{
-    decode_result_register, AceExtension, BaseExtension, FpRegisters, GpRegister, GpRegisters, HartState, HsmExtension,
-    IpiExtension, RfenceExtension, SbiExtension, SrstExtension, TrapReason,
+    decode_result_register, AceExtension, BaseExtension, FpRegisters, GpRegister, GpRegisters, HartArchitecturalState,
+    HartLifecycleState, HartLifecycleStateTransition, HsmExtension, IpiExtension, RfenceExtension, SbiExtension,
+    SrstExtension, TrapReason,
 };
 
 mod riscv;

security-monitor/src/core/architecture/riscv/hart_state.rs → security-monitor/src/core/architecture/riscv/hart_architectural_state.rs

@@ -3,11 +3,11 @@
 // SPDX-License-Identifier: Apache-2.0
 use crate::core::architecture::{FpRegisters, GpRegister, GpRegisters, TrapReason};
 
-/// HartState is the dump state of the processor's core, called in RISC-V a hardware thread (HART).
+/// HartArchitecturalState is the dump state of the processor's core, called in RISC-V a hardware thread (HART).
 #[repr(C)]
-pub struct HartState {
-    // gprs must be the first element in this structure because it is used to calculate the HartState address in the
-    // memory. This address is used by the assembly code.
+pub struct HartArchitecturalState {
+    // gprs must be the first element in this structure because it is used to calculate the HartArchitecturalState
+    // address in the memory. This address is used by the assembly code.
     pub gprs: GpRegisters,
     // floating-point related
     pub fprs: FpRegisters,
@@ -54,9 +54,9 @@ pub struct HartState {
     pub mtval2: usize,
 }
 
-impl HartState {
-    pub fn from_existing(id: usize, existing: &HartState) -> HartState {
-        HartState {
+impl HartArchitecturalState {
+    pub fn from_existing(id: usize, existing: &HartArchitecturalState) -> HartArchitecturalState {
+        HartArchitecturalState {
             id,
             gprs: existing.gprs.clone(),
             // M-mode
@@ -102,8 +102,8 @@ impl HartState {
         }
     }
 
-    pub fn empty(id: usize) -> HartState {
-        HartState {
+    pub fn empty(id: usize) -> HartArchitecturalState {
+        HartArchitecturalState {
             id,
             gprs: GpRegisters::empty(),
             sstatus: 0,
@@ -145,7 +145,7 @@ impl HartState {
     }
 }
 
-impl HartState {
+impl HartArchitecturalState {
     pub fn gpr(&self, register: GpRegister) -> usize {
         self.gprs.get(register)
     }
@@ -154,12 +154,33 @@ impl HartState {
         self.gprs.set(register, value)
     }
 
+    pub fn reset(&mut self) {
+        self.gprs = GpRegisters::empty();
+        self.fprs = FpRegisters::empty();
+        self.fcsr = 0;
+        self.htinst = 0;
+        self.htval = 0;
+        self.sepc = 0;
+        self.scounteren = 0;
+        self.vsie = 0;
+        self.vstvec = 0;
+        self.vsscratch = 0;
+        self.vsepc = 0;
+        self.vscause = 0;
+        self.vstval = 0;
+        self.hvip = 0;
+        self.vsatp = 0;
+        // TODO: what should be the sstatus on reset?
+        // self.sstatus = 0;
+        self.vsstatus = 0;
+    }
+
     pub fn trap_reason(&self) -> TrapReason {
         TrapReason::from_hart_state(self)
     }
 }
 
-impl core::fmt::Debug for HartState {
+impl core::fmt::Debug for HartArchitecturalState {
     fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
         write!(f, "hart_id: {}, ", self.id)?;
         GpRegisters::iter().try_for_each(|x| -> core::fmt::Result {

security-monitor/src/core/architecture/riscv/hart_lifecycle_state.rs

@@ -0,0 +1,24 @@
+// SPDX-FileCopyrightText: 2023 IBM Corporation
+// SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
+// SPDX-License-Identifier: Apache-2.0
+
+/// Hart lifecycle states as documented in the SBI specification of the HSM extension.
+#[derive(PartialEq)]
+pub enum HartLifecycleState {
+    Started,
+    Stopped,
+    StartPending,
+    //
+    // StopPending is never used because the security monitor stops the hart directly and only informs a hypervisor
+    // about it for the bookkeeping pourposes.
+    // StopPending,
+    Suspended,
+    //
+    // SuspendPending is never used because the security monitor stops the hart directly and only informs a hypervisor
+    // about it for the bookkeeping pourposes.
+    // SuspendPending,
+    //
+    // ResumePending is never used because the security monitor stops the hart directly and only informs a hypervisor
+    // about it for the bookkeeping pourposes.
+    // ResumePending,
+}

security-monitor/src/core/architecture/riscv/hart_lifecycle_state_transition.rs

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2023 IBM Corporation
+// SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
+// SPDX-License-Identifier: Apache-2.0
+use crate::core::transformations::{SbiHsmHartStart, SbiHsmHartSuspend};
+
+pub enum HartLifecycleStateTransition {
+    StoppedToStartPending(SbiHsmHartStart),
+    StartedToSuspended(SbiHsmHartSuspend),
+    SuspendedToStarted(),
+    StartedToStopped(),
+}

security-monitor/src/core/architecture/riscv/mod.rs

@@ -4,13 +4,17 @@
 pub use compressed_instructions::decode_result_register;
 pub use fp_registers::FpRegisters;
 pub use gp_registers::{GpRegister, GpRegisters};
-pub use hart_state::HartState;
+pub use hart_architectural_state::HartArchitecturalState;
+pub use hart_lifecycle_state::HartLifecycleState;
+pub use hart_lifecycle_state_transition::HartLifecycleStateTransition;
 pub use sbi::{AceExtension, BaseExtension, HsmExtension, IpiExtension, RfenceExtension, SbiExtension, SrstExtension};
 pub use trap_reason::TrapReason;
 
 mod compressed_instructions;
 mod fp_registers;
 mod gp_registers;
-mod hart_state;
+mod hart_architectural_state;
+mod hart_lifecycle_state;
+mod hart_lifecycle_state_transition;
 mod sbi;
 mod trap_reason;

security-monitor/src/core/architecture/riscv/sbi.rs

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: 2023 IBM Corporation
 // SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
 // SPDX-License-Identifier: Apache-2.0
-use crate::core::architecture::{GpRegister, HartState};
+use crate::core::architecture::{GpRegister, HartArchitecturalState};
 
 #[derive(Debug)]
 pub enum SbiExtension {
@@ -16,7 +16,7 @@ pub enum SbiExtension {
 }
 
 impl SbiExtension {
-    pub fn decode(hart_state: &HartState) -> Self {
+    pub fn decode(hart_state: &HartArchitecturalState) -> Self {
         match (hart_state.gpr(GpRegister::a7), hart_state.gpr(GpRegister::a6)) {
             (AceExtension::EXTID, function_id) => Self::Ace(AceExtension::from_function_id(function_id)),
             (BaseExtension::EXTID, function_id) => Self::Base(BaseExtension::from_function_id(function_id)),

security-monitor/src/core/architecture/riscv/trap_reason.rs

@@ -2,7 +2,7 @@
 // SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
 // SPDX-License-Identifier: Apache-2.0
 use crate::core::architecture::riscv::sbi::SbiExtension;
-use crate::core::architecture::HartState;
+use crate::core::architecture::HartArchitecturalState;
 
 #[derive(Debug)]
 pub enum TrapReason {
@@ -24,7 +24,7 @@ impl TrapReason {
     const GUEST_LOAD_PAGE_FAULT: usize = 21;
     const GUEST_STORE_PAGE_FAULT: usize = 23;
 
-    pub fn from_hart_state(hart_state: &HartState) -> Self {
+    pub fn from_hart_state(hart_state: &HartArchitecturalState) -> Self {
         let mcause = riscv::register::mcause::read();
         if mcause.is_interrupt() {
             return Self::Interrupt;