emulating illegal instructions

Signed-off-by: Wojciech Ozga <woz@zurich.ibm.com>

Makefile

@@ -14,6 +14,8 @@ export QEMU_VERSION						?= 8.2.1
 # Riscv toolchain
 export RISCV_GNU_TOOLCHAIN_SOURCE_DIR	?= $(MAKEFILE_SOURCE_DIR)/riscv-gnu-toolchain/
 export RISCV_GNU_TOOLCHAIN_WORK_DIR		?= /home/woz/yocto/build/tmp/work/riscv64-freedomusdk-linux/opensbi-sifive-hf-prem/1.4/recipe-sysroot-native/usr/bin/riscv64-freedomusdk-linux/
+# export RISCV_GNU_TOOLCHAIN_WORK_DIR		?= /home/woz/yocto/build/tmp/work/hifive_premier_p550-freedomusdk-linux/u-boot-sifive-hf-prem/2024.01/recipe-sysroot-native/usr/bin/riscv64-freedomusdk-linux/
+
 export RISCV_GNU_TOOLCHAIN_VERSION	    ?= riscv64-glibc-ubuntu-22.04-gcc-nightly-2023.09.27-nightly
 # Confidential VMs
 export CONFIDENTIAL_VMS_SOURCE_DIR		?= $(MAKEFILE_SOURCE_DIR)/confidential-vms

confidential-vms/linux_vm/hypervisor_rootfs/run_linux_vm_qemu.sh

@@ -11,7 +11,7 @@ INITRAMFS=/root/linux_vm/rootfs.cpio
 HOST_PORT="$((3000 + RANDOM % 3000))"
 INTERACTIVE="-nographic"
 SMP=1
-MEMORY=1G
+MEMORY=512M
 
 for i in "$@"; do
   case $i in

security-monitor/src/confidential_flow/finite_state_machine.rs

@@ -17,6 +17,7 @@ use crate::confidential_flow::handlers::symmetrical_multiprocessing::{
     Ipi, NoOperation, RemoteFenceI, RemoteSfenceVma, RemoteSfenceVmaAsid, SbiHsmHartStart, SbiHsmHartStatus, SbiHsmHartStop,
     SbiHsmHartSuspend,
 };
+use crate::confidential_flow::handlers::time::SetTimer;
 use crate::confidential_flow::handlers::virtual_instructions::VirtualInstruction;
 use crate::confidential_flow::{ApplyToConfidentialHart, DeclassifyToConfidentialVm};
 use crate::core::architecture::riscv::sbi::BaseExtension::*;
@@ -26,12 +27,16 @@ use crate::core::architecture::riscv::sbi::IpiExtension::*;
 use crate::core::architecture::riscv::sbi::RfenceExtension::*;
 use crate::core::architecture::riscv::sbi::SbiExtension::*;
 use crate::core::architecture::riscv::sbi::SrstExtension::*;
+use crate::core::architecture::riscv::sbi::TimeExtension::*;
+use crate::core::architecture::sbi::CovgExtension;
+use crate::core::architecture::sbi::SbiExtension::Time;
 use crate::core::architecture::TrapCause::*;
 use crate::core::architecture::{GeneralPurposeRegister, HartLifecycleState, TrapCause};
 use crate::core::control_data::{
     ConfidentialHart, ConfidentialHartRemoteCommand, ConfidentialVm, ConfidentialVmId, ControlDataStorage, HardwareHart, HypervisorHart,
     ResumableOperation,
 };
+use crate::core::timer_controller::TimerController;
 use crate::error::Error;
 use crate::non_confidential_flow::{DeclassifyToHypervisor, NonConfidentialFlow};
 
@@ -66,40 +71,22 @@ impl<'a> ConfidentialFlow<'a> {
     /// ConfidentialFlow but accessible to the assembly code performing the context switch.
     #[no_mangle]
     unsafe extern "C" fn route_trap_from_confidential_hart(hardware_hart_pointer: *mut HardwareHart) -> ! {
-        let flow = Self { hardware_hart: unsafe { hardware_hart_pointer.as_mut().expect(Self::CTX_SWITCH_ERROR_MSG) } };
+        let mut flow = Self { hardware_hart: unsafe { hardware_hart_pointer.as_mut().expect(Self::CTX_SWITCH_ERROR_MSG) } };
         assert!(!flow.hardware_hart.confidential_hart().is_dummy());
-        let tt = TrapCause::from_hart_architectural_state(flow.confidential_hart().confidential_hart_state());
-        let a7 = flow.confidential_hart().gprs().read(GeneralPurposeRegister::a7);
-        let a6 = flow.confidential_hart().gprs().read(GeneralPurposeRegister::a6);
-        // match tt {
-        //     Interrupt => {}
-        //     _ => {
-        //         debug!(
-        //             "Enter a7={:x} a6={:x} mstatus={:x} mcause={} mepc={:x}",
-        //             a7,
-        //             a6,
-        //             flow.confidential_hart().csrs().mstatus.read_from_main_memory(),
-        //             flow.confidential_hart().csrs().mcause.read(),
-        //             flow.confidential_hart().csrs().mepc.read_from_main_memory(),
-        //         );
-        //         // crate::debug::__print_hart_state(flow.confidential_hart().confidential_hart_state());
-        //     }
-        // }
 
         use crate::core::architecture::specification::CSR_MSTATUS_MPV;
         if (flow.confidential_hart().confidential_hart_state().csrs().mstatus.read() & 1 << CSR_MSTATUS_MPV) == 0 {
-            crate::debug::__print_hart_state(flow.confidential_hart().confidential_hart_state());
-            debug!("Bug when executing confidential hart {}", flow.confidential_hart().confidential_hart_id());
+            debug!(
+                "Bug when executing confidential hart {} cause={}",
+                flow.confidential_hart().confidential_hart_id(),
+                flow.confidential_hart().confidential_hart_state().csrs().mcause.read()
+            );
             ShutdownRequest::from_confidential_hart(flow.confidential_hart()).handle(flow)
         }
 
-        match tt {
+        match TrapCause::from_hart_architectural_state(flow.confidential_hart().confidential_hart_state()) {
             Interrupt => HandleInterrupt::from_confidential_hart(flow.confidential_hart()).handle(flow),
             IllegalInstruction => DelegateToConfidentialVm::from_confidential_hart(flow.confidential_hart()).handle(flow),
-            LoadAddressMisaligned => DelegateToConfidentialVm::from_confidential_hart(flow.confidential_hart()).handle(flow),
-            StoreAddressMisaligned => DelegateToConfidentialVm::from_confidential_hart(flow.confidential_hart()).handle(flow),
-            LoadAccessFault => DelegateToConfidentialVm::from_confidential_hart(flow.confidential_hart()).handle(flow),
-            StoreAccessFault => DelegateToConfidentialVm::from_confidential_hart(flow.confidential_hart()).handle(flow),
             VsEcall(Base(GetSpecVersion)) => SbiGetSpecVersion::from_confidential_hart(flow.confidential_hart()).handle(flow),
             VsEcall(Base(GetImplId)) => SbiGetImplId::from_confidential_hart(flow.confidential_hart()).handle(flow),
             VsEcall(Base(GetImplVersion)) => SbiGetImplVersion::from_confidential_hart(flow.confidential_hart()).handle(flow),
@@ -127,7 +114,8 @@ impl<'a> ConfidentialFlow<'a> {
             VsEcall(Covg(UnshareMemory)) => UnsharePageRequest::from_confidential_hart(flow.confidential_hart()).handle(flow),
             VsEcall(Covg(RetrieveSecret)) => RetrieveSecretRequest::from_confidential_hart(flow.confidential_hart()).handle(flow),
             VsEcall(Covg(Debug)) => DebugRequest::from_confidential_hart(flow.confidential_hart()).handle(flow),
-            VsEcall(Covg(Time)) => TimeRequest::from_confidential_hart(flow.confidential_hart()).handle(flow),
+            VsEcall(Covg(CovgExtension::Time)) => TimeRequest::from_confidential_hart(flow.confidential_hart()).handle(flow),
+            VsEcall(Time(SetTimer)) => SetTimer::from_confidential_hart(flow.confidential_hart()).handle(flow),
             VsEcall(_) => InvalidCall::from_confidential_hart(flow.confidential_hart()).handle(flow),
             GuestLoadPageFault => MmioLoadRequest::from_confidential_hart(flow.confidential_hart()).handle(flow),
             VirtualInstruction => VirtualInstruction::from_confidential_hart(flow.confidential_hart()).handle(flow),
@@ -162,12 +150,14 @@ impl<'a> ConfidentialFlow<'a> {
     }
 
     /// Moves in the finite state machine (FSM) from the confidential flow into non-confidential flow.
-    pub fn into_non_confidential_flow(self) -> NonConfidentialFlow<'a> {
+    pub fn into_non_confidential_flow(mut self) -> NonConfidentialFlow<'a> {
         // When moving back to the non-confidential flow, we always declassify enabled interrupts and timer configuration. This allows the
         // hypervisor to schedule the confidential VM timer and interrupts. Read the CoVE spec to learn more.
         let declassifier =
             DeclassifyToHypervisor::EnabledInterrupts(ExposeEnabledInterrupts::from_confidential_hart(self.confidential_hart()));
 
+        TimerController::new(&mut self).store_vs_timer();
+
         ControlDataStorage::try_confidential_vm(self.confidential_vm_id(), |mut confidential_vm| {
             // Run heavy context switch when giving back the confidential hart to the confidential VM.
             confidential_vm.return_confidential_hart(self.hardware_hart);
@@ -235,12 +225,11 @@ impl<'a> ConfidentialFlow<'a> {
         // We must restore the control and status registers (CSRs) that might have changed during execution of the security monitor.
         // We call it here because it is just before exiting to the assembly context switch, so we are sure that these CSRs have their
         // final values.
+        let interrupts = (self.confidential_hart().csrs().hvip.read_from_main_memory()
+            | self.confidential_hart().csrs().pending_interrupts)
+            & self.confidential_hart().csrs().allowed_external_interrupts;
+        self.confidential_hart().csrs().hvip.write(interrupts);
         let address = self.confidential_hart_mut().address();
-        self.confidential_hart()
-            .csrs()
-            .hvip
-            .write(self.confidential_hart().csrs().hvip.read_from_main_memory() & (1 << 2 | 1 << 6 | 1 << 10));
-        self.confidential_hart().csrs().vsip.write(self.confidential_hart().csrs().vsip.read_from_main_memory());
         self.confidential_hart().csrs().sscratch.write(address);
         unsafe { exit_to_confidential_hart_asm() }
     }
@@ -341,4 +330,8 @@ impl<'a> ConfidentialFlow<'a> {
     fn hypervisor_hart(&'a self) -> &HypervisorHart {
         &self.hardware_hart.hypervisor_hart()
     }
+
+    pub fn swap_mscratch(&mut self) {
+        self.hardware_hart.swap_mscratch()
+    }
 }

security-monitor/src/confidential_flow/handlers/delegate/mod.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 use crate::confidential_flow::handlers::sbi::SbiResponse;
 use crate::confidential_flow::{ApplyToConfidentialHart, ConfidentialFlow};
+use crate::core::architecture::specification::CAUSE_ILLEGAL_INSTRUCTION;
 use crate::core::architecture::GeneralPurposeRegister;
 use crate::core::control_data::ConfidentialHart;
 use crate::error::Error;
@@ -18,7 +19,8 @@ pub struct DelegateToConfidentialVm {
     mtval: usize,
     vstvec: usize,
     vsstatus: usize,
-    mtinst: usize,
+    inst: usize,
+    inst_len: usize,
 }
 
 impl DelegateToConfidentialVm {
@@ -29,27 +31,91 @@ impl DelegateToConfidentialVm {
         let mtval = confidential_hart.csrs().mtval.read();
         let vstvec = confidential_hart.csrs().vstvec.read();
         let vsstatus = confidential_hart.csrs().vsstatus.read();
-        let mtinst = crate::confidential_flow::handlers::mmio::read_trapped_instruction(confidential_hart).0;
+        let (inst, inst_len) = crate::confidential_flow::handlers::mmio::read_trapped_instruction(confidential_hart);
+        Self { mstatus, mcause, mepc, mtval, vstvec, vsstatus, inst, inst_len }
+    }
 
-        Self { mstatus, mcause, mepc, mtval, vstvec, vsstatus, mtinst }
+    pub fn handle(self, mut confidential_flow: ConfidentialFlow) -> ! {
+        if self.mcause == CAUSE_ILLEGAL_INSTRUCTION.into() {
+            self.emulate_illegal_instruction(confidential_flow)
+        } else {
+            debug!(
+                "Delegating mcause={} mepc={:x} mtinst={:x} mstatus={:x} vstvec={:x} vsstatus={:x} to conf vm",
+                self.mcause, self.mepc, self.inst, self.mstatus, self.vstvec, self.vsstatus
+            );
+            confidential_flow.apply_and_exit_to_confidential_hart(ApplyToConfidentialHart::DelegateToConfidentialVm(self))
+        }
     }
 
-    pub fn handle(self, confidential_flow: ConfidentialFlow) -> ! {
-        debug!(
-            "Delegating mcause={} mepc={:x} mtinst={:x} mstatus={:x} vstvec={:x} vsstatus={:x} to conf vm",
-            self.mcause, self.mepc, self.mtinst, self.mstatus, self.vstvec, self.vsstatus
-        );
-        confidential_flow.apply_and_exit_to_confidential_hart(ApplyToConfidentialHart::DelegateToConfidentialVm(self))
+    pub fn emulate_illegal_instruction(self, mut confidential_flow: ConfidentialFlow) -> ! {
+        use riscv_decode::Instruction::*;
+        let get_rs1_num = |instruction| ((instruction & 0xf8000) >> 15);
+
+        let (csr_value, new_value, do_write, result_gpr_index) = match riscv_decode::decode(self.inst as u32) {
+            Ok(Csrrw(t)) => (self.read_csr(t.csr()), t.rs1() as usize, true, t.rd()),
+            Ok(Csrrs(t)) => {
+                let csr_value = self.read_csr(t.csr());
+                (csr_value, csr_value | (t.rs1() as usize), get_rs1_num(self.inst) != 0, t.rd())
+            }
+            Ok(Csrrc(t)) => {
+                let csr_value = self.read_csr(t.csr());
+                (self.read_csr(t.csr()), csr_value & !(t.rs1() as usize), get_rs1_num(self.inst) != 0, t.rd())
+            }
+            Ok(Csrrwi(t)) => (self.read_csr(t.csr()), get_rs1_num(self.inst), true, t.rd()),
+            Ok(Csrrsi(t)) => {
+                let csr_value = self.read_csr(t.csr());
+                let rs1_num = get_rs1_num(self.inst);
+                (csr_value, csr_value | rs1_num, rs1_num != 0, t.rd())
+            }
+            Ok(Csrrci(t)) => {
+                let csr_value = self.read_csr(t.csr());
+                let rs1_num = get_rs1_num(self.inst);
+                (csr_value, csr_value & !rs1_num, rs1_num != 0, t.rd())
+            }
+            _ => confidential_flow.apply_and_exit_to_confidential_hart(ApplyToConfidentialHart::DelegateToConfidentialVm(self)),
+        };
+
+        if do_write {
+            debug!("Emulate illegal instruction: Not supporting CSR write");
+        }
+
+        match GeneralPurposeRegister::try_from(result_gpr_index as usize) {
+            Ok(v) => {
+                confidential_flow.confidential_hart_mut().gprs_mut().write(v, csr_value as usize);
+            }
+            Err(e) => {
+                debug!("Emulate illegal instruction: Invalid GPR {}", result_gpr_index);
+            }
+        }
+        confidential_flow.confidential_hart_mut().csrs_mut().mepc.add(self.inst_len);
+        confidential_flow.resume_confidential_hart_execution();
+    }
+
+    fn read_csr(&self, csr: u32) -> usize {
+        use crate::core::architecture::specification::*;
+        use crate::core::architecture::CSR;
+
+        if csr == CSR_TIME.into() {
+            return (unsafe { (0x200BFF8 as *const u64).read_volatile() }) as usize;
+        } else if csr == CSR_CYCLE.into() {
+            return CSR.mcycle.read();
+        } else if csr == CSR_INSTRET.into() {
+            return CSR.minstret.read();
+        } else {
+            debug!("Emulate illegal instruction: Unsupported CSR: {:x}", csr);
+            return 0;
+        }
     }
 
     pub fn apply_to_confidential_hart(&self, confidential_hart: &mut ConfidentialHart) {
+        use crate::core::architecture::specification::CSR_MSTATUS_MPP;
         let SR_SPP_MASK = 0x00000100;
         let SR_SIE = 0x00000002;
         let SR_SPIE = 0x00000020;
 
         let mut new_vsstatus = self.vsstatus;
         new_vsstatus &= !SR_SPP_MASK;
-        if self.mstatus & SR_SPP_MASK > 0 {
+        if self.mstatus & (0b11 << CSR_MSTATUS_MPP) == (1 << CSR_MSTATUS_MPP) {
             new_vsstatus |= SR_SPP_MASK;
         }
 
@@ -69,7 +135,7 @@ impl DelegateToConfidentialVm {
         confidential_hart.csrs_mut().vstval.write(self.mtval);
         confidential_hart.csrs_mut().vsepc.write(self.mepc);
         /* Set Guest privilege mode to supervisor */
-        confidential_hart.csrs_mut().mstatus.enable_bits_on_saved_value(SR_SPP_MASK);
+        confidential_hart.csrs_mut().mstatus.enable_bits_on_saved_value((1 << CSR_MSTATUS_MPP));
 
         confidential_hart.csrs_mut().mepc.save_value_in_main_memory(self.vstvec);
     }

security-monitor/src/confidential_flow/handlers/delegate/time.rs

@@ -17,7 +17,6 @@ impl TimeRequest {
     pub fn handle(self, confidential_flow: ConfidentialFlow) -> ! {
         let addr = 0x200BFF8;
         let time = unsafe { (addr as *const u64).read_volatile() } as usize;
-        // let time = CSR.time.read();
         confidential_flow.apply_and_exit_to_confidential_hart(ApplyToConfidentialHart::SbiResponse(SbiResponse::success_with_code(time)))
     }
 }

security-monitor/src/confidential_flow/handlers/interrupts/allow_external_interrupt.rs

@@ -18,6 +18,7 @@ impl AllowExternalInterrupt {
     }
 
     pub fn handle(self, confidential_flow: ConfidentialFlow) -> ! {
+        debug!("enable ext interrupts");
         match ControlDataStorage::try_confidential_vm(confidential_flow.confidential_vm_id(), |mut confidential_vm| {
             Ok(confidential_vm.allow_external_interrupt(self.interrupt_id))
         }) {

security-monitor/src/confidential_flow/handlers/interrupts/expose_enabled_interrupts.rs

@@ -6,19 +6,16 @@ use crate::core::control_data::{ConfidentialHart, HypervisorHart};
 
 pub struct ExposeEnabledInterrupts {
     vsie: usize,
-    // vstimecmp: usize,
+    vstimecmp: usize,
 }
 
 impl ExposeEnabledInterrupts {
     pub fn from_confidential_hart(confidential_hart: &ConfidentialHart) -> Self {
-        Self {
-            vsie: confidential_hart.csrs().vsie.read(),
-            // vstimecmp: confidential_hart.sstc().vstimecmp.read()
-        }
+        Self { vsie: confidential_hart.csrs().vsie.read(), vstimecmp: confidential_hart.csrs().vstimecmp.unwrap_or(usize::MAX - 1) }
     }
 
     pub fn declassify_to_hypervisor_hart(&self, hypervisor_hart: &mut HypervisorHart) {
         hypervisor_hart.shared_memory_mut().write_csr(CSR_VSIE.into(), self.vsie);
-        // hypervisor_hart.shared_memory_mut().write_csr(CSR_VSTIMECMP.into(), self.vstimecmp);
+        hypervisor_hart.shared_memory_mut().write_csr(CSR_VSTIMECMP.into(), self.vstimecmp);
     }
 }