Added a sample Linux kernel-based confidential VM

.github/workflows/build.yml

@@ -43,5 +43,5 @@ jobs:
       run: ACE_DIR=$(pwd)/build/ MAKEFLAGS="--silent -j4" make hypervisor
     - name: build firmware
       run: ACE_DIR=$(pwd)/build/ MAKEFLAGS="--silent -j4" make firmware
-    - name: build cvms
-      run: ACE_DIR=$(pwd)/build/ MAKEFLAGS="--silent -j4" make cvms
+    - name: build confidential_vms
+      run: ACE_DIR=$(pwd)/build/ MAKEFLAGS="--silent -j4" make confidential_vms

.gitignore

@@ -7,6 +7,8 @@ target/*
 security-monitor/target
 
 configurations/overlay/root/harness/baremetal
+confidential-vms/linux_vm/configurations/package_override.dev
+hypervisor/configurations/package_override.dev
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
@@ -15,7 +17,6 @@ Cargo.lock
 # These are backup files generated by rustfmt
 **/*.rs.bk
 
-
 # skip kernel modules
 *.ko
 

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "hypervisor/linux"]
-	path = hypervisor/linux
-	url = https://github.com/torvalds/linux.git
 [submodule "hypervisor/buildroot"]
 	path = hypervisor/buildroot
 	url = https://github.com/buildroot/buildroot.git

Makefile

@@ -6,24 +6,31 @@ MAKEFILE_PATH 							:= $(abspath $(lastword $(MAKEFILE_LIST)))
 MAKEFILE_SOURCE_DIR 					:= $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
 
 export ACE_DIR 							?= $(MAKEFILE_SOURCE_DIR)/build/
+# QEMU
 export QEMU_SOURCE_DIR					?= $(MAKEFILE_SOURCE_DIR)/qemu/
 export QEMU_WORK_DIR					?= $(ACE_DIR)/qemu/
 export QEMU_RISCV_WORK_DIR				?= $(ACE_DIR)/qemu-riscv/
+# Riscv toolchain
 export RISCV_GNU_TOOLCHAIN_SOURCE_DIR	?= $(MAKEFILE_SOURCE_DIR)/riscv-gnu-toolchain/
 export RISCV_GNU_TOOLCHAIN_WORK_DIR		?= $(ACE_DIR)/riscv-gnu-toolchain/
-export CVMS_SOURCE_DIR					?= $(MAKEFILE_SOURCE_DIR)/confidential-vms
-export OVERLAY_ROOT_DIR					?= $(ACE_DIR)/overlay/root
-export LINUX_IMAGE						?= $(ACE_DIR)/linux/arch/riscv/boot/Image
+# Confidential VMs
+export CONFIDENTIAL_VMS_SOURCE_DIR		?= $(MAKEFILE_SOURCE_DIR)/confidential-vms
+# Hypervisor
+export HYPERVISOR_WORK_DIR				?= $(ACE_DIR)/hypervisor/
+export HYPERVISOR_OVERLAY_DIR			?= $(HYPERVISOR_WORK_DIR)/overlay
+export HYPERVISOR_OVERLAY_ROOT_DIR		?= $(HYPERVISOR_OVERLAY_DIR)/root
+export LINUX_IMAGE						?= $(HYPERVISOR_WORK_DIR)/buildroot/images/Image
+# Tools
 export TOOLS_SOURCE_DIR					?= $(MAKEFILE_SOURCE_DIR)/tools
 export TOOLS_WORK_DIR					?= $(ACE_DIR)/tools
-export CROSS_COMPILE					?= riscv64-unknown-linux-gnu-
 
+export CROSS_COMPILE					?= riscv64-unknown-linux-gnu-
 export PLATFORM_RISCV_XLEN				= 64
 export PLATFORM_RISCV_ISA				= rv64gc
 export PLATFORM_RISCV_ABI				= lp64d
 export PATH 							:= $(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)
 
-all: emulator tools firmware cvms
+all: emulator tools firmware confidential_vms
 
 setup:
 	echo $(ACE_DIR)
@@ -42,12 +49,19 @@ devtools: setup
 hypervisor: setup devtools
 	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) $(MAKE) -C hypervisor
 
-new_patches:
-	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) $(MAKE) -C hypervisor new_patches
+confidential_vms: setup devtools hypervisor
+	BIN_DIR="$(OVERLAY_ROOT_DIR)/" RELEASE="" $(MAKE) -C $(CONFIDENTIAL_VMS_SOURCE_DIR)/baremetal/ ;\
+	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) $(MAKE) -C $(CONFIDENTIAL_VMS_SOURCE_DIR)/linux_vm/ buildroot ;\
+	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) $(MAKE) -C $(CONFIDENTIAL_VMS_SOURCE_DIR)/linux_vm/ overlay ;\
+	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) $(MAKE) -C hypervisor rootfs;
+
+hypervisor_dev: 
+	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) $(MAKE) -C hypervisor dev
 
-cvms: setup devtools hypervisor
-	BIN_DIR="$(OVERLAY_ROOT_DIR)/" $(MAKE) -C $(CVMS_SOURCE_DIR)/baremetal/ debug ; \
-	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) $(MAKE) -C hypervisor overlay rootfs;
+dev:
+	$(MAKE) -C $(CONFIDENTIAL_VMS_SOURCE_DIR)/linux_vm/ dev ;\
+	$(MAKE) -C $(CONFIDENTIAL_VMS_SOURCE_DIR)/linux_vm/ overlay ;\
+	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) $(MAKE) -C hypervisor rootfs;
 
 firmware: setup devtools hypervisor
 	PATH="$(RISCV_GNU_TOOLCHAIN_WORK_DIR)/bin:$(PATH)" ACE_DIR=$(ACE_DIR) LINUX_IMAGE=$(LINUX_IMAGE) CROSS_COMPILE=$(CROSS_COMPILE) PLATFORM_RISCV_XLEN=$(PLATFORM_RISCV_XLEN) PLATFORM_RISCV_ISA=$(PLATFORM_RISCV_ISA) PLATFORM_RISCV_ABI=$(PLATFORM_RISCV_ABI) $(MAKE) -C security-monitor opensbi

README.md

@@ -7,7 +7,8 @@ ACE-RISCV is an open-source project, whose goal is to deliver a confidential com
 
 **This is an active research project, without warranties of any kind.** Please read our [paper](https://dl.acm.org/doi/pdf/10.1145/3623652.3623668) to learn about our approach and goals.
 
-We are currently building on RISC-V with hypervisor extentions. We will adapt the AP-TEE extension once it is ratified.
+## Hardware requirements
+We are currently building on RISC-V with hypervisor extentions, physical memory protection (PMP), IOPMP, and supervisor timecmp extension (Sstc). We plan to adapt some of the RISC-V confidential computing extensions, such as [the CoVE extension](https://github.com/riscv-non-isa/riscv-ap-tee/blob/main/specification/riscv-cove.pdf) and [the Smmtt extension](https://github.com/riscv/riscv-smmtt).
 
 ## Quick Start
 Follow instructions to run a sample [confidential workload](harness/baremetal) under an [untrusted Linux-based hypervisor](hypervisor/) in an [emulated RISC-V environment](qemu/). 
@@ -90,6 +91,11 @@ Build the firmware that will boot the system and the security monitor (SM)
 make firmware
 ```
 
+Build test confidential VMs
+```
+make confidential_vms
+```
+
 Build the RISC-V emulator and tools that will simplify running the test environment
 ```
 make emulator
@@ -111,9 +117,14 @@ You should see the output from the boot process and a promt to login to the hype
 # login: root, password: passwd
 ```
 
-To run the sample confidential VM execute:
+To run the sample `baremetal` confidential VM execute:
+```
+./run_baremetal.sh
+```
+
+To run the sample Linux kernel confidential VM execute:
 ```
-./run.sh
+./run_linux_vm.sh
 ```
 
 # License

confidential-vms/README.md

@@ -2,4 +2,7 @@
 This folder contains sample confidential VMs. A confidential VM is a workload that executes confidentially on the ACE infrastructure. The hypervisor, virtual machines, other confidential VMs, untrusted peripherial devices are considered untrusted.
 
 ## Baremetal CVM
-A baremetal CVM is a minimal proof of concept VM that leverages ACE to run confidentially. It is a bare metal application running in virtual supervisor mode that tests presence of certain hypercalls and virtIO.
+A baremetal confidenital VM is a minimal proof of concept VM that leverages ACE to run confidentially. It is a bare metal application running in virtual supervisor mode that tests presence of certain hypercalls and virtIO.
+
+## Linux VM
+It is a proof of concept that linux-based VMs can execute in a trusted execution environment (TEE) provided by ACE.

confidential-vms/baremetal/Cargo.toml

@@ -17,4 +17,4 @@ buddy_system_allocator = "0.9.0"
 sbi = "0.2.0"
 
 # provides macros that help removing boilerplate code in rust error handling
-thiserror-no-std = "2.0" 
+thiserror-no-std = "2.0"

confidential-vms/baremetal/Makefile

@@ -1,29 +1,39 @@
-CARGO = cargo
-RELEASE = --release
-CHAIN=riscv64gc-unknown-none-elf
-TARGET = --target=$(CHAIN)
-RUSTFLAGS = -Clink-arg=-Tsrc/platform/virt.lds -Crelocation-model=pie #-Ccode-model=medium 
-EXEC_NAME = baremetal
-MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
-MAKEFILE_SOURCE_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
-
-ACE_DIR := $(if $(ACE_DIR),$(ACE_DIR),$(MAKEFILE_SOURCE_DIR)target/)
-BIN_DIR := $(if $(BIN_DIR),$(BIN_DIR),$(ACE_DIR)/harness/baremetal/)
-TARGET_DIR := $(ACE_DIR)/harness/baremetal/
-
-all: build
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2023 IBM Corporation
+# SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
+# SPDX-License-Identifier: Apache-2.0
+CARGO 											= cargo
+RELEASE 										?= --release
+CHAIN											= riscv64gc-unknown-none-elf
+TARGET 											= --target=$(CHAIN)
+RUSTFLAGS 										= -Clink-arg=-Tsrc/platform/virt.lds -Crelocation-model=pie #-Ccode-model=medium 
+EXEC_NAME 										= baremetal
+
+ACE_DIR					 						:= $(if $(ACE_DIR),$(ACE_DIR),$(MAKEFILE_SOURCE_DIR)../../build/)
+#
+MAKEFILE_PATH 									:= $(abspath $(lastword $(MAKEFILE_LIST)))
+MAKEFILE_SOURCE_DIR 							:= $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+#
+CONFIDENTIAL_VMS_WORK_DIR 						:= $(ACE_DIR)/confidential_vms/
+CONFIDENTIAL_VMS_BAREMETAL_WORK_DIR				:= $(CONFIDENTIAL_VMS_WORK_DIR)/baremetal/
+#
+HYPERVISOR_OVERLAY_DIR							?= $(ACE_DIR)/hypervisor/overlay
+HYPERVISOR_OVERLAY_ROOT_DIR						?= $(HYPERVISOR_OVERLAY_DIR)/root
+HYPERVISOR_OVERLAY_BAREMETAL_DIR				?= $(HYPERVISOR_OVERLAY_ROOT_DIR)/baremetal
+CONFIDENTIAL_VMS_BAREMETAL_ROOTFS_SOURCE_DIR	?= $(MAKEFILE_SOURCE_DIR)/rootfs
+
+all: overlay
 
 build: fmt
-	@mkdir -p $(TARGET_DIR)
-	@mkdir -p $(BIN_DIR)
-	@RUSTFLAGS='$(RUSTFLAGS)' CARGO_TARGET_DIR=$(TARGET_DIR) INSTALL_DIR=$(ACE_DIR) $(CARGO) build $(RELEASE) $(TARGET)
-	@cp $(TARGET_DIR)/$(CHAIN)/release/$(EXEC_NAME) $(BIN_DIR)/
-
-debug:
-	@mkdir -p $(TARGET_DIR)
-	@mkdir -p $(BIN_DIR)
-	@RUSTFLAGS='$(RUSTFLAGS)' CARGO_TARGET_DIR=$(TARGET_DIR) INSTALL_DIR=$(ACE_DIR) $(CARGO) build $(TARGET)
-	@cp $(TARGET_DIR)/$(CHAIN)/debug/$(EXEC_NAME) $(BIN_DIR)/
+	@mkdir -p $(CONFIDENTIAL_VMS_BAREMETAL_WORK_DIR)
+	@RUSTFLAGS='$(RUSTFLAGS)' CARGO_TARGET_DIR=$(CONFIDENTIAL_VMS_BAREMETAL_WORK_DIR) INSTALL_DIR=$(HYPERVISOR_OVERLAY_BAREMETAL_DIR) $(CARGO) build $(RELEASE) $(TARGET)
+
+overlay: build
+	mkdir -p $(HYPERVISOR_OVERLAY_ROOT_DIR) ;\
+	mkdir -p $(HYPERVISOR_OVERLAY_BAREMETAL_DIR) ;\
+	cp $(CONFIDENTIAL_VMS_BAREMETAL_ROOTFS_SOURCE_DIR)/*.sh $(HYPERVISOR_OVERLAY_ROOT_DIR)/ ;\
+	cp $(CONFIDENTIAL_VMS_BAREMETAL_WORK_DIR)/$(CHAIN)/release/$(EXEC_NAME) $(HYPERVISOR_OVERLAY_BAREMETAL_DIR)/ ;\
+	cp $(CONFIDENTIAL_VMS_BAREMETAL_WORK_DIR)/$(CHAIN)/debug/$(EXEC_NAME) $(HYPERVISOR_OVERLAY_BAREMETAL_DIR)/
 
 doc:
 	@$(CARGO) doc

hypervisor/rootfs/run.sh → confidential-vms/baremetal/rootfs/run_baremetal.sh

@@ -6,4 +6,4 @@
 # this script is used for the development process to run the VM/confidential VM
 . common.sh
 
-run_baremetal
+run_confidential_vm "baremetal/baremetal" 2 128M

hypervisor/rootfs/test_esm.sh → confidential-vms/baremetal/rootfs/test_esm.sh

@@ -5,9 +5,9 @@
 
 . common.sh
 
-run_baremetal
-sleep 15 # wait for the test to finish
-kill_baremetal
+run_confidential_vm "baremetal/baremetal" 2 128M
+sleep 25 # wait for the test to finish
+kill_confidential_vm
 
 result="$(grep 'Hello IBM from confidential VM' guest.log | wc -l)"
 if [[ "$result" -ne 1 ]]; then

confidential-vms/baremetal/src/calls/mod.rs

@@ -7,15 +7,7 @@ use core::arch::asm;
 pub mod ace;
 pub mod sm;
 
-fn ecall(
-    extid: usize,
-    fid: usize,
-    a0: usize,
-    a1: usize,
-    a2: usize,
-    a3: usize,
-    a4: usize,
-) -> Result<usize, usize> {
+fn ecall(extid: usize, fid: usize, a0: usize, a1: usize, a2: usize, a3: usize, a4: usize) -> Result<usize, usize> {
     let (mut error, mut value);
     unsafe {
         asm!("ecall", in("a0") a0, in("a1") a1, in("a2") a2, in("a3") a3, in("a4") a4, in("a6") fid, in("a7") extid, lateout("a0") error, lateout("a1") value)

confidential-vms/baremetal/src/calls/sm.rs

@@ -13,14 +13,5 @@ pub fn esm() -> Result<usize, Error> {
 }
 
 pub fn share_page(paddr: usize, number_of_pages: usize) -> Result<usize, Error> {
-    super::ecall(
-        ACE_EXTID,
-        ACE_SHARE_PAGE_FID,
-        paddr,
-        number_of_pages,
-        0,
-        0,
-        0,
-    )
-    .map_err(|_| Error::SharePageError())
+    super::ecall(ACE_EXTID, ACE_SHARE_PAGE_FID, paddr, number_of_pages, 0, 0, 0).map_err(|_| Error::SharePageError())
 }

confidential-vms/baremetal/src/hal.rs

@@ -1,9 +1,9 @@
 // SPDX-FileCopyrightText: 2023 IBM Corporation
 // SPDX-FileContributor: Wojciech Ozga <woz@zurich.ibm.com>, IBM Research - Zurich
 // SPDX-License-Identifier: Apache-2.0
-use core::{ptr::NonNull, sync::atomic::Ordering};
-use virtio_drivers::PAGE_SIZE;
-use virtio_drivers::{BufferDirection, Hal, PhysAddr};
+use core::ptr::NonNull;
+use core::sync::atomic::Ordering;
+use virtio_drivers::{BufferDirection, Hal, PhysAddr, PAGE_SIZE};
 
 pub struct ScratchPage {
     pub base_paddr: usize,
@@ -60,12 +60,7 @@ unsafe impl Hal for HalSvmImpl {
                 .and_then(|sp| {
                     let position = sp.position;
                     let paddr = sp.base_paddr + position;
-                    sp.translations.push(BufferTranslation {
-                        vaddr,
-                        paddr,
-                        position,
-                        len: buffer.len(),
-                    });
+                    sp.translations.push(BufferTranslation { vaddr, paddr, position, len: buffer.len() });
                     for i in 0..buffer.len() {
                         let input_ptr = (vaddr + i) as *mut u8;
                         let output_ptr = (sp.base_paddr + sp.position) as *mut u8;

confidential-vms/baremetal/src/macros.rs

@@ -28,12 +28,7 @@ macro_rules! println
 fn panic(info: &core::panic::PanicInfo) -> ! {
     print!("Aborting: ");
     if let Some(p) = info.location() {
-        println!(
-            "line {}, file {}: {}",
-            p.line(),
-            p.file(),
-            info.message().unwrap()
-        );
+        println!("line {}, file {}: {}", p.line(), p.file(), info.message().unwrap());
     } else {
         println!("no information available.");
     }

confidential-vms/baremetal/src/main.rs

@@ -21,8 +21,8 @@ mod macros;
 mod calls;
 mod error;
 mod hal;
-mod trap;
 mod sync;
+mod trap;
 mod virtio;
 mod worker;
 
@@ -144,8 +144,7 @@ fn test_virtio(fdt_paddr: usize) -> Result<(), Error> {
 
     let (input_paddr, output_paddr) = prepare_shared_memory()?;
     let input: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(input_paddr as *mut u8, 512) };
-    let mut output: &mut [u8] =
-        unsafe { core::slice::from_raw_parts_mut(output_paddr as *mut u8, 512) };
+    let mut output: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(output_paddr as *mut u8, 512) };
     for x in input.iter_mut() {
         *x = 'I' as u8;
     }
@@ -169,22 +168,10 @@ fn test_virtio(fdt_paddr: usize) -> Result<(), Error> {
 
 fn init_memory(uart: &mut Uart) {
     unsafe {
-        HEAP_ALLOCATOR
-            .lock()
-            .init(_heap_start as usize, _heap_size as usize);
-        uart.println(&format!(
-            "Stack 0x{:x}-0x{:x}",
-            _stack_start as usize, _stack_end as usize
-        ));
-        uart.println(&format!(
-            "DMA   0x{:x}-0x{:x}",
-            _dma_start as usize, _dma_end as usize
-        ));
-        uart.println(&format!(
-            "Heap  0x{:x}-0x{:x}",
-            _heap_start as usize,
-            _heap_start as usize + _heap_size as usize
-        ));
+        HEAP_ALLOCATOR.lock().init(_heap_start as usize, _heap_size as usize);
+        uart.println(&format!("Stack 0x{:x}-0x{:x}", _stack_start as usize, _stack_end as usize));
+        uart.println(&format!("DMA   0x{:x}-0x{:x}", _dma_start as usize, _dma_end as usize));
+        uart.println(&format!("Heap  0x{:x}-0x{:x}", _heap_start as usize, _heap_start as usize + _heap_size as usize));
         let dma_start = (_dma_start as usize + 4096 - 1) & !(4096 - 1);
         crate::DMA_PADDR = Some(AtomicUsize::new(dma_start));
     }
@@ -202,10 +189,7 @@ fn prepare_shared_memory() -> Result<(usize, usize), Error> {
     let pages_to_allocate = 3;
     let paddr = unsafe {
         if let Some(v) = &crate::DMA_PADDR {
-            v.fetch_add(
-                virtio_drivers::PAGE_SIZE * pages_to_allocate,
-                core::sync::atomic::Ordering::SeqCst,
-            )
+            v.fetch_add(virtio_drivers::PAGE_SIZE * pages_to_allocate, core::sync::atomic::Ordering::SeqCst)
         } else {
             return Err(Error::DmaNotInitialized());
         }
@@ -218,14 +202,9 @@ fn prepare_shared_memory() -> Result<(usize, usize), Error> {
     let scratch_paddr = paddr + 2 * 4096;
 
     unsafe {
-        crate::SCRATCH_PAGE = Some(crate::hal::ScratchPage {
-            base_paddr: scratch_paddr,
-            position: 0,
-            translations: alloc::vec![],
-        });
+        crate::SCRATCH_PAGE =
+            Some(crate::hal::ScratchPage { base_paddr: scratch_paddr, position: 0, translations: alloc::vec![] });
     }
 
     Ok((input_paddr, output_paddr))
 }
-
-