Support for CoVE's deployment model 3

arch/riscv/Kconfig

@@ -526,6 +526,13 @@ config RISCV_COVE_GUEST
 	help
 	  Enables support for running TVMs on platforms supporting CoVE.
 
+config RISCV_COVE_GUEST_PROMOTE
+	bool "Automatic promotion of VM to TVM for Confidential VM Extension(CoVE)"
+	default n
+	select RISCV_COVE_GUEST
+	help
+	  VM requests promotion to TVM during early boot on platforms supporting CoVE.
+
 endmenu # "Confidential VM Extension(CoVE) Support"
 
 endmenu # "Platform type"

arch/riscv/include/asm/kvm_cove.h

@@ -19,6 +19,13 @@
 #include <asm/csr.h>
 #include <asm/sbi.h>
 
+#define KVM_COVE_TSM_CAP_PROMOTE_TVM         0x0
+#define KVM_COVE_TSM_CAP_ATTESTATION_LOCAL   0x1
+#define KVM_COVE_TSM_CAP_ATTESTATION_REMOTE  0x2
+#define KVM_COVE_TSM_CAP_AIA                 0x3
+#define KVM_COVE_TSM_CAP_MRIF                0x4
+#define KVM_COVE_TSM_CAP_MEMORY_ALLOCATION   0x5
+
 #define KVM_COVE_PAGE_SIZE_4K	(1UL << 12)
 #define KVM_COVE_PAGE_SIZE_2MB	(1UL << 21)
 #define KVM_COVE_PAGE_SIZE_1GB	(1UL << 30)
@@ -126,11 +133,15 @@ static inline bool is_cove_vcpu(struct kvm_vcpu *vcpu)
 #ifdef CONFIG_RISCV_COVE_HOST
 
 bool kvm_riscv_cove_enabled(void);
+bool kvm_riscv_cove_capability(unsigned long cap);
 int kvm_riscv_cove_init(void);
 
 /* TVM related functions */
 void kvm_riscv_cove_vm_destroy(struct kvm *kvm);
-int kvm_riscv_cove_vm_init(struct kvm *kvm);
+int kvm_riscv_cove_vm_single_step_init(struct kvm_vcpu *vcpu,
+				       unsigned long fdt_address,
+				       unsigned long tap_addr);
+int kvm_riscv_cove_vm_multi_step_init(struct kvm *kvm);
 
 /* TVM VCPU related functions */
 void kvm_riscv_cove_vcpu_destroy(struct kvm_vcpu *vcpu);
@@ -158,13 +169,20 @@ int kvm_riscv_cove_aia_convert_imsic(struct kvm_vcpu *vcpu, phys_addr_t imsic_pa
 int kvm_riscv_cove_vcpu_imsic_addr(struct kvm_vcpu *vcpu);
 #else
 static inline bool kvm_riscv_cove_enabled(void) {return false; };
+static inline bool kvm_riscv_cove_capability(unsigned long cap) { return false; };
 static inline int kvm_riscv_cove_init(void) { return -1; }
 static inline void kvm_riscv_cove_hardware_disable(void) {}
 static inline int kvm_riscv_cove_hardware_enable(void) {return 0; }
 
 /* TVM related functions */
 static inline void kvm_riscv_cove_vm_destroy(struct kvm *kvm) {}
-static inline int kvm_riscv_cove_vm_init(struct kvm *kvm) {return -1; }
+static inline int kvm_riscv_cove_vm_single_step_init(struct kvm_vcpu *vcpu,
+						     unsigned long fdt_address,
+						     unsigned long tap_addr)
+{
+	return -1;
+}
+static inline int kvm_riscv_cove_vm_multi_step_init(struct kvm *kvm) {return -1; }
 
 /* TVM VCPU related functions */
 static inline void kvm_riscv_cove_vcpu_destroy(struct kvm_vcpu *vcpu) {}

arch/riscv/include/asm/kvm_cove_sbi.h

@@ -78,6 +78,10 @@ int sbi_covh_tvm_demote_page(unsigned long tvmid,
 int sbi_covh_tvm_remove_pages(unsigned long tvmid,
 			      unsigned long tvm_base_page_addr,
 			      unsigned long len);
+int sbi_covh_tsm_promote_to_tvm(unsigned long fdt_address,
+				  unsigned long tap_addr,
+				  unsigned long sepc,
+				  unsigned long *tvmid);
 
 /* Functions related to CoVE Interrupt Management(COVI) Extension */
 int sbi_covi_tvm_aia_init(unsigned long tvm_gid, struct sbi_cove_tvm_aia_params *tvm_aia_params);

arch/riscv/include/asm/kvm_vcpu_sbi.h

@@ -68,6 +68,7 @@ extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental;
 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor;
 #ifdef CONFIG_RISCV_COVE_HOST
 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_covg;
+extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_covh;
 #endif
 
 #endif /* __RISCV_KVM_VCPU_SBI_H__ */

arch/riscv/include/asm/sbi.h

@@ -374,6 +374,7 @@ enum sbi_ext_covh_fid {
 	SBI_EXT_COVH_TVM_PROMOTE_PAGE,
 	SBI_EXT_COVH_TVM_DEMOTE_PAGE,
 	SBI_EXT_COVH_TVM_REMOVE_PAGES,
+	SBI_EXT_COVH_PROMOTE_TO_TVM,
 };
 
 enum sbi_ext_covi_fid {
@@ -410,9 +411,15 @@ struct sbi_cove_tsm_info {
 	/* Current state of the TSM */
 	enum sbi_cove_tsm_state tstate;
 
+	/* TSM implementation identifier */
+	uint32_t impl_id;
+
 	/* Version of the loaded TSM */
 	uint32_t version;
 
+	/* Capabilities of the TSM */
+	unsigned long capabilities;
+
 	/* Number of 4K pages required per TVM */
 	unsigned long tvm_pages_needed;
 

arch/riscv/include/uapi/asm/kvm.h

@@ -149,6 +149,7 @@ enum KVM_RISCV_SBI_EXT_ID {
 	KVM_RISCV_SBI_EXT_VENDOR,
 	KVM_RISCV_SBI_EXT_DBCN,
 	KVM_RISCV_SBI_EXT_COVG,
+	KVM_RISCV_SBI_EXT_COVH,
 	KVM_RISCV_SBI_EXT_MAX,
 };
 

arch/riscv/kernel/head.S

@@ -198,6 +198,20 @@ ENTRY(_start_kernel)
 	csrw CSR_IE, zero
 	csrw CSR_IP, zero
 
+#if defined(CONFIG_RISCV_COVE_GUEST_PROMOTE) && !defined(CONFIG_RISCV_M_MODE)
+	mv s0, a0
+	mv s1, a1
+	/* Request hypervisor to promote to TVM */
+	li a7, 0x434F5648        /* SBI_EXT_COVH */
+	li a6, 0x15              /* SBI_EXT_COVH_PROMOTE_TO_TVM */
+	mv a0, a1                /* DTB physical address */
+	la a1, __cove_tap_start	 /* TAP physical address */
+	ecall
+	/* Attestation reflects the result of promotion, so ignore it */
+	mv a0, s0
+	mv a1, s1
+#endif /* CONFIG_RISCV_COVE_GUEST_PROMOTE */
+
 #ifdef CONFIG_RISCV_M_MODE
 	/* flush the instruction cache */
 	fence.i

arch/riscv/kernel/vmlinux.lds.S

@@ -113,6 +113,18 @@ SECTIONS
 	}
 	__init_end = .;
 
+#ifdef CONFIG_RISCV_COVE_GUEST_PROMOTE
+	. = ALIGN(4096);
+	.cove_tvm_attestation_payload : {
+		__cove_tap_start =  .;
+		LONG(0xace0ace0)
+		SHORT(0x0FFA)
+		FILL(0x00)
+		. += 4090;
+		__cove_tap_end =  .;
+	}
+#endif
+
 	/* Start of data section */
 	_sdata = .;
 	RO_DATA(SECTION_ALIGN)

arch/riscv/kvm/Makefile

@@ -31,4 +31,4 @@ kvm-y += aia.o
 kvm-y += aia_device.o
 kvm-y += aia_aplic.o
 kvm-y += aia_imsic.o
-kvm-$(CONFIG_RISCV_COVE_HOST) += cove_sbi.o cove.o vcpu_sbi_covg.o
+kvm-$(CONFIG_RISCV_COVE_HOST) += cove_sbi.o cove.o vcpu_sbi_covg.o vcpu_sbi_covh.o

arch/riscv/kvm/cove.c

@@ -150,6 +150,11 @@ __always_inline bool kvm_riscv_cove_enabled(void)
 	return riscv_cove_enabled;
 }
 
+__always_inline bool kvm_riscv_cove_capability(unsigned long cap)
+{
+	return tinfo.capabilities & BIT(cap);
+}
+
 static void kvm_cove_imsic_clone(void *info)
 {
 	int rc;
@@ -589,9 +594,9 @@ void noinstr kvm_riscv_cove_vcpu_switchto(struct kvm_vcpu *vcpu, struct kvm_cpu_
 
 	/*
 	 * Bind the vsfile here instead during the new vsfile allocation because
-	 * COVH bind call requires the TVM to be in finalized state.
+	 * COVI bind call requires the TVM to be in finalized state.
 	 */
-	if (tvcpuc->imsic.bind_required) {
+	if (kvm_riscv_cove_capability(KVM_COVE_TSM_CAP_AIA) && tvcpuc->imsic.bind_required) {
 		tvcpuc->imsic.bind_required = false;
 		rc = kvm_riscv_cove_vcpu_imsic_bind(vcpu, BIT(tvcpuc->imsic.vsfile_hgei));
 		if (rc) {
@@ -628,12 +633,12 @@ void kvm_riscv_cove_vcpu_destroy(struct kvm_vcpu *vcpu)
 
 int kvm_riscv_cove_vcpu_init(struct kvm_vcpu *vcpu)
 {
-	int rc;
-	struct kvm *kvm;
 	struct kvm_cove_tvm_vcpu_context *tvcpuc;
 	struct kvm_cove_tvm_context *tvmc;
-	struct page *vcpus_page;
 	unsigned long vcpus_phys_addr;
+	struct page *vcpus_page;
+	struct kvm *kvm;
+	int rc;
 
 	if (!vcpu)
 		return -EINVAL;
@@ -654,15 +659,20 @@ int kvm_riscv_cove_vcpu_init(struct kvm_vcpu *vcpu)
 	if (!tvcpuc)
 		return -ENOMEM;
 
+	tvcpuc->vcpu = vcpu;
+	tvcpuc->vcpu_state.npages = tinfo.tvcpu_pages_needed;
+	vcpu->arch.tc = tvcpuc;
+
+	if (!kvm_riscv_cove_capability(KVM_COVE_TSM_CAP_MEMORY_ALLOCATION)) {
+		return 0;
+	}
+
 	vcpus_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
 				 get_order_num_pages(tinfo.tvcpu_pages_needed));
 	if (!vcpus_page) {
 		rc = -ENOMEM;
 		goto alloc_page_failed;
 	}
-
-	tvcpuc->vcpu = vcpu;
-	tvcpuc->vcpu_state.npages = tinfo.tvcpu_pages_needed;
 	tvcpuc->vcpu_state.page = vcpus_page;
 	vcpus_phys_addr = page_to_phys(vcpus_page);
 
@@ -674,8 +684,6 @@ int kvm_riscv_cove_vcpu_init(struct kvm_vcpu *vcpu)
 	if (rc)
 		goto vcpu_create_failed;
 
-	vcpu->arch.tc = tvcpuc;
-
 	return 0;
 
 vcpu_create_failed:
@@ -686,6 +694,7 @@ int kvm_riscv_cove_vcpu_init(struct kvm_vcpu *vcpu)
 	__free_pages(vcpus_page, get_order_num_pages(tinfo.tvcpu_pages_needed));
 
 alloc_page_failed:
+	vcpu->arch.tc = NULL;
 	kfree(tvcpuc);
 	return rc;
 }
@@ -877,14 +886,18 @@ void kvm_riscv_cove_vm_destroy(struct kvm *kvm)
 	kvm_err("Memory reclaim failed with rc %d\n", rc);
 }
 
-int kvm_riscv_cove_vm_init(struct kvm *kvm)
+int kvm_riscv_cove_vm_multi_step_init(struct kvm *kvm)
 {
 	struct kvm_cove_tvm_context *tvmc;
 	struct page *tvms_page, *pgt_page;
 	unsigned long tvm_gid, pgt_phys_addr, tvms_phys_addr;
 	unsigned long gstage_pgd_size = kvm_riscv_gstage_pgd_size();
 	int rc = 0;
 
+	// Multi-step TVM creation requires TSM that supports dynamic page conversion
+	if (!kvm_riscv_cove_capability(KVM_COVE_TSM_CAP_MEMORY_ALLOCATION))
+		return -EOPNOTSUPP;
+
 	tvmc = kzalloc(sizeof(*tvmc), GFP_KERNEL);
 	if (!tvmc)
 		return -ENOMEM;
@@ -980,6 +993,65 @@ int kvm_riscv_cove_vm_init(struct kvm *kvm)
 	return rc;
 }
 
+int kvm_riscv_cove_vm_single_step_init(struct kvm_vcpu *vcpu, unsigned long fdt_address,
+				unsigned long tap_addr)
+{
+	struct kvm_cpu_context *cp = &vcpu->arch.guest_context;
+	unsigned long tvm_gid, target_vcpuid;
+	struct kvm_cove_tvm_context *tvmc;
+	struct kvm_vcpu *target_vcpu;
+	struct kvm *kvm = vcpu->kvm;
+	void *nshmem = nacl_shmem();
+	int rc = 0, gpr_id, offset;
+
+	if (!kvm_riscv_cove_capability(KVM_COVE_TSM_CAP_PROMOTE_TVM))
+		return -EOPNOTSUPP;
+
+	tvmc = kzalloc(sizeof(*tvmc), GFP_KERNEL);
+	if (!tvmc)
+		return -ENOMEM;
+
+	for (gpr_id = 1; gpr_id < 32; gpr_id++) {
+		offset = KVM_ARCH_GUEST_ZERO + gpr_id * sizeof(unsigned long);
+		nacl_shmem_gpr_write_cove(nshmem, offset,
+					  ((unsigned long *)cp)[gpr_id]);
+	}
+	kvm_arch_vcpu_load(vcpu, smp_processor_id());
+	rc = sbi_covh_tsm_promote_to_tvm(fdt_address, tap_addr, cp->sepc, &tvm_gid);
+	if (rc)
+		goto done;
+
+	INIT_LIST_HEAD(&tvmc->measured_pages);
+	INIT_LIST_HEAD(&tvmc->zero_pages);
+	INIT_LIST_HEAD(&tvmc->shared_pages);
+	INIT_LIST_HEAD(&tvmc->reclaim_pending_pages);
+
+	tvmc->tvm_guest_id = tvm_gid;
+	tvmc->kvm = kvm;
+	kvm->arch.tvmc = tvmc;
+
+	kvm_for_each_vcpu(target_vcpuid, target_vcpu, kvm) {
+		rc = kvm_riscv_cove_vcpu_init(target_vcpu);
+		if (rc)
+			goto vcpus_allocated;
+
+		target_vcpu->requests = 0;
+	}
+
+	tvmc->finalized_done = true;
+	kvm_info("Guest VM creation successful with guest id %lx\n", tvm_gid);
+	return 0;
+
+vcpus_allocated:
+	kvm_for_each_vcpu(target_vcpuid, target_vcpu, kvm)
+		if (target_vcpu->arch.tc)
+			kfree(target_vcpu->arch.tc);
+
+done:
+	kfree(tvmc);
+	return rc;
+}
+
 int kvm_riscv_cove_init(void)
 {
 	int rc;

arch/riscv/kvm/cove_sbi.c

@@ -488,3 +488,23 @@ int sbi_covh_tvm_remove_pages(unsigned long tvmid,
 
 	return 0;
 }
+
+int sbi_covh_tsm_promote_to_tvm(unsigned long fdt_address,
+				unsigned long tap_addr,
+				unsigned long sepc,
+				unsigned long *tvmid)
+{
+	struct sbiret ret;
+	int rc = 0;
+
+	ret = sbi_ecall(SBI_EXT_COVH, SBI_EXT_COVH_PROMOTE_TO_TVM, fdt_address,
+			tap_addr, sepc, 0, 0, 0);
+	if (ret.error) {
+		rc = sbi_err_map_linux_errno(ret.error);
+		goto done;
+	}
+
+	*tvmid = ret.value;
+done:
+	return rc;
+}

arch/riscv/kvm/main.c

@@ -31,12 +31,11 @@ int kvm_arch_hardware_enable(void)
 		return rc;
 
 	/*
-	 * We just need to invoke aia enable for CoVE if host is in VS mode
-	 * However, if the host is running in HS mode, we need to initialize
-	 * other CSRs as well for legacy VMs.
-	 * TODO: Handle host in HS mode use case.
+	 * We just need to invoke aia enable for CoVE if host is in VS mode and TSM
+	 * supports AIA (COVI extension). However, if the host is running in HS mode,
+	 * we need to initialize other CSRs as well for legacy VMs.
 	 */
-	if (unlikely(kvm_riscv_cove_enabled()))
+	if (unlikely(kvm_riscv_cove_enabled()) && kvm_riscv_cove_capability(KVM_COVE_TSM_CAP_AIA))
 		goto enable_aia;
 
 	hedeleg = 0;