Introduce the split_external_spt hook and call it within tdp_mmu_set_spte()
for the mirror page table.
tdp_mmu_set_spte() is invoked for SPTE transitions under write mmu_lock.
For the mirror page table, in addition to the valid transitions from a
shadow-present entry to !shadow-present entry, introduce a new valid
transition case for splitting and propagate the transition to the external
page table via the hook split_external_spt.
Signed-off-by: Xiaoyao Li <xiaoyao.li@xxxxxxxxx>
Signed-off-by: Isaku Yamahata <isaku.yamahata@xxxxxxxxx>
Signed-off-by: Yan Zhao <yan.y.zhao@xxxxxxxxx>
---
RFC v2:
- Removed the KVM_BUG_ON() in split_external_spt(). (Rick)
- Add a comment for the KVM_BUG_ON() in tdp_mmu_set_spte(). (Rick)
- Use kvm_x86_call() instead of static_call(). (Binbin)
RFC v1:
- Split patch.
- Dropped invoking hook zap_private_spte and kvm_flush_remote_tlbs() in KVM
MMU core.
---
arch/x86/include/asm/kvm-x86-ops.h | 1 +
arch/x86/include/asm/kvm_host.h | 4 ++++
arch/x86/kvm/mmu/tdp_mmu.c | 29 +++++++++++++++++++++++++----
3 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 18a5c3119e1a..7653a45ad5b2 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -98,6 +98,7 @@ KVM_X86_OP_OPTIONAL(link_external_spt)
KVM_X86_OP_OPTIONAL(set_external_spte)
KVM_X86_OP_OPTIONAL(free_external_spt)
KVM_X86_OP_OPTIONAL(remove_external_spte)
+KVM_X86_OP_OPTIONAL(split_external_spt)
KVM_X86_OP(has_wbinvd_exit)
KVM_X86_OP(get_l2_tsc_offset)
KVM_X86_OP(get_l2_tsc_multiplier)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 823d1aeef2a8..e431ce0e3180 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1839,6 +1839,10 @@ struct kvm_x86_ops {
int (*remove_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
kvm_pfn_t pfn_for_gfn);
+ /* Split the external page table into smaller page tables */
+ int (*split_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
+ void *external_spt);
+
bool (*has_wbinvd_exit)(void);
u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 46b9f276bb6d..a2c6e6e4773f 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -325,6 +325,7 @@ static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
bool shared);
static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(bool mirror);
+static void *get_external_spt(gfn_t gfn, u64 new_spte, int level);
static void tdp_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
@@ -384,6 +385,18 @@ static void remove_external_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,
KVM_BUG_ON(ret, kvm);
}
+static int split_external_spt(struct kvm *kvm, gfn_t gfn, u64 old_spte,
+ u64 new_spte, int level)
+{
+ void *external_spt = get_external_spt(gfn, new_spte, level);
+ int ret;
+
+ KVM_BUG_ON(!external_spt, kvm);
+
+ ret = kvm_x86_call(split_external_spt)(kvm, gfn, level, external_spt);
+
+ return ret;
+}
/**
* handle_removed_pt() - handle a page table removed from the TDP structure
*
@@ -765,12 +778,20 @@ static u64 tdp_mmu_set_spte(struct kvm *kvm, int as_id, tdp_ptep_t sptep,
handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, false);
/*
- * Users that do non-atomic setting of PTEs don't operate on mirror
- * roots, so don't handle it and bug the VM if it's seen.
+ * Propagate changes of SPTE to the external page table under write
+ * mmu_lock.
+ * Current valid transitions:
+ * - present leaf to !present.
+ * - present non-leaf to !present.
+ * - present leaf to present non-leaf (splitting)
*/
if (is_mirror_sptep(sptep)) {
- KVM_BUG_ON(is_shadow_present_pte(new_spte), kvm);
- remove_external_spte(kvm, gfn, old_spte, level);
+ if (!is_shadow_present_pte(new_spte))
+ remove_external_spte(kvm, gfn, old_spte, level);
+ else if (is_last_spte(old_spte, level) && !is_last_spte(new_spte, level))
+ split_external_spt(kvm, gfn, old_spte, new_spte, level);
+ else
+ KVM_BUG_ON(1, kvm);
}
return old_spte;
--
2.43.2
