KVM Userfault consists of a bitmap in userspace that describes which
pages the user wants exits on (when KVM_MEM_USERFAULT is enabled). To
get those exits, the memslot where KVM_MEM_USERFAULT is being enabled
must drop (at least) all of the translations that the bitmap says should
generate faults. Today, simply drop all translations for the memslot. Do
so with a new arch interface, kvm_arch_userfault_enabled(), which can be
specialized in the future by any architecture for which optimizations
make sense.
Make some changes to kvm_set_memory_region() to support setting
KVM_MEM_USERFAULT on KVM_MEM_GUEST_MEMFD memslots, including relaxing
the retrictions on guest_memfd memslots from only deletion to no moving.
Signed-off-by: James Houghton <jthoughton@xxxxxxxxxx>
Signed-off-by: Sean Christopherson <seanjc@xxxxxxxxxx>
---
include/linux/kvm_host.h | 23 ++++++++++++++++++
include/uapi/linux/kvm.h | 5 +++-
virt/kvm/kvm_main.c | 51 ++++++++++++++++++++++++++++++++++++----
3 files changed, 74 insertions(+), 5 deletions(-)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 9a85500cd5c50..bd5fb5ae10d05 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -597,6 +597,7 @@ struct kvm_memory_slot {
unsigned long *dirty_bitmap;
struct kvm_arch_memory_slot arch;
unsigned long userspace_addr;
+ unsigned long __user *userfault_bitmap;
u32 flags;
short id;
u16 as_id;
@@ -1236,6 +1237,20 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm);
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
struct kvm_memory_slot *slot);
+#ifndef __KVM_HAVE_ARCH_USERFAULT_ENABLED
+static inline void kvm_arch_userfault_enabled(struct kvm *kvm,
+ struct kvm_memory_slot *slot)
+{
+ /*
+ * kvm_arch_userfault_enabled() must ensure that new faults on pages
+ * marked as userfault will exit to userspace. Dropping all
+ * translations is sufficient; architectures may choose to optimize
+ * this.
+ */
+ return kvm_arch_flush_shadow_memslot(kvm, slot);
+}
+#endif
+
int kvm_prefetch_pages(struct kvm_memory_slot *slot, gfn_t gfn,
struct page **pages, int nr_pages);
@@ -2524,6 +2539,14 @@ static inline void kvm_prepare_memory_fault_exit(struct kvm_vcpu *vcpu,
if (fault->is_private)
vcpu->run->memory_fault.flags |= KVM_MEMORY_EXIT_FLAG_PRIVATE;
}
+
+bool kvm_do_userfault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
+
+static inline bool kvm_is_userfault_memslot(struct kvm_memory_slot *memslot)
+{
+ return memslot && memslot->flags & KVM_MEM_USERFAULT;
+}
+
#endif
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index d00b85cb168c3..e3b871506ec85 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -40,7 +40,8 @@ struct kvm_userspace_memory_region2 {
__u64 guest_memfd_offset;
__u32 guest_memfd;
__u32 pad1;
- __u64 pad2[14];
+ __u64 userfault_bitmap;
+ __u64 pad2[13];
};
/*
@@ -51,6 +52,7 @@ struct kvm_userspace_memory_region2 {
#define KVM_MEM_LOG_DIRTY_PAGES (1UL << 0)
#define KVM_MEM_READONLY (1UL << 1)
#define KVM_MEM_GUEST_MEMFD (1UL << 2)
+#define KVM_MEM_USERFAULT (1UL << 3)
/* for KVM_IRQ_LINE */
struct kvm_irq_level {
@@ -443,6 +445,7 @@ struct kvm_run {
/* KVM_EXIT_MEMORY_FAULT */
struct {
#define KVM_MEMORY_EXIT_FLAG_PRIVATE (1ULL << 3)
+#define KVM_MEMORY_EXIT_FLAG_USERFAULT (1ULL << 4)
__u64 flags;
__u64 gpa;
__u64 size;
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index eec82775c5bfb..bef6760cd1c0e 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -1747,6 +1747,14 @@ static void kvm_commit_memory_region(struct kvm *kvm,
if (old->dirty_bitmap && !new->dirty_bitmap)
kvm_destroy_dirty_bitmap(old);
+ /*
+ * If KVM_MEM_USERFAULT is being enabled for the slot, drop the
+ * translations that are marked as userfault.
+ */
+ if (!(old_flags & KVM_MEM_USERFAULT) &&
+ (new_flags & KVM_MEM_USERFAULT))
+ kvm_arch_userfault_enabled(kvm, old);
+
/*
* The final quirk. Free the detached, old slot, but only its
* memory, not any metadata. Metadata, including arch specific
@@ -2039,6 +2047,12 @@ static int kvm_set_memory_region(struct kvm *kvm,
if (id < KVM_USER_MEM_SLOTS &&
(mem->memory_size >> PAGE_SHIFT) > KVM_MEM_MAX_NR_PAGES)
return -EINVAL;
+ if (mem->flags & KVM_MEM_USERFAULT &&
+ ((mem->userfault_bitmap != untagged_addr(mem->userfault_bitmap)) ||
+ !access_ok(u64_to_user_ptr(mem->userfault_bitmap),
+ DIV_ROUND_UP(mem->memory_size >> PAGE_SHIFT, BITS_PER_LONG)
+ * sizeof(long))))
+ return -EINVAL;
slots = __kvm_memslots(kvm, as_id);
@@ -2071,14 +2085,15 @@ static int kvm_set_memory_region(struct kvm *kvm,
if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages)
return -EINVAL;
} else { /* Modify an existing slot. */
- /* Private memslots are immutable, they can only be deleted. */
- if (mem->flags & KVM_MEM_GUEST_MEMFD)
- return -EINVAL;
if ((mem->userspace_addr != old->userspace_addr) ||
(npages != old->npages) ||
((mem->flags ^ old->flags) & KVM_MEM_READONLY))
return -EINVAL;
+ /* Moving a guest_memfd memslot isn't supported. */
+ if (base_gfn != old->base_gfn && mem->flags & KVM_MEM_GUEST_MEMFD)
+ return -EINVAL;
+
if (base_gfn != old->base_gfn)
change = KVM_MR_MOVE;
else if (mem->flags != old->flags)
@@ -2102,11 +2117,13 @@ static int kvm_set_memory_region(struct kvm *kvm,
new->npages = npages;
new->flags = mem->flags;
new->userspace_addr = mem->userspace_addr;
- if (mem->flags & KVM_MEM_GUEST_MEMFD) {
+ if (mem->flags & KVM_MEM_GUEST_MEMFD && change == KVM_MR_CREATE) {
r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset);
if (r)
goto out;
}
+ if (mem->flags & KVM_MEM_USERFAULT)
+ new->userfault_bitmap = u64_to_user_ptr(mem->userfault_bitmap);
r = kvm_set_memslot(kvm, old, new, change);
if (r)
@@ -4980,6 +4997,32 @@ static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm)
return cleared;
}
+#ifdef CONFIG_KVM_GENERIC_PAGE_FAULT
+bool kvm_do_userfault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
+{
+ struct kvm_memory_slot *slot = fault->slot;
+ unsigned long __user *user_chunk;
+ unsigned long chunk;
+ gfn_t offset;
+
+ if (!kvm_is_userfault_memslot(slot))
+ return false;
+
+ offset = fault->gfn - slot->base_gfn;
+ user_chunk = slot->userfault_bitmap + (offset / BITS_PER_LONG);
+
+ if (__get_user(chunk, user_chunk))
+ return true;
+
+ if (!test_bit(offset % BITS_PER_LONG, &chunk))
+ return false;
+
+ kvm_prepare_memory_fault_exit(vcpu, fault);
+ vcpu->run->memory_fault.flags |= KVM_MEMORY_EXIT_FLAG_USERFAULT;
+ return true;
+}
+#endif
+
int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
struct kvm_enable_cap *cap)
{
--
2.50.0.rc2.692.g299adb8693-goog
