* Suren Baghdasaryan <surenb@xxxxxxxxxx> [250704 02:07]:
> Utilize per-vma locks to stabilize vma after lookup without taking
> mmap_lock during PROCMAP_QUERY ioctl execution. While we might take
> mmap_lock for reading during contention, we do that momentarily only
> to lock the vma.
> This change is designed to reduce mmap_lock contention and prevent
> PROCMAP_QUERY ioctl calls from blocking address space updates.
>
> Signed-off-by: Suren Baghdasaryan <surenb@xxxxxxxxxx>
> Acked-by: Andrii Nakryiko <andrii@xxxxxxxxxx>
Reviewed-by: Liam R. Howlett <Liam.Howlett@xxxxxxxxxx>
> ---
> fs/proc/task_mmu.c | 60 ++++++++++++++++++++++++++++++++++++----------
> 1 file changed, 48 insertions(+), 12 deletions(-)
>
> diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
> index ff3fe488ce51..0496d5969a51 100644
> --- a/fs/proc/task_mmu.c
> +++ b/fs/proc/task_mmu.c
> @@ -487,28 +487,64 @@ static int pid_maps_open(struct inode *inode, struct file *file)
> PROCMAP_QUERY_VMA_FLAGS \
> )
>
> -static int query_vma_setup(struct mm_struct *mm)
> +#ifdef CONFIG_PER_VMA_LOCK
> +
> +static int query_vma_setup(struct proc_maps_private *priv)
> {
> - return mmap_read_lock_killable(mm);
> + priv->locked_vma = NULL;
> + priv->mmap_locked = false;
> +
> + return 0;
> }
>
> -static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct *vma)
> +static void query_vma_teardown(struct proc_maps_private *priv)
> {
> - mmap_read_unlock(mm);
> + unlock_vma(priv);
> +}
> +
> +static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_private *priv,
> + unsigned long addr)
> +{
> + struct vm_area_struct *vma;
> +
> + rcu_read_lock();
> + vma_iter_init(&priv->iter, priv->mm, addr);
> + vma = get_next_vma(priv, addr);
> + rcu_read_unlock();
> +
> + return vma;
> }
>
> -static struct vm_area_struct *query_vma_find_by_addr(struct mm_struct *mm, unsigned long addr)
> +#else /* CONFIG_PER_VMA_LOCK */
> +
> +static int query_vma_setup(struct proc_maps_private *priv)
> {
> - return find_vma(mm, addr);
> + return mmap_read_lock_killable(priv->mm);
> }
>
> -static struct vm_area_struct *query_matching_vma(struct mm_struct *mm,
> +static void query_vma_teardown(struct proc_maps_private *priv)
> +{
> + mmap_read_unlock(priv->mm);
> +}
> +
> +static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_private *priv,
> + unsigned long addr)
> +{
> + return find_vma(priv->mm, addr);
> +}
> +
> +#endif /* CONFIG_PER_VMA_LOCK */
> +
> +static struct vm_area_struct *query_matching_vma(struct proc_maps_private *priv,
> unsigned long addr, u32 flags)
> {
> struct vm_area_struct *vma;
>
> next_vma:
> - vma = query_vma_find_by_addr(mm, addr);
> + vma = query_vma_find_by_addr(priv, addr);
> + if (IS_ERR(vma))
> + return vma;
> +
> if (!vma)
> goto no_vma;
>
> @@ -584,13 +620,13 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg)
> if (!mm || !mmget_not_zero(mm))
> return -ESRCH;
>
> - err = query_vma_setup(mm);
> + err = query_vma_setup(priv);
> if (err) {
> mmput(mm);
> return err;
> }
>
> - vma = query_matching_vma(mm, karg.query_addr, karg.query_flags);
> + vma = query_matching_vma(priv, karg.query_addr, karg.query_flags);
> if (IS_ERR(vma)) {
> err = PTR_ERR(vma);
> vma = NULL;
> @@ -675,7 +711,7 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg)
> }
>
> /* unlock vma or mmap_lock, and put mm_struct before copying data to user */
> - query_vma_teardown(mm, vma);
> + query_vma_teardown(priv);
> mmput(mm);
>
> if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr),
> @@ -695,7 +731,7 @@ static int do_procmap_query(struct proc_maps_private *priv, void __user *uarg)
> return 0;
>
> out:
> - query_vma_teardown(mm, vma);
> + query_vma_teardown(priv);
> mmput(mm);
> kfree(name_buf);
> return err;
> --
> 2.50.0.727.gbf7dc18ff4-goog
>
