This commit introduces asynchronous writeback to zram, improving the
idle writeback speed.
Key changes include:
* Replacing `submit_bio_wait()` with `submit_bio()` to enable
non-blocking writeback operations.
* Utilizing a dedicated kthread for post-writeback tasks, avoiding
potential lock contention in interrupt context.
* Refactoring writeback-related code into a separate file `zram_wb.c`,
for better organization and maintainability.
Signed-off-by: Richard Chang <richardycc@xxxxxxxxxx>
---
drivers/block/zram/Makefile | 1 +
drivers/block/zram/zram_drv.c | 142 ++++++++--------------
drivers/block/zram/zram_drv.h | 37 ++++++
drivers/block/zram/zram_wb.c | 222 ++++++++++++++++++++++++++++++++++
drivers/block/zram/zram_wb.h | 41 +++++++
5 files changed, 353 insertions(+), 90 deletions(-)
create mode 100644 drivers/block/zram/zram_wb.c
create mode 100644 drivers/block/zram/zram_wb.h
diff --git a/drivers/block/zram/Makefile b/drivers/block/zram/Makefile
index 0fdefd576691..31ee1ed34e17 100644
--- a/drivers/block/zram/Makefile
+++ b/drivers/block/zram/Makefile
@@ -8,5 +8,6 @@ zram-$(CONFIG_ZRAM_BACKEND_LZ4HC) += backend_lz4hc.o
zram-$(CONFIG_ZRAM_BACKEND_ZSTD) += backend_zstd.o
zram-$(CONFIG_ZRAM_BACKEND_DEFLATE) += backend_deflate.o
zram-$(CONFIG_ZRAM_BACKEND_842) += backend_842.o
+zram-$(CONFIG_ZRAM_WRITEBACK) += zram_wb.o
obj-$(CONFIG_ZRAM) += zram.o
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index 54c57103715f..1cdebff13049 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -36,6 +36,7 @@
#include <linux/kernel_read_file.h>
#include "zram_drv.h"
+#include "zram_wb.h"
static DEFINE_IDR(zram_index_idr);
/* idr index must be protected */
@@ -56,7 +57,6 @@ static size_t huge_class_size;
static const struct block_device_operations zram_devops;
-static void zram_free_page(struct zram *zram, size_t index);
static int zram_read_from_zspool(struct zram *zram, struct page *page,
u32 index);
@@ -95,7 +95,7 @@ static __must_check bool zram_slot_trylock(struct zram *zram, u32 index)
return false;
}
-static void zram_slot_lock(struct zram *zram, u32 index)
+void zram_slot_lock(struct zram *zram, u32 index)
{
unsigned long *lock = &zram->table[index].flags;
@@ -104,7 +104,7 @@ static void zram_slot_lock(struct zram *zram, u32 index)
lock_acquired(slot_dep_map(zram, index), _RET_IP_);
}
-static void zram_slot_unlock(struct zram *zram, u32 index)
+void zram_slot_unlock(struct zram *zram, u32 index)
{
unsigned long *lock = &zram->table[index].flags;
@@ -127,19 +127,17 @@ static unsigned long zram_get_handle(struct zram *zram, u32 index)
return zram->table[index].handle;
}
-static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle)
+void zram_set_handle(struct zram *zram, u32 index, unsigned long handle)
{
zram->table[index].handle = handle;
}
-static bool zram_test_flag(struct zram *zram, u32 index,
- enum zram_pageflags flag)
+bool zram_test_flag(struct zram *zram, u32 index, enum zram_pageflags flag)
{
return zram->table[index].flags & BIT(flag);
}
-static void zram_set_flag(struct zram *zram, u32 index,
- enum zram_pageflags flag)
+void zram_set_flag(struct zram *zram, u32 index, enum zram_pageflags flag)
{
zram->table[index].flags |= BIT(flag);
}
@@ -233,22 +231,6 @@ static void zram_accessed(struct zram *zram, u32 index)
}
#if defined CONFIG_ZRAM_WRITEBACK || defined CONFIG_ZRAM_MULTI_COMP
-struct zram_pp_slot {
- unsigned long index;
- struct list_head entry;
-};
-
-/*
- * A post-processing bucket is, essentially, a size class, this defines
- * the range (in bytes) of pp-slots sizes in particular bucket.
- */
-#define PP_BUCKET_SIZE_RANGE 64
-#define NUM_PP_BUCKETS ((PAGE_SIZE / PP_BUCKET_SIZE_RANGE) + 1)
-
-struct zram_pp_ctl {
- struct list_head pp_buckets[NUM_PP_BUCKETS];
-};
-
static struct zram_pp_ctl *init_pp_ctl(void)
{
struct zram_pp_ctl *ctl;
@@ -258,15 +240,20 @@ static struct zram_pp_ctl *init_pp_ctl(void)
if (!ctl)
return NULL;
+ init_completion(&ctl->all_done);
+ atomic_set(&ctl->num_pp_slots, 0);
for (idx = 0; idx < NUM_PP_BUCKETS; idx++)
INIT_LIST_HEAD(&ctl->pp_buckets[idx]);
return ctl;
}
-static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
+static void remove_pp_slot_from_ctl(struct zram_pp_slot *pps)
{
list_del_init(&pps->entry);
+}
+void free_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
+{
zram_slot_lock(zram, pps->index);
zram_clear_flag(zram, pps->index, ZRAM_PP_SLOT);
zram_slot_unlock(zram, pps->index);
@@ -274,7 +261,13 @@ static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
kfree(pps);
}
-static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl)
+static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
+{
+ remove_pp_slot_from_ctl(pps);
+ free_pp_slot(zram, pps);
+}
+
+void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl)
{
u32 idx;
@@ -312,6 +305,7 @@ static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl,
list_add(&pps->entry, &ctl->pp_buckets[bid]);
zram_set_flag(zram, pps->index, ZRAM_PP_SLOT);
+ atomic_inc(&ctl->num_pp_slots);
return true;
}
@@ -713,7 +707,7 @@ static unsigned long alloc_block_bdev(struct zram *zram)
return blk_idx;
}
-static void free_block_bdev(struct zram *zram, unsigned long blk_idx)
+void free_block_bdev(struct zram *zram, unsigned long blk_idx)
{
int was_set;
@@ -737,18 +731,18 @@ static void read_from_bdev_async(struct zram *zram, struct page *page,
static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl)
{
unsigned long blk_idx = 0;
- struct page *page = NULL;
struct zram_pp_slot *pps;
- struct bio_vec bio_vec;
- struct bio bio;
- int ret = 0, err;
+ int ret = 0;
u32 index;
+ int nr_pps = atomic_read(&ctl->num_pp_slots);
- page = alloc_page(GFP_KERNEL);
- if (!page)
- return -ENOMEM;
+ if (!nr_pps)
+ return 0;
while ((pps = select_pp_slot(ctl))) {
+ struct zram_wb_request *req;
+ struct page *page;
+
spin_lock(&zram->wb_limit_lock);
if (zram->wb_limit_enable && !zram->bd_wb_limit) {
spin_unlock(&zram->wb_limit_lock);
@@ -765,6 +759,13 @@ static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl)
}
}
+ req = alloc_wb_request(zram, pps, ctl, blk_idx);
+ if (IS_ERR(req)) {
+ ret = PTR_ERR(req);
+ break;
+ }
+ page = bio_first_page_all(req->bio);
+
index = pps->index;
zram_slot_lock(zram, index);
/*
@@ -779,63 +780,28 @@ static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl)
goto next;
zram_slot_unlock(zram, index);
- bio_init(&bio, zram->bdev, &bio_vec, 1,
- REQ_OP_WRITE | REQ_SYNC);
- bio.bi_iter.bi_sector = blk_idx * (PAGE_SIZE >> 9);
- __bio_add_page(&bio, page, PAGE_SIZE, 0);
-
- /*
- * XXX: A single page IO would be inefficient for write
- * but it would be not bad as starter.
- */
- err = submit_bio_wait(&bio);
- if (err) {
- release_pp_slot(zram, pps);
- /*
- * BIO errors are not fatal, we continue and simply
- * attempt to writeback the remaining objects (pages).
- * At the same time we need to signal user-space that
- * some writes (at least one, but also could be all of
- * them) were not successful and we do so by returning
- * the most recent BIO error.
- */
- ret = err;
- continue;
- }
-
- atomic64_inc(&zram->stats.bd_writes);
- zram_slot_lock(zram, index);
- /*
- * Same as above, we release slot lock during writeback so
- * slot can change under us: slot_free() or slot_free() and
- * reallocation (zram_write_page()). In both cases slot loses
- * ZRAM_PP_SLOT flag. No concurrent post-processing can set
- * ZRAM_PP_SLOT on such slots until current post-processing
- * finishes.
- */
- if (!zram_test_flag(zram, index, ZRAM_PP_SLOT))
- goto next;
-
- zram_free_page(zram, index);
- zram_set_flag(zram, index, ZRAM_WB);
- zram_set_handle(zram, index, blk_idx);
+ nr_pps--;
+ remove_pp_slot_from_ctl(pps);
blk_idx = 0;
- atomic64_inc(&zram->stats.pages_stored);
- spin_lock(&zram->wb_limit_lock);
- if (zram->wb_limit_enable && zram->bd_wb_limit > 0)
- zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);
- spin_unlock(&zram->wb_limit_lock);
+ submit_bio(req->bio);
+ continue;
+
next:
zram_slot_unlock(zram, index);
release_pp_slot(zram, pps);
+ free_wb_request(req);
cond_resched();
}
if (blk_idx)
free_block_bdev(zram, blk_idx);
- if (page)
- __free_page(page);
+
+ if (nr_pps && atomic_sub_and_test(nr_pps, &ctl->num_pp_slots))
+ complete(&ctl->all_done);
+
+ /* wait until all async bios completed */
+ wait_for_completion(&ctl->all_done);
return ret;
}
@@ -1111,8 +1077,6 @@ static int read_from_bdev(struct zram *zram, struct page *page,
{
return -EIO;
}
-
-static void free_block_bdev(struct zram *zram, unsigned long blk_idx) {};
#endif
#ifdef CONFIG_ZRAM_MEMORY_TRACKING
@@ -1620,7 +1584,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize)
return true;
}
-static void zram_free_page(struct zram *zram, size_t index)
+void zram_free_page(struct zram *zram, size_t index)
{
unsigned long handle;
@@ -2247,12 +2211,6 @@ static ssize_t recompress_store(struct device *dev,
goto release_init_lock;
}
- page = alloc_page(GFP_KERNEL);
- if (!page) {
- ret = -ENOMEM;
- goto release_init_lock;
- }
-
ctl = init_pp_ctl();
if (!ctl) {
ret = -ENOMEM;
@@ -2912,6 +2870,9 @@ static int __init zram_init(void)
num_devices--;
}
+ if (setup_zram_writeback())
+ goto out_error;
+
return 0;
out_error:
@@ -2921,6 +2882,7 @@ static int __init zram_init(void)
static void __exit zram_exit(void)
{
+ destroy_zram_writeback();
destroy_devices();
}
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h
index 6cee93f9c0d0..7d9081215668 100644
--- a/drivers/block/zram/zram_drv.h
+++ b/drivers/block/zram/zram_drv.h
@@ -139,4 +139,41 @@ struct zram {
#endif
atomic_t pp_in_progress;
};
+
+void zram_slot_lock(struct zram *zram, u32 index);
+void zram_slot_unlock(struct zram *zram, u32 index);
+void zram_set_handle(struct zram *zram, u32 index, unsigned long handle);
+bool zram_test_flag(struct zram *zram, u32 index, enum zram_pageflags flag);
+void zram_set_flag(struct zram *zram, u32 index, enum zram_pageflags flag);
+void zram_free_page(struct zram *zram, size_t index);
+
+#if defined CONFIG_ZRAM_WRITEBACK || defined CONFIG_ZRAM_MULTI_COMP
+struct zram_pp_slot {
+ unsigned long index;
+ struct list_head entry;
+};
+
+/*
+ * A post-processing bucket is, essentially, a size class, this defines
+ * the range (in bytes) of pp-slots sizes in particular bucket.
+ */
+#define PP_BUCKET_SIZE_RANGE 64
+#define NUM_PP_BUCKETS ((PAGE_SIZE / PP_BUCKET_SIZE_RANGE) + 1)
+
+struct zram_pp_ctl {
+ struct list_head pp_buckets[NUM_PP_BUCKETS];
+ struct completion all_done;
+ atomic_t num_pp_slots;
+};
+
+void free_pp_slot(struct zram *zram, struct zram_pp_slot *pps);
+void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl);
+#endif
+
+#if IS_ENABLED(CONFIG_ZRAM_WRITEBACK)
+void free_block_bdev(struct zram *zram, unsigned long blk_idx);
+#else
+inline void free_block_bdev(struct zram *zram, unsigned long blk_idx) {};
+#endif
+
#endif
diff --git a/drivers/block/zram/zram_wb.c b/drivers/block/zram/zram_wb.c
new file mode 100644
index 000000000000..59f93b22d2bf
--- /dev/null
+++ b/drivers/block/zram/zram_wb.c
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#define KMSG_COMPONENT "zram_wb"
+#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/wait.h>
+#include <linux/freezer.h>
+
+#include "zram_wb.h"
+
+static struct task_struct *wb_thread;
+static DECLARE_WAIT_QUEUE_HEAD(wb_wq);
+static struct zram_wb_request_list wb_req_list;
+
+static void complete_wb_request(struct zram_wb_request *req)
+{
+ struct zram *zram = req->zram;
+ struct zram_pp_slot *pps = req->pps;
+ struct zram_pp_ctl *ctl = req->ppctl;
+ unsigned long index = pps->index;
+ unsigned long blk_idx = req->blk_idx;
+ struct bio *bio = req->bio;
+
+ if (bio->bi_status)
+ goto out_err;
+
+ atomic64_inc(&zram->stats.bd_writes);
+ zram_slot_lock(zram, index);
+
+ /*
+ * Same as above, we release slot lock during writeback so
+ * slot can change under us: slot_free() or slot_free() and
+ * reallocation (zram_write_page()). In both cases slot loses
+ * ZRAM_PP_SLOT flag. No concurrent post-processing can set
+ * ZRAM_PP_SLOT on such slots until current post-processing
+ * finishes.
+ */
+ if (!zram_test_flag(zram, index, ZRAM_PP_SLOT)) {
+ zram_slot_unlock(zram, index);
+ goto out_err;
+ }
+
+ zram_free_page(zram, index);
+ zram_set_flag(zram, index, ZRAM_WB);
+ zram_set_handle(zram, index, blk_idx);
+ atomic64_inc(&zram->stats.pages_stored);
+ spin_lock(&zram->wb_limit_lock);
+ if (zram->wb_limit_enable && zram->bd_wb_limit > 0)
+ zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);
+ spin_unlock(&zram->wb_limit_lock);
+ zram_slot_unlock(zram, index);
+ goto end;
+
+out_err:
+ free_block_bdev(zram, blk_idx);
+end:
+ free_pp_slot(zram, pps);
+ free_wb_request(req);
+
+ if (atomic_dec_and_test(&ctl->num_pp_slots))
+ complete(&ctl->all_done);
+}
+
+static void enqueue_wb_request(struct zram_wb_request_list *req_list,
+ struct zram_wb_request *req)
+{
+ /*
+ * The enqueue path comes from softirq context:
+ * blk_done_softirq -> bio_endio -> zram_writeback_end_io
+ * Use spin_lock_bh for locking.
+ */
+ spin_lock_bh(&req_list->lock);
+ list_add_tail(&req->node, &req_list->head);
+ req_list->count++;
+ spin_unlock_bh(&req_list->lock);
+}
+
+static struct zram_wb_request *dequeue_wb_request(
+ struct zram_wb_request_list *req_list)
+{
+ struct zram_wb_request *req = NULL;
+
+ spin_lock_bh(&req_list->lock);
+ if (!list_empty(&req_list->head)) {
+ req = list_first_entry(&req_list->head,
+ struct zram_wb_request,
+ node);
+ list_del(&req->node);
+ req_list->count--;
+ }
+ spin_unlock_bh(&req_list->lock);
+
+ return req;
+}
+
+static void destroy_wb_request_list(struct zram_wb_request_list *req_list)
+{
+ struct zram_wb_request *req;
+
+ while (!list_empty(&req_list->head)) {
+ req = dequeue_wb_request(req_list);
+ free_block_bdev(req->zram, req->blk_idx);
+ free_wb_request(req);
+ }
+}
+
+static bool wb_ready_to_run(void)
+{
+ int count;
+
+ spin_lock_bh(&wb_req_list.lock);
+ count = wb_req_list.count;
+ spin_unlock_bh(&wb_req_list.lock);
+
+ return count > 0;
+}
+
+static int wb_thread_func(void *data)
+{
+ set_freezable();
+
+ while (!kthread_should_stop()) {
+ wait_event_freezable(wb_wq, wb_ready_to_run());
+
+ while (1) {
+ struct zram_wb_request *req;
+
+ req = dequeue_wb_request(&wb_req_list);
+ if (!req)
+ break;
+ complete_wb_request(req);
+ }
+ }
+ return 0;
+}
+
+void zram_writeback_end_io(struct bio *bio)
+{
+ struct zram_wb_request *req =
+ (struct zram_wb_request *)bio->bi_private;
+
+ enqueue_wb_request(&wb_req_list, req);
+ wake_up(&wb_wq);
+}
+
+struct zram_wb_request *alloc_wb_request(struct zram *zram,
+ struct zram_pp_slot *pps,
+ struct zram_pp_ctl *ppctl,
+ unsigned long blk_idx)
+{
+ struct zram_wb_request *req;
+ struct page *page;
+ struct bio *bio;
+ int err = 0;
+
+ page = alloc_page(GFP_NOIO | __GFP_NOWARN);
+ if (!page)
+ return ERR_PTR(-ENOMEM);
+
+ bio = bio_alloc(zram->bdev, 1, REQ_OP_WRITE, GFP_NOIO | __GFP_NOWARN);
+ if (!bio) {
+ err = -ENOMEM;
+ goto out_free_page;
+ }
+
+ req = kmalloc(sizeof(struct zram_wb_request), GFP_NOIO | __GFP_NOWARN);
+ if (!req) {
+ err = -ENOMEM;
+ goto out_free_bio;
+ }
+ req->zram = zram;
+ req->pps = pps;
+ req->ppctl = ppctl;
+ req->blk_idx = blk_idx;
+ req->bio = bio;
+
+ bio->bi_iter.bi_sector = blk_idx * (PAGE_SIZE >> 9);
+ __bio_add_page(bio, page, PAGE_SIZE, 0);
+ bio->bi_private = req;
+ bio->bi_end_io = zram_writeback_end_io;
+ return req;
+
+out_free_bio:
+ bio_put(bio);
+out_free_page:
+ __free_page(page);
+ return ERR_PTR(err);
+}
+
+void free_wb_request(struct zram_wb_request *req)
+{
+ struct bio *bio = req->bio;
+ struct page *page = bio_first_page_all(bio);
+
+ __free_page(page);
+ bio_put(bio);
+ kfree(req);
+}
+
+int setup_zram_writeback(void)
+{
+ spin_lock_init(&wb_req_list.lock);
+ INIT_LIST_HEAD(&wb_req_list.head);
+ wb_req_list.count = 0;
+
+ wb_thread = kthread_run(wb_thread_func, NULL, "zram_wb_thread");
+ if (IS_ERR(wb_thread)) {
+ pr_err("Unable to create zram_wb_thread\n");
+ return -1;
+ }
+ return 0;
+}
+
+void destroy_zram_writeback(void)
+{
+ kthread_stop(wb_thread);
+ destroy_wb_request_list(&wb_req_list);
+}
+
diff --git a/drivers/block/zram/zram_wb.h b/drivers/block/zram/zram_wb.h
new file mode 100644
index 000000000000..e029b4de0de6
--- /dev/null
+++ b/drivers/block/zram/zram_wb.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef _ZRAM_WRITEBACK_H_
+#define _ZRAM_WRITEBACK_H_
+
+#include <linux/bio.h>
+#include "zram_drv.h"
+
+struct zram_wb_request;
+
+struct zram_wb_request {
+ struct zram *zram;
+ unsigned long blk_idx;
+ struct zram_pp_slot *pps;
+ struct zram_pp_ctl *ppctl;
+ struct bio *bio;
+ struct list_head node;
+};
+
+struct zram_wb_request_list {
+ struct list_head head;
+ int count;
+ spinlock_t lock;
+};
+
+#if IS_ENABLED(CONFIG_ZRAM_WRITEBACK)
+void zram_writeback_end_io(struct bio *bio);
+int setup_zram_writeback(void);
+void destroy_zram_writeback(void);
+struct zram_wb_request *alloc_wb_request(struct zram *zram,
+ struct zram_pp_slot *pps,
+ struct zram_pp_ctl *ppctl,
+ unsigned long blk_idx);
+void free_wb_request(struct zram_wb_request *req);
+#else
+inline void zram_writeback_end_io(struct bio *bio) {}
+inline int setup_zram_writeback(void) { return 0; }
+inline void destroy_zram_writeback(void) {}
+#endif
+
+#endif /* _ZRAM_WRITEBACK_H_ */
--
2.50.0.rc2.696.g1fc2a0284f-goog
