Introduce min_heap_pop_push_eqaware() as a variant of
min_heap_pop_push() that uses the equality-aware version of sift_down,
which is implemented in a top-down manner.
This top-down sift_down reduces the number of comparisons from
O(log2(n)) to O(1) in cases where many elements have equal priority. It
enables more efficient heap construction when the heap contains a large
number of equal elements.
Cc: stable@xxxxxxxxxxxxxxx # 6.11+
Signed-off-by: Kuan-Wei Chiu <visitorckw@xxxxxxxxx>
---
include/linux/min_heap.h | 20 +++++++++++++++-----
lib/min_heap.c | 4 ++--
2 files changed, 17 insertions(+), 7 deletions(-)
diff --git a/include/linux/min_heap.h b/include/linux/min_heap.h
index 6c45d617b027..d7bf8dd0f6b1 100644
--- a/include/linux/min_heap.h
+++ b/include/linux/min_heap.h
@@ -424,15 +424,22 @@ bool __min_heap_pop_inline(min_heap_char *heap, size_t elem_size,
*/
static __always_inline
void __min_heap_pop_push_inline(min_heap_char *heap, const void *element, size_t elem_size,
- const struct min_heap_callbacks *func, void *args)
+ const struct min_heap_callbacks *func, void *args, bool eqaware)
{
+ siftdown_fn_t sift_down = eqaware ? __min_heap_sift_down_eqaware_inline :
+ __min_heap_sift_down_inline;
+
memcpy(heap->data, element, elem_size);
- __min_heap_sift_down_inline(heap, 0, elem_size, func, args);
+ sift_down(heap, 0, elem_size, func, args);
}
#define min_heap_pop_push_inline(_heap, _element, _func, _args) \
__min_heap_pop_push_inline(container_of(&(_heap)->nr, min_heap_char, nr), _element, \
- __minheap_obj_size(_heap), _func, _args)
+ __minheap_obj_size(_heap), _func, _args, false)
+
+#define min_heap_pop_push_eqaware_inline(_heap, _element, _func, _args) \
+ __min_heap_pop_push_inline(container_of(&(_heap)->nr, min_heap_char, nr), _element, \
+ __minheap_obj_size(_heap), _func, _args, true)
/* Push an element on to the heap, O(log2(nr)). */
static __always_inline
@@ -503,7 +510,7 @@ void __min_heapify_all(min_heap_char *heap, size_t elem_size,
bool __min_heap_pop(min_heap_char *heap, size_t elem_size,
const struct min_heap_callbacks *func, void *args, bool eqaware);
void __min_heap_pop_push(min_heap_char *heap, const void *element, size_t elem_size,
- const struct min_heap_callbacks *func, void *args);
+ const struct min_heap_callbacks *func, void *args, bool eqaware);
bool __min_heap_push(min_heap_char *heap, const void *element, size_t elem_size,
const struct min_heap_callbacks *func, void *args);
bool __min_heap_del(min_heap_char *heap, size_t elem_size, size_t idx,
@@ -538,7 +545,10 @@ bool __min_heap_del(min_heap_char *heap, size_t elem_size, size_t idx,
__minheap_obj_size(_heap), _func, _args, true)
#define min_heap_pop_push(_heap, _element, _func, _args) \
__min_heap_pop_push(container_of(&(_heap)->nr, min_heap_char, nr), _element, \
- __minheap_obj_size(_heap), _func, _args)
+ __minheap_obj_size(_heap), _func, _args, false)
+#define min_heap_pop_push_eqaware(_heap, _element, _func, _args) \
+ __min_heap_pop_push(container_of(&(_heap)->nr, min_heap_char, nr), _element, \
+ __minheap_obj_size(_heap), _func, _args, true)
#define min_heap_push(_heap, _element, _func, _args) \
__min_heap_push(container_of(&(_heap)->nr, min_heap_char, nr), _element, \
__minheap_obj_size(_heap), _func, _args)
diff --git a/lib/min_heap.c b/lib/min_heap.c
index dae3ed39421a..a69d8b80d443 100644
--- a/lib/min_heap.c
+++ b/lib/min_heap.c
@@ -56,9 +56,9 @@ bool __min_heap_pop(min_heap_char *heap, size_t elem_size,
EXPORT_SYMBOL(__min_heap_pop);
void __min_heap_pop_push(min_heap_char *heap, const void *element, size_t elem_size,
- const struct min_heap_callbacks *func, void *args)
+ const struct min_heap_callbacks *func, void *args, bool eqaware)
{
- __min_heap_pop_push_inline(heap, element, elem_size, func, args);
+ __min_heap_pop_push_inline(heap, element, elem_size, func, args, eqaware);
}
EXPORT_SYMBOL(__min_heap_pop_push);
--
2.34.1
