On 19/06/2025 14:55, Mark Bloch wrote:
> From: Yevgeny Kliteynik <kliteyn@xxxxxxxxxx>
Vlad wrote the patch, Yevgeny just pused it into our
internal review system, same goes for patch 5 in the series.
v2 will fix both patches.
Mark
>
> Track and grow matcher sizes individually for RX and TX RTCs. This
> allows RX-only or TX-only use cases to effectively halve the device
> resources they use.
>
> For testing we used a simple module that inserts 1M RX-only rules and
> measured the number of pages the device requests, and memory usage as
> reported by `free -h`.
>
> Pages Memory
> Before this patch: 300k 1.5GiB
> After this patch: 160k 900MiB
>
> Signed-off-by: Vlad Dogaru <vdogaru@xxxxxxxxxx>
> Reviewed-by: Yevgeny Kliteynik <kliteyn@xxxxxxxxxx>
> Signed-off-by: Mark Bloch <mbloch@xxxxxxxxxx>
> ---
> .../mellanox/mlx5/core/steering/hws/bwc.c | 213 +++++++++++++-----
> .../mellanox/mlx5/core/steering/hws/bwc.h | 14 +-
> 2 files changed, 167 insertions(+), 60 deletions(-)
>
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
> index 009641e6c874..0a7903cf75e8 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
> @@ -93,12 +93,11 @@ int mlx5hws_bwc_matcher_create_simple(struct mlx5hws_bwc_matcher *bwc_matcher,
>
> hws_bwc_matcher_init_attr(bwc_matcher,
> priority,
> - MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG,
> - MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG,
> + bwc_matcher->rx_size.size_log,
> + bwc_matcher->tx_size.size_log,
> &attr);
>
> bwc_matcher->priority = priority;
> - bwc_matcher->size_log = MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG;
>
> bwc_matcher->size_of_at_array = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM;
> bwc_matcher->at = kcalloc(bwc_matcher->size_of_at_array,
> @@ -150,6 +149,20 @@ int mlx5hws_bwc_matcher_create_simple(struct mlx5hws_bwc_matcher *bwc_matcher,
> return -EINVAL;
> }
>
> +static void
> +hws_bwc_matcher_init_size_rxtx(struct mlx5hws_bwc_matcher_size *size)
> +{
> + size->size_log = MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG;
> + atomic_set(&size->num_of_rules, 0);
> + atomic_set(&size->rehash_required, false);
> +}
> +
> +static void hws_bwc_matcher_init_size(struct mlx5hws_bwc_matcher *bwc_matcher)
> +{
> + hws_bwc_matcher_init_size_rxtx(&bwc_matcher->rx_size);
> + hws_bwc_matcher_init_size_rxtx(&bwc_matcher->tx_size);
> +}
> +
> struct mlx5hws_bwc_matcher *
> mlx5hws_bwc_matcher_create(struct mlx5hws_table *table,
> u32 priority,
> @@ -170,8 +183,7 @@ mlx5hws_bwc_matcher_create(struct mlx5hws_table *table,
> if (!bwc_matcher)
> return NULL;
>
> - atomic_set(&bwc_matcher->num_of_rules, 0);
> - atomic_set(&bwc_matcher->rehash_required, false);
> + hws_bwc_matcher_init_size(bwc_matcher);
>
> /* Check if the required match params can be all matched
> * in single STE, otherwise complex matcher is needed.
> @@ -221,12 +233,13 @@ int mlx5hws_bwc_matcher_destroy_simple(struct mlx5hws_bwc_matcher *bwc_matcher)
>
> int mlx5hws_bwc_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher)
> {
> - u32 num_of_rules = atomic_read(&bwc_matcher->num_of_rules);
> + u32 rx_rules = atomic_read(&bwc_matcher->rx_size.num_of_rules);
> + u32 tx_rules = atomic_read(&bwc_matcher->tx_size.num_of_rules);
>
> - if (num_of_rules)
> + if (rx_rules || tx_rules)
> mlx5hws_err(bwc_matcher->matcher->tbl->ctx,
> - "BWC matcher destroy: matcher still has %d rules\n",
> - num_of_rules);
> + "BWC matcher destroy: matcher still has %u RX and %u TX rules\n",
> + rx_rules, tx_rules);
>
> if (bwc_matcher->complex)
> mlx5hws_bwc_matcher_destroy_complex(bwc_matcher);
> @@ -386,6 +399,16 @@ hws_bwc_rule_destroy_hws_sync(struct mlx5hws_bwc_rule *bwc_rule,
> return 0;
> }
>
> +static void hws_bwc_rule_cnt_dec(struct mlx5hws_bwc_rule *bwc_rule)
> +{
> + struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
> +
> + if (!bwc_rule->skip_rx)
> + atomic_dec(&bwc_matcher->rx_size.num_of_rules);
> + if (!bwc_rule->skip_tx)
> + atomic_dec(&bwc_matcher->tx_size.num_of_rules);
> +}
> +
> int mlx5hws_bwc_rule_destroy_simple(struct mlx5hws_bwc_rule *bwc_rule)
> {
> struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
> @@ -402,7 +425,7 @@ int mlx5hws_bwc_rule_destroy_simple(struct mlx5hws_bwc_rule *bwc_rule)
> mutex_lock(queue_lock);
>
> ret = hws_bwc_rule_destroy_hws_sync(bwc_rule, &attr);
> - atomic_dec(&bwc_matcher->num_of_rules);
> + hws_bwc_rule_cnt_dec(bwc_rule);
> hws_bwc_rule_list_remove(bwc_rule);
>
> mutex_unlock(queue_lock);
> @@ -489,25 +512,27 @@ hws_bwc_rule_update_sync(struct mlx5hws_bwc_rule *bwc_rule,
> }
>
> static bool
> -hws_bwc_matcher_size_maxed_out(struct mlx5hws_bwc_matcher *bwc_matcher)
> +hws_bwc_matcher_size_maxed_out(struct mlx5hws_bwc_matcher *bwc_matcher,
> + struct mlx5hws_bwc_matcher_size *size)
> {
> struct mlx5hws_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps;
>
> /* check the match RTC size */
> - return (bwc_matcher->size_log + MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH +
> + return (size->size_log + MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH +
> MLX5HWS_BWC_MATCHER_SIZE_LOG_STEP) >
> (caps->ste_alloc_log_max - 1);
> }
>
> static bool
> hws_bwc_matcher_rehash_size_needed(struct mlx5hws_bwc_matcher *bwc_matcher,
> + struct mlx5hws_bwc_matcher_size *size,
> u32 num_of_rules)
> {
> - if (unlikely(hws_bwc_matcher_size_maxed_out(bwc_matcher)))
> + if (unlikely(hws_bwc_matcher_size_maxed_out(bwc_matcher, size)))
> return false;
>
> if (unlikely((num_of_rules * 100 / MLX5HWS_BWC_MATCHER_REHASH_PERCENT_TH) >=
> - (1UL << bwc_matcher->size_log)))
> + (1UL << size->size_log)))
> return true;
>
> return false;
> @@ -564,20 +589,21 @@ hws_bwc_matcher_extend_at(struct mlx5hws_bwc_matcher *bwc_matcher,
> }
>
> static int
> -hws_bwc_matcher_extend_size(struct mlx5hws_bwc_matcher *bwc_matcher)
> +hws_bwc_matcher_extend_size(struct mlx5hws_bwc_matcher *bwc_matcher,
> + struct mlx5hws_bwc_matcher_size *size)
> {
> struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
> struct mlx5hws_cmd_query_caps *caps = ctx->caps;
>
> - if (unlikely(hws_bwc_matcher_size_maxed_out(bwc_matcher))) {
> + if (unlikely(hws_bwc_matcher_size_maxed_out(bwc_matcher, size))) {
> mlx5hws_err(ctx, "Can't resize matcher: depth exceeds limit %d\n",
> caps->rtc_log_depth_max);
> return -ENOMEM;
> }
>
> - bwc_matcher->size_log =
> - min(bwc_matcher->size_log + MLX5HWS_BWC_MATCHER_SIZE_LOG_STEP,
> - caps->ste_alloc_log_max - MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH);
> + size->size_log = min(size->size_log + MLX5HWS_BWC_MATCHER_SIZE_LOG_STEP,
> + caps->ste_alloc_log_max -
> + MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH);
>
> return 0;
> }
> @@ -697,8 +723,8 @@ static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher)
>
> hws_bwc_matcher_init_attr(bwc_matcher,
> bwc_matcher->priority,
> - bwc_matcher->size_log,
> - bwc_matcher->size_log,
> + bwc_matcher->rx_size.size_log,
> + bwc_matcher->tx_size.size_log,
> &matcher_attr);
>
> old_matcher = bwc_matcher->matcher;
> @@ -736,21 +762,39 @@ static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher)
> static int
> hws_bwc_matcher_rehash_size(struct mlx5hws_bwc_matcher *bwc_matcher)
> {
> + bool need_rx_rehash, need_tx_rehash;
> int ret;
>
> - /* If the current matcher size is already at its max size, we can't
> - * do the rehash. Skip it and try adding the rule again - perhaps
> - * there was some change.
> + need_rx_rehash = atomic_read(&bwc_matcher->rx_size.rehash_required);
> + need_tx_rehash = atomic_read(&bwc_matcher->tx_size.rehash_required);
> +
> + /* It is possible that another rule has already performed rehash.
> + * Need to check again if we really need rehash.
> */
> - if (hws_bwc_matcher_size_maxed_out(bwc_matcher))
> + if (!need_rx_rehash && !need_tx_rehash)
> return 0;
>
> - /* It is possible that other rule has already performed rehash.
> - * Need to check again if we really need rehash.
> + /* If the current matcher RX/TX size is already at its max size,
> + * it can't be rehashed.
> */
> - if (!atomic_read(&bwc_matcher->rehash_required) &&
> - !hws_bwc_matcher_rehash_size_needed(bwc_matcher,
> - atomic_read(&bwc_matcher->num_of_rules)))
> + if (need_rx_rehash &&
> + hws_bwc_matcher_size_maxed_out(bwc_matcher,
> + &bwc_matcher->rx_size)) {
> + atomic_set(&bwc_matcher->rx_size.rehash_required, false);
> + need_rx_rehash = false;
> + }
> + if (need_tx_rehash &&
> + hws_bwc_matcher_size_maxed_out(bwc_matcher,
> + &bwc_matcher->tx_size)) {
> + atomic_set(&bwc_matcher->tx_size.rehash_required, false);
> + need_tx_rehash = false;
> + }
> +
> + /* If both RX and TX rehash flags are now off, it means that whatever
> + * we wanted to rehash is now at its max size - no rehash can be done.
> + * Return and try adding the rule again - perhaps there was some change.
> + */
> + if (!need_rx_rehash && !need_tx_rehash)
> return 0;
>
> /* Now we're done all the checking - do the rehash:
> @@ -759,12 +803,22 @@ hws_bwc_matcher_rehash_size(struct mlx5hws_bwc_matcher *bwc_matcher)
> * - move all the rules to the new matcher
> * - destroy the old matcher
> */
> + atomic_set(&bwc_matcher->rx_size.rehash_required, false);
> + atomic_set(&bwc_matcher->tx_size.rehash_required, false);
>
> - atomic_set(&bwc_matcher->rehash_required, false);
> + if (need_rx_rehash) {
> + ret = hws_bwc_matcher_extend_size(bwc_matcher,
> + &bwc_matcher->rx_size);
> + if (ret)
> + return ret;
> + }
>
> - ret = hws_bwc_matcher_extend_size(bwc_matcher);
> - if (ret)
> - return ret;
> + if (need_tx_rehash) {
> + ret = hws_bwc_matcher_extend_size(bwc_matcher,
> + &bwc_matcher->tx_size);
> + if (ret)
> + return ret;
> + }
>
> return hws_bwc_matcher_move(bwc_matcher);
> }
> @@ -816,6 +870,62 @@ static int hws_bwc_rule_get_at_idx(struct mlx5hws_bwc_rule *bwc_rule,
> return at_idx;
> }
>
> +static void hws_bwc_rule_cnt_inc_rxtx(struct mlx5hws_bwc_rule *bwc_rule,
> + struct mlx5hws_bwc_matcher_size *size)
> +{
> + u32 num_of_rules = atomic_inc_return(&size->num_of_rules);
> +
> + if (unlikely(hws_bwc_matcher_rehash_size_needed(bwc_rule->bwc_matcher,
> + size, num_of_rules)))
> + atomic_set(&size->rehash_required, true);
> +}
> +
> +static void hws_bwc_rule_cnt_inc(struct mlx5hws_bwc_rule *bwc_rule)
> +{
> + struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
> +
> + if (!bwc_rule->skip_rx)
> + hws_bwc_rule_cnt_inc_rxtx(bwc_rule, &bwc_matcher->rx_size);
> + if (!bwc_rule->skip_tx)
> + hws_bwc_rule_cnt_inc_rxtx(bwc_rule, &bwc_matcher->tx_size);
> +}
> +
> +static int hws_bwc_rule_cnt_inc_with_rehash(struct mlx5hws_bwc_rule *bwc_rule,
> + u16 bwc_queue_idx)
> +{
> + struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
> + struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
> + struct mutex *queue_lock; /* Protect the queue */
> + int ret;
> +
> + hws_bwc_rule_cnt_inc(bwc_rule);
> +
> + if (!atomic_read(&bwc_matcher->rx_size.rehash_required) &&
> + !atomic_read(&bwc_matcher->tx_size.rehash_required))
> + return 0;
> +
> + queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx);
> + mutex_unlock(queue_lock);
> +
> + hws_bwc_lock_all_queues(ctx);
> + ret = hws_bwc_matcher_rehash_size(bwc_matcher);
> + hws_bwc_unlock_all_queues(ctx);
> +
> + mutex_lock(queue_lock);
> +
> + if (likely(!ret))
> + return 0;
> +
> + /* Failed to rehash. Print a diagnostic and rollback the counters. */
> + mlx5hws_err(ctx,
> + "BWC rule insertion: rehash to sizes [%d, %d] failed (%d)\n",
> + bwc_matcher->rx_size.size_log,
> + bwc_matcher->tx_size.size_log, ret);
> + hws_bwc_rule_cnt_dec(bwc_rule);
> +
> + return ret;
> +}
> +
> int mlx5hws_bwc_rule_create_simple(struct mlx5hws_bwc_rule *bwc_rule,
> u32 *match_param,
> struct mlx5hws_rule_action rule_actions[],
> @@ -826,7 +936,6 @@ int mlx5hws_bwc_rule_create_simple(struct mlx5hws_bwc_rule *bwc_rule,
> struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
> struct mlx5hws_rule_attr rule_attr;
> struct mutex *queue_lock; /* Protect the queue */
> - u32 num_of_rules;
> int ret = 0;
> int at_idx;
>
> @@ -844,26 +953,10 @@ int mlx5hws_bwc_rule_create_simple(struct mlx5hws_bwc_rule *bwc_rule,
> return -EINVAL;
> }
>
> - /* check if number of rules require rehash */
> - num_of_rules = atomic_inc_return(&bwc_matcher->num_of_rules);
> -
> - if (unlikely(hws_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules))) {
> + ret = hws_bwc_rule_cnt_inc_with_rehash(bwc_rule, bwc_queue_idx);
> + if (unlikely(ret)) {
> mutex_unlock(queue_lock);
> -
> - hws_bwc_lock_all_queues(ctx);
> - ret = hws_bwc_matcher_rehash_size(bwc_matcher);
> - hws_bwc_unlock_all_queues(ctx);
> -
> - if (ret) {
> - mlx5hws_err(ctx, "BWC rule insertion: rehash size [%d -> %d] failed (%d)\n",
> - bwc_matcher->size_log - MLX5HWS_BWC_MATCHER_SIZE_LOG_STEP,
> - bwc_matcher->size_log,
> - ret);
> - atomic_dec(&bwc_matcher->num_of_rules);
> - return ret;
> - }
> -
> - mutex_lock(queue_lock);
> + return ret;
> }
>
> ret = hws_bwc_rule_create_sync(bwc_rule,
> @@ -881,8 +974,11 @@ int mlx5hws_bwc_rule_create_simple(struct mlx5hws_bwc_rule *bwc_rule,
> * It could be because there was collision, or some other problem.
> * Try rehash by size and insert rule again - last chance.
> */
> + if (!bwc_rule->skip_rx)
> + atomic_set(&bwc_matcher->rx_size.rehash_required, true);
> + if (!bwc_rule->skip_tx)
> + atomic_set(&bwc_matcher->tx_size.rehash_required, true);
>
> - atomic_set(&bwc_matcher->rehash_required, true);
> mutex_unlock(queue_lock);
>
> hws_bwc_lock_all_queues(ctx);
> @@ -891,7 +987,7 @@ int mlx5hws_bwc_rule_create_simple(struct mlx5hws_bwc_rule *bwc_rule,
>
> if (ret) {
> mlx5hws_err(ctx, "BWC rule insertion: rehash failed (%d)\n", ret);
> - atomic_dec(&bwc_matcher->num_of_rules);
> + hws_bwc_rule_cnt_dec(bwc_rule);
> return ret;
> }
>
> @@ -907,7 +1003,7 @@ int mlx5hws_bwc_rule_create_simple(struct mlx5hws_bwc_rule *bwc_rule,
> if (unlikely(ret)) {
> mutex_unlock(queue_lock);
> mlx5hws_err(ctx, "BWC rule insertion failed (%d)\n", ret);
> - atomic_dec(&bwc_matcher->num_of_rules);
> + hws_bwc_rule_cnt_dec(bwc_rule);
> return ret;
> }
>
> @@ -937,6 +1033,9 @@ mlx5hws_bwc_rule_create(struct mlx5hws_bwc_matcher *bwc_matcher,
> if (unlikely(!bwc_rule))
> return NULL;
>
> + mlx5hws_rule_skip(bwc_matcher->matcher, flow_source,
> + &bwc_rule->skip_rx, &bwc_rule->skip_tx);
> +
> bwc_queue_idx = hws_bwc_gen_queue_idx(ctx);
>
> if (bwc_matcher->complex)
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h
> index d21fc247a510..1e9de6b9222c 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.h
> @@ -19,6 +19,13 @@
> #define MLX5HWS_BWC_POLLING_TIMEOUT 60
>
> struct mlx5hws_bwc_matcher_complex_data;
> +
> +struct mlx5hws_bwc_matcher_size {
> + u8 size_log;
> + atomic_t num_of_rules;
> + atomic_t rehash_required;
> +};
> +
> struct mlx5hws_bwc_matcher {
> struct mlx5hws_matcher *matcher;
> struct mlx5hws_match_template *mt;
> @@ -27,10 +34,9 @@ struct mlx5hws_bwc_matcher {
> struct mlx5hws_bwc_matcher *complex_first_bwc_matcher;
> u8 num_of_at;
> u8 size_of_at_array;
> - u8 size_log;
> u32 priority;
> - atomic_t num_of_rules;
> - atomic_t rehash_required;
> + struct mlx5hws_bwc_matcher_size rx_size;
> + struct mlx5hws_bwc_matcher_size tx_size;
> struct list_head *rules;
> };
>
> @@ -40,6 +46,8 @@ struct mlx5hws_bwc_rule {
> struct mlx5hws_bwc_rule *isolated_bwc_rule;
> struct mlx5hws_bwc_complex_rule_hash_node *complex_hash_node;
> u16 bwc_queue_idx;
> + bool skip_rx;
> + bool skip_tx;
> struct list_head list_node;
> };
>
