On Wed, Jun 18, 2025 at 1:58 AM Menglong Dong <menglong8.dong@xxxxxxxxx> wrote:
>
> The function update_prog_stats() will be called in the bpf trampoline.
> Make it always_inline to reduce the overhead.
What kind of difference did you measure ?
> Signed-off-by: Menglong Dong <dongml2@xxxxxxxxxxxxxxx>
> ---
> kernel/bpf/trampoline.c | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
> index c4b1a98ff726..134bcfd00b15 100644
> --- a/kernel/bpf/trampoline.c
> +++ b/kernel/bpf/trampoline.c
> @@ -911,8 +911,8 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
> return bpf_prog_start_time();
> }
>
> -static void notrace update_prog_stats(struct bpf_prog *prog,
> - u64 start)
> +static __always_inline void notrace update_prog_stats(struct bpf_prog *prog,
> + u64 start)
> {
How about the following instead:
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index c4b1a98ff726..728bb2845f41 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -911,28 +911,23 @@ static u64 notrace __bpf_prog_enter_recur(struct
bpf_prog *prog, struct bpf_tram
return bpf_prog_start_time();
}
-static void notrace update_prog_stats(struct bpf_prog *prog,
- u64 start)
+static noinline void notrace __update_prog_stats(struct bpf_prog *prog,
+ u64 start)
{
struct bpf_prog_stats *stats;
-
- if (static_branch_unlikely(&bpf_stats_enabled_key) &&
- /* static_key could be enabled in __bpf_prog_enter*
- * and disabled in __bpf_prog_exit*.
- * And vice versa.
- * Hence check that 'start' is valid.
- */
- start > NO_START_TIME) {
- u64 duration = sched_clock() - start;
- unsigned long flags;
-
- stats = this_cpu_ptr(prog->stats);
- flags = u64_stats_update_begin_irqsave(&stats->syncp);
- u64_stats_inc(&stats->cnt);
- u64_stats_add(&stats->nsecs, duration);
- u64_stats_update_end_irqrestore(&stats->syncp, flags);
- }
+ u64 duration = sched_clock() - start;
+ unsigned long flags;
+
+ stats = this_cpu_ptr(prog->stats);
+ flags = u64_stats_update_begin_irqsave(&stats->syncp);
+ u64_stats_inc(&stats->cnt);
+ u64_stats_add(&stats->nsecs, duration);
+ u64_stats_update_end_irqrestore(&stats->syncp, flags);
}
+#define update_prog_stats(prog, start) \
+ if (static_branch_unlikely(&bpf_stats_enabled_key) && \
+ start > NO_START_TIME) \
+ __update_prog_stats(prog, start)
static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,
struct bpf_tramp_run_ctx *run_ctx)
Maybe
if (start > NO_START_TIME)
should stay within __update_prog_stats().
pls run a few experiments.
