Avoid excessive vzalloc/vfree calls when patching instructions in
do_misc_fixups(). bpf_patch_insn_data() uses vzalloc to allocate new
memory for env->insn_aux_data for each patch as follows:
struct bpf_prog *bpf_patch_insn_data(env, ...)
{
...
new_data = vzalloc(... O(program size) ...);
...
adjust_insn_aux_data(env, new_data, ...);
...
}
void adjust_insn_aux_data(env, new_data, ...)
{
...
memcpy(new_data, env->insn_aux_data);
vfree(env->insn_aux_data);
env->insn_aux_data = new_data;
...
}
The vzalloc/vfree pair is hot in perf report collected for e.g.
pyperf180 test case. It can be replaced with a call to vrealloc in a
hope to reduce the number of actual memory allocations.
This is a stop-gap solution, as bpf_patch_insn_data is still hot in
the profile. More comprehansive solutions had been discussed before
e.g. as in [1].
Perf stat w/o this patch:
$ perf stat -B --all-kernel -r10 -- ./veristat -q pyperf180.bpf.o
...
2201.25 msec task-clock # 0.973 CPUs utilized ( +- 2.20% )
188 context-switches # 85.406 /sec ( +- 9.29% )
15 cpu-migrations # 6.814 /sec ( +- 5.64% )
5 page-faults # 2.271 /sec ( +- 3.27% )
4315057974 instructions # 1.28 insn per cycle
# 0.33 stalled cycles per insn ( +- 0.03% )
3366141387 cycles # 1.529 GHz ( +- 0.21% )
1420810964 stalled-cycles-frontend # 42.21% frontend cycles idle ( +- 0.23% )
1049956791 branches # 476.981 M/sec ( +- 0.03% )
60591781 branch-misses # 5.77% of all branches ( +- 0.07% )
2.2632 +- 0.0527 seconds time elapsed ( +- 2.33% )
Perf stat with this patch:
1132.77 msec task-clock # 0.976 CPUs utilized ( +- 3.47% )
80 context-switches # 70.623 /sec ( +- 31.57% )
1 cpu-migrations # 0.883 /sec ( +- 37.27% )
5 page-faults # 4.414 /sec ( +- 3.59% )
3307816503 instructions # 2.20 insn per cycle
# 0.15 stalled cycles per insn ( +- 0.05% )
1506171011 cycles # 1.330 GHz ( +- 0.55% )
488914539 stalled-cycles-frontend # 32.46% frontend cycles idle ( +- 0.94% )
729783557 branches # 644.246 M/sec ( +- 0.05% )
17312298 branch-misses # 2.37% of all branches ( +- 0.23% )
1.1602 +- 0.0443 seconds time elapsed ( +- 3.82% )
[1] https://lore.kernel.org/bpf/CAEf4BzY_E8MSL4mD0UPuuiDcbJhh9e2xQo2=5w+ppRWWiYSGvQ@xxxxxxxxxxxxxx/
Suggested-by: Alexei Starovoitov <alexei.starovoitov@xxxxxxxxx>
Signed-off-by: Eduard Zingerman <eddyz87@xxxxxxxxx>
---
kernel/bpf/verifier.c | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 69eb2b5c2218..6ef7dc6079a4 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -20699,12 +20699,11 @@ static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
* [0, off) and [off, end) to new locations, so the patched range stays zero
*/
static void adjust_insn_aux_data(struct bpf_verifier_env *env,
- struct bpf_insn_aux_data *new_data,
struct bpf_prog *new_prog, u32 off, u32 cnt)
{
- struct bpf_insn_aux_data *old_data = env->insn_aux_data;
+ struct bpf_insn_aux_data *data = env->insn_aux_data;
struct bpf_insn *insn = new_prog->insnsi;
- u32 old_seen = old_data[off].seen;
+ u32 old_seen = data[off].seen;
u32 prog_len;
int i;
@@ -20712,22 +20711,19 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
* (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
* original insn at old prog.
*/
- old_data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);
+ data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);
if (cnt == 1)
return;
prog_len = new_prog->len;
- memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
- memcpy(new_data + off + cnt - 1, old_data + off,
- sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
+ memmove(data + off + cnt - 1, data + off,
+ sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
for (i = off; i < off + cnt - 1; i++) {
/* Expand insni[off]'s seen count to the patched range. */
- new_data[i].seen = old_seen;
- new_data[i].zext_dst = insn_has_def32(insn + i);
+ data[i].seen = old_seen;
+ data[i].zext_dst = insn_has_def32(insn + i);
}
- env->insn_aux_data = new_data;
- vfree(old_data);
}
static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
@@ -20765,10 +20761,14 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of
struct bpf_insn_aux_data *new_data = NULL;
if (len > 1) {
- new_data = vzalloc(array_size(env->prog->len + len - 1,
- sizeof(struct bpf_insn_aux_data)));
+ new_data = vrealloc(env->insn_aux_data,
+ array_size(env->prog->len + len - 1,
+ sizeof(struct bpf_insn_aux_data)),
+ GFP_KERNEL_ACCOUNT | __GFP_ZERO);
if (!new_data)
return NULL;
+
+ env->insn_aux_data = new_data;
}
new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
@@ -20780,7 +20780,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of
vfree(new_data);
return NULL;
}
- adjust_insn_aux_data(env, new_data, new_prog, off, len);
+ adjust_insn_aux_data(env, new_prog, off, len);
adjust_subprog_starts(env, off, len);
adjust_poke_descs(new_prog, off, len);
return new_prog;
--
2.47.3
