This patch adds support for crafting non-linear skbs in BPF test runs
for tc programs, via a new flag BPF_F_TEST_SKB_NON_LINEAR. When this
flag is set, the size of the linear area is given by ctx->data_end, with
a minimum of ETH_HLEN always pulled in the linear area.
This is particularly useful to test support for non-linear skbs in large
codebases such as Cilium. We've had multiple bugs in the past few years
where we were missing calls to bpf_skb_pull_data(). This support in
BPF_PROG_TEST_RUN would allow us to automatically cover this case in our
BPF tests.
In addition to the selftests introduced later in the series, this patch
was tested by setting BPF_F_TEST_SKB_NON_LINEAR for all tc selftests
programs and checking test failures were expected.
Suggested-by: Daniel Borkmann <daniel@xxxxxxxxxxxxx>
Signed-off-by: Paul Chaignon <paul.chaignon@xxxxxxxxx>
---
include/uapi/linux/bpf.h | 4 ++
net/bpf/test_run.c | 102 ++++++++++++++++++++++++---------
tools/include/uapi/linux/bpf.h | 4 ++
3 files changed, 82 insertions(+), 28 deletions(-)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 233de8677382..34272cd07dd2 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1448,6 +1448,10 @@ enum {
#define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1)
/* If set, apply CHECKSUM_COMPLETE to skb and validate the checksum */
#define BPF_F_TEST_SKB_CHECKSUM_COMPLETE (1U << 2)
+/* If set, skb will be non-linear with the size of the linear area given
+ * by ctx.data_end or at least ETH_HLEN.
+ */
+#define BPF_F_TEST_SKB_NON_LINEAR (1U << 3)
/* type for BPF_ENABLE_STATS */
enum bpf_stats_type {
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index a9c81fec3290..aaa8f93d2fdb 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -660,20 +660,29 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE)
BTF_KFUNCS_END(test_sk_check_kfunc_ids)
static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
- u32 size, u32 headroom, u32 tailroom)
+ u32 size, u32 headroom, u32 tailroom, bool nonlinear)
{
void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
- void *data;
+ void *data, *dst;
if (user_size < ETH_HLEN || user_size > PAGE_SIZE - headroom - tailroom)
return ERR_PTR(-EINVAL);
- size = SKB_DATA_ALIGN(size);
- data = kzalloc(size + headroom + tailroom, GFP_USER);
+ /* In non-linear case, data_in is copied to the paged data */
+ if (nonlinear) {
+ data = alloc_page(GFP_USER);
+ } else {
+ size = SKB_DATA_ALIGN(size);
+ data = kzalloc(size + headroom + tailroom, GFP_USER);
+ }
if (!data)
return ERR_PTR(-ENOMEM);
- if (copy_from_user(data + headroom, data_in, user_size)) {
+ if (nonlinear)
+ dst = page_address(data);
+ else
+ dst = data + headroom;
+ if (copy_from_user(dst, data_in, user_size)) {
kfree(data);
return ERR_PTR(-EFAULT);
}
@@ -910,6 +919,12 @@ static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
/* cb is allowed */
if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
+ offsetof(struct __sk_buff, data_end)))
+ return -EINVAL;
+
+ /* data_end is allowed, but not copied to skb */
+
+ if (!range_is_zero(__skb, offsetofend(struct __sk_buff, data_end),
offsetof(struct __sk_buff, tstamp)))
return -EINVAL;
@@ -984,7 +999,7 @@ static struct proto bpf_dummy_proto = {
int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
union bpf_attr __user *uattr)
{
- bool is_l2 = false, is_direct_pkt_access = false;
+ bool is_l2 = false, is_direct_pkt_access = false, is_nonlinear = false;
struct net *net = current->nsproxy->net_ns;
struct net_device *dev = net->loopback_dev;
u32 size = kattr->test.data_size_in;
@@ -994,25 +1009,14 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
struct sock *sk = NULL;
u32 retval, duration;
int hh_len = ETH_HLEN;
+ int linear_size, ret;
void *data;
- int ret;
- if ((kattr->test.flags & ~BPF_F_TEST_SKB_CHECKSUM_COMPLETE) ||
+ if ((kattr->test.flags & ~(BPF_F_TEST_SKB_CHECKSUM_COMPLETE | BPF_F_TEST_SKB_NON_LINEAR)) ||
kattr->test.cpu || kattr->test.batch_size)
return -EINVAL;
- data = bpf_test_init(kattr, kattr->test.data_size_in,
- size, NET_SKB_PAD + NET_IP_ALIGN,
- SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
- if (IS_ERR(data))
- return PTR_ERR(data);
-
- ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
- if (IS_ERR(ctx)) {
- ret = PTR_ERR(ctx);
- ctx = NULL;
- goto out;
- }
+ is_nonlinear = kattr->test.flags & BPF_F_TEST_SKB_NON_LINEAR;
switch (prog->type) {
case BPF_PROG_TYPE_SCHED_CLS:
@@ -1029,6 +1033,27 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
break;
}
+ if (is_nonlinear && !is_l2)
+ return -EINVAL;
+
+ data = bpf_test_init(kattr, kattr->test.data_size_in,
+ size, NET_SKB_PAD + NET_IP_ALIGN,
+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
+ is_nonlinear);
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
+ ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
+ if (IS_ERR(ctx)) {
+ ret = PTR_ERR(ctx);
+ ctx = NULL;
+ goto out;
+ }
+
+ linear_size = hh_len;
+ if (is_nonlinear && ctx && ctx->data_end > linear_size)
+ linear_size = ctx->data_end;
+
sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1);
if (!sk) {
ret = -ENOMEM;
@@ -1036,15 +1061,32 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
}
sock_init_data(NULL, sk);
- skb = slab_build_skb(data);
+ if (is_nonlinear)
+ skb = alloc_skb(NET_SKB_PAD + NET_IP_ALIGN + size +
+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
+ GFP_USER);
+ else
+ skb = slab_build_skb(data);
if (!skb) {
ret = -ENOMEM;
goto out;
}
+
skb->sk = sk;
skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
- __skb_put(skb, size);
+
+ if (is_nonlinear) {
+ skb_fill_page_desc(skb, 0, data, 0, size);
+ skb->truesize += PAGE_SIZE;
+ skb->data_len = size;
+ skb->len = size;
+
+ /* eth_type_trans expects the Ethernet header in the linear area. */
+ __pskb_pull_tail(skb, linear_size);
+ } else {
+ __skb_put(skb, size);
+ }
data = NULL; /* data released via kfree_skb */
@@ -1127,9 +1169,11 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
convert_skb_to___skb(skb, ctx);
size = skb->len;
- /* bpf program can never convert linear skb to non-linear */
- if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
+ if (skb_is_nonlinear(skb)) {
+ /* bpf program can never convert linear skb to non-linear */
+ WARN_ON_ONCE(!is_nonlinear);
size = skb_headlen(skb);
+ }
ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval,
duration);
if (!ret)
@@ -1139,7 +1183,8 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
if (dev && dev != net->loopback_dev)
dev_put(dev);
kfree_skb(skb);
- kfree(data);
+ if (data)
+ is_nonlinear ? __free_page(data) : kfree(data);
if (sk)
sk_free(sk);
kfree(ctx);
@@ -1265,7 +1310,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
size = max_data_sz;
}
- data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom);
+ data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom, false);
if (IS_ERR(data)) {
ret = PTR_ERR(data);
goto free_ctx;
@@ -1388,7 +1433,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
if (size < ETH_HLEN)
return -EINVAL;
- data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0);
+ data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0, false);
if (IS_ERR(data))
return PTR_ERR(data);
@@ -1661,7 +1706,8 @@ int bpf_prog_test_run_nf(struct bpf_prog *prog,
data = bpf_test_init(kattr, kattr->test.data_size_in, size,
NET_SKB_PAD + NET_IP_ALIGN,
- SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
+ false);
if (IS_ERR(data))
return PTR_ERR(data);
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 233de8677382..34272cd07dd2 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1448,6 +1448,10 @@ enum {
#define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1)
/* If set, apply CHECKSUM_COMPLETE to skb and validate the checksum */
#define BPF_F_TEST_SKB_CHECKSUM_COMPLETE (1U << 2)
+/* If set, skb will be non-linear with the size of the linear area given
+ * by ctx.data_end or at least ETH_HLEN.
+ */
+#define BPF_F_TEST_SKB_NON_LINEAR (1U << 3)
/* type for BPF_ENABLE_STATS */
enum bpf_stats_type {
--
2.43.0
