Add BPF_F_CPU and BPF_F_ALL_CPUS flags support in the following
functions:
* 'bpf_percpu_copy_data()'
* 'bpf_percpu_update_data()'
As a result, all percpu maps have to provide map flags for them.
Signed-off-by: Leon Hwang <leon.hwang@xxxxxxxxx>
---
include/linux/bpf-cgroup.h | 4 ++--
include/linux/bpf.h | 27 ++++++++++++++++++++++-----
kernel/bpf/arraymap.c | 6 +++---
kernel/bpf/hashtab.c | 30 +++++++++++++++---------------
kernel/bpf/local_storage.c | 6 +++---
kernel/bpf/syscall.c | 6 +++---
6 files changed, 48 insertions(+), 31 deletions(-)
diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
index aedf573bdb426..013f4db9903fd 100644
--- a/include/linux/bpf-cgroup.h
+++ b/include/linux/bpf-cgroup.h
@@ -172,7 +172,7 @@ void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage);
int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, struct bpf_map *map);
-int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value);
+int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value, u64 flags);
int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,
void *value, u64 flags);
@@ -467,7 +467,7 @@ static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
static inline void bpf_cgroup_storage_free(
struct bpf_cgroup_storage *storage) {}
static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key,
- void *value) {
+ void *value, u64 flags) {
return 0;
}
static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a78e07b0cb6cb..ef7f7c6a864c2 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -549,10 +549,16 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src
#ifdef CONFIG_BPF_SYSCALL
static inline void bpf_percpu_copy_data(struct bpf_map *map, void __percpu *pptr, void *value,
- u32 size)
+ u32 size, u64 flags)
{
int cpu, off = 0;
+ if (flags & BPF_F_CPU) {
+ cpu = flags >> 32;
+ copy_map_value_long(map, value, per_cpu_ptr(pptr, cpu));
+ return;
+ }
+
for_each_possible_cpu(cpu) {
copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
off += size;
@@ -562,14 +568,25 @@ static inline void bpf_percpu_copy_data(struct bpf_map *map, void __percpu *pptr
void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
static inline void bpf_percpu_update_data(struct bpf_map *map, void __percpu *pptr, void *value,
- u32 size)
+ u32 size, u64 flags)
{
int cpu, off = 0;
+ if (flags & BPF_F_CPU) {
+ cpu = flags >> 32;
+ bpf_obj_free_fields(map->record, per_cpu_ptr(pptr, cpu));
+ bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value, size);
+ return;
+ }
+
for_each_possible_cpu(cpu) {
bpf_obj_free_fields(map->record, per_cpu_ptr(pptr, cpu));
bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size);
- off += size;
+ /* same user-provided value is used if BPF_F_ALL_CPUS is
+ * specified, otherwise value is an array of per-cpu values.
+ */
+ if (!(flags & BPF_F_ALL_CPUS))
+ off += size;
}
}
#endif
@@ -2722,8 +2739,8 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,
struct bpf_func_state *caller,
struct bpf_func_state *callee);
-int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
-int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
+int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 flags);
+int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 flags);
int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
u64 flags);
int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index ed9e47dc4137b..d02cce3202840 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -295,7 +295,7 @@ static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key,
return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu);
}
-int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
+int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)
{
struct bpf_array *array = container_of(map, struct bpf_array, map);
u32 index = *(u32 *)key;
@@ -312,7 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
size = array->elem_size;
rcu_read_lock();
pptr = array->pptrs[index & array->index_mask];
- bpf_percpu_copy_data(map, pptr, value, size);
+ bpf_percpu_copy_data(map, pptr, value, size, map_flags);
rcu_read_unlock();
return 0;
}
@@ -405,7 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
size = array->elem_size;
rcu_read_lock();
pptr = array->pptrs[index & array->index_mask];
- bpf_percpu_update_data(map, pptr, value, size);
+ bpf_percpu_update_data(map, pptr, value, size, map_flags);
rcu_read_unlock();
return 0;
}
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 0a2c1042d5fdb..8955ae8482065 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -937,7 +937,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
}
static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
- void *value, bool onallcpus)
+ void *value, bool onallcpus, u64 map_flags)
{
if (!onallcpus) {
/* copy true value_size bytes */
@@ -945,12 +945,12 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
} else {
u32 size = round_up(htab->map.value_size, 8);
- bpf_percpu_update_data(&htab->map, pptr, value, size);
+ bpf_percpu_update_data(&htab->map, pptr, value, size, map_flags);
}
}
static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
- void *value, bool onallcpus)
+ void *value, bool onallcpus, u64 map_flags)
{
/* When not setting the initial value on all cpus, zero-fill element
* values for other cpus. Otherwise, bpf program has no way to ensure
@@ -968,7 +968,7 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu));
}
} else {
- pcpu_copy_value(htab, pptr, value, onallcpus);
+ pcpu_copy_value(htab, pptr, value, onallcpus, map_flags);
}
}
@@ -980,7 +980,7 @@ static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab)
static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
void *value, u32 key_size, u32 hash,
bool percpu, bool onallcpus,
- struct htab_elem *old_elem)
+ struct htab_elem *old_elem, u64 map_flags)
{
u32 size = htab->map.value_size;
bool prealloc = htab_is_prealloc(htab);
@@ -1038,7 +1038,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
pptr = *(void __percpu **)ptr;
}
- pcpu_init_value(htab, pptr, value, onallcpus);
+ pcpu_init_value(htab, pptr, value, onallcpus, map_flags);
if (!prealloc)
htab_elem_set_ptr(l_new, key_size, pptr);
@@ -1143,7 +1143,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
}
l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,
- l_old);
+ l_old, map_flags);
if (IS_ERR(l_new)) {
/* all pre-allocated elements are in use or memory exhausted */
ret = PTR_ERR(l_new);
@@ -1287,7 +1287,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
/* Update value in-place */
if (percpu) {
pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
- value, onallcpus);
+ value, onallcpus, map_flags);
} else {
void **inner_map_pptr = htab_elem_value(l_old, key_size);
@@ -1296,7 +1296,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
}
} else {
l_new = alloc_htab_elem(htab, key, value, key_size,
- hash, percpu, onallcpus, NULL);
+ hash, percpu, onallcpus, NULL, map_flags);
if (IS_ERR(l_new)) {
ret = PTR_ERR(l_new);
goto err;
@@ -1362,10 +1362,10 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
/* per-cpu hash map can update value in-place */
pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
- value, onallcpus);
+ value, onallcpus, map_flags);
} else {
pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),
- value, onallcpus);
+ value, onallcpus, map_flags);
hlist_nulls_add_head_rcu(&l_new->hash_node, head);
l_new = NULL;
}
@@ -1608,7 +1608,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
void __percpu *pptr;
pptr = htab_elem_get_ptr(l, key_size);
- bpf_percpu_copy_data(&htab->map, pptr, value, roundup_value_size);
+ bpf_percpu_copy_data(&htab->map, pptr, value, roundup_value_size, 0 /* map_flags */);
} else {
void *src = htab_elem_value(l, map->key_size);
@@ -1796,7 +1796,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
void __percpu *pptr;
pptr = htab_elem_get_ptr(l, map->key_size);
- bpf_percpu_copy_data(&htab->map, pptr, dst_val, size);
+ bpf_percpu_copy_data(&htab->map, pptr, dst_val, size, elem_map_flags);
} else {
value = htab_elem_value(l, key_size);
if (is_fd_htab(htab)) {
@@ -2351,7 +2351,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k
return NULL;
}
-int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)
+int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)
{
struct htab_elem *l;
void __percpu *pptr;
@@ -2371,7 +2371,7 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)
* eviction heuristics when user space does a map walk.
*/
pptr = htab_elem_get_ptr(l, map->key_size);
- bpf_percpu_copy_data(map, pptr, value, size);
+ bpf_percpu_copy_data(map, pptr, value, size, map_flags);
ret = 0;
out:
rcu_read_unlock();
diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c
index a1debbd26a415..f63639c79902a 100644
--- a/kernel/bpf/local_storage.c
+++ b/kernel/bpf/local_storage.c
@@ -180,7 +180,7 @@ static long cgroup_storage_update_elem(struct bpf_map *map, void *key,
}
int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key,
- void *value)
+ void *value, u64 map_flags)
{
struct bpf_cgroup_storage_map *map = map_to_storage(_map);
struct bpf_cgroup_storage *storage;
@@ -200,7 +200,7 @@ int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key,
*/
size = round_up(_map->value_size, 8);
pptr = storage->percpu_buf;
- bpf_percpu_copy_data(_map, pptr, value, size);
+ bpf_percpu_copy_data(_map, pptr, value, size, map_flags);
rcu_read_unlock();
return 0;
}
@@ -231,7 +231,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,
*/
size = round_up(_map->value_size, 8);
pptr = storage->percpu_buf;
- bpf_percpu_update_data(_map, pptr, value, size);
+ bpf_percpu_update_data(_map, pptr, value, size, map_flags);
rcu_read_unlock();
return 0;
}
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 50ece48409f3b..726363a64f5af 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -314,11 +314,11 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
bpf_disable_instrumentation();
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
- err = bpf_percpu_hash_copy(map, key, value);
+ err = bpf_percpu_hash_copy(map, key, value, flags);
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
- err = bpf_percpu_array_copy(map, key, value);
+ err = bpf_percpu_array_copy(map, key, value, flags);
} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
- err = bpf_percpu_cgroup_storage_copy(map, key, value);
+ err = bpf_percpu_cgroup_storage_copy(map, key, value, flags);
} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
err = bpf_stackmap_copy(map, key, value);
} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
--
2.50.1
