27 static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size)  in mem_charge()  argument
34 	return map->ops->map_local_storage_charge(smap, owner, size);  in mem_charge()
37 static void mem_uncharge(struct bpf_local_storage_map *smap, void *owner,  in mem_uncharge()  argument
43 		map->ops->map_local_storage_uncharge(smap, owner, size);  in mem_uncharge()
47 owner_storage(struct bpf_local_storage_map *smap, void *owner)  in owner_storage()  argument
51 	return map->ops->map_owner_storage_ptr(owner);  in owner_storage()
75 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,  in bpf_selem_alloc()  argument
80 	if (charge_mem && mem_charge(smap, owner, smap->elem_size))  in bpf_selem_alloc()
110 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_selem_alloc()
246 		/* reuse_now == true only happens when the storage owner  in bpf_selem_free()
291 	void *owner;  in bpf_selem_unlink_storage_nolock()  local
294 	owner = local_storage->owner;  in bpf_selem_unlink_storage_nolock()
296 	/* All uncharging on the owner must be done first.  in bpf_selem_unlink_storage_nolock()
297 	 * The owner may be freed once the last selem is unlinked  in bpf_selem_unlink_storage_nolock()
301 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_selem_unlink_storage_nolock()
306 		mem_uncharge(smap, owner, sizeof(struct bpf_local_storage));  in bpf_selem_unlink_storage_nolock()
307 		local_storage->owner = NULL;  in bpf_selem_unlink_storage_nolock()
309 		/* After this RCU_INIT, owner may be freed and cannot be used */  in bpf_selem_unlink_storage_nolock()
310 		RCU_INIT_POINTER(*owner_storage(smap, owner), NULL);  in bpf_selem_unlink_storage_nolock()
483 int bpf_local_storage_alloc(void *owner,  in bpf_local_storage_alloc()  argument
492 	err = mem_charge(smap, owner, sizeof(*storage));  in bpf_local_storage_alloc()
509 	storage->owner = owner;  in bpf_local_storage_alloc()
515 		(struct bpf_local_storage **)owner_storage(smap, owner);  in bpf_local_storage_alloc()
516 	/* Publish storage to the owner.  in bpf_local_storage_alloc()
517 	 * Instead of using any lock of the kernel object (i.e. owner),  in bpf_local_storage_alloc()
521 	 * From now on, the owner->storage pointer (e.g. sk->sk_bpf_storage)  in bpf_local_storage_alloc()
523 	 * the owner->storage, the storage->lock must be held before  in bpf_local_storage_alloc()
524 	 * setting owner->storage ptr to NULL.  in bpf_local_storage_alloc()
547 	mem_uncharge(smap, owner, sizeof(*storage));  in bpf_local_storage_alloc()
557 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,  in bpf_local_storage_update()  argument
577 	local_storage = rcu_dereference_check(*owner_storage(smap, owner),  in bpf_local_storage_update()
580 		/* Very first elem for the owner */  in bpf_local_storage_update()
585 		selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);  in bpf_local_storage_update()
589 		err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);  in bpf_local_storage_update()
592 			mem_uncharge(smap, owner, smap->elem_size);  in bpf_local_storage_update()
619 	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);  in bpf_local_storage_update()
666 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_local_storage_update()
793  * get a hold of the owner PTR_TO_BTF_ID in any context. eg. bpf_get_current_task_btf.
795  * memory immediately. To be reuse-immediate safe, the owner destruction
883 	 * to the owner->storage or to the map bucket's list.  in bpf_local_storage_map_free()