Lines Matching +full:gen +full:- +full:2
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
30 * ctx - bpf program context
31 * stack - bpf program stack
32 * blob - bpf_attr-s, strings, insns, map data.
42 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field))
46 static int blob_fd_array_off(struct bpf_gen *gen, int index) in blob_fd_array_off() argument
48 return gen->fd_array + index * sizeof(int); in blob_fd_array_off()
51 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) in realloc_insn_buf() argument
53 size_t off = gen->insn_cur - gen->insn_start; in realloc_insn_buf()
56 if (gen->error) in realloc_insn_buf()
57 return gen->error; in realloc_insn_buf()
59 gen->error = -ERANGE; in realloc_insn_buf()
60 return -ERANGE; in realloc_insn_buf()
62 insn_start = realloc(gen->insn_start, off + size); in realloc_insn_buf()
64 gen->error = -ENOMEM; in realloc_insn_buf()
65 free(gen->insn_start); in realloc_insn_buf()
66 gen->insn_start = NULL; in realloc_insn_buf()
67 return -ENOMEM; in realloc_insn_buf()
69 gen->insn_start = insn_start; in realloc_insn_buf()
70 gen->insn_cur = insn_start + off; in realloc_insn_buf()
74 static int realloc_data_buf(struct bpf_gen *gen, __u32 size) in realloc_data_buf() argument
76 size_t off = gen->data_cur - gen->data_start; in realloc_data_buf()
79 if (gen->error) in realloc_data_buf()
80 return gen->error; in realloc_data_buf()
82 gen->error = -ERANGE; in realloc_data_buf()
83 return -ERANGE; in realloc_data_buf()
85 data_start = realloc(gen->data_start, off + size); in realloc_data_buf()
87 gen->error = -ENOMEM; in realloc_data_buf()
88 free(gen->data_start); in realloc_data_buf()
89 gen->data_start = NULL; in realloc_data_buf()
90 return -ENOMEM; in realloc_data_buf()
92 gen->data_start = data_start; in realloc_data_buf()
93 gen->data_cur = data_start + off; in realloc_data_buf()
97 static void emit(struct bpf_gen *gen, struct bpf_insn insn) in emit() argument
99 if (realloc_insn_buf(gen, sizeof(insn))) in emit()
101 memcpy(gen->insn_cur, &insn, sizeof(insn)); in emit()
102 gen->insn_cur += sizeof(insn); in emit()
105 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2) in emit2() argument
107 emit(gen, insn1); in emit2()
108 emit(gen, insn2); in emit2()
111 static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
112 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
114 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) in bpf_gen__init() argument
119 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); in bpf_gen__init()
120 gen->log_level = log_level; in bpf_gen__init()
122 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); in bpf_gen__init()
125 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); in bpf_gen__init()
126 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz)); in bpf_gen__init()
127 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz)); in bpf_gen__init()
128 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); in bpf_gen__init()
129 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); in bpf_gen__init()
134 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, in bpf_gen__init()
136 (nr_progs_sz / 4) * 3 + 2 + in bpf_gen__init()
140 nr_maps * (6 + (gen->log_level ? 6 : 0)))); in bpf_gen__init()
143 gen->cleanup_label = gen->insn_cur - gen->insn_start; in bpf_gen__init()
146 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i)); in bpf_gen__init()
147 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1)); in bpf_gen__init()
148 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); in bpf_gen__init()
151 emit_sys_close_blob(gen, blob_fd_array_off(gen, i)); in bpf_gen__init()
153 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); in bpf_gen__init()
154 emit(gen, BPF_EXIT_INSN()); in bpf_gen__init()
157 static int add_data(struct bpf_gen *gen, const void *data, __u32 size) in add_data() argument
163 if (realloc_data_buf(gen, size8)) in add_data()
165 prev = gen->data_cur; in add_data()
167 memcpy(gen->data_cur, data, size); in add_data()
168 memcpy(gen->data_cur + size, &zero, size8 - size); in add_data()
170 memset(gen->data_cur, 0, size8); in add_data()
172 gen->data_cur += size8; in add_data()
173 return prev - gen->data_start; in add_data()
179 static int add_map_fd(struct bpf_gen *gen) in add_map_fd() argument
181 if (gen->nr_maps == MAX_USED_MAPS) { in add_map_fd()
183 gen->error = -E2BIG; in add_map_fd()
186 return gen->nr_maps++; in add_map_fd()
189 static int add_kfunc_btf_fd(struct bpf_gen *gen) in add_kfunc_btf_fd() argument
193 if (gen->nr_fd_array == MAX_KFUNC_DESCS) { in add_kfunc_btf_fd()
194 cur = add_data(gen, NULL, sizeof(int)); in add_kfunc_btf_fd()
195 return (cur - gen->fd_array) / sizeof(int); in add_kfunc_btf_fd()
197 return MAX_USED_MAPS + gen->nr_fd_array++; in add_kfunc_btf_fd()
205 case 2: return BPF_H; in insn_bytes_to_bpf_size()
207 default: return -1; in insn_bytes_to_bpf_size()
212 static void emit_rel_store(struct bpf_gen *gen, int off, int data) in emit_rel_store() argument
214 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store()
216 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_rel_store()
218 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); in emit_rel_store()
221 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off) in move_blob2blob() argument
223 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2blob()
225 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0)); in move_blob2blob()
226 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2blob()
228 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_blob2blob()
231 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off) in move_blob2ctx() argument
233 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_blob2ctx()
235 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0)); in move_blob2ctx()
236 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); in move_blob2ctx()
239 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off, in move_ctx2blob() argument
242 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off)); in move_ctx2blob()
245 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c in move_ctx2blob()
247 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3)); in move_ctx2blob()
248 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_ctx2blob()
250 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_ctx2blob()
253 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off) in move_stack2blob() argument
255 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); in move_stack2blob()
256 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in move_stack2blob()
258 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); in move_stack2blob()
261 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off) in move_stack2ctx() argument
263 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); in move_stack2ctx()
264 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); in move_stack2ctx()
267 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size) in emit_sys_bpf() argument
269 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd)); in emit_sys_bpf()
270 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, in emit_sys_bpf()
272 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size)); in emit_sys_bpf()
273 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf)); in emit_sys_bpf()
275 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_sys_bpf()
283 static void emit_check_err(struct bpf_gen *gen) in emit_check_err() argument
285 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; in emit_check_err()
291 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); in emit_check_err()
293 gen->error = -ERANGE; in emit_check_err()
294 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); in emit_check_err()
298 /* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */
299 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2, in emit_debug() argument
305 if (!gen->log_level) in emit_debug()
308 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0) in emit_debug()
315 addr = add_data(gen, buf, len); in emit_debug()
317 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_debug()
319 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_debug()
321 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1)); in emit_debug()
323 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2)); in emit_debug()
324 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk)); in emit_debug()
327 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...) in debug_regs() argument
332 emit_debug(gen, reg1, reg2, fmt, args); in debug_regs()
336 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...) in debug_ret() argument
341 emit_debug(gen, BPF_REG_7, -1, fmt, args); in debug_ret()
345 static void __emit_sys_close(struct bpf_gen *gen) in __emit_sys_close() argument
347 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, in __emit_sys_close()
348 /* 2 is the number of the following insns in __emit_sys_close()
351 2 + (gen->log_level ? 6 : 0))); in __emit_sys_close()
352 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1)); in __emit_sys_close()
353 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); in __emit_sys_close()
354 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d"); in __emit_sys_close()
357 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off) in emit_sys_close_stack() argument
359 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off)); in emit_sys_close_stack()
360 __emit_sys_close(gen); in emit_sys_close_stack()
363 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) in emit_sys_close_blob() argument
365 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_sys_close_blob()
367 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0)); in emit_sys_close_blob()
368 __emit_sys_close(gen); in emit_sys_close_blob()
371 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) in bpf_gen__finish() argument
375 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { in bpf_gen__finish()
377 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); in bpf_gen__finish()
378 gen->error = -EFAULT; in bpf_gen__finish()
379 return gen->error; in bpf_gen__finish()
381 emit_sys_close_stack(gen, stack_off(btf_fd)); in bpf_gen__finish()
382 for (i = 0; i < gen->nr_progs; i++) in bpf_gen__finish()
383 move_stack2ctx(gen, in bpf_gen__finish()
385 sizeof(struct bpf_map_desc) * gen->nr_maps + in bpf_gen__finish()
389 for (i = 0; i < gen->nr_maps; i++) in bpf_gen__finish()
390 move_blob2ctx(gen, in bpf_gen__finish()
394 blob_fd_array_off(gen, i)); in bpf_gen__finish()
395 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); in bpf_gen__finish()
396 emit(gen, BPF_EXIT_INSN()); in bpf_gen__finish()
397 pr_debug("gen: finish %s\n", errstr(gen->error)); in bpf_gen__finish()
398 if (!gen->error) { in bpf_gen__finish()
399 struct gen_loader_opts *opts = gen->opts; in bpf_gen__finish()
401 opts->insns = gen->insn_start; in bpf_gen__finish()
402 opts->insns_sz = gen->insn_cur - gen->insn_start; in bpf_gen__finish()
403 opts->data = gen->data_start; in bpf_gen__finish()
404 opts->data_sz = gen->data_cur - gen->data_start; in bpf_gen__finish()
407 if (gen->swapped_endian) { in bpf_gen__finish()
408 struct bpf_insn *insn = (struct bpf_insn *)opts->insns; in bpf_gen__finish()
409 int insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); in bpf_gen__finish()
415 return gen->error; in bpf_gen__finish()
418 void bpf_gen__free(struct bpf_gen *gen) in bpf_gen__free() argument
420 if (!gen) in bpf_gen__free()
422 free(gen->data_start); in bpf_gen__free()
423 free(gen->insn_start); in bpf_gen__free()
424 free(gen); in bpf_gen__free()
428 * Fields of bpf_attr are set to values in native byte-order before being
429 * written to the target-bound data blob, and may need endian conversion.
437 if (gen->swapped_endian) { \
440 case 2: _val = bswap_16(_val); break; \
449 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, in bpf_gen__load_btf() argument
457 btf_data = add_data(gen, btf_raw_data, btf_raw_size); in bpf_gen__load_btf()
460 btf_load_attr = add_data(gen, &attr, attr_size); in bpf_gen__load_btf()
461 pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n", in bpf_gen__load_btf()
465 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4, in bpf_gen__load_btf()
467 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4, in bpf_gen__load_btf()
469 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8, in bpf_gen__load_btf()
472 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data); in bpf_gen__load_btf()
474 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size); in bpf_gen__load_btf()
475 debug_ret(gen, "btf_load size %d", btf_raw_size); in bpf_gen__load_btf()
476 emit_check_err(gen); in bpf_gen__load_btf()
478 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd))); in bpf_gen__load_btf()
481 void bpf_gen__map_create(struct bpf_gen *gen, in bpf_gen__map_create() argument
496 attr.map_flags = tgt_endian(map_attr->map_flags); in bpf_gen__map_create()
497 attr.map_extra = tgt_endian(map_attr->map_extra); in bpf_gen__map_create()
500 attr.numa_node = tgt_endian(map_attr->numa_node); in bpf_gen__map_create()
501 attr.map_ifindex = tgt_endian(map_attr->map_ifindex); in bpf_gen__map_create()
503 attr.btf_key_type_id = tgt_endian(map_attr->btf_key_type_id); in bpf_gen__map_create()
504 attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id); in bpf_gen__map_create()
506 map_create_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_create()
507 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n", in bpf_gen__map_create()
508 map_name, map_idx, map_type, map_attr->btf_value_type_id, in bpf_gen__map_create()
511 if (map_attr->btf_value_type_id) in bpf_gen__map_create()
513 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4, in bpf_gen__map_create()
518 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4, in bpf_gen__map_create()
527 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, in bpf_gen__map_create()
533 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); in bpf_gen__map_create()
534 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", in bpf_gen__map_create()
536 map_attr->btf_value_type_id); in bpf_gen__map_create()
537 emit_check_err(gen); in bpf_gen__map_create()
542 * It's called with -1 to create an inner map. in bpf_gen__map_create()
544 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__map_create()
546 } else if (map_idx != gen->nr_maps) { in bpf_gen__map_create()
547 gen->error = -EDOM; /* internal bug */ in bpf_gen__map_create()
550 /* add_map_fd does gen->nr_maps++ */ in bpf_gen__map_create()
551 idx = add_map_fd(gen); in bpf_gen__map_create()
552 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__map_create()
553 0, 0, 0, blob_fd_array_off(gen, idx))); in bpf_gen__map_create()
554 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); in bpf_gen__map_create()
557 emit_sys_close_stack(gen, stack_off(inner_map_fd)); in bpf_gen__map_create()
560 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, in bpf_gen__record_attach_target() argument
567 gen->attach_kind = kind; in bpf_gen__record_attach_target()
568 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", in bpf_gen__record_attach_target()
570 if (ret >= sizeof(gen->attach_target)) in bpf_gen__record_attach_target()
571 gen->error = -ENOSPC; in bpf_gen__record_attach_target()
574 static void emit_find_attach_target(struct bpf_gen *gen) in emit_find_attach_target() argument
576 int name, len = strlen(gen->attach_target) + 1; in emit_find_attach_target()
578 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind); in emit_find_attach_target()
579 name = add_data(gen, gen->attach_target, len); in emit_find_attach_target()
581 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_find_attach_target()
583 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_find_attach_target()
584 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind)); in emit_find_attach_target()
585 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); in emit_find_attach_target()
586 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); in emit_find_attach_target()
587 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_find_attach_target()
588 debug_ret(gen, "find_by_name_kind(%s,%d)", in emit_find_attach_target()
589 gen->attach_target, gen->attach_kind); in emit_find_attach_target()
590 emit_check_err(gen); in emit_find_attach_target()
591 /* if successful, btf_id is in lower 32-bit of R7 and in emit_find_attach_target()
592 * btf_obj_fd is in upper 32-bit in emit_find_attach_target()
596 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, in bpf_gen__record_extern() argument
601 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo)); in bpf_gen__record_extern()
603 gen->error = -ENOMEM; in bpf_gen__record_extern()
606 gen->relos = relo; in bpf_gen__record_extern()
607 relo += gen->relo_cnt; in bpf_gen__record_extern()
608 relo->name = name; in bpf_gen__record_extern()
609 relo->is_weak = is_weak; in bpf_gen__record_extern()
610 relo->is_typeless = is_typeless; in bpf_gen__record_extern()
611 relo->is_ld64 = is_ld64; in bpf_gen__record_extern()
612 relo->kind = kind; in bpf_gen__record_extern()
613 relo->insn_idx = insn_idx; in bpf_gen__record_extern()
614 gen->relo_cnt++; in bpf_gen__record_extern()
618 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo) in get_ksym_desc() argument
623 for (i = 0; i < gen->nr_ksyms; i++) { in get_ksym_desc()
624 kdesc = &gen->ksyms[i]; in get_ksym_desc()
625 if (kdesc->kind == relo->kind && kdesc->is_ld64 == relo->is_ld64 && in get_ksym_desc()
626 !strcmp(kdesc->name, relo->name)) { in get_ksym_desc()
627 kdesc->ref++; in get_ksym_desc()
631 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc)); in get_ksym_desc()
633 gen->error = -ENOMEM; in get_ksym_desc()
636 gen->ksyms = kdesc; in get_ksym_desc()
637 kdesc = &gen->ksyms[gen->nr_ksyms++]; in get_ksym_desc()
638 kdesc->name = relo->name; in get_ksym_desc()
639 kdesc->kind = relo->kind; in get_ksym_desc()
640 kdesc->ref = 1; in get_ksym_desc()
641 kdesc->off = 0; in get_ksym_desc()
642 kdesc->insn = 0; in get_ksym_desc()
643 kdesc->is_ld64 = relo->is_ld64; in get_ksym_desc()
647 /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
650 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo) in emit_bpf_find_by_name_kind() argument
652 int name_off, len = strlen(relo->name) + 1; in emit_bpf_find_by_name_kind()
654 name_off = add_data(gen, relo->name, len); in emit_bpf_find_by_name_kind()
655 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_find_by_name_kind()
657 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_bpf_find_by_name_kind()
658 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind)); in emit_bpf_find_by_name_kind()
659 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); in emit_bpf_find_by_name_kind()
660 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); in emit_bpf_find_by_name_kind()
661 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_bpf_find_by_name_kind()
662 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind); in emit_bpf_find_by_name_kind()
665 /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
669 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo) in emit_bpf_kallsyms_lookup_name() argument
671 int name_off, len = strlen(relo->name) + 1, res_off; in emit_bpf_kallsyms_lookup_name()
673 name_off = add_data(gen, relo->name, len); in emit_bpf_kallsyms_lookup_name()
674 res_off = add_data(gen, NULL, 8); /* res is u64 */ in emit_bpf_kallsyms_lookup_name()
675 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_kallsyms_lookup_name()
677 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); in emit_bpf_kallsyms_lookup_name()
678 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); in emit_bpf_kallsyms_lookup_name()
679 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE, in emit_bpf_kallsyms_lookup_name()
681 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4)); in emit_bpf_kallsyms_lookup_name()
682 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name)); in emit_bpf_kallsyms_lookup_name()
683 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0)); in emit_bpf_kallsyms_lookup_name()
684 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); in emit_bpf_kallsyms_lookup_name()
685 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind); in emit_bpf_kallsyms_lookup_name()
689 * BPF_REG_8 - pointer to instruction
694 * with index, we get the insn->imm, insn->off pairing that kernel uses for
698 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) in emit_relo_kfunc_btf() argument
703 kdesc = get_ksym_desc(gen, relo); in emit_relo_kfunc_btf()
707 if (kdesc->ref > 1) { in emit_relo_kfunc_btf()
708 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_kfunc_btf()
709 kdesc->insn + offsetof(struct bpf_insn, imm)); in emit_relo_kfunc_btf()
710 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2, in emit_relo_kfunc_btf()
711 kdesc->insn + offsetof(struct bpf_insn, off)); in emit_relo_kfunc_btf()
715 kdesc->insn = insn; in emit_relo_kfunc_btf()
716 emit_bpf_find_by_name_kind(gen, relo); in emit_relo_kfunc_btf()
717 if (!relo->is_weak) in emit_relo_kfunc_btf()
718 emit_check_err(gen); in emit_relo_kfunc_btf()
720 btf_fd_idx = add_kfunc_btf_fd(gen); in emit_relo_kfunc_btf()
723 btf_fd_idx, relo->name); in emit_relo_kfunc_btf()
724 gen->error = -E2BIG; in emit_relo_kfunc_btf()
727 kdesc->off = btf_fd_idx; in emit_relo_kfunc_btf()
729 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); in emit_relo_kfunc_btf()
731 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); in emit_relo_kfunc_btf()
732 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); in emit_relo_kfunc_btf()
734 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10)); in emit_relo_kfunc_btf()
736 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); in emit_relo_kfunc_btf()
738 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); in emit_relo_kfunc_btf()
739 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); in emit_relo_kfunc_btf()
741 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_relo_kfunc_btf()
742 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx))); in emit_relo_kfunc_btf()
744 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0)); in emit_relo_kfunc_btf()
746 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2)); in emit_relo_kfunc_btf()
748 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); in emit_relo_kfunc_btf()
750 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); in emit_relo_kfunc_btf()
752 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx)); in emit_relo_kfunc_btf()
754 if (!gen->log_level) in emit_relo_kfunc_btf()
756 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, in emit_relo_kfunc_btf()
758 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, in emit_relo_kfunc_btf()
760 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d", in emit_relo_kfunc_btf()
761 relo->name, kdesc->ref); in emit_relo_kfunc_btf()
762 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in emit_relo_kfunc_btf()
763 0, 0, 0, blob_fd_array_off(gen, kdesc->off))); in emit_relo_kfunc_btf()
764 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0)); in emit_relo_kfunc_btf()
765 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd", in emit_relo_kfunc_btf()
766 relo->name, kdesc->ref); in emit_relo_kfunc_btf()
769 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo, in emit_ksym_relo_log() argument
772 if (!gen->log_level) in emit_ksym_relo_log()
774 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, in emit_ksym_relo_log()
776 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) + in emit_ksym_relo_log()
778 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d", in emit_ksym_relo_log()
779 relo->is_typeless, relo->is_weak, relo->name, ref); in emit_ksym_relo_log()
780 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); in emit_ksym_relo_log()
781 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg", in emit_ksym_relo_log()
782 relo->is_typeless, relo->is_weak, relo->name, ref); in emit_ksym_relo_log()
786 * BPF_REG_8 - pointer to instruction
788 static void emit_relo_ksym_typeless(struct bpf_gen *gen, in emit_relo_ksym_typeless() argument
793 kdesc = get_ksym_desc(gen, relo); in emit_relo_ksym_typeless()
797 if (kdesc->ref > 1) { in emit_relo_ksym_typeless()
798 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_typeless()
799 kdesc->insn + offsetof(struct bpf_insn, imm)); in emit_relo_ksym_typeless()
800 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_typeless()
801 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); in emit_relo_ksym_typeless()
805 kdesc->insn = insn; in emit_relo_ksym_typeless()
807 kdesc->typeless = true; in emit_relo_ksym_typeless()
808 emit_bpf_kallsyms_lookup_name(gen, relo); in emit_relo_ksym_typeless()
809 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1)); in emit_relo_ksym_typeless()
810 emit_check_err(gen); in emit_relo_ksym_typeless()
812 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm))); in emit_relo_ksym_typeless()
814 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); in emit_relo_ksym_typeless()
815 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, in emit_relo_ksym_typeless()
818 emit_ksym_relo_log(gen, relo, kdesc->ref); in emit_relo_ksym_typeless()
821 static __u32 src_reg_mask(struct bpf_gen *gen) in src_reg_mask() argument
824 return gen->swapped_endian ? 0xf0 : 0x0f; in src_reg_mask()
826 return gen->swapped_endian ? 0x0f : 0xf0; in src_reg_mask()
833 * BPF_REG_8 - pointer to instruction
835 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) in emit_relo_ksym_btf() argument
840 kdesc = get_ksym_desc(gen, relo); in emit_relo_ksym_btf()
844 if (kdesc->ref > 1) { in emit_relo_ksym_btf()
845 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_btf()
846 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); in emit_relo_ksym_btf()
847 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, in emit_relo_ksym_btf()
848 kdesc->insn + offsetof(struct bpf_insn, imm)); in emit_relo_ksym_btf()
852 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3)); in emit_relo_ksym_btf()
856 kdesc->insn = insn; in emit_relo_ksym_btf()
857 emit_bpf_find_by_name_kind(gen, relo); in emit_relo_ksym_btf()
858 if (!relo->is_weak) in emit_relo_ksym_btf()
859 emit_check_err(gen); in emit_relo_ksym_btf()
861 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); in emit_relo_ksym_btf()
863 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); in emit_relo_ksym_btf()
864 …emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0… in emit_relo_ksym_btf()
866 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4)); in emit_relo_ksym_btf()
868 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); in emit_relo_ksym_btf()
870 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); in emit_relo_ksym_btf()
871 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, in emit_relo_ksym_btf()
874 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3)); in emit_relo_ksym_btf()
877 reg_mask = src_reg_mask(gen); in emit_relo_ksym_btf()
878 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); in emit_relo_ksym_btf()
879 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask)); in emit_relo_ksym_btf()
880 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code))); in emit_relo_ksym_btf()
882 emit_ksym_relo_log(gen, relo, kdesc->ref); in emit_relo_ksym_btf()
885 void bpf_gen__record_relo_core(struct bpf_gen *gen, in bpf_gen__record_relo_core() argument
890 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos)); in bpf_gen__record_relo_core()
892 gen->error = -ENOMEM; in bpf_gen__record_relo_core()
895 gen->core_relos = relos; in bpf_gen__record_relo_core()
896 relos += gen->core_relo_cnt; in bpf_gen__record_relo_core()
898 gen->core_relo_cnt++; in bpf_gen__record_relo_core()
901 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) in emit_relo() argument
905 pr_debug("gen: emit_relo (%d): %s at %d %s\n", in emit_relo()
906 relo->kind, relo->name, relo->insn_idx, relo->is_ld64 ? "ld64" : "call"); in emit_relo()
907 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx; in emit_relo()
908 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn)); in emit_relo()
909 if (relo->is_ld64) { in emit_relo()
910 if (relo->is_typeless) in emit_relo()
911 emit_relo_ksym_typeless(gen, relo, insn); in emit_relo()
913 emit_relo_ksym_btf(gen, relo, insn); in emit_relo()
915 emit_relo_kfunc_btf(gen, relo, insn); in emit_relo()
919 static void emit_relos(struct bpf_gen *gen, int insns) in emit_relos() argument
923 for (i = 0; i < gen->relo_cnt; i++) in emit_relos()
924 emit_relo(gen, gen->relos + i, insns); in emit_relos()
927 static void cleanup_core_relo(struct bpf_gen *gen) in cleanup_core_relo() argument
929 if (!gen->core_relo_cnt) in cleanup_core_relo()
931 free(gen->core_relos); in cleanup_core_relo()
932 gen->core_relo_cnt = 0; in cleanup_core_relo()
933 gen->core_relos = NULL; in cleanup_core_relo()
936 static void cleanup_relos(struct bpf_gen *gen, int insns) in cleanup_relos() argument
941 for (i = 0; i < gen->nr_ksyms; i++) { in cleanup_relos()
942 kdesc = &gen->ksyms[i]; in cleanup_relos()
944 if (kdesc->is_ld64 && !kdesc->typeless) { in cleanup_relos()
946 insn = kdesc->insn; in cleanup_relos()
948 emit_sys_close_blob(gen, insn); in cleanup_relos()
949 } else if (!kdesc->is_ld64) { in cleanup_relos()
950 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off)); in cleanup_relos()
951 if (kdesc->off < MAX_FD_ARRAY_SZ) in cleanup_relos()
952 gen->nr_fd_array--; in cleanup_relos()
955 if (gen->nr_ksyms) { in cleanup_relos()
956 free(gen->ksyms); in cleanup_relos()
957 gen->nr_ksyms = 0; in cleanup_relos()
958 gen->ksyms = NULL; in cleanup_relos()
960 if (gen->relo_cnt) { in cleanup_relos()
961 free(gen->relos); in cleanup_relos()
962 gen->relo_cnt = 0; in cleanup_relos()
963 gen->relos = NULL; in cleanup_relos()
965 cleanup_core_relo(gen); in cleanup_relos()
969 static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info, in info_blob_bswap() argument
972 struct bpf_func_info *fi = gen->data_start + func_info; in info_blob_bswap()
973 struct bpf_line_info *li = gen->data_start + line_info; in info_blob_bswap()
974 struct bpf_core_relo *cr = gen->data_start + core_relos; in info_blob_bswap()
977 for (i = 0; i < load_attr->func_info_cnt; i++) in info_blob_bswap()
980 for (i = 0; i < load_attr->line_info_cnt; i++) in info_blob_bswap()
983 for (i = 0; i < gen->core_relo_cnt; i++) in info_blob_bswap()
987 void bpf_gen__prog_load(struct bpf_gen *gen, in bpf_gen__prog_load() argument
992 int func_info_tot_sz = load_attr->func_info_cnt * in bpf_gen__prog_load()
993 load_attr->func_info_rec_size; in bpf_gen__prog_load()
994 int line_info_tot_sz = load_attr->line_info_cnt * in bpf_gen__prog_load()
995 load_attr->line_info_rec_size; in bpf_gen__prog_load()
996 int core_relo_tot_sz = gen->core_relo_cnt * in bpf_gen__prog_load()
1004 license_off = add_data(gen, license, strlen(license) + 1); in bpf_gen__prog_load()
1006 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); in bpf_gen__prog_load()
1007 pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n", in bpf_gen__prog_load()
1011 if (gen->swapped_endian) { in bpf_gen__prog_load()
1012 struct bpf_insn *insn = gen->data_start + insns_off; in bpf_gen__prog_load()
1020 attr.expected_attach_type = tgt_endian(load_attr->expected_attach_type); in bpf_gen__prog_load()
1021 attr.attach_btf_id = tgt_endian(load_attr->attach_btf_id); in bpf_gen__prog_load()
1022 attr.prog_ifindex = tgt_endian(load_attr->prog_ifindex); in bpf_gen__prog_load()
1025 attr.prog_flags = tgt_endian(load_attr->prog_flags); in bpf_gen__prog_load()
1027 attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size); in bpf_gen__prog_load()
1028 attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt); in bpf_gen__prog_load()
1029 func_info = add_data(gen, load_attr->func_info, func_info_tot_sz); in bpf_gen__prog_load()
1030 pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n", in bpf_gen__prog_load()
1031 func_info, load_attr->func_info_cnt, in bpf_gen__prog_load()
1032 load_attr->func_info_rec_size); in bpf_gen__prog_load()
1034 attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size); in bpf_gen__prog_load()
1035 attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt); in bpf_gen__prog_load()
1036 line_info = add_data(gen, load_attr->line_info, line_info_tot_sz); in bpf_gen__prog_load()
1037 pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n", in bpf_gen__prog_load()
1038 line_info, load_attr->line_info_cnt, in bpf_gen__prog_load()
1039 load_attr->line_info_rec_size); in bpf_gen__prog_load()
1042 attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt); in bpf_gen__prog_load()
1043 core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz); in bpf_gen__prog_load()
1044 pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n", in bpf_gen__prog_load()
1045 core_relos, gen->core_relo_cnt, in bpf_gen__prog_load()
1049 if (gen->swapped_endian) in bpf_gen__prog_load()
1050 info_blob_bswap(gen, func_info, line_info, core_relos, load_attr); in bpf_gen__prog_load()
1053 prog_load_attr = add_data(gen, &attr, attr_size); in bpf_gen__prog_load()
1054 pr_debug("gen: prog_load: attr: off %d size %d\n", in bpf_gen__prog_load()
1058 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off); in bpf_gen__prog_load()
1061 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off); in bpf_gen__prog_load()
1064 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); in bpf_gen__prog_load()
1067 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); in bpf_gen__prog_load()
1070 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos); in bpf_gen__prog_load()
1073 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array); in bpf_gen__prog_load()
1076 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4, in bpf_gen__prog_load()
1078 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4, in bpf_gen__prog_load()
1080 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8, in bpf_gen__prog_load()
1083 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4, in bpf_gen__prog_load()
1085 if (gen->attach_kind) { in bpf_gen__prog_load()
1086 emit_find_attach_target(gen); in bpf_gen__prog_load()
1088 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__prog_load()
1090 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in bpf_gen__prog_load()
1092 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); in bpf_gen__prog_load()
1093 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, in bpf_gen__prog_load()
1096 emit_relos(gen, insns_off); in bpf_gen__prog_load()
1098 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); in bpf_gen__prog_load()
1099 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); in bpf_gen__prog_load()
1101 cleanup_relos(gen, insns_off); in bpf_gen__prog_load()
1102 if (gen->attach_kind) { in bpf_gen__prog_load()
1103 emit_sys_close_blob(gen, in bpf_gen__prog_load()
1105 gen->attach_kind = 0; in bpf_gen__prog_load()
1107 emit_check_err(gen); in bpf_gen__prog_load()
1109 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, in bpf_gen__prog_load()
1110 stack_off(prog_fd[gen->nr_progs]))); in bpf_gen__prog_load()
1111 gen->nr_progs++; in bpf_gen__prog_load()
1114 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, in bpf_gen__map_update_elem() argument
1124 value = add_data(gen, pvalue, value_size); in bpf_gen__map_update_elem()
1125 key = add_data(gen, &zero, sizeof(zero)); in bpf_gen__map_update_elem()
1128 * if (ctx->flags & BPF_SKEL_KERNEL) in bpf_gen__map_update_elem()
1134 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, in bpf_gen__map_update_elem()
1138 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); in bpf_gen__map_update_elem()
1139 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, in bpf_gen__map_update_elem()
1141 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); in bpf_gen__map_update_elem()
1142 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, in bpf_gen__map_update_elem()
1144 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); in bpf_gen__map_update_elem()
1145 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); in bpf_gen__map_update_elem()
1146 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); in bpf_gen__map_update_elem()
1147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); in bpf_gen__map_update_elem()
1149 map_update_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_update_elem()
1150 pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n", in bpf_gen__map_update_elem()
1152 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, in bpf_gen__map_update_elem()
1153 blob_fd_array_off(gen, map_idx)); in bpf_gen__map_update_elem()
1154 emit_rel_store(gen, attr_field(map_update_attr, key), key); in bpf_gen__map_update_elem()
1155 emit_rel_store(gen, attr_field(map_update_attr, value), value); in bpf_gen__map_update_elem()
1157 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); in bpf_gen__map_update_elem()
1158 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size); in bpf_gen__map_update_elem()
1159 emit_check_err(gen); in bpf_gen__map_update_elem()
1162 void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, in bpf_gen__populate_outer_map() argument
1173 key = add_data(gen, &tgt_slot, sizeof(tgt_slot)); in bpf_gen__populate_outer_map()
1175 map_update_attr = add_data(gen, &attr, attr_size); in bpf_gen__populate_outer_map()
1176 pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n", in bpf_gen__populate_outer_map()
1178 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, in bpf_gen__populate_outer_map()
1179 blob_fd_array_off(gen, outer_map_idx)); in bpf_gen__populate_outer_map()
1180 emit_rel_store(gen, attr_field(map_update_attr, key), key); in bpf_gen__populate_outer_map()
1181 emit_rel_store(gen, attr_field(map_update_attr, value), in bpf_gen__populate_outer_map()
1182 blob_fd_array_off(gen, inner_map_idx)); in bpf_gen__populate_outer_map()
1185 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); in bpf_gen__populate_outer_map()
1186 debug_ret(gen, "populate_outer_map outer %d key %d inner %d", in bpf_gen__populate_outer_map()
1188 emit_check_err(gen); in bpf_gen__populate_outer_map()
1191 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) in bpf_gen__map_freeze() argument
1198 map_freeze_attr = add_data(gen, &attr, attr_size); in bpf_gen__map_freeze()
1199 pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n", in bpf_gen__map_freeze()
1201 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4, in bpf_gen__map_freeze()
1202 blob_fd_array_off(gen, map_idx)); in bpf_gen__map_freeze()
1204 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size); in bpf_gen__map_freeze()
1205 debug_ret(gen, "map_freeze"); in bpf_gen__map_freeze()
1206 emit_check_err(gen); in bpf_gen__map_freeze()