Lines Matching +full:0 +full:xc2000000
61 #define SEEN_MEM BIT(0) /* use mem[] for temporary storage */
66 #define NVREGS 0xffc0 /* %r6-%r15 */
71 #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */
75 #define REG_0 REG_W0 /* Register 0 */
106 [REG_W0] = 0,
177 unsigned int __disp = (disp) & 0xfff; \
191 unsigned int __imm = (imm) & 0xffff; \
198 long __pcrel = ((pcrel) >> 1) & 0xffff; \
205 _EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \
219 unsigned int __disp = (disp) & 0xfff; \
226 unsigned int __disp_h = _disp & 0xff000; \
227 unsigned int __disp_l = _disp & 0x00fff; \
243 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \
253 (rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \
255 BUILD_BUG_ON(((unsigned long) (imm)) > 0xff); \
261 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\
269 _EMIT6((op) | reg_high(b) << 16 | rel >> 16, rel & 0xffff);\
276 _EMIT6((op) | rel >> 16, rel & 0xffff); \
287 _EMIT6((op) | (__imm >> 16), __imm & 0xffff); \
332 EMIT4(0xb9160000, b1, b1); \
343 return jit->size == 0; in is_first_pass()
403 memset(area, 0, size); in jit_fill_hole()
415 _EMIT6(0xe300f000 | rs << 20 | off, 0x0024); in save_regs()
418 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off); in save_regs()
433 _EMIT6(0xe300f000 | rs << 20 | off, 0x0004); in restore_regs()
436 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off); in restore_regs()
450 return 0; in get_start()
468 #define REGS_RESTORE 0
505 /* brcl 0xf,size */ in bpf_skip()
506 EMIT6_PCREL_RIL(0xc0f4000000, size); in bpf_skip()
509 /* brc 0xf,size */ in bpf_skip()
510 EMIT4_PCREL(0xa7f40000, size); in bpf_skip()
514 /* bcr 0,%0 */ in bpf_skip()
515 _EMIT2(0x0700); in bpf_skip()
538 "bpf_plt_ret: .quad 0\n"
539 "bpf_plt_target: .quad 0\n"
560 /* brcl 0,prologue_plt */ in bpf_jit_prologue()
561 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); in bpf_jit_prologue()
567 _EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT); in bpf_jit_prologue()
586 EMIT4(0xb9040000, REG_15, REG_3); in bpf_jit_prologue()
591 fp->aux->exception_boundary ? NVREGS : 0); in bpf_jit_prologue()
597 /* basr %l,0 */ in bpf_jit_prologue()
598 EMIT2(0x0d00, REG_L, REG_0); in bpf_jit_prologue()
602 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start); in bpf_jit_prologue()
610 EMIT4(0xb9040000, REG_W1, REG_15); in bpf_jit_prologue()
612 EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED); in bpf_jit_prologue()
614 EMIT4_IMM(0xa70b0000, REG_15, -(STK_OFF + stack_depth)); in bpf_jit_prologue()
617 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, in bpf_jit_prologue()
628 EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10); in emit_expoline()
630 EMIT4_PCREL(0xa7f40000, 0); in emit_expoline()
642 _EMIT2(0x07f1); in emit_r1_thunk()
653 EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip); in call_r1()
656 EMIT2(0x0d00, REG_14, REG_1); in call_r1()
666 EMIT4(0xb9040000, REG_2, BPF_REG_0); in bpf_jit_epilogue()
668 save_restore_regs(jit, REGS_RESTORE, stack_depth, 0); in bpf_jit_epilogue()
675 _EMIT2(0x07fe); in bpf_jit_epilogue()
692 regs->gprs[x->data] = 0; in ex_handler_bpf()
728 /* bcr 0,%0 */ in bpf_jit_probe_emit_nop()
729 _EMIT2(0x0700); in bpf_jit_probe_emit_nop()
742 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_load_pre()
756 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_store_pre()
769 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_atomic_pre()
771 EMIT4(0xb9080000, REG_W1, insn->dst_reg); in bpf_jit_probe_atomic_pre()
786 return 0; in bpf_jit_probe_post()
790 return 0; in bpf_jit_probe_post()
795 for (i = 0; i < 2; i++) { in bpf_jit_probe_post()
801 prg = i == 0 ? probe->prg : probe->nop_prg; in bpf_jit_probe_post()
821 return 0; in bpf_jit_probe_post()
830 return 0; in sign_extend()
835 EMIT4(0xb9060000, r, r); in sign_extend()
836 return 0; in sign_extend()
839 EMIT4(0xb9070000, r, r); in sign_extend()
840 return 0; in sign_extend()
843 EMIT4(0xb9140000, r, r); in sign_extend()
844 return 0; in sign_extend()
846 return 0; in sign_extend()
881 case 0: /* DST = (u32) SRC */ in bpf_jit_insn()
883 EMIT4(0xb9160000, dst_reg, src_reg); in bpf_jit_insn()
889 EMIT4(0xb9260000, dst_reg, src_reg); in bpf_jit_insn()
891 EMIT4(0xb9160000, dst_reg, dst_reg); in bpf_jit_insn()
895 EMIT4(0xb9270000, dst_reg, src_reg); in bpf_jit_insn()
897 EMIT4(0xb9160000, dst_reg, dst_reg); in bpf_jit_insn()
906 EMIT4(0xb9020000, dst_reg, src_reg); in bpf_jit_insn()
907 /* brc 8,0f */ in bpf_jit_insn()
909 EMIT4_PCREL_RIC(0xa7040000, 8, 0); in bpf_jit_insn()
911 EMIT6_IMM(0xc0080000, dst_reg, jit->user_arena >> 32); in bpf_jit_insn()
912 /* 0: */ in bpf_jit_insn()
919 case 0: /* DST = SRC */ in bpf_jit_insn()
921 EMIT4(0xb9040000, dst_reg, src_reg); in bpf_jit_insn()
925 EMIT4(0xb9060000, dst_reg, src_reg); in bpf_jit_insn()
929 EMIT4(0xb9070000, dst_reg, src_reg); in bpf_jit_insn()
933 EMIT4(0xb9140000, dst_reg, src_reg); in bpf_jit_insn()
939 EMIT6_IMM(0xc00f0000, dst_reg, imm); in bpf_jit_insn()
945 EMIT6_IMM(0xc0010000, dst_reg, imm); in bpf_jit_insn()
955 imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32; in bpf_jit_insn()
957 EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64)); in bpf_jit_insn()
966 EMIT2(0x1a00, dst_reg, src_reg); in bpf_jit_insn()
971 EMIT4(0xb9080000, dst_reg, src_reg); in bpf_jit_insn()
974 if (imm != 0) { in bpf_jit_insn()
976 EMIT6_IMM(0xc20b0000, dst_reg, imm); in bpf_jit_insn()
984 EMIT6_IMM(0xc2080000, dst_reg, imm); in bpf_jit_insn()
991 EMIT2(0x1b00, dst_reg, src_reg); in bpf_jit_insn()
996 EMIT4(0xb9090000, dst_reg, src_reg); in bpf_jit_insn()
999 if (imm != 0) { in bpf_jit_insn()
1001 EMIT6_IMM(0xc20b0000, dst_reg, -imm); in bpf_jit_insn()
1008 if (imm == -0x80000000) { in bpf_jit_insn()
1009 /* algfi %dst,0x80000000 */ in bpf_jit_insn()
1010 EMIT6_IMM(0xc20a0000, dst_reg, 0x80000000); in bpf_jit_insn()
1013 EMIT6_IMM(0xc2080000, dst_reg, -imm); in bpf_jit_insn()
1021 EMIT4(0xb2520000, dst_reg, src_reg); in bpf_jit_insn()
1026 EMIT4(0xb90c0000, dst_reg, src_reg); in bpf_jit_insn()
1031 EMIT6_IMM(0xc2010000, dst_reg, imm); in bpf_jit_insn()
1039 EMIT6_IMM(0xc2000000, dst_reg, imm); in bpf_jit_insn()
1050 case 0: /* dst = (u32) dst {/,%} (u32) src */ in bpf_jit_insn()
1052 EMIT2(0x1700, REG_W0, REG_W0); in bpf_jit_insn()
1054 EMIT2(0x1800, REG_W1, dst_reg); in bpf_jit_insn()
1056 EMIT4(0xb9970000, REG_W0, src_reg); in bpf_jit_insn()
1060 EMIT4(0xb9140000, REG_W1, dst_reg); in bpf_jit_insn()
1062 EMIT4(0xb91d0000, REG_W0, src_reg); in bpf_jit_insn()
1066 EMIT4(0xb9160000, dst_reg, rc_reg); in bpf_jit_insn()
1077 case 0: /* dst = dst {/,%} src */ in bpf_jit_insn()
1078 /* lghi %w0,0 */ in bpf_jit_insn()
1079 EMIT4_IMM(0xa7090000, REG_W0, 0); in bpf_jit_insn()
1081 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1083 EMIT4(0xb9870000, REG_W0, src_reg); in bpf_jit_insn()
1087 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1089 EMIT4(0xb90d0000, REG_W0, src_reg); in bpf_jit_insn()
1093 EMIT4(0xb9040000, dst_reg, rc_reg); in bpf_jit_insn()
1103 /* lghi %dst,0 */ in bpf_jit_insn()
1104 EMIT4_IMM(0xa7090000, dst_reg, 0); in bpf_jit_insn()
1111 case 0: /* dst = (u32) dst {/,%} (u32) imm */ in bpf_jit_insn()
1113 EMIT2(0x1700, REG_W0, REG_W0); in bpf_jit_insn()
1115 EMIT2(0x1800, REG_W1, dst_reg); in bpf_jit_insn()
1117 EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, in bpf_jit_insn()
1122 EMIT4(0xb9140000, REG_W1, dst_reg); in bpf_jit_insn()
1124 EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0, in bpf_jit_insn()
1130 case 0: /* dst = (u32) dst {/,%} (u32) imm */ in bpf_jit_insn()
1132 EMIT2(0x1700, REG_W0, REG_W0); in bpf_jit_insn()
1134 EMIT2(0x1800, REG_W1, dst_reg); in bpf_jit_insn()
1136 EMIT6_PCREL_RILB(0xc40d0000, dst_reg, in bpf_jit_insn()
1140 EMIT4(0xb9970000, REG_W0, dst_reg); in bpf_jit_insn()
1144 EMIT4(0xb9140000, REG_W1, dst_reg); in bpf_jit_insn()
1146 EMIT6_PCREL_RILB(0xc40c0000, dst_reg, in bpf_jit_insn()
1150 EMIT4(0xb90d0000, REG_W0, dst_reg); in bpf_jit_insn()
1155 EMIT4(0xb9160000, dst_reg, rc_reg); in bpf_jit_insn()
1167 /* lhgi %dst,0 */ in bpf_jit_insn()
1168 EMIT4_IMM(0xa7090000, dst_reg, 0); in bpf_jit_insn()
1173 case 0: /* dst = dst {/,%} imm */ in bpf_jit_insn()
1174 /* lghi %w0,0 */ in bpf_jit_insn()
1175 EMIT4_IMM(0xa7090000, REG_W0, 0); in bpf_jit_insn()
1177 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1179 EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, in bpf_jit_insn()
1184 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1186 EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0, in bpf_jit_insn()
1192 case 0: /* dst = dst {/,%} imm */ in bpf_jit_insn()
1193 /* lghi %w0,0 */ in bpf_jit_insn()
1194 EMIT4_IMM(0xa7090000, REG_W0, 0); in bpf_jit_insn()
1196 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1198 EMIT6_PCREL_RILB(0xc4080000, dst_reg, in bpf_jit_insn()
1202 EMIT4(0xb9870000, REG_W0, dst_reg); in bpf_jit_insn()
1206 EMIT4(0xb9040000, REG_W1, dst_reg); in bpf_jit_insn()
1208 EMIT6_PCREL_RILB(0xc4080000, dst_reg, in bpf_jit_insn()
1212 EMIT4(0xb90d0000, REG_W0, dst_reg); in bpf_jit_insn()
1217 EMIT4(0xb9040000, dst_reg, rc_reg); in bpf_jit_insn()
1225 EMIT2(0x1400, dst_reg, src_reg); in bpf_jit_insn()
1230 EMIT4(0xb9800000, dst_reg, src_reg); in bpf_jit_insn()
1234 EMIT6_IMM(0xc00b0000, dst_reg, imm); in bpf_jit_insn()
1240 EMIT6_DISP_LH(0xe3000000, 0x0080, in bpf_jit_insn()
1245 EMIT6_PCREL_RILB(0xc4080000, REG_W0, in bpf_jit_insn()
1249 EMIT4(0xb9800000, dst_reg, REG_W0); in bpf_jit_insn()
1257 EMIT2(0x1600, dst_reg, src_reg); in bpf_jit_insn()
1262 EMIT4(0xb9810000, dst_reg, src_reg); in bpf_jit_insn()
1266 EMIT6_IMM(0xc00d0000, dst_reg, imm); in bpf_jit_insn()
1272 EMIT6_DISP_LH(0xe3000000, 0x0081, in bpf_jit_insn()
1277 EMIT6_PCREL_RILB(0xc4080000, REG_W0, in bpf_jit_insn()
1281 EMIT4(0xb9810000, dst_reg, REG_W0); in bpf_jit_insn()
1289 EMIT2(0x1700, dst_reg, src_reg); in bpf_jit_insn()
1294 EMIT4(0xb9820000, dst_reg, src_reg); in bpf_jit_insn()
1297 if (imm != 0) { in bpf_jit_insn()
1299 EMIT6_IMM(0xc0070000, dst_reg, imm); in bpf_jit_insn()
1306 EMIT6_DISP_LH(0xe3000000, 0x0082, in bpf_jit_insn()
1311 EMIT6_PCREL_RILB(0xc4080000, REG_W0, in bpf_jit_insn()
1315 EMIT4(0xb9820000, dst_reg, REG_W0); in bpf_jit_insn()
1322 /* sll %dst,0(%src) */ in bpf_jit_insn()
1323 EMIT4_DISP(0x89000000, dst_reg, src_reg, 0); in bpf_jit_insn()
1327 /* sllg %dst,%dst,0(%src) */ in bpf_jit_insn()
1328 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0); in bpf_jit_insn()
1331 if (imm != 0) { in bpf_jit_insn()
1333 EMIT4_DISP(0x89000000, dst_reg, REG_0, imm); in bpf_jit_insn()
1338 if (imm == 0) in bpf_jit_insn()
1341 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm); in bpf_jit_insn()
1347 /* srl %dst,0(%src) */ in bpf_jit_insn()
1348 EMIT4_DISP(0x88000000, dst_reg, src_reg, 0); in bpf_jit_insn()
1352 /* srlg %dst,%dst,0(%src) */ in bpf_jit_insn()
1353 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0); in bpf_jit_insn()
1356 if (imm != 0) { in bpf_jit_insn()
1358 EMIT4_DISP(0x88000000, dst_reg, REG_0, imm); in bpf_jit_insn()
1363 if (imm == 0) in bpf_jit_insn()
1366 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm); in bpf_jit_insn()
1372 /* sra %dst,%dst,0(%src) */ in bpf_jit_insn()
1373 EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0); in bpf_jit_insn()
1377 /* srag %dst,%dst,0(%src) */ in bpf_jit_insn()
1378 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0); in bpf_jit_insn()
1381 if (imm != 0) { in bpf_jit_insn()
1383 EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm); in bpf_jit_insn()
1388 if (imm == 0) in bpf_jit_insn()
1391 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm); in bpf_jit_insn()
1398 EMIT2(0x1300, dst_reg, dst_reg); in bpf_jit_insn()
1403 EMIT4(0xb9030000, dst_reg, dst_reg); in bpf_jit_insn()
1413 EMIT4(0xb9850000, dst_reg, dst_reg); in bpf_jit_insn()
1420 EMIT4(0xb9160000, dst_reg, dst_reg); in bpf_jit_insn()
1431 EMIT4(0xb91f0000, dst_reg, dst_reg); in bpf_jit_insn()
1433 EMIT4_DISP(0x88000000, dst_reg, REG_0, 16); in bpf_jit_insn()
1435 EMIT4(0xb9850000, dst_reg, dst_reg); in bpf_jit_insn()
1441 EMIT4(0xb91f0000, dst_reg, dst_reg); in bpf_jit_insn()
1444 EMIT4(0xb9160000, dst_reg, dst_reg); in bpf_jit_insn()
1448 EMIT4(0xb90f0000, dst_reg, dst_reg); in bpf_jit_insn()
1464 EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, in bpf_jit_insn()
1467 if (err < 0) in bpf_jit_insn()
1475 EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, in bpf_jit_insn()
1478 if (err < 0) in bpf_jit_insn()
1486 EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, in bpf_jit_insn()
1489 if (err < 0) in bpf_jit_insn()
1497 EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, in bpf_jit_insn()
1500 if (err < 0) in bpf_jit_insn()
1507 EMIT4_IMM(0xa7080000, REG_W0, (u8) imm); in bpf_jit_insn()
1510 EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, in bpf_jit_insn()
1513 if (err < 0) in bpf_jit_insn()
1520 EMIT4_IMM(0xa7080000, REG_W0, (u16) imm); in bpf_jit_insn()
1523 EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, in bpf_jit_insn()
1526 if (err < 0) in bpf_jit_insn()
1533 EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm); in bpf_jit_insn()
1536 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, in bpf_jit_insn()
1539 if (err < 0) in bpf_jit_insn()
1546 EMIT6_IMM(0xc0010000, REG_W0, imm); in bpf_jit_insn()
1549 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, in bpf_jit_insn()
1552 if (err < 0) in bpf_jit_insn()
1579 EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \ in bpf_jit_insn()
1583 if (err < 0) \ in bpf_jit_insn()
1586 /* bcr 14,0 - see atomic_fetch_{add,and,or,xor}() */ \ in bpf_jit_insn()
1587 _EMIT2(0x07e0); \ in bpf_jit_insn()
1591 } while (0) in bpf_jit_insn()
1595 EMIT_ATOMIC(0x00fa, 0x00ea); in bpf_jit_insn()
1600 EMIT_ATOMIC(0x00f4, 0x00e4); in bpf_jit_insn()
1605 EMIT_ATOMIC(0x00f6, 0x00e6); in bpf_jit_insn()
1610 EMIT_ATOMIC(0x00f7, 0x00e7); in bpf_jit_insn()
1619 EMIT6_DISP_LH(0xe3000000, in bpf_jit_insn()
1620 is32 ? 0x0058 : 0x0004, REG_W0, REG_0, in bpf_jit_insn()
1629 /* 0: {csy|csg} %w0,%src,off(%arena) */ in bpf_jit_insn()
1630 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, in bpf_jit_insn()
1633 /* brc 4,0b */ in bpf_jit_insn()
1634 EMIT4_PCREL_RIC(0xa7040000, 4, loop_start); in bpf_jit_insn()
1636 EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0); in bpf_jit_insn()
1639 if (err < 0) in bpf_jit_insn()
1642 if (err < 0) in bpf_jit_insn()
1650 /* 0: {csy|csg} %b0,%src,off(%arena) */ in bpf_jit_insn()
1651 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, in bpf_jit_insn()
1655 if (err < 0) in bpf_jit_insn()
1674 EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, in bpf_jit_insn()
1677 if (err < 0) in bpf_jit_insn()
1687 EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off); in bpf_jit_insn()
1689 if (err < 0) in bpf_jit_insn()
1698 EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, in bpf_jit_insn()
1701 if (err < 0) in bpf_jit_insn()
1711 EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off); in bpf_jit_insn()
1713 if (err < 0) in bpf_jit_insn()
1723 EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, in bpf_jit_insn()
1726 if (err < 0) in bpf_jit_insn()
1736 EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off); in bpf_jit_insn()
1738 if (err < 0) in bpf_jit_insn()
1747 EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, in bpf_jit_insn()
1750 if (err < 0) in bpf_jit_insn()
1765 if (ret < 0) in bpf_jit_insn()
1781 _EMIT6(0xd203f000 | STK_OFF_TCCNT, in bpf_jit_insn()
1782 0xf000 | (STK_OFF_TCCNT + STK_OFF + stack_depth)); in bpf_jit_insn()
1790 for (j = 0; j < m->nr_args; j++) { in bpf_jit_insn()
1799 EMIT6_PCREL_RILB(0xc4080000, REG_W1, _EMIT_CONST_U64(func)); in bpf_jit_insn()
1803 EMIT4(0xb9040000, BPF_REG_0, REG_2); in bpf_jit_insn()
1820 EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2, in bpf_jit_insn()
1823 /* clrj %b3,%w1,0xa,out */ in bpf_jit_insn()
1825 EMIT6_PCREL_RIEB(0xec000000, 0x0077, BPF_REG_3, REG_W1, 0xa, in bpf_jit_insn()
1838 EMIT4_IMM(0xa7080000, REG_W0, 1); in bpf_jit_insn()
1840 EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off); in bpf_jit_insn()
1841 /* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */ in bpf_jit_insn()
1843 EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT - 1, in bpf_jit_insn()
1853 EMIT4(0xb9160000, REG_1, BPF_REG_3); in bpf_jit_insn()
1855 EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, REG_1, REG_0, 3); in bpf_jit_insn()
1857 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_1, BPF_REG_2, in bpf_jit_insn()
1859 /* brc 0x8,out */ in bpf_jit_insn()
1861 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg); in bpf_jit_insn()
1866 save_restore_regs(jit, REGS_RESTORE, stack_depth, 0); in bpf_jit_insn()
1873 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0, in bpf_jit_insn()
1878 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start); in bpf_jit_insn()
1879 /* brcl 0xf,__s390_indirect_jump_r1 */ in bpf_jit_insn()
1880 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->r1_thunk_ip); in bpf_jit_insn()
1882 /* bc 0xf,tail_call_start(%r1) */ in bpf_jit_insn()
1883 _EMIT4(0x47f01000 + jit->tail_call_start); in bpf_jit_insn()
1897 last = (i == fp->len - 1) ? 1 : 0; in bpf_jit_insn()
1901 /* brc 0xf, <exit> */ in bpf_jit_insn()
1902 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip); in bpf_jit_insn()
1904 /* brcl 0xf, <exit> */ in bpf_jit_insn()
1905 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip); in bpf_jit_insn()
1915 * 0 | Operands equal | 8 in bpf_jit_insn()
1923 * For example for s390x with offset 0 we jump to the branch in bpf_jit_insn()
1924 * instruction itself (loop) and for BPF with offset 0 we in bpf_jit_insn()
1931 mask = 0xf000; /* j */ in bpf_jit_insn()
1935 mask = 0x2000; /* jh */ in bpf_jit_insn()
1939 mask = 0x4000; /* jl */ in bpf_jit_insn()
1943 mask = 0xa000; /* jhe */ in bpf_jit_insn()
1947 mask = 0xc000; /* jle */ in bpf_jit_insn()
1951 mask = 0x2000; /* jh */ in bpf_jit_insn()
1955 mask = 0x4000; /* jl */ in bpf_jit_insn()
1959 mask = 0xa000; /* jhe */ in bpf_jit_insn()
1963 mask = 0xc000; /* jle */ in bpf_jit_insn()
1967 mask = 0x7000; /* jne */ in bpf_jit_insn()
1971 mask = 0x8000; /* je */ in bpf_jit_insn()
1975 mask = 0x7000; /* jnz */ in bpf_jit_insn()
1978 EMIT6_IMM(0xc00f0000, REG_W1, imm); in bpf_jit_insn()
1980 EMIT2(0x1400, REG_W1, dst_reg); in bpf_jit_insn()
1983 EMIT6_IMM(0xc0010000, REG_W1, imm); in bpf_jit_insn()
1985 EMIT4(0xb9800000, REG_W1, dst_reg); in bpf_jit_insn()
1991 mask = 0x2000; /* jh */ in bpf_jit_insn()
1995 mask = 0x4000; /* jl */ in bpf_jit_insn()
1999 mask = 0xa000; /* jhe */ in bpf_jit_insn()
2003 mask = 0xc000; /* jle */ in bpf_jit_insn()
2007 mask = 0x2000; /* jh */ in bpf_jit_insn()
2011 mask = 0x4000; /* jl */ in bpf_jit_insn()
2015 mask = 0xa000; /* jhe */ in bpf_jit_insn()
2019 mask = 0xc000; /* jle */ in bpf_jit_insn()
2023 mask = 0x7000; /* jne */ in bpf_jit_insn()
2027 mask = 0x8000; /* je */ in bpf_jit_insn()
2034 mask = 0x7000; /* jnz */ in bpf_jit_insn()
2036 EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000), in bpf_jit_insn()
2042 EMIT6_IMM(is_jmp32 ? 0xc20d0000 : 0xc20c0000, in bpf_jit_insn()
2047 EMIT4_PCREL_RIC(0xa7040000, in bpf_jit_insn()
2051 EMIT6_PCREL_RILC(0xc0040000, in bpf_jit_insn()
2058 EMIT6_IMM(0xc0010000, src_reg, imm); in bpf_jit_insn()
2065 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064), in bpf_jit_insn()
2070 EMIT2(0x1900, dst_reg, src_reg); in bpf_jit_insn()
2072 EMIT4(0xb9200000, dst_reg, src_reg); in bpf_jit_insn()
2074 EMIT6_PCREL_RILC(0xc0040000, in bpf_jit_insn()
2083 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065), in bpf_jit_insn()
2088 EMIT2(0x1500, dst_reg, src_reg); in bpf_jit_insn()
2090 EMIT4(0xb9210000, dst_reg, src_reg); in bpf_jit_insn()
2092 EMIT6_PCREL_RILC(0xc0040000, in bpf_jit_insn()
2100 EMIT4_PCREL_RIC(0xa7040000, in bpf_jit_insn()
2105 EMIT6_PCREL_RILC(0xc0040000, in bpf_jit_insn()
2145 if (delta < 0) in bpf_set_addr()
2151 return 0; in bpf_set_addr()
2165 jit->prg = 0; in bpf_jit_prog()
2166 jit->excnt = 0; in bpf_jit_prog()
2174 if (bpf_set_addr(jit, 0) < 0) in bpf_jit_prog()
2176 for (i = 0; i < fp->len; i += insn_count) { in bpf_jit_prog()
2178 if (insn_count < 0) in bpf_jit_prog()
2181 if (bpf_set_addr(jit, i + insn_count) < 0) in bpf_jit_prog()
2202 return 0; in bpf_jit_prog()
2225 for (i = 0; i < fp->len; i++) { in bpf_jit_alloc()
2301 memset(&jit, 0, sizeof(jit)); in bpf_int_jit_compile()
2388 if (err < 0) in bpf_arch_text_poke()
2390 if (insn.opc != (0xc004 | (old_addr ? 0xf0 : 0))) in bpf_arch_text_poke()
2404 if (err < 0) in bpf_arch_text_poke()
2417 insn.opc = 0xc004 | (new_addr ? 0xf0 : 0); in bpf_arch_text_poke()
2423 return 0; in bpf_arch_text_poke()
2463 EMIT6_IMM(0xc00e0000, dst_reg, (val >> 32)); in load_imm64()
2465 EMIT6_IMM(0xc00d0000, dst_reg, val); in load_imm64()
2485 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, REG_0, REG_15, cookie_off); in invoke_bpf_prog()
2488 * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0) in invoke_bpf_prog()
2497 EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off); in invoke_bpf_prog()
2501 EMIT4(0xb9020000, REG_7, REG_2); in invoke_bpf_prog()
2504 EMIT6_PCREL_RILC(0xc0040000, 8, 0); in invoke_bpf_prog()
2513 EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off); in invoke_bpf_prog()
2523 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, in invoke_bpf_prog()
2540 EMIT4(0xb9040000, REG_3, REG_7); in invoke_bpf_prog()
2542 EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off); in invoke_bpf_prog()
2546 return 0; in invoke_bpf_prog()
2602 nr_bpf_args = 0; in __arch_prepare_bpf_trampoline()
2603 for (i = 0; i < m->nr_args; i++) { in __arch_prepare_bpf_trampoline()
2642 EMIT4(0xb9040000, REG_1, REG_15); in __arch_prepare_bpf_trampoline()
2644 EMIT4_IMM(0xa70b0000, REG_15, -tjit->stack_size); in __arch_prepare_bpf_trampoline()
2646 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2649 _EMIT6(0xd203f000 | tjit->tccnt_off, in __arch_prepare_bpf_trampoline()
2650 0xf000 | (tjit->stack_size + STK_OFF_TCCNT)); in __arch_prepare_bpf_trampoline()
2653 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2, in __arch_prepare_bpf_trampoline()
2656 for (i = 0, j = 0; i < m->nr_args; i++) { in __arch_prepare_bpf_trampoline()
2666 EMIT6_DISP_LH(0xe3000000, 0x0024, arg, in __arch_prepare_bpf_trampoline()
2670 _EMIT6(0xd207f000 | bpf_arg_off, in __arch_prepare_bpf_trampoline()
2671 0xf000 | arg); in __arch_prepare_bpf_trampoline()
2675 /* mvc bpf_arg_off(16,%r15),0(%arg) */ in __arch_prepare_bpf_trampoline()
2676 _EMIT6(0xd20ff000 | bpf_arg_off, in __arch_prepare_bpf_trampoline()
2680 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_0, in __arch_prepare_bpf_trampoline()
2682 /* mvc bpf_arg_off(16,%r15),0(%r1) */ in __arch_prepare_bpf_trampoline()
2683 _EMIT6(0xd20ff000 | bpf_arg_off, 0x1000); in __arch_prepare_bpf_trampoline()
2689 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_7, REG_8, REG_15, in __arch_prepare_bpf_trampoline()
2692 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_14, REG_0, REG_15, tjit->r14_off); in __arch_prepare_bpf_trampoline()
2703 EMIT4(0xb9040000, REG_8, REG_0); in __arch_prepare_bpf_trampoline()
2718 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2722 EMIT4_IMM(0xa7090000, REG_0, nr_bpf_args); in __arch_prepare_bpf_trampoline()
2724 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2740 for (i = 0; i < fentry->nr_links; i++) in __arch_prepare_bpf_trampoline()
2747 * retval = 0; in __arch_prepare_bpf_trampoline()
2751 _EMIT6(0xd707f000 | tjit->retval_off, in __arch_prepare_bpf_trampoline()
2752 0xf000 | tjit->retval_off); in __arch_prepare_bpf_trampoline()
2754 for (i = 0; i < fmod_ret->nr_links; i++) { in __arch_prepare_bpf_trampoline()
2764 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_0, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2767 EMIT6_PCREL_RILC(0xc0040000, 7, tjit->do_fexit); in __arch_prepare_bpf_trampoline()
2778 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, in __arch_prepare_bpf_trampoline()
2783 _EMIT6(0xd200f000 | in __arch_prepare_bpf_trampoline()
2786 0xf000 | tjit->orig_stack_args_off); in __arch_prepare_bpf_trampoline()
2788 _EMIT6(0xd203f000 | STK_OFF_TCCNT, 0xf000 | tjit->tccnt_off); in __arch_prepare_bpf_trampoline()
2790 EMIT4(0xb9040000, REG_1, REG_8); in __arch_prepare_bpf_trampoline()
2794 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2803 /* brcl 0,im->ip_epilogue */ in __arch_prepare_bpf_trampoline()
2804 EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); in __arch_prepare_bpf_trampoline()
2809 for (i = 0; i < fexit->nr_links; i++) in __arch_prepare_bpf_trampoline()
2830 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, in __arch_prepare_bpf_trampoline()
2835 EMIT4(0xb9040000, REG_1, REG_8); in __arch_prepare_bpf_trampoline()
2837 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_7, REG_8, REG_15, in __arch_prepare_bpf_trampoline()
2840 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_14, REG_0, REG_15, tjit->r14_off); in __arch_prepare_bpf_trampoline()
2843 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15, in __arch_prepare_bpf_trampoline()
2846 _EMIT6(0xd203f000 | (tjit->stack_size + STK_OFF_TCCNT), in __arch_prepare_bpf_trampoline()
2847 0xf000 | tjit->tccnt_off); in __arch_prepare_bpf_trampoline()
2849 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); in __arch_prepare_bpf_trampoline()
2855 _EMIT2(0x07fe); in __arch_prepare_bpf_trampoline()
2858 _EMIT2(0x07f1); in __arch_prepare_bpf_trampoline()
2862 return 0; in __arch_prepare_bpf_trampoline()
2872 memset(&tjit, 0, sizeof(tjit)); in arch_bpf_trampoline_size()
2877 return ret < 0 ? ret : tjit.common.prg; in arch_bpf_trampoline_size()
2889 memset(&tjit, 0, sizeof(tjit)); in arch_prepare_bpf_trampoline()
2893 if (ret < 0) in arch_prepare_bpf_trampoline()
2902 tjit.common.prg = 0; in arch_prepare_bpf_trampoline()
2907 return ret < 0 ? ret : tjit.common.prg; in arch_prepare_bpf_trampoline()
2941 unsigned long addr, prev_addr = 0; in arch_bpf_stack_walk()
2944 unwind_for_each_frame(&state, NULL, NULL, 0) { in arch_bpf_stack_walk()