283  * Emit an instruction that will be executed unconditionally.
285 static inline void emit(u32 inst, struct jit_ctx *ctx)  in emit()  function
472 	emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx);  in emit_mov_i_no8m()
474 	emit(ARM_MOVW(rd, val & 0xffff), ctx);  in emit_mov_i_no8m()
476 		emit(ARM_MOVT(rd, val >> 16), ctx);  in emit_mov_i_no8m()
485 		emit(ARM_MOV_I(rd, imm12), ctx);  in emit_mov_i()
493 		emit(ARM_BX(tgt_reg), ctx);  in emit_bx_r()
495 		emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx);  in emit_bx_r()
501 	emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx);  in emit_blx_r()
504 	emit(ARM_BLX_R(tgt_reg), ctx);  in emit_blx_r()
529 			emit(sign ? ARM_SDIV(rd, rm, rn) : ARM_UDIV(rd, rm, rn), ctx);  in emit_udivmod()
531 			emit(sign ? ARM_SDIV(ARM_IP, rm, rn) : ARM_UDIV(ARM_IP, rm, rn), ctx);  in emit_udivmod()
532 			emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx);  in emit_udivmod()
547 		emit(ARM_MOV_R(tmp[0], ARM_R1), ctx);  in emit_udivmod()
548 		emit(ARM_MOV_R(ARM_R1, rn), ctx);  in emit_udivmod()
551 		emit(ARM_MOV_R(tmp[1], ARM_R0), ctx);  in emit_udivmod()
552 		emit(ARM_MOV_R(ARM_R0, rm), ctx);  in emit_udivmod()
556 	emit(ARM_PUSH(CALLER_MASK & ~exclude_mask), ctx);  in emit_udivmod()
575 	emit(ARM_POP(CALLER_MASK & ~exclude_mask), ctx);  in emit_udivmod()
579 		emit(ARM_MOV_R(rd, ARM_R0), ctx);  in emit_udivmod()
583 		emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx);  in emit_udivmod()
585 		emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx);  in emit_udivmod()
594 	emit(ARM_PUSH(CALLER_MASK), ctx);  in emit_udivmod64()
607 				emit(ARM_PUSH(BIT(ARM_R0) | BIT(ARM_R1)), ctx);  in emit_udivmod64()
608 			emit(ARM_MOV_R(ARM_R1, rm[0]), ctx);  in emit_udivmod64()
609 			emit(ARM_MOV_R(ARM_R0, rm[1]), ctx);  in emit_udivmod64()
611 				emit(ARM_POP(BIT(ARM_R2) | BIT(ARM_R3)), ctx);  in emit_udivmod64()
619 			emit(ARM_MOV_R(ARM_R3, rn[0]), ctx);  in emit_udivmod64()
620 			emit(ARM_MOV_R(ARM_R2, rn[1]), ctx);  in emit_udivmod64()
644 		emit(ARM_MOV_R(rd[0], ARM_R1), ctx);  in emit_udivmod64()
645 		emit(ARM_MOV_R(rd[1], ARM_R0), ctx);  in emit_udivmod64()
650 		emit(ARM_POP(CALLER_MASK), ctx);  in emit_udivmod64()
652 		emit(ARM_POP(BIT(ARM_R0) | BIT(ARM_R1)), ctx);  in emit_udivmod64()
653 		emit(ARM_ADD_I(ARM_SP, ARM_SP, 8), ctx);  in emit_udivmod64()
655 		emit(ARM_ADD_I(ARM_SP, ARM_SP, 8), ctx);  in emit_udivmod64()
656 		emit(ARM_POP(BIT(ARM_R2) | BIT(ARM_R3)), ctx);  in emit_udivmod64()
673 		emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);  in arm_bpf_get_reg32()
685 			emit(ARM_LDRD_I(tmp[1], ARM_FP,  in arm_bpf_get_reg64()
688 			emit(ARM_LDR_I(tmp[1], ARM_FP,  in arm_bpf_get_reg64()
690 			emit(ARM_LDR_I(tmp[0], ARM_FP,  in arm_bpf_get_reg64()
705 		emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);  in arm_bpf_put_reg32()
707 		emit(ARM_MOV_R(reg, src), ctx);  in arm_bpf_put_reg32()
716 			emit(ARM_STRD_I(src[1], ARM_FP,  in arm_bpf_put_reg64()
719 			emit(ARM_STR_I(src[1], ARM_FP,  in arm_bpf_put_reg64()
721 			emit(ARM_STR_I(src[0], ARM_FP,  in arm_bpf_put_reg64()
726 			emit(ARM_MOV_R(reg[1], src[1]), ctx);  in arm_bpf_put_reg64()
728 			emit(ARM_MOV_R(reg[0], src[0]), ctx);  in arm_bpf_put_reg64()
776 		emit(ARM_ADDS_R(dst, dst, src), ctx);  in emit_a32_add_r()
778 		emit(ARM_ADC_R(dst, dst, src), ctx);  in emit_a32_add_r()
780 		emit(ARM_ADD_R(dst, dst, src), ctx);  in emit_a32_add_r()
793 		emit(ARM_SUBS_R(dst, dst, src), ctx);  in emit_a32_sub_r()
795 		emit(ARM_SBC_R(dst, dst, src), ctx);  in emit_a32_sub_r()
797 		emit(ARM_SUB_R(dst, dst, src), ctx);  in emit_a32_sub_r()
813 		emit(ARM_ORR_R(dst, dst, src), ctx);  in emit_alu_r()
817 		emit(ARM_AND_R(dst, dst, src), ctx);  in emit_alu_r()
821 		emit(ARM_EOR_R(dst, dst, src), ctx);  in emit_alu_r()
825 		emit(ARM_MUL(dst, dst, src), ctx);  in emit_alu_r()
829 		emit(ARM_LSL_R(dst, dst, src), ctx);  in emit_alu_r()
833 		emit(ARM_LSR_R(dst, dst, src), ctx);  in emit_alu_r()
837 		emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx);  in emit_alu_r()
899 		emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx);  in emit_a32_mov_r64()
900 		emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx);  in emit_a32_mov_r64()
902 		emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx);  in emit_a32_mov_r64()
904 		emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx);  in emit_a32_mov_r64()
906 		emit(ARM_MOV_R(dst[0], src[0]), ctx);  in emit_a32_mov_r64()
907 		emit(ARM_MOV_R(dst[1], src[1]), ctx);  in emit_a32_mov_r64()
930 		emit(ARM_LSL_I(rd, rs, 32 - off), ctx);  in emit_a32_movsx_r64()
931 		emit(ARM_ASR_I(rd, rd, 32 - off), ctx);  in emit_a32_movsx_r64()
956 			emit(ARM_ASR_I(tmp[0], rd, 31), ctx);  in emit_a32_movsx_r64()
959 			emit(ARM_ASR_I(dst_hi, rd, 31), ctx);  in emit_a32_movsx_r64()
975 		emit(ARM_LSL_I(rd, rd, val), ctx);  in emit_a32_alu_i()
978 		emit(ARM_LSR_I(rd, rd, val), ctx);  in emit_a32_alu_i()
981 		emit(ARM_ASR_I(rd, rd, val), ctx);  in emit_a32_alu_i()
984 		emit(ARM_RSB_I(rd, rd, val), ctx);  in emit_a32_alu_i()
1001 	emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx);  in emit_a32_neg64()
1002 	emit(ARM_RSC_I(rd[0], rd[0], 0), ctx);  in emit_a32_neg64()
1020 	emit(ARM_SUB_I(ARM_IP, rt, 32), ctx);  in emit_a32_lsh_r64()
1021 	emit(ARM_RSB_I(tmp2[0], rt, 32), ctx);  in emit_a32_lsh_r64()
1022 	emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx);  in emit_a32_lsh_r64()
1023 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx);  in emit_a32_lsh_r64()
1024 	emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx);  in emit_a32_lsh_r64()
1025 	emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx);  in emit_a32_lsh_r64()
1044 	emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);  in emit_a32_arsh_r64()
1045 	emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);  in emit_a32_arsh_r64()
1046 	emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);  in emit_a32_arsh_r64()
1047 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);  in emit_a32_arsh_r64()
1050 	emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx);  in emit_a32_arsh_r64()
1069 	emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);  in emit_a32_rsh_r64()
1070 	emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);  in emit_a32_rsh_r64()
1071 	emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);  in emit_a32_rsh_r64()
1072 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);  in emit_a32_rsh_r64()
1073 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx);  in emit_a32_rsh_r64()
1074 	emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx);  in emit_a32_rsh_r64()
1092 		emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx);  in emit_a32_lsh_i64()
1093 		emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx);  in emit_a32_lsh_i64()
1094 		emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx);  in emit_a32_lsh_i64()
1097 			emit(ARM_MOV_R(rd[0], rd[1]), ctx);  in emit_a32_lsh_i64()
1099 			emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx);  in emit_a32_lsh_i64()
1100 		emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx);  in emit_a32_lsh_i64()
1122 		emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);  in emit_a32_rsh_i64()
1123 		emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);  in emit_a32_rsh_i64()
1124 		emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx);  in emit_a32_rsh_i64()
1126 		emit(ARM_MOV_R(rd[1], rd[0]), ctx);  in emit_a32_rsh_i64()
1127 		emit(ARM_MOV_I(rd[0], 0), ctx);  in emit_a32_rsh_i64()
1129 		emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx);  in emit_a32_rsh_i64()
1130 		emit(ARM_MOV_I(rd[0], 0), ctx);  in emit_a32_rsh_i64()
1152 		emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);  in emit_a32_arsh_i64()
1153 		emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);  in emit_a32_arsh_i64()
1154 		emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx);  in emit_a32_arsh_i64()
1156 		emit(ARM_MOV_R(rd[1], rd[0]), ctx);  in emit_a32_arsh_i64()
1157 		emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);  in emit_a32_arsh_i64()
1159 		emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx);  in emit_a32_arsh_i64()
1160 		emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);  in emit_a32_arsh_i64()
1177 	emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx);  in emit_a32_mul_r64()
1178 	emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx);  in emit_a32_mul_r64()
1179 	emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx);  in emit_a32_mul_r64()
1181 	emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx);  in emit_a32_mul_r64()
1182 	emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx);  in emit_a32_mul_r64()
1236 		emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx);  in emit_str_r()
1243 		emit(ARM_STRB_I(src_lo, rd, off), ctx);  in emit_str_r()
1247 		emit(ARM_STRH_I(src_lo, rd, off), ctx);  in emit_str_r()
1251 		emit(ARM_STR_I(src_lo, rd, off), ctx);  in emit_str_r()
1255 		emit(ARM_STR_I(src_lo, rd, off), ctx);  in emit_str_r()
1256 		emit(ARM_STR_I(src_hi, rd, off + 4), ctx);  in emit_str_r()
1270 		emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);  in emit_ldx_r()
1274 		emit(ARM_MOV_R(tmp[0], rm), ctx);  in emit_ldx_r()
1280 		emit(ARM_LDRB_I(rd[1], rm, off), ctx);  in emit_ldx_r()
1286 		emit(ARM_LDRH_I(rd[1], rm, off), ctx);  in emit_ldx_r()
1292 		emit(ARM_LDR_I(rd[1], rm, off), ctx);  in emit_ldx_r()
1298 		emit(ARM_LDR_I(rd[1], rm, off), ctx);  in emit_ldx_r()
1299 		emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);  in emit_ldx_r()
1320 			emit(ARM_ADD_I(tmp[0], src, add_off), ctx);  in emit_ldsx_r()
1324 			emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);  in emit_ldsx_r()
1333 		emit(ARM_LDRSB_I(rd[1], rm, off), ctx);  in emit_ldsx_r()
1337 		emit(ARM_LDRSH_I(rd[1], rm, off), ctx);  in emit_ldsx_r()
1341 		emit(ARM_LDR_I(rd[1], rm, off), ctx);  in emit_ldsx_r()
1345 	emit(ARM_ASR_I(rd[0], rd[1], 31), ctx);  in emit_ldsx_r()
1356 			emit(ARM_AND_R(ARM_IP, rt, rn), ctx);  in emit_ar_r()
1357 			emit(ARM_AND_R(ARM_LR, rd, rm), ctx);  in emit_ar_r()
1358 			emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx);  in emit_ar_r()
1360 			emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx);  in emit_ar_r()
1370 			emit(ARM_CMP_R(rd, rm), ctx);  in emit_ar_r()
1374 			emit(ARM_CMP_R(rt, rn), ctx);  in emit_ar_r()
1379 		emit(ARM_CMP_R(rn, rt), ctx);  in emit_ar_r()
1381 			emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx);  in emit_ar_r()
1385 		emit(ARM_CMP_R(rt, rn), ctx);  in emit_ar_r()
1387 			emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx);  in emit_ar_r()
1420 	emit(ARM_LDR_I(tmp[1], r_array, off), ctx);  in emit_bpf_tail_call()
1422 	emit(ARM_CMP_R(r_index, tmp[1]), ctx);  in emit_bpf_tail_call()
1435 	emit(ARM_CMP_I(tc[0], hi), ctx);  in emit_bpf_tail_call()
1438 	emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx);  in emit_bpf_tail_call()
1439 	emit(ARM_ADC_I(tc[0], tc[0], 0), ctx);  in emit_bpf_tail_call()
1448 	emit(ARM_ADD_I(tmp[1], r_array, off), ctx);  in emit_bpf_tail_call()
1449 	emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx);  in emit_bpf_tail_call()
1450 	emit(ARM_CMP_I(tmp[1], 0), ctx);  in emit_bpf_tail_call()
1457 	emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx);  in emit_bpf_tail_call()
1458 	emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx);  in emit_bpf_tail_call()
1480 	emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);  in emit_rev16()
1481 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx);  in emit_rev16()
1482 	emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);  in emit_rev16()
1483 	emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx);  in emit_rev16()
1485 	emit(ARM_REV16(rd, rn), ctx);  in emit_rev16()
1495 	emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);  in emit_rev32()
1496 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx);  in emit_rev32()
1497 	emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx);  in emit_rev32()
1499 	emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx);  in emit_rev32()
1500 	emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx);  in emit_rev32()
1501 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx);  in emit_rev32()
1502 	emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);  in emit_rev32()
1503 	emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx);  in emit_rev32()
1504 	emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx);  in emit_rev32()
1505 	emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx);  in emit_rev32()
1508 	emit(ARM_REV(rd, rn), ctx);  in emit_rev32()
1522 	emit(ARM_PUSH(reg_set), ctx);  in emit_push_r64()
1535 	emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx);  in build_prologue()
1536 	emit(ARM_PUSH(reg_set), ctx);  in build_prologue()
1537 	emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx);  in build_prologue()
1539 	emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx);  in build_prologue()
1540 	emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx);  in build_prologue()
1544 	emit(ARM_MOV_I(bpf_r1[0], 0), ctx);  in build_prologue()
1545 	emit(ARM_SUB_I(bpf_r1[1], ARM_SP, SCRATCH_SIZE), ctx);  in build_prologue()
1550 	emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx);  in build_prologue()
1556 	emit(ARM_MOV_I(bpf_r1[1], 0), ctx);  in build_prologue()
1560 	emit(ARM_MOV_R(bpf_r1[1], arm_r0), ctx);  in build_prologue()
1572 	emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx);  in build_epilogue()
1573 	emit(ARM_LDM(ARM_SP, reg_set), ctx);  in build_epilogue()
1576 	emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx);  in build_epilogue()
1577 	emit(ARM_POP(CALLEE_POP_MASK), ctx);  in build_epilogue()
1821 			emit(ARM_MOV_R(rd[0], ARM_LR), ctx);  in build_insn()
1831 			emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx);  in build_insn()
1833 			emit(ARM_UXTH(rd[1], rd[1]), ctx);  in build_insn()
1836 				emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);  in build_insn()
1841 				emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);  in build_insn()
2039 		emit(ARM_B(jmp_offset), ctx);  in build_insn()
2067 		emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean  in build_insn()
2079 		emit(ARM_B(jmp_offset), ctx);  in build_insn()