1*8fb009dcSAndroid Build Coastguard Worker// This file is generated from a similarly-named Perl script in the BoringSSL 2*8fb009dcSAndroid Build Coastguard Worker// source tree. Do not edit by hand. 3*8fb009dcSAndroid Build Coastguard Worker 4*8fb009dcSAndroid Build Coastguard Worker#include <openssl/asm_base.h> 5*8fb009dcSAndroid Build Coastguard Worker 6*8fb009dcSAndroid Build Coastguard Worker#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(_WIN32) 7*8fb009dcSAndroid Build Coastguard Worker#include <openssl/arm_arch.h> 8*8fb009dcSAndroid Build Coastguard Worker#if __ARM_MAX_ARCH__ >= 8 9*8fb009dcSAndroid Build Coastguard Worker 10*8fb009dcSAndroid Build Coastguard Worker.arch armv8-a+crypto 11*8fb009dcSAndroid Build Coastguard Worker.text 12*8fb009dcSAndroid Build Coastguard Worker.globl aes_gcm_enc_kernel 13*8fb009dcSAndroid Build Coastguard Worker 14*8fb009dcSAndroid Build Coastguard Worker.def aes_gcm_enc_kernel 15*8fb009dcSAndroid Build Coastguard Worker .type 32 16*8fb009dcSAndroid Build Coastguard Worker.endef 17*8fb009dcSAndroid Build Coastguard Worker.align 4 18*8fb009dcSAndroid Build Coastguard Workeraes_gcm_enc_kernel: 19*8fb009dcSAndroid Build Coastguard Worker AARCH64_SIGN_LINK_REGISTER 20*8fb009dcSAndroid Build Coastguard Worker stp x29, x30, [sp, #-128]! 21*8fb009dcSAndroid Build Coastguard Worker mov x29, sp 22*8fb009dcSAndroid Build Coastguard Worker stp x19, x20, [sp, #16] 23*8fb009dcSAndroid Build Coastguard Worker mov x16, x4 24*8fb009dcSAndroid Build Coastguard Worker mov x8, x5 25*8fb009dcSAndroid Build Coastguard Worker stp x21, x22, [sp, #32] 26*8fb009dcSAndroid Build Coastguard Worker stp x23, x24, [sp, #48] 27*8fb009dcSAndroid Build Coastguard Worker stp d8, d9, [sp, #64] 28*8fb009dcSAndroid Build Coastguard Worker stp d10, d11, [sp, #80] 29*8fb009dcSAndroid Build Coastguard Worker stp d12, d13, [sp, #96] 30*8fb009dcSAndroid Build Coastguard Worker stp d14, d15, [sp, #112] 31*8fb009dcSAndroid Build Coastguard Worker ldr w17, [x8, #240] 32*8fb009dcSAndroid Build Coastguard Worker add x19, x8, x17, lsl #4 // borrow input_l1 for last key 33*8fb009dcSAndroid Build Coastguard Worker ldp x13, x14, [x19] // load round N keys 34*8fb009dcSAndroid Build Coastguard Worker ldr q31, [x19, #-16] // load round N-1 keys 35*8fb009dcSAndroid Build Coastguard Worker add x4, x0, x1, lsr #3 // end_input_ptr 36*8fb009dcSAndroid Build Coastguard Worker lsr x5, x1, #3 // byte_len 37*8fb009dcSAndroid Build Coastguard Worker mov x15, x5 38*8fb009dcSAndroid Build Coastguard Worker ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 39*8fb009dcSAndroid Build Coastguard Worker ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible 40*8fb009dcSAndroid Build Coastguard Worker sub x5, x5, #1 // byte_len - 1 41*8fb009dcSAndroid Build Coastguard Worker ldr q18, [x8, #0] // load rk0 42*8fb009dcSAndroid Build Coastguard Worker and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) 43*8fb009dcSAndroid Build Coastguard Worker ldr q25, [x8, #112] // load rk7 44*8fb009dcSAndroid Build Coastguard Worker add x5, x5, x0 45*8fb009dcSAndroid Build Coastguard Worker lsr x12, x11, #32 46*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 2 47*8fb009dcSAndroid Build Coastguard Worker orr w11, w11, w11 48*8fb009dcSAndroid Build Coastguard Worker rev w12, w12 // rev_ctr32 49*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 1 50*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 51*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 0 52*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // increment rev_ctr32 53*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 1 54*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 3 55*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 1 56*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 1 57*8fb009dcSAndroid Build Coastguard Worker ldr q19, [x8, #16] // load rk1 58*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 1 59*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 2 60*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 2 61*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 2 62*8fb009dcSAndroid Build Coastguard Worker ldr q20, [x8, #32] // load rk2 63*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 2 64*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 3 65*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 66*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 1 67*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 3 68*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 3 69*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 70*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 0 71*8fb009dcSAndroid Build Coastguard Worker ldr q21, [x8, #48] // load rk3 72*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 73*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 2 74*8fb009dcSAndroid Build Coastguard Worker ldr q24, [x8, #96] // load rk6 75*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 76*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 0 77*8fb009dcSAndroid Build Coastguard Worker ldr q23, [x8, #80] // load rk5 78*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 79*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 1 80*8fb009dcSAndroid Build Coastguard Worker ldr q14, [x6, #48] // load h3l | h3h 81*8fb009dcSAndroid Build Coastguard Worker ext v14.16b, v14.16b, v14.16b, #8 82*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 83*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 0 84*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 85*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 1 86*8fb009dcSAndroid Build Coastguard Worker ldr q22, [x8, #64] // load rk4 87*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 88*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 2 89*8fb009dcSAndroid Build Coastguard Worker ldr q13, [x6, #32] // load h2l | h2h 90*8fb009dcSAndroid Build Coastguard Worker ext v13.16b, v13.16b, v13.16b, #8 91*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 92*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 1 93*8fb009dcSAndroid Build Coastguard Worker ldr q30, [x8, #192] // load rk12 94*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 95*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 2 96*8fb009dcSAndroid Build Coastguard Worker ldr q15, [x6, #80] // load h4l | h4h 97*8fb009dcSAndroid Build Coastguard Worker ext v15.16b, v15.16b, v15.16b, #8 98*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 99*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 3 100*8fb009dcSAndroid Build Coastguard Worker ldr q29, [x8, #176] // load rk11 101*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 102*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 2 103*8fb009dcSAndroid Build Coastguard Worker ldr q26, [x8, #128] // load rk8 104*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 105*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 3 106*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 3 107*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 108*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 3 109*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 110*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 3 111*8fb009dcSAndroid Build Coastguard Worker ld1 { v11.16b}, [x3] 112*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 113*8fb009dcSAndroid Build Coastguard Worker rev64 v11.16b, v11.16b 114*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 115*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 4 116*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 117*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 4 118*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 119*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 4 120*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 121*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 4 122*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 123*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 124*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 5 125*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 126*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 5 127*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 128*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 5 129*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 130*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 5 131*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 132*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 6 133*8fb009dcSAndroid Build Coastguard Worker trn2 v17.2d, v14.2d, v15.2d // h4l | h3l 134*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 135*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 6 136*8fb009dcSAndroid Build Coastguard Worker ldr q27, [x8, #144] // load rk9 137*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 138*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 6 139*8fb009dcSAndroid Build Coastguard Worker ldr q12, [x6] // load h1l | h1h 140*8fb009dcSAndroid Build Coastguard Worker ext v12.16b, v12.16b, v12.16b, #8 141*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 142*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 6 143*8fb009dcSAndroid Build Coastguard Worker ldr q28, [x8, #160] // load rk10 144*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 145*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 7 146*8fb009dcSAndroid Build Coastguard Worker trn1 v9.2d, v14.2d, v15.2d // h4h | h3h 147*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 148*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 7 149*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 150*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 7 151*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 152*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 7 153*8fb009dcSAndroid Build Coastguard Worker trn2 v16.2d, v12.2d, v13.2d // h2l | h1l 154*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 155*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 8 156*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 157*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 8 158*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 159*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 8 160*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 161*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 8 162*8fb009dcSAndroid Build Coastguard Worker b.lt Lenc_finish_first_blocks // branch if AES-128 163*8fb009dcSAndroid Build Coastguard Worker 164*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 165*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 9 166*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 167*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 9 168*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 169*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 9 170*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 171*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 9 172*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 173*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 10 174*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 175*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 10 176*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 177*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 10 178*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 179*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 10 180*8fb009dcSAndroid Build Coastguard Worker b.eq Lenc_finish_first_blocks // branch if AES-192 181*8fb009dcSAndroid Build Coastguard Worker 182*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 183*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 11 184*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 185*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 11 186*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 187*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 11 188*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 189*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 11 190*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 191*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 12 192*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 193*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 12 194*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 195*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 12 196*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 197*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 12 198*8fb009dcSAndroid Build Coastguard Worker 199*8fb009dcSAndroid Build Coastguard WorkerLenc_finish_first_blocks: 200*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // check if we have <= 4 blocks 201*8fb009dcSAndroid Build Coastguard Worker eor v17.16b, v17.16b, v9.16b // h4k | h3k 202*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 2 - round N-1 203*8fb009dcSAndroid Build Coastguard Worker trn1 v8.2d, v12.2d, v13.2d // h2h | h1h 204*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 1 - round N-1 205*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 0 - round N-1 206*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 3 - round N-1 207*8fb009dcSAndroid Build Coastguard Worker eor v16.16b, v16.16b, v8.16b // h2k | h1k 208*8fb009dcSAndroid Build Coastguard Worker b.ge Lenc_tail // handle tail 209*8fb009dcSAndroid Build Coastguard Worker 210*8fb009dcSAndroid Build Coastguard Worker ldp x19, x20, [x0, #16] // AES block 1 - load plaintext 211*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4 212*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0, #0] // AES block 0 - load plaintext 213*8fb009dcSAndroid Build Coastguard Worker ldp x23, x24, [x0, #48] // AES block 3 - load plaintext 214*8fb009dcSAndroid Build Coastguard Worker ldp x21, x22, [x0, #32] // AES block 2 - load plaintext 215*8fb009dcSAndroid Build Coastguard Worker add x0, x0, #64 // AES input_ptr update 216*8fb009dcSAndroid Build Coastguard Worker eor x19, x19, x13 // AES block 1 - round N low 217*8fb009dcSAndroid Build Coastguard Worker eor x20, x20, x14 // AES block 1 - round N high 218*8fb009dcSAndroid Build Coastguard Worker fmov d5, x19 // AES block 1 - mov low 219*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 0 - round N low 220*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 0 - round N high 221*8fb009dcSAndroid Build Coastguard Worker eor x24, x24, x14 // AES block 3 - round N high 222*8fb009dcSAndroid Build Coastguard Worker fmov d4, x6 // AES block 0 - mov low 223*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // check if we have <= 8 blocks 224*8fb009dcSAndroid Build Coastguard Worker fmov v4.d[1], x7 // AES block 0 - mov high 225*8fb009dcSAndroid Build Coastguard Worker eor x23, x23, x13 // AES block 3 - round N low 226*8fb009dcSAndroid Build Coastguard Worker eor x21, x21, x13 // AES block 2 - round N low 227*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x20 // AES block 1 - mov high 228*8fb009dcSAndroid Build Coastguard Worker fmov d6, x21 // AES block 2 - mov low 229*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4 230*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4 231*8fb009dcSAndroid Build Coastguard Worker fmov d7, x23 // AES block 3 - mov low 232*8fb009dcSAndroid Build Coastguard Worker eor x22, x22, x14 // AES block 2 - round N high 233*8fb009dcSAndroid Build Coastguard Worker fmov v6.d[1], x22 // AES block 2 - mov high 234*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v0.16b // AES block 0 - result 235*8fb009dcSAndroid Build Coastguard Worker fmov d0, x10 // CTR block 4 236*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x9 // CTR block 4 237*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 5 238*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 5 239*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v1.16b // AES block 1 - result 240*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 5 241*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 5 242*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 5 243*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 6 244*8fb009dcSAndroid Build Coastguard Worker st1 { v4.16b}, [x2], #16 // AES block 0 - store result 245*8fb009dcSAndroid Build Coastguard Worker fmov v7.d[1], x24 // AES block 3 - mov high 246*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 6 247*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v6.16b, v2.16b // AES block 2 - result 248*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES block 1 - store result 249*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 6 250*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 6 251*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 6 252*8fb009dcSAndroid Build Coastguard Worker st1 { v6.16b}, [x2], #16 // AES block 2 - store result 253*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 7 254*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 7 255*8fb009dcSAndroid Build Coastguard Worker eor v7.16b, v7.16b, v3.16b // AES block 3 - result 256*8fb009dcSAndroid Build Coastguard Worker st1 { v7.16b}, [x2], #16 // AES block 3 - store result 257*8fb009dcSAndroid Build Coastguard Worker b.ge Lenc_prepretail // do prepretail 258*8fb009dcSAndroid Build Coastguard Worker 259*8fb009dcSAndroid Build Coastguard WorkerLenc_main_loop: // main loop start 260*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 261*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 262*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) 263*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 264*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 265*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 4k+3 266*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 267*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 268*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 269*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 270*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 271*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 4k+3 272*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 273*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 274*8fb009dcSAndroid Build Coastguard Worker ldp x23, x24, [x0, #48] // AES block 4k+7 - load plaintext 275*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 276*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 277*8fb009dcSAndroid Build Coastguard Worker ldp x21, x22, [x0, #32] // AES block 4k+6 - load plaintext 278*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 279*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 280*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v11.16b // PRE 1 281*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 282*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 283*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 284*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 285*8fb009dcSAndroid Build Coastguard Worker eor x23, x23, x13 // AES block 4k+7 - round N low 286*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 287*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 288*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH block 4k - mid 289*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high 290*8fb009dcSAndroid Build Coastguard Worker eor x22, x22, x14 // AES block 4k+6 - round N high 291*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH block 4k - mid 292*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 293*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 294*8fb009dcSAndroid Build Coastguard Worker rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) 295*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 296*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 297*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low 298*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid 299*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 300*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 301*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 302*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 303*8fb009dcSAndroid Build Coastguard Worker rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) 304*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high 305*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid 306*8fb009dcSAndroid Build Coastguard Worker rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) 307*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low 308*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high 309*8fb009dcSAndroid Build Coastguard Worker mov d4, v5.d[1] // GHASH block 4k+1 - mid 310*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 311*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 312*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 313*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 314*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low 315*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 316*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 317*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 318*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 319*8fb009dcSAndroid Build Coastguard Worker mov d8, v6.d[1] // GHASH block 4k+2 - mid 320*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 321*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 322*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid 323*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 324*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 325*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 326*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 327*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid 328*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 329*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 330*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid 331*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 332*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 333*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 334*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 335*8fb009dcSAndroid Build Coastguard Worker ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid 336*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 337*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 338*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 339*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 340*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 341*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 342*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 343*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 344*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid 345*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high 346*8fb009dcSAndroid Build Coastguard Worker pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low 347*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 348*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 349*8fb009dcSAndroid Build Coastguard Worker pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low 350*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high 351*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 352*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 353*8fb009dcSAndroid Build Coastguard Worker ldp x19, x20, [x0, #16] // AES block 4k+5 - load plaintext 354*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 355*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 356*8fb009dcSAndroid Build Coastguard Worker mov d4, v7.d[1] // GHASH block 4k+3 - mid 357*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 358*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 359*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low 360*8fb009dcSAndroid Build Coastguard Worker pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid 361*8fb009dcSAndroid Build Coastguard Worker pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high 362*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid 363*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 364*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 365*8fb009dcSAndroid Build Coastguard Worker eor x19, x19, x13 // AES block 4k+5 - round N low 366*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 367*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 368*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid 369*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 370*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 371*8fb009dcSAndroid Build Coastguard Worker eor x21, x21, x13 // AES block 4k+6 - round N low 372*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 373*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 374*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 375*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid 376*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high 377*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 378*8fb009dcSAndroid Build Coastguard Worker fmov d5, x19 // AES block 4k+5 - mov low 379*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0, #0] // AES block 4k+4 - load plaintext 380*8fb009dcSAndroid Build Coastguard Worker b.lt Lenc_main_loop_continue // branch if AES-128 381*8fb009dcSAndroid Build Coastguard Worker 382*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 383*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 384*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 385*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 386*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 387*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 388*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 389*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 390*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 391*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 392*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 393*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 394*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 395*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 396*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 397*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 398*8fb009dcSAndroid Build Coastguard Worker b.eq Lenc_main_loop_continue // branch if AES-192 399*8fb009dcSAndroid Build Coastguard Worker 400*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 401*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 402*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 403*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 404*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 405*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 406*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 407*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 408*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 409*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 410*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 411*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 412*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 413*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 414*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 415*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 416*8fb009dcSAndroid Build Coastguard Worker 417*8fb009dcSAndroid Build Coastguard WorkerLenc_main_loop_continue: 418*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 419*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low 420*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid 421*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+3 422*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 423*8fb009dcSAndroid Build Coastguard Worker add x0, x0, #64 // AES input_ptr update 424*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 425*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+8 426*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 427*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 4k+4 - round N low 428*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up 429*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 4k+4 - round N high 430*8fb009dcSAndroid Build Coastguard Worker fmov d4, x6 // AES block 4k+4 - mov low 431*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+8 432*8fb009dcSAndroid Build Coastguard Worker eor v7.16b, v9.16b, v7.16b // MODULO - fold into mid 433*8fb009dcSAndroid Build Coastguard Worker eor x20, x20, x14 // AES block 4k+5 - round N high 434*8fb009dcSAndroid Build Coastguard Worker eor x24, x24, x14 // AES block 4k+7 - round N high 435*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+8 436*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 4k+4 - round N-1 437*8fb009dcSAndroid Build Coastguard Worker fmov v4.d[1], x7 // AES block 4k+4 - mov high 438*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 439*8fb009dcSAndroid Build Coastguard Worker fmov d7, x23 // AES block 4k+7 - mov low 440*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 4k+5 - round N-1 441*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x20 // AES block 4k+5 - mov high 442*8fb009dcSAndroid Build Coastguard Worker fmov d6, x21 // AES block 4k+6 - mov low 443*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // LOOP CONTROL 444*8fb009dcSAndroid Build Coastguard Worker fmov v6.d[1], x22 // AES block 4k+6 - mov high 445*8fb009dcSAndroid Build Coastguard Worker pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 446*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result 447*8fb009dcSAndroid Build Coastguard Worker fmov d0, x10 // CTR block 4k+8 448*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x9 // CTR block 4k+8 449*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+9 450*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+9 451*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result 452*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 4k+9 453*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+9 454*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 4k+9 455*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 4k+6 - round N-1 456*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+10 457*8fb009dcSAndroid Build Coastguard Worker st1 { v4.16b}, [x2], #16 // AES block 4k+4 - store result 458*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+10 459*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v9.16b // MODULO - fold into low 460*8fb009dcSAndroid Build Coastguard Worker fmov v7.d[1], x24 // AES block 4k+7 - mov high 461*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 462*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES block 4k+5 - store result 463*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+10 464*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 4k+7 - round N-1 465*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result 466*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 4k+10 467*8fb009dcSAndroid Build Coastguard Worker st1 { v6.16b}, [x2], #16 // AES block 4k+6 - store result 468*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 4k+10 469*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+11 470*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 471*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+11 472*8fb009dcSAndroid Build Coastguard Worker eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result 473*8fb009dcSAndroid Build Coastguard Worker st1 { v7.16b}, [x2], #16 // AES block 4k+7 - store result 474*8fb009dcSAndroid Build Coastguard Worker b.lt Lenc_main_loop 475*8fb009dcSAndroid Build Coastguard Worker 476*8fb009dcSAndroid Build Coastguard WorkerLenc_prepretail: // PREPRETAIL 477*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 478*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 479*8fb009dcSAndroid Build Coastguard Worker rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) 480*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 481*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 482*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 4k+3 483*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 484*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 485*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) 486*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 4k+3 487*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 488*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 489*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 490*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 491*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 492*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v11.16b // PRE 1 493*8fb009dcSAndroid Build Coastguard Worker rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) 494*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 495*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 496*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 497*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 498*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH block 4k - mid 499*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 500*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 501*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low 502*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH block 4k - mid 503*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high 504*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 505*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 506*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 507*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 508*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid 509*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 510*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 511*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 512*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 513*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 514*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 515*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid 516*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high 517*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low 518*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 519*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 520*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high 521*8fb009dcSAndroid Build Coastguard Worker mov d4, v5.d[1] // GHASH block 4k+1 - mid 522*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 523*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 524*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low 525*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 526*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 527*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid 528*8fb009dcSAndroid Build Coastguard Worker mov d8, v6.d[1] // GHASH block 4k+2 - mid 529*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 530*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 531*8fb009dcSAndroid Build Coastguard Worker rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) 532*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 533*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 534*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid 535*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid 536*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+3 537*8fb009dcSAndroid Build Coastguard Worker pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low 538*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 539*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 540*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 541*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 542*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid 543*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high 544*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low 545*8fb009dcSAndroid Build Coastguard Worker ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid 546*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 547*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 548*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high 549*8fb009dcSAndroid Build Coastguard Worker mov d4, v7.d[1] // GHASH block 4k+3 - mid 550*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 551*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 552*8fb009dcSAndroid Build Coastguard Worker pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid 553*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid 554*8fb009dcSAndroid Build Coastguard Worker pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high 555*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 556*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 557*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid 558*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid 559*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 560*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 561*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 562*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 563*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 564*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 565*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 566*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 567*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 568*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 569*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 570*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 571*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 572*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high 573*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 574*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 575*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 576*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 577*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 578*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 579*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 580*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid 581*8fb009dcSAndroid Build Coastguard Worker pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low 582*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 583*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 584*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 585*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 586*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 587*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low 588*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 589*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 590*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // karatsuba tidy up 591*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 592*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 593*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v9.1d, v8.1d 594*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 595*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v11.16b 596*8fb009dcSAndroid Build Coastguard Worker b.lt Lenc_finish_prepretail // branch if AES-128 597*8fb009dcSAndroid Build Coastguard Worker 598*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 599*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 600*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 601*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 602*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 603*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 604*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 605*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 606*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 607*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 608*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 609*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 610*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 611*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 612*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 613*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 614*8fb009dcSAndroid Build Coastguard Worker b.eq Lenc_finish_prepretail // branch if AES-192 615*8fb009dcSAndroid Build Coastguard Worker 616*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 617*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 618*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 619*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 620*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 621*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 622*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 623*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 624*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 625*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 626*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 627*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 628*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 629*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 630*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 631*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 632*8fb009dcSAndroid Build Coastguard Worker 633*8fb009dcSAndroid Build Coastguard WorkerLenc_finish_prepretail: 634*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b 635*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b 636*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v10.1d, v8.1d 637*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 638*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 4k+5 - round N-1 639*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v4.16b 640*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 4k+7 - round N-1 641*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 4k+4 - round N-1 642*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 4k+6 - round N-1 643*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b 644*8fb009dcSAndroid Build Coastguard Worker 645*8fb009dcSAndroid Build Coastguard WorkerLenc_tail: // TAIL 646*8fb009dcSAndroid Build Coastguard Worker ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag 647*8fb009dcSAndroid Build Coastguard Worker sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process 648*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0], #16 // AES block 4k+4 - load plaintext 649*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 4k+4 - round N low 650*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 4k+4 - round N high 651*8fb009dcSAndroid Build Coastguard Worker cmp x5, #48 652*8fb009dcSAndroid Build Coastguard Worker fmov d4, x6 // AES block 4k+4 - mov low 653*8fb009dcSAndroid Build Coastguard Worker fmov v4.d[1], x7 // AES block 4k+4 - mov high 654*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v4.16b, v0.16b // AES block 4k+4 - result 655*8fb009dcSAndroid Build Coastguard Worker b.gt Lenc_blocks_more_than_3 656*8fb009dcSAndroid Build Coastguard Worker cmp x5, #32 657*8fb009dcSAndroid Build Coastguard Worker mov v3.16b, v2.16b 658*8fb009dcSAndroid Build Coastguard Worker movi v11.8b, #0 659*8fb009dcSAndroid Build Coastguard Worker movi v9.8b, #0 660*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 661*8fb009dcSAndroid Build Coastguard Worker mov v2.16b, v1.16b 662*8fb009dcSAndroid Build Coastguard Worker movi v10.8b, #0 663*8fb009dcSAndroid Build Coastguard Worker b.gt Lenc_blocks_more_than_2 664*8fb009dcSAndroid Build Coastguard Worker mov v3.16b, v1.16b 665*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 666*8fb009dcSAndroid Build Coastguard Worker cmp x5, #16 667*8fb009dcSAndroid Build Coastguard Worker b.gt Lenc_blocks_more_than_1 668*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 669*8fb009dcSAndroid Build Coastguard Worker b Lenc_blocks_less_than_1 670*8fb009dcSAndroid Build Coastguard WorkerLenc_blocks_more_than_3: // blocks left > 3 671*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES final-3 block - store result 672*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high 673*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-3 block 674*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final-2 block - round N low 675*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 676*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final-2 block - round N high 677*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-3 block - mid 678*8fb009dcSAndroid Build Coastguard Worker fmov d5, x6 // AES final-2 block - mov low 679*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x7 // AES final-2 block - mov high 680*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid 681*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 682*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH final-3 block - mid 683*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low 684*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high 685*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid 686*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v1.16b // AES final-2 block - result 687*8fb009dcSAndroid Build Coastguard WorkerLenc_blocks_more_than_2: // blocks left > 2 688*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES final-2 block - store result 689*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high 690*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-2 block 691*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final-1 block - round N low 692*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 693*8fb009dcSAndroid Build Coastguard Worker fmov d5, x6 // AES final-1 block - mov low 694*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final-1 block - round N high 695*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x7 // AES final-1 block - mov high 696*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 697*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high 698*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-2 block - mid 699*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low 700*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid 701*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v2.16b // AES final-1 block - result 702*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high 703*8fb009dcSAndroid Build Coastguard Worker pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid 704*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low 705*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid 706*8fb009dcSAndroid Build Coastguard WorkerLenc_blocks_more_than_1: // blocks left > 1 707*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES final-1 block - store result 708*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-1 block 709*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0], #16 // AES final block - load input low & high 710*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 711*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 712*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final block - round N low 713*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-1 block - mid 714*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high 715*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final block - round N high 716*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid 717*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high 718*8fb009dcSAndroid Build Coastguard Worker ins v22.d[1], v22.d[0] // GHASH final-1 block - mid 719*8fb009dcSAndroid Build Coastguard Worker fmov d5, x6 // AES final block - mov low 720*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x7 // AES final block - mov high 721*8fb009dcSAndroid Build Coastguard Worker pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid 722*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low 723*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v3.16b // AES final block - result 724*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid 725*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low 726*8fb009dcSAndroid Build Coastguard WorkerLenc_blocks_less_than_1: // blocks left <= 1 727*8fb009dcSAndroid Build Coastguard Worker and x1, x1, #127 // bit_length %= 128 728*8fb009dcSAndroid Build Coastguard Worker mvn x13, xzr // rkN_l = 0xffffffffffffffff 729*8fb009dcSAndroid Build Coastguard Worker sub x1, x1, #128 // bit_length -= 128 730*8fb009dcSAndroid Build Coastguard Worker neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) 731*8fb009dcSAndroid Build Coastguard Worker ld1 { v18.16b}, [x2] // load existing bytes where the possibly partial last block is to be stored 732*8fb009dcSAndroid Build Coastguard Worker mvn x14, xzr // rkN_h = 0xffffffffffffffff 733*8fb009dcSAndroid Build Coastguard Worker and x1, x1, #127 // bit_length %= 128 734*8fb009dcSAndroid Build Coastguard Worker lsr x14, x14, x1 // rkN_h is mask for top 64b of last block 735*8fb009dcSAndroid Build Coastguard Worker cmp x1, #64 736*8fb009dcSAndroid Build Coastguard Worker csel x6, x13, x14, lt 737*8fb009dcSAndroid Build Coastguard Worker csel x7, x14, xzr, lt 738*8fb009dcSAndroid Build Coastguard Worker fmov d0, x6 // ctr0b is mask for last block 739*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x7 740*8fb009dcSAndroid Build Coastguard Worker and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits 741*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final block 742*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 743*8fb009dcSAndroid Build Coastguard Worker bif v5.16b, v18.16b, v0.16b // insert existing bytes in top end of result before storing 744*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high 745*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH final block - mid 746*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 747*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v12.1d // GHASH final block - low 748*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final block - high 749*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH final block - mid 750*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid 751*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final block - low 752*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH final block - mid 753*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 754*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 755*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 756*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up 757*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 758*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 759*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 760*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid 761*8fb009dcSAndroid Build Coastguard Worker pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 762*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 763*8fb009dcSAndroid Build Coastguard Worker str w9, [x16, #12] // store the updated counter 764*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2] // store all 16B 765*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v9.16b // MODULO - fold into low 766*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 767*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 768*8fb009dcSAndroid Build Coastguard Worker rev64 v11.16b, v11.16b 769*8fb009dcSAndroid Build Coastguard Worker mov x0, x15 770*8fb009dcSAndroid Build Coastguard Worker st1 { v11.16b }, [x3] 771*8fb009dcSAndroid Build Coastguard Worker ldp x19, x20, [sp, #16] 772*8fb009dcSAndroid Build Coastguard Worker ldp x21, x22, [sp, #32] 773*8fb009dcSAndroid Build Coastguard Worker ldp x23, x24, [sp, #48] 774*8fb009dcSAndroid Build Coastguard Worker ldp d8, d9, [sp, #64] 775*8fb009dcSAndroid Build Coastguard Worker ldp d10, d11, [sp, #80] 776*8fb009dcSAndroid Build Coastguard Worker ldp d12, d13, [sp, #96] 777*8fb009dcSAndroid Build Coastguard Worker ldp d14, d15, [sp, #112] 778*8fb009dcSAndroid Build Coastguard Worker ldp x29, x30, [sp], #128 779*8fb009dcSAndroid Build Coastguard Worker AARCH64_VALIDATE_LINK_REGISTER 780*8fb009dcSAndroid Build Coastguard Worker ret 781*8fb009dcSAndroid Build Coastguard Worker 782*8fb009dcSAndroid Build Coastguard Worker.globl aes_gcm_dec_kernel 783*8fb009dcSAndroid Build Coastguard Worker 784*8fb009dcSAndroid Build Coastguard Worker.def aes_gcm_dec_kernel 785*8fb009dcSAndroid Build Coastguard Worker .type 32 786*8fb009dcSAndroid Build Coastguard Worker.endef 787*8fb009dcSAndroid Build Coastguard Worker.align 4 788*8fb009dcSAndroid Build Coastguard Workeraes_gcm_dec_kernel: 789*8fb009dcSAndroid Build Coastguard Worker AARCH64_SIGN_LINK_REGISTER 790*8fb009dcSAndroid Build Coastguard Worker stp x29, x30, [sp, #-128]! 791*8fb009dcSAndroid Build Coastguard Worker mov x29, sp 792*8fb009dcSAndroid Build Coastguard Worker stp x19, x20, [sp, #16] 793*8fb009dcSAndroid Build Coastguard Worker mov x16, x4 794*8fb009dcSAndroid Build Coastguard Worker mov x8, x5 795*8fb009dcSAndroid Build Coastguard Worker stp x21, x22, [sp, #32] 796*8fb009dcSAndroid Build Coastguard Worker stp x23, x24, [sp, #48] 797*8fb009dcSAndroid Build Coastguard Worker stp d8, d9, [sp, #64] 798*8fb009dcSAndroid Build Coastguard Worker stp d10, d11, [sp, #80] 799*8fb009dcSAndroid Build Coastguard Worker stp d12, d13, [sp, #96] 800*8fb009dcSAndroid Build Coastguard Worker stp d14, d15, [sp, #112] 801*8fb009dcSAndroid Build Coastguard Worker ldr w17, [x8, #240] 802*8fb009dcSAndroid Build Coastguard Worker add x19, x8, x17, lsl #4 // borrow input_l1 for last key 803*8fb009dcSAndroid Build Coastguard Worker ldp x13, x14, [x19] // load round N keys 804*8fb009dcSAndroid Build Coastguard Worker ldr q31, [x19, #-16] // load round N-1 keys 805*8fb009dcSAndroid Build Coastguard Worker lsr x5, x1, #3 // byte_len 806*8fb009dcSAndroid Build Coastguard Worker mov x15, x5 807*8fb009dcSAndroid Build Coastguard Worker ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 808*8fb009dcSAndroid Build Coastguard Worker ldr q26, [x8, #128] // load rk8 809*8fb009dcSAndroid Build Coastguard Worker sub x5, x5, #1 // byte_len - 1 810*8fb009dcSAndroid Build Coastguard Worker ldr q25, [x8, #112] // load rk7 811*8fb009dcSAndroid Build Coastguard Worker and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) 812*8fb009dcSAndroid Build Coastguard Worker add x4, x0, x1, lsr #3 // end_input_ptr 813*8fb009dcSAndroid Build Coastguard Worker ldr q24, [x8, #96] // load rk6 814*8fb009dcSAndroid Build Coastguard Worker lsr x12, x11, #32 815*8fb009dcSAndroid Build Coastguard Worker ldr q23, [x8, #80] // load rk5 816*8fb009dcSAndroid Build Coastguard Worker orr w11, w11, w11 817*8fb009dcSAndroid Build Coastguard Worker ldr q21, [x8, #48] // load rk3 818*8fb009dcSAndroid Build Coastguard Worker add x5, x5, x0 819*8fb009dcSAndroid Build Coastguard Worker rev w12, w12 // rev_ctr32 820*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // increment rev_ctr32 821*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 3 822*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 1 823*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 1 824*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 1 825*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 1 826*8fb009dcSAndroid Build Coastguard Worker ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible 827*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 1 828*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 2 829*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 2 830*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 2 831*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 2 832*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 2 833*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 3 834*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 3 835*8fb009dcSAndroid Build Coastguard Worker ldr q18, [x8, #0] // load rk0 836*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 3 837*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 3 838*8fb009dcSAndroid Build Coastguard Worker ldr q22, [x8, #64] // load rk4 839*8fb009dcSAndroid Build Coastguard Worker ldr q19, [x8, #16] // load rk1 840*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 841*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 0 842*8fb009dcSAndroid Build Coastguard Worker ldr q14, [x6, #48] // load h3l | h3h 843*8fb009dcSAndroid Build Coastguard Worker ext v14.16b, v14.16b, v14.16b, #8 844*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 845*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 0 846*8fb009dcSAndroid Build Coastguard Worker ldr q15, [x6, #80] // load h4l | h4h 847*8fb009dcSAndroid Build Coastguard Worker ext v15.16b, v15.16b, v15.16b, #8 848*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 849*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 0 850*8fb009dcSAndroid Build Coastguard Worker ldr q13, [x6, #32] // load h2l | h2h 851*8fb009dcSAndroid Build Coastguard Worker ext v13.16b, v13.16b, v13.16b, #8 852*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 853*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 0 854*8fb009dcSAndroid Build Coastguard Worker ldr q20, [x8, #32] // load rk2 855*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 856*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 1 857*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 858*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 1 859*8fb009dcSAndroid Build Coastguard Worker ld1 { v11.16b}, [x3] 860*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 861*8fb009dcSAndroid Build Coastguard Worker rev64 v11.16b, v11.16b 862*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 863*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 1 864*8fb009dcSAndroid Build Coastguard Worker ldr q27, [x8, #144] // load rk9 865*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 866*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 1 867*8fb009dcSAndroid Build Coastguard Worker ldr q30, [x8, #192] // load rk12 868*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 869*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 2 870*8fb009dcSAndroid Build Coastguard Worker ldr q12, [x6] // load h1l | h1h 871*8fb009dcSAndroid Build Coastguard Worker ext v12.16b, v12.16b, v12.16b, #8 872*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 873*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 2 874*8fb009dcSAndroid Build Coastguard Worker ldr q28, [x8, #160] // load rk10 875*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 876*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 2 877*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 878*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 3 879*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 880*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 2 881*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 882*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 3 883*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 884*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 4 885*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 886*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 3 887*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 888*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 3 889*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 890*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 4 891*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 892*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 4 893*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 894*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 4 895*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 896*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 5 897*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 898*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 5 899*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 900*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 5 901*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 902*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 5 903*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 904*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 6 905*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 906*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 6 907*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 908*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 909*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 6 910*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 911*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 6 912*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 913*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 7 914*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 915*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 7 916*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 917*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 7 918*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 919*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 8 920*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 921*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 7 922*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 923*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 8 924*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 925*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 8 926*8fb009dcSAndroid Build Coastguard Worker ldr q29, [x8, #176] // load rk11 927*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 928*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 8 929*8fb009dcSAndroid Build Coastguard Worker b.lt Ldec_finish_first_blocks // branch if AES-128 930*8fb009dcSAndroid Build Coastguard Worker 931*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 932*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 9 933*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 934*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 9 935*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 936*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 9 937*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 938*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 9 939*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 940*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 10 941*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 942*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 10 943*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 944*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 10 945*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 946*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 10 947*8fb009dcSAndroid Build Coastguard Worker b.eq Ldec_finish_first_blocks // branch if AES-192 948*8fb009dcSAndroid Build Coastguard Worker 949*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 950*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 11 951*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 952*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 11 953*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 954*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 11 955*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 956*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 11 957*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 958*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 12 959*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 960*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 12 961*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 962*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 12 963*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 964*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 12 965*8fb009dcSAndroid Build Coastguard Worker 966*8fb009dcSAndroid Build Coastguard WorkerLdec_finish_first_blocks: 967*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // check if we have <= 4 blocks 968*8fb009dcSAndroid Build Coastguard Worker trn1 v9.2d, v14.2d, v15.2d // h4h | h3h 969*8fb009dcSAndroid Build Coastguard Worker trn2 v17.2d, v14.2d, v15.2d // h4l | h3l 970*8fb009dcSAndroid Build Coastguard Worker trn1 v8.2d, v12.2d, v13.2d // h2h | h1h 971*8fb009dcSAndroid Build Coastguard Worker trn2 v16.2d, v12.2d, v13.2d // h2l | h1l 972*8fb009dcSAndroid Build Coastguard Worker eor v17.16b, v17.16b, v9.16b // h4k | h3k 973*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 1 - round N-1 974*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 2 - round N-1 975*8fb009dcSAndroid Build Coastguard Worker eor v16.16b, v16.16b, v8.16b // h2k | h1k 976*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 3 - round N-1 977*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 0 - round N-1 978*8fb009dcSAndroid Build Coastguard Worker b.ge Ldec_tail // handle tail 979*8fb009dcSAndroid Build Coastguard Worker 980*8fb009dcSAndroid Build Coastguard Worker ldr q4, [x0, #0] // AES block 0 - load ciphertext 981*8fb009dcSAndroid Build Coastguard Worker ldr q5, [x0, #16] // AES block 1 - load ciphertext 982*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4 983*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v4.16b, v0.16b // AES block 0 - result 984*8fb009dcSAndroid Build Coastguard Worker eor v1.16b, v5.16b, v1.16b // AES block 1 - result 985*8fb009dcSAndroid Build Coastguard Worker rev64 v5.16b, v5.16b // GHASH block 1 986*8fb009dcSAndroid Build Coastguard Worker ldr q7, [x0, #48] // AES block 3 - load ciphertext 987*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES block 0 - mov high 988*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES block 0 - mov low 989*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v4.16b // GHASH block 0 990*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4 991*8fb009dcSAndroid Build Coastguard Worker fmov d0, x10 // CTR block 4 992*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4 993*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x9 // CTR block 4 994*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 5 995*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 5 996*8fb009dcSAndroid Build Coastguard Worker mov x19, v1.d[0] // AES block 1 - mov low 997*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 5 998*8fb009dcSAndroid Build Coastguard Worker mov x20, v1.d[1] // AES block 1 - mov high 999*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 0 - round N high 1000*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 0 - round N low 1001*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES block 0 - store result 1002*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 5 1003*8fb009dcSAndroid Build Coastguard Worker ldr q6, [x0, #32] // AES block 2 - load ciphertext 1004*8fb009dcSAndroid Build Coastguard Worker add x0, x0, #64 // AES input_ptr update 1005*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 5 1006*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 6 1007*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 6 1008*8fb009dcSAndroid Build Coastguard Worker eor x19, x19, x13 // AES block 1 - round N low 1009*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 6 1010*8fb009dcSAndroid Build Coastguard Worker eor x20, x20, x14 // AES block 1 - round N high 1011*8fb009dcSAndroid Build Coastguard Worker stp x19, x20, [x2], #16 // AES block 1 - store result 1012*8fb009dcSAndroid Build Coastguard Worker eor v2.16b, v6.16b, v2.16b // AES block 2 - result 1013*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // check if we have <= 8 blocks 1014*8fb009dcSAndroid Build Coastguard Worker b.ge Ldec_prepretail // do prepretail 1015*8fb009dcSAndroid Build Coastguard Worker 1016*8fb009dcSAndroid Build Coastguard WorkerLdec_main_loop: // main loop start 1017*8fb009dcSAndroid Build Coastguard Worker mov x21, v2.d[0] // AES block 4k+2 - mov low 1018*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 1019*8fb009dcSAndroid Build Coastguard Worker eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result 1020*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 1021*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 1022*8fb009dcSAndroid Build Coastguard Worker mov x22, v2.d[1] // AES block 4k+2 - mov high 1023*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 1024*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 1025*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 4k+6 1026*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 4k+6 1027*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v11.16b // PRE 1 1028*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+7 1029*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 1030*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 1031*8fb009dcSAndroid Build Coastguard Worker mov x24, v3.d[1] // AES block 4k+3 - mov high 1032*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 1033*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 1034*8fb009dcSAndroid Build Coastguard Worker mov x23, v3.d[0] // AES block 4k+3 - mov low 1035*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high 1036*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH block 4k - mid 1037*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 4k+7 1038*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 1039*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 1040*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+7 1041*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 1042*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 1043*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 4k+7 1044*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 1045*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 1046*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid 1047*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 1048*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 1049*8fb009dcSAndroid Build Coastguard Worker eor x22, x22, x14 // AES block 4k+2 - round N high 1050*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 1051*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 1052*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH block 4k - mid 1053*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 1054*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 1055*8fb009dcSAndroid Build Coastguard Worker rev64 v6.16b, v6.16b // GHASH block 4k+2 1056*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 1057*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 1058*8fb009dcSAndroid Build Coastguard Worker eor x21, x21, x13 // AES block 4k+2 - round N low 1059*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 1060*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 1061*8fb009dcSAndroid Build Coastguard Worker stp x21, x22, [x2], #16 // AES block 4k+2 - store result 1062*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low 1063*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high 1064*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 1065*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 1066*8fb009dcSAndroid Build Coastguard Worker rev64 v7.16b, v7.16b // GHASH block 4k+3 1067*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid 1068*8fb009dcSAndroid Build Coastguard Worker eor x23, x23, x13 // AES block 4k+3 - round N low 1069*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low 1070*8fb009dcSAndroid Build Coastguard Worker eor x24, x24, x14 // AES block 4k+3 - round N high 1071*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high 1072*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 1073*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 1074*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 1075*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 1076*8fb009dcSAndroid Build Coastguard Worker mov d4, v5.d[1] // GHASH block 4k+1 - mid 1077*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 1078*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 1079*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low 1080*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 1081*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 1082*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+7 1083*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 1084*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 1085*8fb009dcSAndroid Build Coastguard Worker mov d8, v6.d[1] // GHASH block 4k+2 - mid 1086*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 1087*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 1088*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid 1089*8fb009dcSAndroid Build Coastguard Worker pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low 1090*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 1091*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 1092*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid 1093*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 1094*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 1095*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 1096*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 1097*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low 1098*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid 1099*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+8 1100*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 1101*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 1102*8fb009dcSAndroid Build Coastguard Worker ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid 1103*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 1104*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 1105*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+8 1106*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 1107*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 1108*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 1109*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 1110*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid 1111*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 1112*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 1113*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high 1114*8fb009dcSAndroid Build Coastguard Worker mov d6, v7.d[1] // GHASH block 4k+3 - mid 1115*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 1116*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 1117*8fb009dcSAndroid Build Coastguard Worker pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid 1118*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 1119*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 1120*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high 1121*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 1122*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 1123*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low 1124*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+8 1125*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid 1126*8fb009dcSAndroid Build Coastguard Worker pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high 1127*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 1128*8fb009dcSAndroid Build Coastguard Worker eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid 1129*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 1130*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 1131*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 1132*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 1133*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high 1134*8fb009dcSAndroid Build Coastguard Worker pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid 1135*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 1136*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 1137*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 1138*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low 1139*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 1140*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 1141*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 1142*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 1143*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 1144*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid 1145*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 1146*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 1147*8fb009dcSAndroid Build Coastguard Worker b.lt Ldec_main_loop_continue // branch if AES-128 1148*8fb009dcSAndroid Build Coastguard Worker 1149*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 1150*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 1151*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 1152*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 1153*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 1154*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 1155*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 1156*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 1157*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 1158*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 1159*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 1160*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 1161*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 1162*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 1163*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 1164*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 1165*8fb009dcSAndroid Build Coastguard Worker b.eq Ldec_main_loop_continue // branch if AES-192 1166*8fb009dcSAndroid Build Coastguard Worker 1167*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 1168*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 1169*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 1170*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 1171*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 1172*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 1173*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 1174*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 1175*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 1176*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 1177*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 1178*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 1179*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 1180*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 1181*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 1182*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 1183*8fb009dcSAndroid Build Coastguard Worker 1184*8fb009dcSAndroid Build Coastguard WorkerLdec_main_loop_continue: 1185*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 1186*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 1187*8fb009dcSAndroid Build Coastguard Worker ldr q4, [x0, #0] // AES block 4k+4 - load ciphertext 1188*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 4k+4 - round N-1 1189*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 1190*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up 1191*8fb009dcSAndroid Build Coastguard Worker ldr q5, [x0, #16] // AES block 4k+5 - load ciphertext 1192*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v4.16b, v0.16b // AES block 4k+4 - result 1193*8fb009dcSAndroid Build Coastguard Worker stp x23, x24, [x2], #16 // AES block 4k+3 - store result 1194*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 1195*8fb009dcSAndroid Build Coastguard Worker ldr q7, [x0, #48] // AES block 4k+7 - load ciphertext 1196*8fb009dcSAndroid Build Coastguard Worker ldr q6, [x0, #32] // AES block 4k+6 - load ciphertext 1197*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES block 4k+4 - mov high 1198*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid 1199*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 4k+5 - round N-1 1200*8fb009dcSAndroid Build Coastguard Worker add x0, x0, #64 // AES input_ptr update 1201*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES block 4k+4 - mov low 1202*8fb009dcSAndroid Build Coastguard Worker fmov d0, x10 // CTR block 4k+8 1203*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x9 // CTR block 4k+8 1204*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 1205*8fb009dcSAndroid Build Coastguard Worker eor v1.16b, v5.16b, v1.16b // AES block 4k+5 - result 1206*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+9 1207*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 4k+6 - round N-1 1208*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+9 1209*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // LOOP CONTROL 1210*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+9 1211*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 4k+4 - round N low 1212*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 4k+4 - round N high 1213*8fb009dcSAndroid Build Coastguard Worker mov x20, v1.d[1] // AES block 4k+5 - mov high 1214*8fb009dcSAndroid Build Coastguard Worker eor v2.16b, v6.16b, v2.16b // AES block 4k+6 - result 1215*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // MODULO - fold into low 1216*8fb009dcSAndroid Build Coastguard Worker mov x19, v1.d[0] // AES block 4k+5 - mov low 1217*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 4k+9 1218*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 1219*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 4k+9 1220*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+10 1221*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+10 1222*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 4k+7 - round N-1 1223*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+10 1224*8fb009dcSAndroid Build Coastguard Worker rev64 v5.16b, v5.16b // GHASH block 4k+5 1225*8fb009dcSAndroid Build Coastguard Worker eor x20, x20, x14 // AES block 4k+5 - round N high 1226*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES block 4k+4 - store result 1227*8fb009dcSAndroid Build Coastguard Worker eor x19, x19, x13 // AES block 4k+5 - round N low 1228*8fb009dcSAndroid Build Coastguard Worker stp x19, x20, [x2], #16 // AES block 4k+5 - store result 1229*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v4.16b // GHASH block 4k+4 1230*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 1231*8fb009dcSAndroid Build Coastguard Worker b.lt Ldec_main_loop 1232*8fb009dcSAndroid Build Coastguard Worker 1233*8fb009dcSAndroid Build Coastguard WorkerLdec_prepretail: // PREPRETAIL 1234*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 1235*8fb009dcSAndroid Build Coastguard Worker mov x21, v2.d[0] // AES block 4k+2 - mov low 1236*8fb009dcSAndroid Build Coastguard Worker eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result 1237*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 1238*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 1239*8fb009dcSAndroid Build Coastguard Worker mov x22, v2.d[1] // AES block 4k+2 - mov high 1240*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 1241*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 1242*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 4k+6 1243*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 4k+6 1244*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+7 1245*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v11.16b // PRE 1 1246*8fb009dcSAndroid Build Coastguard Worker rev64 v6.16b, v6.16b // GHASH block 4k+2 1247*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+7 1248*8fb009dcSAndroid Build Coastguard Worker mov x23, v3.d[0] // AES block 4k+3 - mov low 1249*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 1250*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 1251*8fb009dcSAndroid Build Coastguard Worker mov x24, v3.d[1] // AES block 4k+3 - mov high 1252*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low 1253*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH block 4k - mid 1254*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 4k+7 1255*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high 1256*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 4k+7 1257*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 1258*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 1259*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH block 4k - mid 1260*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 1261*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 1262*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid 1263*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high 1264*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 1265*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 1266*8fb009dcSAndroid Build Coastguard Worker rev64 v7.16b, v7.16b // GHASH block 4k+3 1267*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 1268*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 1269*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid 1270*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high 1271*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low 1272*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 1273*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 1274*8fb009dcSAndroid Build Coastguard Worker mov d4, v5.d[1] // GHASH block 4k+1 - mid 1275*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 1276*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 1277*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 1278*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 1279*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low 1280*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 1281*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 1282*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 1283*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 1284*8fb009dcSAndroid Build Coastguard Worker mov d8, v6.d[1] // GHASH block 4k+2 - mid 1285*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 1286*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 1287*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid 1288*8fb009dcSAndroid Build Coastguard Worker pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low 1289*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 1290*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 1291*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 1292*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 1293*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid 1294*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid 1295*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 1296*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 1297*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low 1298*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 1299*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 1300*8fb009dcSAndroid Build Coastguard Worker pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high 1301*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid 1302*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high 1303*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 1304*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 1305*8fb009dcSAndroid Build Coastguard Worker ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid 1306*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 1307*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 1308*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 1309*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 1310*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high 1311*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low 1312*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 1313*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 1314*8fb009dcSAndroid Build Coastguard Worker mov d6, v7.d[1] // GHASH block 4k+3 - mid 1315*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 1316*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 1317*8fb009dcSAndroid Build Coastguard Worker pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid 1318*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 1319*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 1320*8fb009dcSAndroid Build Coastguard Worker eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid 1321*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 1322*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 1323*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 1324*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 1325*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid 1326*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 1327*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 1328*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 1329*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 1330*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 1331*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 1332*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 1333*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low 1334*8fb009dcSAndroid Build Coastguard Worker pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid 1335*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 1336*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 1337*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 1338*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high 1339*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 1340*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 1341*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 1342*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 1343*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid 1344*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 1345*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 1346*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 1347*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 1348*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 1349*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 1350*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 1351*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 1352*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 1353*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 1354*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 1355*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 1356*8fb009dcSAndroid Build Coastguard Worker b.lt Ldec_finish_prepretail // branch if AES-128 1357*8fb009dcSAndroid Build Coastguard Worker 1358*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 1359*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 1360*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 1361*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 1362*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 1363*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 1364*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 1365*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 1366*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 1367*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 1368*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 1369*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 1370*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 1371*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 1372*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 1373*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 1374*8fb009dcSAndroid Build Coastguard Worker b.eq Ldec_finish_prepretail // branch if AES-192 1375*8fb009dcSAndroid Build Coastguard Worker 1376*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 1377*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 1378*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 1379*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 1380*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 1381*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 1382*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 1383*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 1384*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 1385*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 1386*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 1387*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 1388*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 1389*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 1390*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 1391*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 1392*8fb009dcSAndroid Build Coastguard Worker 1393*8fb009dcSAndroid Build Coastguard WorkerLdec_finish_prepretail: 1394*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up 1395*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 1396*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 1397*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 1398*8fb009dcSAndroid Build Coastguard Worker eor x22, x22, x14 // AES block 4k+2 - round N high 1399*8fb009dcSAndroid Build Coastguard Worker eor x23, x23, x13 // AES block 4k+3 - round N low 1400*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid 1401*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+7 1402*8fb009dcSAndroid Build Coastguard Worker eor x21, x21, x13 // AES block 4k+2 - round N low 1403*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 1404*8fb009dcSAndroid Build Coastguard Worker eor x24, x24, x14 // AES block 4k+3 - round N high 1405*8fb009dcSAndroid Build Coastguard Worker stp x21, x22, [x2], #16 // AES block 4k+2 - store result 1406*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 1407*8fb009dcSAndroid Build Coastguard Worker stp x23, x24, [x2], #16 // AES block 4k+3 - store result 1408*8fb009dcSAndroid Build Coastguard Worker 1409*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // MODULO - fold into low 1410*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 4k+5 - round N-1 1411*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 4k+4 - round N-1 1412*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 4k+7 - round N-1 1413*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 4k+6 - round N-1 1414*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 1415*8fb009dcSAndroid Build Coastguard Worker 1416*8fb009dcSAndroid Build Coastguard WorkerLdec_tail: // TAIL 1417*8fb009dcSAndroid Build Coastguard Worker sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process 1418*8fb009dcSAndroid Build Coastguard Worker ld1 { v5.16b}, [x0], #16 // AES block 4k+4 - load ciphertext 1419*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v5.16b, v0.16b // AES block 4k+4 - result 1420*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES block 4k+4 - mov low 1421*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES block 4k+4 - mov high 1422*8fb009dcSAndroid Build Coastguard Worker ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag 1423*8fb009dcSAndroid Build Coastguard Worker cmp x5, #48 1424*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 4k+4 - round N low 1425*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 4k+4 - round N high 1426*8fb009dcSAndroid Build Coastguard Worker b.gt Ldec_blocks_more_than_3 1427*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 1428*8fb009dcSAndroid Build Coastguard Worker mov v3.16b, v2.16b 1429*8fb009dcSAndroid Build Coastguard Worker movi v10.8b, #0 1430*8fb009dcSAndroid Build Coastguard Worker movi v11.8b, #0 1431*8fb009dcSAndroid Build Coastguard Worker cmp x5, #32 1432*8fb009dcSAndroid Build Coastguard Worker movi v9.8b, #0 1433*8fb009dcSAndroid Build Coastguard Worker mov v2.16b, v1.16b 1434*8fb009dcSAndroid Build Coastguard Worker b.gt Ldec_blocks_more_than_2 1435*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 1436*8fb009dcSAndroid Build Coastguard Worker mov v3.16b, v1.16b 1437*8fb009dcSAndroid Build Coastguard Worker cmp x5, #16 1438*8fb009dcSAndroid Build Coastguard Worker b.gt Ldec_blocks_more_than_1 1439*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 1440*8fb009dcSAndroid Build Coastguard Worker b Ldec_blocks_less_than_1 1441*8fb009dcSAndroid Build Coastguard WorkerLdec_blocks_more_than_3: // blocks left > 3 1442*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-3 block 1443*8fb009dcSAndroid Build Coastguard Worker ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext 1444*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES final-3 block - store result 1445*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH final-3 block - mid 1446*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 1447*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v5.16b, v1.16b // AES final-2 block - result 1448*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-3 block - mid 1449*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES final-2 block - mov low 1450*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES final-2 block - mov high 1451*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid 1452*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 1453*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high 1454*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid 1455*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final-2 block - round N low 1456*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low 1457*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final-2 block - round N high 1458*8fb009dcSAndroid Build Coastguard WorkerLdec_blocks_more_than_2: // blocks left > 2 1459*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-2 block 1460*8fb009dcSAndroid Build Coastguard Worker ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext 1461*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 1462*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES final-2 block - store result 1463*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v5.16b, v2.16b // AES final-1 block - result 1464*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-2 block - mid 1465*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low 1466*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high 1467*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid 1468*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES final-1 block - mov low 1469*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES final-1 block - mov high 1470*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low 1471*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 1472*8fb009dcSAndroid Build Coastguard Worker pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid 1473*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high 1474*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final-1 block - round N low 1475*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid 1476*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final-1 block - round N high 1477*8fb009dcSAndroid Build Coastguard WorkerLdec_blocks_more_than_1: // blocks left > 1 1478*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES final-1 block - store result 1479*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-1 block 1480*8fb009dcSAndroid Build Coastguard Worker ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext 1481*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 1482*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 1483*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-1 block - mid 1484*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v5.16b, v3.16b // AES final block - result 1485*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high 1486*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid 1487*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low 1488*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES final block - mov low 1489*8fb009dcSAndroid Build Coastguard Worker ins v22.d[1], v22.d[0] // GHASH final-1 block - mid 1490*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES final block - mov high 1491*8fb009dcSAndroid Build Coastguard Worker pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid 1492*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final block - round N low 1493*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low 1494*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high 1495*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid 1496*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final block - round N high 1497*8fb009dcSAndroid Build Coastguard WorkerLdec_blocks_less_than_1: // blocks left <= 1 1498*8fb009dcSAndroid Build Coastguard Worker and x1, x1, #127 // bit_length %= 128 1499*8fb009dcSAndroid Build Coastguard Worker mvn x14, xzr // rkN_h = 0xffffffffffffffff 1500*8fb009dcSAndroid Build Coastguard Worker sub x1, x1, #128 // bit_length -= 128 1501*8fb009dcSAndroid Build Coastguard Worker mvn x13, xzr // rkN_l = 0xffffffffffffffff 1502*8fb009dcSAndroid Build Coastguard Worker ldp x4, x5, [x2] // load existing bytes we need to not overwrite 1503*8fb009dcSAndroid Build Coastguard Worker neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) 1504*8fb009dcSAndroid Build Coastguard Worker and x1, x1, #127 // bit_length %= 128 1505*8fb009dcSAndroid Build Coastguard Worker lsr x14, x14, x1 // rkN_h is mask for top 64b of last block 1506*8fb009dcSAndroid Build Coastguard Worker cmp x1, #64 1507*8fb009dcSAndroid Build Coastguard Worker csel x9, x13, x14, lt 1508*8fb009dcSAndroid Build Coastguard Worker csel x10, x14, xzr, lt 1509*8fb009dcSAndroid Build Coastguard Worker fmov d0, x9 // ctr0b is mask for last block 1510*8fb009dcSAndroid Build Coastguard Worker and x6, x6, x9 1511*8fb009dcSAndroid Build Coastguard Worker mov v0.d[1], x10 1512*8fb009dcSAndroid Build Coastguard Worker bic x4, x4, x9 // mask out low existing bytes 1513*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 1514*8fb009dcSAndroid Build Coastguard Worker bic x5, x5, x10 // mask out high existing bytes 1515*8fb009dcSAndroid Build Coastguard Worker orr x6, x6, x4 1516*8fb009dcSAndroid Build Coastguard Worker and x7, x7, x10 1517*8fb009dcSAndroid Build Coastguard Worker orr x7, x7, x5 1518*8fb009dcSAndroid Build Coastguard Worker and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits 1519*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final block 1520*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 1521*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v12.1d // GHASH final block - low 1522*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH final block - mid 1523*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH final block - mid 1524*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high 1525*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid 1526*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final block - high 1527*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final block - low 1528*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH final block - mid 1529*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 1530*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 1531*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 1532*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up 1533*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 1534*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 1535*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 1536*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid 1537*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 1538*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 1539*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // MODULO - fold into low 1540*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2] 1541*8fb009dcSAndroid Build Coastguard Worker str w9, [x16, #12] // store the updated counter 1542*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 1543*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 1544*8fb009dcSAndroid Build Coastguard Worker rev64 v11.16b, v11.16b 1545*8fb009dcSAndroid Build Coastguard Worker mov x0, x15 1546*8fb009dcSAndroid Build Coastguard Worker st1 { v11.16b }, [x3] 1547*8fb009dcSAndroid Build Coastguard Worker ldp x19, x20, [sp, #16] 1548*8fb009dcSAndroid Build Coastguard Worker ldp x21, x22, [sp, #32] 1549*8fb009dcSAndroid Build Coastguard Worker ldp x23, x24, [sp, #48] 1550*8fb009dcSAndroid Build Coastguard Worker ldp d8, d9, [sp, #64] 1551*8fb009dcSAndroid Build Coastguard Worker ldp d10, d11, [sp, #80] 1552*8fb009dcSAndroid Build Coastguard Worker ldp d12, d13, [sp, #96] 1553*8fb009dcSAndroid Build Coastguard Worker ldp d14, d15, [sp, #112] 1554*8fb009dcSAndroid Build Coastguard Worker ldp x29, x30, [sp], #128 1555*8fb009dcSAndroid Build Coastguard Worker AARCH64_VALIDATE_LINK_REGISTER 1556*8fb009dcSAndroid Build Coastguard Worker ret 1557*8fb009dcSAndroid Build Coastguard Worker 1558*8fb009dcSAndroid Build Coastguard Worker#endif 1559*8fb009dcSAndroid Build Coastguard Worker#endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(_WIN32) 1560