1*8fb009dcSAndroid Build Coastguard Worker// This file is generated from a similarly-named Perl script in the BoringSSL 2*8fb009dcSAndroid Build Coastguard Worker// source tree. Do not edit by hand. 3*8fb009dcSAndroid Build Coastguard Worker 4*8fb009dcSAndroid Build Coastguard Worker#include <openssl/asm_base.h> 5*8fb009dcSAndroid Build Coastguard Worker 6*8fb009dcSAndroid Build Coastguard Worker#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__ELF__) 7*8fb009dcSAndroid Build Coastguard Worker#include <openssl/arm_arch.h> 8*8fb009dcSAndroid Build Coastguard Worker#if __ARM_MAX_ARCH__ >= 8 9*8fb009dcSAndroid Build Coastguard Worker 10*8fb009dcSAndroid Build Coastguard Worker.arch armv8-a+crypto 11*8fb009dcSAndroid Build Coastguard Worker.text 12*8fb009dcSAndroid Build Coastguard Worker.globl aes_gcm_enc_kernel 13*8fb009dcSAndroid Build Coastguard Worker.hidden aes_gcm_enc_kernel 14*8fb009dcSAndroid Build Coastguard Worker.type aes_gcm_enc_kernel,%function 15*8fb009dcSAndroid Build Coastguard Worker.align 4 16*8fb009dcSAndroid Build Coastguard Workeraes_gcm_enc_kernel: 17*8fb009dcSAndroid Build Coastguard Worker AARCH64_SIGN_LINK_REGISTER 18*8fb009dcSAndroid Build Coastguard Worker stp x29, x30, [sp, #-128]! 19*8fb009dcSAndroid Build Coastguard Worker mov x29, sp 20*8fb009dcSAndroid Build Coastguard Worker stp x19, x20, [sp, #16] 21*8fb009dcSAndroid Build Coastguard Worker mov x16, x4 22*8fb009dcSAndroid Build Coastguard Worker mov x8, x5 23*8fb009dcSAndroid Build Coastguard Worker stp x21, x22, [sp, #32] 24*8fb009dcSAndroid Build Coastguard Worker stp x23, x24, [sp, #48] 25*8fb009dcSAndroid Build Coastguard Worker stp d8, d9, [sp, #64] 26*8fb009dcSAndroid Build Coastguard Worker stp d10, d11, [sp, #80] 27*8fb009dcSAndroid Build Coastguard Worker stp d12, d13, [sp, #96] 28*8fb009dcSAndroid Build Coastguard Worker stp d14, d15, [sp, #112] 29*8fb009dcSAndroid Build Coastguard Worker ldr w17, [x8, #240] 30*8fb009dcSAndroid Build Coastguard Worker add x19, x8, x17, lsl #4 // borrow input_l1 for last key 31*8fb009dcSAndroid Build Coastguard Worker ldp x13, x14, [x19] // load round N keys 32*8fb009dcSAndroid Build Coastguard Worker ldr q31, [x19, #-16] // load round N-1 keys 33*8fb009dcSAndroid Build Coastguard Worker add x4, x0, x1, lsr #3 // end_input_ptr 34*8fb009dcSAndroid Build Coastguard Worker lsr x5, x1, #3 // byte_len 35*8fb009dcSAndroid Build Coastguard Worker mov x15, x5 36*8fb009dcSAndroid Build Coastguard Worker ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 37*8fb009dcSAndroid Build Coastguard Worker ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible 38*8fb009dcSAndroid Build Coastguard Worker sub x5, x5, #1 // byte_len - 1 39*8fb009dcSAndroid Build Coastguard Worker ldr q18, [x8, #0] // load rk0 40*8fb009dcSAndroid Build Coastguard Worker and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) 41*8fb009dcSAndroid Build Coastguard Worker ldr q25, [x8, #112] // load rk7 42*8fb009dcSAndroid Build Coastguard Worker add x5, x5, x0 43*8fb009dcSAndroid Build Coastguard Worker lsr x12, x11, #32 44*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 2 45*8fb009dcSAndroid Build Coastguard Worker orr w11, w11, w11 46*8fb009dcSAndroid Build Coastguard Worker rev w12, w12 // rev_ctr32 47*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 1 48*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 49*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 0 50*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // increment rev_ctr32 51*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 1 52*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 3 53*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 1 54*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 1 55*8fb009dcSAndroid Build Coastguard Worker ldr q19, [x8, #16] // load rk1 56*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 1 57*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 2 58*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 2 59*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 2 60*8fb009dcSAndroid Build Coastguard Worker ldr q20, [x8, #32] // load rk2 61*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 2 62*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 3 63*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 64*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 1 65*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 3 66*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 3 67*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 68*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 0 69*8fb009dcSAndroid Build Coastguard Worker ldr q21, [x8, #48] // load rk3 70*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 71*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 2 72*8fb009dcSAndroid Build Coastguard Worker ldr q24, [x8, #96] // load rk6 73*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 74*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 0 75*8fb009dcSAndroid Build Coastguard Worker ldr q23, [x8, #80] // load rk5 76*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 77*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 1 78*8fb009dcSAndroid Build Coastguard Worker ldr q14, [x6, #48] // load h3l | h3h 79*8fb009dcSAndroid Build Coastguard Worker ext v14.16b, v14.16b, v14.16b, #8 80*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 81*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 0 82*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 83*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 1 84*8fb009dcSAndroid Build Coastguard Worker ldr q22, [x8, #64] // load rk4 85*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 86*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 2 87*8fb009dcSAndroid Build Coastguard Worker ldr q13, [x6, #32] // load h2l | h2h 88*8fb009dcSAndroid Build Coastguard Worker ext v13.16b, v13.16b, v13.16b, #8 89*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 90*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 1 91*8fb009dcSAndroid Build Coastguard Worker ldr q30, [x8, #192] // load rk12 92*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 93*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 2 94*8fb009dcSAndroid Build Coastguard Worker ldr q15, [x6, #80] // load h4l | h4h 95*8fb009dcSAndroid Build Coastguard Worker ext v15.16b, v15.16b, v15.16b, #8 96*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 97*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 3 98*8fb009dcSAndroid Build Coastguard Worker ldr q29, [x8, #176] // load rk11 99*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 100*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 2 101*8fb009dcSAndroid Build Coastguard Worker ldr q26, [x8, #128] // load rk8 102*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 103*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 3 104*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 3 105*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 106*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 3 107*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 108*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 3 109*8fb009dcSAndroid Build Coastguard Worker ld1 { v11.16b}, [x3] 110*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 111*8fb009dcSAndroid Build Coastguard Worker rev64 v11.16b, v11.16b 112*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 113*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 4 114*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 115*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 4 116*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 117*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 4 118*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 119*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 4 120*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 121*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 122*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 5 123*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 124*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 5 125*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 126*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 5 127*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 128*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 5 129*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 130*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 6 131*8fb009dcSAndroid Build Coastguard Worker trn2 v17.2d, v14.2d, v15.2d // h4l | h3l 132*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 133*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 6 134*8fb009dcSAndroid Build Coastguard Worker ldr q27, [x8, #144] // load rk9 135*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 136*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 6 137*8fb009dcSAndroid Build Coastguard Worker ldr q12, [x6] // load h1l | h1h 138*8fb009dcSAndroid Build Coastguard Worker ext v12.16b, v12.16b, v12.16b, #8 139*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 140*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 6 141*8fb009dcSAndroid Build Coastguard Worker ldr q28, [x8, #160] // load rk10 142*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 143*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 7 144*8fb009dcSAndroid Build Coastguard Worker trn1 v9.2d, v14.2d, v15.2d // h4h | h3h 145*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 146*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 7 147*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 148*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 7 149*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 150*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 7 151*8fb009dcSAndroid Build Coastguard Worker trn2 v16.2d, v12.2d, v13.2d // h2l | h1l 152*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 153*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 8 154*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 155*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 8 156*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 157*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 8 158*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 159*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 8 160*8fb009dcSAndroid Build Coastguard Worker b.lt .Lenc_finish_first_blocks // branch if AES-128 161*8fb009dcSAndroid Build Coastguard Worker 162*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 163*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 9 164*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 165*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 9 166*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 167*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 9 168*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 169*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 9 170*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 171*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 10 172*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 173*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 10 174*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 175*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 10 176*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 177*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 10 178*8fb009dcSAndroid Build Coastguard Worker b.eq .Lenc_finish_first_blocks // branch if AES-192 179*8fb009dcSAndroid Build Coastguard Worker 180*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 181*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 11 182*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 183*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 11 184*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 185*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 11 186*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 187*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 11 188*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 189*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 12 190*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 191*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 12 192*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 193*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 12 194*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 195*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 12 196*8fb009dcSAndroid Build Coastguard Worker 197*8fb009dcSAndroid Build Coastguard Worker.Lenc_finish_first_blocks: 198*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // check if we have <= 4 blocks 199*8fb009dcSAndroid Build Coastguard Worker eor v17.16b, v17.16b, v9.16b // h4k | h3k 200*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 2 - round N-1 201*8fb009dcSAndroid Build Coastguard Worker trn1 v8.2d, v12.2d, v13.2d // h2h | h1h 202*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 1 - round N-1 203*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 0 - round N-1 204*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 3 - round N-1 205*8fb009dcSAndroid Build Coastguard Worker eor v16.16b, v16.16b, v8.16b // h2k | h1k 206*8fb009dcSAndroid Build Coastguard Worker b.ge .Lenc_tail // handle tail 207*8fb009dcSAndroid Build Coastguard Worker 208*8fb009dcSAndroid Build Coastguard Worker ldp x19, x20, [x0, #16] // AES block 1 - load plaintext 209*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4 210*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0, #0] // AES block 0 - load plaintext 211*8fb009dcSAndroid Build Coastguard Worker ldp x23, x24, [x0, #48] // AES block 3 - load plaintext 212*8fb009dcSAndroid Build Coastguard Worker ldp x21, x22, [x0, #32] // AES block 2 - load plaintext 213*8fb009dcSAndroid Build Coastguard Worker add x0, x0, #64 // AES input_ptr update 214*8fb009dcSAndroid Build Coastguard Worker eor x19, x19, x13 // AES block 1 - round N low 215*8fb009dcSAndroid Build Coastguard Worker eor x20, x20, x14 // AES block 1 - round N high 216*8fb009dcSAndroid Build Coastguard Worker fmov d5, x19 // AES block 1 - mov low 217*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 0 - round N low 218*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 0 - round N high 219*8fb009dcSAndroid Build Coastguard Worker eor x24, x24, x14 // AES block 3 - round N high 220*8fb009dcSAndroid Build Coastguard Worker fmov d4, x6 // AES block 0 - mov low 221*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // check if we have <= 8 blocks 222*8fb009dcSAndroid Build Coastguard Worker fmov v4.d[1], x7 // AES block 0 - mov high 223*8fb009dcSAndroid Build Coastguard Worker eor x23, x23, x13 // AES block 3 - round N low 224*8fb009dcSAndroid Build Coastguard Worker eor x21, x21, x13 // AES block 2 - round N low 225*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x20 // AES block 1 - mov high 226*8fb009dcSAndroid Build Coastguard Worker fmov d6, x21 // AES block 2 - mov low 227*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4 228*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4 229*8fb009dcSAndroid Build Coastguard Worker fmov d7, x23 // AES block 3 - mov low 230*8fb009dcSAndroid Build Coastguard Worker eor x22, x22, x14 // AES block 2 - round N high 231*8fb009dcSAndroid Build Coastguard Worker fmov v6.d[1], x22 // AES block 2 - mov high 232*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v0.16b // AES block 0 - result 233*8fb009dcSAndroid Build Coastguard Worker fmov d0, x10 // CTR block 4 234*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x9 // CTR block 4 235*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 5 236*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 5 237*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v1.16b // AES block 1 - result 238*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 5 239*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 5 240*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 5 241*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 6 242*8fb009dcSAndroid Build Coastguard Worker st1 { v4.16b}, [x2], #16 // AES block 0 - store result 243*8fb009dcSAndroid Build Coastguard Worker fmov v7.d[1], x24 // AES block 3 - mov high 244*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 6 245*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v6.16b, v2.16b // AES block 2 - result 246*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES block 1 - store result 247*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 6 248*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 6 249*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 6 250*8fb009dcSAndroid Build Coastguard Worker st1 { v6.16b}, [x2], #16 // AES block 2 - store result 251*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 7 252*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 7 253*8fb009dcSAndroid Build Coastguard Worker eor v7.16b, v7.16b, v3.16b // AES block 3 - result 254*8fb009dcSAndroid Build Coastguard Worker st1 { v7.16b}, [x2], #16 // AES block 3 - store result 255*8fb009dcSAndroid Build Coastguard Worker b.ge .Lenc_prepretail // do prepretail 256*8fb009dcSAndroid Build Coastguard Worker 257*8fb009dcSAndroid Build Coastguard Worker.Lenc_main_loop: // main loop start 258*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 259*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 260*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) 261*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 262*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 263*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 4k+3 264*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 265*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 266*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 267*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 268*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 269*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 4k+3 270*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 271*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 272*8fb009dcSAndroid Build Coastguard Worker ldp x23, x24, [x0, #48] // AES block 4k+7 - load plaintext 273*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 274*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 275*8fb009dcSAndroid Build Coastguard Worker ldp x21, x22, [x0, #32] // AES block 4k+6 - load plaintext 276*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 277*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 278*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v11.16b // PRE 1 279*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 280*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 281*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 282*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 283*8fb009dcSAndroid Build Coastguard Worker eor x23, x23, x13 // AES block 4k+7 - round N low 284*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 285*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 286*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH block 4k - mid 287*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high 288*8fb009dcSAndroid Build Coastguard Worker eor x22, x22, x14 // AES block 4k+6 - round N high 289*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH block 4k - mid 290*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 291*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 292*8fb009dcSAndroid Build Coastguard Worker rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) 293*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 294*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 295*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low 296*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid 297*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 298*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 299*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 300*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 301*8fb009dcSAndroid Build Coastguard Worker rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) 302*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high 303*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid 304*8fb009dcSAndroid Build Coastguard Worker rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) 305*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low 306*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high 307*8fb009dcSAndroid Build Coastguard Worker mov d4, v5.d[1] // GHASH block 4k+1 - mid 308*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 309*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 310*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 311*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 312*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low 313*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 314*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 315*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 316*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 317*8fb009dcSAndroid Build Coastguard Worker mov d8, v6.d[1] // GHASH block 4k+2 - mid 318*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 319*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 320*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid 321*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 322*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 323*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 324*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 325*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid 326*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 327*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 328*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid 329*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 330*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 331*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 332*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 333*8fb009dcSAndroid Build Coastguard Worker ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid 334*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 335*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 336*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 337*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 338*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 339*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 340*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 341*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 342*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid 343*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high 344*8fb009dcSAndroid Build Coastguard Worker pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low 345*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 346*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 347*8fb009dcSAndroid Build Coastguard Worker pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low 348*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high 349*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 350*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 351*8fb009dcSAndroid Build Coastguard Worker ldp x19, x20, [x0, #16] // AES block 4k+5 - load plaintext 352*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 353*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 354*8fb009dcSAndroid Build Coastguard Worker mov d4, v7.d[1] // GHASH block 4k+3 - mid 355*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 356*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 357*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low 358*8fb009dcSAndroid Build Coastguard Worker pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid 359*8fb009dcSAndroid Build Coastguard Worker pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high 360*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid 361*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 362*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 363*8fb009dcSAndroid Build Coastguard Worker eor x19, x19, x13 // AES block 4k+5 - round N low 364*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 365*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 366*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid 367*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 368*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 369*8fb009dcSAndroid Build Coastguard Worker eor x21, x21, x13 // AES block 4k+6 - round N low 370*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 371*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 372*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 373*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid 374*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high 375*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 376*8fb009dcSAndroid Build Coastguard Worker fmov d5, x19 // AES block 4k+5 - mov low 377*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0, #0] // AES block 4k+4 - load plaintext 378*8fb009dcSAndroid Build Coastguard Worker b.lt .Lenc_main_loop_continue // branch if AES-128 379*8fb009dcSAndroid Build Coastguard Worker 380*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 381*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 382*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 383*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 384*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 385*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 386*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 387*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 388*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 389*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 390*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 391*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 392*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 393*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 394*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 395*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 396*8fb009dcSAndroid Build Coastguard Worker b.eq .Lenc_main_loop_continue // branch if AES-192 397*8fb009dcSAndroid Build Coastguard Worker 398*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 399*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 400*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 401*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 402*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 403*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 404*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 405*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 406*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 407*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 408*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 409*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 410*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 411*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 412*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 413*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 414*8fb009dcSAndroid Build Coastguard Worker 415*8fb009dcSAndroid Build Coastguard Worker.Lenc_main_loop_continue: 416*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 417*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low 418*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid 419*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+3 420*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 421*8fb009dcSAndroid Build Coastguard Worker add x0, x0, #64 // AES input_ptr update 422*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 423*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+8 424*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 425*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 4k+4 - round N low 426*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up 427*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 4k+4 - round N high 428*8fb009dcSAndroid Build Coastguard Worker fmov d4, x6 // AES block 4k+4 - mov low 429*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+8 430*8fb009dcSAndroid Build Coastguard Worker eor v7.16b, v9.16b, v7.16b // MODULO - fold into mid 431*8fb009dcSAndroid Build Coastguard Worker eor x20, x20, x14 // AES block 4k+5 - round N high 432*8fb009dcSAndroid Build Coastguard Worker eor x24, x24, x14 // AES block 4k+7 - round N high 433*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+8 434*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 4k+4 - round N-1 435*8fb009dcSAndroid Build Coastguard Worker fmov v4.d[1], x7 // AES block 4k+4 - mov high 436*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 437*8fb009dcSAndroid Build Coastguard Worker fmov d7, x23 // AES block 4k+7 - mov low 438*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 4k+5 - round N-1 439*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x20 // AES block 4k+5 - mov high 440*8fb009dcSAndroid Build Coastguard Worker fmov d6, x21 // AES block 4k+6 - mov low 441*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // .LOOP CONTROL 442*8fb009dcSAndroid Build Coastguard Worker fmov v6.d[1], x22 // AES block 4k+6 - mov high 443*8fb009dcSAndroid Build Coastguard Worker pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 444*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result 445*8fb009dcSAndroid Build Coastguard Worker fmov d0, x10 // CTR block 4k+8 446*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x9 // CTR block 4k+8 447*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+9 448*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+9 449*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result 450*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 4k+9 451*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+9 452*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 4k+9 453*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 4k+6 - round N-1 454*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+10 455*8fb009dcSAndroid Build Coastguard Worker st1 { v4.16b}, [x2], #16 // AES block 4k+4 - store result 456*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+10 457*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v9.16b // MODULO - fold into low 458*8fb009dcSAndroid Build Coastguard Worker fmov v7.d[1], x24 // AES block 4k+7 - mov high 459*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 460*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES block 4k+5 - store result 461*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+10 462*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 4k+7 - round N-1 463*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result 464*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 4k+10 465*8fb009dcSAndroid Build Coastguard Worker st1 { v6.16b}, [x2], #16 // AES block 4k+6 - store result 466*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 4k+10 467*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+11 468*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 469*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+11 470*8fb009dcSAndroid Build Coastguard Worker eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result 471*8fb009dcSAndroid Build Coastguard Worker st1 { v7.16b}, [x2], #16 // AES block 4k+7 - store result 472*8fb009dcSAndroid Build Coastguard Worker b.lt .Lenc_main_loop 473*8fb009dcSAndroid Build Coastguard Worker 474*8fb009dcSAndroid Build Coastguard Worker.Lenc_prepretail: // PREPRETAIL 475*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 476*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 477*8fb009dcSAndroid Build Coastguard Worker rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) 478*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 479*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 480*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 4k+3 481*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 482*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 483*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) 484*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 4k+3 485*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 486*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 487*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 488*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 489*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 490*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v11.16b // PRE 1 491*8fb009dcSAndroid Build Coastguard Worker rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) 492*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 493*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 494*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 495*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 496*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH block 4k - mid 497*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 498*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 499*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low 500*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH block 4k - mid 501*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high 502*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 503*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 504*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 505*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 506*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid 507*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 508*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 509*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 510*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 511*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 512*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 513*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid 514*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high 515*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low 516*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 517*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 518*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high 519*8fb009dcSAndroid Build Coastguard Worker mov d4, v5.d[1] // GHASH block 4k+1 - mid 520*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 521*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 522*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low 523*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 524*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 525*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid 526*8fb009dcSAndroid Build Coastguard Worker mov d8, v6.d[1] // GHASH block 4k+2 - mid 527*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 528*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 529*8fb009dcSAndroid Build Coastguard Worker rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) 530*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 531*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 532*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid 533*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid 534*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+3 535*8fb009dcSAndroid Build Coastguard Worker pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low 536*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 537*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 538*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 539*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 540*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid 541*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high 542*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low 543*8fb009dcSAndroid Build Coastguard Worker ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid 544*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 545*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 546*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high 547*8fb009dcSAndroid Build Coastguard Worker mov d4, v7.d[1] // GHASH block 4k+3 - mid 548*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 549*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 550*8fb009dcSAndroid Build Coastguard Worker pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid 551*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid 552*8fb009dcSAndroid Build Coastguard Worker pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high 553*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 554*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 555*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid 556*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid 557*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 558*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 559*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 560*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 561*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 562*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 563*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 564*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 565*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 566*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 567*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 568*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 569*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 570*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high 571*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 572*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 573*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 574*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 575*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 576*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 577*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 578*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid 579*8fb009dcSAndroid Build Coastguard Worker pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low 580*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 581*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 582*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 583*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 584*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 585*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low 586*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 587*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 588*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // karatsuba tidy up 589*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 590*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 591*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v9.1d, v8.1d 592*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 593*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v11.16b 594*8fb009dcSAndroid Build Coastguard Worker b.lt .Lenc_finish_prepretail // branch if AES-128 595*8fb009dcSAndroid Build Coastguard Worker 596*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 597*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 598*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 599*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 600*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 601*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 602*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 603*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 604*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 605*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 606*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 607*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 608*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 609*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 610*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 611*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 612*8fb009dcSAndroid Build Coastguard Worker b.eq .Lenc_finish_prepretail // branch if AES-192 613*8fb009dcSAndroid Build Coastguard Worker 614*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 615*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 616*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 617*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 618*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 619*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 620*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 621*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 622*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 623*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 624*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 625*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 626*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 627*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 628*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 629*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 630*8fb009dcSAndroid Build Coastguard Worker 631*8fb009dcSAndroid Build Coastguard Worker.Lenc_finish_prepretail: 632*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b 633*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b 634*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v10.1d, v8.1d 635*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 636*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 4k+5 - round N-1 637*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v4.16b 638*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 4k+7 - round N-1 639*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 4k+4 - round N-1 640*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 4k+6 - round N-1 641*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b 642*8fb009dcSAndroid Build Coastguard Worker 643*8fb009dcSAndroid Build Coastguard Worker.Lenc_tail: // TAIL 644*8fb009dcSAndroid Build Coastguard Worker ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag 645*8fb009dcSAndroid Build Coastguard Worker sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process 646*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0], #16 // AES block 4k+4 - load plaintext 647*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 4k+4 - round N low 648*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 4k+4 - round N high 649*8fb009dcSAndroid Build Coastguard Worker cmp x5, #48 650*8fb009dcSAndroid Build Coastguard Worker fmov d4, x6 // AES block 4k+4 - mov low 651*8fb009dcSAndroid Build Coastguard Worker fmov v4.d[1], x7 // AES block 4k+4 - mov high 652*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v4.16b, v0.16b // AES block 4k+4 - result 653*8fb009dcSAndroid Build Coastguard Worker b.gt .Lenc_blocks_more_than_3 654*8fb009dcSAndroid Build Coastguard Worker cmp x5, #32 655*8fb009dcSAndroid Build Coastguard Worker mov v3.16b, v2.16b 656*8fb009dcSAndroid Build Coastguard Worker movi v11.8b, #0 657*8fb009dcSAndroid Build Coastguard Worker movi v9.8b, #0 658*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 659*8fb009dcSAndroid Build Coastguard Worker mov v2.16b, v1.16b 660*8fb009dcSAndroid Build Coastguard Worker movi v10.8b, #0 661*8fb009dcSAndroid Build Coastguard Worker b.gt .Lenc_blocks_more_than_2 662*8fb009dcSAndroid Build Coastguard Worker mov v3.16b, v1.16b 663*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 664*8fb009dcSAndroid Build Coastguard Worker cmp x5, #16 665*8fb009dcSAndroid Build Coastguard Worker b.gt .Lenc_blocks_more_than_1 666*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 667*8fb009dcSAndroid Build Coastguard Worker b .Lenc_blocks_less_than_1 668*8fb009dcSAndroid Build Coastguard Worker.Lenc_blocks_more_than_3: // blocks left > 3 669*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES final-3 block - store result 670*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high 671*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-3 block 672*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final-2 block - round N low 673*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 674*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final-2 block - round N high 675*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-3 block - mid 676*8fb009dcSAndroid Build Coastguard Worker fmov d5, x6 // AES final-2 block - mov low 677*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x7 // AES final-2 block - mov high 678*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid 679*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 680*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH final-3 block - mid 681*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low 682*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high 683*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid 684*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v1.16b // AES final-2 block - result 685*8fb009dcSAndroid Build Coastguard Worker.Lenc_blocks_more_than_2: // blocks left > 2 686*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES final-2 block - store result 687*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high 688*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-2 block 689*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final-1 block - round N low 690*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 691*8fb009dcSAndroid Build Coastguard Worker fmov d5, x6 // AES final-1 block - mov low 692*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final-1 block - round N high 693*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x7 // AES final-1 block - mov high 694*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 695*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high 696*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-2 block - mid 697*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low 698*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid 699*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v2.16b // AES final-1 block - result 700*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high 701*8fb009dcSAndroid Build Coastguard Worker pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid 702*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low 703*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid 704*8fb009dcSAndroid Build Coastguard Worker.Lenc_blocks_more_than_1: // blocks left > 1 705*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2], #16 // AES final-1 block - store result 706*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-1 block 707*8fb009dcSAndroid Build Coastguard Worker ldp x6, x7, [x0], #16 // AES final block - load input low & high 708*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 709*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 710*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final block - round N low 711*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-1 block - mid 712*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high 713*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final block - round N high 714*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid 715*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high 716*8fb009dcSAndroid Build Coastguard Worker ins v22.d[1], v22.d[0] // GHASH final-1 block - mid 717*8fb009dcSAndroid Build Coastguard Worker fmov d5, x6 // AES final block - mov low 718*8fb009dcSAndroid Build Coastguard Worker fmov v5.d[1], x7 // AES final block - mov high 719*8fb009dcSAndroid Build Coastguard Worker pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid 720*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low 721*8fb009dcSAndroid Build Coastguard Worker eor v5.16b, v5.16b, v3.16b // AES final block - result 722*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid 723*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low 724*8fb009dcSAndroid Build Coastguard Worker.Lenc_blocks_less_than_1: // blocks left <= 1 725*8fb009dcSAndroid Build Coastguard Worker and x1, x1, #127 // bit_length %= 128 726*8fb009dcSAndroid Build Coastguard Worker mvn x13, xzr // rkN_l = 0xffffffffffffffff 727*8fb009dcSAndroid Build Coastguard Worker sub x1, x1, #128 // bit_length -= 128 728*8fb009dcSAndroid Build Coastguard Worker neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) 729*8fb009dcSAndroid Build Coastguard Worker ld1 { v18.16b}, [x2] // load existing bytes where the possibly partial last block is to be stored 730*8fb009dcSAndroid Build Coastguard Worker mvn x14, xzr // rkN_h = 0xffffffffffffffff 731*8fb009dcSAndroid Build Coastguard Worker and x1, x1, #127 // bit_length %= 128 732*8fb009dcSAndroid Build Coastguard Worker lsr x14, x14, x1 // rkN_h is mask for top 64b of last block 733*8fb009dcSAndroid Build Coastguard Worker cmp x1, #64 734*8fb009dcSAndroid Build Coastguard Worker csel x6, x13, x14, lt 735*8fb009dcSAndroid Build Coastguard Worker csel x7, x14, xzr, lt 736*8fb009dcSAndroid Build Coastguard Worker fmov d0, x6 // ctr0b is mask for last block 737*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x7 738*8fb009dcSAndroid Build Coastguard Worker and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits 739*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final block 740*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 741*8fb009dcSAndroid Build Coastguard Worker bif v5.16b, v18.16b, v0.16b // insert existing bytes in top end of result before storing 742*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high 743*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH final block - mid 744*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 745*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v12.1d // GHASH final block - low 746*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final block - high 747*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH final block - mid 748*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid 749*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final block - low 750*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH final block - mid 751*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 752*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 753*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 754*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up 755*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 756*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 757*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 758*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid 759*8fb009dcSAndroid Build Coastguard Worker pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 760*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 761*8fb009dcSAndroid Build Coastguard Worker str w9, [x16, #12] // store the updated counter 762*8fb009dcSAndroid Build Coastguard Worker st1 { v5.16b}, [x2] // store all 16B 763*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v9.16b // MODULO - fold into low 764*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 765*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 766*8fb009dcSAndroid Build Coastguard Worker rev64 v11.16b, v11.16b 767*8fb009dcSAndroid Build Coastguard Worker mov x0, x15 768*8fb009dcSAndroid Build Coastguard Worker st1 { v11.16b }, [x3] 769*8fb009dcSAndroid Build Coastguard Worker ldp x19, x20, [sp, #16] 770*8fb009dcSAndroid Build Coastguard Worker ldp x21, x22, [sp, #32] 771*8fb009dcSAndroid Build Coastguard Worker ldp x23, x24, [sp, #48] 772*8fb009dcSAndroid Build Coastguard Worker ldp d8, d9, [sp, #64] 773*8fb009dcSAndroid Build Coastguard Worker ldp d10, d11, [sp, #80] 774*8fb009dcSAndroid Build Coastguard Worker ldp d12, d13, [sp, #96] 775*8fb009dcSAndroid Build Coastguard Worker ldp d14, d15, [sp, #112] 776*8fb009dcSAndroid Build Coastguard Worker ldp x29, x30, [sp], #128 777*8fb009dcSAndroid Build Coastguard Worker AARCH64_VALIDATE_LINK_REGISTER 778*8fb009dcSAndroid Build Coastguard Worker ret 779*8fb009dcSAndroid Build Coastguard Worker.size aes_gcm_enc_kernel,.-aes_gcm_enc_kernel 780*8fb009dcSAndroid Build Coastguard Worker.globl aes_gcm_dec_kernel 781*8fb009dcSAndroid Build Coastguard Worker.hidden aes_gcm_dec_kernel 782*8fb009dcSAndroid Build Coastguard Worker.type aes_gcm_dec_kernel,%function 783*8fb009dcSAndroid Build Coastguard Worker.align 4 784*8fb009dcSAndroid Build Coastguard Workeraes_gcm_dec_kernel: 785*8fb009dcSAndroid Build Coastguard Worker AARCH64_SIGN_LINK_REGISTER 786*8fb009dcSAndroid Build Coastguard Worker stp x29, x30, [sp, #-128]! 787*8fb009dcSAndroid Build Coastguard Worker mov x29, sp 788*8fb009dcSAndroid Build Coastguard Worker stp x19, x20, [sp, #16] 789*8fb009dcSAndroid Build Coastguard Worker mov x16, x4 790*8fb009dcSAndroid Build Coastguard Worker mov x8, x5 791*8fb009dcSAndroid Build Coastguard Worker stp x21, x22, [sp, #32] 792*8fb009dcSAndroid Build Coastguard Worker stp x23, x24, [sp, #48] 793*8fb009dcSAndroid Build Coastguard Worker stp d8, d9, [sp, #64] 794*8fb009dcSAndroid Build Coastguard Worker stp d10, d11, [sp, #80] 795*8fb009dcSAndroid Build Coastguard Worker stp d12, d13, [sp, #96] 796*8fb009dcSAndroid Build Coastguard Worker stp d14, d15, [sp, #112] 797*8fb009dcSAndroid Build Coastguard Worker ldr w17, [x8, #240] 798*8fb009dcSAndroid Build Coastguard Worker add x19, x8, x17, lsl #4 // borrow input_l1 for last key 799*8fb009dcSAndroid Build Coastguard Worker ldp x13, x14, [x19] // load round N keys 800*8fb009dcSAndroid Build Coastguard Worker ldr q31, [x19, #-16] // load round N-1 keys 801*8fb009dcSAndroid Build Coastguard Worker lsr x5, x1, #3 // byte_len 802*8fb009dcSAndroid Build Coastguard Worker mov x15, x5 803*8fb009dcSAndroid Build Coastguard Worker ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 804*8fb009dcSAndroid Build Coastguard Worker ldr q26, [x8, #128] // load rk8 805*8fb009dcSAndroid Build Coastguard Worker sub x5, x5, #1 // byte_len - 1 806*8fb009dcSAndroid Build Coastguard Worker ldr q25, [x8, #112] // load rk7 807*8fb009dcSAndroid Build Coastguard Worker and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) 808*8fb009dcSAndroid Build Coastguard Worker add x4, x0, x1, lsr #3 // end_input_ptr 809*8fb009dcSAndroid Build Coastguard Worker ldr q24, [x8, #96] // load rk6 810*8fb009dcSAndroid Build Coastguard Worker lsr x12, x11, #32 811*8fb009dcSAndroid Build Coastguard Worker ldr q23, [x8, #80] // load rk5 812*8fb009dcSAndroid Build Coastguard Worker orr w11, w11, w11 813*8fb009dcSAndroid Build Coastguard Worker ldr q21, [x8, #48] // load rk3 814*8fb009dcSAndroid Build Coastguard Worker add x5, x5, x0 815*8fb009dcSAndroid Build Coastguard Worker rev w12, w12 // rev_ctr32 816*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // increment rev_ctr32 817*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 3 818*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 1 819*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 1 820*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 1 821*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 1 822*8fb009dcSAndroid Build Coastguard Worker ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible 823*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 1 824*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 2 825*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 2 826*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 2 827*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 2 828*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 2 829*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 3 830*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 3 831*8fb009dcSAndroid Build Coastguard Worker ldr q18, [x8, #0] // load rk0 832*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 3 833*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 3 834*8fb009dcSAndroid Build Coastguard Worker ldr q22, [x8, #64] // load rk4 835*8fb009dcSAndroid Build Coastguard Worker ldr q19, [x8, #16] // load rk1 836*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 837*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 0 838*8fb009dcSAndroid Build Coastguard Worker ldr q14, [x6, #48] // load h3l | h3h 839*8fb009dcSAndroid Build Coastguard Worker ext v14.16b, v14.16b, v14.16b, #8 840*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 841*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 0 842*8fb009dcSAndroid Build Coastguard Worker ldr q15, [x6, #80] // load h4l | h4h 843*8fb009dcSAndroid Build Coastguard Worker ext v15.16b, v15.16b, v15.16b, #8 844*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 845*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 0 846*8fb009dcSAndroid Build Coastguard Worker ldr q13, [x6, #32] // load h2l | h2h 847*8fb009dcSAndroid Build Coastguard Worker ext v13.16b, v13.16b, v13.16b, #8 848*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 849*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 0 850*8fb009dcSAndroid Build Coastguard Worker ldr q20, [x8, #32] // load rk2 851*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 852*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 1 853*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 854*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 1 855*8fb009dcSAndroid Build Coastguard Worker ld1 { v11.16b}, [x3] 856*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 857*8fb009dcSAndroid Build Coastguard Worker rev64 v11.16b, v11.16b 858*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 859*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 1 860*8fb009dcSAndroid Build Coastguard Worker ldr q27, [x8, #144] // load rk9 861*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 862*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 1 863*8fb009dcSAndroid Build Coastguard Worker ldr q30, [x8, #192] // load rk12 864*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 865*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 2 866*8fb009dcSAndroid Build Coastguard Worker ldr q12, [x6] // load h1l | h1h 867*8fb009dcSAndroid Build Coastguard Worker ext v12.16b, v12.16b, v12.16b, #8 868*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 869*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 2 870*8fb009dcSAndroid Build Coastguard Worker ldr q28, [x8, #160] // load rk10 871*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 872*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 2 873*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 874*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 3 875*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 876*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 2 877*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 878*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 3 879*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 880*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 4 881*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 882*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 3 883*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 884*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 3 885*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 886*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 4 887*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 888*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 4 889*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 890*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 4 891*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 892*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 5 893*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 894*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 5 895*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 896*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 5 897*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 898*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 5 899*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 900*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 6 901*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 902*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 6 903*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 904*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 905*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 6 906*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 907*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 6 908*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 909*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 7 910*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 911*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 7 912*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 913*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 7 914*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 915*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 8 916*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 917*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 7 918*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 919*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 8 920*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 921*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 8 922*8fb009dcSAndroid Build Coastguard Worker ldr q29, [x8, #176] // load rk11 923*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 924*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 8 925*8fb009dcSAndroid Build Coastguard Worker b.lt .Ldec_finish_first_blocks // branch if AES-128 926*8fb009dcSAndroid Build Coastguard Worker 927*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 928*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 9 929*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 930*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 9 931*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 932*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 9 933*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 934*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 9 935*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 936*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 10 937*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 938*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 10 939*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 940*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 10 941*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 942*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 10 943*8fb009dcSAndroid Build Coastguard Worker b.eq .Ldec_finish_first_blocks // branch if AES-192 944*8fb009dcSAndroid Build Coastguard Worker 945*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 946*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 11 947*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 948*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 11 949*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 950*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 11 951*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 952*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 11 953*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 954*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 1 - round 12 955*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 956*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 0 - round 12 957*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 958*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 2 - round 12 959*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 960*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 3 - round 12 961*8fb009dcSAndroid Build Coastguard Worker 962*8fb009dcSAndroid Build Coastguard Worker.Ldec_finish_first_blocks: 963*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // check if we have <= 4 blocks 964*8fb009dcSAndroid Build Coastguard Worker trn1 v9.2d, v14.2d, v15.2d // h4h | h3h 965*8fb009dcSAndroid Build Coastguard Worker trn2 v17.2d, v14.2d, v15.2d // h4l | h3l 966*8fb009dcSAndroid Build Coastguard Worker trn1 v8.2d, v12.2d, v13.2d // h2h | h1h 967*8fb009dcSAndroid Build Coastguard Worker trn2 v16.2d, v12.2d, v13.2d // h2l | h1l 968*8fb009dcSAndroid Build Coastguard Worker eor v17.16b, v17.16b, v9.16b // h4k | h3k 969*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 1 - round N-1 970*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 2 - round N-1 971*8fb009dcSAndroid Build Coastguard Worker eor v16.16b, v16.16b, v8.16b // h2k | h1k 972*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 3 - round N-1 973*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 0 - round N-1 974*8fb009dcSAndroid Build Coastguard Worker b.ge .Ldec_tail // handle tail 975*8fb009dcSAndroid Build Coastguard Worker 976*8fb009dcSAndroid Build Coastguard Worker ldr q4, [x0, #0] // AES block 0 - load ciphertext 977*8fb009dcSAndroid Build Coastguard Worker ldr q5, [x0, #16] // AES block 1 - load ciphertext 978*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4 979*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v4.16b, v0.16b // AES block 0 - result 980*8fb009dcSAndroid Build Coastguard Worker eor v1.16b, v5.16b, v1.16b // AES block 1 - result 981*8fb009dcSAndroid Build Coastguard Worker rev64 v5.16b, v5.16b // GHASH block 1 982*8fb009dcSAndroid Build Coastguard Worker ldr q7, [x0, #48] // AES block 3 - load ciphertext 983*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES block 0 - mov high 984*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES block 0 - mov low 985*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v4.16b // GHASH block 0 986*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4 987*8fb009dcSAndroid Build Coastguard Worker fmov d0, x10 // CTR block 4 988*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4 989*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x9 // CTR block 4 990*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 5 991*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 5 992*8fb009dcSAndroid Build Coastguard Worker mov x19, v1.d[0] // AES block 1 - mov low 993*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 5 994*8fb009dcSAndroid Build Coastguard Worker mov x20, v1.d[1] // AES block 1 - mov high 995*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 0 - round N high 996*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 0 - round N low 997*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES block 0 - store result 998*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 5 999*8fb009dcSAndroid Build Coastguard Worker ldr q6, [x0, #32] // AES block 2 - load ciphertext 1000*8fb009dcSAndroid Build Coastguard Worker add x0, x0, #64 // AES input_ptr update 1001*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 5 1002*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 6 1003*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 6 1004*8fb009dcSAndroid Build Coastguard Worker eor x19, x19, x13 // AES block 1 - round N low 1005*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 6 1006*8fb009dcSAndroid Build Coastguard Worker eor x20, x20, x14 // AES block 1 - round N high 1007*8fb009dcSAndroid Build Coastguard Worker stp x19, x20, [x2], #16 // AES block 1 - store result 1008*8fb009dcSAndroid Build Coastguard Worker eor v2.16b, v6.16b, v2.16b // AES block 2 - result 1009*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // check if we have <= 8 blocks 1010*8fb009dcSAndroid Build Coastguard Worker b.ge .Ldec_prepretail // do prepretail 1011*8fb009dcSAndroid Build Coastguard Worker 1012*8fb009dcSAndroid Build Coastguard Worker.Ldec_main_loop: // main loop start 1013*8fb009dcSAndroid Build Coastguard Worker mov x21, v2.d[0] // AES block 4k+2 - mov low 1014*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 1015*8fb009dcSAndroid Build Coastguard Worker eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result 1016*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 1017*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 1018*8fb009dcSAndroid Build Coastguard Worker mov x22, v2.d[1] // AES block 4k+2 - mov high 1019*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 1020*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 1021*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 4k+6 1022*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 4k+6 1023*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v11.16b // PRE 1 1024*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+7 1025*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 1026*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 1027*8fb009dcSAndroid Build Coastguard Worker mov x24, v3.d[1] // AES block 4k+3 - mov high 1028*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 1029*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 1030*8fb009dcSAndroid Build Coastguard Worker mov x23, v3.d[0] // AES block 4k+3 - mov low 1031*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high 1032*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH block 4k - mid 1033*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 4k+7 1034*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 1035*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 1036*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+7 1037*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 1038*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 1039*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 4k+7 1040*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 1041*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 1042*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid 1043*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 1044*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 1045*8fb009dcSAndroid Build Coastguard Worker eor x22, x22, x14 // AES block 4k+2 - round N high 1046*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 1047*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 1048*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH block 4k - mid 1049*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 1050*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 1051*8fb009dcSAndroid Build Coastguard Worker rev64 v6.16b, v6.16b // GHASH block 4k+2 1052*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 1053*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 1054*8fb009dcSAndroid Build Coastguard Worker eor x21, x21, x13 // AES block 4k+2 - round N low 1055*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 1056*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 1057*8fb009dcSAndroid Build Coastguard Worker stp x21, x22, [x2], #16 // AES block 4k+2 - store result 1058*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low 1059*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high 1060*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 1061*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 1062*8fb009dcSAndroid Build Coastguard Worker rev64 v7.16b, v7.16b // GHASH block 4k+3 1063*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid 1064*8fb009dcSAndroid Build Coastguard Worker eor x23, x23, x13 // AES block 4k+3 - round N low 1065*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low 1066*8fb009dcSAndroid Build Coastguard Worker eor x24, x24, x14 // AES block 4k+3 - round N high 1067*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high 1068*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 1069*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 1070*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 1071*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 1072*8fb009dcSAndroid Build Coastguard Worker mov d4, v5.d[1] // GHASH block 4k+1 - mid 1073*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 1074*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 1075*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low 1076*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 1077*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 1078*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+7 1079*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 1080*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 1081*8fb009dcSAndroid Build Coastguard Worker mov d8, v6.d[1] // GHASH block 4k+2 - mid 1082*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 1083*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 1084*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid 1085*8fb009dcSAndroid Build Coastguard Worker pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low 1086*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 1087*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 1088*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid 1089*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 1090*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 1091*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 1092*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 1093*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low 1094*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid 1095*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+8 1096*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 1097*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 1098*8fb009dcSAndroid Build Coastguard Worker ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid 1099*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 1100*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 1101*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+8 1102*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 1103*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 1104*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 1105*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 1106*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid 1107*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 1108*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 1109*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high 1110*8fb009dcSAndroid Build Coastguard Worker mov d6, v7.d[1] // GHASH block 4k+3 - mid 1111*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 1112*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 1113*8fb009dcSAndroid Build Coastguard Worker pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid 1114*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 1115*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 1116*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high 1117*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 1118*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 1119*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low 1120*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+8 1121*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid 1122*8fb009dcSAndroid Build Coastguard Worker pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high 1123*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 1124*8fb009dcSAndroid Build Coastguard Worker eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid 1125*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 1126*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 1127*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 1128*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 1129*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high 1130*8fb009dcSAndroid Build Coastguard Worker pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid 1131*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 1132*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 1133*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 1134*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low 1135*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 1136*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 1137*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 1138*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 1139*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 1140*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid 1141*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 1142*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 1143*8fb009dcSAndroid Build Coastguard Worker b.lt .Ldec_main_loop_continue // branch if AES-128 1144*8fb009dcSAndroid Build Coastguard Worker 1145*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 1146*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 1147*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 1148*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 1149*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 1150*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 1151*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 1152*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 1153*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 1154*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 1155*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 1156*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 1157*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 1158*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 1159*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 1160*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 1161*8fb009dcSAndroid Build Coastguard Worker b.eq .Ldec_main_loop_continue // branch if AES-192 1162*8fb009dcSAndroid Build Coastguard Worker 1163*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 1164*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 1165*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 1166*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 1167*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 1168*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 1169*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 1170*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 1171*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 1172*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 1173*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 1174*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 1175*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 1176*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 1177*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 1178*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 1179*8fb009dcSAndroid Build Coastguard Worker 1180*8fb009dcSAndroid Build Coastguard Worker.Ldec_main_loop_continue: 1181*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 1182*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 1183*8fb009dcSAndroid Build Coastguard Worker ldr q4, [x0, #0] // AES block 4k+4 - load ciphertext 1184*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 4k+4 - round N-1 1185*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 1186*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up 1187*8fb009dcSAndroid Build Coastguard Worker ldr q5, [x0, #16] // AES block 4k+5 - load ciphertext 1188*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v4.16b, v0.16b // AES block 4k+4 - result 1189*8fb009dcSAndroid Build Coastguard Worker stp x23, x24, [x2], #16 // AES block 4k+3 - store result 1190*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 1191*8fb009dcSAndroid Build Coastguard Worker ldr q7, [x0, #48] // AES block 4k+7 - load ciphertext 1192*8fb009dcSAndroid Build Coastguard Worker ldr q6, [x0, #32] // AES block 4k+6 - load ciphertext 1193*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES block 4k+4 - mov high 1194*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid 1195*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 4k+5 - round N-1 1196*8fb009dcSAndroid Build Coastguard Worker add x0, x0, #64 // AES input_ptr update 1197*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES block 4k+4 - mov low 1198*8fb009dcSAndroid Build Coastguard Worker fmov d0, x10 // CTR block 4k+8 1199*8fb009dcSAndroid Build Coastguard Worker fmov v0.d[1], x9 // CTR block 4k+8 1200*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 1201*8fb009dcSAndroid Build Coastguard Worker eor v1.16b, v5.16b, v1.16b // AES block 4k+5 - result 1202*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+9 1203*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 4k+6 - round N-1 1204*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+9 1205*8fb009dcSAndroid Build Coastguard Worker cmp x0, x5 // .LOOP CONTROL 1206*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+9 1207*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 4k+4 - round N low 1208*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 4k+4 - round N high 1209*8fb009dcSAndroid Build Coastguard Worker mov x20, v1.d[1] // AES block 4k+5 - mov high 1210*8fb009dcSAndroid Build Coastguard Worker eor v2.16b, v6.16b, v2.16b // AES block 4k+6 - result 1211*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // MODULO - fold into low 1212*8fb009dcSAndroid Build Coastguard Worker mov x19, v1.d[0] // AES block 4k+5 - mov low 1213*8fb009dcSAndroid Build Coastguard Worker fmov d1, x10 // CTR block 4k+9 1214*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 1215*8fb009dcSAndroid Build Coastguard Worker fmov v1.d[1], x9 // CTR block 4k+9 1216*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+10 1217*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+10 1218*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 4k+7 - round N-1 1219*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+10 1220*8fb009dcSAndroid Build Coastguard Worker rev64 v5.16b, v5.16b // GHASH block 4k+5 1221*8fb009dcSAndroid Build Coastguard Worker eor x20, x20, x14 // AES block 4k+5 - round N high 1222*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES block 4k+4 - store result 1223*8fb009dcSAndroid Build Coastguard Worker eor x19, x19, x13 // AES block 4k+5 - round N low 1224*8fb009dcSAndroid Build Coastguard Worker stp x19, x20, [x2], #16 // AES block 4k+5 - store result 1225*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v4.16b // GHASH block 4k+4 1226*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 1227*8fb009dcSAndroid Build Coastguard Worker b.lt .Ldec_main_loop 1228*8fb009dcSAndroid Build Coastguard Worker 1229*8fb009dcSAndroid Build Coastguard Worker.Ldec_prepretail: // PREPRETAIL 1230*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 1231*8fb009dcSAndroid Build Coastguard Worker mov x21, v2.d[0] // AES block 4k+2 - mov low 1232*8fb009dcSAndroid Build Coastguard Worker eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result 1233*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v18.16b 1234*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 1235*8fb009dcSAndroid Build Coastguard Worker mov x22, v2.d[1] // AES block 4k+2 - mov high 1236*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v18.16b 1237*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 1238*8fb009dcSAndroid Build Coastguard Worker fmov d2, x10 // CTR block 4k+6 1239*8fb009dcSAndroid Build Coastguard Worker fmov v2.d[1], x9 // CTR block 4k+6 1240*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 // CTR block 4k+7 1241*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v11.16b // PRE 1 1242*8fb009dcSAndroid Build Coastguard Worker rev64 v6.16b, v6.16b // GHASH block 4k+2 1243*8fb009dcSAndroid Build Coastguard Worker orr x9, x11, x9, lsl #32 // CTR block 4k+7 1244*8fb009dcSAndroid Build Coastguard Worker mov x23, v3.d[0] // AES block 4k+3 - mov low 1245*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v19.16b 1246*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 1247*8fb009dcSAndroid Build Coastguard Worker mov x24, v3.d[1] // AES block 4k+3 - mov high 1248*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low 1249*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH block 4k - mid 1250*8fb009dcSAndroid Build Coastguard Worker fmov d3, x10 // CTR block 4k+7 1251*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high 1252*8fb009dcSAndroid Build Coastguard Worker fmov v3.d[1], x9 // CTR block 4k+7 1253*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v18.16b 1254*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 1255*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH block 4k - mid 1256*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v19.16b 1257*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 1258*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid 1259*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high 1260*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v19.16b 1261*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 1262*8fb009dcSAndroid Build Coastguard Worker rev64 v7.16b, v7.16b // GHASH block 4k+3 1263*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v18.16b 1264*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 1265*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid 1266*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high 1267*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low 1268*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v19.16b 1269*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 1270*8fb009dcSAndroid Build Coastguard Worker mov d4, v5.d[1] // GHASH block 4k+1 - mid 1271*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v20.16b 1272*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 1273*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v20.16b 1274*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 1275*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low 1276*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v20.16b 1277*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 1278*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v21.16b 1279*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 1280*8fb009dcSAndroid Build Coastguard Worker mov d8, v6.d[1] // GHASH block 4k+2 - mid 1281*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v20.16b 1282*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 1283*8fb009dcSAndroid Build Coastguard Worker eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid 1284*8fb009dcSAndroid Build Coastguard Worker pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low 1285*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v22.16b 1286*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 1287*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v21.16b 1288*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 1289*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid 1290*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid 1291*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v23.16b 1292*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 1293*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low 1294*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v22.16b 1295*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 1296*8fb009dcSAndroid Build Coastguard Worker pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high 1297*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid 1298*8fb009dcSAndroid Build Coastguard Worker pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high 1299*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v23.16b 1300*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 1301*8fb009dcSAndroid Build Coastguard Worker ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid 1302*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v21.16b 1303*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 1304*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v21.16b 1305*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 1306*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high 1307*8fb009dcSAndroid Build Coastguard Worker pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low 1308*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v22.16b 1309*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 1310*8fb009dcSAndroid Build Coastguard Worker mov d6, v7.d[1] // GHASH block 4k+3 - mid 1311*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v22.16b 1312*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 1313*8fb009dcSAndroid Build Coastguard Worker pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid 1314*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v23.16b 1315*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 1316*8fb009dcSAndroid Build Coastguard Worker eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid 1317*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v23.16b 1318*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 1319*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v24.16b 1320*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 1321*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid 1322*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v24.16b 1323*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 1324*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v24.16b 1325*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 1326*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 1327*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v24.16b 1328*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 1329*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low 1330*8fb009dcSAndroid Build Coastguard Worker pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid 1331*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v25.16b 1332*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 1333*8fb009dcSAndroid Build Coastguard Worker cmp x17, #12 // setup flags for AES-128/192/256 check 1334*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high 1335*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v25.16b 1336*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 1337*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v25.16b 1338*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 1339*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid 1340*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v26.16b 1341*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 1342*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v25.16b 1343*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 1344*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 1345*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v26.16b 1346*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 1347*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v26.16b 1348*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 1349*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 1350*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v26.16b 1351*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 1352*8fb009dcSAndroid Build Coastguard Worker b.lt .Ldec_finish_prepretail // branch if AES-128 1353*8fb009dcSAndroid Build Coastguard Worker 1354*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v27.16b 1355*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 1356*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v27.16b 1357*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 1358*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v27.16b 1359*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 1360*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v27.16b 1361*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 1362*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v28.16b 1363*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 1364*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v28.16b 1365*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 1366*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v28.16b 1367*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 1368*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v28.16b 1369*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 1370*8fb009dcSAndroid Build Coastguard Worker b.eq .Ldec_finish_prepretail // branch if AES-192 1371*8fb009dcSAndroid Build Coastguard Worker 1372*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v29.16b 1373*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 1374*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v29.16b 1375*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 1376*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v29.16b 1377*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 1378*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v30.16b 1379*8fb009dcSAndroid Build Coastguard Worker aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 1380*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v29.16b 1381*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 1382*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v30.16b 1383*8fb009dcSAndroid Build Coastguard Worker aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 1384*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v30.16b 1385*8fb009dcSAndroid Build Coastguard Worker aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 1386*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v30.16b 1387*8fb009dcSAndroid Build Coastguard Worker aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 1388*8fb009dcSAndroid Build Coastguard Worker 1389*8fb009dcSAndroid Build Coastguard Worker.Ldec_finish_prepretail: 1390*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up 1391*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 1392*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 1393*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 1394*8fb009dcSAndroid Build Coastguard Worker eor x22, x22, x14 // AES block 4k+2 - round N high 1395*8fb009dcSAndroid Build Coastguard Worker eor x23, x23, x13 // AES block 4k+3 - round N low 1396*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid 1397*8fb009dcSAndroid Build Coastguard Worker add w12, w12, #1 // CTR block 4k+7 1398*8fb009dcSAndroid Build Coastguard Worker eor x21, x21, x13 // AES block 4k+2 - round N low 1399*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 1400*8fb009dcSAndroid Build Coastguard Worker eor x24, x24, x14 // AES block 4k+3 - round N high 1401*8fb009dcSAndroid Build Coastguard Worker stp x21, x22, [x2], #16 // AES block 4k+2 - store result 1402*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 1403*8fb009dcSAndroid Build Coastguard Worker stp x23, x24, [x2], #16 // AES block 4k+3 - store result 1404*8fb009dcSAndroid Build Coastguard Worker 1405*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // MODULO - fold into low 1406*8fb009dcSAndroid Build Coastguard Worker aese v1.16b, v31.16b // AES block 4k+5 - round N-1 1407*8fb009dcSAndroid Build Coastguard Worker aese v0.16b, v31.16b // AES block 4k+4 - round N-1 1408*8fb009dcSAndroid Build Coastguard Worker aese v3.16b, v31.16b // AES block 4k+7 - round N-1 1409*8fb009dcSAndroid Build Coastguard Worker aese v2.16b, v31.16b // AES block 4k+6 - round N-1 1410*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 1411*8fb009dcSAndroid Build Coastguard Worker 1412*8fb009dcSAndroid Build Coastguard Worker.Ldec_tail: // TAIL 1413*8fb009dcSAndroid Build Coastguard Worker sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process 1414*8fb009dcSAndroid Build Coastguard Worker ld1 { v5.16b}, [x0], #16 // AES block 4k+4 - load ciphertext 1415*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v5.16b, v0.16b // AES block 4k+4 - result 1416*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES block 4k+4 - mov low 1417*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES block 4k+4 - mov high 1418*8fb009dcSAndroid Build Coastguard Worker ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag 1419*8fb009dcSAndroid Build Coastguard Worker cmp x5, #48 1420*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES block 4k+4 - round N low 1421*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES block 4k+4 - round N high 1422*8fb009dcSAndroid Build Coastguard Worker b.gt .Ldec_blocks_more_than_3 1423*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 1424*8fb009dcSAndroid Build Coastguard Worker mov v3.16b, v2.16b 1425*8fb009dcSAndroid Build Coastguard Worker movi v10.8b, #0 1426*8fb009dcSAndroid Build Coastguard Worker movi v11.8b, #0 1427*8fb009dcSAndroid Build Coastguard Worker cmp x5, #32 1428*8fb009dcSAndroid Build Coastguard Worker movi v9.8b, #0 1429*8fb009dcSAndroid Build Coastguard Worker mov v2.16b, v1.16b 1430*8fb009dcSAndroid Build Coastguard Worker b.gt .Ldec_blocks_more_than_2 1431*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 1432*8fb009dcSAndroid Build Coastguard Worker mov v3.16b, v1.16b 1433*8fb009dcSAndroid Build Coastguard Worker cmp x5, #16 1434*8fb009dcSAndroid Build Coastguard Worker b.gt .Ldec_blocks_more_than_1 1435*8fb009dcSAndroid Build Coastguard Worker sub w12, w12, #1 1436*8fb009dcSAndroid Build Coastguard Worker b .Ldec_blocks_less_than_1 1437*8fb009dcSAndroid Build Coastguard Worker.Ldec_blocks_more_than_3: // blocks left > 3 1438*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-3 block 1439*8fb009dcSAndroid Build Coastguard Worker ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext 1440*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES final-3 block - store result 1441*8fb009dcSAndroid Build Coastguard Worker mov d10, v17.d[1] // GHASH final-3 block - mid 1442*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 1443*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v5.16b, v1.16b // AES final-2 block - result 1444*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-3 block - mid 1445*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES final-2 block - mov low 1446*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES final-2 block - mov high 1447*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid 1448*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 1449*8fb009dcSAndroid Build Coastguard Worker pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high 1450*8fb009dcSAndroid Build Coastguard Worker pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid 1451*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final-2 block - round N low 1452*8fb009dcSAndroid Build Coastguard Worker pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low 1453*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final-2 block - round N high 1454*8fb009dcSAndroid Build Coastguard Worker.Ldec_blocks_more_than_2: // blocks left > 2 1455*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-2 block 1456*8fb009dcSAndroid Build Coastguard Worker ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext 1457*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 1458*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES final-2 block - store result 1459*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v5.16b, v2.16b // AES final-1 block - result 1460*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-2 block - mid 1461*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low 1462*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high 1463*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid 1464*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES final-1 block - mov low 1465*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES final-1 block - mov high 1466*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low 1467*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 1468*8fb009dcSAndroid Build Coastguard Worker pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid 1469*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high 1470*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final-1 block - round N low 1471*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid 1472*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final-1 block - round N high 1473*8fb009dcSAndroid Build Coastguard Worker.Ldec_blocks_more_than_1: // blocks left > 1 1474*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2], #16 // AES final-1 block - store result 1475*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final-1 block 1476*8fb009dcSAndroid Build Coastguard Worker ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext 1477*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 1478*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0 // suppress further partial tag feed in 1479*8fb009dcSAndroid Build Coastguard Worker mov d22, v4.d[1] // GHASH final-1 block - mid 1480*8fb009dcSAndroid Build Coastguard Worker eor v0.16b, v5.16b, v3.16b // AES final block - result 1481*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high 1482*8fb009dcSAndroid Build Coastguard Worker eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid 1483*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low 1484*8fb009dcSAndroid Build Coastguard Worker mov x6, v0.d[0] // AES final block - mov low 1485*8fb009dcSAndroid Build Coastguard Worker ins v22.d[1], v22.d[0] // GHASH final-1 block - mid 1486*8fb009dcSAndroid Build Coastguard Worker mov x7, v0.d[1] // AES final block - mov high 1487*8fb009dcSAndroid Build Coastguard Worker pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid 1488*8fb009dcSAndroid Build Coastguard Worker eor x6, x6, x13 // AES final block - round N low 1489*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low 1490*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high 1491*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid 1492*8fb009dcSAndroid Build Coastguard Worker eor x7, x7, x14 // AES final block - round N high 1493*8fb009dcSAndroid Build Coastguard Worker.Ldec_blocks_less_than_1: // blocks left <= 1 1494*8fb009dcSAndroid Build Coastguard Worker and x1, x1, #127 // bit_length %= 128 1495*8fb009dcSAndroid Build Coastguard Worker mvn x14, xzr // rkN_h = 0xffffffffffffffff 1496*8fb009dcSAndroid Build Coastguard Worker sub x1, x1, #128 // bit_length -= 128 1497*8fb009dcSAndroid Build Coastguard Worker mvn x13, xzr // rkN_l = 0xffffffffffffffff 1498*8fb009dcSAndroid Build Coastguard Worker ldp x4, x5, [x2] // load existing bytes we need to not overwrite 1499*8fb009dcSAndroid Build Coastguard Worker neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) 1500*8fb009dcSAndroid Build Coastguard Worker and x1, x1, #127 // bit_length %= 128 1501*8fb009dcSAndroid Build Coastguard Worker lsr x14, x14, x1 // rkN_h is mask for top 64b of last block 1502*8fb009dcSAndroid Build Coastguard Worker cmp x1, #64 1503*8fb009dcSAndroid Build Coastguard Worker csel x9, x13, x14, lt 1504*8fb009dcSAndroid Build Coastguard Worker csel x10, x14, xzr, lt 1505*8fb009dcSAndroid Build Coastguard Worker fmov d0, x9 // ctr0b is mask for last block 1506*8fb009dcSAndroid Build Coastguard Worker and x6, x6, x9 1507*8fb009dcSAndroid Build Coastguard Worker mov v0.d[1], x10 1508*8fb009dcSAndroid Build Coastguard Worker bic x4, x4, x9 // mask out low existing bytes 1509*8fb009dcSAndroid Build Coastguard Worker rev w9, w12 1510*8fb009dcSAndroid Build Coastguard Worker bic x5, x5, x10 // mask out high existing bytes 1511*8fb009dcSAndroid Build Coastguard Worker orr x6, x6, x4 1512*8fb009dcSAndroid Build Coastguard Worker and x7, x7, x10 1513*8fb009dcSAndroid Build Coastguard Worker orr x7, x7, x5 1514*8fb009dcSAndroid Build Coastguard Worker and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits 1515*8fb009dcSAndroid Build Coastguard Worker rev64 v4.16b, v5.16b // GHASH final block 1516*8fb009dcSAndroid Build Coastguard Worker eor v4.16b, v4.16b, v8.16b // feed in partial tag 1517*8fb009dcSAndroid Build Coastguard Worker pmull v21.1q, v4.1d, v12.1d // GHASH final block - low 1518*8fb009dcSAndroid Build Coastguard Worker mov d8, v4.d[1] // GHASH final block - mid 1519*8fb009dcSAndroid Build Coastguard Worker eor v8.8b, v8.8b, v4.8b // GHASH final block - mid 1520*8fb009dcSAndroid Build Coastguard Worker pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high 1521*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid 1522*8fb009dcSAndroid Build Coastguard Worker eor v9.16b, v9.16b, v20.16b // GHASH final block - high 1523*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v21.16b // GHASH final block - low 1524*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v8.16b // GHASH final block - mid 1525*8fb009dcSAndroid Build Coastguard Worker movi v8.8b, #0xc2 1526*8fb009dcSAndroid Build Coastguard Worker eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up 1527*8fb009dcSAndroid Build Coastguard Worker shl d8, d8, #56 // mod_constant 1528*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up 1529*8fb009dcSAndroid Build Coastguard Worker pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid 1530*8fb009dcSAndroid Build Coastguard Worker ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment 1531*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid 1532*8fb009dcSAndroid Build Coastguard Worker eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid 1533*8fb009dcSAndroid Build Coastguard Worker pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low 1534*8fb009dcSAndroid Build Coastguard Worker ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment 1535*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v8.16b // MODULO - fold into low 1536*8fb009dcSAndroid Build Coastguard Worker stp x6, x7, [x2] 1537*8fb009dcSAndroid Build Coastguard Worker str w9, [x16, #12] // store the updated counter 1538*8fb009dcSAndroid Build Coastguard Worker eor v11.16b, v11.16b, v10.16b // MODULO - fold into low 1539*8fb009dcSAndroid Build Coastguard Worker ext v11.16b, v11.16b, v11.16b, #8 1540*8fb009dcSAndroid Build Coastguard Worker rev64 v11.16b, v11.16b 1541*8fb009dcSAndroid Build Coastguard Worker mov x0, x15 1542*8fb009dcSAndroid Build Coastguard Worker st1 { v11.16b }, [x3] 1543*8fb009dcSAndroid Build Coastguard Worker ldp x19, x20, [sp, #16] 1544*8fb009dcSAndroid Build Coastguard Worker ldp x21, x22, [sp, #32] 1545*8fb009dcSAndroid Build Coastguard Worker ldp x23, x24, [sp, #48] 1546*8fb009dcSAndroid Build Coastguard Worker ldp d8, d9, [sp, #64] 1547*8fb009dcSAndroid Build Coastguard Worker ldp d10, d11, [sp, #80] 1548*8fb009dcSAndroid Build Coastguard Worker ldp d12, d13, [sp, #96] 1549*8fb009dcSAndroid Build Coastguard Worker ldp d14, d15, [sp, #112] 1550*8fb009dcSAndroid Build Coastguard Worker ldp x29, x30, [sp], #128 1551*8fb009dcSAndroid Build Coastguard Worker AARCH64_VALIDATE_LINK_REGISTER 1552*8fb009dcSAndroid Build Coastguard Worker ret 1553*8fb009dcSAndroid Build Coastguard Worker.size aes_gcm_dec_kernel,.-aes_gcm_dec_kernel 1554*8fb009dcSAndroid Build Coastguard Worker#endif 1555*8fb009dcSAndroid Build Coastguard Worker#endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__ELF__) 1556