Lines Matching +full:1 +full:- +full:9
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 # Accelerated AES-GCM stitched implementation for ppc64le.
5 # Copyright 2024- IBM Inc.
22 # Hash keys = v3 - v14
29 # v31 - counter 1
32 # vs0 - round key 0
35 # This implementation uses stitched AES-GCM approach to improve overall performance.
65 std 0, 16(1)
66 stdu 1,-512(1)
68 SAVE_GPR 14, 112, 1
69 SAVE_GPR 15, 120, 1
70 SAVE_GPR 16, 128, 1
71 SAVE_GPR 17, 136, 1
72 SAVE_GPR 18, 144, 1
73 SAVE_GPR 19, 152, 1
74 SAVE_GPR 20, 160, 1
75 SAVE_GPR 21, 168, 1
76 SAVE_GPR 22, 176, 1
77 SAVE_GPR 23, 184, 1
78 SAVE_GPR 24, 192, 1
80 addi 9, 1, 256
81 SAVE_VRS 20, 0, 9
82 SAVE_VRS 21, 16, 9
83 SAVE_VRS 22, 32, 9
84 SAVE_VRS 23, 48, 9
85 SAVE_VRS 24, 64, 9
86 SAVE_VRS 25, 80, 9
87 SAVE_VRS 26, 96, 9
88 SAVE_VRS 27, 112, 9
89 SAVE_VRS 28, 128, 9
90 SAVE_VRS 29, 144, 9
91 SAVE_VRS 30, 160, 9
92 SAVE_VRS 31, 176, 9
96 addi 9, 1, 256
97 RESTORE_VRS 20, 0, 9
98 RESTORE_VRS 21, 16, 9
99 RESTORE_VRS 22, 32, 9
100 RESTORE_VRS 23, 48, 9
101 RESTORE_VRS 24, 64, 9
102 RESTORE_VRS 25, 80, 9
103 RESTORE_VRS 26, 96, 9
104 RESTORE_VRS 27, 112, 9
105 RESTORE_VRS 28, 128, 9
106 RESTORE_VRS 29, 144, 9
107 RESTORE_VRS 30, 160, 9
108 RESTORE_VRS 31, 176, 9
110 RESTORE_GPR 14, 112, 1
111 RESTORE_GPR 15, 120, 1
112 RESTORE_GPR 16, 128, 1
113 RESTORE_GPR 17, 136, 1
114 RESTORE_GPR 18, 144, 1
115 RESTORE_GPR 19, 152, 1
116 RESTORE_GPR 20, 160, 1
117 RESTORE_GPR 21, 168, 1
118 RESTORE_GPR 22, 176, 1
119 RESTORE_GPR 23, 184, 1
120 RESTORE_GPR 24, 192, 1
122 addi 1, 1, 512
123 ld 0, 16(1)
130 \_VCIPHER \ST+1, \ST+1, \r
138 \_VCIPHER \ST+1, \ST+1, \r
148 xxlor 32+23, 1, 1
175 # Hash keys = v3 - v14
176 # Scratch: v23 - v29
181 vpmsumd 24, 9, \S2
202 vxor 1, 1, 1
203 vsldoi 25, 24, 1, 8 # mL
204 vsldoi 1, 1, 24, 8 # mH
222 vxor 24, 24, 1
237 vxor 1, 1, 1
245 vsldoi 25, 23, 1, 8 # mL
246 vsldoi 26, 1, 23, 8 # mH
263 # Hash keys = v3 - v14
269 # load Hash - h^4, h^3, h^2, h
287 lxvd2x 9+32, 10, 8 # H^3l
319 # Pre-load 8 AES rounds to scratch vectors.
320 xxlor 32+16, 1, 1
329 addi 22, 23, -9 # remaing AES rounds
348 lxv 32+1, 0(10)
349 vcipher 15, 15, 1
352 lxv 32+1, 0(10) # last round key
354 vcipherlast 15, 15, 1
357 stxvb16x 32+15, 0, 9 # store output
359 addi 9, 9, 16
368 addi 5, 5, -16
373 addi 12, 12, -1
393 vspltisb 16, -1
406 addi 22, 23, -1 # loop - 1
411 lxv 32+1, 0(10)
412 vcipher 15, 15, 1
415 lxv 32+1, 0(10) # last round key
416 vcipherlast 15, 15, 1
424 stxvb16x 15+32, 10, 1 # write v15 to stack
425 addi 10, 1, 223
426 addi 12, 9, -1
429 lbzu 22, 1(10)
430 stbu 22, 1(12)
441 add 9, 9, 5
462 # - Compute ghash for a full block. Clear Partial_len and pblock. Update IV.
464 # - Don't compute ghash if not full block. gcm_update will take care of it
490 vspltisb 16, -1
512 addi 22, 23, -1 # loop - 1
517 lxv 32+1, 0(10)
518 vcipher 15, 15, 1
521 lxv 32+1, 0(10) # last round key
522 vcipherlast 15, 15, 1
530 stxvb16x 15+32, 10, 1 # write v15 to stack
531 addi 10, 1, 223
533 addi 15, 9, -1
536 lbzu 22, 1(10)
537 stbu 22, 1(15)
542 add 9, 9, 21
581 # gcm_update(iv, Xi) - compute last hash
591 # load Hash - h^4, h^3, h^2, h
605 li 9, 64
606 lxvb16x 32+6, 9, 3 # load pblock
609 vxor 1, 1, 1
614 vsldoi 15, 13, 1, 8 # mL
615 vsldoi 16, 1, 13, 8 # mH
626 #stxvb16x 32+0, 9, 3
639 # r3 - inp
640 # r4 - out
641 # r5 - len
642 # r6 - AES round keys
643 # r7 - iv and other data
644 # r8 - Xi, HPoli, hash keys
658 # initialize ICB: GHASH( IV ), IV - r7
659 lxvb16x 30+32, 0, 7 # load IV - v30
662 mr 9, 4
664 # counter 1
666 vspltisb 22, 1
667 vsldoi 31, 31, 22,1 # counter 1
674 # load 9 round keys to VSR
676 lxv 1, 16(6) # round key 1
685 # load rounds - 10 (128), 12 (192), 14 (256)
687 li 24, 1 # encrypt
705 divdu 12, 5, 10 # n 128 bytes-blocks
707 addi 12, 12, -1 # loop - 1
717 xxlor 9, 32+22, 32+22 # save last state
721 vxor 15, 15, 29 # IV + round key - add round key 0
739 # Pre-compute first 8 AES state and leave 1/3/5 more rounds
742 addi 22, 23, -9 # process 8 keys
749 lxv 32+1, 0(10) # round key
750 AES_CIPHER_8x vcipher 15 1
753 lxv 32+1, 0(10) # last round key (v1)
762 vcipherlast 15, 15, 1
763 vcipherlast 16, 16, 1
764 vcipherlast 17, 17, 1
765 vcipherlast 18, 18, 1
766 vcipherlast 19, 19, 1
767 vcipherlast 20, 20, 1
768 vcipherlast 21, 21, 1
769 vcipherlast 22, 22, 1
790 stxvb16x 47, 0, 9 # store output
791 stxvb16x 48, 15, 9 # store output
792 stxvb16x 49, 16, 9 # store output
793 stxvb16x 50, 17, 9 # store output
794 stxvb16x 51, 18, 9 # store output
795 stxvb16x 52, 19, 9 # store output
796 stxvb16x 53, 20, 9 # store output
797 stxvb16x 54, 21, 9 # store output
798 addi 9, 9, 128
807 xxlor 32+15, 9, 9 # last state
816 xxlor 9, 32+22, 32+22 # save last state
819 vxor 15, 15, 27 # IV + round key - add round key 0
828 addi 5, 5, -128
835 lxv 32+1, 0(10) # round key
836 AES_CIPHER_8x vcipher 15 1
839 lxv 32+1, 0(10) # last round key (v1)
841 addi 12, 12, -1
846 vcipherlast 15, 15, 1
847 vcipherlast 16, 16, 1
848 vcipherlast 17, 17, 1
849 vcipherlast 18, 18, 1
850 vcipherlast 19, 19, 1
851 vcipherlast 20, 20, 1
852 vcipherlast 21, 21, 1
853 vcipherlast 22, 22, 1
874 stxvb16x 47, 0, 9 # store output
875 stxvb16x 48, 15, 9 # store output
876 stxvb16x 49, 16, 9 # store output
877 stxvb16x 50, 17, 9 # store output
878 stxvb16x 51, 18, 9 # store output
879 stxvb16x 52, 19, 9 # store output
880 stxvb16x 53, 20, 9 # store output
881 stxvb16x 54, 21, 9 # store output
882 addi 9, 9, 128
890 xxlor 30+32, 9, 9 # last ctr
895 addi 5, 5, -128
906 li 24, 1 # encrypt
936 # initialize ICB: GHASH( IV ), IV - r7
937 lxvb16x 30+32, 0, 7 # load IV - v30
940 mr 9, 4
942 # counter 1
944 vspltisb 22, 1
945 vsldoi 31, 31, 22,1 # counter 1
952 # load 9 round keys to VSR
954 lxv 1, 16(6) # round key 1
963 # load rounds - 10 (128), 12 (192), 14 (256)
983 divdu 12, 5, 10 # n 128 bytes-blocks
985 addi 12, 12, -1 # loop - 1
995 xxlor 9, 32+22, 32+22 # save last state
999 vxor 15, 15, 29 # IV + round key - add round key 0
1017 # Pre-compute first 8 AES state and leave 1/3/5 more rounds
1020 addi 22, 23, -9 # process 8 keys
1027 lxv 32+1, 0(10) # round key
1028 AES_CIPHER_8x vcipher 15 1
1031 lxv 32+1, 0(10) # last round key (v1)
1040 vcipherlast 15, 15, 1
1041 vcipherlast 16, 16, 1
1042 vcipherlast 17, 17, 1
1043 vcipherlast 18, 18, 1
1044 vcipherlast 19, 19, 1
1045 vcipherlast 20, 20, 1
1046 vcipherlast 21, 21, 1
1047 vcipherlast 22, 22, 1
1068 stxvb16x 47, 0, 9 # store output
1069 stxvb16x 48, 15, 9 # store output
1070 stxvb16x 49, 16, 9 # store output
1071 stxvb16x 50, 17, 9 # store output
1072 stxvb16x 51, 18, 9 # store output
1073 stxvb16x 52, 19, 9 # store output
1074 stxvb16x 53, 20, 9 # store output
1075 stxvb16x 54, 21, 9 # store output
1077 addi 9, 9, 128
1095 xxlor 32+15, 9, 9 # last state
1104 xxlor 9, 32+22, 32+22 # save last state
1107 vxor 15, 15, 27 # IV + round key - add round key 0
1116 addi 5, 5, -128
1123 lxv 32+1, 0(10) # round key
1124 AES_CIPHER_8x vcipher 15 1
1127 lxv 32+1, 0(10) # last round key (v1)
1129 addi 12, 12, -1
1134 vcipherlast 15, 15, 1
1135 vcipherlast 16, 16, 1
1136 vcipherlast 17, 17, 1
1137 vcipherlast 18, 18, 1
1138 vcipherlast 19, 19, 1
1139 vcipherlast 20, 20, 1
1140 vcipherlast 21, 21, 1
1141 vcipherlast 22, 22, 1
1162 stxvb16x 47, 0, 9 # store output
1163 stxvb16x 48, 15, 9 # store output
1164 stxvb16x 49, 16, 9 # store output
1165 stxvb16x 50, 17, 9 # store output
1166 stxvb16x 51, 18, 9 # store output
1167 stxvb16x 52, 19, 9 # store output
1168 stxvb16x 53, 20, 9 # store output
1169 stxvb16x 54, 21, 9 # store output
1170 addi 9, 9, 128
1188 xxlor 30+32, 9, 9 # last ctr
1193 addi 5, 5, -128