xref: /aosp_15_r20/external/boringssl/src/gen/bcm/p256-x86_64-asm-win.asm (revision 8fb009dc861624b67b6cdb62ea21f0f22d0c584b)
1; This file is generated from a similarly-named Perl script in the BoringSSL
2; source tree. Do not edit by hand.
3
4%ifidn __OUTPUT_FORMAT__, win64
5default	rel
6%define XMMWORD
7%define YMMWORD
8%define ZMMWORD
9%define _CET_ENDBR
10
11%ifdef BORINGSSL_PREFIX
12%include "boringssl_prefix_symbols_nasm.inc"
13%endif
14section	.text code align=64
15
16EXTERN	OPENSSL_ia32cap_P
17
18
19section	.rdata rdata align=8
20ALIGN	64
21$L$poly:
22	DQ	0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001
23
24$L$One:
25	DD	1,1,1,1,1,1,1,1
26$L$Two:
27	DD	2,2,2,2,2,2,2,2
28$L$Three:
29	DD	3,3,3,3,3,3,3,3
30$L$ONE_mont:
31	DQ	0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe
32
33
34$L$ord:
35	DQ	0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000
36$L$ordK:
37	DQ	0xccd1c8aaee00bc4f
38section	.text
39
40
41
42
43global	ecp_nistz256_neg
44
45ALIGN	32
46ecp_nistz256_neg:
47	mov	QWORD[8+rsp],rdi	;WIN64 prologue
48	mov	QWORD[16+rsp],rsi
49	mov	rax,rsp
50$L$SEH_begin_ecp_nistz256_neg:
51	mov	rdi,rcx
52	mov	rsi,rdx
53
54
55
56_CET_ENDBR
57	push	r12
58
59	push	r13
60
61$L$neg_body:
62
63	xor	r8,r8
64	xor	r9,r9
65	xor	r10,r10
66	xor	r11,r11
67	xor	r13,r13
68
69	sub	r8,QWORD[rsi]
70	sbb	r9,QWORD[8+rsi]
71	sbb	r10,QWORD[16+rsi]
72	mov	rax,r8
73	sbb	r11,QWORD[24+rsi]
74	lea	rsi,[$L$poly]
75	mov	rdx,r9
76	sbb	r13,0
77
78	add	r8,QWORD[rsi]
79	mov	rcx,r10
80	adc	r9,QWORD[8+rsi]
81	adc	r10,QWORD[16+rsi]
82	mov	r12,r11
83	adc	r11,QWORD[24+rsi]
84	test	r13,r13
85
86	cmovz	r8,rax
87	cmovz	r9,rdx
88	mov	QWORD[rdi],r8
89	cmovz	r10,rcx
90	mov	QWORD[8+rdi],r9
91	cmovz	r11,r12
92	mov	QWORD[16+rdi],r10
93	mov	QWORD[24+rdi],r11
94
95	mov	r13,QWORD[rsp]
96
97	mov	r12,QWORD[8+rsp]
98
99	lea	rsp,[16+rsp]
100
101$L$neg_epilogue:
102	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
103	mov	rsi,QWORD[16+rsp]
104	ret
105
106$L$SEH_end_ecp_nistz256_neg:
107
108
109
110
111
112
113global	ecp_nistz256_ord_mul_mont
114
115ALIGN	32
116ecp_nistz256_ord_mul_mont:
117	mov	QWORD[8+rsp],rdi	;WIN64 prologue
118	mov	QWORD[16+rsp],rsi
119	mov	rax,rsp
120$L$SEH_begin_ecp_nistz256_ord_mul_mont:
121	mov	rdi,rcx
122	mov	rsi,rdx
123	mov	rdx,r8
124
125
126
127_CET_ENDBR
128	lea	rcx,[OPENSSL_ia32cap_P]
129	mov	rcx,QWORD[8+rcx]
130	and	ecx,0x80100
131	cmp	ecx,0x80100
132	je	NEAR $L$ecp_nistz256_ord_mul_montx
133	push	rbp
134
135	push	rbx
136
137	push	r12
138
139	push	r13
140
141	push	r14
142
143	push	r15
144
145$L$ord_mul_body:
146
147	mov	rax,QWORD[rdx]
148	mov	rbx,rdx
149	lea	r14,[$L$ord]
150	mov	r15,QWORD[$L$ordK]
151
152
153	mov	rcx,rax
154	mul	QWORD[rsi]
155	mov	r8,rax
156	mov	rax,rcx
157	mov	r9,rdx
158
159	mul	QWORD[8+rsi]
160	add	r9,rax
161	mov	rax,rcx
162	adc	rdx,0
163	mov	r10,rdx
164
165	mul	QWORD[16+rsi]
166	add	r10,rax
167	mov	rax,rcx
168	adc	rdx,0
169
170	mov	r13,r8
171	imul	r8,r15
172
173	mov	r11,rdx
174	mul	QWORD[24+rsi]
175	add	r11,rax
176	mov	rax,r8
177	adc	rdx,0
178	mov	r12,rdx
179
180
181	mul	QWORD[r14]
182	mov	rbp,r8
183	add	r13,rax
184	mov	rax,r8
185	adc	rdx,0
186	mov	rcx,rdx
187
188	sub	r10,r8
189	sbb	r8,0
190
191	mul	QWORD[8+r14]
192	add	r9,rcx
193	adc	rdx,0
194	add	r9,rax
195	mov	rax,rbp
196	adc	r10,rdx
197	mov	rdx,rbp
198	adc	r8,0
199
200	shl	rax,32
201	shr	rdx,32
202	sub	r11,rax
203	mov	rax,QWORD[8+rbx]
204	sbb	rbp,rdx
205
206	add	r11,r8
207	adc	r12,rbp
208	adc	r13,0
209
210
211	mov	rcx,rax
212	mul	QWORD[rsi]
213	add	r9,rax
214	mov	rax,rcx
215	adc	rdx,0
216	mov	rbp,rdx
217
218	mul	QWORD[8+rsi]
219	add	r10,rbp
220	adc	rdx,0
221	add	r10,rax
222	mov	rax,rcx
223	adc	rdx,0
224	mov	rbp,rdx
225
226	mul	QWORD[16+rsi]
227	add	r11,rbp
228	adc	rdx,0
229	add	r11,rax
230	mov	rax,rcx
231	adc	rdx,0
232
233	mov	rcx,r9
234	imul	r9,r15
235
236	mov	rbp,rdx
237	mul	QWORD[24+rsi]
238	add	r12,rbp
239	adc	rdx,0
240	xor	r8,r8
241	add	r12,rax
242	mov	rax,r9
243	adc	r13,rdx
244	adc	r8,0
245
246
247	mul	QWORD[r14]
248	mov	rbp,r9
249	add	rcx,rax
250	mov	rax,r9
251	adc	rcx,rdx
252
253	sub	r11,r9
254	sbb	r9,0
255
256	mul	QWORD[8+r14]
257	add	r10,rcx
258	adc	rdx,0
259	add	r10,rax
260	mov	rax,rbp
261	adc	r11,rdx
262	mov	rdx,rbp
263	adc	r9,0
264
265	shl	rax,32
266	shr	rdx,32
267	sub	r12,rax
268	mov	rax,QWORD[16+rbx]
269	sbb	rbp,rdx
270
271	add	r12,r9
272	adc	r13,rbp
273	adc	r8,0
274
275
276	mov	rcx,rax
277	mul	QWORD[rsi]
278	add	r10,rax
279	mov	rax,rcx
280	adc	rdx,0
281	mov	rbp,rdx
282
283	mul	QWORD[8+rsi]
284	add	r11,rbp
285	adc	rdx,0
286	add	r11,rax
287	mov	rax,rcx
288	adc	rdx,0
289	mov	rbp,rdx
290
291	mul	QWORD[16+rsi]
292	add	r12,rbp
293	adc	rdx,0
294	add	r12,rax
295	mov	rax,rcx
296	adc	rdx,0
297
298	mov	rcx,r10
299	imul	r10,r15
300
301	mov	rbp,rdx
302	mul	QWORD[24+rsi]
303	add	r13,rbp
304	adc	rdx,0
305	xor	r9,r9
306	add	r13,rax
307	mov	rax,r10
308	adc	r8,rdx
309	adc	r9,0
310
311
312	mul	QWORD[r14]
313	mov	rbp,r10
314	add	rcx,rax
315	mov	rax,r10
316	adc	rcx,rdx
317
318	sub	r12,r10
319	sbb	r10,0
320
321	mul	QWORD[8+r14]
322	add	r11,rcx
323	adc	rdx,0
324	add	r11,rax
325	mov	rax,rbp
326	adc	r12,rdx
327	mov	rdx,rbp
328	adc	r10,0
329
330	shl	rax,32
331	shr	rdx,32
332	sub	r13,rax
333	mov	rax,QWORD[24+rbx]
334	sbb	rbp,rdx
335
336	add	r13,r10
337	adc	r8,rbp
338	adc	r9,0
339
340
341	mov	rcx,rax
342	mul	QWORD[rsi]
343	add	r11,rax
344	mov	rax,rcx
345	adc	rdx,0
346	mov	rbp,rdx
347
348	mul	QWORD[8+rsi]
349	add	r12,rbp
350	adc	rdx,0
351	add	r12,rax
352	mov	rax,rcx
353	adc	rdx,0
354	mov	rbp,rdx
355
356	mul	QWORD[16+rsi]
357	add	r13,rbp
358	adc	rdx,0
359	add	r13,rax
360	mov	rax,rcx
361	adc	rdx,0
362
363	mov	rcx,r11
364	imul	r11,r15
365
366	mov	rbp,rdx
367	mul	QWORD[24+rsi]
368	add	r8,rbp
369	adc	rdx,0
370	xor	r10,r10
371	add	r8,rax
372	mov	rax,r11
373	adc	r9,rdx
374	adc	r10,0
375
376
377	mul	QWORD[r14]
378	mov	rbp,r11
379	add	rcx,rax
380	mov	rax,r11
381	adc	rcx,rdx
382
383	sub	r13,r11
384	sbb	r11,0
385
386	mul	QWORD[8+r14]
387	add	r12,rcx
388	adc	rdx,0
389	add	r12,rax
390	mov	rax,rbp
391	adc	r13,rdx
392	mov	rdx,rbp
393	adc	r11,0
394
395	shl	rax,32
396	shr	rdx,32
397	sub	r8,rax
398	sbb	rbp,rdx
399
400	add	r8,r11
401	adc	r9,rbp
402	adc	r10,0
403
404
405	mov	rsi,r12
406	sub	r12,QWORD[r14]
407	mov	r11,r13
408	sbb	r13,QWORD[8+r14]
409	mov	rcx,r8
410	sbb	r8,QWORD[16+r14]
411	mov	rbp,r9
412	sbb	r9,QWORD[24+r14]
413	sbb	r10,0
414
415	cmovc	r12,rsi
416	cmovc	r13,r11
417	cmovc	r8,rcx
418	cmovc	r9,rbp
419
420	mov	QWORD[rdi],r12
421	mov	QWORD[8+rdi],r13
422	mov	QWORD[16+rdi],r8
423	mov	QWORD[24+rdi],r9
424
425	mov	r15,QWORD[rsp]
426
427	mov	r14,QWORD[8+rsp]
428
429	mov	r13,QWORD[16+rsp]
430
431	mov	r12,QWORD[24+rsp]
432
433	mov	rbx,QWORD[32+rsp]
434
435	mov	rbp,QWORD[40+rsp]
436
437	lea	rsp,[48+rsp]
438
439$L$ord_mul_epilogue:
440	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
441	mov	rsi,QWORD[16+rsp]
442	ret
443
444$L$SEH_end_ecp_nistz256_ord_mul_mont:
445
446
447
448
449
450
451
452global	ecp_nistz256_ord_sqr_mont
453
454ALIGN	32
455ecp_nistz256_ord_sqr_mont:
456	mov	QWORD[8+rsp],rdi	;WIN64 prologue
457	mov	QWORD[16+rsp],rsi
458	mov	rax,rsp
459$L$SEH_begin_ecp_nistz256_ord_sqr_mont:
460	mov	rdi,rcx
461	mov	rsi,rdx
462	mov	rdx,r8
463
464
465
466_CET_ENDBR
467	lea	rcx,[OPENSSL_ia32cap_P]
468	mov	rcx,QWORD[8+rcx]
469	and	ecx,0x80100
470	cmp	ecx,0x80100
471	je	NEAR $L$ecp_nistz256_ord_sqr_montx
472	push	rbp
473
474	push	rbx
475
476	push	r12
477
478	push	r13
479
480	push	r14
481
482	push	r15
483
484$L$ord_sqr_body:
485
486	mov	r8,QWORD[rsi]
487	mov	rax,QWORD[8+rsi]
488	mov	r14,QWORD[16+rsi]
489	mov	r15,QWORD[24+rsi]
490	lea	rsi,[$L$ord]
491	mov	rbx,rdx
492	jmp	NEAR $L$oop_ord_sqr
493
494ALIGN	32
495$L$oop_ord_sqr:
496
497	mov	rbp,rax
498	mul	r8
499	mov	r9,rax
500DB	102,72,15,110,205
501	mov	rax,r14
502	mov	r10,rdx
503
504	mul	r8
505	add	r10,rax
506	mov	rax,r15
507DB	102,73,15,110,214
508	adc	rdx,0
509	mov	r11,rdx
510
511	mul	r8
512	add	r11,rax
513	mov	rax,r15
514DB	102,73,15,110,223
515	adc	rdx,0
516	mov	r12,rdx
517
518
519	mul	r14
520	mov	r13,rax
521	mov	rax,r14
522	mov	r14,rdx
523
524
525	mul	rbp
526	add	r11,rax
527	mov	rax,r15
528	adc	rdx,0
529	mov	r15,rdx
530
531	mul	rbp
532	add	r12,rax
533	adc	rdx,0
534
535	add	r12,r15
536	adc	r13,rdx
537	adc	r14,0
538
539
540	xor	r15,r15
541	mov	rax,r8
542	add	r9,r9
543	adc	r10,r10
544	adc	r11,r11
545	adc	r12,r12
546	adc	r13,r13
547	adc	r14,r14
548	adc	r15,0
549
550
551	mul	rax
552	mov	r8,rax
553DB	102,72,15,126,200
554	mov	rbp,rdx
555
556	mul	rax
557	add	r9,rbp
558	adc	r10,rax
559DB	102,72,15,126,208
560	adc	rdx,0
561	mov	rbp,rdx
562
563	mul	rax
564	add	r11,rbp
565	adc	r12,rax
566DB	102,72,15,126,216
567	adc	rdx,0
568	mov	rbp,rdx
569
570	mov	rcx,r8
571	imul	r8,QWORD[32+rsi]
572
573	mul	rax
574	add	r13,rbp
575	adc	r14,rax
576	mov	rax,QWORD[rsi]
577	adc	r15,rdx
578
579
580	mul	r8
581	mov	rbp,r8
582	add	rcx,rax
583	mov	rax,QWORD[8+rsi]
584	adc	rcx,rdx
585
586	sub	r10,r8
587	sbb	rbp,0
588
589	mul	r8
590	add	r9,rcx
591	adc	rdx,0
592	add	r9,rax
593	mov	rax,r8
594	adc	r10,rdx
595	mov	rdx,r8
596	adc	rbp,0
597
598	mov	rcx,r9
599	imul	r9,QWORD[32+rsi]
600
601	shl	rax,32
602	shr	rdx,32
603	sub	r11,rax
604	mov	rax,QWORD[rsi]
605	sbb	r8,rdx
606
607	add	r11,rbp
608	adc	r8,0
609
610
611	mul	r9
612	mov	rbp,r9
613	add	rcx,rax
614	mov	rax,QWORD[8+rsi]
615	adc	rcx,rdx
616
617	sub	r11,r9
618	sbb	rbp,0
619
620	mul	r9
621	add	r10,rcx
622	adc	rdx,0
623	add	r10,rax
624	mov	rax,r9
625	adc	r11,rdx
626	mov	rdx,r9
627	adc	rbp,0
628
629	mov	rcx,r10
630	imul	r10,QWORD[32+rsi]
631
632	shl	rax,32
633	shr	rdx,32
634	sub	r8,rax
635	mov	rax,QWORD[rsi]
636	sbb	r9,rdx
637
638	add	r8,rbp
639	adc	r9,0
640
641
642	mul	r10
643	mov	rbp,r10
644	add	rcx,rax
645	mov	rax,QWORD[8+rsi]
646	adc	rcx,rdx
647
648	sub	r8,r10
649	sbb	rbp,0
650
651	mul	r10
652	add	r11,rcx
653	adc	rdx,0
654	add	r11,rax
655	mov	rax,r10
656	adc	r8,rdx
657	mov	rdx,r10
658	adc	rbp,0
659
660	mov	rcx,r11
661	imul	r11,QWORD[32+rsi]
662
663	shl	rax,32
664	shr	rdx,32
665	sub	r9,rax
666	mov	rax,QWORD[rsi]
667	sbb	r10,rdx
668
669	add	r9,rbp
670	adc	r10,0
671
672
673	mul	r11
674	mov	rbp,r11
675	add	rcx,rax
676	mov	rax,QWORD[8+rsi]
677	adc	rcx,rdx
678
679	sub	r9,r11
680	sbb	rbp,0
681
682	mul	r11
683	add	r8,rcx
684	adc	rdx,0
685	add	r8,rax
686	mov	rax,r11
687	adc	r9,rdx
688	mov	rdx,r11
689	adc	rbp,0
690
691	shl	rax,32
692	shr	rdx,32
693	sub	r10,rax
694	sbb	r11,rdx
695
696	add	r10,rbp
697	adc	r11,0
698
699
700	xor	rdx,rdx
701	add	r8,r12
702	adc	r9,r13
703	mov	r12,r8
704	adc	r10,r14
705	adc	r11,r15
706	mov	rax,r9
707	adc	rdx,0
708
709
710	sub	r8,QWORD[rsi]
711	mov	r14,r10
712	sbb	r9,QWORD[8+rsi]
713	sbb	r10,QWORD[16+rsi]
714	mov	r15,r11
715	sbb	r11,QWORD[24+rsi]
716	sbb	rdx,0
717
718	cmovc	r8,r12
719	cmovnc	rax,r9
720	cmovnc	r14,r10
721	cmovnc	r15,r11
722
723	dec	rbx
724	jnz	NEAR $L$oop_ord_sqr
725
726	mov	QWORD[rdi],r8
727	mov	QWORD[8+rdi],rax
728	pxor	xmm1,xmm1
729	mov	QWORD[16+rdi],r14
730	pxor	xmm2,xmm2
731	mov	QWORD[24+rdi],r15
732	pxor	xmm3,xmm3
733
734	mov	r15,QWORD[rsp]
735
736	mov	r14,QWORD[8+rsp]
737
738	mov	r13,QWORD[16+rsp]
739
740	mov	r12,QWORD[24+rsp]
741
742	mov	rbx,QWORD[32+rsp]
743
744	mov	rbp,QWORD[40+rsp]
745
746	lea	rsp,[48+rsp]
747
748$L$ord_sqr_epilogue:
749	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
750	mov	rsi,QWORD[16+rsp]
751	ret
752
753$L$SEH_end_ecp_nistz256_ord_sqr_mont:
754
755
756ALIGN	32
757ecp_nistz256_ord_mul_montx:
758	mov	QWORD[8+rsp],rdi	;WIN64 prologue
759	mov	QWORD[16+rsp],rsi
760	mov	rax,rsp
761$L$SEH_begin_ecp_nistz256_ord_mul_montx:
762	mov	rdi,rcx
763	mov	rsi,rdx
764	mov	rdx,r8
765
766
767
768$L$ecp_nistz256_ord_mul_montx:
769	push	rbp
770
771	push	rbx
772
773	push	r12
774
775	push	r13
776
777	push	r14
778
779	push	r15
780
781$L$ord_mulx_body:
782
783	mov	rbx,rdx
784	mov	rdx,QWORD[rdx]
785	mov	r9,QWORD[rsi]
786	mov	r10,QWORD[8+rsi]
787	mov	r11,QWORD[16+rsi]
788	mov	r12,QWORD[24+rsi]
789	lea	rsi,[((-128))+rsi]
790	lea	r14,[(($L$ord-128))]
791	mov	r15,QWORD[$L$ordK]
792
793
794	mulx	r9,r8,r9
795	mulx	r10,rcx,r10
796	mulx	r11,rbp,r11
797	add	r9,rcx
798	mulx	r12,rcx,r12
799	mov	rdx,r8
800	mulx	rax,rdx,r15
801	adc	r10,rbp
802	adc	r11,rcx
803	adc	r12,0
804
805
806	xor	r13,r13
807	mulx	rbp,rcx,QWORD[((0+128))+r14]
808	adcx	r8,rcx
809	adox	r9,rbp
810
811	mulx	rbp,rcx,QWORD[((8+128))+r14]
812	adcx	r9,rcx
813	adox	r10,rbp
814
815	mulx	rbp,rcx,QWORD[((16+128))+r14]
816	adcx	r10,rcx
817	adox	r11,rbp
818
819	mulx	rbp,rcx,QWORD[((24+128))+r14]
820	mov	rdx,QWORD[8+rbx]
821	adcx	r11,rcx
822	adox	r12,rbp
823	adcx	r12,r8
824	adox	r13,r8
825	adc	r13,0
826
827
828	mulx	rbp,rcx,QWORD[((0+128))+rsi]
829	adcx	r9,rcx
830	adox	r10,rbp
831
832	mulx	rbp,rcx,QWORD[((8+128))+rsi]
833	adcx	r10,rcx
834	adox	r11,rbp
835
836	mulx	rbp,rcx,QWORD[((16+128))+rsi]
837	adcx	r11,rcx
838	adox	r12,rbp
839
840	mulx	rbp,rcx,QWORD[((24+128))+rsi]
841	mov	rdx,r9
842	mulx	rax,rdx,r15
843	adcx	r12,rcx
844	adox	r13,rbp
845
846	adcx	r13,r8
847	adox	r8,r8
848	adc	r8,0
849
850
851	mulx	rbp,rcx,QWORD[((0+128))+r14]
852	adcx	r9,rcx
853	adox	r10,rbp
854
855	mulx	rbp,rcx,QWORD[((8+128))+r14]
856	adcx	r10,rcx
857	adox	r11,rbp
858
859	mulx	rbp,rcx,QWORD[((16+128))+r14]
860	adcx	r11,rcx
861	adox	r12,rbp
862
863	mulx	rbp,rcx,QWORD[((24+128))+r14]
864	mov	rdx,QWORD[16+rbx]
865	adcx	r12,rcx
866	adox	r13,rbp
867	adcx	r13,r9
868	adox	r8,r9
869	adc	r8,0
870
871
872	mulx	rbp,rcx,QWORD[((0+128))+rsi]
873	adcx	r10,rcx
874	adox	r11,rbp
875
876	mulx	rbp,rcx,QWORD[((8+128))+rsi]
877	adcx	r11,rcx
878	adox	r12,rbp
879
880	mulx	rbp,rcx,QWORD[((16+128))+rsi]
881	adcx	r12,rcx
882	adox	r13,rbp
883
884	mulx	rbp,rcx,QWORD[((24+128))+rsi]
885	mov	rdx,r10
886	mulx	rax,rdx,r15
887	adcx	r13,rcx
888	adox	r8,rbp
889
890	adcx	r8,r9
891	adox	r9,r9
892	adc	r9,0
893
894
895	mulx	rbp,rcx,QWORD[((0+128))+r14]
896	adcx	r10,rcx
897	adox	r11,rbp
898
899	mulx	rbp,rcx,QWORD[((8+128))+r14]
900	adcx	r11,rcx
901	adox	r12,rbp
902
903	mulx	rbp,rcx,QWORD[((16+128))+r14]
904	adcx	r12,rcx
905	adox	r13,rbp
906
907	mulx	rbp,rcx,QWORD[((24+128))+r14]
908	mov	rdx,QWORD[24+rbx]
909	adcx	r13,rcx
910	adox	r8,rbp
911	adcx	r8,r10
912	adox	r9,r10
913	adc	r9,0
914
915
916	mulx	rbp,rcx,QWORD[((0+128))+rsi]
917	adcx	r11,rcx
918	adox	r12,rbp
919
920	mulx	rbp,rcx,QWORD[((8+128))+rsi]
921	adcx	r12,rcx
922	adox	r13,rbp
923
924	mulx	rbp,rcx,QWORD[((16+128))+rsi]
925	adcx	r13,rcx
926	adox	r8,rbp
927
928	mulx	rbp,rcx,QWORD[((24+128))+rsi]
929	mov	rdx,r11
930	mulx	rax,rdx,r15
931	adcx	r8,rcx
932	adox	r9,rbp
933
934	adcx	r9,r10
935	adox	r10,r10
936	adc	r10,0
937
938
939	mulx	rbp,rcx,QWORD[((0+128))+r14]
940	adcx	r11,rcx
941	adox	r12,rbp
942
943	mulx	rbp,rcx,QWORD[((8+128))+r14]
944	adcx	r12,rcx
945	adox	r13,rbp
946
947	mulx	rbp,rcx,QWORD[((16+128))+r14]
948	adcx	r13,rcx
949	adox	r8,rbp
950
951	mulx	rbp,rcx,QWORD[((24+128))+r14]
952	lea	r14,[128+r14]
953	mov	rbx,r12
954	adcx	r8,rcx
955	adox	r9,rbp
956	mov	rdx,r13
957	adcx	r9,r11
958	adox	r10,r11
959	adc	r10,0
960
961
962
963	mov	rcx,r8
964	sub	r12,QWORD[r14]
965	sbb	r13,QWORD[8+r14]
966	sbb	r8,QWORD[16+r14]
967	mov	rbp,r9
968	sbb	r9,QWORD[24+r14]
969	sbb	r10,0
970
971	cmovc	r12,rbx
972	cmovc	r13,rdx
973	cmovc	r8,rcx
974	cmovc	r9,rbp
975
976	mov	QWORD[rdi],r12
977	mov	QWORD[8+rdi],r13
978	mov	QWORD[16+rdi],r8
979	mov	QWORD[24+rdi],r9
980
981	mov	r15,QWORD[rsp]
982
983	mov	r14,QWORD[8+rsp]
984
985	mov	r13,QWORD[16+rsp]
986
987	mov	r12,QWORD[24+rsp]
988
989	mov	rbx,QWORD[32+rsp]
990
991	mov	rbp,QWORD[40+rsp]
992
993	lea	rsp,[48+rsp]
994
995$L$ord_mulx_epilogue:
996	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
997	mov	rsi,QWORD[16+rsp]
998	ret
999
1000$L$SEH_end_ecp_nistz256_ord_mul_montx:
1001
1002
1003ALIGN	32
1004ecp_nistz256_ord_sqr_montx:
1005	mov	QWORD[8+rsp],rdi	;WIN64 prologue
1006	mov	QWORD[16+rsp],rsi
1007	mov	rax,rsp
1008$L$SEH_begin_ecp_nistz256_ord_sqr_montx:
1009	mov	rdi,rcx
1010	mov	rsi,rdx
1011	mov	rdx,r8
1012
1013
1014
1015$L$ecp_nistz256_ord_sqr_montx:
1016	push	rbp
1017
1018	push	rbx
1019
1020	push	r12
1021
1022	push	r13
1023
1024	push	r14
1025
1026	push	r15
1027
1028$L$ord_sqrx_body:
1029
1030	mov	rbx,rdx
1031	mov	rdx,QWORD[rsi]
1032	mov	r14,QWORD[8+rsi]
1033	mov	r15,QWORD[16+rsi]
1034	mov	r8,QWORD[24+rsi]
1035	lea	rsi,[$L$ord]
1036	jmp	NEAR $L$oop_ord_sqrx
1037
1038ALIGN	32
1039$L$oop_ord_sqrx:
1040	mulx	r10,r9,r14
1041	mulx	r11,rcx,r15
1042	mov	rax,rdx
1043DB	102,73,15,110,206
1044	mulx	r12,rbp,r8
1045	mov	rdx,r14
1046	add	r10,rcx
1047DB	102,73,15,110,215
1048	adc	r11,rbp
1049	adc	r12,0
1050	xor	r13,r13
1051
1052	mulx	rbp,rcx,r15
1053	adcx	r11,rcx
1054	adox	r12,rbp
1055
1056	mulx	rbp,rcx,r8
1057	mov	rdx,r15
1058	adcx	r12,rcx
1059	adox	r13,rbp
1060	adc	r13,0
1061
1062	mulx	r14,rcx,r8
1063	mov	rdx,rax
1064DB	102,73,15,110,216
1065	xor	r15,r15
1066	adcx	r9,r9
1067	adox	r13,rcx
1068	adcx	r10,r10
1069	adox	r14,r15
1070
1071
1072	mulx	rbp,r8,rdx
1073DB	102,72,15,126,202
1074	adcx	r11,r11
1075	adox	r9,rbp
1076	adcx	r12,r12
1077	mulx	rax,rcx,rdx
1078DB	102,72,15,126,210
1079	adcx	r13,r13
1080	adox	r10,rcx
1081	adcx	r14,r14
1082	mulx	rbp,rcx,rdx
1083	DB	0x67
1084DB	102,72,15,126,218
1085	adox	r11,rax
1086	adcx	r15,r15
1087	adox	r12,rcx
1088	adox	r13,rbp
1089	mulx	rax,rcx,rdx
1090	adox	r14,rcx
1091	adox	r15,rax
1092
1093
1094	mov	rdx,r8
1095	mulx	rcx,rdx,QWORD[32+rsi]
1096
1097	xor	rax,rax
1098	mulx	rbp,rcx,QWORD[rsi]
1099	adcx	r8,rcx
1100	adox	r9,rbp
1101	mulx	rbp,rcx,QWORD[8+rsi]
1102	adcx	r9,rcx
1103	adox	r10,rbp
1104	mulx	rbp,rcx,QWORD[16+rsi]
1105	adcx	r10,rcx
1106	adox	r11,rbp
1107	mulx	rbp,rcx,QWORD[24+rsi]
1108	adcx	r11,rcx
1109	adox	r8,rbp
1110	adcx	r8,rax
1111
1112
1113	mov	rdx,r9
1114	mulx	rcx,rdx,QWORD[32+rsi]
1115
1116	mulx	rbp,rcx,QWORD[rsi]
1117	adox	r9,rcx
1118	adcx	r10,rbp
1119	mulx	rbp,rcx,QWORD[8+rsi]
1120	adox	r10,rcx
1121	adcx	r11,rbp
1122	mulx	rbp,rcx,QWORD[16+rsi]
1123	adox	r11,rcx
1124	adcx	r8,rbp
1125	mulx	rbp,rcx,QWORD[24+rsi]
1126	adox	r8,rcx
1127	adcx	r9,rbp
1128	adox	r9,rax
1129
1130
1131	mov	rdx,r10
1132	mulx	rcx,rdx,QWORD[32+rsi]
1133
1134	mulx	rbp,rcx,QWORD[rsi]
1135	adcx	r10,rcx
1136	adox	r11,rbp
1137	mulx	rbp,rcx,QWORD[8+rsi]
1138	adcx	r11,rcx
1139	adox	r8,rbp
1140	mulx	rbp,rcx,QWORD[16+rsi]
1141	adcx	r8,rcx
1142	adox	r9,rbp
1143	mulx	rbp,rcx,QWORD[24+rsi]
1144	adcx	r9,rcx
1145	adox	r10,rbp
1146	adcx	r10,rax
1147
1148
1149	mov	rdx,r11
1150	mulx	rcx,rdx,QWORD[32+rsi]
1151
1152	mulx	rbp,rcx,QWORD[rsi]
1153	adox	r11,rcx
1154	adcx	r8,rbp
1155	mulx	rbp,rcx,QWORD[8+rsi]
1156	adox	r8,rcx
1157	adcx	r9,rbp
1158	mulx	rbp,rcx,QWORD[16+rsi]
1159	adox	r9,rcx
1160	adcx	r10,rbp
1161	mulx	rbp,rcx,QWORD[24+rsi]
1162	adox	r10,rcx
1163	adcx	r11,rbp
1164	adox	r11,rax
1165
1166
1167	add	r12,r8
1168	adc	r9,r13
1169	mov	rdx,r12
1170	adc	r10,r14
1171	adc	r11,r15
1172	mov	r14,r9
1173	adc	rax,0
1174
1175
1176	sub	r12,QWORD[rsi]
1177	mov	r15,r10
1178	sbb	r9,QWORD[8+rsi]
1179	sbb	r10,QWORD[16+rsi]
1180	mov	r8,r11
1181	sbb	r11,QWORD[24+rsi]
1182	sbb	rax,0
1183
1184	cmovnc	rdx,r12
1185	cmovnc	r14,r9
1186	cmovnc	r15,r10
1187	cmovnc	r8,r11
1188
1189	dec	rbx
1190	jnz	NEAR $L$oop_ord_sqrx
1191
1192	mov	QWORD[rdi],rdx
1193	mov	QWORD[8+rdi],r14
1194	pxor	xmm1,xmm1
1195	mov	QWORD[16+rdi],r15
1196	pxor	xmm2,xmm2
1197	mov	QWORD[24+rdi],r8
1198	pxor	xmm3,xmm3
1199
1200	mov	r15,QWORD[rsp]
1201
1202	mov	r14,QWORD[8+rsp]
1203
1204	mov	r13,QWORD[16+rsp]
1205
1206	mov	r12,QWORD[24+rsp]
1207
1208	mov	rbx,QWORD[32+rsp]
1209
1210	mov	rbp,QWORD[40+rsp]
1211
1212	lea	rsp,[48+rsp]
1213
1214$L$ord_sqrx_epilogue:
1215	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
1216	mov	rsi,QWORD[16+rsp]
1217	ret
1218
1219$L$SEH_end_ecp_nistz256_ord_sqr_montx:
1220
1221
1222
1223
1224
1225
1226global	ecp_nistz256_mul_mont
1227
1228ALIGN	32
1229ecp_nistz256_mul_mont:
1230	mov	QWORD[8+rsp],rdi	;WIN64 prologue
1231	mov	QWORD[16+rsp],rsi
1232	mov	rax,rsp
1233$L$SEH_begin_ecp_nistz256_mul_mont:
1234	mov	rdi,rcx
1235	mov	rsi,rdx
1236	mov	rdx,r8
1237
1238
1239
1240_CET_ENDBR
1241	lea	rcx,[OPENSSL_ia32cap_P]
1242	mov	rcx,QWORD[8+rcx]
1243	and	ecx,0x80100
1244$L$mul_mont:
1245	push	rbp
1246
1247	push	rbx
1248
1249	push	r12
1250
1251	push	r13
1252
1253	push	r14
1254
1255	push	r15
1256
1257$L$mul_body:
1258	cmp	ecx,0x80100
1259	je	NEAR $L$mul_montx
1260	mov	rbx,rdx
1261	mov	rax,QWORD[rdx]
1262	mov	r9,QWORD[rsi]
1263	mov	r10,QWORD[8+rsi]
1264	mov	r11,QWORD[16+rsi]
1265	mov	r12,QWORD[24+rsi]
1266
1267	call	__ecp_nistz256_mul_montq
1268	jmp	NEAR $L$mul_mont_done
1269
1270ALIGN	32
1271$L$mul_montx:
1272	mov	rbx,rdx
1273	mov	rdx,QWORD[rdx]
1274	mov	r9,QWORD[rsi]
1275	mov	r10,QWORD[8+rsi]
1276	mov	r11,QWORD[16+rsi]
1277	mov	r12,QWORD[24+rsi]
1278	lea	rsi,[((-128))+rsi]
1279
1280	call	__ecp_nistz256_mul_montx
1281$L$mul_mont_done:
1282	mov	r15,QWORD[rsp]
1283
1284	mov	r14,QWORD[8+rsp]
1285
1286	mov	r13,QWORD[16+rsp]
1287
1288	mov	r12,QWORD[24+rsp]
1289
1290	mov	rbx,QWORD[32+rsp]
1291
1292	mov	rbp,QWORD[40+rsp]
1293
1294	lea	rsp,[48+rsp]
1295
1296$L$mul_epilogue:
1297	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
1298	mov	rsi,QWORD[16+rsp]
1299	ret
1300
1301$L$SEH_end_ecp_nistz256_mul_mont:
1302
1303
1304ALIGN	32
1305__ecp_nistz256_mul_montq:
1306
1307
1308
1309	mov	rbp,rax
1310	mul	r9
1311	mov	r14,QWORD[(($L$poly+8))]
1312	mov	r8,rax
1313	mov	rax,rbp
1314	mov	r9,rdx
1315
1316	mul	r10
1317	mov	r15,QWORD[(($L$poly+24))]
1318	add	r9,rax
1319	mov	rax,rbp
1320	adc	rdx,0
1321	mov	r10,rdx
1322
1323	mul	r11
1324	add	r10,rax
1325	mov	rax,rbp
1326	adc	rdx,0
1327	mov	r11,rdx
1328
1329	mul	r12
1330	add	r11,rax
1331	mov	rax,r8
1332	adc	rdx,0
1333	xor	r13,r13
1334	mov	r12,rdx
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345	mov	rbp,r8
1346	shl	r8,32
1347	mul	r15
1348	shr	rbp,32
1349	add	r9,r8
1350	adc	r10,rbp
1351	adc	r11,rax
1352	mov	rax,QWORD[8+rbx]
1353	adc	r12,rdx
1354	adc	r13,0
1355	xor	r8,r8
1356
1357
1358
1359	mov	rbp,rax
1360	mul	QWORD[rsi]
1361	add	r9,rax
1362	mov	rax,rbp
1363	adc	rdx,0
1364	mov	rcx,rdx
1365
1366	mul	QWORD[8+rsi]
1367	add	r10,rcx
1368	adc	rdx,0
1369	add	r10,rax
1370	mov	rax,rbp
1371	adc	rdx,0
1372	mov	rcx,rdx
1373
1374	mul	QWORD[16+rsi]
1375	add	r11,rcx
1376	adc	rdx,0
1377	add	r11,rax
1378	mov	rax,rbp
1379	adc	rdx,0
1380	mov	rcx,rdx
1381
1382	mul	QWORD[24+rsi]
1383	add	r12,rcx
1384	adc	rdx,0
1385	add	r12,rax
1386	mov	rax,r9
1387	adc	r13,rdx
1388	adc	r8,0
1389
1390
1391
1392	mov	rbp,r9
1393	shl	r9,32
1394	mul	r15
1395	shr	rbp,32
1396	add	r10,r9
1397	adc	r11,rbp
1398	adc	r12,rax
1399	mov	rax,QWORD[16+rbx]
1400	adc	r13,rdx
1401	adc	r8,0
1402	xor	r9,r9
1403
1404
1405
1406	mov	rbp,rax
1407	mul	QWORD[rsi]
1408	add	r10,rax
1409	mov	rax,rbp
1410	adc	rdx,0
1411	mov	rcx,rdx
1412
1413	mul	QWORD[8+rsi]
1414	add	r11,rcx
1415	adc	rdx,0
1416	add	r11,rax
1417	mov	rax,rbp
1418	adc	rdx,0
1419	mov	rcx,rdx
1420
1421	mul	QWORD[16+rsi]
1422	add	r12,rcx
1423	adc	rdx,0
1424	add	r12,rax
1425	mov	rax,rbp
1426	adc	rdx,0
1427	mov	rcx,rdx
1428
1429	mul	QWORD[24+rsi]
1430	add	r13,rcx
1431	adc	rdx,0
1432	add	r13,rax
1433	mov	rax,r10
1434	adc	r8,rdx
1435	adc	r9,0
1436
1437
1438
1439	mov	rbp,r10
1440	shl	r10,32
1441	mul	r15
1442	shr	rbp,32
1443	add	r11,r10
1444	adc	r12,rbp
1445	adc	r13,rax
1446	mov	rax,QWORD[24+rbx]
1447	adc	r8,rdx
1448	adc	r9,0
1449	xor	r10,r10
1450
1451
1452
1453	mov	rbp,rax
1454	mul	QWORD[rsi]
1455	add	r11,rax
1456	mov	rax,rbp
1457	adc	rdx,0
1458	mov	rcx,rdx
1459
1460	mul	QWORD[8+rsi]
1461	add	r12,rcx
1462	adc	rdx,0
1463	add	r12,rax
1464	mov	rax,rbp
1465	adc	rdx,0
1466	mov	rcx,rdx
1467
1468	mul	QWORD[16+rsi]
1469	add	r13,rcx
1470	adc	rdx,0
1471	add	r13,rax
1472	mov	rax,rbp
1473	adc	rdx,0
1474	mov	rcx,rdx
1475
1476	mul	QWORD[24+rsi]
1477	add	r8,rcx
1478	adc	rdx,0
1479	add	r8,rax
1480	mov	rax,r11
1481	adc	r9,rdx
1482	adc	r10,0
1483
1484
1485
1486	mov	rbp,r11
1487	shl	r11,32
1488	mul	r15
1489	shr	rbp,32
1490	add	r12,r11
1491	adc	r13,rbp
1492	mov	rcx,r12
1493	adc	r8,rax
1494	adc	r9,rdx
1495	mov	rbp,r13
1496	adc	r10,0
1497
1498
1499
1500	sub	r12,-1
1501	mov	rbx,r8
1502	sbb	r13,r14
1503	sbb	r8,0
1504	mov	rdx,r9
1505	sbb	r9,r15
1506	sbb	r10,0
1507
1508	cmovc	r12,rcx
1509	cmovc	r13,rbp
1510	mov	QWORD[rdi],r12
1511	cmovc	r8,rbx
1512	mov	QWORD[8+rdi],r13
1513	cmovc	r9,rdx
1514	mov	QWORD[16+rdi],r8
1515	mov	QWORD[24+rdi],r9
1516
1517	ret
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528global	ecp_nistz256_sqr_mont
1529
1530ALIGN	32
1531ecp_nistz256_sqr_mont:
1532	mov	QWORD[8+rsp],rdi	;WIN64 prologue
1533	mov	QWORD[16+rsp],rsi
1534	mov	rax,rsp
1535$L$SEH_begin_ecp_nistz256_sqr_mont:
1536	mov	rdi,rcx
1537	mov	rsi,rdx
1538
1539
1540
1541_CET_ENDBR
1542	lea	rcx,[OPENSSL_ia32cap_P]
1543	mov	rcx,QWORD[8+rcx]
1544	and	ecx,0x80100
1545	push	rbp
1546
1547	push	rbx
1548
1549	push	r12
1550
1551	push	r13
1552
1553	push	r14
1554
1555	push	r15
1556
1557$L$sqr_body:
1558	cmp	ecx,0x80100
1559	je	NEAR $L$sqr_montx
1560	mov	rax,QWORD[rsi]
1561	mov	r14,QWORD[8+rsi]
1562	mov	r15,QWORD[16+rsi]
1563	mov	r8,QWORD[24+rsi]
1564
1565	call	__ecp_nistz256_sqr_montq
1566	jmp	NEAR $L$sqr_mont_done
1567
1568ALIGN	32
1569$L$sqr_montx:
1570	mov	rdx,QWORD[rsi]
1571	mov	r14,QWORD[8+rsi]
1572	mov	r15,QWORD[16+rsi]
1573	mov	r8,QWORD[24+rsi]
1574	lea	rsi,[((-128))+rsi]
1575
1576	call	__ecp_nistz256_sqr_montx
1577$L$sqr_mont_done:
1578	mov	r15,QWORD[rsp]
1579
1580	mov	r14,QWORD[8+rsp]
1581
1582	mov	r13,QWORD[16+rsp]
1583
1584	mov	r12,QWORD[24+rsp]
1585
1586	mov	rbx,QWORD[32+rsp]
1587
1588	mov	rbp,QWORD[40+rsp]
1589
1590	lea	rsp,[48+rsp]
1591
1592$L$sqr_epilogue:
1593	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
1594	mov	rsi,QWORD[16+rsp]
1595	ret
1596
1597$L$SEH_end_ecp_nistz256_sqr_mont:
1598
1599
1600ALIGN	32
1601__ecp_nistz256_sqr_montq:
1602
1603	mov	r13,rax
1604	mul	r14
1605	mov	r9,rax
1606	mov	rax,r15
1607	mov	r10,rdx
1608
1609	mul	r13
1610	add	r10,rax
1611	mov	rax,r8
1612	adc	rdx,0
1613	mov	r11,rdx
1614
1615	mul	r13
1616	add	r11,rax
1617	mov	rax,r15
1618	adc	rdx,0
1619	mov	r12,rdx
1620
1621
1622	mul	r14
1623	add	r11,rax
1624	mov	rax,r8
1625	adc	rdx,0
1626	mov	rbp,rdx
1627
1628	mul	r14
1629	add	r12,rax
1630	mov	rax,r8
1631	adc	rdx,0
1632	add	r12,rbp
1633	mov	r13,rdx
1634	adc	r13,0
1635
1636
1637	mul	r15
1638	xor	r15,r15
1639	add	r13,rax
1640	mov	rax,QWORD[rsi]
1641	mov	r14,rdx
1642	adc	r14,0
1643
1644	add	r9,r9
1645	adc	r10,r10
1646	adc	r11,r11
1647	adc	r12,r12
1648	adc	r13,r13
1649	adc	r14,r14
1650	adc	r15,0
1651
1652	mul	rax
1653	mov	r8,rax
1654	mov	rax,QWORD[8+rsi]
1655	mov	rcx,rdx
1656
1657	mul	rax
1658	add	r9,rcx
1659	adc	r10,rax
1660	mov	rax,QWORD[16+rsi]
1661	adc	rdx,0
1662	mov	rcx,rdx
1663
1664	mul	rax
1665	add	r11,rcx
1666	adc	r12,rax
1667	mov	rax,QWORD[24+rsi]
1668	adc	rdx,0
1669	mov	rcx,rdx
1670
1671	mul	rax
1672	add	r13,rcx
1673	adc	r14,rax
1674	mov	rax,r8
1675	adc	r15,rdx
1676
1677	mov	rsi,QWORD[(($L$poly+8))]
1678	mov	rbp,QWORD[(($L$poly+24))]
1679
1680
1681
1682
1683	mov	rcx,r8
1684	shl	r8,32
1685	mul	rbp
1686	shr	rcx,32
1687	add	r9,r8
1688	adc	r10,rcx
1689	adc	r11,rax
1690	mov	rax,r9
1691	adc	rdx,0
1692
1693
1694
1695	mov	rcx,r9
1696	shl	r9,32
1697	mov	r8,rdx
1698	mul	rbp
1699	shr	rcx,32
1700	add	r10,r9
1701	adc	r11,rcx
1702	adc	r8,rax
1703	mov	rax,r10
1704	adc	rdx,0
1705
1706
1707
1708	mov	rcx,r10
1709	shl	r10,32
1710	mov	r9,rdx
1711	mul	rbp
1712	shr	rcx,32
1713	add	r11,r10
1714	adc	r8,rcx
1715	adc	r9,rax
1716	mov	rax,r11
1717	adc	rdx,0
1718
1719
1720
1721	mov	rcx,r11
1722	shl	r11,32
1723	mov	r10,rdx
1724	mul	rbp
1725	shr	rcx,32
1726	add	r8,r11
1727	adc	r9,rcx
1728	adc	r10,rax
1729	adc	rdx,0
1730	xor	r11,r11
1731
1732
1733
1734	add	r12,r8
1735	adc	r13,r9
1736	mov	r8,r12
1737	adc	r14,r10
1738	adc	r15,rdx
1739	mov	r9,r13
1740	adc	r11,0
1741
1742	sub	r12,-1
1743	mov	r10,r14
1744	sbb	r13,rsi
1745	sbb	r14,0
1746	mov	rcx,r15
1747	sbb	r15,rbp
1748	sbb	r11,0
1749
1750	cmovc	r12,r8
1751	cmovc	r13,r9
1752	mov	QWORD[rdi],r12
1753	cmovc	r14,r10
1754	mov	QWORD[8+rdi],r13
1755	cmovc	r15,rcx
1756	mov	QWORD[16+rdi],r14
1757	mov	QWORD[24+rdi],r15
1758
1759	ret
1760
1761
1762
1763ALIGN	32
1764__ecp_nistz256_mul_montx:
1765
1766
1767
1768	mulx	r9,r8,r9
1769	mulx	r10,rcx,r10
1770	mov	r14,32
1771	xor	r13,r13
1772	mulx	r11,rbp,r11
1773	mov	r15,QWORD[(($L$poly+24))]
1774	adc	r9,rcx
1775	mulx	r12,rcx,r12
1776	mov	rdx,r8
1777	adc	r10,rbp
1778	shlx	rbp,r8,r14
1779	adc	r11,rcx
1780	shrx	rcx,r8,r14
1781	adc	r12,0
1782
1783
1784
1785	add	r9,rbp
1786	adc	r10,rcx
1787
1788	mulx	rbp,rcx,r15
1789	mov	rdx,QWORD[8+rbx]
1790	adc	r11,rcx
1791	adc	r12,rbp
1792	adc	r13,0
1793	xor	r8,r8
1794
1795
1796
1797	mulx	rbp,rcx,QWORD[((0+128))+rsi]
1798	adcx	r9,rcx
1799	adox	r10,rbp
1800
1801	mulx	rbp,rcx,QWORD[((8+128))+rsi]
1802	adcx	r10,rcx
1803	adox	r11,rbp
1804
1805	mulx	rbp,rcx,QWORD[((16+128))+rsi]
1806	adcx	r11,rcx
1807	adox	r12,rbp
1808
1809	mulx	rbp,rcx,QWORD[((24+128))+rsi]
1810	mov	rdx,r9
1811	adcx	r12,rcx
1812	shlx	rcx,r9,r14
1813	adox	r13,rbp
1814	shrx	rbp,r9,r14
1815
1816	adcx	r13,r8
1817	adox	r8,r8
1818	adc	r8,0
1819
1820
1821
1822	add	r10,rcx
1823	adc	r11,rbp
1824
1825	mulx	rbp,rcx,r15
1826	mov	rdx,QWORD[16+rbx]
1827	adc	r12,rcx
1828	adc	r13,rbp
1829	adc	r8,0
1830	xor	r9,r9
1831
1832
1833
1834	mulx	rbp,rcx,QWORD[((0+128))+rsi]
1835	adcx	r10,rcx
1836	adox	r11,rbp
1837
1838	mulx	rbp,rcx,QWORD[((8+128))+rsi]
1839	adcx	r11,rcx
1840	adox	r12,rbp
1841
1842	mulx	rbp,rcx,QWORD[((16+128))+rsi]
1843	adcx	r12,rcx
1844	adox	r13,rbp
1845
1846	mulx	rbp,rcx,QWORD[((24+128))+rsi]
1847	mov	rdx,r10
1848	adcx	r13,rcx
1849	shlx	rcx,r10,r14
1850	adox	r8,rbp
1851	shrx	rbp,r10,r14
1852
1853	adcx	r8,r9
1854	adox	r9,r9
1855	adc	r9,0
1856
1857
1858
1859	add	r11,rcx
1860	adc	r12,rbp
1861
1862	mulx	rbp,rcx,r15
1863	mov	rdx,QWORD[24+rbx]
1864	adc	r13,rcx
1865	adc	r8,rbp
1866	adc	r9,0
1867	xor	r10,r10
1868
1869
1870
1871	mulx	rbp,rcx,QWORD[((0+128))+rsi]
1872	adcx	r11,rcx
1873	adox	r12,rbp
1874
1875	mulx	rbp,rcx,QWORD[((8+128))+rsi]
1876	adcx	r12,rcx
1877	adox	r13,rbp
1878
1879	mulx	rbp,rcx,QWORD[((16+128))+rsi]
1880	adcx	r13,rcx
1881	adox	r8,rbp
1882
1883	mulx	rbp,rcx,QWORD[((24+128))+rsi]
1884	mov	rdx,r11
1885	adcx	r8,rcx
1886	shlx	rcx,r11,r14
1887	adox	r9,rbp
1888	shrx	rbp,r11,r14
1889
1890	adcx	r9,r10
1891	adox	r10,r10
1892	adc	r10,0
1893
1894
1895
1896	add	r12,rcx
1897	adc	r13,rbp
1898
1899	mulx	rbp,rcx,r15
1900	mov	rbx,r12
1901	mov	r14,QWORD[(($L$poly+8))]
1902	adc	r8,rcx
1903	mov	rdx,r13
1904	adc	r9,rbp
1905	adc	r10,0
1906
1907
1908
1909	xor	eax,eax
1910	mov	rcx,r8
1911	sbb	r12,-1
1912	sbb	r13,r14
1913	sbb	r8,0
1914	mov	rbp,r9
1915	sbb	r9,r15
1916	sbb	r10,0
1917
1918	cmovc	r12,rbx
1919	cmovc	r13,rdx
1920	mov	QWORD[rdi],r12
1921	cmovc	r8,rcx
1922	mov	QWORD[8+rdi],r13
1923	cmovc	r9,rbp
1924	mov	QWORD[16+rdi],r8
1925	mov	QWORD[24+rdi],r9
1926
1927	ret
1928
1929
1930
1931
1932ALIGN	32
1933__ecp_nistz256_sqr_montx:
1934
1935	mulx	r10,r9,r14
1936	mulx	r11,rcx,r15
1937	xor	eax,eax
1938	adc	r10,rcx
1939	mulx	r12,rbp,r8
1940	mov	rdx,r14
1941	adc	r11,rbp
1942	adc	r12,0
1943	xor	r13,r13
1944
1945
1946	mulx	rbp,rcx,r15
1947	adcx	r11,rcx
1948	adox	r12,rbp
1949
1950	mulx	rbp,rcx,r8
1951	mov	rdx,r15
1952	adcx	r12,rcx
1953	adox	r13,rbp
1954	adc	r13,0
1955
1956
1957	mulx	r14,rcx,r8
1958	mov	rdx,QWORD[((0+128))+rsi]
1959	xor	r15,r15
1960	adcx	r9,r9
1961	adox	r13,rcx
1962	adcx	r10,r10
1963	adox	r14,r15
1964
1965	mulx	rbp,r8,rdx
1966	mov	rdx,QWORD[((8+128))+rsi]
1967	adcx	r11,r11
1968	adox	r9,rbp
1969	adcx	r12,r12
1970	mulx	rax,rcx,rdx
1971	mov	rdx,QWORD[((16+128))+rsi]
1972	adcx	r13,r13
1973	adox	r10,rcx
1974	adcx	r14,r14
1975	DB	0x67
1976	mulx	rbp,rcx,rdx
1977	mov	rdx,QWORD[((24+128))+rsi]
1978	adox	r11,rax
1979	adcx	r15,r15
1980	adox	r12,rcx
1981	mov	rsi,32
1982	adox	r13,rbp
1983	DB	0x67,0x67
1984	mulx	rax,rcx,rdx
1985	mov	rdx,QWORD[(($L$poly+24))]
1986	adox	r14,rcx
1987	shlx	rcx,r8,rsi
1988	adox	r15,rax
1989	shrx	rax,r8,rsi
1990	mov	rbp,rdx
1991
1992
1993	add	r9,rcx
1994	adc	r10,rax
1995
1996	mulx	r8,rcx,r8
1997	adc	r11,rcx
1998	shlx	rcx,r9,rsi
1999	adc	r8,0
2000	shrx	rax,r9,rsi
2001
2002
2003	add	r10,rcx
2004	adc	r11,rax
2005
2006	mulx	r9,rcx,r9
2007	adc	r8,rcx
2008	shlx	rcx,r10,rsi
2009	adc	r9,0
2010	shrx	rax,r10,rsi
2011
2012
2013	add	r11,rcx
2014	adc	r8,rax
2015
2016	mulx	r10,rcx,r10
2017	adc	r9,rcx
2018	shlx	rcx,r11,rsi
2019	adc	r10,0
2020	shrx	rax,r11,rsi
2021
2022
2023	add	r8,rcx
2024	adc	r9,rax
2025
2026	mulx	r11,rcx,r11
2027	adc	r10,rcx
2028	adc	r11,0
2029
2030	xor	rdx,rdx
2031	add	r12,r8
2032	mov	rsi,QWORD[(($L$poly+8))]
2033	adc	r13,r9
2034	mov	r8,r12
2035	adc	r14,r10
2036	adc	r15,r11
2037	mov	r9,r13
2038	adc	rdx,0
2039
2040	sub	r12,-1
2041	mov	r10,r14
2042	sbb	r13,rsi
2043	sbb	r14,0
2044	mov	r11,r15
2045	sbb	r15,rbp
2046	sbb	rdx,0
2047
2048	cmovc	r12,r8
2049	cmovc	r13,r9
2050	mov	QWORD[rdi],r12
2051	cmovc	r14,r10
2052	mov	QWORD[8+rdi],r13
2053	cmovc	r15,r11
2054	mov	QWORD[16+rdi],r14
2055	mov	QWORD[24+rdi],r15
2056
2057	ret
2058
2059
2060
2061
2062global	ecp_nistz256_select_w5
2063
2064ALIGN	32
2065ecp_nistz256_select_w5:
2066
2067_CET_ENDBR
2068	lea	rax,[OPENSSL_ia32cap_P]
2069	mov	rax,QWORD[8+rax]
2070	test	eax,32
2071	jnz	NEAR $L$avx2_select_w5
2072	lea	rax,[((-136))+rsp]
2073$L$SEH_begin_ecp_nistz256_select_w5:
2074	DB	0x48,0x8d,0x60,0xe0
2075	DB	0x0f,0x29,0x70,0xe0
2076	DB	0x0f,0x29,0x78,0xf0
2077	DB	0x44,0x0f,0x29,0x00
2078	DB	0x44,0x0f,0x29,0x48,0x10
2079	DB	0x44,0x0f,0x29,0x50,0x20
2080	DB	0x44,0x0f,0x29,0x58,0x30
2081	DB	0x44,0x0f,0x29,0x60,0x40
2082	DB	0x44,0x0f,0x29,0x68,0x50
2083	DB	0x44,0x0f,0x29,0x70,0x60
2084	DB	0x44,0x0f,0x29,0x78,0x70
2085	movdqa	xmm0,XMMWORD[$L$One]
2086	movd	xmm1,r8d
2087
2088	pxor	xmm2,xmm2
2089	pxor	xmm3,xmm3
2090	pxor	xmm4,xmm4
2091	pxor	xmm5,xmm5
2092	pxor	xmm6,xmm6
2093	pxor	xmm7,xmm7
2094
2095	movdqa	xmm8,xmm0
2096	pshufd	xmm1,xmm1,0
2097
2098	mov	rax,16
2099$L$select_loop_sse_w5:
2100
2101	movdqa	xmm15,xmm8
2102	paddd	xmm8,xmm0
2103	pcmpeqd	xmm15,xmm1
2104
2105	movdqa	xmm9,XMMWORD[rdx]
2106	movdqa	xmm10,XMMWORD[16+rdx]
2107	movdqa	xmm11,XMMWORD[32+rdx]
2108	movdqa	xmm12,XMMWORD[48+rdx]
2109	movdqa	xmm13,XMMWORD[64+rdx]
2110	movdqa	xmm14,XMMWORD[80+rdx]
2111	lea	rdx,[96+rdx]
2112
2113	pand	xmm9,xmm15
2114	pand	xmm10,xmm15
2115	por	xmm2,xmm9
2116	pand	xmm11,xmm15
2117	por	xmm3,xmm10
2118	pand	xmm12,xmm15
2119	por	xmm4,xmm11
2120	pand	xmm13,xmm15
2121	por	xmm5,xmm12
2122	pand	xmm14,xmm15
2123	por	xmm6,xmm13
2124	por	xmm7,xmm14
2125
2126	dec	rax
2127	jnz	NEAR $L$select_loop_sse_w5
2128
2129	movdqu	XMMWORD[rcx],xmm2
2130	movdqu	XMMWORD[16+rcx],xmm3
2131	movdqu	XMMWORD[32+rcx],xmm4
2132	movdqu	XMMWORD[48+rcx],xmm5
2133	movdqu	XMMWORD[64+rcx],xmm6
2134	movdqu	XMMWORD[80+rcx],xmm7
2135	movaps	xmm6,XMMWORD[rsp]
2136	movaps	xmm7,XMMWORD[16+rsp]
2137	movaps	xmm8,XMMWORD[32+rsp]
2138	movaps	xmm9,XMMWORD[48+rsp]
2139	movaps	xmm10,XMMWORD[64+rsp]
2140	movaps	xmm11,XMMWORD[80+rsp]
2141	movaps	xmm12,XMMWORD[96+rsp]
2142	movaps	xmm13,XMMWORD[112+rsp]
2143	movaps	xmm14,XMMWORD[128+rsp]
2144	movaps	xmm15,XMMWORD[144+rsp]
2145	lea	rsp,[168+rsp]
2146	ret
2147
2148$L$SEH_end_ecp_nistz256_select_w5:
2149
2150
2151
2152
2153global	ecp_nistz256_select_w7
2154
2155ALIGN	32
2156ecp_nistz256_select_w7:
2157
2158_CET_ENDBR
2159	lea	rax,[OPENSSL_ia32cap_P]
2160	mov	rax,QWORD[8+rax]
2161	test	eax,32
2162	jnz	NEAR $L$avx2_select_w7
2163	lea	rax,[((-136))+rsp]
2164$L$SEH_begin_ecp_nistz256_select_w7:
2165	DB	0x48,0x8d,0x60,0xe0
2166	DB	0x0f,0x29,0x70,0xe0
2167	DB	0x0f,0x29,0x78,0xf0
2168	DB	0x44,0x0f,0x29,0x00
2169	DB	0x44,0x0f,0x29,0x48,0x10
2170	DB	0x44,0x0f,0x29,0x50,0x20
2171	DB	0x44,0x0f,0x29,0x58,0x30
2172	DB	0x44,0x0f,0x29,0x60,0x40
2173	DB	0x44,0x0f,0x29,0x68,0x50
2174	DB	0x44,0x0f,0x29,0x70,0x60
2175	DB	0x44,0x0f,0x29,0x78,0x70
2176	movdqa	xmm8,XMMWORD[$L$One]
2177	movd	xmm1,r8d
2178
2179	pxor	xmm2,xmm2
2180	pxor	xmm3,xmm3
2181	pxor	xmm4,xmm4
2182	pxor	xmm5,xmm5
2183
2184	movdqa	xmm0,xmm8
2185	pshufd	xmm1,xmm1,0
2186	mov	rax,64
2187
2188$L$select_loop_sse_w7:
2189	movdqa	xmm15,xmm8
2190	paddd	xmm8,xmm0
2191	movdqa	xmm9,XMMWORD[rdx]
2192	movdqa	xmm10,XMMWORD[16+rdx]
2193	pcmpeqd	xmm15,xmm1
2194	movdqa	xmm11,XMMWORD[32+rdx]
2195	movdqa	xmm12,XMMWORD[48+rdx]
2196	lea	rdx,[64+rdx]
2197
2198	pand	xmm9,xmm15
2199	pand	xmm10,xmm15
2200	por	xmm2,xmm9
2201	pand	xmm11,xmm15
2202	por	xmm3,xmm10
2203	pand	xmm12,xmm15
2204	por	xmm4,xmm11
2205	prefetcht0	[255+rdx]
2206	por	xmm5,xmm12
2207
2208	dec	rax
2209	jnz	NEAR $L$select_loop_sse_w7
2210
2211	movdqu	XMMWORD[rcx],xmm2
2212	movdqu	XMMWORD[16+rcx],xmm3
2213	movdqu	XMMWORD[32+rcx],xmm4
2214	movdqu	XMMWORD[48+rcx],xmm5
2215	movaps	xmm6,XMMWORD[rsp]
2216	movaps	xmm7,XMMWORD[16+rsp]
2217	movaps	xmm8,XMMWORD[32+rsp]
2218	movaps	xmm9,XMMWORD[48+rsp]
2219	movaps	xmm10,XMMWORD[64+rsp]
2220	movaps	xmm11,XMMWORD[80+rsp]
2221	movaps	xmm12,XMMWORD[96+rsp]
2222	movaps	xmm13,XMMWORD[112+rsp]
2223	movaps	xmm14,XMMWORD[128+rsp]
2224	movaps	xmm15,XMMWORD[144+rsp]
2225	lea	rsp,[168+rsp]
2226	ret
2227
2228$L$SEH_end_ecp_nistz256_select_w7:
2229
2230
2231
2232
2233ALIGN	32
2234ecp_nistz256_avx2_select_w5:
2235
2236$L$avx2_select_w5:
2237	vzeroupper
2238	lea	rax,[((-136))+rsp]
2239	mov	r11,rsp
2240$L$SEH_begin_ecp_nistz256_avx2_select_w5:
2241	DB	0x48,0x8d,0x60,0xe0
2242	DB	0xc5,0xf8,0x29,0x70,0xe0
2243	DB	0xc5,0xf8,0x29,0x78,0xf0
2244	DB	0xc5,0x78,0x29,0x40,0x00
2245	DB	0xc5,0x78,0x29,0x48,0x10
2246	DB	0xc5,0x78,0x29,0x50,0x20
2247	DB	0xc5,0x78,0x29,0x58,0x30
2248	DB	0xc5,0x78,0x29,0x60,0x40
2249	DB	0xc5,0x78,0x29,0x68,0x50
2250	DB	0xc5,0x78,0x29,0x70,0x60
2251	DB	0xc5,0x78,0x29,0x78,0x70
2252	vmovdqa	ymm0,YMMWORD[$L$Two]
2253
2254	vpxor	ymm2,ymm2,ymm2
2255	vpxor	ymm3,ymm3,ymm3
2256	vpxor	ymm4,ymm4,ymm4
2257
2258	vmovdqa	ymm5,YMMWORD[$L$One]
2259	vmovdqa	ymm10,YMMWORD[$L$Two]
2260
2261	vmovd	xmm1,r8d
2262	vpermd	ymm1,ymm2,ymm1
2263
2264	mov	rax,8
2265$L$select_loop_avx2_w5:
2266
2267	vmovdqa	ymm6,YMMWORD[rdx]
2268	vmovdqa	ymm7,YMMWORD[32+rdx]
2269	vmovdqa	ymm8,YMMWORD[64+rdx]
2270
2271	vmovdqa	ymm11,YMMWORD[96+rdx]
2272	vmovdqa	ymm12,YMMWORD[128+rdx]
2273	vmovdqa	ymm13,YMMWORD[160+rdx]
2274
2275	vpcmpeqd	ymm9,ymm5,ymm1
2276	vpcmpeqd	ymm14,ymm10,ymm1
2277
2278	vpaddd	ymm5,ymm5,ymm0
2279	vpaddd	ymm10,ymm10,ymm0
2280	lea	rdx,[192+rdx]
2281
2282	vpand	ymm6,ymm6,ymm9
2283	vpand	ymm7,ymm7,ymm9
2284	vpand	ymm8,ymm8,ymm9
2285	vpand	ymm11,ymm11,ymm14
2286	vpand	ymm12,ymm12,ymm14
2287	vpand	ymm13,ymm13,ymm14
2288
2289	vpxor	ymm2,ymm2,ymm6
2290	vpxor	ymm3,ymm3,ymm7
2291	vpxor	ymm4,ymm4,ymm8
2292	vpxor	ymm2,ymm2,ymm11
2293	vpxor	ymm3,ymm3,ymm12
2294	vpxor	ymm4,ymm4,ymm13
2295
2296	dec	rax
2297	jnz	NEAR $L$select_loop_avx2_w5
2298
2299	vmovdqu	YMMWORD[rcx],ymm2
2300	vmovdqu	YMMWORD[32+rcx],ymm3
2301	vmovdqu	YMMWORD[64+rcx],ymm4
2302	vzeroupper
2303	movaps	xmm6,XMMWORD[rsp]
2304	movaps	xmm7,XMMWORD[16+rsp]
2305	movaps	xmm8,XMMWORD[32+rsp]
2306	movaps	xmm9,XMMWORD[48+rsp]
2307	movaps	xmm10,XMMWORD[64+rsp]
2308	movaps	xmm11,XMMWORD[80+rsp]
2309	movaps	xmm12,XMMWORD[96+rsp]
2310	movaps	xmm13,XMMWORD[112+rsp]
2311	movaps	xmm14,XMMWORD[128+rsp]
2312	movaps	xmm15,XMMWORD[144+rsp]
2313	lea	rsp,[r11]
2314	ret
2315
2316$L$SEH_end_ecp_nistz256_avx2_select_w5:
2317
2318
2319
2320
2321global	ecp_nistz256_avx2_select_w7
2322
2323ALIGN	32
2324ecp_nistz256_avx2_select_w7:
2325
2326$L$avx2_select_w7:
2327_CET_ENDBR
2328	vzeroupper
2329	mov	r11,rsp
2330	lea	rax,[((-136))+rsp]
2331$L$SEH_begin_ecp_nistz256_avx2_select_w7:
2332	DB	0x48,0x8d,0x60,0xe0
2333	DB	0xc5,0xf8,0x29,0x70,0xe0
2334	DB	0xc5,0xf8,0x29,0x78,0xf0
2335	DB	0xc5,0x78,0x29,0x40,0x00
2336	DB	0xc5,0x78,0x29,0x48,0x10
2337	DB	0xc5,0x78,0x29,0x50,0x20
2338	DB	0xc5,0x78,0x29,0x58,0x30
2339	DB	0xc5,0x78,0x29,0x60,0x40
2340	DB	0xc5,0x78,0x29,0x68,0x50
2341	DB	0xc5,0x78,0x29,0x70,0x60
2342	DB	0xc5,0x78,0x29,0x78,0x70
2343	vmovdqa	ymm0,YMMWORD[$L$Three]
2344
2345	vpxor	ymm2,ymm2,ymm2
2346	vpxor	ymm3,ymm3,ymm3
2347
2348	vmovdqa	ymm4,YMMWORD[$L$One]
2349	vmovdqa	ymm8,YMMWORD[$L$Two]
2350	vmovdqa	ymm12,YMMWORD[$L$Three]
2351
2352	vmovd	xmm1,r8d
2353	vpermd	ymm1,ymm2,ymm1
2354
2355
2356	mov	rax,21
2357$L$select_loop_avx2_w7:
2358
2359	vmovdqa	ymm5,YMMWORD[rdx]
2360	vmovdqa	ymm6,YMMWORD[32+rdx]
2361
2362	vmovdqa	ymm9,YMMWORD[64+rdx]
2363	vmovdqa	ymm10,YMMWORD[96+rdx]
2364
2365	vmovdqa	ymm13,YMMWORD[128+rdx]
2366	vmovdqa	ymm14,YMMWORD[160+rdx]
2367
2368	vpcmpeqd	ymm7,ymm4,ymm1
2369	vpcmpeqd	ymm11,ymm8,ymm1
2370	vpcmpeqd	ymm15,ymm12,ymm1
2371
2372	vpaddd	ymm4,ymm4,ymm0
2373	vpaddd	ymm8,ymm8,ymm0
2374	vpaddd	ymm12,ymm12,ymm0
2375	lea	rdx,[192+rdx]
2376
2377	vpand	ymm5,ymm5,ymm7
2378	vpand	ymm6,ymm6,ymm7
2379	vpand	ymm9,ymm9,ymm11
2380	vpand	ymm10,ymm10,ymm11
2381	vpand	ymm13,ymm13,ymm15
2382	vpand	ymm14,ymm14,ymm15
2383
2384	vpxor	ymm2,ymm2,ymm5
2385	vpxor	ymm3,ymm3,ymm6
2386	vpxor	ymm2,ymm2,ymm9
2387	vpxor	ymm3,ymm3,ymm10
2388	vpxor	ymm2,ymm2,ymm13
2389	vpxor	ymm3,ymm3,ymm14
2390
2391	dec	rax
2392	jnz	NEAR $L$select_loop_avx2_w7
2393
2394
2395	vmovdqa	ymm5,YMMWORD[rdx]
2396	vmovdqa	ymm6,YMMWORD[32+rdx]
2397
2398	vpcmpeqd	ymm7,ymm4,ymm1
2399
2400	vpand	ymm5,ymm5,ymm7
2401	vpand	ymm6,ymm6,ymm7
2402
2403	vpxor	ymm2,ymm2,ymm5
2404	vpxor	ymm3,ymm3,ymm6
2405
2406	vmovdqu	YMMWORD[rcx],ymm2
2407	vmovdqu	YMMWORD[32+rcx],ymm3
2408	vzeroupper
2409	movaps	xmm6,XMMWORD[rsp]
2410	movaps	xmm7,XMMWORD[16+rsp]
2411	movaps	xmm8,XMMWORD[32+rsp]
2412	movaps	xmm9,XMMWORD[48+rsp]
2413	movaps	xmm10,XMMWORD[64+rsp]
2414	movaps	xmm11,XMMWORD[80+rsp]
2415	movaps	xmm12,XMMWORD[96+rsp]
2416	movaps	xmm13,XMMWORD[112+rsp]
2417	movaps	xmm14,XMMWORD[128+rsp]
2418	movaps	xmm15,XMMWORD[144+rsp]
2419	lea	rsp,[r11]
2420	ret
2421
2422$L$SEH_end_ecp_nistz256_avx2_select_w7:
2423
2424
2425ALIGN	32
2426__ecp_nistz256_add_toq:
2427
2428	xor	r11,r11
2429	add	r12,QWORD[rbx]
2430	adc	r13,QWORD[8+rbx]
2431	mov	rax,r12
2432	adc	r8,QWORD[16+rbx]
2433	adc	r9,QWORD[24+rbx]
2434	mov	rbp,r13
2435	adc	r11,0
2436
2437	sub	r12,-1
2438	mov	rcx,r8
2439	sbb	r13,r14
2440	sbb	r8,0
2441	mov	r10,r9
2442	sbb	r9,r15
2443	sbb	r11,0
2444
2445	cmovc	r12,rax
2446	cmovc	r13,rbp
2447	mov	QWORD[rdi],r12
2448	cmovc	r8,rcx
2449	mov	QWORD[8+rdi],r13
2450	cmovc	r9,r10
2451	mov	QWORD[16+rdi],r8
2452	mov	QWORD[24+rdi],r9
2453
2454	ret
2455
2456
2457
2458
2459ALIGN	32
2460__ecp_nistz256_sub_fromq:
2461
2462	sub	r12,QWORD[rbx]
2463	sbb	r13,QWORD[8+rbx]
2464	mov	rax,r12
2465	sbb	r8,QWORD[16+rbx]
2466	sbb	r9,QWORD[24+rbx]
2467	mov	rbp,r13
2468	sbb	r11,r11
2469
2470	add	r12,-1
2471	mov	rcx,r8
2472	adc	r13,r14
2473	adc	r8,0
2474	mov	r10,r9
2475	adc	r9,r15
2476	test	r11,r11
2477
2478	cmovz	r12,rax
2479	cmovz	r13,rbp
2480	mov	QWORD[rdi],r12
2481	cmovz	r8,rcx
2482	mov	QWORD[8+rdi],r13
2483	cmovz	r9,r10
2484	mov	QWORD[16+rdi],r8
2485	mov	QWORD[24+rdi],r9
2486
2487	ret
2488
2489
2490
2491
2492ALIGN	32
2493__ecp_nistz256_subq:
2494
2495	sub	rax,r12
2496	sbb	rbp,r13
2497	mov	r12,rax
2498	sbb	rcx,r8
2499	sbb	r10,r9
2500	mov	r13,rbp
2501	sbb	r11,r11
2502
2503	add	rax,-1
2504	mov	r8,rcx
2505	adc	rbp,r14
2506	adc	rcx,0
2507	mov	r9,r10
2508	adc	r10,r15
2509	test	r11,r11
2510
2511	cmovnz	r12,rax
2512	cmovnz	r13,rbp
2513	cmovnz	r8,rcx
2514	cmovnz	r9,r10
2515
2516	ret
2517
2518
2519
2520
2521ALIGN	32
2522__ecp_nistz256_mul_by_2q:
2523
2524	xor	r11,r11
2525	add	r12,r12
2526	adc	r13,r13
2527	mov	rax,r12
2528	adc	r8,r8
2529	adc	r9,r9
2530	mov	rbp,r13
2531	adc	r11,0
2532
2533	sub	r12,-1
2534	mov	rcx,r8
2535	sbb	r13,r14
2536	sbb	r8,0
2537	mov	r10,r9
2538	sbb	r9,r15
2539	sbb	r11,0
2540
2541	cmovc	r12,rax
2542	cmovc	r13,rbp
2543	mov	QWORD[rdi],r12
2544	cmovc	r8,rcx
2545	mov	QWORD[8+rdi],r13
2546	cmovc	r9,r10
2547	mov	QWORD[16+rdi],r8
2548	mov	QWORD[24+rdi],r9
2549
2550	ret
2551
2552
2553global	ecp_nistz256_point_double
2554
2555ALIGN	32
2556ecp_nistz256_point_double:
2557	mov	QWORD[8+rsp],rdi	;WIN64 prologue
2558	mov	QWORD[16+rsp],rsi
2559	mov	rax,rsp
2560$L$SEH_begin_ecp_nistz256_point_double:
2561	mov	rdi,rcx
2562	mov	rsi,rdx
2563
2564
2565
2566_CET_ENDBR
2567	lea	rcx,[OPENSSL_ia32cap_P]
2568	mov	rcx,QWORD[8+rcx]
2569	and	ecx,0x80100
2570	cmp	ecx,0x80100
2571	je	NEAR $L$point_doublex
2572	push	rbp
2573
2574	push	rbx
2575
2576	push	r12
2577
2578	push	r13
2579
2580	push	r14
2581
2582	push	r15
2583
2584	sub	rsp,32*5+8
2585
2586$L$point_doubleq_body:
2587
2588$L$point_double_shortcutq:
2589	movdqu	xmm0,XMMWORD[rsi]
2590	mov	rbx,rsi
2591	movdqu	xmm1,XMMWORD[16+rsi]
2592	mov	r12,QWORD[((32+0))+rsi]
2593	mov	r13,QWORD[((32+8))+rsi]
2594	mov	r8,QWORD[((32+16))+rsi]
2595	mov	r9,QWORD[((32+24))+rsi]
2596	mov	r14,QWORD[(($L$poly+8))]
2597	mov	r15,QWORD[(($L$poly+24))]
2598	movdqa	XMMWORD[96+rsp],xmm0
2599	movdqa	XMMWORD[(96+16)+rsp],xmm1
2600	lea	r10,[32+rdi]
2601	lea	r11,[64+rdi]
2602DB	102,72,15,110,199
2603DB	102,73,15,110,202
2604DB	102,73,15,110,211
2605
2606	lea	rdi,[rsp]
2607	call	__ecp_nistz256_mul_by_2q
2608
2609	mov	rax,QWORD[((64+0))+rsi]
2610	mov	r14,QWORD[((64+8))+rsi]
2611	mov	r15,QWORD[((64+16))+rsi]
2612	mov	r8,QWORD[((64+24))+rsi]
2613	lea	rsi,[((64-0))+rsi]
2614	lea	rdi,[64+rsp]
2615	call	__ecp_nistz256_sqr_montq
2616
2617	mov	rax,QWORD[((0+0))+rsp]
2618	mov	r14,QWORD[((8+0))+rsp]
2619	lea	rsi,[((0+0))+rsp]
2620	mov	r15,QWORD[((16+0))+rsp]
2621	mov	r8,QWORD[((24+0))+rsp]
2622	lea	rdi,[rsp]
2623	call	__ecp_nistz256_sqr_montq
2624
2625	mov	rax,QWORD[32+rbx]
2626	mov	r9,QWORD[((64+0))+rbx]
2627	mov	r10,QWORD[((64+8))+rbx]
2628	mov	r11,QWORD[((64+16))+rbx]
2629	mov	r12,QWORD[((64+24))+rbx]
2630	lea	rsi,[((64-0))+rbx]
2631	lea	rbx,[32+rbx]
2632DB	102,72,15,126,215
2633	call	__ecp_nistz256_mul_montq
2634	call	__ecp_nistz256_mul_by_2q
2635
2636	mov	r12,QWORD[((96+0))+rsp]
2637	mov	r13,QWORD[((96+8))+rsp]
2638	lea	rbx,[64+rsp]
2639	mov	r8,QWORD[((96+16))+rsp]
2640	mov	r9,QWORD[((96+24))+rsp]
2641	lea	rdi,[32+rsp]
2642	call	__ecp_nistz256_add_toq
2643
2644	mov	r12,QWORD[((96+0))+rsp]
2645	mov	r13,QWORD[((96+8))+rsp]
2646	lea	rbx,[64+rsp]
2647	mov	r8,QWORD[((96+16))+rsp]
2648	mov	r9,QWORD[((96+24))+rsp]
2649	lea	rdi,[64+rsp]
2650	call	__ecp_nistz256_sub_fromq
2651
2652	mov	rax,QWORD[((0+0))+rsp]
2653	mov	r14,QWORD[((8+0))+rsp]
2654	lea	rsi,[((0+0))+rsp]
2655	mov	r15,QWORD[((16+0))+rsp]
2656	mov	r8,QWORD[((24+0))+rsp]
2657DB	102,72,15,126,207
2658	call	__ecp_nistz256_sqr_montq
2659	xor	r9,r9
2660	mov	rax,r12
2661	add	r12,-1
2662	mov	r10,r13
2663	adc	r13,rsi
2664	mov	rcx,r14
2665	adc	r14,0
2666	mov	r8,r15
2667	adc	r15,rbp
2668	adc	r9,0
2669	xor	rsi,rsi
2670	test	rax,1
2671
2672	cmovz	r12,rax
2673	cmovz	r13,r10
2674	cmovz	r14,rcx
2675	cmovz	r15,r8
2676	cmovz	r9,rsi
2677
2678	mov	rax,r13
2679	shr	r12,1
2680	shl	rax,63
2681	mov	r10,r14
2682	shr	r13,1
2683	or	r12,rax
2684	shl	r10,63
2685	mov	rcx,r15
2686	shr	r14,1
2687	or	r13,r10
2688	shl	rcx,63
2689	mov	QWORD[rdi],r12
2690	shr	r15,1
2691	mov	QWORD[8+rdi],r13
2692	shl	r9,63
2693	or	r14,rcx
2694	or	r15,r9
2695	mov	QWORD[16+rdi],r14
2696	mov	QWORD[24+rdi],r15
2697	mov	rax,QWORD[64+rsp]
2698	lea	rbx,[64+rsp]
2699	mov	r9,QWORD[((0+32))+rsp]
2700	mov	r10,QWORD[((8+32))+rsp]
2701	lea	rsi,[((0+32))+rsp]
2702	mov	r11,QWORD[((16+32))+rsp]
2703	mov	r12,QWORD[((24+32))+rsp]
2704	lea	rdi,[32+rsp]
2705	call	__ecp_nistz256_mul_montq
2706
2707	lea	rdi,[128+rsp]
2708	call	__ecp_nistz256_mul_by_2q
2709
2710	lea	rbx,[32+rsp]
2711	lea	rdi,[32+rsp]
2712	call	__ecp_nistz256_add_toq
2713
2714	mov	rax,QWORD[96+rsp]
2715	lea	rbx,[96+rsp]
2716	mov	r9,QWORD[((0+0))+rsp]
2717	mov	r10,QWORD[((8+0))+rsp]
2718	lea	rsi,[((0+0))+rsp]
2719	mov	r11,QWORD[((16+0))+rsp]
2720	mov	r12,QWORD[((24+0))+rsp]
2721	lea	rdi,[rsp]
2722	call	__ecp_nistz256_mul_montq
2723
2724	lea	rdi,[128+rsp]
2725	call	__ecp_nistz256_mul_by_2q
2726
2727	mov	rax,QWORD[((0+32))+rsp]
2728	mov	r14,QWORD[((8+32))+rsp]
2729	lea	rsi,[((0+32))+rsp]
2730	mov	r15,QWORD[((16+32))+rsp]
2731	mov	r8,QWORD[((24+32))+rsp]
2732DB	102,72,15,126,199
2733	call	__ecp_nistz256_sqr_montq
2734
2735	lea	rbx,[128+rsp]
2736	mov	r8,r14
2737	mov	r9,r15
2738	mov	r14,rsi
2739	mov	r15,rbp
2740	call	__ecp_nistz256_sub_fromq
2741
2742	mov	rax,QWORD[((0+0))+rsp]
2743	mov	rbp,QWORD[((0+8))+rsp]
2744	mov	rcx,QWORD[((0+16))+rsp]
2745	mov	r10,QWORD[((0+24))+rsp]
2746	lea	rdi,[rsp]
2747	call	__ecp_nistz256_subq
2748
2749	mov	rax,QWORD[32+rsp]
2750	lea	rbx,[32+rsp]
2751	mov	r14,r12
2752	xor	ecx,ecx
2753	mov	QWORD[((0+0))+rsp],r12
2754	mov	r10,r13
2755	mov	QWORD[((0+8))+rsp],r13
2756	cmovz	r11,r8
2757	mov	QWORD[((0+16))+rsp],r8
2758	lea	rsi,[((0-0))+rsp]
2759	cmovz	r12,r9
2760	mov	QWORD[((0+24))+rsp],r9
2761	mov	r9,r14
2762	lea	rdi,[rsp]
2763	call	__ecp_nistz256_mul_montq
2764
2765DB	102,72,15,126,203
2766DB	102,72,15,126,207
2767	call	__ecp_nistz256_sub_fromq
2768
2769	lea	rsi,[((160+56))+rsp]
2770
2771	mov	r15,QWORD[((-48))+rsi]
2772
2773	mov	r14,QWORD[((-40))+rsi]
2774
2775	mov	r13,QWORD[((-32))+rsi]
2776
2777	mov	r12,QWORD[((-24))+rsi]
2778
2779	mov	rbx,QWORD[((-16))+rsi]
2780
2781	mov	rbp,QWORD[((-8))+rsi]
2782
2783	lea	rsp,[rsi]
2784
2785$L$point_doubleq_epilogue:
2786	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
2787	mov	rsi,QWORD[16+rsp]
2788	ret
2789
2790$L$SEH_end_ecp_nistz256_point_double:
2791global	ecp_nistz256_point_add
2792
2793ALIGN	32
2794ecp_nistz256_point_add:
2795	mov	QWORD[8+rsp],rdi	;WIN64 prologue
2796	mov	QWORD[16+rsp],rsi
2797	mov	rax,rsp
2798$L$SEH_begin_ecp_nistz256_point_add:
2799	mov	rdi,rcx
2800	mov	rsi,rdx
2801	mov	rdx,r8
2802
2803
2804
2805_CET_ENDBR
2806	lea	rcx,[OPENSSL_ia32cap_P]
2807	mov	rcx,QWORD[8+rcx]
2808	and	ecx,0x80100
2809	cmp	ecx,0x80100
2810	je	NEAR $L$point_addx
2811	push	rbp
2812
2813	push	rbx
2814
2815	push	r12
2816
2817	push	r13
2818
2819	push	r14
2820
2821	push	r15
2822
2823	sub	rsp,32*18+8
2824
2825$L$point_addq_body:
2826
2827	movdqu	xmm0,XMMWORD[rsi]
2828	movdqu	xmm1,XMMWORD[16+rsi]
2829	movdqu	xmm2,XMMWORD[32+rsi]
2830	movdqu	xmm3,XMMWORD[48+rsi]
2831	movdqu	xmm4,XMMWORD[64+rsi]
2832	movdqu	xmm5,XMMWORD[80+rsi]
2833	mov	rbx,rsi
2834	mov	rsi,rdx
2835	movdqa	XMMWORD[384+rsp],xmm0
2836	movdqa	XMMWORD[(384+16)+rsp],xmm1
2837	movdqa	XMMWORD[416+rsp],xmm2
2838	movdqa	XMMWORD[(416+16)+rsp],xmm3
2839	movdqa	XMMWORD[448+rsp],xmm4
2840	movdqa	XMMWORD[(448+16)+rsp],xmm5
2841	por	xmm5,xmm4
2842
2843	movdqu	xmm0,XMMWORD[rsi]
2844	pshufd	xmm3,xmm5,0xb1
2845	movdqu	xmm1,XMMWORD[16+rsi]
2846	movdqu	xmm2,XMMWORD[32+rsi]
2847	por	xmm5,xmm3
2848	movdqu	xmm3,XMMWORD[48+rsi]
2849	mov	rax,QWORD[((64+0))+rsi]
2850	mov	r14,QWORD[((64+8))+rsi]
2851	mov	r15,QWORD[((64+16))+rsi]
2852	mov	r8,QWORD[((64+24))+rsi]
2853	movdqa	XMMWORD[480+rsp],xmm0
2854	pshufd	xmm4,xmm5,0x1e
2855	movdqa	XMMWORD[(480+16)+rsp],xmm1
2856	movdqu	xmm0,XMMWORD[64+rsi]
2857	movdqu	xmm1,XMMWORD[80+rsi]
2858	movdqa	XMMWORD[512+rsp],xmm2
2859	movdqa	XMMWORD[(512+16)+rsp],xmm3
2860	por	xmm5,xmm4
2861	pxor	xmm4,xmm4
2862	por	xmm1,xmm0
2863DB	102,72,15,110,199
2864
2865	lea	rsi,[((64-0))+rsi]
2866	mov	QWORD[((544+0))+rsp],rax
2867	mov	QWORD[((544+8))+rsp],r14
2868	mov	QWORD[((544+16))+rsp],r15
2869	mov	QWORD[((544+24))+rsp],r8
2870	lea	rdi,[96+rsp]
2871	call	__ecp_nistz256_sqr_montq
2872
2873	pcmpeqd	xmm5,xmm4
2874	pshufd	xmm4,xmm1,0xb1
2875	por	xmm4,xmm1
2876	pshufd	xmm5,xmm5,0
2877	pshufd	xmm3,xmm4,0x1e
2878	por	xmm4,xmm3
2879	pxor	xmm3,xmm3
2880	pcmpeqd	xmm4,xmm3
2881	pshufd	xmm4,xmm4,0
2882	mov	rax,QWORD[((64+0))+rbx]
2883	mov	r14,QWORD[((64+8))+rbx]
2884	mov	r15,QWORD[((64+16))+rbx]
2885	mov	r8,QWORD[((64+24))+rbx]
2886DB	102,72,15,110,203
2887
2888	lea	rsi,[((64-0))+rbx]
2889	lea	rdi,[32+rsp]
2890	call	__ecp_nistz256_sqr_montq
2891
2892	mov	rax,QWORD[544+rsp]
2893	lea	rbx,[544+rsp]
2894	mov	r9,QWORD[((0+96))+rsp]
2895	mov	r10,QWORD[((8+96))+rsp]
2896	lea	rsi,[((0+96))+rsp]
2897	mov	r11,QWORD[((16+96))+rsp]
2898	mov	r12,QWORD[((24+96))+rsp]
2899	lea	rdi,[224+rsp]
2900	call	__ecp_nistz256_mul_montq
2901
2902	mov	rax,QWORD[448+rsp]
2903	lea	rbx,[448+rsp]
2904	mov	r9,QWORD[((0+32))+rsp]
2905	mov	r10,QWORD[((8+32))+rsp]
2906	lea	rsi,[((0+32))+rsp]
2907	mov	r11,QWORD[((16+32))+rsp]
2908	mov	r12,QWORD[((24+32))+rsp]
2909	lea	rdi,[256+rsp]
2910	call	__ecp_nistz256_mul_montq
2911
2912	mov	rax,QWORD[416+rsp]
2913	lea	rbx,[416+rsp]
2914	mov	r9,QWORD[((0+224))+rsp]
2915	mov	r10,QWORD[((8+224))+rsp]
2916	lea	rsi,[((0+224))+rsp]
2917	mov	r11,QWORD[((16+224))+rsp]
2918	mov	r12,QWORD[((24+224))+rsp]
2919	lea	rdi,[224+rsp]
2920	call	__ecp_nistz256_mul_montq
2921
2922	mov	rax,QWORD[512+rsp]
2923	lea	rbx,[512+rsp]
2924	mov	r9,QWORD[((0+256))+rsp]
2925	mov	r10,QWORD[((8+256))+rsp]
2926	lea	rsi,[((0+256))+rsp]
2927	mov	r11,QWORD[((16+256))+rsp]
2928	mov	r12,QWORD[((24+256))+rsp]
2929	lea	rdi,[256+rsp]
2930	call	__ecp_nistz256_mul_montq
2931
2932	lea	rbx,[224+rsp]
2933	lea	rdi,[64+rsp]
2934	call	__ecp_nistz256_sub_fromq
2935
2936	or	r12,r13
2937	movdqa	xmm2,xmm4
2938	or	r12,r8
2939	or	r12,r9
2940	por	xmm2,xmm5
2941DB	102,73,15,110,220
2942
2943	mov	rax,QWORD[384+rsp]
2944	lea	rbx,[384+rsp]
2945	mov	r9,QWORD[((0+96))+rsp]
2946	mov	r10,QWORD[((8+96))+rsp]
2947	lea	rsi,[((0+96))+rsp]
2948	mov	r11,QWORD[((16+96))+rsp]
2949	mov	r12,QWORD[((24+96))+rsp]
2950	lea	rdi,[160+rsp]
2951	call	__ecp_nistz256_mul_montq
2952
2953	mov	rax,QWORD[480+rsp]
2954	lea	rbx,[480+rsp]
2955	mov	r9,QWORD[((0+32))+rsp]
2956	mov	r10,QWORD[((8+32))+rsp]
2957	lea	rsi,[((0+32))+rsp]
2958	mov	r11,QWORD[((16+32))+rsp]
2959	mov	r12,QWORD[((24+32))+rsp]
2960	lea	rdi,[192+rsp]
2961	call	__ecp_nistz256_mul_montq
2962
2963	lea	rbx,[160+rsp]
2964	lea	rdi,[rsp]
2965	call	__ecp_nistz256_sub_fromq
2966
2967	or	r12,r13
2968	or	r12,r8
2969	or	r12,r9
2970
2971DB	102,73,15,126,208
2972DB	102,73,15,126,217
2973	or	r12,r8
2974	DB	0x3e
2975	jnz	NEAR $L$add_proceedq
2976
2977
2978
2979	test	r9,r9
2980	jz	NEAR $L$add_doubleq
2981
2982
2983
2984
2985
2986
2987DB	102,72,15,126,199
2988	pxor	xmm0,xmm0
2989	movdqu	XMMWORD[rdi],xmm0
2990	movdqu	XMMWORD[16+rdi],xmm0
2991	movdqu	XMMWORD[32+rdi],xmm0
2992	movdqu	XMMWORD[48+rdi],xmm0
2993	movdqu	XMMWORD[64+rdi],xmm0
2994	movdqu	XMMWORD[80+rdi],xmm0
2995	jmp	NEAR $L$add_doneq
2996
2997ALIGN	32
2998$L$add_doubleq:
2999DB	102,72,15,126,206
3000DB	102,72,15,126,199
3001	add	rsp,416
3002
3003	jmp	NEAR $L$point_double_shortcutq
3004
3005
3006ALIGN	32
3007$L$add_proceedq:
3008	mov	rax,QWORD[((0+64))+rsp]
3009	mov	r14,QWORD[((8+64))+rsp]
3010	lea	rsi,[((0+64))+rsp]
3011	mov	r15,QWORD[((16+64))+rsp]
3012	mov	r8,QWORD[((24+64))+rsp]
3013	lea	rdi,[96+rsp]
3014	call	__ecp_nistz256_sqr_montq
3015
3016	mov	rax,QWORD[448+rsp]
3017	lea	rbx,[448+rsp]
3018	mov	r9,QWORD[((0+0))+rsp]
3019	mov	r10,QWORD[((8+0))+rsp]
3020	lea	rsi,[((0+0))+rsp]
3021	mov	r11,QWORD[((16+0))+rsp]
3022	mov	r12,QWORD[((24+0))+rsp]
3023	lea	rdi,[352+rsp]
3024	call	__ecp_nistz256_mul_montq
3025
3026	mov	rax,QWORD[((0+0))+rsp]
3027	mov	r14,QWORD[((8+0))+rsp]
3028	lea	rsi,[((0+0))+rsp]
3029	mov	r15,QWORD[((16+0))+rsp]
3030	mov	r8,QWORD[((24+0))+rsp]
3031	lea	rdi,[32+rsp]
3032	call	__ecp_nistz256_sqr_montq
3033
3034	mov	rax,QWORD[544+rsp]
3035	lea	rbx,[544+rsp]
3036	mov	r9,QWORD[((0+352))+rsp]
3037	mov	r10,QWORD[((8+352))+rsp]
3038	lea	rsi,[((0+352))+rsp]
3039	mov	r11,QWORD[((16+352))+rsp]
3040	mov	r12,QWORD[((24+352))+rsp]
3041	lea	rdi,[352+rsp]
3042	call	__ecp_nistz256_mul_montq
3043
3044	mov	rax,QWORD[rsp]
3045	lea	rbx,[rsp]
3046	mov	r9,QWORD[((0+32))+rsp]
3047	mov	r10,QWORD[((8+32))+rsp]
3048	lea	rsi,[((0+32))+rsp]
3049	mov	r11,QWORD[((16+32))+rsp]
3050	mov	r12,QWORD[((24+32))+rsp]
3051	lea	rdi,[128+rsp]
3052	call	__ecp_nistz256_mul_montq
3053
3054	mov	rax,QWORD[160+rsp]
3055	lea	rbx,[160+rsp]
3056	mov	r9,QWORD[((0+32))+rsp]
3057	mov	r10,QWORD[((8+32))+rsp]
3058	lea	rsi,[((0+32))+rsp]
3059	mov	r11,QWORD[((16+32))+rsp]
3060	mov	r12,QWORD[((24+32))+rsp]
3061	lea	rdi,[192+rsp]
3062	call	__ecp_nistz256_mul_montq
3063
3064
3065
3066
3067	xor	r11,r11
3068	add	r12,r12
3069	lea	rsi,[96+rsp]
3070	adc	r13,r13
3071	mov	rax,r12
3072	adc	r8,r8
3073	adc	r9,r9
3074	mov	rbp,r13
3075	adc	r11,0
3076
3077	sub	r12,-1
3078	mov	rcx,r8
3079	sbb	r13,r14
3080	sbb	r8,0
3081	mov	r10,r9
3082	sbb	r9,r15
3083	sbb	r11,0
3084
3085	cmovc	r12,rax
3086	mov	rax,QWORD[rsi]
3087	cmovc	r13,rbp
3088	mov	rbp,QWORD[8+rsi]
3089	cmovc	r8,rcx
3090	mov	rcx,QWORD[16+rsi]
3091	cmovc	r9,r10
3092	mov	r10,QWORD[24+rsi]
3093
3094	call	__ecp_nistz256_subq
3095
3096	lea	rbx,[128+rsp]
3097	lea	rdi,[288+rsp]
3098	call	__ecp_nistz256_sub_fromq
3099
3100	mov	rax,QWORD[((192+0))+rsp]
3101	mov	rbp,QWORD[((192+8))+rsp]
3102	mov	rcx,QWORD[((192+16))+rsp]
3103	mov	r10,QWORD[((192+24))+rsp]
3104	lea	rdi,[320+rsp]
3105
3106	call	__ecp_nistz256_subq
3107
3108	mov	QWORD[rdi],r12
3109	mov	QWORD[8+rdi],r13
3110	mov	QWORD[16+rdi],r8
3111	mov	QWORD[24+rdi],r9
3112	mov	rax,QWORD[128+rsp]
3113	lea	rbx,[128+rsp]
3114	mov	r9,QWORD[((0+224))+rsp]
3115	mov	r10,QWORD[((8+224))+rsp]
3116	lea	rsi,[((0+224))+rsp]
3117	mov	r11,QWORD[((16+224))+rsp]
3118	mov	r12,QWORD[((24+224))+rsp]
3119	lea	rdi,[256+rsp]
3120	call	__ecp_nistz256_mul_montq
3121
3122	mov	rax,QWORD[320+rsp]
3123	lea	rbx,[320+rsp]
3124	mov	r9,QWORD[((0+64))+rsp]
3125	mov	r10,QWORD[((8+64))+rsp]
3126	lea	rsi,[((0+64))+rsp]
3127	mov	r11,QWORD[((16+64))+rsp]
3128	mov	r12,QWORD[((24+64))+rsp]
3129	lea	rdi,[320+rsp]
3130	call	__ecp_nistz256_mul_montq
3131
3132	lea	rbx,[256+rsp]
3133	lea	rdi,[320+rsp]
3134	call	__ecp_nistz256_sub_fromq
3135
3136DB	102,72,15,126,199
3137
3138	movdqa	xmm0,xmm5
3139	movdqa	xmm1,xmm5
3140	pandn	xmm0,XMMWORD[352+rsp]
3141	movdqa	xmm2,xmm5
3142	pandn	xmm1,XMMWORD[((352+16))+rsp]
3143	movdqa	xmm3,xmm5
3144	pand	xmm2,XMMWORD[544+rsp]
3145	pand	xmm3,XMMWORD[((544+16))+rsp]
3146	por	xmm2,xmm0
3147	por	xmm3,xmm1
3148
3149	movdqa	xmm0,xmm4
3150	movdqa	xmm1,xmm4
3151	pandn	xmm0,xmm2
3152	movdqa	xmm2,xmm4
3153	pandn	xmm1,xmm3
3154	movdqa	xmm3,xmm4
3155	pand	xmm2,XMMWORD[448+rsp]
3156	pand	xmm3,XMMWORD[((448+16))+rsp]
3157	por	xmm2,xmm0
3158	por	xmm3,xmm1
3159	movdqu	XMMWORD[64+rdi],xmm2
3160	movdqu	XMMWORD[80+rdi],xmm3
3161
3162	movdqa	xmm0,xmm5
3163	movdqa	xmm1,xmm5
3164	pandn	xmm0,XMMWORD[288+rsp]
3165	movdqa	xmm2,xmm5
3166	pandn	xmm1,XMMWORD[((288+16))+rsp]
3167	movdqa	xmm3,xmm5
3168	pand	xmm2,XMMWORD[480+rsp]
3169	pand	xmm3,XMMWORD[((480+16))+rsp]
3170	por	xmm2,xmm0
3171	por	xmm3,xmm1
3172
3173	movdqa	xmm0,xmm4
3174	movdqa	xmm1,xmm4
3175	pandn	xmm0,xmm2
3176	movdqa	xmm2,xmm4
3177	pandn	xmm1,xmm3
3178	movdqa	xmm3,xmm4
3179	pand	xmm2,XMMWORD[384+rsp]
3180	pand	xmm3,XMMWORD[((384+16))+rsp]
3181	por	xmm2,xmm0
3182	por	xmm3,xmm1
3183	movdqu	XMMWORD[rdi],xmm2
3184	movdqu	XMMWORD[16+rdi],xmm3
3185
3186	movdqa	xmm0,xmm5
3187	movdqa	xmm1,xmm5
3188	pandn	xmm0,XMMWORD[320+rsp]
3189	movdqa	xmm2,xmm5
3190	pandn	xmm1,XMMWORD[((320+16))+rsp]
3191	movdqa	xmm3,xmm5
3192	pand	xmm2,XMMWORD[512+rsp]
3193	pand	xmm3,XMMWORD[((512+16))+rsp]
3194	por	xmm2,xmm0
3195	por	xmm3,xmm1
3196
3197	movdqa	xmm0,xmm4
3198	movdqa	xmm1,xmm4
3199	pandn	xmm0,xmm2
3200	movdqa	xmm2,xmm4
3201	pandn	xmm1,xmm3
3202	movdqa	xmm3,xmm4
3203	pand	xmm2,XMMWORD[416+rsp]
3204	pand	xmm3,XMMWORD[((416+16))+rsp]
3205	por	xmm2,xmm0
3206	por	xmm3,xmm1
3207	movdqu	XMMWORD[32+rdi],xmm2
3208	movdqu	XMMWORD[48+rdi],xmm3
3209
3210$L$add_doneq:
3211	lea	rsi,[((576+56))+rsp]
3212
3213	mov	r15,QWORD[((-48))+rsi]
3214
3215	mov	r14,QWORD[((-40))+rsi]
3216
3217	mov	r13,QWORD[((-32))+rsi]
3218
3219	mov	r12,QWORD[((-24))+rsi]
3220
3221	mov	rbx,QWORD[((-16))+rsi]
3222
3223	mov	rbp,QWORD[((-8))+rsi]
3224
3225	lea	rsp,[rsi]
3226
3227$L$point_addq_epilogue:
3228	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
3229	mov	rsi,QWORD[16+rsp]
3230	ret
3231
3232$L$SEH_end_ecp_nistz256_point_add:
3233global	ecp_nistz256_point_add_affine
3234
3235ALIGN	32
3236ecp_nistz256_point_add_affine:
3237	mov	QWORD[8+rsp],rdi	;WIN64 prologue
3238	mov	QWORD[16+rsp],rsi
3239	mov	rax,rsp
3240$L$SEH_begin_ecp_nistz256_point_add_affine:
3241	mov	rdi,rcx
3242	mov	rsi,rdx
3243	mov	rdx,r8
3244
3245
3246
3247_CET_ENDBR
3248	lea	rcx,[OPENSSL_ia32cap_P]
3249	mov	rcx,QWORD[8+rcx]
3250	and	ecx,0x80100
3251	cmp	ecx,0x80100
3252	je	NEAR $L$point_add_affinex
3253	push	rbp
3254
3255	push	rbx
3256
3257	push	r12
3258
3259	push	r13
3260
3261	push	r14
3262
3263	push	r15
3264
3265	sub	rsp,32*15+8
3266
3267$L$add_affineq_body:
3268
3269	movdqu	xmm0,XMMWORD[rsi]
3270	mov	rbx,rdx
3271	movdqu	xmm1,XMMWORD[16+rsi]
3272	movdqu	xmm2,XMMWORD[32+rsi]
3273	movdqu	xmm3,XMMWORD[48+rsi]
3274	movdqu	xmm4,XMMWORD[64+rsi]
3275	movdqu	xmm5,XMMWORD[80+rsi]
3276	mov	rax,QWORD[((64+0))+rsi]
3277	mov	r14,QWORD[((64+8))+rsi]
3278	mov	r15,QWORD[((64+16))+rsi]
3279	mov	r8,QWORD[((64+24))+rsi]
3280	movdqa	XMMWORD[320+rsp],xmm0
3281	movdqa	XMMWORD[(320+16)+rsp],xmm1
3282	movdqa	XMMWORD[352+rsp],xmm2
3283	movdqa	XMMWORD[(352+16)+rsp],xmm3
3284	movdqa	XMMWORD[384+rsp],xmm4
3285	movdqa	XMMWORD[(384+16)+rsp],xmm5
3286	por	xmm5,xmm4
3287
3288	movdqu	xmm0,XMMWORD[rbx]
3289	pshufd	xmm3,xmm5,0xb1
3290	movdqu	xmm1,XMMWORD[16+rbx]
3291	movdqu	xmm2,XMMWORD[32+rbx]
3292	por	xmm5,xmm3
3293	movdqu	xmm3,XMMWORD[48+rbx]
3294	movdqa	XMMWORD[416+rsp],xmm0
3295	pshufd	xmm4,xmm5,0x1e
3296	movdqa	XMMWORD[(416+16)+rsp],xmm1
3297	por	xmm1,xmm0
3298DB	102,72,15,110,199
3299	movdqa	XMMWORD[448+rsp],xmm2
3300	movdqa	XMMWORD[(448+16)+rsp],xmm3
3301	por	xmm3,xmm2
3302	por	xmm5,xmm4
3303	pxor	xmm4,xmm4
3304	por	xmm3,xmm1
3305
3306	lea	rsi,[((64-0))+rsi]
3307	lea	rdi,[32+rsp]
3308	call	__ecp_nistz256_sqr_montq
3309
3310	pcmpeqd	xmm5,xmm4
3311	pshufd	xmm4,xmm3,0xb1
3312	mov	rax,QWORD[rbx]
3313
3314	mov	r9,r12
3315	por	xmm4,xmm3
3316	pshufd	xmm5,xmm5,0
3317	pshufd	xmm3,xmm4,0x1e
3318	mov	r10,r13
3319	por	xmm4,xmm3
3320	pxor	xmm3,xmm3
3321	mov	r11,r14
3322	pcmpeqd	xmm4,xmm3
3323	pshufd	xmm4,xmm4,0
3324
3325	lea	rsi,[((32-0))+rsp]
3326	mov	r12,r15
3327	lea	rdi,[rsp]
3328	call	__ecp_nistz256_mul_montq
3329
3330	lea	rbx,[320+rsp]
3331	lea	rdi,[64+rsp]
3332	call	__ecp_nistz256_sub_fromq
3333
3334	mov	rax,QWORD[384+rsp]
3335	lea	rbx,[384+rsp]
3336	mov	r9,QWORD[((0+32))+rsp]
3337	mov	r10,QWORD[((8+32))+rsp]
3338	lea	rsi,[((0+32))+rsp]
3339	mov	r11,QWORD[((16+32))+rsp]
3340	mov	r12,QWORD[((24+32))+rsp]
3341	lea	rdi,[32+rsp]
3342	call	__ecp_nistz256_mul_montq
3343
3344	mov	rax,QWORD[384+rsp]
3345	lea	rbx,[384+rsp]
3346	mov	r9,QWORD[((0+64))+rsp]
3347	mov	r10,QWORD[((8+64))+rsp]
3348	lea	rsi,[((0+64))+rsp]
3349	mov	r11,QWORD[((16+64))+rsp]
3350	mov	r12,QWORD[((24+64))+rsp]
3351	lea	rdi,[288+rsp]
3352	call	__ecp_nistz256_mul_montq
3353
3354	mov	rax,QWORD[448+rsp]
3355	lea	rbx,[448+rsp]
3356	mov	r9,QWORD[((0+32))+rsp]
3357	mov	r10,QWORD[((8+32))+rsp]
3358	lea	rsi,[((0+32))+rsp]
3359	mov	r11,QWORD[((16+32))+rsp]
3360	mov	r12,QWORD[((24+32))+rsp]
3361	lea	rdi,[32+rsp]
3362	call	__ecp_nistz256_mul_montq
3363
3364	lea	rbx,[352+rsp]
3365	lea	rdi,[96+rsp]
3366	call	__ecp_nistz256_sub_fromq
3367
3368	mov	rax,QWORD[((0+64))+rsp]
3369	mov	r14,QWORD[((8+64))+rsp]
3370	lea	rsi,[((0+64))+rsp]
3371	mov	r15,QWORD[((16+64))+rsp]
3372	mov	r8,QWORD[((24+64))+rsp]
3373	lea	rdi,[128+rsp]
3374	call	__ecp_nistz256_sqr_montq
3375
3376	mov	rax,QWORD[((0+96))+rsp]
3377	mov	r14,QWORD[((8+96))+rsp]
3378	lea	rsi,[((0+96))+rsp]
3379	mov	r15,QWORD[((16+96))+rsp]
3380	mov	r8,QWORD[((24+96))+rsp]
3381	lea	rdi,[192+rsp]
3382	call	__ecp_nistz256_sqr_montq
3383
3384	mov	rax,QWORD[128+rsp]
3385	lea	rbx,[128+rsp]
3386	mov	r9,QWORD[((0+64))+rsp]
3387	mov	r10,QWORD[((8+64))+rsp]
3388	lea	rsi,[((0+64))+rsp]
3389	mov	r11,QWORD[((16+64))+rsp]
3390	mov	r12,QWORD[((24+64))+rsp]
3391	lea	rdi,[160+rsp]
3392	call	__ecp_nistz256_mul_montq
3393
3394	mov	rax,QWORD[320+rsp]
3395	lea	rbx,[320+rsp]
3396	mov	r9,QWORD[((0+128))+rsp]
3397	mov	r10,QWORD[((8+128))+rsp]
3398	lea	rsi,[((0+128))+rsp]
3399	mov	r11,QWORD[((16+128))+rsp]
3400	mov	r12,QWORD[((24+128))+rsp]
3401	lea	rdi,[rsp]
3402	call	__ecp_nistz256_mul_montq
3403
3404
3405
3406
3407	xor	r11,r11
3408	add	r12,r12
3409	lea	rsi,[192+rsp]
3410	adc	r13,r13
3411	mov	rax,r12
3412	adc	r8,r8
3413	adc	r9,r9
3414	mov	rbp,r13
3415	adc	r11,0
3416
3417	sub	r12,-1
3418	mov	rcx,r8
3419	sbb	r13,r14
3420	sbb	r8,0
3421	mov	r10,r9
3422	sbb	r9,r15
3423	sbb	r11,0
3424
3425	cmovc	r12,rax
3426	mov	rax,QWORD[rsi]
3427	cmovc	r13,rbp
3428	mov	rbp,QWORD[8+rsi]
3429	cmovc	r8,rcx
3430	mov	rcx,QWORD[16+rsi]
3431	cmovc	r9,r10
3432	mov	r10,QWORD[24+rsi]
3433
3434	call	__ecp_nistz256_subq
3435
3436	lea	rbx,[160+rsp]
3437	lea	rdi,[224+rsp]
3438	call	__ecp_nistz256_sub_fromq
3439
3440	mov	rax,QWORD[((0+0))+rsp]
3441	mov	rbp,QWORD[((0+8))+rsp]
3442	mov	rcx,QWORD[((0+16))+rsp]
3443	mov	r10,QWORD[((0+24))+rsp]
3444	lea	rdi,[64+rsp]
3445
3446	call	__ecp_nistz256_subq
3447
3448	mov	QWORD[rdi],r12
3449	mov	QWORD[8+rdi],r13
3450	mov	QWORD[16+rdi],r8
3451	mov	QWORD[24+rdi],r9
3452	mov	rax,QWORD[352+rsp]
3453	lea	rbx,[352+rsp]
3454	mov	r9,QWORD[((0+160))+rsp]
3455	mov	r10,QWORD[((8+160))+rsp]
3456	lea	rsi,[((0+160))+rsp]
3457	mov	r11,QWORD[((16+160))+rsp]
3458	mov	r12,QWORD[((24+160))+rsp]
3459	lea	rdi,[32+rsp]
3460	call	__ecp_nistz256_mul_montq
3461
3462	mov	rax,QWORD[96+rsp]
3463	lea	rbx,[96+rsp]
3464	mov	r9,QWORD[((0+64))+rsp]
3465	mov	r10,QWORD[((8+64))+rsp]
3466	lea	rsi,[((0+64))+rsp]
3467	mov	r11,QWORD[((16+64))+rsp]
3468	mov	r12,QWORD[((24+64))+rsp]
3469	lea	rdi,[64+rsp]
3470	call	__ecp_nistz256_mul_montq
3471
3472	lea	rbx,[32+rsp]
3473	lea	rdi,[256+rsp]
3474	call	__ecp_nistz256_sub_fromq
3475
3476DB	102,72,15,126,199
3477
3478	movdqa	xmm0,xmm5
3479	movdqa	xmm1,xmm5
3480	pandn	xmm0,XMMWORD[288+rsp]
3481	movdqa	xmm2,xmm5
3482	pandn	xmm1,XMMWORD[((288+16))+rsp]
3483	movdqa	xmm3,xmm5
3484	pand	xmm2,XMMWORD[$L$ONE_mont]
3485	pand	xmm3,XMMWORD[(($L$ONE_mont+16))]
3486	por	xmm2,xmm0
3487	por	xmm3,xmm1
3488
3489	movdqa	xmm0,xmm4
3490	movdqa	xmm1,xmm4
3491	pandn	xmm0,xmm2
3492	movdqa	xmm2,xmm4
3493	pandn	xmm1,xmm3
3494	movdqa	xmm3,xmm4
3495	pand	xmm2,XMMWORD[384+rsp]
3496	pand	xmm3,XMMWORD[((384+16))+rsp]
3497	por	xmm2,xmm0
3498	por	xmm3,xmm1
3499	movdqu	XMMWORD[64+rdi],xmm2
3500	movdqu	XMMWORD[80+rdi],xmm3
3501
3502	movdqa	xmm0,xmm5
3503	movdqa	xmm1,xmm5
3504	pandn	xmm0,XMMWORD[224+rsp]
3505	movdqa	xmm2,xmm5
3506	pandn	xmm1,XMMWORD[((224+16))+rsp]
3507	movdqa	xmm3,xmm5
3508	pand	xmm2,XMMWORD[416+rsp]
3509	pand	xmm3,XMMWORD[((416+16))+rsp]
3510	por	xmm2,xmm0
3511	por	xmm3,xmm1
3512
3513	movdqa	xmm0,xmm4
3514	movdqa	xmm1,xmm4
3515	pandn	xmm0,xmm2
3516	movdqa	xmm2,xmm4
3517	pandn	xmm1,xmm3
3518	movdqa	xmm3,xmm4
3519	pand	xmm2,XMMWORD[320+rsp]
3520	pand	xmm3,XMMWORD[((320+16))+rsp]
3521	por	xmm2,xmm0
3522	por	xmm3,xmm1
3523	movdqu	XMMWORD[rdi],xmm2
3524	movdqu	XMMWORD[16+rdi],xmm3
3525
3526	movdqa	xmm0,xmm5
3527	movdqa	xmm1,xmm5
3528	pandn	xmm0,XMMWORD[256+rsp]
3529	movdqa	xmm2,xmm5
3530	pandn	xmm1,XMMWORD[((256+16))+rsp]
3531	movdqa	xmm3,xmm5
3532	pand	xmm2,XMMWORD[448+rsp]
3533	pand	xmm3,XMMWORD[((448+16))+rsp]
3534	por	xmm2,xmm0
3535	por	xmm3,xmm1
3536
3537	movdqa	xmm0,xmm4
3538	movdqa	xmm1,xmm4
3539	pandn	xmm0,xmm2
3540	movdqa	xmm2,xmm4
3541	pandn	xmm1,xmm3
3542	movdqa	xmm3,xmm4
3543	pand	xmm2,XMMWORD[352+rsp]
3544	pand	xmm3,XMMWORD[((352+16))+rsp]
3545	por	xmm2,xmm0
3546	por	xmm3,xmm1
3547	movdqu	XMMWORD[32+rdi],xmm2
3548	movdqu	XMMWORD[48+rdi],xmm3
3549
3550	lea	rsi,[((480+56))+rsp]
3551
3552	mov	r15,QWORD[((-48))+rsi]
3553
3554	mov	r14,QWORD[((-40))+rsi]
3555
3556	mov	r13,QWORD[((-32))+rsi]
3557
3558	mov	r12,QWORD[((-24))+rsi]
3559
3560	mov	rbx,QWORD[((-16))+rsi]
3561
3562	mov	rbp,QWORD[((-8))+rsi]
3563
3564	lea	rsp,[rsi]
3565
3566$L$add_affineq_epilogue:
3567	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
3568	mov	rsi,QWORD[16+rsp]
3569	ret
3570
3571$L$SEH_end_ecp_nistz256_point_add_affine:
3572
3573ALIGN	32
3574__ecp_nistz256_add_tox:
3575
3576	xor	r11,r11
3577	adc	r12,QWORD[rbx]
3578	adc	r13,QWORD[8+rbx]
3579	mov	rax,r12
3580	adc	r8,QWORD[16+rbx]
3581	adc	r9,QWORD[24+rbx]
3582	mov	rbp,r13
3583	adc	r11,0
3584
3585	xor	r10,r10
3586	sbb	r12,-1
3587	mov	rcx,r8
3588	sbb	r13,r14
3589	sbb	r8,0
3590	mov	r10,r9
3591	sbb	r9,r15
3592	sbb	r11,0
3593
3594	cmovc	r12,rax
3595	cmovc	r13,rbp
3596	mov	QWORD[rdi],r12
3597	cmovc	r8,rcx
3598	mov	QWORD[8+rdi],r13
3599	cmovc	r9,r10
3600	mov	QWORD[16+rdi],r8
3601	mov	QWORD[24+rdi],r9
3602
3603	ret
3604
3605
3606
3607
3608ALIGN	32
3609__ecp_nistz256_sub_fromx:
3610
3611	xor	r11,r11
3612	sbb	r12,QWORD[rbx]
3613	sbb	r13,QWORD[8+rbx]
3614	mov	rax,r12
3615	sbb	r8,QWORD[16+rbx]
3616	sbb	r9,QWORD[24+rbx]
3617	mov	rbp,r13
3618	sbb	r11,0
3619
3620	xor	r10,r10
3621	adc	r12,-1
3622	mov	rcx,r8
3623	adc	r13,r14
3624	adc	r8,0
3625	mov	r10,r9
3626	adc	r9,r15
3627
3628	bt	r11,0
3629	cmovnc	r12,rax
3630	cmovnc	r13,rbp
3631	mov	QWORD[rdi],r12
3632	cmovnc	r8,rcx
3633	mov	QWORD[8+rdi],r13
3634	cmovnc	r9,r10
3635	mov	QWORD[16+rdi],r8
3636	mov	QWORD[24+rdi],r9
3637
3638	ret
3639
3640
3641
3642
3643ALIGN	32
3644__ecp_nistz256_subx:
3645
3646	xor	r11,r11
3647	sbb	rax,r12
3648	sbb	rbp,r13
3649	mov	r12,rax
3650	sbb	rcx,r8
3651	sbb	r10,r9
3652	mov	r13,rbp
3653	sbb	r11,0
3654
3655	xor	r9,r9
3656	adc	rax,-1
3657	mov	r8,rcx
3658	adc	rbp,r14
3659	adc	rcx,0
3660	mov	r9,r10
3661	adc	r10,r15
3662
3663	bt	r11,0
3664	cmovc	r12,rax
3665	cmovc	r13,rbp
3666	cmovc	r8,rcx
3667	cmovc	r9,r10
3668
3669	ret
3670
3671
3672
3673
3674ALIGN	32
3675__ecp_nistz256_mul_by_2x:
3676
3677	xor	r11,r11
3678	adc	r12,r12
3679	adc	r13,r13
3680	mov	rax,r12
3681	adc	r8,r8
3682	adc	r9,r9
3683	mov	rbp,r13
3684	adc	r11,0
3685
3686	xor	r10,r10
3687	sbb	r12,-1
3688	mov	rcx,r8
3689	sbb	r13,r14
3690	sbb	r8,0
3691	mov	r10,r9
3692	sbb	r9,r15
3693	sbb	r11,0
3694
3695	cmovc	r12,rax
3696	cmovc	r13,rbp
3697	mov	QWORD[rdi],r12
3698	cmovc	r8,rcx
3699	mov	QWORD[8+rdi],r13
3700	cmovc	r9,r10
3701	mov	QWORD[16+rdi],r8
3702	mov	QWORD[24+rdi],r9
3703
3704	ret
3705
3706
3707
3708ALIGN	32
3709ecp_nistz256_point_doublex:
3710	mov	QWORD[8+rsp],rdi	;WIN64 prologue
3711	mov	QWORD[16+rsp],rsi
3712	mov	rax,rsp
3713$L$SEH_begin_ecp_nistz256_point_doublex:
3714	mov	rdi,rcx
3715	mov	rsi,rdx
3716
3717
3718
3719$L$point_doublex:
3720	push	rbp
3721
3722	push	rbx
3723
3724	push	r12
3725
3726	push	r13
3727
3728	push	r14
3729
3730	push	r15
3731
3732	sub	rsp,32*5+8
3733
3734$L$point_doublex_body:
3735
3736$L$point_double_shortcutx:
3737	movdqu	xmm0,XMMWORD[rsi]
3738	mov	rbx,rsi
3739	movdqu	xmm1,XMMWORD[16+rsi]
3740	mov	r12,QWORD[((32+0))+rsi]
3741	mov	r13,QWORD[((32+8))+rsi]
3742	mov	r8,QWORD[((32+16))+rsi]
3743	mov	r9,QWORD[((32+24))+rsi]
3744	mov	r14,QWORD[(($L$poly+8))]
3745	mov	r15,QWORD[(($L$poly+24))]
3746	movdqa	XMMWORD[96+rsp],xmm0
3747	movdqa	XMMWORD[(96+16)+rsp],xmm1
3748	lea	r10,[32+rdi]
3749	lea	r11,[64+rdi]
3750DB	102,72,15,110,199
3751DB	102,73,15,110,202
3752DB	102,73,15,110,211
3753
3754	lea	rdi,[rsp]
3755	call	__ecp_nistz256_mul_by_2x
3756
3757	mov	rdx,QWORD[((64+0))+rsi]
3758	mov	r14,QWORD[((64+8))+rsi]
3759	mov	r15,QWORD[((64+16))+rsi]
3760	mov	r8,QWORD[((64+24))+rsi]
3761	lea	rsi,[((64-128))+rsi]
3762	lea	rdi,[64+rsp]
3763	call	__ecp_nistz256_sqr_montx
3764
3765	mov	rdx,QWORD[((0+0))+rsp]
3766	mov	r14,QWORD[((8+0))+rsp]
3767	lea	rsi,[((-128+0))+rsp]
3768	mov	r15,QWORD[((16+0))+rsp]
3769	mov	r8,QWORD[((24+0))+rsp]
3770	lea	rdi,[rsp]
3771	call	__ecp_nistz256_sqr_montx
3772
3773	mov	rdx,QWORD[32+rbx]
3774	mov	r9,QWORD[((64+0))+rbx]
3775	mov	r10,QWORD[((64+8))+rbx]
3776	mov	r11,QWORD[((64+16))+rbx]
3777	mov	r12,QWORD[((64+24))+rbx]
3778	lea	rsi,[((64-128))+rbx]
3779	lea	rbx,[32+rbx]
3780DB	102,72,15,126,215
3781	call	__ecp_nistz256_mul_montx
3782	call	__ecp_nistz256_mul_by_2x
3783
3784	mov	r12,QWORD[((96+0))+rsp]
3785	mov	r13,QWORD[((96+8))+rsp]
3786	lea	rbx,[64+rsp]
3787	mov	r8,QWORD[((96+16))+rsp]
3788	mov	r9,QWORD[((96+24))+rsp]
3789	lea	rdi,[32+rsp]
3790	call	__ecp_nistz256_add_tox
3791
3792	mov	r12,QWORD[((96+0))+rsp]
3793	mov	r13,QWORD[((96+8))+rsp]
3794	lea	rbx,[64+rsp]
3795	mov	r8,QWORD[((96+16))+rsp]
3796	mov	r9,QWORD[((96+24))+rsp]
3797	lea	rdi,[64+rsp]
3798	call	__ecp_nistz256_sub_fromx
3799
3800	mov	rdx,QWORD[((0+0))+rsp]
3801	mov	r14,QWORD[((8+0))+rsp]
3802	lea	rsi,[((-128+0))+rsp]
3803	mov	r15,QWORD[((16+0))+rsp]
3804	mov	r8,QWORD[((24+0))+rsp]
3805DB	102,72,15,126,207
3806	call	__ecp_nistz256_sqr_montx
3807	xor	r9,r9
3808	mov	rax,r12
3809	add	r12,-1
3810	mov	r10,r13
3811	adc	r13,rsi
3812	mov	rcx,r14
3813	adc	r14,0
3814	mov	r8,r15
3815	adc	r15,rbp
3816	adc	r9,0
3817	xor	rsi,rsi
3818	test	rax,1
3819
3820	cmovz	r12,rax
3821	cmovz	r13,r10
3822	cmovz	r14,rcx
3823	cmovz	r15,r8
3824	cmovz	r9,rsi
3825
3826	mov	rax,r13
3827	shr	r12,1
3828	shl	rax,63
3829	mov	r10,r14
3830	shr	r13,1
3831	or	r12,rax
3832	shl	r10,63
3833	mov	rcx,r15
3834	shr	r14,1
3835	or	r13,r10
3836	shl	rcx,63
3837	mov	QWORD[rdi],r12
3838	shr	r15,1
3839	mov	QWORD[8+rdi],r13
3840	shl	r9,63
3841	or	r14,rcx
3842	or	r15,r9
3843	mov	QWORD[16+rdi],r14
3844	mov	QWORD[24+rdi],r15
3845	mov	rdx,QWORD[64+rsp]
3846	lea	rbx,[64+rsp]
3847	mov	r9,QWORD[((0+32))+rsp]
3848	mov	r10,QWORD[((8+32))+rsp]
3849	lea	rsi,[((-128+32))+rsp]
3850	mov	r11,QWORD[((16+32))+rsp]
3851	mov	r12,QWORD[((24+32))+rsp]
3852	lea	rdi,[32+rsp]
3853	call	__ecp_nistz256_mul_montx
3854
3855	lea	rdi,[128+rsp]
3856	call	__ecp_nistz256_mul_by_2x
3857
3858	lea	rbx,[32+rsp]
3859	lea	rdi,[32+rsp]
3860	call	__ecp_nistz256_add_tox
3861
3862	mov	rdx,QWORD[96+rsp]
3863	lea	rbx,[96+rsp]
3864	mov	r9,QWORD[((0+0))+rsp]
3865	mov	r10,QWORD[((8+0))+rsp]
3866	lea	rsi,[((-128+0))+rsp]
3867	mov	r11,QWORD[((16+0))+rsp]
3868	mov	r12,QWORD[((24+0))+rsp]
3869	lea	rdi,[rsp]
3870	call	__ecp_nistz256_mul_montx
3871
3872	lea	rdi,[128+rsp]
3873	call	__ecp_nistz256_mul_by_2x
3874
3875	mov	rdx,QWORD[((0+32))+rsp]
3876	mov	r14,QWORD[((8+32))+rsp]
3877	lea	rsi,[((-128+32))+rsp]
3878	mov	r15,QWORD[((16+32))+rsp]
3879	mov	r8,QWORD[((24+32))+rsp]
3880DB	102,72,15,126,199
3881	call	__ecp_nistz256_sqr_montx
3882
3883	lea	rbx,[128+rsp]
3884	mov	r8,r14
3885	mov	r9,r15
3886	mov	r14,rsi
3887	mov	r15,rbp
3888	call	__ecp_nistz256_sub_fromx
3889
3890	mov	rax,QWORD[((0+0))+rsp]
3891	mov	rbp,QWORD[((0+8))+rsp]
3892	mov	rcx,QWORD[((0+16))+rsp]
3893	mov	r10,QWORD[((0+24))+rsp]
3894	lea	rdi,[rsp]
3895	call	__ecp_nistz256_subx
3896
3897	mov	rdx,QWORD[32+rsp]
3898	lea	rbx,[32+rsp]
3899	mov	r14,r12
3900	xor	ecx,ecx
3901	mov	QWORD[((0+0))+rsp],r12
3902	mov	r10,r13
3903	mov	QWORD[((0+8))+rsp],r13
3904	cmovz	r11,r8
3905	mov	QWORD[((0+16))+rsp],r8
3906	lea	rsi,[((0-128))+rsp]
3907	cmovz	r12,r9
3908	mov	QWORD[((0+24))+rsp],r9
3909	mov	r9,r14
3910	lea	rdi,[rsp]
3911	call	__ecp_nistz256_mul_montx
3912
3913DB	102,72,15,126,203
3914DB	102,72,15,126,207
3915	call	__ecp_nistz256_sub_fromx
3916
3917	lea	rsi,[((160+56))+rsp]
3918
3919	mov	r15,QWORD[((-48))+rsi]
3920
3921	mov	r14,QWORD[((-40))+rsi]
3922
3923	mov	r13,QWORD[((-32))+rsi]
3924
3925	mov	r12,QWORD[((-24))+rsi]
3926
3927	mov	rbx,QWORD[((-16))+rsi]
3928
3929	mov	rbp,QWORD[((-8))+rsi]
3930
3931	lea	rsp,[rsi]
3932
3933$L$point_doublex_epilogue:
3934	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
3935	mov	rsi,QWORD[16+rsp]
3936	ret
3937
3938$L$SEH_end_ecp_nistz256_point_doublex:
3939
3940ALIGN	32
3941ecp_nistz256_point_addx:
3942	mov	QWORD[8+rsp],rdi	;WIN64 prologue
3943	mov	QWORD[16+rsp],rsi
3944	mov	rax,rsp
3945$L$SEH_begin_ecp_nistz256_point_addx:
3946	mov	rdi,rcx
3947	mov	rsi,rdx
3948	mov	rdx,r8
3949
3950
3951
3952$L$point_addx:
3953	push	rbp
3954
3955	push	rbx
3956
3957	push	r12
3958
3959	push	r13
3960
3961	push	r14
3962
3963	push	r15
3964
3965	sub	rsp,32*18+8
3966
3967$L$point_addx_body:
3968
3969	movdqu	xmm0,XMMWORD[rsi]
3970	movdqu	xmm1,XMMWORD[16+rsi]
3971	movdqu	xmm2,XMMWORD[32+rsi]
3972	movdqu	xmm3,XMMWORD[48+rsi]
3973	movdqu	xmm4,XMMWORD[64+rsi]
3974	movdqu	xmm5,XMMWORD[80+rsi]
3975	mov	rbx,rsi
3976	mov	rsi,rdx
3977	movdqa	XMMWORD[384+rsp],xmm0
3978	movdqa	XMMWORD[(384+16)+rsp],xmm1
3979	movdqa	XMMWORD[416+rsp],xmm2
3980	movdqa	XMMWORD[(416+16)+rsp],xmm3
3981	movdqa	XMMWORD[448+rsp],xmm4
3982	movdqa	XMMWORD[(448+16)+rsp],xmm5
3983	por	xmm5,xmm4
3984
3985	movdqu	xmm0,XMMWORD[rsi]
3986	pshufd	xmm3,xmm5,0xb1
3987	movdqu	xmm1,XMMWORD[16+rsi]
3988	movdqu	xmm2,XMMWORD[32+rsi]
3989	por	xmm5,xmm3
3990	movdqu	xmm3,XMMWORD[48+rsi]
3991	mov	rdx,QWORD[((64+0))+rsi]
3992	mov	r14,QWORD[((64+8))+rsi]
3993	mov	r15,QWORD[((64+16))+rsi]
3994	mov	r8,QWORD[((64+24))+rsi]
3995	movdqa	XMMWORD[480+rsp],xmm0
3996	pshufd	xmm4,xmm5,0x1e
3997	movdqa	XMMWORD[(480+16)+rsp],xmm1
3998	movdqu	xmm0,XMMWORD[64+rsi]
3999	movdqu	xmm1,XMMWORD[80+rsi]
4000	movdqa	XMMWORD[512+rsp],xmm2
4001	movdqa	XMMWORD[(512+16)+rsp],xmm3
4002	por	xmm5,xmm4
4003	pxor	xmm4,xmm4
4004	por	xmm1,xmm0
4005DB	102,72,15,110,199
4006
4007	lea	rsi,[((64-128))+rsi]
4008	mov	QWORD[((544+0))+rsp],rdx
4009	mov	QWORD[((544+8))+rsp],r14
4010	mov	QWORD[((544+16))+rsp],r15
4011	mov	QWORD[((544+24))+rsp],r8
4012	lea	rdi,[96+rsp]
4013	call	__ecp_nistz256_sqr_montx
4014
4015	pcmpeqd	xmm5,xmm4
4016	pshufd	xmm4,xmm1,0xb1
4017	por	xmm4,xmm1
4018	pshufd	xmm5,xmm5,0
4019	pshufd	xmm3,xmm4,0x1e
4020	por	xmm4,xmm3
4021	pxor	xmm3,xmm3
4022	pcmpeqd	xmm4,xmm3
4023	pshufd	xmm4,xmm4,0
4024	mov	rdx,QWORD[((64+0))+rbx]
4025	mov	r14,QWORD[((64+8))+rbx]
4026	mov	r15,QWORD[((64+16))+rbx]
4027	mov	r8,QWORD[((64+24))+rbx]
4028DB	102,72,15,110,203
4029
4030	lea	rsi,[((64-128))+rbx]
4031	lea	rdi,[32+rsp]
4032	call	__ecp_nistz256_sqr_montx
4033
4034	mov	rdx,QWORD[544+rsp]
4035	lea	rbx,[544+rsp]
4036	mov	r9,QWORD[((0+96))+rsp]
4037	mov	r10,QWORD[((8+96))+rsp]
4038	lea	rsi,[((-128+96))+rsp]
4039	mov	r11,QWORD[((16+96))+rsp]
4040	mov	r12,QWORD[((24+96))+rsp]
4041	lea	rdi,[224+rsp]
4042	call	__ecp_nistz256_mul_montx
4043
4044	mov	rdx,QWORD[448+rsp]
4045	lea	rbx,[448+rsp]
4046	mov	r9,QWORD[((0+32))+rsp]
4047	mov	r10,QWORD[((8+32))+rsp]
4048	lea	rsi,[((-128+32))+rsp]
4049	mov	r11,QWORD[((16+32))+rsp]
4050	mov	r12,QWORD[((24+32))+rsp]
4051	lea	rdi,[256+rsp]
4052	call	__ecp_nistz256_mul_montx
4053
4054	mov	rdx,QWORD[416+rsp]
4055	lea	rbx,[416+rsp]
4056	mov	r9,QWORD[((0+224))+rsp]
4057	mov	r10,QWORD[((8+224))+rsp]
4058	lea	rsi,[((-128+224))+rsp]
4059	mov	r11,QWORD[((16+224))+rsp]
4060	mov	r12,QWORD[((24+224))+rsp]
4061	lea	rdi,[224+rsp]
4062	call	__ecp_nistz256_mul_montx
4063
4064	mov	rdx,QWORD[512+rsp]
4065	lea	rbx,[512+rsp]
4066	mov	r9,QWORD[((0+256))+rsp]
4067	mov	r10,QWORD[((8+256))+rsp]
4068	lea	rsi,[((-128+256))+rsp]
4069	mov	r11,QWORD[((16+256))+rsp]
4070	mov	r12,QWORD[((24+256))+rsp]
4071	lea	rdi,[256+rsp]
4072	call	__ecp_nistz256_mul_montx
4073
4074	lea	rbx,[224+rsp]
4075	lea	rdi,[64+rsp]
4076	call	__ecp_nistz256_sub_fromx
4077
4078	or	r12,r13
4079	movdqa	xmm2,xmm4
4080	or	r12,r8
4081	or	r12,r9
4082	por	xmm2,xmm5
4083DB	102,73,15,110,220
4084
4085	mov	rdx,QWORD[384+rsp]
4086	lea	rbx,[384+rsp]
4087	mov	r9,QWORD[((0+96))+rsp]
4088	mov	r10,QWORD[((8+96))+rsp]
4089	lea	rsi,[((-128+96))+rsp]
4090	mov	r11,QWORD[((16+96))+rsp]
4091	mov	r12,QWORD[((24+96))+rsp]
4092	lea	rdi,[160+rsp]
4093	call	__ecp_nistz256_mul_montx
4094
4095	mov	rdx,QWORD[480+rsp]
4096	lea	rbx,[480+rsp]
4097	mov	r9,QWORD[((0+32))+rsp]
4098	mov	r10,QWORD[((8+32))+rsp]
4099	lea	rsi,[((-128+32))+rsp]
4100	mov	r11,QWORD[((16+32))+rsp]
4101	mov	r12,QWORD[((24+32))+rsp]
4102	lea	rdi,[192+rsp]
4103	call	__ecp_nistz256_mul_montx
4104
4105	lea	rbx,[160+rsp]
4106	lea	rdi,[rsp]
4107	call	__ecp_nistz256_sub_fromx
4108
4109	or	r12,r13
4110	or	r12,r8
4111	or	r12,r9
4112
4113DB	102,73,15,126,208
4114DB	102,73,15,126,217
4115	or	r12,r8
4116	DB	0x3e
4117	jnz	NEAR $L$add_proceedx
4118
4119
4120
4121	test	r9,r9
4122	jz	NEAR $L$add_doublex
4123
4124
4125
4126
4127
4128
4129DB	102,72,15,126,199
4130	pxor	xmm0,xmm0
4131	movdqu	XMMWORD[rdi],xmm0
4132	movdqu	XMMWORD[16+rdi],xmm0
4133	movdqu	XMMWORD[32+rdi],xmm0
4134	movdqu	XMMWORD[48+rdi],xmm0
4135	movdqu	XMMWORD[64+rdi],xmm0
4136	movdqu	XMMWORD[80+rdi],xmm0
4137	jmp	NEAR $L$add_donex
4138
4139ALIGN	32
4140$L$add_doublex:
4141DB	102,72,15,126,206
4142DB	102,72,15,126,199
4143	add	rsp,416
4144
4145	jmp	NEAR $L$point_double_shortcutx
4146
4147
4148ALIGN	32
4149$L$add_proceedx:
4150	mov	rdx,QWORD[((0+64))+rsp]
4151	mov	r14,QWORD[((8+64))+rsp]
4152	lea	rsi,[((-128+64))+rsp]
4153	mov	r15,QWORD[((16+64))+rsp]
4154	mov	r8,QWORD[((24+64))+rsp]
4155	lea	rdi,[96+rsp]
4156	call	__ecp_nistz256_sqr_montx
4157
4158	mov	rdx,QWORD[448+rsp]
4159	lea	rbx,[448+rsp]
4160	mov	r9,QWORD[((0+0))+rsp]
4161	mov	r10,QWORD[((8+0))+rsp]
4162	lea	rsi,[((-128+0))+rsp]
4163	mov	r11,QWORD[((16+0))+rsp]
4164	mov	r12,QWORD[((24+0))+rsp]
4165	lea	rdi,[352+rsp]
4166	call	__ecp_nistz256_mul_montx
4167
4168	mov	rdx,QWORD[((0+0))+rsp]
4169	mov	r14,QWORD[((8+0))+rsp]
4170	lea	rsi,[((-128+0))+rsp]
4171	mov	r15,QWORD[((16+0))+rsp]
4172	mov	r8,QWORD[((24+0))+rsp]
4173	lea	rdi,[32+rsp]
4174	call	__ecp_nistz256_sqr_montx
4175
4176	mov	rdx,QWORD[544+rsp]
4177	lea	rbx,[544+rsp]
4178	mov	r9,QWORD[((0+352))+rsp]
4179	mov	r10,QWORD[((8+352))+rsp]
4180	lea	rsi,[((-128+352))+rsp]
4181	mov	r11,QWORD[((16+352))+rsp]
4182	mov	r12,QWORD[((24+352))+rsp]
4183	lea	rdi,[352+rsp]
4184	call	__ecp_nistz256_mul_montx
4185
4186	mov	rdx,QWORD[rsp]
4187	lea	rbx,[rsp]
4188	mov	r9,QWORD[((0+32))+rsp]
4189	mov	r10,QWORD[((8+32))+rsp]
4190	lea	rsi,[((-128+32))+rsp]
4191	mov	r11,QWORD[((16+32))+rsp]
4192	mov	r12,QWORD[((24+32))+rsp]
4193	lea	rdi,[128+rsp]
4194	call	__ecp_nistz256_mul_montx
4195
4196	mov	rdx,QWORD[160+rsp]
4197	lea	rbx,[160+rsp]
4198	mov	r9,QWORD[((0+32))+rsp]
4199	mov	r10,QWORD[((8+32))+rsp]
4200	lea	rsi,[((-128+32))+rsp]
4201	mov	r11,QWORD[((16+32))+rsp]
4202	mov	r12,QWORD[((24+32))+rsp]
4203	lea	rdi,[192+rsp]
4204	call	__ecp_nistz256_mul_montx
4205
4206
4207
4208
4209	xor	r11,r11
4210	add	r12,r12
4211	lea	rsi,[96+rsp]
4212	adc	r13,r13
4213	mov	rax,r12
4214	adc	r8,r8
4215	adc	r9,r9
4216	mov	rbp,r13
4217	adc	r11,0
4218
4219	sub	r12,-1
4220	mov	rcx,r8
4221	sbb	r13,r14
4222	sbb	r8,0
4223	mov	r10,r9
4224	sbb	r9,r15
4225	sbb	r11,0
4226
4227	cmovc	r12,rax
4228	mov	rax,QWORD[rsi]
4229	cmovc	r13,rbp
4230	mov	rbp,QWORD[8+rsi]
4231	cmovc	r8,rcx
4232	mov	rcx,QWORD[16+rsi]
4233	cmovc	r9,r10
4234	mov	r10,QWORD[24+rsi]
4235
4236	call	__ecp_nistz256_subx
4237
4238	lea	rbx,[128+rsp]
4239	lea	rdi,[288+rsp]
4240	call	__ecp_nistz256_sub_fromx
4241
4242	mov	rax,QWORD[((192+0))+rsp]
4243	mov	rbp,QWORD[((192+8))+rsp]
4244	mov	rcx,QWORD[((192+16))+rsp]
4245	mov	r10,QWORD[((192+24))+rsp]
4246	lea	rdi,[320+rsp]
4247
4248	call	__ecp_nistz256_subx
4249
4250	mov	QWORD[rdi],r12
4251	mov	QWORD[8+rdi],r13
4252	mov	QWORD[16+rdi],r8
4253	mov	QWORD[24+rdi],r9
4254	mov	rdx,QWORD[128+rsp]
4255	lea	rbx,[128+rsp]
4256	mov	r9,QWORD[((0+224))+rsp]
4257	mov	r10,QWORD[((8+224))+rsp]
4258	lea	rsi,[((-128+224))+rsp]
4259	mov	r11,QWORD[((16+224))+rsp]
4260	mov	r12,QWORD[((24+224))+rsp]
4261	lea	rdi,[256+rsp]
4262	call	__ecp_nistz256_mul_montx
4263
4264	mov	rdx,QWORD[320+rsp]
4265	lea	rbx,[320+rsp]
4266	mov	r9,QWORD[((0+64))+rsp]
4267	mov	r10,QWORD[((8+64))+rsp]
4268	lea	rsi,[((-128+64))+rsp]
4269	mov	r11,QWORD[((16+64))+rsp]
4270	mov	r12,QWORD[((24+64))+rsp]
4271	lea	rdi,[320+rsp]
4272	call	__ecp_nistz256_mul_montx
4273
4274	lea	rbx,[256+rsp]
4275	lea	rdi,[320+rsp]
4276	call	__ecp_nistz256_sub_fromx
4277
4278DB	102,72,15,126,199
4279
4280	movdqa	xmm0,xmm5
4281	movdqa	xmm1,xmm5
4282	pandn	xmm0,XMMWORD[352+rsp]
4283	movdqa	xmm2,xmm5
4284	pandn	xmm1,XMMWORD[((352+16))+rsp]
4285	movdqa	xmm3,xmm5
4286	pand	xmm2,XMMWORD[544+rsp]
4287	pand	xmm3,XMMWORD[((544+16))+rsp]
4288	por	xmm2,xmm0
4289	por	xmm3,xmm1
4290
4291	movdqa	xmm0,xmm4
4292	movdqa	xmm1,xmm4
4293	pandn	xmm0,xmm2
4294	movdqa	xmm2,xmm4
4295	pandn	xmm1,xmm3
4296	movdqa	xmm3,xmm4
4297	pand	xmm2,XMMWORD[448+rsp]
4298	pand	xmm3,XMMWORD[((448+16))+rsp]
4299	por	xmm2,xmm0
4300	por	xmm3,xmm1
4301	movdqu	XMMWORD[64+rdi],xmm2
4302	movdqu	XMMWORD[80+rdi],xmm3
4303
4304	movdqa	xmm0,xmm5
4305	movdqa	xmm1,xmm5
4306	pandn	xmm0,XMMWORD[288+rsp]
4307	movdqa	xmm2,xmm5
4308	pandn	xmm1,XMMWORD[((288+16))+rsp]
4309	movdqa	xmm3,xmm5
4310	pand	xmm2,XMMWORD[480+rsp]
4311	pand	xmm3,XMMWORD[((480+16))+rsp]
4312	por	xmm2,xmm0
4313	por	xmm3,xmm1
4314
4315	movdqa	xmm0,xmm4
4316	movdqa	xmm1,xmm4
4317	pandn	xmm0,xmm2
4318	movdqa	xmm2,xmm4
4319	pandn	xmm1,xmm3
4320	movdqa	xmm3,xmm4
4321	pand	xmm2,XMMWORD[384+rsp]
4322	pand	xmm3,XMMWORD[((384+16))+rsp]
4323	por	xmm2,xmm0
4324	por	xmm3,xmm1
4325	movdqu	XMMWORD[rdi],xmm2
4326	movdqu	XMMWORD[16+rdi],xmm3
4327
4328	movdqa	xmm0,xmm5
4329	movdqa	xmm1,xmm5
4330	pandn	xmm0,XMMWORD[320+rsp]
4331	movdqa	xmm2,xmm5
4332	pandn	xmm1,XMMWORD[((320+16))+rsp]
4333	movdqa	xmm3,xmm5
4334	pand	xmm2,XMMWORD[512+rsp]
4335	pand	xmm3,XMMWORD[((512+16))+rsp]
4336	por	xmm2,xmm0
4337	por	xmm3,xmm1
4338
4339	movdqa	xmm0,xmm4
4340	movdqa	xmm1,xmm4
4341	pandn	xmm0,xmm2
4342	movdqa	xmm2,xmm4
4343	pandn	xmm1,xmm3
4344	movdqa	xmm3,xmm4
4345	pand	xmm2,XMMWORD[416+rsp]
4346	pand	xmm3,XMMWORD[((416+16))+rsp]
4347	por	xmm2,xmm0
4348	por	xmm3,xmm1
4349	movdqu	XMMWORD[32+rdi],xmm2
4350	movdqu	XMMWORD[48+rdi],xmm3
4351
4352$L$add_donex:
4353	lea	rsi,[((576+56))+rsp]
4354
4355	mov	r15,QWORD[((-48))+rsi]
4356
4357	mov	r14,QWORD[((-40))+rsi]
4358
4359	mov	r13,QWORD[((-32))+rsi]
4360
4361	mov	r12,QWORD[((-24))+rsi]
4362
4363	mov	rbx,QWORD[((-16))+rsi]
4364
4365	mov	rbp,QWORD[((-8))+rsi]
4366
4367	lea	rsp,[rsi]
4368
4369$L$point_addx_epilogue:
4370	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
4371	mov	rsi,QWORD[16+rsp]
4372	ret
4373
4374$L$SEH_end_ecp_nistz256_point_addx:
4375
4376ALIGN	32
4377ecp_nistz256_point_add_affinex:
4378	mov	QWORD[8+rsp],rdi	;WIN64 prologue
4379	mov	QWORD[16+rsp],rsi
4380	mov	rax,rsp
4381$L$SEH_begin_ecp_nistz256_point_add_affinex:
4382	mov	rdi,rcx
4383	mov	rsi,rdx
4384	mov	rdx,r8
4385
4386
4387
4388$L$point_add_affinex:
4389	push	rbp
4390
4391	push	rbx
4392
4393	push	r12
4394
4395	push	r13
4396
4397	push	r14
4398
4399	push	r15
4400
4401	sub	rsp,32*15+8
4402
4403$L$add_affinex_body:
4404
4405	movdqu	xmm0,XMMWORD[rsi]
4406	mov	rbx,rdx
4407	movdqu	xmm1,XMMWORD[16+rsi]
4408	movdqu	xmm2,XMMWORD[32+rsi]
4409	movdqu	xmm3,XMMWORD[48+rsi]
4410	movdqu	xmm4,XMMWORD[64+rsi]
4411	movdqu	xmm5,XMMWORD[80+rsi]
4412	mov	rdx,QWORD[((64+0))+rsi]
4413	mov	r14,QWORD[((64+8))+rsi]
4414	mov	r15,QWORD[((64+16))+rsi]
4415	mov	r8,QWORD[((64+24))+rsi]
4416	movdqa	XMMWORD[320+rsp],xmm0
4417	movdqa	XMMWORD[(320+16)+rsp],xmm1
4418	movdqa	XMMWORD[352+rsp],xmm2
4419	movdqa	XMMWORD[(352+16)+rsp],xmm3
4420	movdqa	XMMWORD[384+rsp],xmm4
4421	movdqa	XMMWORD[(384+16)+rsp],xmm5
4422	por	xmm5,xmm4
4423
4424	movdqu	xmm0,XMMWORD[rbx]
4425	pshufd	xmm3,xmm5,0xb1
4426	movdqu	xmm1,XMMWORD[16+rbx]
4427	movdqu	xmm2,XMMWORD[32+rbx]
4428	por	xmm5,xmm3
4429	movdqu	xmm3,XMMWORD[48+rbx]
4430	movdqa	XMMWORD[416+rsp],xmm0
4431	pshufd	xmm4,xmm5,0x1e
4432	movdqa	XMMWORD[(416+16)+rsp],xmm1
4433	por	xmm1,xmm0
4434DB	102,72,15,110,199
4435	movdqa	XMMWORD[448+rsp],xmm2
4436	movdqa	XMMWORD[(448+16)+rsp],xmm3
4437	por	xmm3,xmm2
4438	por	xmm5,xmm4
4439	pxor	xmm4,xmm4
4440	por	xmm3,xmm1
4441
4442	lea	rsi,[((64-128))+rsi]
4443	lea	rdi,[32+rsp]
4444	call	__ecp_nistz256_sqr_montx
4445
4446	pcmpeqd	xmm5,xmm4
4447	pshufd	xmm4,xmm3,0xb1
4448	mov	rdx,QWORD[rbx]
4449
4450	mov	r9,r12
4451	por	xmm4,xmm3
4452	pshufd	xmm5,xmm5,0
4453	pshufd	xmm3,xmm4,0x1e
4454	mov	r10,r13
4455	por	xmm4,xmm3
4456	pxor	xmm3,xmm3
4457	mov	r11,r14
4458	pcmpeqd	xmm4,xmm3
4459	pshufd	xmm4,xmm4,0
4460
4461	lea	rsi,[((32-128))+rsp]
4462	mov	r12,r15
4463	lea	rdi,[rsp]
4464	call	__ecp_nistz256_mul_montx
4465
4466	lea	rbx,[320+rsp]
4467	lea	rdi,[64+rsp]
4468	call	__ecp_nistz256_sub_fromx
4469
4470	mov	rdx,QWORD[384+rsp]
4471	lea	rbx,[384+rsp]
4472	mov	r9,QWORD[((0+32))+rsp]
4473	mov	r10,QWORD[((8+32))+rsp]
4474	lea	rsi,[((-128+32))+rsp]
4475	mov	r11,QWORD[((16+32))+rsp]
4476	mov	r12,QWORD[((24+32))+rsp]
4477	lea	rdi,[32+rsp]
4478	call	__ecp_nistz256_mul_montx
4479
4480	mov	rdx,QWORD[384+rsp]
4481	lea	rbx,[384+rsp]
4482	mov	r9,QWORD[((0+64))+rsp]
4483	mov	r10,QWORD[((8+64))+rsp]
4484	lea	rsi,[((-128+64))+rsp]
4485	mov	r11,QWORD[((16+64))+rsp]
4486	mov	r12,QWORD[((24+64))+rsp]
4487	lea	rdi,[288+rsp]
4488	call	__ecp_nistz256_mul_montx
4489
4490	mov	rdx,QWORD[448+rsp]
4491	lea	rbx,[448+rsp]
4492	mov	r9,QWORD[((0+32))+rsp]
4493	mov	r10,QWORD[((8+32))+rsp]
4494	lea	rsi,[((-128+32))+rsp]
4495	mov	r11,QWORD[((16+32))+rsp]
4496	mov	r12,QWORD[((24+32))+rsp]
4497	lea	rdi,[32+rsp]
4498	call	__ecp_nistz256_mul_montx
4499
4500	lea	rbx,[352+rsp]
4501	lea	rdi,[96+rsp]
4502	call	__ecp_nistz256_sub_fromx
4503
4504	mov	rdx,QWORD[((0+64))+rsp]
4505	mov	r14,QWORD[((8+64))+rsp]
4506	lea	rsi,[((-128+64))+rsp]
4507	mov	r15,QWORD[((16+64))+rsp]
4508	mov	r8,QWORD[((24+64))+rsp]
4509	lea	rdi,[128+rsp]
4510	call	__ecp_nistz256_sqr_montx
4511
4512	mov	rdx,QWORD[((0+96))+rsp]
4513	mov	r14,QWORD[((8+96))+rsp]
4514	lea	rsi,[((-128+96))+rsp]
4515	mov	r15,QWORD[((16+96))+rsp]
4516	mov	r8,QWORD[((24+96))+rsp]
4517	lea	rdi,[192+rsp]
4518	call	__ecp_nistz256_sqr_montx
4519
4520	mov	rdx,QWORD[128+rsp]
4521	lea	rbx,[128+rsp]
4522	mov	r9,QWORD[((0+64))+rsp]
4523	mov	r10,QWORD[((8+64))+rsp]
4524	lea	rsi,[((-128+64))+rsp]
4525	mov	r11,QWORD[((16+64))+rsp]
4526	mov	r12,QWORD[((24+64))+rsp]
4527	lea	rdi,[160+rsp]
4528	call	__ecp_nistz256_mul_montx
4529
4530	mov	rdx,QWORD[320+rsp]
4531	lea	rbx,[320+rsp]
4532	mov	r9,QWORD[((0+128))+rsp]
4533	mov	r10,QWORD[((8+128))+rsp]
4534	lea	rsi,[((-128+128))+rsp]
4535	mov	r11,QWORD[((16+128))+rsp]
4536	mov	r12,QWORD[((24+128))+rsp]
4537	lea	rdi,[rsp]
4538	call	__ecp_nistz256_mul_montx
4539
4540
4541
4542
4543	xor	r11,r11
4544	add	r12,r12
4545	lea	rsi,[192+rsp]
4546	adc	r13,r13
4547	mov	rax,r12
4548	adc	r8,r8
4549	adc	r9,r9
4550	mov	rbp,r13
4551	adc	r11,0
4552
4553	sub	r12,-1
4554	mov	rcx,r8
4555	sbb	r13,r14
4556	sbb	r8,0
4557	mov	r10,r9
4558	sbb	r9,r15
4559	sbb	r11,0
4560
4561	cmovc	r12,rax
4562	mov	rax,QWORD[rsi]
4563	cmovc	r13,rbp
4564	mov	rbp,QWORD[8+rsi]
4565	cmovc	r8,rcx
4566	mov	rcx,QWORD[16+rsi]
4567	cmovc	r9,r10
4568	mov	r10,QWORD[24+rsi]
4569
4570	call	__ecp_nistz256_subx
4571
4572	lea	rbx,[160+rsp]
4573	lea	rdi,[224+rsp]
4574	call	__ecp_nistz256_sub_fromx
4575
4576	mov	rax,QWORD[((0+0))+rsp]
4577	mov	rbp,QWORD[((0+8))+rsp]
4578	mov	rcx,QWORD[((0+16))+rsp]
4579	mov	r10,QWORD[((0+24))+rsp]
4580	lea	rdi,[64+rsp]
4581
4582	call	__ecp_nistz256_subx
4583
4584	mov	QWORD[rdi],r12
4585	mov	QWORD[8+rdi],r13
4586	mov	QWORD[16+rdi],r8
4587	mov	QWORD[24+rdi],r9
4588	mov	rdx,QWORD[352+rsp]
4589	lea	rbx,[352+rsp]
4590	mov	r9,QWORD[((0+160))+rsp]
4591	mov	r10,QWORD[((8+160))+rsp]
4592	lea	rsi,[((-128+160))+rsp]
4593	mov	r11,QWORD[((16+160))+rsp]
4594	mov	r12,QWORD[((24+160))+rsp]
4595	lea	rdi,[32+rsp]
4596	call	__ecp_nistz256_mul_montx
4597
4598	mov	rdx,QWORD[96+rsp]
4599	lea	rbx,[96+rsp]
4600	mov	r9,QWORD[((0+64))+rsp]
4601	mov	r10,QWORD[((8+64))+rsp]
4602	lea	rsi,[((-128+64))+rsp]
4603	mov	r11,QWORD[((16+64))+rsp]
4604	mov	r12,QWORD[((24+64))+rsp]
4605	lea	rdi,[64+rsp]
4606	call	__ecp_nistz256_mul_montx
4607
4608	lea	rbx,[32+rsp]
4609	lea	rdi,[256+rsp]
4610	call	__ecp_nistz256_sub_fromx
4611
4612DB	102,72,15,126,199
4613
4614	movdqa	xmm0,xmm5
4615	movdqa	xmm1,xmm5
4616	pandn	xmm0,XMMWORD[288+rsp]
4617	movdqa	xmm2,xmm5
4618	pandn	xmm1,XMMWORD[((288+16))+rsp]
4619	movdqa	xmm3,xmm5
4620	pand	xmm2,XMMWORD[$L$ONE_mont]
4621	pand	xmm3,XMMWORD[(($L$ONE_mont+16))]
4622	por	xmm2,xmm0
4623	por	xmm3,xmm1
4624
4625	movdqa	xmm0,xmm4
4626	movdqa	xmm1,xmm4
4627	pandn	xmm0,xmm2
4628	movdqa	xmm2,xmm4
4629	pandn	xmm1,xmm3
4630	movdqa	xmm3,xmm4
4631	pand	xmm2,XMMWORD[384+rsp]
4632	pand	xmm3,XMMWORD[((384+16))+rsp]
4633	por	xmm2,xmm0
4634	por	xmm3,xmm1
4635	movdqu	XMMWORD[64+rdi],xmm2
4636	movdqu	XMMWORD[80+rdi],xmm3
4637
4638	movdqa	xmm0,xmm5
4639	movdqa	xmm1,xmm5
4640	pandn	xmm0,XMMWORD[224+rsp]
4641	movdqa	xmm2,xmm5
4642	pandn	xmm1,XMMWORD[((224+16))+rsp]
4643	movdqa	xmm3,xmm5
4644	pand	xmm2,XMMWORD[416+rsp]
4645	pand	xmm3,XMMWORD[((416+16))+rsp]
4646	por	xmm2,xmm0
4647	por	xmm3,xmm1
4648
4649	movdqa	xmm0,xmm4
4650	movdqa	xmm1,xmm4
4651	pandn	xmm0,xmm2
4652	movdqa	xmm2,xmm4
4653	pandn	xmm1,xmm3
4654	movdqa	xmm3,xmm4
4655	pand	xmm2,XMMWORD[320+rsp]
4656	pand	xmm3,XMMWORD[((320+16))+rsp]
4657	por	xmm2,xmm0
4658	por	xmm3,xmm1
4659	movdqu	XMMWORD[rdi],xmm2
4660	movdqu	XMMWORD[16+rdi],xmm3
4661
4662	movdqa	xmm0,xmm5
4663	movdqa	xmm1,xmm5
4664	pandn	xmm0,XMMWORD[256+rsp]
4665	movdqa	xmm2,xmm5
4666	pandn	xmm1,XMMWORD[((256+16))+rsp]
4667	movdqa	xmm3,xmm5
4668	pand	xmm2,XMMWORD[448+rsp]
4669	pand	xmm3,XMMWORD[((448+16))+rsp]
4670	por	xmm2,xmm0
4671	por	xmm3,xmm1
4672
4673	movdqa	xmm0,xmm4
4674	movdqa	xmm1,xmm4
4675	pandn	xmm0,xmm2
4676	movdqa	xmm2,xmm4
4677	pandn	xmm1,xmm3
4678	movdqa	xmm3,xmm4
4679	pand	xmm2,XMMWORD[352+rsp]
4680	pand	xmm3,XMMWORD[((352+16))+rsp]
4681	por	xmm2,xmm0
4682	por	xmm3,xmm1
4683	movdqu	XMMWORD[32+rdi],xmm2
4684	movdqu	XMMWORD[48+rdi],xmm3
4685
4686	lea	rsi,[((480+56))+rsp]
4687
4688	mov	r15,QWORD[((-48))+rsi]
4689
4690	mov	r14,QWORD[((-40))+rsi]
4691
4692	mov	r13,QWORD[((-32))+rsi]
4693
4694	mov	r12,QWORD[((-24))+rsi]
4695
4696	mov	rbx,QWORD[((-16))+rsi]
4697
4698	mov	rbp,QWORD[((-8))+rsi]
4699
4700	lea	rsp,[rsi]
4701
4702$L$add_affinex_epilogue:
4703	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
4704	mov	rsi,QWORD[16+rsp]
4705	ret
4706
4707$L$SEH_end_ecp_nistz256_point_add_affinex:
4708EXTERN	__imp_RtlVirtualUnwind
4709
4710
4711ALIGN	16
4712short_handler:
4713	push	rsi
4714	push	rdi
4715	push	rbx
4716	push	rbp
4717	push	r12
4718	push	r13
4719	push	r14
4720	push	r15
4721	pushfq
4722	sub	rsp,64
4723
4724	mov	rax,QWORD[120+r8]
4725	mov	rbx,QWORD[248+r8]
4726
4727	mov	rsi,QWORD[8+r9]
4728	mov	r11,QWORD[56+r9]
4729
4730	mov	r10d,DWORD[r11]
4731	lea	r10,[r10*1+rsi]
4732	cmp	rbx,r10
4733	jb	NEAR $L$common_seh_tail
4734
4735	mov	rax,QWORD[152+r8]
4736
4737	mov	r10d,DWORD[4+r11]
4738	lea	r10,[r10*1+rsi]
4739	cmp	rbx,r10
4740	jae	NEAR $L$common_seh_tail
4741
4742	lea	rax,[16+rax]
4743
4744	mov	r12,QWORD[((-8))+rax]
4745	mov	r13,QWORD[((-16))+rax]
4746	mov	QWORD[216+r8],r12
4747	mov	QWORD[224+r8],r13
4748
4749	jmp	NEAR $L$common_seh_tail
4750
4751
4752
4753ALIGN	16
4754full_handler:
4755	push	rsi
4756	push	rdi
4757	push	rbx
4758	push	rbp
4759	push	r12
4760	push	r13
4761	push	r14
4762	push	r15
4763	pushfq
4764	sub	rsp,64
4765
4766	mov	rax,QWORD[120+r8]
4767	mov	rbx,QWORD[248+r8]
4768
4769	mov	rsi,QWORD[8+r9]
4770	mov	r11,QWORD[56+r9]
4771
4772	mov	r10d,DWORD[r11]
4773	lea	r10,[r10*1+rsi]
4774	cmp	rbx,r10
4775	jb	NEAR $L$common_seh_tail
4776
4777	mov	rax,QWORD[152+r8]
4778
4779	mov	r10d,DWORD[4+r11]
4780	lea	r10,[r10*1+rsi]
4781	cmp	rbx,r10
4782	jae	NEAR $L$common_seh_tail
4783
4784	mov	r10d,DWORD[8+r11]
4785	lea	rax,[r10*1+rax]
4786
4787	mov	rbp,QWORD[((-8))+rax]
4788	mov	rbx,QWORD[((-16))+rax]
4789	mov	r12,QWORD[((-24))+rax]
4790	mov	r13,QWORD[((-32))+rax]
4791	mov	r14,QWORD[((-40))+rax]
4792	mov	r15,QWORD[((-48))+rax]
4793	mov	QWORD[144+r8],rbx
4794	mov	QWORD[160+r8],rbp
4795	mov	QWORD[216+r8],r12
4796	mov	QWORD[224+r8],r13
4797	mov	QWORD[232+r8],r14
4798	mov	QWORD[240+r8],r15
4799
4800$L$common_seh_tail:
4801	mov	rdi,QWORD[8+rax]
4802	mov	rsi,QWORD[16+rax]
4803	mov	QWORD[152+r8],rax
4804	mov	QWORD[168+r8],rsi
4805	mov	QWORD[176+r8],rdi
4806
4807	mov	rdi,QWORD[40+r9]
4808	mov	rsi,r8
4809	mov	ecx,154
4810	DD	0xa548f3fc
4811
4812	mov	rsi,r9
4813	xor	rcx,rcx
4814	mov	rdx,QWORD[8+rsi]
4815	mov	r8,QWORD[rsi]
4816	mov	r9,QWORD[16+rsi]
4817	mov	r10,QWORD[40+rsi]
4818	lea	r11,[56+rsi]
4819	lea	r12,[24+rsi]
4820	mov	QWORD[32+rsp],r10
4821	mov	QWORD[40+rsp],r11
4822	mov	QWORD[48+rsp],r12
4823	mov	QWORD[56+rsp],rcx
4824	call	QWORD[__imp_RtlVirtualUnwind]
4825
4826	mov	eax,1
4827	add	rsp,64
4828	popfq
4829	pop	r15
4830	pop	r14
4831	pop	r13
4832	pop	r12
4833	pop	rbp
4834	pop	rbx
4835	pop	rdi
4836	pop	rsi
4837	ret
4838
4839
4840section	.pdata rdata align=4
4841ALIGN	4
4842	DD	$L$SEH_begin_ecp_nistz256_neg wrt ..imagebase
4843	DD	$L$SEH_end_ecp_nistz256_neg wrt ..imagebase
4844	DD	$L$SEH_info_ecp_nistz256_neg wrt ..imagebase
4845
4846	DD	$L$SEH_begin_ecp_nistz256_ord_mul_mont wrt ..imagebase
4847	DD	$L$SEH_end_ecp_nistz256_ord_mul_mont wrt ..imagebase
4848	DD	$L$SEH_info_ecp_nistz256_ord_mul_mont wrt ..imagebase
4849
4850	DD	$L$SEH_begin_ecp_nistz256_ord_sqr_mont wrt ..imagebase
4851	DD	$L$SEH_end_ecp_nistz256_ord_sqr_mont wrt ..imagebase
4852	DD	$L$SEH_info_ecp_nistz256_ord_sqr_mont wrt ..imagebase
4853	DD	$L$SEH_begin_ecp_nistz256_ord_mul_montx wrt ..imagebase
4854	DD	$L$SEH_end_ecp_nistz256_ord_mul_montx wrt ..imagebase
4855	DD	$L$SEH_info_ecp_nistz256_ord_mul_montx wrt ..imagebase
4856
4857	DD	$L$SEH_begin_ecp_nistz256_ord_sqr_montx wrt ..imagebase
4858	DD	$L$SEH_end_ecp_nistz256_ord_sqr_montx wrt ..imagebase
4859	DD	$L$SEH_info_ecp_nistz256_ord_sqr_montx wrt ..imagebase
4860	DD	$L$SEH_begin_ecp_nistz256_mul_mont wrt ..imagebase
4861	DD	$L$SEH_end_ecp_nistz256_mul_mont wrt ..imagebase
4862	DD	$L$SEH_info_ecp_nistz256_mul_mont wrt ..imagebase
4863
4864	DD	$L$SEH_begin_ecp_nistz256_sqr_mont wrt ..imagebase
4865	DD	$L$SEH_end_ecp_nistz256_sqr_mont wrt ..imagebase
4866	DD	$L$SEH_info_ecp_nistz256_sqr_mont wrt ..imagebase
4867
4868	DD	$L$SEH_begin_ecp_nistz256_select_w5 wrt ..imagebase
4869	DD	$L$SEH_end_ecp_nistz256_select_w5 wrt ..imagebase
4870	DD	$L$SEH_info_ecp_nistz256_select_wX wrt ..imagebase
4871
4872	DD	$L$SEH_begin_ecp_nistz256_select_w7 wrt ..imagebase
4873	DD	$L$SEH_end_ecp_nistz256_select_w7 wrt ..imagebase
4874	DD	$L$SEH_info_ecp_nistz256_select_wX wrt ..imagebase
4875	DD	$L$SEH_begin_ecp_nistz256_avx2_select_w5 wrt ..imagebase
4876	DD	$L$SEH_end_ecp_nistz256_avx2_select_w5 wrt ..imagebase
4877	DD	$L$SEH_info_ecp_nistz256_avx2_select_wX wrt ..imagebase
4878
4879	DD	$L$SEH_begin_ecp_nistz256_avx2_select_w7 wrt ..imagebase
4880	DD	$L$SEH_end_ecp_nistz256_avx2_select_w7 wrt ..imagebase
4881	DD	$L$SEH_info_ecp_nistz256_avx2_select_wX wrt ..imagebase
4882	DD	$L$SEH_begin_ecp_nistz256_point_double wrt ..imagebase
4883	DD	$L$SEH_end_ecp_nistz256_point_double wrt ..imagebase
4884	DD	$L$SEH_info_ecp_nistz256_point_double wrt ..imagebase
4885
4886	DD	$L$SEH_begin_ecp_nistz256_point_add wrt ..imagebase
4887	DD	$L$SEH_end_ecp_nistz256_point_add wrt ..imagebase
4888	DD	$L$SEH_info_ecp_nistz256_point_add wrt ..imagebase
4889
4890	DD	$L$SEH_begin_ecp_nistz256_point_add_affine wrt ..imagebase
4891	DD	$L$SEH_end_ecp_nistz256_point_add_affine wrt ..imagebase
4892	DD	$L$SEH_info_ecp_nistz256_point_add_affine wrt ..imagebase
4893	DD	$L$SEH_begin_ecp_nistz256_point_doublex wrt ..imagebase
4894	DD	$L$SEH_end_ecp_nistz256_point_doublex wrt ..imagebase
4895	DD	$L$SEH_info_ecp_nistz256_point_doublex wrt ..imagebase
4896
4897	DD	$L$SEH_begin_ecp_nistz256_point_addx wrt ..imagebase
4898	DD	$L$SEH_end_ecp_nistz256_point_addx wrt ..imagebase
4899	DD	$L$SEH_info_ecp_nistz256_point_addx wrt ..imagebase
4900
4901	DD	$L$SEH_begin_ecp_nistz256_point_add_affinex wrt ..imagebase
4902	DD	$L$SEH_end_ecp_nistz256_point_add_affinex wrt ..imagebase
4903	DD	$L$SEH_info_ecp_nistz256_point_add_affinex wrt ..imagebase
4904
4905section	.xdata rdata align=8
4906ALIGN	8
4907$L$SEH_info_ecp_nistz256_neg:
4908	DB	9,0,0,0
4909	DD	short_handler wrt ..imagebase
4910	DD	$L$neg_body wrt ..imagebase,$L$neg_epilogue wrt ..imagebase
4911$L$SEH_info_ecp_nistz256_ord_mul_mont:
4912	DB	9,0,0,0
4913	DD	full_handler wrt ..imagebase
4914	DD	$L$ord_mul_body wrt ..imagebase,$L$ord_mul_epilogue wrt ..imagebase
4915	DD	48,0
4916$L$SEH_info_ecp_nistz256_ord_sqr_mont:
4917	DB	9,0,0,0
4918	DD	full_handler wrt ..imagebase
4919	DD	$L$ord_sqr_body wrt ..imagebase,$L$ord_sqr_epilogue wrt ..imagebase
4920	DD	48,0
4921$L$SEH_info_ecp_nistz256_ord_mul_montx:
4922	DB	9,0,0,0
4923	DD	full_handler wrt ..imagebase
4924	DD	$L$ord_mulx_body wrt ..imagebase,$L$ord_mulx_epilogue wrt ..imagebase
4925	DD	48,0
4926$L$SEH_info_ecp_nistz256_ord_sqr_montx:
4927	DB	9,0,0,0
4928	DD	full_handler wrt ..imagebase
4929	DD	$L$ord_sqrx_body wrt ..imagebase,$L$ord_sqrx_epilogue wrt ..imagebase
4930	DD	48,0
4931$L$SEH_info_ecp_nistz256_mul_mont:
4932	DB	9,0,0,0
4933	DD	full_handler wrt ..imagebase
4934	DD	$L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase
4935	DD	48,0
4936$L$SEH_info_ecp_nistz256_sqr_mont:
4937	DB	9,0,0,0
4938	DD	full_handler wrt ..imagebase
4939	DD	$L$sqr_body wrt ..imagebase,$L$sqr_epilogue wrt ..imagebase
4940	DD	48,0
4941$L$SEH_info_ecp_nistz256_select_wX:
4942	DB	0x01,0x33,0x16,0x00
4943	DB	0x33,0xf8,0x09,0x00
4944	DB	0x2e,0xe8,0x08,0x00
4945	DB	0x29,0xd8,0x07,0x00
4946	DB	0x24,0xc8,0x06,0x00
4947	DB	0x1f,0xb8,0x05,0x00
4948	DB	0x1a,0xa8,0x04,0x00
4949	DB	0x15,0x98,0x03,0x00
4950	DB	0x10,0x88,0x02,0x00
4951	DB	0x0c,0x78,0x01,0x00
4952	DB	0x08,0x68,0x00,0x00
4953	DB	0x04,0x01,0x15,0x00
4954ALIGN	8
4955$L$SEH_info_ecp_nistz256_avx2_select_wX:
4956	DB	0x01,0x36,0x17,0x0b
4957	DB	0x36,0xf8,0x09,0x00
4958	DB	0x31,0xe8,0x08,0x00
4959	DB	0x2c,0xd8,0x07,0x00
4960	DB	0x27,0xc8,0x06,0x00
4961	DB	0x22,0xb8,0x05,0x00
4962	DB	0x1d,0xa8,0x04,0x00
4963	DB	0x18,0x98,0x03,0x00
4964	DB	0x13,0x88,0x02,0x00
4965	DB	0x0e,0x78,0x01,0x00
4966	DB	0x09,0x68,0x00,0x00
4967	DB	0x04,0x01,0x15,0x00
4968	DB	0x00,0xb3,0x00,0x00
4969ALIGN	8
4970$L$SEH_info_ecp_nistz256_point_double:
4971	DB	9,0,0,0
4972	DD	full_handler wrt ..imagebase
4973	DD	$L$point_doubleq_body wrt ..imagebase,$L$point_doubleq_epilogue wrt ..imagebase
4974	DD	32*5+56,0
4975$L$SEH_info_ecp_nistz256_point_add:
4976	DB	9,0,0,0
4977	DD	full_handler wrt ..imagebase
4978	DD	$L$point_addq_body wrt ..imagebase,$L$point_addq_epilogue wrt ..imagebase
4979	DD	32*18+56,0
4980$L$SEH_info_ecp_nistz256_point_add_affine:
4981	DB	9,0,0,0
4982	DD	full_handler wrt ..imagebase
4983	DD	$L$add_affineq_body wrt ..imagebase,$L$add_affineq_epilogue wrt ..imagebase
4984	DD	32*15+56,0
4985ALIGN	8
4986$L$SEH_info_ecp_nistz256_point_doublex:
4987	DB	9,0,0,0
4988	DD	full_handler wrt ..imagebase
4989	DD	$L$point_doublex_body wrt ..imagebase,$L$point_doublex_epilogue wrt ..imagebase
4990	DD	32*5+56,0
4991$L$SEH_info_ecp_nistz256_point_addx:
4992	DB	9,0,0,0
4993	DD	full_handler wrt ..imagebase
4994	DD	$L$point_addx_body wrt ..imagebase,$L$point_addx_epilogue wrt ..imagebase
4995	DD	32*18+56,0
4996$L$SEH_info_ecp_nistz256_point_add_affinex:
4997	DB	9,0,0,0
4998	DD	full_handler wrt ..imagebase
4999	DD	$L$add_affinex_body wrt ..imagebase,$L$add_affinex_epilogue wrt ..imagebase
5000	DD	32*15+56,0
5001%else
5002; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738
5003ret
5004%endif
5005