1 /*
2  * Copyright (c) 2022 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22  * IN THE SOFTWARE.
23  */
24 
25 #if defined(__ARM_FEATURE_SVE)
26 
27 template <>
interleave_block(bfloat16 * & out,const float * const * in,size_t width,size_t height,size_t row_offset,bool first)28 void interleave_block<4, 2, VLType::SME, false>(
29   bfloat16 * &out, const float * const *in,
30   size_t width, size_t height, size_t row_offset, bool first
31 )
32 {
33   ARM_COMPUTE_UNUSED(first);
34 
35   __asm__ __volatile__(
36       ".inst 0xd503477f  // SMSTART ZA\n"
37       "cntw x22, ALL, MUL #2\n"
38       "cntw x9\n"
39       "cntw x21, ALL, MUL #2\n"
40       "cntw x19, ALL, MUL #3\n"
41       "sub x20, x22, #0x1\n"
42       ".inst 0x25207817  // ptrue pn15.b\n"
43       "whilelt p1.s, XZR, %x[height]\n"
44       "whilelt p14.s, x9, %x[height]\n"
45       "whilelt p13.s, x21, %x[height]\n"
46       "whilelt p12.s, x19, %x[height]\n"
47       "sub x28, %x[width], #0x1\n"
48       "cntw x19, ALL, MUL #2\n"
49       "ands x27, %x[width], x20\n"
50       "mov x26, %x[in]\n"
51       "add x28, x28, x19\n"
52       "csel x27, x27, x22, NE\n"
53       "add x25, x26, x9, LSL #3\n"
54       "mov x24, #0x0\n"
55       "udiv x28, x28, x19\n"
56       "add x27, x27, #0x1\n"
57       "mov x19, %x[width]\n"
58       "add x23, x25, x9, LSL #3\n"
59       "ptrue p0.b\n"
60       "mov x22, %x[outptr_raw]\n"
61       "mov x21, %x[row_offset]\n"
62       "sub x28, x28, #0x1\n"
63       "lsr x27, x27, #0x1\n"
64       "mov x12, #0x0\n"
65       ".inst 0x25b34713  // whilelt pn11.s, x24, x19, VLx2\n"
66       "add x20, x23, x9, LSL #3\n"
67       "1:"  // Width loop: Preamble: Loop
68       "ldr x19, [x26], #0x8\n"
69       ".inst 0x25306c28  // psel p8.s, p11.s/Z, p1.s[w12]\n"
70       ".inst 0x25306dca  // psel p10.s, p11.s/Z, p14.s[w12]\n"
71       ".inst 0xa0154278  // ld1w { z24.s-z25.s }, pn8.s/Z, [x19, x21, LSL #2]\n"
72       "ldr x19, [x25], #0x8\n"
73       ".inst 0x25306da9  // psel p9.s, p11.s/Z, p13.s[w12]\n"
74       ".inst 0x25306d88  // psel p8.s, p11.s/Z, p12.s[w12]\n"
75       ".inst 0xa0154a62  // ld1w { z2.s-z3.s }, pn10.s/Z, [x19, x21, LSL #2]\n"
76       "ldr x19, [x23], #0x8\n"
77       ".inst 0xa015466a  // ld1w { z10.s-z11.s }, pn9.s/Z, [x19, x21, LSL #2]\n"
78       ".inst 0xc160e318  // bfcvt z24.h, { z24.s-z25.s }\n"
79       ".inst 0xc160e042  // bfcvt z2.h, { z2.s-z3.s }\n"
80       "ldr x19, [x20], #0x8\n"
81       ".inst 0xa015426c  // ld1w { z12.s-z13.s }, pn8.s/Z, [x19, x21, LSL #2]\n"
82       ".inst 0xc160e14a  // bfcvt z10.h, { z10.s-z11.s }\n"
83       ".inst 0xc160e18c  // bfcvt z12.h, { z12.s-z13.s }\n"
84       ".inst 0xc0800300  // mova za0h.s[x12], p0/M, z24.s\n"
85       ".inst 0xc0800044  // mova za1h.s[x12], p0/M, z2.s\n"
86       ".inst 0xc0800148  // mova za2h.s[x12], p0/M, z10.s\n"
87       ".inst 0xc080018c  // mova za3h.s[x12], p0/M, z12.s\n"
88       "add x12, x12, #0x1\n"
89       "cmp x12, x9\n"
90       "blt 1b\n"
91       "incw x21, ALL, MUL #2\n"
92       "incw x24, ALL, MUL #2\n"
93       "cbz x28, 5f\n"
94       "2:"  // Width loop
95       "mov x12, #0x0\n"
96       "3:"  // Width loop: Store: Loop
97       ".inst 0xc0828011  // mova z17.s, p0/M, za0v.s[x12]\n"
98       ".inst 0xc0828095  // mova z21.s, p0/M, za1v.s[x12]\n"
99       ".inst 0xc0828119  // mova z25.s, p0/M, za2v.s[x12]\n"
100       ".inst 0xc082819d  // mova z29.s, p0/M, za3v.s[x12]\n"
101       "add x12, x12, #0x1\n"
102       "cmp x12, x9\n"
103       ".inst 0xa160ded1  // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x22]\n"
104       "addvl x22, x22, #4\n"
105       "blt 3b\n"
106       "mov x26, %x[in]\n"
107       "add x25, x26, x9, LSL #3\n"
108       "mov x19, %x[width]\n"
109       "add x23, x25, x9, LSL #3\n"
110       "mov x12, #0x0\n"
111       ".inst 0x25b34713  // whilelt pn11.s, x24, x19, VLx2\n"
112       "add x20, x23, x9, LSL #3\n"
113       "4:"  // Width loop: Load: Loop
114       "ldr x19, [x26], #0x8\n"
115       ".inst 0x25306c28  // psel p8.s, p11.s/Z, p1.s[w12]\n"
116       ".inst 0x25306dca  // psel p10.s, p11.s/Z, p14.s[w12]\n"
117       ".inst 0xa015426c  // ld1w { z12.s-z13.s }, pn8.s/Z, [x19, x21, LSL #2]\n"
118       "ldr x19, [x25], #0x8\n"
119       ".inst 0x25306da9  // psel p9.s, p11.s/Z, p13.s[w12]\n"
120       ".inst 0x25306d88  // psel p8.s, p11.s/Z, p12.s[w12]\n"
121       ".inst 0xa0154a6e  // ld1w { z14.s-z15.s }, pn10.s/Z, [x19, x21, LSL #2]\n"
122       "ldr x19, [x23], #0x8\n"
123       ".inst 0xa0154672  // ld1w { z18.s-z19.s }, pn9.s/Z, [x19, x21, LSL #2]\n"
124       ".inst 0xc160e18c  // bfcvt z12.h, { z12.s-z13.s }\n"
125       ".inst 0xc160e1ce  // bfcvt z14.h, { z14.s-z15.s }\n"
126       "ldr x19, [x20], #0x8\n"
127       ".inst 0xa015427e  // ld1w { z30.s-z31.s }, pn8.s/Z, [x19, x21, LSL #2]\n"
128       ".inst 0xc160e252  // bfcvt z18.h, { z18.s-z19.s }\n"
129       ".inst 0xc160e3de  // bfcvt z30.h, { z30.s-z31.s }\n"
130       ".inst 0xc0800180  // mova za0h.s[x12], p0/M, z12.s\n"
131       ".inst 0xc08001c4  // mova za1h.s[x12], p0/M, z14.s\n"
132       ".inst 0xc0800248  // mova za2h.s[x12], p0/M, z18.s\n"
133       ".inst 0xc08003cc  // mova za3h.s[x12], p0/M, z30.s\n"
134       "add x12, x12, #0x1\n"
135       "cmp x12, x9\n"
136       "blt 4b\n"
137       "subs x28, x28, #0x1\n"
138       "incw x21, ALL, MUL #2\n"
139       "incw x24, ALL, MUL #2\n"
140       "bgt 2b\n"
141       "5:"  // Width loop: Tails
142       "mov x12, #0x0\n"
143       "6:"  // Width loop: Tails: Loop
144       ".inst 0xc0828011  // mova z17.s, p0/M, za0v.s[x12]\n"
145       ".inst 0xc0828095  // mova z21.s, p0/M, za1v.s[x12]\n"
146       ".inst 0xc0828119  // mova z25.s, p0/M, za2v.s[x12]\n"
147       ".inst 0xc082819d  // mova z29.s, p0/M, za3v.s[x12]\n"
148       "add x12, x12, #0x1\n"
149       "cmp x12, x27\n"
150       ".inst 0xa160ded1  // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x22]\n"
151       "addvl x22, x22, #4\n"
152       "blt 6b\n"
153       "7:"  // End
154       "mov %x[outptr_raw], x22\n"
155       ".inst 0xd503467f  // SMSTOP\n"
156       : [outptr_raw] "+&r" (out)
157       : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width)
158       : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
159     );
160 }
161 
162 #endif  // defined(__ARM_FEATURE_SVE)
163