1 /*
2 * Copyright (c) 2021 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24
25 #pragma once
26
27 #ifdef __aarch64__
28
29 namespace {
30
a64_transpose_interleave_16(uint32_t * out,const uint32_t * in,size_t width,size_t in_stride,size_t height)31 void a64_transpose_interleave_16(uint32_t *out, const uint32_t *in, size_t width, size_t in_stride, size_t height)
32 {
33 size_t out_stride = 4 * height * sizeof(uint32_t);
34
35 __asm__ __volatile__(
36 "cmp %x[height], #0x4\n"
37 "blt 6f\n"
38 "1:" // Main row loop: Head
39 "mov x24, %x[in]\n"
40 "mov x23, %x[out]\n"
41 "add x22, x24, %x[in_stride]\n"
42 "add x21, x22, %x[in_stride]\n"
43 "add x20, x21, %x[in_stride]\n"
44 "add %x[in], x20, %x[in_stride]\n"
45 "sub %x[height], %x[height], #0x4\n"
46 "mov x19, %x[width]\n"
47 "cmp x19, #0x4\n"
48 "blt 3f\n"
49 "2:" // Main row loop: Column loop
50 "ldr q19, [x24], #0x10\n"
51 "sub x19, x19, #0x4\n"
52 "ldr q18, [x22], #0x10\n"
53 "cmp x19, #0x4\n"
54 "ldr q17, [x21], #0x10\n"
55 "ldr q16, [x20], #0x10\n"
56 "str q19, [x23, #0x0]\n"
57 "str q18, [x23, #0x10]\n"
58 "str q17, [x23, #0x20]\n"
59 "str q16, [x23, #0x30]\n"
60 "add x23, x23, %x[out_stride]\n"
61 "bge 2b\n"
62 "3:" // Main row loop: Column loop skip
63 "cmp x19, #0x1\n"
64 "blt 5f\n"
65 "4:" // Main row loop: width 1 loop: loop
66 "ldr s19, [x24], #0x4\n"
67 "sub x19, x19, #0x1\n"
68 "ldr s18, [x22], #0x4\n"
69 "cmp x19, #0x1\n"
70 "ldr s17, [x21], #0x4\n"
71 "ldr s16, [x20], #0x4\n"
72 "str s19, [x23, #0x0]\n"
73 "str s18, [x23, #0x10]\n"
74 "str s17, [x23, #0x20]\n"
75 "str s16, [x23, #0x30]\n"
76 "add x23, x23, #0x4\n"
77 "bge 4b\n"
78 "5:" // Main row loop: width 1 loop: skip
79 "add %x[out], %x[out], #0x40\n"
80 "cmp %x[height], #0x4\n"
81 "bge 1b\n"
82 "cbz %x[height], 12f\n"
83 "6:" // Main loop skip
84
85 "7:" // Tail row loop: Head
86 "mov x24, %x[in]\n"
87 "mov x23, %x[out]\n"
88 "add %x[in], x24, %x[in_stride]\n"
89 "sub %x[height], %x[height], #0x1\n"
90 "mov x19, %x[width]\n"
91 "cmp x19, #0x4\n"
92 "blt 9f\n"
93 "8:" // Tail row loop: Column loop
94 "ldr q16, [x24], #0x10\n"
95 "sub x19, x19, #0x4\n"
96 "cmp x19, #0x4\n"
97 "str q16, [x23, #0x0]\n"
98 "add x23, x23, %x[out_stride]\n"
99 "bge 8b\n"
100 "9:" // Tail row loop: Column loop skip
101 "cmp x19, #0x1\n"
102 "blt 11f\n"
103 "10:" // Tail row loop: width 1 loop: loop
104 "ldr s16, [x24], #0x4\n"
105 "sub x19, x19, #0x1\n"
106 "cmp x19, #0x1\n"
107 "str s16, [x23, #0x0]\n"
108 "add x23, x23, #0x4\n"
109 "bge 10b\n"
110 "11:" // Tail row loop: width 1 loop: skip
111 "add %x[out], %x[out], #0x10\n"
112 "cmp %x[height], #0x1\n"
113 "bge 7b\n"
114 "12:" // Done
115
116 : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out)
117 : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width)
118 : "cc", "memory", "v16", "v17", "v18", "v19", "x19", "x20", "x21", "x22", "x23", "x24"
119 );
120 }
121
122 } // anonymous namespace
123
124 template<>
Transform(float * out,const float * in,int stride,int x0,int xmax,int k0,int kmax)125 void Transform<4, 1, true, VLType::None>(
126 float *out, const float *in, int stride, int x0, int xmax, int k0, int kmax)
127 {
128 a64_transpose_interleave_16(
129 reinterpret_cast<uint32_t *>(out),
130 reinterpret_cast<const uint32_t *>(in + k0 * stride + x0),
131 (xmax-x0) * sizeof(float) / 4,
132 stride * sizeof(float),
133 (kmax-k0)
134 );
135 }
136
137 #endif
138