xref: /aosp_15_r20/external/webp/src/dsp/enc_mips32.c (revision b2055c353e87c8814eb2b6b1b11112a1562253bd)
1 // Copyright 2014 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // MIPS version of speed-critical encoding functions.
11 //
12 // Author(s): Djordje Pesut    ([email protected])
13 //            Jovan Zelincevic ([email protected])
14 //            Slobodan Prijic  ([email protected])
15 
16 #include "src/dsp/dsp.h"
17 
18 #if defined(WEBP_USE_MIPS32)
19 
20 #include "src/dsp/mips_macro.h"
21 #include "src/enc/vp8i_enc.h"
22 #include "src/enc/cost_enc.h"
23 
24 static const int kC1 = WEBP_TRANSFORM_AC3_C1;
25 static const int kC2 = WEBP_TRANSFORM_AC3_C2;
26 
27 // macro for one vertical pass in ITransformOne
28 // MUL macro inlined
29 // temp0..temp15 holds tmp[0]..tmp[15]
30 // A..D - offsets in bytes to load from in buffer
31 // TEMP0..TEMP3 - registers for corresponding tmp elements
32 // TEMP4..TEMP5 - temporary registers
33 #define VERTICAL_PASS(A, B, C, D, TEMP4, TEMP0, TEMP1, TEMP2, TEMP3) \
34   "lh      %[temp16],      " #A "(%[temp20])                 \n\t"          \
35   "lh      %[temp18],      " #B "(%[temp20])                 \n\t"          \
36   "lh      %[temp17],      " #C "(%[temp20])                 \n\t"          \
37   "lh      %[temp19],      " #D "(%[temp20])                 \n\t"          \
38   "addu    %[" #TEMP4 "],    %[temp16],      %[temp18]       \n\t"          \
39   "subu    %[temp16],      %[temp16],      %[temp18]         \n\t"          \
40   "mul     %[" #TEMP0 "],    %[temp17],      %[kC2]          \n\t"          \
41   MUL_SHIFT_C1_IO(temp17, temp18)                                           \
42   MUL_SHIFT_C1(temp18, temp19)                                              \
43   "mul     %[temp19],      %[temp19],      %[kC2]            \n\t"          \
44   "sra     %[" #TEMP0 "],    %[" #TEMP0 "],    16            \n\n"          \
45   "sra     %[temp19],      %[temp19],      16                \n\n"          \
46   "subu    %[" #TEMP2 "],    %[" #TEMP0 "],    %[temp18]     \n\t"          \
47   "addu    %[" #TEMP3 "],    %[temp17],      %[temp19]       \n\t"          \
48   "addu    %[" #TEMP0 "],    %[" #TEMP4 "],    %[" #TEMP3 "] \n\t"          \
49   "addu    %[" #TEMP1 "],    %[temp16],      %[" #TEMP2 "]   \n\t"          \
50   "subu    %[" #TEMP2 "],    %[temp16],      %[" #TEMP2 "]   \n\t"          \
51   "subu    %[" #TEMP3 "],    %[" #TEMP4 "],    %[" #TEMP3 "] \n\t"
52 
53 // macro for one horizontal pass in ITransformOne
54 // MUL and STORE macros inlined
55 // a = clip_8b(a) is replaced with: a = max(a, 0); a = min(a, 255)
56 // temp0..temp15 holds tmp[0]..tmp[15]
57 // A - offset in bytes to load from ref and store to dst buffer
58 // TEMP0, TEMP4, TEMP8 and TEMP12 - registers for corresponding tmp elements
59 #define HORIZONTAL_PASS(A, TEMP0, TEMP4, TEMP8, TEMP12) \
60   "addiu   %[" #TEMP0 "],    %[" #TEMP0 "],    4               \n\t"          \
61   "addu    %[temp16],      %[" #TEMP0 "],    %[" #TEMP8 "]     \n\t"          \
62   "subu    %[temp17],      %[" #TEMP0 "],    %[" #TEMP8 "]     \n\t"          \
63   "mul     %[" #TEMP0 "],    %[" #TEMP4 "],    %[kC2]          \n\t"          \
64   MUL_SHIFT_C1_IO(TEMP4, TEMP8)                                               \
65   MUL_SHIFT_C1(TEMP8, TEMP12)                                                 \
66   "mul     %[" #TEMP12 "],   %[" #TEMP12 "],   %[kC2]          \n\t"          \
67   "sra     %[" #TEMP0 "],    %[" #TEMP0 "],    16              \n\t"          \
68   "sra     %[" #TEMP12 "],   %[" #TEMP12 "],   16              \n\t"          \
69   "subu    %[temp18],      %[" #TEMP0 "],    %[" #TEMP8 "]     \n\t"          \
70   "addu    %[temp19],      %[" #TEMP4 "],    %[" #TEMP12 "]    \n\t"          \
71   "addu    %[" #TEMP0 "],    %[temp16],      %[temp19]         \n\t"          \
72   "addu    %[" #TEMP4 "],    %[temp17],      %[temp18]         \n\t"          \
73   "subu    %[" #TEMP8 "],    %[temp17],      %[temp18]         \n\t"          \
74   "subu    %[" #TEMP12 "],   %[temp16],      %[temp19]         \n\t"          \
75   "lw      %[temp20],      0(%[args])                          \n\t"          \
76   "sra     %[" #TEMP0 "],    %[" #TEMP0 "],    3               \n\t"          \
77   "sra     %[" #TEMP4 "],    %[" #TEMP4 "],    3               \n\t"          \
78   "sra     %[" #TEMP8 "],    %[" #TEMP8 "],    3               \n\t"          \
79   "sra     %[" #TEMP12 "],   %[" #TEMP12 "],   3               \n\t"          \
80   "lbu     %[temp16],      0+" XSTR(BPS) "*" #A "(%[temp20])   \n\t"          \
81   "lbu     %[temp17],      1+" XSTR(BPS) "*" #A "(%[temp20])   \n\t"          \
82   "lbu     %[temp18],      2+" XSTR(BPS) "*" #A "(%[temp20])   \n\t"          \
83   "lbu     %[temp19],      3+" XSTR(BPS) "*" #A "(%[temp20])   \n\t"          \
84   "addu    %[" #TEMP0 "],    %[temp16],      %[" #TEMP0 "]     \n\t"          \
85   "addu    %[" #TEMP4 "],    %[temp17],      %[" #TEMP4 "]     \n\t"          \
86   "addu    %[" #TEMP8 "],    %[temp18],      %[" #TEMP8 "]     \n\t"          \
87   "addu    %[" #TEMP12 "],   %[temp19],      %[" #TEMP12 "]    \n\t"          \
88   "slt     %[temp16],      %[" #TEMP0 "],    $zero             \n\t"          \
89   "slt     %[temp17],      %[" #TEMP4 "],    $zero             \n\t"          \
90   "slt     %[temp18],      %[" #TEMP8 "],    $zero             \n\t"          \
91   "slt     %[temp19],      %[" #TEMP12 "],   $zero             \n\t"          \
92   "movn    %[" #TEMP0 "],    $zero,          %[temp16]         \n\t"          \
93   "movn    %[" #TEMP4 "],    $zero,          %[temp17]         \n\t"          \
94   "movn    %[" #TEMP8 "],    $zero,          %[temp18]         \n\t"          \
95   "movn    %[" #TEMP12 "],   $zero,          %[temp19]         \n\t"          \
96   "addiu   %[temp20],      $zero,          255                 \n\t"          \
97   "slt     %[temp16],      %[" #TEMP0 "],    %[temp20]         \n\t"          \
98   "slt     %[temp17],      %[" #TEMP4 "],    %[temp20]         \n\t"          \
99   "slt     %[temp18],      %[" #TEMP8 "],    %[temp20]         \n\t"          \
100   "slt     %[temp19],      %[" #TEMP12 "],   %[temp20]         \n\t"          \
101   "movz    %[" #TEMP0 "],    %[temp20],      %[temp16]         \n\t"          \
102   "movz    %[" #TEMP4 "],    %[temp20],      %[temp17]         \n\t"          \
103   "lw      %[temp16],      8(%[args])                          \n\t"          \
104   "movz    %[" #TEMP8 "],    %[temp20],      %[temp18]         \n\t"          \
105   "movz    %[" #TEMP12 "],   %[temp20],      %[temp19]         \n\t"          \
106   "sb      %[" #TEMP0 "],    0+" XSTR(BPS) "*" #A "(%[temp16]) \n\t"          \
107   "sb      %[" #TEMP4 "],    1+" XSTR(BPS) "*" #A "(%[temp16]) \n\t"          \
108   "sb      %[" #TEMP8 "],    2+" XSTR(BPS) "*" #A "(%[temp16]) \n\t"          \
109   "sb      %[" #TEMP12 "],   3+" XSTR(BPS) "*" #A "(%[temp16]) \n\t"
110 
111 // Does one or two inverse transforms.
ITransformOne_MIPS32(const uint8_t * ref,const int16_t * in,uint8_t * dst)112 static WEBP_INLINE void ITransformOne_MIPS32(const uint8_t* ref,
113                                              const int16_t* in,
114                                              uint8_t* dst) {
115   int temp0, temp1, temp2, temp3, temp4, temp5, temp6;
116   int temp7, temp8, temp9, temp10, temp11, temp12, temp13;
117   int temp14, temp15, temp16, temp17, temp18, temp19, temp20;
118   const int* args[3] = {(const int*)ref, (const int*)in, (const int*)dst};
119 
120   __asm__ volatile(
121     "lw      %[temp20],      4(%[args])                      \n\t"
122     VERTICAL_PASS(0, 16,  8, 24, temp4,  temp0,  temp1,  temp2,  temp3)
123     VERTICAL_PASS(2, 18, 10, 26, temp8,  temp4,  temp5,  temp6,  temp7)
124     VERTICAL_PASS(4, 20, 12, 28, temp12, temp8,  temp9,  temp10, temp11)
125     VERTICAL_PASS(6, 22, 14, 30, temp20, temp12, temp13, temp14, temp15)
126 
127     HORIZONTAL_PASS(0, temp0, temp4, temp8,  temp12)
128     HORIZONTAL_PASS(1, temp1, temp5, temp9,  temp13)
129     HORIZONTAL_PASS(2, temp2, temp6, temp10, temp14)
130     HORIZONTAL_PASS(3, temp3, temp7, temp11, temp15)
131 
132     : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
133       [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
134       [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
135       [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11),
136       [temp12]"=&r"(temp12), [temp13]"=&r"(temp13), [temp14]"=&r"(temp14),
137       [temp15]"=&r"(temp15), [temp16]"=&r"(temp16), [temp17]"=&r"(temp17),
138       [temp18]"=&r"(temp18), [temp19]"=&r"(temp19), [temp20]"=&r"(temp20)
139     : [args]"r"(args), [kC1]"r"(kC1), [kC2]"r"(kC2)
140     : "memory", "hi", "lo"
141   );
142 }
143 
ITransform_MIPS32(const uint8_t * ref,const int16_t * in,uint8_t * dst,int do_two)144 static void ITransform_MIPS32(const uint8_t* ref, const int16_t* in,
145                               uint8_t* dst, int do_two) {
146   ITransformOne_MIPS32(ref, in, dst);
147   if (do_two) {
148     ITransformOne_MIPS32(ref + 4, in + 16, dst + 4);
149   }
150 }
151 
152 #undef VERTICAL_PASS
153 #undef HORIZONTAL_PASS
154 
155 // macro for one pass through for loop in QuantizeBlock
156 // QUANTDIV macro inlined
157 // J - offset in bytes (kZigzag[n] * 2)
158 // K - offset in bytes (kZigzag[n] * 4)
159 // N - offset in bytes (n * 2)
160 #define QUANTIZE_ONE(J, K, N)                                               \
161   "lh           %[temp0],       " #J "(%[ppin])                     \n\t"   \
162   "lhu          %[temp1],       " #J "(%[ppsharpen])                \n\t"   \
163   "lw           %[temp2],       " #K "(%[ppzthresh])                \n\t"   \
164   "sra          %[sign],        %[temp0],           15              \n\t"   \
165   "xor          %[coeff],       %[temp0],           %[sign]         \n\t"   \
166   "subu         %[coeff],       %[coeff],           %[sign]         \n\t"   \
167   "addu         %[coeff],       %[coeff],           %[temp1]        \n\t"   \
168   "slt          %[temp4],       %[temp2],           %[coeff]        \n\t"   \
169   "addiu        %[temp5],       $zero,              0               \n\t"   \
170   "addiu        %[level],       $zero,              0               \n\t"   \
171   "beqz         %[temp4],       2f                                  \n\t"   \
172   "lhu          %[temp1],       " #J "(%[ppiq])                     \n\t"   \
173   "lw           %[temp2],       " #K "(%[ppbias])                   \n\t"   \
174   "lhu          %[temp3],       " #J "(%[ppq])                      \n\t"   \
175   "mul          %[level],       %[coeff],           %[temp1]        \n\t"   \
176   "addu         %[level],       %[level],           %[temp2]        \n\t"   \
177   "sra          %[level],       %[level],           17              \n\t"   \
178   "slt          %[temp4],       %[max_level],       %[level]        \n\t"   \
179   "movn         %[level],       %[max_level],       %[temp4]        \n\t"   \
180   "xor          %[level],       %[level],           %[sign]         \n\t"   \
181   "subu         %[level],       %[level],           %[sign]         \n\t"   \
182   "mul          %[temp5],       %[level],           %[temp3]        \n\t"   \
183 "2:                                                                 \n\t"   \
184   "sh           %[temp5],       " #J "(%[ppin])                     \n\t"   \
185   "sh           %[level],       " #N "(%[pout])                     \n\t"
186 
QuantizeBlock_MIPS32(int16_t in[16],int16_t out[16],const VP8Matrix * const mtx)187 static int QuantizeBlock_MIPS32(int16_t in[16], int16_t out[16],
188                                 const VP8Matrix* const mtx) {
189   int temp0, temp1, temp2, temp3, temp4, temp5;
190   int sign, coeff, level, i;
191   int max_level = MAX_LEVEL;
192 
193   int16_t* ppin             = &in[0];
194   int16_t* pout             = &out[0];
195   const uint16_t* ppsharpen = &mtx->sharpen_[0];
196   const uint32_t* ppzthresh = &mtx->zthresh_[0];
197   const uint16_t* ppq       = &mtx->q_[0];
198   const uint16_t* ppiq      = &mtx->iq_[0];
199   const uint32_t* ppbias    = &mtx->bias_[0];
200 
201   __asm__ volatile(
202     QUANTIZE_ONE( 0,  0,  0)
203     QUANTIZE_ONE( 2,  4,  2)
204     QUANTIZE_ONE( 8, 16,  4)
205     QUANTIZE_ONE(16, 32,  6)
206     QUANTIZE_ONE(10, 20,  8)
207     QUANTIZE_ONE( 4,  8, 10)
208     QUANTIZE_ONE( 6, 12, 12)
209     QUANTIZE_ONE(12, 24, 14)
210     QUANTIZE_ONE(18, 36, 16)
211     QUANTIZE_ONE(24, 48, 18)
212     QUANTIZE_ONE(26, 52, 20)
213     QUANTIZE_ONE(20, 40, 22)
214     QUANTIZE_ONE(14, 28, 24)
215     QUANTIZE_ONE(22, 44, 26)
216     QUANTIZE_ONE(28, 56, 28)
217     QUANTIZE_ONE(30, 60, 30)
218 
219     : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1),
220       [temp2]"=&r"(temp2), [temp3]"=&r"(temp3),
221       [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
222       [sign]"=&r"(sign), [coeff]"=&r"(coeff),
223       [level]"=&r"(level)
224     : [pout]"r"(pout), [ppin]"r"(ppin),
225       [ppiq]"r"(ppiq), [max_level]"r"(max_level),
226       [ppbias]"r"(ppbias), [ppzthresh]"r"(ppzthresh),
227       [ppsharpen]"r"(ppsharpen), [ppq]"r"(ppq)
228     : "memory", "hi", "lo"
229   );
230 
231   // moved out from macro to increase possibility for earlier breaking
232   for (i = 15; i >= 0; i--) {
233     if (out[i]) return 1;
234   }
235   return 0;
236 }
237 
Quantize2Blocks_MIPS32(int16_t in[32],int16_t out[32],const VP8Matrix * const mtx)238 static int Quantize2Blocks_MIPS32(int16_t in[32], int16_t out[32],
239                                   const VP8Matrix* const mtx) {
240   int nz;
241   nz  = QuantizeBlock_MIPS32(in + 0 * 16, out + 0 * 16, mtx) << 0;
242   nz |= QuantizeBlock_MIPS32(in + 1 * 16, out + 1 * 16, mtx) << 1;
243   return nz;
244 }
245 
246 #undef QUANTIZE_ONE
247 
248 // macro for one horizontal pass in Disto4x4 (TTransform)
249 // two calls of function TTransform are merged into single one
250 // A - offset in bytes to load from a and b buffers
251 // E..H - offsets in bytes to store first results to tmp buffer
252 // E1..H1 - offsets in bytes to store second results to tmp buffer
253 #define HORIZONTAL_PASS(A, E, F, G, H, E1, F1, G1, H1)                  \
254   "lbu    %[temp0],  0+" XSTR(BPS) "*" #A "(%[a])  \n\t"                \
255   "lbu    %[temp1],  1+" XSTR(BPS) "*" #A "(%[a])  \n\t"                \
256   "lbu    %[temp2],  2+" XSTR(BPS) "*" #A "(%[a])  \n\t"                \
257   "lbu    %[temp3],  3+" XSTR(BPS) "*" #A "(%[a])  \n\t"                \
258   "lbu    %[temp4],  0+" XSTR(BPS) "*" #A "(%[b])  \n\t"                \
259   "lbu    %[temp5],  1+" XSTR(BPS) "*" #A "(%[b])  \n\t"                \
260   "lbu    %[temp6],  2+" XSTR(BPS) "*" #A "(%[b])  \n\t"                \
261   "lbu    %[temp7],  3+" XSTR(BPS) "*" #A "(%[b])  \n\t"                \
262   "addu   %[temp8],  %[temp0],    %[temp2]         \n\t"                \
263   "subu   %[temp0],  %[temp0],    %[temp2]         \n\t"                \
264   "addu   %[temp2],  %[temp1],    %[temp3]         \n\t"                \
265   "subu   %[temp1],  %[temp1],    %[temp3]         \n\t"                \
266   "addu   %[temp3],  %[temp4],    %[temp6]         \n\t"                \
267   "subu   %[temp4],  %[temp4],    %[temp6]         \n\t"                \
268   "addu   %[temp6],  %[temp5],    %[temp7]         \n\t"                \
269   "subu   %[temp5],  %[temp5],    %[temp7]         \n\t"                \
270   "addu   %[temp7],  %[temp8],    %[temp2]         \n\t"                \
271   "subu   %[temp2],  %[temp8],    %[temp2]         \n\t"                \
272   "addu   %[temp8],  %[temp0],    %[temp1]         \n\t"                \
273   "subu   %[temp0],  %[temp0],    %[temp1]         \n\t"                \
274   "addu   %[temp1],  %[temp3],    %[temp6]         \n\t"                \
275   "subu   %[temp3],  %[temp3],    %[temp6]         \n\t"                \
276   "addu   %[temp6],  %[temp4],    %[temp5]         \n\t"                \
277   "subu   %[temp4],  %[temp4],    %[temp5]         \n\t"                \
278   "sw     %[temp7],  " #E "(%[tmp])                \n\t"                \
279   "sw     %[temp2],  " #H "(%[tmp])                \n\t"                \
280   "sw     %[temp8],  " #F "(%[tmp])                \n\t"                \
281   "sw     %[temp0],  " #G "(%[tmp])                \n\t"                \
282   "sw     %[temp1],  " #E1 "(%[tmp])               \n\t"                \
283   "sw     %[temp3],  " #H1 "(%[tmp])               \n\t"                \
284   "sw     %[temp6],  " #F1 "(%[tmp])               \n\t"                \
285   "sw     %[temp4],  " #G1 "(%[tmp])               \n\t"
286 
287 // macro for one vertical pass in Disto4x4 (TTransform)
288 // two calls of function TTransform are merged into single one
289 // since only one accu is available in mips32r1 instruction set
290 //   first is done second call of function TTransform and after
291 //   that first one.
292 //   const int sum1 = TTransform(a, w);
293 //   const int sum2 = TTransform(b, w);
294 //   return abs(sum2 - sum1) >> 5;
295 //   (sum2 - sum1) is calculated with madds (sub2) and msubs (sub1)
296 // A..D - offsets in bytes to load first results from tmp buffer
297 // A1..D1 - offsets in bytes to load second results from tmp buffer
298 // E..H - offsets in bytes to load from w buffer
299 #define VERTICAL_PASS(A, B, C, D, A1, B1, C1, D1, E, F, G, H)     \
300   "lw     %[temp0],  " #A1 "(%[tmp])         \n\t"                \
301   "lw     %[temp1],  " #C1 "(%[tmp])         \n\t"                \
302   "lw     %[temp2],  " #B1 "(%[tmp])         \n\t"                \
303   "lw     %[temp3],  " #D1 "(%[tmp])         \n\t"                \
304   "addu   %[temp8],  %[temp0],    %[temp1]   \n\t"                \
305   "subu   %[temp0],  %[temp0],    %[temp1]   \n\t"                \
306   "addu   %[temp1],  %[temp2],    %[temp3]   \n\t"                \
307   "subu   %[temp2],  %[temp2],    %[temp3]   \n\t"                \
308   "addu   %[temp3],  %[temp8],    %[temp1]   \n\t"                \
309   "subu   %[temp8],  %[temp8],    %[temp1]   \n\t"                \
310   "addu   %[temp1],  %[temp0],    %[temp2]   \n\t"                \
311   "subu   %[temp0],  %[temp0],    %[temp2]   \n\t"                \
312   "sra    %[temp4],  %[temp3],    31         \n\t"                \
313   "sra    %[temp5],  %[temp1],    31         \n\t"                \
314   "sra    %[temp6],  %[temp0],    31         \n\t"                \
315   "sra    %[temp7],  %[temp8],    31         \n\t"                \
316   "xor    %[temp3],  %[temp3],    %[temp4]   \n\t"                \
317   "xor    %[temp1],  %[temp1],    %[temp5]   \n\t"                \
318   "xor    %[temp0],  %[temp0],    %[temp6]   \n\t"                \
319   "xor    %[temp8],  %[temp8],    %[temp7]   \n\t"                \
320   "subu   %[temp3],  %[temp3],    %[temp4]   \n\t"                \
321   "subu   %[temp1],  %[temp1],    %[temp5]   \n\t"                \
322   "subu   %[temp0],  %[temp0],    %[temp6]   \n\t"                \
323   "subu   %[temp8],  %[temp8],    %[temp7]   \n\t"                \
324   "lhu    %[temp4],  " #E "(%[w])            \n\t"                \
325   "lhu    %[temp5],  " #F "(%[w])            \n\t"                \
326   "lhu    %[temp6],  " #G "(%[w])            \n\t"                \
327   "lhu    %[temp7],  " #H "(%[w])            \n\t"                \
328   "madd   %[temp4],  %[temp3]                \n\t"                \
329   "madd   %[temp5],  %[temp1]                \n\t"                \
330   "madd   %[temp6],  %[temp0]                \n\t"                \
331   "madd   %[temp7],  %[temp8]                \n\t"                \
332   "lw     %[temp0],  " #A "(%[tmp])          \n\t"                \
333   "lw     %[temp1],  " #C "(%[tmp])          \n\t"                \
334   "lw     %[temp2],  " #B "(%[tmp])          \n\t"                \
335   "lw     %[temp3],  " #D "(%[tmp])          \n\t"                \
336   "addu   %[temp8],  %[temp0],    %[temp1]   \n\t"                \
337   "subu   %[temp0],  %[temp0],    %[temp1]   \n\t"                \
338   "addu   %[temp1],  %[temp2],    %[temp3]   \n\t"                \
339   "subu   %[temp2],  %[temp2],    %[temp3]   \n\t"                \
340   "addu   %[temp3],  %[temp8],    %[temp1]   \n\t"                \
341   "subu   %[temp1],  %[temp8],    %[temp1]   \n\t"                \
342   "addu   %[temp8],  %[temp0],    %[temp2]   \n\t"                \
343   "subu   %[temp0],  %[temp0],    %[temp2]   \n\t"                \
344   "sra    %[temp2],  %[temp3],    31         \n\t"                \
345   "xor    %[temp3],  %[temp3],    %[temp2]   \n\t"                \
346   "subu   %[temp3],  %[temp3],    %[temp2]   \n\t"                \
347   "msub   %[temp4],  %[temp3]                \n\t"                \
348   "sra    %[temp2],  %[temp8],    31         \n\t"                \
349   "sra    %[temp3],  %[temp0],    31         \n\t"                \
350   "sra    %[temp4],  %[temp1],    31         \n\t"                \
351   "xor    %[temp8],  %[temp8],    %[temp2]   \n\t"                \
352   "xor    %[temp0],  %[temp0],    %[temp3]   \n\t"                \
353   "xor    %[temp1],  %[temp1],    %[temp4]   \n\t"                \
354   "subu   %[temp8],  %[temp8],    %[temp2]   \n\t"                \
355   "subu   %[temp0],  %[temp0],    %[temp3]   \n\t"                \
356   "subu   %[temp1],  %[temp1],    %[temp4]   \n\t"                \
357   "msub   %[temp5],  %[temp8]                \n\t"                \
358   "msub   %[temp6],  %[temp0]                \n\t"                \
359   "msub   %[temp7],  %[temp1]                \n\t"
360 
Disto4x4_MIPS32(const uint8_t * const a,const uint8_t * const b,const uint16_t * const w)361 static int Disto4x4_MIPS32(const uint8_t* const a, const uint8_t* const b,
362                            const uint16_t* const w) {
363   int tmp[32];
364   int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8;
365 
366   __asm__ volatile(
367     HORIZONTAL_PASS(0,   0,  4,  8, 12,    64,  68,  72,  76)
368     HORIZONTAL_PASS(1,  16, 20, 24, 28,    80,  84,  88,  92)
369     HORIZONTAL_PASS(2,  32, 36, 40, 44,    96, 100, 104, 108)
370     HORIZONTAL_PASS(3,  48, 52, 56, 60,   112, 116, 120, 124)
371     "mthi   $zero                             \n\t"
372     "mtlo   $zero                             \n\t"
373     VERTICAL_PASS( 0, 16, 32, 48,     64, 80,  96, 112,   0,  8, 16, 24)
374     VERTICAL_PASS( 4, 20, 36, 52,     68, 84, 100, 116,   2, 10, 18, 26)
375     VERTICAL_PASS( 8, 24, 40, 56,     72, 88, 104, 120,   4, 12, 20, 28)
376     VERTICAL_PASS(12, 28, 44, 60,     76, 92, 108, 124,   6, 14, 22, 30)
377     "mflo   %[temp0]                          \n\t"
378     "sra    %[temp1],  %[temp0],  31          \n\t"
379     "xor    %[temp0],  %[temp0],  %[temp1]    \n\t"
380     "subu   %[temp0],  %[temp0],  %[temp1]    \n\t"
381     "sra    %[temp0],  %[temp0],  5           \n\t"
382 
383     : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
384       [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
385       [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8)
386     : [a]"r"(a), [b]"r"(b), [w]"r"(w), [tmp]"r"(tmp)
387     : "memory", "hi", "lo"
388   );
389 
390   return temp0;
391 }
392 
393 #undef VERTICAL_PASS
394 #undef HORIZONTAL_PASS
395 
Disto16x16_MIPS32(const uint8_t * const a,const uint8_t * const b,const uint16_t * const w)396 static int Disto16x16_MIPS32(const uint8_t* const a, const uint8_t* const b,
397                              const uint16_t* const w) {
398   int D = 0;
399   int x, y;
400   for (y = 0; y < 16 * BPS; y += 4 * BPS) {
401     for (x = 0; x < 16; x += 4) {
402       D += Disto4x4_MIPS32(a + x + y, b + x + y, w);
403     }
404   }
405   return D;
406 }
407 
408 // macro for one horizontal pass in FTransform
409 // temp0..temp15 holds tmp[0]..tmp[15]
410 // A - offset in bytes to load from src and ref buffers
411 // TEMP0..TEMP3 - registers for corresponding tmp elements
412 #define HORIZONTAL_PASS(A, TEMP0, TEMP1, TEMP2, TEMP3)                  \
413   "lw     %[" #TEMP1 "],  0(%[args])                           \n\t"    \
414   "lw     %[" #TEMP2 "],  4(%[args])                           \n\t"    \
415   "lbu    %[temp16],    0+" XSTR(BPS) "*" #A "(%[" #TEMP1 "])  \n\t"    \
416   "lbu    %[temp17],    0+" XSTR(BPS) "*" #A "(%[" #TEMP2 "])  \n\t"    \
417   "lbu    %[temp18],    1+" XSTR(BPS) "*" #A "(%[" #TEMP1 "])  \n\t"    \
418   "lbu    %[temp19],    1+" XSTR(BPS) "*" #A "(%[" #TEMP2 "])  \n\t"    \
419   "subu   %[temp20],    %[temp16],    %[temp17]                \n\t"    \
420   "lbu    %[temp16],    2+" XSTR(BPS) "*" #A "(%[" #TEMP1 "])  \n\t"    \
421   "lbu    %[temp17],    2+" XSTR(BPS) "*" #A "(%[" #TEMP2 "])  \n\t"    \
422   "subu   %[" #TEMP0 "],  %[temp18],    %[temp19]              \n\t"    \
423   "lbu    %[temp18],    3+" XSTR(BPS) "*" #A "(%[" #TEMP1 "])  \n\t"    \
424   "lbu    %[temp19],    3+" XSTR(BPS) "*" #A "(%[" #TEMP2 "])  \n\t"    \
425   "subu   %[" #TEMP1 "],  %[temp16],    %[temp17]              \n\t"    \
426   "subu   %[" #TEMP2 "],  %[temp18],    %[temp19]              \n\t"    \
427   "addu   %[" #TEMP3 "],  %[temp20],    %[" #TEMP2 "]          \n\t"    \
428   "subu   %[" #TEMP2 "],  %[temp20],    %[" #TEMP2 "]          \n\t"    \
429   "addu   %[temp20],    %[" #TEMP0 "],  %[" #TEMP1 "]          \n\t"    \
430   "subu   %[" #TEMP0 "],  %[" #TEMP0 "],  %[" #TEMP1 "]        \n\t"    \
431   "mul    %[temp16],    %[" #TEMP2 "],  %[c5352]               \n\t"    \
432   "mul    %[temp17],    %[" #TEMP2 "],  %[c2217]               \n\t"    \
433   "mul    %[temp18],    %[" #TEMP0 "],  %[c5352]               \n\t"    \
434   "mul    %[temp19],    %[" #TEMP0 "],  %[c2217]               \n\t"    \
435   "addu   %[" #TEMP1 "],  %[" #TEMP3 "],  %[temp20]            \n\t"    \
436   "subu   %[temp20],    %[" #TEMP3 "],  %[temp20]              \n\t"    \
437   "sll    %[" #TEMP0 "],  %[" #TEMP1 "],  3                    \n\t"    \
438   "sll    %[" #TEMP2 "],  %[temp20],    3                      \n\t"    \
439   "addiu  %[temp16],    %[temp16],    1812                     \n\t"    \
440   "addiu  %[temp17],    %[temp17],    937                      \n\t"    \
441   "addu   %[temp16],    %[temp16],    %[temp19]                \n\t"    \
442   "subu   %[temp17],    %[temp17],    %[temp18]                \n\t"    \
443   "sra    %[" #TEMP1 "],  %[temp16],    9                      \n\t"    \
444   "sra    %[" #TEMP3 "],  %[temp17],    9                      \n\t"
445 
446 // macro for one vertical pass in FTransform
447 // temp0..temp15 holds tmp[0]..tmp[15]
448 // A..D - offsets in bytes to store to out buffer
449 // TEMP0, TEMP4, TEMP8 and TEMP12 - registers for corresponding tmp elements
450 #define VERTICAL_PASS(A, B, C, D, TEMP0, TEMP4, TEMP8, TEMP12)    \
451   "addu   %[temp16],    %[" #TEMP0 "],  %[" #TEMP12 "]   \n\t"    \
452   "subu   %[temp19],    %[" #TEMP0 "],  %[" #TEMP12 "]   \n\t"    \
453   "addu   %[temp17],    %[" #TEMP4 "],  %[" #TEMP8 "]    \n\t"    \
454   "subu   %[temp18],    %[" #TEMP4 "],  %[" #TEMP8 "]    \n\t"    \
455   "mul    %[" #TEMP8 "],  %[temp19],    %[c2217]         \n\t"    \
456   "mul    %[" #TEMP12 "], %[temp18],    %[c2217]         \n\t"    \
457   "mul    %[" #TEMP4 "],  %[temp19],    %[c5352]         \n\t"    \
458   "mul    %[temp18],    %[temp18],    %[c5352]           \n\t"    \
459   "addiu  %[temp16],    %[temp16],    7                  \n\t"    \
460   "addu   %[" #TEMP0 "],  %[temp16],    %[temp17]        \n\t"    \
461   "sra    %[" #TEMP0 "],  %[" #TEMP0 "],  4              \n\t"    \
462   "addu   %[" #TEMP12 "], %[" #TEMP12 "], %[" #TEMP4 "]  \n\t"    \
463   "subu   %[" #TEMP4 "],  %[temp16],    %[temp17]        \n\t"    \
464   "sra    %[" #TEMP4 "],  %[" #TEMP4 "],  4              \n\t"    \
465   "addiu  %[" #TEMP8 "],  %[" #TEMP8 "],  30000          \n\t"    \
466   "addiu  %[" #TEMP12 "], %[" #TEMP12 "], 12000          \n\t"    \
467   "addiu  %[" #TEMP8 "],  %[" #TEMP8 "],  21000          \n\t"    \
468   "subu   %[" #TEMP8 "],  %[" #TEMP8 "],  %[temp18]      \n\t"    \
469   "sra    %[" #TEMP12 "], %[" #TEMP12 "], 16             \n\t"    \
470   "sra    %[" #TEMP8 "],  %[" #TEMP8 "],  16             \n\t"    \
471   "addiu  %[temp16],    %[" #TEMP12 "], 1                \n\t"    \
472   "movn   %[" #TEMP12 "], %[temp16],    %[temp19]        \n\t"    \
473   "sh     %[" #TEMP0 "],  " #A "(%[temp20])              \n\t"    \
474   "sh     %[" #TEMP4 "],  " #C "(%[temp20])              \n\t"    \
475   "sh     %[" #TEMP8 "],  " #D "(%[temp20])              \n\t"    \
476   "sh     %[" #TEMP12 "], " #B "(%[temp20])              \n\t"
477 
FTransform_MIPS32(const uint8_t * src,const uint8_t * ref,int16_t * out)478 static void FTransform_MIPS32(const uint8_t* src, const uint8_t* ref,
479                               int16_t* out) {
480   int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8;
481   int temp9, temp10, temp11, temp12, temp13, temp14, temp15, temp16;
482   int temp17, temp18, temp19, temp20;
483   const int c2217 = 2217;
484   const int c5352 = 5352;
485   const int* const args[3] =
486       { (const int*)src, (const int*)ref, (const int*)out };
487 
488   __asm__ volatile(
489     HORIZONTAL_PASS(0, temp0,  temp1,  temp2,  temp3)
490     HORIZONTAL_PASS(1, temp4,  temp5,  temp6,  temp7)
491     HORIZONTAL_PASS(2, temp8,  temp9,  temp10, temp11)
492     HORIZONTAL_PASS(3, temp12, temp13, temp14, temp15)
493     "lw   %[temp20],    8(%[args])                     \n\t"
494     VERTICAL_PASS(0,  8, 16, 24, temp0, temp4, temp8,  temp12)
495     VERTICAL_PASS(2, 10, 18, 26, temp1, temp5, temp9,  temp13)
496     VERTICAL_PASS(4, 12, 20, 28, temp2, temp6, temp10, temp14)
497     VERTICAL_PASS(6, 14, 22, 30, temp3, temp7, temp11, temp15)
498 
499     : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
500       [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
501       [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
502       [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11),
503       [temp12]"=&r"(temp12), [temp13]"=&r"(temp13), [temp14]"=&r"(temp14),
504       [temp15]"=&r"(temp15), [temp16]"=&r"(temp16), [temp17]"=&r"(temp17),
505       [temp18]"=&r"(temp18), [temp19]"=&r"(temp19), [temp20]"=&r"(temp20)
506     : [args]"r"(args), [c2217]"r"(c2217), [c5352]"r"(c5352)
507     : "memory", "hi", "lo"
508   );
509 }
510 
511 #undef VERTICAL_PASS
512 #undef HORIZONTAL_PASS
513 
514 #if !defined(WORK_AROUND_GCC)
515 
516 #define GET_SSE_INNER(A, B, C, D)                               \
517   "lbu     %[temp0],    " #A "(%[a])                 \n\t"      \
518   "lbu     %[temp1],    " #A "(%[b])                 \n\t"      \
519   "lbu     %[temp2],    " #B "(%[a])                 \n\t"      \
520   "lbu     %[temp3],    " #B "(%[b])                 \n\t"      \
521   "lbu     %[temp4],    " #C "(%[a])                 \n\t"      \
522   "lbu     %[temp5],    " #C "(%[b])                 \n\t"      \
523   "lbu     %[temp6],    " #D "(%[a])                 \n\t"      \
524   "lbu     %[temp7],    " #D "(%[b])                 \n\t"      \
525   "subu    %[temp0],    %[temp0],     %[temp1]       \n\t"      \
526   "subu    %[temp2],    %[temp2],     %[temp3]       \n\t"      \
527   "subu    %[temp4],    %[temp4],     %[temp5]       \n\t"      \
528   "subu    %[temp6],    %[temp6],     %[temp7]       \n\t"      \
529   "madd    %[temp0],    %[temp0]                     \n\t"      \
530   "madd    %[temp2],    %[temp2]                     \n\t"      \
531   "madd    %[temp4],    %[temp4]                     \n\t"      \
532   "madd    %[temp6],    %[temp6]                     \n\t"
533 
534 #define GET_SSE(A, B, C, D)               \
535   GET_SSE_INNER(A, A + 1, A + 2, A + 3)   \
536   GET_SSE_INNER(B, B + 1, B + 2, B + 3)   \
537   GET_SSE_INNER(C, C + 1, C + 2, C + 3)   \
538   GET_SSE_INNER(D, D + 1, D + 2, D + 3)
539 
SSE16x16_MIPS32(const uint8_t * a,const uint8_t * b)540 static int SSE16x16_MIPS32(const uint8_t* a, const uint8_t* b) {
541   int count;
542   int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
543 
544   __asm__ volatile(
545      "mult   $zero,    $zero                            \n\t"
546 
547      GET_SSE( 0 * BPS, 4 +  0 * BPS, 8 +  0 * BPS, 12 +  0 * BPS)
548      GET_SSE( 1 * BPS, 4 +  1 * BPS, 8 +  1 * BPS, 12 +  1 * BPS)
549      GET_SSE( 2 * BPS, 4 +  2 * BPS, 8 +  2 * BPS, 12 +  2 * BPS)
550      GET_SSE( 3 * BPS, 4 +  3 * BPS, 8 +  3 * BPS, 12 +  3 * BPS)
551      GET_SSE( 4 * BPS, 4 +  4 * BPS, 8 +  4 * BPS, 12 +  4 * BPS)
552      GET_SSE( 5 * BPS, 4 +  5 * BPS, 8 +  5 * BPS, 12 +  5 * BPS)
553      GET_SSE( 6 * BPS, 4 +  6 * BPS, 8 +  6 * BPS, 12 +  6 * BPS)
554      GET_SSE( 7 * BPS, 4 +  7 * BPS, 8 +  7 * BPS, 12 +  7 * BPS)
555      GET_SSE( 8 * BPS, 4 +  8 * BPS, 8 +  8 * BPS, 12 +  8 * BPS)
556      GET_SSE( 9 * BPS, 4 +  9 * BPS, 8 +  9 * BPS, 12 +  9 * BPS)
557      GET_SSE(10 * BPS, 4 + 10 * BPS, 8 + 10 * BPS, 12 + 10 * BPS)
558      GET_SSE(11 * BPS, 4 + 11 * BPS, 8 + 11 * BPS, 12 + 11 * BPS)
559      GET_SSE(12 * BPS, 4 + 12 * BPS, 8 + 12 * BPS, 12 + 12 * BPS)
560      GET_SSE(13 * BPS, 4 + 13 * BPS, 8 + 13 * BPS, 12 + 13 * BPS)
561      GET_SSE(14 * BPS, 4 + 14 * BPS, 8 + 14 * BPS, 12 + 14 * BPS)
562      GET_SSE(15 * BPS, 4 + 15 * BPS, 8 + 15 * BPS, 12 + 15 * BPS)
563 
564     "mflo    %[count]                                   \n\t"
565     : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
566       [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
567       [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [count]"=&r"(count)
568     : [a]"r"(a), [b]"r"(b)
569     : "memory", "hi", "lo"
570   );
571   return count;
572 }
573 
SSE16x8_MIPS32(const uint8_t * a,const uint8_t * b)574 static int SSE16x8_MIPS32(const uint8_t* a, const uint8_t* b) {
575   int count;
576   int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
577 
578   __asm__ volatile(
579      "mult   $zero,    $zero                            \n\t"
580 
581      GET_SSE( 0 * BPS, 4 +  0 * BPS, 8 +  0 * BPS, 12 +  0 * BPS)
582      GET_SSE( 1 * BPS, 4 +  1 * BPS, 8 +  1 * BPS, 12 +  1 * BPS)
583      GET_SSE( 2 * BPS, 4 +  2 * BPS, 8 +  2 * BPS, 12 +  2 * BPS)
584      GET_SSE( 3 * BPS, 4 +  3 * BPS, 8 +  3 * BPS, 12 +  3 * BPS)
585      GET_SSE( 4 * BPS, 4 +  4 * BPS, 8 +  4 * BPS, 12 +  4 * BPS)
586      GET_SSE( 5 * BPS, 4 +  5 * BPS, 8 +  5 * BPS, 12 +  5 * BPS)
587      GET_SSE( 6 * BPS, 4 +  6 * BPS, 8 +  6 * BPS, 12 +  6 * BPS)
588      GET_SSE( 7 * BPS, 4 +  7 * BPS, 8 +  7 * BPS, 12 +  7 * BPS)
589 
590     "mflo    %[count]                                   \n\t"
591     : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
592       [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
593       [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [count]"=&r"(count)
594     : [a]"r"(a), [b]"r"(b)
595     : "memory", "hi", "lo"
596   );
597   return count;
598 }
599 
SSE8x8_MIPS32(const uint8_t * a,const uint8_t * b)600 static int SSE8x8_MIPS32(const uint8_t* a, const uint8_t* b) {
601   int count;
602   int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
603 
604   __asm__ volatile(
605      "mult   $zero,    $zero                            \n\t"
606 
607      GET_SSE(0 * BPS, 4 + 0 * BPS, 1 * BPS, 4 + 1 * BPS)
608      GET_SSE(2 * BPS, 4 + 2 * BPS, 3 * BPS, 4 + 3 * BPS)
609      GET_SSE(4 * BPS, 4 + 4 * BPS, 5 * BPS, 4 + 5 * BPS)
610      GET_SSE(6 * BPS, 4 + 6 * BPS, 7 * BPS, 4 + 7 * BPS)
611 
612     "mflo    %[count]                                   \n\t"
613     : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
614       [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
615       [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [count]"=&r"(count)
616     : [a]"r"(a), [b]"r"(b)
617     : "memory", "hi", "lo"
618   );
619   return count;
620 }
621 
SSE4x4_MIPS32(const uint8_t * a,const uint8_t * b)622 static int SSE4x4_MIPS32(const uint8_t* a, const uint8_t* b) {
623   int count;
624   int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
625 
626   __asm__ volatile(
627      "mult   $zero,    $zero                            \n\t"
628 
629      GET_SSE(0 * BPS, 1 * BPS, 2 * BPS, 3 * BPS)
630 
631     "mflo    %[count]                                   \n\t"
632     : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
633       [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
634       [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [count]"=&r"(count)
635     : [a]"r"(a), [b]"r"(b)
636     : "memory", "hi", "lo"
637   );
638   return count;
639 }
640 
641 #undef GET_SSE
642 #undef GET_SSE_INNER
643 
644 #endif  // !WORK_AROUND_GCC
645 
646 //------------------------------------------------------------------------------
647 // Entry point
648 
649 extern void VP8EncDspInitMIPS32(void);
650 
VP8EncDspInitMIPS32(void)651 WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitMIPS32(void) {
652   VP8ITransform = ITransform_MIPS32;
653   VP8FTransform = FTransform_MIPS32;
654 
655   VP8EncQuantizeBlock = QuantizeBlock_MIPS32;
656   VP8EncQuantize2Blocks = Quantize2Blocks_MIPS32;
657 
658   VP8TDisto4x4 = Disto4x4_MIPS32;
659   VP8TDisto16x16 = Disto16x16_MIPS32;
660 
661 #if !defined(WORK_AROUND_GCC)
662   VP8SSE16x16 = SSE16x16_MIPS32;
663   VP8SSE8x8 = SSE8x8_MIPS32;
664   VP8SSE16x8 = SSE16x8_MIPS32;
665   VP8SSE4x4 = SSE4x4_MIPS32;
666 #endif
667 }
668 
669 #else  // !WEBP_USE_MIPS32
670 
671 WEBP_DSP_INIT_STUB(VP8EncDspInitMIPS32)
672 
673 #endif  // WEBP_USE_MIPS32
674