1 // Copyright 2014 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // Utilities for processing transparent channel.
11 //
12 // Author: Skal ([email protected])
13
14 #include "src/dsp/dsp.h"
15
16 #if defined(WEBP_USE_SSE2)
17 #include <emmintrin.h>
18
19 //------------------------------------------------------------------------------
20
DispatchAlpha_SSE2(const uint8_t * WEBP_RESTRICT alpha,int alpha_stride,int width,int height,uint8_t * WEBP_RESTRICT dst,int dst_stride)21 static int DispatchAlpha_SSE2(const uint8_t* WEBP_RESTRICT alpha,
22 int alpha_stride, int width, int height,
23 uint8_t* WEBP_RESTRICT dst, int dst_stride) {
24 // alpha_and stores an 'and' operation of all the alpha[] values. The final
25 // value is not 0xff if any of the alpha[] is not equal to 0xff.
26 uint32_t alpha_and = 0xff;
27 int i, j;
28 const __m128i zero = _mm_setzero_si128();
29 const __m128i rgb_mask = _mm_set1_epi32((int)0xffffff00); // to preserve RGB
30 const __m128i all_0xff = _mm_set_epi32(0, 0, ~0, ~0);
31 __m128i all_alphas = all_0xff;
32
33 // We must be able to access 3 extra bytes after the last written byte
34 // 'dst[4 * width - 4]', because we don't know if alpha is the first or the
35 // last byte of the quadruplet.
36 const int limit = (width - 1) & ~7;
37
38 for (j = 0; j < height; ++j) {
39 __m128i* out = (__m128i*)dst;
40 for (i = 0; i < limit; i += 8) {
41 // load 8 alpha bytes
42 const __m128i a0 = _mm_loadl_epi64((const __m128i*)&alpha[i]);
43 const __m128i a1 = _mm_unpacklo_epi8(a0, zero);
44 const __m128i a2_lo = _mm_unpacklo_epi16(a1, zero);
45 const __m128i a2_hi = _mm_unpackhi_epi16(a1, zero);
46 // load 8 dst pixels (32 bytes)
47 const __m128i b0_lo = _mm_loadu_si128(out + 0);
48 const __m128i b0_hi = _mm_loadu_si128(out + 1);
49 // mask dst alpha values
50 const __m128i b1_lo = _mm_and_si128(b0_lo, rgb_mask);
51 const __m128i b1_hi = _mm_and_si128(b0_hi, rgb_mask);
52 // combine
53 const __m128i b2_lo = _mm_or_si128(b1_lo, a2_lo);
54 const __m128i b2_hi = _mm_or_si128(b1_hi, a2_hi);
55 // store
56 _mm_storeu_si128(out + 0, b2_lo);
57 _mm_storeu_si128(out + 1, b2_hi);
58 // accumulate eight alpha 'and' in parallel
59 all_alphas = _mm_and_si128(all_alphas, a0);
60 out += 2;
61 }
62 for (; i < width; ++i) {
63 const uint32_t alpha_value = alpha[i];
64 dst[4 * i] = alpha_value;
65 alpha_and &= alpha_value;
66 }
67 alpha += alpha_stride;
68 dst += dst_stride;
69 }
70 // Combine the eight alpha 'and' into a 8-bit mask.
71 alpha_and &= _mm_movemask_epi8(_mm_cmpeq_epi8(all_alphas, all_0xff));
72 return (alpha_and != 0xff);
73 }
74
DispatchAlphaToGreen_SSE2(const uint8_t * WEBP_RESTRICT alpha,int alpha_stride,int width,int height,uint32_t * WEBP_RESTRICT dst,int dst_stride)75 static void DispatchAlphaToGreen_SSE2(const uint8_t* WEBP_RESTRICT alpha,
76 int alpha_stride, int width, int height,
77 uint32_t* WEBP_RESTRICT dst,
78 int dst_stride) {
79 int i, j;
80 const __m128i zero = _mm_setzero_si128();
81 const int limit = width & ~15;
82 for (j = 0; j < height; ++j) {
83 for (i = 0; i < limit; i += 16) { // process 16 alpha bytes
84 const __m128i a0 = _mm_loadu_si128((const __m128i*)&alpha[i]);
85 const __m128i a1 = _mm_unpacklo_epi8(zero, a0); // note the 'zero' first!
86 const __m128i b1 = _mm_unpackhi_epi8(zero, a0);
87 const __m128i a2_lo = _mm_unpacklo_epi16(a1, zero);
88 const __m128i b2_lo = _mm_unpacklo_epi16(b1, zero);
89 const __m128i a2_hi = _mm_unpackhi_epi16(a1, zero);
90 const __m128i b2_hi = _mm_unpackhi_epi16(b1, zero);
91 _mm_storeu_si128((__m128i*)&dst[i + 0], a2_lo);
92 _mm_storeu_si128((__m128i*)&dst[i + 4], a2_hi);
93 _mm_storeu_si128((__m128i*)&dst[i + 8], b2_lo);
94 _mm_storeu_si128((__m128i*)&dst[i + 12], b2_hi);
95 }
96 for (; i < width; ++i) dst[i] = alpha[i] << 8;
97 alpha += alpha_stride;
98 dst += dst_stride;
99 }
100 }
101
ExtractAlpha_SSE2(const uint8_t * WEBP_RESTRICT argb,int argb_stride,int width,int height,uint8_t * WEBP_RESTRICT alpha,int alpha_stride)102 static int ExtractAlpha_SSE2(const uint8_t* WEBP_RESTRICT argb, int argb_stride,
103 int width, int height,
104 uint8_t* WEBP_RESTRICT alpha, int alpha_stride) {
105 // alpha_and stores an 'and' operation of all the alpha[] values. The final
106 // value is not 0xff if any of the alpha[] is not equal to 0xff.
107 uint32_t alpha_and = 0xff;
108 int i, j;
109 const __m128i a_mask = _mm_set1_epi32(0xff); // to preserve alpha
110 const __m128i all_0xff = _mm_set_epi32(0, 0, ~0, ~0);
111 __m128i all_alphas = all_0xff;
112
113 // We must be able to access 3 extra bytes after the last written byte
114 // 'src[4 * width - 4]', because we don't know if alpha is the first or the
115 // last byte of the quadruplet.
116 const int limit = (width - 1) & ~7;
117
118 for (j = 0; j < height; ++j) {
119 const __m128i* src = (const __m128i*)argb;
120 for (i = 0; i < limit; i += 8) {
121 // load 32 argb bytes
122 const __m128i a0 = _mm_loadu_si128(src + 0);
123 const __m128i a1 = _mm_loadu_si128(src + 1);
124 const __m128i b0 = _mm_and_si128(a0, a_mask);
125 const __m128i b1 = _mm_and_si128(a1, a_mask);
126 const __m128i c0 = _mm_packs_epi32(b0, b1);
127 const __m128i d0 = _mm_packus_epi16(c0, c0);
128 // store
129 _mm_storel_epi64((__m128i*)&alpha[i], d0);
130 // accumulate eight alpha 'and' in parallel
131 all_alphas = _mm_and_si128(all_alphas, d0);
132 src += 2;
133 }
134 for (; i < width; ++i) {
135 const uint32_t alpha_value = argb[4 * i];
136 alpha[i] = alpha_value;
137 alpha_and &= alpha_value;
138 }
139 argb += argb_stride;
140 alpha += alpha_stride;
141 }
142 // Combine the eight alpha 'and' into a 8-bit mask.
143 alpha_and &= _mm_movemask_epi8(_mm_cmpeq_epi8(all_alphas, all_0xff));
144 return (alpha_and == 0xff);
145 }
146
ExtractGreen_SSE2(const uint32_t * WEBP_RESTRICT argb,uint8_t * WEBP_RESTRICT alpha,int size)147 static void ExtractGreen_SSE2(const uint32_t* WEBP_RESTRICT argb,
148 uint8_t* WEBP_RESTRICT alpha, int size) {
149 int i;
150 const __m128i mask = _mm_set1_epi32(0xff);
151 const __m128i* src = (const __m128i*)argb;
152
153 for (i = 0; i + 16 <= size; i += 16, src += 4) {
154 const __m128i a0 = _mm_loadu_si128(src + 0);
155 const __m128i a1 = _mm_loadu_si128(src + 1);
156 const __m128i a2 = _mm_loadu_si128(src + 2);
157 const __m128i a3 = _mm_loadu_si128(src + 3);
158 const __m128i b0 = _mm_srli_epi32(a0, 8);
159 const __m128i b1 = _mm_srli_epi32(a1, 8);
160 const __m128i b2 = _mm_srli_epi32(a2, 8);
161 const __m128i b3 = _mm_srli_epi32(a3, 8);
162 const __m128i c0 = _mm_and_si128(b0, mask);
163 const __m128i c1 = _mm_and_si128(b1, mask);
164 const __m128i c2 = _mm_and_si128(b2, mask);
165 const __m128i c3 = _mm_and_si128(b3, mask);
166 const __m128i d0 = _mm_packs_epi32(c0, c1);
167 const __m128i d1 = _mm_packs_epi32(c2, c3);
168 const __m128i e = _mm_packus_epi16(d0, d1);
169 // store
170 _mm_storeu_si128((__m128i*)&alpha[i], e);
171 }
172 if (i + 8 <= size) {
173 const __m128i a0 = _mm_loadu_si128(src + 0);
174 const __m128i a1 = _mm_loadu_si128(src + 1);
175 const __m128i b0 = _mm_srli_epi32(a0, 8);
176 const __m128i b1 = _mm_srli_epi32(a1, 8);
177 const __m128i c0 = _mm_and_si128(b0, mask);
178 const __m128i c1 = _mm_and_si128(b1, mask);
179 const __m128i d = _mm_packs_epi32(c0, c1);
180 const __m128i e = _mm_packus_epi16(d, d);
181 _mm_storel_epi64((__m128i*)&alpha[i], e);
182 i += 8;
183 }
184 for (; i < size; ++i) alpha[i] = argb[i] >> 8;
185 }
186
187 //------------------------------------------------------------------------------
188 // Non-dither premultiplied modes
189
190 #define MULTIPLIER(a) ((a) * 0x8081)
191 #define PREMULTIPLY(x, m) (((x) * (m)) >> 23)
192
193 // We can't use a 'const int' for the SHUFFLE value, because it has to be an
194 // immediate in the _mm_shufflexx_epi16() instruction. We really need a macro.
195 // We use: v / 255 = (v * 0x8081) >> 23, where v = alpha * {r,g,b} is a 16bit
196 // value.
197 #define APPLY_ALPHA(RGBX, SHUFFLE) do { \
198 const __m128i argb0 = _mm_loadu_si128((const __m128i*)&(RGBX)); \
199 const __m128i argb1_lo = _mm_unpacklo_epi8(argb0, zero); \
200 const __m128i argb1_hi = _mm_unpackhi_epi8(argb0, zero); \
201 const __m128i alpha0_lo = _mm_or_si128(argb1_lo, kMask); \
202 const __m128i alpha0_hi = _mm_or_si128(argb1_hi, kMask); \
203 const __m128i alpha1_lo = _mm_shufflelo_epi16(alpha0_lo, SHUFFLE); \
204 const __m128i alpha1_hi = _mm_shufflelo_epi16(alpha0_hi, SHUFFLE); \
205 const __m128i alpha2_lo = _mm_shufflehi_epi16(alpha1_lo, SHUFFLE); \
206 const __m128i alpha2_hi = _mm_shufflehi_epi16(alpha1_hi, SHUFFLE); \
207 /* alpha2 = [ff a0 a0 a0][ff a1 a1 a1] */ \
208 const __m128i A0_lo = _mm_mullo_epi16(alpha2_lo, argb1_lo); \
209 const __m128i A0_hi = _mm_mullo_epi16(alpha2_hi, argb1_hi); \
210 const __m128i A1_lo = _mm_mulhi_epu16(A0_lo, kMult); \
211 const __m128i A1_hi = _mm_mulhi_epu16(A0_hi, kMult); \
212 const __m128i A2_lo = _mm_srli_epi16(A1_lo, 7); \
213 const __m128i A2_hi = _mm_srli_epi16(A1_hi, 7); \
214 const __m128i A3 = _mm_packus_epi16(A2_lo, A2_hi); \
215 _mm_storeu_si128((__m128i*)&(RGBX), A3); \
216 } while (0)
217
ApplyAlphaMultiply_SSE2(uint8_t * rgba,int alpha_first,int w,int h,int stride)218 static void ApplyAlphaMultiply_SSE2(uint8_t* rgba, int alpha_first,
219 int w, int h, int stride) {
220 const __m128i zero = _mm_setzero_si128();
221 const __m128i kMult = _mm_set1_epi16((short)0x8081);
222 const __m128i kMask = _mm_set_epi16(0, 0xff, 0xff, 0, 0, 0xff, 0xff, 0);
223 const int kSpan = 4;
224 while (h-- > 0) {
225 uint32_t* const rgbx = (uint32_t*)rgba;
226 int i;
227 if (!alpha_first) {
228 for (i = 0; i + kSpan <= w; i += kSpan) {
229 APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(2, 3, 3, 3));
230 }
231 } else {
232 for (i = 0; i + kSpan <= w; i += kSpan) {
233 APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 0, 0, 1));
234 }
235 }
236 // Finish with left-overs.
237 for (; i < w; ++i) {
238 uint8_t* const rgb = rgba + (alpha_first ? 1 : 0);
239 const uint8_t* const alpha = rgba + (alpha_first ? 0 : 3);
240 const uint32_t a = alpha[4 * i];
241 if (a != 0xff) {
242 const uint32_t mult = MULTIPLIER(a);
243 rgb[4 * i + 0] = PREMULTIPLY(rgb[4 * i + 0], mult);
244 rgb[4 * i + 1] = PREMULTIPLY(rgb[4 * i + 1], mult);
245 rgb[4 * i + 2] = PREMULTIPLY(rgb[4 * i + 2], mult);
246 }
247 }
248 rgba += stride;
249 }
250 }
251 #undef MULTIPLIER
252 #undef PREMULTIPLY
253
254 //------------------------------------------------------------------------------
255 // Alpha detection
256
HasAlpha8b_SSE2(const uint8_t * src,int length)257 static int HasAlpha8b_SSE2(const uint8_t* src, int length) {
258 const __m128i all_0xff = _mm_set1_epi8((char)0xff);
259 int i = 0;
260 for (; i + 16 <= length; i += 16) {
261 const __m128i v = _mm_loadu_si128((const __m128i*)(src + i));
262 const __m128i bits = _mm_cmpeq_epi8(v, all_0xff);
263 const int mask = _mm_movemask_epi8(bits);
264 if (mask != 0xffff) return 1;
265 }
266 for (; i < length; ++i) if (src[i] != 0xff) return 1;
267 return 0;
268 }
269
HasAlpha32b_SSE2(const uint8_t * src,int length)270 static int HasAlpha32b_SSE2(const uint8_t* src, int length) {
271 const __m128i alpha_mask = _mm_set1_epi32(0xff);
272 const __m128i all_0xff = _mm_set1_epi8((char)0xff);
273 int i = 0;
274 // We don't know if we can access the last 3 bytes after the last alpha
275 // value 'src[4 * length - 4]' (because we don't know if alpha is the first
276 // or the last byte of the quadruplet). Hence the '-3' protection below.
277 length = length * 4 - 3; // size in bytes
278 for (; i + 64 <= length; i += 64) {
279 const __m128i a0 = _mm_loadu_si128((const __m128i*)(src + i + 0));
280 const __m128i a1 = _mm_loadu_si128((const __m128i*)(src + i + 16));
281 const __m128i a2 = _mm_loadu_si128((const __m128i*)(src + i + 32));
282 const __m128i a3 = _mm_loadu_si128((const __m128i*)(src + i + 48));
283 const __m128i b0 = _mm_and_si128(a0, alpha_mask);
284 const __m128i b1 = _mm_and_si128(a1, alpha_mask);
285 const __m128i b2 = _mm_and_si128(a2, alpha_mask);
286 const __m128i b3 = _mm_and_si128(a3, alpha_mask);
287 const __m128i c0 = _mm_packs_epi32(b0, b1);
288 const __m128i c1 = _mm_packs_epi32(b2, b3);
289 const __m128i d = _mm_packus_epi16(c0, c1);
290 const __m128i bits = _mm_cmpeq_epi8(d, all_0xff);
291 const int mask = _mm_movemask_epi8(bits);
292 if (mask != 0xffff) return 1;
293 }
294 for (; i + 32 <= length; i += 32) {
295 const __m128i a0 = _mm_loadu_si128((const __m128i*)(src + i + 0));
296 const __m128i a1 = _mm_loadu_si128((const __m128i*)(src + i + 16));
297 const __m128i b0 = _mm_and_si128(a0, alpha_mask);
298 const __m128i b1 = _mm_and_si128(a1, alpha_mask);
299 const __m128i c = _mm_packs_epi32(b0, b1);
300 const __m128i d = _mm_packus_epi16(c, c);
301 const __m128i bits = _mm_cmpeq_epi8(d, all_0xff);
302 const int mask = _mm_movemask_epi8(bits);
303 if (mask != 0xffff) return 1;
304 }
305 for (; i <= length; i += 4) if (src[i] != 0xff) return 1;
306 return 0;
307 }
308
AlphaReplace_SSE2(uint32_t * src,int length,uint32_t color)309 static void AlphaReplace_SSE2(uint32_t* src, int length, uint32_t color) {
310 const __m128i m_color = _mm_set1_epi32((int)color);
311 const __m128i zero = _mm_setzero_si128();
312 int i = 0;
313 for (; i + 8 <= length; i += 8) {
314 const __m128i a0 = _mm_loadu_si128((const __m128i*)(src + i + 0));
315 const __m128i a1 = _mm_loadu_si128((const __m128i*)(src + i + 4));
316 const __m128i b0 = _mm_srai_epi32(a0, 24);
317 const __m128i b1 = _mm_srai_epi32(a1, 24);
318 const __m128i c0 = _mm_cmpeq_epi32(b0, zero);
319 const __m128i c1 = _mm_cmpeq_epi32(b1, zero);
320 const __m128i d0 = _mm_and_si128(c0, m_color);
321 const __m128i d1 = _mm_and_si128(c1, m_color);
322 const __m128i e0 = _mm_andnot_si128(c0, a0);
323 const __m128i e1 = _mm_andnot_si128(c1, a1);
324 _mm_storeu_si128((__m128i*)(src + i + 0), _mm_or_si128(d0, e0));
325 _mm_storeu_si128((__m128i*)(src + i + 4), _mm_or_si128(d1, e1));
326 }
327 for (; i < length; ++i) if ((src[i] >> 24) == 0) src[i] = color;
328 }
329
330 // -----------------------------------------------------------------------------
331 // Apply alpha value to rows
332
MultARGBRow_SSE2(uint32_t * const ptr,int width,int inverse)333 static void MultARGBRow_SSE2(uint32_t* const ptr, int width, int inverse) {
334 int x = 0;
335 if (!inverse) {
336 const int kSpan = 2;
337 const __m128i zero = _mm_setzero_si128();
338 const __m128i k128 = _mm_set1_epi16(128);
339 const __m128i kMult = _mm_set1_epi16(0x0101);
340 const __m128i kMask = _mm_set_epi16(0, 0xff, 0, 0, 0, 0xff, 0, 0);
341 for (x = 0; x + kSpan <= width; x += kSpan) {
342 // To compute 'result = (int)(a * x / 255. + .5)', we use:
343 // tmp = a * v + 128, result = (tmp * 0x0101u) >> 16
344 const __m128i A0 = _mm_loadl_epi64((const __m128i*)&ptr[x]);
345 const __m128i A1 = _mm_unpacklo_epi8(A0, zero);
346 const __m128i A2 = _mm_or_si128(A1, kMask);
347 const __m128i A3 = _mm_shufflelo_epi16(A2, _MM_SHUFFLE(2, 3, 3, 3));
348 const __m128i A4 = _mm_shufflehi_epi16(A3, _MM_SHUFFLE(2, 3, 3, 3));
349 // here, A4 = [ff a0 a0 a0][ff a1 a1 a1]
350 const __m128i A5 = _mm_mullo_epi16(A4, A1);
351 const __m128i A6 = _mm_add_epi16(A5, k128);
352 const __m128i A7 = _mm_mulhi_epu16(A6, kMult);
353 const __m128i A10 = _mm_packus_epi16(A7, zero);
354 _mm_storel_epi64((__m128i*)&ptr[x], A10);
355 }
356 }
357 width -= x;
358 if (width > 0) WebPMultARGBRow_C(ptr + x, width, inverse);
359 }
360
MultRow_SSE2(uint8_t * WEBP_RESTRICT const ptr,const uint8_t * WEBP_RESTRICT const alpha,int width,int inverse)361 static void MultRow_SSE2(uint8_t* WEBP_RESTRICT const ptr,
362 const uint8_t* WEBP_RESTRICT const alpha,
363 int width, int inverse) {
364 int x = 0;
365 if (!inverse) {
366 const __m128i zero = _mm_setzero_si128();
367 const __m128i k128 = _mm_set1_epi16(128);
368 const __m128i kMult = _mm_set1_epi16(0x0101);
369 for (x = 0; x + 8 <= width; x += 8) {
370 const __m128i v0 = _mm_loadl_epi64((__m128i*)&ptr[x]);
371 const __m128i a0 = _mm_loadl_epi64((const __m128i*)&alpha[x]);
372 const __m128i v1 = _mm_unpacklo_epi8(v0, zero);
373 const __m128i a1 = _mm_unpacklo_epi8(a0, zero);
374 const __m128i v2 = _mm_mullo_epi16(v1, a1);
375 const __m128i v3 = _mm_add_epi16(v2, k128);
376 const __m128i v4 = _mm_mulhi_epu16(v3, kMult);
377 const __m128i v5 = _mm_packus_epi16(v4, zero);
378 _mm_storel_epi64((__m128i*)&ptr[x], v5);
379 }
380 }
381 width -= x;
382 if (width > 0) WebPMultRow_C(ptr + x, alpha + x, width, inverse);
383 }
384
385 //------------------------------------------------------------------------------
386 // Entry point
387
388 extern void WebPInitAlphaProcessingSSE2(void);
389
WebPInitAlphaProcessingSSE2(void)390 WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingSSE2(void) {
391 WebPMultARGBRow = MultARGBRow_SSE2;
392 WebPMultRow = MultRow_SSE2;
393 WebPApplyAlphaMultiply = ApplyAlphaMultiply_SSE2;
394 WebPDispatchAlpha = DispatchAlpha_SSE2;
395 WebPDispatchAlphaToGreen = DispatchAlphaToGreen_SSE2;
396 WebPExtractAlpha = ExtractAlpha_SSE2;
397 WebPExtractGreen = ExtractGreen_SSE2;
398
399 WebPHasAlpha8b = HasAlpha8b_SSE2;
400 WebPHasAlpha32b = HasAlpha32b_SSE2;
401 WebPAlphaReplace = AlphaReplace_SSE2;
402 }
403
404 #else // !WEBP_USE_SSE2
405
406 WEBP_DSP_INIT_STUB(WebPInitAlphaProcessingSSE2)
407
408 #endif // WEBP_USE_SSE2
409