1// Copyright 2022 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert TILE_HEIGHT & (TILE_HEIGHT-1) == 0 and TILE_HEIGHT != 0 7$assert TILE_WIDTH & (TILE_WIDTH-1) == 0 and TILE_WIDTH != 0 8$assert TILE_HEIGHT in [1, 2, 4] 9$assert TILE_WIDTH in [1, 2, 4] 10 11#include <assert.h> 12 13#include <xnnpack/common.h> 14#include <xnnpack/math.h> 15#include <xnnpack/transpose.h> 16 17void xnn_x24_transposec_ukernel__${TILE_HEIGHT}x${TILE_WIDTH}_scalar( 18 const void *input, 19 void * output, 20 size_t input_stride, 21 size_t output_stride, 22 size_t block_width, 23 size_t block_height) 24{ 25 assert(output_stride >= block_height * 3); 26 assert(input_stride >= block_width * 3); 27 28 $if TILE_HEIGHT == 1: 29 const size_t input_reset = ${TILE_WIDTH * 3} - block_height * input_stride; 30 const size_t output_reset = ${TILE_WIDTH} * output_stride - block_height * 3; 31 $else: 32 const size_t input_reset = ${TILE_WIDTH * 3} - round_down_po2(block_height, ${TILE_HEIGHT}) * input_stride; 33 const size_t output_reset = ${TILE_WIDTH} * output_stride - block_height * 3; 34 const size_t input_offset = ${TILE_HEIGHT} * input_stride; 35 36 const uint8_t* i0 = (const uint8_t*) input; 37 $for N in range(1, TILE_HEIGHT): 38 const uint8_t* i${N} = (const uint8_t*) ((uintptr_t) i${N-1} + input_stride); 39 40 uint8_t* o0 = (uint8_t*) output; 41 $for N in range(1, TILE_WIDTH): 42 uint8_t* o${N} = (uint8_t*) ((uintptr_t) o${N-1} + output_stride); 43 44 do { 45 $if TILE_WIDTH > 1: 46 if XNN_UNPREDICTABLE(block_width < 2) { 47 o1 = o0; 48 } 49 $for N in range(2, TILE_WIDTH, 2): 50 if XNN_UNPREDICTABLE(block_width <= ${N}) { 51 o${N} = o0; 52 } 53 if XNN_UNPREDICTABLE(block_width < ${N+2}) { 54 o${N+1} = o0; 55 } 56 size_t bh = block_height; 57 for (; bh >= ${TILE_HEIGHT}; bh -= ${TILE_HEIGHT}) { 58 $for M in reversed(range(TILE_WIDTH)): 59 $POS = 0 60 $for N in range(TILE_HEIGHT): 61 o${M}[${POS}] = i${N}[${M * 3}]; 62 o${M}[${POS + 1}] = i${N}[${M * 3 + 1}]; 63 o${M}[${POS + 2}] = i${N}[${M * 3 + 2}]; 64 $POS += 3 65 o${M} += ${POS}; 66 $for N in range(TILE_HEIGHT): 67 i${N} = (const uint8_t*) ((uintptr_t) i${N} + input_offset); 68 } 69 $if TILE_HEIGHT > 1: 70 const uint8_t* i = i0; 71 $if TILE_HEIGHT > 2: 72 if (bh & 2) { 73 $for M in reversed(range(TILE_WIDTH)): 74 o${M}[0] = i0[${M * 3}]; 75 o${M}[1] = i0[${M * 3 + 1}]; 76 o${M}[2] = i0[${M * 3 + 2}]; 77 o${M}[3] = i1[${M * 3}]; 78 o${M}[4] = i1[${M * 3 + 1}]; 79 o${M}[5] = i1[${M * 3 + 2}]; 80 o${M} += 6; 81 i = i2; 82 } 83 if (bh & 1) { 84 $for M in reversed(range(TILE_WIDTH)): 85 o${M}[0] = i[${M * 3}]; 86 o${M}[1] = i[${M * 3 + 1}]; 87 o${M}[2] = i[${M * 3 + 2}]; 88 o${M} += 3; 89 } 90 91 i0 = (const uint8_t*) ((uintptr_t) i0 + input_reset); 92 $for N in range(1, TILE_HEIGHT): 93 i${N} = (const uint8_t*) ((uintptr_t) i${N-1} + input_stride); 94 $for N in range(TILE_WIDTH): 95 o${N} = (uint8_t*) ((uintptr_t) o${N} + output_reset); 96 block_width = doz(block_width, ${TILE_WIDTH}); 97 } while (block_width != 0); 98} 99