1// Copyright 2021 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert ((TILE_HEIGHT & (TILE_HEIGHT-1) == 0) and TILE_HEIGHT != 0) 7$assert ((TILE_WIDTH & (TILE_WIDTH-1) == 0) and TILE_WIDTH != 0) 8$assert SIZE in [8, 16, 32, 64] 9$assert TYPE in ["int8_t", "int16_t", "int", "float", "int64_t", "double"] 10$assert (TILE_WIDTH * SIZE <= 128) 11$SUFFIX = "float" if TYPE in ["float", "double"] else "int" 12 13#include <assert.h> 14 15#include <xnnpack/common.h> 16#include <xnnpack/math.h> 17#include <xnnpack/transpose.h> 18 19void xnn_x${SIZE}_transposec_ukernel__${TILE_HEIGHT}x${TILE_WIDTH}_scalar_${SUFFIX}( 20 const uint${SIZE}_t *input, 21 uint${SIZE}_t * output, 22 size_t input_stride, 23 size_t output_stride, 24 size_t block_width, 25 size_t block_height) XNN_OOB_READS 26{ 27 assert(output_stride >= block_height * sizeof(${TYPE})); 28 assert(input_stride >= block_width * sizeof(${TYPE})); 29 30 const size_t tile_height = ${TILE_HEIGHT}; 31 const size_t tile_width = ${TILE_WIDTH}; 32 const size_t tile_wbytes = tile_width * sizeof(${TYPE}); 33 $if TILE_HEIGHT == 1: 34 const size_t input_reset = tile_wbytes - block_height * input_stride; 35 const size_t output_reset = tile_width * output_stride - block_height * sizeof(${TYPE}); 36 $else: 37 const size_t input_reset = tile_wbytes - round_down_po2(block_height, tile_height) * input_stride; 38 const size_t output_reset = tile_width * output_stride - round_down_po2(block_height, 2) * sizeof(${TYPE}); 39 const size_t input_offset = tile_height * input_stride; 40 41 const ${TYPE}* i0 = (const ${TYPE}*) input; 42 $for N in range(1, TILE_HEIGHT): 43 const ${TYPE}* i${N} = (const ${TYPE}*) ((uintptr_t) i${N-1} + input_stride); 44 45 ${TYPE}* o0 = (${TYPE}*) output; 46 $for N in range(1, TILE_WIDTH): 47 ${TYPE}* o${N} = (${TYPE}*) ((uintptr_t) o${N-1} + output_stride); 48 49 do { 50 $if TILE_WIDTH > 1: 51 if XNN_UNPREDICTABLE(block_width < 2) { 52 o1 = o0; 53 } 54 $for N in range(2, TILE_WIDTH, 2): 55 if XNN_UNPREDICTABLE(block_width <= ${N}) { 56 o${N} = o0; 57 } 58 if XNN_UNPREDICTABLE(block_width < ${N+2}) { 59 o${N+1} = o0; 60 } 61 size_t bh = block_height; 62 for (; bh >= ${TILE_HEIGHT}; bh -= ${TILE_HEIGHT}) { 63 $for M in reversed(range(TILE_WIDTH)): 64 $for N in range(TILE_HEIGHT): 65 *o${M}++ = i${N}[${M}]; 66 $for N in range(TILE_HEIGHT): 67 i${N} = (const ${TYPE}*) ((uintptr_t) i${N} + input_offset); 68 } 69 $if TILE_HEIGHT > 2: 70 const ${TYPE}* i = i0; 71 if (bh & 2) { 72 $for M in reversed(range(TILE_WIDTH)): 73 o${M}[0] = i0[${M}]; 74 o${M}[1] = i1[${M}]; 75 o${M} += 2; 76 i = i2; 77 } 78 if (bh & 1) { 79 $for M in reversed(range(TILE_WIDTH)): 80 o${M}[0] = i[${M}]; 81 } 82 $elif TILE_HEIGHT > 1: 83 if (bh & 1) { 84 $for M in reversed(range(TILE_WIDTH)): 85 o${M}[0] = i0[${M}]; 86 } 87 88 i0 = (const ${TYPE}*) ((uintptr_t) i0 + input_reset); 89 $for N in range(1, TILE_HEIGHT): 90 i${N} = (const ${TYPE}*) ((uintptr_t) i${N-1} + input_stride); 91 $for N in range(TILE_WIDTH): 92 o${N} = (${TYPE}*) ((uintptr_t) o${N} + output_reset); 93 block_width = doz(block_width, tile_width); 94 } while (block_width != 0); 95} 96