xref: /aosp_15_r20/external/mesa3d/src/compiler/glsl/astc_decoder.glsl (revision 6104692788411f58d303aa86923a9ff6ecaded22)

#version 320 es
precision highp float;
precision highp int;
precision highp usamplerBuffer;
precision highp usampler2D;
precision highp image2D;
precision highp uimage2D;

/* Copyright (c) 2020-2022 Hans-Kristian Arntzen
 * Copyright (c) 2022 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#ifdef VULKAN

precision highp utextureBuffer;
precision highp utexture2DArray;
precision highp uimage2DArray;
precision highp uimage3D;
precision highp utexture3D;

#extension GL_EXT_samplerless_texture_functions : require
layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z = 4) in;

layout(set = 0, binding = 0) writeonly uniform uimage2DArray OutputImage2Darray;
layout(set = 0, binding = 0) writeonly uniform uimage3D OutputImage3D;
layout(set = 0, binding = 1) uniform utexture2DArray PayloadInput2Darray;
layout(set = 0, binding = 1) uniform utexture3D PayloadInput3D;
layout(set = 0, binding = 2) uniform utextureBuffer LUTRemainingBitsToEndpointQuantizer;
layout(set = 0, binding = 3) uniform utextureBuffer LUTEndpointUnquantize;
layout(set = 0, binding = 4) uniform utextureBuffer LUTWeightQuantizer;
layout(set = 0, binding = 5) uniform utextureBuffer LUTWeightUnquantize;
layout(set = 0, binding = 6) uniform utextureBuffer LUTTritQuintDecode;
layout(set = 0, binding = 7) uniform utextureBuffer LUTPartitionTable;

layout(constant_id = 2) const bool DECODE_8BIT = false;

layout(push_constant, std430) uniform pc {
   ivec2 texel_blk_start;
   ivec2 texel_end;
   bool is_3Dimage;
};

#else /* VULKAN */

layout(local_size_x = %u, local_size_y = %u, local_size_z = 4) in;

#define utextureBuffer usamplerBuffer
#define utexture2D usampler2D

layout(binding = 0) uniform utextureBuffer LUTRemainingBitsToEndpointQuantizer;
layout(binding = 1) uniform utextureBuffer LUTEndpointUnquantize;
layout(binding = 2) uniform utextureBuffer LUTWeightQuantizer;
layout(binding = 3) uniform utextureBuffer LUTWeightUnquantize;
layout(binding = 4) uniform utextureBuffer LUTTritQuintDecode;
layout(binding = 5) uniform utexture2D LUTPartitionTable;
layout(binding = 6) uniform utexture2D PayloadInput;

layout(rgba8ui, binding = 7) writeonly uniform uimage2D OutputImage;
const bool DECODE_8BIT = true;

#endif /* VULKAN */

const int MODE_LDR = 0;
const int MODE_HDR = 1;
const int MODE_HDR_LDR_ALPHA = 2;

const uvec4 error_color = uvec4(255, 0, 255, 255);

/* bitextract.h */
int extract_bits(uvec4 payload, int offset, int bits)
{
        int last_offset = offset + bits - 1;
        int result;

        if (bits <= 0)
                result = 0;
        else if ((last_offset >> 5) == (offset >> 5))
                result = int(bitfieldExtract(payload[offset >> 5], offset & 31, bits));
        else
        {
                int first_bits = 32 - (offset & 31);
                int result_first = int(bitfieldExtract(payload[offset >> 5], offset & 31, first_bits));
                int result_second = int(bitfieldExtract(payload[(offset >> 5) + 1], 0, bits - first_bits));
                result = result_first | (result_second << first_bits);
        }
        return result;
}

/* bitextract.h */
int extract_bits_sign(uvec4 payload, int offset, int bits)
{
        int last_offset = offset + bits - 1;
        int result;

        if (bits <= 0)
                result = 0;
        else if ((last_offset >> 5) == (offset >> 5))
                result = bitfieldExtract(int(payload[offset >> 5]), offset & 31, bits);
        else
        {
                int first_bits = 32 - (offset & 31);
                int result_first = int(bitfieldExtract(payload[offset >> 5], offset & 31, first_bits));
                int result_second = bitfieldExtract(int(payload[(offset >> 5) + 1]), 0, bits - first_bits);
                result = result_first | (result_second << first_bits);
        }
        return result;
}

/* bitextract.h */
int extract_bits_reverse(uvec4 payload, int offset, int bits)
{
        int last_offset = offset + bits - 1;
        int result;

        if (bits <= 0)
                result = 0;
        else if ((last_offset >> 5) == (offset >> 5))
                result = int(bitfieldReverse(bitfieldExtract(payload[offset >> 5], offset & 31, bits)) >> (32 - bits));
        else
        {
                int first_bits = 32 - (offset & 31);
                uint result_first = bitfieldExtract(payload[offset >> 5], offset & 31, first_bits);
                uint result_second = bitfieldExtract(payload[(offset >> 5) + 1], 0, bits - first_bits);
                result = int(bitfieldReverse(result_first | (result_second << first_bits)) >> (32 - bits));
        }
        return result;
}

void swap(inout int a, inout int b)
{
    int tmp = a;
    a = b;
    b = tmp;
}

ivec4 build_coord()
{
    ivec2 payload_coord = ivec2(gl_WorkGroupID.xy) * 2;
    payload_coord.x += int(gl_LocalInvocationID.z) & 1;
    payload_coord.y += (int(gl_LocalInvocationID.z) >> 1) & 1;
#ifdef VULKAN
    payload_coord += texel_blk_start;
#endif /* VULKAN */
    ivec2 coord = payload_coord * ivec2(gl_WorkGroupSize.xy);
    coord += ivec2(gl_LocalInvocationID.xy);
    return ivec4(coord, payload_coord);
}

ivec4 interpolate_endpoint(ivec4 ep0, ivec4 ep1, ivec4 weight, int decode_mode)
{
    if (decode_mode == MODE_HDR)
    {
        ep0 <<= 4;
        ep1 <<= 4;
    }
    else if (decode_mode == MODE_HDR_LDR_ALPHA)
    {
        ep0.rgb <<= 4;
        ep1.rgb <<= 4;
        ep0.a *= 0x101;
        ep1.a *= 0x101;
    }
    else if (DECODE_8BIT)
    {
        // This isn't quite right in all cases.
        // In normal ASTC with sRGB, the alpha channel is supposed to
        // be decoded as FP16,
        // even when color components are SRGB 8-bit (?!?!?!?!).
        // This is correct if decode_unorm8 mode is used though,
        // for sanity, we're going to assume unorm8 decoding mode
        // is implied when using sRGB.
        ep0 = (ep0 << 8) | ivec4(0x80);
        ep1 = (ep1 << 8) | ivec4(0x80);
    }
    else
    {
        ep0 *= 0x101;
        ep1 *= 0x101;
    }

    ivec4 color = (ep0 * (64 - weight) + ep1 * weight + 32) >> 6;
    return color;
}

bvec4 bvec_or(bvec4 a, bvec4 b)
{
    return bvec4(ivec4(a) | ivec4(b));
}

uint round_down_quantize_fp16(int color)
{
    // ASTC has a very peculiar way of converting the decoded result to FP16.
    // 0xffff -> 1.0, and for everything else we get roundDownQuantizeFP16(vec4(c) / vec4(0x10000)).
    int msb = findMSB(color);
    int shamt = msb;
    int m = ((color << 10) >> shamt) & 0x3ff;
    int e = msb - 1;
    uint decoded = color == 0xffff ? 0x3c00u : uint(e < 1 ? (color << 8) : (m | (e << 10)));
    return decoded;
}

uvec4 round_down_quantize_fp16(ivec4 color)
{
    // ASTC has a very peculiar way of converting the decoded result to FP16.
    // 0xffff -> 1.0, and for everything else we get roundDownQuantizeFP16(vec4(c) / vec4(0x10000)).
    ivec4 msb = findMSB(color);
    ivec4 shamt = msb;
    ivec4 m = ((color << 10) >> shamt) & 0x3ff;
    ivec4 e = msb - 1;
    uvec4 decoded = uvec4(m | (e << 10));
    uvec4 denorm_decode = uvec4(color << 8);
    decoded = mix(decoded, uvec4(denorm_decode), lessThan(e, ivec4(1)));
    decoded = mix(decoded, uvec4(0x3c00), equal(color, ivec4(0xffff)));
    return decoded;
}

uvec4 decode_fp16(ivec4 color, int decode_mode)
{
    if (decode_mode != MODE_LDR)
    {
        // Interpret the value as FP16, but with some extra fixups along the way to make the interpolation more
        // logarithmic (apparently). From spec:
        ivec4 e = color >> 11;
        ivec4 m = color & 0x7ff;
        ivec4 mt = 4 * m - 512;
        mt = mix(mt, ivec4(3 * m), lessThan(m, ivec4(512)));
        mt = mix(mt, ivec4(5 * m - 2048), greaterThanEqual(m, ivec4(1536)));

        ivec4 decoded = (e << 10) + (mt >> 3);
        // +Inf or NaN are decoded to 0x7bff (max finite value).
        decoded = mix(decoded, ivec4(0x7bff), bvec_or(greaterThan(decoded & 0x7fff, ivec4(0x7c00)), equal(decoded, ivec4(0x7c00))));

        if (decode_mode == MODE_HDR_LDR_ALPHA)
            decoded.a = int(round_down_quantize_fp16(color.a));

        return uvec4(decoded);
    }
    else
    {
        return round_down_quantize_fp16(color);
    }
}

struct BlockMode
{
    ivec2 weight_grid_size;
    int weight_mode_index;
    int num_partitions;
    int seed;
    int cem;
    int config_bits;
    int primary_config_bits;
    bool dual_plane;
    bool void_extent;
};

bool decode_error = false;

BlockMode decode_block_mode(uvec4 payload)
{
    BlockMode mode;
    mode.void_extent = (payload.x & 0x1ffu) == 0x1fcu;
    if (mode.void_extent)
        return mode;

    mode.dual_plane = (payload.x & (1u << 10u)) != 0u;

    uint higher = (payload.x >> 2u) & 3u;
    uint lower = payload.x & 3u;

    if (lower != 0u)
    {
        mode.weight_mode_index = int((payload.x >> 4u) & 1u);
        mode.weight_mode_index |= int((payload.x << 1u) & 6u);
        mode.weight_mode_index |= int((payload.x >> 6u) & 8u);

        if (higher < 2u)
        {
            mode.weight_grid_size.x = int(bitfieldExtract(payload.x, 7, 2) + 4u + 4u * higher);
            mode.weight_grid_size.y = int(bitfieldExtract(payload.x, 5, 2) + 2u);
        }
        else if (higher == 2u)
        {
            mode.weight_grid_size.x = int(bitfieldExtract(payload.x, 5, 2) + 2u);
            mode.weight_grid_size.y = int(bitfieldExtract(payload.x, 7, 2) + 8u);
        }
        else
        {
            if ((payload.x & (1u << 8u)) != 0u)
            {
                mode.weight_grid_size.x = int(bitfieldExtract(payload.x, 7, 1) + 2u);
                mode.weight_grid_size.y = int(bitfieldExtract(payload.x, 5, 2) + 2u);
            }
            else
            {
                mode.weight_grid_size.x = int(bitfieldExtract(payload.x, 5, 2) + 2u);
                mode.weight_grid_size.y = int(bitfieldExtract(payload.x, 7, 1) + 6u);
            }
        }
    }
    else
    {
        int p3 = int(bitfieldExtract(payload.x, 9, 1));
        int hi = int(bitfieldExtract(payload.x, 7, 2));
        int lo = int(bitfieldExtract(payload.x, 5, 2));
        if (hi == 0)
        {
            mode.weight_grid_size.x = 12;
            mode.weight_grid_size.y = lo + 2;
        }
        else if (hi == 1)
        {
            mode.weight_grid_size.x = lo + 2;
            mode.weight_grid_size.y = 12;
        }
        else if (hi == 2)
        {
            mode.dual_plane = false;
            p3 = 0;
            mode.weight_grid_size.x = lo + 6;
            mode.weight_grid_size.y = int(bitfieldExtract(payload.x, 9, 2) + 6u);
        }
        else
        {
            if (lo == 0)
                mode.weight_grid_size = ivec2(6, 10);
            else if (lo == 1)
                mode.weight_grid_size = ivec2(10, 6);
            else
                decode_error = true;
        }

        int p0 = int(bitfieldExtract(payload.x, 4, 1));
        int p1 = int(bitfieldExtract(payload.x, 2, 1));
        int p2 = int(bitfieldExtract(payload.x, 3, 1));
        mode.weight_mode_index = p0 + (p1 << 1) + (p2 << 2) + (p3 << 3);
    }

    // 11 bits for block mode.
    // 2 bits for partition select
    // If partitions > 1:
    //   4 bits CEM selector
    //   If dual_plane:
    //     2 bits of CCS
    // else:
    //   10 for partition seed
    //   2 bits for CEM main selector
    //   If CEM[1:0] = 00:
    //     4 bits for CEM extra selector if all same type.
    //   else:
    //     (1 + 2) * num_partitions if different types.
    //     First 4 bits are encoded next to CEM[1:0], otherwise, packed before weights.
    //   If dual_plane:
    //     2 bits of CCS before extra CEM bits.
    const int CONFIG_BITS_BLOCK = 11;
    const int CONFIG_BITS_PARTITION_MODE = 2;
    const int CONFIG_BITS_SEED = 10;
    const int CONFIG_BITS_PRIMARY_MULTI_CEM = 2;
    const int CONFIG_BITS_CEM = 4;
    const int CONFIG_BITS_EXTRA_CEM_PER_PARTITION = 3;
    const int CONFIG_BITS_CCS = 2;

    mode.num_partitions = int(bitfieldExtract(payload.x, CONFIG_BITS_BLOCK, CONFIG_BITS_PARTITION_MODE)) + 1;

    if (mode.num_partitions > 1)
    {
        mode.seed = int(bitfieldExtract(payload.x, CONFIG_BITS_BLOCK + CONFIG_BITS_PARTITION_MODE, CONFIG_BITS_SEED));
        mode.cem = int(bitfieldExtract(payload.x, CONFIG_BITS_BLOCK + CONFIG_BITS_PARTITION_MODE + CONFIG_BITS_SEED,
                                       CONFIG_BITS_PRIMARY_MULTI_CEM + CONFIG_BITS_CEM));
    }
    else
        mode.cem = int(bitfieldExtract(payload.x, CONFIG_BITS_BLOCK + CONFIG_BITS_PARTITION_MODE, CONFIG_BITS_CEM));

    int config_bits;
    if (mode.num_partitions > 1)
    {
        bool single_cem = (mode.cem & 3) == 0;
        if (single_cem)
        {
            config_bits = CONFIG_BITS_BLOCK + CONFIG_BITS_PARTITION_MODE +
                          CONFIG_BITS_SEED + CONFIG_BITS_PRIMARY_MULTI_CEM + CONFIG_BITS_CEM;
        }
        else
        {
            config_bits = CONFIG_BITS_BLOCK + CONFIG_BITS_PARTITION_MODE +
                          CONFIG_BITS_SEED + CONFIG_BITS_PRIMARY_MULTI_CEM +
                          CONFIG_BITS_EXTRA_CEM_PER_PARTITION * mode.num_partitions;
        }
    }
    else
    {
        config_bits = CONFIG_BITS_BLOCK + CONFIG_BITS_PARTITION_MODE + CONFIG_BITS_CEM;
    }

    // Other config bits are packed before the weights.
    int primary_config_bits;
    if (mode.num_partitions > 1)
    {
        primary_config_bits = CONFIG_BITS_BLOCK + CONFIG_BITS_PARTITION_MODE + CONFIG_BITS_SEED +
                              CONFIG_BITS_PRIMARY_MULTI_CEM + CONFIG_BITS_CEM;
    }
    else
        primary_config_bits = config_bits;

    if (mode.dual_plane)
        config_bits += CONFIG_BITS_CCS;

    // This is not allowed.
    if (any(greaterThan(mode.weight_grid_size, ivec2(gl_WorkGroupSize.xy))))
        decode_error = true;
    if (mode.dual_plane && mode.num_partitions > 3)
        decode_error = true;

    mode.config_bits = config_bits;
    mode.primary_config_bits = primary_config_bits;
    return mode;
}

int idiv3_floor(int v)
{
    return (v * 0x5556) >> 16;
}

int idiv3_ceil(int v)
{
    return idiv3_floor(v + 2);
}

int idiv5_floor(int v)
{
    return (v * 0x3334) >> 16;
}

int idiv5_ceil(int v)
{
    return idiv5_floor(v + 4);
}

uvec4 build_bitmask(int bits)
{
    ivec4 num_bits = ivec4(bits, bits - 32, bits - 64, bits - 96);
    uvec4 mask = uvec4(1) << clamp(num_bits, ivec4(0), ivec4(31));
    mask--;
    mask = mix(mask, uvec4(0xffffffffu), greaterThanEqual(uvec4(bits), uvec4(32, 64, 96, 128)));
    return mask;
}

int decode_integer_sequence(uvec4 payload, int start_bit, int index, ivec3 quant)
{
    int ret;
    if (quant.y != 0)
    {
        // Trit-decoding.
        int block = idiv5_floor(index);
        int offset = index - block * 5;
        start_bit += block * (5 * quant.x + 8);

        int t0_t1_offset = start_bit + (quant.x * 1 + 0);
        int t2_t3_offset = start_bit + (quant.x * 2 + 2);
        int t4_offset    = start_bit + (quant.x * 3 + 4);
        int t5_t6_offset = start_bit + (quant.x * 4 + 5);
        int t7_offset    = start_bit + (quant.x * 5 + 7);

        int t = (extract_bits(payload, t0_t1_offset, 2) << 0) |
                (extract_bits(payload, t2_t3_offset, 2) << 2) |
                (extract_bits(payload, t4_offset, 1) << 4) |
                (extract_bits(payload, t5_t6_offset, 2) << 5) |
                (extract_bits(payload, t7_offset, 1) << 7);

        t = int(texelFetch(LUTTritQuintDecode, t).x);
        t = (t >> (3 * offset)) & 7;

        int m_offset = offset * quant.x;
        m_offset += idiv5_ceil(offset * 8);

        if (quant.x != 0)
        {
            int m = extract_bits(payload, m_offset + start_bit, quant.x);
            ret = (t << quant.x) | m;
        }
        else
            ret = t;
    }
    else if (quant.z != 0)
    {
        // Quint-decoding
        int block = idiv3_floor(index);
        int offset = index - block * 3;
        start_bit += block * (3 * quant.x + 7);

        int q0_q1_q2_offset = start_bit + (quant.x * 1 + 0);
        int q3_q4_offset    = start_bit + (quant.x * 2 + 3);
        int q5_q6_offset    = start_bit + (quant.x * 3 + 5);

        int q = (extract_bits(payload, q0_q1_q2_offset, 3) << 0) |
                (extract_bits(payload, q3_q4_offset, 2) << 3) |
                (extract_bits(payload, q5_q6_offset, 2) << 5);

        q = int(texelFetch(LUTTritQuintDecode, 256 + q).x);
        q = (q >> (3 * offset)) & 7;

        int m_offset = offset * quant.x;
        m_offset += idiv3_ceil(offset * 7);

        if (quant.x != 0)
        {
            int m = extract_bits(payload, m_offset + start_bit, quant.x);
            ret = (q << quant.x) | m;
        }
        else
            ret = q;
    }
    else
    {
        int bit = index * quant.x;
        ret = extract_bits(payload, start_bit + bit, quant.x);
    }
    return ret;
}

ivec2 normalize_coord(ivec2 pixel_coord)
{
    ivec2 D = ivec2((vec2((1024 + ivec2(gl_WorkGroupSize.xy >> 1u))) + 0.5) / vec2(gl_WorkGroupSize.xy - 1u));
    ivec2 c = D * pixel_coord;
    return c;
}

int decode_weight(uvec4 payload, int weight_index, ivec4 quant)
{
    int primary_weight = decode_integer_sequence(payload, 0, weight_index, quant.xyz);
    primary_weight = int(texelFetch(LUTWeightUnquantize, primary_weight + quant.w).x);
    return primary_weight;
}

int decode_weight_bilinear(uvec4 payload, ivec2 coord, int weight_resolution,
                           int stride, int offset, ivec2 fractional, ivec4 quant)
{
    int index = coord.y * weight_resolution + coord.x;
    int p00 = decode_weight(payload, stride * index + offset, quant);
    int p10, p01, p11;

    if (fractional.x != 0)
        p10 = decode_weight(payload, stride * (index + 1) + offset, quant);
    else
        p10 = p00;

    if (fractional.y != 0)
    {
        p01 = decode_weight(payload, stride * (index + weight_resolution) + offset, quant);
        if (fractional.x != 0)
            p11 = decode_weight(payload, stride * (index + weight_resolution + 1) + offset, quant);
        else
            p11 = p01;
    }
    else
    {
        p01 = p00;
        p11 = p10;
    }

    int w11 = (fractional.x * fractional.y + 8) >> 4;
    int w10 = fractional.x - w11;
    int w01 = fractional.y - w11;
    int w00 = 16 - fractional.x - fractional.y + w11;
    return (p00 * w00 + p10 * w10 + p01 * w01 + p11 * w11 + 8) >> 4;
}

ivec4 decode_weights(uvec4 payload, BlockMode mode, ivec2 normalized_pixel, out int weight_cost_bits)
{
    ivec4 quant = ivec4(texelFetch(LUTWeightQuantizer, mode.weight_mode_index));
    int num_weights = mode.weight_grid_size.x * mode.weight_grid_size.y;
    num_weights <<= int(mode.dual_plane);
    weight_cost_bits =
        quant.x * num_weights +
        idiv5_ceil(num_weights * 8 * quant.y) +
        idiv3_ceil(num_weights * 7 * quant.z);

    // Decoders must deal with error conditions and return the correct error color.
    if (weight_cost_bits < 24 || weight_cost_bits > 96 || num_weights > 64)
    {
        decode_error = true;
        return ivec4(0);
    }

    int ccs;
    if (mode.dual_plane)
    {
        int extra_cem_bits = 0;
        if ((mode.cem & 3) != 0)
            extra_cem_bits = max(mode.num_partitions * 3 - 4, 0);
        ccs = extract_bits(payload, 126 - weight_cost_bits - extra_cem_bits, 2);
    }

    payload = bitfieldReverse(payload);
    payload = payload.wzyx;
    payload &= build_bitmask(weight_cost_bits);

    // Scale the normalized coordinate to weight grid.
    ivec2 weight_pixel_fixed_point = (normalized_pixel * (mode.weight_grid_size - 1) + 32) >> 6;
    ivec2 weight_pixel = weight_pixel_fixed_point >> 4;
    ivec2 weight_pixel_fractional = weight_pixel_fixed_point & 0xf;

    ivec4 ret;
    int primary_weight = decode_weight_bilinear(payload, weight_pixel, mode.weight_grid_size.x,
                                                1 << int(mode.dual_plane), 0,
                                                weight_pixel_fractional, quant);
    if (mode.dual_plane)
    {
        int secondary_weight = decode_weight_bilinear(payload, weight_pixel, mode.weight_grid_size.x,
                                                      2, 1,
                                                      weight_pixel_fractional, quant);
        ret = mix(ivec4(primary_weight), ivec4(secondary_weight), equal(ivec4(ccs), ivec4(0, 1, 2, 3)));
    }
    else
        ret = ivec4(primary_weight);

    return ret;
}

void decode_endpoint_ldr_luma_direct(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1)
{
    ep0 = ivec4(ivec3(v0), 0xff);
    ep1 = ivec4(ivec3(v1), 0xff);
}

void decode_endpoint_hdr_luma_direct(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1)
{
    int y0, y1;
    if (v1 >= v0)
    {
        y0 = v0 << 4;
        y1 = v1 << 4;
    }
    else
    {
        y0 = (v1 << 4) + 8;
        y1 = (v0 << 4) - 8;
    }

    ep0 = ivec4(ivec3(y0), 0x780);
    ep1 = ivec4(ivec3(y1), 0x780);
}

void decode_endpoint_hdr_luma_direct_small_range(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1)
{
    int y0, y1, d;

    if ((v0 & 0x80) != 0)
    {
        y0 = ((v1 & 0xe0) << 4) | ((v0 & 0x7f) << 2);
        d = (v1 & 0x1f) << 2;
    }
    else
    {
        y0 = ((v1 & 0xf0) << 4) | ((v0 & 0x7f) << 1);
        d = (v1 & 0x0f)  << 1;
    }

    y1 = min(y0 + d, 0xfff);

    ep0 = ivec4(ivec3(y0), 0x780);
    ep1 = ivec4(ivec3(y1), 0x780);
}

void decode_endpoint_ldr_luma_base_offset(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1)
{
    int l0 = (v0 >> 2) | (v1 & 0xc0);
    int l1 = l0 + (v1 & 0x3f);
    l1 = min(l1, 0xff);
    ep0 = ivec4(ivec3(l0), 0xff);
    ep1 = ivec4(ivec3(l1), 0xff);
}

void decode_endpoint_ldr_luma_alpha_direct(out ivec4 ep0, out ivec4 ep1,
    int v0, int v1, int v2, int v3)
{
    ep0 = ivec4(ivec3(v0), v2);
    ep1 = ivec4(ivec3(v1), v3);
}

ivec4 blue_contract(int r, int g, int b, int a)
{
    ivec4 ret;
    ret.r = (r + b) >> 1;
    ret.g = (g + b) >> 1;
    ret.b = b;
    ret.a = a;
    return ret;
}

void bit_transfer_signed(inout int a, inout int b)
{
    b >>= 1;
    b |= a & 0x80;
    a >>= 1;
    a &= 0x3f;
    a = bitfieldExtract(a, 0, 6);
}

void decode_endpoint_ldr_luma_alpha_base_offset(out ivec4 ep0, out ivec4 ep1,
    int v0, int v1, int v2, int v3)
{
    bit_transfer_signed(v1, v0);
    bit_transfer_signed(v3, v2);
    int v0_v1 = clamp(v0 + v1, 0, 0xff);
    int v2_v3 = clamp(v2 + v3, 0, 0xff);
    v0 = clamp(v0, 0, 0xff);
    v2 = clamp(v2, 0, 0xff);
    ep0 = ivec4(ivec3(v0), v2);
    ep1 = ivec4(ivec3(v0_v1), v2_v3);
}

void decode_endpoint_ldr_rgb_base_scale(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1, int v2, int v3)
{
    ep0 = ivec4((ivec3(v0, v1, v2) * v3) >> 8, 0xff);
    ep1 = ivec4(v0, v1, v2, 0xff);
}

void decode_endpoint_ldr_rgb_base_scale_two_a(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1, int v2, int v3, int v4, int v5)
{
    ep0 = ivec4((ivec3(v0, v1, v2) * v3) >> 8, v4);
    ep1 = ivec4(v0, v1, v2, v5);
}

void decode_endpoint_ldr_rgb_direct(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1, int v2, int v3, int v4, int v5)
{
    int s0 = v0 + v2 + v4;
    int s1 = v1 + v3 + v5;
    if (s1 >= s0)
    {
        ep0 = ivec4(v0, v2, v4, 0xff);
        ep1 = ivec4(v1, v3, v5, 0xff);
    }
    else
    {
        ep0 = blue_contract(v1, v3, v5, 0xff);
        ep1 = blue_contract(v0, v2, v4, 0xff);
    }
}

void decode_endpoint_hdr_rgb_scale(out ivec4 ep0, out ivec4 ep1,
    int v0, int v1, int v2, int v3)
{
    // Mind-numbing weird format, just copy from spec ...
    int mode_value = ((v0 & 0xc0) >> 6) | ((v1 & 0x80) >> 5) | ((v2 & 0x80) >> 4);
    int major_component;
    int mode;

    if ((mode_value & 0xc) != 0xc)
    {
        major_component = mode_value >> 2;
        mode = mode_value & 3;
    }
    else if (mode_value != 0xf)
    {
        major_component = mode_value & 3;
        mode = 4;
    }
    else
    {
        major_component = 0;
        mode = 5;
    }

    int red = v0 & 0x3f;
    int green = v1 & 0x1f;
    int blue = v2 & 0x1f;
    int scale = v3 & 0x1f;

    int x0 = (v1 >> 6) & 1;
    int x1 = (v1 >> 5) & 1;
    int x2 = (v2 >> 6) & 1;
    int x3 = (v2 >> 5) & 1;
    int x4 = (v3 >> 7) & 1;
    int x5 = (v3 >> 6) & 1;
    int x6 = (v3 >> 5) & 1;

    int ohm = 1 << mode;
    if ((ohm & 0x30) != 0) green |= x0 << 6;
    if ((ohm & 0x3a) != 0) green |= x1 << 5;
    if ((ohm & 0x30) != 0) blue |= x2 << 6;
    if ((ohm & 0x3a) != 0) blue |= x3 << 5;
    if ((ohm & 0x3d) != 0) scale |= x6 << 5;
    if ((ohm & 0x2d) != 0) scale |= x5 << 6;
    if ((ohm & 0x04) != 0) scale |= x4 << 7;
    if ((ohm & 0x3b) != 0) red |= x4 << 6;
    if ((ohm & 0x04) != 0) red |= x3 << 6;
    if ((ohm & 0x10) != 0) red |= x5 << 7;
    if ((ohm & 0x0f) != 0) red |= x2 << 7;
    if ((ohm & 0x05) != 0) red |= x1 << 8;
    if ((ohm & 0x0a) != 0) red |= x0 << 8;
    if ((ohm & 0x05) != 0) red |= x0 << 9;
    if ((ohm & 0x02) != 0) red |= x6 << 9;
    if ((ohm & 0x01) != 0) red |= x3 << 10;
    if ((ohm & 0x02) != 0) red |= x5 << 10;

    int shamt = max(mode, 1);
    red <<= shamt;
    green <<= shamt;
    blue <<= shamt;
    scale <<= shamt;

    if (mode != 5)
    {
        green = red - green;
        blue = red - blue;
    }

    if (major_component == 1)
        swap(red, green);
    else if (major_component == 2)
        swap(red, blue);

    ep1 = ivec4(clamp(ivec3(red, green, blue), ivec3(0), ivec3(0xfff)), 0x780);
    ep0 = ivec4(clamp(ivec3(red, green, blue) - scale, ivec3(0), ivec3(0xfff)), 0x780);
}

void decode_endpoint_hdr_rgb_direct(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1, int v2, int v3, int v4, int v5)
{
    int major_component = ((v4 & 0x80) >> 7) | ((v5 & 0x80) >> 6);

    if (major_component == 3)
    {
        ep0 = ivec4(v0 << 4, v2 << 4, (v4 & 0x7f) << 5, 0x780);
        ep1 = ivec4(v1 << 4, v3 << 4, (v5 & 0x7f) << 5, 0x780);
        return;
    }

    int mode = ((v1 & 0x80) >> 7) | ((v2 & 0x80) >> 6) | ((v3 & 0x80) >> 5);
    int va = v0 | ((v1 & 0x40) << 2);
    int vb0 = v2 & 0x3f;
    int vb1 =  v3 & 0x3f;
    int vc = v1 & 0x3f;
    int vd0 = v4 & 0x7f;
    int vd1 = v5 & 0x7f;

    int d_bits = 7 - (mode & 1);
    if ((mode & 5) == 4)
        d_bits -= 2;

    vd0 = bitfieldExtract(vd0, 0, d_bits);
    vd1 = bitfieldExtract(vd1, 0, d_bits);

    int x0 = (v2 >> 6) & 1;
    int x1 = (v3 >> 6) & 1;
    int x2 = (v4 >> 6) & 1;
    int x3 = (v5 >> 6) & 1;
    int x4 = (v4 >> 5) & 1;
    int x5 = (v5 >> 5) & 1;

    int ohm = 1 << mode;
    if ((ohm & 0xa4) != 0) va |= x0 << 9;
    if ((ohm & 0x08) != 0) va |= x2 << 9;
    if ((ohm & 0x50) != 0) va |= x4 << 9;
    if ((ohm & 0x50) != 0) va |= x5 << 10;
    if ((ohm & 0xa0) != 0) va |= x1 << 10;
    if ((ohm & 0xc0) != 0) va |= x2 << 11;

    if ((ohm & 0x04) != 0) vc |= x1 << 6;
    if ((ohm & 0xe8) != 0) vc |= x3 << 6;
    if ((ohm & 0x20) != 0) vc |= x2 << 7;

    if ((ohm & 0x5b) != 0) vb0 |= x0 << 6;
    if ((ohm & 0x5b) != 0) vb1 |= x1 << 6;
    if ((ohm & 0x12) != 0) vb0 |= x2 << 7;
    if ((ohm & 0x12) != 0) vb1 |= x3 << 7;

    int shamt = (mode >> 1) ^ 3;
    va <<= shamt;
    vb0 <<= shamt;
    vb1 <<= shamt;
    vc <<= shamt;
    vd0 <<= shamt;
    vd1 <<= shamt;

    ep1 = ivec4(clamp(ivec3(va, va - vb0, va - vb1), ivec3(0), ivec3(0xfff)), 0x780);
    ep0 = ivec4(clamp(ivec3(va - vc, va - vb0 - vc - vd0, va - vb1 - vc - vd1), ivec3(0), ivec3(0xfff)), 0x780);

    if (major_component == 1)
    {
        swap(ep0.r, ep0.g);
        swap(ep1.r, ep1.g);
    }
    else if (major_component == 2)
    {
        swap(ep0.r, ep0.b);
        swap(ep1.r, ep1.b);
    }
}

void decode_endpoint_ldr_rgb_base_offset(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1, int v2, int v3, int v4, int v5)
{
    bit_transfer_signed(v1, v0);
    bit_transfer_signed(v3, v2);
    bit_transfer_signed(v5, v4);
    if (v1 + v3 + v5 >= 0)
    {
        ep0 = ivec4(v0, v2, v4, 0xff);
        ep1 = ivec4(v0 + v1, v2 + v3, v4 + v5, 0xff);
    }
    else
    {
        ep0 = blue_contract(v0 + v1, v2 + v3, v4 + v5, 0xff);
        ep1 = blue_contract(v0, v2, v4, 0xff);
    }

    ep0.rgb = clamp(ep0.rgb, ivec3(0), ivec3(0xff));
    ep1.rgb = clamp(ep1.rgb, ivec3(0), ivec3(0xff));
}

void decode_endpoint_ldr_rgba_direct(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1, int v2, int v3,
        int v4, int v5, int v6, int v7)
{
    int s0 = v0 + v2 + v4;
    int s1 = v1 + v3 + v5;
    if (s1 >= s0)
    {
        ep0 = ivec4(v0, v2, v4, v6);
        ep1 = ivec4(v1, v3, v5, v7);
    }
    else
    {
        ep0 = blue_contract(v1, v3, v5, v7);
        ep1 = blue_contract(v0, v2, v4, v6);
    }
}

void decode_endpoint_ldr_rgba_base_offset(out ivec4 ep0, out ivec4 ep1,
        int v0, int v1, int v2, int v3, int v4, int v5, int v6, int v7)
{
    bit_transfer_signed(v1, v0);
    bit_transfer_signed(v3, v2);
    bit_transfer_signed(v5, v4);
    bit_transfer_signed(v7, v6);

    if (v1 + v3 + v5 >= 0)
    {
        ep0 = ivec4(v0, v2, v4, v6);
        ep1 = ivec4(v0 + v1, v2 + v3, v4 + v5, v6 + v7);
    }
    else
    {
        ep0 = blue_contract(v0 + v1, v2 + v3, v4 + v5, v6 + v7);
        ep1 = blue_contract(v0, v2, v4, v6);
    }

    ep0 = clamp(ep0, ivec4(0), ivec4(0xff));
    ep1 = clamp(ep1, ivec4(0), ivec4(0xff));
}

void decode_endpoint_hdr_alpha(out int ep0, out int ep1, int v6, int v7)
{
    int mode = ((v6 >> 7) & 1) | ((v7 >> 6) & 2);
    v6 &= 0x7f;
    v7 &= 0x7f;

    if (mode == 3)
    {
        ep0 = v6 << 5;
        ep1 = v7 << 5;
    }
    else
    {
        v6 |= (v7 << (mode + 1)) & 0x780;
        v7 &= 0x3f >> mode;
        v7 ^= 0x20 >> mode;
        v7 -= 0x20 >> mode;
        v6 <<= 4 - mode;
        v7 <<= 4 - mode;
        v7 += v6;
        v7 = clamp(v7, 0, 0xfff);
        ep0 = v6;
        ep1 = v7;
    }
}

void decode_endpoint(out ivec4 ep0, out ivec4 ep1, out int decode_mode,
                     uvec4 payload, int bit_offset, ivec4 quant, int ep_mode,
                     int base_endpoint_index, int num_endpoint_bits)
{
    num_endpoint_bits += bit_offset;
    payload &= build_bitmask(num_endpoint_bits);

    // Could of course use an array, but that doesn't lower nicely to indexed registers on all GPUs.
    int v0, v1, v2, v3, v4, v5, v6, v7;
    int num_values = 2 * ((ep_mode >> 2) + 1);

#define DECODE_EP(i) \
    int(texelFetch(LUTEndpointUnquantize, quant.w + decode_integer_sequence(payload, bit_offset, i + base_endpoint_index, quant.xyz)).x)

    int hi_bits = ep_mode >> 2;
    v0 = DECODE_EP(0);
    v1 = DECODE_EP(1);

    if (hi_bits >= 1)
    {
        v2 = DECODE_EP(2);
        v3 = DECODE_EP(3);
    }

    if (hi_bits >= 2)
    {
        v4 = DECODE_EP(4);
        v5 = DECODE_EP(5);
    }

    if (hi_bits >= 3)
    {
        v6 = DECODE_EP(6);
        v7 = DECODE_EP(7);
    }

    switch (ep_mode)
    {
    case 0:
        decode_endpoint_ldr_luma_direct(ep0, ep1,
            v0, v1);
        decode_mode = MODE_LDR;
        break;

    case 1:
        decode_endpoint_ldr_luma_base_offset(ep0, ep1,
            v0, v1);
        decode_mode = MODE_LDR;
        break;

    case 2:
        decode_endpoint_hdr_luma_direct(ep0, ep1,
            v0, v1);
        decode_mode = MODE_HDR;
        break;

    case 3:
        decode_endpoint_hdr_luma_direct_small_range(ep0, ep1,
            v0, v1);
        decode_mode = MODE_HDR;
        break;

    case 4:
        decode_endpoint_ldr_luma_alpha_direct(ep0, ep1,
            v0, v1, v2, v3);
        decode_mode = MODE_LDR;
        break;

    case 5:
        decode_endpoint_ldr_luma_alpha_base_offset(ep0, ep1,
            v0, v1, v2, v3);
        decode_mode = MODE_LDR;
        break;

    case 6:
        decode_endpoint_ldr_rgb_base_scale(ep0, ep1,
            v0, v1, v2, v3);
        decode_mode = MODE_LDR;
        break;

    case 7:
        decode_endpoint_hdr_rgb_scale(ep0, ep1,
            v0, v1, v2, v3);
        decode_mode = MODE_HDR;
        break;

    case 8:
        decode_endpoint_ldr_rgb_direct(ep0, ep1,
            v0, v1, v2, v3, v4, v5);
        decode_mode = MODE_LDR;
        break;

    case 9:
        decode_endpoint_ldr_rgb_base_offset(ep0, ep1,
            v0, v1, v2, v3, v4, v5);
        decode_mode = MODE_LDR;
        break;

    case 10:
        decode_endpoint_ldr_rgb_base_scale_two_a(ep0, ep1,
            v0, v1, v2, v3, v4, v5);
        decode_mode = MODE_LDR;
        break;

    case 11:
    case 14:
    case 15:
        decode_endpoint_hdr_rgb_direct(ep0, ep1,
            v0, v1, v2, v3, v4, v5);
        if (ep_mode == 14)
        {
            ep0.a = v6;
            ep1.a = v7;
            decode_mode = MODE_HDR_LDR_ALPHA;
        }
        else if (ep_mode == 15)
        {
            decode_endpoint_hdr_alpha(ep0.a, ep1.a, v6, v7);
            decode_mode = MODE_HDR;
        }
        else
            decode_mode = MODE_HDR;
        break;

    case 12:
        decode_endpoint_ldr_rgba_direct(ep0, ep1,
            v0, v1, v2, v3, v4, v5, v6, v7);
        decode_mode = MODE_LDR;
        break;

    case 13:
        decode_endpoint_ldr_rgba_base_offset(ep0, ep1,
            v0, v1, v2, v3, v4, v5, v6, v7);
        decode_mode = MODE_LDR;
        break;
    }

    if (DECODE_8BIT && decode_mode != MODE_LDR)
        decode_error = true;
}

#define CHECK_DECODE_ERROR() do { \
    if (decode_error) \
    { \
        emit_decode_error(coord.xy); \
        return; \
    } \
} while(false)

void emit_decode_error(ivec2 coord)
{
#ifdef VULKAN
    if (is_3Dimage)
        imageStore(OutputImage3D, ivec3(coord, gl_WorkGroupID.z), error_color);
    else
        imageStore(OutputImage2Darray, ivec3(coord, gl_WorkGroupID.z), error_color);
#else /* VULKAN */
    imageStore(OutputImage, coord, error_color);
#endif /* VULKAN */
}

int compute_num_endpoint_pairs(int num_partitions, int cem)
{
    int ret;
    if (num_partitions > 1)
    {
        bool single_cem = (cem & 3) == 0;
        if (single_cem)
            ret = ((cem >> 4) + 1) * num_partitions;
        else
            ret = (cem & 3) * num_partitions + bitCount(bitfieldExtract(uint(cem), 2, num_partitions));
    }
    else
    {
        ret = (cem >> 2) + 1;
    }
    return ret;
}

void decode_cem_base_endpoint(uvec4 payload, int weight_cost_bits, inout int cem, out int base_endpoint_index,
    int num_partitions, int partition_index)
{
    if (num_partitions > 1)
    {
        bool single_cem = (cem & 3) == 0;
        if (single_cem)
        {
            cem >>= 2;
            base_endpoint_index = ((cem >> 2) + 1) * partition_index;
        }
        else
        {
            if (partition_index != 0)
                base_endpoint_index = (cem & 3) * partition_index + bitCount(bitfieldExtract(uint(cem), 2, partition_index));
            else
                base_endpoint_index = 0;

            int base_class = (cem & 3) - 1;
            int extra_cem_bits = num_partitions * 3 - 4;
            int extra_bits = extract_bits(payload, 128 - weight_cost_bits - extra_cem_bits, extra_cem_bits);
            cem = (extra_bits << 4) | (cem >> 2);

            int class_offset_bit = (cem >> partition_index) & 1;
            int ep_bits = (cem >> (num_partitions + 2 * partition_index)) & 3;

            cem = 4 * (base_class + class_offset_bit) + ep_bits;
        }
        base_endpoint_index *= 2;
    }
    else
    {
        base_endpoint_index = 0;
    }
}

ivec4 void_extent_color(uvec4 payload, out int decode_mode)
{
    int min_s = extract_bits(payload, 12, 13);
    int max_s = extract_bits(payload, 12 + 13, 13);
    int min_t = extract_bits(payload, 12 + 2 * 13, 13);
    int max_t = extract_bits(payload, 12 + 3 * 13, 13);

    int reserved = extract_bits(payload, 10, 2);
    if (reserved != 3)
    {
        decode_error = true;
        return ivec4(0);
    }

    if (!all(equal(ivec4(min_s, max_s, min_t, max_t), ivec4((1 << 13) - 1))))
    {
        if (any(greaterThanEqual(ivec2(min_s, min_t), ivec2(max_s, max_t))))
        {
            decode_error = true;
            return ivec4(0);
        }
    }

    decode_mode = (payload.x & (1u << 9)) != 0u ? MODE_HDR : MODE_LDR;

    int r = extract_bits(payload, 64, 16);
    int g = extract_bits(payload, 64 + 16, 16);
    int b = extract_bits(payload, 64 + 32, 16);
    int a = extract_bits(payload, 64 + 48, 16);

    return ivec4(r, g, b, a);
}

void main()
{
    ivec4 coord = build_coord();
#ifdef VULKAN
    if (any(greaterThanEqual(coord.xy, texel_end.xy)))
        return;
#else /* VULKAN */
    if (any(greaterThanEqual(coord.xy, imageSize(OutputImage))))
        return;
#endif /* VULKAN */

    ivec2 pixel_coord = ivec2(gl_LocalInvocationID.xy);
    int linear_pixel = int(gl_WorkGroupSize.x) * pixel_coord.y + pixel_coord.x;
    uvec4 payload;
#ifdef VULKAN
    if (is_3Dimage)
        payload = texelFetch(PayloadInput3D, ivec3(coord.zw, gl_WorkGroupID.z), 0);
    else
        payload = texelFetch(PayloadInput2Darray,ivec3(coord.zw, gl_WorkGroupID.z), 0);
#else /* VULKAN */
    payload = texelFetch(PayloadInput, coord.zw, 0);
#endif /* VULKAN */

    BlockMode block_mode = decode_block_mode(payload);
    CHECK_DECODE_ERROR();

    ivec4 final_color;
    int decode_mode;
    if (block_mode.void_extent)
    {
        final_color = void_extent_color(payload, decode_mode);
        CHECK_DECODE_ERROR();
    }
    else
    {
        int weight_cost_bits;
        ivec4 weights = decode_weights(payload, block_mode, normalize_coord(pixel_coord), weight_cost_bits);

        int partition_index = 0;
        if (block_mode.num_partitions > 1)
        {
            int lut_x = pixel_coord.x + int(gl_WorkGroupSize.x) * (block_mode.seed & 31);
            int lut_y = pixel_coord.y + int(gl_WorkGroupSize.y) * (block_mode.seed >> 5);
#ifdef VULKAN
            int lut_width = int(gl_WorkGroupSize.x) * 32;
            partition_index = int(texelFetch(LUTPartitionTable, lut_y * lut_width + lut_x).x);
#else /* VULKAN */
            partition_index = int(texelFetch(LUTPartitionTable, ivec2(lut_x, lut_y), 0).x);
#endif /* VULKAN */
            partition_index = (partition_index >> (2 * block_mode.num_partitions - 4)) & 3;
        }

        int available_endpoint_bits = max(128 - block_mode.config_bits - weight_cost_bits, 0);

        // In multi-partition mode, the 6-bit CEM field is encoded as
        // First two bits tell if all CEM field are the same, if not we specify a class offset, and N bits
        // after that will offset the class by 1.
        int num_endpoint_pairs = compute_num_endpoint_pairs(block_mode.num_partitions, block_mode.cem);

        // Error color must be emitted if we need more than 18 integer sequence encoded values of color.
        if (num_endpoint_pairs > 9)
        {
            decode_error = true;
            emit_decode_error(coord.xy);
            return;
        }

        ivec4 endpoint_quant = ivec4(texelFetch(LUTRemainingBitsToEndpointQuantizer,
                128 * (num_endpoint_pairs - 1) + available_endpoint_bits));

        // Only read the bits we need for endpoints.
        int num_endpoint_values = num_endpoint_pairs * 2;
        available_endpoint_bits =
            endpoint_quant.x * num_endpoint_values +
            idiv5_ceil(endpoint_quant.y * 8 * num_endpoint_values) +
            idiv3_ceil(endpoint_quant.z * 7 * num_endpoint_values);

        // No space left for color endpoints.
        if (all(equal(endpoint_quant.xyz, ivec3(0))))
        {
            decode_error = true;
            emit_decode_error(coord.xy);
            return;
        }

        int endpoint_bit_offset = block_mode.primary_config_bits;
        ivec4 ep0, ep1;

        // Decode CEM for multi-partition schemes.
        int cem = block_mode.cem;
        int base_endpoint_index;
        decode_cem_base_endpoint(payload, weight_cost_bits, cem, base_endpoint_index,
                                 block_mode.num_partitions, partition_index);

        decode_endpoint(ep0, ep1, decode_mode, payload, endpoint_bit_offset, endpoint_quant,
                        cem, base_endpoint_index, available_endpoint_bits);
        CHECK_DECODE_ERROR();

        final_color = interpolate_endpoint(ep0, ep1, weights, decode_mode);
    }

    if (DECODE_8BIT)
    {
#ifdef VULKAN
        if (is_3Dimage)
            imageStore(OutputImage3D, ivec3(coord.xy, gl_WorkGroupID.z), uvec4(final_color >> 8));
        else
            imageStore(OutputImage2Darray, ivec3(coord.xy, gl_WorkGroupID.z), uvec4(final_color >> 8));
#else /* VULKAN */
        imageStore(OutputImage, coord.xy, uvec4(final_color >> 8));
#endif /* VULKAN */
    }
    else
    {
        uvec4 encoded;
        if (block_mode.void_extent && decode_mode == MODE_HDR)
            encoded = uvec4(final_color);
        else
            encoded = decode_fp16(final_color, decode_mode);
#ifdef VULKAN
        if (is_3Dimage)
            imageStore(OutputImage3D, ivec3(coord.xy, gl_WorkGroupID.z), encoded);
        else
            imageStore(OutputImage2Darray, ivec3(coord.xy, gl_WorkGroupID.z), encoded);
#else /* VULKAN */
        imageStore(OutputImage, coord.xy, encoded);
#endif /* VULKAN */
    }
}