// Copyright 2021 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

$assert DATATYPE in ["QD8", "QC4", "QB4_F16", "QB4_F32"] or REQUANTIZATION in ["FP32", "RNDNU"]
$assert REQUANTIZATION != "FP32" or VARIANT in ["FMAGIC", "IMAGIC", "LRINTF"]
$assert DATATYPE in ["QC8", "QS8", "QU8", "QD8", "QC4", "QB4_F16", "QB4_F32"]
#include <assert.h>
$if VARIANT == "LRINTF":
  #include <math.h>

#include "xnnpack/gemm.h"
#include "xnnpack/math.h"
$if NR % 4 != 0:
  #include "xnnpack/unaligned.h"

$#
$INDENT = 0
$_ = ""
$#
$def SET_INDENT(NEW_INDENT):
  $global INDENT
  $global _
  $INDENT=NEW_INDENT
  $_ = "  " * INDENT
$#
$def BEGIN():
  $SET_INDENT(INDENT + 1)
  $return '{'
$#
$def END():
  $SET_INDENT(INDENT - 1)
  $return _ + '}'
$#

$DATATYPE_SPEC = {"QC8": "qs8_qc8w", "QS8": "qs8", "QU8": "qu8", "QD8": "qd8_f32_qc8w", "QC4": "qd8_f32_qc4w", "QB4_F16": "qd8_f16_qb4w", "QB4_F32": "qd8_f32_qb4w"}[DATATYPE]
$VARIANT_SPEC = "_" + VARIANT.lower() if VARIANT else ""
$REQUANTIZATION_SPEC = "_" + REQUANTIZATION.lower() if REQUANTIZATION else ""
$PARAMS_STRUCT = REQUANTIZATION.lower() + "_scalar"
$PARAMS_TYPE = {"QC8": "union xnn_qs8_qc8w_conv_minmax_params", "QS8": "union xnn_qs8_conv_minmax_params", "QU8": "union xnn_qu8_conv_minmax_params", "QD8": "union xnn_f32_minmax_params", "QC4": "struct xnn_f32_qc4w_minmax_params", "QB4_F16": "struct xnn_f16_qb4w_minmax_params", "QB4_F32": "struct xnn_f32_qb4w_minmax_params"}[DATATYPE]
$SCALING_PARAMS = "xnn_qd8_quantization_params" if DATATYPE in ["QD8", "QC4", "QB4_F16", "QB4_F32"] else ""
$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t"
$OUT_T = {"QC8": "int8_t", "QS8": "int8_t", "QU8": "uint8_t", "QB4_F16": "xnn_float16", "QD8": "float", "QC4": "float", "QB4_F32": "float"}[DATATYPE]
$MIN_F32 = "__builtin_wasm_min_f32" if WASM else "math_min_f32"
$MAX_F32 = "__builtin_wasm_max_f32" if WASM else "math_max_f32"
$BLOCKWISE = True if DATATYPE in ["QB4_F16", "QB4_F32"] else False
void xnn_${DATATYPE_SPEC}_gemm_minmax${REQUANTIZATION_SPEC}_ukernel_${MR}x${NR}__${"wasm" if WASM else "scalar"}${VARIANT_SPEC}(
    size_t mr,
    size_t nc,
    size_t kc,
    const ${XINT8_T}* restrict a,
    size_t a_stride,
    const void* restrict w,
    ${OUT_T}* restrict c,
    size_t cm_stride,
    size_t cn_stride,
    $if DATATYPE in ["QD8", "QC4", "QB4_F16", "QB4_F32"]:
      const ${PARAMS_TYPE} params[restrict XNN_MIN_ELEMENTS(1)],
      const struct ${SCALING_PARAMS} quantization_params[restrict XNN_MIN_ELEMENTS(1)])
    $else:
      const ${PARAMS_TYPE} params[restrict XNN_MIN_ELEMENTS(1)])
{
  assert(mr != 0);
  assert(mr <= ${MR});
  assert(nc != 0);
  assert(kc != 0);
  $if BLOCKWISE:
    size_t bl = params->scalar.blocksize;
    assert(bl <= round_up_po2(kc, 2));
    assert(bl != 0);
    assert(bl % 32 == 0);

  const ${XINT8_T}* a0 = a;
  ${OUT_T}* c0 = c;
  $for M in range(1, MR):
    const ${XINT8_T}* a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M-1} + a_stride);
    ${OUT_T}* c${M} = (${OUT_T}*) ((uintptr_t) c${M-1} + cm_stride);
    $if M % 2 == 0:
      if XNN_UNPREDICTABLE(mr <= ${M}) {
        a${M} = a${M-1};
        c${M} = c${M-1};
      }
    $elif M + 1 == MR:
      if XNN_UNPREDICTABLE(mr != ${M+1}) {
        a${M} = a${M-1};
        c${M} = c${M-1};
      }
    $else:
      if XNN_UNPREDICTABLE(mr < ${M+1}) {
        a${M} = a${M-1};
        c${M} = c${M-1};
      }

  $if DATATYPE not in ["QD8", "QC4", "QB4_F16", "QB4_F32"]:
    $if REQUANTIZATION == "FP32":
      const int32_t output_min_less_zero_point = (int32_t) params->${PARAMS_STRUCT}.output_min - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      const int32_t output_max_less_zero_point = (int32_t) params->${PARAMS_STRUCT}.output_max - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      $if DATATYPE != "QC8":
        const float vscale = params->${PARAMS_STRUCT}.scale;
      $if VARIANT == "FMAGIC":
        const float voutput_min_less_zero_point = output_min_less_zero_point;
        const float voutput_max_less_zero_point = output_max_less_zero_point;
        const float vmagic_bias = 12582912.0f;
        const int32_t vmagic_bias_less_output_zero_point = INT32_C(0x4B400000) - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      $elif VARIANT == "IMAGIC":
        const float vmagic_bias = 12582912.0f;
        const int32_t vmagic_min = (int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point);
        const int32_t vmagic_max = (int32_t) float_as_uint32(12582912.0f + output_max_less_zero_point);
        const int32_t vmagic_bias_less_zero_point = INT32_C(0x4B400000) - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      $elif VARIANT == "LRINTF":
        const float voutput_min_less_zero_point = output_min_less_zero_point;
        const float voutput_max_less_zero_point = output_max_less_zero_point;
        const int32_t voutput_zero_point = params->${PARAMS_STRUCT}.output_zero_point;
    $elif REQUANTIZATION == "RNDNU":
      const int32_t vmultiplier = params->${PARAMS_STRUCT}.multiplier;
      const int64_t vrounding = params->${PARAMS_STRUCT}.rounding;
      const uint32_t vshift = params->${PARAMS_STRUCT}.shift;
      const int32_t voutput_min_less_zero_point = (int32_t) params->${PARAMS_STRUCT}.output_min - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      const int32_t voutput_max_less_zero_point = (int32_t) params->${PARAMS_STRUCT}.output_max - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      const int32_t voutput_zero_point = params->${PARAMS_STRUCT}.output_zero_point;

  $if DATATYPE == "QU8":
    const int32_t vb_zero_point = params->${PARAMS_STRUCT}.kernel_zero_point;
  $elif DATATYPE in ["QC4", "QB4_F16", "QB4_F32"]:
    kc = round_up_po2(kc, 2);
  do {
    $if DATATYPE in ["QD8", "QC4", "QB4_F16", "QB4_F32"]:
      $if NR % 4 != 0:
        $for N in range(NR):
          $if BLOCKWISE:
            const float vksum${N} = unaligned_indexed_load_f32(w, ${N});
          $else:
            const int32_t vksum${N} = unaligned_indexed_load_s32(w, ${N});
      $else:
        $for N in range(NR):
          $if BLOCKWISE:
            const float vksum${N} = ((const float*) w)[${N}];
          $else:
            const int32_t vksum${N} = ((const int32_t*) w)[${N}];
      $for M in range(MR):
        $if BLOCKWISE:
          const float vinput_zero_point${M} = (const float) quantization_params[${M}].zero_point;
        $else:
          const int32_t vinput_zero_point${M} = quantization_params[${M}].zero_point;
        $for N in range(NR):
          $if BLOCKWISE:
            float vout${M}x${N} = vksum${N} * vinput_zero_point${M};
          $else:
            int32_t vacc${M}x${N} = vksum${N} * vinput_zero_point${M};
    $else:
      $for N in range(NR):
        $if NR % 4 != 0:
          int32_t vacc0x${N} = unaligned_indexed_load_s32(w, ${N});
        $else:
          int32_t vacc0x${N} = ((const int32_t*) w)[${N}];
      $for M in range(1, MR):
        $for N in range(NR):
          int32_t vacc${M}x${N} = vacc0x${N};
    $if BLOCKWISE:
      w = (const float*) w + ${NR};
    $else:
      w = (const int32_t*) w + ${NR};

    $if BLOCKWISE:
      for (size_t kb=0; kb < kc; kb += bl) {
        $for M in range(MR):
          $for N in range(NR):
            int32_t vacc${M}x${N} = 0;
        size_t k = bl;
      $SET_INDENT(1)
    $else:
      size_t k = kc;
    $if DATATYPE in ["QC4", "QB4_F16", "QB4_F32"]:
      ${_}for (; k >= 2 * sizeof(uint8_t); k -= 2 * sizeof(uint8_t)) {
        $for M in range(MR):
          ${_}const int32_t va${M}c0 = (int32_t) a${M}[0];
          ${_}const int32_t va${M}c1 = (int32_t) a${M}[1];
          ${_}a${M} += 2;

        $for N in range(NR):
          ${_}const uint8_t vbi${N} = ((const uint8_t*) w)[${N}];
        ${_}w = (const uint8_t*) w + ${NR};
        $for N in range(NR):
          ${_}const int32_t vb${N}c0 = (int32_t) (int8_t) (vbi${N} << 4);
          ${_}const int32_t vb${N}c1 = (int32_t) (int8_t) (vbi${N} & 0xF0);

        $for M in range(MR):
          $for N in range(NR):
            ${_}vacc${M}x${N} += va${M}c0 * vb${N}c0;
        $for M in range(MR):
          $for N in range(NR):
            ${_}vacc${M}x${N} += va${M}c1 * vb${N}c1;
      }
    $else:
      do {
        $for M in range(MR):
          $if DATATYPE == "QU8":
            const int32_t va${M} = (int32_t) (uint32_t) *a${M}++;
          $else:
            const int32_t va${M} = (int32_t) *a${M}++;

        $for N in range(NR):
          $if DATATYPE == "QU8":
            const int32_t vb${N} = (int32_t) (uint32_t) ((const uint8_t*) w)[${N}] - vb_zero_point;
          $else:
            const int32_t vb${N} = (int32_t) ((const int8_t*) w)[${N}];
        w = (const ${XINT8_T}*) w + ${NR};

        $for M in range(MR):
          $for N in range(NR):
            vacc${M}x${N} += va${M} * vb${N};

        k -= sizeof(${XINT8_T});
      } while (k != 0);
    $if BLOCKWISE:
      // accumulate in float
        $for M in range(MR):
          $for N in range(NR):
            float vf${M}x${N} = vacc${M}x${N};
            $if M == 0:
              $if NR % 4 != 0:
                const float vfilter_output_scale${N} = math_cvt_fp32_bf16(unaligned_indexed_load_u16(w, ${N}));
              $else:
                const float vfilter_output_scale${N} = math_cvt_fp32_bf16(((const uint16_t*) w)[${N}]);

        $for M in range(MR):
          $for N in range(NR):
            vf${M}x${N} *= vfilter_output_scale${N};
            vout${M}x${N} += vf${M}x${N};
        w = (const uint16_t*) w + ${NR};
      }

    $if DATATYPE in ["QD8", "QC4", "QB4_F16", "QB4_F32"]:
      $for M in range(MR):
        $for N in range(NR):
          $if not BLOCKWISE:
            $if DATATYPE == "QC4":
              float vout${M}x${N} = (float) math_asr_s32(vacc${M}x${N}, 4);
            $else:
              float vout${M}x${N} = (float) vacc${M}x${N};

      $for M in range(MR):
        const float vinput_scale${M} = quantization_params[${M}].inv_scale;
        $for N in range(NR):
          vout${M}x${N} *= vinput_scale${M};

      $if not BLOCKWISE:
        $for N in range(NR):
          $if NR % 4 != 0:
            const float vfilter_output_scale${N} = unaligned_indexed_load_f32(w, ${N});
          $else:
            const float vfilter_output_scale${N} = ((const float*) w)[${N}];
          $for M in range(MR):
            vout${M}x${N} *= vfilter_output_scale${N};

      $if BLOCKWISE:
        $for N in range(NR):
          $if NR % 4 != 0:
            const float vbias${N} = unaligned_indexed_load_f32(w, ${N});
          $else:
            const float vbias${N} = ((const float*) w)[${N}];
          $for M in range(MR):
            vout${M}x${N} += vbias${N};
      $else:
        $for N in range(NR):
          $if NR % 4 != 0:
            const float vbias${N} = unaligned_indexed_load_f32(w, ${NR + N});
          $else:
            const float vbias${N} = ((const float*) w)[${NR + N}];
          $for M in range(MR):
            vout${M}x${N} += vbias${N};

      $if BLOCKWISE:
        w = (const float*) w + ${NR};
      $else:
        w = (const float*) w + ${NR * 2};

      $if DATATYPE in ["QB4_F16"]:
        const float voutput_min = xnn_float16_to_float(params->scalar.min);
      $else:
        const float voutput_min = params->scalar.min;
      $for N in range(NR):
        $for M in range(MR):
          vout${M}x${N} = ${MAX_F32}(vout${M}x${N}, voutput_min);

      $if DATATYPE in ["QB4_F16"]:
        const float voutput_max = xnn_float16_to_float(params->scalar.max);
      $else:
        const float voutput_max = params->scalar.max;
      $for N in range(NR):
        $for M in range(MR):
          vout${M}x${N} = ${MIN_F32}(vout${M}x${N}, voutput_max);

      if XNN_LIKELY(nc >= ${NR}) {
        $for M in range(MR):
          $for N in range(NR):
            $if DATATYPE in ["QB4_F16"]:
              c${M}[${N}] = xnn_float16_from_float(vout${M}x${N});
            $else:
              c${M}[${N}] = vout${M}x${N};

        $for M in range(MR):
          a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} - kc);

        $for M in range(MR):
          $if DATATYPE in ["QB4_F16"]:
            c${M} = (${OUT_T}*) ((uintptr_t) c${M} + cn_stride);
          $else:
            c${M} = (float*) ((uintptr_t) c${M} + cn_stride);

        nc -= ${NR};
      } else {
        $for LOG2N in reversed(range(NR.bit_length() - 1)):
          if (nc & ${1 << LOG2N}) {
            $for M in range(MR):
              $for N in range(1 << LOG2N):
                $if DATATYPE in ["QB4_F16"]:
                  c${M}[${N}] = xnn_float16_from_float(vout${M}x${N});
                $else:
                  c${M}[${N}] = vout${M}x${N};
              $if LOG2N != 0:
                $for N in range(NR - (1 << LOG2N) - 1):
                  vout${M}x${N} = vout${M}x${N + (1 << LOG2N)};
                c${M} += ${1 << LOG2N};
          }

        nc = 0;
      }
    $else:
      $if REQUANTIZATION == "FP32":
        $for M in range(MR):
          $for N in range(NR):
            float vfpacc${M}x${N} = (float) vacc${M}x${N};

        $if DATATYPE == "QC8":
          $if NR % 4 != 0:
            $for N in range(NR):
              const float vscale${N} = unaligned_indexed_load_f32(w, ${N});
              $for M in range(MR):
                vfpacc${M}x${N} *= vscale${N};
          $else:
            $for N in range(NR):
              const float vscale${N} = ((const float*) w)[${N}];
              $for M in range(MR):
                vfpacc${M}x${N} *= vscale${N};
          w = (const void*) ((const float*) w + ${NR});
        $else:
          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} *= vscale;

        $if VARIANT == "FMAGIC":
          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} = ${MAX_F32}(vfpacc${M}x${N}, voutput_min_less_zero_point);

          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} = ${MIN_F32}(vfpacc${M}x${N}, voutput_max_less_zero_point);

          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} += vmagic_bias;

          $for M in range(MR):
            $for N in range(NR):
              int32_t vout${M}x${N} = (int32_t) float_as_uint32(vfpacc${M}x${N}) - vmagic_bias_less_output_zero_point;
        $elif VARIANT == "IMAGIC":
          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} += vmagic_bias;

          $for M in range(MR):
            $for N in range(NR):
              int32_t vout${M}x${N} = (int32_t) float_as_uint32(vfpacc${M}x${N});

          $for M in range(MR):
            $for N in range(NR):
              vout${M}x${N} = math_max_s32(vout${M}x${N}, vmagic_min);

          $for M in range(MR):
            $for N in range(NR):
              vout${M}x${N} = math_min_s32(vout${M}x${N}, vmagic_max);

          $for M in range(MR):
            $for N in range(NR):
              vout${M}x${N} -= vmagic_bias_less_zero_point;
        $elif VARIANT == "LRINTF":
          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} = ${MAX_F32}(vfpacc${M}x${N}, voutput_min_less_zero_point);

          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} = ${MIN_F32}(vfpacc${M}x${N}, voutput_max_less_zero_point);

          $for M in range(MR):
            $for N in range(NR):
              const int32_t vrndacc${M}x${N} = (int32_t) lrintf(vfpacc${M}x${N});

          $for M in range(MR):
            $for N in range(NR):
              int32_t vout${M}x${N} = vrndacc${M}x${N} + voutput_zero_point;
      $elif REQUANTIZATION == "RNDNU":
        $for M in range(MR):
          $for N in range(NR):
            const int64_t vextacc${M}x${N} = math_mulext_s32(vacc${M}x${N}, vmultiplier) + vrounding;

        $for M in range(MR):
          $for N in range(NR):
            int32_t vout${M}x${N} = (int32_t) math_asr_s64(vextacc${M}x${N}, vshift);

        $for M in range(MR):
          $for N in range(NR):
            vout${M}x${N} = math_max_s32(vout${M}x${N}, voutput_min_less_zero_point);

        $for M in range(MR):
          $for N in range(NR):
            vout${M}x${N} = math_min_s32(vout${M}x${N}, voutput_max_less_zero_point);

        $for M in range(MR):
          $for N in range(NR):
            vout${M}x${N} += voutput_zero_point;

      if XNN_LIKELY(nc >= ${NR}) {
        $for M in range(MR):
          $for N in range(NR):
            c${M}[${N}] = (${XINT8_T}) vout${M}x${N};

        $for M in range(MR):
          a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} - kc);

        $for M in range(MR):
          c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride);

        nc -= ${NR};
      } else {
        $for LOG2N in reversed(range(NR.bit_length() - 1)):
          if (nc & ${1 << LOG2N}) {
            $for M in range(MR):
              $for N in range(1 << LOG2N):
                c${M}[${N}] = (${XINT8_T}) vout${M}x${N};
              $if LOG2N != 0:
                $for N in range(NR - (1 << LOG2N) - 1):
                  vout${M}x${N} = vout${M}x${N + (1 << LOG2N)};
                c${M} += ${1 << LOG2N};
          }

        nc = 0;
      }
  } while (nc != 0);
}