// Copyright 2021 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

$assert REQUANTIZATION != "FP32" or VARIANT in ["FMAGIC", "IMAGIC", "LRINTF"]
$assert DATATYPE in ["QC8", "QS8", "QU8", "QD8"]
$assert DATATYPE in ["QD8"] or REQUANTIZATION in ["FP32", "RNDNU"]
#include <assert.h>
$if VARIANT == "LRINTF":
  #include <math.h>

#include "xnnpack/math.h"
#include "xnnpack/gemm.h"
$if NR % 4 != 0:
  #include "xnnpack/unaligned.h"


$DATATYPE_SPEC = {"QC8": "qs8_qc8w", "QS8": "qs8", "QU8": "qu8", "QD8": "qd8_f32_qc8w"}[DATATYPE]
$REQUANTIZATION_SPEC = "_" + REQUANTIZATION.lower() if REQUANTIZATION else ""
$PARAMS_STRUCT = REQUANTIZATION.lower() + "_scalar"
$PARAMS_TYPE = {"QC8": "union xnn_qs8_qc8w_conv_minmax_params", "QS8": "union xnn_qs8_conv_minmax_params", "QU8": "union xnn_qu8_conv_minmax_params", "QD8": "union xnn_f32_minmax_params"}[DATATYPE]
$SCALING_PARAMS = "xnn_qd8_quantization_params" if DATATYPE in ["QD8", "QC4"] else ""
$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t"
$OUT_T = XINT8_T if REQUANTIZATION else "float"
$MIN_F32 = "__builtin_wasm_min_f32" if WASM else "math_min_f32"
$MAX_F32 = "__builtin_wasm_max_f32" if WASM else "math_max_f32"
void xnn_${DATATYPE_SPEC}_igemm_minmax${REQUANTIZATION_SPEC}_ukernel_${MR}x${NR}__${"wasm" if WASM else "scalar"}${"_" + VARIANT.lower() if VARIANT else ""}(
    size_t mr,
    size_t nc,
    size_t kc,
    size_t ks,
    const ${XINT8_T}** restrict a,
    const void* restrict w,
    ${OUT_T}* restrict c,
    size_t cm_stride,
    size_t cn_stride,
    size_t a_offset,
    const ${XINT8_T}* zero,
    $if DATATYPE in ["QD8"]:
      const int8_t* zero_data,
    $if DATATYPE in ["QD8", "QC4"]:
      const ${PARAMS_TYPE} params[restrict XNN_MIN_ELEMENTS(1)],
      const struct ${SCALING_PARAMS} quantization_params[restrict XNN_MIN_ELEMENTS(1)])
    $else:
      const ${PARAMS_TYPE} params[restrict XNN_MIN_ELEMENTS(1)])
{
  assert(mr != 0);
  assert(mr <= ${MR});
  assert(nc != 0);
  assert(kc != 0);
  assert(ks != 0);
  assert(ks % (${MR} * sizeof(void*)) == 0);
  assert(a != NULL);
  assert(w != NULL);
  assert(c != NULL);

  ${OUT_T}* c0 = c;
  $for M in range(1, MR):
    ${OUT_T}* c${M} = (${OUT_T}*) ((uintptr_t) c${M-1} + cm_stride);
    $if M % 2 == 0:
      if XNN_UNPREDICTABLE(mr <= ${M}) {
        c${M} = c${M-1};
      }
    $elif M + 1 == MR:
      if XNN_UNPREDICTABLE(mr != ${M+1}) {
        c${M} = c${M-1};
      }
    $else:
      if XNN_UNPREDICTABLE(mr < ${M+1}) {
        c${M} = c${M-1};
      }

  $if DATATYPE != "QD8":
    $if REQUANTIZATION == "FP32":
      const int32_t output_min_less_zero_point = (int32_t) params->${PARAMS_STRUCT}.output_min - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      const int32_t output_max_less_zero_point = (int32_t) params->${PARAMS_STRUCT}.output_max - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      $if DATATYPE != "QC8":
        const float vscale = params->${PARAMS_STRUCT}.scale;
      $if VARIANT == "FMAGIC":
        const float voutput_min_less_zero_point = output_min_less_zero_point;
        const float voutput_max_less_zero_point = output_max_less_zero_point;
        const float vmagic_bias = 12582912.0f;
        const int32_t vmagic_bias_less_output_zero_point = INT32_C(0x4B400000) - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      $elif VARIANT == "IMAGIC":
        const float vmagic_bias = 12582912.0f;
        const int32_t vmagic_min = (int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point);
        const int32_t vmagic_max = (int32_t) float_as_uint32(12582912.0f + output_max_less_zero_point);
        const int32_t vmagic_bias_less_zero_point = INT32_C(0x4B400000) - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      $elif VARIANT == "LRINTF":
        const float voutput_min_less_zero_point = output_min_less_zero_point;
        const float voutput_max_less_zero_point = output_max_less_zero_point;
        const int32_t voutput_zero_point = params->${PARAMS_STRUCT}.output_zero_point;
    $elif REQUANTIZATION == "RNDNU":
      const int32_t vmultiplier = params->${PARAMS_STRUCT}.multiplier;
      const int64_t vrounding = params->${PARAMS_STRUCT}.rounding;
      const uint32_t vshift = params->${PARAMS_STRUCT}.shift;
      const int32_t voutput_min_less_zero_point = (int32_t) params->${PARAMS_STRUCT}.output_min - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      const int32_t voutput_max_less_zero_point = (int32_t) params->${PARAMS_STRUCT}.output_max - (int32_t) params->${PARAMS_STRUCT}.output_zero_point;
      const int32_t voutput_zero_point = params->${PARAMS_STRUCT}.output_zero_point;

  $if DATATYPE == "QU8":
    const int32_t vb_zero_point = params->${PARAMS_STRUCT}.kernel_zero_point;
  do {
    $if DATATYPE == "QD8":
      $if NR % 4 != 0:
        $for N in range(NR):
          const int32_t vksum${N} = unaligned_indexed_load_s32(w, ${N});
      $else:
        $for N in range(NR):
          const int32_t vksum${N} = ((const int32_t*) w)[${N}];
      const int32_t vinput_zero_point = quantization_params->zero_point;
      $for M in range(MR):
        $for N in range(NR):
          int32_t vacc${M}x${N} = vksum${N} * vinput_zero_point;
    $else:
      $if NR % 4 != 0:
        $for N in range(NR):
          int32_t vacc0x${N} = unaligned_indexed_load_s32(w, ${N});
      $else:
        $for N in range(NR):
          int32_t vacc0x${N} = ((const int32_t*) w)[${N}];
      $for M in range(1, MR):
        $for N in range(NR):
          int32_t vacc${M}x${N} = vacc0x${N};
    w = (const void*) ((const int32_t*) w + ${NR});

    size_t p = ks;
    do {
      $for M in range(MR):
        const ${XINT8_T}* restrict a${M} = a[${M}];
        assert(a${M} != NULL);
        if XNN_UNPREDICTABLE(a${M} != zero) {
          a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} + a_offset);
        $if DATATYPE == "QD8":
          } else {
            a${M} = zero_data;
        }
      a += ${MR};

      size_t k = kc;
      do {
        $for M in range(MR):
          $if DATATYPE == "QU8":
            const int32_t va${M} = (int32_t) (uint32_t) *a${M}++;
          $else:
            const int32_t va${M} = (int32_t) *a${M}++;

        $for N in range(NR):
          $if DATATYPE == "QU8":
            const int32_t vb${N} = (int32_t) (uint32_t) ((const uint8_t*) w)[${N}] - vb_zero_point;
          $else:
            const int32_t vb${N} = (int32_t) ((const int8_t*) w)[${N}];
        w = (const void*) ((const ${XINT8_T}*) w + ${NR});

        $for M in range(MR):
          $for N in range(NR):
            vacc${M}x${N} += va${M} * vb${N};

        k -= sizeof(${XINT8_T});
      } while (k != 0);
      p -= ${MR} * sizeof(void*);
    } while (p != 0);

    $if DATATYPE == "QD8":
      $for M in range(MR):
        $for N in range(NR):
          float vout${M}x${N} = (float) vacc${M}x${N};

      const float vinput_scale = quantization_params->inv_scale;
      $for M in range(MR):
        $for N in range(NR):
          vout${M}x${N} *= vinput_scale;

      $for N in range(NR):
        $if NR % 4 != 0:
          const float vfilter_output_scale${N} = unaligned_indexed_load_f32(w, ${N});
        $else:
          const float vfilter_output_scale${N} = ((const float*) w)[${N}];
        $for M in range(MR):
          vout${M}x${N} *= vfilter_output_scale${N};

      $for N in range(NR):
        $if NR % 4 != 0:
          const float vbias${N} = unaligned_indexed_load_f32(w, ${NR + N});
        $else:
          const float vbias${N} = ((const float*) w)[${NR + N}];
        $for M in range(MR):
          vout${M}x${N} += vbias${N};

      w = (const float*) w + ${NR * 2};

      const float voutput_min = params->scalar.min;
      $for N in range(NR):
        $for M in range(MR):
          vout${M}x${N} = ${MAX_F32}(vout${M}x${N}, voutput_min);

      const float voutput_max = params->scalar.max;
      $for N in range(NR):
        $for M in range(MR):
          vout${M}x${N} = ${MIN_F32}(vout${M}x${N}, voutput_max);

      if XNN_LIKELY(nc >= ${NR}) {
        $for M in reversed(range(MR)):
          $for N in range(NR):
            c${M}[${N}] = vout${M}x${N};

        $for M in range(MR):
          c${M} = (float*) ((uintptr_t) c${M} + cn_stride);

        a = (const ${XINT8_T}**restrict) ((uintptr_t) a - ks);
        nc -= ${NR};
      } else {
        $for LOG2N in reversed(range(NR.bit_length() - 1)):
          if (nc & ${1 << LOG2N}) {
            $for M in reversed(range(MR)):
              $for N in range(1 << LOG2N):
                c${M}[${N}] = vout${M}x${N};
              $if LOG2N != 0:
                $for N in range(NR - (1 << LOG2N) - 1):
                  vout${M}x${N} = vout${M}x${N + (1 << LOG2N)};
                c${M} += ${1 << LOG2N};
          }

        nc = 0;
      }
    $else:
      $if REQUANTIZATION == "FP32":
        $for M in range(MR):
          $for N in range(NR):
            float vfpacc${M}x${N} = (float) vacc${M}x${N};

        $if DATATYPE == "QC8":
          $if NR % 4 != 0:
            $for N in range(NR):
              const float vscale${N} = unaligned_indexed_load_f32(w, ${N});
              $for M in range(MR):
                vfpacc${M}x${N} *= vscale${N};
          $else:
            $for N in range(NR):
              const float vscale${N} = ((const float*) w)[${N}];
              $for M in range(MR):
                vfpacc${M}x${N} *= vscale${N};
          w = (const void*) ((const float*) w + ${NR});
        $else:
          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} *= vscale;

        $if VARIANT == "FMAGIC":
          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} = ${MAX_F32}(vfpacc${M}x${N}, voutput_min_less_zero_point);

          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} = ${MIN_F32}(vfpacc${M}x${N}, voutput_max_less_zero_point);

          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} += vmagic_bias;

          $for M in range(MR):
            $for N in range(NR):
              int32_t vout${M}x${N} = (int32_t) float_as_uint32(vfpacc${M}x${N}) - vmagic_bias_less_output_zero_point;
        $elif VARIANT == "IMAGIC":
          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} += vmagic_bias;

          $for M in range(MR):
            $for N in range(NR):
              int32_t vout${M}x${N} = (int32_t) float_as_uint32(vfpacc${M}x${N});

          $for M in range(MR):
            $for N in range(NR):
              vout${M}x${N} = math_max_s32(vout${M}x${N}, vmagic_min);

          $for M in range(MR):
            $for N in range(NR):
              vout${M}x${N} = math_min_s32(vout${M}x${N}, vmagic_max);

          $for M in range(MR):
            $for N in range(NR):
              vout${M}x${N} -= vmagic_bias_less_zero_point;
        $elif VARIANT == "LRINTF":
          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} = ${MAX_F32}(vfpacc${M}x${N}, voutput_min_less_zero_point);

          $for M in range(MR):
            $for N in range(NR):
              vfpacc${M}x${N} = ${MIN_F32}(vfpacc${M}x${N}, voutput_max_less_zero_point);

          $for M in range(MR):
            $for N in range(NR):
              const int32_t vrndacc${M}x${N} = (int32_t) lrintf(vfpacc${M}x${N});

          $for M in range(MR):
            $for N in range(NR):
              int32_t vout${M}x${N} = vrndacc${M}x${N} + voutput_zero_point;
      $elif REQUANTIZATION == "RNDNU":
        $for M in range(MR):
          $for N in range(NR):
            const int64_t vextacc${M}x${N} = math_mulext_s32(vacc${M}x${N}, vmultiplier) + vrounding;

        $for M in range(MR):
          $for N in range(NR):
            int32_t vout${M}x${N} = (int32_t) math_asr_s64(vextacc${M}x${N}, vshift);

        $for M in range(MR):
          $for N in range(NR):
            vout${M}x${N} = math_max_s32(vout${M}x${N}, voutput_min_less_zero_point);

        $for M in range(MR):
          $for N in range(NR):
            vout${M}x${N} = math_min_s32(vout${M}x${N}, voutput_max_less_zero_point);

        $for M in range(MR):
          $for N in range(NR):
            vout${M}x${N} += voutput_zero_point;

      if XNN_LIKELY(nc >= ${NR}) {
        $for M in reversed(range(MR)):
          $for N in range(NR):
            c${M}[${N}] = (${XINT8_T}) vout${M}x${N};

        $for M in reversed(range(MR)):
          c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride);

        a = (const ${XINT8_T}**restrict) ((uintptr_t) a - ks);
        nc -= ${NR};
      } else {
        $for LOG2N in reversed(range(NR.bit_length() - 1)):
          if (nc & ${1 << LOG2N}) {
            $for M in reversed(range(MR)):
              $for N in range(1 << LOG2N):
                c${M}[${N}] = (${XINT8_T}) vout${M}x${N};
              $if LOG2N != 0:
                $for N in range(NR - (1 << LOG2N) - 1):
                  vout${M}x${N} = vout${M}x${N + (1 << LOG2N)};
                c${M} += ${1 << LOG2N};
          }

        nc = 0;
      }
  } while (nc != 0);
}