// Copyright 2019 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

#include <assert.h>
#include <inttypes.h>
#include <math.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

#include "xnnpack.h"
#include "xnnpack/allocator.h"
#include "xnnpack/cache.h"
#include "xnnpack/common.h"
#include "xnnpack/compute.h"
#include "xnnpack/config-types.h"
#include "xnnpack/config.h"
#include "xnnpack/log.h"
#include "xnnpack/math.h"
#include "xnnpack/microfnptr.h"
#include "xnnpack/microkernel-type.h"
#include "xnnpack/operator-type.h"
#include "xnnpack/operator-utils.h"
#include "xnnpack/operator.h"
#include "xnnpack/pack.h"
#include "xnnpack/params.h"
#include "pthreadpool.h"

static enum xnn_status create_spmm_path(
    const uint32_t kernel_height,
    const uint32_t kernel_width,
    const uint32_t groups,
    const size_t group_input_channels,
    const size_t group_output_channels,
    const void* kernel,
    const void* bias,
    const uint32_t log2_filter_element_size,
    const xnn_analyze_spmm_w_fn xnn_analyze_spmm,
    const xnn_pack_spmm_w_fn xnn_pack_spmm,
    const struct xnn_spmm_config* spmm_config,
    const struct xnn_spmm_config* spmm2_config,
    const struct xnn_spmm_config* spmm4_config,
    const enum xnn_operator_type operator_type,
    const xnn_operator_t convolution_op)
{
  assert(spmm_config != NULL);
  assert(kernel_height == 1);
  assert(kernel_width == 1);
  assert(groups == 1);

  // Count number of non-zero values.
  struct xnn_spmm_packing_params spmm_packing_params;

  xnn_analyze_spmm(group_output_channels, group_input_channels, kernel, &spmm_packing_params);

  size_t num_nonzeroes = spmm_packing_params.num_nonzeroes;
  size_t num_nonzero_blocks2 = spmm_packing_params.num_nonzero_blocks2;
  size_t num_nonzero_blocks4 = spmm_packing_params.num_nonzero_blocks4;
  size_t num_block2_nonzeroes = spmm_packing_params.num_block2_nonzeroes;
  size_t num_block4_nonzeroes = spmm_packing_params.num_block4_nonzeroes;

  // Select block encoding when 2 or 4 channels have non-zero values.
  size_t output_channels_block_size = 1;
  size_t num_output_channel_blocks = group_output_channels;
  size_t num_nonzero_values = num_nonzeroes;
  size_t num_nonzero_blocks = num_nonzeroes;
  if (num_block4_nonzeroes * 5 >= num_nonzero_blocks4 * 18 && spmm4_config != NULL && spmm4_config->ukernel != NULL) {
    // 4-channel blocks have 90%+ non-zeroes

    output_channels_block_size = 4;
    num_output_channel_blocks = num_output_channel_blocks / 4 + num_output_channel_blocks % 4;
    spmm_config = spmm4_config;
    // Non-zeroes which don't fit into whole 4-channel blocks, processed one-by-one
    const size_t num_remaining_nonzeroes = num_nonzeroes - num_block4_nonzeroes;
    num_nonzero_values = num_nonzero_blocks4 * 4 + num_remaining_nonzeroes;
    num_nonzero_blocks = num_nonzero_blocks4 + num_remaining_nonzeroes;
  } else if (num_block2_nonzeroes * 5 >= num_nonzero_blocks2 * 9 && spmm2_config != NULL && spmm2_config->ukernel != NULL) {
    // 2-channel blocks have 90%+ non-zeroes

    output_channels_block_size = 2;
    num_output_channel_blocks = num_output_channel_blocks / 2 + num_output_channel_blocks % 2;
    spmm_config = spmm2_config;
    // Non-zeroes which don't fit into whole 2-channel blocks, processed one-by-one
    const size_t num_remaining_nonzeroes = num_nonzeroes - num_block2_nonzeroes;
    num_nonzero_values = num_nonzero_blocks2 * 2 + num_remaining_nonzeroes;
    num_nonzero_blocks = num_nonzero_blocks2 + num_remaining_nonzeroes;
  }

  // Sparse representation of weights consists of four components:
  // 1. An array of int32_t values storing scaled [by sizeof(input element)] difference between input channels
  //    corresponding to successive non-zero blocks.  Used by setup to compute (array 2).
  // 2. An array of int32_t values storing increment for input pointer after each processed tile. This array is
  //    derived from scaled difference in array 1 using parameters to setup function.
  // 3. An array of uint32_t values storing the number of non-zero kernel elements per each output channel.
  // 4. An array of float or fp16 values storing all bias elements (group_output_channels) and non-zero kernel elements.
  //    All elements within non-zero block are assumed to be non-zero.

  const size_t packed_weights_size =
    num_nonzero_blocks * 2 * sizeof(int32_t) +
    num_output_channel_blocks * sizeof(uint32_t) +
    ((group_output_channels + num_nonzero_values) << log2_filter_element_size) + XNN_EXTRA_BYTES;

  convolution_op->packed_weights.pointer = xnn_allocate_simd_memory(packed_weights_size);
  if (convolution_op->packed_weights.pointer == NULL) {
    xnn_log_error(
      "failed to allocate %zu bytes for %s operator packed weights",
      packed_weights_size, xnn_operator_type_to_string(operator_type));
    return xnn_status_out_of_memory;
  }
  xnn_log_debug("allocated %zu bytes for packed weights in %s operator",
    packed_weights_size, xnn_operator_type_to_string(operator_type));

  convolution_op->num_nonzero_values = num_nonzero_values;
  convolution_op->num_nonzero_blocks = num_nonzero_blocks;
  convolution_op->num_output_channel_blocks = num_output_channel_blocks;

  int32_t* input_channel_diffs = (int32_t*) convolution_op->packed_weights.pointer;
  int32_t* input_increments = (int32_t*) (input_channel_diffs + num_nonzero_blocks);
  uint32_t* output_channel_nonzeros = (uint32_t*) (input_increments + num_nonzero_blocks);
  void* nonzero_values = (void*) (output_channel_nonzeros + num_output_channel_blocks);

  memset(output_channel_nonzeros, 0, num_output_channel_blocks * sizeof(uint32_t));

  size_t first_ic = 0;
  enum xnn_status status = xnn_pack_spmm(
        group_output_channels,
        output_channels_block_size,
        group_input_channels,
        kernel,
        bias,
        input_channel_diffs,
        output_channel_nonzeros,
        nonzero_values,
        &first_ic);

  if (status != xnn_status_success) {
    goto error;
  }
  convolution_op->first_input_channel = first_ic;

  convolution_op->ukernel.spmm = (struct xnn_ukernel_spmm) {
    .function = spmm_config->ukernel,
    .mr = spmm_config->mr,
  };
  return xnn_status_success;

error:
  xnn_release_simd_memory(convolution_op->packed_weights.pointer);
  return status;
}

static enum xnn_status create_conv2d_hwc2chw_path(
    const uint32_t kernel_height,
    const uint32_t kernel_width,
    const uint32_t groups,
    const size_t group_input_channels,
    const size_t group_output_channels,
    const size_t output_height_tile,
    const size_t output_channel_tile,
    const void* kernel,
    const void* bias,
    const uint32_t log2_filter_element_size,
    const xnn_pack_dconv_oki_w_fn xnn_pack_dconv_oki_w,
    const xnn_conv_hwc2chw_ukernel_fn conv_hwc2chw_ukernel,
    const enum xnn_operator_type operator_type,
    const xnn_operator_t convolution_op)
{
  assert(conv_hwc2chw_ukernel != NULL);

  const size_t packed_group_output_channels = round_up(group_output_channels, output_channel_tile);
  const size_t packed_weights_size = (groups * packed_group_output_channels *
    (group_input_channels * kernel_height * kernel_width + 1 /* bias */)) << log2_filter_element_size;
  const size_t aligned_total_weights_size = round_up_po2(packed_weights_size, XNN_ALLOCATION_ALIGNMENT);
  void* weights_ptr = xnn_get_pointer_to_write_weights(convolution_op, aligned_total_weights_size, 0);
  if (weights_ptr == NULL) {
    xnn_log_error("failed to reserve or allocate %zu bytes for %s operator conv2d_hwc2chw packed weights",
                  aligned_total_weights_size,
                  xnn_operator_type_to_string(operator_type));
    return xnn_status_out_of_memory;
  }
  xnn_log_debug("allocated %zu bytes for packed weights in %s operator",
    aligned_total_weights_size, xnn_operator_type_to_string(operator_type));

  xnn_pack_dconv_oki_w(
    group_output_channels,
    group_input_channels,
    output_channel_tile,
    kernel_height, kernel_width,
    kernel, bias, weights_ptr, NULL);

  if (use_weights_cache(convolution_op)) {
    struct xnn_weights_cache_look_up_key cache_key;
    cache_key.seed = group_input_channels ^ group_output_channels ^ output_channel_tile;
    cache_key.kernel = kernel;
    cache_key.bias = bias;
    convolution_op->packed_weights.offset = xnn_look_up_or_insert_weights_cache(
        convolution_op->weights_cache, &cache_key, weights_ptr, aligned_total_weights_size);
  }

  convolution_op->ukernel.conv2d = (struct xnn_ukernel_conv2d) {
    .hwc2chw_fn = conv_hwc2chw_ukernel,
    .output_height_tile = output_height_tile,
    .output_channel_tile = output_channel_tile,
  };
  return xnn_status_success;
}

static enum xnn_status create_dwconv_path(
    const uint32_t kernel_height,
    const uint32_t kernel_width,
    const uint32_t groups,
    const void* kernel,
    const void* bias,
    const uint32_t flags,
    const uint32_t log2_filter_element_size,
    const xnn_pack_chw_dwconv_hwg_w_fn pack_chw_dwconv_hwg_w,
    const xnn_pack_chw_dwconv_ghw_w_fn pack_chw_dwconv_ghw_w,
    const size_t output_width_tile,
    const xnn_dwconv2d_chw_ukernel_fn dwconv_ukernel,
    const enum xnn_operator_type operator_type,
    const xnn_operator_t convolution_op)
{
  assert(dwconv_ukernel != NULL);

  const size_t packed_weights_size = (groups * (kernel_height * kernel_width + 1 /* bias */)) << log2_filter_element_size;
  const size_t aligned_total_weights_size = round_up_po2(packed_weights_size, XNN_ALLOCATION_ALIGNMENT);
  void* weights_ptr = xnn_get_pointer_to_write_weights(
      convolution_op, aligned_total_weights_size, 0);
  if (weights_ptr == NULL) {
    xnn_log_error("failed to reserve or allocated %zu bytes for %s operator dwconv packed weights",
                  aligned_total_weights_size, xnn_operator_type_to_string(operator_type));
    return xnn_status_out_of_memory;
  }
  xnn_log_debug("allocated %zu bytes for packed weights in %s operator",
                aligned_total_weights_size, xnn_operator_type_to_string(operator_type));

  uint32_t cache_seed = kernel_height ^ kernel_width ^ groups;
  if (flags & XNN_FLAG_DEPTHWISE_CONVOLUTION) {
    pack_chw_dwconv_hwg_w(
      kernel_height * kernel_width, groups,
      kernel, bias, weights_ptr, NULL);
  } else {
    cache_seed = ~cache_seed;
    pack_chw_dwconv_ghw_w(
      kernel_height * kernel_width, groups,
      kernel, bias, weights_ptr, NULL);
  }

  if (use_weights_cache(convolution_op)) {
    struct xnn_weights_cache_look_up_key cache_key;
    cache_key.seed = cache_seed;
    cache_key.kernel = kernel;
    cache_key.bias = bias;
    convolution_op->packed_weights.offset = xnn_look_up_or_insert_weights_cache(
        convolution_op->weights_cache, &cache_key, weights_ptr, aligned_total_weights_size);
  }

  convolution_op->ukernel.dwconv2d = (struct xnn_ukernel_dwconv2d) {
    .chw_fn = dwconv_ukernel,
    .output_width_tile = output_width_tile,
  };

  return xnn_status_success;
}

enum xnn_status xnn_create_convolution2d_nchw_f16(
    uint32_t input_padding_top,
    uint32_t input_padding_right,
    uint32_t input_padding_bottom,
    uint32_t input_padding_left,
    uint32_t kernel_height,
    uint32_t kernel_width,
    uint32_t subsampling_height,
    uint32_t subsampling_width,
    uint32_t dilation_height,
    uint32_t dilation_width,
    uint32_t groups,
    size_t group_input_channels,
    size_t group_output_channels,
    size_t input_channel_stride,
    size_t output_channel_stride,
    const void* kernel,
    const void* bias,
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_code_cache_t code_cache,
    xnn_weights_cache_t weights_cache,
    xnn_operator_t* convolution_op_out)
{
  xnn_operator_t convolution_op = NULL;
  enum xnn_status status = xnn_status_uninitialized;

  const size_t log2_filter_element_size = XNN_LOG2_SIZEOF_HALF;
  const enum xnn_operator_type operator_type = xnn_operator_type_convolution_nchw_f16;

  if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
    xnn_log_error("failed to create %s operator: XNNPACK is not initialized",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  status = xnn_status_invalid_parameter;

  if (kernel_width == 0 || kernel_height == 0) {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 "x%" PRIu32 " kernel: kernel dimensions must be non-zero",
      xnn_operator_type_to_string(operator_type), kernel_width, kernel_height);
    goto error;
  }

  if (subsampling_width == 0 || subsampling_height == 0) {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 "x%" PRIu32 " subsampling: subsampling dimensions must be non-zero",
      xnn_operator_type_to_string(operator_type), subsampling_width, subsampling_height);
    goto error;
  }

  if (dilation_width == 0 || dilation_height == 0) {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 "x%" PRIu32 " dilation: dilation dimensions must be non-zero",
      xnn_operator_type_to_string(operator_type), dilation_width, dilation_height);
    goto error;
  }

  if (groups == 0) {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 " groups: number of groups must be non-zero",
      xnn_operator_type_to_string(operator_type), groups);
    goto error;
  }

  if (group_input_channels == 0) {
    xnn_log_error(
      "failed to create %s operator with %zu input channels per group: number of channels must be non-zero",
      xnn_operator_type_to_string(operator_type), group_input_channels);
    goto error;
  }

  if (group_output_channels == 0) {
    xnn_log_error(
      "failed to create %s operator with %zu output channels per group: number of channels must be non-zero",
      xnn_operator_type_to_string(operator_type), group_output_channels);
    goto error;
  }

  const size_t input_channels = groups * group_input_channels;
  if (input_channel_stride < input_channels) {
    xnn_log_error(
      "failed to create %s operator with input channel stride of %zu: "
      "stride must be at least as large as the number of input channels (%" PRIu32 "x%zu)",
      xnn_operator_type_to_string(operator_type),
      input_channel_stride, groups, group_input_channels);
    goto error;
  }

  const size_t output_channels = groups * group_output_channels;
  if (output_channel_stride < output_channels) {
    xnn_log_error(
      "failed to create %s operator with output channel stride of %zu: "
      "stride must be at least as large as the number of output channels (%" PRIu32 "x%zu)",
      xnn_operator_type_to_string(operator_type),
      output_channel_stride, groups, group_output_channels);
    goto error;
  }

  if (isnan(output_min)) {
    xnn_log_error(
      "failed to create %s operator with NaN output lower bound: lower bound must be non-NaN",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  if (isnan(output_max)) {
    xnn_log_error(
      "failed to create %s operator with NaN output upper bound: upper bound must be non-NaN",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  const xnn_float16 fp16_output_min = xnn_float16_from_float(output_min);
  const xnn_float16 fp16_output_max = xnn_float16_from_float(output_max);
  const float rounded_output_min = xnn_float16_to_float(fp16_output_min);
  const float rounded_output_max = xnn_float16_to_float(fp16_output_max);
  if (rounded_output_min >= rounded_output_max) {
    xnn_log_error(
      "failed to create %s operator with [%.7g, %.7g] output range: lower bound must be below upper bound",
      xnn_operator_type_to_string(operator_type), rounded_output_min, rounded_output_max);
    goto error;
  }

  if ((flags & XNN_FLAG_DEPTHWISE_CONVOLUTION) != 0 && group_input_channels != 1) {
    xnn_log_error(
      "failed to create depthwise %s operator with %zu input channels per group: "
      "depthwise convolution must have exactly 1 input channel per group",
      xnn_operator_type_to_string(operator_type), group_input_channels);
    goto error;
  }

  status = xnn_status_unsupported_hardware;

  const struct xnn_spmm_config* spmm_config = xnn_init_f16_spmm_config();
  if (spmm_config == NULL) {
    xnn_log_error("failed to create %s operator: operations on data type are not supported",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  const struct xnn_dwconv2d_chw_config* dwconv2d_chw_config = xnn_init_f16_dwconv2d_chw_config();
  if (dwconv2d_chw_config == NULL) {
    xnn_log_error("failed to create %s operator: operations on data type are not supported",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  status = xnn_status_unsupported_parameter;

  enum xnn_microkernel_type ukernel_type;
  const struct xnn_dwconv2d_chw_parameters* dwconv2d_parameters = NULL;
  // Supported cases:
  // + 1x1 convolution (no groups)
  // + 3x3 stride-2 with 3 input channels and NHWC input layout
  // + 3x3 stride-2 depthwise convolution with horizontal padding 1 & no vertical padding
  // + 3x3 stride-1 depthwise convolution with horizontal padding 1 & no vertical padding
  // + 5x5 stride-2 depthwise convolution with horizontal padding 2 & no vertical padding
  // + 5x5 stride-1 depthwise convolution with horizontal padding 2 & no vertical padding
  const bool any_padding = (input_padding_left | input_padding_top | input_padding_right | input_padding_bottom) != 0;
  const bool is_1x1 = kernel_width == 1 && kernel_height == 1 && subsampling_height == 1 && subsampling_width == 1;
  const bool is_3x3 = kernel_width == 3 && kernel_height == 3 && dilation_height == 1 && dilation_width == 1;
  const bool is_5x5 = kernel_width == 5 && kernel_height == 5 && dilation_height == 1 && dilation_width == 1;
  const bool nhwc_input = (flags & XNN_FLAG_INPUT_NHWC) != 0;
  if (is_1x1 && !any_padding && !nhwc_input && groups == 1) {
    ukernel_type = xnn_microkernel_type_spmm;
  } else if (is_3x3 && subsampling_height == 2 && subsampling_width == 2 &&
    input_padding_top == 1 && input_padding_left == 1 && input_padding_bottom == 1 && input_padding_right == 1 &&
    nhwc_input && groups == 1)
  {
    ukernel_type = xnn_microkernel_type_conv2d_hwc2chw;
  } else if (is_3x3 && subsampling_height == 1 && subsampling_width == 1 &&
    input_padding_top == 1 && input_padding_left == 1 && input_padding_bottom == 1 && input_padding_right == 1 &&
    !nhwc_input && group_input_channels == 1 && group_output_channels == 1)
  {
    ukernel_type = xnn_microkernel_type_dwconv;
    dwconv2d_parameters = &dwconv2d_chw_config->dwconv2d_chw_3x3;
  } else if (is_3x3 && subsampling_height == 2 && subsampling_width == 2 &&
    (input_padding_top == 0 || input_padding_top == 1) && input_padding_left == 1 && input_padding_bottom == 1 && input_padding_right == 1 &&
    !nhwc_input && group_input_channels == 1 && group_output_channels == 1)
  {
    ukernel_type = xnn_microkernel_type_dwconv;
    dwconv2d_parameters = &dwconv2d_chw_config->dwconv2d_chw_3x3s2;
  } else if (is_5x5 && subsampling_height == 1 && subsampling_width == 1 &&
    input_padding_top == 2 && input_padding_left == 2 && input_padding_bottom == 2 && input_padding_right == 2 &&
    !nhwc_input && group_input_channels == 1 && group_output_channels == 1)
  {
    ukernel_type = xnn_microkernel_type_dwconv;
    dwconv2d_parameters = &dwconv2d_chw_config->dwconv2d_chw_5x5;
  } else if (is_5x5 && subsampling_height == 2 && subsampling_width == 2 &&
    (input_padding_top == 1 || input_padding_top == 2) && input_padding_left == 2 && input_padding_bottom == 2 && input_padding_right == 2 &&
    !nhwc_input && group_input_channels == 1 && group_output_channels == 1)
  {
    ukernel_type = xnn_microkernel_type_dwconv;
    dwconv2d_parameters = &dwconv2d_chw_config->dwconv2d_chw_5x5s2;
  } else {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 "x%" PRIu32 " kernel, %"PRIu32 "x%" PRIu32 " subsampling, %"PRIu32 "x%" PRIu32 " dilation"
      ", %" PRIu32 "+%" PRIu32 "x%" PRIu32 "+%" PRIu32" padding, %" PRIu32 "x%zu input channels, and %" PRIu32 "x%zu output channels: "
      "only selected convolution parameters are supported",
      xnn_operator_type_to_string(operator_type),
      kernel_width, kernel_height, subsampling_width, subsampling_height, dilation_width, dilation_height,
      input_padding_top, input_padding_left, input_padding_bottom, input_padding_right,
      groups, group_input_channels, groups, group_output_channels);
    goto error;
  }

  status = xnn_status_out_of_memory;

  convolution_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));
  if (convolution_op == NULL) {
    xnn_log_error(
      "failed to allocate %zu bytes for %s operator descriptor",
      sizeof(struct xnn_operator), xnn_operator_type_to_string(operator_type));
    goto error;
  }

  if (ukernel_type != xnn_microkernel_type_spmm) {
    convolution_op->weights_cache = weights_cache;
  }

  switch (ukernel_type) {
    case xnn_microkernel_type_spmm:
    {
      xnn_analyze_spmm_w_fn xnn_analyze_spmm;
      xnn_pack_spmm_w_fn xnn_pack_spmm;
      if (flags & XNN_FLAG_FP32_STATIC_WEIGHTS) {
        xnn_analyze_spmm = (xnn_analyze_spmm_w_fn) xnn_analyze_f32_spmm_w;
        xnn_pack_spmm = (xnn_pack_spmm_w_fn) xnn_pack_f32_to_f16_spmm_w;
      } else {
        xnn_analyze_spmm = (xnn_analyze_spmm_w_fn) xnn_analyze_f16_spmm_w;
        xnn_pack_spmm = (xnn_pack_spmm_w_fn) xnn_pack_f16_spmm_w;
      }

      spmm_config->init.f16(&convolution_op->params.f16_minmax, fp16_output_min, fp16_output_max);

      status = create_spmm_path(
          kernel_height, kernel_width, groups,
          group_input_channels, group_output_channels,
          kernel, bias, log2_filter_element_size,
          xnn_analyze_spmm, xnn_pack_spmm,
          spmm_config, NULL, NULL,
          operator_type, convolution_op);
      if (status != xnn_status_success) {
        goto error;
      }
      break;
    }
    case xnn_microkernel_type_conv2d_hwc2chw:
    {
      assert(groups == 1);
      xnn_pack_dconv_oki_w_fn xnn_pack_dconv_oki_w = (xnn_pack_dconv_oki_w_fn) xnn_pack_f16_dconv_oki_w;
      if (flags & XNN_FLAG_FP32_STATIC_WEIGHTS) {
        xnn_pack_dconv_oki_w = (xnn_pack_dconv_oki_w_fn) xnn_pack_f32_to_f16_dconv_oki_w;
      }

      const struct xnn_conv_hwc2chw_config* conv_hwc2chw_config = xnn_init_f16_conv_hwc2chw_3x3c3s2_config();
      if (conv_hwc2chw_config == NULL) {
        status = xnn_status_unsupported_hardware;
        xnn_log_error("failed to create %s operator: operations on data type are not supported",
                      xnn_operator_type_to_string(operator_type));
        goto error;
      }

      conv_hwc2chw_config->init.f16(&convolution_op->params.f16_minmax, fp16_output_min, fp16_output_max);

      status = create_conv2d_hwc2chw_path(
          kernel_height, kernel_width, groups,
          group_input_channels,
          group_output_channels,
          conv_hwc2chw_config->output_height_tile,
          conv_hwc2chw_config->output_channel_tile,
          kernel, bias, log2_filter_element_size,
          xnn_pack_dconv_oki_w,
          conv_hwc2chw_config->ukernel_with_symm_padding,
          operator_type, convolution_op);
      if (status != xnn_status_success) {
        goto error;
      }
      break;
    }
    case xnn_microkernel_type_dwconv:
    {
      xnn_pack_chw_dwconv_hwg_w_fn pack_chw_dwconv_hwg_w = (xnn_pack_chw_dwconv_hwg_w_fn) xnn_pack_f16_chw_dwconv_hwg_w;
      xnn_pack_chw_dwconv_ghw_w_fn pack_chw_dwconv_ghw_w = (xnn_pack_chw_dwconv_ghw_w_fn) xnn_pack_f16_chw_dwconv_ghw_w;
      if (flags & XNN_FLAG_FP32_STATIC_WEIGHTS) {
        pack_chw_dwconv_hwg_w = (xnn_pack_chw_dwconv_hwg_w_fn) xnn_pack_f32_to_f16_chw_dwconv_hwg_w;
        pack_chw_dwconv_ghw_w = (xnn_pack_chw_dwconv_ghw_w_fn) xnn_pack_f32_to_f16_chw_dwconv_ghw_w;
      }

      dwconv2d_parameters->init.f16(&convolution_op->params.f16_chw, fp16_output_min, fp16_output_max);

      status = create_dwconv_path(
          kernel_height, kernel_width, groups,
          kernel, bias, flags, log2_filter_element_size,
          pack_chw_dwconv_hwg_w,
          pack_chw_dwconv_ghw_w,
          dwconv2d_parameters->output_width_tile,
          dwconv2d_parameters->ukernel,
          operator_type, convolution_op);
      if (status != xnn_status_success) {
        goto error;
      }
      break;
    }
    default:
      XNN_UNREACHABLE;
  }

  convolution_op->padding_top = input_padding_top;
  convolution_op->padding_right = input_padding_right;
  convolution_op->padding_bottom = input_padding_bottom;
  convolution_op->padding_left = input_padding_left;

  convolution_op->kernel_height = kernel_height;
  convolution_op->kernel_width = kernel_width;
  convolution_op->stride_height = subsampling_height;
  convolution_op->stride_width = subsampling_width;
  convolution_op->dilation_height = dilation_height;
  convolution_op->dilation_width = dilation_width;
  convolution_op->groups = groups;
  convolution_op->group_input_channels = group_input_channels;
  convolution_op->group_output_channels = group_output_channels;
  convolution_op->input_pixel_stride = input_channel_stride;
  convolution_op->output_pixel_stride = output_channel_stride;

  convolution_op->type = operator_type;
  convolution_op->ukernel.type = ukernel_type;
  convolution_op->flags = flags;

  convolution_op->state = xnn_run_state_invalid;
  *convolution_op_out = convolution_op;
  return xnn_status_success;

error:
  xnn_delete_operator(convolution_op);
  return status;
}


enum xnn_status xnn_create_convolution2d_nchw_f32(
    uint32_t input_padding_top,
    uint32_t input_padding_right,
    uint32_t input_padding_bottom,
    uint32_t input_padding_left,
    uint32_t kernel_height,
    uint32_t kernel_width,
    uint32_t subsampling_height,
    uint32_t subsampling_width,
    uint32_t dilation_height,
    uint32_t dilation_width,
    uint32_t groups,
    size_t group_input_channels,
    size_t group_output_channels,
    size_t input_channel_stride,
    size_t output_channel_stride,
    const float* kernel,
    const float* bias,
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_code_cache_t code_cache,
    xnn_weights_cache_t weights_cache,
    xnn_operator_t* convolution_op_out)
{
  xnn_operator_t convolution_op = NULL;
  enum xnn_status status = xnn_status_uninitialized;
  const size_t log2_filter_element_size = XNN_LOG2_SIZEOF_FLOAT;
  const enum xnn_operator_type operator_type = xnn_operator_type_convolution_nchw_f32;

  if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
    xnn_log_error("failed to create %s operator: XNNPACK is not initialized",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  status = xnn_status_invalid_parameter;

  if (kernel_width == 0 || kernel_height == 0) {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 "x%" PRIu32 " kernel: kernel dimensions must be non-zero",
      xnn_operator_type_to_string(operator_type), kernel_width, kernel_height);
    goto error;
  }

  if (subsampling_width == 0 || subsampling_height == 0) {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 "x%" PRIu32 " subsampling: subsampling dimensions must be non-zero",
      xnn_operator_type_to_string(operator_type), subsampling_width, subsampling_height);
    goto error;
  }

  if (dilation_width == 0 || dilation_height == 0) {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 "x%" PRIu32 " dilation: dilation dimensions must be non-zero",
      xnn_operator_type_to_string(operator_type), dilation_width, dilation_height);
    goto error;
  }

  if (groups == 0) {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 " groups: number of groups must be non-zero",
      xnn_operator_type_to_string(operator_type), groups);
    goto error;
  }

  if (group_input_channels == 0) {
    xnn_log_error(
      "failed to create %s operator with %zu input channels per group: number of channels must be non-zero",
      xnn_operator_type_to_string(operator_type), group_input_channels);
    goto error;
  }

  if (group_output_channels == 0) {
    xnn_log_error(
      "failed to create %s operator with %zu output channels per group: number of channels must be non-zero",
      xnn_operator_type_to_string(operator_type), group_output_channels);
    goto error;
  }

  const size_t input_channels = groups * group_input_channels;
  if (input_channel_stride < input_channels) {
    xnn_log_error(
      "failed to create %s operator with input channel stride of %zu: "
      "stride must be at least as large as the number of input channels (%" PRIu32 "x%zu)",
      xnn_operator_type_to_string(operator_type),
      input_channel_stride, groups, group_input_channels);
    goto error;
  }

  const size_t output_channels = groups * group_output_channels;
  if (output_channel_stride < output_channels) {
    xnn_log_error(
      "failed to create %s operator with output channel stride of %zu: "
      "stride must be at least as large as the number of output channels (%" PRIu32 "x%zu)",
      xnn_operator_type_to_string(operator_type),
      output_channel_stride, groups, group_output_channels);
    goto error;
  }

  if (isnan(output_min)) {
    xnn_log_error(
      "failed to create %s operator with NaN output lower bound: lower bound must be non-NaN",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  if (isnan(output_max)) {
    xnn_log_error(
      "failed to create %s operator with NaN output upper bound: upper bound must be non-NaN",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  if (output_min > output_max) {
    xnn_log_error(
      "failed to create %s operator with [%.7g, %.7g] output range: lower bound must be less than or equal to upper bound",
      xnn_operator_type_to_string(operator_type), output_min, output_max);
    goto error;
  }

  if ((flags & XNN_FLAG_DEPTHWISE_CONVOLUTION) != 0 && group_input_channels != 1) {
    xnn_log_error(
      "failed to create depthwise %s operator with %zu input channels per group: "
      "depthwise convolution must have exactly 1 input channel per group",
      xnn_operator_type_to_string(operator_type), group_input_channels);
    goto error;
  }

  status = xnn_status_unsupported_hardware;

  const struct xnn_dwconv2d_chw_config* dwconv2d_chw_config = xnn_init_f32_dwconv2d_chw_config();
  if (dwconv2d_chw_config == NULL) {
    xnn_log_error("failed to create %s operator: operations on data type are not supported",
      xnn_operator_type_to_string(operator_type));
    goto error;
  }

  status = xnn_status_unsupported_parameter;

  enum xnn_microkernel_type ukernel_type;
  const struct xnn_dwconv2d_chw_parameters* dwconv2d_parameters = NULL;
  // Supported cases:
  // + 1x1 convolution (no groups)
  // + 3x3 stride-2 with 3 input channels and NHWC input layout
  // + 3x3 stride-2 depthwise convolution with horizontal padding 1 & no vertical padding
  // + 3x3 stride-1 depthwise convolution with horizontal padding 1 & no vertical padding
  // + 5x5 stride-2 depthwise convolution with horizontal padding 2 & no vertical padding
  // + 5x5 stride-1 depthwise convolution with horizontal padding 2 & no vertical padding
  const bool any_padding = (input_padding_left | input_padding_top | input_padding_right | input_padding_bottom) != 0;
  const bool is_1x1 = kernel_width == 1 && kernel_height == 1 && subsampling_height == 1 && subsampling_width == 1;
  const bool is_3x3 = kernel_width == 3 && kernel_height == 3 && dilation_height == 1 && dilation_width == 1;
  const bool is_5x5 = kernel_width == 5 && kernel_height == 5 && dilation_height == 1 && dilation_width == 1;
  const bool nhwc_input = (flags & XNN_FLAG_INPUT_NHWC) != 0;
  if (is_1x1 && !any_padding && !nhwc_input && groups == 1) {
    ukernel_type = xnn_microkernel_type_spmm;
  } else if (is_3x3 && subsampling_height == 2 && subsampling_width == 2 &&
    input_padding_top == 1 && input_padding_left == 1 && input_padding_bottom == 1 && input_padding_right == 1 &&
    nhwc_input && groups == 1)
  {
    ukernel_type = xnn_microkernel_type_conv2d_hwc2chw;
  } else if (is_3x3 && subsampling_height == 1 && subsampling_width == 1 &&
    input_padding_top == 1 && input_padding_left == 1 && input_padding_bottom == 1 && input_padding_right == 1 &&
    !nhwc_input && group_input_channels == 1 && group_output_channels == 1)
  {
    ukernel_type = xnn_microkernel_type_dwconv;
    dwconv2d_parameters = &dwconv2d_chw_config->dwconv2d_chw_3x3;
  } else if (is_3x3 && subsampling_height == 2 && subsampling_width == 2 &&
    (input_padding_top == 0 || input_padding_top == 1) && input_padding_left == 1 && input_padding_bottom == 1 && input_padding_right == 1 &&
    !nhwc_input && group_input_channels == 1 && group_output_channels == 1)
  {
    ukernel_type = xnn_microkernel_type_dwconv;
    dwconv2d_parameters = &dwconv2d_chw_config->dwconv2d_chw_3x3s2;
  } else if (is_5x5 && subsampling_height == 1 && subsampling_width == 1 &&
    input_padding_top == 2 && input_padding_left == 2 && input_padding_bottom == 2 && input_padding_right == 2 &&
    !nhwc_input && group_input_channels == 1 && group_output_channels == 1)
  {
    ukernel_type = xnn_microkernel_type_dwconv;
    dwconv2d_parameters = &dwconv2d_chw_config->dwconv2d_chw_5x5;
  } else if (is_5x5 && subsampling_height == 2 && subsampling_width == 2 &&
    (input_padding_top == 1 || input_padding_top == 2) && input_padding_left == 2 && input_padding_bottom == 2 && input_padding_right == 2 &&
    !nhwc_input && group_input_channels == 1 && group_output_channels == 1)
  {
    ukernel_type = xnn_microkernel_type_dwconv;
    dwconv2d_parameters = &dwconv2d_chw_config->dwconv2d_chw_5x5s2;
  } else {
    xnn_log_error(
      "failed to create %s operator with %" PRIu32 "x%" PRIu32 " kernel, %"PRIu32 "x%" PRIu32 " subsampling, %"PRIu32 "x%" PRIu32 " dilation"
      ", %" PRIu32 "+%" PRIu32 "x%" PRIu32 "+%" PRIu32" padding, %" PRIu32 "x%zu input channels, and %" PRIu32 "x%zu output channels: "
      "only selected convolution parameters are supported",
      xnn_operator_type_to_string(operator_type),
      kernel_width, kernel_height, subsampling_width, subsampling_height, dilation_width, dilation_height,
      input_padding_top, input_padding_left, input_padding_bottom, input_padding_right,
      groups, group_input_channels, groups, group_output_channels);
    goto error;
  }

  status = xnn_status_out_of_memory;

  convolution_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));
  if (convolution_op == NULL) {
    xnn_log_error(
      "failed to allocate %zu bytes for %s operator descriptor",
      sizeof(struct xnn_operator), xnn_operator_type_to_string(operator_type));
    goto error;
  }

  if (ukernel_type != xnn_microkernel_type_spmm) {
    convolution_op->weights_cache = weights_cache;
  }

  const struct xnn_spmm_config* spmm_config = xnn_init_f32_spmm_config();
  if (spmm_config == NULL) {
    xnn_log_error(
      "failed to create %s operator: unsupported hardware configuration",
      xnn_operator_type_to_string(xnn_operator_type_convolution_nchw_f32));
    return xnn_status_unsupported_hardware;
  }
  const struct xnn_spmm_config* spmm2_config = xnn_init_f32_spmm2_config();
  if (spmm2_config == NULL) {
    xnn_log_error(
      "failed to create %s operator: unsupported hardware configuration",
      xnn_operator_type_to_string(xnn_operator_type_convolution_nchw_f32));
    return xnn_status_unsupported_hardware;
  }
  const struct xnn_spmm_config* spmm4_config = xnn_init_f32_spmm4_config();
  if (spmm4_config == NULL) {
    xnn_log_error(
      "failed to create %s operator: unsupported hardware configuration",
      xnn_operator_type_to_string(xnn_operator_type_convolution_nchw_f32));
    return xnn_status_unsupported_hardware;
  }

  switch (ukernel_type) {
    case xnn_microkernel_type_spmm:
    {
      const xnn_analyze_spmm_w_fn xnn_analyze_spmm = (xnn_analyze_spmm_w_fn) xnn_analyze_f32_spmm_w;
      const xnn_pack_spmm_w_fn xnn_pack_spmm = (xnn_pack_spmm_w_fn) xnn_pack_f32_spmm_w;

      spmm_config->init.f32(&convolution_op->params.f32_minmax, output_min, output_max);

      status = create_spmm_path(
          kernel_height, kernel_width, groups,
          group_input_channels, group_output_channels,
          kernel, bias, log2_filter_element_size,
          xnn_analyze_spmm, xnn_pack_spmm,
          spmm_config, spmm2_config, spmm4_config,
          operator_type, convolution_op);
      if (status != xnn_status_success) {
        goto error;
      }
      break;
    }
    case xnn_microkernel_type_conv2d_hwc2chw:
    {
      const struct xnn_conv_hwc2chw_config* conv_hwc2chw_config = xnn_init_f32_conv_hwc2chw_3x3c3s2_config();
      if (conv_hwc2chw_config == NULL) {
        status = xnn_status_unsupported_hardware;
        xnn_log_error("failed to create %s operator: operations on data type are not supported",
                      xnn_operator_type_to_string(operator_type));
        goto error;
      }

      conv_hwc2chw_config->init.f32(&convolution_op->params.f32_minmax, output_min, output_max);

      status = create_conv2d_hwc2chw_path(
          kernel_height, kernel_width, groups,
          group_input_channels,
          group_output_channels,
          conv_hwc2chw_config->output_height_tile,
          conv_hwc2chw_config->output_channel_tile,
          kernel, bias, log2_filter_element_size,
          (xnn_pack_dconv_oki_w_fn) xnn_pack_f32_dconv_oki_w,
          conv_hwc2chw_config->ukernel_with_symm_padding,
          operator_type, convolution_op);
      if (status != xnn_status_success) {
        goto error;
      }
      break;
    }
    case xnn_microkernel_type_dwconv:
    {
      dwconv2d_parameters->init.f32(&convolution_op->params.f32_chw, output_min, output_max);

      status = create_dwconv_path(
          kernel_height, kernel_width, groups,
          kernel, bias, flags, log2_filter_element_size,
          (xnn_pack_chw_dwconv_hwg_w_fn) xnn_pack_f32_chw_dwconv_hwg_w,
          (xnn_pack_chw_dwconv_ghw_w_fn) xnn_pack_f32_chw_dwconv_ghw_w,
          dwconv2d_parameters->output_width_tile,
          dwconv2d_parameters->ukernel,
          operator_type, convolution_op);
      if (status != xnn_status_success) {
        goto error;
      }
      break;
    }
    default:
      XNN_UNREACHABLE;
  }

  convolution_op->padding_top = input_padding_top;
  convolution_op->padding_right = input_padding_right;
  convolution_op->padding_bottom = input_padding_bottom;
  convolution_op->padding_left = input_padding_left;

  convolution_op->kernel_height = kernel_height;
  convolution_op->kernel_width = kernel_width;
  convolution_op->stride_height = subsampling_height;
  convolution_op->stride_width = subsampling_width;
  convolution_op->dilation_height = dilation_height;
  convolution_op->dilation_width = dilation_width;
  convolution_op->groups = groups;
  convolution_op->group_input_channels = group_input_channels;
  convolution_op->group_output_channels = group_output_channels;
  convolution_op->input_pixel_stride = input_channel_stride;
  convolution_op->output_pixel_stride = output_channel_stride;

  convolution_op->type = operator_type;
  convolution_op->ukernel.type = ukernel_type;
  convolution_op->flags = flags;

  convolution_op->state = xnn_run_state_invalid;

  *convolution_op_out = convolution_op;
  return xnn_status_success;

error:
  xnn_delete_operator(convolution_op);
  return status;
}

static enum xnn_status reshape_convolution2d_nchw(
  xnn_operator_t convolution_op,
  enum xnn_operator_type expected_operator_type,
  size_t batch_size,
  size_t input_height,
  size_t input_width,
  uint32_t log2_input_element_size,
  uint32_t log2_filter_element_size,
  uint32_t bias_element_size,
  uint32_t log2_output_element_size,
  const void* params,
  void* chw_params,
  size_t* output_height_out,
  size_t* output_width_out,
  pthreadpool_t threadpool)
{
  if (convolution_op->type != expected_operator_type) {
    xnn_log_error("failed to reshape operator: operator type mismatch (expected %s, got %s)",
      xnn_operator_type_to_string(expected_operator_type),
      xnn_operator_type_to_string(convolution_op->type));
    return xnn_status_invalid_parameter;
  }
  convolution_op->state = xnn_run_state_invalid;

  if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
    xnn_log_error("failed to reshape %s operator: XNNPACK is not initialized",
      xnn_operator_type_to_string(convolution_op->type));
    return xnn_status_uninitialized;
  }

  if (input_width == 0 || input_height == 0) {
    xnn_log_error(
      "failed to reshape %s operator with %zux%zu input: input dimensions must be non-zero",
      xnn_operator_type_to_string(convolution_op->type), input_width, input_height);
    return xnn_status_invalid_parameter;
  }

  if (batch_size == 0) {
    convolution_op->state = xnn_run_state_skip;
    return xnn_status_success;
  }

  convolution_op->batch_size = batch_size;
  convolution_op->input_height = input_height;
  convolution_op->input_width = input_width;

  const size_t output_height = xnn_compute_convolution_output_dimension(
      convolution_op->padding_top + input_height + convolution_op->padding_bottom,
      convolution_op->kernel_height,
      convolution_op->dilation_height,
      convolution_op->stride_height);
  if (output_height_out != NULL) {
    *output_height_out = output_height;
  }
  const size_t output_width = xnn_compute_convolution_output_dimension(
      convolution_op->padding_left + input_width + convolution_op->padding_right,
      convolution_op->kernel_width,
      convolution_op->dilation_width,
      convolution_op->stride_width);
  if (output_width_out != NULL) {
    *output_width_out = output_width;
  }

  const size_t input_batch_stride = (input_height * input_width * convolution_op->input_pixel_stride) << log2_input_element_size;
  const size_t output_batch_stride = (output_height * output_width * convolution_op->output_pixel_stride) << log2_output_element_size;
  const size_t num_threads = pthreadpool_get_threads_count(threadpool);
  switch (convolution_op->ukernel.type) {
    case xnn_microkernel_type_spmm:
    {
      const size_t num_nonzero_values = convolution_op->num_nonzero_values;
      const size_t num_nonzero_blocks = convolution_op->num_nonzero_blocks;
      const size_t num_output_channel_blocks = convolution_op->num_output_channel_blocks;

      convolution_op->num_nonzero_values = num_nonzero_values;
      convolution_op->num_nonzero_blocks = num_nonzero_blocks;
      convolution_op->num_output_channel_blocks = num_output_channel_blocks;

      const int32_t* input_channel_diffs = (const int32_t*) packed_weights(convolution_op);
      int32_t* input_increments = ((int32_t*) packed_weights(convolution_op)) + num_nonzero_blocks;
      const uint32_t* output_channel_nonzeros = (uint32_t*) (input_increments + num_nonzero_blocks);
      const void* nonzero_values = (const void*) (output_channel_nonzeros + num_output_channel_blocks);

      // Scale input_channel_diffs by input_size to compute input_increments;
      const size_t input_size = input_height * input_width;
      for (size_t i = 0; i < num_nonzero_blocks; i++) {
        const int32_t diff = input_channel_diffs[i];
        const int64_t increment = (int64_t) diff * input_size;
        if ((int64_t) (int32_t) increment != increment) {
          xnn_log_error(
            "failed to reshape %s operator with sparse kernel representation: input increment exceeds int32_t range",
            xnn_operator_type_to_string(convolution_op->type));
          return xnn_status_unsupported_parameter;
        }
        input_increments[i] = (int32_t) increment;
      }

      convolution_op->context.spmm = (struct spmm_context) {
          .n = convolution_op->group_output_channels,
          .scaled_m = input_size << log2_input_element_size,
          .nonzero_weights = nonzero_values,
          .input_increments = input_increments,
          .output_channel_nonzeros = output_channel_nonzeros,
          .batched_input_stride = input_batch_stride,
          .batched_output_stride = output_batch_stride,
          .ukernel = convolution_op->ukernel.spmm.function,
      };
      memcpy(&convolution_op->context.spmm.params, params, sizeof(convolution_op->context.spmm.params));

      const size_t mr = convolution_op->ukernel.spmm.mr;
      size_t mc = input_size;
      if (num_threads > 1) {
        const size_t target_tiles_per_thread = 5;
        const size_t max_mc = divide_round_up(input_size, num_threads * target_tiles_per_thread);
        if (max_mc < mc) {
          mc = min(mc, divide_round_up(mc, max_mc * mr) * mr);
        }
      }
      convolution_op->compute[0].type = xnn_parallelization_type_2d_tile_1d;
      convolution_op->compute[0].task_2d_tile_1d = (pthreadpool_task_2d_tile_1d_t) xnn_compute_spmm;
      convolution_op->compute[0].range[0] = batch_size;
      convolution_op->compute[0].range[1] = input_size << log2_input_element_size;
      convolution_op->compute[0].tile[0] = mc << log2_input_element_size;
      convolution_op->state = xnn_run_state_needs_setup;

      return xnn_status_success;
    }
    case xnn_microkernel_type_conv2d_hwc2chw:
    {
      const size_t zero_size = (input_width * convolution_op->group_input_channels << log2_input_element_size) + XNN_EXTRA_BYTES;

      // Note: zero buffer must be SIMD-aligned, so we can't use xnn_reallocate_memory
      xnn_release_simd_memory(convolution_op->zero_buffer);
      convolution_op->zero_buffer = xnn_allocate_zero_simd_memory(zero_size);
      if (convolution_op->zero_buffer == NULL) {
        xnn_log_error(
          "failed to allocate %zu bytes for %s operator zero padding",
          sizeof(struct xnn_operator), xnn_operator_type_to_string(convolution_op->type));
        return xnn_status_out_of_memory;
      }

      convolution_op->context.conv2d = (struct conv2d_context) {
        .input_height = input_height,
        .input_width = input_width,
        .input_batch_stride = input_batch_stride,
        .zero = convolution_op->zero_buffer,
        .packed_weights = packed_weights(convolution_op),
        .output_batch_stride = output_batch_stride,
        .input_padding_top = convolution_op->padding_top,
        .output_channels = convolution_op->group_output_channels,
        .output_height_stride = output_width << log2_output_element_size,
        .output_channel_stride = output_height * output_width << log2_output_element_size,
        .hwc2chw_ukernel = convolution_op->ukernel.conv2d.hwc2chw_fn,
      };
      memcpy(&convolution_op->context.conv2d.params, params, sizeof(convolution_op->context.conv2d.params));

      const size_t output_height_tile = convolution_op->ukernel.conv2d.output_height_tile;
      size_t output_height_slice = output_height;
      if (num_threads > 1) {
        const size_t target_tiles_per_thread = 5;
        const size_t max_output_height_slice = divide_round_up(output_height, num_threads * target_tiles_per_thread);
        if (max_output_height_slice < output_height_slice) {
          output_height_slice = min(output_height_slice,
            divide_round_up(output_height_slice, max_output_height_slice * output_height_tile) * output_height_tile);
        }
      }
      convolution_op->compute[0].type = xnn_parallelization_type_2d_tile_1d;
      convolution_op->compute[0].task_2d_tile_1d = (pthreadpool_task_2d_tile_1d_t) xnn_compute_conv2d_hwc2chw;
      convolution_op->compute[0].range[0] = batch_size;
      convolution_op->compute[0].range[1] = output_height;
      convolution_op->compute[0].tile[0] = output_height_slice;
      convolution_op->state = xnn_run_state_needs_setup;

      return xnn_status_success;
    }
    case xnn_microkernel_type_dwconv:
    {
      const size_t zero_size = (input_width << log2_input_element_size) + 2 * XNN_EXTRA_BYTES;

      // Note: zero buffer must be SIMD-aligned, so we can't use xnn_reallocate_memory
      xnn_release_simd_memory(convolution_op->zero_buffer);
      convolution_op->zero_buffer = xnn_allocate_zero_simd_memory(zero_size);
      if (convolution_op->zero_buffer == NULL) {
        xnn_log_error(
          "failed to allocate %zu bytes for %s operator zero padding",
          sizeof(struct xnn_operator), xnn_operator_type_to_string(convolution_op->type));
        return xnn_status_out_of_memory;
      }

      convolution_op->context.dwconv2d = (struct dwconv2d_context) {
        .input_height = input_height,
        .input_width = input_width << log2_input_element_size,
        .zero = convolution_op->zero_buffer,
        .input_padding_top = convolution_op->padding_top,
        .input_channel_stride = input_height * input_width << log2_input_element_size,
        .input_batch_stride = input_batch_stride,
        .packed_weights = packed_weights(convolution_op),
        .weights_channel_stride = bias_element_size +
          (convolution_op->kernel_height * convolution_op->kernel_width << log2_filter_element_size),
        .output_channel_stride = output_height * output_width << log2_output_element_size,
        .output_batch_stride = output_batch_stride,
        .chw_ukernel = convolution_op->ukernel.dwconv2d.chw_fn,
      };
      memcpy(&convolution_op->context.dwconv2d.params, chw_params, sizeof(convolution_op->context.dwconv2d.params));

      convolution_op->compute[0].type = xnn_parallelization_type_2d;
      convolution_op->compute[0].task_2d = (pthreadpool_task_2d_t) xnn_compute_dwconv2d_chw;
      convolution_op->compute[0].range[0] = batch_size;
      convolution_op->compute[0].range[1] = convolution_op->groups;
      convolution_op->state = xnn_run_state_needs_setup;

      return xnn_status_success;
    }
    default:
      XNN_UNREACHABLE;
  }
}

enum xnn_status xnn_reshape_convolution2d_nchw_f16(
    xnn_operator_t convolution_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t* output_height_out,
    size_t* output_width_out,
    pthreadpool_t threadpool)
{
  return reshape_convolution2d_nchw(
    convolution_op,
    xnn_operator_type_convolution_nchw_f16,
    batch_size, input_height, input_width,
    /*log2_input_element_size=*/XNN_LOG2_SIZEOF_HALF,
    /*log2_filter_element_size=*/XNN_LOG2_SIZEOF_HALF,
    /*bias_element_size=*/sizeof(uint16_t),
    /*log2_output_element_size=*/XNN_LOG2_SIZEOF_HALF,
    &convolution_op->params.f16_minmax,
    &convolution_op->params.f16_chw,
    output_height_out, output_width_out,
    threadpool);
}

enum xnn_status xnn_reshape_convolution2d_nchw_f32(
    xnn_operator_t convolution_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t* output_height_out,
    size_t* output_width_out,
    pthreadpool_t threadpool)
{
  return reshape_convolution2d_nchw(
    convolution_op,
    xnn_operator_type_convolution_nchw_f32,
    batch_size, input_height, input_width,
    /*log2_input_element_size=*/XNN_LOG2_SIZEOF_FLOAT,
    /*log2_filter_element_size=*/XNN_LOG2_SIZEOF_FLOAT,
    /*bias_element_size=*/sizeof(float),
    /*log2_output_element_size=*/XNN_LOG2_SIZEOF_FLOAT,
    &convolution_op->params.f32_minmax,
    &convolution_op->params.f32_chw,
    output_height_out, output_width_out,
    threadpool);
}

static enum xnn_status setup_convolution2d_nchw(
  xnn_operator_t convolution_op,
  enum xnn_operator_type expected_operator_type,
  const void* input,
  void* output)
{
  if (convolution_op->type != expected_operator_type) {
    xnn_log_error("failed to setup operator: operator type mismatch (expected %s, got %s)",
      xnn_operator_type_to_string(expected_operator_type),
      xnn_operator_type_to_string(convolution_op->type));
    return xnn_status_invalid_parameter;
  }

  if (convolution_op->weights_cache != NULL && !xnn_weights_cache_is_finalized(convolution_op->weights_cache)) {
    xnn_log_error("failed to setup %s operator: weights cache is not finalized",
      xnn_operator_type_to_string(expected_operator_type));
    return xnn_status_invalid_state;
  }

  switch (convolution_op->state) {
    case xnn_run_state_skip:
      return xnn_status_success;
    case xnn_run_state_invalid:
      xnn_log_error(
        "failed to setup %s operator: operator has not been reshaped yet",
        xnn_operator_type_to_string(convolution_op->type));
      return xnn_status_invalid_state;
    case xnn_run_state_needs_setup:
      // Operator has been reshaped, but not setup, continue with setup.
    case xnn_run_state_ready:
      // Operator has been reshaped, and we are setting up with different pointers.
      break;
  }

  switch (convolution_op->ukernel.type) {
    case xnn_microkernel_type_spmm:
    {
      convolution_op->context.spmm.input = (const void*) ((uintptr_t) input + (convolution_op->first_input_channel *
                                                                               convolution_op->context.spmm.scaled_m));
      convolution_op->context.spmm.output = output;
      break;
    }
    case xnn_microkernel_type_conv2d_hwc2chw:
    {
      convolution_op->context.conv2d.input = input;
      convolution_op->context.conv2d.output = output;
      break;
    }
    case xnn_microkernel_type_dwconv:
    {
      convolution_op->context.dwconv2d.input = input;
      convolution_op->context.dwconv2d.output = output;
      break;
    }
    default:
      XNN_UNREACHABLE;
  }

  convolution_op->state = xnn_run_state_ready;

  return xnn_status_success;
}

enum xnn_status xnn_setup_convolution2d_nchw_f16(
    xnn_operator_t convolution_op,
    const void* input,
    void* output)
{
  return setup_convolution2d_nchw(
    convolution_op,
    xnn_operator_type_convolution_nchw_f16,
    input, output);
}

enum xnn_status xnn_setup_convolution2d_nchw_f32(
    xnn_operator_t convolution_op,
    const float* input,
    float* output)
{
  return setup_convolution2d_nchw(
    convolution_op,
    xnn_operator_type_convolution_nchw_f32,
    input, output);
}
