/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2020-2021 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
; generated by igemm_codegen.py (c8c86649c68a788f18be9f7a599a555f08903048)
;
.macro .mdiv_u32_ss s_quot s_numer s_magic s_shift s_tmp
    s_mul_hi_u32 s[\s_tmp], s[\s_magic], s[\s_numer]
    s_add_u32 s[\s_tmp], s[\s_tmp], s[\s_numer]
    s_lshr_b32 s[\s_quot], s[\s_tmp], s[\s_shift]
.endm

.macro .mdiv_u32_rem_ss s_rem s_quot s_numer s_magic s_shift s_denom s_tmp
    .mdiv_u32_ss \s_quot,\s_numer,\s_magic,\s_shift,\s_tmp
    s_mul_i32 s[\s_tmp], s[\s_denom], s[\s_quot]
    s_sub_u32 s[\s_rem], s[\s_numer], s[\s_tmp]
.endm

.macro .mdiv_u32_vs v_quot v_numer s_magic s_shift v_tmp
    v_mul_hi_u32 v[\v_tmp], s[\s_magic], v[\v_numer]
    v_add_u32 v[\v_tmp], v[\v_tmp], v[\v_numer]
    v_lshrrev_b32 v[\v_quot], s[\s_shift], v[\v_tmp]
.endm

.macro .mdiv_u32_rem_vs v_rem v_quot v_numer s_magic s_shift s_denom v_tmp
    .mdiv_u32_vs \v_quot,\v_numer,\s_magic,\s_shift,\v_tmp
    v_mul_lo_u32 v[\v_tmp], s[\s_denom], v[\v_quot]
    v_sub_u32 v[\v_rem], v[\v_numer], v[\v_tmp]
.endm

.macro .v_clear_acc_c a, num
    _a = \a
    .rept \num
        v_accvgpr_write_b32 a[_a], 0
        _a = _a + 1
    .endr
.endm

.macro .v_clear_nc vid, num
    _v = \vid
    .rept \num
        v_mov_b32 v[_v], 0
        _v = _v + 1
    .endr
.endm

;----------------------------------------------------------
; starting of kernel igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8
; tensor_layout              : 'nchw'
; gemm_m_per_block           : 256
; gemm_n_per_block           : 128
; gemm_k_per_block           : 32
; wave_tile_m                : 32
; wave_step_m                : 2
; wave_repeat_m              : 2
; wave_tile_n                : 32
; wave_step_n                : 1
; wave_repeat_n              : 2
; wave_tile_k                : 8
; tensor_a_thread_lengths    : [1, 8, 4, 1]
; tensor_a_cluster_lengths   : [1, 4, 1, 64]
; tensor_b_thread_lengths    : [1, 1, 16, 1]
; tensor_b_cluster_lengths   : [1, 32, 1, 8]
; direction                  : 'fwd'
; precision                  : 'fp16'
; nxb                        : 1
; nxe                        : 0
; 
; block_size                 : 256
; lds_total                  : 32768
; 
.set k_p_in, 0
.set k_p_wei, 8
.set k_p_out, 16
.set k_hi, 24
.set k_wi, 28
.set k_n, 32
.set k_k, 36
.set k_c, 40
.set k_ho, 44
.set k_wo, 48
.set k_stride_h, 52
.set k_stride_w, 56
.set k_dilation_h, 60
.set k_dilation_w, 64
.set k_pad_h, 68
.set k_pad_w, 72
.set k_y, 76
.set k_x, 80
.set k_group, 84
.set k_magic_0, 88
.set k_magic_1, 92
.set k_magic_2, 96
.set k_magic_3, 100
.set k_magic_4, 104
.set k_magic_5, 108
.set k_magic_6, 112
.set k_shift_pack_0, 116
.set k_shift_pack_1, 120
.set k__pack_0, 124
.set k_end, 128

.set s_ka, 0
.set s_bx, 2
.set s_by, 3
.set s_p_in, 4
.set s_p_wei, 8
.set s_p_out, 12
.set s_hi, 16
.set s_wi, 17
.set s_n, 18
.set s_k, 19
.set s_c, 20
.set s_group, 21
.set s_wei_stride_k0, 22
.set s_stride_hw, 23
.set s_in_stride_c, 24
.set s_in_stride_n, 25
.set s_in_stride_n0, 26
.set s_out_stride_k, 27
.set s_out_stride_n, 28
.set s_in_stride_c_c1, 29
.set s_in_stride_c_c0_c1_diff, 30
.set s_block_gtc_ig, 31
.set s_block_gtc_ik, 32
.set s_block_gtc_in0, 33
.set s_block_gtc_in1b, 34
.set s_move_slice_k_c1e, 35
.set s_knum, 3
.set s_gemm_k_num_c1, 36
.set s_kitr, 1
.set s_in_offset, 37
.set s_wei_offset, 51
.set s_k_padded, 53
.set s_shift_pack_0, 14
.set s_shift_pack_1, 15
.set s_magic_2, 29
.set s_magic_3, 30
.set s_magic_4, 35
.set s_magic_5, 36
.set s_magic_6, 33
.set s_tmp, 54
.set s_magic_0, 10
.set s_magic_1, 11
.set s_end, 60

.set v_c, 0  ; coalescing:64, needed:0, resuable:72
.set v_a, 0
.set v_b, 16
.set v_gld_a, 24
.set v_gld_b, 40
.set v_sst_a_os, 56
.set v_sst_b_os, 57
.set v_sld_a_os, 58
.set v_sld_b_os, 59
.set v_in_os, 60
.set v_in_os_base, 61
.set v_wei_os, 62
.set v_gtc_ta_ik1, 63
.set v_gtc_ta_ik0, 64
.set v_gtc_ta_ic1e, 65
.set v_gtc_ta_ic0, 66
.set v_gtc_tb_in1b, 67
.set v_gtc_tb_in0, 68
.set v_gtc_tb_ic1e, 69
.set v_gtc_tb_in1, 70
.set v_gtc_tb_ib, 71
.set v_co_sst, 72
.set v_co_sld, 73
.set v_out_os, 74
.set v_out_in0, 75
.set v_out_in1b, 76
.set v_out_in1, 77
.set v_in_iho, 78
.set v_in_iwo, 79
.set v_in_ihi, 80
.set v_in_iwi, 81
.set v_move_slice_k_ic1, 69
.set v_gemm_in, 82
.set v_gemm_im, 83
.set v_out_iho, 84
.set v_out_iwo, 85
.set v_co_sub_m_index, 86
.set v_co_sub_n_index, 87
.set v_cur_k, 88
.set v_tmp, 90
.set v_end, 128

.set a_c, 0
.set a_end, 128

.text
.globl igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8
.p2align 8
.type igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8,@function
igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8:
    ; gemm_m_unmerge_cluster:0, gemm_n_unmerge_cluster:0, gemm_k_unmerge_cluster:0
    s_load_dwordx2  s[s_p_in+0:s_p_in+1],    s[s_ka+0:s_ka+1],    0+k_p_in
    s_load_dwordx2  s[s_p_wei+0:s_p_wei+1],   s[s_ka+0:s_ka+1],    0+k_p_wei
    s_load_dwordx2  s[s_p_out+0:s_p_out+1],   s[s_ka+0:s_ka+1],    0+k_p_out
    s_load_dwordx4 s[s_hi+0:s_hi+3],    s[s_ka+0:s_ka+1],    0+k_hi
    s_load_dword s[s_c],    s[s_ka+0:s_ka+1],    0+k_c
    s_load_dword s[s_group],    s[s_ka+0:s_ka+1],    0+k_group
    s_load_dwordx2 s[s_magic_0+0:s_magic_0+1],  s[s_ka+0:s_ka+1],  0+k_magic_0
    s_load_dwordx2 s[s_tmp+2:s_tmp+3],  s[s_ka+0:s_ka+1],  0+k_magic_2
    s_load_dwordx2 s[s_tmp+4:s_tmp+5],  s[s_ka+0:s_ka+1],  0+k_magic_4
    s_load_dword s[s_magic_6],  s[s_ka+0:s_ka+1],  0+k_magic_6
    s_load_dwordx2 s[s_shift_pack_0+0:s_shift_pack_0+1], s[s_ka+0:s_ka+1],  0+k_shift_pack_0
    ; wei(c0, c1e, k0, k1) thread_lengths: 1x8x4x1, cluster_lengths:1x4x1x64
    v_mov_b32 v[v_tmp], v0
    v_and_b32 v[v_gtc_ta_ic1e], 3, v[v_tmp]
    v_lshlrev_b32 v[v_gtc_ta_ic1e], 3, v[v_gtc_ta_ic1e]
    v_lshrrev_b32 v[v_tmp], 2, v[v_tmp]
    v_mov_b32 v[v_gtc_ta_ic0], 0
    v_and_b32 v[v_gtc_ta_ik1], 63, v[v_tmp]
    v_lshrrev_b32 v[v_tmp], 6, v[v_tmp]
    v_mov_b32 v[v_gtc_ta_ik0], 0

    ; in(c0, c1e, n0, n1b), thread_lengths: 1x1x16x1, cluster_lengths:1x32x1x8
    v_mov_b32 v[v_tmp], v0
    v_and_b32 v[v_gtc_tb_in1b], 7, v[v_tmp]
    v_lshrrev_b32 v[v_tmp], 3, v[v_tmp]
    v_mov_b32 v[v_gtc_tb_in0], 0
    v_and_b32 v[v_gtc_tb_ic1e], 31, v[v_tmp]

    s_mov_b32 s[s_p_in+3], 0x27000
    s_waitcnt lgkmcnt(0)

    s_mov_b32 s[s_magic_2], s[s_tmp+2]
    s_mov_b32 s[s_magic_3], s[s_tmp+3]
    s_mov_b32 s[s_magic_4], s[s_tmp+4]
    s_mov_b32 s[s_magic_5], s[s_tmp+5]
    ; calculate index
    s_lshl_b32 s[s_wei_stride_k0], s[s_c], 6
    s_mul_i32 s[s_stride_hw], s[s_hi], s[s_wi]
    s_mov_b32 s[s_out_stride_k],       s[s_stride_hw]
    s_mov_b32 s[s_in_stride_c],       s[s_stride_hw]
    s_mul_i32 s[s_tmp], s[s_c], s[s_group]
    s_mul_i32 s[s_in_stride_n], s[s_tmp], s[s_stride_hw]
    s_lshl_b32 s[s_in_stride_n0], s[s_in_stride_n], 3 
    s_mul_i32 s[s_tmp], s[s_group], s[s_k]
    s_mul_i32 s[s_out_stride_n], s[s_tmp], s[s_stride_hw]
    s_mul_i32  s[s_tmp], s[s_n], s[s_in_stride_n]
    s_mul_i32  s[s_tmp+1], s[s_n], s[s_out_stride_n]
    s_lshl_b32 s[s_tmp+4], s[s_tmp], 1
    s_lshl_b32 s[s_tmp+5], s[s_tmp+1], 1
    s_mul_i32 s[s_tmp], s[s_by], s[s_tmp+4]
    s_mul_hi_u32 s[s_tmp+1], s[s_by], s[s_tmp+4]
    s_add_u32 s[s_p_in], s[s_p_in], s[s_tmp]
    s_addc_u32 s[s_p_in+1], s[s_p_in+1], s[s_tmp+1]
    s_mul_i32 s[s_tmp], s[s_by], s[s_tmp+5]
    s_mul_hi_u32 s[s_tmp+1], s[s_by], s[s_tmp+5]
    s_add_u32 s[s_p_out], s[s_p_out], s[s_tmp]
    s_addc_u32 s[s_p_out+1], s[s_p_out+1], s[s_tmp+1]
    s_mov_b32 s[s_knum], s[s_c]

    ; pad k if need
    s_add_u32 s[s_tmp], 255, s[s_k]
    s_lshr_b32 s[s_tmp], s[s_tmp], 8
    s_lshl_b32 s[s_k_padded], s[s_tmp], 8

    ; gemm_m_per_block:256, gemm_n_per_block:128, source_access_order:1
    s_mul_i32 s[s_tmp], s[s_stride_hw], s[s_n]
    s_mul_i32 s[s_tmp+1], s[s_tmp], s[s_k_padded]
    s_lshr_b32 s[0], s[s_tmp+1], 15
    s_bfe_u32 s[s_tmp+3], s[s_shift_pack_1], 0x00080010 ; offset:16, width:8
    .mdiv_u32_rem_ss s_tmp+4,s_block_gtc_ig,s_bx,s_magic_6,s_tmp+3,0,s_tmp
    s_mov_b32 s[s_bx], s[s_tmp+4]
    s_lshr_b32 s[0], s[s_k_padded], 8
    s_bfe_u32 s[s_tmp+3], s[s_shift_pack_0], 0x00080000 ; offset:0, width:8
    .mdiv_u32_rem_ss s_tmp+5,s_tmp+4,s_bx,s_magic_0,s_tmp+3,0,s_tmp
    ; s_tmp+4:block_gtc_in, s_tmp+5:block_gtc_im
    s_lshl_b32 s[s_block_gtc_ik], s[s_tmp+5], 8
    s_mov_b32 s[0], s[s_stride_hw] ; total number of n1b
    s_bfe_u32 s[s_tmp+3], s[s_shift_pack_0], 0x00080008 ; offset:8, width:8
    .mdiv_u32_rem_ss s_block_gtc_in1b,s_block_gtc_in0,s_tmp+4,s_magic_1,s_tmp+3,0,s_tmp
    s_lshl_b32 s[s_block_gtc_in1b], s[s_block_gtc_in1b], 3
    s_lshl_b32 s[s_block_gtc_in0], s[s_block_gtc_in0], 4

    ; in c1e transform
    ; in n1b transform
    v_add_u32 v[v_tmp+5], s[s_block_gtc_in1b], v[v_gtc_tb_in1b]
    s_bfe_u32 s[s_tmp+3], s[s_shift_pack_1], 0x00080000 ; offset:0, width:8
    .mdiv_u32_rem_vs v_tmp+4,v_gtc_tb_in1,v_tmp+5,s_magic_4,s_tmp+3,s_stride_hw,v_tmp
    s_bfe_u32 s[s_tmp+3], s[s_shift_pack_1], 0x00080008 ; offset:8, width:8
    .mdiv_u32_rem_vs v_in_iwi,v_in_ihi,v_tmp+4,s_magic_5,s_tmp+3,s_wi,v_tmp
    ; calculate in offset
    s_mul_i32 s[s_p_in+2], s[s_in_stride_n], s[s_n]
    s_lshl_b32 s[s_p_in+2], s[s_p_in+2], 1
    s_mul_i32 s[s_tmp+5], s[s_c], s[s_in_stride_c]
    s_lshl_b32 s[s_block_gtc_ig], s[s_block_gtc_ig], 1
    s_mul_i32 s[s_tmp], s[s_block_gtc_ig], s[s_tmp+5]
    s_mul_hi_u32 s[s_tmp+1], s[s_block_gtc_ig], s[s_tmp+5]
    s_sub_u32 s[s_p_in+2], s[s_p_in+2], s[s_tmp]
    s_add_u32 s[s_p_in], s[s_p_in], s[s_tmp]
    s_addc_u32 s[s_p_in+1], s[s_p_in+1], s[s_tmp+1]
    s_lshl_b32 s[s_tmp+3], s[s_block_gtc_in0], 4
    s_mul_i32 s[s_tmp], s[s_in_stride_n], s[s_tmp+3]
    s_mul_hi_u32 s[s_tmp+1], s[s_in_stride_n], s[s_tmp+3]
    s_sub_u32 s[s_p_in+2], s[s_p_in+2], s[s_tmp]
    s_add_u32 s[s_p_in], s[s_p_in], s[s_tmp]
    s_addc_u32 s[s_p_in+1], s[s_p_in+1], s[s_tmp+1]

    v_mov_b32 v[v_tmp], v[v_gtc_tb_ic1e]
    v_mul_lo_u32 v[v_tmp], s[s_in_stride_c], v[v_tmp]
    v_mov_b32 v[v_tmp+1], v[v_gtc_tb_in1]
    v_mul_lo_u32 v[v_tmp+1], s[s_in_stride_n], v[v_tmp+1]
    v_add_lshl_u32 v[v_tmp+4], v[v_tmp], v[v_tmp+1], 1
    v_mad_u32_u24 v[v_tmp], v[v_in_ihi], s[s_wi], v[v_in_iwi]
    v_lshl_add_u32 v[v_in_os], v[v_tmp], 1, v[v_tmp+4]

    s_lshl_b32 s[s_in_stride_n0], s[s_in_stride_n0], 1

    s_mul_i32 s[s_in_offset+0], 2, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+1], 3, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+2], 4, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+3], 5, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+4], 6, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+5], 7, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+6], 8, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+7], 9, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+8], 10, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+9], 11, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+10], 12, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+11], 13, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+12], 14, s[s_in_stride_n0]
    s_mul_i32 s[s_in_offset+13], 15, s[s_in_stride_n0]
    ; load input
    buffer_load_short_d16 v[v_gld_b+0], v[v_in_os], s[s_p_in:s_p_in+3], 0 offen offset:0
    buffer_load_short_d16 v[v_gld_b+1], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_stride_n0] offen offset:0
    buffer_load_short_d16 v[v_gld_b+2], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+0] offen offset:0
    buffer_load_short_d16 v[v_gld_b+3], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+1] offen offset:0
    buffer_load_short_d16 v[v_gld_b+4], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+2] offen offset:0
    buffer_load_short_d16 v[v_gld_b+5], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+3] offen offset:0
    buffer_load_short_d16 v[v_gld_b+6], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+4] offen offset:0
    buffer_load_short_d16 v[v_gld_b+7], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+5] offen offset:0
    buffer_load_short_d16 v[v_gld_b+8], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+6] offen offset:0
    buffer_load_short_d16 v[v_gld_b+9], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+7] offen offset:0
    buffer_load_short_d16 v[v_gld_b+10], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+8] offen offset:0
    buffer_load_short_d16 v[v_gld_b+11], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+9] offen offset:0
    buffer_load_short_d16 v[v_gld_b+12], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+10] offen offset:0
    buffer_load_short_d16 v[v_gld_b+13], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+11] offen offset:0
    buffer_load_short_d16 v[v_gld_b+14], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+12] offen offset:0
    buffer_load_short_d16 v[v_gld_b+15], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+13] offen offset:0

    ; config for weight range
    s_mul_i32 s[s_p_wei+2], s[s_c], s[s_k]
    s_lshl_b32 s[s_p_wei+2], s[s_p_wei+2], 1
    s_mov_b32 s[s_p_wei+3], 0x27000
    ; calculate wei offset
    s_mul_i32 s[s_tmp+2], s[s_k], s[s_c]
    s_mul_i32 s[s_tmp], s[s_block_gtc_ig], s[s_tmp+2]
    s_mul_hi_u32 s[s_tmp+1], s[s_block_gtc_ig], s[s_tmp+2]
    s_add_u32 s[s_p_wei], s[s_p_wei], s[s_tmp]
    s_addc_u32 s[s_p_wei+1], s[s_p_wei+1], s[s_tmp+1]
    v_mov_b32 v[v_tmp], v[v_gtc_ta_ik1]
    v_add_u32 v[v_cur_k], s[s_block_gtc_ik], v[v_tmp]
    v_mul_lo_u32 v[v_tmp], s[s_c], v[v_cur_k]
    v_add_lshl_u32 v[v_wei_os], v[v_tmp], v[v_gtc_ta_ic1e], 1

    s_lshl_b32 s[s_wei_stride_k0], s[s_wei_stride_k0], 1

    s_mul_i32 s[s_tmp], s[s_wei_stride_k0], 2
    s_mov_b32 s[s_wei_offset+0], s[s_tmp]
    s_mul_i32 s[s_tmp], s[s_wei_stride_k0], 3
    s_mov_b32 s[s_wei_offset+1], s[s_tmp]
    ; load weight
    buffer_load_dwordx4 v[v_gld_a+0:v_gld_a+0+3], v[v_wei_os], s[s_p_wei:s_p_wei+3], 0 offen offset:0
    buffer_load_dwordx4 v[v_gld_a+4:v_gld_a+4+3], v[v_wei_os], s[s_p_wei:s_p_wei+3], s[s_wei_stride_k0] offen offset:0
    buffer_load_dwordx4 v[v_gld_a+8:v_gld_a+8+3], v[v_wei_os], s[s_p_wei:s_p_wei+3], s[s_wei_offset+0] offen offset:0
    buffer_load_dwordx4 v[v_gld_a+12:v_gld_a+12+3], v[v_wei_os], s[s_p_wei:s_p_wei+3], s[s_wei_offset+1] offen offset:0

    v_mov_b32 v[v_tmp+5], v0
    ; xdlops mapping, get source matrix gemm index
    v_and_b32 v[v_gemm_in], 31, v[v_tmp+5]           ; block_n index 
    v_and_b32 v[v_gemm_im], 31, v[v_tmp+5]           ; block_m index 
    v_lshrrev_b32 v[v_tmp+5], 5, v[v_tmp+5]
    v_and_b32 v[v_tmp + 0], 1, v[v_tmp+5]          ; block_k_per_wave index
    v_lshl_or_b32 v[v_gemm_in], v[v_tmp + 0], 7, v[v_gemm_in]
    v_lshl_or_b32 v[v_gemm_im], v[v_tmp + 0], 8, v[v_gemm_im]
    v_lshrrev_b32 v[v_tmp+5], 1, v[v_tmp+5]
    v_and_b32 v[v_tmp + 2], 1, v[v_tmp+5]  ; waves_per_n index
    v_lshl_or_b32 v[v_gemm_in], v[v_tmp + 2], 5, v[v_gemm_in]
    v_lshrrev_b32 v[v_tmp+5], 1, v[v_tmp+5]
    v_and_b32 v[v_tmp + 3], 1, v[v_tmp+5]  ; waves_per_m index
    v_lshl_or_b32 v[v_gemm_im], v[v_tmp + 3], 6, v[v_gemm_im]

    v_mov_b32 v[v_tmp+5], v0
    ; xdlops mapping, get dst matrix gemm index
    v_and_b32 v[v_tmp+0], 31, v[v_tmp+5]
    v_lshrrev_b32 v[v_tmp+5], 5, v[v_tmp+5]
    v_and_b32 v[v_tmp+1], 1, v[v_tmp+5]
    v_lshrrev_b32 v[v_tmp+5], 1, v[v_tmp+5]
    v_mov_b32 v[v_co_sst], v[v_tmp+0]
    v_lshlrev_b32 v[v_co_sld], 2, v[v_tmp+1]
    v_and_b32 v[v_tmp+0], 1, v[v_tmp+5]
    v_lshrrev_b32 v[v_tmp+5], 1, v[v_tmp+5]
    v_and_b32 v[v_tmp+1], 1, v[v_tmp+5]
    v_lshl_or_b32 v[v_co_sst], v[v_tmp+0], 5, v[v_co_sst]
    v_lshl_or_b32 v[v_co_sld], v[v_tmp+1], 6, v[v_co_sld]

    ; LDS store, in: c0,c1e,n0,n1b: 1x1x16x1, 1x32x1x8, order:4
    v_lshlrev_b32 v[v_tmp], 2, v[v_gtc_tb_in1b]
    v_lshrrev_b32 v[v_tmp+1], 2, v[v_gtc_tb_ic1e]
    v_lshl_add_u32 v[v_tmp], v[v_tmp+1], 9, v[v_tmp]
    v_and_b32 v[v_tmp+1], 3, v[v_gtc_tb_ic1e]
    v_add_u32 v[v_tmp], v[v_tmp], v[v_tmp+1]
    v_lshlrev_b32 v[v_sst_b_os], 1, v[v_tmp]
    v_add_u32 v[v_sst_b_os], 16384, v[v_sst_b_os]

    ; LDS store, wei: c0,c1e,c0,c1: 1x8x4x1, 1x4x1x64, order:0
    v_lshlrev_b32 v[v_tmp], 2, v[v_gtc_ta_ik1]
    v_lshrrev_b32 v[v_tmp+1], 2, v[v_gtc_ta_ic1e]
    v_lshl_add_u32 v[v_tmp], v[v_tmp+1], 10, v[v_tmp]
    v_and_b32 v[v_tmp+1], 3, v[v_gtc_ta_ic1e]
    v_add_u32 v[v_tmp], v[v_tmp], v[v_tmp+1]
    v_lshlrev_b32 v[v_sst_a_os], 1, v[v_tmp]

    ; LDS load
    v_lshlrev_b32 v[v_sld_b_os], 3, v[v_gemm_in]
    v_lshlrev_b32 v[v_sld_a_os], 3, v[v_gemm_im]
    v_add_u32 v[v_sld_b_os], 16384, v[v_sld_b_os]

    v_mov_b32 v[v_gemm_in], v[v_co_sst]
    v_mov_b32 v[v_gemm_im], v[v_co_sld]
    ; init_co_lds_offset for xdlops
    v_lshrrev_b32 v[v_tmp], 2, v[v_gemm_im]
    v_and_b32 v[v_tmp], 1, v[v_tmp]   ; thread id of lanegroup_m_per_cluster
    v_lshlrev_b32 v[v_co_sst], 2, v[v_tmp]
    v_lshrrev_b32 v[v_tmp+2], 6, v[v_gemm_im]  ; thread id of waves_per_m
    v_lshl_or_b32 v[v_co_sst], v[v_tmp+2], 6, v[v_co_sst]
    v_lshrrev_b32 v[v_tmp], 2, v[v_co_sst]
    v_lshlrev_b32 v[v_tmp+1], 2, v[v_gemm_in]   ; implicit transpose with m granularity:4 while store
    v_lshl_or_b32 v[v_co_sst], v[v_tmp], 9, v[v_tmp+1]
    v_lshlrev_b32 v[v_co_sst], 1, v[v_co_sst]
    v_lshlrev_b32 v[v_co_sld], 3, v[0]
    ; init_co_sub_m_index xdlops, block_size:256, macro-tile:256x128 sub_m_index:[0, 4]
    ; g_mr:2, g_ms:1, g_mw:1, g_mb:1, g_mt:1 | l_mr:1, l_ms:2, l_mw:1, l_mb:4, l_mt:4 | n_mc:2, n_ml:1, n_mv:2
    ; nd_stride:[2, 1, 4, 1, 2, 2, 1]
    v_lshrrev_b32 v[v_co_sub_m_index], 7, v[0]   ; get tid along m
    v_and_b32 v[v_tmp+0], 1, v[v_co_sub_m_index]                   ; => x_mc
    v_mov_b32 v[v_co_sub_m_index], v[v_tmp+0]      ; => accumulate x_mc
    v_lshlrev_b32 v[v_co_sub_m_index], 2, v[v_co_sub_m_index]
    ; init_co_sub_n_index xdlops
    v_and_b32 v[v_co_sub_n_index], 127, v[0]

    ; output offset
    s_mul_i32 s[s_tmp+2], s[s_k], s[s_out_stride_k]
    s_mul_i32 s[s_tmp], s[s_block_gtc_ig], s[s_tmp+2]
    s_mul_hi_u32 s[s_tmp+1], s[s_block_gtc_ig], s[s_tmp+2]
    s_add_u32 s[s_p_out], s[s_p_out], s[s_tmp]
    s_addc_u32 s[s_p_out+1], s[s_p_out+1], s[s_tmp+1]
    s_lshl_b32 s[s_tmp+3], s[s_block_gtc_in0], 4
    s_mul_i32 s[s_tmp], s[s_out_stride_n], s[s_tmp+3]
    s_mul_hi_u32 s[s_tmp+1], s[s_out_stride_n], s[s_tmp+3]
    s_add_u32 s[s_p_out], s[s_p_out], s[s_tmp]
    s_addc_u32 s[s_p_out+1], s[s_p_out+1], s[s_tmp+1]

    s_lshl_b32 s[s_tmp+3], s[s_block_gtc_ik], 1
    s_mul_i32 s[s_tmp], s[s_out_stride_k], s[s_tmp+3]
    s_mul_hi_u32 s[s_tmp+1], s[s_out_stride_k], s[s_tmp+3]
    s_add_u32 s[s_p_out], s[s_p_out], s[s_tmp]
    s_addc_u32 s[s_p_out+1], s[s_p_out+1], s[s_tmp+1]

    ; compute v_co_sub_n_index along n0 x n1b : 16x8
    v_and_b32 v[v_out_in1b], 7, v[v_co_sub_n_index]     ; => N1B
    v_lshrrev_b32 v[v_out_in0], 3, v[v_co_sub_n_index]  ; => N0
    ;   compute from n1b
    v_add_u32 v[v_tmp+5], s[s_block_gtc_in1b], v[v_out_in1b]
    s_bfe_u32 s[s_tmp+3], s[s_shift_pack_1], 0x00080000 ; offset:0, width:8
    .mdiv_u32_rem_vs v_tmp+4,v_out_in1,v_tmp+5,s_magic_4,s_tmp+3,s_stride_hw,v_tmp
    s_bfe_u32 s[s_tmp+3], s[s_shift_pack_1], 0x00080008 ; offset:8, width:8
    .mdiv_u32_rem_vs v_out_iwo,v_out_iho,v_tmp+4,s_magic_5,s_tmp+3,s_wi,v_tmp


    ; add in_in0, in_in1
    v_lshl_or_b32 v[v_tmp+1], v[v_out_in0], 3, v[v_out_in1]
    v_mul_lo_u32 v[v_out_os], s[s_out_stride_n], v[v_tmp+1]
    ; add i_k
    v_mul_lo_u32 v[v_tmp], s[s_out_stride_k], v[v_co_sub_m_index]
    v_add_u32 v[v_out_os], v[v_out_os], v[v_tmp]
    ; add ho, wo
    v_mul_lo_u32 v[v_tmp+1], s[s_wi], v[v_out_iho]
    v_add3_u32 v[v_out_os], v[v_out_os], v[v_tmp+1], v[v_out_iwo]
    v_lshlrev_b32 v[v_out_os], 1, v[v_out_os]
    ; move slice stride
    s_mov_b32 s[s_move_slice_k_c1e], 32

    s_mul_i32 s[s_in_stride_c_c1], s[s_move_slice_k_c1e], s[s_stride_hw]  ; might be 0 or larger

    s_lshl_b32 s[s_in_stride_c_c1], s[s_in_stride_c_c1], 1
    s_lshl_b32 s[s_in_stride_c_c0_c1_diff], s[s_in_stride_c_c0_c1_diff], 1
    s_lshl_b32 s[s_in_stride_c], s[s_in_stride_c], 1
    s_lshl_b32 s[s_c], s[s_c], 1
    s_lshl_b32 s[s_out_stride_k], s[s_out_stride_k], 1
    s_lshl_b32 s[s_move_slice_k_c1e], s[s_move_slice_k_c1e], 1
    s_mov_b32 s[s_p_out+2], 0xffffffff
    s_mov_b32 s[s_p_out+3], 0x27000
    ; start MFMA loop, 32x32 wave tile with 2x2 repeat, 2x1 step
    s_waitcnt vmcnt(4)
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+0] 
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+0+1] offset:64
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+2] offset:128
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+2+1] offset:192
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+4] offset:256
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+4+1] offset:320
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+6] offset:384
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+6+1] offset:448
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+8] offset:512
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+8+1] offset:576
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+10] offset:640
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+10+1] offset:704
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+12] offset:768
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+12+1] offset:832
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+14] offset:896
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+14+1] offset:960

    s_waitcnt vmcnt(0)
    ds_write_b64 v[v_sst_a_os], v[v_gld_a:v_gld_a+1] offset:0
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+4:v_gld_a+5] offset:512
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+8:v_gld_a+9] offset:1024
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+12:v_gld_a+13] offset:1536
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+2:v_gld_a+3] offset:2048
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+6:v_gld_a+7] offset:2560
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+10:v_gld_a+11] offset:3072
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+14:v_gld_a+15] offset:3584

    .v_clear_acc_c a_c, 128
    ; make sure acc WAR harzard, at least 1 nop for src_c
    s_sub_i32 s[s_kitr], s[s_knum], 32
    s_cmp_gt_i32 s[s_kitr], 0
    s_cbranch_scc0 L_igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8_mfma_end

    v_add_u32 v[v_in_os], s[s_in_stride_c_c1], v[v_in_os]
    ; move slice window for weight
    v_add_u32 v[v_wei_os],  s[s_move_slice_k_c1e], v[v_wei_os]
    s_waitcnt lgkmcnt(0)
    s_barrier
L_igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8_mfma_body:
    ; do fma accumulate with unroll 32
    ds_read2_b64 v[v_a+0:v_a+3], v[v_sld_a_os], offset0:0, offset1:32
    ds_read_b64 v[v_b:v_b+1], v[v_sld_b_os] 
    ds_read_b64 v[v_b+2:v_b+2+1], v[v_sld_b_os] offset:512
    ds_read2_b64 v[v_a+4+0:v_a+4+3], v[v_sld_a_os], offset0:128, offset1:160
    s_waitcnt lgkmcnt(2)
    v_mfma_f32_32x32x8f16 a[a_c+0:a_c+15], v[v_a+0:v_a+1], v[v_b+0:v_b+1], a[a_c+0:a_c+15]     ; repeat:0x0, step:0x0, num_a_c:16
    buffer_load_short_d16 v[v_gld_b+0], v[v_in_os], s[s_p_in:s_p_in+3], 0 offen offset:0
    buffer_load_short_d16 v[v_gld_b+1], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_stride_n0] offen offset:0
    v_mfma_f32_32x32x8f16 a[a_c+16:a_c+31], v[v_a+2:v_a+3], v[v_b+0:v_b+1], a[a_c+16:a_c+31]     ; repeat:0x0, step:1x0, num_a_c:16
    buffer_load_short_d16 v[v_gld_b+2], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+0] offen offset:0
    buffer_load_short_d16 v[v_gld_b+3], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+1] offen offset:0
    v_add_u32 v[v_tmp+4], 4096, v[v_sld_a_os]
    ds_read2_b64 v[v_a+8+0:v_a+8+3], v[v_tmp+4], offset0:0, offset1:32 ; load i_k:1 into local buffer 1, repeat 0
    ds_read_b64 v[v_b+4:v_b+4+1], v[v_sld_b_os] offset:2048 ; load i_k:1 into local buffer 1, repeat 0
    s_waitcnt lgkmcnt(3)
    v_mfma_f32_32x32x8f16 a[a_c+32:a_c+47], v[v_a+0:v_a+1], v[v_b+2:v_b+3], a[a_c+32:a_c+47]     ; repeat:0x1, step:0x0, num_a_c:16
    buffer_load_short_d16 v[v_gld_b+4], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+2] offen offset:0
    buffer_load_short_d16 v[v_gld_b+5], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+3] offen offset:0
    v_mfma_f32_32x32x8f16 a[a_c+48:a_c+63], v[v_a+2:v_a+3], v[v_b+2:v_b+3], a[a_c+48:a_c+63]     ; repeat:0x1, step:1x0, num_a_c:16
    buffer_load_short_d16 v[v_gld_b+6], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+4] offen offset:0
    buffer_load_short_d16 v[v_gld_b+7], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+5] offen offset:0
    ds_read_b64 v[v_b+6:v_b+6+1], v[v_sld_b_os] offset:2560 ; load i_k:1 into local buffer 1, repeat 1
    ds_read2_b64 v[v_a+12+0:v_a+12+3], v[v_tmp+4], offset0:128, offset1:160 ; load i_k:1 into local buffer 1, repeat 1
    s_waitcnt lgkmcnt(4)
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+4:v_a+5], v[v_b+0:v_b+1], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    buffer_load_short_d16 v[v_gld_b+8], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+6] offen offset:0
    buffer_load_short_d16 v[v_gld_b+9], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+7] offen offset:0
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+6:v_a+7], v[v_b+0:v_b+1], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16
    buffer_load_short_d16 v[v_gld_b+10], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+8] offen offset:0
    buffer_load_short_d16 v[v_gld_b+11], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+9] offen offset:0
    v_add_u32 v[v_tmp+4], 4096, v[v_tmp+4]
    ds_read2_b64 v[v_a+0:v_a+3], v[v_tmp+4], offset0:0, offset1:32 ; load i_k:2 into local buffer 0, repeat 0
    ds_read_b64 v[v_b:v_b+1], v[v_sld_b_os] offset:4096 ; load i_k:2 into local buffer 0, repeat 0
    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+4:v_a+5], v[v_b+2:v_b+3], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    buffer_load_short_d16 v[v_gld_b+12], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+10] offen offset:0
    buffer_load_short_d16 v[v_gld_b+13], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+11] offen offset:0
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+6:v_a+7], v[v_b+2:v_b+3], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16
    buffer_load_short_d16 v[v_gld_b+14], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+12] offen offset:0
    buffer_load_short_d16 v[v_gld_b+15], v[v_in_os], s[s_p_in:s_p_in+3], s[s_in_offset+13] offen offset:0
    ds_read_b64 v[v_b+2:v_b+2+1], v[v_sld_b_os] offset:4608 ; load i_k:2 into local buffer 0, repeat 1
    s_waitcnt lgkmcnt(5)
    v_mfma_f32_32x32x8f16 a[a_c+0:a_c+15], v[v_a+8:v_a+9], v[v_b+4:v_b+5], a[a_c+0:a_c+15]     ; repeat:0x0, step:0x0, num_a_c:16
    buffer_load_dwordx4 v[v_gld_a+0:v_gld_a+0+3], v[v_wei_os], s[s_p_wei:s_p_wei+3], 0 offen offset:0
    buffer_load_dwordx4 v[v_gld_a+4:v_gld_a+4+3], v[v_wei_os], s[s_p_wei:s_p_wei+3], s[s_wei_stride_k0] offen offset:0
    v_mfma_f32_32x32x8f16 a[a_c+16:a_c+31], v[v_a+10:v_a+11], v[v_b+4:v_b+5], a[a_c+16:a_c+31]     ; repeat:0x0, step:1x0, num_a_c:16
    buffer_load_dwordx4 v[v_gld_a+8:v_gld_a+8+3], v[v_wei_os], s[s_p_wei:s_p_wei+3], s[s_wei_offset+0] offen offset:0
    buffer_load_dwordx4 v[v_gld_a+12:v_gld_a+12+3], v[v_wei_os], s[s_p_wei:s_p_wei+3], s[s_wei_offset+1] offen offset:0
    ds_read2_b64 v[v_a+4+0:v_a+4+3], v[v_tmp+4], offset0:128, offset1:160 ; load i_k:2 into local buffer 0, repeat 1
    s_waitcnt lgkmcnt(5)
    v_mfma_f32_32x32x8f16 a[a_c+32:a_c+47], v[v_a+8:v_a+9], v[v_b+6:v_b+7], a[a_c+32:a_c+47]     ; repeat:0x1, step:0x0, num_a_c:16
    v_add_u32 v[v_in_os], s[s_in_stride_c_c1], v[v_in_os]
    v_mfma_f32_32x32x8f16 a[a_c+48:a_c+63], v[v_a+10:v_a+11], v[v_b+6:v_b+7], a[a_c+48:a_c+63]     ; repeat:0x1, step:1x0, num_a_c:16
    v_add_u32 v[v_wei_os],  s[s_move_slice_k_c1e], v[v_wei_os]
    v_add_u32 v[v_tmp+4], 4096, v[v_tmp+4]
    ds_read2_b64 v[v_a+8+0:v_a+8+3], v[v_tmp+4], offset0:0, offset1:32 ; load i_k:3 into local buffer 1, repeat 0
    s_waitcnt lgkmcnt(5)
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+12:v_a+13], v[v_b+4:v_b+5], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+14:v_a+15], v[v_b+4:v_b+5], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16
    ds_read_b64 v[v_b+4:v_b+4+1], v[v_sld_b_os] offset:6144 ; load i_k:3 into local buffer 1, repeat 0
    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+12:v_a+13], v[v_b+6:v_b+7], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+14:v_a+15], v[v_b+6:v_b+7], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16
    ds_read_b64 v[v_b+6:v_b+6+1], v[v_sld_b_os] offset:6656 ; load i_k:3 into local buffer 1, repeat 1
    ds_read2_b64 v[v_a+12+0:v_a+12+3], v[v_tmp+4], offset0:128, offset1:160 ; load i_k:3 into local buffer 1, repeat 1
    s_waitcnt lgkmcnt(0)
    s_barrier
    s_waitcnt vmcnt(4)
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+0]
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+0+1] offset:64
    v_mfma_f32_32x32x8f16 a[a_c+0:a_c+15], v[v_a+0:v_a+1], v[v_b+0:v_b+1], a[a_c+0:a_c+15]     ; repeat:0x0, step:0x0, num_a_c:16
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+2] offset:128
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+2+1] offset:192
    v_mfma_f32_32x32x8f16 a[a_c+16:a_c+31], v[v_a+2:v_a+3], v[v_b+0:v_b+1], a[a_c+16:a_c+31]     ; repeat:0x0, step:1x0, num_a_c:16
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+4] offset:256
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+4+1] offset:320
    v_mfma_f32_32x32x8f16 a[a_c+32:a_c+47], v[v_a+0:v_a+1], v[v_b+2:v_b+3], a[a_c+32:a_c+47]     ; repeat:0x1, step:0x0, num_a_c:16
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+6] offset:384
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+6+1] offset:448
    v_mfma_f32_32x32x8f16 a[a_c+48:a_c+63], v[v_a+2:v_a+3], v[v_b+2:v_b+3], a[a_c+48:a_c+63]     ; repeat:0x1, step:1x0, num_a_c:16
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+8] offset:512
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+8+1] offset:576
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+4:v_a+5], v[v_b+0:v_b+1], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+10] offset:640
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+10+1] offset:704
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+6:v_a+7], v[v_b+0:v_b+1], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+12] offset:768
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+12+1] offset:832
    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+4:v_a+5], v[v_b+2:v_b+3], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+14] offset:896
    ds_write_b16 v[v_sst_b_os], v[v_gld_b+14+1] offset:960
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+6:v_a+7], v[v_b+2:v_b+3], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16
    s_waitcnt vmcnt(0)
    ds_write_b64 v[v_sst_a_os], v[v_gld_a:v_gld_a+1] offset:0
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+4:v_gld_a+5] offset:512
    v_mfma_f32_32x32x8f16 a[a_c+0:a_c+15], v[v_a+8:v_a+9], v[v_b+4:v_b+5], a[a_c+0:a_c+15]     ; repeat:0x0, step:0x0, num_a_c:16
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+8:v_gld_a+9] offset:1024
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+12:v_gld_a+13] offset:1536
    v_mfma_f32_32x32x8f16 a[a_c+16:a_c+31], v[v_a+10:v_a+11], v[v_b+4:v_b+5], a[a_c+16:a_c+31]     ; repeat:0x0, step:1x0, num_a_c:16
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+2:v_gld_a+3] offset:2048
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+6:v_gld_a+7] offset:2560
    v_mfma_f32_32x32x8f16 a[a_c+32:a_c+47], v[v_a+8:v_a+9], v[v_b+6:v_b+7], a[a_c+32:a_c+47]     ; repeat:0x1, step:0x0, num_a_c:16
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+10:v_gld_a+11] offset:3072
    ds_write_b64 v[v_sst_a_os], v[v_gld_a+14:v_gld_a+15] offset:3584
    v_mfma_f32_32x32x8f16 a[a_c+48:a_c+63], v[v_a+10:v_a+11], v[v_b+6:v_b+7], a[a_c+48:a_c+63]     ; repeat:0x1, step:1x0, num_a_c:16
    s_sub_i32 s[s_kitr], s[s_kitr], 32
    s_cmp_gt_i32 s[s_kitr], 0
    s_cbranch_scc0 L_igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8_mfma_finishing
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+12:v_a+13], v[v_b+4:v_b+5], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+14:v_a+15], v[v_b+4:v_b+5], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+12:v_a+13], v[v_b+6:v_b+7], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+14:v_a+15], v[v_b+6:v_b+7], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16
    s_waitcnt lgkmcnt(0)
    s_barrier
    s_branch L_igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8_mfma_body
L_igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8_mfma_finishing:
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+12:v_a+13], v[v_b+4:v_b+5], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+14:v_a+15], v[v_b+4:v_b+5], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+12:v_a+13], v[v_b+6:v_b+7], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+14:v_a+15], v[v_b+6:v_b+7], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16
L_igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8_mfma_end:
    s_waitcnt lgkmcnt(0)
    s_barrier
    ds_read2_b64 v[v_a+0:v_a+3], v[v_sld_a_os], offset0:0, offset1:32
    ds_read_b64 v[v_b:v_b+1], v[v_sld_b_os] 
    ds_read_b64 v[v_b+2:v_b+2+1], v[v_sld_b_os] offset:512
    ds_read2_b64 v[v_a+4+0:v_a+4+3], v[v_sld_a_os], offset0:128, offset1:160
    ; k iteration : 0
    s_waitcnt lgkmcnt(2)
    v_mfma_f32_32x32x8f16 a[a_c+0:a_c+15], v[v_a+0:v_a+1], v[v_b+0:v_b+1], a[a_c+0:a_c+15]     ; repeat:0x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+16:a_c+31], v[v_a+2:v_a+3], v[v_b+0:v_b+1], a[a_c+16:a_c+31]     ; repeat:0x0, step:1x0, num_a_c:16
    v_add_u32 v[v_tmp+4], 4096, v[v_sld_a_os]
    ds_read2_b64 v[v_a+8+0:v_a+8+3], v[v_tmp+4], offset0:0, offset1:32 ; load i_k:1 into local buffer 1, repeat 0
    ds_read_b64 v[v_b+4:v_b+4+1], v[v_sld_b_os] offset:2048 ; load i_k:1 into local buffer 1, repeat 0

    s_waitcnt lgkmcnt(3)
    v_mfma_f32_32x32x8f16 a[a_c+32:a_c+47], v[v_a+0:v_a+1], v[v_b+2:v_b+3], a[a_c+32:a_c+47]     ; repeat:0x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+48:a_c+63], v[v_a+2:v_a+3], v[v_b+2:v_b+3], a[a_c+48:a_c+63]     ; repeat:0x1, step:1x0, num_a_c:16
    ds_read_b64 v[v_b+6:v_b+6+1], v[v_sld_b_os] offset:2560 ; load i_k:1 into local buffer 1, repeat 1
    ds_read2_b64 v[v_a+12+0:v_a+12+3], v[v_tmp+4], offset0:128, offset1:160 ; load i_k:1 into local buffer 1, repeat 1

    s_waitcnt lgkmcnt(4)
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+4:v_a+5], v[v_b+0:v_b+1], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+6:v_a+7], v[v_b+0:v_b+1], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16
    v_add_u32 v[v_tmp+4], 4096, v[v_tmp+4]
    ds_read2_b64 v[v_a+0:v_a+3], v[v_tmp+4], offset0:0, offset1:32 ; load i_k:2 into local buffer 0, repeat 0
    ds_read_b64 v[v_b:v_b+1], v[v_sld_b_os] offset:4096 ; load i_k:2 into local buffer 0, repeat 0

    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+4:v_a+5], v[v_b+2:v_b+3], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+6:v_a+7], v[v_b+2:v_b+3], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16
    ds_read_b64 v[v_b+2:v_b+2+1], v[v_sld_b_os] offset:4608 ; load i_k:2 into local buffer 0, repeat 1

    ; k iteration : 1
    s_waitcnt lgkmcnt(5)
    v_mfma_f32_32x32x8f16 a[a_c+0:a_c+15], v[v_a+8:v_a+9], v[v_b+4:v_b+5], a[a_c+0:a_c+15]     ; repeat:0x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+16:a_c+31], v[v_a+10:v_a+11], v[v_b+4:v_b+5], a[a_c+16:a_c+31]     ; repeat:0x0, step:1x0, num_a_c:16
    ds_read2_b64 v[v_a+4+0:v_a+4+3], v[v_tmp+4], offset0:128, offset1:160 ; load i_k:2 into local buffer 0, repeat 1

    s_waitcnt lgkmcnt(5)
    v_mfma_f32_32x32x8f16 a[a_c+32:a_c+47], v[v_a+8:v_a+9], v[v_b+6:v_b+7], a[a_c+32:a_c+47]     ; repeat:0x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+48:a_c+63], v[v_a+10:v_a+11], v[v_b+6:v_b+7], a[a_c+48:a_c+63]     ; repeat:0x1, step:1x0, num_a_c:16
    v_add_u32 v[v_tmp+4], 4096, v[v_tmp+4]
    ds_read2_b64 v[v_a+8+0:v_a+8+3], v[v_tmp+4], offset0:0, offset1:32 ; load i_k:3 into local buffer 1, repeat 0

    s_waitcnt lgkmcnt(5)
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+12:v_a+13], v[v_b+4:v_b+5], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+14:v_a+15], v[v_b+4:v_b+5], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16
    ds_read_b64 v[v_b+4:v_b+4+1], v[v_sld_b_os] offset:6144 ; load i_k:3 into local buffer 1, repeat 0

    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+12:v_a+13], v[v_b+6:v_b+7], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+14:v_a+15], v[v_b+6:v_b+7], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16
    ds_read_b64 v[v_b+6:v_b+6+1], v[v_sld_b_os] offset:6656 ; load i_k:3 into local buffer 1, repeat 1
    ds_read2_b64 v[v_a+12+0:v_a+12+3], v[v_tmp+4], offset0:128, offset1:160 ; load i_k:3 into local buffer 1, repeat 1

    ; k iteration : 30
    s_waitcnt lgkmcnt(6)
    v_mfma_f32_32x32x8f16 a[a_c+0:a_c+15], v[v_a+0:v_a+1], v[v_b+0:v_b+1], a[a_c+0:a_c+15]     ; repeat:0x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+16:a_c+31], v[v_a+2:v_a+3], v[v_b+0:v_b+1], a[a_c+16:a_c+31]     ; repeat:0x0, step:1x0, num_a_c:16

    s_waitcnt lgkmcnt(5)
    v_mfma_f32_32x32x8f16 a[a_c+32:a_c+47], v[v_a+0:v_a+1], v[v_b+2:v_b+3], a[a_c+32:a_c+47]     ; repeat:0x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+48:a_c+63], v[v_a+2:v_a+3], v[v_b+2:v_b+3], a[a_c+48:a_c+63]     ; repeat:0x1, step:1x0, num_a_c:16

    s_waitcnt lgkmcnt(4)
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+4:v_a+5], v[v_b+0:v_b+1], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+6:v_a+7], v[v_b+0:v_b+1], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16

    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+4:v_a+5], v[v_b+2:v_b+3], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+6:v_a+7], v[v_b+2:v_b+3], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16
    ; k iteration : 31
    s_waitcnt lgkmcnt(2)
    v_mfma_f32_32x32x8f16 a[a_c+0:a_c+15], v[v_a+8:v_a+9], v[v_b+4:v_b+5], a[a_c+0:a_c+15]     ; repeat:0x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+16:a_c+31], v[v_a+10:v_a+11], v[v_b+4:v_b+5], a[a_c+16:a_c+31]     ; repeat:0x0, step:1x0, num_a_c:16

    s_waitcnt lgkmcnt(1)
    v_mfma_f32_32x32x8f16 a[a_c+32:a_c+47], v[v_a+8:v_a+9], v[v_b+6:v_b+7], a[a_c+32:a_c+47]     ; repeat:0x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+48:a_c+63], v[v_a+10:v_a+11], v[v_b+6:v_b+7], a[a_c+48:a_c+63]     ; repeat:0x1, step:1x0, num_a_c:16

    s_waitcnt lgkmcnt(0)
    v_mfma_f32_32x32x8f16 a[a_c+64:a_c+79], v[v_a+12:v_a+13], v[v_b+4:v_b+5], a[a_c+64:a_c+79]     ; repeat:1x0, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+80:a_c+95], v[v_a+14:v_a+15], v[v_b+4:v_b+5], a[a_c+80:a_c+95]     ; repeat:1x0, step:1x0, num_a_c:16

    v_mfma_f32_32x32x8f16 a[a_c+96:a_c+111], v[v_a+12:v_a+13], v[v_b+6:v_b+7], a[a_c+96:a_c+111]     ; repeat:1x1, step:0x0, num_a_c:16
    v_mfma_f32_32x32x8f16 a[a_c+112:a_c+127], v[v_a+14:v_a+15], v[v_b+6:v_b+7], a[a_c+112:a_c+127]     ; repeat:1x1, step:1x0, num_a_c:16

    s_nop 15
    s_nop 2
    ; coalescing store, mapping:mt_m:256, mt_n:128, wt_m:32, wt_n:32, ws:4, r_m:2, r_n:2, s_m:2, s_n:1 | 32x32x8, lanegroup_m_tcbw:4x2x4x1, lanegroup_n_tcbw:1x32x1x1
    ; coalescing_groups:2, num_dword_per_group:64
    ; init_co_sub_m_index xdlops, block_size:256, macro-tile:256x128 sub_m_index:[0, 4]
    ; g_mr:2, g_ms:1, g_mw:1, g_mb:1, g_mt:1 | l_mr:1, l_ms:2, l_mw:1, l_mb:4, l_mt:4 | n_mc:2, n_ml:1, n_mv:2
    ; nd_stride:[2, 1, 4, 1, 2, 2, 1]
    ; start group 0, i_g_mr:0, i_g_ms:0, i_g_mw:0, i_g_mb:0, i_g_mt:0, m index start from 0
    s_barrier
    v_accvgpr_read_b32 v[v_c], a[a_c]
    v_accvgpr_read_b32 v[v_c+1], a[a_c+1]
    v_accvgpr_read_b32 v[v_c+2], a[a_c+2]
    v_accvgpr_read_b32 v[v_c+3], a[a_c+3]
    v_cvt_f16_f32_e32 v[v_c], v[v_c]
    v_cvt_f16_f32_e32 v[v_c+1], v[v_c+1]
    v_cvt_f16_f32_e32 v[v_c+2], v[v_c+2]
    v_cvt_f16_f32_e32 v[v_c+3], v[v_c+3]
    v_pack_b32_f16 v[v_c], v[v_c], v[v_c+1]
    v_pack_b32_f16 v[v_c+1], v[v_c+2], v[v_c+3]
    ds_write_b64 v[v_co_sst], v[v_c:v_c+1]    ; idword:0(0,0),  0x0 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:0  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+4], a[a_c+32]
    v_accvgpr_read_b32 v[v_c+5], a[a_c+33]
    v_accvgpr_read_b32 v[v_c+6], a[a_c+34]
    v_accvgpr_read_b32 v[v_c+7], a[a_c+35]
    v_cvt_f16_f32_e32 v[v_c+4], v[v_c+4]
    v_cvt_f16_f32_e32 v[v_c+5], v[v_c+5]
    v_cvt_f16_f32_e32 v[v_c+6], v[v_c+6]
    v_cvt_f16_f32_e32 v[v_c+7], v[v_c+7]
    v_pack_b32_f16 v[v_c+4], v[v_c+4], v[v_c+5]
    v_pack_b32_f16 v[v_c+5], v[v_c+6], v[v_c+7]
    ds_write_b64 v[v_co_sst], v[v_c+4:v_c+4+1] offset:512   ; idword:64(0,64),  0x64 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:0  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+8], a[a_c+4]
    v_accvgpr_read_b32 v[v_c+9], a[a_c+5]
    v_accvgpr_read_b32 v[v_c+10], a[a_c+6]
    v_accvgpr_read_b32 v[v_c+11], a[a_c+7]
    v_cvt_f16_f32_e32 v[v_c+8], v[v_c+8]
    v_cvt_f16_f32_e32 v[v_c+9], v[v_c+9]
    v_cvt_f16_f32_e32 v[v_c+10], v[v_c+10]
    v_cvt_f16_f32_e32 v[v_c+11], v[v_c+11]
    v_pack_b32_f16 v[v_c+8], v[v_c+8], v[v_c+9]
    v_pack_b32_f16 v[v_c+9], v[v_c+10], v[v_c+11]
    ds_write_b64 v[v_co_sst], v[v_c+8:v_c+8+1] offset:2048   ; idword:256(2,0),  2x0 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:1  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+12], a[a_c+36]
    v_accvgpr_read_b32 v[v_c+13], a[a_c+37]
    v_accvgpr_read_b32 v[v_c+14], a[a_c+38]
    v_accvgpr_read_b32 v[v_c+15], a[a_c+39]
    v_cvt_f16_f32_e32 v[v_c+12], v[v_c+12]
    v_cvt_f16_f32_e32 v[v_c+13], v[v_c+13]
    v_cvt_f16_f32_e32 v[v_c+14], v[v_c+14]
    v_cvt_f16_f32_e32 v[v_c+15], v[v_c+15]
    v_pack_b32_f16 v[v_c+12], v[v_c+12], v[v_c+13]
    v_pack_b32_f16 v[v_c+13], v[v_c+14], v[v_c+15]
    ds_write_b64 v[v_co_sst], v[v_c+12:v_c+12+1] offset:2560   ; idword:320(2,64),  2x64 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:1  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+16], a[a_c+8]
    v_accvgpr_read_b32 v[v_c+17], a[a_c+9]
    v_accvgpr_read_b32 v[v_c+18], a[a_c+10]
    v_accvgpr_read_b32 v[v_c+19], a[a_c+11]
    v_cvt_f16_f32_e32 v[v_c+16], v[v_c+16]
    v_cvt_f16_f32_e32 v[v_c+17], v[v_c+17]
    v_cvt_f16_f32_e32 v[v_c+18], v[v_c+18]
    v_cvt_f16_f32_e32 v[v_c+19], v[v_c+19]
    v_pack_b32_f16 v[v_c+16], v[v_c+16], v[v_c+17]
    v_pack_b32_f16 v[v_c+17], v[v_c+18], v[v_c+19]
    ds_write_b64 v[v_co_sst], v[v_c+16:v_c+16+1] offset:4096   ; idword:512(4,0),  4x0 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:2  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+20], a[a_c+40]
    v_accvgpr_read_b32 v[v_c+21], a[a_c+41]
    v_accvgpr_read_b32 v[v_c+22], a[a_c+42]
    v_accvgpr_read_b32 v[v_c+23], a[a_c+43]
    v_cvt_f16_f32_e32 v[v_c+20], v[v_c+20]
    v_cvt_f16_f32_e32 v[v_c+21], v[v_c+21]
    v_cvt_f16_f32_e32 v[v_c+22], v[v_c+22]
    v_cvt_f16_f32_e32 v[v_c+23], v[v_c+23]
    v_pack_b32_f16 v[v_c+20], v[v_c+20], v[v_c+21]
    v_pack_b32_f16 v[v_c+21], v[v_c+22], v[v_c+23]
    ds_write_b64 v[v_co_sst], v[v_c+20:v_c+20+1] offset:4608   ; idword:576(4,64),  4x64 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:2  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+24], a[a_c+12]
    v_accvgpr_read_b32 v[v_c+25], a[a_c+13]
    v_accvgpr_read_b32 v[v_c+26], a[a_c+14]
    v_accvgpr_read_b32 v[v_c+27], a[a_c+15]
    v_cvt_f16_f32_e32 v[v_c+24], v[v_c+24]
    v_cvt_f16_f32_e32 v[v_c+25], v[v_c+25]
    v_cvt_f16_f32_e32 v[v_c+26], v[v_c+26]
    v_cvt_f16_f32_e32 v[v_c+27], v[v_c+27]
    v_pack_b32_f16 v[v_c+24], v[v_c+24], v[v_c+25]
    v_pack_b32_f16 v[v_c+25], v[v_c+26], v[v_c+27]
    ds_write_b64 v[v_co_sst], v[v_c+24:v_c+24+1] offset:6144   ; idword:768(6,0),  6x0 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:3  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+28], a[a_c+44]
    v_accvgpr_read_b32 v[v_c+29], a[a_c+45]
    v_accvgpr_read_b32 v[v_c+30], a[a_c+46]
    v_accvgpr_read_b32 v[v_c+31], a[a_c+47]
    v_cvt_f16_f32_e32 v[v_c+28], v[v_c+28]
    v_cvt_f16_f32_e32 v[v_c+29], v[v_c+29]
    v_cvt_f16_f32_e32 v[v_c+30], v[v_c+30]
    v_cvt_f16_f32_e32 v[v_c+31], v[v_c+31]
    v_pack_b32_f16 v[v_c+28], v[v_c+28], v[v_c+29]
    v_pack_b32_f16 v[v_c+29], v[v_c+30], v[v_c+31]
    ds_write_b64 v[v_co_sst], v[v_c+28:v_c+28+1] offset:6656   ; idword:832(6,64),  6x64 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:3  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+32], a[a_c+16]
    v_accvgpr_read_b32 v[v_c+33], a[a_c+17]
    v_accvgpr_read_b32 v[v_c+34], a[a_c+18]
    v_accvgpr_read_b32 v[v_c+35], a[a_c+19]
    v_cvt_f16_f32_e32 v[v_c+32], v[v_c+32]
    v_cvt_f16_f32_e32 v[v_c+33], v[v_c+33]
    v_cvt_f16_f32_e32 v[v_c+34], v[v_c+34]
    v_cvt_f16_f32_e32 v[v_c+35], v[v_c+35]
    v_pack_b32_f16 v[v_c+32], v[v_c+32], v[v_c+33]
    v_pack_b32_f16 v[v_c+33], v[v_c+34], v[v_c+35]
    ds_write_b64 v[v_co_sst], v[v_c+32:v_c+32+1] offset:8192   ; idword:1024(8,0),  8x0 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:0  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+36], a[a_c+48]
    v_accvgpr_read_b32 v[v_c+37], a[a_c+49]
    v_accvgpr_read_b32 v[v_c+38], a[a_c+50]
    v_accvgpr_read_b32 v[v_c+39], a[a_c+51]
    v_cvt_f16_f32_e32 v[v_c+36], v[v_c+36]
    v_cvt_f16_f32_e32 v[v_c+37], v[v_c+37]
    v_cvt_f16_f32_e32 v[v_c+38], v[v_c+38]
    v_cvt_f16_f32_e32 v[v_c+39], v[v_c+39]
    v_pack_b32_f16 v[v_c+36], v[v_c+36], v[v_c+37]
    v_pack_b32_f16 v[v_c+37], v[v_c+38], v[v_c+39]
    ds_write_b64 v[v_co_sst], v[v_c+36:v_c+36+1] offset:8704   ; idword:1088(8,64),  8x64 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:0  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+40], a[a_c+20]
    v_accvgpr_read_b32 v[v_c+41], a[a_c+21]
    v_accvgpr_read_b32 v[v_c+42], a[a_c+22]
    v_accvgpr_read_b32 v[v_c+43], a[a_c+23]
    v_cvt_f16_f32_e32 v[v_c+40], v[v_c+40]
    v_cvt_f16_f32_e32 v[v_c+41], v[v_c+41]
    v_cvt_f16_f32_e32 v[v_c+42], v[v_c+42]
    v_cvt_f16_f32_e32 v[v_c+43], v[v_c+43]
    v_pack_b32_f16 v[v_c+40], v[v_c+40], v[v_c+41]
    v_pack_b32_f16 v[v_c+41], v[v_c+42], v[v_c+43]
    ds_write_b64 v[v_co_sst], v[v_c+40:v_c+40+1] offset:10240   ; idword:1280(10,0),  10x0 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:1  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+44], a[a_c+52]
    v_accvgpr_read_b32 v[v_c+45], a[a_c+53]
    v_accvgpr_read_b32 v[v_c+46], a[a_c+54]
    v_accvgpr_read_b32 v[v_c+47], a[a_c+55]
    v_cvt_f16_f32_e32 v[v_c+44], v[v_c+44]
    v_cvt_f16_f32_e32 v[v_c+45], v[v_c+45]
    v_cvt_f16_f32_e32 v[v_c+46], v[v_c+46]
    v_cvt_f16_f32_e32 v[v_c+47], v[v_c+47]
    v_pack_b32_f16 v[v_c+44], v[v_c+44], v[v_c+45]
    v_pack_b32_f16 v[v_c+45], v[v_c+46], v[v_c+47]
    ds_write_b64 v[v_co_sst], v[v_c+44:v_c+44+1] offset:10752   ; idword:1344(10,64),  10x64 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:1  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+48], a[a_c+24]
    v_accvgpr_read_b32 v[v_c+49], a[a_c+25]
    v_accvgpr_read_b32 v[v_c+50], a[a_c+26]
    v_accvgpr_read_b32 v[v_c+51], a[a_c+27]
    v_cvt_f16_f32_e32 v[v_c+48], v[v_c+48]
    v_cvt_f16_f32_e32 v[v_c+49], v[v_c+49]
    v_cvt_f16_f32_e32 v[v_c+50], v[v_c+50]
    v_cvt_f16_f32_e32 v[v_c+51], v[v_c+51]
    v_pack_b32_f16 v[v_c+48], v[v_c+48], v[v_c+49]
    v_pack_b32_f16 v[v_c+49], v[v_c+50], v[v_c+51]
    ds_write_b64 v[v_co_sst], v[v_c+48:v_c+48+1] offset:12288   ; idword:1536(12,0),  12x0 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:2  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+52], a[a_c+56]
    v_accvgpr_read_b32 v[v_c+53], a[a_c+57]
    v_accvgpr_read_b32 v[v_c+54], a[a_c+58]
    v_accvgpr_read_b32 v[v_c+55], a[a_c+59]
    v_cvt_f16_f32_e32 v[v_c+52], v[v_c+52]
    v_cvt_f16_f32_e32 v[v_c+53], v[v_c+53]
    v_cvt_f16_f32_e32 v[v_c+54], v[v_c+54]
    v_cvt_f16_f32_e32 v[v_c+55], v[v_c+55]
    v_pack_b32_f16 v[v_c+52], v[v_c+52], v[v_c+53]
    v_pack_b32_f16 v[v_c+53], v[v_c+54], v[v_c+55]
    ds_write_b64 v[v_co_sst], v[v_c+52:v_c+52+1] offset:12800   ; idword:1600(12,64),  12x64 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:2  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+56], a[a_c+28]
    v_accvgpr_read_b32 v[v_c+57], a[a_c+29]
    v_accvgpr_read_b32 v[v_c+58], a[a_c+30]
    v_accvgpr_read_b32 v[v_c+59], a[a_c+31]
    v_cvt_f16_f32_e32 v[v_c+56], v[v_c+56]
    v_cvt_f16_f32_e32 v[v_c+57], v[v_c+57]
    v_cvt_f16_f32_e32 v[v_c+58], v[v_c+58]
    v_cvt_f16_f32_e32 v[v_c+59], v[v_c+59]
    v_pack_b32_f16 v[v_c+56], v[v_c+56], v[v_c+57]
    v_pack_b32_f16 v[v_c+57], v[v_c+58], v[v_c+59]
    ds_write_b64 v[v_co_sst], v[v_c+56:v_c+56+1] offset:14336   ; idword:1792(14,0),  14x0 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:3  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+60], a[a_c+60]
    v_accvgpr_read_b32 v[v_c+61], a[a_c+61]
    v_accvgpr_read_b32 v[v_c+62], a[a_c+62]
    v_accvgpr_read_b32 v[v_c+63], a[a_c+63]
    v_cvt_f16_f32_e32 v[v_c+60], v[v_c+60]
    v_cvt_f16_f32_e32 v[v_c+61], v[v_c+61]
    v_cvt_f16_f32_e32 v[v_c+62], v[v_c+62]
    v_cvt_f16_f32_e32 v[v_c+63], v[v_c+63]
    v_pack_b32_f16 v[v_c+60], v[v_c+60], v[v_c+61]
    v_pack_b32_f16 v[v_c+61], v[v_c+62], v[v_c+63]
    ds_write_b64 v[v_co_sst], v[v_c+60:v_c+60+1] offset:14848   ; idword:1856(14,64),  14x64 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:3  x  i_nr:1, i_ns:0, i_nw:0
    s_waitcnt lgkmcnt(0)
    s_barrier
    ;   load from lds
    ds_read_b64 v[v_c:v_c+1], v[v_co_sld] 
    ds_read_b64 v[v_c+2:v_c+2+1], v[v_co_sld] offset:2048
    ds_read_b64 v[v_c+4:v_c+4+1], v[v_co_sld] offset:4096
    ds_read_b64 v[v_c+6:v_c+6+1], v[v_co_sld] offset:6144
    ds_read_b64 v[v_c+8:v_c+8+1], v[v_co_sld] offset:8192
    ds_read_b64 v[v_c+10:v_c+10+1], v[v_co_sld] offset:10240
    ds_read_b64 v[v_c+12:v_c+12+1], v[v_co_sld] offset:12288
    ds_read_b64 v[v_c+14:v_c+14+1], v[v_co_sld] offset:14336
    ds_read_b64 v[v_c+16:v_c+16+1], v[v_co_sld] offset:16384
    ds_read_b64 v[v_c+18:v_c+18+1], v[v_co_sld] offset:18432
    ds_read_b64 v[v_c+20:v_c+20+1], v[v_co_sld] offset:20480
    ds_read_b64 v[v_c+22:v_c+22+1], v[v_co_sld] offset:22528
    ds_read_b64 v[v_c+24:v_c+24+1], v[v_co_sld] offset:24576
    ds_read_b64 v[v_c+26:v_c+26+1], v[v_co_sld] offset:26624
    ds_read_b64 v[v_c+28:v_c+28+1], v[v_co_sld] offset:28672
    ds_read_b64 v[v_c+30:v_c+30+1], v[v_co_sld] offset:30720
    ;   store to global, m index start from 0, m0:0, m1:0
    s_mov_b32 s[s_tmp], 0   ; i_m:0(i_m0:0,i_m1:0)
    v_add_u32 v[v_cur_k], s[s_block_gtc_ik], v[v_co_sub_m_index]
    v_mov_b32 v[v_tmp], v[v_cur_k]
    s_waitcnt lgkmcnt(15)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mov_b32 s[s_tmp], s[s_out_stride_k]   ; i_m:1(i_m0:0,i_m1:1)
    v_add_u32 v[v_tmp], 1, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 2, s[s_out_stride_k]   ; i_m:2(i_m0:0,i_m1:2)
    v_add_u32 v[v_tmp], 2, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+1], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 3, s[s_out_stride_k]   ; i_m:3(i_m0:0,i_m1:3)
    v_add_u32 v[v_tmp], 3, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+1], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 8, s[s_out_stride_k]   ; i_m:8(i_m0:0,i_m1:8)
    v_add_u32 v[v_tmp], 8, v[v_cur_k]
    s_waitcnt lgkmcnt(14)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+2], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 9, s[s_out_stride_k]   ; i_m:9(i_m0:0,i_m1:9)
    v_add_u32 v[v_tmp], 9, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+2], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 10, s[s_out_stride_k]   ; i_m:10(i_m0:0,i_m1:10)
    v_add_u32 v[v_tmp], 10, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+3], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 11, s[s_out_stride_k]   ; i_m:11(i_m0:0,i_m1:11)
    v_add_u32 v[v_tmp], 11, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+3], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 16, s[s_out_stride_k]   ; i_m:16(i_m0:0,i_m1:16)
    v_add_u32 v[v_tmp], 16, v[v_cur_k]
    s_waitcnt lgkmcnt(13)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+4], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 17, s[s_out_stride_k]   ; i_m:17(i_m0:0,i_m1:17)
    v_add_u32 v[v_tmp], 17, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+4], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 18, s[s_out_stride_k]   ; i_m:18(i_m0:0,i_m1:18)
    v_add_u32 v[v_tmp], 18, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+5], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 19, s[s_out_stride_k]   ; i_m:19(i_m0:0,i_m1:19)
    v_add_u32 v[v_tmp], 19, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+5], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 24, s[s_out_stride_k]   ; i_m:24(i_m0:0,i_m1:24)
    v_add_u32 v[v_tmp], 24, v[v_cur_k]
    s_waitcnt lgkmcnt(12)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+6], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 25, s[s_out_stride_k]   ; i_m:25(i_m0:0,i_m1:25)
    v_add_u32 v[v_tmp], 25, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+6], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 26, s[s_out_stride_k]   ; i_m:26(i_m0:0,i_m1:26)
    v_add_u32 v[v_tmp], 26, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+7], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 27, s[s_out_stride_k]   ; i_m:27(i_m0:0,i_m1:27)
    v_add_u32 v[v_tmp], 27, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+7], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 32, s[s_out_stride_k]   ; i_m:32(i_m0:0,i_m1:32)
    v_add_u32 v[v_tmp], 32, v[v_cur_k]
    s_waitcnt lgkmcnt(11)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+8], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 33, s[s_out_stride_k]   ; i_m:33(i_m0:0,i_m1:33)
    v_add_u32 v[v_tmp], 33, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+8], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 34, s[s_out_stride_k]   ; i_m:34(i_m0:0,i_m1:34)
    v_add_u32 v[v_tmp], 34, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+9], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 35, s[s_out_stride_k]   ; i_m:35(i_m0:0,i_m1:35)
    v_add_u32 v[v_tmp], 35, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+9], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 40, s[s_out_stride_k]   ; i_m:40(i_m0:0,i_m1:40)
    v_add_u32 v[v_tmp], 40, v[v_cur_k]
    s_waitcnt lgkmcnt(10)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+10], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 41, s[s_out_stride_k]   ; i_m:41(i_m0:0,i_m1:41)
    v_add_u32 v[v_tmp], 41, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+10], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 42, s[s_out_stride_k]   ; i_m:42(i_m0:0,i_m1:42)
    v_add_u32 v[v_tmp], 42, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+11], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 43, s[s_out_stride_k]   ; i_m:43(i_m0:0,i_m1:43)
    v_add_u32 v[v_tmp], 43, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+11], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 48, s[s_out_stride_k]   ; i_m:48(i_m0:0,i_m1:48)
    v_add_u32 v[v_tmp], 48, v[v_cur_k]
    s_waitcnt lgkmcnt(9)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+12], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 49, s[s_out_stride_k]   ; i_m:49(i_m0:0,i_m1:49)
    v_add_u32 v[v_tmp], 49, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+12], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 50, s[s_out_stride_k]   ; i_m:50(i_m0:0,i_m1:50)
    v_add_u32 v[v_tmp], 50, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+13], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 51, s[s_out_stride_k]   ; i_m:51(i_m0:0,i_m1:51)
    v_add_u32 v[v_tmp], 51, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+13], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 56, s[s_out_stride_k]   ; i_m:56(i_m0:0,i_m1:56)
    v_add_u32 v[v_tmp], 56, v[v_cur_k]
    s_waitcnt lgkmcnt(8)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+14], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 57, s[s_out_stride_k]   ; i_m:57(i_m0:0,i_m1:57)
    v_add_u32 v[v_tmp], 57, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+14], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 58, s[s_out_stride_k]   ; i_m:58(i_m0:0,i_m1:58)
    v_add_u32 v[v_tmp], 58, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+15], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 59, s[s_out_stride_k]   ; i_m:59(i_m0:0,i_m1:59)
    v_add_u32 v[v_tmp], 59, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+15], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 64, s[s_out_stride_k]   ; i_m:64(i_m0:1,i_m1:0)
    v_add_u32 v[v_tmp], 64, v[v_cur_k]
    s_waitcnt lgkmcnt(7)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+16], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 65, s[s_out_stride_k]   ; i_m:65(i_m0:1,i_m1:1)
    v_add_u32 v[v_tmp], 65, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+16], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 66, s[s_out_stride_k]   ; i_m:66(i_m0:1,i_m1:2)
    v_add_u32 v[v_tmp], 66, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+17], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 67, s[s_out_stride_k]   ; i_m:67(i_m0:1,i_m1:3)
    v_add_u32 v[v_tmp], 67, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+17], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 72, s[s_out_stride_k]   ; i_m:72(i_m0:1,i_m1:8)
    v_add_u32 v[v_tmp], 72, v[v_cur_k]
    s_waitcnt lgkmcnt(6)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+18], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 73, s[s_out_stride_k]   ; i_m:73(i_m0:1,i_m1:9)
    v_add_u32 v[v_tmp], 73, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+18], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 74, s[s_out_stride_k]   ; i_m:74(i_m0:1,i_m1:10)
    v_add_u32 v[v_tmp], 74, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+19], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 75, s[s_out_stride_k]   ; i_m:75(i_m0:1,i_m1:11)
    v_add_u32 v[v_tmp], 75, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+19], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 80, s[s_out_stride_k]   ; i_m:80(i_m0:1,i_m1:16)
    v_add_u32 v[v_tmp], 80, v[v_cur_k]
    s_waitcnt lgkmcnt(5)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+20], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 81, s[s_out_stride_k]   ; i_m:81(i_m0:1,i_m1:17)
    v_add_u32 v[v_tmp], 81, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+20], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 82, s[s_out_stride_k]   ; i_m:82(i_m0:1,i_m1:18)
    v_add_u32 v[v_tmp], 82, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+21], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 83, s[s_out_stride_k]   ; i_m:83(i_m0:1,i_m1:19)
    v_add_u32 v[v_tmp], 83, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+21], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 88, s[s_out_stride_k]   ; i_m:88(i_m0:1,i_m1:24)
    v_add_u32 v[v_tmp], 88, v[v_cur_k]
    s_waitcnt lgkmcnt(4)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+22], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 89, s[s_out_stride_k]   ; i_m:89(i_m0:1,i_m1:25)
    v_add_u32 v[v_tmp], 89, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+22], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 90, s[s_out_stride_k]   ; i_m:90(i_m0:1,i_m1:26)
    v_add_u32 v[v_tmp], 90, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+23], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 91, s[s_out_stride_k]   ; i_m:91(i_m0:1,i_m1:27)
    v_add_u32 v[v_tmp], 91, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+23], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 96, s[s_out_stride_k]   ; i_m:96(i_m0:1,i_m1:32)
    v_add_u32 v[v_tmp], 96, v[v_cur_k]
    s_waitcnt lgkmcnt(3)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+24], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 97, s[s_out_stride_k]   ; i_m:97(i_m0:1,i_m1:33)
    v_add_u32 v[v_tmp], 97, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+24], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 98, s[s_out_stride_k]   ; i_m:98(i_m0:1,i_m1:34)
    v_add_u32 v[v_tmp], 98, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+25], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 99, s[s_out_stride_k]   ; i_m:99(i_m0:1,i_m1:35)
    v_add_u32 v[v_tmp], 99, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+25], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 104, s[s_out_stride_k]   ; i_m:104(i_m0:1,i_m1:40)
    v_add_u32 v[v_tmp], 104, v[v_cur_k]
    s_waitcnt lgkmcnt(2)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+26], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 105, s[s_out_stride_k]   ; i_m:105(i_m0:1,i_m1:41)
    v_add_u32 v[v_tmp], 105, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+26], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 106, s[s_out_stride_k]   ; i_m:106(i_m0:1,i_m1:42)
    v_add_u32 v[v_tmp], 106, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+27], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 107, s[s_out_stride_k]   ; i_m:107(i_m0:1,i_m1:43)
    v_add_u32 v[v_tmp], 107, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+27], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 112, s[s_out_stride_k]   ; i_m:112(i_m0:1,i_m1:48)
    v_add_u32 v[v_tmp], 112, v[v_cur_k]
    s_waitcnt lgkmcnt(1)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+28], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 113, s[s_out_stride_k]   ; i_m:113(i_m0:1,i_m1:49)
    v_add_u32 v[v_tmp], 113, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+28], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 114, s[s_out_stride_k]   ; i_m:114(i_m0:1,i_m1:50)
    v_add_u32 v[v_tmp], 114, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+29], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 115, s[s_out_stride_k]   ; i_m:115(i_m0:1,i_m1:51)
    v_add_u32 v[v_tmp], 115, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+29], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 120, s[s_out_stride_k]   ; i_m:120(i_m0:1,i_m1:56)
    v_add_u32 v[v_tmp], 120, v[v_cur_k]
    s_waitcnt lgkmcnt(0)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+30], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 121, s[s_out_stride_k]   ; i_m:121(i_m0:1,i_m1:57)
    v_add_u32 v[v_tmp], 121, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+30], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 122, s[s_out_stride_k]   ; i_m:122(i_m0:1,i_m1:58)
    v_add_u32 v[v_tmp], 122, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+31], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 123, s[s_out_stride_k]   ; i_m:123(i_m0:1,i_m1:59)
    v_add_u32 v[v_tmp], 123, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+31], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    ; start group 1, i_g_mr:1, i_g_ms:0, i_g_mw:0, i_g_mb:0, i_g_mt:0, m index start from 128
    s_barrier
    v_accvgpr_read_b32 v[v_c], a[a_c+64]
    v_accvgpr_read_b32 v[v_c+1], a[a_c+65]
    v_accvgpr_read_b32 v[v_c+2], a[a_c+66]
    v_accvgpr_read_b32 v[v_c+3], a[a_c+67]
    v_cvt_f16_f32_e32 v[v_c], v[v_c]
    v_cvt_f16_f32_e32 v[v_c+1], v[v_c+1]
    v_cvt_f16_f32_e32 v[v_c+2], v[v_c+2]
    v_cvt_f16_f32_e32 v[v_c+3], v[v_c+3]
    v_pack_b32_f16 v[v_c], v[v_c], v[v_c+1]
    v_pack_b32_f16 v[v_c+1], v[v_c+2], v[v_c+3]
    ds_write_b64 v[v_co_sst], v[v_c:v_c+1]    ; idword:0(0,0),  0x0 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:0  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+4], a[a_c+96]
    v_accvgpr_read_b32 v[v_c+5], a[a_c+97]
    v_accvgpr_read_b32 v[v_c+6], a[a_c+98]
    v_accvgpr_read_b32 v[v_c+7], a[a_c+99]
    v_cvt_f16_f32_e32 v[v_c+4], v[v_c+4]
    v_cvt_f16_f32_e32 v[v_c+5], v[v_c+5]
    v_cvt_f16_f32_e32 v[v_c+6], v[v_c+6]
    v_cvt_f16_f32_e32 v[v_c+7], v[v_c+7]
    v_pack_b32_f16 v[v_c+4], v[v_c+4], v[v_c+5]
    v_pack_b32_f16 v[v_c+5], v[v_c+6], v[v_c+7]
    ds_write_b64 v[v_co_sst], v[v_c+4:v_c+4+1] offset:512   ; idword:64(0,64),  0x64 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:0  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+8], a[a_c+68]
    v_accvgpr_read_b32 v[v_c+9], a[a_c+69]
    v_accvgpr_read_b32 v[v_c+10], a[a_c+70]
    v_accvgpr_read_b32 v[v_c+11], a[a_c+71]
    v_cvt_f16_f32_e32 v[v_c+8], v[v_c+8]
    v_cvt_f16_f32_e32 v[v_c+9], v[v_c+9]
    v_cvt_f16_f32_e32 v[v_c+10], v[v_c+10]
    v_cvt_f16_f32_e32 v[v_c+11], v[v_c+11]
    v_pack_b32_f16 v[v_c+8], v[v_c+8], v[v_c+9]
    v_pack_b32_f16 v[v_c+9], v[v_c+10], v[v_c+11]
    ds_write_b64 v[v_co_sst], v[v_c+8:v_c+8+1] offset:2048   ; idword:256(2,0),  2x0 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:1  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+12], a[a_c+100]
    v_accvgpr_read_b32 v[v_c+13], a[a_c+101]
    v_accvgpr_read_b32 v[v_c+14], a[a_c+102]
    v_accvgpr_read_b32 v[v_c+15], a[a_c+103]
    v_cvt_f16_f32_e32 v[v_c+12], v[v_c+12]
    v_cvt_f16_f32_e32 v[v_c+13], v[v_c+13]
    v_cvt_f16_f32_e32 v[v_c+14], v[v_c+14]
    v_cvt_f16_f32_e32 v[v_c+15], v[v_c+15]
    v_pack_b32_f16 v[v_c+12], v[v_c+12], v[v_c+13]
    v_pack_b32_f16 v[v_c+13], v[v_c+14], v[v_c+15]
    ds_write_b64 v[v_co_sst], v[v_c+12:v_c+12+1] offset:2560   ; idword:320(2,64),  2x64 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:1  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+16], a[a_c+72]
    v_accvgpr_read_b32 v[v_c+17], a[a_c+73]
    v_accvgpr_read_b32 v[v_c+18], a[a_c+74]
    v_accvgpr_read_b32 v[v_c+19], a[a_c+75]
    v_cvt_f16_f32_e32 v[v_c+16], v[v_c+16]
    v_cvt_f16_f32_e32 v[v_c+17], v[v_c+17]
    v_cvt_f16_f32_e32 v[v_c+18], v[v_c+18]
    v_cvt_f16_f32_e32 v[v_c+19], v[v_c+19]
    v_pack_b32_f16 v[v_c+16], v[v_c+16], v[v_c+17]
    v_pack_b32_f16 v[v_c+17], v[v_c+18], v[v_c+19]
    ds_write_b64 v[v_co_sst], v[v_c+16:v_c+16+1] offset:4096   ; idword:512(4,0),  4x0 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:2  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+20], a[a_c+104]
    v_accvgpr_read_b32 v[v_c+21], a[a_c+105]
    v_accvgpr_read_b32 v[v_c+22], a[a_c+106]
    v_accvgpr_read_b32 v[v_c+23], a[a_c+107]
    v_cvt_f16_f32_e32 v[v_c+20], v[v_c+20]
    v_cvt_f16_f32_e32 v[v_c+21], v[v_c+21]
    v_cvt_f16_f32_e32 v[v_c+22], v[v_c+22]
    v_cvt_f16_f32_e32 v[v_c+23], v[v_c+23]
    v_pack_b32_f16 v[v_c+20], v[v_c+20], v[v_c+21]
    v_pack_b32_f16 v[v_c+21], v[v_c+22], v[v_c+23]
    ds_write_b64 v[v_co_sst], v[v_c+20:v_c+20+1] offset:4608   ; idword:576(4,64),  4x64 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:2  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+24], a[a_c+76]
    v_accvgpr_read_b32 v[v_c+25], a[a_c+77]
    v_accvgpr_read_b32 v[v_c+26], a[a_c+78]
    v_accvgpr_read_b32 v[v_c+27], a[a_c+79]
    v_cvt_f16_f32_e32 v[v_c+24], v[v_c+24]
    v_cvt_f16_f32_e32 v[v_c+25], v[v_c+25]
    v_cvt_f16_f32_e32 v[v_c+26], v[v_c+26]
    v_cvt_f16_f32_e32 v[v_c+27], v[v_c+27]
    v_pack_b32_f16 v[v_c+24], v[v_c+24], v[v_c+25]
    v_pack_b32_f16 v[v_c+25], v[v_c+26], v[v_c+27]
    ds_write_b64 v[v_co_sst], v[v_c+24:v_c+24+1] offset:6144   ; idword:768(6,0),  6x0 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:3  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+28], a[a_c+108]
    v_accvgpr_read_b32 v[v_c+29], a[a_c+109]
    v_accvgpr_read_b32 v[v_c+30], a[a_c+110]
    v_accvgpr_read_b32 v[v_c+31], a[a_c+111]
    v_cvt_f16_f32_e32 v[v_c+28], v[v_c+28]
    v_cvt_f16_f32_e32 v[v_c+29], v[v_c+29]
    v_cvt_f16_f32_e32 v[v_c+30], v[v_c+30]
    v_cvt_f16_f32_e32 v[v_c+31], v[v_c+31]
    v_pack_b32_f16 v[v_c+28], v[v_c+28], v[v_c+29]
    v_pack_b32_f16 v[v_c+29], v[v_c+30], v[v_c+31]
    ds_write_b64 v[v_co_sst], v[v_c+28:v_c+28+1] offset:6656   ; idword:832(6,64),  6x64 | /4, i_mr:0, i_ms:0, i_mw:0, i_mb:3  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+32], a[a_c+80]
    v_accvgpr_read_b32 v[v_c+33], a[a_c+81]
    v_accvgpr_read_b32 v[v_c+34], a[a_c+82]
    v_accvgpr_read_b32 v[v_c+35], a[a_c+83]
    v_cvt_f16_f32_e32 v[v_c+32], v[v_c+32]
    v_cvt_f16_f32_e32 v[v_c+33], v[v_c+33]
    v_cvt_f16_f32_e32 v[v_c+34], v[v_c+34]
    v_cvt_f16_f32_e32 v[v_c+35], v[v_c+35]
    v_pack_b32_f16 v[v_c+32], v[v_c+32], v[v_c+33]
    v_pack_b32_f16 v[v_c+33], v[v_c+34], v[v_c+35]
    ds_write_b64 v[v_co_sst], v[v_c+32:v_c+32+1] offset:8192   ; idword:1024(8,0),  8x0 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:0  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+36], a[a_c+112]
    v_accvgpr_read_b32 v[v_c+37], a[a_c+113]
    v_accvgpr_read_b32 v[v_c+38], a[a_c+114]
    v_accvgpr_read_b32 v[v_c+39], a[a_c+115]
    v_cvt_f16_f32_e32 v[v_c+36], v[v_c+36]
    v_cvt_f16_f32_e32 v[v_c+37], v[v_c+37]
    v_cvt_f16_f32_e32 v[v_c+38], v[v_c+38]
    v_cvt_f16_f32_e32 v[v_c+39], v[v_c+39]
    v_pack_b32_f16 v[v_c+36], v[v_c+36], v[v_c+37]
    v_pack_b32_f16 v[v_c+37], v[v_c+38], v[v_c+39]
    ds_write_b64 v[v_co_sst], v[v_c+36:v_c+36+1] offset:8704   ; idword:1088(8,64),  8x64 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:0  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+40], a[a_c+84]
    v_accvgpr_read_b32 v[v_c+41], a[a_c+85]
    v_accvgpr_read_b32 v[v_c+42], a[a_c+86]
    v_accvgpr_read_b32 v[v_c+43], a[a_c+87]
    v_cvt_f16_f32_e32 v[v_c+40], v[v_c+40]
    v_cvt_f16_f32_e32 v[v_c+41], v[v_c+41]
    v_cvt_f16_f32_e32 v[v_c+42], v[v_c+42]
    v_cvt_f16_f32_e32 v[v_c+43], v[v_c+43]
    v_pack_b32_f16 v[v_c+40], v[v_c+40], v[v_c+41]
    v_pack_b32_f16 v[v_c+41], v[v_c+42], v[v_c+43]
    ds_write_b64 v[v_co_sst], v[v_c+40:v_c+40+1] offset:10240   ; idword:1280(10,0),  10x0 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:1  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+44], a[a_c+116]
    v_accvgpr_read_b32 v[v_c+45], a[a_c+117]
    v_accvgpr_read_b32 v[v_c+46], a[a_c+118]
    v_accvgpr_read_b32 v[v_c+47], a[a_c+119]
    v_cvt_f16_f32_e32 v[v_c+44], v[v_c+44]
    v_cvt_f16_f32_e32 v[v_c+45], v[v_c+45]
    v_cvt_f16_f32_e32 v[v_c+46], v[v_c+46]
    v_cvt_f16_f32_e32 v[v_c+47], v[v_c+47]
    v_pack_b32_f16 v[v_c+44], v[v_c+44], v[v_c+45]
    v_pack_b32_f16 v[v_c+45], v[v_c+46], v[v_c+47]
    ds_write_b64 v[v_co_sst], v[v_c+44:v_c+44+1] offset:10752   ; idword:1344(10,64),  10x64 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:1  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+48], a[a_c+88]
    v_accvgpr_read_b32 v[v_c+49], a[a_c+89]
    v_accvgpr_read_b32 v[v_c+50], a[a_c+90]
    v_accvgpr_read_b32 v[v_c+51], a[a_c+91]
    v_cvt_f16_f32_e32 v[v_c+48], v[v_c+48]
    v_cvt_f16_f32_e32 v[v_c+49], v[v_c+49]
    v_cvt_f16_f32_e32 v[v_c+50], v[v_c+50]
    v_cvt_f16_f32_e32 v[v_c+51], v[v_c+51]
    v_pack_b32_f16 v[v_c+48], v[v_c+48], v[v_c+49]
    v_pack_b32_f16 v[v_c+49], v[v_c+50], v[v_c+51]
    ds_write_b64 v[v_co_sst], v[v_c+48:v_c+48+1] offset:12288   ; idword:1536(12,0),  12x0 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:2  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+52], a[a_c+120]
    v_accvgpr_read_b32 v[v_c+53], a[a_c+121]
    v_accvgpr_read_b32 v[v_c+54], a[a_c+122]
    v_accvgpr_read_b32 v[v_c+55], a[a_c+123]
    v_cvt_f16_f32_e32 v[v_c+52], v[v_c+52]
    v_cvt_f16_f32_e32 v[v_c+53], v[v_c+53]
    v_cvt_f16_f32_e32 v[v_c+54], v[v_c+54]
    v_cvt_f16_f32_e32 v[v_c+55], v[v_c+55]
    v_pack_b32_f16 v[v_c+52], v[v_c+52], v[v_c+53]
    v_pack_b32_f16 v[v_c+53], v[v_c+54], v[v_c+55]
    ds_write_b64 v[v_co_sst], v[v_c+52:v_c+52+1] offset:12800   ; idword:1600(12,64),  12x64 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:2  x  i_nr:1, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+56], a[a_c+92]
    v_accvgpr_read_b32 v[v_c+57], a[a_c+93]
    v_accvgpr_read_b32 v[v_c+58], a[a_c+94]
    v_accvgpr_read_b32 v[v_c+59], a[a_c+95]
    v_cvt_f16_f32_e32 v[v_c+56], v[v_c+56]
    v_cvt_f16_f32_e32 v[v_c+57], v[v_c+57]
    v_cvt_f16_f32_e32 v[v_c+58], v[v_c+58]
    v_cvt_f16_f32_e32 v[v_c+59], v[v_c+59]
    v_pack_b32_f16 v[v_c+56], v[v_c+56], v[v_c+57]
    v_pack_b32_f16 v[v_c+57], v[v_c+58], v[v_c+59]
    ds_write_b64 v[v_co_sst], v[v_c+56:v_c+56+1] offset:14336   ; idword:1792(14,0),  14x0 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:3  x  i_nr:0, i_ns:0, i_nw:0
    v_accvgpr_read_b32 v[v_c+60], a[a_c+124]
    v_accvgpr_read_b32 v[v_c+61], a[a_c+125]
    v_accvgpr_read_b32 v[v_c+62], a[a_c+126]
    v_accvgpr_read_b32 v[v_c+63], a[a_c+127]
    v_cvt_f16_f32_e32 v[v_c+60], v[v_c+60]
    v_cvt_f16_f32_e32 v[v_c+61], v[v_c+61]
    v_cvt_f16_f32_e32 v[v_c+62], v[v_c+62]
    v_cvt_f16_f32_e32 v[v_c+63], v[v_c+63]
    v_pack_b32_f16 v[v_c+60], v[v_c+60], v[v_c+61]
    v_pack_b32_f16 v[v_c+61], v[v_c+62], v[v_c+63]
    ds_write_b64 v[v_co_sst], v[v_c+60:v_c+60+1] offset:14848   ; idword:1856(14,64),  14x64 | /4, i_mr:0, i_ms:1, i_mw:0, i_mb:3  x  i_nr:1, i_ns:0, i_nw:0
    s_waitcnt lgkmcnt(0)
    s_barrier
    ;   load from lds
    ds_read_b64 v[v_c:v_c+1], v[v_co_sld] 
    ds_read_b64 v[v_c+2:v_c+2+1], v[v_co_sld] offset:2048
    ds_read_b64 v[v_c+4:v_c+4+1], v[v_co_sld] offset:4096
    ds_read_b64 v[v_c+6:v_c+6+1], v[v_co_sld] offset:6144
    ds_read_b64 v[v_c+8:v_c+8+1], v[v_co_sld] offset:8192
    ds_read_b64 v[v_c+10:v_c+10+1], v[v_co_sld] offset:10240
    ds_read_b64 v[v_c+12:v_c+12+1], v[v_co_sld] offset:12288
    ds_read_b64 v[v_c+14:v_c+14+1], v[v_co_sld] offset:14336
    ds_read_b64 v[v_c+16:v_c+16+1], v[v_co_sld] offset:16384
    ds_read_b64 v[v_c+18:v_c+18+1], v[v_co_sld] offset:18432
    ds_read_b64 v[v_c+20:v_c+20+1], v[v_co_sld] offset:20480
    ds_read_b64 v[v_c+22:v_c+22+1], v[v_co_sld] offset:22528
    ds_read_b64 v[v_c+24:v_c+24+1], v[v_co_sld] offset:24576
    ds_read_b64 v[v_c+26:v_c+26+1], v[v_co_sld] offset:26624
    ds_read_b64 v[v_c+28:v_c+28+1], v[v_co_sld] offset:28672
    ds_read_b64 v[v_c+30:v_c+30+1], v[v_co_sld] offset:30720
    ;   store to global, m index start from 128, m0:2, m1:0
    s_mul_i32 s[s_tmp], 128, s[s_out_stride_k]   ; i_m:128(i_m0:2,i_m1:0)
    v_add_u32 v[v_tmp], 128, v[v_cur_k]
    s_waitcnt lgkmcnt(15)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 129, s[s_out_stride_k]   ; i_m:129(i_m0:2,i_m1:1)
    v_add_u32 v[v_tmp], 129, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 130, s[s_out_stride_k]   ; i_m:130(i_m0:2,i_m1:2)
    v_add_u32 v[v_tmp], 130, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+1], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 131, s[s_out_stride_k]   ; i_m:131(i_m0:2,i_m1:3)
    v_add_u32 v[v_tmp], 131, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+1], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 136, s[s_out_stride_k]   ; i_m:136(i_m0:2,i_m1:8)
    v_add_u32 v[v_tmp], 136, v[v_cur_k]
    s_waitcnt lgkmcnt(14)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+2], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 137, s[s_out_stride_k]   ; i_m:137(i_m0:2,i_m1:9)
    v_add_u32 v[v_tmp], 137, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+2], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 138, s[s_out_stride_k]   ; i_m:138(i_m0:2,i_m1:10)
    v_add_u32 v[v_tmp], 138, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+3], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 139, s[s_out_stride_k]   ; i_m:139(i_m0:2,i_m1:11)
    v_add_u32 v[v_tmp], 139, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+3], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 144, s[s_out_stride_k]   ; i_m:144(i_m0:2,i_m1:16)
    v_add_u32 v[v_tmp], 144, v[v_cur_k]
    s_waitcnt lgkmcnt(13)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+4], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 145, s[s_out_stride_k]   ; i_m:145(i_m0:2,i_m1:17)
    v_add_u32 v[v_tmp], 145, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+4], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 146, s[s_out_stride_k]   ; i_m:146(i_m0:2,i_m1:18)
    v_add_u32 v[v_tmp], 146, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+5], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 147, s[s_out_stride_k]   ; i_m:147(i_m0:2,i_m1:19)
    v_add_u32 v[v_tmp], 147, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+5], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 152, s[s_out_stride_k]   ; i_m:152(i_m0:2,i_m1:24)
    v_add_u32 v[v_tmp], 152, v[v_cur_k]
    s_waitcnt lgkmcnt(12)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+6], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 153, s[s_out_stride_k]   ; i_m:153(i_m0:2,i_m1:25)
    v_add_u32 v[v_tmp], 153, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+6], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 154, s[s_out_stride_k]   ; i_m:154(i_m0:2,i_m1:26)
    v_add_u32 v[v_tmp], 154, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+7], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 155, s[s_out_stride_k]   ; i_m:155(i_m0:2,i_m1:27)
    v_add_u32 v[v_tmp], 155, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+7], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 160, s[s_out_stride_k]   ; i_m:160(i_m0:2,i_m1:32)
    v_add_u32 v[v_tmp], 160, v[v_cur_k]
    s_waitcnt lgkmcnt(11)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+8], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 161, s[s_out_stride_k]   ; i_m:161(i_m0:2,i_m1:33)
    v_add_u32 v[v_tmp], 161, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+8], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 162, s[s_out_stride_k]   ; i_m:162(i_m0:2,i_m1:34)
    v_add_u32 v[v_tmp], 162, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+9], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 163, s[s_out_stride_k]   ; i_m:163(i_m0:2,i_m1:35)
    v_add_u32 v[v_tmp], 163, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+9], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 168, s[s_out_stride_k]   ; i_m:168(i_m0:2,i_m1:40)
    v_add_u32 v[v_tmp], 168, v[v_cur_k]
    s_waitcnt lgkmcnt(10)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+10], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 169, s[s_out_stride_k]   ; i_m:169(i_m0:2,i_m1:41)
    v_add_u32 v[v_tmp], 169, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+10], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 170, s[s_out_stride_k]   ; i_m:170(i_m0:2,i_m1:42)
    v_add_u32 v[v_tmp], 170, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+11], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 171, s[s_out_stride_k]   ; i_m:171(i_m0:2,i_m1:43)
    v_add_u32 v[v_tmp], 171, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+11], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 176, s[s_out_stride_k]   ; i_m:176(i_m0:2,i_m1:48)
    v_add_u32 v[v_tmp], 176, v[v_cur_k]
    s_waitcnt lgkmcnt(9)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+12], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 177, s[s_out_stride_k]   ; i_m:177(i_m0:2,i_m1:49)
    v_add_u32 v[v_tmp], 177, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+12], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 178, s[s_out_stride_k]   ; i_m:178(i_m0:2,i_m1:50)
    v_add_u32 v[v_tmp], 178, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+13], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 179, s[s_out_stride_k]   ; i_m:179(i_m0:2,i_m1:51)
    v_add_u32 v[v_tmp], 179, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+13], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 184, s[s_out_stride_k]   ; i_m:184(i_m0:2,i_m1:56)
    v_add_u32 v[v_tmp], 184, v[v_cur_k]
    s_waitcnt lgkmcnt(8)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+14], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 185, s[s_out_stride_k]   ; i_m:185(i_m0:2,i_m1:57)
    v_add_u32 v[v_tmp], 185, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+14], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 186, s[s_out_stride_k]   ; i_m:186(i_m0:2,i_m1:58)
    v_add_u32 v[v_tmp], 186, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+15], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 187, s[s_out_stride_k]   ; i_m:187(i_m0:2,i_m1:59)
    v_add_u32 v[v_tmp], 187, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+15], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 192, s[s_out_stride_k]   ; i_m:192(i_m0:3,i_m1:0)
    v_add_u32 v[v_tmp], 192, v[v_cur_k]
    s_waitcnt lgkmcnt(7)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+16], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 193, s[s_out_stride_k]   ; i_m:193(i_m0:3,i_m1:1)
    v_add_u32 v[v_tmp], 193, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+16], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 194, s[s_out_stride_k]   ; i_m:194(i_m0:3,i_m1:2)
    v_add_u32 v[v_tmp], 194, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+17], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 195, s[s_out_stride_k]   ; i_m:195(i_m0:3,i_m1:3)
    v_add_u32 v[v_tmp], 195, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+17], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 200, s[s_out_stride_k]   ; i_m:200(i_m0:3,i_m1:8)
    v_add_u32 v[v_tmp], 200, v[v_cur_k]
    s_waitcnt lgkmcnt(6)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+18], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 201, s[s_out_stride_k]   ; i_m:201(i_m0:3,i_m1:9)
    v_add_u32 v[v_tmp], 201, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+18], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 202, s[s_out_stride_k]   ; i_m:202(i_m0:3,i_m1:10)
    v_add_u32 v[v_tmp], 202, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+19], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 203, s[s_out_stride_k]   ; i_m:203(i_m0:3,i_m1:11)
    v_add_u32 v[v_tmp], 203, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+19], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 208, s[s_out_stride_k]   ; i_m:208(i_m0:3,i_m1:16)
    v_add_u32 v[v_tmp], 208, v[v_cur_k]
    s_waitcnt lgkmcnt(5)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+20], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 209, s[s_out_stride_k]   ; i_m:209(i_m0:3,i_m1:17)
    v_add_u32 v[v_tmp], 209, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+20], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 210, s[s_out_stride_k]   ; i_m:210(i_m0:3,i_m1:18)
    v_add_u32 v[v_tmp], 210, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+21], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 211, s[s_out_stride_k]   ; i_m:211(i_m0:3,i_m1:19)
    v_add_u32 v[v_tmp], 211, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+21], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 216, s[s_out_stride_k]   ; i_m:216(i_m0:3,i_m1:24)
    v_add_u32 v[v_tmp], 216, v[v_cur_k]
    s_waitcnt lgkmcnt(4)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+22], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 217, s[s_out_stride_k]   ; i_m:217(i_m0:3,i_m1:25)
    v_add_u32 v[v_tmp], 217, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+22], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 218, s[s_out_stride_k]   ; i_m:218(i_m0:3,i_m1:26)
    v_add_u32 v[v_tmp], 218, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+23], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 219, s[s_out_stride_k]   ; i_m:219(i_m0:3,i_m1:27)
    v_add_u32 v[v_tmp], 219, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+23], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 224, s[s_out_stride_k]   ; i_m:224(i_m0:3,i_m1:32)
    v_add_u32 v[v_tmp], 224, v[v_cur_k]
    s_waitcnt lgkmcnt(3)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+24], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 225, s[s_out_stride_k]   ; i_m:225(i_m0:3,i_m1:33)
    v_add_u32 v[v_tmp], 225, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+24], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 226, s[s_out_stride_k]   ; i_m:226(i_m0:3,i_m1:34)
    v_add_u32 v[v_tmp], 226, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+25], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 227, s[s_out_stride_k]   ; i_m:227(i_m0:3,i_m1:35)
    v_add_u32 v[v_tmp], 227, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+25], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 232, s[s_out_stride_k]   ; i_m:232(i_m0:3,i_m1:40)
    v_add_u32 v[v_tmp], 232, v[v_cur_k]
    s_waitcnt lgkmcnt(2)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+26], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 233, s[s_out_stride_k]   ; i_m:233(i_m0:3,i_m1:41)
    v_add_u32 v[v_tmp], 233, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+26], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 234, s[s_out_stride_k]   ; i_m:234(i_m0:3,i_m1:42)
    v_add_u32 v[v_tmp], 234, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+27], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 235, s[s_out_stride_k]   ; i_m:235(i_m0:3,i_m1:43)
    v_add_u32 v[v_tmp], 235, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+27], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 240, s[s_out_stride_k]   ; i_m:240(i_m0:3,i_m1:48)
    v_add_u32 v[v_tmp], 240, v[v_cur_k]
    s_waitcnt lgkmcnt(1)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+28], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 241, s[s_out_stride_k]   ; i_m:241(i_m0:3,i_m1:49)
    v_add_u32 v[v_tmp], 241, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+28], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 242, s[s_out_stride_k]   ; i_m:242(i_m0:3,i_m1:50)
    v_add_u32 v[v_tmp], 242, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+29], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 243, s[s_out_stride_k]   ; i_m:243(i_m0:3,i_m1:51)
    v_add_u32 v[v_tmp], 243, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+29], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 248, s[s_out_stride_k]   ; i_m:248(i_m0:3,i_m1:56)
    v_add_u32 v[v_tmp], 248, v[v_cur_k]
    s_waitcnt lgkmcnt(0)
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+30], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 249, s[s_out_stride_k]   ; i_m:249(i_m0:3,i_m1:57)
    v_add_u32 v[v_tmp], 249, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+30], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 250, s[s_out_stride_k]   ; i_m:250(i_m0:3,i_m1:58)
    v_add_u32 v[v_tmp], 250, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short v[v_c+31], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
    s_mul_i32 s[s_tmp], 251, s[s_out_stride_k]   ; i_m:251(i_m0:3,i_m1:59)
    v_add_u32 v[v_tmp], 251, v[v_cur_k]
    v_cmp_gt_u32 vcc, s[s_k], v[v_tmp]
    s_and_saveexec_b64 s[s_tmp+4:s_tmp+5], vcc
    buffer_store_short_d16_hi v[v_c+31], v[v_out_os], s[s_p_out:s_p_out+3], s[s_tmp] offen offset:0
    s_or_b64 exec, exec, s[s_tmp+4:s_tmp+5]
L_igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8_out:
    s_endpgm
.rodata
.p2align 6
.amdhsa_kernel igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8
    .amdhsa_group_segment_fixed_size 32768
    .amdhsa_user_sgpr_kernarg_segment_ptr 1
    .amdhsa_system_sgpr_workgroup_id_x 1
    .amdhsa_system_sgpr_workgroup_id_y 1
    .amdhsa_system_vgpr_workitem_id 0
    .amdhsa_next_free_vgpr 128
    .amdhsa_next_free_sgpr 60
    .amdhsa_ieee_mode 0
    .amdhsa_dx10_clamp 0
.end_amdhsa_kernel

.amdgpu_metadata
---
amdhsa.version: [ 1, 0 ]
amdhsa.kernels:
  - .name: igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8
    .symbol: igemm_fwd_gtcx_nchw_fp16_bx1_ex0_bt256x128x32_wt32x32x8_ws2x1_wr2x2_ta1x8x4x1_1x4x1x64_tb1x1x16x1_1x32x1x8.kd
    .sgpr_count: 66
    .vgpr_count: 128
    .kernarg_segment_align: 8
    .kernarg_segment_size: 128
    .group_segment_fixed_size: 32768
    .private_segment_fixed_size: 0
    .wavefront_size: 64
    .reqd_workgroup_size : [256, 1, 1]
    .max_flat_workgroup_size: 256
    .args:
    - { .name: p_in      , .size: 8, .offset:   0, .value_kind: global_buffer, .value_type: f32, .address_space: global, .is_const: true}
    - { .name: p_wei     , .size: 8, .offset:   8, .value_kind: global_buffer, .value_type: f32, .address_space: global, .is_const: true}
    - { .name: p_out     , .size: 8, .offset:  16, .value_kind: global_buffer, .value_type: f32, .address_space: global, .is_const: false}
    - { .name: hi        , .size: 4, .offset:  24, .value_kind: by_value, .value_type: i32}
    - { .name: wi        , .size: 4, .offset:  28, .value_kind: by_value, .value_type: i32}
    - { .name: n_         , .size: 4, .offset:  32, .value_kind: by_value, .value_type: i32}
    - { .name: k         , .size: 4, .offset:  36, .value_kind: by_value, .value_type: i32}
    - { .name: c         , .size: 4, .offset:  40, .value_kind: by_value, .value_type: i32}
    - { .name: ho        , .size: 4, .offset:  44, .value_kind: by_value, .value_type: i32}
    - { .name: wo        , .size: 4, .offset:  48, .value_kind: by_value, .value_type: i32}
    - { .name: stride_h  , .size: 4, .offset:  52, .value_kind: by_value, .value_type: i32}
    - { .name: stride_w  , .size: 4, .offset:  56, .value_kind: by_value, .value_type: i32}
    - { .name: dilation_h, .size: 4, .offset:  60, .value_kind: by_value, .value_type: i32}
    - { .name: dilation_w, .size: 4, .offset:  64, .value_kind: by_value, .value_type: i32}
    - { .name: pad_h     , .size: 4, .offset:  68, .value_kind: by_value, .value_type: i32}
    - { .name: pad_w     , .size: 4, .offset:  72, .value_kind: by_value, .value_type: i32}
    - { .name: y_         , .size: 4, .offset:  76, .value_kind: by_value, .value_type: i32}
    - { .name: x         , .size: 4, .offset:  80, .value_kind: by_value, .value_type: i32}
    - { .name: group     , .size: 4, .offset:  84, .value_kind: by_value, .value_type: i32}
    - { .name: magic_0   , .size: 4, .offset:  88, .value_kind: by_value, .value_type: i32}
    - { .name: magic_1   , .size: 4, .offset:  92, .value_kind: by_value, .value_type: i32}
    - { .name: magic_2   , .size: 4, .offset:  96, .value_kind: by_value, .value_type: i32}
    - { .name: magic_3   , .size: 4, .offset: 100, .value_kind: by_value, .value_type: i32}
    - { .name: magic_4   , .size: 4, .offset: 104, .value_kind: by_value, .value_type: i32}
    - { .name: magic_5   , .size: 4, .offset: 108, .value_kind: by_value, .value_type: i32}
    - { .name: magic_6   , .size: 4, .offset: 112, .value_kind: by_value, .value_type: i32}
    - { .name: shift_pack_0, .size: 4, .offset: 116, .value_kind: by_value, .value_type: i32}
    - { .name: shift_pack_1, .size: 4, .offset: 120, .value_kind: by_value, .value_type: i32}
    - { .name: __pack_0  , .size: 4, .offset: 124, .value_kind: by_value, .value_type: i32}
...
.end_amdgpu_metadata
