/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2020-2021 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
; generated by igemm_codegen.py (6612fa4de68b883367c3388e88a0aa56d301a99e)
;
.macro .v_u32_div v_q, v_n, v_d, v_tmp4, s_tmp4
    v_cvt_f32_u32 v[\v_tmp4+0], v[\v_d]
    v_sub_i32 v[\v_tmp4+3], 0, v[\v_d]
    v_rcp_iflag_f32 v[\v_tmp4+0], v[\v_tmp4+0]
    s_nop 0
    v_mul_f32 v[\v_tmp4+0], 0x4f7ffffe, v[\v_tmp4+0]
    v_cvt_u32_f32 v[\v_tmp4+0], v[\v_tmp4+0]
    v_mul_lo_u32 v[\v_tmp4+1], v[\v_tmp4+3], v[\v_tmp4+0]
    v_mul_hi_u32 v[\v_tmp4+1], v[\v_tmp4+0], v[\v_tmp4+1]
    v_add_u32 v[\v_tmp4+0], v[\v_tmp4+0], v[\v_tmp4+1]
    v_mul_hi_u32 v[\v_tmp4+0], v[\v_n], v[\v_tmp4+0]
    v_mul_lo_u32 v[\v_tmp4+1], v[\v_tmp4+0], v[\v_d]
    v_sub_u32 v[\v_q], v[\v_n], v[\v_tmp4+1]
    v_add_u32 v[\v_tmp4+2], 1, v[\v_tmp4+0]
    v_cmp_le_u32 vcc, v[\v_d], v[\v_q]
    v_subrev_u32 v[\v_tmp4+1], v[\v_d], v[\v_q]
    s_nop 0
    v_cndmask_b32 v[\v_tmp4+0], v[\v_tmp4+0], v[\v_tmp4+2], vcc
    v_cndmask_b32 v[\v_q], v[\v_q], v[\v_tmp4+1], vcc
    v_add_u32 v[\v_tmp4+1], 1, v[\v_tmp4+0]
    v_cmp_le_u32 vcc, v[\v_d], v[\v_q]
    s_nop 1
    v_cndmask_b32 v[\v_q], v[\v_tmp4+0], v[\v_tmp4+1], vcc
.endm

.macro .v_u32_div_vs v_q, v_n, s_d, v_tmp4, s_tmp4
    v_cvt_f32_u32 v[\v_tmp4+0], s[\s_d]
    s_sub_i32 s[\s_tmp4+0], 0, s[\s_d]
    v_rcp_iflag_f32 v[\v_tmp4+0], v[\v_tmp4+0]
    s_nop 0
    v_mul_f32 v[\v_tmp4+0], 0x4f7ffffe, v[\v_tmp4+0]
    v_cvt_u32_f32 v[\v_tmp4+0], v[\v_tmp4+0]
    v_mul_lo_u32 v[\v_tmp4+1], s[\s_tmp4+0], v[\v_tmp4+0]
    v_mul_hi_u32 v[\v_tmp4+1], v[\v_tmp4+0], v[\v_tmp4+1]
    v_add_u32 v[\v_tmp4+0], v[\v_tmp4+0], v[\v_tmp4+1]
    v_mul_hi_u32 v[\v_tmp4+0], v[\v_n], v[\v_tmp4+0]
    v_mul_lo_u32 v[\v_tmp4+1], v[\v_tmp4+0], s[\s_d]
    v_sub_u32 v[\v_q], v[\v_n], v[\v_tmp4+1]
    v_add_u32 v[\v_tmp4+2], 1, v[\v_tmp4+0]
    v_cmp_le_u32 vcc, s[\s_d], v[\v_q]
    v_subrev_u32 v[\v_tmp4+1], s[\s_d], v[\v_q]
    s_nop 0
    v_cndmask_b32 v[\v_tmp4+0], v[\v_tmp4+0], v[\v_tmp4+2], vcc
    v_cndmask_b32 v[\v_q], v[\v_q], v[\v_tmp4+1], vcc
    v_add_u32 v[\v_tmp4+1], 1, v[\v_tmp4+0]
    v_cmp_le_u32 vcc, s[\s_d], v[\v_q]
    s_nop 1
    v_cndmask_b32 v[\v_q], v[\v_tmp4+0], v[\v_tmp4+1], vcc
.endm

.macro .v_u32_div_ss v_q, s_n, s_d, v_tmp4, s_tmp4
    v_cvt_f32_u32 v[\v_tmp4+0], s[\s_d]
    s_sub_i32 s[\s_tmp4+0], 0, s[\s_d]
    v_rcp_iflag_f32 v[\v_tmp4+0], v[\v_tmp4+0]
    s_nop 0
    v_mul_f32 v[\v_tmp4+0], 0x4f7ffffe, v[\v_tmp4+0]
    v_cvt_u32_f32 v[\v_tmp4+0], v[\v_tmp4+0]
    v_mul_lo_u32 v[\v_tmp4+1], s[\s_tmp4+0], v[\v_tmp4+0]
    v_mul_hi_u32 v[\v_tmp4+1], v[\v_tmp4+0], v[\v_tmp4+1]
    v_add_u32 v[\v_tmp4+0], v[\v_tmp4+0], v[\v_tmp4+1]
    v_mul_hi_u32 v[\v_tmp4+0], s[\s_n], v[\v_tmp4+0]
    v_mul_lo_u32 v[\v_tmp4+1], v[\v_tmp4+0], s[\s_d]
    v_sub_u32 v[\v_q], s[\s_n], v[\v_tmp4+1]
    v_add_u32 v[\v_tmp4+2], 1, v[\v_tmp4+0]
    v_cmp_le_u32 vcc, s[\s_d], v[\v_q]
    v_subrev_u32 v[\v_tmp4+1], s[\s_d], v[\v_q]
    s_nop 0
    v_cndmask_b32 v[\v_tmp4+0], v[\v_tmp4+0], v[\v_tmp4+2], vcc
    v_cndmask_b32 v[\v_q], v[\v_q], v[\v_tmp4+1], vcc
    v_add_u32 v[\v_tmp4+1], 1, v[\v_tmp4+0]
    v_cmp_le_u32 vcc, s[\s_d], v[\v_q]
    s_nop 1
    v_cndmask_b32 v[\v_q], v[\v_tmp4+0], v[\v_tmp4+1], vcc
.endm

.macro .v_u32_div_rem v_r, v_q, v_n, v_d, v_tmp4, s_tmp4
    .v_u32_div \v_q, \v_n, \v_d, \v_tmp4, \s_tmp4
    v_mul_lo_u32 v[\v_tmp4], v[\v_d], v[\v_q]
    v_sub_u32 v[\v_r], v[\v_n], v[\v_tmp4]
.endm

.macro .v_u32_div_rem_vs v_r, v_q, v_n, s_d, v_tmp4, s_tmp4
    .v_u32_div_vs \v_q, \v_n, \s_d, \v_tmp4, \s_tmp4
    v_mul_lo_u32 v[\v_tmp4], s[\s_d], v[\v_q]
    v_sub_u32 v[\v_r], v[\v_n], v[\v_tmp4]
    s_nop 0
.endm

.macro .v_u32_div_rem_ss s_r, s_q, s_n, s_d, v_q, v_tmp4, s_tmp4
    .v_u32_div_ss \v_q, \s_n, \s_d, \v_tmp4, \s_tmp4
    s_nop 0
    v_readfirstlane_b32 s[\s_q], v[\v_q]
    s_mul_i32 s[\s_tmp4], s[\s_d], s[\s_q]
    s_sub_i32 s[\s_r], s[\s_n], s[\s_tmp4]
.endm

.macro .mdiv_u32_si s_quot s_numer magic shift s_tmp
    s_mul_hi_u32 s[\s_tmp], \magic, s[\s_numer]
    s_add_u32 s[\s_tmp], s[\s_tmp], s[\s_numer]
    s_lshr_b32 s[\s_quot], s[\s_tmp], \shift
.endm

.macro .mdiv_u32_vi v_quot v_numer magic shift v_tmp
    v_mul_hi_u32 v[\v_tmp], \magic, v[\v_numer]
    v_add_u32 v[\v_tmp], v[\v_tmp], v[\v_numer]
    v_lshrrev_b32 v[\v_quot], \shift, v[\v_tmp]
.endm

.macro .mdiv_u32_rem_vi v_rem v_quot v_numer magic shift denom v_tmp
    .mdiv_u32_vi \v_quot,\v_numer,\magic,\shift,\v_tmp
    v_mul_lo_u32 v[\v_tmp], \denom, v[\v_quot]
    v_sub_u32 v[\v_rem], v[\v_numer], v[\v_tmp]
.endm

.macro .mdiv_u32_ss s_quot s_numer s_magic s_shift s_tmp
    s_mul_hi_u32 s[\s_tmp], s[\s_magic], s[\s_numer]
    s_add_u32 s[\s_tmp], s[\s_tmp], s[\s_numer]
    s_lshr_b32 s[\s_quot], s[\s_tmp], s[\s_shift]
.endm

.macro .mdiv_u32_rem_ss s_rem s_quot s_numer s_magic s_shift s_denom s_tmp
    .mdiv_u32_ss \s_quot,\s_numer,\s_magic,\s_shift,\s_tmp
    s_mul_i32 s[\s_tmp], s[\s_denom], s[\s_quot]
    s_sub_u32 s[\s_rem], s[\s_numer], s[\s_tmp]
.endm

.macro .mdiv_u32_vs v_quot v_numer s_magic s_shift v_tmp
    v_mul_hi_u32 v[\v_tmp], s[\s_magic], v[\v_numer]
    v_add_u32 v[\v_tmp], v[\v_tmp], v[\v_numer]
    v_lshrrev_b32 v[\v_quot], s[\s_shift], v[\v_tmp]
.endm

.macro .mdiv_u32_rem_vs v_rem v_quot v_numer s_magic s_shift s_denom v_tmp
    .mdiv_u32_vs \v_quot,\v_numer,\s_magic,\s_shift,\v_tmp
    v_mul_lo_u32 v[\v_tmp], s[\s_denom], v[\v_quot]
    v_sub_u32 v[\v_rem], v[\v_numer], v[\v_tmp]
.endm

.macro .v_clear_acc_c a, num
    _a = \a
    .rept \num
        v_accvgpr_write_b32 a[_a], 0
        _a = _a + 1
    .endr
.endm

.macro .v_clear_nc vid, num
    _v = \vid
    .rept \num
        v_mov_b32 v[_v], 0
        _v = _v + 1
    .endr
.endm

.macro .v_fma_4x4_s8 c, a, b
    v_mac_f32 v[\c], v[\a], v[\b]
    v_mac_f32 v[\c+1], v[\a], v[\b+1]
    v_mac_f32 v[\c+2], v[\a], v[\b+2]
    v_mac_f32 v[\c+3], v[\a], v[\b+3]
    v_mac_f32 v[\c+8], v[\a+1], v[\b]
    v_mac_f32 v[\c+9], v[\a+1], v[\b+1]
    v_mac_f32 v[\c+10], v[\a+1], v[\b+2]
    v_mac_f32 v[\c+11], v[\a+1], v[\b+3]
    v_mac_f32 v[\c+16], v[\a+2], v[\b]
    v_mac_f32 v[\c+17], v[\a+2], v[\b+1]
    v_mac_f32 v[\c+18], v[\a+2], v[\b+2]
    v_mac_f32 v[\c+19], v[\a+2], v[\b+3]
    v_mac_f32 v[\c+24], v[\a+3], v[\b]
    v_mac_f32 v[\c+25], v[\a+3], v[\b+1]
    v_mac_f32 v[\c+26], v[\a+3], v[\b+2]
    v_mac_f32 v[\c+27], v[\a+3], v[\b+3]
.endm

.macro .v_fma_2x4_s8 c, a, b
    v_mac_f32 v[\c], v[\a], v[\b]
    v_mac_f32 v[\c+1], v[\a], v[\b+1]
    v_mac_f32 v[\c+2], v[\a], v[\b+2]
    v_mac_f32 v[\c+3], v[\a], v[\b+3]
    v_mac_f32 v[\c+8], v[\a+1], v[\b]
    v_mac_f32 v[\c+9], v[\a+1], v[\b+1]
    v_mac_f32 v[\c+10], v[\a+1], v[\b+2]
    v_mac_f32 v[\c+11], v[\a+1], v[\b+3]
.endm

.macro .v_fma_4x2_s4 c, a, b
    v_mac_f32 v[\c], v[\a], v[\b]
    v_mac_f32 v[\c+1], v[\a], v[\b+1]
    v_mac_f32 v[\c+4], v[\a+1], v[\b]
    v_mac_f32 v[\c+5], v[\a+1], v[\b+1]
    v_mac_f32 v[\c+8], v[\a+2], v[\b]
    v_mac_f32 v[\c+9], v[\a+2], v[\b+1]
    v_mac_f32 v[\c+12], v[\a+3], v[\b]
    v_mac_f32 v[\c+13], v[\a+3], v[\b+1]
.endm

.macro .v_fma_2x2_s4 c, a, b
    v_mac_f32 v[\c], v[\a], v[\b]
    v_mac_f32 v[\c+1], v[\a], v[\b+1]
    v_mac_f32 v[\c+4], v[\a+1], v[\b]
    v_mac_f32 v[\c+5], v[\a+1], v[\b+1]
.endm

.ifndef force_sc0_sc1
.set force_sc0_sc1, 0
.endif

.macro buffer_store_byte_m dst, addr, base, modifier:vararg
.if force_sc0_sc1 == 1
    buffer_store_byte \dst, \addr, \base, \modifier sc0 sc1
.else
    buffer_store_byte \dst, \addr, \base, \modifier
.endif
.endm

.macro buffer_store_byte_d16_hi_m dst, addr, base, modifier:vararg
.if force_sc0_sc1 == 1
    buffer_store_byte_d16_hi \dst, \addr, \base, \modifier sc0 sc1
.else
    buffer_store_byte_d16_hi \dst, \addr, \base, \modifier
.endif
.endm

.macro buffer_store_short_m dst, addr, base, modifier:vararg
.if force_sc0_sc1 == 1
    buffer_store_short \dst, \addr, \base, \modifier sc0 sc1
.else
    buffer_store_short \dst, \addr, \base, \modifier
.endif
.endm

.macro buffer_store_short_d16_hi_m dst, addr, base, modifier:vararg
.if force_sc0_sc1 == 1
    buffer_store_short_d16_hi \dst, \addr, \base, \modifier sc0 sc1
.else
    buffer_store_short_d16_hi \dst, \addr, \base, \modifier
.endif
.endm

.macro buffer_store_dword_m dst, addr, base, modifier:vararg
.if force_sc0_sc1 == 1
    buffer_store_dword \dst, \addr, \base, \modifier sc0 sc1
.else
    buffer_store_dword \dst, \addr, \base, \modifier
.endif
.endm

.macro buffer_store_dwordx2_m dst, addr, base, modifier:vararg
.if force_sc0_sc1 == 1
    buffer_store_dwordx2 \dst, \addr, \base, \modifier sc0 sc1
.else
    buffer_store_dwordx2 \dst, \addr, \base, \modifier
.endif
.endm

.macro buffer_store_dwordx3_m dst, addr, base, modifier:vararg
.if force_sc0_sc1 == 1
    buffer_store_dwordx3 \dst, \addr, \base, \modifier sc0 sc1
.else
    buffer_store_dwordx3 \dst, \addr, \base, \modifier
.endif
.endm

.macro buffer_store_dwordx4_m dst, addr, base, modifier:vararg
.if force_sc0_sc1 == 1
    buffer_store_dwordx4 \dst, \addr, \base, \modifier sc0 sc1
.else
    buffer_store_dwordx4 \dst, \addr, \base, \modifier
.endif
.endm

.ifndef atomic_add_using_cas
.set atomic_add_using_cas, 0
.endif

.macro buffer_atomic_add_f32_m data, addr, base, other:vararg
.if atomic_add_using_cas == 1
    s_mov_b64 s[0:1], exec
Label_cas_loop_start_\@:
    buffer_load_dword v[v_tmp+2], \addr, \base, \other sc1
    s_waitcnt vmcnt(0)
    v_mov_b32 v[v_tmp+3] v[v_tmp+2]
    v_add_f32 v[v_tmp+2], \data, v[v_tmp+3]
    buffer_atomic_cmpswap v[v_tmp+2:v_tmp+3], \addr, \base, \other sc0
    s_waitcnt vmcnt(0)
    v_cmp_ne_u32 s[s_tmp+2:s_tmp+3], v[v_tmp+2], v[v_tmp+3]
    s_and_saveexec_b64 s[s_tmp+2:s_tmp+3], s[s_tmp+2:s_tmp+3]
    s_cbranch_execnz Label_cas_loop_start_\@
Label_cas_loop_end_\@:
    s_mov_b64 exec, s[0:1]
    s_nop 0
.else
    buffer_atomic_add_f32 \data, \addr, \base, \other
.endif
.endm

.macro buffer_atomic_pk_add_f16_m data, addr, base, other:vararg
.if atomic_add_using_cas == 1
    s_mov_b64 s[0:1], exec
Label_cas_loop_start_\@:
    buffer_load_dword v[v_tmp+2], \addr, \base, \other sc1
    s_waitcnt vmcnt(0)
    v_mov_b32 v[v_tmp+3] v[v_tmp+2]
    v_pk_add_f16 v[v_tmp+2], \data, v[v_tmp+3]
    buffer_atomic_cmpswap v[v_tmp+2:v_tmp+3], \addr, \base, \other sc0
    s_waitcnt vmcnt(0)
    v_cmp_ne_u32 s[s_tmp+2:s_tmp+3], v[v_tmp+2], v[v_tmp+3]
    s_and_saveexec_b64 s[s_tmp+2:s_tmp+3], s[s_tmp+2:s_tmp+3]
    s_cbranch_execnz Label_cas_loop_start_\@
Label_cas_loop_end_\@:
    s_mov_b64 exec, s[0:1]
    s_nop 0
.else
    buffer_atomic_pk_add_f16 \data, \addr, \base, \other
.endif
.endm

