/*========================== begin_copyright_notice ============================

Copyright (C) 2017-2024 Intel Corporation

SPDX-License-Identifier: MIT

============================= end_copyright_notice ===========================*/

#include "IGC/common/StringMacros.hpp"
#include "common/LLVMWarningsPush.hpp"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvmWrapper/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
#include "llvm/Analysis/TargetFolder.h"
#include "llvmWrapper/IR/DerivedTypes.h"
#include "llvmWrapper/IR/Instructions.h"
#include "llvmWrapper/IR/Intrinsics.h"
#include "llvmWrapper/Support/Alignment.h"
#include "common/LLVMWarningsPop.hpp"
#include "common/LLVMUtils.h"
#include "common/IGCIRBuilder.h"
#include "GenISAIntrinsics/GenIntrinsics.h"
#include "Compiler/CISACodeGen/ShaderCodeGen.hpp"
#include "Compiler/IGCPassSupport.h"
#include "Compiler/MetaDataUtilsWrapper.h"
#include "Compiler/CISACodeGen/Emu64OpsPass.h"
#include "Probe/Assertion.h"

using namespace llvm;
using namespace IGC;
using namespace IGC::IGCMD;


using std::ldexp;


namespace {

    typedef llvm::IGCIRBuilder<TargetFolder> BuilderType;
    typedef std::pair<Value*, Value*> ValuePair;

    class InstExpander;
    class Preprocessor;

    class Emu64Ops : public FunctionPass {
        friend class InstExpander;
        friend class Preprocessor;

        const DataLayout* DL;
        IGC::CodeGenContext* CGC;

        BuilderType* IRB;
        InstExpander* Expander;

        LLVMContext* TheContext;
        Module* TheModule;
        Function* TheFunction;

        typedef DenseMap<Value*, ValuePair> ValueMapTy;
        ValueMapTy ValueMap;

        // Special bitcasts of 64-bit arguments, which need special handling as we
        // cannot replace argument type.
        SmallPtrSet<BitCastInst*, 8> Arg64Casts;

        SmallPtrSet<Instruction*, 32> DeadInsts;

        SmallPtrSet<Instruction*, 16> Int64Insts;
        bool doNotDeleteInstr = false;
    public:

        static char ID;

        Emu64Ops() : FunctionPass(ID), DL(nullptr), CGC(nullptr), IRB(nullptr),
            Expander(nullptr), TheContext(nullptr), TheModule(nullptr),
            TheFunction(nullptr) {
            initializeEmu64OpsPass(*PassRegistry::getPassRegistry());
        }

        bool runOnFunction(Function& F) override;

        StringRef getPassName() const override { return "Emu64Ops"; }

    private:
        void getAnalysisUsage(AnalysisUsage& AU) const override {
            AU.addRequired<CodeGenContextWrapper>();
            AU.addRequired<MetaDataUtilsWrapper>();
        }

        LLVMContext* getContext() const { return TheContext; }
        Module* getModule() const { return TheModule; }
        Function* getFunction() const { return TheFunction; }
        bool hasPtr64() const {
            return (DL->getPointerSizeInBits() == 64 ||
                DL->getPointerSizeInBits(ADDRESS_SPACE_GLOBAL) == 64 ||
                DL->getPointerSizeInBits(ADDRESS_SPACE_CONSTANT) == 64);
        }
        bool isPtr64(const PointerType* PtrTy) const {
            return CGC->getRegisterPointerSizeInBits(PtrTy->getAddressSpace()) == 64;
        }
        bool isInt64(const Type* Ty) const { return Ty->isIntegerTy(64); }
        bool isInt64(const Value* V) const { return isInt64(V->getType()); }

        bool isArg64Cast(BitCastInst* BC) const { return Arg64Casts.count(BC) != 0; }

        Type* getV2Int32Ty(unsigned NumElts = 1) const {
            return IGCLLVM::FixedVectorType::get(IRB->getInt32Ty(), NumElts * 2);
        }

        ValuePair getExpandedValues(Value* V);
        void setExpandedValues(Value* V, Value* Lo, Value* Hi);

        alignment_t getAlignment(LoadInst* LD) const {
            auto Align = IGCLLVM::getAlignmentValue(LD);
            if (Align == 0)
                Align = DL->getABITypeAlign(LD->getType()).value();
            return Align;
        }

        void copyKnownMetadata(Instruction* NewI, Instruction* OldI) const
        {
            unsigned LscCacheCtrlID =
                OldI->getContext().getMDKindID("lsc.cache.ctrl");
            SmallVector<std::pair<unsigned, MDNode*>, 8> MD;
            OldI->getAllMetadata(MD);
            for (const auto& MDPair : MD)
            {
                unsigned ID = MDPair.first;
                MDNode* N = MDPair.second;
                if (ID == LscCacheCtrlID)
                    NewI->setMetadata(ID, N);
            }
        }

        void dupMemoryAttribute(LoadInst* NewLD, LoadInst* RefLD, unsigned Off) const {
            auto alignment = getAlignment(RefLD);

            NewLD->setVolatile(RefLD->isVolatile());
            NewLD->setAlignment(IGCLLVM::getAlign(MinAlign(alignment, Off)));
            NewLD->setOrdering(RefLD->getOrdering());
            NewLD->setSyncScopeID(RefLD->getSyncScopeID());
            copyKnownMetadata(NewLD, RefLD);
        }

        void dupMemoryAttribute(StoreInst* NewST, StoreInst* RefST, unsigned Off) const {
            auto alignment = IGCLLVM::getAlignmentValue(RefST);

            NewST->setVolatile(RefST->isVolatile());
            NewST->setAlignment(IGCLLVM::getAlign(MinAlign(alignment, Off)));
            NewST->setOrdering(RefST->getOrdering());
            NewST->setSyncScopeID(RefST->getSyncScopeID());
            copyKnownMetadata(NewST, RefST);
        }

        bool expandArguments(Function& F);
        bool preparePHIs(Function& F);
        bool expandInsts(Function& F);
        bool populatePHIs(Function& F);
        bool removeDeadInsts();
        bool hasOpcodeInt64Support(unsigned int opcode) const;
        void addInstToInt64InstrList(Instruction* inst) { Int64Insts.insert(inst); }
        bool eraseInstrFromInt64InstrList(Instruction* inst) { return Int64Insts.erase(inst); }
    };

    class InstExpander : public InstVisitor<InstExpander, bool> {
        friend class InstVisitor<InstExpander, bool>;

        Emu64Ops* Emu;
        BuilderType* IRB;

    public:
        InstExpander(Emu64Ops* E, BuilderType* B) : Emu(E), IRB(B) {}

        bool expand(Instruction* I);

    private:
        bool visitInstruction(Instruction&);

        bool visitRet(ReturnInst&);
        bool visitBr(BranchInst&) { return false; }
        bool visitSwitch(SwitchInst&) { return false; }
        bool visitIndirectBr(IndirectBrInst&) { return false; }
        bool visitInvoke(InvokeInst&) { return false; }
        bool visitResume(ResumeInst&) { return false; }
        bool visitUnreachable(UnreachableInst&) { return false; }
        bool visitFreeze(FreezeInst&);

        bool visitAdd(BinaryOperator&);
        bool visitFAdd(BinaryOperator&) { return false; }
        bool visitSub(BinaryOperator&);
        bool visitFSub(BinaryOperator&) { return false; }
        bool visitMul(BinaryOperator&);
        bool visitFMul(BinaryOperator&) { return false; }
        bool visitSDiv(BinaryOperator&);
        bool visitUDiv(BinaryOperator&);
        bool visitFDiv(BinaryOperator&) { return false; }
        bool visitSRem(BinaryOperator&);
        bool visitURem(BinaryOperator&);
        bool visitFRem(BinaryOperator&) { return false; }
        bool visitFNeg(UnaryOperator&) { return false; }

        bool visitShl(BinaryOperator&);
        bool visitLShr(BinaryOperator&);
        bool visitAShr(BinaryOperator&);
        bool visitAnd(BinaryOperator&);
        bool visitOr(BinaryOperator&);
        bool visitXor(BinaryOperator&);

        bool visitAlloca(AllocaInst&) { return false; }
        bool visitLoad(LoadInst&);
        bool visitStore(StoreInst&);
        bool visitGetElementPtr(GetElementPtrInst&) { return false; }
        bool visitFence(FenceInst&) { return false; }
        bool visitAtomicCmpXchg(AtomicCmpXchgInst&);
        bool visitAtomicRMW(AtomicRMWInst&);

        bool visitTrunc(TruncInst&);
        bool visitSExt(SExtInst&);
        bool visitZExt(ZExtInst&);
        bool visitFPToUI(FPToUIInst&);
        bool visitFPToSI(FPToSIInst&);
        bool visitUIToFP(UIToFPInst&);
        bool visitSIToFP(SIToFPInst&);
        bool visitFPTrunc(FPTruncInst&) { return false; }
        bool visitFPExt(FPExtInst&) { return false; }
        bool visitPtrToInt(PtrToIntInst&);
        bool visitIntToPtr(IntToPtrInst&);
        bool visitBitCast(BitCastInst&);
        bool visitAddrSpaceCast(AddrSpaceCastInst&) { return false; }

        bool visitICmp(ICmpInst&);
        bool visitFCmp(FCmpInst&) { return false; }
        bool visitPHI(PHINode&);
        bool visitCall(CallInst&);
        bool visitSelect(SelectInst&);

        bool visitVAArg(VAArgInst&);
        bool visitExtractElement(ExtractElementInst&);
        bool visitInsertElement(InsertElementInst&);
        bool visitShuffleVector(ShuffleVectorInst&) { return false; }
        bool visitExtractValue(ExtractValueInst&);
        bool visitInsertValue(InsertValueInst&);
        bool visitLandingPad(LandingPadInst&);

        Value* convertUIToFP32(Type* DstTy, Value* Lo, Value* Hi, Instruction* Pos);
        bool hasHWSupport(BinaryOperator&);
        bool needsLoHiSplitting(Instruction&);
    };

    class Preprocessor {
        Emu64Ops* Emu;
        BuilderType* IRB;

    public:
        Preprocessor(Emu64Ops* E, BuilderType* B) : Emu(E), IRB(B) {}

        bool preprocess(Function& F) {
            bool Changed = false;
            // Preprocess additions with overflow.
            for (auto& BB : F) {
                for (auto BI = BB.begin(), BE = BB.end(); BI != BE; /*EMPTY*/) {
                    IntrinsicInst* II = dyn_cast<IntrinsicInst>(BI);
                    if (!II || II->getIntrinsicID() != Intrinsic::uadd_with_overflow ||
                        !II->getArgOperand(0)->getType()->isIntegerTy(64)) {
                        ++BI;
                        continue;
                    }

                    IRB->SetInsertPoint(II);

                    Value* LHS = II->getArgOperand(0);
                    Value* RHS = II->getArgOperand(1);
                    Value* Res = IRB->CreateAdd(LHS, RHS);
                    Value* Overflow = IRB->CreateICmpULT(Res, LHS);

                    for (auto UI = II->user_begin(),
                        UE = II->user_end(); UI != UE; /*EMPTY*/) {
                        User* U = *UI++;
                        ExtractValueInst* Ex = cast<ExtractValueInst>(U);
                        IGC_ASSERT(nullptr != Ex);
                        IGC_ASSERT(Ex->getNumIndices() == 1);

                        unsigned Idx = *Ex->idx_begin();
                        IGC_ASSERT(Idx == 0 || Idx == 1);
                        Ex->replaceAllUsesWith((Idx == 0) ? Res : Overflow);
                        Ex->eraseFromParent();
                    }
                    IGC_ASSERT(II->user_empty());
                    ++BI;
                    II->eraseFromParent();
                    Changed = true;
                }
            }
            // Preprocess non-LOAD/-STORE pointer usage if there's 64-bit pointer.
            IGC_ASSERT(nullptr != Emu);
            if (Emu->hasPtr64()) {
                for (auto& BB : F) {
                    SmallVector<Instruction*, 16> LocalDeadInsts;
                    for (auto BI = BB.begin(), BE = BB.end(); BI != BE; ++BI) {
                        switch (BI->getOpcode()) {
                        default: // By default, NOTHING!
                            break;
                        case Instruction::ICmp: {
                            ICmpInst* Cmp = cast<ICmpInst>(BI);
                            IGC_ASSERT(nullptr != Cmp);
                            PointerType* PtrTy =
                                dyn_cast<PointerType>(Cmp->getOperand(0)->getType());
                            if (!PtrTy || !Emu->isPtr64(PtrTy))
                                continue;

                            IRB->SetInsertPoint(Cmp);

                            Value* LHS = Cmp->getOperand(0);
                            Value* RHS = Cmp->getOperand(1);
                            LHS = IRB->CreatePtrToInt(LHS, IRB->getInt64Ty());
                            RHS = IRB->CreatePtrToInt(RHS, IRB->getInt64Ty());
                            Cmp->setOperand(0, LHS);
                            Cmp->setOperand(1, RHS);

                            Changed = true;
                            break;
                        }
                        case Instruction::Select: {
                            SelectInst* SI = cast<SelectInst>(BI);
                            PointerType* PtrTy = dyn_cast<PointerType>(SI->getType());
                            if (!PtrTy || !Emu->isPtr64(PtrTy))
                                continue;

                            IRB->SetInsertPoint(SI);

                            Value* TVal = SI->getTrueValue();
                            Value* FVal = SI->getFalseValue();
                            TVal = IRB->CreatePtrToInt(TVal, IRB->getInt64Ty());
                            FVal = IRB->CreatePtrToInt(FVal, IRB->getInt64Ty());
                            Value* NewPtr = IRB->CreateSelect(SI->getCondition(), TVal, FVal);
                            NewPtr = IRB->CreateIntToPtr(NewPtr, PtrTy);
                            SI->replaceAllUsesWith(NewPtr);
                            LocalDeadInsts.push_back(SI);

                            Changed = true;
                            break;
                        }
                        case Instruction::Load: {
                            LoadInst* LD = cast<LoadInst>(BI);
                            PointerType* PtrTy = dyn_cast<PointerType>(LD->getType());
                            if (!PtrTy || !Emu->isPtr64(PtrTy))
                                continue;

                            IRB->SetInsertPoint(LD);

                            // Cast the original pointer to pointer to pointer to i64.
                            Value* OldPtr = LD->getPointerOperand();
                            PointerType* OldPtrTy = cast<PointerType>(OldPtr->getType());
                            PointerType* NewPtrTy =
                                IRB->getInt64Ty()->getPointerTo(OldPtrTy->getAddressSpace());
                            Value* NewPtr = IRB->CreateBitCast(OldPtr, NewPtrTy);
                            // Create new load.
                            LoadInst* NewLD = IRB->CreateLoad(IRB->getInt64Ty(), NewPtr);
                            Emu->dupMemoryAttribute(NewLD, LD, 0);
                            // Cast the load i64 back to pointer.
                            Value* NewVal = IRB->CreateIntToPtr(NewLD, PtrTy);
                            LD->replaceAllUsesWith(NewVal);
                            LocalDeadInsts.push_back(LD);

                            Changed = true;
                            break;
                        }
                        case Instruction::Store: {
                            StoreInst* ST = cast<StoreInst>(BI);
                            PointerType* PtrTy =
                                dyn_cast<PointerType>(ST->getValueOperand()->getType());
                            if (!PtrTy || !Emu->isPtr64(PtrTy))
                                continue;

                            IRB->SetInsertPoint(ST);

                            // Cast the pointer to pointer to pointer to i64.
                            Value* OldPtr = ST->getPointerOperand();
                            PointerType* OldPtrTy = cast<PointerType>(OldPtr->getType());
                            PointerType* NewPtrTy =
                                IRB->getInt64Ty()->getPointerTo(OldPtrTy->getAddressSpace());
                            Value* NewPtr = IRB->CreateBitCast(OldPtr, NewPtrTy);
                            // Cast the pointer to be stored into i64.
                            Value* OldVal = ST->getValueOperand();
                            Value* NewVal = IRB->CreatePtrToInt(OldVal, IRB->getInt64Ty());
                            // Create new store.
                            StoreInst* NewST = IRB->CreateStore(NewVal, NewPtr);
                            Emu->dupMemoryAttribute(NewST, ST, 0);
                            LocalDeadInsts.push_back(ST);

                            Changed = true;
                            break;
                        }
                        case Instruction::IntToPtr: {
                            IntToPtrInst* I2P = cast<IntToPtrInst>(BI);
                            Value* Src = I2P->getOperand(0);
                            PointerType* PtrTy = cast<PointerType>(I2P->getType());
                            if (!Emu->isPtr64(PtrTy) && !Emu->isInt64(Src))
                                continue;

                            IRB->SetInsertPoint(I2P);
                            unsigned int ptrSize = Emu->CGC->getRegisterPointerSizeInBits(PtrTy->getAddressSpace());

                            Src = IRB->CreateZExtOrTrunc(Src, IRB->getIntNTy(ptrSize));
                            I2P->setOperand(0, Src);

                            Changed = true;
                            break;
                        }
                        case Instruction::PtrToInt: {
                            PtrToIntInst* P2I = cast<PtrToIntInst>(BI);
                            Value* Src = P2I->getOperand(0);
                            PointerType* PtrTy = cast<PointerType>(Src->getType());
                            if (!Emu->isPtr64(PtrTy) || Emu->isInt64(Src))
                                continue;

                            IRB->SetInsertPoint(P2I);

                            Value* NewVal = IRB->CreatePtrToInt(Src, IRB->getInt64Ty());
                            NewVal = IRB->CreateZExtOrTrunc(NewVal, P2I->getType());
                            P2I->replaceAllUsesWith(NewVal);
                            LocalDeadInsts.push_back(P2I);

                            Changed = true;
                            break;
                        }
                        }
                    }
                    // Remove dead instructions.
                    for (auto I : LocalDeadInsts)
                        I->eraseFromParent();
                }
            }

            return Changed;
        }
    };

    char Emu64Ops::ID = 0;

} // End anonymous namespace

FunctionPass* createEmu64OpsPass() {
    return new Emu64Ops();
}

#define PASS_FLAG     "igc-emu64ops"
#define PASS_DESC     "IGC 64-bit ops emulation"
#define PASS_CFG_ONLY false
#define PASS_ANALYSIS false
IGC_INITIALIZE_PASS_BEGIN(Emu64Ops, PASS_FLAG, PASS_DESC, PASS_CFG_ONLY, PASS_ANALYSIS)
IGC_INITIALIZE_PASS_DEPENDENCY(CodeGenContextWrapper)
IGC_INITIALIZE_PASS_DEPENDENCY(MetaDataUtilsWrapper)
IGC_INITIALIZE_PASS_END(Emu64Ops, PASS_FLAG, PASS_DESC, PASS_CFG_ONLY, PASS_ANALYSIS)

bool Emu64Ops::runOnFunction(Function& F) {
    // Skip non-kernel function.
    MetaDataUtils* MDU = nullptr;
    MDU = getAnalysis<MetaDataUtilsWrapper>().getMetaDataUtils();
    IGC_ASSERT(nullptr != MDU);
    auto FII = MDU->findFunctionsInfoItem(&F);
    if (FII == MDU->end_FunctionsInfo())
        return false;

    DL = &F.getParent()->getDataLayout();
    CGC = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();

    BuilderType TheBuilder(F.getContext(), TargetFolder(*DL));
    InstExpander TheExpander(this, &TheBuilder);
    Preprocessor ThePreprocessor(this, &TheBuilder);
    IRB = &TheBuilder;
    Expander = &TheExpander;

    TheContext = &F.getContext();
    TheModule = F.getParent();
    TheFunction = &F;

    ValueMap.clear();
    Arg64Casts.clear();
    DeadInsts.clear();
    Int64Insts.clear();

    bool Changed = false;
    Changed |= ThePreprocessor.preprocess(F);
    Changed |= expandArguments(F);
    Changed |= preparePHIs(F);
    Changed |= expandInsts(F);
    Changed |= populatePHIs(F);
    Changed |= removeDeadInsts();

    return Changed;
}

ValuePair Emu64Ops::getExpandedValues(Value* V) {
    auto [VMI, New] = ValueMap.insert(std::make_pair(V, ValuePair()));
    if (!New)
        return VMI->second;

    if (dyn_cast<ConstantInt>(V)) {
        Value* Lo = IRB->CreateTrunc(V, IRB->getInt32Ty());
        Value* Hi = IRB->CreateTrunc(IRB->CreateLShr(V, 32), IRB->getInt32Ty());
        VMI->second = std::make_pair(Lo, Hi);
        return VMI->second;
    }

    if (dyn_cast<UndefValue>(V)) {
        Value* Lo = UndefValue::get(IRB->getInt32Ty());
        Value* Hi = UndefValue::get(IRB->getInt32Ty());
        VMI->second = std::make_pair(Lo, Hi);
        return VMI->second;
    }

    if (ConstantExpr * CE = dyn_cast<ConstantExpr>(V)) {
        Value* Lo = nullptr;
        Value* Hi = nullptr;
        if (isa<PtrToIntOperator>(CE)) {
            V = IRB->CreateBitCast(V, getV2Int32Ty());
            Lo = IRB->CreateExtractElement(V, IRB->getInt32(0));
            Hi = IRB->CreateExtractElement(V, IRB->getInt32(1));
        }
        else {
            Lo = IRB->CreateTrunc(V, IRB->getInt32Ty());
            Hi = IRB->CreateTrunc(IRB->CreateLShr(V, 32), IRB->getInt32Ty());
        }
        VMI->second = std::make_pair(Lo, Hi);
        return VMI->second;
    }

    IGC_ASSERT_UNREACHABLE(); // TODO: NOT IMPLEMENTED!
}

void Emu64Ops::setExpandedValues(Value* V, Value* Lo, Value* Hi) {
    ValuePair Pair = std::make_pair(Lo, Hi);
    ValueMap.insert(std::make_pair(V, Pair));
}

bool Emu64Ops::expandArguments(Function& F) {
    Instruction* Pos = &F.getEntryBlock().front();
    IRB->SetInsertPoint(Pos);

    bool Changed = false;
    for (auto& Arg : F.args()) {
        if (!isInt64(&Arg))
            continue;
        Value* V = IRB->CreateBitCast(&Arg, getV2Int32Ty());
        Value* Lo = IRB->CreateExtractElement(V, IRB->getInt32(0));
        Value* Hi = IRB->CreateExtractElement(V, IRB->getInt32(1));
        setExpandedValues(&Arg, Lo, Hi);
        Arg64Casts.insert(cast<BitCastInst>(V));
        Changed = true;
    }

    return Changed;
}

bool Emu64Ops::preparePHIs(Function& F) {
    bool Changed = false;
    for (auto& BB : F) {
        PHINode* PN = nullptr;
        for (auto BI = BB.begin(); (PN = dyn_cast<PHINode>(BI)) != nullptr; ++BI) {
            if (!isInt64(PN))
                continue;
            IRB->SetInsertPoint(PN);

            unsigned NumIncomingValues = PN->getNumIncomingValues();
            Value* Lo = IRB->CreatePHI(IRB->getInt32Ty(), NumIncomingValues);
            Value* Hi = IRB->CreatePHI(IRB->getInt32Ty(), NumIncomingValues);
            setExpandedValues(PN, Lo, Hi);

            DeadInsts.insert(PN);
            Changed = true;
        }
    }

    return Changed;
}

bool Emu64Ops::expandInsts(Function& F) {
    ReversePostOrderTraversal<Function*> RPOT(&F);
    bool Changed = false;
    for (auto& BB : RPOT) {
        for (auto BI = BB->begin(), BE = BB->end(); BI != BE; /*EMPTY*/) {
            Instruction* I = &(*BI++);
            doNotDeleteInstr = false;
            CGC->metrics.StatBeginEmuFunc(I);
            bool LocalChanged = Expander->expand(I);
            Changed |= LocalChanged;

            if (LocalChanged) {
                CGC->metrics.StatEndEmuFunc(I);
                BI = std::next(BasicBlock::iterator(I));
                BE = I->getParent()->end();
                DeadInsts.insert(I);
            }
            if (doNotDeleteInstr)
                BE = I->getParent()->end();
        }
    }

    return Changed;
}

bool Emu64Ops::populatePHIs(Function& F) {
    bool Changed = false;
    for (auto& BB : F) {
        PHINode* PN;
        for (auto BI = BB.begin(); (PN = dyn_cast<PHINode>(BI)) != nullptr; ++BI) {
            if (!isInt64(PN))
                continue;
            auto [L, H] = getExpandedValues(PN);

            PHINode* Lo = cast<PHINode>(L);
            PHINode* Hi = cast<PHINode>(H);
            for (auto& Op : PN->operands()) {
                BasicBlock* BB = PN->getIncomingBlock(Op);
                std::tie(L, H) = getExpandedValues(Op.get());
                Lo->addIncoming(L, BB);
                Hi->addIncoming(H, BB);
            }
            Changed = true;
        }
    }
    return Changed;
}

bool Emu64Ops::removeDeadInsts() {
    bool Changed = false;
    for (auto* I : DeadInsts) {
        Type* Ty = I->getType();
        if (!Ty->isVoidTy())
            I->replaceAllUsesWith(UndefValue::get(Ty));
        I->eraseFromParent();
        Changed = true;
    }
    return Changed;
}

bool Emu64Ops::hasOpcodeInt64Support(unsigned int opcode) const
{
    // PVC-B0 only
    return ((opcode == llvm::Instruction::Shl) ||
            (opcode == llvm::Instruction::LShr) ||
            (opcode == llvm::Instruction::AShr) ||
            (CGC->platform.hasInt64Add() && (opcode == llvm::Instruction::Add || opcode == llvm::Instruction::Sub)));
}

bool InstExpander::expand(Instruction* I) {
    IRB->SetInsertPoint(I);

    if (!visit(*I))
        return false;

    return true;
}

bool InstExpander::visitInstruction(Instruction& I) {
    IGC_ASSERT_MESSAGE(0, "UNKNOWN INSTRUCTION is BEING EXPANDED!");
    return false;
}

bool InstExpander::visitRet(ReturnInst& RI) {
    if (Value* V = RI.getReturnValue())
    {
        IGC_ASSERT(nullptr != Emu);
        if (Emu->isInt64(V))
        {
            auto [Lo, Hi] = Emu->getExpandedValues(V);
            Type* V2I32Ty = Emu->getV2Int32Ty();
            Value* NewVal = UndefValue::get(V2I32Ty);
            NewVal = IRB->CreateInsertElement(NewVal, Lo, IRB->getInt32(0));
            NewVal = IRB->CreateInsertElement(NewVal, Hi, IRB->getInt32(1));
            NewVal = IRB->CreateBitCast(NewVal, IRB->getInt64Ty());

            IRB->SetInsertPoint(&RI);
            (void) IRB->CreateRet(NewVal);
            return true;
        }
    }
    return false;
}

bool InstExpander::visitFreeze(FreezeInst& FI) {
    IGC_ASSERT(nullptr != Emu);
    // Skip if it's not 64-bit integer.
    if (!Emu->isInt64(&FI))
        return false;

    Value* Src = FI.getOperand(0);
    IGC_ASSERT(nullptr != Src);

    auto [Lo, Hi] = Emu->getExpandedValues(Src);

    Value* newLo = IRB->CreateFreeze(Lo);
    Value* newHi = IRB->CreateFreeze(Hi);

    Emu->setExpandedValues(&FI, newLo, newHi);
    return true;
}

bool InstExpander::visitAdd(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    if (Emu->CGC->platform.hasInt64Add()) {
        if (hasHWSupport(BinOp))
            return false;
    }

    auto [L0, H0] = Emu->getExpandedValues(BinOp.getOperand(0));
    auto [L1, H1] = Emu->getExpandedValues(BinOp.getOperand(1));

    GenISAIntrinsic::ID GIID = GenISAIntrinsic::GenISA_add_pair;
    Function* IFunc = GenISAIntrinsic::getDeclaration(Emu->getModule(), GIID);
    IGC_ASSERT(nullptr != IRB);
    Value* V = IRB->CreateCall4(IFunc, L0, H0, L1, H1);
    Value* Lo = IRB->CreateExtractValue(V, 0);
    Value* Hi = IRB->CreateExtractValue(V, 1);

    Emu->setExpandedValues(&BinOp, Lo, Hi);
    return true;
}

bool InstExpander::visitSub(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    if (Emu->CGC->platform.hasInt64Add()) {
        if (hasHWSupport(BinOp))
            return false;
    }

    auto [L0, H0] = Emu->getExpandedValues(BinOp.getOperand(0));
    auto [L1, H1] = Emu->getExpandedValues(BinOp.getOperand(1));

    GenISAIntrinsic::ID GIID = GenISAIntrinsic::GenISA_sub_pair;
    Function* IFunc = GenISAIntrinsic::getDeclaration(Emu->getModule(), GIID);
    IGC_ASSERT(nullptr != IRB);
    Value* V = IRB->CreateCall4(IFunc, L0, H0, L1, H1);
    Value* Lo = IRB->CreateExtractValue(V, 0);
    Value* Hi = IRB->CreateExtractValue(V, 1);

    Emu->setExpandedValues(&BinOp, Lo, Hi);
    return true;
}

bool InstExpander::visitMul(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    auto [L0, H0] = Emu->getExpandedValues(BinOp.getOperand(0));
    auto [L1, H1] = Emu->getExpandedValues(BinOp.getOperand(1));

    GenISAIntrinsic::ID GIID = GenISAIntrinsic::GenISA_mul_pair;
    Function* IFunc = GenISAIntrinsic::getDeclaration(Emu->getModule(), GIID);
    IGC_ASSERT(nullptr != IRB);
    Value* V = IRB->CreateCall4(IFunc, L0, H0, L1, H1);
    Value* Lo = IRB->CreateExtractValue(V, 0);
    Value* Hi = IRB->CreateExtractValue(V, 1);

    Emu->setExpandedValues(&BinOp, Lo, Hi);
    return true;
}

bool InstExpander::visitSDiv(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT_MESSAGE(false == Emu->isInt64(&BinOp), "There should not be `sdiv` which is already emulated by library call.");
    return false;
}

bool InstExpander::visitUDiv(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT_MESSAGE(false == Emu->isInt64(&BinOp), "There should not be `udiv` which is already emulated by library call.");
    return false;
}

bool InstExpander::visitSRem(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT_MESSAGE(false == Emu->isInt64(&BinOp), "There should not be `srem` which is already emulated by library call.");
    return false;
}

bool InstExpander::visitURem(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT_MESSAGE(false == Emu->isInt64(&BinOp), "There should not be `urem` which is already emulated by library call.");
    return false;
}

bool InstExpander::visitShl(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    if (hasHWSupport(BinOp))
        return false;

    auto [Lo, Hi] = Emu->getExpandedValues(BinOp.getOperand(0));
    Value* ShAmt = nullptr;
    std::tie(ShAmt, std::ignore) = Emu->getExpandedValues(BinOp.getOperand(1));

    BasicBlock* OldBB = BinOp.getParent();
    BasicBlock* InnerTBB = nullptr;
    BasicBlock* InnerFBB = nullptr;
    PHINode* InnerResLo = nullptr;
    PHINode* InnerResHi = nullptr;
    Value* ResLo = nullptr;
    Value* ResHi = nullptr;
    DebugLoc BinOpDebugLoc = BinOp.getDebugLoc();

    Value* Cond = nullptr;

    // Only LSB of ShAmt is used for shift
    ShAmt = IRB->CreateAnd(ShAmt, 63);
    if (!isa<ConstantInt>(ShAmt)) {
        // Create outer if-endif to handle the special case where `ShAmt` is zero.
        // We have option to handle that with `((S >> (~ShAmt)) >> 1)`. However, as
        // a zero `ShAmt` is a very rare case so that the outer branch should be
        // uniform one in most cases.
        Value* NE = IRB->CreateICmpNE(ShAmt, Constant::getNullValue(ShAmt->getType()));
        BasicBlock* JointBB = OldBB->splitBasicBlock(&BinOp);
        ResLo = PHINode::Create(IRB->getInt32Ty(), 2, ".shl.outer.merge.lo", &BinOp);
        cast<Instruction>(ResLo)->setDebugLoc(BinOpDebugLoc);
        ResHi = PHINode::Create(IRB->getInt32Ty(), 2, ".shl.outer.merge.hi", &BinOp);
        cast<Instruction>(ResHi)->setDebugLoc(BinOpDebugLoc);

        BasicBlock* TrueBB = BasicBlock::Create(*Emu->getContext(),
            ".shl.outer.true.branch");
        TrueBB->insertInto(Emu->getFunction(), JointBB);
        Instruction* TrueJmp = BranchInst::Create(JointBB, TrueBB);
        TrueJmp->setDebugLoc(BinOpDebugLoc);

        OldBB->getTerminator()->eraseFromParent();
        Instruction* TempOldBBBranchInst = BranchInst::Create(TrueBB, JointBB, NE, OldBB);
        TempOldBBBranchInst->setDebugLoc(BinOpDebugLoc);

        // Create the inner branch.
        IRB->SetInsertPoint(&(*TrueBB->begin()));

        Cond = IRB->CreateICmpULT(ShAmt, IRB->getInt32(32));
        // Prepare to generate branches to handle the case where `ShAmt` is less
        // than 32 (true branch) or otherwise (false branch).
        BasicBlock* InnerJBB = TrueBB->splitBasicBlock(TrueJmp);
        InnerResLo = PHINode::Create(IRB->getInt32Ty(), 2, ".shl.merge.inner.lo", TrueJmp);
        InnerResLo->setDebugLoc(BinOpDebugLoc);
        InnerResHi = PHINode::Create(IRB->getInt32Ty(), 2, ".shl.merge.inner.hi", TrueJmp);
        InnerResHi->setDebugLoc(BinOpDebugLoc);

        InnerTBB = BasicBlock::Create(*Emu->getContext(), ".shl.inner.true.branch");
        InnerTBB->insertInto(Emu->getFunction(), InnerJBB);
        Instruction* TempInnerTBBBranchInst = BranchInst::Create(InnerJBB, InnerTBB);
        TempInnerTBBBranchInst->setDebugLoc(BinOpDebugLoc);

        InnerFBB = BasicBlock::Create(*Emu->getContext(), ".shl.inner.false.branch");
        InnerFBB->insertInto(Emu->getFunction(), InnerJBB);
        Instruction* TempInnerFBBBranchInst = BranchInst::Create(InnerJBB, InnerFBB);
        TempInnerFBBBranchInst->setDebugLoc(BinOpDebugLoc);

        TrueBB->getTerminator()->eraseFromParent();
        Instruction* TempTrueBBBranchInst = BranchInst::Create(InnerTBB, InnerFBB, Cond, TrueBB);
        TempTrueBBBranchInst->setDebugLoc(BinOpDebugLoc);

        // The result is the same as the source if ShAmt is 0, i.e. NE is
        // false.
        cast<PHINode>(ResLo)->addIncoming(Lo, OldBB);
        cast<PHINode>(ResHi)->addIncoming(Hi, OldBB);
        // Prepare the result from inner if-else-endif.
        cast<PHINode>(ResLo)->addIncoming(InnerResLo, InnerJBB);
        cast<PHINode>(ResHi)->addIncoming(InnerResHi, InnerJBB);
    }
    else
        Cond = IRB->CreateICmpULT(ShAmt, IRB->getInt32(32));

    if (Cond == IRB->getTrue() || InnerTBB) {
        if (InnerTBB) IRB->SetInsertPoint(&(*InnerTBB->begin()));

        Value* L = IRB->CreateShl(Lo, ShAmt);
        Value* Amt = IRB->CreateSub(IRB->getInt32(32), ShAmt);
        Value* T0 = IRB->CreateLShr(Lo, Amt);
        Value* H = IRB->CreateShl(Hi, ShAmt);
        H = IRB->CreateOr(H, T0);

        if (InnerTBB) {
            IGC_ASSERT(isa<PHINode>(InnerResLo));
            IGC_ASSERT(isa<PHINode>(InnerResHi));
            InnerResLo->addIncoming(L, InnerTBB);
            InnerResHi->addIncoming(H, InnerTBB);
        }
        else {
            ResLo = L;
            ResHi = H;
        }
    }

    if (Cond == IRB->getFalse() || InnerFBB) {
        if (InnerFBB) IRB->SetInsertPoint(&(*InnerFBB->begin()));

        Value* L = IRB->getInt32(0);
        Value* Amt = IRB->CreateSub(ShAmt, IRB->getInt32(32));
        Value* H = IRB->CreateShl(Lo, Amt);

        if (InnerFBB) {
            IGC_ASSERT(isa<PHINode>(InnerResLo));
            IGC_ASSERT(isa<PHINode>(InnerResHi));
            InnerResLo->addIncoming(L, InnerFBB);
            InnerResHi->addIncoming(H, InnerFBB);
        }
        else {
            ResLo = L;
            ResHi = H;
        }
    }

    Emu->setExpandedValues(&BinOp, ResLo, ResHi);
    return true;
}

bool InstExpander::visitLShr(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    if (hasHWSupport(BinOp))
        return false;

    auto [Lo, Hi] = Emu->getExpandedValues(BinOp.getOperand(0));
    Value* ShAmt = nullptr;
    std::tie(ShAmt, std::ignore) = Emu->getExpandedValues(BinOp.getOperand(1));

    BasicBlock* OldBB = BinOp.getParent();
    BasicBlock* InnerTBB = nullptr;
    BasicBlock* InnerFBB = nullptr;
    PHINode* InnerResLo = nullptr;
    PHINode* InnerResHi = nullptr;
    Value* ResLo = nullptr;
    Value* ResHi = nullptr;
    DebugLoc BinOpDebugLoc = BinOp.getDebugLoc();

    Value* Cond = nullptr;

    // Only LSB of ShAmt is used for shift
    ShAmt = IRB->CreateAnd(ShAmt, 63);
    if (!isa<ConstantInt>(ShAmt)) {
        // Create outer if-endif to handle the special case where `ShAmt` is zero.
        // We have option to handle that with `((S >> (~ShAmt)) >> 1)`. However, as
        // a zero `ShAmt` is a very rare case so that the outer branch should be
        // uniform one in most cases.
        Value* NE = IRB->CreateICmpNE(ShAmt, Constant::getNullValue(ShAmt->getType()));
        BasicBlock* JointBB = OldBB->splitBasicBlock(&BinOp);
        ResLo = PHINode::Create(IRB->getInt32Ty(), 2, ".lshr.outer.merge.lo", &BinOp);
        cast<Instruction>(ResLo)->setDebugLoc(BinOpDebugLoc);
        ResHi = PHINode::Create(IRB->getInt32Ty(), 2, ".lshr.outer.merge.hi", &BinOp);
        cast<Instruction>(ResHi)->setDebugLoc(BinOpDebugLoc);

        BasicBlock* TrueBB = BasicBlock::Create(*Emu->getContext(),
            ".lshr.outer.true.branch");
        TrueBB->insertInto(Emu->getFunction(), JointBB);
        Instruction* TrueJmp = BranchInst::Create(JointBB, TrueBB);
        TrueJmp->setDebugLoc(BinOpDebugLoc);

        OldBB->getTerminator()->eraseFromParent();
        Instruction* TempOldBBBranchInst = BranchInst::Create(TrueBB, JointBB, NE, OldBB);
        TempOldBBBranchInst->setDebugLoc(BinOpDebugLoc);

        // Create the inner branch.
        IRB->SetInsertPoint(&(*TrueBB->begin()));

        Cond = IRB->CreateICmpULT(ShAmt, IRB->getInt32(32));
        // Prepare to generate branches to handle the case where `ShAmt` is less
        // than 32 (true branch) or otherwise (false branch).
        BasicBlock* InnerJBB = TrueBB->splitBasicBlock(TrueJmp);
        InnerResLo = PHINode::Create(IRB->getInt32Ty(), 2, ".lshr.merge.inner.lo", TrueJmp);
        InnerResLo->setDebugLoc(BinOpDebugLoc);
        InnerResHi = PHINode::Create(IRB->getInt32Ty(), 2, ".lshr.merge.inner.hi", TrueJmp);
        InnerResHi->setDebugLoc(BinOpDebugLoc);

        InnerTBB = BasicBlock::Create(*Emu->getContext(), ".lshr.inner.true.branch");
        InnerTBB->insertInto(Emu->getFunction(), InnerJBB);
        Instruction* TempInnerTBBBranchInst = BranchInst::Create(InnerJBB, InnerTBB);
        TempInnerTBBBranchInst->setDebugLoc(BinOpDebugLoc);

        InnerFBB = BasicBlock::Create(*Emu->getContext(), ".lshr.inner.false.branch");
        InnerFBB->insertInto(Emu->getFunction(), InnerJBB);
        Instruction* TempInnerFBBBranchInst = BranchInst::Create(InnerJBB, InnerFBB);
        TempInnerFBBBranchInst->setDebugLoc(BinOpDebugLoc);

        TrueBB->getTerminator()->eraseFromParent();
        Instruction* TempTrueBBBranchInst = BranchInst::Create(InnerTBB, InnerFBB, Cond, TrueBB);
        TempTrueBBBranchInst->setDebugLoc(BinOpDebugLoc);

        // The result is the same as the source if ShAmt is 0, i.e. NE is
        // false.
        cast<PHINode>(ResLo)->addIncoming(Lo, OldBB);
        cast<PHINode>(ResHi)->addIncoming(Hi, OldBB);
        // Prepare the result from inner if-else-endif.
        cast<PHINode>(ResLo)->addIncoming(InnerResLo, InnerJBB);
        cast<PHINode>(ResHi)->addIncoming(InnerResHi, InnerJBB);
    }
    else
        Cond = IRB->CreateICmpULT(ShAmt, IRB->getInt32(32));

    if (Cond == IRB->getTrue() || InnerTBB) {
        if (InnerTBB) IRB->SetInsertPoint(&(*InnerTBB->begin()));

        Value* H = IRB->CreateLShr(Hi, ShAmt);

        // Avoid using IRB->CreateNeg(ShAmt) as shift amount, because (-ShAmt) will
        // be interpret as a number bigger than 32. LLVM might figure out this out
        // and thus makes shift result undefined!
        Value* Amt = IRB->CreateSub(IRB->getInt32(32), ShAmt);

        Value* T0 = IRB->CreateShl(Hi, Amt);
        Value* L = IRB->CreateLShr(Lo, ShAmt);
        L = IRB->CreateOr(L, T0);

        if (InnerTBB) {
            InnerResLo->addIncoming(L, InnerTBB);
            InnerResHi->addIncoming(H, InnerTBB);
        }
        else {
            ResLo = L;
            ResHi = H;
        }
    }

    if (Cond == IRB->getFalse() || InnerFBB) {
        if (InnerFBB) IRB->SetInsertPoint(&(*InnerFBB->begin()));

        Value* H = IRB->getInt32(0);

        // Avoid using ShAmt as shift amount as it is bigger than 32. LLVM might figure out
        // this shift amount is bigger than 32 and thus makes the shift result undefined.
        Value* Amt = IRB->CreateSub(ShAmt, IRB->getInt32(32));

        Value* L = IRB->CreateLShr(Hi, Amt);

        if (InnerFBB) {
            InnerResLo->addIncoming(L, InnerFBB);
            InnerResHi->addIncoming(H, InnerFBB);
        }
        else {
            ResLo = L;
            ResHi = H;
        }
    }

    Emu->setExpandedValues(&BinOp, ResLo, ResHi);
    return true;
}

bool InstExpander::visitAShr(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    if (hasHWSupport(BinOp))
        return false;

    auto [Lo, Hi] = Emu->getExpandedValues(BinOp.getOperand(0));
    Value* ShAmt = nullptr;
    std::tie(ShAmt, std::ignore) = Emu->getExpandedValues(BinOp.getOperand(1));

    BasicBlock* OldBB = BinOp.getParent();
    BasicBlock* InnerTBB = nullptr;
    BasicBlock* InnerFBB = nullptr;
    PHINode* InnerResLo = nullptr;
    PHINode* InnerResHi = nullptr;
    Value* ResLo = nullptr;
    Value* ResHi = nullptr;
    DebugLoc BinOpDebugLoc = BinOp.getDebugLoc();

    Value* Cond = nullptr;

    // Only LSB of ShAmt is used for shift
    ShAmt = IRB->CreateAnd(ShAmt, 63);
    if (!isa<ConstantInt>(ShAmt)) {
        // Create outer if-endif to handle the special case where `ShAmt` is zero.
        // We have option to handle that with `((S >> (~ShAmt)) >> 1)`. However, as
        // a zero `ShAmt` is a very rare case so that the outer branch should be
        // uniform one in most cases.
        Value* NE = IRB->CreateICmpNE(ShAmt, Constant::getNullValue(ShAmt->getType()));
        BasicBlock* JointBB = OldBB->splitBasicBlock(&BinOp);
        ResLo = PHINode::Create(IRB->getInt32Ty(), 2, ".ashr.outer.merge.lo", &BinOp);
        cast<Instruction>(ResLo)->setDebugLoc(BinOpDebugLoc);
        ResHi = PHINode::Create(IRB->getInt32Ty(), 2, ".ashr.outer.merge.hi", &BinOp);
        cast<Instruction>(ResHi)->setDebugLoc(BinOpDebugLoc);

        BasicBlock* TrueBB = BasicBlock::Create(*Emu->getContext(),
            ".ashr.outer.true.branch");
        TrueBB->insertInto(Emu->getFunction(), JointBB);
        Instruction* TrueJmp = BranchInst::Create(JointBB, TrueBB);
        TrueJmp->setDebugLoc(BinOpDebugLoc);

        OldBB->getTerminator()->eraseFromParent();
        Instruction* TempOldBBBranchInst = BranchInst::Create(TrueBB, JointBB, NE, OldBB);
        TempOldBBBranchInst->setDebugLoc(BinOpDebugLoc);

        // Create the inner branch.
        IRB->SetInsertPoint(&(*TrueBB->begin()));

        Cond = IRB->CreateICmpULT(ShAmt, IRB->getInt32(32));
        // Prepare to generate branches to handle the case where `ShAmt` is less
        // than 32 or otherwise.
        BasicBlock* InnerJBB = TrueBB->splitBasicBlock(TrueJmp);
        InnerResLo = PHINode::Create(IRB->getInt32Ty(), 2, ".ashr.merge.inner.lo", TrueJmp);
        InnerResLo->setDebugLoc(BinOpDebugLoc);
        InnerResHi = PHINode::Create(IRB->getInt32Ty(), 2, ".ashr.merge.inner.hi", TrueJmp);
        InnerResHi->setDebugLoc(BinOpDebugLoc);

        InnerTBB = BasicBlock::Create(*Emu->getContext(), ".ashr.inner.true.branch");
        InnerTBB->insertInto(Emu->getFunction(), InnerJBB);
        Instruction* TempInnerTBBBranchInst = BranchInst::Create(InnerJBB, InnerTBB);
        TempInnerTBBBranchInst->setDebugLoc(BinOpDebugLoc);

        InnerFBB = BasicBlock::Create(*Emu->getContext(), ".ashr.inner.false.branch");
        InnerFBB->insertInto(Emu->getFunction(), InnerJBB);
        Instruction* TempInnerFBBBranchInst = BranchInst::Create(InnerJBB, InnerFBB);
        TempInnerFBBBranchInst->setDebugLoc(BinOp.getDebugLoc());

        TrueBB->getTerminator()->eraseFromParent();
        Instruction* TempTrueBBBranchInst = BranchInst::Create(InnerTBB, InnerFBB, Cond, TrueBB);
        TempTrueBBBranchInst->setDebugLoc(BinOpDebugLoc);

        // The result is the same as the source if ShAmt is 0, i.e. NE is
        // false.
        cast<PHINode>(ResLo)->addIncoming(Lo, OldBB);
        cast<PHINode>(ResHi)->addIncoming(Hi, OldBB);
        // Prepare the result from inner if-else-endif.
        cast<PHINode>(ResLo)->addIncoming(InnerResLo, InnerJBB);
        cast<PHINode>(ResHi)->addIncoming(InnerResHi, InnerJBB);
    }
    else
        Cond = IRB->CreateICmpULT(ShAmt, IRB->getInt32(32));

    if (Cond == IRB->getTrue() || InnerTBB) {
        if (InnerTBB) IRB->SetInsertPoint(&(*InnerTBB->begin()));

        Value* H = IRB->CreateAShr(Hi, ShAmt);
        Value* Amt = IRB->CreateSub(IRB->getInt32(32), ShAmt);
        Value* T0 = IRB->CreateShl(Hi, Amt);
        Value* L = IRB->CreateLShr(Lo, ShAmt);
        L = IRB->CreateOr(L, T0);

        if (InnerTBB) {
            InnerResLo->addIncoming(L, InnerTBB);
            InnerResHi->addIncoming(H, InnerTBB);
        }
        else {
            ResLo = L;
            ResHi = H;
        }
    }

    if (Cond == IRB->getFalse() || InnerFBB) {
        if (InnerFBB) IRB->SetInsertPoint(&(*InnerFBB->begin()));

        Value* H = IRB->CreateAShr(Hi, 31);
        Value* Amt = IRB->CreateSub(ShAmt, IRB->getInt32(32));
        Value* L = IRB->CreateAShr(Hi, Amt);

        if (InnerFBB) {
            InnerResLo->addIncoming(L, InnerFBB);
            InnerResHi->addIncoming(H, InnerFBB);
        }
        else {
            ResLo = L;
            ResHi = H;
        }
    }

    Emu->setExpandedValues(&BinOp, ResLo, ResHi);
    return true;
}

bool InstExpander::visitAnd(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    auto [L0, H0] = Emu->getExpandedValues(BinOp.getOperand(0));
    auto [L1, H1] = Emu->getExpandedValues(BinOp.getOperand(1));

    Value* Lo = IRB->CreateAnd(L0, L1);
    Value* Hi = IRB->CreateAnd(H0, H1);

    Emu->setExpandedValues(&BinOp, Lo, Hi);
    return true;
}

bool InstExpander::visitOr(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    auto [L0, H0] = Emu->getExpandedValues(BinOp.getOperand(0));
    auto [L1, H1] = Emu->getExpandedValues(BinOp.getOperand(1));

    Value* Lo = IRB->CreateOr(L0, L1);
    Value* Hi = IRB->CreateOr(H0, H1);

    Emu->setExpandedValues(&BinOp, Lo, Hi);
    return true;
}

bool InstExpander::visitXor(BinaryOperator& BinOp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BinOp))
        return false;

    auto [L0, H0] = Emu->getExpandedValues(BinOp.getOperand(0));
    auto [L1, H1] = Emu->getExpandedValues(BinOp.getOperand(1));

    Value* Lo = IRB->CreateXor(L0, L1);
    Value* Hi = IRB->CreateXor(H0, H1);

    Emu->setExpandedValues(&BinOp, Lo, Hi);
    return true;
}

bool InstExpander::visitLoad(LoadInst& LD) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&LD))
        return false;

    Value* OldPtr = LD.getPointerOperand();
    PointerType* OldPtrTy = cast<PointerType>(OldPtr->getType());
    PointerType* NewPtrTy =
        Emu->getV2Int32Ty()->getPointerTo(OldPtrTy->getAddressSpace());
    Value* NewPtr = IRB->CreatePointerCast(OldPtr, NewPtrTy);

    LoadInst* NewLD = IRB->CreateLoad(Emu->getV2Int32Ty(), NewPtr);
    Emu->dupMemoryAttribute(NewLD, &LD, 0);

    Value* Lo = IRB->CreateExtractElement(NewLD, IRB->getInt32(0));
    Value* Hi = IRB->CreateExtractElement(NewLD, IRB->getInt32(1));

    Emu->setExpandedValues(&LD, Lo, Hi);
    return true;
}

bool InstExpander::visitStore(StoreInst& ST) {
    Value* V = ST.getValueOperand();
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(V))
        return false;

    auto [Lo, Hi] = Emu->getExpandedValues(V);

    Type* V2I32Ty = Emu->getV2Int32Ty();
    Value* NewVal = UndefValue::get(V2I32Ty);
    NewVal = IRB->CreateInsertElement(NewVal, Lo, IRB->getInt32(0));
    NewVal = IRB->CreateInsertElement(NewVal, Hi, IRB->getInt32(1));

    Value* OldPtr = ST.getPointerOperand();
    PointerType* OldPtrTy = cast<PointerType>(OldPtr->getType());
    PointerType* NewPtrTy = V2I32Ty->getPointerTo(OldPtrTy->getAddressSpace());
    Value* NewPtr = IRB->CreatePointerCast(OldPtr, NewPtrTy);

    StoreInst* NewST = IRB->CreateStore(NewVal, NewPtr);
    Emu->dupMemoryAttribute(NewST, &ST, 0);

    return true;
}

bool InstExpander::visitAtomicCmpXchg(AtomicCmpXchgInst& ACXI) {
    Value* V = ACXI.getCompareOperand();
    IGC_ASSERT(nullptr != V);
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT_MESSAGE(false == Emu->isInt64(V), "TODO: NOT IMPLEMENTED YET!");

    return false;
}

bool InstExpander::visitAtomicRMW(AtomicRMWInst& RMW) {
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT_MESSAGE(false == Emu->isInt64(&RMW), "TODO: NOT IMPLEMENTED YET!");
    return false;
}

bool InstExpander::visitTrunc(TruncInst& TI) {
    Value* Src = TI.getOperand(0);
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT(nullptr != Src);
    if (!Emu->isInt64(Src))
        return false;

    Value* Lo = nullptr;
    std::tie(Lo, std::ignore) = Emu->getExpandedValues(Src);
    IGC_ASSERT(nullptr != Lo);
    IGC_ASSERT(nullptr != Lo->getType());
    IGC_ASSERT(nullptr != TI.getType());
    IGC_ASSERT(Lo->getType()->getScalarSizeInBits() >= TI.getType()->getScalarSizeInBits());

    if (Lo->getType()->getScalarSizeInBits() !=
        TI.getType()->getScalarSizeInBits())
        Lo = IRB->CreateTrunc(Lo, TI.getType());

    TI.replaceAllUsesWith(Lo);
    return true;
}

bool InstExpander::visitSExt(SExtInst& SEI) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&SEI))
        return false;

    if (!needsLoHiSplitting(SEI))
        return false;

    Value* Src = SEI.getOperand(0);
    Type* SrcTy = SEI.getSrcTy();
    IGC_ASSERT(nullptr != SrcTy);
    unsigned SrcWidth = SrcTy->getIntegerBitWidth();
    IGC_ASSERT(SrcWidth <= 32);

    Value* Lo = Src;
    if (SrcWidth < 32)
        Lo = IRB->CreateSExt(Lo, IRB->getInt32Ty());
    Value* Hi = IRB->CreateAShr(Lo, IRB->getInt32(31));

    Emu->setExpandedValues(&SEI, Lo, Hi);
    return true;
}

bool InstExpander::visitZExt(ZExtInst& ZEI) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&ZEI))
        return false;

    if (!needsLoHiSplitting(ZEI))
        return false;

    Value* Src = ZEI.getOperand(0);
    Type* SrcTy = ZEI.getSrcTy();
    IGC_ASSERT(nullptr != SrcTy);
    unsigned SrcWidth = SrcTy->getIntegerBitWidth();
    IGC_ASSERT(SrcWidth <= 32);

    Value* Lo = Src;
    if (SrcWidth < 32)
        Lo = IRB->CreateZExt(Lo, IRB->getInt32Ty());
    Value* Hi = IRB->getInt32(0);

    Emu->setExpandedValues(&ZEI, Lo, Hi);
    return true;
}

bool InstExpander::visitFPToUI(FPToUIInst& F2U) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&F2U))
        return false;

    IGCLLVM::Intrinsic IID;
    Value* Src = F2U.getOperand(0);
    IGC_ASSERT(nullptr != Src);
    Type* SrcTy = Src->getType();
    IGC_ASSERT(nullptr != SrcTy);

    if (SrcTy->isHalfTy()) {
        // Convert half directly into 32-bit integer.
        Value* Lo = IRB->CreateFPToUI(Src, IRB->getInt32Ty());
        Value* Hi = Constant::getNullValue(IRB->getInt32Ty());
        Emu->setExpandedValues(&F2U, Lo, Hi);
        return true;
    }

    IID = Intrinsic::trunc;
    Function* Trunc = Intrinsic::getDeclaration(Emu->getModule(), IID, SrcTy);
    IID = Intrinsic::fma;
    Function* Fma = Intrinsic::getDeclaration(Emu->getModule(), IID, SrcTy);

    Value* FC0 = ConstantFP::get(SrcTy, ldexp(1., -32));
    Value* FC1 = ConstantFP::get(SrcTy, ldexp(-1., 32));

    Value* HiF = nullptr;
    Value* Hi = nullptr;
    if (SrcTy->isDoubleTy()) {
        // HW doesn't support rounding on double but does support it on float,
        // We have to build our own based on integer conversion.
        Hi = IRB->CreateFPToUI(IRB->CreateFMul(Src, FC0), IRB->getInt32Ty());
        HiF = IRB->CreateUIToFP(Hi, SrcTy);
    }
    else {
        HiF = IRB->CreateCall(Trunc, IRB->CreateFMul(Src, FC0));
        Hi = IRB->CreateFPToUI(HiF, IRB->getInt32Ty());
    }
    Value* LoF = IRB->CreateCall3(Fma, HiF, FC1, Src);
    Value* Lo = IRB->CreateFPToUI(LoF, IRB->getInt32Ty());

    Emu->setExpandedValues(&F2U, Lo, Hi);
    return true;
}

bool InstExpander::visitFPToSI(FPToSIInst& F2S) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&F2S))
        return false;

    IGCLLVM::Intrinsic IID;
    Value* Src = F2S.getOperand(0);
    IGC_ASSERT(nullptr != Src);
    Type* SrcTy = Src->getType();
    IGC_ASSERT(nullptr != SrcTy);

    if (SrcTy->isHalfTy()) {
        // Convert half directly into 32-bit integer.
        Value* Lo = IRB->CreateFPToSI(Src, IRB->getInt32Ty());
        Value* Hi = IRB->CreateAShr(Lo, 31);
        Emu->setExpandedValues(&F2S, Lo, Hi);
        return true;
    }

    Value* Sign = nullptr;
    if (SrcTy->isDoubleTy()) {
        Sign = IRB->CreateBitCast(Src, Emu->getV2Int32Ty());
        Sign = IRB->CreateExtractElement(Sign, IRB->getInt32(1));
    }
    else {
        IGC_ASSERT_MESSAGE(SrcTy->isFloatTy(), "Unknown float type!");
        Sign = IRB->CreateBitCast(Src, IRB->getInt32Ty());
    }
    Sign = IRB->CreateAShr(Sign, 31);

    IID = Intrinsic::fabs;
    Function* FAbs = Intrinsic::getDeclaration(Emu->getModule(), IID, SrcTy);
    IID = Intrinsic::trunc;
    Function* Trunc = Intrinsic::getDeclaration(Emu->getModule(), IID, SrcTy);
    IID = Intrinsic::fma;
    Function* Fma = Intrinsic::getDeclaration(Emu->getModule(), IID, SrcTy);

    Value* FC0 = ConstantFP::get(SrcTy, ldexp(1., -32));
    Value* FC1 = ConstantFP::get(SrcTy, ldexp(-1., 32));

    Src = IRB->CreateCall(FAbs, Src);
    Value* HiF = nullptr;
    Value* Hi = nullptr;
    if (SrcTy->isDoubleTy()) {
        // HW doesn't support rounding on double but does support it on float,
        // We have to build our own based on integer conversion.
        Hi = IRB->CreateFPToUI(IRB->CreateFMul(Src, FC0), IRB->getInt32Ty());
        HiF = IRB->CreateUIToFP(Hi, SrcTy);
    }
    else {
        HiF = IRB->CreateCall(Trunc, IRB->CreateFMul(Src, FC0));
        Hi = IRB->CreateFPToUI(HiF, IRB->getInt32Ty());
    }
    Value* LoF = IRB->CreateCall3(Fma, HiF, FC1, Src);
    Value* Lo = IRB->CreateFPToUI(LoF, IRB->getInt32Ty());

    Lo = IRB->CreateXor(Lo, Sign);
    Hi = IRB->CreateXor(Hi, Sign);

    GenISAIntrinsic::ID GIID = GenISAIntrinsic::GenISA_sub_pair;
    Function* IFunc = GenISAIntrinsic::getDeclaration(Emu->getModule(), GIID);
    Value* V = IRB->CreateCall4(IFunc, Lo, Hi, Sign, Sign);
    Lo = IRB->CreateExtractValue(V, 0);
    Hi = IRB->CreateExtractValue(V, 1);

    Emu->setExpandedValues(&F2S, Lo, Hi);
    return true;
}

Value* InstExpander::convertUIToFP32(Type* DstTy, Value* Lo, Value* Hi, Instruction* Pos) {
    IGC_ASSERT(nullptr != Pos);
    IGC_ASSERT(nullptr != IRB);

    IGCLLVM::Intrinsic IID;
    IID = Intrinsic::ctlz;
    Function* Lzd = Intrinsic::getDeclaration(Emu->getModule(), IID, Lo->getType());

    Value* ShAmt = IRB->CreateCall2(Lzd, Hi, IRB->getFalse());
    // Check ShAmt == 32
    Value* NE = IRB->CreateICmpNE(ShAmt, IRB->getInt32(32));

    BasicBlock* OldBB = Pos->getParent();
    IGC_ASSERT(nullptr != OldBB);
    BasicBlock* JointBB = OldBB->splitBasicBlock(Pos);

    // Set insert point to Pos in the new block JointBB
    IRB->SetInsertPoint(Pos);
    DebugLoc PosDebugLoc = Pos->getDebugLoc();

    PHINode* Res = PHINode::Create(IRB->getInt32Ty(), 2, ".u2f.outer.merge", Pos);
    Res->setDebugLoc(PosDebugLoc);

    {
        BuilderType::InsertPointGuard Guard(*IRB);

        BasicBlock* TrueBB = BasicBlock::Create(*Emu->getContext(), ".u2f.outer.true.branch");
        TrueBB->insertInto(Emu->getFunction(), JointBB);
        Instruction* TrueJmp = BranchInst::Create(JointBB, TrueBB);
        TrueJmp->setDebugLoc(PosDebugLoc);

        OldBB->getTerminator()->eraseFromParent();
        Instruction* TempOldBBBranchInst = BranchInst::Create(TrueBB, JointBB, NE, OldBB);
        TempOldBBBranchInst->setDebugLoc(PosDebugLoc);

        IRB->SetInsertPoint(&(*TrueBB->begin()));
        // Check ShAmt == 0
        NE = IRB->CreateICmpNE(ShAmt, Constant::getNullValue(ShAmt->getType()));

        BasicBlock* InnerJBB = TrueBB->splitBasicBlock(TrueJmp);
        PHINode* InnerResHi = PHINode::Create(IRB->getInt32Ty(), 2, ".u2f.inner.merge.hi", TrueJmp);
        InnerResHi->setDebugLoc(PosDebugLoc);
        PHINode* InnerResLo = PHINode::Create(IRB->getInt32Ty(), 2, ".u2f.inner.merge.lo", TrueJmp);
        InnerResLo->setDebugLoc(PosDebugLoc);

        BasicBlock* InnerTBB = BasicBlock::Create(*Emu->getContext(), ".u2f.inner.true.branch");
        InnerTBB->insertInto(Emu->getFunction(), InnerJBB);
        Instruction* TempInnerTBBBranchInst = BranchInst::Create(InnerJBB, InnerTBB);
        TempInnerTBBBranchInst->setDebugLoc(PosDebugLoc);

        TrueBB->getTerminator()->eraseFromParent();
        Instruction* TempTrueBBBranchInst = BranchInst::Create(InnerTBB, InnerJBB, NE, TrueBB);
        TempTrueBBBranchInst->setDebugLoc(PosDebugLoc);

        IRB->SetInsertPoint(&(*InnerTBB->begin()));
        // 0 < ShAmt < 32
        Value* L = IRB->CreateShl(Lo, ShAmt);
        Value* H = IRB->CreateShl(Hi, ShAmt);
        Value* S = IRB->CreateSub(IRB->getInt32(32), ShAmt);
        Value* T0 = IRB->CreateLShr(Lo, S);
        H = IRB->CreateOr(H, T0);

        InnerResHi->addIncoming(Hi, TrueBB);
        InnerResLo->addIncoming(Lo, TrueBB);

        InnerResHi->addIncoming(H, InnerTBB);
        InnerResLo->addIncoming(L, InnerTBB);

        Instruction* InnerJmp = InnerJBB->getTerminator();
        IRB->SetInsertPoint(InnerJmp);
        // Check InnerResLo != 0 to round correctly.
        NE = IRB->CreateICmpNE(InnerResLo, Constant::getNullValue(InnerResLo->getType()));

        BasicBlock* RoundingJBB = InnerJBB->splitBasicBlock(InnerJmp);
        PHINode* RoundingRes = PHINode::Create(IRB->getInt32Ty(), 2, ".u2f.rounding.merge.hi", InnerJmp);
        RoundingRes->setDebugLoc(PosDebugLoc);

        BasicBlock* RoundingBB = BasicBlock::Create(*Emu->getContext(), ".u2f.rounding.branch");
        RoundingBB->insertInto(Emu->getFunction(), RoundingJBB);
        Instruction* TempRoundingBBBranchInst = BranchInst::Create(RoundingJBB, RoundingBB);
        TempRoundingBBBranchInst->setDebugLoc(PosDebugLoc);

        InnerJBB->getTerminator()->eraseFromParent();
        Instruction* TempInnerJBBBranchInst = BranchInst::Create(RoundingBB, RoundingJBB, NE, InnerJBB);
        TempInnerJBBBranchInst->setDebugLoc(PosDebugLoc);

        // Rounding
        IRB->SetInsertPoint(&(*RoundingBB->begin()));
        Value* Rounded = IRB->CreateOr(InnerResHi, IRB->getInt32(1));

        RoundingRes->addIncoming(InnerResHi, InnerJBB);
        RoundingRes->addIncoming(Rounded, RoundingBB);

        Res->addIncoming(Lo, OldBB);
        Res->addIncoming(RoundingRes, RoundingJBB);
    }

    Value* NewVal = IRB->CreateUIToFP(Res, DstTy);
    // Adjust
    Value* Exp = IRB->CreateSub(IRB->getInt32(127 + 32), ShAmt);
    Exp = IRB->CreateShl(Exp, IRB->getInt32(23));
    Exp = IRB->CreateBitCast(Exp, NewVal->getType());
    NewVal = IRB->CreateFMul(NewVal, Exp);

    return NewVal;
}

bool InstExpander::visitUIToFP(UIToFPInst& U2F) {
    Value* Src = U2F.getOperand(0);
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(Src))
        return false;

    auto [Lo, Hi] = Emu->getExpandedValues(Src);

    Type* DstTy = U2F.getType();
    Value* NewVal = nullptr;

    if (ConstantInt * CI = dyn_cast<ConstantInt>(Hi)) {
        if (CI->isNullValue()) {
            NewVal = IRB->CreateUIToFP(Lo, DstTy);
            U2F.replaceAllUsesWith(NewVal);
            return true;
        }
    }

    if (DstTy->isDoubleTy()) {
        IGCLLVM::Intrinsic IID = Intrinsic::fma;
        Function* Fma = Intrinsic::getDeclaration(Emu->getModule(), IID, DstTy);
        Value* FC0 = ConstantFP::get(DstTy, ldexp(1., 32));
        Value* LoF = IRB->CreateUIToFP(Lo, DstTy);
        Value* HiF = IRB->CreateUIToFP(Hi, DstTy);
        NewVal = IRB->CreateCall3(Fma, HiF, FC0, LoF);
    }
    else {
        NewVal = convertUIToFP32(IRB->getFloatTy(), Lo, Hi, &U2F);
        // It's OK to apply the same approach in `convertUIToFP32` to convert 64-bit
        // integer into half. But, it would introduce a little more instructions to
        // properly round the remaining 48 bits into the high 16 bits. Instead, that
        // 64-bit integer is firstly converted into `float` and then converted into
        // `half`.
        if (DstTy->isHalfTy())
            NewVal = IRB->CreateFPTrunc(NewVal, DstTy);
    }

    U2F.replaceAllUsesWith(NewVal);
    return true;
}

bool InstExpander::visitSIToFP(SIToFPInst& S2F) {
    Value* Src = S2F.getOperand(0);
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(Src))
        return false;

    auto [Lo, Hi] = Emu->getExpandedValues(Src);

    Type* DstTy = S2F.getType();
    Value* NewVal = nullptr;

    if (ConstantInt * CI = dyn_cast<ConstantInt>(Hi)) {
        if (CI->isNullValue()) {
            NewVal = IRB->CreateUIToFP(Lo, DstTy);
            S2F.replaceAllUsesWith(NewVal);
            return true;
        }
        if (CI->isMinusOne()) {
            NewVal = IRB->CreateUIToFP(Lo, DstTy);
            NewVal = IRB->CreateFNeg(NewVal);
            S2F.replaceAllUsesWith(NewVal);
            return true;
        }
    }

    if (DstTy->isDoubleTy()) {
        IGCLLVM::Intrinsic IID = Intrinsic::fma;
        Function* Fma = Intrinsic::getDeclaration(Emu->getModule(), IID, DstTy);
        Value* FC0 = ConstantFP::get(DstTy, ldexp(1., 32));
        Value* LoF = IRB->CreateUIToFP(Lo, DstTy);
        Value* HiF = IRB->CreateSIToFP(Hi, DstTy);
        NewVal = IRB->CreateCall3(Fma, HiF, FC0, LoF);
    }
    else {
        GenISAIntrinsic::ID GIID = GenISAIntrinsic::GenISA_sub_pair;
        Function* SubPair = GenISAIntrinsic::getDeclaration(Emu->getModule(), GIID);

        Value* Sign = IRB->CreateAShr(Hi, 31);

        // Get abs(x) = (x ^ s) - s;
        Lo = IRB->CreateXor(Lo, Sign);
        Hi = IRB->CreateXor(Hi, Sign);
        Value* V = IRB->CreateCall4(SubPair, Lo, Hi, Sign, Sign);
        Lo = IRB->CreateExtractValue(V, 0);
        Hi = IRB->CreateExtractValue(V, 1);

        NewVal = convertUIToFP32(IRB->getFloatTy(), Lo, Hi, &S2F);
        NewVal = IRB->CreateBitCast(NewVal, IRB->getInt32Ty());
        Sign = IRB->CreateAnd(Sign, IRB->getInt32(0x80000000));
        NewVal = IRB->CreateOr(NewVal, Sign);
        NewVal = IRB->CreateBitCast(NewVal, IRB->getFloatTy());
        if (DstTy->isHalfTy())
            NewVal = IRB->CreateFPTrunc(NewVal, DstTy);
    }

    S2F.replaceAllUsesWith(NewVal);
    return true;
}

bool InstExpander::visitPtrToInt(PtrToIntInst& P2I) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&P2I))
        return false;

    Value* Ptr = P2I.getOperand(0);

    GenISAIntrinsic::ID GIID = GenISAIntrinsic::GenISA_ptr_to_pair;
    Function* IFunc = GenISAIntrinsic::getDeclaration(Emu->getModule(), GIID, Ptr->getType());
    Value* V = IRB->CreateCall(IFunc, Ptr);
    Value* Lo = IRB->CreateExtractValue(V, 0);
    Value* Hi = IRB->CreateExtractValue(V, 1);

    Emu->setExpandedValues(&P2I, Lo, Hi);
    return true;
}

bool InstExpander::visitIntToPtr(IntToPtrInst& I2P) {
    Value* Src = I2P.getOperand(0);
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(Src))
        return false;

    auto [Lo, Hi] = Emu->getExpandedValues(Src);

    GenISAIntrinsic::ID GIID = GenISAIntrinsic::GenISA_pair_to_ptr;
    Function* IFunc = GenISAIntrinsic::getDeclaration(Emu->getModule(), GIID, I2P.getType());
    Value* Ptr = IRB->CreateCall2(IFunc, Lo, Hi);
    Ptr = IRB->CreateBitCast(Ptr, I2P.getType());

    I2P.replaceAllUsesWith(Ptr);
    return true;
}

bool InstExpander::visitBitCast(BitCastInst& BC) {
    Value* Src = BC.getOperand(0);
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&BC) && !Emu->isInt64(Src))
        return false;

    if (Emu->isInt64(&BC)) {
        if (Emu->isInt64(Src)) {
            // Somehow there are still 'bitcast's from i64 to i64. Directly
            // re-use the already expanded values.
            auto [Lo, Hi] = Emu->getExpandedValues(Src);
            Emu->setExpandedValues(&BC, Lo, Hi);
            return true;
        }
        Src = IRB->CreateBitCast(Src, Emu->getV2Int32Ty());
        Value* Lo = IRB->CreateExtractElement(Src, IRB->getInt32(0));
        Value* Hi = IRB->CreateExtractElement(Src, IRB->getInt32(1));

        Emu->setExpandedValues(&BC, Lo, Hi);
        return true;
    }

    IGC_ASSERT(Emu->isInt64(Src));

    // Skip argument which is already prepared specially.
    if (Emu->isArg64Cast(&BC))
        return false;

    auto [Lo, Hi] = Emu->getExpandedValues(Src);

    Type* V2I32Ty = Emu->getV2Int32Ty();
    Value* NewVal = UndefValue::get(V2I32Ty);
    NewVal = IRB->CreateInsertElement(NewVal, Lo, IRB->getInt32(0));
    NewVal = IRB->CreateInsertElement(NewVal, Hi, IRB->getInt32(1));
    NewVal = IRB->CreateBitCast(NewVal, BC.getType());

    BC.replaceAllUsesWith(NewVal);
    return true;
}

bool InstExpander::visitICmp(ICmpInst& Cmp) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(Cmp.getOperand(0)))
        return false;

    auto Pred = Cmp.getPredicate();
    auto [L0, H0] = Emu->getExpandedValues(Cmp.getOperand(0));
    auto [L1, H1] = Emu->getExpandedValues(Cmp.getOperand(1));

    Value* T0 = nullptr, * T1 = nullptr, * T2 = nullptr, * T3 = nullptr, * Res = nullptr;
    switch (Pred) {
    case CmpInst::ICMP_EQ:
        T0 = IRB->CreateICmpEQ(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        Res = IRB->CreateAnd(T1, T0);
        break;
    case CmpInst::ICMP_NE:
        T0 = IRB->CreateICmpNE(L0, L1),
            T1 = IRB->CreateICmpNE(H0, H1);
        Res = IRB->CreateOr(T1, T0);
        break;
    case CmpInst::ICMP_UGT:
        T0 = IRB->CreateICmpUGT(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        T2 = IRB->CreateAnd(T1, T0);
        T3 = IRB->CreateICmpUGT(H0, H1);
        Res = IRB->CreateOr(T2, T3);
        break;
    case CmpInst::ICMP_UGE:
        T0 = IRB->CreateICmpUGE(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        T2 = IRB->CreateAnd(T1, T0);
        T3 = IRB->CreateICmpUGT(H0, H1);
        Res = IRB->CreateOr(T2, T3);
        break;
    case CmpInst::ICMP_ULT:
        T0 = IRB->CreateICmpULT(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        T2 = IRB->CreateAnd(T1, T0);
        T3 = IRB->CreateICmpULT(H0, H1);
        Res = IRB->CreateOr(T2, T3);
        break;
    case CmpInst::ICMP_ULE:
        T0 = IRB->CreateICmpULE(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        T2 = IRB->CreateAnd(T1, T0);
        T3 = IRB->CreateICmpULT(H0, H1);
        Res = IRB->CreateOr(T2, T3);
        break;
    case CmpInst::ICMP_SGT:
        T0 = IRB->CreateICmpUGT(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        T2 = IRB->CreateAnd(T1, T0);
        T3 = IRB->CreateICmpSGT(H0, H1);
        Res = IRB->CreateOr(T2, T3);
        break;
    case CmpInst::ICMP_SGE:
        T0 = IRB->CreateICmpUGE(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        T2 = IRB->CreateAnd(T1, T0);
        T3 = IRB->CreateICmpSGT(H0, H1);
        Res = IRB->CreateOr(T2, T3);
        break;
    case CmpInst::ICMP_SLT:
        T0 = IRB->CreateICmpULT(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        T2 = IRB->CreateAnd(T1, T0);
        T3 = IRB->CreateICmpSLT(H0, H1);
        Res = IRB->CreateOr(T2, T3);
        break;
    case CmpInst::ICMP_SLE:
        T0 = IRB->CreateICmpULE(L0, L1);
        T1 = IRB->CreateICmpEQ(H0, H1);
        T2 = IRB->CreateAnd(T1, T0);
        T3 = IRB->CreateICmpSLT(H0, H1);
        Res = IRB->CreateOr(T2, T3);
        break;
    default:
        IGC_ASSERT_EXIT_MESSAGE(0, "Invalid ICmp predicate");
        break;
    }

    IGC_ASSERT(nullptr != Res);

    Cmp.replaceAllUsesWith(Res);
    return true;
}

bool InstExpander::visitPHI(PHINode& PN) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&PN))
        return false;
    auto [Lo, Hi] = Emu->getExpandedValues(&PN);
    IGC_ASSERT(nullptr != Lo);
    IGC_ASSERT(nullptr != Hi);
    return false;
}

bool InstExpander::visitCall(CallInst& Call) {

    // lambdas for splitting and combining i64 to <2 x i32>
    auto Combine2xi32Toi64 = [this](Value* val)
    {
        IGC_ASSERT(nullptr != Emu);
        IGC_ASSERT(Emu->isInt64(val));
        auto [InputLo, InputHi] = Emu->getExpandedValues(val);
        Type* V2I32Ty = Emu->getV2Int32Ty();
        Value* NewVal = UndefValue::get(V2I32Ty);
        NewVal = IRB->CreateInsertElement(NewVal, InputLo, IRB->getInt32(0));
        NewVal = IRB->CreateInsertElement(NewVal, InputHi, IRB->getInt32(1));
        NewVal = IRB->CreateBitCast(NewVal, IRB->getInt64Ty());
        return NewVal;
    };
    auto Spliti64To2xi32 = [this](Value* retVal)
    {
        IGC_ASSERT(nullptr != Emu);
        IGC_ASSERT(Emu->isInt64(retVal));
        Value* V = IRB->CreateBitCast(retVal, Emu->getV2Int32Ty());
        auto Lo = IRB->CreateExtractElement(V, IRB->getInt32(0));
        auto Hi = IRB->CreateExtractElement(V, IRB->getInt32(1));
        return std::make_pair(Lo, Hi);
    };

    IGC_ASSERT(nullptr != Emu);

    const Function* F = Call.getCalledFunction();
    bool doInt64BitCall = Emu->isInt64(&Call);
    if (!doInt64BitCall) {
        for (auto& Op : Call.operands()) {
            if (Emu->isInt64(Op.get())) {
                doInt64BitCall = true;
                break;
            }
        }
    }
    if (!doInt64BitCall) {
        return false;
    }
    if (F && F->isDeclaration()) {
        Intrinsic::ID IntrID = F->getIntrinsicID();
        switch (IntrID) {
        default:
            break;
            // Ignore the following intrinsics in CG.
        case Intrinsic::assume:
        case Intrinsic::expect:
        case Intrinsic::dbg_declare:
        case Intrinsic::dbg_value:
        case Intrinsic::var_annotation:
        case Intrinsic::ptr_annotation:
        case Intrinsic::annotation:
        case Intrinsic::lifetime_start:
        case Intrinsic::lifetime_end:
        case Intrinsic::invariant_start:
        case Intrinsic::invariant_end:
            return false;
        // emulate @llvm.abs.i64
        case Intrinsic::abs:
        {
            Value* OldVal = Call.getArgOperand(0);
            auto [Lo, Hi] = Emu->getExpandedValues(OldVal);

            Value* Cmp = IRB->CreateICmpSLT(Hi, IRB->getInt32(0));

            GenISAIntrinsic::ID GIID = GenISAIntrinsic::GenISA_sub_pair;
            Function* IFunc = GenISAIntrinsic::getDeclaration(Emu->getModule(), GIID);
            Value* Sub = IRB->CreateCall4(IFunc, IRB->getInt32(0), IRB->getInt32(0), Lo, Hi);

            Value* SubLo = IRB->CreateExtractValue(Sub, 0);
            Value* SubHi = IRB->CreateExtractValue(Sub, 1);

            Value* SelectLo = IRB->CreateSelect(Cmp, SubLo, Lo);
            Value* SelectHo = IRB->CreateSelect(Cmp, SubHi, Hi);

            Emu->setExpandedValues(&Call, SelectLo, SelectHo);
            return true;
        }
        }
    }

    // Recreate Call with its operands/result emulated
    // TODO: Investigate whether we should replace the call with two
    // i32-operating calls for Lo and Hi instead (at least for certain
    // intrinsics)
    auto* CallCopy = Call.clone();
    IGC_ASSERT(nullptr != CallCopy);
    CallCopy->insertBefore(&Call);
    IRB->SetInsertPoint(CallCopy);
    for (int argNo=0, sz = (int)IGCLLVM::getNumArgOperands(&Call); argNo < sz; ++argNo)
    {
        Value* OldVal = Call.getArgOperand(argNo);
        if (Emu->isInt64(OldVal))
        {
            Value* NewVal = nullptr;

            // If this operand has been combined (same value is used more than
            // once. for example,  mul %12  %12), do not recombine them.
            for (int i = 0; i < argNo; ++i)
            {
                if (Call.getArgOperand(i) == OldVal)
                {
                    NewVal = CallCopy->getOperand(i);
                    break;
                }
            }
            if (NewVal == nullptr)
            {
                NewVal = Combine2xi32Toi64(OldVal);
            }
            CallCopy->setOperand(argNo, NewVal);
        }
    }

    // For int64 return value, split it right after CallCopy
    if (Emu->isInt64(&Call))
    {
        IRB->SetInsertPoint(&Call);
        auto [OutputLo, OutputHi] = Spliti64To2xi32(CallCopy);
        Emu->setExpandedValues(CallCopy, OutputLo, OutputHi);
    }
    Call.replaceAllUsesWith(CallCopy);
    return true;
}


bool InstExpander::visitSelect(SelectInst& SI) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&SI))
        return false;

    Value* Cond = SI.getOperand(0);
    auto [L0, H0] = Emu->getExpandedValues(SI.getOperand(1));
    auto [L1, H1] = Emu->getExpandedValues(SI.getOperand(2));

    IGC_ASSERT(nullptr != IRB);
    Value* Lo = IRB->CreateSelect(Cond, L0, L1);
    Value* Hi = IRB->CreateSelect(Cond, H0, H1);

    Emu->setExpandedValues(&SI, Lo, Hi);
    return true;
}

bool InstExpander::visitVAArg(VAArgInst& VAAI) {
    // TODO: Add i64 emulation support.
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT_MESSAGE(false == Emu->isInt64(&VAAI), "TODO: NOT IMPLEMENTED YET!");
    return false;
}

bool InstExpander::visitExtractElement(ExtractElementInst& EEI) {
    // Fix index operand if necessary.
    IGC_ASSERT(nullptr != Emu);
    if (Emu->isInt64(EEI.getIndexOperand())) {
        Value* L = nullptr;
        std::tie(L, std::ignore) = Emu->getExpandedValues(EEI.getIndexOperand());
        EEI.setOperand(1, L);
    }

    // Skip if it's not 64-bit integer.
    if (!Emu->isInt64(&EEI))
        return false;

    // NOTE: This is NOT the efficient way to handle that and needs revising
    // later.

    Value* V = EEI.getVectorOperand();
    unsigned NumElts = (unsigned)cast<IGCLLVM::FixedVectorType>(V->getType())->getNumElements();
    V = IRB->CreateBitCast(V, Emu->getV2Int32Ty(NumElts));
    // Re-calculate indices to Lo and Hi parts.
    Value* Idx = EEI.getIndexOperand();
    Idx = IRB->CreateZExt(Idx, IRB->getInt32Ty());
    Idx = IRB->CreateMul(Idx, IRB->getInt32(2));
    Value* IdxLo = IRB->CreateAdd(Idx, IRB->getInt32(0));
    Value* IdxHi = IRB->CreateAdd(Idx, IRB->getInt32(1));
    Value* Lo = IRB->CreateExtractElement(V, IdxLo);
    Value* Hi = IRB->CreateExtractElement(V, IdxHi);

    Emu->setExpandedValues(&EEI, Lo, Hi);
    return true;
}

bool InstExpander::visitInsertElement(InsertElementInst& IEI) {
    // Fix index operand if necessary.
    IGC_ASSERT(nullptr != Emu);
    if (Emu->isInt64(IEI.getOperand(2))) {
        Value* L = nullptr;
        std::tie(L, std::ignore) = Emu->getExpandedValues(IEI.getOperand(2));
        IEI.setOperand(2, L);
    }

    // Skip if it's not 64-bit integer.
    Value* V = IEI.getOperand(1);
    if (!Emu->isInt64(V))
        return false;

    // NOTE: This is NOT the efficient way to handle that and needs revising
    // later.

    auto [Lo, Hi] = Emu->getExpandedValues(V);

    // Create the emulated vector.
    Value* NewVal = IEI.getOperand(0);
    unsigned NumElts = (unsigned)cast<IGCLLVM::FixedVectorType>(NewVal->getType())->getNumElements();
    NewVal = IRB->CreateBitCast(NewVal, Emu->getV2Int32Ty(NumElts));
    // Re-calculate indices to Lo and Hi parts.
    Value* Idx = IEI.getOperand(2);
    Idx = IRB->CreateMul(Idx, IRB->getInt32(2));
    Value* IdxLo = IRB->CreateAdd(Idx, IRB->getInt32(0));
    Value* IdxHi = IRB->CreateAdd(Idx, IRB->getInt32(1));
    // Insert Lo and Hi parts into the emulated vector.
    NewVal = IRB->CreateInsertElement(NewVal, Lo, IdxLo);
    NewVal = IRB->CreateInsertElement(NewVal, Hi, IdxHi);
    NewVal = IRB->CreateBitCast(NewVal, IEI.getType());

    IEI.replaceAllUsesWith(NewVal);
    return true;
}

bool InstExpander::visitExtractValue(ExtractValueInst& EVI) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(&EVI))
        return false;
    IGC_ASSERT(EVI.getNumIndices() == 1);

    auto* EVICopy = EVI.clone();
    EVICopy->insertBefore(&EVI);
    IRB->SetInsertPoint(&EVI);
    Value* OutputLo = nullptr, * OutputHi = nullptr;
    Value* V = IRB->CreateBitCast(EVICopy, Emu->getV2Int32Ty());
    OutputLo = IRB->CreateExtractElement(V, IRB->getInt32(0));
    OutputHi = IRB->CreateExtractElement(V, IRB->getInt32(1));
    Emu->setExpandedValues(EVICopy, OutputLo, OutputHi);
    EVI.replaceAllUsesWith(EVICopy);
    return true;
}

bool InstExpander::visitInsertValue(InsertValueInst& IVI) {
    IGC_ASSERT(nullptr != Emu);
    if (!Emu->isInt64(IVI.getOperand(1)))
        return false;
    IGC_ASSERT(IVI.getNumIndices() == 1);

    auto* IVICopy = IVI.clone();
    IVICopy->insertBefore(&IVI);
    IRB->SetInsertPoint(IVICopy);
    Value* In64 = IVI.getOperand(1);
    auto [InputLo, InputHi] = Emu->getExpandedValues(In64);
    Type* V2I32Ty = Emu->getV2Int32Ty();
    Value* NewVal = UndefValue::get(V2I32Ty);
    NewVal = IRB->CreateInsertElement(NewVal, InputLo, IRB->getInt32(0));
    NewVal = IRB->CreateInsertElement(NewVal, InputHi, IRB->getInt32(1));
    NewVal = IRB->CreateBitCast(NewVal, IRB->getInt64Ty());
    IVICopy->setOperand(1, NewVal);
    IVI.replaceAllUsesWith(IVICopy);
    return true;
}

bool InstExpander::visitLandingPad(LandingPadInst& LPI) {
    // TODO: Add i64 emulation support.
    IGC_ASSERT(nullptr != Emu);
    IGC_ASSERT(1 < LPI.getNumOperands());
    IGC_ASSERT_MESSAGE(false == Emu->isInt64(LPI.getOperand(1)), "TODO: NOT IMPLEMENTED YET!");
    return false;
}

bool InstExpander::hasHWSupport(BinaryOperator& BinOp) {

    if (Emu->CGC->platform.hasPartialInt64Support())
    {
        if (Emu->eraseInstrFromInt64InstrList(&BinOp))
        {
            // No need to reconstruct the 64b operands
            // the former instruction SExt or ZExt has not been changed.

            // Just do not delete an original instruction
            // and split the result into Lo and Hi part.
        }
        else
        {
            // reconstruct the 64b operands
            for (uint operandId = 0; operandId < 2; operandId++)
            {
                auto [Lo, Hi] = Emu->getExpandedValues(BinOp.getOperand(operandId));
                Type* V2I32Ty = Emu->getV2Int32Ty();
                Value* reconstrut64bOperands = UndefValue::get(V2I32Ty);
                reconstrut64bOperands = IRB->CreateInsertElement(reconstrut64bOperands, Lo, IRB->getInt32(0));
                reconstrut64bOperands = IRB->CreateInsertElement(reconstrut64bOperands, Hi, IRB->getInt32(1));
                reconstrut64bOperands = IRB->CreateBitCast(reconstrut64bOperands, IRB->getInt64Ty());
                BinOp.setOperand(operandId, reconstrut64bOperands);
            }
        }

        // Leave, do not delete an original instruction
        BasicBlock::iterator BI = std::next(BasicBlock::iterator(&BinOp));
        IRB->SetInsertPoint(&*BI);

        // Split the result into Lo and Hi part.
        Value* V = IRB->CreateBitCast(&BinOp, Emu->getV2Int32Ty());
        Value* outputLo = IRB->CreateExtractElement(V, IRB->getInt32(0));
        Value* outputHi = IRB->CreateExtractElement(V, IRB->getInt32(1));
        Emu->setExpandedValues(&BinOp, outputLo, outputHi);

        Emu->doNotDeleteInstr = true;
        return true;
    }
    return false;
}

bool InstExpander::needsLoHiSplitting(Instruction& inst) {

    bool splittingRequired = true;
    if (Emu->CGC->platform.hasPartialInt64Support())
    {
        bool doNextStep = true;

        // Do not split into Lo and Hi parts if the result of CastInst:
        // (A) is used only by the opcode with HW native Int64 support, AND
        //      eg. %9 = shl nsw i64 %conv.i1, 1  ;  % 5 = add i64 % 4, % conv.i1
        // (B) is used by the binary instr w/ HW Int64 support and all operands come from SExt,ZExt or
        //     one is a constant value
        //      eg. %1 = shl nsw i64 %conv.i1, 3  ;  %shr.i.i = lshr i64 % cond.i9.i, % shr.mask.i.i

        for (auto* U : inst.users())
        {
            if (Instruction * _instr = dyn_cast<Instruction>(U))
            {
                if (!Emu->hasOpcodeInt64Support(_instr->getOpcode())) // (A)
                {
                    doNextStep = false;
                    break;
                }

                for (auto& Op : _instr->operands())
                {
                    if (!isa<ConstantInt>(Op.get()) &&
                        !isa<SExtInst>(Op.get()) &&
                        !isa<ZExtInst>(Op.get())) // (B)
                    {
                        doNextStep = false;
                        break;
                    }
                }
            }
        }

        if (doNextStep)
        {
            for (auto* U : inst.users())
            {
                if (Instruction * binaryInstr = dyn_cast<BinaryOperator>(U))
                {
                    Emu->addInstToInt64InstrList(binaryInstr);
                    splittingRequired = false;
                }
            }
        }
    }
    return splittingRequired;
}
