//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements hazard recognizers for scheduling on GCN processors. // //===----------------------------------------------------------------------===// #include "GCNHazardRecognizer.h" #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIMachineFunctionInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/TargetParser/TargetParser.h" using namespace llvm; namespace { struct MFMAPaddingRatioParser : public cl::parser { MFMAPaddingRatioParser(cl::Option &O) : cl::parser(O) {} bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) { if (Arg.getAsInteger(0, Value)) return O.error("'" + Arg + "' value invalid for uint argument!"); if (Value > 100) return O.error("'" + Arg + "' value must be in the range [0, 100]!"); return false; } }; } // end anonymous namespace static cl::opt MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")); // This is intended for debugging purposes only. static cl::opt NopPadding("amdgpu-snop-padding", cl::init(0), cl::Hidden, cl::desc("Insert a s_nop x before every instruction")); //===----------------------------------------------------------------------===// // Hazard Recognizer Implementation //===----------------------------------------------------------------------===// static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, const GCNSubtarget &ST); GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : IsHazardRecognizerMode(false), CurrCycleInstr(nullptr), MF(MF), ST(MF.getSubtarget()), TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) { MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST); } void GCNHazardRecognizer::Reset() { EmittedInstrs.clear(); } void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { EmitInstruction(SU->getInstr()); } void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { CurrCycleInstr = MI; } static bool isDivFMas(unsigned Opcode) { return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; } static bool isSGetReg(unsigned Opcode) { return Opcode == AMDGPU::S_GETREG_B32; } static bool isSSetReg(unsigned Opcode) { switch (Opcode) { case AMDGPU::S_SETREG_B32: case AMDGPU::S_SETREG_B32_mode: case AMDGPU::S_SETREG_IMM32_B32: case AMDGPU::S_SETREG_IMM32_B32_mode: return true; } return false; } static bool isRWLane(unsigned Opcode) { return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; } static bool isRFE(unsigned Opcode) { return Opcode == AMDGPU::S_RFE_B64; } static bool isSMovRel(unsigned Opcode) { switch (Opcode) { case AMDGPU::S_MOVRELS_B32: case AMDGPU::S_MOVRELS_B64: case AMDGPU::S_MOVRELD_B32: case AMDGPU::S_MOVRELD_B64: return true; default: return false; } } static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI) { if (TII.isAlwaysGDS(MI.getOpcode())) return true; switch (MI.getOpcode()) { case AMDGPU::S_SENDMSG: case AMDGPU::S_SENDMSGHALT: case AMDGPU::S_TTRACEDATA: return true; // These DS opcodes don't support GDS. case AMDGPU::DS_NOP: case AMDGPU::DS_PERMUTE_B32: case AMDGPU::DS_BPERMUTE_B32: return false; default: if (TII.isDS(MI.getOpcode())) { int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::gds); if (MI.getOperand(GDS).getImm()) return true; } return false; } } static bool isPermlane(const MachineInstr &MI) { unsigned Opcode = MI.getOpcode(); return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || Opcode == AMDGPU::V_PERMLANE64_B32 || Opcode == AMDGPU::V_PERMLANEX16_B32_e64 || Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 || Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 || Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 || Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 || Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 || Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64; } static bool isLdsDma(const MachineInstr &MI) { return SIInstrInfo::isVALU(MI) && (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)); } static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) { const MachineOperand *RegOp = TII->getNamedOperand(RegInstr, AMDGPU::OpName::simm16); return std::get<0>(AMDGPU::Hwreg::HwregEncoding::decode(RegOp->getImm())); } ScheduleHazardRecognizer::HazardType GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { MachineInstr *MI = SU->getInstr(); // If we are not in "HazardRecognizerMode" and therefore not being run from // the scheduler, track possible stalls from hazards but don't insert noops. auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard; if (MI->isBundle()) return NoHazard; if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) return HazardType; if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) return HazardType; if (checkFPAtomicToDenormModeHazard(MI) > 0) return HazardType; if (ST.hasNoDataDepHazard()) return NoHazard; if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0) return HazardType; if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) return HazardType; if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) return HazardType; if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) return HazardType; if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) return HazardType; if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) return HazardType; if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) return HazardType; if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) return HazardType; if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) return HazardType; if (((ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) || MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 || MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) || (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) && checkReadM0Hazards(MI) > 0) return HazardType; if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) return HazardType; if ((SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) return HazardType; if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) return HazardType; return NoHazard; } static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity) { while (Quantity > 0) { unsigned Arg = std::min(Quantity, 8u); Quantity -= Arg; BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) .addImm(Arg - 1); } } unsigned GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const { const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI); assert(TSchedModel.getWriteProcResBegin(SC) != TSchedModel.getWriteProcResEnd(SC)); return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle; } void GCNHazardRecognizer::processBundle() { MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end(); // Check bundled MachineInstr's for hazards. for (; MI != E && MI->isInsideBundle(); ++MI) { CurrCycleInstr = &*MI; unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr); if (IsHazardRecognizerMode) { fixHazards(CurrCycleInstr); insertNoopsInBundle(CurrCycleInstr, TII, WaitStates); } // It’s unnecessary to track more than MaxLookAhead instructions. Since we // include the bundled MI directly after, only add a maximum of // (MaxLookAhead - 1) noops to EmittedInstrs. for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i) EmittedInstrs.push_front(nullptr); EmittedInstrs.push_front(CurrCycleInstr); EmittedInstrs.resize(MaxLookAhead); } CurrCycleInstr = nullptr; } void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) { assert(IsHazardRecognizerMode); unsigned NumPreNoops = PreEmitNoops(MI); EmitNoops(NumPreNoops); if (MI->isInsideBundle()) insertNoopsInBundle(MI, TII, NumPreNoops); else TII.insertNoops(*MI->getParent(), MachineBasicBlock::iterator(MI), NumPreNoops); EmitInstruction(MI); AdvanceCycle(); } unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { IsHazardRecognizerMode = true; CurrCycleInstr = MI; unsigned W = PreEmitNoopsCommon(MI); fixHazards(MI); CurrCycleInstr = nullptr; return std::max(W, NopPadding.getValue()); } unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { if (MI->isBundle()) return 0; int WaitStates = 0; if (SIInstrInfo::isSMRD(*MI)) return std::max(WaitStates, checkSMRDHazards(MI)); if (ST.hasNSAtoVMEMBug()) WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); if (ST.hasNoDataDepHazard()) return WaitStates; if (SIInstrInfo::isVMEM(*MI)) WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); if (SIInstrInfo::isVALU(*MI)) WaitStates = std::max(WaitStates, checkVALUHazards(MI)); if (SIInstrInfo::isDPP(*MI)) WaitStates = std::max(WaitStates, checkDPPHazards(MI)); if (isDivFMas(MI->getOpcode())) WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); if (isRWLane(MI->getOpcode())) WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); if (MI->isInlineAsm()) return std::max(WaitStates, checkInlineAsmHazards(MI)); if (isSGetReg(MI->getOpcode())) return std::max(WaitStates, checkGetRegHazards(MI)); if (isSSetReg(MI->getOpcode())) return std::max(WaitStates, checkSetRegHazards(MI)); if (isRFE(MI->getOpcode())) return std::max(WaitStates, checkRFEHazards(MI)); if ((ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) || MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 || MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) || (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) return std::max(WaitStates, checkReadM0Hazards(MI)); if (SIInstrInfo::isMAI(*MI)) return std::max(WaitStates, checkMAIHazards(MI)); if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI)) return std::max(WaitStates, checkMAILdStHazards(MI)); if (ST.hasGFX950Insts() && isPermlane(*MI)) return std::max(WaitStates, checkPermlaneHazards(MI)); return WaitStates; } void GCNHazardRecognizer::EmitNoop() { EmittedInstrs.push_front(nullptr); } void GCNHazardRecognizer::AdvanceCycle() { // When the scheduler detects a stall, it will call AdvanceCycle() without // emitting any instructions. if (!CurrCycleInstr) { EmittedInstrs.push_front(nullptr); return; } if (CurrCycleInstr->isBundle()) { processBundle(); return; } unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr); if (!NumWaitStates) { CurrCycleInstr = nullptr; return; } // Keep track of emitted instructions EmittedInstrs.push_front(CurrCycleInstr); // Add a nullptr for each additional wait state after the first. Make sure // not to add more than getMaxLookAhead() items to the list, since we // truncate the list to that size right after this loop. for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead()); i < e; ++i) { EmittedInstrs.push_front(nullptr); } // getMaxLookahead() is the largest number of wait states we will ever need // to insert, so there is no point in keeping track of more than that many // wait states. EmittedInstrs.resize(getMaxLookAhead()); CurrCycleInstr = nullptr; } void GCNHazardRecognizer::RecedeCycle() { assert(!IsHazardRecognizerMode && "Bottom-up scheduling shouldn't run in hazard recognizer mode"); } //===----------------------------------------------------------------------===// // Helper Functions //===----------------------------------------------------------------------===// using HazardFnResult = enum { HazardFound, HazardExpired, NoHazardFound }; using IsExpiredFn = function_ref; using GetNumWaitStatesFn = function_ref; // Search for a hazard in a block and its predecessors. template static bool hasHazard(StateT State, function_ref IsHazard, function_ref UpdateState, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, DenseSet &Visited) { for (auto E = MBB->instr_rend(); I != E; ++I) { // No need to look at parent BUNDLE instructions. if (I->isBundle()) continue; switch (IsHazard(State, *I)) { case HazardFound: return true; case HazardExpired: return false; default: // Continue search break; } if (I->isInlineAsm() || I->isMetaInstruction()) continue; UpdateState(State, *I); } for (MachineBasicBlock *Pred : MBB->predecessors()) { if (!Visited.insert(Pred).second) continue; if (hasHazard(State, IsHazard, UpdateState, Pred, Pred->instr_rbegin(), Visited)) return true; } return false; } // Returns a minimum wait states since \p I walking all predecessors. // Only scans until \p IsExpired does not return true. // Can only be run in a hazard recognizer mode. static int getWaitStatesSince( GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet &Visited, GetNumWaitStatesFn GetNumWaitStates = SIInstrInfo::getNumWaitStates) { for (auto E = MBB->instr_rend(); I != E; ++I) { // Don't add WaitStates for parent BUNDLE instructions. if (I->isBundle()) continue; if (IsHazard(*I)) return WaitStates; if (I->isInlineAsm()) continue; WaitStates += GetNumWaitStates(*I); if (IsExpired(*I, WaitStates)) return std::numeric_limits::max(); } int MinWaitStates = std::numeric_limits::max(); for (MachineBasicBlock *Pred : MBB->predecessors()) { if (!Visited.insert(Pred).second) continue; int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), WaitStates, IsExpired, Visited, GetNumWaitStates); MinWaitStates = std::min(MinWaitStates, W); } return MinWaitStates; } static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, const MachineInstr *MI, IsExpiredFn IsExpired) { DenseSet Visited; return getWaitStatesSince(IsHazard, MI->getParent(), std::next(MI->getReverseIterator()), 0, IsExpired, Visited); } int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) { if (IsHazardRecognizerMode) { auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) { return WaitStates >= Limit; }; return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn); } int WaitStates = 0; for (MachineInstr *MI : EmittedInstrs) { if (MI) { if (IsHazard(*MI)) return WaitStates; if (MI->isInlineAsm()) continue; } ++WaitStates; if (WaitStates >= Limit) break; } return std::numeric_limits::max(); } int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit) { const SIRegisterInfo *TRI = ST.getRegisterInfo(); auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) { return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI); }; return getWaitStatesSince(IsHazardFn, Limit); } int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit) { auto IsHazardFn = [IsHazard](const MachineInstr &MI) { return isSSetReg(MI.getOpcode()) && IsHazard(MI); }; return getWaitStatesSince(IsHazardFn, Limit); } //===----------------------------------------------------------------------===// // No-op Hazard Detection //===----------------------------------------------------------------------===// static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg) { for (MCRegUnit Unit : TRI.regunits(Reg)) BV.set(Unit); } static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range Ops, BitVector &DefSet, BitVector &UseSet) { for (const MachineOperand &Op : Ops) { if (Op.isReg()) addRegUnits(TRI, Op.isDef() ? DefSet : UseSet, Op.getReg().asMCReg()); } } void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { addRegsToSet(TRI, MI.operands(), ClauseDefs, ClauseUses); } static bool breaksSMEMSoftClause(MachineInstr *MI) { return !SIInstrInfo::isSMRD(*MI); } static bool breaksVMEMSoftClause(MachineInstr *MI) { return !SIInstrInfo::isVMEM(*MI); } int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { // SMEM soft clause are only present on VI+, and only matter if xnack is // enabled. if (!ST.isXNACKEnabled()) return 0; bool IsSMRD = TII.isSMRD(*MEM); resetClause(); // A soft-clause is any group of consecutive SMEM instructions. The // instructions in this group may return out of order and/or may be // replayed (i.e. the same instruction issued more than once). // // In order to handle these situations correctly we need to make sure that // when a clause has more than one instruction, no instruction in the clause // writes to a register that is read by another instruction in the clause // (including itself). If we encounter this situation, we need to break the // clause by inserting a non SMEM instruction. for (MachineInstr *MI : EmittedInstrs) { // When we hit a non-SMEM instruction then we have passed the start of the // clause and we can stop. if (!MI) break; if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) break; addClauseInst(*MI); } if (ClauseDefs.none()) return 0; // We need to make sure not to put loads and stores in the same clause if they // use the same address. For now, just start a new clause whenever we see a // store. if (MEM->mayStore()) return 1; addClauseInst(*MEM); // If the set of defs and uses intersect then we cannot add this instruction // to the clause, so we have a hazard. return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0; } int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { int WaitStatesNeeded = 0; WaitStatesNeeded = checkSoftClauseHazards(SMRD); // This SMRD hazard only affects SI. if (!ST.hasSMRDReadVALUDefHazard()) return WaitStatesNeeded; // A read of an SGPR by SMRD instruction requires 4 wait states when the // SGPR was written by a VALU instruction. int SmrdSgprWaitStates = 4; auto IsHazardDefFn = [this](const MachineInstr &MI) { return TII.isVALU(MI); }; auto IsBufferHazardDefFn = [this](const MachineInstr &MI) { return TII.isSALU(MI); }; bool IsBufferSMRD = TII.isBufferSMRD(*SMRD); for (const MachineOperand &Use : SMRD->uses()) { if (!Use.isReg()) continue; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, SmrdSgprWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); // This fixes what appears to be undocumented hardware behavior in SI where // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor // needs some number of nops in between. We don't know how many we need, but // let's use 4. This wasn't discovered before probably because the only // case when this happens is when we expand a 64-bit pointer into a full // descriptor and use s_buffer_load_dword instead of s_load_dword, which was // probably never encountered in the closed-source land. if (IsBufferSMRD) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsBufferHazardDefFn, SmrdSgprWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } } return WaitStatesNeeded; } int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { if (!ST.hasVMEMReadSGPRVALUDefHazard()) return 0; int WaitStatesNeeded = checkSoftClauseHazards(VMEM); // A read of an SGPR by a VMEM instruction requires 5 wait states when the // SGPR was written by a VALU Instruction. const int VmemSgprWaitStates = 5; auto IsHazardDefFn = [this](const MachineInstr &MI) { return TII.isVALU(MI); }; for (const MachineOperand &Use : VMEM->uses()) { if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg())) continue; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, VmemSgprWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } return WaitStatesNeeded; } int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIInstrInfo *TII = ST.getInstrInfo(); // Check for DPP VGPR read after VALU VGPR write and EXEC write. int DppVgprWaitStates = 2; int DppExecWaitStates = 5; int WaitStatesNeeded = 0; auto IsHazardDefFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; for (const MachineOperand &Use : DPP->uses()) { if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg())) continue; int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef( Use.getReg(), [](const MachineInstr &) { return true; }, DppVgprWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } WaitStatesNeeded = std::max( WaitStatesNeeded, DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, DppExecWaitStates)); return WaitStatesNeeded; } int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) { const SIInstrInfo *TII = ST.getInstrInfo(); // v_div_fmas requires 4 wait states after a write to vcc from a VALU // instruction. const int DivFMasWaitStates = 4; auto IsHazardDefFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, DivFMasWaitStates); return DivFMasWaitStates - WaitStatesNeeded; } int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) { const SIInstrInfo *TII = ST.getInstrInfo(); unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr); const int GetRegWaitStates = 2; auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) { return GetRegHWReg == getHWReg(TII, MI); }; int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates); return GetRegWaitStates - WaitStatesNeeded; } int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { const SIInstrInfo *TII = ST.getInstrInfo(); unsigned HWReg = getHWReg(TII, *SetRegInstr); const int SetRegWaitStates = ST.getSetRegWaitStates(); auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) { return HWReg == getHWReg(TII, MI); }; int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates); return SetRegWaitStates - WaitStatesNeeded; } int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { if (!MI.mayStore()) return -1; const SIInstrInfo *TII = ST.getInstrInfo(); unsigned Opcode = MI.getOpcode(); const MCInstrDesc &Desc = MI.getDesc(); int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); int VDataRCID = -1; if (VDataIdx != -1) VDataRCID = Desc.operands()[VDataIdx].RegClass; if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { // There is no hazard if the instruction does not use vector regs // (like wbinvl1) if (VDataIdx == -1) return -1; // For MUBUF/MTBUF instructions this hazard only exists if the // instruction is not using a register in the soffset field. const MachineOperand *SOffset = TII->getNamedOperand(MI, AMDGPU::OpName::soffset); // If we have no soffset operand, then assume this field has been // hardcoded to zero. if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && (!SOffset || !SOffset->isReg())) return VDataIdx; } // MIMG instructions create a hazard if they don't use a 256-bit T# and // the store size is greater than 8 bytes and they have more than two bits // of their dmask set. // All our MIMG definitions use a 256-bit T#, so we can skip checking for them. if (TII->isMIMG(MI)) { int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); assert(SRsrcIdx != -1 && AMDGPU::getRegBitWidth(Desc.operands()[SRsrcIdx].RegClass) == 256); (void)SRsrcIdx; } if (TII->isFLAT(MI)) { // There is no hazard if the instruction does not use vector regs if (VDataIdx == -1) return -1; if (AMDGPU::getRegBitWidth(VDataRCID) > 64) return VDataIdx; } return -1; } int GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI) { // Helper to check for the hazard where VMEM instructions that store more than // 8 bytes can have there store data over written by the next instruction. const SIRegisterInfo *TRI = ST.getRegisterInfo(); const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1; int WaitStatesNeeded = 0; if (!TRI->isVectorRegister(MRI, Def.getReg())) return WaitStatesNeeded; Register Reg = Def.getReg(); auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) { int DataIdx = createsVALUHazard(MI); return DataIdx >= 0 && TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg); }; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); return WaitStatesNeeded; } /// Dest sel forwarding issue occurs if additional logic is needed to swizzle / /// pack the computed value into correct bit position of the dest register. This /// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with /// dst_sel that is not aligned to the register. This function analayzes the \p /// MI and \returns an operand with dst forwarding issue, or nullptr if /// none exists. static const MachineOperand * getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) { if (!SIInstrInfo::isVALU(MI)) return nullptr; const SIInstrInfo *TII = ST.getInstrInfo(); unsigned Opcode = MI.getOpcode(); // There are three different types of instructions // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3 // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and // op_sel[3:2] // != 0 if (SIInstrInfo::isSDWA(MI)) { // Type 1: SDWA with dst_sel != DWORD if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel)) if (DstSel->getImm() != AMDGPU::SDWA::DWORD) return TII->getNamedOperand(MI, AMDGPU::OpName::vdst); } AMDGPU::FPType IsFP4OrFP8ConvOpc = AMDGPU::getFPDstSelType(Opcode); if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel)) { // Type 2: VOP3 which write the hi bits if (TII->getNamedImmOperand(MI, AMDGPU::OpName::src0_modifiers) & SISrcMods::DST_OP_SEL) return TII->getNamedOperand(MI, AMDGPU::OpName::vdst); // Type 3: FP8DstSelInst with op_sel[3:2] != 0) if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP8 && (TII->getNamedImmOperand(MI, AMDGPU::OpName::src2_modifiers) & SISrcMods::OP_SEL_0)) return TII->getNamedOperand(MI, AMDGPU::OpName::vdst); } // Special case: nop is required for all the opsel values for fp4 sr variant // cvt scale instructions if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP4) return TII->getNamedOperand(MI, AMDGPU::OpName::vdst); return nullptr; } /// Checks whether the provided \p MI "consumes" the operand with a Dest sel /// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit /// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW) static bool consumesDstSelForwardingOperand(const MachineInstr *VALU, const MachineOperand *Dst, const SIRegisterInfo *TRI) { // We must consider implicit reads of the VALU. SDWA with dst_sel and // UNUSED_PRESERVE will implicitly read the result from forwarded dest, // and we must account for that hazard. // We also must account for WAW hazards. In particular, WAW with dest // preserve semantics (e.g. VOP3 with op_sel, VOP2 && // !zeroesHigh16BitsOfDest) will read the forwarded dest for parity // check for ECC. Without accounting for this hazard, the ECC will be // wrong. // TODO: limit to RAW (including implicit reads) + problematic WAW (i.e. // complete zeroesHigh16BitsOfDest) for (auto &Operand : VALU->operands()) { if (Operand.isReg() && TRI->regsOverlap(Dst->getReg(), Operand.getReg())) { return true; } } return false; } int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) { int WaitStatesNeeded = 0; if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) { const int TransDefWaitstates = 1; auto IsTransDefFn = [this, VALU](const MachineInstr &MI) { if (!SIInstrInfo::isTRANS(MI)) return false; const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIInstrInfo *TII = ST.getInstrInfo(); Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg(); for (const MachineOperand &Use : VALU->explicit_uses()) { if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg())) return true; } return false; }; int WaitStatesNeededForDef = TransDefWaitstates - getWaitStatesSince(IsTransDefFn, TransDefWaitstates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); } if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) { const int Shift16DefWaitstates = 1; auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) { const SIRegisterInfo *TRI = ST.getRegisterInfo(); const MachineOperand *ForwardedDst = getDstSelForwardingOperand(ProducerMI, ST); if (ForwardedDst) { return consumesDstSelForwardingOperand(VALU, ForwardedDst, TRI); } if (ProducerMI.isInlineAsm()) { // Assume inline asm has dst forwarding hazard for (auto &Def : ProducerMI.all_defs()) { if (consumesDstSelForwardingOperand(VALU, &Def, TRI)) return true; } } return false; }; int WaitStatesNeededForDef = Shift16DefWaitstates - getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); } if (ST.hasVDecCoExecHazard()) { const int VALUWriteSGPRVALUReadWaitstates = 2; const int VALUWriteEXECRWLane = 4; const int VALUWriteVGPRReadlaneRead = 1; const SIRegisterInfo *TRI = ST.getRegisterInfo(); const MachineRegisterInfo &MRI = MF.getRegInfo(); Register UseReg; auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) { if (!SIInstrInfo::isVALU(MI)) return false; return MI.modifiesRegister(UseReg, TRI); }; for (const MachineOperand &Use : VALU->explicit_uses()) { if (!Use.isReg()) continue; UseReg = Use.getReg(); if (TRI->isSGPRReg(MRI, UseReg)) { int WaitStatesNeededForDef = VALUWriteSGPRVALUReadWaitstates - getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); } } if (VALU->readsRegister(AMDGPU::VCC, TRI)) { UseReg = AMDGPU::VCC; int WaitStatesNeededForDef = VALUWriteSGPRVALUReadWaitstates - getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); } switch (VALU->getOpcode()) { case AMDGPU::V_READLANE_B32: case AMDGPU::V_READFIRSTLANE_B32: { MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0); UseReg = Src->getReg(); int WaitStatesNeededForDef = VALUWriteVGPRReadlaneRead - getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); } [[fallthrough]]; case AMDGPU::V_WRITELANE_B32: { UseReg = AMDGPU::EXEC; int WaitStatesNeededForDef = VALUWriteEXECRWLane - getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); break; } default: break; } } // This checks for the hazard where VMEM instructions that store more than // 8 bytes can have there store data over written by the next instruction. if (!ST.has12DWordStoreHazard()) return WaitStatesNeeded; const MachineRegisterInfo &MRI = MF.getRegInfo(); for (const MachineOperand &Def : VALU->defs()) { WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI)); } return WaitStatesNeeded; } int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) { // This checks for hazards associated with inline asm statements. // Since inline asms can contain just about anything, we use this // to call/leverage other check*Hazard routines. Note that // this function doesn't attempt to address all possible inline asm // hazards (good luck), but is a collection of what has been // problematic thus far. // see checkVALUHazards() if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() && !ST.hasCvtScaleForwardingHazard()) return 0; const MachineRegisterInfo &MRI = MF.getRegInfo(); int WaitStatesNeeded = 0; for (const MachineOperand &Op : llvm::drop_begin(IA->operands(), InlineAsm::MIOp_FirstOperand)) { if (Op.isReg() && Op.isDef()) { if (!TRI.isVectorRegister(MRI, Op.getReg())) continue; if (ST.has12DWordStoreHazard()) { WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI)); } } } if (ST.hasDstSelForwardingHazard()) { const int Shift16DefWaitstates = 1; auto IsShift16BitDefFn = [this, &IA](const MachineInstr &ProducerMI) { const MachineOperand *Dst = getDstSelForwardingOperand(ProducerMI, ST); // Assume inline asm reads the dst if (Dst) return IA->modifiesRegister(Dst->getReg(), &TRI) || IA->readsRegister(Dst->getReg(), &TRI); if (ProducerMI.isInlineAsm()) { // If MI is inline asm, assume it has dst forwarding hazard for (auto &Def : ProducerMI.all_defs()) { if (IA->modifiesRegister(Def.getReg(), &TRI) || IA->readsRegister(Def.getReg(), &TRI)) { return true; } } } return false; }; int WaitStatesNeededForDef = Shift16DefWaitstates - getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); } return WaitStatesNeeded; } int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) { const SIInstrInfo *TII = ST.getInstrInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo(); const MachineRegisterInfo &MRI = MF.getRegInfo(); const MachineOperand *LaneSelectOp = TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg())) return 0; Register LaneSelectReg = LaneSelectOp->getReg(); auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; const int RWLaneWaitStates = 4; int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn, RWLaneWaitStates); return RWLaneWaitStates - WaitStatesSince; } int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { if (!ST.hasRFEHazards()) return 0; const SIInstrInfo *TII = ST.getInstrInfo(); const int RFEWaitStates = 1; auto IsHazardFn = [TII](const MachineInstr &MI) { return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; }; int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates); return RFEWaitStates - WaitStatesNeeded; } int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { const SIInstrInfo *TII = ST.getInstrInfo(); const int ReadM0WaitStates = 1; auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); }; return ReadM0WaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates); } void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { fixVMEMtoScalarWriteHazards(MI); fixVcmpxPermlaneHazards(MI); fixSMEMtoVectorWriteHazards(MI); fixVcmpxExecWARHazard(MI); fixLdsBranchVmemWARHazard(MI); if (ST.hasLdsDirect()) { fixLdsDirectVALUHazard(MI); fixLdsDirectVMEMHazard(MI); } fixVALUPartialForwardingHazard(MI); fixVALUTransUseHazard(MI); fixWMMAHazards(MI); fixShift64HighRegBug(MI); fixVALUMaskWriteHazard(MI); fixRequiredExportPriority(MI); } static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI, const MachineInstr &MI) { return (TII.isVOPC(MI) || (MI.isCompare() && (TII.isVOP3(MI) || TII.isSDWA(MI)))) && MI.modifiesRegister(AMDGPU::EXEC, &TRI); } bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) return false; const SIInstrInfo *TII = ST.getInstrInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo(); auto IsHazardFn = [TII, TRI](const MachineInstr &MI) { return isVCmpXWritesExec(*TII, *TRI, MI); }; auto IsExpiredFn = [](const MachineInstr &MI, int) { unsigned Opc = MI.getOpcode(); return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa; }; if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == std::numeric_limits::max()) return false; // V_NOP will be discarded by SQ. // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE* // which is always a VGPR and available. auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); Register Reg = Src0->getReg(); bool IsUndef = Src0->isUndef(); BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_MOV_B32_e32)) .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0)) .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill); return true; } bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { if (!ST.hasVMEMtoScalarWriteHazard()) return false; assert(!ST.hasExtendedWaitCounts()); if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) return false; if (MI->getNumDefs() == 0) return false; const SIRegisterInfo *TRI = ST.getRegisterInfo(); auto IsHazardFn = [TRI, MI](const MachineInstr &I) { if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I)) return false; for (const MachineOperand &Def : MI->defs()) { const MachineOperand *Op = I.findRegisterUseOperand(Def.getReg(), TRI, false); if (!Op) continue; return true; } return false; }; auto IsExpiredFn = [](const MachineInstr &MI, int) { return SIInstrInfo::isVALU(MI) || (MI.getOpcode() == AMDGPU::S_WAITCNT && !MI.getOperand(0).getImm()) || (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && AMDGPU::DepCtr::decodeFieldVmVsrc(MI.getOperand(0).getImm()) == 0); }; if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == std::numeric_limits::max()) return false; const SIInstrInfo *TII = ST.getInstrInfo(); BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::S_WAITCNT_DEPCTR)) .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0)); return true; } bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { if (!ST.hasSMEMtoVectorWriteHazard()) return false; assert(!ST.hasExtendedWaitCounts()); if (!SIInstrInfo::isVALU(*MI)) return false; AMDGPU::OpName SDSTName; switch (MI->getOpcode()) { case AMDGPU::V_READLANE_B32: case AMDGPU::V_READFIRSTLANE_B32: SDSTName = AMDGPU::OpName::vdst; break; default: SDSTName = AMDGPU::OpName::sdst; break; } const SIInstrInfo *TII = ST.getInstrInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo(); const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); if (!SDST) { for (const auto &MO : MI->implicit_operands()) { if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg()))) { SDST = &MO; break; } } } if (!SDST) return false; const Register SDSTReg = SDST->getReg(); auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) { return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI); }; auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) { if (TII->isSALU(MI)) { switch (MI.getOpcode()) { case AMDGPU::S_SETVSKIP: case AMDGPU::S_VERSION: case AMDGPU::S_WAITCNT_VSCNT: case AMDGPU::S_WAITCNT_VMCNT: case AMDGPU::S_WAITCNT_EXPCNT: // These instructions cannot not mitigate the hazard. return false; case AMDGPU::S_WAITCNT_LGKMCNT: // Reducing lgkmcnt count to 0 always mitigates the hazard. return (MI.getOperand(1).getImm() == 0) && (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); case AMDGPU::S_WAITCNT: { const int64_t Imm = MI.getOperand(0).getImm(); AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); // DsCnt corresponds to LGKMCnt here. return (Decoded.DsCnt == 0); } default: // SOPP instructions cannot mitigate the hazard. if (TII->isSOPP(MI)) return false; // At this point the SALU can be assumed to mitigate the hazard // because either: // (a) it is independent of the at risk SMEM (breaking chain), // or // (b) it is dependent on the SMEM, in which case an appropriate // s_waitcnt lgkmcnt _must_ exist between it and the at risk // SMEM instruction. return true; } } return false; }; if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == std::numeric_limits::max()) return false; BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) .addImm(0); return true; } bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { if (!ST.hasVcmpxExecWARHazard()) return false; assert(!ST.hasExtendedWaitCounts()); if (!SIInstrInfo::isVALU(*MI)) return false; const SIRegisterInfo *TRI = ST.getRegisterInfo(); if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) return false; auto IsHazardFn = [TRI](const MachineInstr &I) { if (SIInstrInfo::isVALU(I)) return false; return I.readsRegister(AMDGPU::EXEC, TRI); }; const SIInstrInfo *TII = ST.getInstrInfo(); auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) { if (SIInstrInfo::isVALU(MI)) { if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) return true; for (auto MO : MI.implicit_operands()) if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg()))) return true; } if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && AMDGPU::DepCtr::decodeFieldSaSdst(MI.getOperand(0).getImm()) == 0) return true; return false; }; if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == std::numeric_limits::max()) return false; BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::S_WAITCNT_DEPCTR)) .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0)); return true; } static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, const GCNSubtarget &ST) { if (!ST.hasLdsBranchVmemWARHazard()) return false; // Check if the necessary condition for the hazard is met: both LDS and VMEM // instructions need to appear in the same function. bool HasLds = false; bool HasVmem = false; for (auto &MBB : MF) { for (auto &MI : MBB) { HasLds |= SIInstrInfo::isDS(MI); HasVmem |= (SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) || SIInstrInfo::isSegmentSpecificFLAT(MI); if (HasLds && HasVmem) return true; } } return false; } static bool isStoreCountWaitZero(const MachineInstr &I) { return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && !I.getOperand(1).getImm(); } bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { if (!RunLdsBranchVmemWARHazardFixup) return false; assert(ST.hasLdsBranchVmemWARHazard()); assert(!ST.hasExtendedWaitCounts()); auto IsHazardInst = [](const MachineInstr &MI) { if (SIInstrInfo::isDS(MI)) return 1; if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) || SIInstrInfo::isSegmentSpecificFLAT(MI)) return 2; return 0; }; auto InstType = IsHazardInst(*MI); if (!InstType) return false; auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) { return IsHazardInst(I) || isStoreCountWaitZero(I); }; auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) { if (!I.isBranch()) return false; auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) { auto InstType2 = IsHazardInst(I); return InstType2 && InstType != InstType2; }; auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) { auto InstType2 = IsHazardInst(I); if (InstType == InstType2) return true; return isStoreCountWaitZero(I); }; return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) != std::numeric_limits::max(); }; if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == std::numeric_limits::max()) return false; const SIInstrInfo *TII = ST.getInstrInfo(); BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::S_WAITCNT_VSCNT)) .addReg(AMDGPU::SGPR_NULL, RegState::Undef) .addImm(0); return true; } bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) { if (!SIInstrInfo::isLDSDIR(*MI)) return false; const int NoHazardWaitStates = 15; const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); const Register VDSTReg = VDST->getReg(); bool VisitedTrans = false; auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) { if (!SIInstrInfo::isVALU(I)) return false; VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I); // Cover both WAR and WAW return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI); }; auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) { if (WaitStates >= NoHazardWaitStates) return true; // Instructions which cause va_vdst==0 expire hazard return SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I); }; auto GetWaitStatesFn = [](const MachineInstr &MI) { return SIInstrInfo::isVALU(MI) ? 1 : 0; }; DenseSet Visited; auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(), std::next(MI->getReverseIterator()), 0, IsExpiredFn, Visited, GetWaitStatesFn); // Transcendentals can execute in parallel to other VALUs. // This makes va_vdst count unusable with a mixture of VALU and TRANS. if (VisitedTrans) Count = 0; MachineOperand *WaitVdstOp = TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst); WaitVdstOp->setImm(std::min(Count, NoHazardWaitStates)); return true; } bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) { if (!SIInstrInfo::isLDSDIR(*MI)) return false; const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); const Register VDSTReg = VDST->getReg(); auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) { if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I)) return false; return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI); }; bool LdsdirCanWait = ST.hasLdsWaitVMSRC(); // TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT // according to the type of VMEM instruction. auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) { return SIInstrInfo::isVALU(I) || SIInstrInfo::isEXP(I) || (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) || (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && AMDGPU::DepCtr::decodeFieldVmVsrc(I.getOperand(0).getImm()) == 0) || (LdsdirCanWait && SIInstrInfo::isLDSDIR(I) && !TII.getNamedOperand(I, AMDGPU::OpName::waitvsrc)->getImm()); }; if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == std::numeric_limits::max()) return false; if (LdsdirCanWait) { TII.getNamedOperand(*MI, AMDGPU::OpName::waitvsrc)->setImm(0); } else { BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_WAITCNT_DEPCTR)) .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0)); } return true; } bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) { if (!ST.hasVALUPartialForwardingHazard()) return false; assert(!ST.hasExtendedWaitCounts()); if (!ST.isWave64() || !SIInstrInfo::isVALU(*MI)) return false; SmallSetVector SrcVGPRs; for (const MachineOperand &Use : MI->explicit_uses()) { if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg())) SrcVGPRs.insert(Use.getReg()); } // Only applies with >= 2 unique VGPR sources if (SrcVGPRs.size() <= 1) return false; // Look for the following pattern: // Va <- VALU [PreExecPos] // intv1 // Exec <- SALU [ExecPos] // intv2 // Vb <- VALU [PostExecPos] // intv3 // MI Va, Vb (WaitState = 0) // // Where: // intv1 + intv2 <= 2 VALUs // intv3 <= 4 VALUs // // If found, insert an appropriate S_WAITCNT_DEPCTR before MI. const int Intv1plus2MaxVALUs = 2; const int Intv3MaxVALUs = 4; const int IntvMaxVALUs = 6; const int NoHazardVALUWaitStates = IntvMaxVALUs + 2; struct StateType { SmallDenseMap DefPos; int ExecPos = std::numeric_limits::max(); int VALUs = 0; }; StateType State; // This overloads expiry testing with all the hazard detection auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) { // Too many VALU states have passed if (State.VALUs > NoHazardVALUWaitStates) return HazardExpired; // Instructions which cause va_vdst==0 expire hazard if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && AMDGPU::DepCtr::decodeFieldVaVdst(I.getOperand(0).getImm()) == 0)) return HazardExpired; // Track registers writes bool Changed = false; if (SIInstrInfo::isVALU(I)) { for (Register Src : SrcVGPRs) { if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) { State.DefPos[Src] = State.VALUs; Changed = true; } } } else if (SIInstrInfo::isSALU(I)) { if (State.ExecPos == std::numeric_limits::max()) { if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) { State.ExecPos = State.VALUs; Changed = true; } } } // Early expiration: too many VALUs in intv3 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty()) return HazardExpired; // Only evaluate state if something changed if (!Changed) return NoHazardFound; // Determine positions of VALUs pre/post exec change if (State.ExecPos == std::numeric_limits::max()) return NoHazardFound; int PreExecPos = std::numeric_limits::max(); int PostExecPos = std::numeric_limits::max(); for (auto Entry : State.DefPos) { int DefVALUs = Entry.second; if (DefVALUs != std::numeric_limits::max()) { if (DefVALUs >= State.ExecPos) PreExecPos = std::min(PreExecPos, DefVALUs); else PostExecPos = std::min(PostExecPos, DefVALUs); } } // Need a VALUs post exec change if (PostExecPos == std::numeric_limits::max()) return NoHazardFound; // Too many VALUs in intv3? int Intv3VALUs = PostExecPos; if (Intv3VALUs > Intv3MaxVALUs) return HazardExpired; // Too many VALUs in intv2? int Intv2VALUs = (State.ExecPos - PostExecPos) - 1; if (Intv2VALUs > Intv1plus2MaxVALUs) return HazardExpired; // Need a VALUs pre exec change if (PreExecPos == std::numeric_limits::max()) return NoHazardFound; // Too many VALUs in intv1? int Intv1VALUs = PreExecPos - State.ExecPos; if (Intv1VALUs > Intv1plus2MaxVALUs) return HazardExpired; // Too many VALUs in intv1 + intv2 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs) return HazardExpired; return HazardFound; }; auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { if (SIInstrInfo::isVALU(MI)) State.VALUs += 1; }; DenseSet Visited; if (!hasHazard(State, IsHazardFn, UpdateStateFn, MI->getParent(), std::next(MI->getReverseIterator()), Visited)) return false; BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_WAITCNT_DEPCTR)) .addImm(0x0fff); return true; } bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) { if (!ST.hasVALUTransUseHazard()) return false; assert(!ST.hasExtendedWaitCounts()); if (!SIInstrInfo::isVALU(*MI)) return false; SmallSet SrcVGPRs; for (const MachineOperand &Use : MI->explicit_uses()) { if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg())) SrcVGPRs.insert(Use.getReg()); } // Look for the following pattern: // Va <- TRANS VALU // intv // MI Va (WaitState = 0) // // Where: // intv <= 5 VALUs / 1 TRANS // // If found, insert an appropriate S_WAITCNT_DEPCTR before MI. const int IntvMaxVALUs = 5; const int IntvMaxTRANS = 1; struct StateType { int VALUs = 0; int TRANS = 0; }; StateType State; // This overloads expiry testing with all the hazard detection auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) { // Too many VALU states have passed if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS) return HazardExpired; // Instructions which cause va_vdst==0 expire hazard if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && I.getOperand(0).getImm() == 0x0fff)) return HazardExpired; // Track registers writes if (SIInstrInfo::isTRANS(I)) { for (Register Src : SrcVGPRs) { if (I.modifiesRegister(Src, &TRI)) { return HazardFound; } } } return NoHazardFound; }; auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { if (SIInstrInfo::isVALU(MI)) State.VALUs += 1; if (SIInstrInfo::isTRANS(MI)) State.TRANS += 1; }; DenseSet Visited; if (!hasHazard(State, IsHazardFn, UpdateStateFn, MI->getParent(), std::next(MI->getReverseIterator()), Visited)) return false; // Hazard is observed - insert a wait on va_dst counter to ensure hazard is // avoided. BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_WAITCNT_DEPCTR)) .addImm(AMDGPU::DepCtr::encodeFieldVaVdst(0)); return true; } bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) { if (!SIInstrInfo::isWMMA(*MI) && !SIInstrInfo::isSWMMAC(*MI)) return false; const SIInstrInfo *TII = ST.getInstrInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo(); auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) { if (!SIInstrInfo::isWMMA(I) && !SIInstrInfo::isSWMMAC(I)) return false; // Src0(matrix A) or Src1(matrix B) of the current wmma instruction overlaps // with the dest(matrix D) of the previous wmma. const Register CurSrc0Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg(); const Register CurSrc1Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); const Register PrevDstReg = TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg(); if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) || TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) { return true; } // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall) // but Index can't overlap with PrevDstReg. if (AMDGPU::isGFX12Plus(ST)) { if (SIInstrInfo::isSWMMAC(*MI)) { const Register CurIndex = TII->getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg(); if (TRI->regsOverlap(PrevDstReg, CurIndex)) return true; } return false; } return false; }; auto IsExpiredFn = [](const MachineInstr &I, int) { return SIInstrInfo::isVALU(I); }; if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == std::numeric_limits::max()) return false; BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32)); return true; } bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) { if (!ST.hasShift64HighRegBug()) return false; assert(!ST.hasExtendedWaitCounts()); switch (MI->getOpcode()) { default: return false; case AMDGPU::V_LSHLREV_B64_e64: case AMDGPU::V_LSHRREV_B64_e64: case AMDGPU::V_ASHRREV_I64_e64: break; } MachineOperand *Amt = TII.getNamedOperand(*MI, AMDGPU::OpName::src0); if (!Amt->isReg()) return false; Register AmtReg = Amt->getReg(); const MachineRegisterInfo &MRI = MF.getRegInfo(); // Check if this is a last VGPR in the allocation block. if (!TRI.isVGPR(MRI, AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7) return false; if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(AmtReg + 1)) return false; MachineOperand *Src1 = TII.getNamedOperand(*MI, AMDGPU::OpName::src1); bool OverlappedSrc = Src1->isReg() && TRI.regsOverlap(Src1->getReg(), AmtReg); bool OverlappedDst = MI->modifiesRegister(AmtReg, &TRI); bool Overlapped = OverlappedSrc || OverlappedDst; assert(!OverlappedDst || !OverlappedSrc || Src1->getReg() == MI->getOperand(0).getReg()); assert(ST.needsAlignedVGPRs()); static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1); Register NewReg; for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass : AMDGPU::VGPR_32RegClass) { if (!MI->modifiesRegister(Reg, &TRI) && !MI->readsRegister(Reg, &TRI)) { NewReg = Reg; break; } } Register NewAmt = Overlapped ? (Register)TRI.getSubReg(NewReg, AMDGPU::sub1) : NewReg; Register NewAmtLo; if (Overlapped) NewAmtLo = TRI.getSubReg(NewReg, AMDGPU::sub0); DebugLoc DL = MI->getDebugLoc(); MachineBasicBlock *MBB = MI->getParent(); // Insert a full wait count because found register might be pending a wait. BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_WAITCNT)) .addImm(0); // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them. if (Overlapped) runOnInstruction( BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmtLo) .addDef(AmtReg - 1) .addReg(AmtReg - 1, RegState::Undef) .addReg(NewAmtLo, RegState::Undef)); runOnInstruction(BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt) .addDef(AmtReg) .addReg(AmtReg, RegState::Undef) .addReg(NewAmt, RegState::Undef)); // Instructions emitted after the current instruction will be processed by the // parent loop of the hazard recognizer in a natural way. BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32), AmtReg) .addDef(NewAmt) .addReg(NewAmt) .addReg(AmtReg); if (Overlapped) BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32), AmtReg - 1) .addDef(NewAmtLo) .addReg(NewAmtLo) .addReg(AmtReg - 1); // Re-running hazard recognizer on the modified instruction is not necessary, // inserted V_SWAP_B32 has already both read and write new registers so // hazards related to these register has already been handled. Amt->setReg(NewAmt); Amt->setIsKill(false); // We do not update liveness, so verifier may see it as undef. Amt->setIsUndef(); if (OverlappedDst) MI->getOperand(0).setReg(NewReg); if (OverlappedSrc) { Src1->setReg(NewReg); Src1->setIsKill(false); Src1->setIsUndef(); } return true; } int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { int NSAtoVMEMWaitStates = 1; if (!ST.hasNSAtoVMEMBug()) return 0; if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) return 0; const SIInstrInfo *TII = ST.getInstrInfo(); const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); if (!Offset || (Offset->getImm() & 6) == 0) return 0; auto IsHazardFn = [TII](const MachineInstr &I) { if (!SIInstrInfo::isMIMG(I)) return false; const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode()); return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && TII->getInstSizeInBytes(I) >= 16; }; return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1); } int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { int FPAtomicToDenormModeWaitStates = 3; if (!ST.hasFPAtomicToDenormModeHazard()) return 0; assert(!ST.hasExtendedWaitCounts()); if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) return 0; auto IsHazardFn = [](const MachineInstr &I) { if (!SIInstrInfo::isVMEM(I)) return false; return SIInstrInfo::isFPAtomic(I); }; auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) { if (WaitStates >= 3 || SIInstrInfo::isVALU(MI)) return true; switch (MI.getOpcode()) { case AMDGPU::S_WAITCNT: case AMDGPU::S_WAITCNT_VSCNT: case AMDGPU::S_WAITCNT_VMCNT: case AMDGPU::S_WAITCNT_EXPCNT: case AMDGPU::S_WAITCNT_LGKMCNT: case AMDGPU::S_WAIT_IDLE: return true; default: break; } return false; }; return FPAtomicToDenormModeWaitStates - ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); } int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { assert(SIInstrInfo::isMAI(*MI)); return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); } int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) { // Early exit if no padding is requested. if (MFMAPaddingRatio == 0) return 0; const SIMachineFunctionInfo *MFI = MF.getInfo(); if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2) return 0; int NeighborMFMALatency = 0; auto IsNeighboringMFMA = [&NeighborMFMALatency, this](const MachineInstr &MI) { if (!SIInstrInfo::isMFMA(MI)) return false; NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI); return true; }; const int MaxMFMAPipelineWaitStates = 16; int WaitStatesSinceNeighborMFMA = getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates); int NeighborMFMAPaddingNeeded = (NeighborMFMALatency * MFMAPaddingRatio / 100) - WaitStatesSinceNeighborMFMA; return std::max(0, NeighborMFMAPaddingNeeded); } int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { int WaitStatesNeeded = 0; unsigned Opc = MI->getOpcode(); auto IsVALUFn = [](const MachineInstr &MI) { return SIInstrInfo::isVALU(MI) || MI.isInlineAsm(); }; if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write const int LegacyVALUWritesVGPRWaitStates = 2; const int VALUWritesExecWaitStates = 4; const int MaxWaitStates = 4; int WaitStatesNeededForUse = VALUWritesExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); if (WaitStatesNeeded < MaxWaitStates) { for (const MachineOperand &Use : MI->explicit_uses()) { const int MaxWaitStates = 2; if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg())) continue; int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates - getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); if (WaitStatesNeeded == MaxWaitStates) break; } } } for (const MachineOperand &Op : MI->explicit_operands()) { if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg())) continue; if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) continue; const int MFMAWritesAGPROverlappedSrcABWaitStates = 4; const int MFMAWritesAGPROverlappedSrcCWaitStates = 2; const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4; const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10; const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18; const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1; const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7; const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15; const int MaxWaitStates = 18; Register Reg = Op.getReg(); unsigned HazardDefLatency = 0; auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency, this](const MachineInstr &MI) { if (!SIInstrInfo::isMFMA(MI)) return false; Register DstReg = MI.getOperand(0).getReg(); if (DstReg == Reg) return false; HazardDefLatency = std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); return TRI.regsOverlap(DstReg, Reg); }; int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates); int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates; int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); int OpNo = Op.getOperandNo(); if (OpNo == SrcCIdx) { NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates; } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { switch (HazardDefLatency) { case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates; break; case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates; break; case 16: [[fallthrough]]; default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates; break; } } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { switch (HazardDefLatency) { case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates; break; case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates; break; case 16: [[fallthrough]]; default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates; break; } } int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); if (WaitStatesNeeded == MaxWaitStates) return WaitStatesNeeded; // Early exit. auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) { if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) return false; Register DstReg = MI.getOperand(0).getReg(); return TRI.regsOverlap(Reg, DstReg); }; const int AccVGPRWriteMFMAReadSrcCWaitStates = 1; const int AccVGPRWriteMFMAReadSrcABWaitStates = 3; const int AccVGPRWriteAccVgprReadWaitStates = 3; NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates; if (OpNo == SrcCIdx) NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates; else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates; WaitStatesNeededForUse = NeedWaitStates - getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); if (WaitStatesNeeded == MaxWaitStates) return WaitStatesNeeded; // Early exit. } if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0; const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5; const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13; const int MaxWaitStates = 13; Register DstReg = MI->getOperand(0).getReg(); unsigned HazardDefLatency = 0; auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency, this](const MachineInstr &MI) { if (!SIInstrInfo::isMFMA(MI)) return false; Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); HazardDefLatency = std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); return TRI.regsOverlap(Reg, DstReg); }; int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates); int NeedWaitStates; switch (HazardDefLatency) { case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates; break; case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates; break; case 16: [[fallthrough]]; default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates; break; } int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince; WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } // Pad neighboring MFMA with noops for better inter-wave performance. WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); return WaitStatesNeeded; } static int GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses, bool IsGFX950) { // xdl def cycles | gfx940 | gfx950 // 2 pass | 3 4 // 4 pass | 5 6 // 8 pass | 9 10 // 16 pass | 17 18 return NumPasses + 1 + IsGFX950; } static int GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses, bool IsGFX950) { // xdl def cycles | gfx940 | gfx950 // 2 pass | 3 3 // 4 pass | 5 6 // 8 pass | 9 10 // 16 pass | 17 18 return NumPasses + 1 + (NumPasses != 2 && IsGFX950); } static int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) { // 2 pass -> 2 // 4 pass -> 4 // 8 pass -> 8 // 16 pass -> 16 return NumPasses; } static int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) { // 2 pass -> 4 // 4 pass -> 6 // 8 pass -> 10 // 16 pass -> 18 return NumPasses + 2; } static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses, bool IsGFX950) { // xdl def cycles | gfx942 | gfx950 // 2 pass | 5 5 // 4 pass | 7 8 // 8 pass | 11 12 // 16 pass | 19 20 return NumPasses + 3 + (NumPasses != 2 && IsGFX950); } int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { int WaitStatesNeeded = 0; unsigned Opc = MI->getOpcode(); auto IsLegacyVALUFn = [](const MachineInstr &MI) { return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI); }; auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) { return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) && !SIInstrInfo::isDOT(MI); }; if (!SIInstrInfo::isMFMA(*MI)) return WaitStatesNeeded; const int VALUWritesExecWaitStates = 4; int WaitStatesNeededForUse = VALUWritesExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, VALUWritesExecWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); // Loop for both DGEMM and S/HGEMM 2nd instruction. for (const MachineOperand &Use : MI->explicit_uses()) { const int LegacyVALUNotDotWritesVGPRWaitStates = 2; const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2; const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8; const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16; const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3; const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9; const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17; const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9; const int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17; const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4; const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5; const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11; const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19; const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6; const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11; const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19; const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4; const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2; const int MaxWaitStates = 19; if (!Use.isReg()) continue; Register Reg = Use.getReg(); bool FullReg; const MachineInstr *MI1; auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1, this](const MachineInstr &MI) { if (!SIInstrInfo::isMFMA(MI)) return false; Register DstReg = MI.getOperand(0).getReg(); FullReg = (DstReg == Reg); MI1 = &MI; return TRI.regsOverlap(DstReg, Reg); }; WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates - getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); int NumWaitStates = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates); if (NumWaitStates == std::numeric_limits::max()) continue; int OpNo = Use.getOperandNo(); unsigned Opc1 = MI1->getOpcode(); int NeedWaitStates = 0; if (OpNo == SrcCIdx) { if (!SIInstrInfo::isDGEMM(Opc) && (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) { NeedWaitStates = 0; } else if (FullReg) { if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates; else if (ST.hasGFX940Insts() && TSchedModel.computeInstrLatency(MI1) == 2) NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates; } else { switch (Opc1) { case AMDGPU::V_MFMA_F64_16X16X4F64_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: if (!TII.isXDL(*MI)) NeedWaitStates = ST.hasGFX950Insts() ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates : DMFMA16x16WritesVGPROverlappedSrcCWaitStates; break; case AMDGPU::V_MFMA_F64_4X4X4F64_e64: case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: if (!TII.isXDL(*MI)) NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; break; default: int NumPasses = TSchedModel.computeInstrLatency(MI1); if (ST.hasGFX940Insts()) { if (TII.isXDL(*MI) && !TII.isXDL(*MI1)) break; NeedWaitStates = TII.isXDL(*MI1) ? (TII.isXDL(*MI) ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates( NumPasses, ST.hasGFX950Insts())) : GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates( NumPasses); break; } switch (NumPasses) { case 2: NeedWaitStates = SIInstrInfo::isDGEMM(Opc) ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; break; case 8: NeedWaitStates = SIInstrInfo::isDGEMM(Opc) ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; break; case 16: NeedWaitStates = SIInstrInfo::isDGEMM(Opc) ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; break; default: llvm_unreachable("unexpected number of passes"); } } } } else { switch (Opc1) { case AMDGPU::V_MFMA_F64_16X16X4F64_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: NeedWaitStates = ST.hasGFX950Insts() ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates; break; case AMDGPU::V_MFMA_F64_4X4X4F64_e64: case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; break; default: int NumPasses = TSchedModel.computeInstrLatency(MI1); if (ST.hasGFX940Insts()) { NeedWaitStates = TII.isXDL(*MI1) ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates( NumPasses); break; } switch (NumPasses) { case 2: NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates; break; case 4: llvm_unreachable("unexpected number of passes for mfma"); case 8: NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates; break; case 16: default: NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates; } } } if (WaitStatesNeeded >= NeedWaitStates) continue; WaitStatesNeededForUse = NeedWaitStates - NumWaitStates; WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); if (WaitStatesNeeded == MaxWaitStates) break; } // Pad neighboring MFMA with noops for better inter-wave performance. WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); return WaitStatesNeeded; } int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards() if (!ST.hasMAIInsts() || ST.hasGFX90AInsts()) return 0; int WaitStatesNeeded = 0; auto IsAccVgprReadFn = [](const MachineInstr &MI) { return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; }; for (const MachineOperand &Op : MI->explicit_uses()) { if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg())) continue; Register Reg = Op.getReg(); const int AccVgprReadLdStWaitStates = 2; const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1; const int MaxWaitStates = 2; int WaitStatesNeededForUse = AccVgprReadLdStWaitStates - getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); if (WaitStatesNeeded == MaxWaitStates) return WaitStatesNeeded; // Early exit. auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) { if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) return false; auto IsVALUFn = [](const MachineInstr &MI) { return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI); }; return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) < std::numeric_limits::max(); }; WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates - getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } return WaitStatesNeeded; } int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) { assert(!ST.hasVcmpxPermlaneHazard() && "this is a different vcmpx+permlane hazard"); const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIInstrInfo *TII = ST.getInstrInfo(); auto IsVCmpXWritesExecFn = [TII, TRI](const MachineInstr &MI) { return isVCmpXWritesExec(*TII, *TRI, MI); }; auto IsVALUFn = [](const MachineInstr &MI) { return SIInstrInfo::isVALU(MI); }; const int VCmpXWritesExecWaitStates = 4; const int VALUWritesVDstWaitStates = 2; int WaitStatesNeeded = 0; for (const MachineOperand &Op : MI->explicit_uses()) { if (!Op.isReg() || !TRI->isVGPR(MF.getRegInfo(), Op.getReg())) continue; Register Reg = Op.getReg(); int WaitStatesSinceDef = VALUWritesVDstWaitStates - getWaitStatesSinceDef(Reg, IsVALUFn, /*MaxWaitStates=*/VALUWritesVDstWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesSinceDef); if (WaitStatesNeeded >= VALUWritesVDstWaitStates) break; } int VCmpXHazardWaits = VCmpXWritesExecWaitStates - getWaitStatesSince(IsVCmpXWritesExecFn, VCmpXWritesExecWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, VCmpXHazardWaits); return WaitStatesNeeded; } static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) { // 2 pass -> 4 // 4 pass -> 6 // 8 pass -> 10 // 16 pass -> 18 return NumPasses + 2; } static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses, bool IsGFX950) { // xdl def cycles | gfx942 | gfx950 // 2 pass | 5 5 // 4 pass | 7 8 // 8 pass | 11 12 // 16 pass | 19 20 return NumPasses + 3 + (NumPasses != 2 && IsGFX950); } static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses, bool IsGFX950) { // xdl def cycles | gfx942 | gfx950 // 2 pass | 5 5 // 4 pass | 7 8 // 8 pass | 11 12 // 16 pass | 19 20 return NumPasses + 3 + (NumPasses != 2 && IsGFX950); } static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) { // 2 pass -> 4 // 4 pass -> 6 // 8 pass -> 10 // 16 pass -> 18 return NumPasses + 2; } int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { if (!ST.hasGFX90AInsts()) return 0; auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { return SIInstrInfo::isDGEMM(MI.getOpcode()); }; // This is checked in checkMAIHazards90A() if (SIInstrInfo::isMFMA(*MI)) return 0; const MachineRegisterInfo &MRI = MF.getRegInfo(); int WaitStatesNeeded = 0; bool IsMem = SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI); bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI); bool IsVALU = SIInstrInfo::isVALU(*MI); const MachineInstr *MFMA = nullptr; unsigned Reg; auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { if (!SIInstrInfo::isMFMA(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) return false; MFMA = &MI; return true; }; const MachineInstr *DOT = nullptr; auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) { if (!SIInstrInfo::isDOT(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) return false; DOT = &MI; return true; }; bool DGEMMAfterVALUWrite = false; auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) { // Found DGEMM on reverse traversal to def. if (SIInstrInfo::isDGEMM(MI.getOpcode())) DGEMMAfterVALUWrite = true; // Only hazard if register is defined by a VALU and a DGEMM is found after // after the def. if (!TII.isVALU(MI) || !DGEMMAfterVALUWrite) return false; return true; }; int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::src2); if (IsMemOrExport || IsVALU) { const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5; const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11; const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19; const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9; const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18; const int DMFMA4x4WriteVgprVALUReadWaitStates = 6; const int DMFMA16x16WriteVgprVALUReadWaitStates = 11; const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19; const int DotWriteSameDotReadSrcAB = 3; const int DotWriteDifferentVALURead = 3; const int DMFMABetweenVALUWriteVMEMRead = 2; const int MaxWaitStates = 19; for (const MachineOperand &Use : MI->explicit_uses()) { if (!Use.isReg()) continue; Reg = Use.getReg(); DOT = nullptr; int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, MaxWaitStates); if (DOT) { int NeedWaitStates = 0; if (DOT->getOpcode() == MI->getOpcode()) { if (&Use - &MI->getOperand(0) != SrcCIdx) NeedWaitStates = DotWriteSameDotReadSrcAB; } else { NeedWaitStates = DotWriteDifferentVALURead; } int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } // Workaround for HW data hazard bug observed only in GFX90A. When there // is a DGEMM instruction in-between a VALU and a VMEM instruction it // causes the SQ to incorrectly not insert two wait states between the two // instructions needed to avoid data hazard. if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) { DGEMMAfterVALUWrite = false; if (TRI.isVectorRegister(MRI, Reg)) { int WaitStatesNeededForUse = DMFMABetweenVALUWriteVMEMRead - getWaitStatesSinceDef(Reg, IsDGEMMHazard, DMFMABetweenVALUWriteVMEMRead); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } } MFMA = nullptr; WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); if (!MFMA) continue; unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); int NumPasses = HazardDefLatency; int NeedWaitStates = MaxWaitStates; if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { switch (HazardDefLatency) { case 4: NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates : DMFMA4x4WriteVgprVALUReadWaitStates; break; case 8: case 16: NeedWaitStates = IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates : (ST.hasGFX950Insts() ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates : DMFMA16x16WriteVgprVALUReadWaitStates); break; default: llvm_unreachable("unexpected dgemm"); } } else if (ST.hasGFX940Insts()) { NeedWaitStates = TII.isXDL(*MFMA) ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates( NumPasses); } else { switch (HazardDefLatency) { case 2: NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates; break; case 8: NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates; break; case 16: NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates; break; default: llvm_unreachable("unexpected number of passes for mfma"); } } int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); if (WaitStatesNeeded == MaxWaitStates) break; } } unsigned Opc = MI->getOpcode(); const int DMFMAToFMA64WaitStates = 2; if ((Opc == AMDGPU::V_FMA_F64_e64 || Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || Opc == AMDGPU::V_FMAC_F64_dpp) && WaitStatesNeeded < DMFMAToFMA64WaitStates) { int WaitStatesNeededForUse = DMFMAToFMA64WaitStates - getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } if (!IsVALU && !IsMemOrExport) return WaitStatesNeeded; for (const MachineOperand &Def : MI->defs()) { const int SMFMA4x4WriteVgprVALUWawWaitStates = 5; const int SMFMA16x16WriteVgprVALUWawWaitStates = 11; const int SMFMA32x32WriteVgprVALUWawWaitStates = 19; const int SMFMA4x4ReadVgprVALUWarWaitStates = 1; const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3; const int SMFMA16x16ReadVgprVALUWarWaitStates = 7; const int SMFMA32x32ReadVgprVALUWarWaitStates = 15; const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6; const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11; const int DotWriteDifferentVALUWrite = 3; const int MaxWaitStates = 19; const int MaxWarWaitStates = 15; Reg = Def.getReg(); DOT = nullptr; int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, MaxWaitStates); if (DOT && DOT->getOpcode() != MI->getOpcode()) WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite - WaitStatesSinceDef); MFMA = nullptr; WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); if (MFMA) { int NeedWaitStates = MaxWaitStates; int NumPasses = TSchedModel.computeInstrLatency(MFMA); if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { switch (NumPasses) { case 4: NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates; break; case 8: case 16: NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates; break; default: llvm_unreachable("unexpected number of cycles for dgemm"); } } else if (ST.hasGFX940Insts()) { NeedWaitStates = TII.isXDL(*MFMA) ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses); } else { switch (NumPasses) { case 2: NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates; break; case 8: NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates; break; case 16: NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates; break; default: llvm_unreachable("Unexpected number of passes for mfma"); } } int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); if (WaitStatesNeeded == MaxWaitStates) break; } auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) || !MI.readsRegister(Reg, &TRI)) return false; if (ST.hasGFX940Insts() && !TII.isXDL(MI)) return false; const MachineOperand *SrcC = TII.getNamedOperand(MI, AMDGPU::OpName::src2); assert(SrcC); if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg)) return false; MFMA = &MI; return true; }; MFMA = nullptr; int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn, MaxWarWaitStates); if (!MFMA) continue; unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); int NeedWaitStates = MaxWaitStates; switch (HazardDefLatency) { case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; break; case 4: assert(ST.hasGFX940Insts()); NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates; break; case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates; break; case 16: [[fallthrough]]; default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates; break; } int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse; WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); } return WaitStatesNeeded; } bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) { if (!SU->isInstr()) return false; const MachineInstr *MAI = nullptr; auto IsMFMAFn = [&MAI](const MachineInstr &MI) { MAI = nullptr; if (SIInstrInfo::isMFMA(MI)) MAI = &MI; return MAI != nullptr; }; MachineInstr *MI = SU->getInstr(); if (IsMFMAFn(*MI)) { int W = getWaitStatesSince(IsMFMAFn, 16); if (MAI) return W < (int)TSchedModel.computeInstrLatency(MAI); } return false; } // Adjust global offsets for instructions bundled with S_GETPC_B64 after // insertion of a new instruction. static void updateGetPCBundle(MachineInstr *NewMI) { if (!NewMI->isBundled()) return; // Find start of bundle. auto I = NewMI->getIterator(); while (I->isBundledWithPred()) I--; if (I->isBundle()) I++; // Bail if this is not an S_GETPC bundle. if (I->getOpcode() != AMDGPU::S_GETPC_B64) return; // Update offsets of any references in the bundle. const unsigned NewBytes = 4; assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && "Unexpected instruction insertion in bundle"); auto NextMI = std::next(NewMI->getIterator()); auto End = NewMI->getParent()->end(); while (NextMI != End && NextMI->isBundledWithPred()) { for (auto &Operand : NextMI->operands()) { if (Operand.isGlobal()) Operand.setOffset(Operand.getOffset() + NewBytes); } NextMI++; } } bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) { if (!ST.hasVALUMaskWriteHazard()) return false; assert(!ST.hasExtendedWaitCounts()); if (!ST.isWave64() || !SIInstrInfo::isSALU(*MI)) return false; // The hazard sequence is three instructions: // 1. VALU reads SGPR as mask // 2. SALU writes SGPR // 3. SALU reads SGPR // The hazard can expire if the distance between 2 and 3 is sufficient. // In practice this happens <10% of the time, hence this always assumes // the hazard exists if 1 and 2 are present to avoid searching. const MachineOperand *SDSTOp = TII.getNamedOperand(*MI, AMDGPU::OpName::sdst); if (!SDSTOp || !SDSTOp->isReg()) return false; const Register HazardReg = SDSTOp->getReg(); if (HazardReg == AMDGPU::EXEC || HazardReg == AMDGPU::EXEC_LO || HazardReg == AMDGPU::EXEC_HI || HazardReg == AMDGPU::M0) return false; auto IsHazardFn = [HazardReg, this](const MachineInstr &I) { switch (I.getOpcode()) { case AMDGPU::V_ADDC_U32_e32: case AMDGPU::V_ADDC_U32_dpp: case AMDGPU::V_CNDMASK_B16_t16_e32: case AMDGPU::V_CNDMASK_B16_fake16_e32: case AMDGPU::V_CNDMASK_B16_t16_dpp: case AMDGPU::V_CNDMASK_B16_fake16_dpp: case AMDGPU::V_CNDMASK_B32_e32: case AMDGPU::V_CNDMASK_B32_dpp: case AMDGPU::V_DIV_FMAS_F32_e64: case AMDGPU::V_DIV_FMAS_F64_e64: case AMDGPU::V_SUBB_U32_e32: case AMDGPU::V_SUBB_U32_dpp: case AMDGPU::V_SUBBREV_U32_e32: case AMDGPU::V_SUBBREV_U32_dpp: // These implicitly read VCC as mask source. return HazardReg == AMDGPU::VCC || HazardReg == AMDGPU::VCC_LO || HazardReg == AMDGPU::VCC_HI; case AMDGPU::V_ADDC_U32_e64: case AMDGPU::V_ADDC_U32_e64_dpp: case AMDGPU::V_CNDMASK_B16_t16_e64: case AMDGPU::V_CNDMASK_B16_fake16_e64: case AMDGPU::V_CNDMASK_B16_t16_e64_dpp: case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp: case AMDGPU::V_CNDMASK_B32_e64: case AMDGPU::V_CNDMASK_B32_e64_dpp: case AMDGPU::V_SUBB_U32_e64: case AMDGPU::V_SUBB_U32_e64_dpp: case AMDGPU::V_SUBBREV_U32_e64: case AMDGPU::V_SUBBREV_U32_e64_dpp: { // Only check mask register overlaps. const MachineOperand *SSRCOp = TII.getNamedOperand(I, AMDGPU::OpName::src2); assert(SSRCOp); return TRI.regsOverlap(SSRCOp->getReg(), HazardReg); } default: return false; } }; const MachineRegisterInfo &MRI = MF.getRegInfo(); auto IsExpiredFn = [&MRI, this](const MachineInstr &I, int) { // s_waitcnt_depctr sa_sdst(0) mitigates hazard. if (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && AMDGPU::DepCtr::decodeFieldSaSdst(I.getOperand(0).getImm()) == 0) return true; // VALU access to any SGPR or literal constant other than HazardReg // mitigates hazard. No need to check HazardReg here as this will // only be called when !IsHazardFn. if (!SIInstrInfo::isVALU(I)) return false; for (int OpNo = 0, End = I.getNumOperands(); OpNo < End; ++OpNo) { const MachineOperand &Op = I.getOperand(OpNo); if (Op.isReg()) { Register OpReg = Op.getReg(); // Only consider uses if (!Op.isUse()) continue; // Ignore EXEC if (OpReg == AMDGPU::EXEC || OpReg == AMDGPU::EXEC_LO || OpReg == AMDGPU::EXEC_HI) continue; // Ignore all implicit uses except VCC if (Op.isImplicit()) { if (OpReg == AMDGPU::VCC || OpReg == AMDGPU::VCC_LO || OpReg == AMDGPU::VCC_HI) return true; continue; } if (TRI.isSGPRReg(MRI, OpReg)) return true; } else { const MCInstrDesc &InstDesc = I.getDesc(); const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; if (!TII.isInlineConstant(Op, OpInfo)) return true; } } return false; }; // Check for hazard if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == std::numeric_limits::max()) return false; auto NextMI = std::next(MI->getIterator()); // Add s_waitcnt_depctr sa_sdst(0) after SALU write. auto NewMI = BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(), TII.get(AMDGPU::S_WAITCNT_DEPCTR)) .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0)); // SALU write may be s_getpc in a bundle. updateGetPCBundle(NewMI); return true; } static bool ensureEntrySetPrio(MachineFunction *MF, int Priority, const SIInstrInfo &TII) { MachineBasicBlock &EntryMBB = MF->front(); if (EntryMBB.begin() != EntryMBB.end()) { auto &EntryMI = *EntryMBB.begin(); if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO && EntryMI.getOperand(0).getImm() >= Priority) return false; } BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), TII.get(AMDGPU::S_SETPRIO)) .addImm(Priority); return true; } bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) { if (!ST.hasRequiredExportPriority()) return false; // Assume the following shader types will never have exports, // and avoid adding or adjusting S_SETPRIO. MachineBasicBlock *MBB = MI->getParent(); MachineFunction *MF = MBB->getParent(); auto CC = MF->getFunction().getCallingConv(); switch (CC) { case CallingConv::AMDGPU_CS: case CallingConv::AMDGPU_CS_Chain: case CallingConv::AMDGPU_CS_ChainPreserve: case CallingConv::AMDGPU_KERNEL: return false; default: break; } const int MaxPriority = 3; const int NormalPriority = 2; const int PostExportPriority = 0; auto It = MI->getIterator(); switch (MI->getOpcode()) { case AMDGPU::S_ENDPGM: case AMDGPU::S_ENDPGM_SAVED: case AMDGPU::S_ENDPGM_ORDERED_PS_DONE: case AMDGPU::SI_RETURN_TO_EPILOG: // Ensure shader with calls raises priority at entry. // This ensures correct priority if exports exist in callee. if (MF->getFrameInfo().hasCalls()) return ensureEntrySetPrio(MF, NormalPriority, TII); return false; case AMDGPU::S_SETPRIO: { // Raise minimum priority unless in workaround. auto &PrioOp = MI->getOperand(0); int Prio = PrioOp.getImm(); bool InWA = (Prio == PostExportPriority) && (It != MBB->begin() && TII.isEXP(*std::prev(It))); if (InWA || Prio >= NormalPriority) return false; PrioOp.setImm(std::min(Prio + NormalPriority, MaxPriority)); return true; } default: if (!TII.isEXP(*MI)) return false; break; } // Check entry priority at each export (as there will only be a few). // Note: amdgpu_gfx can only be a callee, so defer to caller setprio. bool Changed = false; if (CC != CallingConv::AMDGPU_Gfx) Changed = ensureEntrySetPrio(MF, NormalPriority, TII); auto NextMI = std::next(It); bool EndOfShader = false; if (NextMI != MBB->end()) { // Only need WA at end of sequence of exports. if (TII.isEXP(*NextMI)) return Changed; // Assume appropriate S_SETPRIO after export means WA already applied. if (NextMI->getOpcode() == AMDGPU::S_SETPRIO && NextMI->getOperand(0).getImm() == PostExportPriority) return Changed; EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM; } const DebugLoc &DL = MI->getDebugLoc(); // Lower priority. BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_SETPRIO)) .addImm(PostExportPriority); if (!EndOfShader) { // Wait for exports to complete. BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_WAITCNT_EXPCNT)) .addReg(AMDGPU::SGPR_NULL) .addImm(0); } BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_NOP)).addImm(0); BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_NOP)).addImm(0); if (!EndOfShader) { // Return to normal (higher) priority. BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_SETPRIO)) .addImm(NormalPriority); } return true; }