35 #include "debug/PM4PacketProcessor.hh"
42 #include "enums/GfxVersion.hh"
96 addr = (((
addr >> 12) << 3) << 12) | low_bits;
112 return result->second;
150 "%d, pipe %d queue: %d size: %d\n",
id,
q->base(),
q->offset(),
151 q->me(),
q->pipe(),
q->queue(),
q->size());
157 q->wptr(wptrOffset *
sizeof(uint32_t));
159 if (!
q->processing()) {
169 q->id(),
q->rptr(),
q->wptr());
171 if (
q->rptr() <
q->wptr()) {
184 q->processing(
false);
200 void *dmaBuffer =
nullptr;
206 if (
header.count != 0x3fff) {
207 q->incRptr((
header.count + 1) *
sizeof(uint32_t));
214 [ = ] (
const uint64_t &)
223 [ = ] (
const uint64_t &)
232 [ = ] (
const uint64_t &)
241 [ = ] (
const uint64_t &)
250 [ = ] (
const uint64_t &)
259 [ = ] (
const uint64_t &)
268 [ = ] (
const uint64_t &)
276 [ = ] (
const uint64_t &)
285 [ = ] (
const uint64_t &)
294 [ = ] (
const uint64_t &)
303 [ = ] (
const uint64_t &)
312 q->incRptr((
header.count + 1) *
sizeof(uint32_t));
317 warn(
"PM4 packet opcode 0x%x not supported.\n",
header.opcode);
320 q->incRptr((
header.count + 1) *
sizeof(uint32_t));
362 "pipe: %d, queueSlot: %d, queueType: %d, allocFormat: %d, "
363 "engineSel: %d, numQueues: %d, checkDisable: %d, doorbellOffset:"
364 " %d, mqdAddr: %lx, wptrAddr: %lx\n", pkt->
queueSel, pkt->
vmid,
375 "Mapping mqd from %p %p (vmid %d - last vmid %d).\n",
384 [ = ] (
const uint32_t &) {
395 [ = ] (
const uint32_t &) {
432 mqd_size, 8, GfxVersion::gfx900,
offset,
450 "%#x/%#x ib: %#x/%#x size: %d ctrl: %#x rptr wb addr: %#lx\n",
477 "%d destSel %d dataSel %d, address %p data %p, intCtx %p\n",
482 "PM4 release_mem destSel 0 bypasses caches to MC.\n");
490 panic(
"Unimplemented PM4ReleaseMem.dataSelect");
501 "pipe: %d, queueSlot:%d\n",
q->id(), pkt->
intCtxId,
q->me(),
502 q->pipe(),
q->queue());
506 ringId = (
q->queue() << 4) | (
q->me() << 2) |
q->pipe();
530 "pasid: %p doorbellOffset0 %p \n",
576 for (
auto id : iter.second) {
580 if (
queues[
id]->privileged()) {
588 96 *
sizeof(uint32_t));
591 [ = ] (
const uint32_t &) {
597 hsa_pp.unsetDeviceQueueDesc(
id, 8);
603 panic(
"Unrecognized options\n");
638 scratch_base + 0xFFFFFFFF);
655 q->wptr(pkt->
ibSize *
sizeof(uint32_t));
671 q->wptr(pkt->
ibSize *
sizeof(uint32_t));
723 " %d command: %d, pasid: %d, doorbellOffset: %d, engineSel: %d "
737 panic(
"query_status with interruptSel %d command %d not supported",
754 switch (mmio_offset) {
1014 int num_queues =
queues.size();
1015 Addr id[num_queues];
1016 Addr mqd_base[num_queues];
1018 Addr rptr[num_queues];
1019 Addr wptr[num_queues];
1020 Addr ib_base[num_queues];
1021 Addr ib_rptr[num_queues];
1022 Addr ib_wptr[num_queues];
1024 bool processing[num_queues];
1025 bool ib[num_queues];
1028 for (
auto iter :
queues) {
1031 mqd_base[
i] =
q->mqdBase();
1032 bool cur_state =
q->ib();
1034 base[
i] =
q->base() >> 8;
1035 rptr[
i] =
q->getRptr();
1036 wptr[
i] =
q->getWptr();
1038 ib_base[
i] =
q->ibBase();
1039 ib_rptr[
i] =
q->getRptr();
1040 ib_wptr[
i] =
q->getWptr();
1043 processing[
i] =
q->processing();
1071 Addr id[num_queues];
1072 Addr mqd_base[num_queues];
1074 Addr rptr[num_queues];
1075 Addr wptr[num_queues];
1076 Addr ib_base[num_queues];
1077 Addr ib_rptr[num_queues];
1078 Addr ib_wptr[num_queues];
1080 bool processing[num_queues];
1081 bool ib[num_queues];
1095 for (
int i = 0;
i < num_queues;
i++) {
1101 mqd->
rptr = rptr[
i];
1112 queues[
id[
i]]->processing(processing[
i]);
Device model for an AMD GPU.
void insertQId(uint16_t vmid, int id)
void setDoorbellType(uint32_t offset, QueueType qt)
Set handles to GPU blocks.
void deallocateAllQueues()
std::unordered_map< uint16_t, std::set< int > > & getUsedVMIDs()
GPUCommandProcessor * CP()
uint16_t getVMID(Addr doorbell)
AMDGPUInterruptHandler * getIH()
Get handles to GPU blocks.
void deallocateVmid(uint16_t vmid)
void mapDoorbellToVMID(Addr doorbell, uint16_t vmid)
void setRegVal(uint32_t addr, uint32_t value)
uint16_t allocateVMID(uint16_t pasid)
void deallocatePasid(uint16_t pasid)
SDMAEngine * getSDMAById(int id)
void setSDMAEngine(Addr offset, SDMAEngine *eng)
void prepareInterruptCookie(ContextID cntxtId, uint32_t ring_id, uint32_t client_id, uint32_t source_id)
void submitInterruptCookie()
Translation range generators.
bool inAGP(Addr vaddr)
Methods for resolving apertures.
void setPageTableBase(uint16_t vmid, Addr ptBase)
Page table base/start accessors for user VMIDs.
void serialize(CheckpointOut &cp) const override
Serialize an object.
void unserialize(CheckpointIn &cp) override
Unserialize an object.
Wraps a std::function object in a DmaCallback.
void dmaReadVirt(Addr host_addr, unsigned size, DmaCallback *cb, void *data, Tick delay=0)
Initiate a DMA read from virtual address host_addr.
void dmaWriteVirt(Addr host_addr, unsigned size, DmaCallback *b, void *data, Tick delay=0)
Initiate a DMA write from virtual address host_addr.
HSAPacketProcessor & hsaPacketProc()
void setDeviceQueueDesc(uint64_t hostReadIndexPointer, uint64_t basePointer, uint64_t queue_id, uint32_t size, int doorbellSize, GfxVersion gfxVersion, Addr offset=0, uint64_t rd_idx=0)
void writeMMIO(PacketPtr pkt, Addr mmio_offset)
void setRbWptrPollAddrLo(uint32_t data)
void decodeHeader(PM4Queue *q, PM4Header header)
This method calls other PM4 packet processing methods based on the header of a PM4 packet.
void unserialize(CheckpointIn &cp) override
Unserialize an object.
void setRbWptrHi(uint32_t data)
void mapKiq(Addr offset)
The first compute queue, the Kernel Interface Queueu a.k.a.
Addr getGARTAddr(Addr addr) const
void writeDataDone(PM4Queue *q, PM4WriteData *pkt, Addr addr)
void setRbWptrLo(uint32_t data)
void switchBuffer(PM4Queue *q, PM4SwitchBuf *pkt)
void setGPUDevice(AMDGPUDevice *gpu_device)
void serialize(CheckpointOut &cp) const override
Serialize an object.
void setRbCntl(uint32_t data)
uint32_t getKiqDoorbellOffset()
void setHqdPqWptrLo(uint32_t data)
std::unordered_map< uint32_t, PM4Queue * > queuesMap
void setUconfigReg(PM4Queue *q, PM4SetUconfigReg *pkt)
void queryStatus(PM4Queue *q, PM4QueryStatus *pkt)
void releaseMem(PM4Queue *q, PM4ReleaseMem *pkt)
void releaseMemDone(PM4Queue *q, PM4ReleaseMem *pkt, Addr addr)
void setHqdPqRptrReportAddr(uint32_t data)
void updateReadIndex(Addr offset, uint64_t rd_idx)
Update read index on doorbell rings.
void writeData(PM4Queue *q, PM4WriteData *pkt)
void setRbBaseHi(uint32_t data)
void setRbVmid(uint32_t data)
void setHqdActive(uint32_t data)
void processSDMAMQD(PM4MapQueues *pkt, PM4Queue *q, Addr addr, SDMAQueueDesc *mqd, uint16_t vmid)
void process(PM4Queue *q, Addr wptrOffset)
This method start processing a PM4Queue from the current read pointer to the newly communicated write...
void setHqdPqControl(uint32_t data)
void setRbBaseLo(uint32_t data)
void setHqdIbCtrl(uint32_t data)
void setRbRptrAddrHi(uint32_t data)
void setHqdPqWptrPollAddr(uint32_t data)
void newQueue(QueueDesc *q, Addr offset, PM4MapQueues *pkt=nullptr, int id=-1)
This method creates a new PM4Queue based on a queue descriptor and an offset.
void unmapQueues(PM4Queue *q, PM4UnmapQueues *pkt)
void queryStatusDone(PM4Queue *q, PM4QueryStatus *pkt)
void setRbDoorbellRangeLo(uint32_t data)
void waitRegMem(PM4Queue *q, PM4WaitRegMem *pkt)
void setHqdPqBaseHi(uint32_t data)
void runList(PM4Queue *q, PM4RunList *pkt)
void decodeNext(PM4Queue *q)
This method decodes the next packet in a PM4Queue.
void mapPq(Addr offset)
The first graphics queue, the Primary Queueu a.k.a.
void setHqdVmid(uint32_t data)
void setHqdPqDoorbellCtrl(uint32_t data)
void setHqdPqBase(uint32_t data)
void setRbDoorbellRangeHi(uint32_t data)
uint32_t getPqDoorbellOffset()
void doneMQDWrite(Addr mqdAddr, Addr addr)
std::unordered_map< uint16_t, PM4Queue * > queues
void indirectBuffer(PM4Queue *q, PM4IndirectBuf *pkt)
PM4PacketProcessor(const PM4PacketProcessorParams &p)
void setHqdPqPtr(uint32_t data)
void setHqdPqRptrReportAddrHi(uint32_t data)
void mapQueues(PM4Queue *q, PM4MapQueues *pkt)
TranslationGenPtr translate(Addr vaddr, Addr size) override
Method for functional translation.
void processMQD(PM4MapQueues *pkt, PM4Queue *q, Addr addr, QueueDesc *mqd, uint16_t vmid)
void setRbRptrAddrLo(uint32_t data)
void mapProcess(PM4Queue *q, PM4MapProcess *pkt)
void setRbDoorbellCntrl(uint32_t data)
PM4Queue * getQueue(Addr offset, bool gfx=false)
Based on an offset communicated through doorbell write, the PM4PacketProcessor identifies which queue...
AddrRangeList getAddrRanges() const override
Every PIO device is obliged to provide an implementation that returns the address ranges the device r...
void setHqdPqWptrPollAddrHi(uint32_t data)
void setHqdPqWptrHi(uint32_t data)
void setRbWptrPollAddrHi(uint32_t data)
Class defining a PM4 queue.
A Packet is used to encapsulate a transfer between two objects in the memory system (e....
T getLE() const
Get the data in the packet byte swapped from little endian to host endian.
System DMA Engine class for AMD dGPU.
void registerRLCQueue(Addr doorbell, Addr rb_base, uint32_t size, Addr rptr_wb_addr)
Methods for RLC queues.
void setLdsApe(Addr base, Addr limit)
void setScratchApe(Addr base, Addr limit)
void setHwReg(int regIdx, uint32_t val)
The GPUCommandProcessor (CP) is responsible for accepting commands, in the form of HSA AQL packets,...
constexpr T bits(T val, unsigned first, unsigned last)
Extract the bitfield from position 'first' to 'last' (inclusive) from 'val' and right justify it.
#define panic(...)
This implements a cprintf based panic() function.
#define UNSERIALIZE_ARRAY(member, size)
#define SERIALIZE_ARRAY(member, size)
Reference material can be found at the JEDEC website: UFS standard http://www.jedec....
struct gem5::GEM5_PACKED PM4WriteData
struct gem5::GEM5_PACKED PM4WaitRegMem
std::ostream CheckpointOut
struct gem5::GEM5_PACKED PM4RunList
uint64_t Addr
Address type This will probably be moved somewhere else in the near future.
@ SOC15_IH_CLIENTID_GRBM_CP
struct gem5::GEM5_PACKED PM4ReleaseMem
struct gem5::GEM5_PACKED PM4SwitchBuf
struct gem5::GEM5_PACKED PM4MapQueues
struct gem5::GEM5_PACKED PM4MapProcess
struct gem5::GEM5_PACKED SDMAQueueDesc
Queue descriptor for SDMA-based user queues (RLC queues).
struct gem5::GEM5_PACKED PM4UnmapQueues
struct gem5::GEM5_PACKED PM4SetUconfigReg
struct gem5::GEM5_PACKED PM4QueryStatus
struct gem5::GEM5_PACKED QueueDesc
Queue descriptor with relevant MQD attributes.
std::unique_ptr< TranslationGen > TranslationGenPtr
struct gem5::GEM5_PACKED PM4IndirectBuf
Declaration of the Packet class.
#define PACKET3_SET_UCONFIG_REG_START
Value from vega10/pm4_header.h.
#define mmCP_RB_DOORBELL_CONTROL
#define mmCP_RB0_RPTR_ADDR_HI
#define mmCP_HQD_PQ_RPTR_REPORT_ADDR
#define mmCP_HQD_PQ_DOORBELL_CONTROL
#define mmCP_HQD_PQ_WPTR_POLL_ADDR
#define mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI
#define mmCP_RB_DOORBELL_RANGE_UPPER
#define mmCP_HQD_IB_CONTROL
#define mmCP_RB_WPTR_POLL_ADDR_LO
#define mmCP_HQD_PQ_BASE_HI
#define mmCP_HQD_PQ_WPTR_HI
#define mmCP_HQD_PQ_CONTROL
#define mmCP_RB_DOORBELL_RANGE_LOWER
#define mmCP_RB_WPTR_POLL_ADDR_HI
#define mmCP_RB0_RPTR_ADDR
#define mmCP_HQD_PQ_WPTR_POLL_ADDR_HI
#define mmCP_HQD_PQ_WPTR_LO
#define UNSERIALIZE_SCALAR(scalar)
#define SERIALIZE_SCALAR(scalar)
uint32_t sdmax_rlcx_ib_base_lo
uint32_t sdmax_rlcx_rb_rptr
uint32_t sdmax_rlcx_rb_rptr_addr_hi
uint32_t sdmax_rlcx_rb_cntl
uint32_t hqd_pq_wptr_poll_addr_hi
uint32_t sdmax_rlcx_rb_wptr_hi
uint32_t sdmax_rlcx_ib_base_hi
uint32_t hqd_pq_wptr_poll_addr_lo
uint32_t hqd_pq_rptr_report_addr_hi
uint32_t hqd_pq_rptr_report_addr_lo
uint32_t hqd_pq_doorbell_control
uint32_t sdmax_rlcx_rb_rptr_addr_lo
uint32_t sdmax_rlcx_rb_wptr
uint32_t sdmax_rlcx_rb_rptr_hi
uint64_t completionSignal