51#include "debug/DMACopyEngine.hh"
52#include "debug/Drain.hh"
55#include "params/CopyEngine.hh"
69 regs.chanCount =
p.ChanCnt;
73 if (
regs.chanCount > 64)
74 fatal(
"CopyEngine interface doesn't support more than 64 DMA engines\n");
76 for (
int x = 0;
x <
regs.chanCount;
x++) {
90 addrCompleteEvent([
this]{ fetchAddrComplete(); },
name()),
91 readCompleteEvent([
this]{ readCopyBytesComplete(); },
name()),
92 writeCompleteEvent([
this]{ writeCopyBytesComplete(); },
name()),
93 statusCompleteEvent([
this]{ writeStatusComplete(); },
name())
96 cr.status.dma_transfer_status(3);
98 cr.completionAddr = 0;
100 curDmaDesc =
new DmaDesc;
101 memset(curDmaDesc, 0,
sizeof(DmaDesc));
102 copyBuffer =
new uint8_t[
ce->params().XferCap];
107 for (
int x = 0;
x <
chan.size();
x++) {
121 if (if_name !=
"dma") {
125 if (idx >=
static_cast<int>(
chan.size())) {
126 panic(
"CopyEngine::getPort: unknown index %d\n", idx);
129 return chan[idx]->getPort();
143 if (
cr.command.start_dma()) {
145 cr.status.dma_transfer_status(0);
150 }
else if (
cr.command.append_dma()) {
157 }
else if (
cr.command.reset_dma()) {
161 cr.status.dma_transfer_status(3);
164 }
else if (
cr.command.resume_dma() ||
cr.command.abort_dma() ||
165 cr.command.suspend_dma())
166 panic(
"Resume, Abort, and Suspend are not supported\n");
177 panic(
"Invalid PCI memory access to unmapped memory.\n");
183 if (size !=
sizeof(uint64_t) && size !=
sizeof(uint32_t) &&
184 size !=
sizeof(uint16_t) && size !=
sizeof(uint8_t)) {
185 panic(
"Unknown size for MMIO access: %d\n", pkt->
getSize());
188 DPRINTF(DMACopyEngine,
"Read device register %#X size: %d\n", daddr, size);
197 assert(size ==
sizeof(
regs.chanCount));
201 assert(size ==
sizeof(
regs.xferCap));
205 assert(size ==
sizeof(uint8_t));
207 regs.intrctrl.master_int_enable(0);
210 assert(size ==
sizeof(
regs.attnStatus));
215 panic(
"Read request to unknown register number: %#x\n", daddr);
225 while (daddr >= 0x80) {
230 if (chanid >=
regs.chanCount)
231 panic(
"Access to channel %d (device only configured for %d channels)",
232 chanid,
regs.chanCount);
237 chan[chanid]->channelRead(pkt, daddr, size);
248 assert(size ==
sizeof(uint16_t));
249 pkt->
setLE<uint16_t>(
cr.ctrl());
253 assert(size ==
sizeof(uint64_t));
254 pkt->
setLE<uint64_t>(
cr.status() | (
busy ? 0 : 1));
257 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
258 if (size ==
sizeof(uint64_t))
259 pkt->
setLE<uint64_t>(
cr.descChainAddr);
264 assert(size ==
sizeof(uint32_t));
265 pkt->
setLE<uint32_t>(
bits(
cr.descChainAddr,32,63));
268 assert(size ==
sizeof(uint8_t));
269 pkt->
setLE<uint32_t>(
cr.command());
272 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
273 if (size ==
sizeof(uint64_t))
274 pkt->
setLE<uint64_t>(
cr.completionAddr);
276 pkt->
setLE<uint32_t>(
bits(
cr.completionAddr,0,31));
279 assert(size ==
sizeof(uint32_t));
280 pkt->
setLE<uint32_t>(
bits(
cr.completionAddr,32,63));
283 assert(size ==
sizeof(uint32_t));
284 pkt->
setLE<uint32_t>(
cr.error());
287 panic(
"Read request to unknown channel register number: (%d)%#x\n",
301 panic(
"Invalid PCI memory access to unmapped memory.\n");
312 if (size ==
sizeof(uint64_t)) {
313 [[maybe_unused]] uint64_t
val = pkt->
getLE<uint64_t>();
314 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
316 }
else if (size ==
sizeof(uint32_t)) {
317 [[maybe_unused]] uint32_t
val = pkt->
getLE<uint32_t>();
318 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
320 }
else if (size ==
sizeof(uint16_t)) {
321 [[maybe_unused]] uint16_t
val = pkt->
getLE<uint16_t>();
322 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
324 }
else if (size ==
sizeof(uint8_t)) {
325 [[maybe_unused]] uint8_t
val = pkt->
getLE<uint8_t>();
326 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
329 panic(
"Unknown size for MMIO access: %d\n", size);
337 DPRINTF(DMACopyEngine,
"Warning, ignorning write to register %x\n",
341 regs.intrctrl.master_int_enable(
bits(pkt->
getLE<uint8_t>(), 0, 1));
344 panic(
"Read request to unknown register number: %#x\n", daddr);
353 while (daddr >= 0x80) {
358 if (chanid >=
regs.chanCount)
359 panic(
"Access to channel %d (device only configured for %d channels)",
360 chanid,
regs.chanCount);
365 chan[chanid]->channelWrite(pkt, daddr, size);
376 assert(size ==
sizeof(uint16_t));
378 old_int_disable =
cr.ctrl.interrupt_disable();
379 cr.ctrl(pkt->
getLE<uint16_t>());
380 if (
cr.ctrl.interrupt_disable())
381 cr.ctrl.interrupt_disable(0);
383 cr.ctrl.interrupt_disable(old_int_disable);
386 assert(size ==
sizeof(uint64_t));
387 DPRINTF(DMACopyEngine,
"Warning, ignorning write to register %x\n",
391 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
392 if (size ==
sizeof(uint64_t))
393 cr.descChainAddr = pkt->
getLE<uint64_t>();
395 cr.descChainAddr = (uint64_t)pkt->
getLE<uint32_t>() |
396 (
cr.descChainAddr &
~mask(32));
397 DPRINTF(DMACopyEngine,
"Chain Address %x\n",
cr.descChainAddr);
400 assert(size ==
sizeof(uint32_t));
401 cr.descChainAddr = ((uint64_t)pkt->
getLE<uint32_t>() << 32) |
402 (
cr.descChainAddr &
mask(32));
403 DPRINTF(DMACopyEngine,
"Chain Address %x\n",
cr.descChainAddr);
406 assert(size ==
sizeof(uint8_t));
407 cr.command(pkt->
getLE<uint8_t>());
411 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
412 if (size ==
sizeof(uint64_t))
413 cr.completionAddr = pkt->
getLE<uint64_t>();
415 cr.completionAddr = pkt->
getLE<uint32_t>() |
416 (
cr.completionAddr & ~
mask(32));
419 assert(size ==
sizeof(uint32_t));
420 cr.completionAddr = ((uint64_t)pkt->
getLE<uint32_t>() <<32) |
421 (
cr.completionAddr &
mask(32));
424 assert(size ==
sizeof(uint32_t));
425 cr.error(~pkt->
getLE<uint32_t>() &
cr.error());
428 panic(
"Read request to unknown channel register number: (%d)%#x\n",
435 const uint8_t &channel_count)
438 "Number of bytes copied by each engine"),
440 "Number of copies processed by each engine")
455 DPRINTF(DMACopyEngine,
"Reading descriptor from at memory location %#x(%#x)\n",
456 address,
ce->pciToDma(address));
460 DPRINTF(DMACopyEngine,
"dmaAction: %#x, %d bytes, to addr %#x\n",
472 DPRINTF(DMACopyEngine,
"Read of descriptor complete\n");
475 DPRINTF(DMACopyEngine,
"Got NULL descriptor, skipping\n");
478 panic(
"Shouldn't be able to get here\n");
491 panic(
"Descriptor has flag other that completion status set\n");
501 DPRINTF(DMACopyEngine,
"Reading %d bytes from buffer to memory location %#x(%#x)\n",
511 DPRINTF(DMACopyEngine,
"Read of bytes to copy complete\n");
521 DPRINTF(DMACopyEngine,
"Writing %d bytes from buffer to memory location %#x(%#x)\n",
535 DPRINTF(DMACopyEngine,
"Write of bytes to copy complete user1: %#x\n",
583 DPRINTF(DMACopyEngine,
"Writing completion status %#x to address %#x(%#x)\n",
585 ce->pciToDma(
cr.completionAddr));
588 ce->pciToDma(
cr.completionAddr),
596 DPRINTF(DMACopyEngine,
"Writing completion status complete\n");
603 DPRINTF(DMACopyEngine,
"Fetching next address...\n");
606 ce->pciToDma(address + offsetof(
DmaDesc, next)),
614 DPRINTF(DMACopyEngine,
"Fetching next address complete: %#x\n",
617 DPRINTF(DMACopyEngine,
"Got NULL descriptor, nothing more to do\n");
633 DPRINTF(Drain,
"CopyEngine done draining, processing drain event\n");
646 DPRINTF(Drain,
"CopyEngineChannel not drained\n");
656 for (
int x =0;
x <
chan.size();
x++)
665 for (
int x = 0;
x <
chan.size();
x++)
727 panic(
"Unknown state for CopyEngineChannel\n");
734 DPRINTF(DMACopyEngine,
"Restarting state machine at state %d\n",
nextState);
void unserialize(CheckpointIn &cp) override
Unserialize an object.
EventFunctionWrapper addrCompleteEvent
void restartStateMachine()
EventFunctionWrapper statusCompleteEvent
copy_engine_reg::DmaDesc * curDmaDesc
void continueProcessing()
void writeStatusComplete()
void fetchNextAddr(Addr address)
void writeCopyBytesComplete()
virtual ~CopyEngineChannel()
CopyEngineChannel(CopyEngine *_ce, int cid)
EventFunctionWrapper readCompleteEvent
void fetchDescriptor(Addr address)
void channelWrite(PacketPtr pkt, Addr daddr, int size)
void readCopyBytesComplete()
void writeCompletionStatus()
EventFunctionWrapper writeCompleteEvent
DrainState drain() override
Draining is the process of clearing out the states of SimObjects.These are the SimObjects that are pa...
copy_engine_reg::ChanRegs cr
void serialize(CheckpointOut &cp) const override
Serialize an object.
void drainResume() override
Resume execution after a successful drain.
void channelRead(PacketPtr pkt, Addr daddr, int size)
EventFunctionWrapper fetchCompleteEvent
uint64_t completionDataReg
Tick write(PacketPtr pkt) override
Pure virtual function that the device must implement.
Tick read(PacketPtr pkt) override
Pure virtual function that the device must implement.
void unserialize(CheckpointIn &cp) override
Unserialize an object.
void serialize(CheckpointOut &cp) const override
Serialize an object.
std::vector< CopyEngineChannel * > chan
Port & getPort(const std::string &if_name, PortID idx=InvalidPortID) override
Get a port with a given name and index.
CopyEngine(const Params ¶ms)
gem5::CopyEngine::CopyEngineStats copyEngineStats
copy_engine_reg::Regs regs
Port & getPort(const std::string &if_name, PortID idx=InvalidPortID) override
Get a port with a given name and index.
A Packet is used to encapsulate a transfer between two objects in the memory system (e....
void setLE(T v)
Set the value in the data pointer to v as little endian.
void makeAtomicResponse()
T getLE() const
Get the data in the packet byte swapped from little endian to host endian.
void serialize(CheckpointOut &cp) const override
Serialize this object to the given output stream.
bool getBAR(Addr addr, int &num, Addr &offs)
Which base address register (if any) maps the given address?
void unserialize(CheckpointIn &cp) override
Reconstruct the state of this object from a checkpoint.
PciEndpoint(const PciEndpointParams ¶ms)
Constructor for PCI Dev.
Ports are used to interface objects to each other.
#define ADD_STAT(n,...)
Convenience macro to add a stat to a statistics group.
constexpr int findMsbSet(uint64_t val)
Returns the bit position of the MSB that is set in the input.
constexpr T bits(T val, unsigned first, unsigned last)
Extract the bitfield from position 'first' to 'last' (inclusive) from 'val' and right justify it.
void signalDrainDone() const
Signal that an object is drained.
DrainState drainState() const
Return the current drain state of an object.
DrainState
Object drain/handover states.
@ Draining
Draining buffers pending serialization/handover.
@ Running
Running normally.
@ Drained
Buffers drained, ready for serialization/handover.
#define panic(...)
This implements a cprintf based panic() function.
#define fatal(...)
This implements a cprintf based fatal() function.
void serializeSection(CheckpointOut &cp, const char *name) const
Serialize an object into a new section.
#define UNSERIALIZE_ARRAY(member, size)
#define SERIALIZE_ARRAY(member, size)
void unserializeSection(CheckpointIn &cp, const char *name)
Unserialize an a child object.
const Params & params() const
const uint32_t GEN_XFERCAP
const uint32_t DESC_CTRL_NULL
const uint32_t CHAN_STATUS
const uint32_t DESC_CTRL_CP_STS
const uint32_t GEN_ATTNSTATUS
const uint32_t GEN_CHANCOUNT
const uint32_t CHAN_CMPLNADDR_HIGH
const uint32_t CHAN_CONTROL
const uint32_t CHAN_CMPLNADDR
const uint32_t CHAN_CHAINADDR
const uint32_t CHAN_CHAINADDR_HIGH
const uint32_t GEN_INTRCTRL
const uint32_t CHAN_ERROR
const uint32_t CHAN_COMMAND
const FlagsType total
Print the total.
Copyright (c) 2024 Arm Limited All rights reserved.
std::ostream CheckpointOut
uint64_t Addr
Address type This will probably be moved somewhere else in the near future.
int16_t PortID
Port index/ID type, and a symbolic name for an invalid port id.
void arrayParamOut(CheckpointOut &cp, const std::string &name, const CircleBuf< T > ¶m)
uint64_t Tick
Tick count type.
std::string csprintf(const char *format, const Args &...args)
void arrayParamIn(CheckpointIn &cp, const std::string &name, CircleBuf< T > ¶m)
Declaration of the Packet class.
#define UNSERIALIZE_SCALAR(scalar)
#define SERIALIZE_SCALAR(scalar)
statistics::Vector bytesCopied
statistics::Vector copiesProcessed
CopyEngineStats(statistics::Group *parent, const uint8_t &channel_count)
const std::string & name()