51#include "debug/DMACopyEngine.hh"
52#include "debug/Drain.hh"
55#include "params/CopyEngine.hh"
62using namespace copy_engine_reg;
66 copyEngineStats(this,
p.ChanCnt)
74 fatal(
"CopyEngine interface doesn't support more than 64 DMA engines\n");
84 : cePort(_ce, _ce->sys),
85 ce(_ce), channelId(cid), busy(false), underReset(false),
86 refreshNext(false), latBeforeBegin(
ce->params().latBeforeBegin),
87 latAfterCompletion(
ce->params().latAfterCompletion),
88 completionDataReg(0), nextState(Idle),
90 addrCompleteEvent([
this]{ fetchAddrComplete(); },
name()),
91 readCompleteEvent([
this]{ readCopyBytesComplete(); },
name()),
92 writeCompleteEvent([
this]{ writeCopyBytesComplete(); },
name()),
93 statusCompleteEvent([
this]{ writeStatusComplete(); },
name())
96 cr.status.dma_transfer_status(3);
98 cr.completionAddr = 0;
100 curDmaDesc =
new DmaDesc;
101 memset(curDmaDesc, 0,
sizeof(DmaDesc));
102 copyBuffer =
new uint8_t[
ce->params().XferCap];
107 for (
int x = 0;
x <
chan.size();
x++) {
115 delete [] copyBuffer;
121 if (if_name !=
"dma") {
125 if (idx >=
static_cast<int>(
chan.size())) {
126 panic(
"CopyEngine::getPort: unknown index %d\n", idx);
129 return chan[idx]->getPort();
143 if (cr.command.start_dma()) {
145 cr.status.dma_transfer_status(0);
146 nextState = DescriptorFetch;
147 fetchAddress = cr.descChainAddr;
149 fetchDescriptor(cr.descChainAddr);
150 }
else if (cr.command.append_dma()) {
152 nextState = AddressFetch;
154 fetchNextAddr(lastDescriptorAddr);
157 }
else if (cr.command.reset_dma()) {
161 cr.status.dma_transfer_status(3);
164 }
else if (cr.command.resume_dma() || cr.command.abort_dma() ||
165 cr.command.suspend_dma())
166 panic(
"Resume, Abort, and Suspend are not supported\n");
177 panic(
"Invalid PCI memory access to unmapped memory.\n");
183 if (size !=
sizeof(uint64_t) && size !=
sizeof(uint32_t) &&
184 size !=
sizeof(uint16_t) && size !=
sizeof(uint8_t)) {
185 panic(
"Unknown size for MMIO access: %d\n", pkt->
getSize());
188 DPRINTF(DMACopyEngine,
"Read device register %#X size: %d\n", daddr, size);
205 assert(size ==
sizeof(uint8_t));
215 panic(
"Read request to unknown register number: %#x\n", daddr);
225 while (daddr >= 0x80) {
231 panic(
"Access to channel %d (device only configured for %d channels)",
237 chan[chanid]->channelRead(pkt, daddr, size);
248 assert(size ==
sizeof(uint16_t));
249 pkt->
setLE<uint16_t>(cr.ctrl());
253 assert(size ==
sizeof(uint64_t));
254 pkt->
setLE<uint64_t>(cr.status() | (busy ? 0 : 1));
257 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
258 if (size ==
sizeof(uint64_t))
259 pkt->
setLE<uint64_t>(cr.descChainAddr);
261 pkt->
setLE<uint32_t>(
bits(cr.descChainAddr,0,31));
264 assert(size ==
sizeof(uint32_t));
265 pkt->
setLE<uint32_t>(
bits(cr.descChainAddr,32,63));
268 assert(size ==
sizeof(uint8_t));
269 pkt->
setLE<uint32_t>(cr.command());
272 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
273 if (size ==
sizeof(uint64_t))
274 pkt->
setLE<uint64_t>(cr.completionAddr);
276 pkt->
setLE<uint32_t>(
bits(cr.completionAddr,0,31));
279 assert(size ==
sizeof(uint32_t));
280 pkt->
setLE<uint32_t>(
bits(cr.completionAddr,32,63));
283 assert(size ==
sizeof(uint32_t));
284 pkt->
setLE<uint32_t>(cr.error());
287 panic(
"Read request to unknown channel register number: (%d)%#x\n",
301 panic(
"Invalid PCI memory access to unmapped memory.\n");
312 if (size ==
sizeof(uint64_t)) {
313 [[maybe_unused]] uint64_t
val = pkt->
getLE<uint64_t>();
314 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
316 }
else if (size ==
sizeof(uint32_t)) {
317 [[maybe_unused]] uint32_t
val = pkt->
getLE<uint32_t>();
318 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
320 }
else if (size ==
sizeof(uint16_t)) {
321 [[maybe_unused]] uint16_t
val = pkt->
getLE<uint16_t>();
322 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
324 }
else if (size ==
sizeof(uint8_t)) {
325 [[maybe_unused]] uint8_t
val = pkt->
getLE<uint8_t>();
326 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
329 panic(
"Unknown size for MMIO access: %d\n", size);
337 DPRINTF(DMACopyEngine,
"Warning, ignorning write to register %x\n",
344 panic(
"Read request to unknown register number: %#x\n", daddr);
353 while (daddr >= 0x80) {
359 panic(
"Access to channel %d (device only configured for %d channels)",
365 chan[chanid]->channelWrite(pkt, daddr, size);
376 assert(size ==
sizeof(uint16_t));
378 old_int_disable = cr.ctrl.interrupt_disable();
379 cr.ctrl(pkt->
getLE<uint16_t>());
380 if (cr.ctrl.interrupt_disable())
381 cr.ctrl.interrupt_disable(0);
383 cr.ctrl.interrupt_disable(old_int_disable);
386 assert(size ==
sizeof(uint64_t));
387 DPRINTF(DMACopyEngine,
"Warning, ignorning write to register %x\n",
391 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
392 if (size ==
sizeof(uint64_t))
393 cr.descChainAddr = pkt->
getLE<uint64_t>();
395 cr.descChainAddr = (uint64_t)pkt->
getLE<uint32_t>() |
396 (cr.descChainAddr & ~mask(32));
397 DPRINTF(DMACopyEngine,
"Chain Address %x\n", cr.descChainAddr);
400 assert(size ==
sizeof(uint32_t));
401 cr.descChainAddr = ((uint64_t)pkt->
getLE<uint32_t>() << 32) |
402 (cr.descChainAddr &
mask(32));
403 DPRINTF(DMACopyEngine,
"Chain Address %x\n", cr.descChainAddr);
406 assert(size ==
sizeof(uint8_t));
407 cr.command(pkt->
getLE<uint8_t>());
411 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
412 if (size ==
sizeof(uint64_t))
413 cr.completionAddr = pkt->
getLE<uint64_t>();
415 cr.completionAddr = pkt->
getLE<uint32_t>() |
416 (cr.completionAddr & ~
mask(32));
419 assert(size ==
sizeof(uint32_t));
420 cr.completionAddr = ((uint64_t)pkt->
getLE<uint32_t>() <<32) |
421 (cr.completionAddr &
mask(32));
424 assert(size ==
sizeof(uint32_t));
425 cr.error(~pkt->
getLE<uint32_t>() & cr.error());
428 panic(
"Read request to unknown channel register number: (%d)%#x\n",
435 const uint8_t &channel_count)
436 : statistics::
Group(parent,
"CopyEngine"),
437 ADD_STAT(bytesCopied, statistics::units::Byte::get(),
438 "Number of bytes copied by each engine"),
439 ADD_STAT(copiesProcessed, statistics::units::Count::get(),
440 "Number of copies processed by each engine")
455 DPRINTF(DMACopyEngine,
"Reading descriptor from at memory location %#x(%#x)\n",
456 address,
ce->pciToDma(address));
460 DPRINTF(DMACopyEngine,
"dmaAction: %#x, %d bytes, to addr %#x\n",
461 ce->pciToDma(address),
sizeof(
DmaDesc), curDmaDesc);
464 sizeof(
DmaDesc), &fetchCompleteEvent,
465 (uint8_t*)curDmaDesc, latBeforeBegin);
466 lastDescriptorAddr = address;
472 DPRINTF(DMACopyEngine,
"Read of descriptor complete\n");
475 DPRINTF(DMACopyEngine,
"Got NULL descriptor, skipping\n");
478 panic(
"Shouldn't be able to get here\n");
479 nextState = CompletionWrite;
480 if (inDrain())
return;
481 writeCompletionStatus();
491 panic(
"Descriptor has flag other that completion status set\n");
494 if (inDrain())
return;
501 DPRINTF(DMACopyEngine,
"Reading %d bytes from buffer to memory location %#x(%#x)\n",
502 curDmaDesc->len, curDmaDesc->dest,
503 ce->pciToDma(curDmaDesc->src));
505 curDmaDesc->len, &readCompleteEvent, copyBuffer, 0);
511 DPRINTF(DMACopyEngine,
"Read of bytes to copy complete\n");
513 nextState = DMAWrite;
514 if (inDrain())
return;
521 DPRINTF(DMACopyEngine,
"Writing %d bytes from buffer to memory location %#x(%#x)\n",
522 curDmaDesc->len, curDmaDesc->dest,
523 ce->pciToDma(curDmaDesc->dest));
526 curDmaDesc->len, &writeCompleteEvent, copyBuffer, 0);
528 ce->copyEngineStats.bytesCopied[channelId] += curDmaDesc->len;
529 ce->copyEngineStats.copiesProcessed[channelId]++;
535 DPRINTF(DMACopyEngine,
"Write of bytes to copy complete user1: %#x\n",
538 cr.status.compl_desc_addr(lastDescriptorAddr >> 6);
539 completionDataReg = cr.status() | 1;
542 nextState = CompletionWrite;
543 if (inDrain())
return;
544 writeCompletionStatus();
548 continueProcessing();
564 if (curDmaDesc->next) {
565 nextState = DescriptorFetch;
566 fetchAddress = curDmaDesc->next;
567 if (inDrain())
return;
568 fetchDescriptor(curDmaDesc->next);
569 }
else if (refreshNext) {
570 nextState = AddressFetch;
572 if (inDrain())
return;
573 fetchNextAddr(lastDescriptorAddr);
583 DPRINTF(DMACopyEngine,
"Writing completion status %#x to address %#x(%#x)\n",
584 completionDataReg, cr.completionAddr,
585 ce->pciToDma(cr.completionAddr));
588 ce->pciToDma(cr.completionAddr),
589 sizeof(completionDataReg), &statusCompleteEvent,
590 (uint8_t*)&completionDataReg, latAfterCompletion);
596 DPRINTF(DMACopyEngine,
"Writing completion status complete\n");
597 continueProcessing();
603 DPRINTF(DMACopyEngine,
"Fetching next address...\n");
606 ce->pciToDma(address + offsetof(
DmaDesc, next)),
607 sizeof(
Addr), &addrCompleteEvent,
608 (uint8_t*)curDmaDesc + offsetof(
DmaDesc, next), 0);
614 DPRINTF(DMACopyEngine,
"Fetching next address complete: %#x\n",
616 if (!curDmaDesc->next) {
617 DPRINTF(DMACopyEngine,
"Got NULL descriptor, nothing more to do\n");
623 nextState = DescriptorFetch;
624 fetchAddress = curDmaDesc->next;
625 if (inDrain())
return;
626 fetchDescriptor(curDmaDesc->next);
633 DPRINTF(Drain,
"CopyEngine done draining, processing drain event\n");
646 DPRINTF(Drain,
"CopyEngineChannel not drained\n");
656 for (
int x =0;
x <
chan.size();
x++)
665 for (
int x = 0;
x <
chan.size();
x++)
679 int nextState = this->nextState;
710 fetchNextAddr(lastDescriptorAddr);
712 case DescriptorFetch:
713 fetchDescriptor(fetchAddress);
721 case CompletionWrite:
722 writeCompletionStatus();
727 panic(
"Unknown state for CopyEngineChannel\n");
734 DPRINTF(DMACopyEngine,
"Restarting state machine at state %d\n", nextState);
735 restartStateMachine();
void unserialize(CheckpointIn &cp) override
Unserialize an object.
void restartStateMachine()
void continueProcessing()
void writeStatusComplete()
void fetchNextAddr(Addr address)
void writeCopyBytesComplete()
virtual ~CopyEngineChannel()
CopyEngineChannel(CopyEngine *_ce, int cid)
void fetchDescriptor(Addr address)
void channelWrite(PacketPtr pkt, Addr daddr, int size)
void readCopyBytesComplete()
void writeCompletionStatus()
DrainState drain() override
Draining is the process of clearing out the states of SimObjects.These are the SimObjects that are pa...
void serialize(CheckpointOut &cp) const override
Serialize an object.
void drainResume() override
Resume execution after a successful drain.
void channelRead(PacketPtr pkt, Addr daddr, int size)
Tick write(PacketPtr pkt) override
Pure virtual function that the device must implement.
Tick read(PacketPtr pkt) override
Pure virtual function that the device must implement.
void unserialize(CheckpointIn &cp) override
Unserialize an object.
void serialize(CheckpointOut &cp) const override
Serialize an object.
std::vector< CopyEngineChannel * > chan
Port & getPort(const std::string &if_name, PortID idx=InvalidPortID) override
Get a port with a given name and index.
CopyEngine(const Params ¶ms)
copy_engine_reg::Regs regs
Port & getPort(const std::string &if_name, PortID idx=InvalidPortID) override
Get a port with a given name and index.
A Packet is used to encapsulate a transfer between two objects in the memory system (e....
void setLE(T v)
Set the value in the data pointer to v as little endian.
void makeAtomicResponse()
T getLE() const
Get the data in the packet byte swapped from little endian to host endian.
PCI device, base implementation is only config space.
void unserialize(CheckpointIn &cp) override
Reconstruct the state of this object from a checkpoint.
void serialize(CheckpointOut &cp) const override
Serialize this object to the given output stream.
bool getBAR(Addr addr, int &num, Addr &offs)
Which base address register (if any) maps the given address?
Ports are used to interface objects to each other.
Derived & flags(Flags _flags)
Set the flags and marks this stat to print at the end of simulation.
Derived & init(size_type size)
Set this vector to have the given size.
#define ADD_STAT(n,...)
Convenience macro to add a stat to a statistics group.
constexpr int findMsbSet(uint64_t val)
Returns the bit position of the MSB that is set in the input.
constexpr T bits(T val, unsigned first, unsigned last)
Extract the bitfield from position 'first' to 'last' (inclusive) from 'val' and right justify it.
void signalDrainDone() const
Signal that an object is drained.
DrainState drainState() const
Return the current drain state of an object.
DrainState
Object drain/handover states.
@ Draining
Draining buffers pending serialization/handover.
@ Running
Running normally.
@ Drained
Buffers drained, ready for serialization/handover.
#define panic(...)
This implements a cprintf based panic() function.
#define fatal(...)
This implements a cprintf based fatal() function.
void serializeSection(CheckpointOut &cp, const char *name) const
Serialize an object into a new section.
#define UNSERIALIZE_ARRAY(member, size)
#define SERIALIZE_ARRAY(member, size)
void unserializeSection(CheckpointIn &cp, const char *name)
Unserialize an a child object.
const uint32_t GEN_XFERCAP
const uint32_t DESC_CTRL_NULL
const uint32_t CHAN_STATUS
const uint32_t DESC_CTRL_CP_STS
const uint32_t GEN_ATTNSTATUS
const uint32_t GEN_CHANCOUNT
const uint32_t CHAN_CMPLNADDR_HIGH
const uint32_t CHAN_CONTROL
const uint32_t CHAN_CMPLNADDR
const uint32_t CHAN_CHAINADDR
const uint32_t CHAN_CHAINADDR_HIGH
const uint32_t GEN_INTRCTRL
const uint32_t CHAN_ERROR
const uint32_t CHAN_COMMAND
const FlagsType total
Print the total.
Copyright (c) 2024 - Pranith Kumar Copyright (c) 2020 Inria All rights reserved.
std::ostream CheckpointOut
uint64_t Addr
Address type This will probably be moved somewhere else in the near future.
int16_t PortID
Port index/ID type, and a symbolic name for an invalid port id.
void arrayParamOut(CheckpointOut &cp, const std::string &name, const CircleBuf< T > ¶m)
uint64_t Tick
Tick count type.
std::string csprintf(const char *format, const Args &...args)
void arrayParamIn(CheckpointIn &cp, const std::string &name, CircleBuf< T > ¶m)
Declaration of the Packet class.
#define UNSERIALIZE_SCALAR(scalar)
#define SERIALIZE_SCALAR(scalar)
statistics::Vector bytesCopied
statistics::Vector copiesProcessed
CopyEngineStats(statistics::Group *parent, const uint8_t &channel_count)
void unserialize(CheckpointIn &cp) override
Unserialize an object.
void serialize(CheckpointOut &cp) const override
Serialize an object.
const std::string & name()