51 #include "debug/DMACopyEngine.hh"
52 #include "debug/Drain.hh"
55 #include "params/CopyEngine.hh"
62 using namespace copy_engine_reg;
66 copyEngineStats(this,
p.ChanCnt)
74 fatal(
"CopyEngine interface doesn't support more than 64 DMA engines\n");
84 : cePort(_ce, _ce->sys),
85 ce(_ce), channelId(cid), busy(false), underReset(false),
86 refreshNext(false), latBeforeBegin(
ce->params().latBeforeBegin),
87 latAfterCompletion(
ce->params().latAfterCompletion),
88 completionDataReg(0), nextState(Idle),
90 addrCompleteEvent([
this]{ fetchAddrComplete(); },
name()),
91 readCompleteEvent([
this]{ readCopyBytesComplete(); },
name()),
92 writeCompleteEvent([
this]{ writeCopyBytesComplete(); },
name()),
93 statusCompleteEvent([
this]{ writeStatusComplete(); },
name())
96 cr.status.dma_transfer_status(3);
98 cr.completionAddr = 0;
100 curDmaDesc =
new DmaDesc;
101 memset(curDmaDesc, 0,
sizeof(DmaDesc));
102 copyBuffer =
new uint8_t[
ce->params().XferCap];
107 for (
int x = 0;
x <
chan.size();
x++) {
115 delete [] copyBuffer;
121 if (if_name !=
"dma") {
125 if (idx >=
static_cast<int>(
chan.size())) {
126 panic(
"CopyEngine::getPort: unknown index %d\n", idx);
129 return chan[idx]->getPort();
143 if (cr.command.start_dma()) {
145 cr.status.dma_transfer_status(0);
146 nextState = DescriptorFetch;
147 fetchAddress = cr.descChainAddr;
149 fetchDescriptor(cr.descChainAddr);
150 }
else if (cr.command.append_dma()) {
152 nextState = AddressFetch;
154 fetchNextAddr(lastDescriptorAddr);
157 }
else if (cr.command.reset_dma()) {
161 cr.status.dma_transfer_status(3);
164 }
else if (cr.command.resume_dma() || cr.command.abort_dma() ||
165 cr.command.suspend_dma())
166 panic(
"Resume, Abort, and Suspend are not supported\n");
177 panic(
"Invalid PCI memory access to unmapped memory.\n");
183 if (size !=
sizeof(uint64_t) && size !=
sizeof(uint32_t) &&
184 size !=
sizeof(uint16_t) && size !=
sizeof(uint8_t)) {
185 panic(
"Unknown size for MMIO access: %d\n", pkt->
getSize());
188 DPRINTF(DMACopyEngine,
"Read device register %#X size: %d\n", daddr, size);
205 assert(size ==
sizeof(uint8_t));
215 panic(
"Read request to unknown register number: %#x\n", daddr);
225 while (daddr >= 0x80) {
231 panic(
"Access to channel %d (device only configured for %d channels)",
237 chan[chanid]->channelRead(pkt, daddr, size);
248 assert(size ==
sizeof(uint16_t));
249 pkt->
setLE<uint16_t>(cr.ctrl());
253 assert(size ==
sizeof(uint64_t));
254 pkt->
setLE<uint64_t>(cr.status() | (busy ? 0 : 1));
257 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
258 if (size ==
sizeof(uint64_t))
259 pkt->
setLE<uint64_t>(cr.descChainAddr);
261 pkt->
setLE<uint32_t>(
bits(cr.descChainAddr,0,31));
264 assert(size ==
sizeof(uint32_t));
265 pkt->
setLE<uint32_t>(
bits(cr.descChainAddr,32,63));
268 assert(size ==
sizeof(uint8_t));
269 pkt->
setLE<uint32_t>(cr.command());
272 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
273 if (size ==
sizeof(uint64_t))
274 pkt->
setLE<uint64_t>(cr.completionAddr);
276 pkt->
setLE<uint32_t>(
bits(cr.completionAddr,0,31));
279 assert(size ==
sizeof(uint32_t));
280 pkt->
setLE<uint32_t>(
bits(cr.completionAddr,32,63));
283 assert(size ==
sizeof(uint32_t));
284 pkt->
setLE<uint32_t>(cr.error());
287 panic(
"Read request to unknown channel register number: (%d)%#x\n",
301 panic(
"Invalid PCI memory access to unmapped memory.\n");
312 if (size ==
sizeof(uint64_t)) {
313 GEM5_VAR_USED uint64_t
val = pkt->
getLE<uint64_t>();
314 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
316 }
else if (size ==
sizeof(uint32_t)) {
317 GEM5_VAR_USED uint32_t
val = pkt->
getLE<uint32_t>();
318 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
320 }
else if (size ==
sizeof(uint16_t)) {
321 GEM5_VAR_USED uint16_t
val = pkt->
getLE<uint16_t>();
322 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
324 }
else if (size ==
sizeof(uint8_t)) {
325 GEM5_VAR_USED uint8_t
val = pkt->
getLE<uint8_t>();
326 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
329 panic(
"Unknown size for MMIO access: %d\n", size);
337 DPRINTF(DMACopyEngine,
"Warning, ignorning write to register %x\n",
344 panic(
"Read request to unknown register number: %#x\n", daddr);
353 while (daddr >= 0x80) {
359 panic(
"Access to channel %d (device only configured for %d channels)",
365 chan[chanid]->channelWrite(pkt, daddr, size);
376 assert(size ==
sizeof(uint16_t));
378 old_int_disable = cr.ctrl.interrupt_disable();
379 cr.ctrl(pkt->
getLE<uint16_t>());
380 if (cr.ctrl.interrupt_disable())
381 cr.ctrl.interrupt_disable(0);
383 cr.ctrl.interrupt_disable(old_int_disable);
386 assert(size ==
sizeof(uint64_t));
387 DPRINTF(DMACopyEngine,
"Warning, ignorning write to register %x\n",
391 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
392 if (size ==
sizeof(uint64_t))
393 cr.descChainAddr = pkt->
getLE<uint64_t>();
395 cr.descChainAddr = (uint64_t)pkt->
getLE<uint32_t>() |
396 (cr.descChainAddr & ~
mask(32));
397 DPRINTF(DMACopyEngine,
"Chain Address %x\n", cr.descChainAddr);
400 assert(size ==
sizeof(uint32_t));
401 cr.descChainAddr = ((uint64_t)pkt->
getLE<uint32_t>() << 32) |
402 (cr.descChainAddr &
mask(32));
403 DPRINTF(DMACopyEngine,
"Chain Address %x\n", cr.descChainAddr);
406 assert(size ==
sizeof(uint8_t));
407 cr.command(pkt->
getLE<uint8_t>());
411 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
412 if (size ==
sizeof(uint64_t))
413 cr.completionAddr = pkt->
getLE<uint64_t>();
415 cr.completionAddr = pkt->
getLE<uint32_t>() |
416 (cr.completionAddr & ~
mask(32));
419 assert(size ==
sizeof(uint32_t));
420 cr.completionAddr = ((uint64_t)pkt->
getLE<uint32_t>() <<32) |
421 (cr.completionAddr &
mask(32));
424 assert(size ==
sizeof(uint32_t));
425 cr.error(~pkt->
getLE<uint32_t>() & cr.error());
428 panic(
"Read request to unknown channel register number: (%d)%#x\n",
435 const uint8_t &channel_count)
436 : statistics::
Group(parent,
"CopyEngine"),
437 ADD_STAT(bytesCopied, statistics::units::Byte::get(),
438 "Number of bytes copied by each engine"),
439 ADD_STAT(copiesProcessed, statistics::units::Count::get(),
440 "Number of copies processed by each engine")
455 DPRINTF(DMACopyEngine,
"Reading descriptor from at memory location %#x(%#x)\n",
456 address,
ce->pciToDma(address));
460 DPRINTF(DMACopyEngine,
"dmaAction: %#x, %d bytes, to addr %#x\n",
461 ce->pciToDma(address),
sizeof(
DmaDesc), curDmaDesc);
464 sizeof(
DmaDesc), &fetchCompleteEvent,
465 (uint8_t*)curDmaDesc, latBeforeBegin);
466 lastDescriptorAddr = address;
472 DPRINTF(DMACopyEngine,
"Read of descriptor complete\n");
475 DPRINTF(DMACopyEngine,
"Got NULL descriptor, skipping\n");
478 panic(
"Shouldn't be able to get here\n");
479 nextState = CompletionWrite;
480 if (inDrain())
return;
481 writeCompletionStatus();
491 panic(
"Descriptor has flag other that completion status set\n");
494 if (inDrain())
return;
501 DPRINTF(DMACopyEngine,
"Reading %d bytes from buffer to memory location %#x(%#x)\n",
502 curDmaDesc->len, curDmaDesc->dest,
503 ce->pciToDma(curDmaDesc->src));
505 curDmaDesc->len, &readCompleteEvent, copyBuffer, 0);
511 DPRINTF(DMACopyEngine,
"Read of bytes to copy complete\n");
513 nextState = DMAWrite;
514 if (inDrain())
return;
521 DPRINTF(DMACopyEngine,
"Writing %d bytes from buffer to memory location %#x(%#x)\n",
522 curDmaDesc->len, curDmaDesc->dest,
523 ce->pciToDma(curDmaDesc->dest));
526 curDmaDesc->len, &writeCompleteEvent, copyBuffer, 0);
528 ce->copyEngineStats.bytesCopied[channelId] += curDmaDesc->len;
529 ce->copyEngineStats.copiesProcessed[channelId]++;
535 DPRINTF(DMACopyEngine,
"Write of bytes to copy complete user1: %#x\n",
538 cr.status.compl_desc_addr(lastDescriptorAddr >> 6);
539 completionDataReg = cr.status() | 1;
542 nextState = CompletionWrite;
543 if (inDrain())
return;
544 writeCompletionStatus();
548 continueProcessing();
564 if (curDmaDesc->next) {
565 nextState = DescriptorFetch;
566 fetchAddress = curDmaDesc->next;
567 if (inDrain())
return;
568 fetchDescriptor(curDmaDesc->next);
569 }
else if (refreshNext) {
570 nextState = AddressFetch;
572 if (inDrain())
return;
573 fetchNextAddr(lastDescriptorAddr);
583 DPRINTF(DMACopyEngine,
"Writing completion status %#x to address %#x(%#x)\n",
584 completionDataReg, cr.completionAddr,
585 ce->pciToDma(cr.completionAddr));
588 ce->pciToDma(cr.completionAddr),
589 sizeof(completionDataReg), &statusCompleteEvent,
590 (uint8_t*)&completionDataReg, latAfterCompletion);
596 DPRINTF(DMACopyEngine,
"Writing completion status complete\n");
597 continueProcessing();
603 DPRINTF(DMACopyEngine,
"Fetching next address...\n");
606 ce->pciToDma(address + offsetof(
DmaDesc, next)),
607 sizeof(
Addr), &addrCompleteEvent,
608 (uint8_t*)curDmaDesc + offsetof(
DmaDesc, next), 0);
614 DPRINTF(DMACopyEngine,
"Fetching next address complete: %#x\n",
616 if (!curDmaDesc->next) {
617 DPRINTF(DMACopyEngine,
"Got NULL descriptor, nothing more to do\n");
623 nextState = DescriptorFetch;
624 fetchAddress = curDmaDesc->next;
625 if (inDrain())
return;
626 fetchDescriptor(curDmaDesc->next);
633 DPRINTF(Drain,
"CopyEngine done draining, processing drain event\n");
646 DPRINTF(Drain,
"CopyEngineChannel not drained\n");
656 for (
int x =0;
x <
chan.size();
x++)
665 for (
int x = 0;
x <
chan.size();
x++)
679 int nextState = this->nextState;
710 fetchNextAddr(lastDescriptorAddr);
712 case DescriptorFetch:
713 fetchDescriptor(fetchAddress);
721 case CompletionWrite:
722 writeCompletionStatus();
727 panic(
"Unknown state for CopyEngineChannel\n");
734 DPRINTF(DMACopyEngine,
"Restarting state machine at state %d\n", nextState);
735 restartStateMachine();