50 #include "debug/DMACopyEngine.hh"
51 #include "debug/Drain.hh"
54 #include "params/CopyEngine.hh"
69 fatal(
"CopyEngine interface doesn't support more than 64 DMA engines\n");
79 : cePort(_ce, _ce->sys),
80 ce(_ce), channelId(cid), busy(false), underReset(false),
81 refreshNext(false), latBeforeBegin(
ce->params()->latBeforeBegin),
82 latAfterCompletion(
ce->params()->latAfterCompletion),
83 completionDataReg(0), nextState(Idle),
85 addrCompleteEvent([
this]{ fetchAddrComplete(); },
name()),
86 readCompleteEvent([
this]{ readCopyBytesComplete(); },
name()),
87 writeCompleteEvent([
this]{ writeCopyBytesComplete(); },
name()),
88 statusCompleteEvent([
this]{ writeStatusComplete(); },
name())
91 cr.status.dma_transfer_status(3);
93 cr.completionAddr = 0;
96 memset(curDmaDesc, 0,
sizeof(
DmaDesc));
97 copyBuffer =
new uint8_t[
ce->params()->XferCap];
102 for (
int x = 0;
x <
chan.size();
x++) {
110 delete [] copyBuffer;
116 if (if_name !=
"dma") {
120 if (idx >=
static_cast<int>(
chan.size())) {
121 panic(
"CopyEngine::getPort: unknown index %d\n", idx);
124 return chan[idx]->getPort();
138 if (cr.command.start_dma()) {
140 cr.status.dma_transfer_status(0);
141 nextState = DescriptorFetch;
142 fetchAddress = cr.descChainAddr;
144 fetchDescriptor(cr.descChainAddr);
145 }
else if (cr.command.append_dma()) {
147 nextState = AddressFetch;
149 fetchNextAddr(lastDescriptorAddr);
152 }
else if (cr.command.reset_dma()) {
156 cr.status.dma_transfer_status(3);
159 }
else if (cr.command.resume_dma() || cr.command.abort_dma() ||
160 cr.command.suspend_dma())
161 panic(
"Resume, Abort, and Suspend are not supported\n");
172 panic(
"Invalid PCI memory access to unmapped memory.\n");
178 if (size !=
sizeof(uint64_t) && size !=
sizeof(uint32_t) &&
179 size !=
sizeof(uint16_t) && size !=
sizeof(uint8_t)) {
180 panic(
"Unknown size for MMIO access: %d\n", pkt->
getSize());
183 DPRINTF(DMACopyEngine,
"Read device register %#X size: %d\n", daddr, size);
200 assert(size ==
sizeof(uint8_t));
210 panic(
"Read request to unknown register number: %#x\n", daddr);
220 while (daddr >= 0x80) {
226 panic(
"Access to channel %d (device only configured for %d channels)",
232 chan[chanid]->channelRead(pkt, daddr, size);
243 assert(size ==
sizeof(uint16_t));
244 pkt->
setLE<uint16_t>(cr.ctrl());
248 assert(size ==
sizeof(uint64_t));
249 pkt->
setLE<uint64_t>(cr.status() | (busy ? 0 : 1));
252 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
253 if (size ==
sizeof(uint64_t))
254 pkt->
setLE<uint64_t>(cr.descChainAddr);
256 pkt->
setLE<uint32_t>(
bits(cr.descChainAddr,0,31));
259 assert(size ==
sizeof(uint32_t));
260 pkt->
setLE<uint32_t>(
bits(cr.descChainAddr,32,63));
263 assert(size ==
sizeof(uint8_t));
264 pkt->
setLE<uint32_t>(cr.command());
267 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
268 if (size ==
sizeof(uint64_t))
269 pkt->
setLE<uint64_t>(cr.completionAddr);
271 pkt->
setLE<uint32_t>(
bits(cr.completionAddr,0,31));
274 assert(size ==
sizeof(uint32_t));
275 pkt->
setLE<uint32_t>(
bits(cr.completionAddr,32,63));
278 assert(size ==
sizeof(uint32_t));
279 pkt->
setLE<uint32_t>(cr.error());
282 panic(
"Read request to unknown channel register number: (%d)%#x\n",
296 panic(
"Invalid PCI memory access to unmapped memory.\n");
307 if (size ==
sizeof(uint64_t)) {
308 uint64_t
val M5_VAR_USED = pkt->
getLE<uint64_t>();
309 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
311 }
else if (size ==
sizeof(uint32_t)) {
312 uint32_t
val M5_VAR_USED = pkt->
getLE<uint32_t>();
313 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
315 }
else if (size ==
sizeof(uint16_t)) {
316 uint16_t
val M5_VAR_USED = pkt->
getLE<uint16_t>();
317 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
319 }
else if (size ==
sizeof(uint8_t)) {
320 uint8_t
val M5_VAR_USED = pkt->
getLE<uint8_t>();
321 DPRINTF(DMACopyEngine,
"Wrote device register %#X value %#X\n",
324 panic(
"Unknown size for MMIO access: %d\n", size);
332 DPRINTF(DMACopyEngine,
"Warning, ignorning write to register %x\n",
339 panic(
"Read request to unknown register number: %#x\n", daddr);
348 while (daddr >= 0x80) {
354 panic(
"Access to channel %d (device only configured for %d channels)",
360 chan[chanid]->channelWrite(pkt, daddr, size);
371 assert(size ==
sizeof(uint16_t));
373 old_int_disable = cr.ctrl.interrupt_disable();
374 cr.ctrl(pkt->
getLE<uint16_t>());
375 if (cr.ctrl.interrupt_disable())
376 cr.ctrl.interrupt_disable(0);
378 cr.ctrl.interrupt_disable(old_int_disable);
381 assert(size ==
sizeof(uint64_t));
382 DPRINTF(DMACopyEngine,
"Warning, ignorning write to register %x\n",
386 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
387 if (size ==
sizeof(uint64_t))
388 cr.descChainAddr = pkt->
getLE<uint64_t>();
390 cr.descChainAddr = (uint64_t)pkt->
getLE<uint32_t>() |
391 (cr.descChainAddr & ~
mask(32));
392 DPRINTF(DMACopyEngine,
"Chain Address %x\n", cr.descChainAddr);
395 assert(size ==
sizeof(uint32_t));
396 cr.descChainAddr = ((uint64_t)pkt->
getLE<uint32_t>() << 32) |
397 (cr.descChainAddr &
mask(32));
398 DPRINTF(DMACopyEngine,
"Chain Address %x\n", cr.descChainAddr);
401 assert(size ==
sizeof(uint8_t));
402 cr.command(pkt->
getLE<uint8_t>());
406 assert(size ==
sizeof(uint64_t) || size ==
sizeof(uint32_t));
407 if (size ==
sizeof(uint64_t))
408 cr.completionAddr = pkt->
getLE<uint64_t>();
410 cr.completionAddr = pkt->
getLE<uint32_t>() |
411 (cr.completionAddr & ~
mask(32));
414 assert(size ==
sizeof(uint32_t));
415 cr.completionAddr = ((uint64_t)pkt->
getLE<uint32_t>() <<32) |
416 (cr.completionAddr &
mask(32));
419 assert(size ==
sizeof(uint32_t));
420 cr.error(~pkt->
getLE<uint32_t>() & cr.error());
423 panic(
"Read request to unknown channel register number: (%d)%#x\n",
433 using namespace Stats;
437 .
desc(
"Number of bytes copied by each engine")
443 .
desc(
"Number of copies processed by each engine")
451 DPRINTF(DMACopyEngine,
"Reading descriptor from at memory location %#x(%#x)\n",
452 address,
ce->pciToDma(address));
456 DPRINTF(DMACopyEngine,
"dmaAction: %#x, %d bytes, to addr %#x\n",
457 ce->pciToDma(address),
sizeof(
DmaDesc), curDmaDesc);
460 sizeof(
DmaDesc), &fetchCompleteEvent,
461 (uint8_t*)curDmaDesc, latBeforeBegin);
462 lastDescriptorAddr = address;
468 DPRINTF(DMACopyEngine,
"Read of descriptor complete\n");
471 DPRINTF(DMACopyEngine,
"Got NULL descriptor, skipping\n");
474 panic(
"Shouldn't be able to get here\n");
475 nextState = CompletionWrite;
476 if (inDrain())
return;
477 writeCompletionStatus();
487 panic(
"Descriptor has flag other that completion status set\n");
490 if (inDrain())
return;
497 DPRINTF(DMACopyEngine,
"Reading %d bytes from buffer to memory location %#x(%#x)\n",
498 curDmaDesc->len, curDmaDesc->dest,
499 ce->pciToDma(curDmaDesc->src));
501 curDmaDesc->len, &readCompleteEvent, copyBuffer, 0);
507 DPRINTF(DMACopyEngine,
"Read of bytes to copy complete\n");
509 nextState = DMAWrite;
510 if (inDrain())
return;
517 DPRINTF(DMACopyEngine,
"Writing %d bytes from buffer to memory location %#x(%#x)\n",
518 curDmaDesc->len, curDmaDesc->dest,
519 ce->pciToDma(curDmaDesc->dest));
522 curDmaDesc->len, &writeCompleteEvent, copyBuffer, 0);
524 ce->bytesCopied[channelId] += curDmaDesc->len;
525 ce->copiesProcessed[channelId]++;
531 DPRINTF(DMACopyEngine,
"Write of bytes to copy complete user1: %#x\n",
534 cr.status.compl_desc_addr(lastDescriptorAddr >> 6);
535 completionDataReg = cr.status() | 1;
538 nextState = CompletionWrite;
539 if (inDrain())
return;
540 writeCompletionStatus();
544 continueProcessing();
560 if (curDmaDesc->next) {
561 nextState = DescriptorFetch;
562 fetchAddress = curDmaDesc->next;
563 if (inDrain())
return;
564 fetchDescriptor(curDmaDesc->next);
565 }
else if (refreshNext) {
566 nextState = AddressFetch;
568 if (inDrain())
return;
569 fetchNextAddr(lastDescriptorAddr);
579 DPRINTF(DMACopyEngine,
"Writing completion status %#x to address %#x(%#x)\n",
580 completionDataReg, cr.completionAddr,
581 ce->pciToDma(cr.completionAddr));
584 ce->pciToDma(cr.completionAddr),
585 sizeof(completionDataReg), &statusCompleteEvent,
586 (uint8_t*)&completionDataReg, latAfterCompletion);
592 DPRINTF(DMACopyEngine,
"Writing completion status complete\n");
593 continueProcessing();
599 DPRINTF(DMACopyEngine,
"Fetching next address...\n");
602 ce->pciToDma(address + offsetof(
DmaDesc, next)),
603 sizeof(
Addr), &addrCompleteEvent,
604 (uint8_t*)curDmaDesc + offsetof(
DmaDesc, next), 0);
610 DPRINTF(DMACopyEngine,
"Fetching next address complete: %#x\n",
612 if (!curDmaDesc->next) {
613 DPRINTF(DMACopyEngine,
"Got NULL descriptor, nothing more to do\n");
619 nextState = DescriptorFetch;
620 fetchAddress = curDmaDesc->next;
621 if (inDrain())
return;
622 fetchDescriptor(curDmaDesc->next);
629 DPRINTF(Drain,
"CopyEngine done draining, processing drain event\n");
642 DPRINTF(Drain,
"CopyEngineChannel not drained\n");
652 for (
int x =0;
x <
chan.size();
x++)
661 for (
int x = 0;
x <
chan.size();
x++)
675 int nextState = this->nextState;
706 fetchNextAddr(lastDescriptorAddr);
708 case DescriptorFetch:
709 fetchDescriptor(fetchAddress);
717 case CompletionWrite:
718 writeCompletionStatus();
723 panic(
"Unknown state for CopyEngineChannel\n");
730 DPRINTF(DMACopyEngine,
"Restarting state machine at state %d\n", nextState);
731 restartStateMachine();
735 CopyEngineParams::create()