tlb.cc

Go to the documentation of this file.

/*
 * Copyright (c) 2021 Advanced Micro Devices, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
 
#include "arch/amdgpu/vega/tlb.hh"
 
#include <cmath>
#include <cstring>
 
#include "arch/amdgpu/common/gpu_translation_state.hh"
#include "arch/amdgpu/vega/faults.hh"
#include "arch/amdgpu/vega/pagetable_walker.hh"
#include "debug/GPUPrefetch.hh"
#include "debug/GPUTLB.hh"
#include "dev/amdgpu/amdgpu_device.hh"
 
namespace gem5
{
namespace VegaISA
{
 
// we have no limit for the number of translations we send
// downstream as we depend on the limit of the coalescer
// above us
GpuTLB::GpuTLB(const VegaGPUTLBParams &p)
    :  ClockedObject(p), walker(p.walker),
      gpuDevice(p.gpu_device), size(p.size), stats(this),
      cleanupEvent([this]{ cleanup(); }, name(), false,
                   Event::Maximum_Pri)
{
    assoc = p.assoc;
    assert(assoc <= size);
    numSets = size/assoc;
    allocationPolicy = p.allocationPolicy;
    hasMemSidePort = false;
 
    tlb.assign(size, VegaTlbEntry());
 
    freeList.resize(numSets);
    entryList.resize(numSets);
 
    for (int set = 0; set < numSets; ++set) {
        for (int way = 0; way < assoc; ++way) {
            int x = set * assoc + way;
            freeList[set].push_back(&tlb.at(x));
        }
    }
 
    FA = (size == assoc);
    setMask = numSets - 1;
 
    maxCoalescedReqs = p.maxOutstandingReqs;
 
 
    outstandingReqs = 0;
    hitLatency = p.hitLatency;
    missLatency1 = p.missLatency1;
    missLatency2 = p.missLatency2;
 
    // create the response ports based on the number of connected ports
    for (size_t i = 0; i < p.port_cpu_side_ports_connection_count; ++i) {
        cpuSidePort.push_back(new CpuSidePort(csprintf("%s-port%d",
                              name(), i), this, i));
    }
 
    // create the requestor ports based on the number of connected ports
    for (size_t i = 0; i < p.port_mem_side_ports_connection_count; ++i) {
        memSidePort.push_back(new MemSidePort(csprintf("%s-port%d",
                              name(), i), this, i));
    }
 
    // assuming one walker per TLB, set our walker's TLB to this TLB.
    walker->setTLB(this);
 
    // gpuDevice should be non-null in full system only and is set by GpuTLB
    // params from the config file.
    if (gpuDevice) {
        gpuDevice->getVM().registerTLB(this);
    }
}
 
GpuTLB::~GpuTLB()
{
}
 
Port &
GpuTLB::getPort(const std::string &if_name, PortID idx)
{
    if (if_name == "cpu_side_ports") {
        if (idx >= static_cast<PortID>(cpuSidePort.size())) {
            panic("TLBCoalescer::getPort: unknown index %d\n", idx);
        }
 
        return *cpuSidePort[idx];
    } else if (if_name == "mem_side_ports") {
        if (idx >= static_cast<PortID>(memSidePort.size())) {
            panic("TLBCoalescer::getPort: unknown index %d\n", idx);
        }
 
        hasMemSidePort = true;
 
        return *memSidePort[idx];
    } else {
        panic("TLBCoalescer::getPort: unknown port %s\n", if_name);
    }
}
 
Fault
GpuTLB::createPagefault(Addr vaddr, Mode mode)
{
    DPRINTF(GPUTLB, "GPUTLB: Raising page fault.\n");
    ExceptionCode code;
    if (mode == BaseMMU::Read)
        code = ExceptionCode::LOAD_PAGE;
    else if (mode == BaseMMU::Write)
        code = ExceptionCode::STORE_PAGE;
    else
        code = ExceptionCode::INST_PAGE;
    return std::make_shared<PageFault>(vaddr, code, true, mode, true);
}
 
Addr
GpuTLB::pageAlign(Addr vaddr)
{
    Addr pageMask = mask(VegaISA::PageShift);
    return (vaddr & ~pageMask);
}
 
int
GpuTLB::getSet(Addr va, unsigned int page_shift)
{
    return (va >> page_shift) & setMask;
}
 
VegaTlbEntry*
GpuTLB::insert(Addr vpn, VegaTlbEntry &entry)
{
    VegaTlbEntry *newEntry = nullptr;
 
    int set = getSet(entry.vaddr, entry.logBytes);
 
    if (!freeList[set].empty()) {
        newEntry = freeList[set].front();
        freeList[set].pop_front();
    } else {
        newEntry = entryList[set].back();
        entryList[set].pop_back();
    }
 
    *newEntry = entry;
    entryList[set].push_front(newEntry);
 
    DPRINTF(GPUTLB, "Inserted %#lx -> %#lx of size %#lx into set %d\n",
            newEntry->vaddr, newEntry->paddr, entry.size(), set);
 
    return newEntry;
}
 
GpuTLB::EntryList::iterator
GpuTLB::lookupIt(Addr va, unsigned int ps, bool update_lru)
{
    int set = getSet(va, ps);
 
    if (FA) {
        assert(!set);
    }
 
    auto entry = entryList[set].begin();
    for (; entry != entryList[set].end(); ++entry) {
        int page_size = (*entry)->size();
 
        if ((*entry)->vaddr <= va && (*entry)->vaddr + page_size > va &&
            ps == (*entry)->logBytes) {
            DPRINTF(GPUTLB, "Matched vaddr %#x to entry starting at %#x "
                    "with size %#x.\n", va, (*entry)->vaddr, page_size);
 
            if (update_lru) {
                entryList[set].push_front(*entry);
                entryList[set].erase(entry);
                entry = entryList[set].begin();
            }
 
            break;
        }
    }
 
    return entry;
}
 
VegaTlbEntry*
GpuTLB::lookup(Addr va, bool update_lru)
{
    for (auto ps : logPageShiftList) {
        int set = getSet(va, ps);
 
        auto entry = lookupIt(va, ps, update_lru);
 
        if (entry == entryList[set].end())
            continue;
        else
            return *entry;
    }
    return nullptr;
}
 
void
GpuTLB::invalidateAll()
{
    DPRINTF(GPUTLB, "Invalidating all entries.\n");
 
    for (int i = 0; i < numSets; ++i) {
        while (!entryList[i].empty()) {
            VegaTlbEntry *entry = entryList[i].front();
            entryList[i].pop_front();
            freeList[i].push_back(entry);
        }
    }
}
 
void
GpuTLB::demapPage(Addr va, uint64_t asn)
{
    DPRINTF(GPUTLB, "Demapping vaddr %#x.\n", va);
    for (auto ps : logPageShiftList) {
        int set = getSet(va, ps);
        auto entry = lookupIt(va, ps, false);
 
        if (entry != entryList[set].end()) {
            freeList[set].push_back(*entry);
            entryList[set].erase(entry);
        }
    }
}
 
 

VegaTlbEntry *
GpuTLB::tlbLookup(const RequestPtr &req, bool update_stats)
{
    if (req->hasNoAddr()) {
        return NULL;
    }
    Addr vaddr = req->getVaddr();
    Addr alignedVaddr = pageAlign(vaddr);
    DPRINTF(GPUTLB, "TLB Lookup for vaddr %#x.\n", vaddr);
 
    //update LRU stack on a hit
    VegaTlbEntry *entry = lookup(alignedVaddr, true);
 
    if (!update_stats) {
        // functional tlb access for memory initialization
        // i.e., memory seeding or instr. seeding -> don't update
        // TLB and stats
        return entry;
    }
 
    stats.localNumTLBAccesses++;
 
    if (!entry) {
        stats.localNumTLBMisses++;
    } else {
        stats.localNumTLBHits++;
    }
 
    return entry;
}
 
Walker*
GpuTLB::getWalker()
{
    return walker;
}
 
 
void
GpuTLB::serialize(CheckpointOut &cp) const
{
}
 
void
GpuTLB::unserialize(CheckpointIn &cp)
{
}

 
void
GpuTLB::issueTLBLookup(PacketPtr pkt)
{
    assert(pkt);
    assert(pkt->senderState);

    Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                    VegaISA::PageBytes);
 
    GpuTranslationState *sender_state =
            safe_cast<GpuTranslationState*>(pkt->senderState);
 
    bool update_stats = !sender_state->isPrefetch;
 
    DPRINTF(GPUTLB, "Translation req. for virt. page addr %#x\n",
            virt_page_addr);
 
    int req_cnt = sender_state->reqCnt.back();
 
    if (update_stats) {
        stats.accessCycles -= (curCycle() * req_cnt);
        stats.localCycles -= curCycle();
        stats.globalNumTLBAccesses += req_cnt;
    }
 
    tlbOutcome lookup_outcome = TLB_MISS;
    const RequestPtr &tmp_req = pkt->req;
 
    // Access the TLB and figure out if it's a hit or a miss.
    auto entry = tlbLookup(tmp_req, update_stats);
    if (entry || pkt->req->hasNoAddr()) {
        // Put the entry in SenderState
        lookup_outcome = TLB_HIT;
        if (pkt->req->hasNoAddr()) {
            sender_state->tlbEntry =
                new VegaTlbEntry(1 /* VMID */, 0, 0, 0, 0);
            // set false because we shouldn't go to
            // host memory for a memtime request
            pkt->req->setSystemReq(false);
        } else {
            VegaTlbEntry *entry = lookup(virt_page_addr, false);
            assert(entry);
 
            // Set if this is a system request
            pkt->req->setSystemReq(entry->pte.s);
 
            sender_state->tlbEntry =
                new VegaTlbEntry(*entry);
        }
 
        if (update_stats) {
            // the reqCnt has an entry per level, so its size tells us
            // which level we are in
            sender_state->hitLevel = sender_state->reqCnt.size();
            stats.globalNumTLBHits += req_cnt;
        }
    } else {
        if (update_stats)
            stats.globalNumTLBMisses += req_cnt;
    }
 
    /*
     * We now know the TLB lookup outcome (if it's a hit or a miss), as
     * well as the TLB access latency.
     *
     * We create and schedule a new TLBEvent which will help us take the
     * appropriate actions (e.g., update TLB on a hit, send request to
     * lower level TLB on a miss, or start a page walk if this was the
     * last-level TLB)
     */
    TLBEvent *tlb_event =
        new TLBEvent(this, virt_page_addr, lookup_outcome, pkt);
 
    if (translationReturnEvent.count(virt_page_addr)) {
        panic("Virtual Page Address %#x already has a return event\n",
              virt_page_addr);
    }
 
    translationReturnEvent[virt_page_addr] = tlb_event;
    assert(tlb_event);
 
    DPRINTF(GPUTLB, "schedule translationReturnEvent @ curTick %d\n",
            curTick() + cyclesToTicks(Cycles(hitLatency)));
 
    schedule(tlb_event, curTick() + cyclesToTicks(Cycles(hitLatency)));
}
 
GpuTLB::TLBEvent::TLBEvent(GpuTLB* _tlb, Addr _addr,
    tlbOutcome tlb_outcome, PacketPtr _pkt)
        : Event(CPU_Tick_Pri), tlb(_tlb), virtPageAddr(_addr),
          outcome(tlb_outcome), pkt(_pkt)
{
}

void
GpuTLB::pagingProtectionChecks(PacketPtr pkt, VegaTlbEntry * tlb_entry,
        Mode mode)
{
    // Do paging protection checks.
    bool badWrite = (!tlb_entry->writable());
 
    if (mode == BaseMMU::Write && badWrite) {
        // The page must have been present to get into the TLB in
        // the first place. We'll assume the reserved bits are
        // fine even though we're not checking them.
        fatal("Page fault on addr %lx PTE=%#lx", pkt->req->getVaddr(),
                (uint64_t)tlb_entry->pte);
    }
}
 
void
GpuTLB::walkerResponse(VegaTlbEntry& entry, PacketPtr pkt)
{
    DPRINTF(GPUTLB, "WalkerResponse for %#lx. Entry: (%#lx, %#lx, %#lx)\n",
            pkt->req->getVaddr(), entry.vaddr, entry.paddr, entry.size());
 
    Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                    VegaISA::PageBytes);
 
    Addr page_addr = entry.pte.ppn << VegaISA::PageShift;
    Addr paddr = page_addr + (entry.vaddr & mask(entry.logBytes));
    pkt->req->setPaddr(paddr);
    pkt->req->setSystemReq(entry.pte.s);
 
    GpuTranslationState *sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);
    sender_state->tlbEntry = new VegaTlbEntry(entry);
 
    handleTranslationReturn(virt_page_addr, TLB_MISS, pkt);
}

void
GpuTLB::handleTranslationReturn(Addr virt_page_addr,
    tlbOutcome tlb_outcome, PacketPtr pkt)
{
    assert(pkt);
    Addr vaddr = pkt->req->getVaddr();
 
    GpuTranslationState *sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);
 
    Mode mode = sender_state->tlbMode;
 
    VegaTlbEntry *local_entry, *new_entry;
 
    int req_cnt = sender_state->reqCnt.back();
    bool update_stats = !sender_state->isPrefetch;
 
    if (update_stats) {
        stats.accessCycles += (req_cnt * curCycle());
        stats.localCycles += curCycle();
    }
 
    if (tlb_outcome == TLB_HIT) {
        DPRINTF(GPUTLB, "Translation Done - TLB Hit for addr %#x\n",
            vaddr);
        local_entry = safe_cast<VegaTlbEntry *>(sender_state->tlbEntry);
    } else {
        DPRINTF(GPUTLB, "Translation Done - TLB Miss for addr %#x\n",
            vaddr);

        new_entry = safe_cast<VegaTlbEntry *>(sender_state->tlbEntry);
        assert(new_entry);
        local_entry = new_entry;
 
        if (allocationPolicy) {
            assert(new_entry->pte);
            DPRINTF(GPUTLB, "allocating entry w/ addr %#lx of size %#lx\n",
                    virt_page_addr, new_entry->size());
 
            local_entry = insert(virt_page_addr, *new_entry);
        }
 
        assert(local_entry);
    }

    DPRINTF(GPUTLB, "Entry found with vaddr %#x,  doing protection checks "
            "while paddr was %#x.\n", local_entry->vaddr,
            local_entry->paddr);
 
    pagingProtectionChecks(pkt, local_entry, mode);
    int page_size = local_entry->size();
    Addr paddr = local_entry->paddr + (vaddr & (page_size - 1));
    DPRINTF(GPUTLB, "Translated %#x -> %#x.\n", vaddr, paddr);
 
    // Since this packet will be sent through the cpu side port, it must be
    // converted to a response pkt if it is not one already
    if (pkt->isRequest()) {
        pkt->makeTimingResponse();
    }
 
    pkt->req->setPaddr(paddr);
 
    if (local_entry->uncacheable()) {
         pkt->req->setFlags(Request::UNCACHEABLE);
    }
 
    //send packet back to coalescer
    cpuSidePort[0]->sendTimingResp(pkt);
    //schedule cleanup event
    cleanupQueue.push(virt_page_addr);
 
    DPRINTF(GPUTLB, "Scheduled %#lx for cleanup\n", virt_page_addr);
 
    // schedule this only once per cycle.
    // The check is required because we might have multiple translations
    // returning the same cycle
    // this is a maximum priority event and must be on the same cycle
    // as the cleanup event in TLBCoalescer to avoid a race with
    // IssueProbeEvent caused by TLBCoalescer::MemSidePort::recvReqRetry
    if (!cleanupEvent.scheduled())
        schedule(cleanupEvent, curTick());
}

void
GpuTLB::translationReturn(Addr virtPageAddr, tlbOutcome outcome,
                          PacketPtr pkt)
{
    DPRINTF(GPUTLB, "Triggered TLBEvent for addr %#x\n", virtPageAddr);
 
    assert(translationReturnEvent[virtPageAddr]);
    assert(pkt);
 
    GpuTranslationState *tmp_sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);
 
    int req_cnt = tmp_sender_state->reqCnt.back();
    bool update_stats = !tmp_sender_state->isPrefetch;
 
 
    if (outcome == TLB_HIT) {
        handleTranslationReturn(virtPageAddr, TLB_HIT, pkt);
 
    } else if (outcome == TLB_MISS) {
 
        DPRINTF(GPUTLB, "This is a TLB miss\n");
        if (hasMemSidePort) {
            // the one cyle added here represent the delay from when we get
            // the reply back till when we propagate it to the coalescer
            // above.

            tmp_sender_state->deviceId = 1;
            tmp_sender_state->pasId = 0;
 
            if (!memSidePort[0]->sendTimingReq(pkt)) {
                DPRINTF(GPUTLB, "Failed sending translation request to "
                        "lower level TLB for addr %#x\n", virtPageAddr);
 
                memSidePort[0]->retries.push_back(pkt);
            } else {
                DPRINTF(GPUTLB, "Sent translation request to lower level "
                        "TLB for addr %#x\n", virtPageAddr);
            }
        } else {
            //this is the last level TLB. Start a page walk
            DPRINTF(GPUTLB, "Last level TLB - start a page walk for "
                    "addr %#x\n", virtPageAddr);
 
            if (update_stats)
                stats.pageTableCycles -= (req_cnt*curCycle());
 
            TLBEvent *tlb_event = translationReturnEvent[virtPageAddr];
            assert(tlb_event);
            tlb_event->updateOutcome(PAGE_WALK);
            schedule(tlb_event,
                     curTick() + cyclesToTicks(Cycles(missLatency2)));
        }
    } else if (outcome == PAGE_WALK) {
        if (update_stats)
            stats.pageTableCycles += (req_cnt*curCycle());
 
        // Need to access the page table and update the TLB
        DPRINTF(GPUTLB, "Doing a page walk for address %#x\n",
                virtPageAddr);
 
        Addr base = gpuDevice->getVM().getPageTableBase(1);
        Addr vaddr = pkt->req->getVaddr();
        walker->setDevRequestor(gpuDevice->vramRequestorId());
 
        // Do page table walk
        walker->startTiming(pkt, base, vaddr, BaseMMU::Mode::Read);
    } else if (outcome == MISS_RETURN) {
        handleTranslationReturn(virtPageAddr, TLB_MISS, pkt);
    } else {
        panic("Unexpected TLB outcome %d", outcome);
    }
}
 
void
GpuTLB::TLBEvent::process()
{
    tlb->translationReturn(virtPageAddr, outcome, pkt);
}
 
const char*
GpuTLB::TLBEvent::description() const
{
    return "trigger translationDoneEvent";
}
 
void
GpuTLB::TLBEvent::updateOutcome(tlbOutcome _outcome)
{
    outcome = _outcome;
}
 
Addr
GpuTLB::TLBEvent::getTLBEventVaddr()
{
    return virtPageAddr;
}

bool
GpuTLB::CpuSidePort::recvTimingReq(PacketPtr pkt)
{
    bool ret = false;
    [[maybe_unused]] Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                                     VegaISA::PageBytes);
 
    if (tlb->outstandingReqs < tlb->maxCoalescedReqs) {
        assert(!tlb->translationReturnEvent.count(virt_page_addr));
        tlb->issueTLBLookup(pkt);
        // update number of outstanding translation requests
        tlb->outstandingReqs++;
        ret = true;
    } else {
        DPRINTF(GPUTLB, "Reached maxCoalescedReqs number %d\n",
                tlb->outstandingReqs);
        tlb->stats.maxDownstreamReached++;
        ret = false;
 
    }
 
    if (tlb->outstandingReqs > tlb->stats.outstandingReqsMax.value())
        tlb->stats.outstandingReqsMax = tlb->outstandingReqs;
 
    return ret;
}

void
GpuTLB::handleFuncTranslationReturn(PacketPtr pkt, tlbOutcome tlb_outcome)
{
    GpuTranslationState *sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);
 
    Mode mode = sender_state->tlbMode;
    Addr vaddr = pkt->req->getVaddr();
 
    VegaTlbEntry *local_entry, *new_entry;
 
    if (tlb_outcome == TLB_HIT) {
        DPRINTF(GPUTLB, "Functional Translation Done - TLB hit for addr "
                "%#x\n", vaddr);
 
        local_entry = safe_cast<VegaTlbEntry *>(sender_state->tlbEntry);
    } else {
        DPRINTF(GPUTLB, "Functional Translation Done - TLB miss for addr "
                "%#x\n", vaddr);

        new_entry = safe_cast<VegaTlbEntry *>(sender_state->tlbEntry);
        assert(new_entry);
        local_entry = new_entry;
 
        if (allocationPolicy) {
            Addr virt_page_addr = roundDown(vaddr, VegaISA::PageBytes);
 
            DPRINTF(GPUTLB, "allocating entry w/ addr %#lx\n",
                    virt_page_addr);
 
            local_entry = insert(virt_page_addr, *new_entry);
        }
 
        assert(local_entry);
    }
 
    DPRINTF(GPUTLB, "Entry found with vaddr %#x, doing protection checks "
            "while paddr was %#x.\n", local_entry->vaddr,
            local_entry->paddr);

    if (!sender_state->isPrefetch && sender_state->tlbEntry)
        pagingProtectionChecks(pkt, local_entry, mode);
 
    int page_size = local_entry->size();
    Addr paddr = local_entry->paddr + (vaddr & (page_size - 1));
    DPRINTF(GPUTLB, "Translated %#x -> %#x.\n", vaddr, paddr);
 
    pkt->req->setPaddr(paddr);
 
    if (local_entry->uncacheable())
         pkt->req->setFlags(Request::UNCACHEABLE);
}
 
// This is used for atomic translations. Need to
// make it all happen during the same cycle.
void
GpuTLB::CpuSidePort::recvFunctional(PacketPtr pkt)
{
    GpuTranslationState *sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);
 
    bool update_stats = !sender_state->isPrefetch;
 
    Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                    VegaISA::PageBytes);
 
    // do the TLB lookup without updating the stats
    bool success = tlb->tlbLookup(pkt->req, update_stats);
    tlbOutcome tlb_outcome = success ? TLB_HIT : TLB_MISS;
 
    // functional mode means no coalescing
    // global metrics are the same as the local metrics
    if (update_stats) {
        tlb->stats.globalNumTLBAccesses++;
 
        if (success) {
            sender_state->hitLevel = sender_state->reqCnt.size();
            tlb->stats.globalNumTLBHits++;
        } else {
            tlb->stats.globalNumTLBMisses++;
        }
    }
 
    if (!success) {
        if (tlb->hasMemSidePort) {
            // there is a TLB below -> propagate down the TLB hierarchy
            tlb->memSidePort[0]->sendFunctional(pkt);
            // If no valid translation from a prefetch, then just return
            if (sender_state->isPrefetch && !pkt->req->hasPaddr())
                return;
        } else {
            // Need to access the page table and update the TLB
            DPRINTF(GPUTLB, "Doing a page walk for address %#x\n",
                    virt_page_addr);
 
            Addr vaddr = pkt->req->getVaddr();
            [[maybe_unused]] Addr alignedVaddr =
                tlb->pageAlign(virt_page_addr);
            assert(alignedVaddr == virt_page_addr);
 
            unsigned logBytes;
            PageTableEntry pte;
 
            // Initialize walker state for VMID
            Addr base = tlb->gpuDevice->getVM().getPageTableBase(1);
            tlb->walker->setDevRequestor(tlb->gpuDevice->vramRequestorId());
 
            // Do page table walk
            Fault fault = tlb->walker->startFunctional(base, vaddr, pte,
                                                       logBytes,
                                                       BaseMMU::Mode::Read);
            if (fault != NoFault) {
                fatal("Translation fault in TLB at %d!", __LINE__);
            }
 
            // PPN is already shifted by fragment so we only shift by native
            // page size. Fragment is still used via logBytes to select lower
            // bits from vaddr.
            Addr page_addr = pte.ppn << PageShift;
            Addr paddr = page_addr + (vaddr & mask(logBytes));
            Addr alignedPaddr = tlb->pageAlign(paddr);
            pkt->req->setPaddr(paddr);
            pkt->req->setSystemReq(pte.s);
 
            if (!sender_state->isPrefetch) {
                assert(paddr);
 
                DPRINTF(GPUTLB, "Mapping %#x to %#x\n", vaddr, paddr);
 
                sender_state->tlbEntry =
                    new VegaTlbEntry(1 /* VMID */, vaddr & (~mask(logBytes)),
                                 alignedPaddr, logBytes, pte);
            } else {
                // If this was a prefetch, then do the normal thing if it
                // was a successful translation.  Otherwise, send an empty
                // TLB entry back so that it can be figured out as empty
                // and handled accordingly.
                if (paddr) {
                    DPRINTF(GPUTLB, "Mapping %#x to %#x\n", vaddr, paddr);
 
                    sender_state->tlbEntry =
                        new VegaTlbEntry(1 /* VMID */,
                                     vaddr & (~mask(logBytes)),
                                     alignedPaddr, logBytes, pte);
                } else {
                    DPRINTF(GPUPrefetch, "Prefetch failed %#x\n", vaddr);
 
                    sender_state->tlbEntry = nullptr;
 
                    return;
                }
            }
        }
    } else {
        VegaTlbEntry *entry = tlb->lookup(virt_page_addr, update_stats);
        assert(entry);
 
        if (sender_state->isPrefetch) {
            DPRINTF(GPUPrefetch, "Functional Hit for vaddr %#x\n",
                    entry->vaddr);
        }
 
        sender_state->tlbEntry = new VegaTlbEntry(*entry);
    }
 
    // This is the function that would populate pkt->req with the paddr of
    // the translation. But if no translation happens (i.e Prefetch fails)
    // then the early returns in the above code wiill keep this function
    // from executing.
    tlb->handleFuncTranslationReturn(pkt, tlb_outcome);
}
 
void
GpuTLB::CpuSidePort::recvReqRetry()
{
    // The CPUSidePort never sends anything but replies. No retries
    // expected.
    panic("recvReqRetry called");
}
 
AddrRangeList
GpuTLB::CpuSidePort::getAddrRanges() const
{
    // currently not checked by the requestor
    AddrRangeList ranges;
 
    return ranges;
}

bool
GpuTLB::MemSidePort::recvTimingResp(PacketPtr pkt)
{
    Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                    VegaISA::PageBytes);
 
    DPRINTF(GPUTLB, "MemSidePort recvTiming for virt_page_addr %#x\n",
            virt_page_addr);
 
    TLBEvent *tlb_event = tlb->translationReturnEvent[virt_page_addr];
    assert(tlb_event);
    assert(virt_page_addr == tlb_event->getTLBEventVaddr());
 
    tlb_event->updateOutcome(MISS_RETURN);
    tlb->schedule(tlb_event, curTick()+tlb->clockPeriod());
 
    return true;
}
 
void
GpuTLB::MemSidePort::recvReqRetry()
{
    // No retries should reach the TLB. The retries
    // should only reach the TLBCoalescer.
    panic("recvReqRetry called");
}
 
void
GpuTLB::cleanup()
{
    while (!cleanupQueue.empty()) {
        Addr cleanup_addr = cleanupQueue.front();
        cleanupQueue.pop();
 
        DPRINTF(GPUTLB, "Deleting return event for %#lx\n", cleanup_addr);
 
        // delete TLBEvent
        TLBEvent * old_tlb_event = translationReturnEvent[cleanup_addr];
        delete old_tlb_event;
        translationReturnEvent.erase(cleanup_addr);
 
        // update number of outstanding requests
        outstandingReqs--;
    }

    for (int i = 0; i < cpuSidePort.size(); ++i) {
        cpuSidePort[i]->sendRetryReq();
    }
}
 
GpuTLB::VegaTLBStats::VegaTLBStats(statistics::Group *parent)
    : statistics::Group(parent),
      ADD_STAT(maxDownstreamReached, "Number of refused translation requests"),
      ADD_STAT(outstandingReqsMax, "Maximum count in coalesced request queue"),
      ADD_STAT(localNumTLBAccesses, "Number of TLB accesses"),
      ADD_STAT(localNumTLBHits, "Number of TLB hits"),
      ADD_STAT(localNumTLBMisses, "Number of TLB misses"),
      ADD_STAT(localTLBMissRate, "TLB miss rate"),
      ADD_STAT(globalNumTLBAccesses, "Number of TLB accesses"),
      ADD_STAT(globalNumTLBHits, "Number of TLB hits"),
      ADD_STAT(globalNumTLBMisses, "Number of TLB misses"),
      ADD_STAT(globalTLBMissRate, "TLB miss rate"),
      ADD_STAT(accessCycles, "Cycles spent accessing this TLB level"),
      ADD_STAT(pageTableCycles, "Cycles spent accessing the page table"),
      ADD_STAT(localCycles, "Number of cycles spent in queue for all "
               "incoming reqs"),
      ADD_STAT(localLatency, "Avg. latency over incoming coalesced reqs")
{
    localTLBMissRate = 100 * localNumTLBMisses / localNumTLBAccesses;
    globalTLBMissRate = 100 * globalNumTLBMisses / globalNumTLBAccesses;
 
    localLatency = localCycles / localNumTLBAccesses;
}
 
} // namespace VegaISA
} // namespace gem5