lsq_unit.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2012-2014,2017-2018,2020 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Copyright (c) 2004-2006 The Regents of The University of Michigan
 * Copyright (c) 2013 Advanced Micro Devices, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#ifndef __CPU_O3_LSQ_UNIT_HH__
#define __CPU_O3_LSQ_UNIT_HH__
 
#include <algorithm>
#include <cstring>
#include <map>
#include <memory>
#include <queue>
 
#include "arch/generic/debugfaults.hh"
#include "arch/generic/vec_reg.hh"
#include "arch/locked_mem.hh"
#include "config/the_isa.hh"
#include "cpu/inst_seq.hh"
#include "cpu/timebuf.hh"
#include "debug/HtmCpu.hh"
#include "debug/LSQUnit.hh"
#include "mem/packet.hh"
#include "mem/port.hh"
 
struct DerivO3CPUParams;
#include "base/circular_queue.hh"
 
template <class Impl>
class LSQUnit
{
  public:
    static constexpr auto MaxDataBytes = MaxVecRegLenInBytes;
 
    typedef typename Impl::O3CPU O3CPU;
    typedef typename Impl::DynInstPtr DynInstPtr;
    typedef typename Impl::CPUPol::IEW IEW;
    typedef typename Impl::CPUPol::LSQ LSQ;
    typedef typename Impl::CPUPol::IssueStruct IssueStruct;
 
    using LSQSenderState = typename LSQ::LSQSenderState;
    using LSQRequest = typename Impl::CPUPol::LSQ::LSQRequest;
  private:
    class LSQEntry
    {
      private:
        DynInstPtr inst;
        LSQRequest* req;
        uint32_t _size;
        bool _valid;
      public:
        LSQEntry()
            : inst(nullptr), req(nullptr), _size(0), _valid(false)
        {
        }
 
        ~LSQEntry()
        {
            inst = nullptr;
            if (req != nullptr) {
                req->freeLSQEntry();
                req = nullptr;
            }
        }
 
        void
        clear()
        {
            inst = nullptr;
            if (req != nullptr) {
                req->freeLSQEntry();
            }
            req = nullptr;
            _valid = false;
            _size = 0;
        }
 
        void
        set(const DynInstPtr& inst)
        {
            assert(!_valid);
            this->inst = inst;
            _valid = true;
            _size = 0;
        }
        LSQRequest* request() { return req; }
        void setRequest(LSQRequest* r) { req = r; }
        bool hasRequest() { return req != nullptr; }
        bool valid() const { return _valid; }
        uint32_t& size() { return _size; }
        const uint32_t& size() const { return _size; }
        const DynInstPtr& instruction() const { return inst; }
    };
 
    class SQEntry : public LSQEntry
    {
      private:
        char _data[MaxDataBytes];
        bool _canWB;
        bool _committed;
        bool _completed;
        bool _isAllZeros;
      public:
        static constexpr size_t DataSize = sizeof(_data);
        SQEntry()
            : _canWB(false), _committed(false), _completed(false),
              _isAllZeros(false)
        {
            std::memset(_data, 0, DataSize);
        }
 
        ~SQEntry()
        {
        }
 
        void
        set(const DynInstPtr& inst)
        {
            LSQEntry::set(inst);
        }
 
        void
        clear()
        {
            LSQEntry::clear();
            _canWB = _completed = _committed = _isAllZeros = false;
        }
        bool& canWB() { return _canWB; }
        const bool& canWB() const { return _canWB; }
        bool& completed() { return _completed; }
        const bool& completed() const { return _completed; }
        bool& committed() { return _committed; }
        const bool& committed() const { return _committed; }
        bool& isAllZeros() { return _isAllZeros; }
        const bool& isAllZeros() const { return _isAllZeros; }
        char* data() { return _data; }
        const char* data() const { return _data; }
    };
    using LQEntry = LSQEntry;
 
    enum class AddrRangeCoverage
    {
        PartialAddrRangeCoverage, /* Two ranges partly overlap */
        FullAddrRangeCoverage, /* One range fully covers another */
        NoAddrRangeCoverage /* Two ranges are disjoint */
    };
 
  public:
    using LoadQueue = CircularQueue<LQEntry>;
    using StoreQueue = CircularQueue<SQEntry>;
 
  public:
    LSQUnit(uint32_t lqEntries, uint32_t sqEntries);
 
    LSQUnit(const LSQUnit &l): stats(nullptr)
    {
        panic("LSQUnit is not copy-able");
    }
 
    void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
            LSQ *lsq_ptr, unsigned id);
 
    std::string name() const;
 
    void setDcachePort(RequestPort *dcache_port);
 
    void drainSanityCheck() const;
 
    void takeOverFrom();
 
    void insert(const DynInstPtr &inst);
    void insertLoad(const DynInstPtr &load_inst);
    void insertStore(const DynInstPtr &store_inst);
 
    Fault checkViolations(typename LoadQueue::iterator& loadIt,
            const DynInstPtr& inst);
 
    void checkSnoop(PacketPtr pkt);
 
    Fault executeLoad(const DynInstPtr &inst);
 
    Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; }
    Fault executeStore(const DynInstPtr &inst);
 
    void commitLoad();
    void commitLoads(InstSeqNum &youngest_inst);
 
    void commitStores(InstSeqNum &youngest_inst);
 
    void writebackStores();
 
    void completeDataAccess(PacketPtr pkt);
 
    void squash(const InstSeqNum &squashed_num);
 
    bool violation() { return memDepViolator; }
 
    DynInstPtr getMemDepViolator();
 
    unsigned numFreeLoadEntries();
 
    unsigned numFreeStoreEntries();
 
    int numLoads() { return loads; }
 
    int numStores() { return stores; }
 
    // hardware transactional memory
    int numHtmStarts() const { return htmStarts; }
    int numHtmStops() const { return htmStops; }
    void resetHtmStartsStops() { htmStarts = htmStops = 0; }
    uint64_t getLatestHtmUid() const
    {
        const auto& htm_cpt = cpu->tcBase(lsqID)->getHtmCheckpointPtr();
        return htm_cpt->getHtmUid();
    }
    void setLastRetiredHtmUid(uint64_t htm_uid)
    {
        assert(htm_uid >= lastRetiredHtmUid);
        lastRetiredHtmUid = htm_uid;
    }
 
    bool isFull() { return lqFull() || sqFull(); }
 
    bool isEmpty() const { return lqEmpty() && sqEmpty(); }
 
    bool lqFull() { return loadQueue.full(); }
 
    bool sqFull() { return storeQueue.full(); }
 
    bool lqEmpty() const { return loads == 0; }
 
    bool sqEmpty() const { return stores == 0; }
 
    unsigned getCount() { return loads + stores; }
 
    bool hasStoresToWB() { return storesToWB; }
 
    int numStoresToWB() { return storesToWB; }
 
    bool
    willWB()
    {
        return storeWBIt.dereferenceable() &&
                        storeWBIt->valid() &&
                        storeWBIt->canWB() &&
                        !storeWBIt->completed() &&
                        !isStoreBlocked;
    }
 
    void recvRetry();
 
    unsigned int cacheLineSize();
  private:
    void resetState();
 
    void writeback(const DynInstPtr &inst, PacketPtr pkt);
 
    void writebackBlockedStore();
 
    void completeStore(typename StoreQueue::iterator store_idx);
 
    void storePostSend();
 
  public:
    bool trySendPacket(bool isLoad, PacketPtr data_pkt);
 
 
    void dumpInsts() const;
 
    void schedule(Event& ev, Tick when) { cpu->schedule(ev, when); }
 
    BaseTLB* dTLB() { return cpu->dtb; }
 
  private:
    O3CPU *cpu;
 
    IEW *iewStage;
 
    LSQ *lsq;
 
    RequestPort *dcachePort;
 
    class LQSenderState : public LSQSenderState
    {
        using LSQSenderState::alive;
      public:
        LQSenderState(typename LoadQueue::iterator idx_)
            : LSQSenderState(idx_->request(), true), idx(idx_) { }
 
        typename LoadQueue::iterator idx;
        //virtual LSQRequest* request() { return idx->request(); }
        virtual void
        complete()
        {
            //if (alive())
            //  idx->request()->senderState(nullptr);
        }
    };
 
    class SQSenderState : public LSQSenderState
    {
        using LSQSenderState::alive;
      public:
        SQSenderState(typename StoreQueue::iterator idx_)
            : LSQSenderState(idx_->request(), false), idx(idx_) { }
        typename StoreQueue::iterator idx;
        //virtual LSQRequest* request() { return idx->request(); }
        virtual void
        complete()
        {
            //if (alive())
            //   idx->request()->senderState(nullptr);
        }
    };
 
    class WritebackEvent : public Event
    {
      public:
        WritebackEvent(const DynInstPtr &_inst, PacketPtr pkt,
                LSQUnit *lsq_ptr);
 
        void process();
 
        const char *description() const;
 
      private:
        DynInstPtr inst;
 
        PacketPtr pkt;
 
        LSQUnit<Impl> *lsqPtr;
    };
 
  public:
    bool recvTimingResp(PacketPtr pkt);
 
  private:
    ThreadID lsqID;
  public:
    CircularQueue<SQEntry> storeQueue;
 
    LoadQueue loadQueue;
 
  private:
    unsigned depCheckShift;
 
    bool checkLoads;
 
    int loads;
    int stores;
    int storesToWB;
 
    // hardware transactional memory
    // nesting depth
    int htmStarts;
    int htmStops;
    // sanity checks and debugging
    uint64_t lastRetiredHtmUid;
 
    typename StoreQueue::iterator storeWBIt;
 
    Addr cacheBlockMask;
 
    typename TimeBuffer<IssueStruct>::wire fromIssue;
 
    bool stalled;
    InstSeqNum stallingStoreIsn;
    int stallingLoadIdx;
 
    PacketPtr retryPkt;
 
    bool isStoreBlocked;
 
    bool storeInFlight;
 
    DynInstPtr memDepViolator;
 
    bool hasPendingRequest;
 
    LSQRequest* pendingRequest;
 
    bool needsTSO;
 
  protected:
    // Will also need how many read/write ports the Dcache has.  Or keep track
    // of that in stage that is one level up, and only call executeLoad/Store
    // the appropriate number of times.
    struct LSQUnitStats : public Stats::Group{
        LSQUnitStats(Stats::Group *parent);
 
        Stats::Scalar forwLoads;
 
        Stats::Scalar squashedLoads;
 
        Stats::Scalar ignoredResponses;
 
        Stats::Scalar memOrderViolation;
 
        Stats::Scalar squashedStores;
 
        Stats::Scalar rescheduledLoads;
 
        Stats::Scalar blockedByCache;
    } stats;
 
  public:
    Fault read(LSQRequest *req, int load_idx);
 
    Fault write(LSQRequest *req, uint8_t *data, int store_idx);
 
    int getLoadHead() { return loadQueue.head(); }
 
    InstSeqNum
    getLoadHeadSeqNum()
    {
        return loadQueue.front().valid()
            ? loadQueue.front().instruction()->seqNum
            : 0;
    }
 
    int getStoreHead() { return storeQueue.head(); }
    InstSeqNum
    getStoreHeadSeqNum()
    {
        return storeQueue.front().valid()
            ? storeQueue.front().instruction()->seqNum
            : 0;
    }
 
    bool isStalled()  { return stalled; }
  public:
    typedef typename CircularQueue<LQEntry>::iterator LQIterator;
    typedef typename CircularQueue<SQEntry>::iterator SQIterator;
    typedef CircularQueue<LQEntry> LQueue;
    typedef CircularQueue<SQEntry> SQueue;
};
 
template <class Impl>
Fault
LSQUnit<Impl>::read(LSQRequest *req, int load_idx)
{
    LQEntry& load_req = loadQueue[load_idx];
    const DynInstPtr& load_inst = load_req.instruction();
 
    load_req.setRequest(req);
    assert(load_inst);
 
    assert(!load_inst->isExecuted());
 
    // Make sure this isn't a strictly ordered load
    // A bit of a hackish way to get strictly ordered accesses to work
    // only if they're at the head of the LSQ and are ready to commit
    // (at the head of the ROB too).
 
    if (req->mainRequest()->isStrictlyOrdered() &&
        (load_idx != loadQueue.head() || !load_inst->isAtCommit())) {
        // Tell IQ/mem dep unit that this instruction will need to be
        // rescheduled eventually
        iewStage->rescheduleMemInst(load_inst);
        load_inst->clearIssued();
        load_inst->effAddrValid(false);
        ++stats.rescheduledLoads;
        DPRINTF(LSQUnit, "Strictly ordered load [sn:%lli] PC %s\n",
                load_inst->seqNum, load_inst->pcState());
 
        // Must delete request now that it wasn't handed off to
        // memory.  This is quite ugly.  @todo: Figure out the proper
        // place to really handle request deletes.
        load_req.setRequest(nullptr);
        req->discard();
        return std::make_shared<GenericISA::M5PanicFault>(
            "Strictly ordered load [sn:%llx] PC %s\n",
            load_inst->seqNum, load_inst->pcState());
    }
 
    DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, "
            "storeHead: %i addr: %#x%s\n",
            load_idx - 1, load_inst->sqIt._idx, storeQueue.head() - 1,
            req->mainRequest()->getPaddr(), req->isSplit() ? " split" : "");
 
    if (req->mainRequest()->isLLSC()) {
        // Disable recording the result temporarily.  Writing to misc
        // regs normally updates the result, but this is not the
        // desired behavior when handling store conditionals.
        load_inst->recordResult(false);
        TheISA::handleLockedRead(load_inst.get(), req->mainRequest());
        load_inst->recordResult(true);
    }
 
    if (req->mainRequest()->isLocalAccess()) {
        assert(!load_inst->memData);
        assert(!load_inst->inHtmTransactionalState());
        load_inst->memData = new uint8_t[MaxDataBytes];
 
        ThreadContext *thread = cpu->tcBase(lsqID);
        PacketPtr main_pkt = new Packet(req->mainRequest(), MemCmd::ReadReq);
 
        main_pkt->dataStatic(load_inst->memData);
 
        Cycles delay = req->mainRequest()->localAccessor(thread, main_pkt);
 
        WritebackEvent *wb = new WritebackEvent(load_inst, main_pkt, this);
        cpu->schedule(wb, cpu->clockEdge(delay));
        return NoFault;
    }
 
    // hardware transactional memory
    if (req->mainRequest()->isHTMStart() || req->mainRequest()->isHTMCommit())
    {
        // don't want to send nested transactionStarts and
        // transactionStops outside of core, e.g. to Ruby
        if (req->mainRequest()->getFlags().isSet(Request::NO_ACCESS)) {
            Cycles delay(0);
            PacketPtr data_pkt =
                new Packet(req->mainRequest(), MemCmd::ReadReq);
 
            // Allocate memory if this is the first time a load is issued.
            if (!load_inst->memData) {
                load_inst->memData =
                    new uint8_t[req->mainRequest()->getSize()];
                // sanity checks espect zero in request's data
                memset(load_inst->memData, 0, req->mainRequest()->getSize());
            }
 
            data_pkt->dataStatic(load_inst->memData);
            if (load_inst->inHtmTransactionalState()) {
                data_pkt->setHtmTransactional(
                    load_inst->getHtmTransactionUid());
            }
            data_pkt->makeResponse();
 
            WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this);
            cpu->schedule(wb, cpu->clockEdge(delay));
            return NoFault;
        }
    }
 
    // Check the SQ for any previous stores that might lead to forwarding
    auto store_it = load_inst->sqIt;
    assert (store_it >= storeWBIt);
    // End once we've reached the top of the LSQ
    while (store_it != storeWBIt) {
        // Move the index to one younger
        store_it--;
        assert(store_it->valid());
        assert(store_it->instruction()->seqNum < load_inst->seqNum);
        int store_size = store_it->size();
 
        // Cache maintenance instructions go down via the store
        // path but they carry no data and they shouldn't be
        // considered for forwarding
        if (store_size != 0 && !store_it->instruction()->strictlyOrdered() &&
            !(store_it->request()->mainRequest() &&
              store_it->request()->mainRequest()->isCacheMaintenance())) {
            assert(store_it->instruction()->effAddrValid());
 
            // Check if the store data is within the lower and upper bounds of
            // addresses that the request needs.
            auto req_s = req->mainRequest()->getVaddr();
            auto req_e = req_s + req->mainRequest()->getSize();
            auto st_s = store_it->instruction()->effAddr;
            auto st_e = st_s + store_size;
 
            bool store_has_lower_limit = req_s >= st_s;
            bool store_has_upper_limit = req_e <= st_e;
            bool lower_load_has_store_part = req_s < st_e;
            bool upper_load_has_store_part = req_e > st_s;
 
            auto coverage = AddrRangeCoverage::NoAddrRangeCoverage;
 
            // If the store entry is not atomic (atomic does not have valid
            // data), the store has all of the data needed, and
            // the load is not LLSC, then
            // we can forward data from the store to the load
            if (!store_it->instruction()->isAtomic() &&
                store_has_lower_limit && store_has_upper_limit &&
                !req->mainRequest()->isLLSC()) {
 
                const auto& store_req = store_it->request()->mainRequest();
                coverage = store_req->isMasked() ?
                    AddrRangeCoverage::PartialAddrRangeCoverage :
                    AddrRangeCoverage::FullAddrRangeCoverage;
            } else if (
                // This is the partial store-load forwarding case where a store
                // has only part of the load's data and the load isn't LLSC
                (!req->mainRequest()->isLLSC() &&
                 ((store_has_lower_limit && lower_load_has_store_part) ||
                  (store_has_upper_limit && upper_load_has_store_part) ||
                  (lower_load_has_store_part && upper_load_has_store_part))) ||
                // The load is LLSC, and the store has all or part of the
                // load's data
                (req->mainRequest()->isLLSC() &&
                 ((store_has_lower_limit || upper_load_has_store_part) &&
                  (store_has_upper_limit || lower_load_has_store_part))) ||
                // The store entry is atomic and has all or part of the load's
                // data
                (store_it->instruction()->isAtomic() &&
                 ((store_has_lower_limit || upper_load_has_store_part) &&
                  (store_has_upper_limit || lower_load_has_store_part)))) {
 
                coverage = AddrRangeCoverage::PartialAddrRangeCoverage;
            }
 
            if (coverage == AddrRangeCoverage::FullAddrRangeCoverage) {
                // Get shift amount for offset into the store's data.
                int shift_amt = req->mainRequest()->getVaddr() -
                    store_it->instruction()->effAddr;
 
                // Allocate memory if this is the first time a load is issued.
                if (!load_inst->memData) {
                    load_inst->memData =
                        new uint8_t[req->mainRequest()->getSize()];
                }
                if (store_it->isAllZeros())
                    memset(load_inst->memData, 0,
                            req->mainRequest()->getSize());
                else
                    memcpy(load_inst->memData,
                        store_it->data() + shift_amt,
                        req->mainRequest()->getSize());
 
                DPRINTF(LSQUnit, "Forwarding from store idx %i to load to "
                        "addr %#x\n", store_it._idx,
                        req->mainRequest()->getVaddr());
 
                PacketPtr data_pkt = new Packet(req->mainRequest(),
                        MemCmd::ReadReq);
                data_pkt->dataStatic(load_inst->memData);
 
                // hardware transactional memory
                // Store to load forwarding within a transaction
                // This should be okay because the store will be sent to
                // the memory subsystem and subsequently get added to the
                // write set of the transaction. The write set has a stronger
                // property than the read set, so the load doesn't necessarily
                // have to be there.
                assert(!req->mainRequest()->isHTMCmd());
                if (load_inst->inHtmTransactionalState()) {
                    assert (!storeQueue[store_it._idx].completed());
                    assert (
                        storeQueue[store_it._idx].instruction()->
                          inHtmTransactionalState());
                    assert (
                        load_inst->getHtmTransactionUid() ==
                        storeQueue[store_it._idx].instruction()->
                          getHtmTransactionUid());
                    data_pkt->setHtmTransactional(
                        load_inst->getHtmTransactionUid());
                    DPRINTF(HtmCpu, "HTM LD (ST2LDF) "
                      "pc=0x%lx - vaddr=0x%lx - "
                      "paddr=0x%lx - htmUid=%u\n",
                      load_inst->instAddr(),
                      data_pkt->req->hasVaddr() ?
                        data_pkt->req->getVaddr() : 0lu,
                      data_pkt->getAddr(),
                      load_inst->getHtmTransactionUid());
                }
 
                if (req->isAnyOutstandingRequest()) {
                    assert(req->_numOutstandingPackets > 0);
                    // There are memory requests packets in flight already.
                    // This may happen if the store was not complete the
                    // first time this load got executed. Signal the senderSate
                    // that response packets should be discarded.
                    req->discardSenderState();
                }
 
                WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt,
                        this);
 
                // We'll say this has a 1 cycle load-store forwarding latency
                // for now.
                // @todo: Need to make this a parameter.
                cpu->schedule(wb, curTick());
 
                // Don't need to do anything special for split loads.
                ++stats.forwLoads;
 
                return NoFault;
            } else if (coverage == AddrRangeCoverage::PartialAddrRangeCoverage) {
                // If it's already been written back, then don't worry about
                // stalling on it.
                if (store_it->completed()) {
                    panic("Should not check one of these");
                    continue;
                }
 
                // Must stall load and force it to retry, so long as it's the
                // oldest load that needs to do so.
                if (!stalled ||
                    (stalled &&
                     load_inst->seqNum <
                     loadQueue[stallingLoadIdx].instruction()->seqNum)) {
                    stalled = true;
                    stallingStoreIsn = store_it->instruction()->seqNum;
                    stallingLoadIdx = load_idx;
                }
 
                // Tell IQ/mem dep unit that this instruction will need to be
                // rescheduled eventually
                iewStage->rescheduleMemInst(load_inst);
                load_inst->clearIssued();
                load_inst->effAddrValid(false);
                ++stats.rescheduledLoads;
 
                // Do not generate a writeback event as this instruction is not
                // complete.
                DPRINTF(LSQUnit, "Load-store forwarding mis-match. "
                        "Store idx %i to load addr %#x\n",
                        store_it._idx, req->mainRequest()->getVaddr());
 
                // Must discard the request.
                req->discard();
                load_req.setRequest(nullptr);
                return NoFault;
            }
        }
    }
 
    // If there's no forwarding case, then go access memory
    DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n",
            load_inst->seqNum, load_inst->pcState());
 
    // Allocate memory if this is the first time a load is issued.
    if (!load_inst->memData) {
        load_inst->memData = new uint8_t[req->mainRequest()->getSize()];
    }
 
 
    // hardware transactional memory
    if (req->mainRequest()->isHTMCmd()) {
        // this is a simple sanity check
        // the Ruby cache controller will set
        // memData to 0x0ul if successful.
        *load_inst->memData = (uint64_t) 0x1ull;
    }
 
    // For now, load throughput is constrained by the number of
    // load FUs only, and loads do not consume a cache port (only
    // stores do).
    // @todo We should account for cache port contention
    // and arbitrate between loads and stores.
 
    // if we the cache is not blocked, do cache access
    if (req->senderState() == nullptr) {
        LQSenderState *state = new LQSenderState(
                loadQueue.getIterator(load_idx));
        state->isLoad = true;
        state->inst = load_inst;
        state->isSplit = req->isSplit();
        req->senderState(state);
    }
    req->buildPackets();
    req->sendPacketToCache();
    if (!req->isSent())
        iewStage->blockMemInst(load_inst);
 
    return NoFault;
}
 
template <class Impl>
Fault
LSQUnit<Impl>::write(LSQRequest *req, uint8_t *data, int store_idx)
{
    assert(storeQueue[store_idx].valid());
 
    DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x | storeHead:%i "
            "[sn:%llu]\n",
            store_idx - 1, req->request()->getPaddr(), storeQueue.head() - 1,
            storeQueue[store_idx].instruction()->seqNum);
 
    storeQueue[store_idx].setRequest(req);
    unsigned size = req->_size;
    storeQueue[store_idx].size() = size;
    bool store_no_data =
        req->mainRequest()->getFlags() & Request::STORE_NO_DATA;
    storeQueue[store_idx].isAllZeros() = store_no_data;
    assert(size <= SQEntry::DataSize || store_no_data);
 
    // copy data into the storeQueue only if the store request has valid data
    if (!(req->request()->getFlags() & Request::CACHE_BLOCK_ZERO) &&
        !req->request()->isCacheMaintenance() &&
        !req->request()->isAtomic())
        memcpy(storeQueue[store_idx].data(), data, size);
 
    // This function only writes the data to the store queue, so no fault
    // can happen here.
    return NoFault;
}
 
#endif // __CPU_O3_LSQ_UNIT_HH__