lsq_unit.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2012-2014,2017-2018,2020 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Copyright (c) 2004-2006 The Regents of The University of Michigan
 * Copyright (c) 2013 Advanced Micro Devices, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __CPU_O3_LSQ_UNIT_HH__
#define __CPU_O3_LSQ_UNIT_HH__

#include <algorithm>
#include <cstring>
#include <map>
#include <queue>

#include "arch/generic/debugfaults.hh"
#include "arch/generic/vec_reg.hh"
#include "arch/isa_traits.hh"
#include "arch/locked_mem.hh"
#include "config/the_isa.hh"
#include "cpu/inst_seq.hh"
#include "cpu/timebuf.hh"
#include "debug/LSQUnit.hh"
#include "mem/packet.hh"
#include "mem/port.hh"

struct DerivO3CPUParams;
#include "base/circular_queue.hh"

template <class Impl>
class LSQUnit
{
  public:
    static constexpr auto MaxDataBytes = MaxVecRegLenInBytes;

    typedef typename Impl::O3CPU O3CPU;
    typedef typename Impl::DynInstPtr DynInstPtr;
    typedef typename Impl::CPUPol::IEW IEW;
    typedef typename Impl::CPUPol::LSQ LSQ;
    typedef typename Impl::CPUPol::IssueStruct IssueStruct;

    using LSQSenderState = typename LSQ::LSQSenderState;
    using LSQRequest = typename Impl::CPUPol::LSQ::LSQRequest;
  private:
    class LSQEntry
    {
      private:
        DynInstPtr inst;
        LSQRequest* req;
        uint32_t _size;
        bool _valid;
      public:
        LSQEntry()
            : inst(nullptr), req(nullptr), _size(0), _valid(false)
        {
        }

        ~LSQEntry()
        {
            inst = nullptr;
            if (req != nullptr) {
                req->freeLSQEntry();
                req = nullptr;
            }
        }

        void
        clear()
        {
            inst = nullptr;
            if (req != nullptr) {
                req->freeLSQEntry();
            }
            req = nullptr;
            _valid = false;
            _size = 0;
        }

        void
        set(const DynInstPtr& inst)
        {
            assert(!_valid);
            this->inst = inst;
            _valid = true;
            _size = 0;
        }
        LSQRequest* request() { return req; }
        void setRequest(LSQRequest* r) { req = r; }
        bool hasRequest() { return req != nullptr; }
        bool valid() const { return _valid; }
        uint32_t& size() { return _size; }
        const uint32_t& size() const { return _size; }
        const DynInstPtr& instruction() const { return inst; }
    };

    class SQEntry : public LSQEntry
    {
      private:
        char _data[MaxDataBytes];
        bool _canWB;
        bool _committed;
        bool _completed;
        bool _isAllZeros;
      public:
        static constexpr size_t DataSize = sizeof(_data);
        SQEntry()
            : _canWB(false), _committed(false), _completed(false),
              _isAllZeros(false)
        {
            std::memset(_data, 0, DataSize);
        }

        ~SQEntry()
        {
        }

        void
        set(const DynInstPtr& inst)
        {
            LSQEntry::set(inst);
        }

        void
        clear()
        {
            LSQEntry::clear();
            _canWB = _completed = _committed = _isAllZeros = false;
        }
        bool& canWB() { return _canWB; }
        const bool& canWB() const { return _canWB; }
        bool& completed() { return _completed; }
        const bool& completed() const { return _completed; }
        bool& committed() { return _committed; }
        const bool& committed() const { return _committed; }
        bool& isAllZeros() { return _isAllZeros; }
        const bool& isAllZeros() const { return _isAllZeros; }
        char* data() { return _data; }
        const char* data() const { return _data; }
    };
    using LQEntry = LSQEntry;

    enum class AddrRangeCoverage
    {
        PartialAddrRangeCoverage, /* Two ranges partly overlap */
        FullAddrRangeCoverage, /* One range fully covers another */
        NoAddrRangeCoverage /* Two ranges are disjoint */
    };

  public:
    using LoadQueue = CircularQueue<LQEntry>;
    using StoreQueue = CircularQueue<SQEntry>;

  public:
    LSQUnit(uint32_t lqEntries, uint32_t sqEntries);

    LSQUnit(const LSQUnit &l) { panic("LSQUnit is not copy-able"); }

    void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
            LSQ *lsq_ptr, unsigned id);

    std::string name() const;

    void regStats();

    void setDcachePort(MasterPort *dcache_port);

    void drainSanityCheck() const;

    void takeOverFrom();

    void insert(const DynInstPtr &inst);
    void insertLoad(const DynInstPtr &load_inst);
    void insertStore(const DynInstPtr &store_inst);

    Fault checkViolations(typename LoadQueue::iterator& loadIt,
            const DynInstPtr& inst);

    void checkSnoop(PacketPtr pkt);

    Fault executeLoad(const DynInstPtr &inst);

    Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; }
    Fault executeStore(const DynInstPtr &inst);

    void commitLoad();
    void commitLoads(InstSeqNum &youngest_inst);

    void commitStores(InstSeqNum &youngest_inst);

    void writebackStores();

    void completeDataAccess(PacketPtr pkt);

    void squash(const InstSeqNum &squashed_num);

    bool violation() { return memDepViolator; }

    DynInstPtr getMemDepViolator();

    unsigned numFreeLoadEntries();

    unsigned numFreeStoreEntries();

    int numLoads() { return loads; }

    int numStores() { return stores; }

    bool isFull() { return lqFull() || sqFull(); }

    bool isEmpty() const { return lqEmpty() && sqEmpty(); }

    bool lqFull() { return loadQueue.full(); }

    bool sqFull() { return storeQueue.full(); }

    bool lqEmpty() const { return loads == 0; }

    bool sqEmpty() const { return stores == 0; }

    unsigned getCount() { return loads + stores; }

    bool hasStoresToWB() { return storesToWB; }

    int numStoresToWB() { return storesToWB; }

    bool
    willWB()
    {
        return storeWBIt.dereferenceable() &&
                        storeWBIt->valid() &&
                        storeWBIt->canWB() &&
                        !storeWBIt->completed() &&
                        !isStoreBlocked;
    }

    void recvRetry();

    unsigned int cacheLineSize();
  private:
    void resetState();

    void writeback(const DynInstPtr &inst, PacketPtr pkt);

    void writebackBlockedStore();

    void completeStore(typename StoreQueue::iterator store_idx);

    void storePostSend();

  public:
    bool trySendPacket(bool isLoad, PacketPtr data_pkt);


    void dumpInsts() const;

    void schedule(Event& ev, Tick when) { cpu->schedule(ev, when); }

    BaseTLB* dTLB() { return cpu->dtb; }

  private:
    O3CPU *cpu;

    IEW *iewStage;

    LSQ *lsq;

    MasterPort *dcachePort;

    class LQSenderState : public LSQSenderState
    {
        using LSQSenderState::alive;
      public:
        LQSenderState(typename LoadQueue::iterator idx_)
            : LSQSenderState(idx_->request(), true), idx(idx_) { }

        typename LoadQueue::iterator idx;
        //virtual LSQRequest* request() { return idx->request(); }
        virtual void
        complete()
        {
            //if (alive())
            //  idx->request()->senderState(nullptr);
        }
    };

    class SQSenderState : public LSQSenderState
    {
        using LSQSenderState::alive;
      public:
        SQSenderState(typename StoreQueue::iterator idx_)
            : LSQSenderState(idx_->request(), false), idx(idx_) { }
        typename StoreQueue::iterator idx;
        //virtual LSQRequest* request() { return idx->request(); }
        virtual void
        complete()
        {
            //if (alive())
            //   idx->request()->senderState(nullptr);
        }
    };

    class WritebackEvent : public Event
    {
      public:
        WritebackEvent(const DynInstPtr &_inst, PacketPtr pkt,
                LSQUnit *lsq_ptr);

        void process();

        const char *description() const;

      private:
        DynInstPtr inst;

        PacketPtr pkt;

        LSQUnit<Impl> *lsqPtr;
    };

  public:
    bool recvTimingResp(PacketPtr pkt);

  private:
    ThreadID lsqID;
  public:
    CircularQueue<SQEntry> storeQueue;

    LoadQueue loadQueue;

  private:
    unsigned depCheckShift;

    bool checkLoads;

    int loads;
    int stores;
    int storesToWB;

    typename StoreQueue::iterator storeWBIt;

    Addr cacheBlockMask;

    typename TimeBuffer<IssueStruct>::wire fromIssue;

    bool stalled;
    InstSeqNum stallingStoreIsn;
    int stallingLoadIdx;

    PacketPtr retryPkt;

    bool isStoreBlocked;

    bool storeInFlight;

    DynInstPtr memDepViolator;

    bool hasPendingRequest;

    LSQRequest* pendingRequest;

    bool needsTSO;

    // Will also need how many read/write ports the Dcache has.  Or keep track
    // of that in stage that is one level up, and only call executeLoad/Store
    // the appropriate number of times.
    Stats::Scalar lsqForwLoads;

    Stats::Scalar invAddrLoads;

    Stats::Scalar lsqSquashedLoads;

    Stats::Scalar lsqIgnoredResponses;

    Stats::Scalar lsqMemOrderViolation;

    Stats::Scalar lsqSquashedStores;

    Stats::Scalar invAddrSwpfs;

    Stats::Scalar lsqBlockedLoads;

    Stats::Scalar lsqRescheduledLoads;

    Stats::Scalar lsqCacheBlocked;

  public:
    Fault read(LSQRequest *req, int load_idx);

    Fault write(LSQRequest *req, uint8_t *data, int store_idx);

    int getLoadHead() { return loadQueue.head(); }

    InstSeqNum
    getLoadHeadSeqNum()
    {
        return loadQueue.front().valid()
            ? loadQueue.front().instruction()->seqNum
            : 0;
    }

    int getStoreHead() { return storeQueue.head(); }
    InstSeqNum
    getStoreHeadSeqNum()
    {
        return storeQueue.front().valid()
            ? storeQueue.front().instruction()->seqNum
            : 0;
    }

    bool isStalled()  { return stalled; }
  public:
    typedef typename CircularQueue<LQEntry>::iterator LQIterator;
    typedef typename CircularQueue<SQEntry>::iterator SQIterator;
    typedef CircularQueue<LQEntry> LQueue;
    typedef CircularQueue<SQEntry> SQueue;
};

template <class Impl>
Fault
LSQUnit<Impl>::read(LSQRequest *req, int load_idx)
{
    LQEntry& load_req = loadQueue[load_idx];
    const DynInstPtr& load_inst = load_req.instruction();

    load_req.setRequest(req);
    assert(load_inst);

    assert(!load_inst->isExecuted());

    // Make sure this isn't a strictly ordered load
    // A bit of a hackish way to get strictly ordered accesses to work
    // only if they're at the head of the LSQ and are ready to commit
    // (at the head of the ROB too).

    if (req->mainRequest()->isStrictlyOrdered() &&
        (load_idx != loadQueue.head() || !load_inst->isAtCommit())) {
        // Tell IQ/mem dep unit that this instruction will need to be
        // rescheduled eventually
        iewStage->rescheduleMemInst(load_inst);
        load_inst->clearIssued();
        load_inst->effAddrValid(false);
        ++lsqRescheduledLoads;
        DPRINTF(LSQUnit, "Strictly ordered load [sn:%lli] PC %s\n",
                load_inst->seqNum, load_inst->pcState());

        // Must delete request now that it wasn't handed off to
        // memory.  This is quite ugly.  @todo: Figure out the proper
        // place to really handle request deletes.
        load_req.setRequest(nullptr);
        req->discard();
        return std::make_shared<GenericISA::M5PanicFault>(
            "Strictly ordered load [sn:%llx] PC %s\n",
            load_inst->seqNum, load_inst->pcState());
    }

    DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, "
            "storeHead: %i addr: %#x%s\n",
            load_idx - 1, load_inst->sqIt._idx, storeQueue.head() - 1,
            req->mainRequest()->getPaddr(), req->isSplit() ? " split" : "");

    if (req->mainRequest()->isLLSC()) {
        // Disable recording the result temporarily.  Writing to misc
        // regs normally updates the result, but this is not the
        // desired behavior when handling store conditionals.
        load_inst->recordResult(false);
        TheISA::handleLockedRead(load_inst.get(), req->mainRequest());
        load_inst->recordResult(true);
    }

    if (req->mainRequest()->isLocalAccess()) {
        assert(!load_inst->memData);
        load_inst->memData = new uint8_t[MaxDataBytes];

        ThreadContext *thread = cpu->tcBase(lsqID);
        PacketPtr main_pkt = new Packet(req->mainRequest(), MemCmd::ReadReq);

        main_pkt->dataStatic(load_inst->memData);

        Cycles delay = req->mainRequest()->localAccessor(thread, main_pkt);

        WritebackEvent *wb = new WritebackEvent(load_inst, main_pkt, this);
        cpu->schedule(wb, cpu->clockEdge(delay));
        return NoFault;
    }

    // Check the SQ for any previous stores that might lead to forwarding
    auto store_it = load_inst->sqIt;
    assert (store_it >= storeWBIt);
    // End once we've reached the top of the LSQ
    while (store_it != storeWBIt) {
        // Move the index to one younger
        store_it--;
        assert(store_it->valid());
        assert(store_it->instruction()->seqNum < load_inst->seqNum);
        int store_size = store_it->size();

        // Cache maintenance instructions go down via the store
        // path but they carry no data and they shouldn't be
        // considered for forwarding
        if (store_size != 0 && !store_it->instruction()->strictlyOrdered() &&
            !(store_it->request()->mainRequest() &&
              store_it->request()->mainRequest()->isCacheMaintenance())) {
            assert(store_it->instruction()->effAddrValid());

            // Check if the store data is within the lower and upper bounds of
            // addresses that the request needs.
            auto req_s = req->mainRequest()->getVaddr();
            auto req_e = req_s + req->mainRequest()->getSize();
            auto st_s = store_it->instruction()->effAddr;
            auto st_e = st_s + store_size;

            bool store_has_lower_limit = req_s >= st_s;
            bool store_has_upper_limit = req_e <= st_e;
            bool lower_load_has_store_part = req_s < st_e;
            bool upper_load_has_store_part = req_e > st_s;

            auto coverage = AddrRangeCoverage::NoAddrRangeCoverage;

            // If the store entry is not atomic (atomic does not have valid
            // data), the store has all of the data needed, and
            // the load is not LLSC, then
            // we can forward data from the store to the load
            if (!store_it->instruction()->isAtomic() &&
                store_has_lower_limit && store_has_upper_limit &&
                !req->mainRequest()->isLLSC()) {

                const auto& store_req = store_it->request()->mainRequest();
                coverage = store_req->isMasked() ?
                    AddrRangeCoverage::PartialAddrRangeCoverage :
                    AddrRangeCoverage::FullAddrRangeCoverage;
            } else if (
                // This is the partial store-load forwarding case where a store
                // has only part of the load's data and the load isn't LLSC
                (!req->mainRequest()->isLLSC() &&
                 ((store_has_lower_limit && lower_load_has_store_part) ||
                  (store_has_upper_limit && upper_load_has_store_part) ||
                  (lower_load_has_store_part && upper_load_has_store_part))) ||
                // The load is LLSC, and the store has all or part of the
                // load's data
                (req->mainRequest()->isLLSC() &&
                 ((store_has_lower_limit || upper_load_has_store_part) &&
                  (store_has_upper_limit || lower_load_has_store_part))) ||
                // The store entry is atomic and has all or part of the load's
                // data
                (store_it->instruction()->isAtomic() &&
                 ((store_has_lower_limit || upper_load_has_store_part) &&
                  (store_has_upper_limit || lower_load_has_store_part)))) {

                coverage = AddrRangeCoverage::PartialAddrRangeCoverage;
            }

            if (coverage == AddrRangeCoverage::FullAddrRangeCoverage) {
                // Get shift amount for offset into the store's data.
                int shift_amt = req->mainRequest()->getVaddr() -
                    store_it->instruction()->effAddr;

                // Allocate memory if this is the first time a load is issued.
                if (!load_inst->memData) {
                    load_inst->memData =
                        new uint8_t[req->mainRequest()->getSize()];
                }
                if (store_it->isAllZeros())
                    memset(load_inst->memData, 0,
                            req->mainRequest()->getSize());
                else
                    memcpy(load_inst->memData,
                        store_it->data() + shift_amt,
                        req->mainRequest()->getSize());

                DPRINTF(LSQUnit, "Forwarding from store idx %i to load to "
                        "addr %#x\n", store_it._idx,
                        req->mainRequest()->getVaddr());

                PacketPtr data_pkt = new Packet(req->mainRequest(),
                        MemCmd::ReadReq);
                data_pkt->dataStatic(load_inst->memData);

                if (req->isAnyOutstandingRequest()) {
                    assert(req->_numOutstandingPackets > 0);
                    // There are memory requests packets in flight already.
                    // This may happen if the store was not complete the
                    // first time this load got executed. Signal the senderSate
                    // that response packets should be discarded.
                    req->discardSenderState();
                }

                WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt,
                        this);

                // We'll say this has a 1 cycle load-store forwarding latency
                // for now.
                // @todo: Need to make this a parameter.
                cpu->schedule(wb, curTick());

                // Don't need to do anything special for split loads.
                ++lsqForwLoads;

                return NoFault;
            } else if (coverage == AddrRangeCoverage::PartialAddrRangeCoverage) {
                // If it's already been written back, then don't worry about
                // stalling on it.
                if (store_it->completed()) {
                    panic("Should not check one of these");
                    continue;
                }

                // Must stall load and force it to retry, so long as it's the
                // oldest load that needs to do so.
                if (!stalled ||
                    (stalled &&
                     load_inst->seqNum <
                     loadQueue[stallingLoadIdx].instruction()->seqNum)) {
                    stalled = true;
                    stallingStoreIsn = store_it->instruction()->seqNum;
                    stallingLoadIdx = load_idx;
                }

                // Tell IQ/mem dep unit that this instruction will need to be
                // rescheduled eventually
                iewStage->rescheduleMemInst(load_inst);
                load_inst->clearIssued();
                load_inst->effAddrValid(false);
                ++lsqRescheduledLoads;

                // Do not generate a writeback event as this instruction is not
                // complete.
                DPRINTF(LSQUnit, "Load-store forwarding mis-match. "
                        "Store idx %i to load addr %#x\n",
                        store_it._idx, req->mainRequest()->getVaddr());

                // Must discard the request.
                req->discard();
                load_req.setRequest(nullptr);
                return NoFault;
            }
        }
    }

    // If there's no forwarding case, then go access memory
    DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n",
            load_inst->seqNum, load_inst->pcState());

    // Allocate memory if this is the first time a load is issued.
    if (!load_inst->memData) {
        load_inst->memData = new uint8_t[req->mainRequest()->getSize()];
    }

    // For now, load throughput is constrained by the number of
    // load FUs only, and loads do not consume a cache port (only
    // stores do).
    // @todo We should account for cache port contention
    // and arbitrate between loads and stores.

    // if we the cache is not blocked, do cache access
    if (req->senderState() == nullptr) {
        LQSenderState *state = new LQSenderState(
                loadQueue.getIterator(load_idx));
        state->isLoad = true;
        state->inst = load_inst;
        state->isSplit = req->isSplit();
        req->senderState(state);
    }
    req->buildPackets();
    req->sendPacketToCache();
    if (!req->isSent())
        iewStage->blockMemInst(load_inst);

    return NoFault;
}

template <class Impl>
Fault
LSQUnit<Impl>::write(LSQRequest *req, uint8_t *data, int store_idx)
{
    assert(storeQueue[store_idx].valid());

    DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x | storeHead:%i "
            "[sn:%llu]\n",
            store_idx - 1, req->request()->getPaddr(), storeQueue.head() - 1,
            storeQueue[store_idx].instruction()->seqNum);

    storeQueue[store_idx].setRequest(req);
    unsigned size = req->_size;
    storeQueue[store_idx].size() = size;
    bool store_no_data =
        req->mainRequest()->getFlags() & Request::STORE_NO_DATA;
    storeQueue[store_idx].isAllZeros() = store_no_data;
    assert(size <= SQEntry::DataSize || store_no_data);

    // copy data into the storeQueue only if the store request has valid data
    if (!(req->request()->getFlags() & Request::CACHE_BLOCK_ZERO) &&
        !req->request()->isCacheMaintenance() &&
        !req->request()->isAtomic())
        memcpy(storeQueue[store_idx].data(), data, size);

    // This function only writes the data to the store queue, so no fault
    // can happen here.
    return NoFault;
}

#endif // __CPU_O3_LSQ_UNIT_HH__