lsq.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2011-2012, 2014, 2018-2019, 2021 ARM Limited
 * Copyright (c) 2013 Advanced Micro Devices, Inc.
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Copyright (c) 2004-2006 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#ifndef __CPU_O3_LSQ_HH__
#define __CPU_O3_LSQ_HH__
 
#include <cassert>
#include <cstdint>
#include <list>
#include <map>
#include <queue>
#include <vector>
 
#include "arch/generic/mmu.hh"
#include "arch/generic/tlb.hh"
#include "base/flags.hh"
#include "base/statistics.hh"
#include "base/types.hh"
#include "cpu/inst_seq.hh"
#include "cpu/o3/dyn_inst_ptr.hh"
#include "cpu/utils.hh"
#include "enums/SMTQueuePolicy.hh"
#include "mem/port.hh"
#include "sim/sim_object.hh"
 
namespace gem5
{
 
struct BaseO3CPUParams;
 
namespace o3
{
 
class CPU;
class IEW;
class LSQUnit;
 
class LSQ
{
  public:
    class LSQRequest;

    class DcachePort : public RequestPort
    {
      protected:

        LSQ *lsq;
        CPU *cpu;
 
      public:
        DcachePort(LSQ *_lsq, CPU *_cpu);
 
        struct DcachePortStats : public statistics::Group
        {
            DcachePortStats(CPU* cpu);
 
            /* Number of recieved responses */
            statistics::Scalar numRecvResp;
 
            /* Number of recieved response bytes */
            statistics::Scalar numRecvRespBytes;
 
            /* Average bandwidth of received responses */
            statistics::Formula recvRespAvgBW;
 
            /* Average pkt size per received response */
            statistics::Formula recvRespAvgSize;
 
            /* Average rate of received responses per cycle */
            statistics::Formula recvRespAvgRate;
 
            /* Average retry rate per received response */
            statistics::Formula recvRespAvgRetryRate;
 
            /* Number of retry responses sent */
            statistics::Scalar numSendRetryResp;
        } dcachePortStats;
 
      protected:

        virtual bool recvTimingResp(PacketPtr pkt);
        virtual void recvTimingSnoopReq(PacketPtr pkt);
 
        virtual void
        recvFunctionalSnoop(PacketPtr pkt)
        {
            // @todo: Is there a need for potential invalidation here?
        }

        virtual void recvReqRetry();

        virtual bool isSnooping() const { return true; }

        bool throttleReadResp(PacketPtr pkt);
    };

    class LSQRequest : public BaseMMU::Translation, public Packet::SenderState
    {
      protected:
        typedef uint32_t FlagsStorage;
        typedef Flags<FlagsStorage> FlagsType;
 
        enum Flag : FlagsStorage
        {
            IsLoad              = 0x00000001,
            WriteBackToRegister = 0x00000002,
            Delayed             = 0x00000004,
            IsSplit             = 0x00000008,
            TranslationStarted  = 0x00000010,
            TranslationFinished = 0x00000020,
            Sent                = 0x00000040,
            Retry               = 0x00000080,
            Complete            = 0x00000100,
            TranslationSquashed = 0x00000200,
            Discarded           = 0x00000400,
            LSQEntryFreed       = 0x00000800,
            WritebackScheduled  = 0x00001000,
            WritebackDone       = 0x00002000,
            IsAtomic            = 0x00004000
        };
        FlagsType flags;
 
        enum class State
        {
            NotIssued,
            Translation,
            Request,
            Fault,
            PartialFault,
        };
        State _state;
        void setState(const State& newState) { _state = newState; }
 
        uint32_t numTranslatedFragments;
        uint32_t numInTranslationFragments;
 
 
        void markDelayed() override { flags.set(Flag::Delayed); }
        bool isDelayed() { return flags.isSet(Flag::Delayed); }
 
      public:
        LSQUnit& _port;
        const DynInstPtr _inst;
        uint32_t _taskId;
        PacketDataPtr _data;
        std::vector<PacketPtr> _packets;
        std::vector<RequestPtr> _reqs;
        std::vector<Fault> _fault;
        uint64_t* _res;
        const Addr _addr;
        const uint32_t _size;
        const Request::Flags _flags;
        std::vector<bool> _byteEnable;
        uint32_t _numOutstandingPackets;
        AtomicOpFunctorPtr _amo_op;
        bool _hasStaleTranslation;
 
      protected:
        LSQUnit* lsqUnit() { return &_port; }
        LSQRequest(LSQUnit* port, const DynInstPtr& inst, bool isLoad);
        LSQRequest(LSQUnit* port, const DynInstPtr& inst, bool isLoad,
                const Addr& addr, const uint32_t& size,
                const Request::Flags& flags_, PacketDataPtr data=nullptr,
                uint64_t* res=nullptr, AtomicOpFunctorPtr amo_op=nullptr,
                bool stale_translation=false);
 
        bool
        isLoad() const
        {
            return flags.isSet(Flag::IsLoad);
        }
 
        bool
        isAtomic() const
        {
            return flags.isSet(Flag::IsAtomic);
        }

        void install();
 
        bool squashed() const override;
 

        void
        release(Flag reason)
        {
            assert(reason == Flag::LSQEntryFreed || reason == Flag::Discarded);
            if (!isAnyOutstandingRequest()) {
                delete this;
            } else {
                flags.set(reason);
            }
        }

        void addReq(Addr addr, unsigned size,
                const std::vector<bool>& byte_enable);

        virtual ~LSQRequest();
 
      public:
        void
        setContext(const ContextID& context_id)
        {
            req()->setContext(context_id);
        }
 
        const DynInstPtr& instruction() { return _inst; }
 
        bool hasStaleTranslation() const { return _hasStaleTranslation; }
 
        virtual void markAsStaleTranslation() = 0;

        void
        setVirt(Addr vaddr, unsigned size, Request::Flags flags_,
                RequestorID requestor_id, Addr pc)
        {
            req()->setVirt(vaddr, size, flags_, requestor_id, pc);
        }
 
        ContextID contextId() const;
 
        void
        taskId(const uint32_t& v)
        {
            _taskId = v;
            for (auto& r: _reqs)
                r->taskId(v);
        }
 
        uint32_t taskId() const { return _taskId; }
 
        RequestPtr req(int idx = 0) { return _reqs.at(idx); }
        const RequestPtr req(int idx = 0) const { return _reqs.at(idx); }
 
        Addr getVaddr(int idx = 0) const { return req(idx)->getVaddr(); }
        virtual void initiateTranslation() = 0;
 
        PacketPtr packet(int idx = 0) { return _packets.at(idx); }
 
        virtual PacketPtr
        mainPacket()
        {
            assert (_packets.size() == 1);
            return packet();
        }
 
        virtual RequestPtr
        mainReq()
        {
            assert (_reqs.size() == 1);
            return req();
        }

        bool
        isAnyOutstandingRequest()
        {
            return numInTranslationFragments > 0 ||
                _numOutstandingPackets > 0 ||
                (flags.isSet(Flag::WritebackScheduled) &&
                 !flags.isSet(Flag::WritebackDone));
        }

        bool
        isReleased()
        {
            return flags.isSet(Flag::LSQEntryFreed) ||
                flags.isSet(Flag::Discarded);
        }
 
        bool
        isSplit() const
        {
            return flags.isSet(Flag::IsSplit);
        }
 
        bool
        needWBToRegister() const
        {
            return flags.isSet(Flag::WriteBackToRegister);
        }

        virtual bool recvTimingResp(PacketPtr pkt) = 0;
        virtual void sendPacketToCache() = 0;
        virtual void buildPackets() = 0;

        virtual Cycles handleLocalAccess(
                gem5::ThreadContext *thread, PacketPtr pkt) = 0;

        virtual bool isCacheBlockHit(Addr blockAddr, Addr cacheBlockMask) = 0;

        void
        packetSent()
        {
            flags.set(Flag::Sent);
        }

        void
        packetNotSent()
        {
            flags.set(Flag::Retry);
            flags.clear(Flag::Sent);
        }
 
        void sendFragmentToTranslation(int i);
        bool
        isComplete()
        {
            return flags.isSet(Flag::Complete);
        }
 
        bool
        isInTranslation()
        {
            return _state == State::Translation;
        }
 
        bool
        isTranslationComplete()
        {
            return flags.isSet(Flag::TranslationStarted) &&
                   !isInTranslation();
        }
 
        bool
        isTranslationBlocked()
        {
            return _state == State::Translation &&
                flags.isSet(Flag::TranslationStarted) &&
                !flags.isSet(Flag::TranslationFinished);
        }
 
        bool
        isSent()
        {
            return flags.isSet(Flag::Sent);
        }
 
        bool
        isPartialFault()
        {
            return _state == State::PartialFault;
        }
 
        bool
        isMemAccessRequired()
        {
            return (_state == State::Request ||
                    (isPartialFault() && isLoad()));
        }
 
        void
        setStateToFault()
        {
            setState(State::Fault);
        }

        void
        freeLSQEntry()
        {
            release(Flag::LSQEntryFreed);
        }

        void
        discard()
        {
            release(Flag::Discarded);
        }
 
        void
        packetReplied()
        {
            assert(_numOutstandingPackets > 0);
            _numOutstandingPackets--;
            if (_numOutstandingPackets == 0 && isReleased())
                delete this;
        }
 
        void
        writebackScheduled()
        {
            assert(!flags.isSet(Flag::WritebackScheduled));
            flags.set(Flag::WritebackScheduled);
        }
 
        void
        writebackDone()
        {
            flags.set(Flag::WritebackDone);
            /* If the lsq resources are already free */
            if (_numOutstandingPackets == 0 && isReleased()) {
                delete this;
            }
        }
 
        void
        squashTranslation()
        {
            assert(numInTranslationFragments == 0);
            flags.set(Flag::TranslationSquashed);
            /* If we are on our own, self-destruct. */
            if (isReleased()) {
                delete this;
            }
        }
 
        void
        complete()
        {
            flags.set(Flag::Complete);
        }
 
        virtual std::string name() const { return "LSQRequest"; }
    };
 
    class SingleDataRequest : public LSQRequest
    {
      public:
        SingleDataRequest(LSQUnit* port, const DynInstPtr& inst,
                bool isLoad, const Addr& addr, const uint32_t& size,
                const Request::Flags& flags_, PacketDataPtr data=nullptr,
                uint64_t* res=nullptr, AtomicOpFunctorPtr amo_op=nullptr) :
            LSQRequest(port, inst, isLoad, addr, size, flags_, data, res,
                       std::move(amo_op)) {}
 
        virtual ~SingleDataRequest() {}
        virtual void markAsStaleTranslation();
        virtual void initiateTranslation();
        virtual void finish(const Fault &fault, const RequestPtr &req,
                gem5::ThreadContext* tc, BaseMMU::Mode mode);
        virtual bool recvTimingResp(PacketPtr pkt);
        virtual void sendPacketToCache();
        virtual void buildPackets();
        virtual Cycles handleLocalAccess(
                gem5::ThreadContext *thread, PacketPtr pkt);
        virtual bool isCacheBlockHit(Addr blockAddr, Addr cacheBlockMask);
        virtual std::string name() const { return "SingleDataRequest"; }
    };
 
    // This class extends SingleDataRequest for the purpose
    // of allowing special requests (eg Hardware transactional memory, TLB
    // shootdowns) to bypass irrelevant system elements like translation &
    // squashing.
    class UnsquashableDirectRequest : public SingleDataRequest
    {
      public:
        UnsquashableDirectRequest(LSQUnit* port, const DynInstPtr& inst,
                const Request::Flags& flags_);
        inline virtual ~UnsquashableDirectRequest() {}
        virtual void initiateTranslation();
        virtual void markAsStaleTranslation();
        virtual void finish(const Fault &fault, const RequestPtr &req,
                gem5::ThreadContext* tc, BaseMMU::Mode mode);
        virtual std::string
        name() const
        {
            return "UnsquashableDirectRequest";
        }
    };
 
    class SplitDataRequest : public LSQRequest
    {
      protected:
        uint32_t numFragments;
        uint32_t numReceivedPackets;
        RequestPtr _mainReq;
        PacketPtr _mainPacket;
 
      public:
        SplitDataRequest(LSQUnit* port, const DynInstPtr& inst,
                bool isLoad, const Addr& addr, const uint32_t& size,
                const Request::Flags & flags_, PacketDataPtr data=nullptr,
                uint64_t* res=nullptr) :
            LSQRequest(port, inst, isLoad, addr, size, flags_, data, res,
                       nullptr),
            numFragments(0),
            numReceivedPackets(0),
            _mainReq(nullptr),
            _mainPacket(nullptr)
        {
            flags.set(Flag::IsSplit);
        }
        virtual ~SplitDataRequest()
        {
            if (_mainReq) {
                _mainReq = nullptr;
            }
            if (_mainPacket) {
                delete _mainPacket;
                _mainPacket = nullptr;
            }
        }
        virtual void markAsStaleTranslation();
        virtual void finish(const Fault &fault, const RequestPtr &req,
                gem5::ThreadContext* tc, BaseMMU::Mode mode);
        virtual bool recvTimingResp(PacketPtr pkt);
        virtual void initiateTranslation();
        virtual void sendPacketToCache();
        virtual void buildPackets();
 
        virtual Cycles handleLocalAccess(
                gem5::ThreadContext *thread, PacketPtr pkt);
        virtual bool isCacheBlockHit(Addr blockAddr, Addr cacheBlockMask);
 
        virtual RequestPtr mainReq();
        virtual PacketPtr mainPacket();
        virtual std::string name() const { return "SplitDataRequest"; }
    };

    LSQ(CPU *cpu_ptr, IEW *iew_ptr, const BaseO3CPUParams &params);

    std::string name() const;

    void setActiveThreads(std::list<ThreadID> *at_ptr);

    void drainSanityCheck() const;
    bool isDrained() const;
    void takeOverFrom();

    int entryAmount(ThreadID num_threads);

    void tick();

    void insertLoad(const DynInstPtr &load_inst);
    void insertStore(const DynInstPtr &store_inst);

    Fault executeLoad(const DynInstPtr &inst);

    Fault executeStore(const DynInstPtr &inst);

    void commitLoads(InstSeqNum &youngest_inst, ThreadID tid);

    void commitStores(InstSeqNum &youngest_inst, ThreadID tid);

    void writebackStores();
    void writebackStores(ThreadID tid);

    void squash(const InstSeqNum &squashed_num, ThreadID tid);

    bool violation();

    bool violation(ThreadID tid);

    DynInstPtr getMemDepViolator(ThreadID tid);

    int getLoadHead(ThreadID tid);

    InstSeqNum getLoadHeadSeqNum(ThreadID tid);

    int getStoreHead(ThreadID tid);

    InstSeqNum getStoreHeadSeqNum(ThreadID tid);

    int getCount();
    int getCount(ThreadID tid);

    int numLoads();
    int numLoads(ThreadID tid);

    int numStores();
    int numStores(ThreadID tid);
 
 
    // hardware transactional memory
 
    int numHtmStarts(ThreadID tid) const;
    int numHtmStops(ThreadID tid) const;
    void resetHtmStartsStops(ThreadID tid);
    uint64_t getLatestHtmUid(ThreadID tid) const;
    void setLastRetiredHtmUid(ThreadID tid, uint64_t htmUid);

    unsigned numFreeLoadEntries();

    unsigned numFreeStoreEntries();

    unsigned numFreeEntries(ThreadID tid);

    unsigned numFreeLoadEntries(ThreadID tid);

    unsigned numFreeStoreEntries(ThreadID tid);

    bool isFull();
    bool isFull(ThreadID tid);

    bool isEmpty() const;
    bool lqEmpty() const;
    bool sqEmpty() const;

    bool lqFull();
    bool lqFull(ThreadID tid);

    bool sqFull();
    bool sqFull(ThreadID tid);

    bool isStalled();
    bool isStalled(ThreadID tid);

    bool hasStoresToWB();

    bool hasStoresToWB(ThreadID tid);

    int numStoresToWB(ThreadID tid);

    bool willWB();
    bool willWB(ThreadID tid);

    void dumpInsts() const;
    void dumpInsts(ThreadID tid) const;

    Fault read(LSQRequest* request, ssize_t load_idx);

    Fault write(LSQRequest* request, uint8_t *data, ssize_t store_idx);

    void checkStaleTranslations();

    void recvReqRetry();
 
    void completeDataAccess(PacketPtr pkt);
    bool recvTimingResp(PacketPtr pkt);
 
    void recvTimingSnoopReq(PacketPtr pkt);
 
    Fault pushRequest(const DynInstPtr& inst, bool isLoad, uint8_t *data,
                      unsigned int size, Addr addr, Request::Flags flags,
                      uint64_t *res, AtomicOpFunctorPtr amo_op,
                      const std::vector<bool>& byte_enable);

    CPU *cpu;

    IEW *iewStage;

    bool cacheBlocked() const;
    void cacheBlocked(bool v);
    bool cachePortAvailable(bool is_load) const;
    void cachePortBusy(bool is_load);
 
    RequestPort &getDataPort() { return dcachePort; }
 
    void sendRetryResp();
 
  protected:
    bool _cacheBlocked;
    int cacheStorePorts;
    int usedStorePorts;
    int cacheLoadPorts;
    int usedLoadPorts;

    bool waitingForStaleTranslation;
    Addr staleTranslationWaitTxnId;

    SMTQueuePolicy lsqPolicy;

    static uint32_t
    maxLSQAllocation(SMTQueuePolicy pol, uint32_t entries,
            uint32_t numThreads, uint32_t SMTThreshold)
    {
        if (pol == SMTQueuePolicy::Dynamic) {
            return entries;
        } else if (pol == SMTQueuePolicy::Partitioned) {
            //@todo:make work if part_amt doesnt divide evenly.
            return entries / numThreads;
        } else if (pol == SMTQueuePolicy::Threshold) {
            //Divide up by threshold amount
            //@todo: Should threads check the max and the total
            //amount of the LSQ
            return SMTThreshold;
        }
        return 0;
    }

    std::list<ThreadID> *activeThreads;

    unsigned LQEntries;
    unsigned SQEntries;

    unsigned maxLQEntries;

    unsigned maxSQEntries;

    DcachePort dcachePort;

    std::vector<LSQUnit> thread;

    ThreadID numThreads;

    const bool recvRespThrottling;
    const unsigned recvRespMaxCachelines;
    const unsigned recvRespBufferSize;
    unsigned recvRespBytes;
    unsigned recvRespPendBytes;
    unsigned recvRespCachelines;
    Addr recvRespLastCachelineAddr;
    Cycles recvRespLastActiveCycle;
    EventFunctionWrapper retryRespEvent;
};
 
} // namespace o3
} // namespace gem5
 
#endif // __CPU_O3_LSQ_HH__