lsq.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2011-2012, 2014, 2018-2019 ARM Limited
 * Copyright (c) 2013 Advanced Micro Devices, Inc.
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Copyright (c) 2004-2006 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __CPU_O3_LSQ_HH__
#define __CPU_O3_LSQ_HH__

#include <map>
#include <queue>

#include "arch/generic/tlb.hh"
#include "cpu/inst_seq.hh"
#include "cpu/o3/lsq_unit.hh"
#include "cpu/utils.hh"
#include "enums/SMTQueuePolicy.hh"
#include "mem/port.hh"
#include "sim/sim_object.hh"

struct DerivO3CPUParams;

template <class Impl>
class FullO3CPU;

template <class Impl>
class LSQ

{
  public:
    typedef typename Impl::O3CPU O3CPU;
    typedef typename Impl::DynInstPtr DynInstPtr;
    typedef typename Impl::CPUPol::IEW IEW;
    typedef typename Impl::CPUPol::LSQUnit LSQUnit;

    class LSQRequest;
    class LSQSenderState : public Packet::SenderState
    {
      protected:
        LSQRequest* _request;

        LSQSenderState(LSQRequest* request, bool isLoad_)
            : _request(request), mainPkt(nullptr), pendingPacket(nullptr),
              outstanding(0), isLoad(isLoad_), needWB(isLoad_), isSplit(false),
              pktToSend(false), deleted(false)
          { }
      public:

        DynInstPtr inst;
        PacketPtr mainPkt;
        PacketPtr pendingPacket;
        uint8_t outstanding;
        bool isLoad;
        bool needWB;
        bool isSplit;
        bool pktToSend;
        bool deleted;
        ContextID contextId() { return inst->contextId(); }

        inline bool isComplete() { return outstanding == 0; }
        inline void deleteRequest() { deleted = true; }
        inline bool alive() { return !deleted; }
        LSQRequest* request() { return _request; }
        virtual void complete() = 0;
        void writebackDone() { _request->writebackDone(); }
    };

    class DcachePort : public MasterPort
    {
      protected:

        LSQ<Impl> *lsq;
        FullO3CPU<Impl> *cpu;

      public:
        DcachePort(LSQ<Impl> *_lsq, FullO3CPU<Impl>* _cpu)
            : MasterPort(_cpu->name() + ".dcache_port", _cpu), lsq(_lsq),
              cpu(_cpu)
        { }

      protected:

        virtual bool recvTimingResp(PacketPtr pkt);
        virtual void recvTimingSnoopReq(PacketPtr pkt);

        virtual void recvFunctionalSnoop(PacketPtr pkt)
        {
            // @todo: Is there a need for potential invalidation here?
        }

        virtual void recvReqRetry();

        virtual bool isSnooping() const { return true; }
    };

    class LSQRequest : public BaseTLB::Translation
    {
      protected:
        typedef uint32_t FlagsStorage;
        typedef ::Flags<FlagsStorage> FlagsType;

        enum Flag : FlagsStorage
        {
            IsLoad              = 0x00000001,
            WbStore             = 0x00000002,
            Delayed             = 0x00000004,
            IsSplit             = 0x00000008,
            TranslationStarted  = 0x00000010,
            TranslationFinished = 0x00000020,
            Sent                = 0x00000040,
            Retry               = 0x00000080,
            Complete            = 0x00000100,
            TranslationSquashed = 0x00000200,
            Discarded           = 0x00000400,
            LSQEntryFreed       = 0x00000800,
            WritebackScheduled  = 0x00001000,
            WritebackDone       = 0x00002000,
            IsAtomic            = 0x00004000
        };
        FlagsType flags;

        enum class State
        {
            NotIssued,
            Translation,
            Request,
            Fault,
            PartialFault,
        };
        State _state;
        LSQSenderState* _senderState;
        void setState(const State& newState) { _state = newState; }

        uint32_t numTranslatedFragments;
        uint32_t numInTranslationFragments;

        uint32_t _entryIdx;

        void markDelayed() override { flags.set(Flag::Delayed); }
        bool isDelayed() { return flags.isSet(Flag::Delayed); }

      public:
        LSQUnit& _port;
        const DynInstPtr _inst;
        uint32_t _taskId;
        PacketDataPtr _data;
        std::vector<PacketPtr> _packets;
        std::vector<RequestPtr> _requests;
        std::vector<Fault> _fault;
        uint64_t* _res;
        const Addr _addr;
        const uint32_t _size;
        const Request::Flags _flags;
        std::vector<bool> _byteEnable;
        uint32_t _numOutstandingPackets;
        AtomicOpFunctorPtr _amo_op;
      protected:
        LSQUnit* lsqUnit() { return &_port; }
        LSQRequest(LSQUnit* port, const DynInstPtr& inst, bool isLoad) :
            _state(State::NotIssued), _senderState(nullptr),
            _port(*port), _inst(inst), _data(nullptr),
            _res(nullptr), _addr(0), _size(0), _flags(0),
            _numOutstandingPackets(0), _amo_op(nullptr)
        {
            flags.set(Flag::IsLoad, isLoad);
            flags.set(Flag::WbStore,
                      _inst->isStoreConditional() || _inst->isAtomic());
            flags.set(Flag::IsAtomic, _inst->isAtomic());
            install();
        }
        LSQRequest(LSQUnit* port, const DynInstPtr& inst, bool isLoad,
                   const Addr& addr, const uint32_t& size,
                   const Request::Flags& flags_,
                   PacketDataPtr data = nullptr, uint64_t* res = nullptr,
                   AtomicOpFunctorPtr amo_op = nullptr)
            : _state(State::NotIssued), _senderState(nullptr),
            numTranslatedFragments(0),
            numInTranslationFragments(0),
            _port(*port), _inst(inst), _data(data),
            _res(res), _addr(addr), _size(size),
            _flags(flags_),
            _numOutstandingPackets(0),
            _amo_op(std::move(amo_op))
        {
            flags.set(Flag::IsLoad, isLoad);
            flags.set(Flag::WbStore,
                      _inst->isStoreConditional() || _inst->isAtomic());
            flags.set(Flag::IsAtomic, _inst->isAtomic());
            install();
        }

        bool
        isLoad() const
        {
            return flags.isSet(Flag::IsLoad);
        }

        bool
        isAtomic() const
        {
            return flags.isSet(Flag::IsAtomic);
        }

        void install()
        {
            if (isLoad()) {
                _port.loadQueue[_inst->lqIdx].setRequest(this);
            } else {
                // Store, StoreConditional, and Atomic requests are pushed
                // to this storeQueue
                _port.storeQueue[_inst->sqIdx].setRequest(this);
            }
        }
        virtual bool
        squashed() const override
        {
            return _inst->isSquashed();
        }

        bool
        isReleased()
        {
            return flags.isSet(Flag::LSQEntryFreed) ||
                flags.isSet(Flag::Discarded);
        }

        void release(Flag reason)
        {
            assert(reason == Flag::LSQEntryFreed || reason == Flag::Discarded);
            if (!isAnyOutstandingRequest()) {
                delete this;
            } else {
                if (_senderState) {
                    _senderState->deleteRequest();
                }
                flags.set(reason);
            }
        }

        void
        addRequest(Addr addr, unsigned size,
                   const std::vector<bool>& byte_enable)
        {
            if (byte_enable.empty() ||
                isAnyActiveElement(byte_enable.begin(), byte_enable.end())) {
                auto request = std::make_shared<Request>(
                        addr, size, _flags, _inst->masterId(),
                        _inst->instAddr(), _inst->contextId(),
                        std::move(_amo_op));
                if (!byte_enable.empty()) {
                    request->setByteEnable(byte_enable);
                }
                _requests.push_back(request);
            }
        }

        virtual ~LSQRequest()
        {
            assert(!isAnyOutstandingRequest());
            _inst->savedReq = nullptr;
            if (_senderState)
                delete _senderState;

            for (auto r: _packets)
                delete r;
        };


      public:
        void
        setContext(const ContextID& context_id)
        {
            request()->setContext(context_id);
        }

        const DynInstPtr&
        instruction()
        {
            return _inst;
        }

        void
        setVirt(Addr vaddr, unsigned size, Request::Flags flags_,
                MasterID mid, Addr pc)
        {
            request()->setVirt(vaddr, size, flags_, mid, pc);
        }

        void
        taskId(const uint32_t& v)
        {
            _taskId = v;
            for (auto& r: _requests)
                r->taskId(v);
        }

        uint32_t taskId() const { return _taskId; }
        RequestPtr request(int idx = 0) { return _requests.at(idx); }

        const RequestPtr
        request(int idx = 0) const
        {
            return _requests.at(idx);
        }

        Addr getVaddr(int idx = 0) const { return request(idx)->getVaddr(); }
        virtual void initiateTranslation() = 0;

        PacketPtr packet(int idx = 0) { return _packets.at(idx); }

        virtual PacketPtr
        mainPacket()
        {
            assert (_packets.size() == 1);
            return packet();
        }

        virtual RequestPtr
        mainRequest()
        {
            assert (_requests.size() == 1);
            return request();
        }

        void
        senderState(LSQSenderState* st)
        {
            _senderState = st;
            for (auto& pkt: _packets) {
                if (pkt)
                    pkt->senderState = st;
            }
        }

        const LSQSenderState*
        senderState() const
        {
            return _senderState;
        }

        void
        discardSenderState()
        {
            assert(_senderState);
            _senderState->deleteRequest();
        }

        bool
        isAnyOutstandingRequest()
        {
            return numInTranslationFragments > 0 ||
                _numOutstandingPackets > 0 ||
                (flags.isSet(Flag::WritebackScheduled) &&
                 !flags.isSet(Flag::WritebackDone));
        }

        bool
        isSplit() const
        {
            return flags.isSet(Flag::IsSplit);
        }
        virtual bool recvTimingResp(PacketPtr pkt) = 0;
        virtual void sendPacketToCache() = 0;
        virtual void buildPackets() = 0;

        virtual Cycles handleLocalAccess(
                ThreadContext *thread, PacketPtr pkt) = 0;

        virtual bool isCacheBlockHit(Addr blockAddr, Addr cacheBlockMask) = 0;

        void
        packetSent()
        {
            flags.set(Flag::Sent);
        }
        void
        packetNotSent()
        {
            flags.set(Flag::Retry);
            flags.clear(Flag::Sent);
        }

        void sendFragmentToTranslation(int i);
        bool
        isComplete()
        {
            return flags.isSet(Flag::Complete);
        }

        bool
        isInTranslation()
        {
            return _state == State::Translation;
        }

        bool
        isTranslationComplete()
        {
            return flags.isSet(Flag::TranslationStarted) &&
                   !isInTranslation();
        }

        bool
        isTranslationBlocked()
        {
            return _state == State::Translation &&
                flags.isSet(Flag::TranslationStarted) &&
                !flags.isSet(Flag::TranslationFinished);
        }

        bool
        isSent()
        {
            return flags.isSet(Flag::Sent);
        }

        bool
        isPartialFault()
        {
            return _state == State::PartialFault;
        }

        bool
        isMemAccessRequired()
        {
            return (_state == State::Request ||
                    (isPartialFault() && isLoad()));
        }

        void
        setStateToFault()
        {
            setState(State::Fault);
        }

        void
        freeLSQEntry()
        {
            release(Flag::LSQEntryFreed);
        }

        void
        discard()
        {
            release(Flag::Discarded);
        }

        void
        packetReplied()
        {
            assert(_numOutstandingPackets > 0);
            _numOutstandingPackets--;
            if (_numOutstandingPackets == 0 && isReleased())
                delete this;
        }

        void
        writebackScheduled()
        {
            assert(!flags.isSet(Flag::WritebackScheduled));
            flags.set(Flag::WritebackScheduled);
        }

        void
        writebackDone()
        {
            flags.set(Flag::WritebackDone);
            /* If the lsq resources are already free */
            if (isReleased()) {
                delete this;
            }
        }

        void
        squashTranslation()
        {
            assert(numInTranslationFragments == 0);
            flags.set(Flag::TranslationSquashed);
            /* If we are on our own, self-destruct. */
            if (isReleased()) {
                delete this;
            }
        }

        void
        complete()
        {
            flags.set(Flag::Complete);
        }
    };

    class SingleDataRequest : public LSQRequest
    {
      protected:
        /* Given that we are inside templates, children need explicit
         * declaration of the names in the parent class. */
        using Flag = typename LSQRequest::Flag;
        using State = typename LSQRequest::State;
        using LSQRequest::_addr;
        using LSQRequest::_fault;
        using LSQRequest::_flags;
        using LSQRequest::_size;
        using LSQRequest::_byteEnable;
        using LSQRequest::_requests;
        using LSQRequest::_inst;
        using LSQRequest::_packets;
        using LSQRequest::_port;
        using LSQRequest::_res;
        using LSQRequest::_taskId;
        using LSQRequest::_senderState;
        using LSQRequest::_state;
        using LSQRequest::flags;
        using LSQRequest::isLoad;
        using LSQRequest::isTranslationComplete;
        using LSQRequest::lsqUnit;
        using LSQRequest::request;
        using LSQRequest::sendFragmentToTranslation;
        using LSQRequest::setState;
        using LSQRequest::numInTranslationFragments;
        using LSQRequest::numTranslatedFragments;
        using LSQRequest::_numOutstandingPackets;
        using LSQRequest::_amo_op;
      public:
        SingleDataRequest(LSQUnit* port, const DynInstPtr& inst, bool isLoad,
                          const Addr& addr, const uint32_t& size,
                          const Request::Flags& flags_,
                          PacketDataPtr data = nullptr,
                          uint64_t* res = nullptr,
                          AtomicOpFunctorPtr amo_op = nullptr) :
            LSQRequest(port, inst, isLoad, addr, size, flags_, data, res,
                       std::move(amo_op)) {}

        inline virtual ~SingleDataRequest() {}
        virtual void initiateTranslation();
        virtual void finish(const Fault &fault, const RequestPtr &req,
                ThreadContext* tc, BaseTLB::Mode mode);
        virtual bool recvTimingResp(PacketPtr pkt);
        virtual void sendPacketToCache();
        virtual void buildPackets();
        virtual Cycles handleLocalAccess(ThreadContext *thread, PacketPtr pkt);
        virtual bool isCacheBlockHit(Addr blockAddr, Addr cacheBlockMask);
    };

    class SplitDataRequest : public LSQRequest
    {
      protected:
        /* Given that we are inside templates, children need explicit
         * declaration of the names in the parent class. */
        using Flag = typename LSQRequest::Flag;
        using State = typename LSQRequest::State;
        using LSQRequest::_addr;
        using LSQRequest::_data;
        using LSQRequest::_fault;
        using LSQRequest::_flags;
        using LSQRequest::_inst;
        using LSQRequest::_packets;
        using LSQRequest::_port;
        using LSQRequest::_requests;
        using LSQRequest::_res;
        using LSQRequest::_byteEnable;
        using LSQRequest::_senderState;
        using LSQRequest::_size;
        using LSQRequest::_state;
        using LSQRequest::_taskId;
        using LSQRequest::flags;
        using LSQRequest::isLoad;
        using LSQRequest::isTranslationComplete;
        using LSQRequest::lsqUnit;
        using LSQRequest::numInTranslationFragments;
        using LSQRequest::numTranslatedFragments;
        using LSQRequest::request;
        using LSQRequest::sendFragmentToTranslation;
        using LSQRequest::setState;
        using LSQRequest::_numOutstandingPackets;

        uint32_t numFragments;
        uint32_t numReceivedPackets;
        RequestPtr mainReq;
        PacketPtr _mainPacket;

      public:
        SplitDataRequest(LSQUnit* port, const DynInstPtr& inst, bool isLoad,
                         const Addr& addr, const uint32_t& size,
                         const Request::Flags & flags_,
                         PacketDataPtr data = nullptr,
                         uint64_t* res = nullptr) :
            LSQRequest(port, inst, isLoad, addr, size, flags_, data, res,
                       nullptr),
            numFragments(0),
            numReceivedPackets(0),
            mainReq(nullptr),
            _mainPacket(nullptr)
        {
            flags.set(Flag::IsSplit);
        }
        virtual ~SplitDataRequest()
        {
            if (mainReq) {
                mainReq = nullptr;
            }
            if (_mainPacket) {
                delete _mainPacket;
                _mainPacket = nullptr;
            }
        }
        virtual void finish(const Fault &fault, const RequestPtr &req,
                ThreadContext* tc, BaseTLB::Mode mode);
        virtual bool recvTimingResp(PacketPtr pkt);
        virtual void initiateTranslation();
        virtual void sendPacketToCache();
        virtual void buildPackets();

        virtual Cycles handleLocalAccess(ThreadContext *thread, PacketPtr pkt);
        virtual bool isCacheBlockHit(Addr blockAddr, Addr cacheBlockMask);

        virtual RequestPtr mainRequest();
        virtual PacketPtr mainPacket();
    };

    LSQ(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params);
    ~LSQ() { }

    std::string name() const;

    void regStats();

    void setActiveThreads(std::list<ThreadID> *at_ptr);

    void drainSanityCheck() const;
    bool isDrained() const;
    void takeOverFrom();

    int entryAmount(ThreadID num_threads);

    void tick();

    void insertLoad(const DynInstPtr &load_inst);
    void insertStore(const DynInstPtr &store_inst);

    Fault executeLoad(const DynInstPtr &inst);

    Fault executeStore(const DynInstPtr &inst);

    void commitLoads(InstSeqNum &youngest_inst, ThreadID tid)
    { thread.at(tid).commitLoads(youngest_inst); }

    void commitStores(InstSeqNum &youngest_inst, ThreadID tid)
    { thread.at(tid).commitStores(youngest_inst); }

    void writebackStores();
    void writebackStores(ThreadID tid);

    void
    squash(const InstSeqNum &squashed_num, ThreadID tid)
    {
        thread.at(tid).squash(squashed_num);
    }

    bool violation();
    bool violation(ThreadID tid) { return thread.at(tid).violation(); }

    DynInstPtr
    getMemDepViolator(ThreadID tid)
    {
        return thread.at(tid).getMemDepViolator();
    }

    int getLoadHead(ThreadID tid) { return thread.at(tid).getLoadHead(); }

    InstSeqNum
    getLoadHeadSeqNum(ThreadID tid)
    {
        return thread.at(tid).getLoadHeadSeqNum();
    }

    int getStoreHead(ThreadID tid) { return thread.at(tid).getStoreHead(); }

    InstSeqNum
    getStoreHeadSeqNum(ThreadID tid)
    {
        return thread.at(tid).getStoreHeadSeqNum();
    }

    int getCount();
    int getCount(ThreadID tid) { return thread.at(tid).getCount(); }

    int numLoads();
    int numLoads(ThreadID tid) { return thread.at(tid).numLoads(); }

    int numStores();
    int numStores(ThreadID tid) { return thread.at(tid).numStores(); }

    unsigned numFreeLoadEntries();

    unsigned numFreeStoreEntries();

    unsigned numFreeEntries(ThreadID tid);

    unsigned numFreeLoadEntries(ThreadID tid);

    unsigned numFreeStoreEntries(ThreadID tid);

    bool isFull();
    bool isFull(ThreadID tid);

    bool isEmpty() const;
    bool lqEmpty() const;
    bool sqEmpty() const;

    bool lqFull();
    bool lqFull(ThreadID tid);

    bool sqFull();
    bool sqFull(ThreadID tid);

    bool isStalled();
    bool isStalled(ThreadID tid);

    bool hasStoresToWB();

    bool hasStoresToWB(ThreadID tid) { return thread.at(tid).hasStoresToWB(); }

    int numStoresToWB(ThreadID tid) { return thread.at(tid).numStoresToWB(); }

    bool willWB();
    bool willWB(ThreadID tid) { return thread.at(tid).willWB(); }

    void dumpInsts() const;
    void dumpInsts(ThreadID tid) const { thread.at(tid).dumpInsts(); }

    Fault read(LSQRequest* req, int load_idx);

    Fault write(LSQRequest* req, uint8_t *data, int store_idx);

    void recvReqRetry();

    void completeDataAccess(PacketPtr pkt);
    bool recvTimingResp(PacketPtr pkt);

    void recvTimingSnoopReq(PacketPtr pkt);

    Fault pushRequest(const DynInstPtr& inst, bool isLoad, uint8_t *data,
                      unsigned int size, Addr addr, Request::Flags flags,
                      uint64_t *res, AtomicOpFunctorPtr amo_op,
                      const std::vector<bool>& byte_enable);

    O3CPU *cpu;

    IEW *iewStage;

    bool cacheBlocked() const;
    void cacheBlocked(bool v);
    bool cachePortAvailable(bool is_load) const;
    void cachePortBusy(bool is_load);

    MasterPort &getDataPort() { return dcachePort; }

  protected:
    bool _cacheBlocked;
    int cacheStorePorts;
    int usedStorePorts;
    int cacheLoadPorts;
    int usedLoadPorts;


    SMTQueuePolicy lsqPolicy;

    static uint32_t
    maxLSQAllocation(SMTQueuePolicy pol, uint32_t entries,
            uint32_t numThreads, uint32_t SMTThreshold)
    {
        if (pol == SMTQueuePolicy::Dynamic) {
            return entries;
        } else if (pol == SMTQueuePolicy::Partitioned) {
            //@todo:make work if part_amt doesnt divide evenly.
            return entries / numThreads;
        } else if (pol == SMTQueuePolicy::Threshold) {
            //Divide up by threshold amount
            //@todo: Should threads check the max and the total
            //amount of the LSQ
            return SMTThreshold;
        }
        return 0;
    }

    std::list<ThreadID> *activeThreads;

    unsigned LQEntries;
    unsigned SQEntries;

    unsigned maxLQEntries;

    unsigned maxSQEntries;

    DcachePort dcachePort;

    std::vector<LSQUnit> thread;

    ThreadID numThreads;
};

template <class Impl>
Fault
LSQ<Impl>::read(LSQRequest* req, int load_idx)
{
    ThreadID tid = cpu->contextToThread(req->request()->contextId());

    return thread.at(tid).read(req, load_idx);
}

template <class Impl>
Fault
LSQ<Impl>::write(LSQRequest* req, uint8_t *data, int store_idx)
{
    ThreadID tid = cpu->contextToThread(req->request()->contextId());

    return thread.at(tid).write(req, data, store_idx);
}

#endif // __CPU_O3_LSQ_HH__