mem_ctrl.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2020 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#include "debug/QOS.hh"
#include "mem/qos/policy.hh"
#include "mem/qos/q_policy.hh"
#include "params/QoSMemCtrl.hh"
#include "sim/clocked_object.hh"
#include "sim/system.hh"
 
#include <unordered_map>
#include <vector>
#include <deque>
 
#ifndef __MEM_QOS_MEM_CTRL_HH__
#define __MEM_QOS_MEM_CTRL_HH__
 
namespace QoS {
 
class MemCtrl : public ClockedObject
{
  public:
    enum BusState { READ, WRITE };
 
  protected:
    const std::unique_ptr<Policy> policy;
 
    const std::unique_ptr<TurnaroundPolicy> turnPolicy;
 
    const std::unique_ptr<QueuePolicy> queuePolicy;
 
    const uint8_t _numPriorities;
 
    const bool qosPriorityEscalation;
 
    const bool qosSyncroScheduler;
 
    std::unordered_map<RequestorID, const std::string> requestors;
 
    std::unordered_map<RequestorID, std::vector<uint64_t> > packetPriorities;
 
    std::unordered_map<RequestorID,
            std::unordered_map<uint64_t, std::deque<uint64_t>> > requestTimes;
 
    std::vector<Tick> serviceTick;
 
    std::vector<uint64_t> readQueueSizes;
 
    std::vector<uint64_t> writeQueueSizes;
 
    uint64_t totalReadQueueSize;
 
    uint64_t totalWriteQueueSize;
 
    BusState busState;
 
    BusState busStateNext;
 
    struct MemCtrlStats : public Stats::Group
    {
        MemCtrlStats(MemCtrl &mc);
 
        void regStats() override;
 
        const MemCtrl &memCtrl;
 
        Stats::VectorStandardDeviation avgPriority;
        Stats::VectorStandardDeviation avgPriorityDistance;
 
        Stats::Vector priorityMinLatency;
        Stats::Vector priorityMaxLatency;
        Stats::Scalar numReadWriteTurnArounds;
        Stats::Scalar numWriteReadTurnArounds;
        Stats::Scalar numStayReadState;
        Stats::Scalar numStayWriteState;
    } stats;
 
    System* _system;
 
    void addRequestor(const RequestorID id);
 
    void logRequest(BusState dir, RequestorID id, uint8_t qos,
                    Addr addr, uint64_t entries);
 
    void logResponse(BusState dir, RequestorID id, uint8_t qos,
                     Addr addr, uint64_t entries, double delay);
 
    template<typename Queues>
    uint8_t qosSchedule(std::initializer_list<Queues*> queues_ptr,
                        uint64_t queue_entry_size, const PacketPtr pkt);
 
    using SimObject::schedule;
    uint8_t schedule(RequestorID id, uint64_t data);
    uint8_t schedule(const PacketPtr pkt);
 
    BusState selectNextBusState();
 
    void setCurrentBusState() { busState = busStateNext; }
 
    void recordTurnaroundStats();
 
    template<typename Queues>
    void escalate(std::initializer_list<Queues*> queues,
                  uint64_t queue_entry_size,
                  RequestorID id, uint8_t tgt_prio);
 
    template<typename Queues>
    void escalateQueues(Queues& queues, uint64_t queue_entry_size,
                        RequestorID id, uint8_t curr_prio, uint8_t tgt_prio);
 
  public:
    MemCtrl(const QoSMemCtrlParams*);
 
    virtual ~MemCtrl();
 
    BusState getBusState() const { return busState; }
 
    BusState getBusStateNext() const { return busStateNext; }
 
    bool hasRequestor(RequestorID id) const
    {
        return requestors.find(id) != requestors.end();
    }
 
    uint64_t getReadQueueSize(const uint8_t prio) const
    { return readQueueSizes[prio]; }
 
    uint64_t getWriteQueueSize(const uint8_t prio) const
    { return writeQueueSizes[prio]; }
 
    uint64_t getTotalReadQueueSize() const { return totalReadQueueSize; }
 
    uint64_t getTotalWriteQueueSize() const { return totalWriteQueueSize; }
 
    Tick getServiceTick(const uint8_t prio) const { return serviceTick[prio]; }
 
    uint8_t numPriorities() const { return _numPriorities; }
 
    System* system() const { return _system; }
};
 
template<typename Queues>
void
MemCtrl::escalateQueues(Queues& queues, uint64_t queue_entry_size,
                        RequestorID id, uint8_t curr_prio, uint8_t tgt_prio)
{
    auto it = queues[curr_prio].begin();
    while (it != queues[curr_prio].end()) {
        // No packets left to move
        if (packetPriorities[id][curr_prio] == 0)
            break;
 
        auto pkt = *it;
 
        DPRINTF(QOS,
                "QoSMemCtrl::escalate checking priority %d packet "
                "id %d address %d\n", curr_prio,
                pkt->requestorId(), pkt->getAddr());
 
        // Found a packet to move
        if (pkt->requestorId() == id) {
 
            uint64_t moved_entries = divCeil(pkt->getSize(),
                                             queue_entry_size);
 
            DPRINTF(QOS,
                    "QoSMemCtrl::escalate Requestor %s [id %d] moving "
                    "packet addr %d size %d (p size %d) from priority %d "
                    "to priority %d - "
                    "this requestor packets %d (entries to move %d)\n",
                    requestors[id], id, pkt->getAddr(),
                    pkt->getSize(),
                    queue_entry_size, curr_prio, tgt_prio,
                    packetPriorities[id][curr_prio], moved_entries);
 
 
            if (pkt->isRead()) {
                panic_if(readQueueSizes[curr_prio] < moved_entries,
                         "QoSMemCtrl::escalate requestor %s negative READ "
                         "packets for priority %d",
                        requestors[id], tgt_prio);
                readQueueSizes[curr_prio] -= moved_entries;
                readQueueSizes[tgt_prio] += moved_entries;
            } else if (pkt->isWrite()) {
                panic_if(writeQueueSizes[curr_prio] < moved_entries,
                         "QoSMemCtrl::escalate requestor %s negative WRITE "
                         "packets for priority %d",
                        requestors[id], tgt_prio);
                writeQueueSizes[curr_prio] -= moved_entries;
                writeQueueSizes[tgt_prio] += moved_entries;
            }
 
            // Change QoS priority and move packet
            pkt->qosValue(tgt_prio);
            queues[tgt_prio].push_back(pkt);
 
            // Erase element from source packet queue, this will
            // increment the iterator
            it = queues[curr_prio].erase(it);
            panic_if(packetPriorities[id][curr_prio] < moved_entries,
                     "QoSMemCtrl::escalate requestor %s negative packets "
                     "for priority %d",
                     requestors[id], tgt_prio);
 
            packetPriorities[id][curr_prio] -= moved_entries;
            packetPriorities[id][tgt_prio] += moved_entries;
        } else {
            // Increment iterator to next location in the queue
            it++;
        }
    }
}
 
template<typename Queues>
void
MemCtrl::escalate(std::initializer_list<Queues*> queues,
                  uint64_t queue_entry_size,
                  RequestorID id, uint8_t tgt_prio)
{
    // If needed, initialize all counters and statistics
    // for this requestor
    addRequestor(id);
 
    DPRINTF(QOS,
            "QoSMemCtrl::escalate Requestor %s [id %d] to priority "
            "%d (currently %d packets)\n",requestors[id], id, tgt_prio,
            packetPriorities[id][tgt_prio]);
 
    for (uint8_t curr_prio = 0; curr_prio < numPriorities(); ++curr_prio) {
        // Skip target priority
        if (curr_prio == tgt_prio)
            continue;
 
        // Process other priority packet
        while (packetPriorities[id][curr_prio] > 0) {
            DPRINTF(QOS,
                    "QoSMemCtrl::escalate MID %d checking priority %d "
                    "(packets %d)- current packets in prio %d:  %d\n"
                    "\t(source read %d source write %d target read %d, "
                    "target write %d)\n",
                    id, curr_prio, packetPriorities[id][curr_prio],
                    tgt_prio, packetPriorities[id][tgt_prio],
                    readQueueSizes[curr_prio],
                    writeQueueSizes[curr_prio], readQueueSizes[tgt_prio],
                    writeQueueSizes[tgt_prio]);
 
            // Check both read and write queue
            for (auto q : queues) {
                escalateQueues(*q, queue_entry_size, id,
                               curr_prio, tgt_prio);
            }
        }
    }
 
    DPRINTF(QOS,
            "QoSMemCtrl::escalate Completed requestor %s [id %d] to priority "
            "%d (now %d packets)\n\t(total read %d, total write %d)\n",
            requestors[id], id, tgt_prio, packetPriorities[id][tgt_prio],
            readQueueSizes[tgt_prio], writeQueueSizes[tgt_prio]);
}
 
template<typename Queues>
uint8_t
MemCtrl::qosSchedule(std::initializer_list<Queues*> queues,
                     const uint64_t queue_entry_size,
                     const PacketPtr pkt)
{
    // Schedule packet.
    uint8_t pkt_priority = schedule(pkt);
 
    assert(pkt_priority < numPriorities());
 
    pkt->qosValue(pkt_priority);
 
    if (qosSyncroScheduler) {
        // Call the scheduling function on all other requestors.
        for (const auto& requestor : requestors) {
 
            if (requestor.first == pkt->requestorId())
                continue;
 
            uint8_t prio = schedule(requestor.first, 0);
 
            if (qosPriorityEscalation) {
                DPRINTF(QOS,
                        "QoSMemCtrl::qosSchedule: (syncro) escalating "
                        "REQUESTOR %s to assigned priority %d\n",
                        _system->getRequestorName(requestor.first),
                        prio);
                escalate(queues, queue_entry_size, requestor.first, prio);
            }
        }
    }
 
    if (qosPriorityEscalation) {
        DPRINTF(QOS,
                "QoSMemCtrl::qosSchedule: escalating "
                "REQUESTOR %s to assigned priority %d\n",
                _system->getRequestorName(pkt->requestorId()),
                pkt_priority);
        escalate(queues, queue_entry_size, pkt->requestorId(), pkt_priority);
    }
 
    // Update last service tick for selected priority
    serviceTick[pkt_priority] = curTick();
 
    return pkt_priority;
}
 
} // namespace QoS
 
#endif /* __MEM_QOS_MEM_CTRL_HH__ */