mem_ctrl.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2019 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "debug/QOS.hh"
#include "mem/abstract_mem.hh"
#include "mem/qos/q_policy.hh"
#include "mem/qos/policy.hh"
#include "params/QoSMemCtrl.hh"
#include "sim/system.hh"

#include <unordered_map>
#include <vector>
#include <deque>

#ifndef __MEM_QOS_MEM_CTRL_HH__
#define __MEM_QOS_MEM_CTRL_HH__

namespace QoS {

class MemCtrl: public AbstractMemory
{
  public:
    enum BusState { READ, WRITE };

  protected:
    const std::unique_ptr<Policy> policy;

    const std::unique_ptr<TurnaroundPolicy> turnPolicy;

    const std::unique_ptr<QueuePolicy> queuePolicy;

    const uint8_t _numPriorities;

    const bool qosPriorityEscalation;

    const bool qosSyncroScheduler;

    std::unordered_map<MasterID, const std::string> masters;

    std::unordered_map<MasterID, std::vector<uint64_t> > packetPriorities;

    std::unordered_map<MasterID,
            std::unordered_map<uint64_t, std::deque<uint64_t>> > requestTimes;

    std::vector<Tick> serviceTick;

    std::vector<uint64_t> readQueueSizes;

    std::vector<uint64_t> writeQueueSizes;

    uint64_t totalReadQueueSize;

    uint64_t totalWriteQueueSize;

    BusState busState;

    BusState busStateNext;

    struct MemCtrlStats : public Stats::Group
    {
        MemCtrlStats(MemCtrl &mc);

        void regStats() override;

        const MemCtrl &memCtrl;

        Stats::VectorStandardDeviation avgPriority;
        Stats::VectorStandardDeviation avgPriorityDistance;

        Stats::Vector priorityMinLatency;
        Stats::Vector priorityMaxLatency;
        Stats::Scalar numReadWriteTurnArounds;
        Stats::Scalar numWriteReadTurnArounds;
        Stats::Scalar numStayReadState;
        Stats::Scalar numStayWriteState;
    } stats;

    void addMaster(const MasterID m_id);

    void logRequest(BusState dir, MasterID m_id, uint8_t qos,
                    Addr addr, uint64_t entries);

    void logResponse(BusState dir, MasterID m_id, uint8_t qos,
                     Addr addr, uint64_t entries, double delay);

    template<typename Queues>
    uint8_t qosSchedule(std::initializer_list<Queues*> queues_ptr,
                        uint64_t queue_entry_size, const PacketPtr pkt);

    using SimObject::schedule;
    uint8_t schedule(MasterID m_id, uint64_t data);
    uint8_t schedule(const PacketPtr pkt);

    BusState selectNextBusState();

    void setCurrentBusState() { busState = busStateNext; }

    void recordTurnaroundStats();

    template<typename Queues>
    void escalate(std::initializer_list<Queues*> queues,
                  uint64_t queue_entry_size,
                  MasterID m_id, uint8_t tgt_prio);

    template<typename Queues>
    void escalateQueues(Queues& queues, uint64_t queue_entry_size,
                        MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio);

  public:
    MemCtrl(const QoSMemCtrlParams*);

    virtual ~MemCtrl();

    void init() override;

    BusState getBusState() const { return busState; }

    BusState getBusStateNext() const { return busStateNext; }

    bool hasMaster(MasterID m_id) const
    {
        return masters.find(m_id) != masters.end();
    }

    uint64_t getReadQueueSize(const uint8_t prio) const
    { return readQueueSizes[prio]; }

    uint64_t getWriteQueueSize(const uint8_t prio) const
    { return writeQueueSizes[prio]; }

    uint64_t getTotalReadQueueSize() const { return totalReadQueueSize; }

    uint64_t getTotalWriteQueueSize() const { return totalWriteQueueSize; }

    Tick getServiceTick(const uint8_t prio) const { return serviceTick[prio]; }

    uint8_t numPriorities() const { return _numPriorities; }
};

template<typename Queues>
void
MemCtrl::escalateQueues(Queues& queues, uint64_t queue_entry_size,
                        MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio)
{
    auto it = queues[curr_prio].begin();
    while (it != queues[curr_prio].end()) {
        // No packets left to move
        if (packetPriorities[m_id][curr_prio] == 0)
            break;

        auto pkt = *it;

        DPRINTF(QOS,
                "QoSMemCtrl::escalate checking priority %d packet "
                "m_id %d address %d\n", curr_prio,
                pkt->masterId(), pkt->getAddr());

        // Found a packet to move
        if (pkt->masterId() == m_id) {

            uint64_t moved_entries = divCeil(pkt->getSize(),
                                             queue_entry_size);

            DPRINTF(QOS,
                    "QoSMemCtrl::escalate Master %s [id %d] moving "
                    "packet addr %d size %d (p size %d) from priority %d "
                    "to priority %d - "
                    "this master packets %d (entries to move %d)\n",
                    masters[m_id], m_id, pkt->getAddr(),
                    pkt->getSize(),
                    queue_entry_size, curr_prio, tgt_prio,
                    packetPriorities[m_id][curr_prio], moved_entries);


            if (pkt->isRead()) {
                panic_if(readQueueSizes[curr_prio] < moved_entries,
                         "QoSMemCtrl::escalate master %s negative READ "
                         "packets for priority %d",
                        masters[m_id], tgt_prio);
                readQueueSizes[curr_prio] -= moved_entries;
                readQueueSizes[tgt_prio] += moved_entries;
            } else if (pkt->isWrite()) {
                panic_if(writeQueueSizes[curr_prio] < moved_entries,
                         "QoSMemCtrl::escalate master %s negative WRITE "
                         "packets for priority %d",
                        masters[m_id], tgt_prio);
                writeQueueSizes[curr_prio] -= moved_entries;
                writeQueueSizes[tgt_prio] += moved_entries;
            }

            // Change QoS priority and move packet
            pkt->qosValue(tgt_prio);
            queues[tgt_prio].push_back(pkt);

            // Erase element from source packet queue, this will
            // increment the iterator
            it = queues[curr_prio].erase(it);
            panic_if(packetPriorities[m_id][curr_prio] < moved_entries,
                     "QoSMemCtrl::escalate master %s negative packets "
                     "for priority %d",
                     masters[m_id], tgt_prio);

            packetPriorities[m_id][curr_prio] -= moved_entries;
            packetPriorities[m_id][tgt_prio] += moved_entries;
        } else {
            // Increment iterator to next location in the queue
            it++;
        }
    }
}

template<typename Queues>
void
MemCtrl::escalate(std::initializer_list<Queues*> queues,
                  uint64_t queue_entry_size,
                  MasterID m_id, uint8_t tgt_prio)
{
    // If needed, initialize all counters and statistics
    // for this master
    addMaster(m_id);

    DPRINTF(QOS,
            "QoSMemCtrl::escalate Master %s [id %d] to priority "
            "%d (currently %d packets)\n",masters[m_id], m_id, tgt_prio,
            packetPriorities[m_id][tgt_prio]);

    for (uint8_t curr_prio = 0; curr_prio < numPriorities(); ++curr_prio) {
        // Skip target priority
        if (curr_prio == tgt_prio)
            continue;

        // Process other priority packet
        while (packetPriorities[m_id][curr_prio] > 0) {
            DPRINTF(QOS,
                    "QoSMemCtrl::escalate MID %d checking priority %d "
                    "(packets %d)- current packets in prio %d:  %d\n"
                    "\t(source read %d source write %d target read %d, "
                    "target write %d)\n",
                    m_id, curr_prio, packetPriorities[m_id][curr_prio],
                    tgt_prio, packetPriorities[m_id][tgt_prio],
                    readQueueSizes[curr_prio],
                    writeQueueSizes[curr_prio], readQueueSizes[tgt_prio],
                    writeQueueSizes[tgt_prio]);

            // Check both read and write queue
            for (auto q : queues) {
                escalateQueues(*q, queue_entry_size, m_id,
                               curr_prio, tgt_prio);
            }
        }
    }

    DPRINTF(QOS,
            "QoSMemCtrl::escalate Completed master %s [id %d] to priority %d "
            "(now %d packets)\n\t(total read %d, total write %d)\n",
            masters[m_id], m_id, tgt_prio, packetPriorities[m_id][tgt_prio],
            readQueueSizes[tgt_prio], writeQueueSizes[tgt_prio]);
}

template<typename Queues>
uint8_t
MemCtrl::qosSchedule(std::initializer_list<Queues*> queues,
                     const uint64_t queue_entry_size,
                     const PacketPtr pkt)
{
    // Schedule packet.
    uint8_t pkt_priority = schedule(pkt);

    assert(pkt_priority < numPriorities());

    pkt->qosValue(pkt_priority);

    if (qosSyncroScheduler) {
        // Call the scheduling function on all other masters.
        for (const auto& m : masters) {

            if (m.first == pkt->masterId())
                continue;

            uint8_t prio = schedule(m.first, 0);

            if (qosPriorityEscalation) {
                DPRINTF(QOS,
                        "QoSMemCtrl::qosSchedule: (syncro) escalating "
                        "MASTER %s to assigned priority %d\n",
                        _system->getMasterName(m.first),
                        prio);
                escalate(queues, queue_entry_size, m.first, prio);
            }
        }
    }

    if (qosPriorityEscalation) {
        DPRINTF(QOS,
                "QoSMemCtrl::qosSchedule: escalating "
                "MASTER %s to assigned priority %d\n",
                _system->getMasterName(pkt->masterId()),
                pkt_priority);
        escalate(queues, queue_entry_size, pkt->masterId(), pkt_priority);
    }

    // Update last service tick for selected priority
    serviceTick[pkt_priority] = curTick();

    return pkt_priority;
}

} // namespace QoS

#endif /* __MEM_QOS_MEM_CTRL_HH__ */