49 #include "debug/LLSC.hh"
50 #include "debug/MemoryAccess.hh"
51 #include "debug/ProtocolTrace.hh"
52 #include "debug/RubyHitMiss.hh"
53 #include "debug/RubySequencer.hh"
54 #include "debug/RubyStats.hh"
57 #include "mem/ruby/protocol/PrefetchBit.hh"
58 #include "mem/ruby/protocol/RubyAccessMode.hh"
71 :
RubyPort(
p), m_IncompleteTimes(MachineType_NUM),
72 deadlockCheckEvent([this]{
wakeup(); },
"Sequencer deadlock check")
74 m_outstanding_count = 0;
76 m_dataCache_ptr =
p.dcache;
77 m_max_outstanding_requests =
p.max_outstanding_requests;
78 m_deadlock_threshold =
p.deadlock_threshold;
81 assert(m_max_outstanding_requests > 0);
82 assert(m_deadlock_threshold > 0);
84 m_unaddressedTransactionCnt = 0;
86 m_runningGarnetStandalone =
p.garnet_standalone;
92 m_outstandReqHist.init(10);
93 m_latencyHist.init(10);
94 m_hitLatencyHist.init(10);
95 m_missLatencyHist.init(10);
97 for (
int i = 0;
i < RubyRequestType_NUM;
i++) {
99 m_typeLatencyHist[
i]->init(10);
102 m_hitTypeLatencyHist[
i]->init(10);
105 m_missTypeLatencyHist[
i]->init(10);
108 for (
int i = 0;
i < MachineType_NUM;
i++) {
110 m_hitMachLatencyHist[
i]->init(10);
113 m_missMachLatencyHist[
i]->init(10);
116 m_IssueToInitialDelayHist[
i]->init(10);
119 m_InitialToForwardDelayHist[
i]->init(10);
121 m_ForwardToFirstResponseDelayHist.push_back(
123 m_ForwardToFirstResponseDelayHist[
i]->init(10);
125 m_FirstResponseToCompletionDelayHist.push_back(
127 m_FirstResponseToCompletionDelayHist[
i]->init(10);
130 for (
int i = 0;
i < RubyRequestType_NUM;
i++) {
131 m_hitTypeMachLatencyHist.push_back(
133 m_missTypeMachLatencyHist.push_back(
136 for (
int j = 0;
j < MachineType_NUM;
j++) {
137 m_hitTypeMachLatencyHist[
i].push_back(
new statistics::Histogram());
138 m_hitTypeMachLatencyHist[
i][
j]->init(10);
140 m_missTypeMachLatencyHist[
i].push_back(
141 new statistics::Histogram());
142 m_missTypeMachLatencyHist[
i][
j]->init(10);
156 "%s must have a dcache object to support LLSC requests.",
name());
160 DPRINTF(LLSC,
"LLSC Monitor - inserting load linked - "
161 "addr=0x%lx - cpu=%u\n", claddr,
m_version);
174 DPRINTF(LLSC,
"LLSC Monitor - clearing due to store - "
175 "addr=0x%lx - cpu=%u\n", claddr,
m_version);
183 "%s must have a dcache object to support LLSC requests.",
name());
188 DPRINTF(LLSC,
"LLSC Monitor - clearing due to "
189 "store conditional - "
190 "addr=0x%lx - cpu=%u\n",
236 for (
const auto &seq_req : table_entry.second) {
240 panic(
"Possible Deadlock detected. Aborting!\n version: %d "
241 "request.paddr: 0x%x m_readRequestTable: %d current time: "
242 "%u issue_time: %d difference: %d\n",
m_version,
243 seq_req.pkt->getAddr(), table_entry.second.size(),
248 total_outstanding += table_entry.second.size();
265 for (
const auto& seq_req : table_entry.second) {
266 if (seq_req.functionalWrite(func_pkt))
280 for (
int i = 0;
i < RubyRequestType_NUM;
i++) {
284 for (
int j = 0;
j < MachineType_NUM;
j++) {
290 for (
int i = 0;
i < MachineType_NUM;
i++) {
307 RubyRequestType secondary_type)
316 assert(primary_type == secondary_type);
318 switch (primary_type) {
319 case RubyRequestType_TLBI_EXT_SYNC_COMP:
322 case RubyRequestType_TLBI:
323 case RubyRequestType_TLBI_SYNC:
328 [[maybe_unused]]
auto insert_data = \
329 m_UnaddressedRequestTable.emplace(
332 pkt, primary_type, secondary_type,
curCycle()));
335 assert(insert_data.second &&
336 "Another TLBI request with the same ID exists");
338 DPRINTF(RubySequencer,
"Inserting TLBI request %016x\n",
345 panic(
"Unexpected TLBI RubyRequestType");
348 return RequestStatus_Ready;
355 seq_req_list.emplace_back(pkt, primary_type,
359 if (seq_req_list.size() > 1) {
360 return RequestStatus_Aliased;
365 return RequestStatus_Ready;
376 const MachineType respondingMach,
377 bool isExternalHit,
Cycles initialRequestTime,
378 Cycles forwardRequestTime,
386 Cycles total_lat = completion_time - issued_time;
388 if ((initialRequestTime != 0) && (initialRequestTime < issued_time)) {
397 DPRINTFR(ProtocolTrace,
"%15s %3s %10s%20s %6s>%-6s %s %d cycles\n",
408 if (respondingMach != MachineType_NUM) {
412 if ((issued_time <= initialRequestTime) &&
413 (initialRequestTime <= forwardRequestTime) &&
414 (forwardRequestTime <= firstResponseTime) &&
415 (firstResponseTime <= completion_time)) {
418 initialRequestTime - issued_time);
420 forwardRequestTime - initialRequestTime);
422 firstResponseTime - forwardRequestTime);
424 completion_time - firstResponseTime);
433 if (respondingMach != MachineType_NUM) {
449 const bool externalHit,
const MachineType mach,
450 const Cycles initialRequestTime,
451 const Cycles forwardRequestTime,
452 const Cycles firstResponseTime,
466 bool ruby_request =
true;
467 int aliased_stores = 0;
468 int aliased_loads = 0;
469 while (!seq_req_list.empty()) {
472 if (noCoales && !ruby_request) {
481 assert(seq_req.
m_type != RubyRequestType_LD);
482 assert(seq_req.
m_type != RubyRequestType_Load_Linked);
483 assert(seq_req.
m_type != RubyRequestType_IFETCH);
487 if ((seq_req.
m_type != RubyRequestType_LD) &&
488 (seq_req.
m_type != RubyRequestType_Load_Linked) &&
489 (seq_req.
m_type != RubyRequestType_IFETCH)) {
493 if (seq_req.
m_type != RubyRequestType_Store_Conditional) {
501 seq_req.
pkt->
req->setExtraData(success ? 1 : 0);
507 if (seq_req.
m_type == RubyRequestType_Locked_RMW_Read) {
515 }
else if (seq_req.
m_type == RubyRequestType_Locked_RMW_Write) {
521 initialRequestTime, forwardRequestTime,
528 initialRequestTime, forwardRequestTime,
529 firstResponseTime, !ruby_request);
530 ruby_request =
false;
533 assert(!ruby_request);
537 initialRequestTime, forwardRequestTime,
538 firstResponseTime, !ruby_request);
540 seq_req_list.pop_front();
544 if (seq_req_list.empty()) {
551 bool externalHit,
const MachineType mach,
552 Cycles initialRequestTime,
553 Cycles forwardRequestTime,
567 bool ruby_request =
true;
568 int aliased_loads = 0;
569 while (!seq_req_list.empty()) {
572 assert((seq_req.
m_type == RubyRequestType_LD) ||
573 (seq_req.
m_type == RubyRequestType_Load_Linked) ||
574 (seq_req.
m_type == RubyRequestType_IFETCH));
578 if ((seq_req.
m_type != RubyRequestType_LD) &&
579 (seq_req.
m_type != RubyRequestType_Load_Linked) &&
580 (seq_req.
m_type != RubyRequestType_IFETCH)) {
587 initialRequestTime, forwardRequestTime,
592 initialRequestTime, forwardRequestTime,
593 firstResponseTime, !ruby_request);
594 ruby_request =
false;
595 seq_req_list.pop_front();
599 if (seq_req_list.empty()) {
607 const MachineType mach,
const bool externalHit,
608 const Cycles initialRequestTime,
609 const Cycles forwardRequestTime,
610 const Cycles firstResponseTime,
611 const bool was_coalesced)
613 warn_once(
"Replacement policy updates recently became the responsibility "
614 "of SLICC state machines. Make sure to setMRU() near callbacks "
630 if (
type == RubyRequestType_Load_Linked) {
635 DPRINTF(RubyHitMiss,
"Cache %s at %#x\n",
636 externalHit ?
"miss" :
"hit",
643 if ((
type == RubyRequestType_LD) ||
644 (
type == RubyRequestType_IFETCH) ||
645 (
type == RubyRequestType_RMW_Read) ||
646 (
type == RubyRequestType_Locked_RMW_Read) ||
647 (
type == RubyRequestType_Load_Linked)) {
651 }
else if (pkt->
req->isSwap()) {
657 data.setData(&overwrite_val[0],
664 DPRINTF(RubySequencer,
"AMO original data %s\n",
data);
669 }
else if (
type != RubyRequestType_Store_Conditional || llscSuccess) {
681 DPRINTF(RubySequencer,
"hitCallback %s 0x%x using RubyTester\n",
685 assert(testerSenderState);
693 rs->m_cache_recorder->enqueueNextFetchRequest();
696 rs->m_cache_recorder->enqueueNextFlushRequest();
705 RubyRequestType reqType,
706 const MachineType mach,
707 const Cycles initialRequestTime,
708 const Cycles forwardRequestTime,
709 const Cycles firstResponseTime)
711 DPRINTF(RubySequencer,
"unaddressedCallback ID:%08x type:%d\n",
712 unaddressedReqId, reqType);
715 case RubyRequestType_TLBI_EXT_SYNC:
725 case RubyRequestType_TLBI:
726 case RubyRequestType_TLBI_SYNC:
737 assert(seq_req.
m_type == reqType);
749 panic(
"Unexpected TLBI RubyRequestType");
766 !pkt->
req->isHTMAbort()) {
767 return RequestStatus_BufferFull;
770 RubyRequestType primary_type = RubyRequestType_NULL;
771 RubyRequestType secondary_type = RubyRequestType_NULL;
785 DPRINTF(RubySequencer,
"Issuing SC\n");
786 primary_type = RubyRequestType_Store_Conditional;
787 #if defined (PROTOCOL_MESI_Three_Level) || defined (PROTOCOL_MESI_Three_Level_HTM)
788 secondary_type = RubyRequestType_Store_Conditional;
790 secondary_type = RubyRequestType_ST;
793 DPRINTF(RubySequencer,
"Issuing LL\n");
795 primary_type = RubyRequestType_Load_Linked;
796 secondary_type = RubyRequestType_LD;
798 }
else if (pkt->
req->isLockedRMW()) {
806 DPRINTF(RubySequencer,
"Issuing Locked RMW Write\n");
807 primary_type = RubyRequestType_Locked_RMW_Write;
809 DPRINTF(RubySequencer,
"Issuing Locked RMW Read\n");
811 primary_type = RubyRequestType_Locked_RMW_Read;
813 secondary_type = RubyRequestType_ST;
814 }
else if (pkt->
req->isTlbiCmd()) {
816 DPRINTF(RubySequencer,
"Issuing TLBI\n");
827 primary_type = secondary_type = RubyRequestType_ST;
828 }
else if (pkt->
isRead()) {
830 if (pkt->
req->isHTMCmd()) {
832 }
else if (pkt->
req->isInstFetch()) {
833 primary_type = secondary_type = RubyRequestType_IFETCH;
835 if (pkt->
req->isReadModifyWrite()) {
836 primary_type = RubyRequestType_RMW_Read;
837 secondary_type = RubyRequestType_ST;
839 primary_type = secondary_type = RubyRequestType_LD;
843 primary_type = secondary_type = RubyRequestType_FLUSH;
845 panic(
"Unsupported ruby packet type\n");
850 if (!pkt->
req->isMemMgmt() &&
852 (primary_type != RubyRequestType_Locked_RMW_Write)) {
856 return RequestStatus_Aliased;
862 if (
status != RequestStatus_Ready &&
status != RequestStatus_Aliased)
866 if (
status != RequestStatus_Aliased)
870 return RequestStatus_Issued;
884 if (pkt->
req->hasPC()) {
885 pc = pkt->
req->getPC();
890 std::shared_ptr<RubyRequest> msg;
891 if (pkt->
req->isMemMgmt()) {
892 msg = std::make_shared<RubyRequest>(
clockEdge(),
894 RubyAccessMode_Supervisor, pkt,
897 DPRINTFR(ProtocolTrace,
"%15s %3s %10s%20s %6s>%-6s %s\n",
899 RubyRequestType_to_string(secondary_type));
901 if (pkt->
req->isTlbiCmd()) {
902 msg->m_isTlbi =
true;
903 switch (secondary_type) {
904 case RubyRequestType_TLBI_EXT_SYNC_COMP:
905 msg->m_tlbiTransactionUid = pkt->
req->getExtraData();
907 case RubyRequestType_TLBI:
908 case RubyRequestType_TLBI_SYNC:
909 msg->m_tlbiTransactionUid = \
910 getCurrentUnaddressedTransactionID();
913 panic(
"Unexpected TLBI RubyRequestType");
915 DPRINTF(RubySequencer,
"Issuing TLBI %016x\n",
916 msg->m_tlbiTransactionUid);
921 RubyAccessMode_Supervisor, pkt,
922 PrefetchBit_No, proc_id, core_id);
924 DPRINTFR(ProtocolTrace,
"%15s %3s %10s%20s %6s>%-6s %#x %s\n",
927 RubyRequestType_to_string(secondary_type));
934 msg->m_htmFromTransaction =
true;
946 template <
class KEY,
class VALUE>
948 operator<<(std::ostream &out,
const std::unordered_map<KEY, VALUE> &map)
950 for (
const auto &table_entry : map) {
951 out <<
"[ " << table_entry.first <<
" =";
952 for (
const auto &seq_req : table_entry.second) {
953 out <<
" " << RubyRequestType_to_string(seq_req.m_second_type);
972 DPRINTF(RubyStats,
"Recorded statistic: %s\n",
973 SequencerRequestType_to_string(requestType));
989 uint64_t aligned_txid = \
992 if (aligned_txid > 0xFFFFFFFFull) {
1001 uint64_t(
m_version & 0xFFFFFFFF) << 32) |
Cycles curCycle() const
Determine the current cycle, corresponding to a tick aligned to a clock edge.
Tick clockEdge(Cycles cycles=Cycles(0)) const
Determine the tick when a cycle begins, by default the current one, but the argument also enables the...
Tick cyclesToTicks(Cycles c) const
Cycles is a wrapper class for representing cycle counts, i.e.
virtual std::string name() const
A Packet is used to encapsulate a transfer between two objects in the memory system (e....
AtomicOpFunctor * getAtomicOp() const
Accessor function to atomic op.
T * findNextSenderState() const
Go through the sender state stack and return the first instance that is of type T (as determined by a...
bool isHtmTransactional() const
Returns whether or not this packet/request originates in the CPU executing in transactional mode,...
void setData(const uint8_t *p)
Copy data into the packet from the provided pointer.
uint64_t getHtmTransactionUid() const
If a packet/request originates in a CPU executing in transactional mode, i.e.
const std::string & cmdString() const
Return the string name of the cmd field (for debugging and tracing).
RequestPtr req
A pointer to the original request.
bool isMaskedWrite() const
void writeData(uint8_t *p) const
Copy data from the packet to the memory at the provided pointer.
bool isLocked(int context) const
void setLocked(int context)
virtual void notifyCoalesced(const Addr &addr, const RubyRequestType &type, const RequestPtr &req, const DataBlock &data_blk, const bool &was_miss)
Notifies controller of a request coalesced at the sequencer.
virtual Cycles mandatoryQueueLatency(const RubyRequestType ¶m_type)
bool isBlocked(Addr) const
void blockOnQueue(Addr, MessageBuffer *)
void clearLockedAll(int context)
AbstractCacheEntry * lookup(Addr address)
void enqueue(MsgPtr message, Tick curTime, Tick delta)
void ruby_hit_callback(PacketPtr pkt)
void ruby_unaddressed_callback(PacketPtr pkt)
void ruby_stale_translation_callback(Addr txnId)
RubySystem * m_ruby_system
virtual int functionalWrite(Packet *func_pkt)
AbstractController * m_controller
void ruby_eviction_callback(Addr address)
MessageBuffer * m_mandatory_q_ptr
static bool getWarmupEnabled()
static uint32_t getBlockSizeBits()
static bool getCooldownEnabled()
statistics::Histogram m_outstandReqHist
Histogram for number of outstanding requests per cycle.
void resetStats() override
Callback to reset stats.
Cycles m_deadlock_threshold
std::unordered_map< Addr, std::list< SequencerRequest > > m_RequestTable
virtual bool empty() const
void writeCallback(Addr address, DataBlock &data, const bool externalHit=false, const MachineType mach=MachineType_NUM, const Cycles initialRequestTime=Cycles(0), const Cycles forwardRequestTime=Cycles(0), const Cycles firstResponseTime=Cycles(0), const bool noCoales=false)
std::vector< statistics::Counter > m_IncompleteTimes
bool llscCheckMonitor(const Addr)
Searches for cache line address in the global monitor tagged with this Sequencer object's version id.
virtual int functionalWrite(Packet *func_pkt) override
std::vector< statistics::Histogram * > m_InitialToForwardDelayHist
std::vector< statistics::Histogram * > m_hitMachLatencyHist
Histograms for profiling the latencies for requests that did not required external messages.
void recordMissLatency(SequencerRequest *srequest, bool llscSuccess, const MachineType respondingMach, bool isExternalHit, Cycles initialRequestTime, Cycles forwardRequestTime, Cycles firstResponseTime)
std::vector< statistics::Histogram * > m_typeLatencyHist
void writeCallbackScFail(Addr address, DataBlock &data)
Proxy function to writeCallback that first invalidates the line address in the local monitor.
CacheMemory * m_dataCache_ptr
std::vector< statistics::Histogram * > m_FirstResponseToCompletionDelayHist
void incrementUnaddressedTransactionCnt()
Increment the unaddressed transaction counter.
void hitCallback(SequencerRequest *srequest, DataBlock &data, bool llscSuccess, const MachineType mach, const bool externalHit, const Cycles initialRequestTime, const Cycles forwardRequestTime, const Cycles firstResponseTime, const bool was_coalesced)
void llscLoadLinked(const Addr)
Places the cache line address into the global monitor tagged with this Sequencer object's version id.
uint64_t getCurrentUnaddressedTransactionID() const
Generate the current unaddressed transaction ID based on the counter and the Sequencer object's versi...
Sequencer(const Params &)
statistics::Histogram m_latencyHist
Histogram for holding latency profile of all requests.
void issueRequest(PacketPtr pkt, RubyRequestType type)
bool llscStoreConditional(const Addr)
Searches for cache line address in the global monitor tagged with this Sequencer object's version id.
void unaddressedCallback(Addr unaddressedReqId, RubyRequestType requestType, const MachineType mach=MachineType_NUM, const Cycles initialRequestTime=Cycles(0), const Cycles forwardRequestTime=Cycles(0), const Cycles firstResponseTime=Cycles(0))
virtual RequestStatus insertRequest(PacketPtr pkt, RubyRequestType primary_type, RubyRequestType secondary_type)
EventFunctionWrapper deadlockCheckEvent
std::unordered_map< uint64_t, SequencerRequest > m_UnaddressedRequestTable
uint64_t m_unaddressedTransactionCnt
std::vector< statistics::Histogram * > m_hitTypeLatencyHist
statistics::Histogram m_hitLatencyHist
Histogram for holding latency profile of all requests that hit in the controller connected to this se...
RequestStatus makeRequest(PacketPtr pkt) override
int m_max_outstanding_requests
void llscClearLocalMonitor()
Removes all addresses from the local monitor.
void recordRequestType(SequencerRequestType requestType)
std::vector< statistics::Histogram * > m_IssueToInitialDelayHist
Histograms for recording the breakdown of miss latency.
virtual void print(std::ostream &out) const
std::vector< std::vector< statistics::Histogram * > > m_hitTypeMachLatencyHist
void llscClearMonitor(const Addr)
Removes the cache line address from the global monitor.
void evictionCallback(Addr address)
std::vector< statistics::Histogram * > m_ForwardToFirstResponseDelayHist
void readCallback(Addr address, DataBlock &data, const bool externalHit=false, const MachineType mach=MachineType_NUM, const Cycles initialRequestTime=Cycles(0), const Cycles forwardRequestTime=Cycles(0), const Cycles firstResponseTime=Cycles(0))
std::vector< statistics::Histogram * > m_missMachLatencyHist
Histograms for profiling the latencies for requests that required external messages.
std::vector< statistics::Histogram * > m_missTypeLatencyHist
statistics::Histogram m_missLatencyHist
Histogram for holding latency profile of all requests that miss in the controller connected to this s...
std::vector< std::vector< statistics::Histogram * > > m_missTypeMachLatencyHist
void mergeFrom(const DataBlock &data)
void sample(const U &v, int n=1)
Add a value to the distribtion n times.
void reset()
Reset stat value to default.
DrainState drainState() const
Return the current drain state of an object.
@ Draining
Draining buffers pending serialization/handover.
bool scheduled() const
Determine if the current event is scheduled.
void schedule(Event &event, Tick when)
#define panic(...)
This implements a cprintf based panic() function.
#define fatal_if(cond,...)
Conditional fatal macro that checks the supplied condition and only causes a fatal error if the condi...
RubyRequestType tlbiCmdToRubyRequestType(const Packet *pkt)
bool isTlbiCmdRequest(RubyRequestType type)
Addr makeLineAddress(Addr addr)
Addr getOffset(Addr addr)
RubyRequestType htmCmdToRubyRequestType(const Packet *pkt)
std::ostream & operator<<(std::ostream &os, const BoolVec &myvector)
std::string printAddress(Addr addr)
Reference material can be found at the JEDEC website: UFS standard http://www.jedec....
Tick curTick()
The universal simulation clock.
uint64_t Addr
Address type This will probably be moved somewhere else in the near future.
uint64_t Tick
Tick count type.
int ContextID
Globally unique thread context ID.
const ContextID InvalidContextID
Declaration of the Packet class.
RubyRequestType m_second_type