table_walker.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2010-2016, 2019 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#ifndef __ARCH_ARM_TABLE_WALKER_HH__
#define __ARCH_ARM_TABLE_WALKER_HH__
 
#include <list>
 
#include "arch/arm/faults.hh"
#include "arch/arm/miscregs.hh"
#include "arch/arm/system.hh"
#include "arch/arm/tlb.hh"
#include "mem/request.hh"
#include "params/ArmTableWalker.hh"
#include "sim/clocked_object.hh"
#include "sim/eventq.hh"
 
class ThreadContext;
 
class DmaPort;
 
namespace ArmISA {
class Translation;
class TLB;
class Stage2MMU;
 
class TableWalker : public ClockedObject
{
  public:
    class WalkerState;
 
    class DescriptorBase {
      public:
        DescriptorBase() : lookupLevel(L0) {}
 
        LookupLevel lookupLevel;
 
        virtual Addr pfn() const = 0;
        virtual TlbEntry::DomainType domain() const = 0;
        virtual bool xn() const = 0;
        virtual uint8_t ap() const = 0;
        virtual bool global(WalkerState *currState) const = 0;
        virtual uint8_t offsetBits() const = 0;
        virtual bool secure(bool have_security, WalkerState *currState) const = 0;
        virtual std::string dbgHeader() const = 0;
        virtual uint64_t getRawData() const = 0;
        virtual uint8_t texcb() const
        {
            panic("texcb() not implemented for this class\n");
        }
        virtual bool shareable() const
        {
            panic("shareable() not implemented for this class\n");
        }
    };
 
    class L1Descriptor : public DescriptorBase {
      public:
        enum EntryType {
            Ignore,
            PageTable,
            Section,
            Reserved
        };
 
        uint32_t data;
 
        bool _dirty;
 
        L1Descriptor() : data(0), _dirty(false)
        {
            lookupLevel = L1;
        }
 
        virtual uint64_t getRawData() const
        {
            return (data);
        }
 
        virtual std::string dbgHeader() const
        {
            return "Inserting Section Descriptor into TLB\n";
        }
 
        virtual uint8_t offsetBits() const
        {
            return 20;
        }
 
        EntryType type() const
        {
            return (EntryType)(data & 0x3);
        }
 
        bool supersection() const
        {
            return bits(data, 18);
        }
 
        Addr paddr() const
        {
            if (supersection())
                panic("Super sections not implemented\n");
            return mbits(data, 31, 20);
        }
        Addr paddr(Addr va) const
        {
            if (supersection())
                panic("Super sections not implemented\n");
            return mbits(data, 31, 20) | mbits(va, 19, 0);
        }
 
 
        Addr pfn() const
        {
            if (supersection())
                panic("Super sections not implemented\n");
            return bits(data, 31, 20);
        }
 
        bool global(WalkerState *currState) const
        {
            return !bits(data, 17);
        }
 
        bool xn() const
        {
            return bits(data, 4);
        }
 
        uint8_t ap() const
        {
            return (bits(data, 15) << 2) | bits(data, 11, 10);
        }
 
        TlbEntry::DomainType domain() const
        {
            return static_cast<TlbEntry::DomainType>(bits(data, 8, 5));
        }
 
        Addr l2Addr() const
        {
            return mbits(data, 31, 10);
        }
 
        uint8_t texcb() const
        {
            return bits(data, 2) | bits(data, 3) << 1 | bits(data, 14, 12) << 2;
        }
 
        bool shareable() const
        {
            return bits(data, 16);
        }
 
        void setAp0()
        {
            data |= 1 << 10;
            _dirty = true;
        }
 
        bool dirty() const
        {
            return _dirty;
        }
 
        bool secure(bool have_security, WalkerState *currState) const
        {
            if (have_security && currState->secureLookup) {
                if (type() == PageTable)
                    return !bits(data, 3);
                else
                    return !bits(data, 19);
            }
            return false;
        }
    };
 
    class L2Descriptor : public DescriptorBase {
      public:
        uint32_t     data;
        L1Descriptor *l1Parent;
 
        bool _dirty;
 
        L2Descriptor() : data(0), l1Parent(nullptr), _dirty(false)
        {
            lookupLevel = L2;
        }
 
        L2Descriptor(L1Descriptor &parent) : data(0), l1Parent(&parent),
                                             _dirty(false)
        {
            lookupLevel = L2;
        }
 
        virtual uint64_t getRawData() const
        {
            return (data);
        }
 
        virtual std::string dbgHeader() const
        {
            return "Inserting L2 Descriptor into TLB\n";
        }
 
        virtual TlbEntry::DomainType domain() const
        {
            return l1Parent->domain();
        }
 
        bool secure(bool have_security, WalkerState *currState) const
        {
            return l1Parent->secure(have_security, currState);
        }
 
        virtual uint8_t offsetBits() const
        {
            return large() ? 16 : 12;
        }
 
        bool invalid() const
        {
            return bits(data, 1, 0) == 0;
        }
 
        bool large() const
        {
            return bits(data, 1) == 0;
        }
 
        bool xn() const
        {
            return large() ? bits(data, 15) : bits(data, 0);
        }
 
        bool global(WalkerState *currState) const
        {
            return !bits(data, 11);
        }
 
        uint8_t ap() const
        {
           return bits(data, 5, 4) | (bits(data, 9) << 2);
        }
 
        uint8_t texcb() const
        {
            return large() ?
                (bits(data, 2) | (bits(data, 3) << 1) | (bits(data, 14, 12) << 2)) :
                (bits(data, 2) | (bits(data, 3) << 1) | (bits(data, 8, 6) << 2));
        }
 
        Addr pfn() const
        {
            return large() ? bits(data, 31, 16) : bits(data, 31, 12);
        }
 
        Addr paddr(Addr va) const
        {
            if (large())
                return mbits(data, 31, 16) | mbits(va, 15, 0);
            else
                return mbits(data, 31, 12) | mbits(va, 11, 0);
        }
 
        bool shareable() const
        {
            return bits(data, 10);
        }
 
        void setAp0()
        {
            data |= 1 << 4;
            _dirty = true;
        }
 
        bool dirty() const
        {
            return _dirty;
        }
 
    };
 
    // Granule sizes for AArch64 long descriptors
    enum GrainSize {
        Grain4KB  = 12,
        Grain16KB = 14,
        Grain64KB = 16,
        ReservedGrain = 0
    };
 
    class LongDescriptor : public DescriptorBase {
      public:
        enum EntryType {
            Invalid,
            Table,
            Block,
            Page
        };
 
        LongDescriptor()
          : data(0), _dirty(false), aarch64(false), grainSize(Grain4KB),
            physAddrRange(0)
        {}
 
        uint64_t data;
 
        bool _dirty;
 
        bool aarch64;
 
        GrainSize grainSize;
 
        uint8_t physAddrRange;
 
 
        virtual uint64_t getRawData() const
        {
            return (data);
        }
 
        virtual std::string dbgHeader() const
        {
            if (type() == LongDescriptor::Page) {
                assert(lookupLevel == L3);
                return "Inserting Page descriptor into TLB\n";
            } else {
                assert(lookupLevel < L3);
                return "Inserting Block descriptor into TLB\n";
            }
        }
 
        bool secure(bool have_security, WalkerState *currState) const
        {
            assert(type() == Block || type() == Page);
            return have_security && (currState->secureLookup && !bits(data, 5));
        }
 
        EntryType type() const
        {
            switch (bits(data, 1, 0)) {
              case 0x1:
                // In AArch64 blocks are not allowed at L0 for the
                // 4 KiB granule and at L1 for 16/64 KiB granules
                switch (grainSize) {
                  case Grain4KB:
                    if (lookupLevel == L0 || lookupLevel == L3)
                        return Invalid;
                    else
                        return Block;
 
                  case Grain16KB:
                    if (lookupLevel == L2)
                        return Block;
                    else
                        return Invalid;
 
                  case Grain64KB:
                    // With Armv8.2-LPA (52bit PA) L1 Block descriptors
                    // are allowed for 64KiB granule
                    if ((lookupLevel == L1 && physAddrRange == 52) ||
                        lookupLevel == L2)
                        return Block;
                    else
                        return Invalid;
 
                  default:
                    return Invalid;
                }
              case 0x3:
                return lookupLevel == L3 ? Page : Table;
              default:
                return Invalid;
            }
        }
 
        uint8_t offsetBits() const
        {
            if (type() == Block) {
                switch (grainSize) {
                    case Grain4KB:
                        return lookupLevel == L1 ? 30 /* 1 GiB */
                                                 : 21 /* 2 MiB */;
                    case Grain16KB:
                        return 25  /* 32 MiB */;
                    case Grain64KB:
                        return lookupLevel == L1 ? 42 /* 4 TiB */
                                                 : 29 /* 512 MiB */;
                    default:
                        panic("Invalid AArch64 VM granule size\n");
                }
            } else if (type() == Page) {
                switch (grainSize) {
                    case Grain4KB:
                    case Grain16KB:
                    case Grain64KB:
                        return grainSize; /* enum -> uint okay */
                    default:
                        panic("Invalid AArch64 VM granule size\n");
                }
            } else {
                panic("AArch64 page table entry must be block or page\n");
            }
        }
 
        Addr pfn() const
        {
            return paddr() >> offsetBits();
        }
 
        Addr paddr() const
        {
            Addr addr = 0;
            if (aarch64) {
                addr = mbits(data, 47, offsetBits());
                if (physAddrRange == 52 && grainSize == Grain64KB) {
                    addr |= bits(data, 15, 12) << 48;
                }
            } else {
                addr = mbits(data, 39, offsetBits());
            }
            return addr;
        }
 
        Addr nextTableAddr() const
        {
            assert(type() == Table);
            Addr table_address = 0;
            if (aarch64) {
                table_address = mbits(data, 47, grainSize);
                // Using 52bit if Armv8.2-LPA is implemented
                if (physAddrRange == 52 && grainSize == Grain64KB)
                    table_address |= bits(data, 15, 12) << 48;
            } else {
                table_address = mbits(data, 39, 12);
            }
 
            return table_address;
        }
 
        Addr nextDescAddr(Addr va) const
        {
            assert(type() == Table);
            Addr pa = 0;
            if (aarch64) {
                int stride = grainSize - 3;
                int va_lo = stride * (3 - (lookupLevel + 1)) + grainSize;
                int va_hi = va_lo + stride - 1;
                pa = nextTableAddr() | (bits(va, va_hi, va_lo) << 3);
            } else {
                if (lookupLevel == L1)
                    pa = nextTableAddr() | (bits(va, 29, 21) << 3);
                else  // lookupLevel == L2
                    pa = nextTableAddr() | (bits(va, 20, 12) << 3);
            }
            return pa;
        }
 
        bool xn() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 54);
        }
 
        bool pxn() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 53);
        }
 
        bool contiguousHint() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 52);
        }
 
        bool global(WalkerState *currState) const
        {
            assert(currState && (type() == Block || type() == Page));
            if (!currState->aarch64 && (currState->isSecure &&
                                        !currState->secureLookup)) {
                return false;  // ARM ARM issue C B3.6.3
            } else if (currState->aarch64) {
                if (currState->el == EL2 || currState->el == EL3) {
                    return true;  // By default translations are treated as global
                                  // in AArch64 EL2 and EL3
                } else if (currState->isSecure && !currState->secureLookup) {
                    return false;
                }
            }
            return !bits(data, 11);
        }
 
        bool af() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 10);
        }
 
        uint8_t sh() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 9, 8);
        }
 
        uint8_t ap() const
        {
            assert(type() == Block || type() == Page);
            // Long descriptors only support the AP[2:1] scheme
            return bits(data, 7, 6);
        }
 
        bool rw() const
        {
            assert(type() == Block || type() == Page);
            return !bits(data, 7);
        }
 
        bool user() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 6);
        }
 
        static uint8_t ap(bool rw, bool user)
        {
            return ((!rw) << 2) | (user << 1);
        }
 
        TlbEntry::DomainType domain() const
        {
            // Long-desc. format only supports Client domain
            assert(type() == Block || type() == Page);
            return TlbEntry::DomainType::Client;
        }
 
        uint8_t attrIndx() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 4, 2);
        }
 
        uint8_t memAttr() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 5, 2);
        }
 
        void setAf()
        {
            data |= 1 << 10;
            _dirty = true;
        }
 
        bool dirty() const
        {
            return _dirty;
        }
 
        bool secureTable() const
        {
            assert(type() == Table);
            return !bits(data, 63);
        }
 
        uint8_t apTable() const
        {
            assert(type() == Table);
            return bits(data, 62, 61);
        }
 
        uint8_t rwTable() const
        {
            assert(type() == Table);
            return !bits(data, 62);
        }
 
        uint8_t userTable() const
        {
            assert(type() == Table);
            return !bits(data, 61);
        }
 
        bool xnTable() const
        {
            assert(type() == Table);
            return bits(data, 60);
        }
 
        bool pxnTable() const
        {
            assert(type() == Table);
            return bits(data, 59);
        }
    };
 
    class WalkerState
    {
      public:
        ThreadContext *tc;
 
        bool aarch64;
 
        ExceptionLevel el;
 
        int physAddrRange;
 
        RequestPtr req;
 
        uint16_t asid;
        uint8_t vmid;
        bool    isHyp;
 
        TLB::Translation *transState;
 
        Fault fault;
 
        Addr vaddr;
 
        Addr vaddr_tainted;
 
        SCTLR sctlr;
 
        SCR scr;
 
        CPSR cpsr;
 
        union {
            TTBCR ttbcr; // AArch32 translations
            TCR tcr;     // AArch64 translations
        };
 
        HTCR htcr;
 
        HCR  hcr;
 
        VTCR_t vtcr;
 
        bool isWrite;
 
        bool isFetch;
 
        bool isSecure;
 
        bool isUncacheable;
 
        bool secureLookup;
        bool rwTable;
        bool userTable;
        bool xnTable;
        bool pxnTable;
 
        bool hpd;
 
        bool stage2Req;
 
        TLB::Translation *stage2Tran;
 
        bool timing;
 
        bool functional;
 
        BaseTLB::Mode mode;
 
        TLB::ArmTranslationType tranType;
 
        L1Descriptor l1Desc;
        L2Descriptor l2Desc;
 
        LongDescriptor longDesc;
 
        bool delayed;
 
        TableWalker *tableWalker;
 
        Tick startTime;
 
        unsigned levels;
 
        void doL1Descriptor();
        void doL2Descriptor();
 
        void doLongDescriptor();
 
        WalkerState();
 
        std::string name() const { return tableWalker->name(); }
    };
 
  protected:
 
    std::list<WalkerState *> stateQueues[MAX_LOOKUP_LEVELS];
 
    std::list<WalkerState *> pendingQueue;
 
    Stage2MMU *stage2Mmu;
 
    DmaPort* port;
 
    RequestorID requestorId;
 
    const bool isStage2;
 
    TLB *tlb;
 
    SCTLR sctlr;
 
    WalkerState *currState;
 
    bool pending;
 
    unsigned numSquashable;
 
    bool haveSecurity;
    bool _haveLPAE;
    bool _haveVirtualization;
    uint8_t _physAddrRange;
    bool _haveLargeAsid64;
 
   struct TableWalkerStats : public Stats::Group {
        TableWalkerStats(Stats::Group *parent);
        Stats::Scalar walks;
        Stats::Scalar walksShortDescriptor;
        Stats::Scalar walksLongDescriptor;
        Stats::Vector walksShortTerminatedAtLevel;
        Stats::Vector walksLongTerminatedAtLevel;
        Stats::Scalar squashedBefore;
        Stats::Scalar squashedAfter;
        Stats::Histogram walkWaitTime;
        Stats::Histogram walkServiceTime;
        Stats::Histogram pendingWalks; // essentially "L" of queueing theory
        Stats::Vector pageSizes;
        Stats::Vector2d requestOrigin;
    } stats;
 
    mutable unsigned pendingReqs;
    mutable Tick pendingChangeTick;
 
    static const unsigned REQUESTED = 0;
    static const unsigned COMPLETED = 1;
 
  public:
    PARAMS(ArmTableWalker);
    TableWalker(const Params &p);
    virtual ~TableWalker();
 
    void init() override;
 
    bool haveLPAE() const { return _haveLPAE; }
    bool haveVirtualization() const { return _haveVirtualization; }
    bool haveLargeAsid64() const { return _haveLargeAsid64; }
    uint8_t physAddrRange() const { return _physAddrRange; }
    void completeDrain();
    DrainState drain() override;
    void drainResume() override;
 
    Port &getPort(const std::string &if_name,
                  PortID idx=InvalidPortID) override;
 
    Fault walk(const RequestPtr &req, ThreadContext *tc,
               uint16_t asid, uint8_t _vmid,
               bool _isHyp, TLB::Mode mode, TLB::Translation *_trans,
               bool timing, bool functional, bool secure,
               TLB::ArmTranslationType tranType, bool _stage2Req);
 
    void setTlb(TLB *_tlb) { tlb = _tlb; }
    TLB* getTlb() { return tlb; }
    void setMMU(Stage2MMU *m, RequestorID requestor_id);
    void memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
                  uint8_t texcb, bool s);
    void memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
                      LongDescriptor &lDescriptor);
    void memAttrsAArch64(ThreadContext *tc, TlbEntry &te,
                         LongDescriptor &lDescriptor);
 
    static LookupLevel toLookupLevel(uint8_t lookup_level_as_int);
 
  private:
 
    void doL1Descriptor();
    void doL1DescriptorWrapper();
    EventFunctionWrapper doL1DescEvent;
 
    void doL2Descriptor();
    void doL2DescriptorWrapper();
    EventFunctionWrapper doL2DescEvent;
 
    void doLongDescriptor();
 
    void doL0LongDescriptorWrapper();
    EventFunctionWrapper doL0LongDescEvent;
    void doL1LongDescriptorWrapper();
    EventFunctionWrapper doL1LongDescEvent;
    void doL2LongDescriptorWrapper();
    EventFunctionWrapper doL2LongDescEvent;
    void doL3LongDescriptorWrapper();
    EventFunctionWrapper doL3LongDescEvent;
 
    void doLongDescriptorWrapper(LookupLevel curr_lookup_level);
    Event* LongDescEventByLevel[4];
 
    bool fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
        Request::Flags flags, int queueIndex, Event *event,
        void (TableWalker::*doDescriptor)());
 
    Fault generateLongDescFault(ArmFault::FaultSource src);
 
    void insertTableEntry(DescriptorBase &descriptor, bool longDescriptor);
 
    Fault processWalk();
    Fault processWalkLPAE();
 
    bool checkVAddrSizeFaultAArch64(Addr addr, int top_bit,
        GrainSize granule, int tsz, bool low_range);
 
    bool checkAddrSizeFaultAArch64(Addr addr, int pa_range);
 
    Fault processWalkAArch64();
    void processWalkWrapper();
    EventFunctionWrapper doProcessEvent;
 
    void nextWalk(ThreadContext *tc);
 
    void pendingChange();
 
    static uint8_t pageSizeNtoStatBin(uint8_t N);
 
    Fault testWalk(Addr pa, Addr size, TlbEntry::DomainType domain,
                   LookupLevel lookup_level);
};
 
} // namespace ArmISA
 
#endif //__ARCH_ARM_TABLE_WALKER_HH__