table_walker.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2010-2016, 2019 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Ali Saidi
 *          Giacomo Gabrielli
 */

#ifndef __ARCH_ARM_TABLE_WALKER_HH__
#define __ARCH_ARM_TABLE_WALKER_HH__

#include <list>

#include "arch/arm/faults.hh"
#include "arch/arm/miscregs.hh"
#include "arch/arm/system.hh"
#include "arch/arm/tlb.hh"
#include "mem/request.hh"
#include "params/ArmTableWalker.hh"
#include "sim/clocked_object.hh"
#include "sim/eventq.hh"

class ThreadContext;

class DmaPort;

namespace ArmISA {
class Translation;
class TLB;
class Stage2MMU;

class TableWalker : public ClockedObject
{
  public:
    class WalkerState;

    class DescriptorBase {
      public:
        DescriptorBase() : lookupLevel(L0) {}

        LookupLevel lookupLevel;

        virtual Addr pfn() const = 0;
        virtual TlbEntry::DomainType domain() const = 0;
        virtual bool xn() const = 0;
        virtual uint8_t ap() const = 0;
        virtual bool global(WalkerState *currState) const = 0;
        virtual uint8_t offsetBits() const = 0;
        virtual bool secure(bool have_security, WalkerState *currState) const = 0;
        virtual std::string dbgHeader() const = 0;
        virtual uint64_t getRawData() const = 0;
        virtual uint8_t texcb() const
        {
            panic("texcb() not implemented for this class\n");
        }
        virtual bool shareable() const
        {
            panic("shareable() not implemented for this class\n");
        }
    };

    class L1Descriptor : public DescriptorBase {
      public:
        enum EntryType {
            Ignore,
            PageTable,
            Section,
            Reserved
        };

        uint32_t data;

        bool _dirty;

        L1Descriptor() : data(0), _dirty(false)
        {
            lookupLevel = L1;
        }

        virtual uint64_t getRawData() const
        {
            return (data);
        }

        virtual std::string dbgHeader() const
        {
            return "Inserting Section Descriptor into TLB\n";
        }

        virtual uint8_t offsetBits() const
        {
            return 20;
        }

        EntryType type() const
        {
            return (EntryType)(data & 0x3);
        }

        bool supersection() const
        {
            return bits(data, 18);
        }

        Addr paddr() const
        {
            if (supersection())
                panic("Super sections not implemented\n");
            return mbits(data, 31, 20);
        }
        Addr paddr(Addr va) const
        {
            if (supersection())
                panic("Super sections not implemented\n");
            return mbits(data, 31, 20) | mbits(va, 19, 0);
        }


        Addr pfn() const
        {
            if (supersection())
                panic("Super sections not implemented\n");
            return bits(data, 31, 20);
        }

        bool global(WalkerState *currState) const
        {
            return !bits(data, 17);
        }

        bool xn() const
        {
            return bits(data, 4);
        }

        uint8_t ap() const
        {
            return (bits(data, 15) << 2) | bits(data, 11, 10);
        }

        TlbEntry::DomainType domain() const
        {
            return static_cast<TlbEntry::DomainType>(bits(data, 8, 5));
        }

        Addr l2Addr() const
        {
            return mbits(data, 31, 10);
        }

        uint8_t texcb() const
        {
            return bits(data, 2) | bits(data, 3) << 1 | bits(data, 14, 12) << 2;
        }

        bool shareable() const
        {
            return bits(data, 16);
        }

        void setAp0()
        {
            data |= 1 << 10;
            _dirty = true;
        }

        bool dirty() const
        {
            return _dirty;
        }

        bool secure(bool have_security, WalkerState *currState) const
        {
            if (have_security) {
                if (type() == PageTable)
                    return !bits(data, 3);
                else
                    return !bits(data, 19);
            }
            return false;
        }
    };

    class L2Descriptor : public DescriptorBase {
      public:
        uint32_t     data;
        L1Descriptor *l1Parent;

        bool _dirty;

        L2Descriptor() : data(0), l1Parent(nullptr), _dirty(false)
        {
            lookupLevel = L2;
        }

        L2Descriptor(L1Descriptor &parent) : data(0), l1Parent(&parent),
                                             _dirty(false)
        {
            lookupLevel = L2;
        }

        virtual uint64_t getRawData() const
        {
            return (data);
        }

        virtual std::string dbgHeader() const
        {
            return "Inserting L2 Descriptor into TLB\n";
        }

        virtual TlbEntry::DomainType domain() const
        {
            return l1Parent->domain();
        }

        bool secure(bool have_security, WalkerState *currState) const
        {
            return l1Parent->secure(have_security, currState);
        }

        virtual uint8_t offsetBits() const
        {
            return large() ? 16 : 12;
        }

        bool invalid() const
        {
            return bits(data, 1, 0) == 0;
        }

        bool large() const
        {
            return bits(data, 1) == 0;
        }

        bool xn() const
        {
            return large() ? bits(data, 15) : bits(data, 0);
        }

        bool global(WalkerState *currState) const
        {
            return !bits(data, 11);
        }

        uint8_t ap() const
        {
           return bits(data, 5, 4) | (bits(data, 9) << 2);
        }

        uint8_t texcb() const
        {
            return large() ?
                (bits(data, 2) | (bits(data, 3) << 1) | (bits(data, 14, 12) << 2)) :
                (bits(data, 2) | (bits(data, 3) << 1) | (bits(data, 8, 6) << 2));
        }

        Addr pfn() const
        {
            return large() ? bits(data, 31, 16) : bits(data, 31, 12);
        }

        Addr paddr(Addr va) const
        {
            if (large())
                return mbits(data, 31, 16) | mbits(va, 15, 0);
            else
                return mbits(data, 31, 12) | mbits(va, 11, 0);
        }

        bool shareable() const
        {
            return bits(data, 10);
        }

        void setAp0()
        {
            data |= 1 << 4;
            _dirty = true;
        }

        bool dirty() const
        {
            return _dirty;
        }

    };

    // Granule sizes for AArch64 long descriptors
    enum GrainSize {
        Grain4KB  = 12,
        Grain16KB = 14,
        Grain64KB = 16,
        ReservedGrain = 0
    };

    class LongDescriptor : public DescriptorBase {
      public:
        enum EntryType {
            Invalid,
            Table,
            Block,
            Page
        };

        LongDescriptor() : data(0), _dirty(false) {}

        uint64_t data;

        bool _dirty;

        virtual uint64_t getRawData() const
        {
            return (data);
        }

        virtual std::string dbgHeader() const
        {
            if (type() == LongDescriptor::Page) {
                assert(lookupLevel == L3);
                return "Inserting Page descriptor into TLB\n";
            } else {
                assert(lookupLevel < L3);
                return "Inserting Block descriptor into TLB\n";
            }
        }

        bool secure(bool have_security, WalkerState *currState) const
        {
            assert(type() == Block || type() == Page);
            return have_security && (currState->secureLookup && !bits(data, 5));
        }

        bool aarch64;

        GrainSize grainSize;

        EntryType type() const
        {
            switch (bits(data, 1, 0)) {
              case 0x1:
                // In AArch64 blocks are not allowed at L0 for the 4 KB granule
                // and at L1 for 16/64 KB granules
                if (grainSize > Grain4KB)
                    return lookupLevel == L2 ? Block : Invalid;
                return lookupLevel == L0 || lookupLevel == L3 ? Invalid : Block;
              case 0x3:
                return lookupLevel == L3 ? Page : Table;
              default:
                return Invalid;
            }
        }

        uint8_t offsetBits() const
        {
            if (type() == Block) {
                switch (grainSize) {
                    case Grain4KB:
                        return lookupLevel == L1 ? 30 /* 1 GB */
                                                 : 21 /* 2 MB */;
                    case Grain16KB:
                        return 25  /* 32 MB */;
                    case Grain64KB:
                        return 29 /* 512 MB */;
                    default:
                        panic("Invalid AArch64 VM granule size\n");
                }
            } else if (type() == Page) {
                switch (grainSize) {
                    case Grain4KB:
                    case Grain16KB:
                    case Grain64KB:
                        return grainSize; /* enum -> uint okay */
                    default:
                        panic("Invalid AArch64 VM granule size\n");
                }
            } else {
                panic("AArch64 page table entry must be block or page\n");
            }
        }

        Addr pfn() const
        {
            if (aarch64)
                return bits(data, 47, offsetBits());
            return bits(data, 39, offsetBits());
        }

        Addr paddr(Addr va) const
        {
            int n = offsetBits();
            if (aarch64)
                return mbits(data, 47, n) | mbits(va, n - 1, 0);
            return mbits(data, 39, n) | mbits(va, n - 1, 0);
        }

        Addr paddr() const
        {
            if (aarch64)
                return mbits(data, 47, offsetBits());
            return mbits(data, 39, offsetBits());
        }

        Addr nextTableAddr() const
        {
            assert(type() == Table);
            if (aarch64)
                return mbits(data, 47, grainSize);
            else
                return mbits(data, 39, 12);
        }

        Addr nextDescAddr(Addr va) const
        {
            assert(type() == Table);
            Addr pa = 0;
            if (aarch64) {
                int stride = grainSize - 3;
                int va_lo = stride * (3 - (lookupLevel + 1)) + grainSize;
                int va_hi = va_lo + stride - 1;
                pa = nextTableAddr() | (bits(va, va_hi, va_lo) << 3);
            } else {
                if (lookupLevel == L1)
                    pa = nextTableAddr() | (bits(va, 29, 21) << 3);
                else  // lookupLevel == L2
                    pa = nextTableAddr() | (bits(va, 20, 12) << 3);
            }
            return pa;
        }

        bool xn() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 54);
        }

        bool pxn() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 53);
        }

        bool contiguousHint() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 52);
        }

        bool global(WalkerState *currState) const
        {
            assert(currState && (type() == Block || type() == Page));
            if (!currState->aarch64 && (currState->isSecure &&
                                        !currState->secureLookup)) {
                return false;  // ARM ARM issue C B3.6.3
            } else if (currState->aarch64) {
                if (currState->el == EL2 || currState->el == EL3) {
                    return true;  // By default translations are treated as global
                                  // in AArch64 EL2 and EL3
                } else if (currState->isSecure && !currState->secureLookup) {
                    return false;
                }
            }
            return !bits(data, 11);
        }

        bool af() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 10);
        }

        uint8_t sh() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 9, 8);
        }

        uint8_t ap() const
        {
            assert(type() == Block || type() == Page);
            // Long descriptors only support the AP[2:1] scheme
            return bits(data, 7, 6);
        }

        bool rw() const
        {
            assert(type() == Block || type() == Page);
            return !bits(data, 7);
        }

        bool user() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 6);
        }

        static uint8_t ap(bool rw, bool user)
        {
            return ((!rw) << 2) | (user << 1);
        }

        TlbEntry::DomainType domain() const
        {
            // Long-desc. format only supports Client domain
            assert(type() == Block || type() == Page);
            return TlbEntry::DomainType::Client;
        }

        uint8_t attrIndx() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 4, 2);
        }

        uint8_t memAttr() const
        {
            assert(type() == Block || type() == Page);
            return bits(data, 5, 2);
        }

        void setAf()
        {
            data |= 1 << 10;
            _dirty = true;
        }

        bool dirty() const
        {
            return _dirty;
        }

        bool secureTable() const
        {
            assert(type() == Table);
            return !bits(data, 63);
        }

        uint8_t apTable() const
        {
            assert(type() == Table);
            return bits(data, 62, 61);
        }

        uint8_t rwTable() const
        {
            assert(type() == Table);
            return !bits(data, 62);
        }

        uint8_t userTable() const
        {
            assert(type() == Table);
            return !bits(data, 61);
        }

        bool xnTable() const
        {
            assert(type() == Table);
            return bits(data, 60);
        }

        bool pxnTable() const
        {
            assert(type() == Table);
            return bits(data, 59);
        }
    };

    class WalkerState
    {
      public:
        ThreadContext *tc;

        bool aarch64;

        ExceptionLevel el;

        int physAddrRange;

        RequestPtr req;

        uint16_t asid;
        uint8_t vmid;
        bool    isHyp;

        TLB::Translation *transState;

        Fault fault;

        Addr vaddr;

        Addr vaddr_tainted;

        SCTLR sctlr;

        SCR scr;

        CPSR cpsr;

        union {
            TTBCR ttbcr; // AArch32 translations
            TCR tcr;     // AArch64 translations
        };

        HTCR htcr;

        HCR  hcr;

        VTCR_t vtcr;

        bool isWrite;

        bool isFetch;

        bool isSecure;

        bool isUncacheable;

        bool secureLookup;
        bool rwTable;
        bool userTable;
        bool xnTable;
        bool pxnTable;

        bool hpd;

        bool stage2Req;

        TLB::Translation *stage2Tran;

        bool timing;

        bool functional;

        BaseTLB::Mode mode;

        TLB::ArmTranslationType tranType;

        L1Descriptor l1Desc;
        L2Descriptor l2Desc;

        LongDescriptor longDesc;

        bool delayed;

        TableWalker *tableWalker;

        Tick startTime;

        unsigned levels;

        void doL1Descriptor();
        void doL2Descriptor();

        void doLongDescriptor();

        WalkerState();

        std::string name() const { return tableWalker->name(); }
    };

  protected:

    std::list<WalkerState *> stateQueues[MAX_LOOKUP_LEVELS];

    std::list<WalkerState *> pendingQueue;

    Stage2MMU *stage2Mmu;

    DmaPort* port;

    MasterID masterId;

    const bool isStage2;

    TLB *tlb;

    SCTLR sctlr;

    WalkerState *currState;

    bool pending;

    unsigned numSquashable;

    bool haveSecurity;
    bool _haveLPAE;
    bool _haveVirtualization;
    uint8_t physAddrRange;
    bool _haveLargeAsid64;

    Stats::Scalar statWalks;
    Stats::Scalar statWalksShortDescriptor;
    Stats::Scalar statWalksLongDescriptor;
    Stats::Vector statWalksShortTerminatedAtLevel;
    Stats::Vector statWalksLongTerminatedAtLevel;
    Stats::Scalar statSquashedBefore;
    Stats::Scalar statSquashedAfter;
    Stats::Histogram statWalkWaitTime;
    Stats::Histogram statWalkServiceTime;
    Stats::Histogram statPendingWalks; // essentially "L" of queueing theory
    Stats::Vector statPageSizes;
    Stats::Vector2d statRequestOrigin;

    mutable unsigned pendingReqs;
    mutable Tick pendingChangeTick;

    static const unsigned REQUESTED = 0;
    static const unsigned COMPLETED = 1;

  public:
   typedef ArmTableWalkerParams Params;
    TableWalker(const Params *p);
    virtual ~TableWalker();

    const Params *
    params() const
    {
        return dynamic_cast<const Params *>(_params);
    }

    void init() override;

    bool haveLPAE() const { return _haveLPAE; }
    bool haveVirtualization() const { return _haveVirtualization; }
    bool haveLargeAsid64() const { return _haveLargeAsid64; }
    void completeDrain();
    DrainState drain() override;
    void drainResume() override;

    Port &getPort(const std::string &if_name,
                  PortID idx=InvalidPortID) override;

    void regStats() override;

    Fault walk(const RequestPtr &req, ThreadContext *tc,
               uint16_t asid, uint8_t _vmid,
               bool _isHyp, TLB::Mode mode, TLB::Translation *_trans,
               bool timing, bool functional, bool secure,
               TLB::ArmTranslationType tranType, bool _stage2Req);

    void setTlb(TLB *_tlb) { tlb = _tlb; }
    TLB* getTlb() { return tlb; }
    void setMMU(Stage2MMU *m, MasterID master_id);
    void memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
                  uint8_t texcb, bool s);
    void memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
                      LongDescriptor &lDescriptor);
    void memAttrsAArch64(ThreadContext *tc, TlbEntry &te,
                         LongDescriptor &lDescriptor);

    static LookupLevel toLookupLevel(uint8_t lookup_level_as_int);

  private:

    void doL1Descriptor();
    void doL1DescriptorWrapper();
    EventFunctionWrapper doL1DescEvent;

    void doL2Descriptor();
    void doL2DescriptorWrapper();
    EventFunctionWrapper doL2DescEvent;

    void doLongDescriptor();

    void doL0LongDescriptorWrapper();
    EventFunctionWrapper doL0LongDescEvent;
    void doL1LongDescriptorWrapper();
    EventFunctionWrapper doL1LongDescEvent;
    void doL2LongDescriptorWrapper();
    EventFunctionWrapper doL2LongDescEvent;
    void doL3LongDescriptorWrapper();
    EventFunctionWrapper doL3LongDescEvent;

    void doLongDescriptorWrapper(LookupLevel curr_lookup_level);
    Event* LongDescEventByLevel[4];

    bool fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
        Request::Flags flags, int queueIndex, Event *event,
        void (TableWalker::*doDescriptor)());

    Fault generateLongDescFault(ArmFault::FaultSource src);

    void insertTableEntry(DescriptorBase &descriptor, bool longDescriptor);

    Fault processWalk();
    Fault processWalkLPAE();
    static unsigned adjustTableSizeAArch64(unsigned tsz);
    static bool checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange);
    Fault processWalkAArch64();
    void processWalkWrapper();
    EventFunctionWrapper doProcessEvent;

    void nextWalk(ThreadContext *tc);

    void pendingChange();

    static uint8_t pageSizeNtoStatBin(uint8_t N);

    Fault testWalk(Addr pa, Addr size, TlbEntry::DomainType domain,
                   LookupLevel lookup_level);
};

} // namespace ArmISA

#endif //__ARCH_ARM_TABLE_WALKER_HH__