operand.hh

Go to the documentation of this file.

/*
 * Copyright (c) 2017-2021 Advanced Micro Devices, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
 
#ifndef __ARCH_VEGA_OPERAND_HH__
#define __ARCH_VEGA_OPERAND_HH__
 
#include <array>
 
#include "arch/amdgpu/vega/gpu_registers.hh"
#include "arch/generic/vec_reg.hh"
#include "debug/GPUTrace.hh"
#include "gpu-compute/scalar_register_file.hh"
#include "gpu-compute/shader.hh"
#include "gpu-compute/vector_register_file.hh"
#include "gpu-compute/wavefront.hh"
 
namespace gem5
{

 
namespace VegaISA
{
    template<typename T> struct OpTraits { typedef float FloatT; };
    template<> struct OpTraits<ScalarRegF64> { typedef double FloatT; };
    template<> struct OpTraits<ScalarRegU64> { typedef double FloatT; };
 
    class Operand
    {
      public:
        Operand() = delete;
 
        Operand(GPUDynInstPtr gpuDynInst, int opIdx)
            : _gpuDynInst(gpuDynInst), _opIdx(opIdx)
        {
            assert(_gpuDynInst);
            assert(_opIdx >= 0);
        }

        virtual void read() = 0;
        virtual void write() = 0;
 
      protected:
        GPUDynInstPtr _gpuDynInst;
        int _opIdx;
    };
 
    template<typename DataType, bool Const, size_t NumDwords>
    class ScalarOperand;
 
    template<typename DataType, bool Const,
        size_t NumDwords = sizeof(DataType) / sizeof(VecElemU32)>
    class VecOperand final : public Operand
    {
      static_assert(NumDwords >= 1 && NumDwords <= MaxOperandDwords,
            "Incorrect number of DWORDS for VEGA operand.");
 
      public:
        VecOperand() = delete;
 
        VecOperand(GPUDynInstPtr gpuDynInst, int opIdx)
            : Operand(gpuDynInst, opIdx), scalar(false), absMod(false),
              negMod(false), scRegData(gpuDynInst, _opIdx),
              vrfData{{ nullptr }}
        {
            vecReg.zero();
        }
 
        ~VecOperand()
        {
        }

        void
        readSrc()
        {
            if (isVectorReg(_opIdx)) {
                _opIdx = opSelectorToRegIdx(_opIdx, _gpuDynInst->wavefront()
                    ->reservedScalarRegs);
                read();
            } else {
                readScalar();
            }
        }

        void
        read() override
        {
            assert(_gpuDynInst);
            assert(_gpuDynInst->wavefront());
            assert(_gpuDynInst->computeUnit());
            Wavefront *wf = _gpuDynInst->wavefront();
            ComputeUnit *cu = _gpuDynInst->computeUnit();
 
            for (auto i = 0; i < NumDwords; ++i) {
                int vgprIdx = cu->registerManager->mapVgpr(wf, _opIdx + i);
                vrfData[i] = &cu->vrf[wf->simdId]->readWriteable(vgprIdx);
 
                DPRINTF(GPUVRF, "Read v[%d]\n", vgprIdx);
                DPRINTF(GPUTrace, "wave[%d] Read v[%d] by instruction %s\n",
                        wf->wfDynId, vgprIdx,
                        _gpuDynInst->disassemble().c_str());
                cu->vrf[wf->simdId]->printReg(wf, vgprIdx);
            }
 
            if (NumDwords == 1) {
                assert(vrfData[0]);
                auto vgpr = vecReg.template as<DataType>();
                auto reg_file_vgpr = vrfData[0]->template as<VecElemU32>();
                for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
                    std::memcpy((void*)&vgpr[lane],
                        (void*)&reg_file_vgpr[lane], sizeof(DataType));
                }
            } else if (NumDwords == 2) {
                assert(vrfData[0]);
                assert(vrfData[1]);
                auto vgpr = vecReg.template as<VecElemU64>();
                auto reg_file_vgpr0 = vrfData[0]->template as<VecElemU32>();
                auto reg_file_vgpr1 = vrfData[1]->template as<VecElemU32>();
 
                for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
                    VecElemU64 tmp_val(0);
                    ((VecElemU32*)&tmp_val)[0] = reg_file_vgpr0[lane];
                    ((VecElemU32*)&tmp_val)[1] = reg_file_vgpr1[lane];
                    vgpr[lane] = tmp_val;
                }
            }
        }

        void
        write() override
        {
            assert(_gpuDynInst);
            assert(_gpuDynInst->wavefront());
            assert(_gpuDynInst->computeUnit());
            Wavefront *wf = _gpuDynInst->wavefront();
            ComputeUnit *cu = _gpuDynInst->computeUnit();
            VectorMask &exec_mask = _gpuDynInst->isLoad()
                ? _gpuDynInst->exec_mask : wf->execMask();
 
            if (NumDwords == 1) {
                int vgprIdx = cu->registerManager->mapVgpr(wf, _opIdx);
                vrfData[0] = &cu->vrf[wf->simdId]->readWriteable(vgprIdx);
                assert(vrfData[0]);
                auto reg_file_vgpr = vrfData[0]->template as<VecElemU32>();
                auto vgpr = vecReg.template as<DataType>();
 
                for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
                    if (exec_mask[lane] || _gpuDynInst->ignoreExec()) {
                        std::memcpy((void*)&reg_file_vgpr[lane],
                            (void*)&vgpr[lane], sizeof(DataType));
                    }
                }
 
                DPRINTF(GPUVRF, "Write v[%d]\n", vgprIdx);
                DPRINTF(GPUTrace, "wave[%d] Write v[%d] by instruction %s\n",
                        wf->wfDynId, vgprIdx,
                        _gpuDynInst->disassemble().c_str());
                cu->vrf[wf->simdId]->printReg(wf, vgprIdx);
            } else if (NumDwords == 2) {
                int vgprIdx0 = cu->registerManager->mapVgpr(wf, _opIdx);
                int vgprIdx1 = cu->registerManager->mapVgpr(wf, _opIdx + 1);
                vrfData[0] = &cu->vrf[wf->simdId]->readWriteable(vgprIdx0);
                vrfData[1] = &cu->vrf[wf->simdId]->readWriteable(vgprIdx1);
                assert(vrfData[0]);
                assert(vrfData[1]);
                auto reg_file_vgpr0 = vrfData[0]->template as<VecElemU32>();
                auto reg_file_vgpr1 = vrfData[1]->template as<VecElemU32>();
                auto vgpr = vecReg.template as<VecElemU64>();
 
                for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
                    if (exec_mask[lane] || _gpuDynInst->ignoreExec()) {
                        reg_file_vgpr0[lane] = ((VecElemU32*)&vgpr[lane])[0];
                        reg_file_vgpr1[lane] = ((VecElemU32*)&vgpr[lane])[1];
                    }
                }
 
                DPRINTF(GPUVRF, "Write v[%d:%d]\n", vgprIdx0, vgprIdx1);
                DPRINTF(GPUTrace, "wave[%d] Write v[%d:%d] by instruction "
                        "%s\n", wf->wfDynId, vgprIdx0, vgprIdx1,
                        _gpuDynInst->disassemble().c_str());
                cu->vrf[wf->simdId]->printReg(wf, vgprIdx0);
                cu->vrf[wf->simdId]->printReg(wf, vgprIdx1);
            }
        }
 
        void
        negModifier()
        {
            negMod = true;
        }
 
        void
        absModifier()
        {
            absMod = true;
        }

        template<bool Condition = (NumDwords == 1 || NumDwords == 2) && Const>
        typename std::enable_if<Condition, const DataType>::type
        operator[](size_t idx) const
        {
            assert(idx < NumVecElemPerVecReg);
 
            if (scalar) {
                DataType ret_val = scRegData.rawData();
 
                if (absMod) {
                    assert(std::is_floating_point_v<DataType>);
                    ret_val = std::fabs(ret_val);
                }
 
                if (negMod) {
                    assert(std::is_floating_point_v<DataType>);
                    ret_val = -ret_val;
                }
 
                return ret_val;
            } else {
                auto vgpr = vecReg.template as<DataType>();
                DataType ret_val = vgpr[idx];
 
                if (absMod) {
                    assert(std::is_floating_point_v<DataType>);
                    ret_val = std::fabs(ret_val);
                }
 
                if (negMod) {
                    assert(std::is_floating_point_v<DataType>);
                    ret_val = -ret_val;
                }
 
                return ret_val;
            }
        }

        template<bool Condition = (NumDwords == 1 || NumDwords == 2) && !Const>
        typename std::enable_if<Condition, DataType&>::type
        operator[](size_t idx)
        {
            assert(!scalar);
            assert(idx < NumVecElemPerVecReg);
 
            return vecReg.template as<DataType>()[idx];
        }
 
        private:
          void
          readScalar()
          {
              scalar = true;
              scRegData.read();
          }
 
          using VecRegCont =
              VecRegContainer<sizeof(DataType) * NumVecElemPerVecReg>;

          bool scalar;
          bool absMod;
          bool negMod;
          VecRegCont vecReg;
          ScalarOperand<DataType, Const, NumDwords> scRegData;
          std::array<VecRegContainerU32*, NumDwords> vrfData;
    };
 
    template<typename DataType, bool Const,
        size_t NumDwords = sizeof(DataType) / sizeof(ScalarRegU32)>
    class ScalarOperand final : public Operand
    {
      static_assert(NumDwords >= 1 && NumDwords <= MaxOperandDwords,
            "Incorrect number of DWORDS for VEGA operand.");
      public:
        ScalarOperand() = delete;
 
        ScalarOperand(GPUDynInstPtr gpuDynInst, int opIdx)
            : Operand(gpuDynInst, opIdx)
        {
            std::memset(srfData.data(), 0, NumDwords * sizeof(ScalarRegU32));
        }
 
        ~ScalarOperand()
        {
        }

        template<bool Condition = NumDwords == 1 || NumDwords == 2>
        typename std::enable_if<Condition, DataType>::type
        rawData() const
        {
            assert(sizeof(DataType) <= sizeof(srfData));
            DataType raw_data((DataType)0);
            std::memcpy((void*)&raw_data, (void*)srfData.data(),
                sizeof(DataType));
 
            return raw_data;
        }
 
        void*
        rawDataPtr()
        {
            return (void*)srfData.data();
        }
 
        void
        read() override
        {
            Wavefront *wf = _gpuDynInst->wavefront();
            ComputeUnit *cu = _gpuDynInst->computeUnit();
 
            if (!isScalarReg(_opIdx)) {
                readSpecialVal();
            } else {
                for (auto i = 0; i < NumDwords; ++i) {
                    int sgprIdx = regIdx(i);
                    srfData[i] = cu->srf[wf->simdId]->read(sgprIdx);
                    DPRINTF(GPUSRF, "Read s[%d]\n", sgprIdx);
                    DPRINTF(GPUTrace, "wave[%d] Read s[%d] by instruction "
                            "%s\n", wf->wfDynId, sgprIdx,
                            _gpuDynInst->disassemble().c_str());
                    cu->srf[wf->simdId]->printReg(wf, sgprIdx);
                }
            }
        }
 
        void
        write() override
        {
            Wavefront *wf = _gpuDynInst->wavefront();
            ComputeUnit *cu = _gpuDynInst->computeUnit();
 
            if (!isScalarReg(_opIdx)) {
                if (_opIdx == REG_EXEC_LO) {
                    ScalarRegU64 new_exec_mask_val
                        = wf->execMask().to_ullong();
                    if (NumDwords == 1) {
                        std::memcpy((void*)&new_exec_mask_val,
                            (void*)srfData.data(), sizeof(VecElemU32));
                    } else if (NumDwords == 2) {
                        std::memcpy((void*)&new_exec_mask_val,
                            (void*)srfData.data(), sizeof(VecElemU64));
                    } else {
                        panic("Trying to write more than 2 DWORDS to EXEC\n");
                    }
                    VectorMask new_exec_mask(new_exec_mask_val);
                    wf->execMask() = new_exec_mask;
                    DPRINTF(GPUSRF, "Write EXEC\n");
                    DPRINTF(GPUSRF, "EXEC = %#x\n", new_exec_mask_val);
                } else if (_opIdx == REG_EXEC_HI) {
                    assert(NumDwords == 1);
                    ScalarRegU32 new_exec_mask_hi_val(0);
                    ScalarRegU64 new_exec_mask_val
                        = wf->execMask().to_ullong();
                    std::memcpy((void*)&new_exec_mask_hi_val,
                        (void*)srfData.data(), sizeof(new_exec_mask_hi_val));
                    replaceBits(new_exec_mask_val, 63, 32,
                                new_exec_mask_hi_val);
                    VectorMask new_exec_mask(new_exec_mask_val);
                    wf->execMask() = new_exec_mask;
                    DPRINTF(GPUSRF, "Write EXEC\n");
                    DPRINTF(GPUSRF, "EXEC = %#x\n", new_exec_mask_val);
                } else {
                    _gpuDynInst->writeMiscReg(_opIdx, srfData[0]);
                }
            } else {
                for (auto i = 0; i < NumDwords; ++i) {
                    int sgprIdx = regIdx(i);
                    auto &sgpr = cu->srf[wf->simdId]->readWriteable(sgprIdx);
                    if (_gpuDynInst->isLoad()) {
                        assert(sizeof(DataType) <= sizeof(ScalarRegU64));
                        sgpr = reinterpret_cast<ScalarRegU32*>(
                            _gpuDynInst->scalar_data)[i];
                    } else {
                        sgpr = srfData[i];
                    }
                    DPRINTF(GPUSRF, "Write s[%d]\n", sgprIdx);
                    DPRINTF(GPUTrace, "wave[%d] Write s[%d] by instruction "
                            "%s\n", wf->wfDynId, sgprIdx,
                            _gpuDynInst->disassemble().c_str());
                    cu->srf[wf->simdId]->printReg(wf, sgprIdx);
                }
            }
        }

        template<bool Condition = NumDwords == 1 || NumDwords == 2>
        typename std::enable_if<Condition, void>::type
        setBit(int bit, int bit_val)
        {
            GEM5_ALIGNED(8) DataType &sgpr = *((DataType*)srfData.data());
            replaceBits(sgpr, bit, bit_val);
        }
 
        template<bool Condition = (NumDwords == 1 || NumDwords == 2) && !Const>
        typename std::enable_if<Condition, ScalarOperand&>::type
        operator=(DataType rhs)
        {
            std::memcpy((void*)srfData.data(), (void*)&rhs, sizeof(DataType));
            return *this;
        }
 
      private:
        void
        readSpecialVal()
        {
            assert(NumDwords == 1 || NumDwords == 2);
 
            if (_opIdx >= REG_INT_CONST_POS_MIN &&
                _opIdx <= REG_INT_CONST_NEG_MAX) {
                assert(sizeof(DataType) <= sizeof(srfData));
                DataType misc_val(0);
                assert(isConstVal(_opIdx));
                misc_val = (DataType)_gpuDynInst
                    ->readConstVal<DataType>(_opIdx);
                std::memcpy((void*)srfData.data(), (void*)&misc_val,
                            sizeof(DataType));
 
                return;
            }
 
            if (_opIdx == REG_M0 || _opIdx == REG_ZERO || _opIdx == REG_SCC) {
                assert(sizeof(DataType) <= sizeof(srfData));
                DataType misc_val(0);
                misc_val = (DataType)_gpuDynInst->readMiscReg(_opIdx);
                std::memcpy((void*)srfData.data(), (void*)&misc_val,
                            sizeof(DataType));
 
                return;
            }
 
            switch(_opIdx) {
              case REG_EXEC_LO:
                {
                    if constexpr (NumDwords == 2) {
                        ScalarRegU64 exec_mask = _gpuDynInst->wavefront()->
                            execMask().to_ullong();
                        std::memcpy((void*)srfData.data(), (void*)&exec_mask,
                            sizeof(exec_mask));
                        DPRINTF(GPUSRF, "Read EXEC\n");
                        DPRINTF(GPUSRF, "EXEC = %#x\n", exec_mask);
                    } else {
                        ScalarRegU64 exec_mask = _gpuDynInst->wavefront()->
                            execMask().to_ullong();
 
                        ScalarRegU32 exec_mask_lo = bits(exec_mask, 31, 0);
                        std::memcpy((void*)srfData.data(),
                            (void*)&exec_mask_lo, sizeof(exec_mask_lo));
                        DPRINTF(GPUSRF, "Read EXEC_LO\n");
                        DPRINTF(GPUSRF, "EXEC_LO = %#x\n", exec_mask_lo);
                    }
                }
                break;
              case REG_EXEC_HI:
                {
                    assert(NumDwords == 1);
                    ScalarRegU64 exec_mask = _gpuDynInst->wavefront()
                        ->execMask().to_ullong();
 
                    ScalarRegU32 exec_mask_hi = bits(exec_mask, 63, 32);
                    std::memcpy((void*)srfData.data(), (void*)&exec_mask_hi,
                                sizeof(exec_mask_hi));
                    DPRINTF(GPUSRF, "Read EXEC_HI\n");
                    DPRINTF(GPUSRF, "EXEC_HI = %#x\n", exec_mask_hi);
                }
                break;
              case REG_SRC_SWDA:
              case REG_SRC_DPP:
              case REG_SRC_LITERAL:
                srfData[0] = _gpuDynInst->srcLiteral();
                if constexpr (NumDwords == 2) {
                    if constexpr (std::is_integral_v<DataType>) {
                        if constexpr (std::is_signed_v<DataType>) {
                            if (bits(srfData[0], 31, 31) == 1) {
                                srfData[1] = 0xffffffff;
                            } else {
                                srfData[1] = 0;
                            }
                        } else {
                            srfData[1] = 0;
                        }
                    } else {
                        srfData[1] = _gpuDynInst->srcLiteral();
                        srfData[0] = 0;
                    }
                }
                break;
              case REG_SHARED_BASE:
                {
                    assert(NumDwords == 2);
                    if constexpr (NumDwords == 2) {
                        ComputeUnit *cu = _gpuDynInst->computeUnit();
                        ScalarRegU64 shared_base = cu->shader->ldsApe().base;
                        std::memcpy((void*)srfData.data(), (void*)&shared_base,
                                sizeof(srfData));
                        DPRINTF(GPUSRF, "Read SHARED_BASE = %#x\n",
                                shared_base);
                    }
                }
                break;
              case REG_SHARED_LIMIT:
                {
                    assert(NumDwords == 2);
                    if constexpr (NumDwords == 2) {
                        ComputeUnit *cu = _gpuDynInst->computeUnit();
                        ScalarRegU64 shared_limit = cu->shader->ldsApe().limit;
                        std::memcpy((void*)srfData.data(),
                                (void*)&shared_limit, sizeof(srfData));
                        DPRINTF(GPUSRF, "Read SHARED_LIMIT = %#x\n",
                                shared_limit);
                    }
                }
                break;
              case REG_PRIVATE_BASE:
                {
                    assert(NumDwords == 2);
                    if constexpr (NumDwords == 2) {
                        ComputeUnit *cu = _gpuDynInst->computeUnit();
                        ScalarRegU64 priv_base = cu->shader->scratchApe().base;
                        std::memcpy((void*)srfData.data(), (void*)&priv_base,
                                sizeof(srfData));
                        DPRINTF(GPUSRF, "Read PRIVATE_BASE = %#x\n",
                                priv_base);
                    }
                }
                break;
              case REG_PRIVATE_LIMIT:
                {
                    assert(NumDwords == 2);
                    if constexpr (NumDwords == 2) {
                        ComputeUnit *cu = _gpuDynInst->computeUnit();
                        ScalarRegU64 priv_limit =
                            cu->shader->scratchApe().limit;
                        std::memcpy((void*)srfData.data(), (void*)&priv_limit,
                                sizeof(srfData));
                        DPRINTF(GPUSRF, "Read PRIVATE_LIMIT = %#x\n",
                                priv_limit);
                    }
                }
                break;
              case REG_POS_HALF:
                {
                    typename OpTraits<DataType>::FloatT pos_half = 0.5;
                    std::memcpy((void*)srfData.data(), (void*)&pos_half,
                        sizeof(pos_half));
 
                }
                break;
              case REG_NEG_HALF:
                {
                    typename OpTraits<DataType>::FloatT neg_half = -0.5;
                    std::memcpy((void*)srfData.data(), (void*)&neg_half,
                        sizeof(neg_half));
                }
                break;
              case REG_POS_ONE:
                {
                    typename OpTraits<DataType>::FloatT pos_one = 1.0;
                    std::memcpy(srfData.data(), &pos_one, sizeof(pos_one));
                }
                break;
              case REG_NEG_ONE:
                {
                    typename OpTraits<DataType>::FloatT neg_one = -1.0;
                    std::memcpy(srfData.data(), &neg_one, sizeof(neg_one));
                }
                break;
              case REG_POS_TWO:
                {
                    typename OpTraits<DataType>::FloatT pos_two = 2.0;
                    std::memcpy(srfData.data(), &pos_two, sizeof(pos_two));
                }
                break;
              case REG_NEG_TWO:
                {
                    typename OpTraits<DataType>::FloatT neg_two = -2.0;
                    std::memcpy(srfData.data(), &neg_two, sizeof(neg_two));
                }
                break;
              case REG_POS_FOUR:
                {
                    typename OpTraits<DataType>::FloatT pos_four = 4.0;
                    std::memcpy(srfData.data(), &pos_four, sizeof(pos_four));
                }
                break;
              case REG_NEG_FOUR:
                {
                    typename OpTraits<DataType>::FloatT neg_four = -4.0;
                    std::memcpy((void*)srfData.data(), (void*)&neg_four ,
                        sizeof(neg_four));
                }
                break;
                case REG_PI:
                {
                    assert(sizeof(DataType) == sizeof(ScalarRegF64)
                        || sizeof(DataType) == sizeof(ScalarRegF32));
 
                    const ScalarRegU32 pi_u32(0x3e22f983UL);
                    const ScalarRegU64 pi_u64(0x3fc45f306dc9c882ULL);
 
                    if (sizeof(DataType) == sizeof(ScalarRegF64)) {
                        std::memcpy((void*)srfData.data(),
                            (void*)&pi_u64, sizeof(pi_u64));
                    } else {
                        std::memcpy((void*)srfData.data(),
                            (void*)&pi_u32, sizeof(pi_u32));
                    }
                }
                break;
              default:
                panic("Invalid special register index: %d\n", _opIdx);
                break;
            }
        }

        int
        regIdx(int dword) const
        {
            Wavefront *wf = _gpuDynInst->wavefront();
            ComputeUnit *cu = _gpuDynInst->computeUnit();
            int sgprIdx(-1);
 
            if (_opIdx == REG_VCC_HI) {
                sgprIdx = cu->registerManager
                    ->mapSgpr(wf, wf->reservedScalarRegs - 1 + dword);
            } else if (_opIdx == REG_VCC_LO) {
                sgprIdx = cu->registerManager
                    ->mapSgpr(wf, wf->reservedScalarRegs - 2 + dword);
            } else if (_opIdx == REG_FLAT_SCRATCH_HI) {
                sgprIdx = cu->registerManager
                    ->mapSgpr(wf, wf->reservedScalarRegs - 3 + dword);
            } else if (_opIdx == REG_FLAT_SCRATCH_LO) {
                assert(NumDwords == 1);
                sgprIdx = cu->registerManager
                    ->mapSgpr(wf, wf->reservedScalarRegs - 4 + dword);
            } else {
                sgprIdx = cu->registerManager->mapSgpr(wf, _opIdx + dword);
            }
 
            assert(sgprIdx > -1);
 
            return sgprIdx;
        }

        GEM5_ALIGNED(8) std::array<ScalarRegU32, NumDwords> srfData;
    };
 
    // typedefs for the various sizes/types of scalar operands
    using ScalarOperandU8 = ScalarOperand<ScalarRegU8, false, 1>;
    using ScalarOperandI8 = ScalarOperand<ScalarRegI8, false, 1>;
    using ScalarOperandU16 = ScalarOperand<ScalarRegU16, false, 1>;
    using ScalarOperandI16 = ScalarOperand<ScalarRegI16, false, 1>;
    using ScalarOperandU32 = ScalarOperand<ScalarRegU32, false>;
    using ScalarOperandI32 = ScalarOperand<ScalarRegI32, false>;
    using ScalarOperandF32 = ScalarOperand<ScalarRegF32, false>;
    using ScalarOperandU64 = ScalarOperand<ScalarRegU64, false>;
    using ScalarOperandI64 = ScalarOperand<ScalarRegI64, false>;
    using ScalarOperandF64 = ScalarOperand<ScalarRegF64, false>;
    using ScalarOperandU128 = ScalarOperand<ScalarRegU32, false, 4>;
    using ScalarOperandU256 = ScalarOperand<ScalarRegU32, false, 8>;
    using ScalarOperandU512 = ScalarOperand<ScalarRegU32, false, 16>;
    // non-writeable versions of scalar operands
    using ConstScalarOperandU8 = ScalarOperand<ScalarRegU8, true, 1>;
    using ConstScalarOperandI8 = ScalarOperand<ScalarRegI8, true, 1>;
    using ConstScalarOperandU16 = ScalarOperand<ScalarRegU16, true, 1>;
    using ConstScalarOperandI16 = ScalarOperand<ScalarRegI16, true, 1>;
    using ConstScalarOperandU32 = ScalarOperand<ScalarRegU32, true>;
    using ConstScalarOperandI32 = ScalarOperand<ScalarRegI32, true>;
    using ConstScalarOperandF32 = ScalarOperand<ScalarRegF32, true>;
    using ConstScalarOperandU64 = ScalarOperand<ScalarRegU64, true>;
    using ConstScalarOperandI64 = ScalarOperand<ScalarRegI64, true>;
    using ConstScalarOperandF64 = ScalarOperand<ScalarRegF64, true>;
    using ConstScalarOperandU128 = ScalarOperand<ScalarRegU32, true, 4>;
    using ConstScalarOperandU256 = ScalarOperand<ScalarRegU32, true, 8>;
    using ConstScalarOperandU512 = ScalarOperand<ScalarRegU32, true, 16>;
    // typedefs for the various sizes/types of vector operands
    using VecOperandU8 = VecOperand<VecElemU8, false, 1>;
    using VecOperandI8 = VecOperand<VecElemI8, false, 1>;
    using VecOperandU16 = VecOperand<VecElemU16, false, 1>;
    using VecOperandI16 = VecOperand<VecElemI16, false, 1>;
    using VecOperandU32 = VecOperand<VecElemU32, false>;
    using VecOperandI32 = VecOperand<VecElemI32, false>;
    using VecOperandF32 = VecOperand<VecElemF32, false>;
    using VecOperandU64 = VecOperand<VecElemU64, false>;
    using VecOperandF64 = VecOperand<VecElemF64, false>;
    using VecOperandI64 = VecOperand<VecElemI64, false>;
    using VecOperandU96 = VecOperand<VecElemU32, false, 3>;
    using VecOperandU128 = VecOperand<VecElemU32, false, 4>;
    using VecOperandU256 = VecOperand<VecElemU32, false, 8>;
    using VecOperandU512 = VecOperand<VecElemU32, false, 16>;
    // non-writeable versions of vector operands
    using ConstVecOperandU8 = VecOperand<VecElemU8, true, 1>;
    using ConstVecOperandI8 = VecOperand<VecElemI8, true, 1>;
    using ConstVecOperandU16 = VecOperand<VecElemU16, true, 1>;
    using ConstVecOperandI16 = VecOperand<VecElemI16, true, 1>;
    using ConstVecOperandU32 = VecOperand<VecElemU32, true>;
    using ConstVecOperandI32 = VecOperand<VecElemI32, true>;
    using ConstVecOperandF32 = VecOperand<VecElemF32, true>;
    using ConstVecOperandU64 = VecOperand<VecElemU64, true>;
    using ConstVecOperandI64 = VecOperand<VecElemI64, true>;
    using ConstVecOperandF64 = VecOperand<VecElemF64, true>;
    using ConstVecOperandU96 = VecOperand<VecElemU32, true, 3>;
    using ConstVecOperandU128 = VecOperand<VecElemU32, true, 4>;
    using ConstVecOperandU256 = VecOperand<VecElemU32, true, 8>;
    using ConstVecOperandU512 = VecOperand<VecElemU32, true, 16>;
 
 
// Helper class for using multiple VecElemU32 to represent data types which
// do not divide a dword evenly.
template<int BITS, int ELEM_SIZE>
class PackedReg
{
    // Logical view is:
    // dword N, dword N - 1, ..., dword 1, dword 0.
    // Within each dword, the element starts at [ELEM_SIZE:0]. For example,
    // for ELEM_SIZE = 6 for fp6 types, [5:0] is the first value, [11:6] is
    // the second, and so forth. For 6 bits specifically, the 6th element
    // spans dword 0 and dword 1.
    static_assert(BITS % 32 == 0);
    static_assert(BITS % ELEM_SIZE == 0);
    static_assert(ELEM_SIZE <= 32);
 
    static constexpr int NumDwords = BITS / 32;
    uint32_t dwords[NumDwords] = {};
 
  public:
    PackedReg() = default;
 
    void
    setDword(int dw, uint32_t value)
    {
        assert(dw < NumDwords);
        dwords[dw] = value;
    }
 
    uint32_t
    getDword(int dw)
    {
        assert(dw < NumDwords);
        return dwords[dw];
    }
 
    uint32_t
    getElem(int elem)
    {
        assert(elem < (BITS / ELEM_SIZE));
 
        // Get the upper/lower *bit* location of the element.
        int ubit, lbit;
        ubit = elem * ELEM_SIZE + (ELEM_SIZE - 1);
        lbit = elem * ELEM_SIZE;
 
        // Convert the bit locations to upper/lower dwords. It is possible
        // to span two dwords but this does not have to support spanning
        // more than two dwords.
        int udw, ldw;
        udw = ubit / 32;
        ldw = lbit / 32;
        assert(udw == ldw || udw == ldw + 1);
 
        if (udw == ldw) {
            // Easy case, just shift the dword value and mask to get value.
            int dw_lbit = lbit % 32;
 
            uint32_t elem_mask = (1UL << ELEM_SIZE) - 1;
            uint32_t rv = (dwords[ldw] >> dw_lbit) & elem_mask;
 
            return rv;
        }
 
        // Harder case. To make it easier put into a quad word and shift
        // that variable instead of trying to work with two.
        uint64_t qword =
            uint64_t(dwords[udw]) << 32 | uint64_t(dwords[ldw]);
 
        int qw_lbit = lbit % 32;
 
        uint64_t elem_mask = (1ULL << ELEM_SIZE) - 1;
        uint32_t rv = uint32_t((qword >> qw_lbit) & elem_mask);
 
        return rv;
    }
 
    void
    setElem(int elem, uint32_t value)
    {
        assert(elem < (BITS / ELEM_SIZE));
 
        // Get the upper/lower *bit* location of the element.
        int ubit, lbit;
        ubit = elem * ELEM_SIZE + (ELEM_SIZE - 1);
        lbit = elem * ELEM_SIZE;
 
        // Convert the bit locations to upper/lower dwords. It is possible
        // to span two dwords but this does not have to support spanning
        // more than two dwords.
        int udw, ldw;
        udw = ubit / 32;
        ldw = lbit / 32;
        assert(udw == ldw || udw == ldw + 1);
 
        if (udw == ldw) {
            // Easy case, just shift the dword value and mask to get value.
            int dw_lbit = lbit % 32;
 
            // Make sure the value is not going to clobber another element.
            uint32_t elem_mask = (1UL << ELEM_SIZE) - 1;
            value &= elem_mask;
 
            // Clear the bits we are setting.
            elem_mask <<= dw_lbit;
            dwords[ldw] &= ~elem_mask;
 
            value <<= dw_lbit;
            dwords[ldw] |= value;
 
            return;
        }
 
        // Harder case. Put the two dwords in a quad word and manipulate that.
        // Then place the two new dwords back into the storage.
        uint64_t qword =
            uint64_t(dwords[udw]) << 32 | uint64_t(dwords[ldw]);
 
        int qw_lbit = lbit % 32;
 
        // Make sure the value is not going to clobber another element.
        uint64_t elem_mask = (1ULL << ELEM_SIZE) - 1;
        value &= elem_mask;
 
        // Clear the bits where the value goes so that operator| can be used.
        elem_mask <<= qw_lbit;
        qword &= ~elem_mask;
 
        // Promote to 64-bit to prevent shifting out of range
        uint64_t value64 = value;
        value64 <<= qw_lbit;
        qword |= value64;
 
        dwords[udw] = uint32_t(qword >> 32);
        dwords[ldw] = uint32_t(qword & mask(32));
    }
};
 
}
 
} // namespace gem5
 
#endif // __ARCH_VEGA_OPERAND_HH__