gpu_compute_driver.cc

Go to the documentation of this file.

/*
 * Copyright (c) 2015-2018 Advanced Micro Devices, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
 
#include "gpu-compute/gpu_compute_driver.hh"
 
#include <memory>
 
#include "arch/x86/page_size.hh"
#include "base/compiler.hh"
#include "base/logging.hh"
#include "base/trace.hh"
#include "cpu/thread_context.hh"
#include "debug/GPUDriver.hh"
#include "debug/GPUShader.hh"
#include "dev/hsa/hsa_packet_processor.hh"
#include "dev/hsa/kfd_event_defines.h"
#include "dev/hsa/kfd_ioctl.h"
#include "gpu-compute/gpu_command_processor.hh"
#include "gpu-compute/shader.hh"
#include "mem/port_proxy.hh"
#include "mem/se_translating_port_proxy.hh"
#include "mem/translating_port_proxy.hh"
#include "params/GPUComputeDriver.hh"
#include "sim/full_system.hh"
#include "sim/process.hh"
#include "sim/se_workload.hh"
#include "sim/syscall_emul_buf.hh"
 
namespace gem5
{
 
GPUComputeDriver::GPUComputeDriver(const Params &p)
    : EmulatedDriver(p), device(p.device), queueId(0),
      isdGPU(p.isdGPU), gfxVersion(p.gfxVersion), dGPUPoolID(p.dGPUPoolID),
      eventPage(0), eventSlotIndex(0)
{
    device->attachDriver(this);
    DPRINTF(GPUDriver, "Constructing KFD: device\n");
 
    // Convert the 3 bit mtype specified in Shader.py to the proper type
    // used for requests.
    if (MtypeFlags::SHARED & p.m_type)
        defaultMtype.set(Request::SHARED);
 
    if (MtypeFlags::READ_WRITE & p.m_type)
        defaultMtype.set(Request::READ_WRITE);
 
    if (MtypeFlags::CACHED & p.m_type)
        defaultMtype.set(Request::CACHED);
}
 
const char*
GPUComputeDriver::DriverWakeupEvent::description() const
{
    return "DriverWakeupEvent";
}
 
int
GPUComputeDriver::open(ThreadContext *tc, int mode, int flags)
{
    DPRINTF(GPUDriver, "Opened %s\n", filename);
    auto process = tc->getProcessPtr();
    auto device_fd_entry = std::make_shared<DeviceFDEntry>(this, filename);
    int tgt_fd = process->fds->allocFD(device_fd_entry);
    return tgt_fd;
}
 
Addr
GPUComputeDriver::mmap(ThreadContext *tc, Addr start, uint64_t length,
                       int prot, int tgt_flags, int tgt_fd, off_t offset)
{
    auto process = tc->getProcessPtr();
    auto mem_state = process->memState;
 
    Addr pg_off = offset >> PAGE_SHIFT;
    Addr mmap_type = pg_off & KFD_MMAP_TYPE_MASK;
    DPRINTF(GPUDriver, "amdkfd mmap (start: %p, length: 0x%x,"
            "offset: 0x%x)\n", start, length, offset);
 
    switch(mmap_type) {
        case KFD_MMAP_TYPE_DOORBELL:
            DPRINTF(GPUDriver, "amdkfd mmap type DOORBELL offset\n");
            start = mem_state->extendMmap(length);
            process->pTable->map(start, device->hsaPacketProc().pioAddr,
                    length, false);
            break;
        case KFD_MMAP_TYPE_EVENTS:
            DPRINTF(GPUDriver, "amdkfd mmap type EVENTS offset\n");
            panic_if(start != 0,
                     "Start address should be provided by KFD\n");
            panic_if(length != 8 * KFD_SIGNAL_EVENT_LIMIT,
                     "Requested length %d, expected length %d; length "
                     "mismatch\n", length, 8* KFD_SIGNAL_EVENT_LIMIT);
            if (!eventPage) {
                eventPage = mem_state->extendMmap(length);
                start = eventPage;
            }
            break;
        default:
            warn_once("Unrecognized kfd mmap type %llx\n", mmap_type);
            break;
    }
 
    return start;
}
 
void
GPUComputeDriver::allocateQueue(PortProxy &mem_proxy, Addr ioc_buf)
{
    TypedBufferArg<kfd_ioctl_create_queue_args> args(ioc_buf);
    args.copyIn(mem_proxy);
 
    if ((doorbellSize() * queueId) > 4096) {
        fatal("%s: Exceeded maximum number of HSA queues allowed\n", name());
    }
 
    args->doorbell_offset = (KFD_MMAP_TYPE_DOORBELL |
        KFD_MMAP_GPU_ID(args->gpu_id)) << PAGE_SHIFT;
 
    // for vega offset needs to include exact value of doorbell
    if (doorbellSize())
        args->doorbell_offset += queueId * doorbellSize();
 
    args->queue_id = queueId++;
    auto &hsa_pp = device->hsaPacketProc();
    hsa_pp.setDeviceQueueDesc(args->read_pointer_address,
                              args->ring_base_address, args->queue_id,
                              args->ring_size, doorbellSize(), gfxVersion);
    args.copyOut(mem_proxy);
}
 
void
GPUComputeDriver::DriverWakeupEvent::scheduleWakeup(Tick wakeup_delay)
{
    assert(driver);
    driver->schedule(this, curTick() + wakeup_delay);
}
 
void
GPUComputeDriver::signalWakeupEvent(uint32_t event_id)
{
    panic_if(event_id >= eventSlotIndex,
        "Trying wakeup on an event that is not yet created\n");
    if (ETable[event_id].threadWaiting) {
        panic_if(!ETable[event_id].tc,
                 "No thread context to wake up\n");
        ThreadContext *tc = ETable[event_id].tc;
        DPRINTF(GPUDriver,
                "Signal event: Waking up CPU %d\n", tc->cpuId());
        // Remove events that can wakeup this thread
        TCEvents[tc].clearEvents();
        // Now wakeup this thread
        tc->activate();
    } else {
       // This may be a race condition between an ioctl call asking to wait on
       // this event and this signalWakeupEvent. Taking care of this race
       // condition here by setting the event here. The ioctl call should take
       // the necessary action when waiting on an already set event.  However,
       // this may be a genuine instance in which the runtime has decided not
       // to wait on this event. But since we cannot distinguish this case with
       // the race condition, we are any way setting the event.
       ETable[event_id].setEvent = true;
    }
}
 
void
GPUComputeDriver::DriverWakeupEvent::process()
{
    DPRINTF(GPUDriver,
            "Timer event: Waking up CPU %d\n", tc->cpuId());
    // Remove events that can wakeup this thread
    driver->TCEvents[tc].clearEvents();
    // Now wakeup this thread
    tc->activate();
}
 
int
GPUComputeDriver::ioctl(ThreadContext *tc, unsigned req, Addr ioc_buf)
{
    TranslatingPortProxy fs_proxy(tc);
    SETranslatingPortProxy se_proxy(tc);
    PortProxy &virt_proxy = FullSystem ? fs_proxy : se_proxy;
    auto process = tc->getProcessPtr();
    auto mem_state = process->memState;
 
    switch (req) {
        case AMDKFD_IOC_GET_VERSION:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_GET_VERSION\n");
 
            TypedBufferArg<kfd_ioctl_get_version_args> args(ioc_buf);
            args->major_version = KFD_IOCTL_MAJOR_VERSION;
            args->minor_version = KFD_IOCTL_MINOR_VERSION;
 
            args.copyOut(virt_proxy);
          }
          break;
        case AMDKFD_IOC_CREATE_QUEUE:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_CREATE_QUEUE\n");
 
            allocateQueue(virt_proxy, ioc_buf);
 
            DPRINTF(GPUDriver, "Creating queue %d\n", queueId);
          }
          break;
        case AMDKFD_IOC_DESTROY_QUEUE:
          {
            TypedBufferArg<kfd_ioctl_destroy_queue_args> args(ioc_buf);
            args.copyIn(virt_proxy);
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_DESTROY_QUEUE;" \
                    "queue offset %d\n", args->queue_id);
            device->hsaPacketProc().unsetDeviceQueueDesc(args->queue_id,
                                                         doorbellSize());
          }
          break;
        case AMDKFD_IOC_SET_MEMORY_POLICY:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_SET_MEMORY_POLICY\n");
          }
          break;
        case AMDKFD_IOC_GET_CLOCK_COUNTERS:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_GET_CLOCK_COUNTERS\n");
 
            TypedBufferArg<kfd_ioctl_get_clock_counters_args> args(ioc_buf);
            args.copyIn(virt_proxy);
 
            // Set nanosecond resolution
            args->system_clock_freq = 1000000000;
 
            uint64_t elapsed_nsec = curTick() / sim_clock::as_int::ns;
            args->gpu_clock_counter = elapsed_nsec;
            args->cpu_clock_counter = elapsed_nsec;
            args->system_clock_counter = elapsed_nsec;
 
            args.copyOut(virt_proxy);
          }
          break;
        case AMDKFD_IOC_GET_PROCESS_APERTURES:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_GET_PROCESS_APERTURES\n");
 
            TypedBufferArg<kfd_ioctl_get_process_apertures_args> args(ioc_buf);
            args->num_of_nodes = 1;
 
            for (int i = 0; i < args->num_of_nodes; ++i) {
                switch (gfxVersion) {
                  case GfxVersion::gfx801:
                  case GfxVersion::gfx803:
                    args->process_apertures[i].scratch_base =
                        scratchApeBase(i + 1);
                    args->process_apertures[i].lds_base =
                        ldsApeBase(i + 1);
                    break;
                  case GfxVersion::gfx900:
                    args->process_apertures[i].scratch_base =
                        scratchApeBaseV9();
                    args->process_apertures[i].lds_base =
                        ldsApeBaseV9();
                    break;
                  default:
                    fatal("Invalid gfx version\n");
                }
 
                // GFX8 and GFX9 set lds and scratch limits the same way
                args->process_apertures[i].scratch_limit =
                    scratchApeLimit(args->process_apertures[i].scratch_base);
 
                args->process_apertures[i].lds_limit =
                    ldsApeLimit(args->process_apertures[i].lds_base);
 
                switch (gfxVersion) {
                  case GfxVersion::gfx801:
                    args->process_apertures[i].gpuvm_base =
                        gpuVmApeBase(i + 1);
                    args->process_apertures[i].gpuvm_limit =
                        gpuVmApeLimit(args->process_apertures[i].gpuvm_base);
                    break;
                  case GfxVersion::gfx803:
                  case GfxVersion::gfx900:
                  case GfxVersion::gfx902:
                    // Taken from SVM_USE_BASE in Linux kernel
                    args->process_apertures[i].gpuvm_base = 0x1000000ull;
                    // Taken from AMDGPU_GMC_HOLE_START in Linux kernel
                    args->process_apertures[i].gpuvm_limit =
                        0x0000800000000000ULL - 1;
                    break;
                  default:
                    fatal("Invalid gfx version");
                }
 
                // NOTE: Must match ID populated by hsaTopology.py
                //
                // https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver/
                // blob/6a986c0943e9acd8c4c0cf2a9d510ff42167b43f/include/uapi/
                // linux/kfd_ioctl.h#L564
                //
                // The gpu_id is a device identifier used by the driver for
                // ioctls that allocate arguments. Each device has an unique
                // id composed out of a non-zero base and an offset.
                if (isdGPU) {
                    switch (gfxVersion) {
                      case GfxVersion::gfx803:
                        args->process_apertures[i].gpu_id = 50156;
                        break;
                      case GfxVersion::gfx900:
                        args->process_apertures[i].gpu_id = 22124;
                        break;
                      default:
                        fatal("Invalid gfx version for dGPU\n");
                    }
                } else {
                    switch (gfxVersion) {
                      case GfxVersion::gfx801:
                      case GfxVersion::gfx902:
                        args->process_apertures[i].gpu_id = 2765;
                        break;
                      default:
                        fatal("Invalid gfx version for APU\n");
                    }
                }
 
                DPRINTF(GPUDriver, "GPUVM base for node[%i] = %#x\n", i,
                        args->process_apertures[i].gpuvm_base);
                DPRINTF(GPUDriver, "GPUVM limit for node[%i] = %#x\n", i,
                        args->process_apertures[i].gpuvm_limit);
 
                DPRINTF(GPUDriver, "LDS base for node[%i] = %#x\n", i,
                        args->process_apertures[i].lds_base);
                DPRINTF(GPUDriver, "LDS limit for node[%i] = %#x\n", i,
                        args->process_apertures[i].lds_limit);
 
                DPRINTF(GPUDriver, "Scratch base for node[%i] = %#x\n", i,
                        args->process_apertures[i].scratch_base);
                DPRINTF(GPUDriver, "Scratch limit for node[%i] = %#x\n", i,
                        args->process_apertures[i].scratch_limit);
 
                assert(bits<Addr>(args->process_apertures[i].scratch_base, 63,
                       47) != 0x1ffff);
                assert(bits<Addr>(args->process_apertures[i].scratch_base, 63,
                       47) != 0);
                assert(bits<Addr>(args->process_apertures[i].scratch_limit, 63,
                       47) != 0x1ffff);
                assert(bits<Addr>(args->process_apertures[i].scratch_limit, 63,
                       47) != 0);
                assert(bits<Addr>(args->process_apertures[i].lds_base, 63,
                       47) != 0x1ffff);
                assert(bits<Addr>(args->process_apertures[i].lds_base, 63,
                       47) != 0);
                assert(bits<Addr>(args->process_apertures[i].lds_limit, 63,
                       47) != 0x1ffff);
                assert(bits<Addr>(args->process_apertures[i].lds_limit, 63,
                       47) != 0);
            }
 
            args.copyOut(virt_proxy);
          }
          break;
        case AMDKFD_IOC_UPDATE_QUEUE:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_UPDATE_QUEUE\n");
          }
          break;
        case AMDKFD_IOC_CREATE_EVENT:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_CREATE_EVENT\n");
 
            TypedBufferArg<kfd_ioctl_create_event_args> args(ioc_buf);
            args.copyIn(virt_proxy);
            if (args->event_type != KFD_IOC_EVENT_SIGNAL) {
                warn("Signal events are only supported currently\n");
            } else if (eventSlotIndex == SLOTS_PER_PAGE) {
                fatal("Signal event wasn't created; signal limit reached\n");
            }
            // Currently, we allocate only one signal_page for events.
            // Note that this signal page is of size 8 * KFD_SIGNAL_EVENT_LIMIT
            uint64_t page_index = 0;
            args->event_page_offset = (page_index | KFD_MMAP_TYPE_EVENTS);
            args->event_page_offset <<= PAGE_SHIFT;
            // TODO: Currently we support only signal events, hence using
            // the same ID for both signal slot and event slot
            args->event_slot_index = eventSlotIndex;
            args->event_id = eventSlotIndex++;
            args->event_trigger_data = args->event_id;
            DPRINTF(GPUDriver, "amdkfd create events"
                    "(event_id: 0x%x, offset: 0x%x)\n",
                    args->event_id, args->event_page_offset);
            // Since eventSlotIndex is increased everytime a new event is
            // created ETable at eventSlotIndex(event_id) is guaranteed to be
            // empty. In a future implementation that reuses deleted event_ids,
            // we should check if event table at this
            // eventSlotIndex(event_id) is empty before inserting a new event
            // table entry
            ETable.emplace(std::pair<uint32_t, ETEntry>(args->event_id, {}));
            args.copyOut(virt_proxy);
          }
          break;
        case AMDKFD_IOC_DESTROY_EVENT:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_DESTROY_EVENT\n");
            TypedBufferArg<kfd_ioctl_destroy_event_args> args(ioc_buf);
            args.copyIn(virt_proxy);
            DPRINTF(GPUDriver, "amdkfd destroying event %d\n", args->event_id);
            fatal_if(ETable.count(args->event_id) == 0,
                     "Event ID invalid, cannot destroy this event\n");
            ETable.erase(args->event_id);
          }
          break;
        case AMDKFD_IOC_SET_EVENT:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_SET_EVENTS\n");
            TypedBufferArg<kfd_ioctl_set_event_args> args(ioc_buf);
            args.copyIn(virt_proxy);
            DPRINTF(GPUDriver, "amdkfd set event %d\n", args->event_id);
            fatal_if(ETable.count(args->event_id) == 0,
                     "Event ID invlaid, cannot set this event\n");
            ETable[args->event_id].setEvent = true;
            signalWakeupEvent(args->event_id);
          }
          break;
        case AMDKFD_IOC_RESET_EVENT:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_RESET_EVENT\n");
          }
          break;
        case AMDKFD_IOC_WAIT_EVENTS:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_WAIT_EVENTS\n");
            TypedBufferArg<kfd_ioctl_wait_events_args> args(ioc_buf);
            args.copyIn(virt_proxy);
            kfd_event_data *events =
                (kfd_event_data *)args->events_ptr;
            DPRINTF(GPUDriver, "amdkfd wait for events"
                    "(wait on all: %d, timeout : %d, num_events: %s)\n",
                    args->wait_for_all, args->timeout, args->num_events);
            panic_if(args->wait_for_all != 0 && args->num_events > 1,
                    "Wait for all events not supported\n");
            bool should_sleep = true;
            if (TCEvents.count(tc) == 0) {
                // This thread context trying to wait on an event for the first
                // time, initialize it.
                TCEvents.emplace(std::piecewise_construct, std::make_tuple(tc),
                                 std::make_tuple(this, tc));
                DPRINTF(GPUDriver, "\tamdkfd creating event list"
                        " for thread  %d\n", tc->cpuId());
            }
            panic_if(TCEvents[tc].signalEvents.size() != 0,
                     "There are %d events that put this thread to sleep,"
                     " this thread should not be running\n",
                     TCEvents[tc].signalEvents.size());
            for (int i = 0; i < args->num_events; i++) {
                panic_if(!events,
                         "Event pointer invalid\n");
                Addr eventDataAddr = (Addr)(events + i);
                TypedBufferArg<kfd_event_data> EventData(
                    eventDataAddr, sizeof(kfd_event_data));
                EventData.copyIn(virt_proxy);
                DPRINTF(GPUDriver,
                        "\tamdkfd wait for event %d\n", EventData->event_id);
                panic_if(ETable.count(EventData->event_id) == 0,
                         "Event ID invalid, cannot set this event\n");
                if (ETable[EventData->event_id].threadWaiting)
                         warn("Multiple threads waiting on the same event\n");
                if (ETable[EventData->event_id].setEvent) {
                    // If event is already set, the event has already happened.
                    // Just unset the event and dont put this thread to sleep.
                    ETable[EventData->event_id].setEvent = false;
                    should_sleep = false;
                }
                if (should_sleep) {
                    // Put this thread to sleep
                    ETable[EventData->event_id].threadWaiting = true;
                    ETable[EventData->event_id].tc = tc;
                    TCEvents[tc].signalEvents.insert(EventData->event_id);
                }
            }
 
            // TODO: Return the correct wait_result back. Currently, returning
            // success for both KFD_WAIT_TIMEOUT and KFD_WAIT_COMPLETE.
            // Ideally, this needs to be done after the event is triggered and
            // after the thread is woken up.
            args->wait_result = 0;
            args.copyOut(virt_proxy);
            if (should_sleep) {
                // Put this thread to sleep
                sleepCPU(tc, args->timeout);
            } else {
                // Remove events that tried to put this thread to sleep
                TCEvents[tc].clearEvents();
            }
          }
          break;
        case AMDKFD_IOC_DBG_REGISTER:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_DBG_REGISTER\n");
          }
          break;
        case AMDKFD_IOC_DBG_UNREGISTER:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_DBG_UNREGISTER\n");
          }
          break;
        case AMDKFD_IOC_DBG_ADDRESS_WATCH:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_DBG_ADDRESS_WATCH\n");
          }
          break;
        case AMDKFD_IOC_DBG_WAVE_CONTROL:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_DBG_WAVE_CONTROL\n");
          }
          break;
        case AMDKFD_IOC_SET_SCRATCH_BACKING_VA:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_SET_SCRATCH_BACKING_VA\n");
          }
          break;
        case AMDKFD_IOC_GET_TILE_CONFIG:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_GET_TILE_CONFIG\n");
          }
          break;
        case AMDKFD_IOC_SET_TRAP_HANDLER:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_SET_TRAP_HANDLER\n");
          }
          break;
        case AMDKFD_IOC_GET_PROCESS_APERTURES_NEW:
          {
            DPRINTF(GPUDriver,
                    "ioctl: AMDKFD_IOC_GET_PROCESS_APERTURES_NEW\n");
 
            TypedBufferArg<kfd_ioctl_get_process_apertures_new_args>
                ioc_args(ioc_buf);
 
            ioc_args.copyIn(virt_proxy);
            ioc_args->num_of_nodes = 1;
 
            for (int i = 0; i < ioc_args->num_of_nodes; ++i) {
                TypedBufferArg<kfd_process_device_apertures> ape_args
                    (ioc_args->kfd_process_device_apertures_ptr);
 
                switch (gfxVersion) {
                  case GfxVersion::gfx801:
                  case GfxVersion::gfx803:
                    ape_args->scratch_base = scratchApeBase(i + 1);
                    ape_args->lds_base = ldsApeBase(i + 1);
                    break;
                  case GfxVersion::gfx900:
                    ape_args->scratch_base = scratchApeBaseV9();
                    ape_args->lds_base = ldsApeBaseV9();
                    break;
                  default:
                    fatal("Invalid gfx version\n");
                }
 
                // GFX8 and GFX9 set lds and scratch limits the same way
                ape_args->scratch_limit =
                    scratchApeLimit(ape_args->scratch_base);
                ape_args->lds_limit = ldsApeLimit(ape_args->lds_base);
 
                switch (gfxVersion) {
                  case GfxVersion::gfx801:
                    ape_args->gpuvm_base = gpuVmApeBase(i + 1);
                    ape_args->gpuvm_limit =
                        gpuVmApeLimit(ape_args->gpuvm_base);
                    break;
                  case GfxVersion::gfx803:
                  case GfxVersion::gfx900:
                  case GfxVersion::gfx902:
                    // Taken from SVM_USE_BASE in Linux kernel
                    ape_args->gpuvm_base = 0x1000000ull;
                    // Taken from AMDGPU_GMC_HOLE_START in Linux kernel
                    ape_args->gpuvm_limit = 0x0000800000000000ULL - 1;
                    break;
                  default:
                    fatal("Invalid gfx version\n");
                }
 
                // NOTE: Must match ID populated by hsaTopology.py
                if (isdGPU) {
                    switch (gfxVersion) {
                      case GfxVersion::gfx803:
                        ape_args->gpu_id = 50156;
                        break;
                      case GfxVersion::gfx900:
                        ape_args->gpu_id = 22124;
                        break;
                      default:
                        fatal("Invalid gfx version for dGPU\n");
                    }
                } else {
                    switch (gfxVersion) {
                      case GfxVersion::gfx801:
                      case GfxVersion::gfx902:
                        ape_args->gpu_id = 2765;
                        break;
                      default:
                        fatal("Invalid gfx version for APU\n");
                    }
                }
 
                assert(bits<Addr>(ape_args->scratch_base, 63, 47) != 0x1ffff);
                assert(bits<Addr>(ape_args->scratch_base, 63, 47) != 0);
                assert(bits<Addr>(ape_args->scratch_limit, 63, 47) != 0x1ffff);
                assert(bits<Addr>(ape_args->scratch_limit, 63, 47) != 0);
                assert(bits<Addr>(ape_args->lds_base, 63, 47) != 0x1ffff);
                assert(bits<Addr>(ape_args->lds_base, 63, 47) != 0);
                assert(bits<Addr>(ape_args->lds_limit, 63, 47) != 0x1ffff);
                assert(bits<Addr>(ape_args->lds_limit, 63, 47) != 0);
 
                ape_args.copyOut(virt_proxy);
            }
 
            ioc_args.copyOut(virt_proxy);
          }
          break;
        case AMDKFD_IOC_ACQUIRE_VM:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_ACQUIRE_VM\n");
          }
          break;
        case AMDKFD_IOC_ALLOC_MEMORY_OF_GPU:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_ALLOC_MEMORY_OF_GPU\n");
            TypedBufferArg<kfd_ioctl_alloc_memory_of_gpu_args> args(ioc_buf);
            args.copyIn(virt_proxy);
 
            assert(isdGPU || gfxVersion == GfxVersion::gfx902);
            assert((args->va_addr % TheISA::PageBytes) == 0);
            [[maybe_unused]] Addr mmap_offset = 0;
 
            Request::CacheCoherenceFlags mtype = defaultMtype;
            Addr pa_addr = 0;
 
            int npages = divCeil(args->size, (int64_t)TheISA::PageBytes);
            bool cacheable = true;
 
            if (KFD_IOC_ALLOC_MEM_FLAGS_VRAM & args->flags) {
                DPRINTF(GPUDriver, "amdkfd allocation type: VRAM\n");
                args->mmap_offset = args->va_addr;
                // VRAM allocations are device memory mapped into GPUVM
                // space.
                //
                // We can't rely on the lazy host allocator (fixupFault) to
                // handle this mapping since it needs to be placed in dGPU
                // framebuffer memory.  The lazy allocator will try to place
                // this in host memory.
                //
                // TODO: We don't have the appropriate bifurcation of the
                // physical address space with different memory controllers
                // yet.  This is where we will explicitly add the PT maps to
                // dGPU memory in the future.
                //
                // Bind the VA space to the dGPU physical memory pool.  Mark
                // this region as Uncacheable.  The Uncacheable flag is only
                // really used by the CPU and is ignored by the GPU. We mark
                // this as uncacheable from the CPU so that we can implement
                // direct CPU framebuffer access similar to what we currently
                // offer in real HW through the so-called Large BAR feature.
                pa_addr = process->seWorkload->allocPhysPages(
                        npages, dGPUPoolID);
                //
                // TODO: Uncacheable accesses need to be supported by the
                // CPU-side protocol for this to work correctly.  I believe
                // it only works right now if the physical memory is MMIO
                cacheable = false;
 
                DPRINTF(GPUDriver, "Mapping VA %p to framebuffer PA %p size "
                        "%d\n", args->va_addr, pa_addr, args->size);
 
            } else if (KFD_IOC_ALLOC_MEM_FLAGS_USERPTR & args->flags) {
                DPRINTF(GPUDriver, "amdkfd allocation type: USERPTR\n");
                mmap_offset = args->mmap_offset;
                // USERPTR allocations are system memory mapped into GPUVM
                // space.  The user provides the driver with the pointer.
                pa_addr = process->seWorkload->allocPhysPages(npages);
 
                DPRINTF(GPUDriver, "Mapping VA %p to framebuffer PA %p size "
                        "%d\n", args->va_addr, pa_addr, args->size);
 
                // If the HSA runtime requests system coherent memory, than we
                // need to explicity mark this region as uncacheable from the
                // perspective of the GPU.
                if (args->flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)
                    mtype.clear();
 
            } else if (KFD_IOC_ALLOC_MEM_FLAGS_GTT & args->flags) {
                DPRINTF(GPUDriver, "amdkfd allocation type: GTT\n");
                args->mmap_offset = args->va_addr;
                // GTT allocations are system memory mapped into GPUVM space.
                // It's different than a USERPTR allocation since the driver
                // itself allocates the physical memory on the host.
                //
                // We will lazily map it into host memory on first touch.  The
                // fixupFault will find the original SVM aperture mapped to the
                // host.
                pa_addr = process->seWorkload->allocPhysPages(npages);
 
                DPRINTF(GPUDriver, "Mapping VA %p to framebuffer PA %p size "
                        "%d\n", args->va_addr, pa_addr, args->size);
 
                // If the HSA runtime requests system coherent memory, than we
                // need to explicity mark this region as uncacheable from the
                // perspective of the GPU.
                if (args->flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)
                    mtype.clear();
 
                // Note that for GTT the thunk layer needs to call mmap on the
                // driver FD later if it wants the host to have access to this
                // memory (which it probably does).  This will be ignored.
            } else if (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL & args->flags) {
                DPRINTF(GPUDriver, "amdkfd allocation type: DOORBELL\n");
                // DOORBELL allocations are the queue doorbells that are
                // memory mapped into GPUVM space.
                //
                // Explicitly map this virtual address to our PIO doorbell
                // interface in the page tables (non-cacheable)
                pa_addr = device->hsaPacketProc().pioAddr;
                cacheable = false;
            }
 
            DPRINTF(GPUDriver, "amdkfd allocation arguments: va_addr %p "
                    "size %lu, mmap_offset %p, gpu_id %d\n",
                    args->va_addr, args->size, mmap_offset, args->gpu_id);
 
            // Bind selected physical memory to provided virtual address range
            // in X86 page tables.
            process->pTable->map(args->va_addr, pa_addr, args->size,
                cacheable);
 
            // We keep track of allocated regions of GPU mapped memory,
            // just like the driver would.  This allows us to provide the
            // user with a unique handle for a given allocation.  The user
            // will only provide us with a handle after allocation and expect
            // us to be able to use said handle to extract all the properties
            // of the region.
            //
            // This is a simplified version of regular system VMAs, but for
            // GPUVM space (none of the clobber/remap nonsense we find in real
            // OS managed memory).
            allocateGpuVma(mtype, args->va_addr, args->size);
 
            // Used by the runtime to uniquely identify this allocation.
            // We can just use the starting address of the VMA region.
            args->handle= args->va_addr;
            args.copyOut(virt_proxy);
          }
          break;
        case AMDKFD_IOC_FREE_MEMORY_OF_GPU:
          {
            DPRINTF(GPUDriver, "ioctl: AMDKFD_IOC_FREE_MEMORY_OF_GPU\n");
            TypedBufferArg<kfd_ioctl_free_memory_of_gpu_args> args(ioc_buf);
            args.copyIn(virt_proxy);
 
            assert(isdGPU);
            DPRINTF(GPUDriver, "amdkfd free arguments: handle %p ",
                    args->handle);
 
            // We don't recycle physical pages in SE mode
            Addr size = deallocateGpuVma(args->handle);
            process->pTable->unmap(args->handle, size);
 
            // TODO: IOMMU and GPUTLBs do not seem to correctly support
            // shootdown.  This is also a potential issue for APU systems
            // that perform unmap or remap with system memory.
            tc->getMMUPtr()->flushAll();
 
            args.copyOut(virt_proxy);
          }
          break;
        case AMDKFD_IOC_MAP_MEMORY_TO_GPU:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_MAP_MEMORY_TO_GPU\n");
          }
          break;
        case AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU\n");
          }
          break;
        case AMDKFD_IOC_SET_CU_MASK:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_SET_CU_MASK\n");
          }
          break;
        case AMDKFD_IOC_GET_QUEUE_WAVE_STATE:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_GET_QUEUE_WAVE_STATE\n");
          }
          break;
        case AMDKFD_IOC_GET_DMABUF_INFO:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_GET_DMABUF_INFO\n");
          }
          break;
        case AMDKFD_IOC_IMPORT_DMABUF:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_IMPORT_DMABUF\n");
          }
          break;
        case AMDKFD_IOC_ALLOC_QUEUE_GWS:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_ALLOC_QUEUE_GWS\n");
          }
          break;
        case AMDKFD_IOC_SMI_EVENTS:
          {
            warn("unimplemented ioctl: AMDKFD_IOC_SMI_EVENTS\n");
          }
          break;
        default:
          fatal("%s: bad ioctl %d\n", req);
          break;
    }
    return 0;
}
 
void
GPUComputeDriver::sleepCPU(ThreadContext *tc, uint32_t milliSecTimeout)
{
    // Convert millisecs to ticks
    Tick wakeup_delay((uint64_t)milliSecTimeout * 1000000000);
    assert(TCEvents.count(tc) == 1);
    TCEvents[tc].timerEvent.scheduleWakeup(wakeup_delay);
    tc->suspend();
    DPRINTF(GPUDriver,
            "CPU %d is put to sleep\n", tc->cpuId());
}
 
Addr
GPUComputeDriver::gpuVmApeBase(int gpuNum) const
{
    return ((Addr)gpuNum << 61) + 0x1000000000000L;
}
 
Addr
GPUComputeDriver::gpuVmApeLimit(Addr apeBase) const
{
    return (apeBase & 0xFFFFFF0000000000UL) | 0xFFFFFFFFFFL;
}
 
Addr
GPUComputeDriver::scratchApeBase(int gpuNum) const
{
    return ((Addr)gpuNum << 61) + 0x100000000L;
}
 
// Used for GFX9 devices
// From drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c in the Linux kernel
Addr
GPUComputeDriver::scratchApeBaseV9() const
{
    return ((Addr)0x1 << 48);
}
 
Addr
GPUComputeDriver::scratchApeLimit(Addr apeBase) const
{
    return (apeBase & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF;
}
 
Addr
GPUComputeDriver::ldsApeBase(int gpuNum) const
{
    return ((Addr)gpuNum << 61) + 0x0;
}
 
//Used for GFX9 devices
// From drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c in the Linux kernel
Addr
GPUComputeDriver::ldsApeBaseV9() const
{
    return ((Addr)0x2 << 48);
}
 
Addr
GPUComputeDriver::ldsApeLimit(Addr apeBase) const
{
    return (apeBase & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF;
}
 
void
GPUComputeDriver::allocateGpuVma(Request::CacheCoherenceFlags mtype,
                                 Addr start, Addr length)
{
    AddrRange range = AddrRange(start, start + length);
    DPRINTF(GPUDriver, "Registering [%p - %p] with MTYPE %d\n",
            range.start(), range.end(), mtype);
    fatal_if(gpuVmas.insert(range, mtype) == gpuVmas.end(),
             "Attempted to double register Mtypes for [%p - %p]\n",
             range.start(), range.end());
}
 
Addr
GPUComputeDriver::deallocateGpuVma(Addr start)
{
    auto vma = gpuVmas.contains(start);
    assert(vma != gpuVmas.end());
    assert((vma->first.start() == start));
    Addr size = vma->first.size();
    DPRINTF(GPUDriver, "Unregistering [%p - %p]\n", vma->first.start(),
            vma->first.end());
    gpuVmas.erase(vma);
    return size;
}
 
void
GPUComputeDriver::setMtype(RequestPtr req)
{
    // If we are a dGPU then set the MTYPE from our VMAs.
    if (isdGPU) {
        AddrRange range = RangeSize(req->getVaddr(), req->getSize());
        auto vma = gpuVmas.contains(range);
        assert(vma != gpuVmas.end());
        DPRINTF(GPUShader, "Setting req from [%p - %p] MTYPE %d\n"
                "%d\n", range.start(), range.end(), vma->second);
        req->setCacheCoherenceFlags(vma->second);
    // APUs always get the default MTYPE
    } else {
        req->setCacheCoherenceFlags(defaultMtype);
    }
}
 
} // namespace gem5