// SPDX-License-Identifier: MIT
/*
* Copyright © 2021 Intel Corporation
*/
#include "xe_device.h"
#include <linux/aperture.h>
#include <linux/delay.h>
#include <linux/fault-inject.h>
#include <linux/units.h>
#include <drm/drm_client.h>
#include <drm/drm_gem_ttm_helper.h>
#include <drm/drm_ioctl.h>
#include <drm/drm_managed.h>
#include <drm/drm_pagemap_util.h>
#include <drm/drm_print.h>
#include <kunit/static_stub.h>
#include <uapi/drm/xe_drm.h>
#include "display/xe_display.h"
#include "instructions/xe_gpu_commands.h"
#include "regs/xe_gt_regs.h"
#include "regs/xe_regs.h"
#include "xe_bo.h"
#include "xe_bo_evict.h"
#include "xe_configfs.h"
#include "xe_debugfs.h"
#include "xe_defaults.h"
#include "xe_devcoredump.h"
#include "xe_device_sysfs.h"
#include "xe_dma_buf.h"
#include "xe_drm_client.h"
#include "xe_drv.h"
#include "xe_exec.h"
#include "xe_exec_queue.h"
#include "xe_force_wake.h"
#include "xe_ggtt.h"
#include "xe_gt.h"
#include "xe_gt_mcr.h"
#include "xe_gt_printk.h"
#include "xe_gt_sriov_vf.h"
#include "xe_guc.h"
#include "xe_guc_pc.h"
#include "xe_hw_engine_group.h"
#include "xe_hwmon.h"
#include "xe_i2c.h"
#include "xe_irq.h"
#include "xe_late_bind_fw.h"
#include "xe_mmio.h"
#include "xe_module.h"
#include "xe_nvm.h"
#include "xe_oa.h"
#include "xe_observation.h"
#include "xe_pagefault.h"
#include "xe_pat.h"
#include "xe_pcode.h"
#include "xe_pm.h"
#include "xe_pmu.h"
#include "xe_psmi.h"
#include "xe_pxp.h"
#include "xe_query.h"
#include "xe_ras.h"
#include "xe_shrinker.h"
#include "xe_soc_remapper.h"
#include "xe_survivability_mode.h"
#include "xe_sriov.h"
#include "xe_svm.h"
#include "xe_sysctrl.h"
#include "xe_tile.h"
#include "xe_ttm_stolen_mgr.h"
#include "xe_ttm_sys_mgr.h"
#include "xe_vm.h"
#include "xe_vm_madvise.h"
#include "xe_vram.h"
#include "xe_vram_types.h"
#include "xe_vsec.h"
#include "xe_wait_user_fence.h"
#include "xe_wa.h"
#include <generated/xe_device_wa_oob.h>
#include <generated/xe_wa_oob.h>
static int xe_file_open(struct drm_device *dev, struct drm_file *file)
{
struct xe_device *xe = to_xe_device(dev);
struct xe_drm_client *client;
struct xe_file *xef;
int ret = -ENOMEM;
struct task_struct *task = NULL;
xef = kzalloc_obj(*xef);
if (!xef)
return ret;
client = xe_drm_client_alloc();
if (!client) {
kfree(xef);
return ret;
}
xef->drm = file;
xef->client = client;
xef->xe = xe;
mutex_init(&xef->vm.lock);
xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1);
mutex_init(&xef->exec_queue.lock);
xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
file->driver_priv = xef;
kref_init(&xef->refcount);
task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
if (task) {
xef->process_name = kstrdup(task->comm, GFP_KERNEL);
xef->pid = task->pid;
put_task_struct(task);
}
return 0;
}
static void xe_file_destroy(struct kref *ref)
{
struct xe_file *xef = container_of(ref, struct xe_file, refcount);
xa_destroy(&xef->exec_queue.xa);
mutex_destroy(&xef->exec_queue.lock);
xa_destroy(&xef->vm.xa);
mutex_destroy(&xef->vm.lock);
xe_drm_client_put(xef->client);
kfree(xef->process_name);
kfree(xef);
}
/**
* xe_file_get() - Take a reference to the xe file object
* @xef: Pointer to the xe file
*
* Anyone with a pointer to xef must take a reference to the xe file
* object using this call.
*
* Return: xe file pointer
*/
struct xe_file *xe_file_get(struct xe_file *xef)
{
kref_get(&xef->refcount);
return xef;
}
/**
* xe_file_put() - Drop a reference to the xe file object
* @xef: Pointer to the xe file
*
* Used to drop reference to the xef object
*/
void xe_file_put(struct xe_file *xef)
{
kref_put(&xef->refcount, xe_file_destroy);
}
static void xe_file_close(struct drm_device *dev, struct drm_file *file)
{
struct xe_device *xe = to_xe_device(dev);
struct xe_file *xef = file->driver_priv;
struct xe_vm *vm;
struct xe_exec_queue *q;
unsigned long idx;
guard(xe_pm_runtime)(xe);
/*
* No need for exec_queue.lock here as there is no contention for it
* when FD is closing as IOCTLs presumably can't be modifying the
* xarray. Taking exec_queue.lock here causes undue dependency on
* vm->lock taken during xe_exec_queue_kill().
*/
xa_for_each(&xef->exec_queue.xa, idx, q) {
if (q->vm && q->hwe->hw_engine_group)
xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
xe_exec_queue_kill(q);
xe_exec_queue_put(q);
}
xa_for_each(&xef->vm.xa, idx, vm)
xe_vm_close_and_put(vm);
xe_file_put(xef);
}
static const struct drm_ioctl_desc xe_ioctls[] = {
DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl,
DRM_RENDER_ALLOW),
};
static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
struct drm_file *file_priv = file->private_data;
struct xe_device *xe = to_xe_device(file_priv->minor->dev);
long ret;
if (xe_device_wedged(xe))
return -ECANCELED;
ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
if (ret >= 0)
ret = drm_ioctl(file, cmd, arg);
return ret;
}
#ifdef CONFIG_COMPAT
static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
struct drm_file *file_priv = file->private_data;
struct xe_device *xe = to_xe_device(file_priv->minor->dev);
long ret;
if (xe_device_wedged(xe))
return -ECANCELED;
ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
if (ret >= 0)
ret = drm_compat_ioctl(file, cmd, arg);
return ret;
}
#else
/* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
#define xe_drm_compat_ioctl NULL
#endif
static void barrier_open(struct vm_area_struct *vma)
{
drm_dev_get(vma->vm_private_data);
}
static void barrier_close(struct vm_area_struct *vma)
{
drm_dev_put(vma->vm_private_data);
}
static void barrier_release_dummy_page(struct drm_device *dev, void *res)
{
struct page *dummy_page = (struct page *)res;
__free_page(dummy_page);
}
static vm_fault_t barrier_fault(struct vm_fault *vmf)
{
struct drm_device *dev = vmf->vma->vm_private_data;
struct vm_area_struct *vma = vmf->vma;
vm_fault_t ret = VM_FAULT_NOPAGE;
pgprot_t prot;
int idx;
prot = vma_get_page_prot(vma);
if (drm_dev_enter(dev, &idx)) {
unsigned long pfn;
#define LAST_DB_PAGE_OFFSET 0x7ff001
pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +
LAST_DB_PAGE_OFFSET);
ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn,
pgprot_noncached(prot));
drm_dev_exit(idx);
} else {
struct page *page;
/* Allocate new dummy page to map all the VA range in this VMA to it*/
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (!page)
return VM_FAULT_OOM;
/* Set the page to be freed using drmm release action */
if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))
return VM_FAULT_OOM;
ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page),
prot);
}
return ret;
}
static const struct vm_operations_struct vm_ops_barrier = {
.open = barrier_open,
.close = barrier_close,
.fault = barrier_fault,
};
static int xe_pci_barrier_mmap(struct file *filp,
struct vm_area_struct *vma)
{
struct drm_file *priv = filp->private_data;
struct drm_device *dev = priv->minor->dev;
struct xe_device *xe = to_xe_device(dev);
if (!IS_DGFX(xe))
return -EINVAL;
if (vma->vm_end - vma->vm_start > SZ_4K)
return -EINVAL;
if (vma_is_cow_mapping(vma))
return -EINVAL;
if (vma->vm_flags & (VM_READ | VM_EXEC))
return -EINVAL;
vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
vma->vm_ops = &vm_ops_barrier;
vma->vm_private_data = dev;
drm_dev_get(vma->vm_private_data);
return 0;
}
static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct drm_file *priv = filp->private_data;
struct drm_device *dev = priv->minor->dev;
if (drm_dev_is_unplugged(dev))
return -ENODEV;
switch (vma->vm_pgoff) {
case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:
return xe_pci_barrier_mmap(filp, vma);
}
return drm_gem_mmap(filp, vma);
}
static const struct file_operations xe_driver_fops = {
.owner = THIS_MODULE,
.open = drm_open,
.release = drm_release_noglobal,
.unlocked_ioctl = xe_drm_ioctl,
.mmap = xe_mmap,
.poll = drm_poll,
.read = drm_read,
.compat_ioctl = xe_drm_compat_ioctl,
.llseek = noop_llseek,
#ifdef CONFIG_PROC_FS
.show_fdinfo = drm_show_fdinfo,
#endif
.fop_flags = FOP_UNSIGNED_OFFSET,
};
/**
* xe_is_xe_file() - Is the file an xe device file?
* @file: The file.
*
* Checks whether the file is opened against
* an xe device.
*
* Return: %true if an xe file, %false if not.
*/
bool xe_is_xe_file(const struct file *file)
{
return file->f_op == &xe_driver_fops;
}
static const struct drm_driver regular_driver = {
.driver_features =
XE_DISPLAY_DRIVER_FEATURES |
DRIVER_GEM |
DRIVER_RENDER | DRIVER_SYNCOBJ |
DRIVER_SYNCOBJ_TIMELINE,
.open = xe_file_open,
.postclose = xe_file_close,
.gem_prime_import = xe_gem_prime_import,
.dumb_create = xe_bo_dumb_create,
.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
#ifdef CONFIG_PROC_FS
.show_fdinfo = xe_drm_client_fdinfo,
#endif
.ioctls = xe_ioctls,
.num_ioctls = ARRAY_SIZE(xe_ioctls),
.fops = &xe_driver_fops,
.name = DRIVER_NAME,
.desc = DRIVER_DESC,
.major = DRIVER_MAJOR,
.minor = DRIVER_MINOR,
.patchlevel = DRIVER_PATCHLEVEL,
XE_DISPLAY_DRIVER_OPS,
};
#ifdef CONFIG_PCI_IOV
static const struct drm_ioctl_desc xe_ioctls_admin_only[] = {
DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
};
static const struct drm_driver admin_only_driver = {
.driver_features =
DRIVER_GEM | DRIVER_RENDER,
.open = xe_file_open,
.postclose = xe_file_close,
.ioctls = xe_ioctls_admin_only,
.num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only),
.fops = &xe_driver_fops,
.name = DRIVER_NAME,
.desc = DRIVER_DESC,
.major = DRIVER_MAJOR,
.minor = DRIVER_MINOR,
.patchlevel = DRIVER_PATCHLEVEL,
};
/**
* xe_device_is_admin_only() - Check whether device is admin only or not.
* @xe: the &xe_device to check
*
* Return: true if the device is admin only, false otherwise.
*/
bool xe_device_is_admin_only(const struct xe_device *xe)
{
KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe);
return xe->drm.driver == &admin_only_driver;
}
#endif
static void xe_device_destroy(struct drm_device *dev, void *dummy)
{
struct xe_device *xe = to_xe_device(dev);
xe_bo_dev_fini(&xe->bo_device);
if (xe->preempt_fence_wq)
destroy_workqueue(xe->preempt_fence_wq);
if (xe->ordered_wq)
destroy_workqueue(xe->ordered_wq);
if (xe->unordered_wq)
destroy_workqueue(xe->unordered_wq);
if (xe->destroy_wq)
destroy_workqueue(xe->destroy_wq);
ttm_device_fini(&xe->ttm);
}
/**
* xe_device_create() - Create a new &xe_device instance
* @pdev: the parent &pci_dev
*
* Allocate and initialize a device managed Xe device structure.
*
* Return: pointer to new &xe_device on success, or ERR_PTR on failure.
*/
struct xe_device *xe_device_create(struct pci_dev *pdev)
{
const struct drm_driver *driver = ®ular_driver;
struct xe_device *xe;
int err;
#ifdef CONFIG_PCI_IOV
/*
* Since XE device is not initialized yet, read from configfs
* directly to decide whether we are in admin-only PF mode or not.
*/
if (xe_configfs_admin_only_pf(pdev))
driver = &admin_only_driver;
#endif
err = aperture_remove_conflicting_pci_devices(pdev, driver->name);
if (err)
return ERR_PTR(err);
xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm);
if (IS_ERR(xe))
return xe;
err = xe_device_init_early(xe);
if (err)
return ERR_PTR(err);
return xe;
}
ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
static void xe_device_parse_modparam(struct xe_device *xe)
{
xe->atomic_svm_timeslice_ms = 5;
xe->min_run_period_lr_ms = 5;
xe->info.num_pf_work = xe_modparam.num_pf_work;
if (xe->info.num_pf_work < 1)
xe->info.num_pf_work = 1;
else if (xe->info.num_pf_work > XE_PAGEFAULT_WORK_MAX)
xe->info.num_pf_work = XE_PAGEFAULT_WORK_MAX;
}
/**
* xe_device_init_early() - Initialize a new &xe_device instance
* @xe: the &xe_device to initialize
*
* Return: 0 on success or a negative error code on failure.
*/
int xe_device_init_early(struct xe_device *xe)
{
int err;
err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,
xe->drm.anon_inode->i_mapping,
xe->drm.vma_offset_manager,
TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
if (err)
return err;
xe_bo_dev_init(&xe->bo_device);
err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
if (err)
return err;
err = xe_shrinker_create(xe);
if (err)
return err;
xe_device_parse_modparam(xe);
err = xe_irq_init(xe);
if (err)
return err;
xe_validation_device_init(&xe->val);
init_waitqueue_head(&xe->ufence_wq);
init_rwsem(&xe->usm.lock);
err = xe_pagemap_shrinker_create(xe);
if (err)
return err;
xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
/* Trigger a large asid and an early asid wrap. */
u32 asid;
BUILD_BUG_ON(XE_MAX_ASID < 2);
err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,
XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
&xe->usm.next_asid, GFP_KERNEL);
drm_WARN_ON(&xe->drm, err);
if (err >= 0)
xa_erase(&xe->usm.asid_to_vm, asid);
}
err = xe_bo_pinned_init(xe);
if (err)
return err;
xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
WQ_MEM_RECLAIM);
xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0);
xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU | WQ_MEM_RECLAIM, 0);
if (!xe->ordered_wq || !xe->unordered_wq ||
!xe->preempt_fence_wq || !xe->destroy_wq) {
/*
* Cleanup done in xe_device_destroy via
* drmm_add_action_or_reset register above
*/
drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
return -ENOMEM;
}
err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
if (err)
return err;
err = xe_pm_init_early(xe);
if (err)
return err;
return 0;
}
static bool xe_driver_flr_disabled(struct xe_device *xe)
{
if (IS_SRIOV_VF(xe))
return true;
if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) {
drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n");
return true;
}
return false;
}
/*
* The driver-initiated FLR is the highest level of reset that we can trigger
* from within the driver. It is different from the PCI FLR in that it doesn't
* fully reset the SGUnit and doesn't modify the PCI config space and therefore
* it doesn't require a re-enumeration of the PCI BARs. However, the
* driver-initiated FLR does still cause a reset of both GT and display and a
* memory wipe of local and stolen memory, so recovery would require a full HW
* re-init and saving/restoring (or re-populating) the wiped memory. Since we
* perform the FLR as the very last action before releasing access to the HW
* during the driver release flow, we don't attempt recovery at all, because
* if/when a new instance of Xe is bound to the device it will do a full
* re-init anyway.
*/
static void __xe_driver_flr(struct xe_device *xe)
{
const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */
struct xe_mmio *mmio = xe_root_tile_mmio(xe);
int ret;
drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
/*
* Make sure any pending FLR requests have cleared by waiting for the
* FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
* to make sure it's not still set from a prior attempt (it's a write to
* clear bit).
* Note that we should never be in a situation where a previous attempt
* is still pending (unless the HW is totally dead), but better to be
* safe in case something unexpected happens
*/
ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
if (ret) {
drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
return;
}
xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
/* Trigger the actual Driver-FLR */
xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR);
/* Wait for hardware teardown to complete */
ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
if (ret) {
drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
return;
}
/* Wait for hardware/firmware re-init to complete */
ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
flr_timeout, NULL, false);
if (ret) {
drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
return;
}
/* Clear sticky completion status */
xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
}
static void xe_driver_flr(struct xe_device *xe)
{
if (xe_driver_flr_disabled(xe))
return;
__xe_driver_flr(xe);
}
static void xe_driver_flr_fini(void *arg)
{
struct xe_device *xe = arg;
if (xe->needs_flr_on_fini)
xe_driver_flr(xe);
}
static void xe_device_sanitize(void *arg)
{
struct xe_device *xe = arg;
struct xe_gt *gt;
u8 id;
for_each_gt(gt, xe, id)
xe_gt_sanitize(gt);
}
static int xe_set_dma_info(struct xe_device *xe)
{
unsigned int mask_size = xe->info.dma_mask_size;
int err;
dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));
err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
if (err)
goto mask_err;
err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
if (err)
goto mask_err;
return 0;
mask_err:
drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
return err;
}
static void assert_lmem_ready(struct xe_device *xe)
{
if (!IS_DGFX(xe) || IS_SRIOV_VF(xe))
return;
xe_assert(xe, xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) &
LMEM_INIT);
}
static void vf_update_device_info(struct xe_device *xe)
{
xe_assert(xe, IS_SRIOV_VF(xe));
/* disable features that are not available/applicable to VFs */
xe->info.probe_display = 0;
xe->info.has_heci_cscfi = 0;
xe->info.has_heci_gscfi = 0;
xe->info.has_i2c = 0;
xe->info.has_late_bind = 0;
xe->info.skip_guc_pc = 1;
xe->info.skip_pcode = 1;
xe->info.has_drm_ras = false;
}
static int xe_device_vram_alloc(struct xe_device *xe)
{
struct xe_vram_region *vram;
if (!IS_DGFX(xe))
return 0;
vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL);
if (!vram)
return -ENOMEM;
xe->mem.vram = vram;
return 0;
}
/**
* xe_device_probe_early: Device early probe
* @xe: xe device instance
*
* Initialize MMIO resources that don't require any
* knowledge about tile count. Also initialize pcode and
* check vram initialization on root tile.
*
* Return: 0 on success, error code on failure
*/
int xe_device_probe_early(struct xe_device *xe)
{
int err;
xe_wa_device_init(xe);
xe_wa_process_device_oob(xe);
err = xe_mmio_probe_early(xe);
if (err)
return err;
xe_sriov_probe_early(xe);
if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) {
xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n");
return -ENODEV;
}
if (IS_SRIOV_VF(xe))
vf_update_device_info(xe);
/*
* Check for pcode uncore_init status to confirm if the SoC
* initialization is complete. Until done, any MMIO or lmem access from
* the driver will be blocked
*/
err = xe_pcode_probe_early(xe);
if (err || xe_survivability_mode_is_requested(xe)) {
int save_err = err;
/*
* Try to leave device in survivability mode if device is
* possible, but still return the previous error for error
* propagation
*/
err = xe_survivability_mode_boot_enable(xe);
if (err)
return err;
return save_err;
}
/*
* Make sure the lmem is initialized and ready to use. xe_pcode_ready()
* is flagged after full initialization is complete. Assert if lmem is
* not initialized.
*/
assert_lmem_ready(xe);
xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ?
XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode;
drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n",
xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode));
err = xe_device_vram_alloc(xe);
if (err)
return err;
return 0;
}
ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */
static int probe_has_flat_ccs(struct xe_device *xe)
{
struct xe_gt *gt;
u32 reg;
/* Always enabled/disabled, no runtime check to do */
if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))
return 0;
gt = xe_root_mmio_gt(xe);
if (!gt)
return 0;
CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
if (!fw_ref.domains)
return -ETIMEDOUT;
reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
if (!xe->info.has_flat_ccs)
drm_dbg(&xe->drm,
"Flat CCS has been disabled in bios, May lead to performance impact");
return 0;
}
/*
* Detect if the driver is being run on pre-production hardware. We don't
* keep workarounds for pre-production hardware long term, so print an
* error and add taint if we're being loaded on a pre-production platform
* for which the pre-prod workarounds have already been removed.
*
* The general policy is that we'll remove any workarounds that only apply to
* pre-production hardware around the time force_probe restrictions are lifted
* for a platform of the next major IP generation (for example, Xe2 pre-prod
* workarounds should be removed around the time the first Xe3 platforms have
* force_probe lifted).
*/
static void detect_preproduction_hw(struct xe_device *xe)
{
struct xe_gt *gt;
int id;
/*
* SR-IOV VFs don't have access to the FUSE2 register, so we can't
* check pre-production status there. But the host OS will notice
* and report the pre-production status, which should be enough to
* help us catch mistaken use of pre-production hardware.
*/
if (IS_SRIOV_VF(xe))
return;
/*
* The "SW_CAP" fuse contains a bit indicating whether the device is a
* production or pre-production device. This fuse is reflected through
* the GT "FUSE2" register, even though the contents of the fuse are
* not GT-specific. Every GT's reflection of this fuse should show the
* same value, so we'll just use the first available GT for lookup.
*/
for_each_gt(gt, xe, id)
break;
if (!gt)
return;
CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) {
xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n");
return;
}
if (xe_mmio_read32(>->mmio, FUSE2) & PRODUCTION_HW)
return;
xe_info(xe, "Pre-production hardware detected.\n");
if (!xe->info.has_pre_prod_wa) {
xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n");
add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
}
}
static void xe_device_wedged_fini(struct drm_device *drm, void *arg)
{
struct xe_device *xe = arg;
if (atomic_read(&xe->wedged.flag))
xe_pm_runtime_put(xe);
}
#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE
static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe)
{
int err;
err = drmm_mutex_init(&xe->drm, &xe->page_size_alloc_ctrl.lock);
if (err)
return err;
xe->page_size_alloc_ctrl.mode = XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE;
xe->page_size_alloc_ctrl.cur_index = 0;
return 0;
}
#else
static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe)
{
return 0;
}
#endif
int xe_device_probe(struct xe_device *xe)
{
struct xe_tile *tile;
struct xe_gt *gt;
int err;
u8 id;
xe_pat_init_early(xe);
err = xe_sriov_init(xe);
if (err)
return err;
xe->info.mem_region_mask = 1;
err = xe_set_dma_info(xe);
if (err)
return err;
err = xe_mmio_probe_tiles(xe);
if (err)
return err;
for_each_gt(gt, xe, id) {
err = xe_gt_init_early(gt);
if (err)
return err;
}
/*
* Wa_16029380221: The affected GT will always use non-coherent
* access to page tables, so we must do uncached writes from the
* CPU.
*/
for_each_gt(gt, xe, id)
if (XE_GT_WA(gt, 16029380221))
xe->info.has_cached_pt = false;
for_each_tile(tile, xe, id) {
err = xe_ggtt_init_early(tile->mem.ggtt);
if (err)
return err;
}
/*
* From here on, if a step fails, make sure a Driver-FLR is triggereed
*/
err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);
if (err)
return err;
err = probe_has_flat_ccs(xe);
if (err)
return err;
err = xe_vram_probe(xe);
if (err)
return err;
for_each_tile(tile, xe, id) {
err = xe_tile_init_noalloc(tile);
if (err)
return err;
}
/*
* Allow allocations only now to ensure xe_display_init_early()
* is the first to allocate, always.
*/
err = xe_ttm_sys_mgr_init(xe);
if (err)
return err;
/* Allocate and map stolen after potential VRAM resize */
err = xe_ttm_stolen_mgr_init(xe);
if (err)
return err;
err = xe_soc_remapper_init(xe);
if (err)
return err;
err = xe_sysctrl_init(xe);
if (err)
return err;
xe_ras_init(xe);
/*
* Now that GT is initialized (TTM in particular),
* we can try to init display, and inherit the initial fb.
* This is the reason the first allocation needs to be done
* inside display.
*/
err = xe_display_init_early(xe);
if (err)
return err;
for_each_tile(tile, xe, id) {
err = xe_tile_init(tile);
if (err)
return err;
}
err = xe_irq_install(xe);
if (err)
return err;
for_each_gt(gt, xe, id) {
err = xe_gt_init(gt);
if (err)
return err;
}
err = xe_pagefault_init(xe);
if (err)
return err;
if (xe->tiles->media_gt &&
XE_GT_WA(xe->tiles->media_gt, 15015404425_disable))
XE_DEVICE_WA_DISABLE(xe, 15015404425);
err = xe_devcoredump_init(xe);
if (err)
return err;
xe_nvm_init(xe);
err = xe_heci_gsc_init(xe);
if (err)
return err;
err = xe_late_bind_init(&xe->late_bind);
if (err)
return err;
err = xe_oa_init(xe);
if (err)
return err;
err = xe_display_init(xe);
if (err)
return err;
err = xe_pxp_init(xe);
if (err)
return err;
err = xe_psmi_init(xe);
if (err)
return err;
err = xe_debug_page_size_alloc_ctrl_init(xe);
if (err)
return err;
err = drm_dev_register(&xe->drm, 0);
if (err)
return err;
xe_display_register(xe);
err = xe_oa_register(xe);
if (err)
goto err_unregister_display;
err = xe_pmu_register(&xe->pmu);
if (err)
goto err_unregister_display;
err = xe_device_sysfs_init(xe);
if (err)
goto err_unregister_display;
xe_debugfs_register(xe);
err = xe_hwmon_register(xe);
if (err)
goto err_unregister_display;
err = xe_i2c_probe(xe);
if (err)
goto err_unregister_display;
for_each_gt(gt, xe, id)
xe_gt_sanitize_freq(gt);
xe_vsec_init(xe);
err = xe_sriov_init_late(xe);
if (err)
goto err_unregister_display;
detect_preproduction_hw(xe);
err = drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe);
if (err)
goto err_unregister_display;
/*
* Process and log any errors detected by hardware. Possible results can
* include declaring the device as wedged, which must be done only after
* xe_device_wedged_fini() is registered.
*/
xe_ras_process_errors(xe);
return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
err_unregister_display:
xe_display_unregister(xe);
drm_dev_unregister(&xe->drm);
return err;
}
void xe_device_remove(struct xe_device *xe)
{
xe_display_unregister(xe);
drm_dev_unplug(&xe->drm);
xe_bo_pci_dev_remove_all(xe);
}
void xe_device_shutdown(struct xe_device *xe)
{
struct xe_gt *gt;
u8 id;
drm_dbg(&xe->drm, "Shutting down device\n");
xe_display_shutdown(xe);
xe_irq_suspend(xe);
for_each_gt(gt, xe, id)
xe_gt_shutdown(gt);
xe_display_shutdown_late(xe);
if (!xe_driver_flr_disabled(xe)) {
/* BOOM! */
__xe_driver_flr(xe);
}
}
/**
* xe_device_wmb() - Device specific write memory barrier
* @xe: the &xe_device
*
* While wmb() is sufficient for a barrier if we use system memory, on discrete
* platforms with device memory we additionally need to issue a register write.
* Since it doesn't matter which register we write to, use the read-only VF_CAP
* register that is also marked as accessible by the VFs.
*/
void xe_device_wmb(struct xe_device *xe)
{
wmb();
if (IS_DGFX(xe))
xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0);
}
/*
* Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt.
*/
static void tdf_request_sync(struct xe_device *xe)
{
struct xe_gt *gt;
u8 id;
for_each_gt_with_type(gt, xe, id, BIT(XE_GT_TYPE_MAIN)) {
CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
if (!fw_ref.domains)
return;
xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
/*
* FIXME: We can likely do better here with our choice of
* timeout. Currently we just assume the worst case, i.e. 150us,
* which is believed to be sufficient to cover the worst case
* scenario on current platforms if all cache entries are
* transient and need to be flushed..
*/
if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
300, NULL, false))
xe_gt_err_once(gt, "TD flush timeout\n");
}
}
/**
* xe_device_is_l2_flush_optimized - if L2 flush is optimized by HW
* @xe: The device to check.
*
* Return: true if the HW device optimizing L2 flush, false otherwise.
*/
bool xe_device_is_l2_flush_optimized(struct xe_device *xe)
{
/* XA is *always* flushed, like at the end-of-submssion (and maybe other
* places), just that internally as an optimisation hw doesn't need to make
* that a full flush (which will also include XA) when Media is
* off/powergated, since it doesn't need to worry about GT caches vs Media
* coherency, and only CPU vs GPU coherency, so can make that flush a
* targeted XA flush, since stuff tagged with XA now means it's shared with
* the CPU. The main implication is that we now need to somehow flush non-XA before
* freeing system memory pages, otherwise dirty cachelines could be flushed after the free
* (like if Media suddenly turns on and does a full flush)
*/
if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe))
return true;
return false;
}
void xe_device_l2_flush(struct xe_device *xe)
{
struct xe_gt *gt;
gt = xe_root_mmio_gt(xe);
if (!gt)
return;
if (!XE_GT_WA(gt, 16023588340))
return;
CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
if (!fw_ref.domains)
return;
spin_lock(>->global_invl_lock);
xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1);
if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true))
xe_gt_err_once(gt, "Global invalidation timeout\n");
spin_unlock(>->global_invl_lock);
}
/**
* xe_device_td_flush() - Flush transient L3 cache entries
* @xe: The device
*
* Display engine has direct access to memory and is never coherent with L3/L4
* caches (or CPU caches), however KMD is responsible for specifically flushing
* transient L3 GPU cache entries prior to the flip sequence to ensure scanout
* can happen from such a surface without seeing corruption.
*
* Display surfaces can be tagged as transient by mapping it using one of the
* various L3:XD PAT index modes on Xe2.
*
* Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed
* at the end of each submission via PIPE_CONTROL for compute/render, since SA
* Media is not coherent with L3 and we want to support render-vs-media
* usescases. For other engines like copy/blt the HW internally forces uncached
* behaviour, hence why we can skip the TDF on such platforms.
*/
void xe_device_td_flush(struct xe_device *xe)
{
struct xe_gt *root_gt;
/*
* From Xe3p onward the HW takes care of flush of TD entries also along
* with flushing XA entries, which will be at the usual sync points,
* like at the end of submission, so no manual flush is needed here.
*/
if (GRAPHICS_VER(xe) >= 35)
return;
if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)
return;
root_gt = xe_root_mmio_gt(xe);
if (!root_gt)
return;
if (XE_GT_WA(root_gt, 16023588340)) {
/* A transient flush is not sufficient: flush the L2 */
xe_device_l2_flush(xe);
} else {
xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc);
tdf_request_sync(xe);
xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc);
}
}
u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
{
return xe_device_has_flat_ccs(xe) ?
DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
}
/**
* xe_device_assert_mem_access - Inspect the current runtime_pm state.
* @xe: xe device instance
*
* To be used before any kind of memory access. It will splat a debug warning
* if the device is currently sleeping. But it doesn't guarantee in any way
* that the device is going to remain awake. Xe PM runtime get and put
* functions might be added to the outer bound of the memory access, while
* this check is intended for inner usage to splat some warning if the worst
* case has just happened.
*/
void xe_device_assert_mem_access(struct xe_device *xe)
{
xe_assert(xe, !xe_pm_runtime_suspended(xe));
}
void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
{
struct xe_gt *gt;
u8 id;
drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);
drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);
for_each_gt(gt, xe, id) {
drm_printf(p, "GT id: %u\n", id);
drm_printf(p, "\tTile: %u\n", gt->tile->id);
drm_printf(p, "\tType: %s\n",
gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
drm_printf(p, "\tIP ver: %u.%u.%u\n",
REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock);
}
}
u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
{
return sign_extend64(address, xe->info.va_bits - 1);
}
u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
{
return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
}
/**
* DOC: Xe Device Wedging
*
* Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst.
* When device is in wedged state, every IOCTL will be blocked and GT cannot
* be used. The conditions under which the driver declares the device wedged
* depend on the wedged mode configuration (see &enum xe_wedged_mode). The
* default recovery method for a wedged state is rebind/bus-reset.
*
* Another recovery method is vendor-specific. Below are the cases that send
* ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent.
*
* Case: Firmware Flash
* --------------------
*
* Identification Hint
* +++++++++++++++++++
*
* ``WEDGED=vendor-specific`` drm device wedged uevent with
* :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify
* admin/userspace consumer about the need for a firmware flash.
*
* Recovery Procedure
* ++++++++++++++++++
*
* Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow
* the below steps
*
* - Check Runtime Survivability mode sysfs.
* If enabled, firmware flash is required to recover the device.
*
* /sys/bus/pci/devices/<device>/survivability_mode
*
* - Admin/userspace consumer can use firmware flashing tools like fwupd to flash
* firmware and restore device to normal operation.
*/
/**
* xe_device_set_wedged_method - Set wedged recovery method
* @xe: xe device instance
* @method: recovery method to set
*
* Set wedged recovery method to be sent in drm wedged uevent.
*/
void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method)
{
xe->wedged.method = method;
}
/**
* xe_device_declare_wedged - Declare device wedged
* @xe: xe device instance
*
* This is a final state that can only be cleared with the recovery method
* specified in the drm wedged uevent. The method can be set using
* xe_device_set_wedged_method before declaring the device as wedged. If no method
* is set, reprobe (unbind/re-bind) will be sent by default.
*
* In this state every IOCTL will be blocked so the GT cannot be used.
* In general it will be called upon any critical error such as gt reset
* failure or guc loading failure. Userspace will be notified of this state
* through device wedged uevent.
* If xe.wedged module parameter is set to 2, this function will be called
* on every single execution timeout (a.k.a. GPU hang) right after devcoredump
* snapshot capture. In this mode, GT reset won't be attempted so the state of
* the issue is preserved for further debugging.
*/
void xe_device_declare_wedged(struct xe_device *xe)
{
struct xe_gt *gt;
u8 id;
if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) {
drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");
return;
}
if (!atomic_xchg(&xe->wedged.flag, 1)) {
xe->needs_flr_on_fini = true;
xe_pm_runtime_get_noresume(xe);
drm_err(&xe->drm,
"CRITICAL: Xe has declared device %s as wedged.\n"
"IOCTLs and executions are blocked.\n"
"For recovery procedure, refer to https://docs.kernel.org/gpu/drm-uapi.html#device-wedging\n"
"Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
dev_name(xe->drm.dev));
}
for_each_gt(gt, xe, id)
xe_gt_declare_wedged(gt);
if (xe_device_wedged(xe)) {
/*
* XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET is intended for debugging
* hangs, so wedge the device with 'none' recovery method and have
* it available to the user for debugging.
*/
if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET)
xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_NONE);
/* If no wedge recovery method is set, use default */
else if (!xe->wedged.method)
xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND |
DRM_WEDGE_RECOVERY_BUS_RESET);
/* Notify userspace of wedged device */
drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL);
}
}
/**
* xe_device_validate_wedged_mode - Check if given mode is supported
* @xe: the &xe_device
* @mode: requested mode to validate
*
* Check whether the provided wedged mode is supported.
*
* Return: 0 if mode is supported, error code otherwise.
*/
int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode)
{
if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) {
drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode);
return -EINVAL;
} else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) ||
(IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) {
drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n",
mode, xe_wedged_mode_to_string(mode),
xe_sriov_mode_to_string(xe_device_sriov_mode(xe)));
return -EPERM;
}
return 0;
}
/**
* xe_wedged_mode_to_string - Convert enum value to string.
* @mode: the &xe_wedged_mode to convert
*
* Returns: wedged mode as a user friendly string.
*/
const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode)
{
switch (mode) {
case XE_WEDGED_MODE_NEVER:
return "never";
case XE_WEDGED_MODE_UPON_CRITICAL_ERROR:
return "upon-critical-error";
case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET:
return "upon-any-hang-no-reset";
default:
return "<invalid>";
}
}
/**
* xe_device_asid_to_vm() - Find VM from ASID
* @xe: the &xe_device
* @asid: Address space ID
*
* Find a VM from ASID and take a reference to VM which caller must drop.
* Reclaim safe.
*
* Return: VM on success, ERR_PTR on failure
*/
struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid)
{
struct xe_vm *vm;
down_read(&xe->usm.lock);
vm = xa_load(&xe->usm.asid_to_vm, asid);
if (vm)
xe_vm_get(vm);
else
vm = ERR_PTR(-EINVAL);
up_read(&xe->usm.lock);
return vm;
}
