// SPDX-License-Identifier: MIT
/*
* Copyright(c) 2024 Intel Corporation.
*/
#include "xe_pxp.h"
#include <drm/drm_managed.h>
#include <uapi/drm/xe_drm.h>
#include <linux/device.h>
#include "xe_bo.h"
#include "xe_bo_types.h"
#include "xe_device_types.h"
#include "xe_exec_queue.h"
#include "xe_force_wake.h"
#include "xe_guc_exec_queue_types.h"
#include "xe_guc_submit.h"
#include "xe_gsc_proxy.h"
#include "xe_gt_types.h"
#include "xe_huc.h"
#include "xe_hw_engine.h"
#include "xe_mmio.h"
#include "xe_pm.h"
#include "xe_pxp_submit.h"
#include "xe_pxp_types.h"
#include "xe_uc_fw.h"
#include "regs/xe_irq_regs.h"
#include "regs/xe_pxp_regs.h"
/**
* DOC: PXP
*
* PXP (Protected Xe Path) allows execution and flip to display of protected
* (i.e. encrypted) objects. This feature is currently only supported in
* integrated parts.
*/
#define ARB_SESSION DRM_XE_PXP_HWDRM_DEFAULT_SESSION /* shorter define */
/*
* A submission to GSC can take up to 250ms to complete, so use a 300ms
* timeout for activation where only one of those is involved. Termination
* additionally requires a submission to VCS and an interaction with KCR, so
* bump the timeout to 500ms for that.
*/
#define PXP_ACTIVATION_TIMEOUT_MS 300
#define PXP_TERMINATION_TIMEOUT_MS 500
bool xe_pxp_is_supported(const struct xe_device *xe)
{
return xe->info.has_pxp && IS_ENABLED(CONFIG_INTEL_MEI_GSC_PROXY);
}
bool xe_pxp_is_enabled(const struct xe_pxp *pxp)
{
return pxp;
}
static bool pxp_prerequisites_done(const struct xe_pxp *pxp)
{
struct xe_gt *gt = pxp->gt;
bool huc_ok;
bool ready;
CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
/*
* If force_wake fails we could falsely report the prerequisites as not
* done even if they are; the consequence of this would be that the
* callers won't go ahead with using PXP, but if force_wake doesn't work
* the GT is very likely in a bad state so not really a problem to abort
* PXP. Therefore, we can just log the force_wake error and not escalate
* it.
*/
XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL));
/*
* PXP requires GSC proxy to be initialized. On platforms where the HuC
* is loaded by the kernel driver (i.e., pre media 35) PXP also requires
* the HuC to be authenticated by GSC.
*/
huc_ok = MEDIA_VER(gt_to_xe(gt)) >= 35 ||
xe_huc_is_authenticated(>->uc.huc, XE_HUC_AUTH_VIA_GSC);
ready = huc_ok && xe_gsc_proxy_init_done(>->uc.gsc);
return ready;
}
/**
* xe_pxp_get_readiness_status - check whether PXP is ready for userspace use
* @pxp: the xe_pxp pointer (can be NULL if PXP is disabled)
*
* Returns: 0 if PXP is not ready yet, 1 if it is ready, a negative errno value
* if PXP is not supported/enabled or if something went wrong in the
* initialization of the prerequisites. Note that the return values of this
* function follow the uapi (see drm_xe_query_pxp_status), so they can be used
* directly in the query ioctl.
*/
int xe_pxp_get_readiness_status(struct xe_pxp *pxp)
{
int ret = 0;
if (!xe_pxp_is_enabled(pxp))
return -ENODEV;
/* If the GSC FW is in an error state, PXP will never work */
if (xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status))
return -EIO;
/* Same for HuC FW, but only if the kernel owns HuC-loading (i.e. pre-NVL) */
if (MEDIA_VER(gt_to_xe(pxp->gt)) < 35 &&
xe_uc_fw_status_to_error(pxp->gt->uc.huc.fw.status))
return -EIO;
guard(xe_pm_runtime)(pxp->xe);
/* PXP requires both HuC loaded and GSC proxy initialized */
if (pxp_prerequisites_done(pxp))
ret = 1;
return ret;
}
static bool pxp_session_is_in_play(struct xe_pxp *pxp, u32 id)
{
struct xe_gt *gt = pxp->gt;
return xe_mmio_read32(>->mmio, KCR_SIP) & BIT(id);
}
static int pxp_wait_for_session_state(struct xe_pxp *pxp, u32 id, bool in_play)
{
struct xe_gt *gt = pxp->gt;
u32 mask = BIT(id);
return xe_mmio_wait32(>->mmio, KCR_SIP, mask, in_play ? mask : 0,
250, NULL, false);
}
static void pxp_invalidate_queues(struct xe_pxp *pxp);
static int pxp_terminate_hw(struct xe_pxp *pxp)
{
struct xe_gt *gt = pxp->gt;
int ret = 0;
drm_dbg(&pxp->xe->drm, "Terminating PXP\n");
CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
return -EIO;
/* terminate the hw session */
ret = xe_pxp_submit_session_termination(pxp, ARB_SESSION);
if (ret)
return ret;
ret = pxp_wait_for_session_state(pxp, ARB_SESSION, false);
if (ret)
return ret;
/* Trigger full HW cleanup */
xe_mmio_write32(>->mmio, KCR_GLOBAL_TERMINATE, 1);
/* now we can tell the GSC to clean up its own state */
return xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION);
}
static void mark_termination_in_progress(struct xe_pxp *pxp)
{
lockdep_assert_held(&pxp->mutex);
reinit_completion(&pxp->termination);
pxp->status = XE_PXP_TERMINATION_IN_PROGRESS;
}
static bool pxp_prep_for_termination(struct xe_pxp *pxp)
{
lockdep_assert_held(&pxp->mutex);
if (pxp->status == XE_PXP_ACTIVE)
pxp->key_instance++;
/*
* we'll mark the status as needing termination on resume, so no need to
* emit a termination now.
*/
if (pxp->status == XE_PXP_SUSPENDED)
return false;
/*
* If we have a termination already in progress, we need to wait for
* it to complete before queueing another one. Once the first
* termination is completed we'll set the state back to
* NEEDS_TERMINATION and leave it to the pxp start code to issue it.
*/
if (pxp->status == XE_PXP_TERMINATION_IN_PROGRESS) {
pxp->status = XE_PXP_NEEDS_ADDITIONAL_TERMINATION;
return false;
}
mark_termination_in_progress(pxp);
return true;
}
static void pxp_terminate(struct xe_pxp *pxp, bool hw_only)
{
struct xe_device *xe = pxp->xe;
int ret = 0;
if (!wait_for_completion_timeout(&pxp->activation,
msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS)))
drm_err(&xe->drm, "failed to wait for PXP start before termination\n");
if (!hw_only) {
bool prep_ok;
mutex_lock(&pxp->mutex);
prep_ok = pxp_prep_for_termination(pxp);
mutex_unlock(&pxp->mutex);
if (!prep_ok)
return;
pxp_invalidate_queues(pxp);
} else {
/*
* The caller of the HW-only termination should have already
* called pxp_prep_for_termination and marked the termination as
* in progress.
*/
xe_assert(xe, !completion_done(&pxp->termination));
}
ret = pxp_terminate_hw(pxp);
if (ret) {
drm_err(&xe->drm, "PXP termination failed: %pe\n", ERR_PTR(ret));
mutex_lock(&pxp->mutex);
pxp->status = XE_PXP_ERROR;
complete_all(&pxp->termination);
mutex_unlock(&pxp->mutex);
}
}
static void pxp_terminate_complete(struct xe_pxp *pxp)
{
/*
* We expect PXP to be in one of 3 states when we get here:
* - XE_PXP_TERMINATION_IN_PROGRESS: a single termination event was
* requested and it is now completing, so we're ready to start.
* - XE_PXP_NEEDS_ADDITIONAL_TERMINATION: a second termination was
* requested while the first one was still being processed.
* - XE_PXP_SUSPENDED: PXP is now suspended, so we defer everything to
* when we come back on resume.
*/
mutex_lock(&pxp->mutex);
switch (pxp->status) {
case XE_PXP_TERMINATION_IN_PROGRESS:
pxp->status = XE_PXP_READY_TO_START;
break;
case XE_PXP_NEEDS_ADDITIONAL_TERMINATION:
pxp->status = XE_PXP_NEEDS_TERMINATION;
break;
case XE_PXP_SUSPENDED:
/* Nothing to do */
break;
default:
drm_err(&pxp->xe->drm,
"PXP termination complete while status was %u\n",
pxp->status);
}
complete_all(&pxp->termination);
mutex_unlock(&pxp->mutex);
}
static void pxp_events_work(struct work_struct *work)
{
struct xe_pxp *pxp = container_of(work, typeof(*pxp), events.work);
struct xe_device *xe = pxp->xe;
bool hw_only = false;
u32 events = 0;
events = atomic_xchg(&pxp->events.pending, 0);
if (!events)
return;
/*
* If the termination request comes from an irq while we're suspending,
* then we can defer it to the resume path instead of waking the device
* up.
* In the case of the termination on resume the pm reference is taken
* in xe_pxp_pm_resume() and released here.
* Note that we do not expect both events to be set at the same time,
* but if it does happen due to a spurious interrupt we want to behave
* as if the only request we got was the one from the resume path; this
* is because the termination prep has already been done in
* xe_pxp_pm_resume() and it is impossible for any PXP operations to
* occur between the prep and the termination completion, so there is no
* need for a new SW prep.
*/
if (events & PXP_TERMINATION_REQUEST_ON_RESUME) {
events &= ~PXP_TERMINATION_REQUEST_IRQ;
hw_only = true;
}
if ((events & PXP_TERMINATION_REQUEST_IRQ) && !xe_pm_runtime_get_if_active(xe))
return;
if (events & PXP_TERMINATION_REQUEST) {
events &= ~PXP_TERMINATION_COMPLETE_IRQ;
pxp_terminate(pxp, hw_only);
}
if (events & PXP_TERMINATION_COMPLETE_IRQ)
pxp_terminate_complete(pxp);
if (events & PXP_TERMINATION_REQUEST)
xe_pm_runtime_put(xe);
}
/**
* xe_pxp_irq_handler - Handles PXP interrupts.
* @xe: the xe_device structure
* @iir: interrupt vector
*/
void xe_pxp_irq_handler(struct xe_device *xe, u16 iir)
{
struct xe_pxp *pxp = xe->pxp;
if (!xe_pxp_is_enabled(pxp)) {
drm_err(&xe->drm, "PXP irq 0x%x received with PXP disabled!\n", iir);
return;
}
if (unlikely(!iir))
return;
if (iir & (KCR_PXP_STATE_TERMINATED_INTERRUPT |
KCR_APP_TERMINATED_PER_FW_REQ_INTERRUPT))
atomic_or(PXP_TERMINATION_REQUEST_IRQ, &pxp->events.pending);
if (iir & KCR_PXP_STATE_RESET_COMPLETE_INTERRUPT)
atomic_or(PXP_TERMINATION_COMPLETE_IRQ, &pxp->events.pending);
if (atomic_read(&pxp->events.pending))
queue_work(pxp->events.wq, &pxp->events.work);
}
static int kcr_pxp_set_status(const struct xe_pxp *pxp, bool enable)
{
u32 val = enable ? REG_MASKED_FIELD_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :
REG_MASKED_FIELD_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);
CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
return -EIO;
xe_mmio_write32(&pxp->gt->mmio, KCR_INIT, val);
return 0;
}
static int kcr_pxp_enable(const struct xe_pxp *pxp)
{
return kcr_pxp_set_status(pxp, true);
}
static int kcr_pxp_disable(const struct xe_pxp *pxp)
{
return kcr_pxp_set_status(pxp, false);
}
static void pxp_fini(void *arg)
{
struct xe_pxp *pxp = arg;
destroy_workqueue(pxp->events.wq);
xe_pxp_destroy_execution_resources(pxp);
/* no need to explicitly disable KCR since we're going to do an FLR */
}
/**
* xe_pxp_init - initialize PXP support
* @xe: the xe_device structure
*
* Initialize the HW state and allocate the objects required for PXP support.
* Note that some of the requirement for PXP support (GSC proxy init, HuC auth)
* are performed asynchronously as part of the GSC init. PXP can only be used
* after both this function and the async worker have completed.
*
* Returns 0 if PXP is not supported or if PXP initialization is successful,
* other errno value if there is an error during the init.
*/
int xe_pxp_init(struct xe_device *xe)
{
struct xe_gt *gt = xe->tiles[0].media_gt;
bool gsc_ok, huc_ok;
struct xe_pxp *pxp;
int err;
if (!xe_pxp_is_supported(xe))
return 0;
/* we only support PXP on single tile devices with a media GT */
if (xe->info.tile_count > 1 || !gt)
return 0;
/* The GSCCS is required for submissions to the GSC FW */
if (!(gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)))
return 0;
/* PXP requires GSC FW to be available. Pre-NVL it also requires HuC FW */
gsc_ok = xe_uc_fw_is_loadable(>->uc.gsc.fw);
huc_ok = MEDIA_VER(xe) >= 35 || xe_uc_fw_is_loadable(>->uc.huc.fw);
if (!gsc_ok || !huc_ok) {
drm_info(&xe->drm, "Skipping PXP due to unsatisfied FW deps - GSC=%s, HuC=%s\n",
str_yes_no(gsc_ok),
MEDIA_VER(xe) >= 35 ? "not needed" : str_yes_no(huc_ok));
return 0;
}
/*
* On PTL, older GSC FWs have a bug that can cause them to crash during
* PXP invalidation events, which leads to a complete loss of power
* management on the media GT. Therefore, we can't use PXP on FWs that
* have this bug, which was fixed in PTL GSC build 1396.
*/
if (xe->info.platform == XE_PANTHERLAKE &&
gt->uc.gsc.fw.versions.found[XE_UC_FW_VER_RELEASE].build < 1396) {
drm_info(&xe->drm, "PXP requires PTL GSC build 1396 or newer\n");
return 0;
}
pxp = drmm_kzalloc(&xe->drm, sizeof(struct xe_pxp), GFP_KERNEL);
if (!pxp) {
err = -ENOMEM;
goto out;
}
INIT_LIST_HEAD(&pxp->queues.list);
spin_lock_init(&pxp->queues.lock);
INIT_WORK(&pxp->events.work, pxp_events_work);
pxp->xe = xe;
pxp->gt = gt;
pxp->key_instance = 1;
pxp->last_suspend_key_instance = 1;
/*
* we'll use the completions to check if there is an action pending,
* so we start them as completed and we reinit it when an action is
* triggered.
*/
init_completion(&pxp->activation);
init_completion(&pxp->termination);
complete_all(&pxp->termination);
complete_all(&pxp->activation);
mutex_init(&pxp->mutex);
pxp->events.wq = alloc_ordered_workqueue("pxp-wq", 0);
if (!pxp->events.wq) {
err = -ENOMEM;
goto out_free;
}
err = kcr_pxp_enable(pxp);
if (err)
goto out_wq;
err = xe_pxp_allocate_execution_resources(pxp);
if (err)
goto out_kcr_disable;
xe->pxp = pxp;
return devm_add_action_or_reset(xe->drm.dev, pxp_fini, pxp);
out_kcr_disable:
kcr_pxp_disable(pxp);
out_wq:
destroy_workqueue(pxp->events.wq);
out_free:
drmm_kfree(&xe->drm, pxp);
out:
drm_err(&xe->drm, "PXP initialization failed: %pe\n", ERR_PTR(err));
return err;
}
static int __pxp_start_arb_session(struct xe_pxp *pxp)
{
int ret;
CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
return -EIO;
if (pxp_session_is_in_play(pxp, ARB_SESSION))
return -EEXIST;
ret = xe_pxp_submit_session_init(&pxp->gsc_res, ARB_SESSION);
if (ret) {
drm_err(&pxp->xe->drm, "Failed to init PXP arb session: %pe\n", ERR_PTR(ret));
return ret;
}
ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true);
if (ret) {
drm_err(&pxp->xe->drm, "PXP ARB session failed to go in play%pe\n", ERR_PTR(ret));
return ret;
}
drm_dbg(&pxp->xe->drm, "PXP ARB session is active\n");
return 0;
}
/**
* xe_pxp_exec_queue_set_type - Mark a queue as using PXP
* @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled)
* @q: the queue to mark as using PXP
* @type: the type of PXP session this queue will use
*
* Returns 0 if the selected PXP type is supported, -ENODEV otherwise.
*/
int xe_pxp_exec_queue_set_type(struct xe_pxp *pxp, struct xe_exec_queue *q, u8 type)
{
if (!xe_pxp_is_enabled(pxp))
return -ENODEV;
/* we only support HWDRM sessions right now */
xe_assert(pxp->xe, type == DRM_XE_PXP_TYPE_HWDRM);
q->pxp.type = type;
return 0;
}
static int __exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q)
{
int ret = 0;
/*
* A queue can be added to the list only if the PXP is in active status,
* otherwise the termination might not handle it correctly.
*/
mutex_lock(&pxp->mutex);
if (pxp->status == XE_PXP_ACTIVE) {
spin_lock_irq(&pxp->queues.lock);
list_add_tail(&q->pxp.link, &pxp->queues.list);
spin_unlock_irq(&pxp->queues.lock);
} else if (pxp->status == XE_PXP_ERROR || pxp->status == XE_PXP_SUSPENDED) {
ret = -EIO;
} else {
ret = -EBUSY; /* try again later */
}
mutex_unlock(&pxp->mutex);
return ret;
}
static int pxp_start(struct xe_pxp *pxp, u8 type)
{
int ret = 0;
bool restart;
if (!xe_pxp_is_enabled(pxp))
return -ENODEV;
/* we only support HWDRM sessions right now */
xe_assert(pxp->xe, type == DRM_XE_PXP_TYPE_HWDRM);
/* get_readiness_status() returns 0 for in-progress and 1 for done */
ret = xe_pxp_get_readiness_status(pxp);
if (ret <= 0)
return ret ?: -EBUSY;
ret = 0;
wait_for_idle:
/*
* if there is an action in progress, wait for it. We need to wait
* outside the lock because the completion is done from within the lock.
* Note that the two actions should never be pending at the same time.
*/
if (!wait_for_completion_timeout(&pxp->termination,
msecs_to_jiffies(PXP_TERMINATION_TIMEOUT_MS)))
return -ETIMEDOUT;
if (!wait_for_completion_timeout(&pxp->activation,
msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS)))
return -ETIMEDOUT;
restart = false;
mutex_lock(&pxp->mutex);
/* If PXP is not already active, turn it on */
switch (pxp->status) {
case XE_PXP_ERROR:
ret = -EIO;
goto out_unlock;
case XE_PXP_ACTIVE:
goto out_unlock;
case XE_PXP_READY_TO_START:
pxp->status = XE_PXP_START_IN_PROGRESS;
reinit_completion(&pxp->activation);
break;
case XE_PXP_START_IN_PROGRESS:
/* If a start is in progress then the completion must not be done */
XE_WARN_ON(completion_done(&pxp->activation));
restart = true;
goto out_unlock;
case XE_PXP_NEEDS_TERMINATION:
mark_termination_in_progress(pxp);
break;
case XE_PXP_TERMINATION_IN_PROGRESS:
case XE_PXP_NEEDS_ADDITIONAL_TERMINATION:
/* If a termination is in progress then the completion must not be done */
XE_WARN_ON(completion_done(&pxp->termination));
restart = true;
goto out_unlock;
case XE_PXP_SUSPENDED:
default:
drm_err(&pxp->xe->drm, "unexpected state during PXP start: %u\n", pxp->status);
ret = -EIO;
goto out_unlock;
}
mutex_unlock(&pxp->mutex);
if (!completion_done(&pxp->termination)) {
ret = pxp_terminate_hw(pxp);
if (ret) {
drm_err(&pxp->xe->drm, "PXP termination failed before start\n");
mutex_lock(&pxp->mutex);
pxp->status = XE_PXP_ERROR;
complete_all(&pxp->termination);
goto out_unlock;
}
goto wait_for_idle;
}
/* All the cases except for start should have exited earlier */
XE_WARN_ON(completion_done(&pxp->activation));
ret = __pxp_start_arb_session(pxp);
mutex_lock(&pxp->mutex);
complete_all(&pxp->activation);
/*
* Any other process should wait until the state goes away from
* XE_PXP_START_IN_PROGRESS, so if the state is not that something went
* wrong. Mark the status as needing termination and try again.
*/
if (pxp->status != XE_PXP_START_IN_PROGRESS) {
drm_err(&pxp->xe->drm, "unexpected state after PXP start: %u\n", pxp->status);
pxp->status = XE_PXP_NEEDS_TERMINATION;
restart = true;
goto out_unlock;
}
/* If everything went ok, update the status and add the queue to the list */
if (!ret)
pxp->status = XE_PXP_ACTIVE;
else
pxp->status = XE_PXP_ERROR;
out_unlock:
mutex_unlock(&pxp->mutex);
if (restart)
goto wait_for_idle;
return ret;
}
/**
* xe_pxp_exec_queue_add - add a queue to the PXP list
* @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled)
* @q: the queue to add to the list
*
* If PXP is enabled and the prerequisites are done, start the PXP default
* session (if not already running) and add the queue to the PXP list.
*
* Returns 0 if the PXP session is running and the queue is in the list,
* -ENODEV if PXP is disabled, -EBUSY if the PXP prerequisites are not done,
* other errno value if something goes wrong during the session start.
*/
int xe_pxp_exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q)
{
int ret;
if (!xe_pxp_is_enabled(pxp))
return -ENODEV;
/*
* Runtime suspend kills PXP, so we take a reference to prevent it from
* happening while we have active queues that use PXP
*/
xe_pm_runtime_get(pxp->xe);
start:
ret = pxp_start(pxp, q->pxp.type);
if (!ret) {
ret = __exec_queue_add(pxp, q);
if (ret == -EBUSY)
goto start;
}
/*
* in the successful case the PM ref is released from
* xe_pxp_exec_queue_remove
*/
if (ret)
xe_pm_runtime_put(pxp->xe);
return ret;
}
ALLOW_ERROR_INJECTION(xe_pxp_exec_queue_add, ERRNO);
static void __pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q, bool lock)
{
bool need_pm_put = false;
if (!xe_pxp_is_enabled(pxp))
return;
if (lock)
spin_lock_irq(&pxp->queues.lock);
if (!list_empty(&q->pxp.link)) {
list_del_init(&q->pxp.link);
need_pm_put = true;
}
q->pxp.type = DRM_XE_PXP_TYPE_NONE;
if (lock)
spin_unlock_irq(&pxp->queues.lock);
if (need_pm_put)
xe_pm_runtime_put(pxp->xe);
}
/**
* xe_pxp_exec_queue_remove - remove a queue from the PXP list
* @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled)
* @q: the queue to remove from the list
*
* If PXP is enabled and the exec_queue is in the list, the queue will be
* removed from the list and its PM reference will be released. It is safe to
* call this function multiple times for the same queue.
*/
void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q)
{
__pxp_exec_queue_remove(pxp, q, true);
}
static void pxp_invalidate_queues(struct xe_pxp *pxp)
{
struct xe_exec_queue *tmp, *q;
LIST_HEAD(to_clean);
spin_lock_irq(&pxp->queues.lock);
list_for_each_entry_safe(q, tmp, &pxp->queues.list, pxp.link) {
q = xe_exec_queue_get_unless_zero(q);
if (!q)
continue;
list_move_tail(&q->pxp.link, &to_clean);
}
spin_unlock_irq(&pxp->queues.lock);
list_for_each_entry_safe(q, tmp, &to_clean, pxp.link) {
drm_dbg(&pxp->xe->drm,
"Killing queue due to PXP termination: eclass=%s, guc_id=%d\n",
xe_hw_engine_class_to_str(q->class), q->guc->id);
xe_exec_queue_kill(q);
/*
* We hold a ref to the queue so there is no risk of racing with
* the calls to exec_queue_remove coming from exec_queue_destroy.
*/
__pxp_exec_queue_remove(pxp, q, false);
xe_exec_queue_put(q);
}
}
/**
* xe_pxp_key_assign - mark a BO as using the current PXP key iteration
* @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled)
* @bo: the BO to mark
*
* Returns: -ENODEV if PXP is disabled, 0 otherwise.
*/
int xe_pxp_key_assign(struct xe_pxp *pxp, struct xe_bo *bo)
{
if (!xe_pxp_is_enabled(pxp))
return -ENODEV;
xe_assert(pxp->xe, !bo->pxp_key_instance);
/*
* Note that the PXP key handling is inherently racey, because the key
* can theoretically change at any time (although it's unlikely to do
* so without triggers), even right after we copy it. Taking a lock
* wouldn't help because the value might still change as soon as we
* release the lock.
* Userspace needs to handle the fact that their BOs can go invalid at
* any point.
*/
bo->pxp_key_instance = pxp->key_instance;
return 0;
}
/**
* xe_pxp_bo_key_check - check if the key used by a xe_bo is valid
* @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled)
* @bo: the BO we want to check
*
* Checks whether a BO was encrypted with the current key or an obsolete one.
*
* Returns: 0 if the key is valid, -ENODEV if PXP is disabled, -EINVAL if the
* BO is not using PXP, -ENOEXEC if the key is not valid.
*/
int xe_pxp_bo_key_check(struct xe_pxp *pxp, struct xe_bo *bo)
{
if (!xe_pxp_is_enabled(pxp))
return -ENODEV;
if (!xe_bo_is_protected(bo))
return -EINVAL;
xe_assert(pxp->xe, bo->pxp_key_instance);
/*
* Note that the PXP key handling is inherently racey, because the key
* can theoretically change at any time (although it's unlikely to do
* so without triggers), even right after we check it. Taking a lock
* wouldn't help because the value might still change as soon as we
* release the lock.
* We mitigate the risk by checking the key at multiple points (on each
* submission involving the BO and right before flipping it on the
* display), but there is still a very small chance that we could
* operate on an invalid BO for a single submission or a single frame
* flip. This is a compromise made to protect the encrypted data (which
* is what the key termination is for).
*/
if (bo->pxp_key_instance != pxp->key_instance)
return -ENOEXEC;
return 0;
}
/**
* xe_pxp_obj_key_check - check if the key used by a drm_gem_obj is valid
* @obj: the drm_gem_obj we want to check
*
* Checks whether a drm_gem_obj was encrypted with the current key or an
* obsolete one.
*
* Returns: 0 if the key is valid, -ENODEV if PXP is disabled, -EINVAL if the
* obj is not using PXP, -ENOEXEC if the key is not valid.
*/
int xe_pxp_obj_key_check(struct drm_gem_object *obj)
{
struct xe_bo *bo = gem_to_xe_bo(obj);
struct xe_device *xe = xe_bo_device(bo);
struct xe_pxp *pxp = xe->pxp;
return xe_pxp_bo_key_check(pxp, bo);
}
/**
* xe_pxp_pm_suspend - prepare PXP for HW suspend
* @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled)
*
* Makes sure all PXP actions have completed and invalidates all PXP queues
* and objects before we go into a suspend state.
*
* Returns: 0 if successful, a negative errno value otherwise.
*/
int xe_pxp_pm_suspend(struct xe_pxp *pxp)
{
bool needs_queue_inval = false;
int ret = 0;
if (!xe_pxp_is_enabled(pxp))
return 0;
wait_for_activation:
if (!wait_for_completion_timeout(&pxp->activation,
msecs_to_jiffies(PXP_ACTIVATION_TIMEOUT_MS)))
ret = -ETIMEDOUT;
mutex_lock(&pxp->mutex);
switch (pxp->status) {
case XE_PXP_ERROR:
case XE_PXP_READY_TO_START:
case XE_PXP_SUSPENDED:
case XE_PXP_TERMINATION_IN_PROGRESS:
case XE_PXP_NEEDS_ADDITIONAL_TERMINATION:
/*
* If PXP is not running there is nothing to cleanup. If there
* is a termination pending then no need to issue another one.
*/
break;
case XE_PXP_START_IN_PROGRESS:
mutex_unlock(&pxp->mutex);
goto wait_for_activation;
case XE_PXP_NEEDS_TERMINATION:
/* If PXP was never used we can skip the cleanup */
if (pxp->key_instance == pxp->last_suspend_key_instance)
break;
fallthrough;
case XE_PXP_ACTIVE:
pxp->key_instance++;
pxp->needs_termination_on_resume = true;
needs_queue_inval = true;
break;
}
/*
* We set this even if we were in error state, hoping the suspend clears
* the error. Worse case we fail again and go in error state again.
*/
pxp->status = XE_PXP_SUSPENDED;
mutex_unlock(&pxp->mutex);
if (needs_queue_inval)
pxp_invalidate_queues(pxp);
/*
* if there is a termination in progress, wait for it.
* We need to wait outside the lock because the completion is done from
* within the lock
*/
if (!wait_for_completion_timeout(&pxp->termination,
msecs_to_jiffies(PXP_TERMINATION_TIMEOUT_MS)))
ret = -ETIMEDOUT;
pxp->last_suspend_key_instance = pxp->key_instance;
return ret;
}
/**
* xe_pxp_pm_resume - re-init PXP after HW suspend
* @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled)
*/
void xe_pxp_pm_resume(struct xe_pxp *pxp)
{
bool has_pm = false;
int err;
if (!xe_pxp_is_enabled(pxp))
return;
err = kcr_pxp_enable(pxp);
/*
* We want to avoid the device runtime suspending before we're done with
* the termination queued below, so we need a runtime PM reference; we
* can't call the rpm functions from within the PXP lock, so we take the
* ref here. Note that we don't want the rpm resume code to actually run
* here as that would call back into this function, but as long as we
* don't enable DPM_FLAG_SMART_SUSPEND (which we currently do not) we're
* guaranteed to not be runtime suspended at this point, so we can
* safely use the get_noresume variant.
*/
if (pxp->needs_termination_on_resume) {
has_pm = true;
xe_assert(pxp->xe, !dev_pm_smart_suspend(pxp->xe->drm.dev));
xe_pm_runtime_get_noresume(pxp->xe);
}
mutex_lock(&pxp->mutex);
xe_assert(pxp->xe, pxp->status == XE_PXP_SUSPENDED);
if (err) {
pxp->status = XE_PXP_ERROR;
} else {
pxp->status = XE_PXP_NEEDS_TERMINATION;
if (pxp->needs_termination_on_resume) {
pxp->needs_termination_on_resume = false;
/*
* We can't call pxp_terminate_hw directly from here
* because we're not allowed to do allocations within
* the rpm resume call, so we defer the termination to
* the worker that we use for the termination irqs.
* However, we do not want any PXP ops to go through
* between the suspend completing and the worker
* starting, so we need to do the termination prep
* immediately, which will mark the termination as in
* progress and stall PXP ops.
*/
if (pxp_prep_for_termination(pxp)) {
has_pm = false; /* move PM ref ownership to worker */
atomic_or(PXP_TERMINATION_REQUEST_ON_RESUME, &pxp->events.pending);
queue_work(pxp->events.wq, &pxp->events.work);
}
}
}
mutex_unlock(&pxp->mutex);
if (has_pm)
xe_pm_runtime_put(pxp->xe);
}
