// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
#include "msm_gem.h"
#include "msm_mmu.h"
#include "msm_gpu_trace.h"
#include "a6xx_gpu.h"
#include "a6xx_gmu.xml.h"
#include <linux/bitfield.h>
#include <linux/devfreq.h>
#include <linux/firmware/qcom/qcom_scm.h>
#include <linux/pm_domain.h>
#include <linux/soc/qcom/llcc-qcom.h>
#define GPU_PAS_ID 13
static void a8xx_aperture_slice_set(struct msm_gpu *gpu, enum adreno_pipe pipe, u32 slice)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
u32 val;
val = A8XX_CP_APERTURE_CNTL_HOST_PIPEID(pipe) | A8XX_CP_APERTURE_CNTL_HOST_SLICEID(slice);
if (a6xx_gpu->cached_aperture == val)
return;
gpu_write(gpu, REG_A8XX_CP_APERTURE_CNTL_HOST, val);
a6xx_gpu->cached_aperture = val;
}
static void a8xx_aperture_acquire(struct msm_gpu *gpu, enum adreno_pipe pipe, unsigned long *flags)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
spin_lock_irqsave(&a6xx_gpu->aperture_lock, *flags);
a8xx_aperture_slice_set(gpu, pipe, 0);
}
static void a8xx_aperture_release(struct msm_gpu *gpu, unsigned long flags)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
spin_unlock_irqrestore(&a6xx_gpu->aperture_lock, flags);
}
static void a8xx_aperture_clear(struct msm_gpu *gpu)
{
unsigned long flags;
a8xx_aperture_acquire(gpu, PIPE_NONE, &flags);
a8xx_aperture_release(gpu, flags);
}
static void a8xx_write_pipe(struct msm_gpu *gpu, enum adreno_pipe pipe, u32 offset, u32 data)
{
unsigned long flags;
a8xx_aperture_acquire(gpu, pipe, &flags);
gpu_write(gpu, offset, data);
a8xx_aperture_release(gpu, flags);
}
static u32 a8xx_read_pipe_slice(struct msm_gpu *gpu, enum adreno_pipe pipe, u32 slice, u32 offset)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
unsigned long flags;
u32 val;
spin_lock_irqsave(&a6xx_gpu->aperture_lock, flags);
a8xx_aperture_slice_set(gpu, pipe, slice);
val = gpu_read(gpu, offset);
spin_unlock_irqrestore(&a6xx_gpu->aperture_lock, flags);
return val;
}
void a8xx_gpu_get_slice_info(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
const struct a6xx_info *info = adreno_gpu->info->a6xx;
struct device *dev = &gpu->pdev->dev;
u32 slice_mask;
if (adreno_gpu->info->family < ADRENO_8XX_GEN1)
return;
if (a6xx_gpu->slice_mask)
return;
slice_mask = GENMASK(info->max_slices - 1, 0);
/* GEN1 doesn't support partial slice configurations */
if (adreno_gpu->info->family == ADRENO_8XX_GEN1) {
a6xx_gpu->slice_mask = slice_mask;
return;
}
slice_mask &= a6xx_cx_misc_read(a6xx_gpu,
REG_A8XX_CX_MISC_SLICE_ENABLE_FINAL);
a6xx_gpu->slice_mask = slice_mask;
/* Chip ID depends on the number of slices available. So update it */
adreno_gpu->chip_id |= FIELD_PREP(GENMASK(7, 4), hweight32(slice_mask));
/* Update the gpu-name to reflect the slice config: */
const char *name = devm_kasprintf(dev, GFP_KERNEL,
"%"ADRENO_CHIPID_FMT,
ADRENO_CHIPID_ARGS(adreno_gpu->chip_id));
if (name) {
devm_kfree(dev, adreno_gpu->base.name);
adreno_gpu->base.name = name;
}
}
static u32 a8xx_get_first_slice(struct a6xx_gpu *a6xx_gpu)
{
return ffs(a6xx_gpu->slice_mask) - 1;
}
static inline bool _a8xx_check_idle(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
/* Check that the GMU is idle */
if (!a6xx_gmu_isidle(&a6xx_gpu->gmu))
return false;
/* Check that the CX master is idle */
if (gpu_read(gpu, REG_A8XX_RBBM_STATUS) &
~A8XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
return false;
return !(gpu_read(gpu, REG_A8XX_RBBM_INT_0_STATUS) &
A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
}
static bool a8xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
{
/* wait for CP to drain ringbuffer: */
if (!adreno_idle(gpu, ring))
return false;
if (spin_until(_a8xx_check_idle(gpu))) {
DRM_ERROR(
"%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
gpu->name, __builtin_return_address(0),
gpu_read(gpu, REG_A8XX_RBBM_STATUS),
gpu_read(gpu, REG_A8XX_RBBM_INT_0_STATUS),
gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
return false;
}
return true;
}
void a8xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
uint32_t wptr;
unsigned long flags;
spin_lock_irqsave(&ring->preempt_lock, flags);
/* Copy the shadow to the actual register */
ring->cur = ring->next;
/* Make sure to wrap wptr if we need to */
wptr = get_wptr(ring);
/* Update HW if this is the current ring and we are not in preempt*/
if (!a6xx_in_preempt(a6xx_gpu)) {
if (a6xx_gpu->cur_ring == ring)
a6xx_fenced_write(a6xx_gpu, REG_A6XX_CP_RB_WPTR, wptr, BIT(0), false);
else
ring->restore_wptr = true;
} else {
ring->restore_wptr = true;
}
spin_unlock_irqrestore(&ring->preempt_lock, flags);
}
static void a8xx_set_hwcg(struct msm_gpu *gpu, bool state)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
u32 val;
if (adreno_is_x285(adreno_gpu) && state)
gpu_write(gpu, REG_A8XX_RBBM_CGC_0_PC, 0x00000702);
gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL,
state ? adreno_gpu->info->a6xx->gmu_cgc_mode : 0);
gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL,
state ? 0x110111 : 0);
gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL,
state ? 0x55555 : 0);
gpu_write(gpu, REG_A8XX_RBBM_CLOCK_CNTL_GLOBAL, 1);
gpu_write(gpu, REG_A8XX_RBBM_CGC_GLOBAL_LOAD_CMD, !!state);
if (state) {
gpu_write(gpu, REG_A8XX_RBBM_CGC_P2S_TRIG_CMD, 1);
if (gpu_poll_timeout(gpu, REG_A8XX_RBBM_CGC_P2S_STATUS, val,
val & A8XX_RBBM_CGC_P2S_STATUS_TXDONE, 1, 10)) {
dev_err(&gpu->pdev->dev, "RBBM_CGC_P2S_STATUS TXDONE Poll failed\n");
return;
}
gpu_write(gpu, REG_A8XX_RBBM_CLOCK_CNTL_GLOBAL, 0);
} else {
/*
* GMU enables clk gating in GBIF during boot up. So,
* override that here when hwcg feature is disabled
*/
gpu_rmw(gpu, REG_A6XX_GBIF_CX_CONFIG, BIT(0), 0);
}
}
static void a8xx_set_cp_protect(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
const struct adreno_protect *protect = adreno_gpu->info->a6xx->protect;
u32 cntl, final_cfg;
unsigned int i;
cntl = A8XX_CP_PROTECT_CNTL_PIPE_ACCESS_PROT_EN |
A8XX_CP_PROTECT_CNTL_PIPE_ACCESS_FAULT_ON_VIOL_EN |
A8XX_CP_PROTECT_CNTL_PIPE_LAST_SPAN_INF_RANGE |
A8XX_CP_PROTECT_CNTL_PIPE_HALT_SQE_RANGE__MASK;
/*
* Enable access protection to privileged registers, fault on an access
* protect violation and select the last span to protect from the start
* address all the way to the end of the register address space
*/
a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_CP_PROTECT_CNTL_PIPE, cntl);
a8xx_write_pipe(gpu, PIPE_BV, REG_A8XX_CP_PROTECT_CNTL_PIPE, cntl);
a8xx_aperture_clear(gpu);
for (i = 0; i < protect->count; i++) {
/* Intentionally skip writing to some registers */
if (protect->regs[i]) {
gpu_write(gpu, REG_A8XX_CP_PROTECT_GLOBAL(i), protect->regs[i]);
final_cfg = protect->regs[i];
}
}
/*
* Last span feature is only supported on PIPE specific register.
* So update those here
*/
a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_CP_PROTECT_PIPE(15), final_cfg);
a8xx_write_pipe(gpu, PIPE_BV, REG_A8XX_CP_PROTECT_PIPE(15), final_cfg);
a8xx_aperture_clear(gpu);
}
static void a8xx_set_ubwc_config(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
const struct qcom_ubwc_cfg_data *cfg = adreno_gpu->ubwc_config;
u32 level2_swizzling_dis = !(qcom_ubwc_swizzle(cfg) & UBWC_SWIZZLE_ENABLE_LVL2);
u32 level3_swizzling_dis = !(qcom_ubwc_swizzle(cfg) & UBWC_SWIZZLE_ENABLE_LVL3);
bool rgba8888_lossless = false, fp16compoptdis = false;
bool yuvnotcomptofc = false, min_acc_len_64b = false;
bool rgb565_predicator = false;
bool amsbc = qcom_ubwc_enable_amsbc(cfg);
bool ubwc_mode = qcom_ubwc_get_ubwc_mode(cfg);
u32 ubwc_version = cfg->ubwc_enc_version;
u32 hbb, hbb_hi, hbb_lo, mode;
u8 uavflagprd_inv = 2;
if (ubwc_version > UBWC_6_0)
dev_err(&gpu->pdev->dev, "Unknown UBWC version: 0x%x\n", ubwc_version);
if (ubwc_version == UBWC_6_0)
yuvnotcomptofc = true;
if (ubwc_version < UBWC_5_0 &&
ubwc_version >= UBWC_4_0)
rgba8888_lossless = true;
if (ubwc_version < UBWC_4_3)
fp16compoptdis = true;
if (cfg->ubwc_enc_version >= UBWC_4_0)
rgb565_predicator = true;
if (ubwc_version < UBWC_3_0)
dev_err(&gpu->pdev->dev, "Unsupported UBWC version: 0x%x\n", ubwc_version);
mode = qcom_ubwc_version_tag(cfg);
/*
* We subtract 13 from the highest bank bit (13 is the minimum value
* allowed by hw) and write the lowest two bits of the remaining value
* as hbb_lo and the one above it as hbb_hi to the hardware.
*/
WARN_ON(cfg->highest_bank_bit < 13);
hbb = cfg->highest_bank_bit - 13;
hbb_hi = hbb >> 2;
hbb_lo = hbb & 3;
a8xx_write_pipe(gpu, PIPE_BV, REG_A8XX_GRAS_NC_MODE_CNTL,
hbb << 5 |
level3_swizzling_dis << 4 |
level2_swizzling_dis << 3);
a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_GRAS_NC_MODE_CNTL,
hbb << 5 |
level3_swizzling_dis << 4 |
level2_swizzling_dis << 3);
a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_RB_CCU_NC_MODE_CNTL,
yuvnotcomptofc << 6 |
level3_swizzling_dis << 5 |
level2_swizzling_dis << 4 |
hbb_hi << 3 |
hbb_lo << 1);
a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_RB_CMP_NC_MODE_CNTL,
mode << 15 |
yuvnotcomptofc << 6 |
rgba8888_lossless << 4 |
fp16compoptdis << 3 |
rgb565_predicator << 2 |
amsbc << 1 |
min_acc_len_64b);
a8xx_aperture_clear(gpu);
gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL,
level3_swizzling_dis << 13 |
level2_swizzling_dis << 12 |
hbb_hi << 10 |
uavflagprd_inv << 4 |
min_acc_len_64b << 3 |
hbb_lo << 1 | ubwc_mode);
gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL,
level3_swizzling_dis << 7 |
level2_swizzling_dis << 6 |
hbb_hi << 4 |
min_acc_len_64b << 3 |
hbb_lo << 1 | ubwc_mode);
}
static void a8xx_nonctxt_config(struct msm_gpu *gpu, u32 *gmem_protect)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
const struct a6xx_info *info = adreno_gpu->info->a6xx;
const struct adreno_reglist_pipe *regs = info->nonctxt_reglist;
unsigned int pipe_id, i;
unsigned long flags;
for (pipe_id = PIPE_NONE; pipe_id <= PIPE_DDE_BV; pipe_id++) {
/* We don't have support for LPAC yet */
if (pipe_id == PIPE_LPAC)
continue;
a8xx_aperture_acquire(gpu, pipe_id, &flags);
for (i = 0; regs[i].offset; i++) {
if (!(BIT(pipe_id) & regs[i].pipe))
continue;
if (regs[i].offset == REG_A8XX_RB_GC_GMEM_PROTECT)
*gmem_protect = regs[i].value;
gpu_write(gpu, regs[i].offset, regs[i].value);
}
a8xx_aperture_release(gpu, flags);
}
a8xx_aperture_clear(gpu);
}
static void a8xx_patch_pwrup_reglist(struct msm_gpu *gpu)
{
const struct adreno_reglist_pipe_list *dyn_pwrup_reglist;
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
const struct adreno_reglist_list *reglist;
void *ptr = a6xx_gpu->pwrup_reglist_ptr;
struct cpu_gpu_lock *lock = ptr;
u32 *dest = (u32 *)&lock->regs[0];
u32 dyn_pwrup_reglist_count = 0;
int i;
lock->gpu_req = lock->cpu_req = lock->turn = 0;
reglist = adreno_gpu->info->a6xx->ifpc_reglist;
if (reglist) {
lock->ifpc_list_len = reglist->count;
/*
* For each entry in each of the lists, write the offset and the current
* register value into the GPU buffer
*/
for (i = 0; i < reglist->count; i++) {
*dest++ = reglist->regs[i];
*dest++ = gpu_read(gpu, reglist->regs[i]);
}
}
reglist = adreno_gpu->info->a6xx->pwrup_reglist;
if (reglist) {
lock->preemption_list_len = reglist->count;
for (i = 0; i < reglist->count; i++) {
*dest++ = reglist->regs[i];
*dest++ = gpu_read(gpu, reglist->regs[i]);
}
}
/*
* The overall register list is composed of
* 1. Static IFPC-only registers
* 2. Static IFPC + preemption registers
* 3. Dynamic IFPC + preemption registers (ex: perfcounter selects)
*
* The first two lists are static. Size of these lists are stored as
* number of pairs in ifpc_list_len and preemption_list_len
* respectively. With concurrent binning, Some of the perfcounter
* registers being virtualized, CP needs to know the pipe id to program
* the aperture inorder to restore the same. Thus, third list is a
* dynamic list with triplets as
* (<aperture, shifted 12 bits> <address> <data>), and the length is
* stored as number for triplets in dynamic_list_len.
*/
dyn_pwrup_reglist = adreno_gpu->info->a6xx->dyn_pwrup_reglist;
if (!dyn_pwrup_reglist)
goto done;
for (u32 pipe_id = PIPE_BR; pipe_id <= PIPE_DDE_BV; pipe_id++) {
for (i = 0; i < dyn_pwrup_reglist->count; i++) {
if (!(dyn_pwrup_reglist->regs[i].pipe & BIT(pipe_id)))
continue;
*dest++ = A8XX_CP_APERTURE_CNTL_HOST_PIPEID(pipe_id);
*dest++ = dyn_pwrup_reglist->regs[i].offset;
*dest++ = a8xx_read_pipe_slice(gpu,
pipe_id,
a8xx_get_first_slice(a6xx_gpu),
dyn_pwrup_reglist->regs[i].offset);
dyn_pwrup_reglist_count++;
}
}
lock->dynamic_list_len = dyn_pwrup_reglist_count;
a6xx_gpu->dynamic_sel_reglist_offset = dyn_pwrup_reglist_count;
done:
a8xx_aperture_clear(gpu);
}
static int a8xx_preempt_start(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
struct msm_ringbuffer *ring = gpu->rb[0];
if (gpu->nr_rings <= 1)
return 0;
/* Turn CP protection off */
OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
OUT_RING(ring, 0);
a6xx_emit_set_pseudo_reg(ring, a6xx_gpu, NULL);
a6xx_flush_yield(gpu, ring);
return a8xx_idle(gpu, ring) ? 0 : -EINVAL;
}
static int a8xx_cp_init(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
struct msm_ringbuffer *ring = gpu->rb[0];
u32 mask;
/* Disable concurrent binning before sending CP init */
OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
OUT_RING(ring, BIT(27));
OUT_PKT7(ring, CP_ME_INIT, 7);
/* Use multiple HW contexts */
mask = BIT(0);
/* Enable error detection */
mask |= BIT(1);
/* Set default reset state */
mask |= BIT(3);
/* Disable save/restore of performance counters across preemption */
mask |= BIT(6);
/* Enable the register init list with the spinlock */
mask |= BIT(8);
OUT_RING(ring, mask);
/* Enable multiple hardware contexts */
OUT_RING(ring, 0x00000003);
/* Enable error detection */
OUT_RING(ring, 0x20000000);
/* Operation mode mask */
OUT_RING(ring, 0x00000002);
/* Lo address */
OUT_RING(ring, lower_32_bits(a6xx_gpu->pwrup_reglist_iova));
/* Hi address */
OUT_RING(ring, upper_32_bits(a6xx_gpu->pwrup_reglist_iova));
/* Enable dyn pwrup list with triplets (offset, value, pipe) */
OUT_RING(ring, BIT(31));
a6xx_flush(gpu, ring);
return a8xx_idle(gpu, ring) ? 0 : -EINVAL;
}
#define A8XX_INT_MASK \
(A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR | \
A6XX_RBBM_INT_0_MASK_CP_SW | \
A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT | \
A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS | \
A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR | \
A6XX_RBBM_INT_0_MASK_TSBWRITEERROR | \
A6XX_RBBM_INT_0_MASK_SWFUSEVIOLATION)
#define A8XX_APRIV_MASK \
(A8XX_CP_APRIV_CNTL_PIPE_ICACHE | \
A8XX_CP_APRIV_CNTL_PIPE_RBFETCH | \
A8XX_CP_APRIV_CNTL_PIPE_RBPRIVLEVEL | \
A8XX_CP_APRIV_CNTL_PIPE_RBRPWB)
#define A8XX_BR_APRIV_MASK \
(A8XX_APRIV_MASK | \
A8XX_CP_APRIV_CNTL_PIPE_CDREAD | \
A8XX_CP_APRIV_CNTL_PIPE_CDWRITE)
#define A8XX_CP_GLOBAL_INT_MASK \
(A8XX_CP_GLOBAL_INT_MASK_HWFAULTBR | \
A8XX_CP_GLOBAL_INT_MASK_HWFAULTBV | \
A8XX_CP_GLOBAL_INT_MASK_HWFAULTLPAC | \
A8XX_CP_GLOBAL_INT_MASK_HWFAULTAQE0 | \
A8XX_CP_GLOBAL_INT_MASK_HWFAULTAQE1 | \
A8XX_CP_GLOBAL_INT_MASK_HWFAULTDDEBR | \
A8XX_CP_GLOBAL_INT_MASK_HWFAULTDDEBV | \
A8XX_CP_GLOBAL_INT_MASK_SWFAULTBR | \
A8XX_CP_GLOBAL_INT_MASK_SWFAULTBV | \
A8XX_CP_GLOBAL_INT_MASK_SWFAULTLPAC | \
A8XX_CP_GLOBAL_INT_MASK_SWFAULTAQE0 | \
A8XX_CP_GLOBAL_INT_MASK_SWFAULTAQE1 | \
A8XX_CP_GLOBAL_INT_MASK_SWFAULTDDEBR | \
A8XX_CP_GLOBAL_INT_MASK_SWFAULTDDEBV)
#define A8XX_CP_INTERRUPT_STATUS_MASK_PIPE \
(A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFRBWRAP | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFIB1WRAP | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFIB2WRAP | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFIB3WRAP | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFSDSWRAP | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFMRBWRAP | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_CSFVSDWRAP | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_OPCODEERROR | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VSDPARITYERROR | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_REGISTERPROTECTIONERROR | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_ILLEGALINSTRUCTION | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_SMMUFAULT | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VBIFRESPCLIENT| \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VBIFRESPTYPE | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VBIFRESPREAD | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_VBIFRESP | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_RTWROVF | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_LRZRTWROVF | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_LRZRTREFCNTOVF | \
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE_LRZRTCLRRESMISS)
#define A8XX_CP_HW_FAULT_STATUS_MASK_PIPE \
(A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFRBFAULT | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFIB1FAULT | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFIB2FAULT | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFIB3FAULT | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFSDSFAULT | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFMRBFAULT | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_CSFVSDFAULT | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_SQEREADBURSTOVF | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_EVENTENGINEOVF | \
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE_UCODEERROR)
static int hw_init(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
unsigned int pipe_id, i;
u32 gmem_protect = 0;
u64 gmem_range_min;
int ret;
ret = a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
if (ret)
return ret;
/* Clear the cached value to force aperture configuration next time */
a6xx_gpu->cached_aperture = UINT_MAX;
a8xx_aperture_clear(gpu);
/* Clear GBIF halt in case GX domain was not collapsed */
gpu_write(gpu, REG_A6XX_GBIF_HALT, 0);
gpu_read(gpu, REG_A6XX_GBIF_HALT);
gpu_write(gpu, REG_A8XX_RBBM_GBIF_HALT, 0);
gpu_read(gpu, REG_A8XX_RBBM_GBIF_HALT);
gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
/*
* Disable the trusted memory range - we don't actually supported secure
* memory rendering at this point in time and we don't want to block off
* part of the virtual memory space.
*/
gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE, 0x00000000);
gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
/* Make all blocks contribute to the GPU BUSY perf counter */
gpu_write(gpu, REG_A8XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
/* Setup GMEM Range in UCHE */
gmem_range_min = SZ_64M;
/* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
gpu_write64(gpu, REG_A8XX_UCHE_CCHE_GC_GMEM_RANGE_MIN, gmem_range_min);
gpu_write64(gpu, REG_A8XX_SP_HLSQ_GC_GMEM_RANGE_MIN, gmem_range_min);
/* Setup UCHE Trap region */
gpu_write64(gpu, REG_A8XX_UCHE_TRAP_BASE, adreno_gpu->uche_trap_base);
gpu_write64(gpu, REG_A8XX_UCHE_WRITE_THRU_BASE, adreno_gpu->uche_trap_base);
gpu_write64(gpu, REG_A8XX_UCHE_CCHE_TRAP_BASE, adreno_gpu->uche_trap_base);
gpu_write64(gpu, REG_A8XX_UCHE_CCHE_WRITE_THRU_BASE, adreno_gpu->uche_trap_base);
/* Turn on performance counters */
gpu_write(gpu, REG_A8XX_RBBM_PERFCTR_CNTL, 0x1);
gpu_write(gpu, REG_A8XX_RBBM_SLICE_PERFCTR_CNTL, 0x1);
/* Turn on the IFPC counter (countable 4 on XOCLK1) */
gmu_write(&a6xx_gpu->gmu, REG_A8XX_GMU_CX_GMU_POWER_COUNTER_SELECT_XOCLK_1,
FIELD_PREP(GENMASK(7, 0), 0x4));
/* Select CP0 to always count cycles */
gpu_write(gpu, REG_A8XX_CP_PERFCTR_CP_SEL(0), 1);
a8xx_set_ubwc_config(gpu);
/* Set weights for bicubic filtering */
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(0), 0);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(1), 0x3fe05ff4);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(2), 0x3fa0ebee);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(3), 0x3f5193ed);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(4), 0x3f0243f0);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(5), 0x00000000);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(6), 0x3fd093e8);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(7), 0x3f4133dc);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(8), 0x3ea1dfdb);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(9), 0x3e0283e0);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(10), 0x0000ac2b);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(11), 0x0000f01d);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(12), 0x00114412);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(13), 0x0021980a);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(14), 0x0051ec05);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(15), 0x0000380e);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(16), 0x3ff09001);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(17), 0x3fc10bfa);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(18), 0x3f9193f7);
gpu_write(gpu, REG_A8XX_TPL1_BICUBIC_WEIGHTS_TABLE(19), 0x3f7227f7);
gpu_write(gpu, REG_A8XX_UCHE_CLIENT_PF, BIT(7) | 0x1);
a8xx_nonctxt_config(gpu, &gmem_protect);
/* Enable fault detection */
gpu_write(gpu, REG_A8XX_RBBM_INTERFACE_HANG_INT_CNTL, BIT(30) | 0xcfffff);
gpu_write(gpu, REG_A8XX_RBBM_SLICE_INTERFACE_HANG_INT_CNTL, BIT(30));
/* Set up the CX GMU counter 0 to count busy ticks */
gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xff000000);
/* Enable the power counter */
gmu_rmw(gmu, REG_A8XX_GMU_CX_GMU_POWER_COUNTER_SELECT_XOCLK_0, 0xff, BIT(5));
gmu_write(gmu, REG_A8XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1);
/* Protect registers from the CP */
a8xx_set_cp_protect(gpu);
/* Enable the GMEM save/restore feature for preemption */
a8xx_write_pipe(gpu, PIPE_BR, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE, 1);
for (pipe_id = PIPE_BR; pipe_id <= PIPE_DDE_BV; pipe_id++) {
u32 apriv_mask = A8XX_APRIV_MASK;
unsigned long flags;
if (pipe_id == PIPE_LPAC)
continue;
if (pipe_id == PIPE_BR)
apriv_mask = A8XX_BR_APRIV_MASK;
a8xx_aperture_acquire(gpu, pipe_id, &flags);
gpu_write(gpu, REG_A8XX_CP_APRIV_CNTL_PIPE, apriv_mask);
gpu_write(gpu, REG_A8XX_CP_INTERRUPT_STATUS_MASK_PIPE,
A8XX_CP_INTERRUPT_STATUS_MASK_PIPE);
gpu_write(gpu, REG_A8XX_CP_HW_FAULT_STATUS_MASK_PIPE,
A8XX_CP_HW_FAULT_STATUS_MASK_PIPE);
a8xx_aperture_release(gpu, flags);
}
a8xx_aperture_clear(gpu);
/* Enable interrupts */
gpu_write(gpu, REG_A8XX_CP_INTERRUPT_STATUS_MASK_GLOBAL, A8XX_CP_GLOBAL_INT_MASK);
gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, A8XX_INT_MASK);
ret = adreno_hw_init(gpu);
if (ret)
goto out;
gpu_write64(gpu, REG_A8XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova);
if (a6xx_gpu->aqe_iova)
gpu_write64(gpu, REG_A8XX_CP_AQE_INSTR_BASE_0, a6xx_gpu->aqe_iova);
/* Set the ringbuffer address */
gpu_write64(gpu, REG_A6XX_CP_RB_BASE, gpu->rb[0]->iova);
gpu_write(gpu, REG_A6XX_CP_RB_CNTL, MSM_GPU_RB_CNTL_DEFAULT);
/* Configure the RPTR shadow if needed: */
gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR, shadowptr(a6xx_gpu, gpu->rb[0]));
gpu_write64(gpu, REG_A8XX_CP_RB_RPTR_ADDR_BV, rbmemptr(gpu->rb[0], bv_rptr));
a8xx_preempt_hw_init(gpu);
for (i = 0; i < gpu->nr_rings; i++)
a6xx_gpu->shadow[i] = 0;
/* Always come up on rb 0 */
a6xx_gpu->cur_ring = gpu->rb[0];
for (i = 0; i < gpu->nr_rings; i++)
gpu->rb[i]->cur_ctx_seqno = 0;
/* Enable the SQE_to start the CP engine */
gpu_write(gpu, REG_A8XX_CP_SQE_CNTL, 1);
ret = a8xx_cp_init(gpu);
if (ret)
goto out;
/*
* Try to load a zap shader into the secure world. If successful
* we can use the CP to switch out of secure mode. If not then we
* have no resource but to try to switch ourselves out manually. If we
* guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
* be blocked and a permissions violation will soon follow.
*/
ret = a6xx_zap_shader_init(gpu);
if (!ret) {
OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
OUT_RING(gpu->rb[0], 0x00000000);
a6xx_flush(gpu, gpu->rb[0]);
if (!a8xx_idle(gpu, gpu->rb[0]))
return -EINVAL;
} else if (ret == -ENODEV) {
/*
* This device does not use zap shader (but print a warning
* just in case someone got their dt wrong.. hopefully they
* have a debug UART to realize the error of their ways...
* if you mess this up you are about to crash horribly)
*/
dev_warn_once(gpu->dev->dev,
"Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
ret = 0;
} else {
return ret;
}
/*
* GMEM_PROTECT register should be programmed after GPU is transitioned to
* non-secure mode
*/
a8xx_write_pipe(gpu, PIPE_BR, REG_A8XX_RB_GC_GMEM_PROTECT, gmem_protect);
WARN_ON(!gmem_protect);
a8xx_aperture_clear(gpu);
if (!a6xx_gpu->pwrup_reglist_emitted) {
a8xx_patch_pwrup_reglist(gpu);
a6xx_gpu->pwrup_reglist_emitted = true;
}
/* Enable hardware clockgating */
a8xx_set_hwcg(gpu, true);
out:
/* Last step - yield the ringbuffer */
a8xx_preempt_start(gpu);
/*
* Tell the GMU that we are done touching the GPU and it can start power
* management
*/
a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
if (!ret && msm_gpu_sysprof_no_perfcntr_zap(gpu)) {
ret = a6xx_gmu_set_oob(gmu, GMU_OOB_PERFCOUNTER_SET);
if (!ret)
set_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status);
}
return ret;
}
int a8xx_hw_init(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
int ret;
mutex_lock(&a6xx_gpu->gmu.lock);
ret = hw_init(gpu);
mutex_unlock(&a6xx_gpu->gmu.lock);
return ret;
}
static void a8xx_dump(struct msm_gpu *gpu)
{
DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n", gpu_read(gpu, REG_A8XX_RBBM_STATUS));
adreno_dump(gpu);
}
void a8xx_recover(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
int active_submits;
adreno_dump_info(gpu);
/*
* To handle recovery specific sequences during the rpm suspend we are
* about to trigger
*/
a6xx_gpu->hung = true;
if (adreno_gpu->funcs->gx_is_on(adreno_gpu)) {
/*
* Sometimes crashstate capture is skipped, so SQE should be
* halted here again
*/
gpu_write(gpu, REG_A8XX_CP_SQE_CNTL, 3);
if (hang_debug)
a8xx_dump(gpu);
}
pm_runtime_dont_use_autosuspend(&gpu->pdev->dev);
/* active_submit won't change until we make a submission */
mutex_lock(&gpu->active_lock);
active_submits = gpu->active_submits;
/*
* Temporarily clear active_submits count to silence a WARN() in the
* runtime suspend cb
*/
gpu->active_submits = 0;
reinit_completion(&gmu->pd_gate);
dev_pm_genpd_add_notifier(gmu->cxpd, &gmu->pd_nb);
dev_pm_genpd_synced_poweroff(gmu->cxpd);
/* Drop the rpm refcount from active submits */
if (active_submits)
pm_runtime_put(&gpu->pdev->dev);
/* And the final one from recover worker */
pm_runtime_put_sync(&gpu->pdev->dev);
if (!wait_for_completion_timeout(&gmu->pd_gate, msecs_to_jiffies(1000)))
DRM_DEV_ERROR(&gpu->pdev->dev, "cx gdsc didn't collapse\n");
dev_pm_genpd_remove_notifier(gmu->cxpd);
pm_runtime_use_autosuspend(&gpu->pdev->dev);
if (active_submits)
pm_runtime_get(&gpu->pdev->dev);
pm_runtime_get_sync(&gpu->pdev->dev);
gpu->active_submits = active_submits;
mutex_unlock(&gpu->active_lock);
msm_gpu_hw_init(gpu);
a6xx_gpu->hung = false;
}
static const char *a8xx_uche_fault_block(struct msm_gpu *gpu, u32 mid)
{
static const char * const uche_clients[] = {
"BR_VFD", "BR_SP", "BR_VSC", "BR_VPC", "BR_HLSQ", "BR_PC", "BR_LRZ", "BR_TP",
"BV_VFD", "BV_SP", "BV_VSC", "BV_VPC", "BV_HLSQ", "BV_PC", "BV_LRZ", "BV_TP",
"STCHE",
};
static const char * const uche_clients_lpac[] = {
"-", "SP_LPAC", "-", "-", "HLSQ_LPAC", "-", "-", "TP_LPAC",
};
u32 val;
/*
* The source of the data depends on the mid ID read from FSYNR1.
* and the client ID read from the UCHE block
*/
val = gpu_read(gpu, REG_A8XX_UCHE_CLIENT_PF);
val &= GENMASK(6, 0);
/* mid=3 refers to BR or BV */
if (mid == 3) {
if (val < ARRAY_SIZE(uche_clients))
return uche_clients[val];
else
return "UCHE";
}
/* mid=8 refers to LPAC */
if (mid == 8) {
if (val < ARRAY_SIZE(uche_clients_lpac))
return uche_clients_lpac[val];
else
return "UCHE_LPAC";
}
return "Unknown";
}
static const char *a8xx_fault_block(struct msm_gpu *gpu, u32 id)
{
switch (id) {
case 0x0:
return "CP";
case 0x1:
return "UCHE: Unknown";
case 0x2:
return "UCHE_LPAC: Unknown";
case 0x3:
case 0x8:
return a8xx_uche_fault_block(gpu, id);
case 0x4:
return "CCU";
case 0x5:
return "Flag cache";
case 0x6:
return "PREFETCH";
case 0x7:
return "GMU";
case 0x9:
return "UCHE_HPAC";
}
return "Unknown";
}
int a8xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
{
struct msm_gpu *gpu = arg;
struct adreno_smmu_fault_info *info = data;
const char *block = "unknown";
u32 scratch[] = {
gpu_read(gpu, REG_A8XX_CP_SCRATCH_GLOBAL(0)),
gpu_read(gpu, REG_A8XX_CP_SCRATCH_GLOBAL(1)),
gpu_read(gpu, REG_A8XX_CP_SCRATCH_GLOBAL(2)),
gpu_read(gpu, REG_A8XX_CP_SCRATCH_GLOBAL(3)),
};
if (info)
block = a8xx_fault_block(gpu, info->fsynr1 & 0xff);
return adreno_fault_handler(gpu, iova, flags, info, block, scratch);
}
static void a8xx_cp_hw_err_irq(struct msm_gpu *gpu)
{
u32 status = gpu_read(gpu, REG_A8XX_CP_INTERRUPT_STATUS_GLOBAL);
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
u32 slice = a8xx_get_first_slice(a6xx_gpu);
u32 hw_fault_mask = GENMASK(6, 0);
u32 sw_fault_mask = GENMASK(22, 16);
u32 pipe = 0;
dev_err_ratelimited(&gpu->pdev->dev, "CP Fault Global INT status: 0x%x\n", status);
if (status & (A8XX_CP_GLOBAL_INT_MASK_HWFAULTBR |
A8XX_CP_GLOBAL_INT_MASK_SWFAULTBR))
pipe |= BIT(PIPE_BR);
if (status & (A8XX_CP_GLOBAL_INT_MASK_HWFAULTBV |
A8XX_CP_GLOBAL_INT_MASK_SWFAULTBV))
pipe |= BIT(PIPE_BV);
if (!pipe) {
dev_err_ratelimited(&gpu->pdev->dev, "CP Fault Unknown pipe\n");
goto out;
}
for (unsigned int pipe_id = PIPE_NONE; pipe_id <= PIPE_DDE_BV; pipe_id++) {
if (!(BIT(pipe_id) & pipe))
continue;
if (hw_fault_mask & status) {
status = a8xx_read_pipe_slice(gpu, pipe_id, slice,
REG_A8XX_CP_HW_FAULT_STATUS_PIPE);
dev_err_ratelimited(&gpu->pdev->dev,
"CP HW FAULT pipe: %u status: 0x%x\n", pipe_id, status);
}
if (sw_fault_mask & status) {
status = a8xx_read_pipe_slice(gpu, pipe_id, slice,
REG_A8XX_CP_INTERRUPT_STATUS_PIPE);
dev_err_ratelimited(&gpu->pdev->dev,
"CP SW FAULT pipe: %u status: 0x%x\n", pipe_id, status);
if (status & BIT(8)) {
a8xx_write_pipe(gpu, pipe_id, REG_A8XX_CP_SQE_STAT_ADDR_PIPE, 1);
status = a8xx_read_pipe_slice(gpu, pipe_id, slice,
REG_A8XX_CP_SQE_STAT_DATA_PIPE);
dev_err_ratelimited(&gpu->pdev->dev,
"CP Opcode error, opcode=0x%x\n", status);
}
if (status & BIT(10)) {
status = a8xx_read_pipe_slice(gpu, pipe_id, slice,
REG_A8XX_CP_PROTECT_STATUS_PIPE);
dev_err_ratelimited(&gpu->pdev->dev,
"CP REG PROTECT error, status=0x%x\n", status);
}
}
}
out:
/* Turn off interrupts to avoid triggering recovery again */
a8xx_aperture_clear(gpu);
gpu_write(gpu, REG_A8XX_CP_INTERRUPT_STATUS_MASK_GLOBAL, 0);
gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, 0);
kthread_queue_work(gpu->worker, &gpu->recover_work);
}
static u32 gpu_periph_read(struct msm_gpu *gpu, u32 dbg_offset)
{
gpu_write(gpu, REG_A8XX_CP_SQE_UCODE_DBG_ADDR_PIPE, dbg_offset);
return gpu_read(gpu, REG_A8XX_CP_SQE_UCODE_DBG_DATA_PIPE);
}
static u64 gpu_periph_read64(struct msm_gpu *gpu, u32 dbg_offset)
{
u64 lo, hi;
lo = gpu_periph_read(gpu, dbg_offset);
hi = gpu_periph_read(gpu, dbg_offset + 1);
return (hi << 32) | lo;
}
#define CP_PERIPH_IB1_BASE_LO 0x7005
#define CP_PERIPH_IB1_BASE_HI 0x7006
#define CP_PERIPH_IB1_SIZE 0x7007
#define CP_PERIPH_IB1_OFFSET 0x7008
#define CP_PERIPH_IB2_BASE_LO 0x7009
#define CP_PERIPH_IB2_BASE_HI 0x700a
#define CP_PERIPH_IB2_SIZE 0x700b
#define CP_PERIPH_IB2_OFFSET 0x700c
#define CP_PERIPH_IB3_BASE_LO 0x700d
#define CP_PERIPH_IB3_BASE_HI 0x700e
#define CP_PERIPH_IB3_SIZE 0x700f
#define CP_PERIPH_IB3_OFFSET 0x7010
static void a8xx_fault_detect_irq(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
unsigned long flags;
/*
* If stalled on SMMU fault, we could trip the GPU's hang detection,
* but the fault handler will trigger the devcore dump, and we want
* to otherwise resume normally rather than killing the submit, so
* just bail.
*/
if (gpu_read(gpu, REG_A8XX_RBBM_MISC_STATUS) & A8XX_RBBM_MISC_STATUS_SMMU_STALLED_ON_FAULT)
return;
/*
* Force the GPU to stay on until after we finish
* collecting information
*/
if (!adreno_has_gmu_wrapper(adreno_gpu))
gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);
DRM_DEV_ERROR(&gpu->pdev->dev,
"gpu fault ring %d fence %x status %8.8X gfx_status %8.8X\n",
ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0,
gpu_read(gpu, REG_A8XX_RBBM_STATUS), gpu_read(gpu, REG_A8XX_RBBM_GFX_STATUS));
a8xx_aperture_acquire(gpu, PIPE_BR, &flags);
DRM_DEV_ERROR(&gpu->pdev->dev,
"BR: status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x ib3 %16.16llX/%4.4x\n",
gpu_read(gpu, REG_A8XX_RBBM_GFX_BR_STATUS),
gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
gpu_periph_read64(gpu, CP_PERIPH_IB1_BASE_LO),
gpu_periph_read(gpu, CP_PERIPH_IB1_OFFSET),
gpu_periph_read64(gpu, CP_PERIPH_IB2_BASE_LO),
gpu_periph_read(gpu, CP_PERIPH_IB2_OFFSET),
gpu_periph_read64(gpu, CP_PERIPH_IB3_BASE_LO),
gpu_periph_read(gpu, CP_PERIPH_IB3_OFFSET));
a8xx_aperture_release(gpu, flags);
a8xx_aperture_acquire(gpu, PIPE_BV, &flags);
DRM_DEV_ERROR(&gpu->pdev->dev,
"BV: status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x ib3 %16.16llX/%4.4x\n",
gpu_read(gpu, REG_A8XX_RBBM_GFX_BV_STATUS),
gpu_read(gpu, REG_A8XX_CP_RB_RPTR_BV),
gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
gpu_periph_read64(gpu, CP_PERIPH_IB1_BASE_LO),
gpu_periph_read(gpu, CP_PERIPH_IB1_OFFSET),
gpu_periph_read64(gpu, CP_PERIPH_IB2_BASE_LO),
gpu_periph_read(gpu, CP_PERIPH_IB2_OFFSET),
gpu_periph_read64(gpu, CP_PERIPH_IB3_BASE_LO),
gpu_periph_read(gpu, CP_PERIPH_IB3_OFFSET));
a8xx_aperture_release(gpu, flags);
a8xx_aperture_clear(gpu);
/* Turn off the hangcheck timer to keep it from bothering us */
timer_delete(&gpu->hangcheck_timer);
kthread_queue_work(gpu->worker, &gpu->recover_work);
}
static void a8xx_sw_fuse_violation_irq(struct msm_gpu *gpu)
{
u32 status;
status = gpu_read(gpu, REG_A8XX_RBBM_SW_FUSE_INT_STATUS);
gpu_write(gpu, REG_A8XX_RBBM_SW_FUSE_INT_MASK, 0);
dev_err_ratelimited(&gpu->pdev->dev, "SW fuse violation status=%8.8x\n", status);
/*
* Ignore FASTBLEND violations, because the HW will silently fall back
* to legacy blending.
*/
if (status & (A7XX_CX_MISC_SW_FUSE_VALUE_RAYTRACING |
A7XX_CX_MISC_SW_FUSE_VALUE_LPAC)) {
timer_delete(&gpu->hangcheck_timer);
kthread_queue_work(gpu->worker, &gpu->recover_work);
}
}
irqreturn_t a8xx_irq(struct msm_gpu *gpu)
{
struct msm_drm_private *priv = gpu->dev->dev_private;
u32 status = gpu_read(gpu, REG_A8XX_RBBM_INT_0_STATUS);
gpu_write(gpu, REG_A8XX_RBBM_INT_CLEAR_CMD, status);
if (priv->disable_err_irq) {
/* Turn off interrupts to avoid interrupt storm */
gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS);
status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS;
}
if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
a8xx_fault_detect_irq(gpu);
if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR) {
u32 rl0, rl1;
rl0 = gpu_read(gpu, REG_A8XX_CP_RL_ERROR_DETAILS_0);
rl1 = gpu_read(gpu, REG_A8XX_CP_RL_ERROR_DETAILS_1);
dev_err_ratelimited(&gpu->pdev->dev,
"CP | AHB bus error RL_ERROR_0: %x, RL_ERROR_1: %x\n", rl0, rl1);
}
if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
a8xx_cp_hw_err_irq(gpu);
if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
if (status & A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Trap interrupt\n");
if (status & A6XX_RBBM_INT_0_MASK_SWFUSEVIOLATION)
a8xx_sw_fuse_violation_irq(gpu);
if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) {
msm_gpu_retire(gpu);
a8xx_preempt_trigger(gpu);
}
if (status & A6XX_RBBM_INT_0_MASK_CP_SW)
a8xx_preempt_irq(gpu);
return IRQ_HANDLED;
}
void a8xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
{
struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
struct msm_gpu *gpu = &adreno_gpu->base;
if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {
u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
gpu_scid &= GENMASK(5, 0);
gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL1,
FIELD_PREP(GENMASK(29, 24), gpu_scid) |
FIELD_PREP(GENMASK(23, 18), gpu_scid) |
FIELD_PREP(GENMASK(17, 12), gpu_scid) |
FIELD_PREP(GENMASK(11, 6), gpu_scid) |
FIELD_PREP(GENMASK(5, 0), gpu_scid));
gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL0,
FIELD_PREP(GENMASK(27, 22), gpu_scid) |
FIELD_PREP(GENMASK(21, 16), gpu_scid) |
FIELD_PREP(GENMASK(15, 10), gpu_scid) |
BIT(8));
}
llcc_slice_activate(a6xx_gpu->htw_llc_slice);
}
#define GBIF_CLIENT_HALT_MASK BIT(0)
#define GBIF_ARB_HALT_MASK BIT(1)
#define VBIF_XIN_HALT_CTRL0_MASK GENMASK(3, 0)
#define VBIF_RESET_ACK_MASK 0xF0
#define GPR0_GBIF_HALT_REQUEST 0x1E0
void a8xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off)
{
struct msm_gpu *gpu = &adreno_gpu->base;
if (gx_off) {
/* Halt the gx side of GBIF */
gpu_write(gpu, REG_A8XX_RBBM_GBIF_HALT, 1);
spin_until(gpu_read(gpu, REG_A8XX_RBBM_GBIF_HALT_ACK) & 1);
}
/* Halt new client requests on GBIF */
gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK);
spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
(GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK);
/* Halt all AXI requests on GBIF */
gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK);
spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
(GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK);
/* The GBIF halt needs to be explicitly cleared */
gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0);
}
u64 a8xx_gmu_get_timestamp(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
u64 count_hi, count_lo, temp;
do {
count_hi = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_H);
count_lo = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_L);
temp = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_H);
} while (unlikely(count_hi != temp));
return (count_hi << 32) | count_lo;
}
u64 a8xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
u64 busy_cycles;
/* 19.2MHz */
*out_sample_rate = 19200000;
busy_cycles = gmu_read64(&a6xx_gpu->gmu,
REG_A8XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L,
REG_A8XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
return busy_cycles;
}
bool a8xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
{
return true;
}
void a8xx_perfcntr_flush(struct msm_gpu *gpu)
{
u32 val;
/*
* Flush delta counters (both perf counters and pipe stats) present in
* RBBM_S and RBBM_US to perf RAM logic to get the latest data.
*/
gpu_write(gpu, REG_A8XX_RBBM_PERFCTR_FLUSH_HOST_CMD, BIT(0));
gpu_write(gpu, REG_A8XX_RBBM_SLICE_PERFCTR_FLUSH_HOST_CMD, BIT(0));
/* Ensure all writes are posted before polling status register */
wmb();
if (gpu_poll_timeout(gpu, REG_A8XX_RBBM_PERFCTR_FLUSH_HOST_STATUS, val,
val & BIT(0), 100, 100 * 1000)) {
dev_err(&gpu->pdev->dev, "Perfcounter flush timed out: status=0x%08x\n", val);
}
}
