// SPDX-License-Identifier: GPL-2.0-only
/*
* CXL EDAC memory feature driver.
*
* Copyright (c) 2024-2025 HiSilicon Limited.
*
* - Supports functions to configure EDAC features of the
* CXL memory devices.
* - Registers with the EDAC device subsystem driver to expose
* the features sysfs attributes to the user for configuring
* CXL memory RAS feature.
*/
#include <linux/cleanup.h>
#include <linux/edac.h>
#include <linux/limits.h>
#include <linux/unaligned.h>
#include <linux/xarray.h>
#include <cxl/features.h>
#include <cxl.h>
#include <cxlmem.h>
#include "core.h"
#include "trace.h"
#define CXL_NR_EDAC_DEV_FEATURES 7
#define CXL_SCRUB_NO_REGION -1
struct cxl_patrol_scrub_context {
u8 instance;
u16 get_feat_size;
u16 set_feat_size;
u8 get_version;
u8 set_version;
u16 effects;
struct cxl_memdev *cxlmd;
struct cxl_region *cxlr;
};
/*
* See CXL spec rev 3.2 @8.2.10.9.11.1 Table 8-222 Device Patrol Scrub Control
* Feature Readable Attributes.
*/
struct cxl_scrub_rd_attrbs {
u8 scrub_cycle_cap;
__le16 scrub_cycle_hours;
u8 scrub_flags;
} __packed;
/*
* See CXL spec rev 3.2 @8.2.10.9.11.1 Table 8-223 Device Patrol Scrub Control
* Feature Writable Attributes.
*/
struct cxl_scrub_wr_attrbs {
u8 scrub_cycle_hours;
u8 scrub_flags;
} __packed;
#define CXL_SCRUB_CONTROL_CHANGEABLE BIT(0)
#define CXL_SCRUB_CONTROL_REALTIME BIT(1)
#define CXL_SCRUB_CONTROL_CYCLE_MASK GENMASK(7, 0)
#define CXL_SCRUB_CONTROL_MIN_CYCLE_MASK GENMASK(15, 8)
#define CXL_SCRUB_CONTROL_ENABLE BIT(0)
#define CXL_GET_SCRUB_CYCLE_CHANGEABLE(cap) \
FIELD_GET(CXL_SCRUB_CONTROL_CHANGEABLE, cap)
#define CXL_GET_SCRUB_CYCLE(cycle) \
FIELD_GET(CXL_SCRUB_CONTROL_CYCLE_MASK, cycle)
#define CXL_GET_SCRUB_MIN_CYCLE(cycle) \
FIELD_GET(CXL_SCRUB_CONTROL_MIN_CYCLE_MASK, cycle)
#define CXL_GET_SCRUB_EN_STS(flags) FIELD_GET(CXL_SCRUB_CONTROL_ENABLE, flags)
#define CXL_SET_SCRUB_CYCLE(cycle) \
FIELD_PREP(CXL_SCRUB_CONTROL_CYCLE_MASK, cycle)
#define CXL_SET_SCRUB_EN(en) FIELD_PREP(CXL_SCRUB_CONTROL_ENABLE, en)
static int cxl_mem_scrub_get_attrbs(struct cxl_mailbox *cxl_mbox, u8 *cap,
u16 *cycle, u8 *flags, u8 *min_cycle)
{
size_t rd_data_size = sizeof(struct cxl_scrub_rd_attrbs);
size_t data_size;
struct cxl_scrub_rd_attrbs *rd_attrbs __free(kfree) =
kzalloc(rd_data_size, GFP_KERNEL);
if (!rd_attrbs)
return -ENOMEM;
data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_PATROL_SCRUB_UUID,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, NULL);
if (!data_size)
return -EIO;
*cap = rd_attrbs->scrub_cycle_cap;
*cycle = le16_to_cpu(rd_attrbs->scrub_cycle_hours);
*flags = rd_attrbs->scrub_flags;
if (min_cycle)
*min_cycle = CXL_GET_SCRUB_MIN_CYCLE(*cycle);
return 0;
}
static int cxl_scrub_get_attrbs(struct cxl_patrol_scrub_context *cxl_ps_ctx,
u8 *cap, u16 *cycle, u8 *flags, u8 *min_cycle)
{
struct cxl_mailbox *cxl_mbox;
struct cxl_region_params *p;
struct cxl_memdev *cxlmd;
struct cxl_region *cxlr;
u8 min_scrub_cycle = 0;
int i, ret;
if (!cxl_ps_ctx->cxlr) {
cxl_mbox = &cxl_ps_ctx->cxlmd->cxlds->cxl_mbox;
return cxl_mem_scrub_get_attrbs(cxl_mbox, cap, cycle,
flags, min_cycle);
}
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return ret;
cxlr = cxl_ps_ctx->cxlr;
p = &cxlr->params;
for (i = 0; i < p->nr_targets; i++) {
struct cxl_endpoint_decoder *cxled = p->targets[i];
cxlmd = cxled_to_memdev(cxled);
cxl_mbox = &cxlmd->cxlds->cxl_mbox;
ret = cxl_mem_scrub_get_attrbs(cxl_mbox, cap, cycle, flags,
min_cycle);
if (ret)
return ret;
/*
* The min_scrub_cycle of a region is the max of minimum scrub
* cycles supported by memdevs that back the region.
*/
if (min_cycle)
min_scrub_cycle = max(*min_cycle, min_scrub_cycle);
}
if (min_cycle)
*min_cycle = min_scrub_cycle;
return 0;
}
static int cxl_scrub_set_attrbs_region(struct device *dev,
struct cxl_patrol_scrub_context *cxl_ps_ctx,
u8 cycle, u8 flags)
{
struct cxl_scrub_wr_attrbs wr_attrbs;
struct cxl_mailbox *cxl_mbox;
struct cxl_region_params *p;
struct cxl_memdev *cxlmd;
struct cxl_region *cxlr;
int ret, i;
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return ret;
cxlr = cxl_ps_ctx->cxlr;
p = &cxlr->params;
wr_attrbs.scrub_cycle_hours = cycle;
wr_attrbs.scrub_flags = flags;
for (i = 0; i < p->nr_targets; i++) {
struct cxl_endpoint_decoder *cxled = p->targets[i];
cxlmd = cxled_to_memdev(cxled);
cxl_mbox = &cxlmd->cxlds->cxl_mbox;
ret = cxl_set_feature(cxl_mbox, &CXL_FEAT_PATROL_SCRUB_UUID,
cxl_ps_ctx->set_version, &wr_attrbs,
sizeof(wr_attrbs),
CXL_SET_FEAT_FLAG_DATA_SAVED_ACROSS_RESET,
0, NULL);
if (ret)
return ret;
if (cycle != cxlmd->scrub_cycle) {
if (cxlmd->scrub_region_id != CXL_SCRUB_NO_REGION)
dev_info(dev,
"Device scrub rate(%d hours) set by region%d rate overwritten by region%d scrub rate(%d hours)\n",
cxlmd->scrub_cycle,
cxlmd->scrub_region_id, cxlr->id,
cycle);
cxlmd->scrub_cycle = cycle;
cxlmd->scrub_region_id = cxlr->id;
}
}
return 0;
}
static int cxl_scrub_set_attrbs_device(struct device *dev,
struct cxl_patrol_scrub_context *cxl_ps_ctx,
u8 cycle, u8 flags)
{
struct cxl_scrub_wr_attrbs wr_attrbs;
struct cxl_mailbox *cxl_mbox;
struct cxl_memdev *cxlmd;
int ret;
wr_attrbs.scrub_cycle_hours = cycle;
wr_attrbs.scrub_flags = flags;
cxlmd = cxl_ps_ctx->cxlmd;
cxl_mbox = &cxlmd->cxlds->cxl_mbox;
ret = cxl_set_feature(cxl_mbox, &CXL_FEAT_PATROL_SCRUB_UUID,
cxl_ps_ctx->set_version, &wr_attrbs,
sizeof(wr_attrbs),
CXL_SET_FEAT_FLAG_DATA_SAVED_ACROSS_RESET, 0,
NULL);
if (ret)
return ret;
if (cycle != cxlmd->scrub_cycle) {
if (cxlmd->scrub_region_id != CXL_SCRUB_NO_REGION)
dev_info(dev,
"Device scrub rate(%d hours) set by region%d rate overwritten with device local scrub rate(%d hours)\n",
cxlmd->scrub_cycle, cxlmd->scrub_region_id,
cycle);
cxlmd->scrub_cycle = cycle;
cxlmd->scrub_region_id = CXL_SCRUB_NO_REGION;
}
return 0;
}
static int cxl_scrub_set_attrbs(struct device *dev,
struct cxl_patrol_scrub_context *cxl_ps_ctx,
u8 cycle, u8 flags)
{
if (cxl_ps_ctx->cxlr)
return cxl_scrub_set_attrbs_region(dev, cxl_ps_ctx, cycle, flags);
return cxl_scrub_set_attrbs_device(dev, cxl_ps_ctx, cycle, flags);
}
static int cxl_patrol_scrub_get_enabled_bg(struct device *dev, void *drv_data,
bool *enabled)
{
struct cxl_patrol_scrub_context *ctx = drv_data;
u8 cap, flags;
u16 cycle;
int ret;
ret = cxl_scrub_get_attrbs(ctx, &cap, &cycle, &flags, NULL);
if (ret)
return ret;
*enabled = CXL_GET_SCRUB_EN_STS(flags);
return 0;
}
static int cxl_patrol_scrub_set_enabled_bg(struct device *dev, void *drv_data,
bool enable)
{
struct cxl_patrol_scrub_context *ctx = drv_data;
u8 cap, flags, wr_cycle;
u16 rd_cycle;
int ret;
if (!capable(CAP_SYS_RAWIO))
return -EPERM;
ret = cxl_scrub_get_attrbs(ctx, &cap, &rd_cycle, &flags, NULL);
if (ret)
return ret;
wr_cycle = CXL_GET_SCRUB_CYCLE(rd_cycle);
flags = CXL_SET_SCRUB_EN(enable);
return cxl_scrub_set_attrbs(dev, ctx, wr_cycle, flags);
}
static int cxl_patrol_scrub_get_min_scrub_cycle(struct device *dev,
void *drv_data, u32 *min)
{
struct cxl_patrol_scrub_context *ctx = drv_data;
u8 cap, flags, min_cycle;
u16 cycle;
int ret;
ret = cxl_scrub_get_attrbs(ctx, &cap, &cycle, &flags, &min_cycle);
if (ret)
return ret;
*min = min_cycle * 3600;
return 0;
}
static int cxl_patrol_scrub_get_max_scrub_cycle(struct device *dev,
void *drv_data, u32 *max)
{
*max = U8_MAX * 3600; /* Max set by register size */
return 0;
}
static int cxl_patrol_scrub_get_scrub_cycle(struct device *dev, void *drv_data,
u32 *scrub_cycle_secs)
{
struct cxl_patrol_scrub_context *ctx = drv_data;
u8 cap, flags;
u16 cycle;
int ret;
ret = cxl_scrub_get_attrbs(ctx, &cap, &cycle, &flags, NULL);
if (ret)
return ret;
*scrub_cycle_secs = CXL_GET_SCRUB_CYCLE(cycle) * 3600;
return 0;
}
static int cxl_patrol_scrub_set_scrub_cycle(struct device *dev, void *drv_data,
u32 scrub_cycle_secs)
{
struct cxl_patrol_scrub_context *ctx = drv_data;
u8 scrub_cycle_hours = scrub_cycle_secs / 3600;
u8 cap, wr_cycle, flags, min_cycle;
u16 rd_cycle;
int ret;
if (!capable(CAP_SYS_RAWIO))
return -EPERM;
ret = cxl_scrub_get_attrbs(ctx, &cap, &rd_cycle, &flags, &min_cycle);
if (ret)
return ret;
if (!CXL_GET_SCRUB_CYCLE_CHANGEABLE(cap))
return -EOPNOTSUPP;
if (scrub_cycle_hours < min_cycle) {
dev_dbg(dev, "Invalid CXL patrol scrub cycle(%d) to set\n",
scrub_cycle_hours);
dev_dbg(dev,
"Minimum supported CXL patrol scrub cycle in hour %d\n",
min_cycle);
return -EINVAL;
}
wr_cycle = CXL_SET_SCRUB_CYCLE(scrub_cycle_hours);
return cxl_scrub_set_attrbs(dev, ctx, wr_cycle, flags);
}
static const struct edac_scrub_ops cxl_ps_scrub_ops = {
.get_enabled_bg = cxl_patrol_scrub_get_enabled_bg,
.set_enabled_bg = cxl_patrol_scrub_set_enabled_bg,
.get_min_cycle = cxl_patrol_scrub_get_min_scrub_cycle,
.get_max_cycle = cxl_patrol_scrub_get_max_scrub_cycle,
.get_cycle_duration = cxl_patrol_scrub_get_scrub_cycle,
.set_cycle_duration = cxl_patrol_scrub_set_scrub_cycle,
};
static int cxl_memdev_scrub_init(struct cxl_memdev *cxlmd,
struct edac_dev_feature *ras_feature,
u8 scrub_inst)
{
struct cxl_patrol_scrub_context *cxl_ps_ctx;
struct cxl_feat_entry *feat_entry;
u8 cap, flags;
u16 cycle;
int rc;
feat_entry = cxl_feature_info(to_cxlfs(cxlmd->cxlds),
&CXL_FEAT_PATROL_SCRUB_UUID);
if (IS_ERR(feat_entry))
return -EOPNOTSUPP;
if (!(le32_to_cpu(feat_entry->flags) & CXL_FEATURE_F_CHANGEABLE))
return -EOPNOTSUPP;
cxl_ps_ctx = devm_kzalloc(&cxlmd->dev, sizeof(*cxl_ps_ctx), GFP_KERNEL);
if (!cxl_ps_ctx)
return -ENOMEM;
*cxl_ps_ctx = (struct cxl_patrol_scrub_context){
.get_feat_size = le16_to_cpu(feat_entry->get_feat_size),
.set_feat_size = le16_to_cpu(feat_entry->set_feat_size),
.get_version = feat_entry->get_feat_ver,
.set_version = feat_entry->set_feat_ver,
.effects = le16_to_cpu(feat_entry->effects),
.instance = scrub_inst,
.cxlmd = cxlmd,
};
rc = cxl_mem_scrub_get_attrbs(&cxlmd->cxlds->cxl_mbox, &cap, &cycle,
&flags, NULL);
if (rc)
return rc;
cxlmd->scrub_cycle = CXL_GET_SCRUB_CYCLE(cycle);
cxlmd->scrub_region_id = CXL_SCRUB_NO_REGION;
ras_feature->ft_type = RAS_FEAT_SCRUB;
ras_feature->instance = cxl_ps_ctx->instance;
ras_feature->scrub_ops = &cxl_ps_scrub_ops;
ras_feature->ctx = cxl_ps_ctx;
return 0;
}
static int cxl_region_scrub_init(struct cxl_region *cxlr,
struct edac_dev_feature *ras_feature,
u8 scrub_inst)
{
struct cxl_patrol_scrub_context *cxl_ps_ctx;
struct cxl_region_params *p = &cxlr->params;
struct cxl_feat_entry *feat_entry = NULL;
struct cxl_memdev *cxlmd;
u8 cap, flags;
u16 cycle;
int i, rc;
/*
* The cxl_region_rwsem must be held if the code below is used in a context
* other than when the region is in the probe state, as shown here.
*/
for (i = 0; i < p->nr_targets; i++) {
struct cxl_endpoint_decoder *cxled = p->targets[i];
cxlmd = cxled_to_memdev(cxled);
feat_entry = cxl_feature_info(to_cxlfs(cxlmd->cxlds),
&CXL_FEAT_PATROL_SCRUB_UUID);
if (IS_ERR(feat_entry))
return -EOPNOTSUPP;
if (!(le32_to_cpu(feat_entry->flags) &
CXL_FEATURE_F_CHANGEABLE))
return -EOPNOTSUPP;
rc = cxl_mem_scrub_get_attrbs(&cxlmd->cxlds->cxl_mbox, &cap,
&cycle, &flags, NULL);
if (rc)
return rc;
cxlmd->scrub_cycle = CXL_GET_SCRUB_CYCLE(cycle);
cxlmd->scrub_region_id = CXL_SCRUB_NO_REGION;
}
cxl_ps_ctx = devm_kzalloc(&cxlr->dev, sizeof(*cxl_ps_ctx), GFP_KERNEL);
if (!cxl_ps_ctx)
return -ENOMEM;
*cxl_ps_ctx = (struct cxl_patrol_scrub_context){
.get_feat_size = le16_to_cpu(feat_entry->get_feat_size),
.set_feat_size = le16_to_cpu(feat_entry->set_feat_size),
.get_version = feat_entry->get_feat_ver,
.set_version = feat_entry->set_feat_ver,
.effects = le16_to_cpu(feat_entry->effects),
.instance = scrub_inst,
.cxlr = cxlr,
};
ras_feature->ft_type = RAS_FEAT_SCRUB;
ras_feature->instance = cxl_ps_ctx->instance;
ras_feature->scrub_ops = &cxl_ps_scrub_ops;
ras_feature->ctx = cxl_ps_ctx;
return 0;
}
struct cxl_ecs_context {
u16 num_media_frus;
u16 get_feat_size;
u16 set_feat_size;
u8 get_version;
u8 set_version;
u16 effects;
struct cxl_memdev *cxlmd;
};
/*
* See CXL spec rev 3.2 @8.2.10.9.11.2 Table 8-225 DDR5 ECS Control Feature
* Readable Attributes.
*/
struct cxl_ecs_fru_rd_attrbs {
u8 ecs_cap;
__le16 ecs_config;
u8 ecs_flags;
} __packed;
struct cxl_ecs_rd_attrbs {
u8 ecs_log_cap;
struct cxl_ecs_fru_rd_attrbs fru_attrbs[];
} __packed;
/*
* See CXL spec rev 3.2 @8.2.10.9.11.2 Table 8-226 DDR5 ECS Control Feature
* Writable Attributes.
*/
struct cxl_ecs_fru_wr_attrbs {
__le16 ecs_config;
} __packed;
struct cxl_ecs_wr_attrbs {
u8 ecs_log_cap;
struct cxl_ecs_fru_wr_attrbs fru_attrbs[];
} __packed;
#define CXL_ECS_LOG_ENTRY_TYPE_MASK GENMASK(1, 0)
#define CXL_ECS_REALTIME_REPORT_CAP_MASK BIT(0)
#define CXL_ECS_THRESHOLD_COUNT_MASK GENMASK(2, 0)
#define CXL_ECS_COUNT_MODE_MASK BIT(3)
#define CXL_ECS_RESET_COUNTER_MASK BIT(4)
#define CXL_ECS_RESET_COUNTER 1
enum {
ECS_THRESHOLD_256 = 256,
ECS_THRESHOLD_1024 = 1024,
ECS_THRESHOLD_4096 = 4096,
};
enum {
ECS_THRESHOLD_IDX_256 = 3,
ECS_THRESHOLD_IDX_1024 = 4,
ECS_THRESHOLD_IDX_4096 = 5,
};
static const u16 ecs_supp_threshold[] = {
[ECS_THRESHOLD_IDX_256] = 256,
[ECS_THRESHOLD_IDX_1024] = 1024,
[ECS_THRESHOLD_IDX_4096] = 4096,
};
enum {
ECS_LOG_ENTRY_TYPE_DRAM = 0x0,
ECS_LOG_ENTRY_TYPE_MEM_MEDIA_FRU = 0x1,
};
enum cxl_ecs_count_mode {
ECS_MODE_COUNTS_ROWS = 0,
ECS_MODE_COUNTS_CODEWORDS = 1,
};
static int cxl_mem_ecs_get_attrbs(struct device *dev,
struct cxl_ecs_context *cxl_ecs_ctx,
int fru_id, u8 *log_cap, u16 *config)
{
struct cxl_memdev *cxlmd = cxl_ecs_ctx->cxlmd;
struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
struct cxl_ecs_fru_rd_attrbs *fru_rd_attrbs;
size_t rd_data_size;
size_t data_size;
rd_data_size = cxl_ecs_ctx->get_feat_size;
struct cxl_ecs_rd_attrbs *rd_attrbs __free(kvfree) =
kvzalloc(rd_data_size, GFP_KERNEL);
if (!rd_attrbs)
return -ENOMEM;
data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_ECS_UUID,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, NULL);
if (!data_size)
return -EIO;
fru_rd_attrbs = rd_attrbs->fru_attrbs;
*log_cap = rd_attrbs->ecs_log_cap;
*config = le16_to_cpu(fru_rd_attrbs[fru_id].ecs_config);
return 0;
}
static int cxl_mem_ecs_set_attrbs(struct device *dev,
struct cxl_ecs_context *cxl_ecs_ctx,
int fru_id, u8 log_cap, u16 config)
{
struct cxl_memdev *cxlmd = cxl_ecs_ctx->cxlmd;
struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
struct cxl_ecs_fru_rd_attrbs *fru_rd_attrbs;
struct cxl_ecs_fru_wr_attrbs *fru_wr_attrbs;
size_t rd_data_size, wr_data_size;
u16 num_media_frus, count;
size_t data_size;
num_media_frus = cxl_ecs_ctx->num_media_frus;
rd_data_size = cxl_ecs_ctx->get_feat_size;
wr_data_size = cxl_ecs_ctx->set_feat_size;
struct cxl_ecs_rd_attrbs *rd_attrbs __free(kvfree) =
kvzalloc(rd_data_size, GFP_KERNEL);
if (!rd_attrbs)
return -ENOMEM;
data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_ECS_UUID,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, NULL);
if (!data_size)
return -EIO;
struct cxl_ecs_wr_attrbs *wr_attrbs __free(kvfree) =
kvzalloc(wr_data_size, GFP_KERNEL);
if (!wr_attrbs)
return -ENOMEM;
/*
* Fill writable attributes from the current attributes read
* for all the media FRUs.
*/
fru_rd_attrbs = rd_attrbs->fru_attrbs;
fru_wr_attrbs = wr_attrbs->fru_attrbs;
wr_attrbs->ecs_log_cap = log_cap;
for (count = 0; count < num_media_frus; count++)
fru_wr_attrbs[count].ecs_config =
fru_rd_attrbs[count].ecs_config;
fru_wr_attrbs[fru_id].ecs_config = cpu_to_le16(config);
return cxl_set_feature(cxl_mbox, &CXL_FEAT_ECS_UUID,
cxl_ecs_ctx->set_version, wr_attrbs,
wr_data_size,
CXL_SET_FEAT_FLAG_DATA_SAVED_ACROSS_RESET,
0, NULL);
}
static u8 cxl_get_ecs_log_entry_type(u8 log_cap, u16 config)
{
return FIELD_GET(CXL_ECS_LOG_ENTRY_TYPE_MASK, log_cap);
}
static u16 cxl_get_ecs_threshold(u8 log_cap, u16 config)
{
u8 index = FIELD_GET(CXL_ECS_THRESHOLD_COUNT_MASK, config);
return ecs_supp_threshold[index];
}
static u8 cxl_get_ecs_count_mode(u8 log_cap, u16 config)
{
return FIELD_GET(CXL_ECS_COUNT_MODE_MASK, config);
}
#define CXL_ECS_GET_ATTR(attrb) \
static int cxl_ecs_get_##attrb(struct device *dev, void *drv_data, \
int fru_id, u32 *val) \
{ \
struct cxl_ecs_context *ctx = drv_data; \
u8 log_cap; \
u16 config; \
int ret; \
\
ret = cxl_mem_ecs_get_attrbs(dev, ctx, fru_id, &log_cap, \
&config); \
if (ret) \
return ret; \
\
*val = cxl_get_ecs_##attrb(log_cap, config); \
\
return 0; \
}
CXL_ECS_GET_ATTR(log_entry_type)
CXL_ECS_GET_ATTR(count_mode)
CXL_ECS_GET_ATTR(threshold)
static int cxl_set_ecs_log_entry_type(struct device *dev, u8 *log_cap,
u16 *config, u32 val)
{
if (val != ECS_LOG_ENTRY_TYPE_DRAM &&
val != ECS_LOG_ENTRY_TYPE_MEM_MEDIA_FRU)
return -EINVAL;
*log_cap = FIELD_PREP(CXL_ECS_LOG_ENTRY_TYPE_MASK, val);
return 0;
}
static int cxl_set_ecs_threshold(struct device *dev, u8 *log_cap, u16 *config,
u32 val)
{
*config &= ~CXL_ECS_THRESHOLD_COUNT_MASK;
switch (val) {
case ECS_THRESHOLD_256:
*config |= FIELD_PREP(CXL_ECS_THRESHOLD_COUNT_MASK,
ECS_THRESHOLD_IDX_256);
break;
case ECS_THRESHOLD_1024:
*config |= FIELD_PREP(CXL_ECS_THRESHOLD_COUNT_MASK,
ECS_THRESHOLD_IDX_1024);
break;
case ECS_THRESHOLD_4096:
*config |= FIELD_PREP(CXL_ECS_THRESHOLD_COUNT_MASK,
ECS_THRESHOLD_IDX_4096);
break;
default:
dev_dbg(dev, "Invalid CXL ECS threshold count(%u) to set\n",
val);
dev_dbg(dev, "Supported ECS threshold counts: %u, %u, %u\n",
ECS_THRESHOLD_256, ECS_THRESHOLD_1024,
ECS_THRESHOLD_4096);
return -EINVAL;
}
return 0;
}
static int cxl_set_ecs_count_mode(struct device *dev, u8 *log_cap, u16 *config,
u32 val)
{
if (val != ECS_MODE_COUNTS_ROWS && val != ECS_MODE_COUNTS_CODEWORDS) {
dev_dbg(dev, "Invalid CXL ECS scrub mode(%d) to set\n", val);
dev_dbg(dev,
"Supported ECS Modes: 0: ECS counts rows with errors,"
" 1: ECS counts codewords with errors\n");
return -EINVAL;
}
*config &= ~CXL_ECS_COUNT_MODE_MASK;
*config |= FIELD_PREP(CXL_ECS_COUNT_MODE_MASK, val);
return 0;
}
static int cxl_set_ecs_reset_counter(struct device *dev, u8 *log_cap,
u16 *config, u32 val)
{
if (val != CXL_ECS_RESET_COUNTER)
return -EINVAL;
*config &= ~CXL_ECS_RESET_COUNTER_MASK;
*config |= FIELD_PREP(CXL_ECS_RESET_COUNTER_MASK, val);
return 0;
}
#define CXL_ECS_SET_ATTR(attrb) \
static int cxl_ecs_set_##attrb(struct device *dev, void *drv_data, \
int fru_id, u32 val) \
{ \
struct cxl_ecs_context *ctx = drv_data; \
u8 log_cap; \
u16 config; \
int ret; \
\
if (!capable(CAP_SYS_RAWIO)) \
return -EPERM; \
\
ret = cxl_mem_ecs_get_attrbs(dev, ctx, fru_id, &log_cap, \
&config); \
if (ret) \
return ret; \
\
ret = cxl_set_ecs_##attrb(dev, &log_cap, &config, val); \
if (ret) \
return ret; \
\
return cxl_mem_ecs_set_attrbs(dev, ctx, fru_id, log_cap, \
config); \
}
CXL_ECS_SET_ATTR(log_entry_type)
CXL_ECS_SET_ATTR(count_mode)
CXL_ECS_SET_ATTR(reset_counter)
CXL_ECS_SET_ATTR(threshold)
static const struct edac_ecs_ops cxl_ecs_ops = {
.get_log_entry_type = cxl_ecs_get_log_entry_type,
.set_log_entry_type = cxl_ecs_set_log_entry_type,
.get_mode = cxl_ecs_get_count_mode,
.set_mode = cxl_ecs_set_count_mode,
.reset = cxl_ecs_set_reset_counter,
.get_threshold = cxl_ecs_get_threshold,
.set_threshold = cxl_ecs_set_threshold,
};
static int cxl_memdev_ecs_init(struct cxl_memdev *cxlmd,
struct edac_dev_feature *ras_feature)
{
struct cxl_ecs_context *cxl_ecs_ctx;
struct cxl_feat_entry *feat_entry;
int num_media_frus;
feat_entry =
cxl_feature_info(to_cxlfs(cxlmd->cxlds), &CXL_FEAT_ECS_UUID);
if (IS_ERR(feat_entry))
return -EOPNOTSUPP;
if (!(le32_to_cpu(feat_entry->flags) & CXL_FEATURE_F_CHANGEABLE))
return -EOPNOTSUPP;
num_media_frus = (le16_to_cpu(feat_entry->get_feat_size) -
sizeof(struct cxl_ecs_rd_attrbs)) /
sizeof(struct cxl_ecs_fru_rd_attrbs);
if (!num_media_frus)
return -EOPNOTSUPP;
cxl_ecs_ctx =
devm_kzalloc(&cxlmd->dev, sizeof(*cxl_ecs_ctx), GFP_KERNEL);
if (!cxl_ecs_ctx)
return -ENOMEM;
*cxl_ecs_ctx = (struct cxl_ecs_context){
.get_feat_size = le16_to_cpu(feat_entry->get_feat_size),
.set_feat_size = le16_to_cpu(feat_entry->set_feat_size),
.get_version = feat_entry->get_feat_ver,
.set_version = feat_entry->set_feat_ver,
.effects = le16_to_cpu(feat_entry->effects),
.num_media_frus = num_media_frus,
.cxlmd = cxlmd,
};
ras_feature->ft_type = RAS_FEAT_ECS;
ras_feature->ecs_ops = &cxl_ecs_ops;
ras_feature->ctx = cxl_ecs_ctx;
ras_feature->ecs_info.num_media_frus = num_media_frus;
return 0;
}
/*
* Perform Maintenance CXL 3.2 Spec 8.2.10.7.1
*/
/*
* Perform Maintenance input payload
* CXL rev 3.2 section 8.2.10.7.1 Table 8-117
*/
struct cxl_mbox_maintenance_hdr {
u8 op_class;
u8 op_subclass;
} __packed;
static int cxl_perform_maintenance(struct cxl_mailbox *cxl_mbox, u8 class,
u8 subclass, void *data_in,
size_t data_in_size)
{
struct cxl_memdev_maintenance_pi {
struct cxl_mbox_maintenance_hdr hdr;
u8 data[];
} __packed;
struct cxl_mbox_cmd mbox_cmd;
size_t hdr_size;
struct cxl_memdev_maintenance_pi *pi __free(kvfree) =
kvzalloc(cxl_mbox->payload_size, GFP_KERNEL);
if (!pi)
return -ENOMEM;
pi->hdr.op_class = class;
pi->hdr.op_subclass = subclass;
hdr_size = sizeof(pi->hdr);
/*
* Check minimum mbox payload size is available for
* the maintenance data transfer.
*/
if (hdr_size + data_in_size > cxl_mbox->payload_size)
return -ENOMEM;
memcpy(pi->data, data_in, data_in_size);
mbox_cmd = (struct cxl_mbox_cmd){
.opcode = CXL_MBOX_OP_DO_MAINTENANCE,
.size_in = hdr_size + data_in_size,
.payload_in = pi,
};
return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
}
/*
* Support for finding a memory operation attributes
* are from the current boot or not.
*/
struct cxl_mem_err_rec {
struct xarray rec_gen_media;
struct xarray rec_dram;
};
enum cxl_mem_repair_type {
CXL_PPR,
CXL_CACHELINE_SPARING,
CXL_ROW_SPARING,
CXL_BANK_SPARING,
CXL_RANK_SPARING,
CXL_REPAIR_MAX,
};
/**
* struct cxl_mem_repair_attrbs - CXL memory repair attributes
* @dpa: DPA of memory to repair
* @nibble_mask: nibble mask, identifies one or more nibbles on the memory bus
* @row: row of memory to repair
* @column: column of memory to repair
* @channel: channel of memory to repair
* @sub_channel: sub channel of memory to repair
* @rank: rank of memory to repair
* @bank_group: bank group of memory to repair
* @bank: bank of memory to repair
* @repair_type: repair type. For eg. PPR, memory sparing etc.
*/
struct cxl_mem_repair_attrbs {
u64 dpa;
u32 nibble_mask;
u32 row;
u16 column;
u8 channel;
u8 sub_channel;
u8 rank;
u8 bank_group;
u8 bank;
enum cxl_mem_repair_type repair_type;
};
static struct cxl_event_gen_media *
cxl_find_rec_gen_media(struct cxl_memdev *cxlmd,
struct cxl_mem_repair_attrbs *attrbs)
{
struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array;
struct cxl_event_gen_media *rec;
if (!array_rec)
return NULL;
rec = xa_load(&array_rec->rec_gen_media, attrbs->dpa);
if (!rec)
return NULL;
if (attrbs->repair_type == CXL_PPR)
return rec;
return NULL;
}
static struct cxl_event_dram *
cxl_find_rec_dram(struct cxl_memdev *cxlmd,
struct cxl_mem_repair_attrbs *attrbs)
{
struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array;
struct cxl_event_dram *rec;
u16 validity_flags;
if (!array_rec)
return NULL;
rec = xa_load(&array_rec->rec_dram, attrbs->dpa);
if (!rec)
return NULL;
validity_flags = get_unaligned_le16(rec->media_hdr.validity_flags);
if (!(validity_flags & CXL_DER_VALID_CHANNEL) ||
!(validity_flags & CXL_DER_VALID_RANK))
return NULL;
switch (attrbs->repair_type) {
case CXL_PPR:
if (!(validity_flags & CXL_DER_VALID_NIBBLE) ||
get_unaligned_le24(rec->nibble_mask) == attrbs->nibble_mask)
return rec;
break;
case CXL_CACHELINE_SPARING:
if (!(validity_flags & CXL_DER_VALID_BANK_GROUP) ||
!(validity_flags & CXL_DER_VALID_BANK) ||
!(validity_flags & CXL_DER_VALID_ROW) ||
!(validity_flags & CXL_DER_VALID_COLUMN))
return NULL;
if (rec->media_hdr.channel == attrbs->channel &&
rec->media_hdr.rank == attrbs->rank &&
rec->bank_group == attrbs->bank_group &&
rec->bank == attrbs->bank &&
get_unaligned_le24(rec->row) == attrbs->row &&
get_unaligned_le16(rec->column) == attrbs->column &&
(!(validity_flags & CXL_DER_VALID_NIBBLE) ||
get_unaligned_le24(rec->nibble_mask) ==
attrbs->nibble_mask) &&
(!(validity_flags & CXL_DER_VALID_SUB_CHANNEL) ||
rec->sub_channel == attrbs->sub_channel))
return rec;
break;
case CXL_ROW_SPARING:
if (!(validity_flags & CXL_DER_VALID_BANK_GROUP) ||
!(validity_flags & CXL_DER_VALID_BANK) ||
!(validity_flags & CXL_DER_VALID_ROW))
return NULL;
if (rec->media_hdr.channel == attrbs->channel &&
rec->media_hdr.rank == attrbs->rank &&
rec->bank_group == attrbs->bank_group &&
rec->bank == attrbs->bank &&
get_unaligned_le24(rec->row) == attrbs->row &&
(!(validity_flags & CXL_DER_VALID_NIBBLE) ||
get_unaligned_le24(rec->nibble_mask) ==
attrbs->nibble_mask))
return rec;
break;
case CXL_BANK_SPARING:
if (!(validity_flags & CXL_DER_VALID_BANK_GROUP) ||
!(validity_flags & CXL_DER_VALID_BANK))
return NULL;
if (rec->media_hdr.channel == attrbs->channel &&
rec->media_hdr.rank == attrbs->rank &&
rec->bank_group == attrbs->bank_group &&
rec->bank == attrbs->bank &&
(!(validity_flags & CXL_DER_VALID_NIBBLE) ||
get_unaligned_le24(rec->nibble_mask) ==
attrbs->nibble_mask))
return rec;
break;
case CXL_RANK_SPARING:
if (rec->media_hdr.channel == attrbs->channel &&
rec->media_hdr.rank == attrbs->rank &&
(!(validity_flags & CXL_DER_VALID_NIBBLE) ||
get_unaligned_le24(rec->nibble_mask) ==
attrbs->nibble_mask))
return rec;
break;
default:
return NULL;
}
return NULL;
}
#define CXL_MAX_STORAGE_DAYS 10
#define CXL_MAX_STORAGE_TIME_SECS (CXL_MAX_STORAGE_DAYS * 24 * 60 * 60)
static void cxl_del_expired_gmedia_recs(struct xarray *rec_xarray,
struct cxl_event_gen_media *cur_rec)
{
u64 cur_ts = le64_to_cpu(cur_rec->media_hdr.hdr.timestamp);
struct cxl_event_gen_media *rec;
unsigned long index;
u64 delta_ts_secs;
xa_for_each(rec_xarray, index, rec) {
delta_ts_secs = (cur_ts -
le64_to_cpu(rec->media_hdr.hdr.timestamp)) / 1000000000ULL;
if (delta_ts_secs >= CXL_MAX_STORAGE_TIME_SECS) {
xa_erase(rec_xarray, index);
kfree(rec);
}
}
}
static void cxl_del_expired_dram_recs(struct xarray *rec_xarray,
struct cxl_event_dram *cur_rec)
{
u64 cur_ts = le64_to_cpu(cur_rec->media_hdr.hdr.timestamp);
struct cxl_event_dram *rec;
unsigned long index;
u64 delta_secs;
xa_for_each(rec_xarray, index, rec) {
delta_secs = (cur_ts -
le64_to_cpu(rec->media_hdr.hdr.timestamp)) / 1000000000ULL;
if (delta_secs >= CXL_MAX_STORAGE_TIME_SECS) {
xa_erase(rec_xarray, index);
kfree(rec);
}
}
}
#define CXL_MAX_REC_STORAGE_COUNT 200
static void cxl_del_overflow_old_recs(struct xarray *rec_xarray)
{
void *err_rec;
unsigned long index, count = 0;
xa_for_each(rec_xarray, index, err_rec)
count++;
if (count <= CXL_MAX_REC_STORAGE_COUNT)
return;
count -= CXL_MAX_REC_STORAGE_COUNT;
xa_for_each(rec_xarray, index, err_rec) {
xa_erase(rec_xarray, index);
kfree(err_rec);
count--;
if (!count)
break;
}
}
int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, union cxl_event *evt)
{
struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array;
struct cxl_event_gen_media *rec;
void *old_rec;
if (!IS_ENABLED(CONFIG_CXL_EDAC_MEM_REPAIR) || !array_rec)
return 0;
rec = kmemdup(&evt->gen_media, sizeof(*rec), GFP_KERNEL);
if (!rec)
return -ENOMEM;
old_rec = xa_store(&array_rec->rec_gen_media,
le64_to_cpu(rec->media_hdr.phys_addr), rec,
GFP_KERNEL);
if (xa_is_err(old_rec)) {
kfree(rec);
return xa_err(old_rec);
}
kfree(old_rec);
cxl_del_expired_gmedia_recs(&array_rec->rec_gen_media, rec);
cxl_del_overflow_old_recs(&array_rec->rec_gen_media);
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_store_rec_gen_media, "CXL");
int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt)
{
struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array;
struct cxl_event_dram *rec;
void *old_rec;
if (!IS_ENABLED(CONFIG_CXL_EDAC_MEM_REPAIR) || !array_rec)
return 0;
rec = kmemdup(&evt->dram, sizeof(*rec), GFP_KERNEL);
if (!rec)
return -ENOMEM;
old_rec = xa_store(&array_rec->rec_dram,
le64_to_cpu(rec->media_hdr.phys_addr), rec,
GFP_KERNEL);
if (xa_is_err(old_rec)) {
kfree(rec);
return xa_err(old_rec);
}
kfree(old_rec);
cxl_del_expired_dram_recs(&array_rec->rec_dram, rec);
cxl_del_overflow_old_recs(&array_rec->rec_dram);
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_store_rec_dram, "CXL");
static bool cxl_is_memdev_memory_online(const struct cxl_memdev *cxlmd)
{
struct cxl_port *port = cxlmd->endpoint;
if (port && cxl_num_decoders_committed(port))
return true;
return false;
}
/*
* CXL memory sparing control
*/
enum cxl_mem_sparing_granularity {
CXL_MEM_SPARING_CACHELINE,
CXL_MEM_SPARING_ROW,
CXL_MEM_SPARING_BANK,
CXL_MEM_SPARING_RANK,
CXL_MEM_SPARING_MAX
};
struct cxl_mem_sparing_context {
struct cxl_memdev *cxlmd;
uuid_t repair_uuid;
u16 get_feat_size;
u16 set_feat_size;
u16 effects;
u8 instance;
u8 get_version;
u8 set_version;
u8 op_class;
u8 op_subclass;
bool cap_safe_when_in_use;
bool cap_hard_sparing;
bool cap_soft_sparing;
u8 channel;
u8 rank;
u8 bank_group;
u32 nibble_mask;
u64 dpa;
u32 row;
u16 column;
u8 bank;
u8 sub_channel;
enum edac_mem_repair_type repair_type;
bool persist_mode;
};
#define CXL_SPARING_RD_CAP_SAFE_IN_USE_MASK BIT(0)
#define CXL_SPARING_RD_CAP_HARD_SPARING_MASK BIT(1)
#define CXL_SPARING_RD_CAP_SOFT_SPARING_MASK BIT(2)
#define CXL_SPARING_WR_DEVICE_INITIATED_MASK BIT(0)
#define CXL_SPARING_QUERY_RESOURCE_FLAG BIT(0)
#define CXL_SET_HARD_SPARING_FLAG BIT(1)
#define CXL_SPARING_SUB_CHNL_VALID_FLAG BIT(2)
#define CXL_SPARING_NIB_MASK_VALID_FLAG BIT(3)
#define CXL_GET_SPARING_SAFE_IN_USE(flags) \
(FIELD_GET(CXL_SPARING_RD_CAP_SAFE_IN_USE_MASK, \
flags) ^ 1)
#define CXL_GET_CAP_HARD_SPARING(flags) \
FIELD_GET(CXL_SPARING_RD_CAP_HARD_SPARING_MASK, \
flags)
#define CXL_GET_CAP_SOFT_SPARING(flags) \
FIELD_GET(CXL_SPARING_RD_CAP_SOFT_SPARING_MASK, \
flags)
#define CXL_SET_SPARING_QUERY_RESOURCE(val) \
FIELD_PREP(CXL_SPARING_QUERY_RESOURCE_FLAG, val)
#define CXL_SET_HARD_SPARING(val) \
FIELD_PREP(CXL_SET_HARD_SPARING_FLAG, val)
#define CXL_SET_SPARING_SUB_CHNL_VALID(val) \
FIELD_PREP(CXL_SPARING_SUB_CHNL_VALID_FLAG, val)
#define CXL_SET_SPARING_NIB_MASK_VALID(val) \
FIELD_PREP(CXL_SPARING_NIB_MASK_VALID_FLAG, val)
/*
* See CXL spec rev 3.2 @8.2.10.7.2.3 Table 8-134 Memory Sparing Feature
* Readable Attributes.
*/
struct cxl_memdev_repair_rd_attrbs_hdr {
u8 max_op_latency;
__le16 op_cap;
__le16 op_mode;
u8 op_class;
u8 op_subclass;
u8 rsvd[9];
} __packed;
struct cxl_memdev_sparing_rd_attrbs {
struct cxl_memdev_repair_rd_attrbs_hdr hdr;
u8 rsvd;
__le16 restriction_flags;
} __packed;
/*
* See CXL spec rev 3.2 @8.2.10.7.1.4 Table 8-120 Memory Sparing Input Payload.
*/
struct cxl_memdev_sparing_in_payload {
u8 flags;
u8 channel;
u8 rank;
u8 nibble_mask[3];
u8 bank_group;
u8 bank;
u8 row[3];
__le16 column;
u8 sub_channel;
} __packed;
static int
cxl_mem_sparing_get_attrbs(struct cxl_mem_sparing_context *cxl_sparing_ctx)
{
size_t rd_data_size = sizeof(struct cxl_memdev_sparing_rd_attrbs);
struct cxl_memdev *cxlmd = cxl_sparing_ctx->cxlmd;
struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
u16 restriction_flags;
size_t data_size;
u16 return_code;
struct cxl_memdev_sparing_rd_attrbs *rd_attrbs __free(kfree) =
kzalloc(rd_data_size, GFP_KERNEL);
if (!rd_attrbs)
return -ENOMEM;
data_size = cxl_get_feature(cxl_mbox, &cxl_sparing_ctx->repair_uuid,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, &return_code);
if (!data_size)
return -EIO;
cxl_sparing_ctx->op_class = rd_attrbs->hdr.op_class;
cxl_sparing_ctx->op_subclass = rd_attrbs->hdr.op_subclass;
restriction_flags = le16_to_cpu(rd_attrbs->restriction_flags);
cxl_sparing_ctx->cap_safe_when_in_use =
CXL_GET_SPARING_SAFE_IN_USE(restriction_flags);
cxl_sparing_ctx->cap_hard_sparing =
CXL_GET_CAP_HARD_SPARING(restriction_flags);
cxl_sparing_ctx->cap_soft_sparing =
CXL_GET_CAP_SOFT_SPARING(restriction_flags);
return 0;
}
static struct cxl_event_dram *
cxl_mem_get_rec_dram(struct cxl_memdev *cxlmd,
struct cxl_mem_sparing_context *ctx)
{
struct cxl_mem_repair_attrbs attrbs = { 0 };
attrbs.dpa = ctx->dpa;
attrbs.channel = ctx->channel;
attrbs.rank = ctx->rank;
attrbs.nibble_mask = ctx->nibble_mask;
switch (ctx->repair_type) {
case EDAC_REPAIR_CACHELINE_SPARING:
attrbs.repair_type = CXL_CACHELINE_SPARING;
attrbs.bank_group = ctx->bank_group;
attrbs.bank = ctx->bank;
attrbs.row = ctx->row;
attrbs.column = ctx->column;
attrbs.sub_channel = ctx->sub_channel;
break;
case EDAC_REPAIR_ROW_SPARING:
attrbs.repair_type = CXL_ROW_SPARING;
attrbs.bank_group = ctx->bank_group;
attrbs.bank = ctx->bank;
attrbs.row = ctx->row;
break;
case EDAC_REPAIR_BANK_SPARING:
attrbs.repair_type = CXL_BANK_SPARING;
attrbs.bank_group = ctx->bank_group;
attrbs.bank = ctx->bank;
break;
case EDAC_REPAIR_RANK_SPARING:
attrbs.repair_type = CXL_RANK_SPARING;
break;
default:
return NULL;
}
return cxl_find_rec_dram(cxlmd, &attrbs);
}
static int
cxl_mem_perform_sparing(struct device *dev,
struct cxl_mem_sparing_context *cxl_sparing_ctx)
{
struct cxl_memdev *cxlmd = cxl_sparing_ctx->cxlmd;
struct cxl_memdev_sparing_in_payload sparing_pi;
struct cxl_event_dram *rec = NULL;
u16 validity_flags = 0;
int ret;
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem)))
return ret;
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
return ret;
if (!cxl_sparing_ctx->cap_safe_when_in_use) {
/* Memory to repair must be offline */
if (cxl_is_memdev_memory_online(cxlmd))
return -EBUSY;
} else {
if (cxl_is_memdev_memory_online(cxlmd)) {
rec = cxl_mem_get_rec_dram(cxlmd, cxl_sparing_ctx);
if (!rec)
return -EINVAL;
if (!get_unaligned_le16(rec->media_hdr.validity_flags))
return -EINVAL;
}
}
memset(&sparing_pi, 0, sizeof(sparing_pi));
sparing_pi.flags = CXL_SET_SPARING_QUERY_RESOURCE(0);
if (cxl_sparing_ctx->persist_mode)
sparing_pi.flags |= CXL_SET_HARD_SPARING(1);
if (rec)
validity_flags = get_unaligned_le16(rec->media_hdr.validity_flags);
switch (cxl_sparing_ctx->repair_type) {
case EDAC_REPAIR_CACHELINE_SPARING:
sparing_pi.column = cpu_to_le16(cxl_sparing_ctx->column);
if (!rec || (validity_flags & CXL_DER_VALID_SUB_CHANNEL)) {
sparing_pi.flags |= CXL_SET_SPARING_SUB_CHNL_VALID(1);
sparing_pi.sub_channel = cxl_sparing_ctx->sub_channel;
}
fallthrough;
case EDAC_REPAIR_ROW_SPARING:
put_unaligned_le24(cxl_sparing_ctx->row, sparing_pi.row);
fallthrough;
case EDAC_REPAIR_BANK_SPARING:
sparing_pi.bank_group = cxl_sparing_ctx->bank_group;
sparing_pi.bank = cxl_sparing_ctx->bank;
fallthrough;
case EDAC_REPAIR_RANK_SPARING:
sparing_pi.rank = cxl_sparing_ctx->rank;
fallthrough;
default:
sparing_pi.channel = cxl_sparing_ctx->channel;
if ((rec && (validity_flags & CXL_DER_VALID_NIBBLE)) ||
(!rec && (!cxl_sparing_ctx->nibble_mask ||
(cxl_sparing_ctx->nibble_mask & 0xFFFFFF)))) {
sparing_pi.flags |= CXL_SET_SPARING_NIB_MASK_VALID(1);
put_unaligned_le24(cxl_sparing_ctx->nibble_mask,
sparing_pi.nibble_mask);
}
break;
}
return cxl_perform_maintenance(&cxlmd->cxlds->cxl_mbox,
cxl_sparing_ctx->op_class,
cxl_sparing_ctx->op_subclass,
&sparing_pi, sizeof(sparing_pi));
}
static int cxl_mem_sparing_get_repair_type(struct device *dev, void *drv_data,
const char **repair_type)
{
struct cxl_mem_sparing_context *ctx = drv_data;
switch (ctx->repair_type) {
case EDAC_REPAIR_CACHELINE_SPARING:
case EDAC_REPAIR_ROW_SPARING:
case EDAC_REPAIR_BANK_SPARING:
case EDAC_REPAIR_RANK_SPARING:
*repair_type = edac_repair_type[ctx->repair_type];
break;
default:
return -EINVAL;
}
return 0;
}
#define CXL_SPARING_GET_ATTR(attrb, data_type) \
static int cxl_mem_sparing_get_##attrb( \
struct device *dev, void *drv_data, data_type *val) \
{ \
struct cxl_mem_sparing_context *ctx = drv_data; \
\
*val = ctx->attrb; \
\
return 0; \
}
CXL_SPARING_GET_ATTR(persist_mode, bool)
CXL_SPARING_GET_ATTR(dpa, u64)
CXL_SPARING_GET_ATTR(nibble_mask, u32)
CXL_SPARING_GET_ATTR(bank_group, u32)
CXL_SPARING_GET_ATTR(bank, u32)
CXL_SPARING_GET_ATTR(rank, u32)
CXL_SPARING_GET_ATTR(row, u32)
CXL_SPARING_GET_ATTR(column, u32)
CXL_SPARING_GET_ATTR(channel, u32)
CXL_SPARING_GET_ATTR(sub_channel, u32)
#define CXL_SPARING_SET_ATTR(attrb, data_type) \
static int cxl_mem_sparing_set_##attrb(struct device *dev, \
void *drv_data, data_type val) \
{ \
struct cxl_mem_sparing_context *ctx = drv_data; \
\
ctx->attrb = val; \
\
return 0; \
}
CXL_SPARING_SET_ATTR(nibble_mask, u32)
CXL_SPARING_SET_ATTR(bank_group, u32)
CXL_SPARING_SET_ATTR(bank, u32)
CXL_SPARING_SET_ATTR(rank, u32)
CXL_SPARING_SET_ATTR(row, u32)
CXL_SPARING_SET_ATTR(column, u32)
CXL_SPARING_SET_ATTR(channel, u32)
CXL_SPARING_SET_ATTR(sub_channel, u32)
static int cxl_mem_sparing_set_persist_mode(struct device *dev, void *drv_data,
bool persist_mode)
{
struct cxl_mem_sparing_context *ctx = drv_data;
if ((persist_mode && ctx->cap_hard_sparing) ||
(!persist_mode && ctx->cap_soft_sparing))
ctx->persist_mode = persist_mode;
else
return -EOPNOTSUPP;
return 0;
}
static int cxl_get_mem_sparing_safe_when_in_use(struct device *dev,
void *drv_data, bool *safe)
{
struct cxl_mem_sparing_context *ctx = drv_data;
*safe = ctx->cap_safe_when_in_use;
return 0;
}
static int cxl_mem_sparing_get_min_dpa(struct device *dev, void *drv_data,
u64 *min_dpa)
{
struct cxl_mem_sparing_context *ctx = drv_data;
struct cxl_memdev *cxlmd = ctx->cxlmd;
struct cxl_dev_state *cxlds = cxlmd->cxlds;
*min_dpa = cxlds->dpa_res.start;
return 0;
}
static int cxl_mem_sparing_get_max_dpa(struct device *dev, void *drv_data,
u64 *max_dpa)
{
struct cxl_mem_sparing_context *ctx = drv_data;
struct cxl_memdev *cxlmd = ctx->cxlmd;
struct cxl_dev_state *cxlds = cxlmd->cxlds;
*max_dpa = cxlds->dpa_res.end;
return 0;
}
static int cxl_mem_sparing_set_dpa(struct device *dev, void *drv_data, u64 dpa)
{
struct cxl_mem_sparing_context *ctx = drv_data;
struct cxl_memdev *cxlmd = ctx->cxlmd;
struct cxl_dev_state *cxlds = cxlmd->cxlds;
if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa))
return -EINVAL;
ctx->dpa = dpa;
return 0;
}
static int cxl_do_mem_sparing(struct device *dev, void *drv_data, u32 val)
{
struct cxl_mem_sparing_context *ctx = drv_data;
if (val != EDAC_DO_MEM_REPAIR)
return -EINVAL;
return cxl_mem_perform_sparing(dev, ctx);
}
#define RANK_OPS \
.get_repair_type = cxl_mem_sparing_get_repair_type, \
.get_persist_mode = cxl_mem_sparing_get_persist_mode, \
.set_persist_mode = cxl_mem_sparing_set_persist_mode, \
.get_repair_safe_when_in_use = cxl_get_mem_sparing_safe_when_in_use, \
.get_min_dpa = cxl_mem_sparing_get_min_dpa, \
.get_max_dpa = cxl_mem_sparing_get_max_dpa, \
.get_dpa = cxl_mem_sparing_get_dpa, \
.set_dpa = cxl_mem_sparing_set_dpa, \
.get_nibble_mask = cxl_mem_sparing_get_nibble_mask, \
.set_nibble_mask = cxl_mem_sparing_set_nibble_mask, \
.get_rank = cxl_mem_sparing_get_rank, \
.set_rank = cxl_mem_sparing_set_rank, \
.get_channel = cxl_mem_sparing_get_channel, \
.set_channel = cxl_mem_sparing_set_channel, \
.do_repair = cxl_do_mem_sparing
#define BANK_OPS \
RANK_OPS, .get_bank_group = cxl_mem_sparing_get_bank_group, \
.set_bank_group = cxl_mem_sparing_set_bank_group, \
.get_bank = cxl_mem_sparing_get_bank, \
.set_bank = cxl_mem_sparing_set_bank
#define ROW_OPS \
BANK_OPS, .get_row = cxl_mem_sparing_get_row, \
.set_row = cxl_mem_sparing_set_row
#define CACHELINE_OPS \
ROW_OPS, .get_column = cxl_mem_sparing_get_column, \
.set_column = cxl_mem_sparing_set_column, \
.get_sub_channel = cxl_mem_sparing_get_sub_channel, \
.set_sub_channel = cxl_mem_sparing_set_sub_channel
static const struct edac_mem_repair_ops cxl_rank_sparing_ops = {
RANK_OPS,
};
static const struct edac_mem_repair_ops cxl_bank_sparing_ops = {
BANK_OPS,
};
static const struct edac_mem_repair_ops cxl_row_sparing_ops = {
ROW_OPS,
};
static const struct edac_mem_repair_ops cxl_cacheline_sparing_ops = {
CACHELINE_OPS,
};
struct cxl_mem_sparing_desc {
const uuid_t repair_uuid;
enum edac_mem_repair_type repair_type;
const struct edac_mem_repair_ops *repair_ops;
};
static const struct cxl_mem_sparing_desc mem_sparing_desc[] = {
{
.repair_uuid = CXL_FEAT_CACHELINE_SPARING_UUID,
.repair_type = EDAC_REPAIR_CACHELINE_SPARING,
.repair_ops = &cxl_cacheline_sparing_ops,
},
{
.repair_uuid = CXL_FEAT_ROW_SPARING_UUID,
.repair_type = EDAC_REPAIR_ROW_SPARING,
.repair_ops = &cxl_row_sparing_ops,
},
{
.repair_uuid = CXL_FEAT_BANK_SPARING_UUID,
.repair_type = EDAC_REPAIR_BANK_SPARING,
.repair_ops = &cxl_bank_sparing_ops,
},
{
.repair_uuid = CXL_FEAT_RANK_SPARING_UUID,
.repair_type = EDAC_REPAIR_RANK_SPARING,
.repair_ops = &cxl_rank_sparing_ops,
},
};
static int cxl_memdev_sparing_init(struct cxl_memdev *cxlmd,
struct edac_dev_feature *ras_feature,
const struct cxl_mem_sparing_desc *desc,
u8 repair_inst)
{
struct cxl_mem_sparing_context *cxl_sparing_ctx;
struct cxl_feat_entry *feat_entry;
int ret;
feat_entry = cxl_feature_info(to_cxlfs(cxlmd->cxlds),
&desc->repair_uuid);
if (IS_ERR(feat_entry))
return -EOPNOTSUPP;
if (!(le32_to_cpu(feat_entry->flags) & CXL_FEATURE_F_CHANGEABLE))
return -EOPNOTSUPP;
cxl_sparing_ctx = devm_kzalloc(&cxlmd->dev, sizeof(*cxl_sparing_ctx),
GFP_KERNEL);
if (!cxl_sparing_ctx)
return -ENOMEM;
*cxl_sparing_ctx = (struct cxl_mem_sparing_context){
.get_feat_size = le16_to_cpu(feat_entry->get_feat_size),
.set_feat_size = le16_to_cpu(feat_entry->set_feat_size),
.get_version = feat_entry->get_feat_ver,
.set_version = feat_entry->set_feat_ver,
.effects = le16_to_cpu(feat_entry->effects),
.cxlmd = cxlmd,
.repair_type = desc->repair_type,
.instance = repair_inst++,
};
uuid_copy(&cxl_sparing_ctx->repair_uuid, &desc->repair_uuid);
ret = cxl_mem_sparing_get_attrbs(cxl_sparing_ctx);
if (ret)
return ret;
if ((cxl_sparing_ctx->cap_soft_sparing &&
cxl_sparing_ctx->cap_hard_sparing) ||
cxl_sparing_ctx->cap_soft_sparing)
cxl_sparing_ctx->persist_mode = 0;
else if (cxl_sparing_ctx->cap_hard_sparing)
cxl_sparing_ctx->persist_mode = 1;
else
return -EOPNOTSUPP;
ras_feature->ft_type = RAS_FEAT_MEM_REPAIR;
ras_feature->instance = cxl_sparing_ctx->instance;
ras_feature->mem_repair_ops = desc->repair_ops;
ras_feature->ctx = cxl_sparing_ctx;
return 0;
}
/*
* CXL memory soft PPR & hard PPR control
*/
struct cxl_ppr_context {
uuid_t repair_uuid;
u8 instance;
u16 get_feat_size;
u16 set_feat_size;
u8 get_version;
u8 set_version;
u16 effects;
u8 op_class;
u8 op_subclass;
bool cap_dpa;
bool cap_nib_mask;
bool media_accessible;
bool data_retained;
struct cxl_memdev *cxlmd;
enum edac_mem_repair_type repair_type;
bool persist_mode;
u64 dpa;
u32 nibble_mask;
};
/*
* See CXL rev 3.2 @8.2.10.7.2.1 Table 8-128 sPPR Feature Readable Attributes
*
* See CXL rev 3.2 @8.2.10.7.2.2 Table 8-131 hPPR Feature Readable Attributes
*/
#define CXL_PPR_OP_CAP_DEVICE_INITIATED BIT(0)
#define CXL_PPR_OP_MODE_DEV_INITIATED BIT(0)
#define CXL_PPR_FLAG_DPA_SUPPORT_MASK BIT(0)
#define CXL_PPR_FLAG_NIB_SUPPORT_MASK BIT(1)
#define CXL_PPR_FLAG_MEM_SPARING_EV_REC_SUPPORT_MASK BIT(2)
#define CXL_PPR_FLAG_DEV_INITED_PPR_AT_BOOT_CAP_MASK BIT(3)
#define CXL_PPR_RESTRICTION_FLAG_MEDIA_ACCESSIBLE_MASK BIT(0)
#define CXL_PPR_RESTRICTION_FLAG_DATA_RETAINED_MASK BIT(2)
#define CXL_PPR_SPARING_EV_REC_EN_MASK BIT(0)
#define CXL_PPR_DEV_INITED_PPR_AT_BOOT_EN_MASK BIT(1)
#define CXL_PPR_GET_CAP_DPA(flags) \
FIELD_GET(CXL_PPR_FLAG_DPA_SUPPORT_MASK, flags)
#define CXL_PPR_GET_CAP_NIB_MASK(flags) \
FIELD_GET(CXL_PPR_FLAG_NIB_SUPPORT_MASK, flags)
#define CXL_PPR_GET_MEDIA_ACCESSIBLE(restriction_flags) \
(FIELD_GET(CXL_PPR_RESTRICTION_FLAG_MEDIA_ACCESSIBLE_MASK, \
restriction_flags) ^ 1)
#define CXL_PPR_GET_DATA_RETAINED(restriction_flags) \
(FIELD_GET(CXL_PPR_RESTRICTION_FLAG_DATA_RETAINED_MASK, \
restriction_flags) ^ 1)
struct cxl_memdev_ppr_rd_attrbs {
struct cxl_memdev_repair_rd_attrbs_hdr hdr;
u8 ppr_flags;
__le16 restriction_flags;
u8 ppr_op_mode;
} __packed;
/*
* See CXL rev 3.2 @8.2.10.7.1.2 Table 8-118 sPPR Maintenance Input Payload
*
* See CXL rev 3.2 @8.2.10.7.1.3 Table 8-119 hPPR Maintenance Input Payload
*/
struct cxl_memdev_ppr_maintenance_attrbs {
u8 flags;
__le64 dpa;
u8 nibble_mask[3];
} __packed;
static int cxl_mem_ppr_get_attrbs(struct cxl_ppr_context *cxl_ppr_ctx)
{
size_t rd_data_size = sizeof(struct cxl_memdev_ppr_rd_attrbs);
struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
u16 restriction_flags;
size_t data_size;
u16 return_code;
struct cxl_memdev_ppr_rd_attrbs *rd_attrbs __free(kfree) =
kmalloc(rd_data_size, GFP_KERNEL);
if (!rd_attrbs)
return -ENOMEM;
data_size = cxl_get_feature(cxl_mbox, &cxl_ppr_ctx->repair_uuid,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, &return_code);
if (!data_size)
return -EIO;
cxl_ppr_ctx->op_class = rd_attrbs->hdr.op_class;
cxl_ppr_ctx->op_subclass = rd_attrbs->hdr.op_subclass;
cxl_ppr_ctx->cap_dpa = CXL_PPR_GET_CAP_DPA(rd_attrbs->ppr_flags);
cxl_ppr_ctx->cap_nib_mask =
CXL_PPR_GET_CAP_NIB_MASK(rd_attrbs->ppr_flags);
restriction_flags = le16_to_cpu(rd_attrbs->restriction_flags);
cxl_ppr_ctx->media_accessible =
CXL_PPR_GET_MEDIA_ACCESSIBLE(restriction_flags);
cxl_ppr_ctx->data_retained =
CXL_PPR_GET_DATA_RETAINED(restriction_flags);
return 0;
}
static int cxl_mem_perform_ppr(struct cxl_ppr_context *cxl_ppr_ctx)
{
struct cxl_memdev_ppr_maintenance_attrbs maintenance_attrbs;
struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
struct cxl_mem_repair_attrbs attrbs = { 0 };
int ret;
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem)))
return ret;
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
return ret;
if (!cxl_ppr_ctx->media_accessible || !cxl_ppr_ctx->data_retained) {
/* Memory to repair must be offline */
if (cxl_is_memdev_memory_online(cxlmd))
return -EBUSY;
} else {
if (cxl_is_memdev_memory_online(cxlmd)) {
/* Check memory to repair is from the current boot */
attrbs.repair_type = CXL_PPR;
attrbs.dpa = cxl_ppr_ctx->dpa;
attrbs.nibble_mask = cxl_ppr_ctx->nibble_mask;
if (!cxl_find_rec_dram(cxlmd, &attrbs) &&
!cxl_find_rec_gen_media(cxlmd, &attrbs))
return -EINVAL;
}
}
memset(&maintenance_attrbs, 0, sizeof(maintenance_attrbs));
maintenance_attrbs.flags = 0;
maintenance_attrbs.dpa = cpu_to_le64(cxl_ppr_ctx->dpa);
put_unaligned_le24(cxl_ppr_ctx->nibble_mask,
maintenance_attrbs.nibble_mask);
return cxl_perform_maintenance(&cxlmd->cxlds->cxl_mbox,
cxl_ppr_ctx->op_class,
cxl_ppr_ctx->op_subclass,
&maintenance_attrbs,
sizeof(maintenance_attrbs));
}
static int cxl_ppr_get_repair_type(struct device *dev, void *drv_data,
const char **repair_type)
{
*repair_type = edac_repair_type[EDAC_REPAIR_PPR];
return 0;
}
static int cxl_ppr_get_persist_mode(struct device *dev, void *drv_data,
bool *persist_mode)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
*persist_mode = cxl_ppr_ctx->persist_mode;
return 0;
}
static int cxl_get_ppr_safe_when_in_use(struct device *dev, void *drv_data,
bool *safe)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
*safe = cxl_ppr_ctx->media_accessible & cxl_ppr_ctx->data_retained;
return 0;
}
static int cxl_ppr_get_min_dpa(struct device *dev, void *drv_data, u64 *min_dpa)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
struct cxl_dev_state *cxlds = cxlmd->cxlds;
*min_dpa = cxlds->dpa_res.start;
return 0;
}
static int cxl_ppr_get_max_dpa(struct device *dev, void *drv_data, u64 *max_dpa)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
struct cxl_dev_state *cxlds = cxlmd->cxlds;
*max_dpa = cxlds->dpa_res.end;
return 0;
}
static int cxl_ppr_get_dpa(struct device *dev, void *drv_data, u64 *dpa)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
*dpa = cxl_ppr_ctx->dpa;
return 0;
}
static int cxl_ppr_set_dpa(struct device *dev, void *drv_data, u64 dpa)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
struct cxl_dev_state *cxlds = cxlmd->cxlds;
if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa))
return -EINVAL;
cxl_ppr_ctx->dpa = dpa;
return 0;
}
static int cxl_ppr_get_nibble_mask(struct device *dev, void *drv_data,
u32 *nibble_mask)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
*nibble_mask = cxl_ppr_ctx->nibble_mask;
return 0;
}
static int cxl_ppr_set_nibble_mask(struct device *dev, void *drv_data,
u32 nibble_mask)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
cxl_ppr_ctx->nibble_mask = nibble_mask;
return 0;
}
static int cxl_do_ppr(struct device *dev, void *drv_data, u32 val)
{
struct cxl_ppr_context *cxl_ppr_ctx = drv_data;
struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
struct cxl_dev_state *cxlds = cxlmd->cxlds;
if (val != EDAC_DO_MEM_REPAIR ||
!cxl_resource_contains_addr(&cxlds->dpa_res, cxl_ppr_ctx->dpa))
return -EINVAL;
return cxl_mem_perform_ppr(cxl_ppr_ctx);
}
static const struct edac_mem_repair_ops cxl_sppr_ops = {
.get_repair_type = cxl_ppr_get_repair_type,
.get_persist_mode = cxl_ppr_get_persist_mode,
.get_repair_safe_when_in_use = cxl_get_ppr_safe_when_in_use,
.get_min_dpa = cxl_ppr_get_min_dpa,
.get_max_dpa = cxl_ppr_get_max_dpa,
.get_dpa = cxl_ppr_get_dpa,
.set_dpa = cxl_ppr_set_dpa,
.get_nibble_mask = cxl_ppr_get_nibble_mask,
.set_nibble_mask = cxl_ppr_set_nibble_mask,
.do_repair = cxl_do_ppr,
};
static int cxl_memdev_soft_ppr_init(struct cxl_memdev *cxlmd,
struct edac_dev_feature *ras_feature,
u8 repair_inst)
{
struct cxl_ppr_context *cxl_sppr_ctx;
struct cxl_feat_entry *feat_entry;
int ret;
feat_entry = cxl_feature_info(to_cxlfs(cxlmd->cxlds),
&CXL_FEAT_SPPR_UUID);
if (IS_ERR(feat_entry))
return -EOPNOTSUPP;
if (!(le32_to_cpu(feat_entry->flags) & CXL_FEATURE_F_CHANGEABLE))
return -EOPNOTSUPP;
cxl_sppr_ctx =
devm_kzalloc(&cxlmd->dev, sizeof(*cxl_sppr_ctx), GFP_KERNEL);
if (!cxl_sppr_ctx)
return -ENOMEM;
*cxl_sppr_ctx = (struct cxl_ppr_context){
.get_feat_size = le16_to_cpu(feat_entry->get_feat_size),
.set_feat_size = le16_to_cpu(feat_entry->set_feat_size),
.get_version = feat_entry->get_feat_ver,
.set_version = feat_entry->set_feat_ver,
.effects = le16_to_cpu(feat_entry->effects),
.cxlmd = cxlmd,
.repair_type = EDAC_REPAIR_PPR,
.persist_mode = 0,
.instance = repair_inst,
};
uuid_copy(&cxl_sppr_ctx->repair_uuid, &CXL_FEAT_SPPR_UUID);
ret = cxl_mem_ppr_get_attrbs(cxl_sppr_ctx);
if (ret)
return ret;
ras_feature->ft_type = RAS_FEAT_MEM_REPAIR;
ras_feature->instance = cxl_sppr_ctx->instance;
ras_feature->mem_repair_ops = &cxl_sppr_ops;
ras_feature->ctx = cxl_sppr_ctx;
return 0;
}
static void err_rec_free(void *_cxlmd)
{
struct cxl_memdev *cxlmd = _cxlmd;
struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array;
struct cxl_event_gen_media *rec_gen_media;
struct cxl_event_dram *rec_dram;
unsigned long index;
cxlmd->err_rec_array = NULL;
xa_for_each(&array_rec->rec_dram, index, rec_dram)
kfree(rec_dram);
xa_destroy(&array_rec->rec_dram);
xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media)
kfree(rec_gen_media);
xa_destroy(&array_rec->rec_gen_media);
kfree(array_rec);
}
static int devm_cxl_memdev_setup_err_rec(struct cxl_memdev *cxlmd)
{
struct cxl_mem_err_rec *array_rec = kzalloc_obj(*array_rec);
if (!array_rec)
return -ENOMEM;
xa_init(&array_rec->rec_gen_media);
xa_init(&array_rec->rec_dram);
cxlmd->err_rec_array = array_rec;
return devm_add_action_or_reset(&cxlmd->dev, err_rec_free, cxlmd);
}
int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd)
{
struct edac_dev_feature ras_features[CXL_NR_EDAC_DEV_FEATURES];
int num_ras_features = 0;
u8 repair_inst = 0;
int rc;
if (IS_ENABLED(CONFIG_CXL_EDAC_SCRUB)) {
rc = cxl_memdev_scrub_init(cxlmd, &ras_features[num_ras_features], 0);
if (rc < 0 && rc != -EOPNOTSUPP)
return rc;
if (rc != -EOPNOTSUPP)
num_ras_features++;
}
if (IS_ENABLED(CONFIG_CXL_EDAC_ECS)) {
rc = cxl_memdev_ecs_init(cxlmd, &ras_features[num_ras_features]);
if (rc < 0 && rc != -EOPNOTSUPP)
return rc;
if (rc != -EOPNOTSUPP)
num_ras_features++;
}
if (IS_ENABLED(CONFIG_CXL_EDAC_MEM_REPAIR)) {
for (int i = 0; i < CXL_MEM_SPARING_MAX; i++) {
rc = cxl_memdev_sparing_init(cxlmd,
&ras_features[num_ras_features],
&mem_sparing_desc[i], repair_inst);
if (rc == -EOPNOTSUPP)
continue;
if (rc < 0)
return rc;
repair_inst++;
num_ras_features++;
}
rc = cxl_memdev_soft_ppr_init(cxlmd, &ras_features[num_ras_features],
repair_inst);
if (rc < 0 && rc != -EOPNOTSUPP)
return rc;
if (rc != -EOPNOTSUPP) {
repair_inst++;
num_ras_features++;
}
if (repair_inst) {
rc = devm_cxl_memdev_setup_err_rec(cxlmd);
if (rc)
return rc;
}
}
if (!num_ras_features)
return -EINVAL;
char *cxl_dev_name __free(kfree) =
kasprintf(GFP_KERNEL, "cxl_%s", dev_name(&cxlmd->dev));
if (!cxl_dev_name)
return -ENOMEM;
return edac_dev_register(&cxlmd->dev, cxl_dev_name, NULL,
num_ras_features, ras_features);
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_memdev_edac_register, "CXL");
int devm_cxl_region_edac_register(struct cxl_region *cxlr)
{
struct edac_dev_feature ras_features[CXL_NR_EDAC_DEV_FEATURES];
int num_ras_features = 0;
int rc;
if (!IS_ENABLED(CONFIG_CXL_EDAC_SCRUB))
return 0;
rc = cxl_region_scrub_init(cxlr, &ras_features[num_ras_features], 0);
if (rc < 0)
return rc;
num_ras_features++;
char *cxl_dev_name __free(kfree) =
kasprintf(GFP_KERNEL, "cxl_%s", dev_name(&cxlr->dev));
if (!cxl_dev_name)
return -ENOMEM;
return edac_dev_register(&cxlr->dev, cxl_dev_name, NULL,
num_ras_features, ras_features);
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_region_edac_register, "CXL");
