From: Moshe Shemesh <mo...@mellanox.com>
Use devlink_health_report() to report any symptom of FW issue as FW
counter miss or new health syndrom.
The FW issues detected in mlx5 during poll_health which is called in
timer atomic context and so health work queue is used to schedule the
reports.
Signed-off-by: Moshe Shemesh <mo...@mellanox.com>
Signed-off-by: Eran Ben Elisha <era...@mellanox.com>
Signed-off-by: Saeed Mahameed <sae...@mellanox.com>
---
.../net/ethernet/mellanox/mlx5/core/health.c | 33 +++++++++++++++++++
include/linux/mlx5/driver.h | 2 ++
2 files changed, 35 insertions(+)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c
b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index 34b8252afad5..03b9fc9ebd6e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -497,6 +497,29 @@ mlx5_fw_reporter_dump(struct devlink_health_reporter
*reporter,
return mlx5_fw_tracer_get_saved_traces_objects(dev->tracer, fmsg);
}
+static void mlx5_fw_reporter_err_work(struct work_struct *work)
+{
+ struct mlx5_fw_reporter_ctx fw_reporter_ctx;
+ struct mlx5_core_health *health;
+
+ health = container_of(work, struct mlx5_core_health, report_work);
+
+ if (IS_ERR_OR_NULL(health->fw_reporter))
+ return;
+
+ fw_reporter_ctx.err_synd = health->synd;
+ fw_reporter_ctx.miss_counter = health->miss_counter;
+ if (fw_reporter_ctx.err_synd) {
+ devlink_health_report(health->fw_reporter,
+ "FW syndrome reported", &fw_reporter_ctx);
+ return;
+ }
+ if (fw_reporter_ctx.miss_counter)
+ devlink_health_report(health->fw_reporter,
+ "FW miss counter reported",
+ &fw_reporter_ctx);
+}
+
static const struct devlink_health_reporter_ops mlx5_fw_reporter_ops = {
.name = "fw",
.diagnose = mlx5_fw_reporter_diagnose,
@@ -554,8 +577,10 @@ static void poll_health(struct timer_list *t)
{
struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
struct mlx5_core_health *health = &dev->priv.health;
+ struct health_buffer __iomem *h = health->health;
u32 fatal_error;
u32 count;
+ u8 prev_synd;
if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
goto out;
@@ -570,8 +595,14 @@ static void poll_health(struct timer_list *t)
if (health->miss_counter == MAX_MISSES) {
mlx5_core_err(dev, "device's health compromised - reached miss
count\n");
mlx5_print_health_info(dev);
+ queue_work(health->wq, &health->report_work);
}
+ prev_synd = health->synd;
+ health->synd = ioread8(&h->synd);
+ if (health->synd && health->synd != prev_synd)
+ queue_work(health->wq, &health->report_work);
+
fatal_error = check_fatal_sensors(dev);
if (fatal_error && !health->fatal_error) {
@@ -621,6 +652,7 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
spin_lock_irqsave(&health->wq_lock, flags);
set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
spin_unlock_irqrestore(&health->wq_lock, flags);
+ cancel_work_sync(&health->report_work);
cancel_work_sync(&health->work);
}
@@ -658,6 +690,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev)
return -ENOMEM;
spin_lock_init(&health->wq_lock);
INIT_WORK(&health->work, health_care);
+ INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work);
health->crdump = NULL;
health->info_buf = kmalloc(HEALTH_INFO_MAX_BUFF, GFP_KERNEL);
if (!health->info_buf)
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index ebda70984601..604079b4706c 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -437,12 +437,14 @@ struct mlx5_core_health {
struct timer_list timer;
u32 prev;
int miss_counter;
+ u8 synd;
u32 fatal_error;
/* wq spinlock to synchronize draining */
spinlock_t wq_lock;
struct workqueue_struct *wq;
unsigned long flags;
struct work_struct work;
+ struct work_struct report_work;
struct delayed_work recover_work;
struct mlx5_fw_crdump *crdump;
char *info_buf;
--
2.20.1
