Author: hselasky
Date: Fri Mar 30 19:17:09 2018
New Revision: 331809
URL: https://svnweb.freebsd.org/changeset/base/331809
Log:
MFC r331443:
Improve support for health recovery in mlx5core.
This patch accumulates the following Linux commits:
- 04c0c1ab38e95105d950db5b84e727637e149ce7
net/mlx5: PCI error recovery health care simulation
- 0179720d6be2096b8d0a4d143254ff9e77747daa
net/mlx5: Introduce trigger_health_work function
- 3fece5d676939f42f434c63dfe1bd42d7d94e6f0
net/mlx5: Continue health polling until it is explicitly stopped
Submitted by: Matthew Finlay <m...@mellanox.com>
Sponsored by: Mellanox Technologies
Modified:
stable/11/sys/dev/mlx5/driver.h
stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h
stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c
stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/dev/mlx5/driver.h
==============================================================================
--- stable/11/sys/dev/mlx5/driver.h Fri Mar 30 19:15:04 2018
(r331808)
+++ stable/11/sys/dev/mlx5/driver.h Fri Mar 30 19:17:09 2018
(r331809)
@@ -498,6 +498,7 @@ struct mlx5_core_health {
struct workqueue_struct *wq;
unsigned long flags;
struct work_struct work;
+ struct delayed_work recover_work;
};
#define MLX5_CQ_LINEAR_ARRAY_SIZE 1024
@@ -887,6 +888,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev);
void mlx5_start_health_poll(struct mlx5_core_dev *dev);
void mlx5_stop_health_poll(struct mlx5_core_dev *dev);
void mlx5_drain_health_wq(struct mlx5_core_dev *dev);
+void mlx5_trigger_health_work(struct mlx5_core_dev *dev);
#define mlx5_buf_alloc_node(dev, size, direct, buf, node) \
mlx5_buf_alloc(dev, size, direct, buf)
Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h Fri Mar 30 19:15:04
2018 (r331808)
+++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h Fri Mar 30 19:17:09
2018 (r331809)
@@ -74,6 +74,7 @@ void mlx5_core_event(struct mlx5_core_dev *dev, enum m
unsigned long param);
void mlx5_enter_error_state(struct mlx5_core_dev *dev);
void mlx5_disable_device(struct mlx5_core_dev *dev);
+void mlx5_recover_device(struct mlx5_core_dev *dev);
void mlx5e_init(void);
void mlx5e_cleanup(void);
Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c Fri Mar 30 19:15:04
2018 (r331808)
+++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c Fri Mar 30 19:17:09
2018 (r331809)
@@ -40,14 +40,15 @@
enum {
MLX5_NIC_IFC_FULL = 0,
MLX5_NIC_IFC_DISABLED = 1,
- MLX5_NIC_IFC_NO_DRAM_NIC = 2
+ MLX5_NIC_IFC_NO_DRAM_NIC = 2,
+ MLX5_NIC_IFC_INVALID = 3,
};
enum {
MLX5_DROP_NEW_HEALTH_WORK,
};
-static u8 get_nic_interface(struct mlx5_core_dev *dev)
+static u8 get_nic_state(struct mlx5_core_dev *dev)
{
return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
}
@@ -80,7 +81,7 @@ static int in_fatal(struct mlx5_core_dev *dev)
struct mlx5_core_health *health = &dev->priv.health;
struct mlx5_health_buffer __iomem *h = health->health;
- if (get_nic_interface(dev) == MLX5_NIC_IFC_DISABLED)
+ if (get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
return 1;
if (ioread32be(&h->fw_ver) == 0xffffffff)
@@ -112,9 +113,9 @@ unlock:
static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
{
- u8 nic_interface = get_nic_interface(dev);
+ u8 nic_state = get_nic_state(dev);
- switch (nic_interface) {
+ switch (nic_state) {
case MLX5_NIC_IFC_FULL:
mlx5_core_warn(dev, "Expected to see disabled NIC but it is
full driver\n");
break;
@@ -128,23 +129,58 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev
break;
default:
mlx5_core_warn(dev, "Expected to see disabled NIC but it is has
invalid value %d\n",
- nic_interface);
+ nic_state);
}
mlx5_disable_device(dev);
}
+static void health_recover(struct work_struct *work)
+{
+ struct mlx5_core_health *health;
+ struct delayed_work *dwork;
+ struct mlx5_core_dev *dev;
+ struct mlx5_priv *priv;
+ u8 nic_state;
+
+ dwork = container_of(work, struct delayed_work, work);
+ health = container_of(dwork, struct mlx5_core_health, recover_work);
+ priv = container_of(health, struct mlx5_priv, health);
+ dev = container_of(priv, struct mlx5_core_dev, priv);
+
+ nic_state = get_nic_state(dev);
+ if (nic_state == MLX5_NIC_IFC_INVALID) {
+ dev_err(&dev->pdev->dev, "health recovery flow aborted since
the nic state is invalid\n");
+ return;
+ }
+
+ dev_err(&dev->pdev->dev, "starting health recovery flow\n");
+ mlx5_recover_device(dev);
+}
+
+/* How much time to wait until health resetting the driver (in msecs) */
+#define MLX5_RECOVERY_DELAY_MSECS 60000
static void health_care(struct work_struct *work)
{
+ unsigned long recover_delay =
msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS);
struct mlx5_core_health *health;
struct mlx5_core_dev *dev;
struct mlx5_priv *priv;
+ unsigned long flags;
health = container_of(work, struct mlx5_core_health, work);
priv = container_of(health, struct mlx5_priv, health);
dev = container_of(priv, struct mlx5_core_dev, priv);
mlx5_core_warn(dev, "handling bad device here\n");
mlx5_handle_bad_state(dev);
+
+ spin_lock_irqsave(&health->wq_lock, flags);
+ if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
+ schedule_delayed_work(&health->recover_work, recover_delay);
+ else
+ dev_err(&dev->pdev->dev,
+ "new health works are not permitted at this stage\n");
+ spin_unlock_irqrestore(&health->wq_lock, flags);
}
static int get_next_poll_jiffies(void)
@@ -158,6 +194,20 @@ static int get_next_poll_jiffies(void)
return next;
}
+void mlx5_trigger_health_work(struct mlx5_core_dev *dev)
+{
+ struct mlx5_core_health *health = &dev->priv.health;
+ unsigned long flags;
+
+ spin_lock_irqsave(&health->wq_lock, flags);
+ if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
+ queue_work(health->wq, &health->work);
+ else
+ dev_err(&dev->pdev->dev,
+ "new health works are not permitted at this stage\n");
+ spin_unlock_irqrestore(&health->wq_lock, flags);
+}
+
static const char *hsynd_str(u8 synd)
{
switch (synd) {
@@ -224,10 +274,8 @@ static void poll_health(unsigned long data)
if (dev->state != MLX5_DEVICE_STATE_UP)
return;
- if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
- mod_timer(&health->timer, get_next_poll_jiffies());
- return;
- }
+ if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
+ goto out;
count = ioread32be(health->health_counter);
if (count == health->prev)
@@ -239,21 +287,16 @@ static void poll_health(unsigned long data)
if (health->miss_counter == MAX_MISSES) {
mlx5_core_err(dev, "device's health compromised - reached miss
count\n");
print_health_info(dev);
- } else {
- mod_timer(&health->timer, get_next_poll_jiffies());
}
if (in_fatal(dev) && !health->sick) {
health->sick = true;
print_health_info(dev);
- spin_lock(&health->wq_lock);
- if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
- queue_work(health->wq, &health->work);
- else
- dev_err(&dev->pdev->dev,
- "new health works are not permitted at this
stage\n");
- spin_unlock(&health->wq_lock);
+ mlx5_trigger_health_work(dev);
}
+
+out:
+ mod_timer(&health->timer, get_next_poll_jiffies());
}
void mlx5_start_health_poll(struct mlx5_core_dev *dev)
@@ -281,10 +324,12 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev)
void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
{
struct mlx5_core_health *health = &dev->priv.health;
+ unsigned long flags;
- spin_lock(&health->wq_lock);
+ spin_lock_irqsave(&health->wq_lock, flags);
set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
- spin_unlock(&health->wq_lock);
+ spin_unlock_irqrestore(&health->wq_lock, flags);
+ cancel_delayed_work_sync(&health->recover_work);
cancel_work_sync(&health->work);
}
@@ -316,6 +361,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev)
spin_lock_init(&health->wq_lock);
INIT_WORK(&health->work, health_care);
+ INIT_DELAYED_WORK(&health->recover_work, health_recover);
return 0;
}
Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c Fri Mar 30 19:15:04
2018 (r331808)
+++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c Fri Mar 30 19:17:09
2018 (r331809)
@@ -1257,11 +1257,6 @@ static pci_ers_result_t mlx5_pci_slot_reset(struct pci
return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
}
-void mlx5_disable_device(struct mlx5_core_dev *dev)
-{
- mlx5_pci_err_detected(dev->pdev, 0);
-}
-
/* wait for the device to show vital signs. For now we check
* that we can read the device ID and that the health buffer
* shows a non zero value which is different than 0xffffffff
@@ -1376,6 +1371,18 @@ static const struct pci_device_id mlx5_core_pci_table[
};
MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
+
+void mlx5_disable_device(struct mlx5_core_dev *dev)
+{
+ mlx5_pci_err_detected(dev->pdev, 0);
+}
+
+void mlx5_recover_device(struct mlx5_core_dev *dev)
+{
+ mlx5_pci_disable_device(dev);
+ if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED)
+ mlx5_pci_resume(dev->pdev);
+}
struct pci_driver mlx5_core_driver = {
.name = DRIVER_NAME,
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
