@@ -56,7 +56,9 @@ MODULE_DESCRIPTION("core kernel InfiniBand API");
 MODULE_LICENSE("Dual BSD/GPL");
 
 struct workqueue_struct *ib_comp_wq;
+EXPORT_SYMBOL_GPL(ib_comp_wq);
 struct workqueue_struct *ib_comp_unbound_wq;
+EXPORT_SYMBOL_GPL(ib_comp_unbound_wq);
 struct workqueue_struct *ib_wq;
 EXPORT_SYMBOL_GPL(ib_wq);
 static struct workqueue_struct *ib_unreg_wq;

@@ -619,6 +619,10 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 	int npolled;
 
 	spin_lock_irqsave(&cq->lock, flags);
+	if (mlx5_core_is_shutting_down(mdev)) {
+		spin_unlock_irqrestore(&cq->lock, flags);
+		return 0;
+	}
 	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
 		/* make sure no soft wqe's are waiting */
 		if (unlikely(!list_empty(&cq->wc_list)))

@@ -654,6 +658,10 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 	int ret = 0;
 
 	spin_lock_irqsave(&cq->lock, irq_flags);
+	if (mlx5_core_is_shutting_down(mdev)) {
+		spin_unlock_irqrestore(&cq->lock, irq_flags);
+		return 0;
+	}
 	if (cq->notify_flags != IB_CQ_NEXT_COMP)
 		cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
 

@@ -661,6 +669,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 		ret = 1;
 	spin_unlock_irqrestore(&cq->lock, irq_flags);
 
+	if (mlx5_core_is_shutting_down(mdev))
+		return ret;
+
 	mlx5_cq_arm(&cq->mcq,
 		    (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
 		    MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,

@@ -84,6 +84,13 @@ static LIST_HEAD(mlx5_ib_dev_list);
  */
 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
 
+static void mlx5_ib_shutdown_quiesce(void)
+{
+	flush_workqueue(ib_comp_wq);
+	flush_workqueue(ib_comp_unbound_wq);
+	flush_workqueue(mlx5_ib_event_wq);
+}
+
 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
 {
 	struct mlx5_ib_dev *dev;

@@ -2976,6 +2983,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
 	unsigned long flags_cq;
 	unsigned long flags;
 
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		return;
+
 	INIT_LIST_HEAD(&cq_armed_list);
 
 	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/

@@ -3200,6 +3210,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)
 	struct mlx5_ib_dev *ibdev = work->dev;
 	struct ib_event ibev;
 
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		goto out;
+
 	ibev.event = IB_EVENT_DEVICE_FATAL;
 	mlx5_ib_handle_internal_error(ibdev);
 	ibev.element.port_num = (u8)(unsigned long)work->param;

@@ -3222,10 +3235,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb,
 				   unsigned long event, void *param)
 {
 	struct mlx5_ib_event_work *work;
+	struct mlx5_ib_dev *ibdev;
 
 	if (event != MLX5_DEV_EVENT_SYS_ERROR)
 		return NOTIFY_DONE;
 
+	ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events);
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		return NOTIFY_OK;
+
 	work = kmalloc_obj(*work, GFP_ATOMIC);
 	if (!work)
 		return NOTIFY_DONE;

@@ -5529,6 +5547,7 @@ static int __init mlx5_ib_init(void)
 	if (ret)
 		goto drv_err;
 
+	mlx5_rdma_shutdown_quiesce = mlx5_ib_shutdown_quiesce;
 	return 0;
 
 drv_err:

@@ -5547,6 +5566,7 @@ static int __init mlx5_ib_init(void)
 
 static void __exit mlx5_ib_cleanup(void)
 {
+	mlx5_rdma_shutdown_quiesce = NULL;
 	mlx5_data_direct_driver_unregister();
 	auxiliary_driver_unregister(&mlx5r_driver);
 	auxiliary_driver_unregister(&mlx5r_mp_driver);

@@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)
 		mlx5_cmd_flush(dev);
 	}
 
+	if (mlx5_core_is_shutting_down(dev))
+		return;
+
 	mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
 }
 

@@ -2186,12 +2186,22 @@ static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
 	return 0;
 }
 
+void (*mlx5_rdma_shutdown_quiesce)(void);
+EXPORT_SYMBOL_GPL(mlx5_rdma_shutdown_quiesce);
+
 static void shutdown(struct pci_dev *pdev)
 {
 	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
 	int err;
 
 	mlx5_core_info(dev, "Shutdown was called\n");
+	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
+	/*
+	 * Ensure in-flight CQ pollers observe SHUTTING_DOWN before
+	 * fast_unload tears the device down.
+	 */
+	if (mlx5_rdma_shutdown_quiesce)
+		mlx5_rdma_shutdown_quiesce();
 	set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
 	mlx5_drain_fw_reset(dev);
 	mlx5_drain_health_wq(dev);

@@ -111,6 +111,9 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev)
 	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
 	struct mlx5_core_dev *mdev = sf_dev->mdev;
 
+	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &mdev->intf_state);
+	if (mlx5_rdma_shutdown_quiesce)
+		mlx5_rdma_shutdown_quiesce();
 	set_bit(MLX5_BREAK_FW_WAIT, &mdev->intf_state);
 	mlx5_drain_health_wq(mdev);
 	mlx5_unload_one(mdev, false);

@@ -653,8 +653,14 @@ enum mlx5_device_state {
 enum mlx5_interface_state {
 	MLX5_INTERFACE_STATE_UP = BIT(0),
 	MLX5_BREAK_FW_WAIT = BIT(1),
+	MLX5_INTERFACE_STATE_SHUTTING_DOWN = BIT(2),
 };
 
+/* Optional hook installed by mlx5_ib to flush CQ poll work after the
+ * SHUTTING_DOWN flag is set and before teardown continues.
+ */
+extern void (*mlx5_rdma_shutdown_quiesce)(void);
+
 enum mlx5_pci_status {
 	MLX5_PCI_STATUS_DISABLED,
 	MLX5_PCI_STATUS_ENABLED,

@@ -1220,6 +1226,11 @@ static inline bool mlx5_core_is_pf(const struct mlx5_core_dev *dev)
 	return dev->coredev_type == MLX5_COREDEV_PF;
 }
 
+static inline bool mlx5_core_is_shutting_down(struct mlx5_core_dev *dev)
+{
+	return test_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
+}
+
 static inline bool mlx5_core_is_vf(const struct mlx5_core_dev *dev)
 {
 	return dev->coredev_type == MLX5_COREDEV_VF;