diff --git a/net/rds/connection.c b/net/rds/connection.c
index a44aa4d2a5e8..c3b3d756c52e 100644
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -48,6 +48,8 @@
/* converting this to RCU is a chore for another day.. */
static DEFINE_SPINLOCK(rds_conn_lock);
static unsigned long rds_conn_count;
+/* woken whenever a transport's t_conn_count drops to zero */
+static DECLARE_WAIT_QUEUE_HEAD(rds_conn_freed_waitq);
static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES];
static struct kmem_cache *rds_conn_slab;
@@ -341,6 +343,7 @@ static struct rds_connection *__rds_conn_create(struct net *net,
parent->c_passive = conn;
rds_cong_add_conn(conn);
rds_conn_count++;
+ atomic_inc(&conn->c_trans->t_conn_count);
}
} else {
/* Creating normal conn */@@ -359,6 +362,7 @@ static struct rds_connection *__rds_conn_create(struct net *net,
hlist_add_head_rcu(&conn->c_hash_node, head);
rds_cong_add_conn(conn);
rds_conn_count++;
+ atomic_inc(&conn->c_trans->t_conn_count);
}
}
spin_unlock_irqrestore(&rds_conn_lock, flags);
@@ -584,6 +588,7 @@ static void rds_conn_destroy_fini(struct kref *kref)
struct rds_connection *conn = container_of(kref, struct rds_connection,
c_refcount);
int npaths = (conn->c_trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);
+ struct rds_transport *trans = conn->c_trans;
unsigned long flags;
int i;
@@ -596,7 +601,51 @@ static void rds_conn_destroy_fini(struct kref *kref)
spin_lock_irqsave(&rds_conn_lock, flags);
rds_conn_count--;
spin_unlock_irqrestore(&rds_conn_lock, flags);
+
+ /* only after everything the transport module owns has been
+ * freed above may its unload proceed
+ */
+ if (!atomic_dec_return(&trans->t_conn_count))
+ wake_up_all(&rds_conn_freed_waitq);
+}
+
+/* Wait for all of @trans's connections to be freed; the free runs
+ * asynchronously once rds_conn_destroy() has quiesced a connection.
+ * Called on transport module unload, after the transport has destroyed
+ * all of its connections. A connection reference can be held for an
+ * application-controlled time - an unread datagram pins the inc that
+ * carries it, and thus the connection - so the wait is unbounded: the
+ * frees that run after unload call into this module's text (conn_free,
+ * inc_free) and free into its slabs, so proceeding while any remain
+ * would be a use-after-free, not a leak. Warn periodically so a stuck
+ * count is diagnosable, but never stop waiting. This matches the
+ * historical RDS contract that teardown does not discard queued data.
+ */
+void rds_conn_wait_conns_freed(struct rds_transport *trans,
+ void (*resweep)(void))
+{
+ unsigned long warn_interval =
+ msecs_to_jiffies(RDS_CONN_FREE_WARN_INTERVAL_MS);
+ unsigned long warn_at = jiffies + warn_interval;
+
+ while (!wait_event_timeout(rds_conn_freed_waitq,
+ !atomic_read(&trans->t_conn_count),
+ msecs_to_jiffies(RDS_CONN_FREE_POLL_MS))) {
+ /* A transport whose teardown is asynchronous (IB moves a
+ * connection off its device from the shutdown work) gives
+ * us a resweep to destroy what has arrived since.
+ */
+ if (resweep)
+ resweep();
+ if (time_after_eq(jiffies, warn_at)) {
+ pr_warn("RDS/%s: still waiting for %d connection(s) to be freed before unload\n",
+ trans->t_name,
+ atomic_read(&trans->t_conn_count));
+ warn_at = jiffies + warn_interval;
+ }
+ }
}
+EXPORT_SYMBOL_GPL(rds_conn_wait_conns_freed);
void rds_conn_get(struct rds_connection *conn)
{diff --git a/net/rds/ib.c b/net/rds/ib.c
index 9fe3b9951bd3..3fc2de9d19d5 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -537,7 +537,24 @@ void rds_ib_exit(void)
rds_info_deregister_func(RDS6_INFO_IB_CONNECTIONS, rds6_ib_ic_info);
#endif
rds_ib_unregister_client();
+
+ /* rds_ib_dev_shutdown() only dropped the connections still
+ * attached to a device; each moves itself to ib_nodev_conns
+ * from its shutdown work. Destroy what is there now and keep
+ * sweeping the list while the wait sees connections outstanding,
+ * so a late arrival is destroyed rather than waited on forever.
+ */
rds_ib_destroy_nodev_conns();
+ rds_conn_wait_conns_freed(&rds_ib_transport,
+ rds_ib_destroy_nodev_conns);
+
+ /* Tearing down the last connection may have dropped the final
+ * reference on a device, deferring rds_ib_dev_free() to rds_wq.
+ * Drain it before the module goes away; it queues nothing
+ * further on rds_wq.
+ */
+ flush_workqueue(rds_wq);
+
rds_ib_sysctl_exit();
rds_ib_recv_exit();
rds_trans_unregister(&rds_ib_transport);
diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c
index db7e92e7bd29..a9b27f06cbfc 100644
--- a/net/rds/ib_rdma.c
+++ b/net/rds/ib_rdma.c
@@ -165,7 +165,7 @@ void rds_ib_destroy_nodev_conns(void)
/* avoid calling conn_destroy with irqs off */
spin_lock_irq(&ib_nodev_conns_lock);
- list_splice(&ib_nodev_conns, &tmp_list);
+ list_splice_init(&ib_nodev_conns, &tmp_list);
spin_unlock_irq(&ib_nodev_conns_lock);
list_for_each_entry_safe(ic, _ic, &tmp_list, ib_node)
diff --git a/net/rds/loop.c b/net/rds/loop.c
index e6b0750bbeda..fd774f8080d0 100644
--- a/net/rds/loop.c
+++ b/net/rds/loop.c
@@ -195,6 +195,8 @@ void rds_loop_exit(void)
WARN_ON(lc->conn->c_passive);
rds_conn_destroy(lc->conn);
}
+
+ rds_conn_wait_conns_freed(&rds_loop_transport, NULL);
}
static void rds_loop_kill_conns(struct net *net)
diff --git a/net/rds/rds.h b/net/rds/rds.h
index 49629108c22a..8a969444e698 100644
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -557,6 +557,12 @@ struct rds_transport {
unsigned int t_prefer_loopback:1,
t_mp_capable:1;
unsigned int t_type;
+ /* Connections of this transport not yet freed; freeing runs
+ * asynchronously once rds_conn_destroy() has quiesced a
+ * connection, so transport module unload has to wait for this
+ * to reach zero (rds_conn_wait_conns_freed()).
+ */
+ atomic_t t_conn_count;
int (*laddr_check)(struct net *net, const struct in6_addr *addr,
__u32 scope_id);@@ -834,6 +840,13 @@ void rds_conn_shutdown(struct rds_conn_path *cpath);
void rds_conn_destroy(struct rds_connection *conn);
void rds_conn_get(struct rds_connection *conn);
void rds_conn_put(struct rds_connection *conn);
+/* transport unload waits for its connections to be freed, polling at
+ * the first interval and warning at the second
+ */
+#define RDS_CONN_FREE_POLL_MS 100
+#define RDS_CONN_FREE_WARN_INTERVAL_MS 10000
+void rds_conn_wait_conns_freed(struct rds_transport *trans,
+ void (*resweep)(void));
void rds_conn_drop(struct rds_connection *conn);
void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
void rds_conn_connect_if_down(struct rds_connection *conn);
diff --git a/net/rds/tcp.c b/net/rds/tcp.c
index 774a71f88d37..826e4629e4ee 100644
--- a/net/rds/tcp.c
+++ b/net/rds/tcp.c
@@ -805,6 +805,7 @@ static void rds_tcp_exit(void)
#endif
unregister_pernet_device(&rds_tcp_net_ops);
rds_tcp_destroy_conns();
+ rds_conn_wait_conns_freed(&rds_tcp_transport, NULL);
rds_trans_unregister(&rds_tcp_transport);
rds_tcp_recv_exit();
kmem_cache_destroy(rds_tcp_conn_slab);
--
2.25.1