diff --git a/net/rds/connection.c b/net/rds/connection.c
index 97d470242d0f..258a6bc3c573 100644
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -231,6 +231,7 @@ static struct rds_connection *__rds_conn_create(struct net *net,
goto out;
}
+ kref_init(&conn->c_refcount);
INIT_HLIST_NODE(&conn->c_hash_node);
conn->c_laddr = *laddr;
conn->c_isv6 = !ipv6_addr_v4mapped(laddr);
@@ -477,9 +478,10 @@ void rds_conn_shutdown(struct rds_conn_path *cp)
* Quiesce the reconnect timer before bailing
* out, though. When a pending destroy did
* suppress the queue, no later pass runs, and
- * rds_conn_path_destroy() is about to flush
- * cp_down_w and free the path: it must not
- * find cp_conn_w still armed. A successor
+ * rds_conn_path_quiesce() is about to flush
+ * cp_down_w, ahead of the path's deferred
+ * free: it must not find cp_conn_w still
+ * armed. A successor
* pass, when there is one, re-arms the
* reconnect from its own tail.
*/
@@ -526,10 +528,12 @@ void rds_conn_shutdown(struct rds_conn_path *cp)
conn->c_trans->conn_slots_available(conn, false);
}
-/* destroy a single rds_conn_path. rds_conn_destroy() iterates over
- * all paths using rds_conn_path_destroy()
+/* quiesce a single rds_conn_path: shut it down and tear down any
+ * queued messages. rds_conn_destroy() iterates over all paths using
+ * rds_conn_path_quiesce(); the transport state and the workqueue are
+ * freed later, from rds_conn_path_free().
*/
-static void rds_conn_path_destroy(struct rds_conn_path *cp)
+static void rds_conn_path_quiesce(struct rds_conn_path *cp)
{
struct rds_message *rm, *rtmp;
@@ -558,6 +562,16 @@ static void rds_conn_path_destroy(struct rds_conn_path *cp)
WARN_ON(delayed_work_pending(&cp->cp_recv_w));
WARN_ON(delayed_work_pending(&cp->cp_conn_w));
WARN_ON(work_pending(&cp->cp_down_w));
+}
+
+/* free a quiesced rds_conn_path's transport state and workqueue; runs
+ * from rds_conn_destroy_fini() once the last connection reference is
+ * dropped.
+ */
+static void rds_conn_path_free(struct rds_conn_path *cp)
+{
+ if (!cp->cp_transport_data)
+ return;
if (cp->cp_wq != rds_wq) {
destroy_workqueue(cp->cp_wq);@@ -567,16 +581,55 @@ static void rds_conn_path_destroy(struct rds_conn_path *cp)
cp->cp_conn->c_trans->conn_free(cp->cp_transport_data);
}
+/* Free a connection. This runs from rds_conn_put() when the last
+ * reference is dropped, after rds_conn_destroy() has quiesced the
+ * connection and dropped the initial reference.
+ */
+static void rds_conn_destroy_fini(struct kref *kref)
+{
+ struct rds_connection *conn = container_of(kref, struct rds_connection,
+ c_refcount);
+ int npaths = (conn->c_trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);
+ unsigned long flags;
+ int i;
+
+ for (i = 0; i < npaths; i++)
+ rds_conn_path_free(&conn->c_path[i]);
+
+ kfree(conn->c_path);
+ kmem_cache_free(rds_conn_slab, conn);
+
+ spin_lock_irqsave(&rds_conn_lock, flags);
+ rds_conn_count--;
+ spin_unlock_irqrestore(&rds_conn_lock, flags);
+}
+
+void rds_conn_get(struct rds_connection *conn)
+{
+ kref_get(&conn->c_refcount);
+}
+EXPORT_SYMBOL_GPL(rds_conn_get);
+
+void rds_conn_put(struct rds_connection *conn)
+{
+ kref_put(&conn->c_refcount, rds_conn_destroy_fini);
+}
+EXPORT_SYMBOL_GPL(rds_conn_put);
+
/*
* Stop and free a connection.
*
- * This can only be used in very limited circumstances. It assumes that once
- * the conn has been shutdown that no one else is referencing the connection.
- * We can only ensure this in the rmmod path in the current code.
+ * Quiesces the connection synchronously (workers cancelled, transport
+ * connections shut down, queued messages dropped) and drops the
+ * initial reference. Only the first call for a connection does any
+ * of that: a later one finds c_destroy_in_prog already set and
+ * returns at once, without waiting for the first. The memory -
+ * including the transport's per-connection state and the path
+ * workqueues - is freed once the last rds_conn_put() runs, which may
+ * be after this returns.
*/
void rds_conn_destroy(struct rds_connection *conn)
{
- unsigned long flags;
int i;
struct rds_conn_path *cp;
int npaths = (conn->c_trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);@@ -591,11 +644,23 @@ void rds_conn_destroy(struct rds_connection *conn)
* bar the exemptions noted at c_destroy_in_prog) from queueing
* new work on the path workqueues once we start cancelling and
* destroying them.
+ *
+ * Only the first caller proceeds. No caller in this tree hands
+ * a connection to rds_conn_destroy() twice - every teardown
+ * sweep unlinks or claims its node before the destroy - so this
+ * documents the contract rather than closing a path: a second
+ * caller returns at once, without quiescing or waiting. The
+ * unhash also happens under rds_conn_lock, so a looked-up conn
+ * can never be quiesced twice.
*/
+ spin_lock_irq(&rds_conn_lock);
+ if (conn->c_destroy_in_prog) {
+ spin_unlock_irq(&rds_conn_lock);
+ return;
+ }
WRITE_ONCE(conn->c_destroy_in_prog, true);
/* Ensure conn will not be scheduled for reconnect */
- spin_lock_irq(&rds_conn_lock);
hlist_del_init_rcu(&conn->c_hash_node);
spin_unlock_irq(&rds_conn_lock);
synchronize_rcu();@@ -603,7 +668,7 @@ void rds_conn_destroy(struct rds_connection *conn)
/* shut the connection down */
for (i = 0; i < npaths; i++) {
cp = &conn->c_path[i];
- rds_conn_path_destroy(cp);
+ rds_conn_path_quiesce(cp);
BUG_ON(!list_empty(&cp->cp_retrans));
}
@@ -614,12 +679,10 @@ void rds_conn_destroy(struct rds_connection *conn)
*/
rds_cong_remove_conn(conn);
- kfree(conn->c_path);
- kmem_cache_free(rds_conn_slab, conn);
-
- spin_lock_irqsave(&rds_conn_lock, flags);
- rds_conn_count--;
- spin_unlock_irqrestore(&rds_conn_lock, flags);
+ /* drop the initial reference; the connection is freed from
+ * rds_conn_destroy_fini() once every holder has dropped theirs
+ */
+ rds_conn_put(conn);
}
EXPORT_SYMBOL_GPL(rds_conn_destroy);
diff --git a/net/rds/rds.h b/net/rds/rds.h
index 7e0364fe7b56..6cc5ea183491 100644
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -137,6 +137,12 @@ struct rds_conn_path {
/* One rds_connection per RDS address pair */
struct rds_connection {
struct hlist_node c_hash_node;
+ /* rds_conn_destroy() quiesces the connection synchronously;
+ * freeing it - the connection memory, the path workqueues and
+ * the transport's per-connection state - is deferred until the
+ * last reference is dropped via rds_conn_put().
+ */
+ struct kref c_refcount;
struct in6_addr c_laddr;
struct in6_addr c_faddr;
int c_dev_if; /* ifindex used for this conn */@@ -147,8 +153,10 @@ struct rds_connection {
c_pad_to_32:29;
int c_npaths;
bool c_with_sport_idx;
- /* Set once, by rds_conn_destroy(), before it cancels the path
- * works; read through rds_destroy_pending(). A site that arms
+ /* Set once, by rds_conn_destroy() under rds_conn_lock - a
+ * test-and-set, so a second destroy of the same connection
+ * returns at once - before it cancels the path works. Read
+ * through rds_destroy_pending(). A site that arms
* a path work must test the predicate and queue the work inside
* one rcu_read_lock() section: the synchronize_rcu() that
* follows the store is what keeps a queue issued after the@@ -833,6 +841,13 @@ struct rds_connection *rds_conn_create_outgoing(struct net *net,
u8 tos, gfp_t gfp, int dev_if);
void rds_conn_shutdown(struct rds_conn_path *cpath);
void rds_conn_destroy(struct rds_connection *conn);
+void rds_conn_get(struct rds_connection *conn);
+void rds_conn_put(struct rds_connection *conn);
+/* take a reference unless the connection is already being freed */
+static inline bool rds_conn_get_unless_zero(struct rds_connection *conn)
+{
+ return kref_get_unless_zero(&conn->c_refcount);
+}
void rds_conn_drop(struct rds_connection *conn);
void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
void rds_conn_connect_if_down(struct rds_connection *conn);--
2.50.1