Il giorno 17 dic 2016, alle ore 01:12, Jens Axboe [off-list ref] ha =

scritto:

=20
This adds a set of hooks that intercepts the blk-mq path of
allocating/inserting/issuing/completing requests, allowing
us to develop a scheduler within that framework.
=20
We reuse the existing elevator scheduler API on the registration
side, but augment that with the scheduler flagging support for
the blk-mq interfce, and with a separate set of ops hooks for MQ
devices.
=20
Schedulers can opt in to using shadow requests. Shadow requests
are internal requests that the scheduler uses for for the allocate
and insert part, which are then mapped to a real driver request
at dispatch time. This is needed to separate the device queue depth
from the pool of requests that the scheduler has to work with.
=20
Signed-off-by: Jens Axboe <axboe@fb.com>

...
=20
+struct request *blk_mq_sched_get_request(struct request_queue *q,
+					 struct bio *bio,
+					 unsigned int op,
+					 struct blk_mq_alloc_data *data)
+{
+	struct elevator_queue *e =3D q->elevator;
+	struct blk_mq_hw_ctx *hctx;
+	struct blk_mq_ctx *ctx;
+	struct request *rq;
+
+	blk_queue_enter_live(q);
+	ctx =3D blk_mq_get_ctx(q);
+	hctx =3D blk_mq_map_queue(q, ctx->cpu);
+
+	blk_mq_set_alloc_data(data, q, 0, ctx, hctx);
+
+	if (e && e->type->ops.mq.get_request)
+		rq =3D e->type->ops.mq.get_request(q, op, data);

bio is not passed to the scheduler here.  Yet bfq uses bio to get the
blkcg (invoking bio_blkcg).  I'm not finding any workaround.

+	else
+		rq =3D __blk_mq_alloc_request(data, op);
+
+	if (rq) {
+		rq->elv.icq =3D NULL;
+		if (e && e->type->icq_cache)
+			blk_mq_sched_assign_ioc(q, rq, bio);

bfq needs rq->elv.icq to be consistent in bfq_get_request, but the
needed initialization seems to occur only after mq.get_request is
invoked.

Note: to minimize latency, I'm reporting immediately each problem that
apparently cannot be solved by just modifying bfq.  But, if the
resulting higher number of micro-emails is annoying for you, I can
buffer my questions, and send you cumulative emails less frequently.

Thanks,
Paolo

+		data->hctx->queued++;
+		return rq;
+	}
+
+	blk_queue_exit(q);
+	return NULL;
+}
+
+void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
+{
+	struct elevator_queue *e =3D hctx->queue->elevator;
+	LIST_HEAD(rq_list);
+
+	if (unlikely(blk_mq_hctx_stopped(hctx)))
+		return;
+
+	hctx->run++;
+
+	/*
+	 * If we have previous entries on our dispatch list, grab them =

first for

+	 * more fair dispatch.
+	 */
+	if (!list_empty_careful(&hctx->dispatch)) {
+		spin_lock(&hctx->lock);
+		if (!list_empty(&hctx->dispatch))
+			list_splice_init(&hctx->dispatch, &rq_list);
+		spin_unlock(&hctx->lock);
+	}
+
+	/*
+	 * Only ask the scheduler for requests, if we didn't have =

residual

+	 * requests from the dispatch list. This is to avoid the case =

where

+	 * we only ever dispatch a fraction of the requests available =

because

+	 * of low device queue depth. Once we pull requests out of the =

IO

+	 * scheduler, we can no longer merge or sort them. So it's best =

to

+	 * leave them there for as long as we can. Mark the hw queue as
+	 * needing a restart in that case.
+	 */
+	if (list_empty(&rq_list)) {
+		if (e && e->type->ops.mq.dispatch_requests)
+			e->type->ops.mq.dispatch_requests(hctx, =

&rq_list);

+		else
+			blk_mq_flush_busy_ctxs(hctx, &rq_list);
+	} else if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
+		set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
+
+	blk_mq_dispatch_rq_list(hctx, &rq_list);
+}
+
+bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio)
+{
+	struct request *rq;
+	int ret;
+
+	ret =3D elv_merge(q, &rq, bio);
+	if (ret =3D=3D ELEVATOR_BACK_MERGE) {
+		if (bio_attempt_back_merge(q, rq, bio)) {
+			if (!attempt_back_merge(q, rq))
+				elv_merged_request(q, rq, ret);
+			return true;
+		}
+	} else if (ret =3D=3D ELEVATOR_FRONT_MERGE) {
+		if (bio_attempt_front_merge(q, rq, bio)) {
+			if (!attempt_front_merge(q, rq))
+				elv_merged_request(q, rq, ret);
+			return true;
+		}
+	}
+
+	return false;
+}
+EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
+
+bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio =

*bio)

+{
+	struct elevator_queue *e =3D q->elevator;
+
+	if (e->type->ops.mq.bio_merge) {
+		struct blk_mq_ctx *ctx =3D blk_mq_get_ctx(q);
+		struct blk_mq_hw_ctx *hctx =3D blk_mq_map_queue(q, =

ctx->cpu);

+
+		blk_mq_put_ctx(ctx);
+		return e->type->ops.mq.bio_merge(hctx, bio);
+	}
+
+	return false;
+}
+
+bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct =

request *rq)

unsigned int numa_node);

+void blk_mq_sched_free_requests(struct blk_mq_tags *tags);
+
+int blk_mq_sched_init_hctx_data(struct request_queue *q, size_t size,
+				int (*init)(struct blk_mq_hw_ctx *),
+				void (*exit)(struct blk_mq_hw_ctx *));
+
+void blk_mq_sched_free_hctx_data(struct request_queue *q,
+				 void (*exit)(struct blk_mq_hw_ctx *));
+
+void blk_mq_sched_free_shadow_request(struct blk_mq_tags *tags,
+				      struct request *rq);
+struct request *blk_mq_sched_alloc_shadow_request(struct =

request_queue *q,

+						  struct =

blk_mq_alloc_data *data,

+						  struct blk_mq_tags =

*tags,

+						  atomic_t *wait_index);
+struct request *
+blk_mq_sched_request_from_shadow(struct blk_mq_hw_ctx *hctx,
+				 struct request *(*get_sched_rq)(struct =

blk_mq_hw_ctx *));

+struct request *
+__blk_mq_sched_request_from_shadow(struct blk_mq_hw_ctx *hctx,
+				   struct request *sched_rq);
+
+struct request *blk_mq_sched_get_request(struct request_queue *q, =

struct bio *bio, unsigned int op, struct blk_mq_alloc_data *data);

+
+void __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx);
+void blk_mq_sched_request_inserted(struct request *rq);
+bool blk_mq_sched_try_merge(struct request_queue *q, struct bio =

*bio);

+bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio =

*bio);

+bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct =

request *rq);

+
+void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx);
+
+static inline bool
+blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
+{
+	struct elevator_queue *e =3D q->elevator;
+
+	if (!e || blk_queue_nomerges(q) || !bio_mergeable(bio))
+		return false;
+
+	return __blk_mq_sched_bio_merge(q, bio);
+}
+
+static inline int blk_mq_sched_get_rq_priv(struct request_queue *q,
+					   struct request *rq)
+{
+	struct elevator_queue *e =3D q->elevator;
+
+	if (e && e->type->ops.mq.get_rq_priv)
+		return e->type->ops.mq.get_rq_priv(q, rq);
+
+	return 0;
+}
+
+static inline void blk_mq_sched_put_rq_priv(struct request_queue *q,
+					    struct request *rq)
+{
+	struct elevator_queue *e =3D q->elevator;
+
+	if (e && e->type->ops.mq.put_rq_priv)
+		e->type->ops.mq.put_rq_priv(q, rq);
+}
+
+static inline void blk_mq_sched_put_request(struct request *rq)
+{
+	struct request_queue *q =3D rq->q;
+	struct elevator_queue *e =3D q->elevator;
+	bool do_free =3D true;
+
+	wbt_done(q->rq_wb, &rq->issue_stat);
+
+	if (rq->rq_flags & RQF_ELVPRIV) {
+		blk_mq_sched_put_rq_priv(rq->q, rq);
+		if (rq->elv.icq) {
+			put_io_context(rq->elv.icq->ioc);
+			rq->elv.icq =3D NULL;
+		}
+	}
+
+	if (e && e->type->ops.mq.put_request)
+		do_free =3D !e->type->ops.mq.put_request(rq);
+	if (do_free)
+		blk_mq_finish_request(rq);
+}
+
+static inline void
+blk_mq_sched_insert_request(struct request *rq, bool at_head, bool =

run_queue,

+			    bool async)
+{
+	struct request_queue *q =3D rq->q;
+	struct elevator_queue *e =3D q->elevator;
+	struct blk_mq_ctx *ctx =3D rq->mq_ctx;
+	struct blk_mq_hw_ctx *hctx =3D blk_mq_map_queue(q, ctx->cpu);
+
+	if (e && e->type->ops.mq.insert_requests) {
+		LIST_HEAD(list);
+
+		list_add(&rq->queuelist, &list);
+		e->type->ops.mq.insert_requests(hctx, &list, at_head);
+	} else {
+		spin_lock(&ctx->lock);
+		__blk_mq_insert_request(hctx, rq, at_head);
+		spin_unlock(&ctx->lock);
+	}
+
+	if (run_queue)
+		blk_mq_run_hw_queue(hctx, async);
+}
+
+static inline void
+blk_mq_sched_insert_requests(struct request_queue *q, struct =

blk_mq_ctx *ctx,

+			     struct list_head *list, bool =

run_queue_async)

+{
+	struct blk_mq_hw_ctx *hctx =3D blk_mq_map_queue(q, ctx->cpu);
+	struct elevator_queue *e =3D hctx->queue->elevator;
+
+	if (e && e->type->ops.mq.insert_requests)
+		e->type->ops.mq.insert_requests(hctx, list, false);
+	else
+		blk_mq_insert_requests(hctx, ctx, list);
+
+	blk_mq_run_hw_queue(hctx, run_queue_async);
+}
+
+static inline void
+blk_mq_sched_dispatch_shadow_requests(struct blk_mq_hw_ctx *hctx,
+				      struct list_head *rq_list,
+				      struct request =

*(*get_sched_rq)(struct blk_mq_hw_ctx *))

+{
+	do {
+		struct request *rq;
+
+		rq =3D blk_mq_sched_request_from_shadow(hctx, =

get_sched_rq);

+		if (!rq)
+			break;
+
+		list_add_tail(&rq->queuelist, rq_list);
+	} while (1);
+}
+
+static inline bool
+blk_mq_sched_allow_merge(struct request_queue *q, struct request *rq,
+			 struct bio *bio)
+{
+	struct elevator_queue *e =3D q->elevator;
+
+	if (e && e->type->ops.mq.allow_merge)
+		return e->type->ops.mq.allow_merge(q, rq, bio);
+
+	return true;
+}
+
+static inline void
+blk_mq_sched_completed_request(struct blk_mq_hw_ctx *hctx, struct =

request *rq)

request_queue *q, int rw,

}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
=20
-void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx, struct =

blk_mq_ctx *ctx,

-			   struct request *rq)
+void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct =

blk_mq_ctx *ctx,

*hctx, struct blk_mq_ctx *ctx,

	blk_queue_exit(q);
}
=20
-static void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *hctx,
+static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
				     struct request *rq)
{
	struct blk_mq_ctx *ctx =3D rq->mq_ctx;
=20
	ctx->rq_completed[rq_is_sync(rq)]++;
-	__blk_mq_free_request(hctx, ctx, rq);
+	__blk_mq_finish_request(hctx, ctx, rq);
+}
+
+void blk_mq_finish_request(struct request *rq)
+{
+	blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, =

rq->mq_ctx->cpu), rq);

}
=20
void blk_mq_free_request(struct request *rq)
{
-	blk_mq_free_hctx_request(blk_mq_map_queue(rq->q, =

rq->mq_ctx->cpu), rq);

request *rq)

work_struct *work)

=20
		rq->rq_flags &=3D ~RQF_SOFTBARRIER;
		list_del_init(&rq->queuelist);
-		blk_mq_insert_request(rq, true, false, false);
+		blk_mq_sched_insert_request(rq, true, false, false);
	}
=20
	while (!list_empty(&rq_list)) {
		rq =3D list_entry(rq_list.next, struct request, =

queuelist);

request_queue *q,

*hctx, struct list_head *list)

	return ret !=3D BLK_MQ_RQ_QUEUE_BUSY;
}
=20
-/*
- * Run this hardware queue, pulling any software queues mapped to it =

in.

- * Note that this function currently has various problems around =

ordering

- * of IO. In particular, we'd like FIFO behaviour on handling =

existing

blk_mq_hw_ctx *hctx)

blk_mq_hw_ctx *hctx, struct request *rq,

	blk_mq_hctx_mark_pending(hctx, ctx);
}
=20
-void blk_mq_insert_request(struct request *rq, bool at_head, bool =

run_queue,

-			   bool async)
-{
-	struct blk_mq_ctx *ctx =3D rq->mq_ctx;
-	struct request_queue *q =3D rq->q;
-	struct blk_mq_hw_ctx *hctx =3D blk_mq_map_queue(q, ctx->cpu);
-
-	spin_lock(&ctx->lock);
-	__blk_mq_insert_request(hctx, rq, at_head);
-	spin_unlock(&ctx->lock);
-
-	if (run_queue)
-		blk_mq_run_hw_queue(hctx, async);
-}
-
-static void blk_mq_insert_requests(struct request_queue *q,
-				     struct blk_mq_ctx *ctx,
-				     struct list_head *list,
-				     int depth,
-				     bool from_schedule)
+void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct =

blk_mq_ctx *ctx,

+			    struct list_head *list)
=20
{
-	struct blk_mq_hw_ctx *hctx =3D blk_mq_map_queue(q, ctx->cpu);
-
-	trace_block_unplug(q, depth, !from_schedule);
-
	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu =

is

request_queue *q,

	}
	blk_mq_hctx_mark_pending(hctx, ctx);
	spin_unlock(&ctx->lock);
-
-	blk_mq_run_hw_queue(hctx, from_schedule);
}
=20
static int plug_ctx_cmp(void *priv, struct list_head *a, struct =

list_head *b)

*plug, bool from_schedule)

		BUG_ON(!rq->q);
		if (rq->mq_ctx !=3D this_ctx) {
			if (this_ctx) {
-				blk_mq_insert_requests(this_q, this_ctx,
-							&ctx_list, =

depth,

-							from_schedule);
+				trace_block_unplug(this_q, depth, =

from_schedule);

+				blk_mq_sched_insert_requests(this_q, =

this_ctx,

+								=

&ctx_list,

+								=

from_schedule);

*plug, bool from_schedule)

	 * on 'ctx_list'. Do those.
	 */
	if (this_ctx) {
-		blk_mq_insert_requests(this_q, this_ctx, &ctx_list, =

depth,

-				       from_schedule);
+		trace_block_unplug(this_q, depth, from_schedule);
+		blk_mq_sched_insert_requests(this_q, this_ctx, =

&ctx_list,

blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,

		}
=20
		spin_unlock(&ctx->lock);
-		__blk_mq_free_request(hctx, ctx, rq);
+		__blk_mq_finish_request(hctx, ctx, rq);
		return true;
	}
}
=20
-static struct request *blk_mq_map_request(struct request_queue *q,
-					  struct bio *bio,
-					  struct blk_mq_alloc_data =

*data)

-{
-	struct blk_mq_hw_ctx *hctx;
-	struct blk_mq_ctx *ctx;
-	struct request *rq;
-
-	blk_queue_enter_live(q);
-	ctx =3D blk_mq_get_ctx(q);
-	hctx =3D blk_mq_map_queue(q, ctx->cpu);
-
-	trace_block_getrq(q, bio, bio->bi_opf);
-	blk_mq_set_alloc_data(data, q, 0, ctx, hctx);
-	rq =3D __blk_mq_alloc_request(data, bio->bi_opf);
-
-	data->hctx->queued++;
-	return rq;
-}
-
static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t =

*cookie)

{
-	int ret;
	struct request_queue *q =3D rq->q;
-	struct blk_mq_hw_ctx *hctx =3D blk_mq_map_queue(q, =

rq->mq_ctx->cpu);

	struct blk_mq_queue_data bd =3D {
		.rq =3D rq,
		.list =3D NULL,
		.last =3D 1
	};
-	blk_qc_t new_cookie =3D blk_tag_to_qc_t(rq->tag, =

hctx->queue_num);

request *rq, blk_qc_t *cookie)

request_queue *q, struct bio *bio)

	    blk_attempt_plug_merge(q, bio, &request_count, =

&same_queue_rq))

request_queue *q, struct bio *bio)

		goto done;
	}
=20
+	if (q->elevator) {
+		blk_mq_put_ctx(data.ctx);
+		blk_mq_bio_to_request(rq, bio);
+		blk_mq_sched_insert_request(rq, false, true, true);
+		goto done;
+	}
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware =

immediately. For

request_queue *q, struct bio *bio)

request_queue *q, struct bio *bio)

		return cookie;
	}
=20
+	if (q->elevator) {
+		blk_mq_put_ctx(data.ctx);
+		blk_mq_bio_to_request(rq, bio);
+		blk_mq_sched_insert_request(rq, false, true, true);
+		goto done;
+	}
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware =

immediately. For

request_queue *q, struct bio *bio)

*set, struct blk_mq_tags *tags,

blk_mq_tag_set *set,

 */
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct =

request *rq,

				bool at_head);
-
+void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct =

blk_mq_ctx *ctx,

blk_mq_alloc_data *data,

 */
void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx =

*ctx,

			struct request *rq, unsigned int op);
-void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx, struct =

blk_mq_ctx *ctx,

+void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct =

blk_mq_ctx *ctx,

request *rq, struct bio *bio)

	struct request_queue *q =3D rq->q;
	struct elevator_queue *e =3D q->elevator;
=20
-	if (e->type->ops.sq.elevator_allow_bio_merge_fn)
+	if (e->uses_mq && e->type->ops.mq.allow_merge)
+		return e->type->ops.mq.allow_merge(q, rq, bio);
+	else if (!e->uses_mq && =

e->type->ops.sq.elevator_allow_bio_merge_fn)

		return e->type->ops.sq.elevator_allow_bio_merge_fn(q, =

rq, bio);

request_queue *q,

*name)

struct request *rq)

struct request *rq)

	hash_add(e->hash, &rq->hash, rq_hash_key(rq));
	rq->rq_flags |=3D RQF_HASHED;
}
+EXPORT_SYMBOL_GPL(elv_rqhash_add);
=20
void elv_rqhash_reposition(struct request_queue *q, struct request =

*rq)

request **req, struct bio *bio)

request **req, struct bio *bio)

 *
 * Returns true if we merged, false otherwise
 */
-static bool elv_attempt_insert_merge(struct request_queue *q,
-				     struct request *rq)
+bool elv_attempt_insert_merge(struct request_queue *q, struct request =

*rq)

struct request *rq, int type)

struct request *rq,

			     struct request *next)
{
	struct elevator_queue *e =3D q->elevator;
-	const int next_sorted =3D next->rq_flags & RQF_SORTED;
-
-	if (next_sorted && e->type->ops.sq.elevator_merge_req_fn)
-		e->type->ops.sq.elevator_merge_req_fn(q, rq, next);
+	bool next_sorted =3D false;
+
+	if (e->uses_mq && e->type->ops.mq.requests_merged)
+		e->type->ops.mq.requests_merged(q, rq, next);
+	else if (e->type->ops.sq.elevator_merge_req_fn) {
+		next_sorted =3D next->rq_flags & RQF_SORTED;
+		if (next_sorted)
+			e->type->ops.sq.elevator_merge_req_fn(q, rq, =

next);

struct request *rq,

request_queue *q, struct request *rq)

request_queue *q, struct request *rq)

struct request *rq,

{
	struct elevator_queue *e =3D q->elevator;
=20
+	if (WARN_ON_ONCE(e->uses_mq))
+		return 0;
+
	if (e->type->ops.sq.elevator_set_req_fn)
		return e->type->ops.sq.elevator_set_req_fn(q, rq, bio, =

gfp_mask);

struct request *rq)

unsigned int op)

*q, struct request *rq)

e->type->ops.sq.elevator_registered_fn)

elevator_type *new_e)

{
	struct elevator_queue *old =3D q->elevator;
-	bool registered =3D old->registered;
+	bool old_registered =3D false;
	int err;
=20
+	if (q->mq_ops) {
+		blk_mq_freeze_queue(q);
+		blk_mq_quiesce_queue(q);
+	}
+
	/*
	 * Turn on BYPASS and drain all requests w/ elevator private =

data.

	 * Block layer doesn't call into a quiesced elevator - all =

requests

*q, struct elevator_type *new_e)

	 * using INSERT_BACK.  All requests have SOFTBARRIER set and no
	 * merge happens either.
	 */
-	blk_queue_bypass_start(q);
+	if (old) {
+		old_registered =3D old->registered;
=20
-	/* unregister and clear all auxiliary data of the old elevator =

*/

-	if (registered)
-		elv_unregister_queue(q);
+		if (!q->mq_ops)
+			blk_queue_bypass_start(q);
=20
-	spin_lock_irq(q->queue_lock);
-	ioc_clear_queue(q);
-	spin_unlock_irq(q->queue_lock);
+		/* unregister and clear all auxiliary data of the old =

elevator */

+		if (old_registered)
+			elv_unregister_queue(q);
+
+		spin_lock_irq(q->queue_lock);
+		ioc_clear_queue(q);
+		spin_unlock_irq(q->queue_lock);
+	}
=20
	/* allocate, init and register new elevator */
-	err =3D new_e->ops.sq.elevator_init_fn(q, new_e);
-	if (err)
-		goto fail_init;
+	if (new_e) {
+		if (new_e->uses_mq)
+			err =3D new_e->ops.mq.init_sched(q, new_e);
+		else
+			err =3D new_e->ops.sq.elevator_init_fn(q, =

new_e);

+		if (err)
+			goto fail_init;
=20
-	if (registered) {
		err =3D elv_register_queue(q);
		if (err)
			goto fail_register;
-	}
+	} else
+		q->elevator =3D NULL;
=20
	/* done, kill the old one and finish */
-	elevator_exit(old);
-	blk_queue_bypass_end(q);
+	if (old) {
+		elevator_exit(old);
+		if (!q->mq_ops)
+			blk_queue_bypass_end(q);
+	}
+
+	if (q->mq_ops) {
+		blk_mq_unfreeze_queue(q);
+		blk_mq_start_stopped_hw_queues(q, true);
+	}
=20
-	blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
+	if (new_e)
+		blk_add_trace_msg(q, "elv switch: %s", =

new_e->elevator_name);

*q, struct elevator_type *new_e)

request_queue *q, const char *name)

request_queue *q, const char *name)

		return -EINVAL;
	}
=20
-	if (!strcmp(elevator_name, q->elevator->type->elevator_name)) {
+	if (q->elevator &&
+	    !strcmp(elevator_name, q->elevator->type->elevator_name)) {
		elevator_put(e);
		return 0;
	}
=20
+	if (!e->uses_mq && q->mq_ops) {
+		printk(KERN_ERR "blk-mq-sched: elv %s does not support =

mq\n", elevator_name);

+		elevator_put(e);
+		return -EINVAL;
+	}
+	if (e->uses_mq && !q->mq_ops) {
+		printk(KERN_ERR "blk-mq-sched: elv %s is for mq\n", =

elevator_name);

*q, const char *name,

*q, const char *name,

ssize_t elv_iosched_show(struct request_queue *q, char *name)
{
	struct elevator_queue *e =3D q->elevator;
-	struct elevator_type *elv;
+	struct elevator_type *elv =3D NULL;
	struct elevator_type *__e;
	int len =3D 0;
=20
-	if (!q->elevator || !blk_queue_stackable(q))
+	if (!blk_queue_stackable(q))
		return sprintf(name, "none\n");
=20
-	elv =3D e->type;
+	if (!q->elevator)
+		len +=3D sprintf(name+len, "[none] ");
+	else
+		elv =3D e->type;
=20
	spin_lock(&elv_list_lock);
	list_for_each_entry(__e, &elv_list, list) {
-		if (!strcmp(elv->elevator_name, __e->elevator_name))
+		if (elv && !strcmp(elv->elevator_name, =

__e->elevator_name)) {

			len +=3D sprintf(name+len, "[%s] ", =

elv->elevator_name);

-		else
+			continue;
+		}
+		if (__e->uses_mq && q->mq_ops)
+			len +=3D sprintf(name+len, "%s ", =

__e->elevator_name);

+		else if (!__e->uses_mq && !q->mq_ops)
			len +=3D sprintf(name+len, "%s ", =

__e->elevator_name);

*set);

=20
void blk_mq_flush_plug_list(struct blk_plug *plug, bool =

from_schedule);

*);

+	void (*exit_sched)(struct elevator_queue *);
+
+	bool (*allow_merge)(struct request_queue *, struct request *, =

struct bio *);

+	bool (*bio_merge)(struct blk_mq_hw_ctx *, struct bio *);
+	int (*request_merge)(struct request_queue *q, struct request **, =

struct bio *);

+	void (*request_merged)(struct request_queue *, struct request *, =

int);

+	void (*requests_merged)(struct request_queue *, struct request =

*, struct request *);

+	struct request *(*get_request)(struct request_queue *, unsigned =

int, struct blk_mq_alloc_data *);

+	bool (*put_request)(struct request *);
+	void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head =

*, bool);

+	void (*dispatch_requests)(struct blk_mq_hw_ctx *, struct =

list_head *);

+	bool (*has_work)(struct blk_mq_hw_ctx *);
+	void (*completed_request)(struct blk_mq_hw_ctx *, struct request =

*);

+	void (*started_request)(struct request *);
+	void (*requeue_request)(struct request *);
+	struct request *(*former_request)(struct request_queue *, struct =

request *);

+	struct request *(*next_request)(struct request_queue *, struct =

request *);

request_queue *, struct request *,

extern void elv_merged_request(struct request_queue *, struct request =

*, int);

extern void elv_bio_merged(struct request_queue *q, struct request *,
				struct bio *);
+extern bool elv_attempt_insert_merge(struct request_queue *, struct =

request *);

extern void elv_requeue_request(struct request_queue *, struct request =

*);

extern struct request *elv_former_request(struct request_queue *, =

struct request *);

extern struct request *elv_latter_request(struct request_queue *, =

struct request *);

--=20
2.7.4
=20