Re: [v6, 1/2] cxl: Add mechanism for delivering AFU driver specific events

On Jun 16, 2016, at 9:13 AM, Philippe Bergheaud =

[off-list ref] wrote:

=20
This adds an afu_driver_ops structure with fetch_event() and
event_delivered() callbacks. An AFU driver such as cxlflash can fill
this out and associate it with a context to enable passing custom
AFU specific events to userspace.
=20
This also adds a new kernel API function cxl_context_pending_events(),
that the AFU driver can use to notify the cxl driver that new specific
events are ready to be delivered, and wake up anyone waiting on the
context wait queue.
=20
The current count of AFU driver specific events is stored in the field
afu_driver_events of the context structure.
=20
The cxl driver checks the afu_driver_events count during poll, select,
read, etc. calls to check if an AFU driver specific event is pending,
and calls fetch_event() to obtain and deliver that event. This way,
the cxl driver takes care of all the usual locking semantics around =

these

calls and handles all the generic cxl events, so that the AFU driver =

only

needs to worry about it's own events.
=20
fetch_event() return a struct cxl_event_afu_driver_reserved, allocated
by the AFU driver, and filled in with the specific event information =

and

size. Data size should not be greater than CXL_MAX_EVENT_DATA_SIZE.
=20
Th cxl driver prepends an appropriate cxl event header, copies the =

event

to userspace, and finally calls event_delivered() to return the status =

of

the operation to the AFU driver. The event is identified by the =

context

and cxl_event_afu_driver_reserved pointers.
=20
Since AFU drivers provide their own means for userspace to obtain the
AFU file descriptor (i.e. cxlflash uses an ioctl on their scsi file
descriptor to obtain the AFU file descriptor) and the generic cxl =

driver

will never use this event, the ABI of the event is up to each =

individual

AFU driver.
=20
Signed-off-by: Philippe Bergheaud <redacted>
---
Changes since v5:
- s/deliver_event/fetch_event/
- Fixed the handling of fetch_event errors
- Documented the return codes of event_delivered
=20
Changes since v4:
- Addressed comments from Vaibhav:
 - Changed struct cxl_event_afu_driver_reserved from
   { __u64 reserved[4]; } to { size_t data_size; u8 data[]; }
 - Modified deliver_event to return a struct =

cxl_event_afu_driver_reserved

 - Added new callback event_delivered
 - Added static function afu_driver_event_copy
=20
Changes since v3:
- Removed driver ops callback ctx_event_pending
- Created cxl function cxl_context_pending_events
- Created cxl function cxl_unset_driver_ops
- Added atomic event counter afu_driver_events
=20
Changes since v2:
- Fixed some typos spotted by Matt Ochs
=20
Changes since v1:
- Rebased on upstream
- Bumped cxl api version to 3
- Addressed comments from mpe:
 - Clarified commit message & some comments
 - Mentioned 'cxlflash' as a possible user of this event
 - Check driver ops on registration and warn if missing calls
 - Remove redundant checks where driver ops is used
 - Simplified ctx_event_pending and removed underscore version
 - Changed deliver_event to take the context as the first argument
=20
drivers/misc/cxl/Kconfig |  5 +++++
drivers/misc/cxl/api.c   | 27 ++++++++++++++++++++++
drivers/misc/cxl/cxl.h   |  7 +++++-
drivers/misc/cxl/file.c  | 58 =

+++++++++++++++++++++++++++++++++++++++++-------

include/misc/cxl.h       | 53 =

+++++++++++++++++++++++++++++++++++++++++++

file *file)

vm_area_struct *vm)

	return cxl_context_iomap(ctx, vm);
}
=20
+static inline bool ctx_event_pending(struct cxl_context *ctx)
+{
+	if (ctx->pending_irq || ctx->pending_fault || =

ctx->pending_afu_err)

+		return true;
+
+	if (ctx->afu_driver_ops && atomic_read(&ctx->afu_driver_events))
+		return true;
+
+	return false;
+}
+
unsigned int afu_poll(struct file *file, struct poll_table_struct =

*poll)

poll_table_struct *poll)

	pr_devel("afu_poll wait done pe: %i\n", ctx->pe);
=20
	spin_lock_irqsave(&ctx->lock, flags);
-	if (ctx->pending_irq || ctx->pending_fault ||
-	    ctx->pending_afu_err)
+	if (ctx_event_pending(ctx))
		mask |=3D POLLIN | POLLRDNORM;
	else if (ctx->status =3D=3D CLOSED)
		/* Only error on closed when there are no futher events =

pending

poll_table_struct *poll)

	return mask;
}
=20
-static inline int ctx_event_pending(struct cxl_context *ctx)
+static ssize_t afu_driver_event_copy(struct cxl_context *ctx,
+				     char __user *buf,
+				     struct cxl_event *event,
+				     struct =

cxl_event_afu_driver_reserved *pl)

*buf, size_t count,

=20
	for (;;) {
		prepare_to_wait(&ctx->wq, &wait, TASK_INTERRUPTIBLE);
-		if (ctx_event_pending(ctx))
+		if (ctx_event_pending(ctx) || (ctx->status =3D=3D =

CLOSED))

*buf, size_t count,

	memset(&event, 0, sizeof(event));
	event.header.process_element =3D ctx->pe;
	event.header.size =3D sizeof(struct cxl_event_header);
-	if (ctx->pending_irq) {
+	if (ctx->afu_driver_ops && atomic_read(&ctx->afu_driver_events)) =

{

+		pr_devel("afu_read delivering AFU driver specific =

event\n");

+		pl =3D ctx->afu_driver_ops->fetch_event(ctx);
+		atomic_dec(&ctx->afu_driver_events);
+		event.header.type =3D CXL_EVENT_AFU_DRIVER;
+	} else if (ctx->pending_irq) {
		pr_devel("afu_read delivering AFU interrupt\n");
		event.header.size +=3D sizeof(struct =

cxl_event_afu_interrupt);

*buf, size_t count,

some

*afu,

 */
ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t =

count);

=20
+/*
+ * AFU driver ops allow an AFU driver to create their own events to =

pass to

+ * userspace through the file descriptor as a simpler alternative to =

overriding

+ * the read() and poll() calls that works with the generic cxl =

events. These

+ * events are given priority over the generic cxl events, so they =

will be

+ * delivered first if multiple types of events are pending.
+ *
+ * The AFU driver must call cxl_context_events_pending() to notify =

the cxl

+ * driver that new events are ready to be delivered for a specific =

context.

+ * cxl_context_events_pending() will adjust the current count of AFU =

driver

+ * events for this context, and wake up anyone waiting on the context =

wait

+ * queue.
+ *
+ * The cxl driver will then call fetch_event() to get a structure =

defining

+ * the size and address of the driver specific event data. Data size =

cannot

+ * be greater than CXL_MAX_EVENT_DATA_SIZE. The cxl driver will build =

a cxl

+ * header with type and process_element fields filled in, and =

header.size

+ * set to sizeof(struct cxl_event_header) + data_size.
+ *
+ * fetch_event() is called with a spin lock held, so it must not =

sleep.

+ *
+ * The cxl driver will then deliver the event to userspace, and =

finally

+ * call event_delivered() to return the status of the operation, =

identified

+ * by its context and AFU driver event data pointers:
+ *   0        Success
+ *   -EFAULT  copy_to_user() has failed
+ *   -EINVAL  Event data pointer is NULL, or
+ *            event data size is greater than CXL_MAX_EVENT_DATA_SIZE
+ */
+struct cxl_afu_driver_ops {
+	struct cxl_event_afu_driver_reserved *(*fetch_event) (
+						struct cxl_context =

*ctx);

+	void (*event_delivered) (struct cxl_context *ctx,
+				 struct cxl_event_afu_driver_reserved =

*event,

+				 int rc);
+};
+
+/*
+ * Associate the above driver ops with a specific context.
+ * Reset the current count of AFU driver events.
+ */
+void cxl_set_driver_ops(struct cxl_context *ctx,
+			struct cxl_afu_driver_ops *ops);
+/*
+ * Remove the driver ops from a specific context.
+ * The current count of AFU driver events must be 0.
+ */
+int cxl_unset_driver_ops(struct cxl_context *ctx);
+
+/* Notify cxl driver that new events are ready to be delivered for =

context */

Any reason this can't be larger if the user buffer is 4K?

Having a cap is good, and our current plans for exploiting this will
not use anywhere near this limit, however I feel that since we've
moved to a dynamic model, the size limit should be a bit more
accommodating - especially since changing this later would require
a version bump.

How about 2K?

+
+struct cxl_event_afu_driver_reserved {
+	/*
+	 * Defines the buffer passed to the cxl driver by the AFU =

driver.

+	 *
+	 * This is not ABI since the event header.size passed to the =

user for

+	 * existing events is set in the read call to =

sizeof(cxl_event_header)

+	 * + sizeof(whatever event is being dispatched) and the user is =

already

+	 * required to use a 4K buffer on the read call.
+	 *
+	 * Of course the contents will be ABI, but that's up the AFU =

driver.

+	 *
+	 * data_size is limited to MAX_EVENT_DATA_SIZE to prevent abuse.
+	 */
+	size_t data_size;
+	u8 data[];
+};
+
struct cxl_event {
	struct cxl_event_header header;
	union {
		struct cxl_event_afu_interrupt irq;
		struct cxl_event_data_storage fault;
		struct cxl_event_afu_error afu_error;
+		struct cxl_event_afu_driver_reserved afu_driver_event;
	};
};
=20
--=20
2.1.0
=20