diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst
index dbe462040999..a91f6a152dff 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -349,6 +349,8 @@ Code Seq# Include File Comments
<mailto:luzmaximilian@gmail.com>
0xA6 00-0F uapi/linux/alloc_tag.h Memory allocation profiling
<mailto:surenb@google.com>
+0xA7 00-0F uapi/linux/bao.h Bao hypervisor
+ <mailto:info@bao-project.org>
0xAA 00-3F linux/uapi/linux/userfaultfd.h
0xAB 00-1F linux/nbd.h
0xAC 00-1F linux/raw.hdiff --git a/drivers/virt/bao/Kconfig b/drivers/virt/bao/Kconfig
index 4f7929d57475..ab08a20db8c4 100644
--- a/drivers/virt/bao/Kconfig
+++ b/drivers/virt/bao/Kconfig
@@ -1,3 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
source "drivers/virt/bao/ipcshmem/Kconfig"
+
+source "drivers/virt/bao/io-dispatcher/Kconfig"
diff --git a/drivers/virt/bao/Makefile b/drivers/virt/bao/Makefile
index 68f5d3f282c4..c463f04cf206 100644
--- a/drivers/virt/bao/Makefile
+++ b/drivers/virt/bao/Makefile
@@ -1,3 +1,4 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_BAO_SHMEM) += ipcshmem/
+obj-$(CONFIG_BAO_IO_DISPATCHER) += io-dispatcher/
diff --git a/drivers/virt/bao/bao_hypercall.h b/drivers/virt/bao/bao_hypercall.h
index 7a68c29feb48..18d7102a54fc 100644
--- a/drivers/virt/bao/bao_hypercall.h
+++ b/drivers/virt/bao/bao_hypercall.h
@@ -24,6 +24,33 @@
/* IPC through shared-memory hypercall ID */
#define BAO_IPCSHMEM_HYPERCALL_ID 0x1
+/* Remote I/O hypercall ID */
+#define BAO_REMIO_HYPERCALL_ID 0x2
+
+/**
+ * struct bao_remio_hypercall_ctx - Remote I/O hypercall context
+ * @dm_id: Device model identifier
+ * @addr: Target address
+ * @op: Operation code
+ * @value: Value to read/write
+ * @access_width: Access width in bytes
+ * @request_id: Request identifier
+ * @npend_req: Number of pending requests
+ *
+ * @dm_id, @addr, @op, @value and @request_id are passed to the hypervisor;
+ * @addr, @op, @value, @access_width, @request_id and @npend_req are updated
+ * with the values it returns.
+ */
+struct bao_remio_hypercall_ctx {
+ u64 dm_id;
+ u64 addr;
+ u64 op;
+ u64 value;
+ u64 access_width;
+ u64 request_id;
+ u64 npend_req;
+};
+
#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
#include <linux/arm-smccc.h>@@ -55,6 +82,44 @@ static inline unsigned long bao_ipcshmem_hypercall(unsigned long ipcshmem_id)
return res.a0;
}
+#ifdef CONFIG_ARM64
+/**
+ * bao_remio_hypercall - Issue a Remote I/O hypercall
+ * @ctx: Hypercall context, updated with the values returned by the hypervisor
+ *
+ * The hypervisor returns the request in x1-x6 on top of the status in x0,
+ * which is only expressible with an SMCCC v1.2 call.
+ *
+ * Return: The hypervisor status code, 0 on success.
+ */
+static inline unsigned long
+bao_remio_hypercall(struct bao_remio_hypercall_ctx *ctx)
+{
+ struct arm_smccc_1_2_regs args = {
+ .a0 = BAO_HYPERCALL_FID(BAO_REMIO_HYPERCALL_ID),
+ .a1 = ctx->dm_id,
+ .a2 = ctx->addr,
+ .a3 = ctx->op,
+ .a4 = ctx->value,
+ .a5 = ctx->request_id,
+ };
+ struct arm_smccc_1_2_regs res;
+
+ arm_smccc_1_2_hvc(&args, &res);
+ if (res.a0)
+ return res.a0;
+
+ ctx->addr = res.a1;
+ ctx->op = res.a2;
+ ctx->value = res.a3;
+ ctx->access_width = res.a4;
+ ctx->request_id = res.a5;
+ ctx->npend_req = res.a6;
+
+ return 0;
+}
+#endif /* CONFIG_ARM64 */
+
#elif defined(CONFIG_RISCV)
#include <asm/sbi.h>@@ -92,6 +157,49 @@ static inline unsigned long bao_ipcshmem_hypercall(unsigned long ipcshmem_id)
return ret.value;
}
+/**
+ * bao_remio_hypercall - Issue a Remote I/O hypercall
+ * @ctx: Hypercall context, updated with the values returned by the hypervisor
+ *
+ * The hypervisor returns the request in a2-a7 on top of the SBI error in a0
+ * and the hypercall status in a1, which sbi_ecall() cannot express as it only
+ * exposes a0 and a1.
+ *
+ * Return: The hypervisor status code, 0 on success, or the SBI error if the
+ * extension is not implemented.
+ */
+static inline unsigned long
+bao_remio_hypercall(struct bao_remio_hypercall_ctx *ctx)
+{
+ register unsigned long a0 asm("a0") = ctx->dm_id;
+ register unsigned long a1 asm("a1") = ctx->addr;
+ register unsigned long a2 asm("a2") = ctx->op;
+ register unsigned long a3 asm("a3") = ctx->value;
+ register unsigned long a4 asm("a4") = ctx->request_id;
+ register unsigned long a5 asm("a5") = 0;
+ register unsigned long a6 asm("a6") = BAO_REMIO_HYPERCALL_ID;
+ register unsigned long a7 asm("a7") = BAO_SBI_EXT_ID;
+
+ asm volatile("ecall"
+ : "+r"(a0), "+r"(a1), "+r"(a2), "+r"(a3), "+r"(a4),
+ "+r"(a5), "+r"(a6), "+r"(a7)
+ :
+ : "memory");
+ if (a0)
+ return a0;
+ if (a1)
+ return a1;
+
+ ctx->addr = a2;
+ ctx->op = a3;
+ ctx->value = a4;
+ ctx->access_width = a5;
+ ctx->request_id = a6;
+ ctx->npend_req = a7;
+
+ return 0;
+}
+
#endif
#endif /* __BAO_HYPERCALL_H */diff --git a/drivers/virt/bao/io-dispatcher/Kconfig b/drivers/virt/bao/io-dispatcher/Kconfig
new file mode 100644
index 000000000000..a321bfb029f8
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/Kconfig
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: GPL-2.0
+config BAO_IO_DISPATCHER
+ tristate "Bao Hypervisor I/O Dispatcher"
+ depends on ARM64 || RISCV_SBI
+ select EVENTFD
+ help
+ The Bao I/O Dispatcher is a kernel module for backend Linux VMs
+ running under the Bao hypervisor. It establishes the connection
+ between the Remote I/O system (Bao's mechanism for forwarding
+ I/O requests from frontend VMs to the backend VMs) and the
+ VirtIO backend device.
+
+ This provides a unified API to support various VirtIO backends,
+ allowing Bao guests to perform I/O through the hypervisor
+ transparently.
+
+ To compile this driver as a module, choose M here: the module
+ will be called bao_io_dispatcher.
+
+ If unsure, say N.
diff --git a/drivers/virt/bao/io-dispatcher/Makefile b/drivers/virt/bao/io-dispatcher/Makefile
new file mode 100644
index 000000000000..b5fb18577bca
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_BAO_IO_DISPATCHER) += bao_io_dispatcher.o
+bao_io_dispatcher-y := driver.o dm.o intc.o io_client.o io_dispatcher.o \
+ ioeventfd.o irqfd.o
diff --git a/drivers/virt/bao/io-dispatcher/bao_drv.h b/drivers/virt/bao/io-dispatcher/bao_drv.h
new file mode 100644
index 000000000000..3065493413b4
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/bao_drv.h
@@ -0,0 +1,401 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Provides some definitions for the Bao Hypervisor modules
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ */
+
+#ifndef __BAO_DRV_H
+#define __BAO_DRV_H
+
+#include <linux/bao.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/rwsem.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+#include <linux/xarray.h>
+#include "../bao_hypercall.h"
+
+/* Room for a "bao-" prefixed name and a full u32 DM id. */
+#define BAO_NAME_MAX_LEN 32
+
+/*
+ * Upper bound on I/O requests queued but not yet consumed by a client. Caps the
+ * memory a misbehaving frontend can pin by flooding the backend with requests.
+ */
+#define BAO_IO_CLIENT_MAX_REQUESTS 1024
+
+/*
+ * Upper bounds on the ioeventfds and irqfds a DM can register, so userspace
+ * cannot grow the lists without limit (a device model needs a few per queue).
+ */
+#define BAO_IOEVENTFD_MAX 256
+#define BAO_IRQFD_MAX 256
+
+/* Bit in &bao_io_client.flags: the client is being torn down. */
+#define BAO_IO_CLIENT_DESTROYING 0U
+
+struct bao_dm;
+struct bao_io_client;
+
+typedef int (*bao_io_client_handler_t)(struct bao_io_client *client,
+ struct bao_virtio_request *req);
+typedef void (*bao_intc_handler_t)(struct bao_dm *dm);
+
+/*
+ * Operations of the Remote I/O hypercall that never reach userspace, on top of
+ * the BAO_IO_WRITE and BAO_IO_READ requests of the UAPI: fetch the next pending
+ * request, and notify the frontend of a completed backend operation.
+ */
+#define BAO_IO_ASK 2
+#define BAO_IO_NOTIFY 3
+
+/**
+ * struct bao_io_client - Bao I/O client
+ * @name: Client name
+ * @dm: The DM that the client belongs to
+ * @list: List node for this bao_io_client
+ * @is_control: If this client is the control client
+ * @flags: Flags (BAO_IO_CLIENT_*)
+ * @virtio_requests: List of pending I/O requests
+ * @nr_requests: Number of entries in @virtio_requests
+ * @virtio_requests_lock: Protects @virtio_requests and @nr_requests
+ * @range_list: I/O ranges
+ * @range_lock: Protects @range_list
+ * @handler: I/O request handler for this client
+ * @thread: Kernel thread executing the handler
+ * @wq: Wait queue used for thread parking
+ * @priv: Private data for the handler
+ */
+struct bao_io_client {
+ char name[BAO_NAME_MAX_LEN];
+ struct bao_dm *dm;
+ struct list_head list;
+ bool is_control;
+ unsigned long flags;
+ struct list_head virtio_requests;
+ unsigned int nr_requests;
+ /* protects virtio_requests and nr_requests */
+ struct mutex virtio_requests_lock;
+ struct list_head range_list;
+ /* protects range_list */
+ struct rw_semaphore range_lock;
+ bao_io_client_handler_t handler;
+ struct task_struct *thread;
+ wait_queue_head_t wq;
+ void *priv;
+};
+
+/**
+ * struct bao_dm - Bao backend device model (DM)
+ * @info: DM information (id, shmem_addr, shmem_size, irq)
+ * @name: Backing storage for the control client name
+ * @ioeventfds: List of all ioeventfds
+ * @nr_ioeventfds: Number of entries in @ioeventfds
+ * @ioeventfds_lock: Protects @ioeventfds and @nr_ioeventfds
+ * @ioeventfd_client: Ioeventfd client
+ * @irqfds: List of all irqfds
+ * @nr_irqfds: Number of entries in @irqfds
+ * @irqfds_lock: Protects @irqfds and @nr_irqfds
+ * @irqfd_server: Workqueue responsible for irqfd handling
+ * @io_clients_lock: Protects @io_clients
+ * @io_clients: List of all bao_io_client
+ * @control_client: Control client
+ * @inflight: Requests handed to userspace and not completed yet, indexed by
+ * request id
+ * @io_wq: Workqueue dispatching the I/O requests of this DM
+ * @io_work: Work item queued on @io_wq by the notification interrupt
+ * @intc_handler: Per-DM interrupt controller dispatch callback
+ * @intc_name: Backing storage for the request_irq() name
+ * @virq: Linux interrupt number of the hypervisor notification line
+ */
+struct bao_dm {
+ struct bao_dm_info info;
+ char name[BAO_NAME_MAX_LEN];
+
+ struct list_head ioeventfds;
+ unsigned int nr_ioeventfds;
+ /* protects ioeventfds and nr_ioeventfds */
+ struct mutex ioeventfds_lock;
+ struct bao_io_client *ioeventfd_client;
+
+ struct list_head irqfds;
+ unsigned int nr_irqfds;
+ /* protects irqfds and nr_irqfds */
+ struct mutex irqfds_lock;
+ struct workqueue_struct *irqfd_server;
+
+ /* protects io_clients */
+ struct rw_semaphore io_clients_lock;
+ struct list_head io_clients;
+ struct bao_io_client *control_client;
+ struct xarray inflight;
+
+ struct workqueue_struct *io_wq;
+ struct work_struct io_work;
+
+ bao_intc_handler_t intc_handler;
+ char intc_name[BAO_NAME_MAX_LEN];
+ unsigned int virq;
+};
+
+/**
+ * struct bao_io_range - Represents a range of I/O addresses
+ * @list: List node for linking multiple ranges
+ * @start: Start address of the range
+ * @end: End address of the range (inclusive)
+ */
+struct bao_io_range {
+ struct list_head list;
+ u64 start;
+ u64 end;
+};
+
+/**
+ * bao_dm_create - Create a backend device model (DM)
+ * @info: DM information (id, shmem_addr, shmem_size, irq)
+ *
+ * Return: Pointer to the created DM on success, ERR_PTR() on error.
+ */
+struct bao_dm *bao_dm_create(struct bao_dm_info *info);
+
+/**
+ * bao_dm_create_fd - Create a DM and bind it to a new file descriptor
+ * @info: DM information (id, shmem_addr, shmem_size, irq)
+ *
+ * Instantiates a DM, registers its interrupt and installs an anonymous inode so
+ * the DM lives exactly as long as the returned descriptor is open. Invoked from
+ * the /dev/bao control device on BAO_IOCTL_CREATE_DM.
+ *
+ * Return: A new file descriptor on success, negative error code on failure.
+ */
+int bao_dm_create_fd(struct bao_dm_info *info);
+
+/**
+ * bao_dm_destroy - Destroy a backend device model (DM)
+ * @dm: DM to be destroyed
+ */
+void bao_dm_destroy(struct bao_dm *dm);
+
+/**
+ * bao_io_client_create - Create a backend I/O client
+ * @dm: DM this client belongs to
+ * @handler: I/O client handler for requests
+ * @data: Private data passed to the handler
+ * @is_control: True if this is the control client
+ * @name: Name of the I/O client
+ *
+ * Return: Pointer to the created I/O client, NULL on failure.
+ */
+struct bao_io_client *bao_io_client_create(struct bao_dm *dm,
+ bao_io_client_handler_t handler,
+ void *data, bool is_control,
+ const char *name);
+
+/**
+ * bao_io_clients_destroy - Destroy all I/O clients of a DM
+ * @dm: DM whose I/O clients are to be destroyed
+ */
+void bao_io_clients_destroy(struct bao_dm *dm);
+
+/**
+ * bao_io_client_attach - Wait until an I/O client has requests to process
+ * @client: I/O client to attach
+ *
+ * Sleeps until a request is queued on @client, the client is being destroyed
+ * or, for a kernel client, its thread is asked to stop.
+ *
+ * Return: 0 when requests are pending, -ERESTARTSYS if interrupted by a
+ * signal, -EPERM if the client is going away.
+ */
+int bao_io_client_attach(struct bao_io_client *client);
+
+/**
+ * bao_io_client_range_add - Add an I/O range to monitor in a client
+ * @client: I/O client
+ * @start: Start address of the range
+ * @end: End address of the range (inclusive)
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int bao_io_client_range_add(struct bao_io_client *client, u64 start, u64 end);
+
+/**
+ * bao_io_client_range_del - Remove an I/O range from a client
+ * @client: I/O client
+ * @start: Start address of the range
+ * @end: End address of the range (inclusive)
+ */
+void bao_io_client_range_del(struct bao_io_client *client, u64 start, u64 end);
+
+/**
+ * bao_io_client_request - Retrieve the oldest I/O request from a client
+ * @client: I/O client
+ * @req: Pointer to virtio request structure to fill
+ *
+ * Return: 0 on success, -EAGAIN if no request is available.
+ */
+int bao_io_client_request(struct bao_io_client *client,
+ struct bao_virtio_request *req);
+
+/**
+ * bao_io_client_push_request - Push an I/O request into a client
+ * @client: I/O client
+ * @req: I/O request to push
+ *
+ * Return: True if a request was pushed, false otherwise.
+ */
+bool bao_io_client_push_request(struct bao_io_client *client,
+ struct bao_virtio_request *req);
+
+/**
+ * bao_io_client_pop_request - Pop the oldest I/O request from a client
+ * @client: I/O client
+ * @req: Buffer to store the popped request
+ *
+ * Return: True if a request was popped, false if the list was empty.
+ */
+bool bao_io_client_pop_request(struct bao_io_client *client,
+ struct bao_virtio_request *req);
+
+/**
+ * bao_io_client_find - Find the I/O client for a given request
+ * @dm: DM that the I/O request belongs to
+ * @req: I/O request to locate
+ *
+ * Return: Pointer to the I/O client handling the request, NULL if none found.
+ */
+struct bao_io_client *bao_io_client_find(struct bao_dm *dm,
+ struct bao_virtio_request *req);
+
+/**
+ * bao_ioeventfd_client_init - Initialize the Ioeventfd client for a DM
+ * @dm: DM that the Ioeventfd client belongs to
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int bao_ioeventfd_client_init(struct bao_dm *dm);
+
+/**
+ * bao_ioeventfd_client_destroy - Destroy the Ioeventfd client for a DM
+ * @dm: DM that the Ioeventfd client belongs to
+ */
+void bao_ioeventfd_client_destroy(struct bao_dm *dm);
+
+/**
+ * bao_ioeventfd_client_config - Configure an Ioeventfd client
+ * @dm: DM that the Ioeventfd client belongs to
+ * @config: Ioeventfd configuration to apply
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int bao_ioeventfd_client_config(struct bao_dm *dm,
+ struct bao_ioeventfd *config);
+
+/**
+ * bao_irqfd_server_init - Initialize the Irqfd server for a DM
+ * @dm: DM that the Irqfd server belongs to
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int bao_irqfd_server_init(struct bao_dm *dm);
+
+/**
+ * bao_irqfd_server_destroy - Destroy the Irqfd server for a DM
+ * @dm: DM that the Irqfd server belongs to
+ */
+void bao_irqfd_server_destroy(struct bao_dm *dm);
+
+/**
+ * bao_irqfd_server_config - Configure an Irqfd server
+ * @dm: DM that the Irqfd server belongs to
+ * @config: Irqfd configuration to apply
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int bao_irqfd_server_config(struct bao_dm *dm, struct bao_irqfd *config);
+
+/**
+ * bao_io_dispatcher_init - Initialize the I/O Dispatcher for a DM
+ * @dm: DM to initialize on the I/O Dispatcher
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int bao_io_dispatcher_init(struct bao_dm *dm);
+
+/**
+ * bao_io_dispatcher_destroy - Destroy the I/O Dispatcher for a DM
+ * @dm: DM to destroy on the I/O Dispatcher
+ */
+void bao_io_dispatcher_destroy(struct bao_dm *dm);
+
+/**
+ * bao_dispatch_io - Acquire and dispatch one I/O request from the hypervisor
+ * @dm: DM whose I/O clients will handle the request
+ *
+ * A request that cannot be handed to a client is completed back to the
+ * hypervisor with an error so the frontend is resumed, and fetching goes on.
+ *
+ * Return: The number of requests still pending, or a negative error code when
+ * no request could be fetched.
+ */
+int bao_dispatch_io(struct bao_dm *dm);
+
+/**
+ * bao_io_request_complete_error - Complete a request the backend cannot serve
+ * @dm: DM owning the request
+ * @req: The request that could not be handed to userspace or a handler
+ *
+ * Completes @req back to the hypervisor with a zero value so the frontend
+ * making the I/O access is resumed instead of hanging forever.
+ */
+void bao_io_request_complete_error(struct bao_dm *dm,
+ struct bao_virtio_request *req);
+
+/**
+ * bao_io_dispatcher_pause - Pause the I/O Dispatcher for a DM
+ * @dm: DM to pause
+ */
+void bao_io_dispatcher_pause(struct bao_dm *dm);
+
+/**
+ * bao_io_dispatcher_resume - Resume the I/O Dispatcher for a DM
+ * @dm: DM to resume
+ */
+void bao_io_dispatcher_resume(struct bao_dm *dm);
+
+/**
+ * bao_intc_init - Register the interrupt controller for a DM
+ * @dm: DM that the interrupt controller belongs to
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int bao_intc_init(struct bao_dm *dm);
+
+/**
+ * bao_intc_destroy - Unregister the interrupt controller for a DM
+ * @dm: DM that the interrupt controller belongs to
+ */
+void bao_intc_destroy(struct bao_dm *dm);
+
+/**
+ * bao_intc_setup_handler - Setup the interrupt controller handler
+ * @dm: DM that the interrupt controller belongs to
+ * @handler: Function pointer to the interrupt handler
+ */
+void bao_intc_setup_handler(struct bao_dm *dm, bao_intc_handler_t handler);
+
+/**
+ * bao_intc_remove_handler - Remove the interrupt controller handler
+ * @dm: DM that the interrupt controller belongs to
+ */
+void bao_intc_remove_handler(struct bao_dm *dm);
+
+#endif /* __BAO_DRV_H */diff --git a/drivers/virt/bao/io-dispatcher/dm.c b/drivers/virt/bao/io-dispatcher/dm.c
new file mode 100644
index 000000000000..700e46974297
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/dm.c
@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Bao Hypervisor Backend Device Model (DM)
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/anon_inodes.h>
+#include <linux/capability.h>
+#include <linux/err.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/ioport.h>
+#include <linux/mm.h>
+#include <linux/security.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/uaccess.h>
+#include <linux/wordpart.h>
+#include <linux/xarray.h>
+#include "bao_drv.h"
+
+/*
+ * Ids of the DMs that have a file descriptor. A DM id names a device model of
+ * the hypervisor, which must be served by a single descriptor: two would both
+ * fetch and complete its requests.
+ */
+static DEFINE_XARRAY(bao_dm_ids);
+
+/*
+ * Requests handed to userspace by IO_CLIENT_ATTACH are tracked until
+ * userspace completes them, so that a request the VMM never completes (it
+ * exited or crashed) is still completed when the DM goes away, instead of
+ * leaving the frontend vCPU waiting for it forever.
+ */
+static int bao_dm_inflight_add(struct bao_dm *dm,
+ const struct bao_virtio_request *req)
+{
+ struct bao_virtio_request *copy;
+ int ret;
+
+ copy = kmemdup(req, sizeof(*req), GFP_KERNEL);
+ if (!copy)
+ return -ENOMEM;
+
+ ret = xa_insert(&dm->inflight, req->request_id, copy, GFP_KERNEL);
+ if (ret)
+ kfree(copy);
+
+ return ret;
+}
+
+static void bao_dm_inflight_complete_all(struct bao_dm *dm)
+{
+ struct bao_virtio_request *req;
+ unsigned long id;
+
+ xa_for_each(&dm->inflight, id, req) {
+ xa_erase(&dm->inflight, id);
+ bao_io_request_complete_error(dm, req);
+ kfree(req);
+ }
+ xa_destroy(&dm->inflight);
+}
+
+static int bao_dm_release(struct inode *inode, struct file *filp)
+{
+ struct bao_dm *dm = filp->private_data;
+ struct bao_dm_info info;
+
+ if (WARN_ON_ONCE(!dm))
+ return 0;
+
+ info = dm->info;
+ filp->private_data = NULL;
+ bao_intc_destroy(dm);
+ bao_dm_destroy(dm);
+ release_mem_region(info.shmem_addr, info.shmem_size);
+ xa_erase(&bao_dm_ids, info.id);
+
+ return 0;
+}
+
+static long bao_dm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+ struct bao_dm *dm = filp->private_data;
+ int rc;
+
+ if (WARN_ON_ONCE(!dm))
+ return -ENODEV;
+
+ switch (cmd) {
+ case BAO_IOCTL_DM_GET_INFO: {
+ struct bao_dm_info info;
+
+ /* Zero-fill so nothing uninitialized is leaked to userspace. */
+ memset(&info, 0, sizeof(info));
+ info.shmem_addr = dm->info.shmem_addr;
+ info.shmem_size = dm->info.shmem_size;
+ info.id = dm->info.id;
+ info.irq = dm->info.irq;
+
+ if (copy_to_user((void __user *)arg, &info, sizeof(info)))
+ return -EFAULT;
+
+ rc = 0;
+ break;
+ }
+ case BAO_IOCTL_IO_CLIENT_ATTACH: {
+ struct bao_virtio_request req;
+
+ if (!dm->control_client) {
+ rc = -ENOENT;
+ break;
+ }
+
+ rc = bao_io_client_attach(dm->control_client);
+ if (rc)
+ break;
+
+ rc = bao_io_client_request(dm->control_client, &req);
+ if (rc)
+ break;
+
+ rc = bao_dm_inflight_add(dm, &req);
+ if (rc) {
+ bao_io_request_complete_error(dm, &req);
+ break;
+ }
+
+ if (copy_to_user((void __user *)arg, &req, sizeof(req))) {
+ /*
+ * The request is already off the queue and userspace
+ * will never see it: complete it with an error so the
+ * frontend is resumed instead of hanging forever.
+ */
+ kfree(xa_erase(&dm->inflight, req.request_id));
+ bao_io_request_complete_error(dm, &req);
+ rc = -EFAULT;
+ }
+ break;
+ }
+ case BAO_IOCTL_IO_REQUEST_COMPLETE: {
+ struct bao_virtio_request req, *inflight;
+ struct bao_remio_hypercall_ctx ctx;
+
+ if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
+ return -EFAULT;
+
+ /*
+ * Only a request handed out by IO_CLIENT_ATTACH, a write or a
+ * read, can be completed. The other operations of the hypercall
+ * belong to the kernel: an ASK from here would fetch a request
+ * and lose it, leaving its frontend blocked.
+ */
+ if (req.op != BAO_IO_WRITE && req.op != BAO_IO_READ)
+ return -EINVAL;
+
+ inflight = xa_erase(&dm->inflight, req.request_id);
+ if (!inflight)
+ return -ENOENT;
+
+ /*
+ * Only the value comes from userspace. The DM is the one bound
+ * to this fd and the rest is the request as it was handed out,
+ * so a client can neither complete requests of another DM nor
+ * change the operation being completed.
+ */
+ ctx.dm_id = dm->info.id;
+ ctx.addr = inflight->addr;
+ ctx.op = inflight->op;
+ ctx.value = req.value;
+ ctx.access_width = inflight->access_width;
+ ctx.request_id = inflight->request_id;
+ kfree(inflight);
+
+ rc = bao_remio_hypercall(&ctx) ? -EIO : 0;
+ break;
+ }
+ case BAO_IOCTL_IOEVENTFD: {
+ struct bao_ioeventfd ioeventfd;
+
+ if (copy_from_user(&ioeventfd, (void __user *)arg,
+ sizeof(struct bao_ioeventfd)))
+ return -EFAULT;
+
+ rc = bao_ioeventfd_client_config(dm, &ioeventfd);
+ break;
+ }
+ case BAO_IOCTL_IRQFD: {
+ struct bao_irqfd irqfd;
+
+ if (copy_from_user(&irqfd, (void __user *)arg,
+ sizeof(struct bao_irqfd)))
+ return -EFAULT;
+
+ rc = bao_irqfd_server_config(dm, &irqfd);
+ break;
+ }
+ default:
+ rc = -ENOTTY;
+ break;
+ }
+
+ return rc;
+}
+
+/**
+ * bao_dm_mmap_prepare - Map the DM shared memory into userspace
+ * @desc: Description of the mapping being set up
+ *
+ * The region is mapped from offset 0, in whole or in part. Only shared
+ * mappings are allowed: what is written to a private copy of the region
+ * would never reach the frontend.
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+static int bao_dm_mmap_prepare(struct vm_area_desc *desc)
+{
+ struct bao_dm *dm = desc->file->private_data;
+
+ if (WARN_ON_ONCE(!dm))
+ return -ENODEV;
+
+ if (!vma_desc_test(desc, VMA_MAYSHARE_BIT))
+ return -EINVAL;
+
+ if (desc->pgoff || vma_desc_size(desc) > dm->info.shmem_size)
+ return -EINVAL;
+
+ mmap_action_remap_full(desc, PHYS_PFN(dm->info.shmem_addr));
+
+ return 0;
+}
+
+static loff_t bao_dm_llseek(struct file *file, loff_t offset, int whence)
+{
+ struct bao_dm *dm = file->private_data;
+
+ if (WARN_ON_ONCE(!dm))
+ return -ENODEV;
+
+ return fixed_size_llseek(file, offset, whence, dm->info.shmem_size);
+}
+
+static const struct file_operations bao_dm_fops = {
+ .owner = THIS_MODULE,
+ .release = bao_dm_release,
+ .unlocked_ioctl = bao_dm_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+ .llseek = bao_dm_llseek,
+ .mmap_prepare = bao_dm_mmap_prepare,
+};
+
+struct bao_dm *bao_dm_create(struct bao_dm_info *info)
+{
+ struct bao_dm *dm;
+ int ret;
+
+ if (WARN_ON(!info))
+ return ERR_PTR(-EINVAL);
+
+ dm = kzalloc_obj(*dm);
+ if (!dm)
+ return ERR_PTR(-ENOMEM);
+
+ INIT_LIST_HEAD(&dm->io_clients);
+ init_rwsem(&dm->io_clients_lock);
+ xa_init(&dm->inflight);
+
+ dm->info = *info;
+
+ ret = bao_io_dispatcher_init(dm);
+ if (ret) {
+ pr_err("failed to init I/O dispatcher for DM %u\n",
+ dm->info.id);
+ goto err_free;
+ }
+
+ snprintf(dm->name, sizeof(dm->name), "bao-ioctlc%u", dm->info.id);
+ dm->control_client = bao_io_client_create(dm, NULL, NULL, true,
+ dm->name);
+ if (!dm->control_client) {
+ pr_err("failed to create control client for DM %u\n",
+ dm->info.id);
+ ret = -ENOMEM;
+ goto err_destroy_dispatcher;
+ }
+
+ ret = bao_ioeventfd_client_init(dm);
+ if (ret) {
+ pr_err("failed to initialize ioeventfd for DM %u\n",
+ dm->info.id);
+ goto err_destroy_io_clients;
+ }
+
+ ret = bao_irqfd_server_init(dm);
+ if (ret) {
+ pr_err("failed to initialize irqfd for DM %u\n",
+ dm->info.id);
+ goto err_destroy_io_clients;
+ }
+
+ return dm;
+
+err_destroy_io_clients:
+ bao_io_clients_destroy(dm);
+err_destroy_dispatcher:
+ bao_io_dispatcher_destroy(dm);
+err_free:
+ kfree(dm);
+
+ return ERR_PTR(ret);
+}
+
+void bao_dm_destroy(struct bao_dm *dm)
+{
+ if (WARN_ON_ONCE(!dm))
+ return;
+
+ bao_dm_inflight_complete_all(dm);
+ bao_irqfd_server_destroy(dm);
+ bao_io_clients_destroy(dm);
+ bao_io_dispatcher_destroy(dm);
+
+ kfree(dm);
+}
+
+/**
+ * bao_dm_info_valid - Check the shared-memory region of a DM description
+ * @info: DM description supplied by userspace
+ *
+ * The region must be non-empty, page-aligned, must not wrap around and must be
+ * addressable. It is memory the hypervisor shares between the backend and its
+ * frontend, never part of this guest's RAM, so anything overlapping System RAM
+ * is refused: the descriptor is later mapped into userspace and must not be a
+ * way to reach kernel memory.
+ *
+ * Return: True if the description can be used.
+ */
+static bool bao_dm_info_valid(const struct bao_dm_info *info)
+{
+ u64 end;
+
+ if (!info->shmem_size || !PAGE_ALIGNED(info->shmem_addr) ||
+ !PAGE_ALIGNED(info->shmem_size))
+ return false;
+
+ end = info->shmem_addr + info->shmem_size - 1;
+ if (end < info->shmem_addr)
+ return false;
+
+ if (sizeof(phys_addr_t) < sizeof(u64) && upper_32_bits(end))
+ return false;
+
+ if (region_intersects(info->shmem_addr, info->shmem_size,
+ IORESOURCE_SYSTEM_RAM,
+ IORES_DESC_NONE) != REGION_DISJOINT)
+ return false;
+
+ return true;
+}
+
+int bao_dm_create_fd(struct bao_dm_info *info)
+{
+ struct bao_dm *dm;
+ int fd, ret;
+
+ if (WARN_ON(!info))
+ return -EINVAL;
+
+ /*
+ * Creating a DM maps a physical region chosen by userspace and claims
+ * an interrupt line: like /dev/mem, that takes CAP_SYS_RAWIO and is
+ * not available under lockdown.
+ */
+ if (!capable(CAP_SYS_RAWIO))
+ return -EPERM;
+
+ ret = security_locked_down(LOCKDOWN_DEV_MEM);
+ if (ret)
+ return ret;
+
+ if (!bao_dm_info_valid(info))
+ return -EINVAL;
+
+ /* Claim the id; it is released once the DM is fully torn down. */
+ ret = xa_insert(&bao_dm_ids, info->id, NULL, GFP_KERNEL);
+ if (ret)
+ return ret == -EBUSY ? -EEXIST : ret;
+
+ /*
+ * Claim the region as a driver claims its MMIO, so that it cannot be
+ * a range another driver owns nor the region of another DM.
+ */
+ if (!request_mem_region(info->shmem_addr, info->shmem_size, "bao-dm")) {
+ ret = -EBUSY;
+ goto err_release_id;
+ }
+
+ dm = bao_dm_create(info);
+ if (IS_ERR(dm)) {
+ ret = PTR_ERR(dm);
+ goto err_release_region;
+ }
+
+ ret = bao_intc_init(dm);
+ if (ret) {
+ pr_err("failed to register interrupt %u for DM %u: %d\n",
+ info->irq, info->id, ret);
+ goto err_destroy_dm;
+ }
+
+ fd = anon_inode_getfd("[bao-dm]", &bao_dm_fops, dm, O_RDWR | O_CLOEXEC);
+ if (fd < 0) {
+ ret = fd;
+ goto err_destroy_intc;
+ }
+
+ return fd;
+
+err_destroy_intc:
+ bao_intc_destroy(dm);
+err_destroy_dm:
+ bao_dm_destroy(dm);
+err_release_region:
+ release_mem_region(info->shmem_addr, info->shmem_size);
+err_release_id:
+ xa_erase(&bao_dm_ids, info->id);
+ return ret;
+}diff --git a/drivers/virt/bao/io-dispatcher/driver.c b/drivers/virt/bao/io-dispatcher/driver.c
new file mode 100644
index 000000000000..037787c8be6a
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/driver.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Bao Hypervisor I/O Dispatcher Kernel Driver
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ *
+ * The set of device models a backend guest serves is a pure software contract
+ * between the Bao hypervisor and its userspace VMM, so it is not described in
+ * the device tree. Following the model used by other hypervisor drivers (e.g.
+ * drivers/virt/acrn), the driver exposes a single /dev/bao control device and
+ * the VMM instantiates each device model from its own configuration through the
+ * BAO_IOCTL_CREATE_DM ioctl, which returns a per-DM file descriptor.
+ */
+
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include "bao_drv.h"
+
+static long bao_ctl_ioctl(struct file *filp, unsigned int cmd,
+ unsigned long arg)
+{
+ struct bao_dm_info info;
+
+ switch (cmd) {
+ case BAO_IOCTL_CREATE_DM:
+ if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
+ return -EFAULT;
+
+ return bao_dm_create_fd(&info);
+ default:
+ return -ENOTTY;
+ }
+}
+
+static const struct file_operations bao_ctl_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = bao_ctl_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+ .llseek = noop_llseek,
+};
+
+static struct miscdevice bao_ctl_dev = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "bao",
+ .fops = &bao_ctl_fops,
+};
+
+static int __init bao_io_dispatcher_driver_init(void)
+{
+ return misc_register(&bao_ctl_dev);
+}
+
+static void __exit bao_io_dispatcher_driver_exit(void)
+{
+ misc_deregister(&bao_ctl_dev);
+}
+
+module_init(bao_io_dispatcher_driver_init);
+module_exit(bao_io_dispatcher_driver_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("João Peixoto <jpeixoto@osyx.tech>");
+MODULE_AUTHOR("David Cerdeira <davidmcerdeira@osyx.tech>");
+MODULE_AUTHOR("José Martins <jose@osyx.tech>");
+MODULE_DESCRIPTION("Bao Hypervisor I/O Dispatcher Kernel Driver");diff --git a/drivers/virt/bao/io-dispatcher/intc.c b/drivers/virt/bao/io-dispatcher/intc.c
new file mode 100644
index 000000000000..27b17e35a38a
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/intc.c
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Bao Hypervisor I/O Dispatcher Interrupt Controller
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include "bao_drv.h"
+
+/**
+ * bao_interrupt_handler - Top-level interrupt handler for Bao DM
+ * @irq: Interrupt number
+ * @dev: Pointer to the Bao device model (struct bao_dm)
+ *
+ * Invokes the DM's registered interrupt controller handler, if any.
+ *
+ * Return: IRQ_HANDLED
+ */
+static irqreturn_t bao_interrupt_handler(int irq, void *dev)
+{
+ struct bao_dm *dm = dev;
+ bao_intc_handler_t handler = READ_ONCE(dm->intc_handler);
+
+ if (handler)
+ handler(dm);
+
+ return IRQ_HANDLED;
+}
+
+void bao_intc_setup_handler(struct bao_dm *dm, bao_intc_handler_t handler)
+{
+ if (WARN_ON_ONCE(!dm))
+ return;
+
+ WRITE_ONCE(dm->intc_handler, handler);
+}
+
+void bao_intc_remove_handler(struct bao_dm *dm)
+{
+ if (WARN_ON_ONCE(!dm))
+ return;
+
+ WRITE_ONCE(dm->intc_handler, NULL);
+}
+
+/**
+ * bao_intc_map_irq - Map the hypervisor notification line to a Linux IRQ
+ * @line: Line of the system's root interrupt controller (e.g. a GIC SPI on
+ * arm64, a PLIC source on riscv)
+ *
+ * The I/O dispatcher is not a device-tree device, so its notification
+ * interrupt cannot be resolved from an "interrupts" property. Resolve it
+ * against the root interrupt parent instead, i.e. the controller the device
+ * tree's root node points at, which is the one any device without an explicit
+ * interrupt-parent would use. Only the controller's "#interrupt-cells" is
+ * needed to build the specifier: a single-cell controller (e.g. a PLIC) takes
+ * the line directly, a two-cell one (e.g. an APLIC) the line and the trigger
+ * type, and a GIC-style three-cell one an edge-triggered SPI.
+ *
+ * The irqchip drivers trust the specifiers they are given, which normally come
+ * from the device tree, so the line is checked against the number of sources
+ * the controller declares (a PLIC or APLIC) or against the SPI range (a GIC)
+ * before it is mapped.
+ *
+ * Return: A Linux virtual IRQ number on success, negative error code on failure.
+ */
+static int bao_intc_map_irq(u32 line)
+{
+ struct of_phandle_args oirq = {};
+ struct device_node *parent;
+ unsigned int virq;
+ u32 cells, nr_sources;
+ int ret;
+
+ parent = of_irq_find_parent(of_root);
+ if (!parent) {
+ pr_err("no root interrupt-parent in the device tree\n");
+ return -ENODEV;
+ }
+
+ ret = of_property_read_u32(parent, "#interrupt-cells", &cells);
+ if (ret)
+ goto out_put;
+
+ oirq.np = parent;
+ oirq.args_count = cells;
+
+ switch (cells) {
+ case 1:
+ /* PLIC: sources 1 to riscv,ndev */
+ if (of_property_read_u32(parent, "riscv,ndev", &nr_sources) ||
+ !line || line > nr_sources)
+ goto out_bad_line;
+ oirq.args[0] = line;
+ break;
+ case 2:
+ /* APLIC: sources 1 to riscv,num-sources */
+ if (of_property_read_u32(parent, "riscv,num-sources",
+ &nr_sources) ||
+ !line || line > nr_sources)
+ goto out_bad_line;
+ oirq.args[0] = line;
+ oirq.args[1] = IRQ_TYPE_EDGE_RISING;
+ break;
+ case 3:
+ case 4:
+ /* GIC: SPIs 0 to 987, i.e. INTIDs 32 to 1019 */
+ if (line > 987)
+ goto out_bad_line;
+ oirq.args[0] = GIC_SPI;
+ oirq.args[1] = line;
+ oirq.args[2] = IRQ_TYPE_EDGE_RISING;
+ break;
+ default:
+ pr_err("unsupported #interrupt-cells = %u on %pOF\n",
+ cells, parent);
+ ret = -EINVAL;
+ goto out_put;
+ }
+
+ virq = irq_create_of_mapping(&oirq);
+ ret = virq ? (int)virq : -EINVAL;
+ goto out_put;
+
+out_bad_line:
+ pr_err("line %u is not a valid source of %pOF\n", line, parent);
+ ret = -EINVAL;
+out_put:
+ of_node_put(parent);
+ return ret;
+}
+
+int bao_intc_init(struct bao_dm *dm)
+{
+ int virq, ret;
+
+ if (WARN_ON_ONCE(!dm))
+ return -EINVAL;
+
+ virq = bao_intc_map_irq(dm->info.irq);
+ if (virq < 0)
+ return virq;
+
+ dm->virq = virq;
+
+ scnprintf(dm->intc_name, sizeof(dm->intc_name), "bao-iodintc%u",
+ dm->info.id);
+
+ /*
+ * The line is requested exclusively: one that is already in use (a
+ * device interrupt, another DM's line) is refused rather than shared.
+ *
+ * The mapping is never disposed, here or in bao_intc_destroy():
+ * irq_create_of_mapping() hands back the existing mapping when the line
+ * is already mapped and does not count its users, so disposing it could
+ * free a descriptor someone else still uses. A mapping left behind is
+ * reused the next time the same line is mapped.
+ */
+ ret = request_irq(dm->virq, bao_interrupt_handler, 0, dm->intc_name,
+ dm);
+ if (ret)
+ dm->virq = 0;
+
+ return ret;
+}
+
+void bao_intc_destroy(struct bao_dm *dm)
+{
+ if (WARN_ON_ONCE(!dm))
+ return;
+
+ free_irq(dm->virq, dm);
+}diff --git a/drivers/virt/bao/io-dispatcher/io_client.c b/drivers/virt/bao/io-dispatcher/io_client.c
new file mode 100644
index 000000000000..4ef3013f79c9
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/io_client.c
@@ -0,0 +1,430 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Bao Hypervisor I/O Client
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kthread.h>
+#include <linux/slab.h>
+#include "bao_drv.h"
+
+/**
+ * struct bao_io_request - Bao I/O request structure
+ * @list: List node linking all requests
+ * @virtio_request: The VirtIO request payload
+ *
+ * Represents a single I/O request for a Bao I/O client.
+ */
+struct bao_io_request {
+ struct list_head list;
+ struct bao_virtio_request virtio_request;
+};
+
+/**
+ * bao_io_client_has_pending_requests - Check if an I/O client has pending requests
+ * @client: The bao_io_client to check
+ *
+ * Return: True if has pending I/O requests, false otherwise.
+ */
+static inline bool
+bao_io_client_has_pending_requests(struct bao_io_client *client)
+{
+ if (WARN_ON_ONCE(!client))
+ return false;
+
+ return !list_empty(&client->virtio_requests);
+}
+
+/**
+ * bao_io_client_is_destroying - Check if an I/O client is being destroyed
+ * @client: The bao_io_client to check
+ *
+ * Return: True if the client is being destroyed, false otherwise.
+ */
+static inline bool bao_io_client_is_destroying(struct bao_io_client *client)
+{
+ if (WARN_ON_ONCE(!client))
+ return true;
+
+ return test_bit(BAO_IO_CLIENT_DESTROYING, &client->flags);
+}
+
+bool bao_io_client_push_request(struct bao_io_client *client,
+ struct bao_virtio_request *req)
+{
+ struct bao_io_request *io_req;
+
+ if (WARN_ON_ONCE(!client || !req))
+ return false;
+
+ io_req = kzalloc_obj(*io_req);
+ if (!io_req)
+ return false;
+
+ io_req->virtio_request = *req;
+
+ mutex_lock(&client->virtio_requests_lock);
+ if (client->nr_requests >= BAO_IO_CLIENT_MAX_REQUESTS) {
+ mutex_unlock(&client->virtio_requests_lock);
+ kfree(io_req);
+ return false;
+ }
+ list_add_tail(&io_req->list, &client->virtio_requests);
+ client->nr_requests++;
+ mutex_unlock(&client->virtio_requests_lock);
+
+ return true;
+}
+
+bool bao_io_client_pop_request(struct bao_io_client *client,
+ struct bao_virtio_request *ret)
+{
+ struct bao_io_request *req;
+
+ if (WARN_ON_ONCE(!client || !ret))
+ return false;
+
+ mutex_lock(&client->virtio_requests_lock);
+
+ req = list_first_entry_or_null(&client->virtio_requests,
+ struct bao_io_request, list);
+ if (!req) {
+ mutex_unlock(&client->virtio_requests_lock);
+ return false;
+ }
+
+ list_del(&req->list);
+ if (!WARN_ON_ONCE(client->nr_requests == 0))
+ client->nr_requests--;
+ *ret = req->virtio_request;
+
+ mutex_unlock(&client->virtio_requests_lock);
+
+ kfree(req);
+
+ return true;
+}
+
+/**
+ * bao_io_client_destroy - Destroy an I/O client
+ * @client: The bao_io_client to destroy
+ */
+static void bao_io_client_destroy(struct bao_io_client *client)
+{
+ struct bao_io_range *range;
+ struct bao_io_range *next;
+ struct bao_io_request *io_req;
+ struct bao_io_request *io_next;
+ struct bao_dm *dm;
+
+ if (WARN_ON_ONCE(!client))
+ return;
+
+ dm = client->dm;
+
+ bao_io_dispatcher_pause(dm);
+
+ set_bit(BAO_IO_CLIENT_DESTROYING, &client->flags);
+
+ if (client->is_control) {
+ wake_up_interruptible(&client->wq);
+ } else {
+ bao_ioeventfd_client_destroy(dm);
+ if (client->thread)
+ kthread_stop(client->thread);
+ }
+
+ down_write(&client->range_lock);
+ list_for_each_entry_safe(range, next, &client->range_list, list) {
+ list_del(&range->list);
+ kfree(range);
+ }
+ up_write(&client->range_lock);
+
+ down_write(&dm->io_clients_lock);
+ if (client->is_control)
+ dm->control_client = NULL;
+ else
+ dm->ioeventfd_client = NULL;
+
+ list_del(&client->list);
+ up_write(&dm->io_clients_lock);
+
+ bao_io_dispatcher_resume(dm);
+
+ /*
+ * Requests still queued will never be served: complete them back to
+ * the hypervisor with an error so the frontends waiting on them are
+ * resumed, then free them.
+ */
+ mutex_lock(&client->virtio_requests_lock);
+ list_for_each_entry_safe(io_req, io_next, &client->virtio_requests,
+ list) {
+ list_del(&io_req->list);
+ bao_io_request_complete_error(dm, &io_req->virtio_request);
+ kfree(io_req);
+ }
+ client->nr_requests = 0;
+ mutex_unlock(&client->virtio_requests_lock);
+
+ kfree(client);
+}
+
+void bao_io_clients_destroy(struct bao_dm *dm)
+{
+ struct bao_io_client *client, *next;
+
+ if (WARN_ON_ONCE(!dm))
+ return;
+
+ list_for_each_entry_safe(client, next, &dm->io_clients, list) {
+ bao_io_client_destroy(client);
+ }
+}
+
+int bao_io_client_attach(struct bao_io_client *client)
+{
+ int ret;
+
+ if (WARN_ON_ONCE(!client))
+ return -EINVAL;
+
+ if (client->is_control) {
+ ret = wait_event_interruptible(client->wq,
+ bao_io_client_has_pending_requests(client) ||
+ bao_io_client_is_destroying(client));
+ /* Let a caught signal restart the syscall instead of erroring. */
+ if (ret)
+ return ret;
+ if (bao_io_client_is_destroying(client))
+ return -EPERM;
+ } else {
+ /*
+ * The kernel thread only ever leaves through kthread_stop(),
+ * which wakes it up; teardown sets BAO_IO_CLIENT_DESTROYING
+ * and then stops the thread, so there is no need to wait on
+ * the flag here.
+ */
+ ret = wait_event_interruptible(client->wq,
+ bao_io_client_has_pending_requests(client) ||
+ kthread_should_stop());
+ if (ret)
+ return ret;
+ if (kthread_should_stop())
+ return -EPERM;
+ }
+
+ return 0;
+}
+
+/**
+ * bao_io_client_kernel_thread - Thread for processing a kernel I/O client
+ * @data: Pointer to the bao_io_client structure
+ *
+ * Runs the client handler on every queued request and completes the request
+ * back to the hypervisor. The thread never exits on its own: kthread_stop()
+ * relies on the task still being around, so errors are logged and the thread
+ * goes back to waiting for requests.
+ *
+ * Return: 0 on completion
+ */
+static int bao_io_client_kernel_thread(void *data)
+{
+ struct bao_io_client *client = data;
+ struct bao_virtio_request req;
+ struct bao_remio_hypercall_ctx ctx;
+ int ret;
+
+ if (WARN_ON_ONCE(!client))
+ return -EINVAL;
+
+ while (!kthread_should_stop()) {
+ if (bao_io_client_attach(client))
+ continue;
+
+ while (bao_io_client_pop_request(client, &req)) {
+ ret = client->handler(client, &req);
+ if (ret < 0) {
+ pr_warn_ratelimited("%s: handler returned %d\n",
+ client->name, ret);
+ bao_io_request_complete_error(client->dm,
+ &req);
+ continue;
+ }
+
+ ctx.dm_id = client->dm->info.id;
+ ctx.op = req.op;
+ ctx.addr = req.addr;
+ ctx.value = req.value;
+ ctx.access_width = req.access_width;
+ ctx.request_id = req.request_id;
+
+ if (bao_remio_hypercall(&ctx))
+ pr_warn_ratelimited("%s: failed to complete request %llu\n",
+ client->name,
+ req.request_id);
+ }
+ }
+
+ return 0;
+}
+
+struct bao_io_client *bao_io_client_create(struct bao_dm *dm,
+ bao_io_client_handler_t handler,
+ void *data, bool is_control,
+ const char *name)
+{
+ struct bao_io_client *client;
+
+ if (WARN_ON_ONCE(!dm || !name))
+ return NULL;
+
+ if (!handler && !is_control)
+ return NULL;
+
+ client = kzalloc_obj(*client);
+ if (!client)
+ return NULL;
+
+ client->handler = handler;
+ client->dm = dm;
+ client->priv = data;
+ client->is_control = is_control;
+ strscpy(client->name, name, sizeof(client->name));
+
+ INIT_LIST_HEAD(&client->virtio_requests);
+ mutex_init(&client->virtio_requests_lock);
+ init_rwsem(&client->range_lock);
+ INIT_LIST_HEAD(&client->range_list);
+ init_waitqueue_head(&client->wq);
+
+ if (client->handler) {
+ client->thread = kthread_run(bao_io_client_kernel_thread,
+ client, "%s", client->name);
+ if (IS_ERR(client->thread)) {
+ kfree(client);
+ return NULL;
+ }
+ }
+
+ down_write(&dm->io_clients_lock);
+ if (is_control)
+ dm->control_client = client;
+ else
+ dm->ioeventfd_client = client;
+
+ list_add(&client->list, &dm->io_clients);
+ up_write(&dm->io_clients_lock);
+
+ return client;
+}
+
+int bao_io_client_request(struct bao_io_client *client,
+ struct bao_virtio_request *req)
+{
+ if (WARN_ON_ONCE(!client))
+ return -EINVAL;
+
+ if (!bao_io_client_pop_request(client, req))
+ return -EAGAIN;
+
+ return 0;
+}
+
+int bao_io_client_range_add(struct bao_io_client *client, u64 start, u64 end)
+{
+ struct bao_io_range *range;
+
+ if (WARN_ON_ONCE(!client))
+ return -EINVAL;
+
+ if (end < start)
+ return -EINVAL;
+
+ range = kzalloc_obj(*range);
+ if (!range)
+ return -ENOMEM;
+
+ range->start = start;
+ range->end = end;
+
+ down_write(&client->range_lock);
+ list_add(&range->list, &client->range_list);
+ up_write(&client->range_lock);
+
+ return 0;
+}
+
+void bao_io_client_range_del(struct bao_io_client *client, u64 start, u64 end)
+{
+ struct bao_io_range *range;
+ struct bao_io_range *tmp;
+
+ if (WARN_ON_ONCE(!client))
+ return;
+
+ down_write(&client->range_lock);
+ list_for_each_entry_safe(range, tmp, &client->range_list, list) {
+ if (range->start == start && range->end == end) {
+ list_del(&range->list);
+ kfree(range);
+ break;
+ }
+ }
+ up_write(&client->range_lock);
+}
+
+/**
+ * bao_io_request_in_range - Check if the I/O request is in the range
+ * @range: The I/O request range
+ * @req: The I/O request to be checked
+ *
+ * Return: True if the I/O request is in the range, false otherwise
+ */
+static bool bao_io_request_in_range(struct bao_io_range *range,
+ struct bao_virtio_request *req)
+{
+ if (WARN_ON_ONCE(!range || !req))
+ return false;
+
+ if (req->addr >= range->start &&
+ (req->addr + req->access_width - 1) <= range->end)
+ return true;
+
+ return false;
+}
+
+struct bao_io_client *bao_io_client_find(struct bao_dm *dm,
+ struct bao_virtio_request *req)
+{
+ struct bao_io_client *client;
+ struct bao_io_client *found = NULL;
+ struct bao_io_range *range;
+
+ if (WARN_ON_ONCE(!dm || !req))
+ return NULL;
+
+ list_for_each_entry(client, &dm->io_clients, list) {
+ down_read(&client->range_lock);
+ list_for_each_entry(range, &client->range_list, list) {
+ if (bao_io_request_in_range(range, req)) {
+ found = client;
+ break;
+ }
+ }
+ up_read(&client->range_lock);
+
+ if (found)
+ break;
+ }
+
+ return found ? found : dm->control_client;
+}diff --git a/drivers/virt/bao/io-dispatcher/io_dispatcher.c b/drivers/virt/bao/io-dispatcher/io_dispatcher.c
new file mode 100644
index 000000000000..c098721f2d08
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/io_dispatcher.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Bao Hypervisor I/O Dispatcher
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/workqueue.h>
+#include "bao_drv.h"
+
+void bao_io_dispatcher_destroy(struct bao_dm *dm)
+{
+ if (WARN_ON_ONCE(!dm))
+ return;
+
+ if (!dm->io_wq)
+ return;
+
+ bao_io_dispatcher_pause(dm);
+
+ destroy_workqueue(dm->io_wq);
+ dm->io_wq = NULL;
+}
+
+void bao_io_request_complete_error(struct bao_dm *dm,
+ struct bao_virtio_request *req)
+{
+ struct bao_remio_hypercall_ctx ctx = {
+ .dm_id = dm->info.id,
+ .op = req->op,
+ .addr = req->addr,
+ .value = 0,
+ .access_width = req->access_width,
+ .request_id = req->request_id,
+ };
+
+ bao_remio_hypercall(&ctx);
+}
+
+int bao_dispatch_io(struct bao_dm *dm)
+{
+ struct bao_io_client *client;
+ struct bao_remio_hypercall_ctx ctx;
+ struct bao_virtio_request req;
+
+ if (WARN_ON_ONCE(!dm))
+ return -EINVAL;
+
+ ctx.dm_id = dm->info.id;
+ ctx.op = BAO_IO_ASK;
+ ctx.addr = 0;
+ ctx.value = 0;
+ ctx.request_id = 0;
+ ctx.access_width = 0;
+ ctx.npend_req = 0;
+
+ if (bao_remio_hypercall(&ctx))
+ return -EIO;
+
+ /* Only "more" or "no more" matters; keep the count in the int range. */
+ ctx.npend_req = min_t(u64, ctx.npend_req, INT_MAX);
+
+ req.dm_id = ctx.dm_id;
+ req.op = ctx.op;
+ req.addr = ctx.addr;
+ req.value = ctx.value;
+ req.access_width = ctx.access_width;
+ req.request_id = ctx.request_id;
+
+ /*
+ * A request that cannot be delivered is completed with an error and
+ * fetching goes on: the notification interrupt is edge-triggered, so
+ * stopping here would strand the requests still pending.
+ */
+ down_read(&dm->io_clients_lock);
+ client = bao_io_client_find(dm, &req);
+ if (!client) {
+ up_read(&dm->io_clients_lock);
+ pr_warn_ratelimited("DM %u: no client for request %llu at %#llx\n",
+ dm->info.id, req.request_id, req.addr);
+ bao_io_request_complete_error(dm, &req);
+ return ctx.npend_req;
+ }
+
+ if (!bao_io_client_push_request(client, &req)) {
+ up_read(&dm->io_clients_lock);
+ pr_warn_ratelimited("DM %u: dropping request %llu, client queue full\n",
+ dm->info.id, req.request_id);
+ bao_io_request_complete_error(dm, &req);
+ return ctx.npend_req;
+ }
+
+ wake_up_interruptible(&client->wq);
+ up_read(&dm->io_clients_lock);
+
+ return ctx.npend_req;
+}
+
+/**
+ * io_dispatcher - Workqueue handler for dispatching I/O
+ * @work: Work struct representing this dispatch operation
+ *
+ * Handles all pending I/O requests for the associated Bao DM.
+ * Executed in process context by the workqueue.
+ */
+static void io_dispatcher(struct work_struct *work)
+{
+ struct bao_dm *dm = container_of(work, struct bao_dm, io_work);
+
+ while (bao_dispatch_io(dm) > 0)
+ cond_resched();
+}
+
+/**
+ * io_dispatcher_intc_handler - Interrupt handler for I/O requests
+ * @dm: Bao device model that triggered the interrupt
+ *
+ * Invoked by the interrupt controller when a new I/O request is available.
+ * Queues the DM's work item onto its I/O dispatcher workqueue for processing
+ * in process context.
+ */
+static void io_dispatcher_intc_handler(struct bao_dm *dm)
+{
+ queue_work(dm->io_wq, &dm->io_work);
+}
+
+void bao_io_dispatcher_pause(struct bao_dm *dm)
+{
+ if (WARN_ON_ONCE(!dm || !dm->io_wq))
+ return;
+
+ bao_intc_remove_handler(dm);
+
+ drain_workqueue(dm->io_wq);
+}
+
+void bao_io_dispatcher_resume(struct bao_dm *dm)
+{
+ if (WARN_ON_ONCE(!dm || !dm->io_wq))
+ return;
+
+ bao_intc_setup_handler(dm, io_dispatcher_intc_handler);
+
+ queue_work(dm->io_wq, &dm->io_work);
+}
+
+int bao_io_dispatcher_init(struct bao_dm *dm)
+{
+ if (WARN_ON_ONCE(!dm))
+ return -EINVAL;
+
+ if (dm->io_wq)
+ return -EBUSY;
+
+ dm->io_wq = alloc_workqueue("bao-iodwq%u",
+ WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_PERCPU, 1,
+ dm->info.id);
+ if (!dm->io_wq)
+ return -ENOMEM;
+
+ INIT_WORK(&dm->io_work, io_dispatcher);
+
+ bao_intc_setup_handler(dm, io_dispatcher_intc_handler);
+
+ return 0;
+}diff --git a/drivers/virt/bao/io-dispatcher/ioeventfd.c b/drivers/virt/bao/io-dispatcher/ioeventfd.c
new file mode 100644
index 000000000000..dcf195669ebb
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/ioeventfd.c
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Bao Hypervisor Ioeventfd Client
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ */
+
+#include <linux/eventfd.h>
+#include <linux/slab.h>
+#include "bao_drv.h"
+
+/**
+ * struct ioeventfd - Properties of an I/O eventfd
+ * @list: List node linking this ioeventfd
+ * @eventfd: Associated eventfd context
+ * @addr: Start address of the I/O range
+ * @data: Data used for matching (if not wildcard)
+ * @length: Length of the I/O range
+ * @wildcard: True if data matching is not required
+ *
+ * Represents an I/O eventfd registered for a Bao device model.
+ */
+struct ioeventfd {
+ struct list_head list;
+ struct eventfd_ctx *eventfd;
+ u64 addr;
+ u64 data;
+ int length;
+ bool wildcard;
+};
+
+/**
+ * bao_ioeventfd_shutdown - Release and remove an ioeventfd
+ * @dm: Bao device model owning the ioeventfd
+ * @p: Ioeventfd to shut down
+ */
+static void bao_ioeventfd_shutdown(struct bao_dm *dm, struct ioeventfd *p)
+{
+ lockdep_assert_held(&dm->ioeventfds_lock);
+
+ if (WARN_ON_ONCE(!p))
+ return;
+
+ eventfd_ctx_put(p->eventfd);
+ list_del(&p->list);
+ dm->nr_ioeventfds--;
+ kfree(p);
+}
+
+/**
+ * bao_ioeventfd_config_valid - Validate ioeventfd configuration
+ * @config: Ioeventfd configuration
+ *
+ * Return: True if config is non-NULL, only known flags are set, the reserved
+ * field is zero, address+length does not wrap, and length is 1, 2, 4, or 8
+ * bytes.
+ */
+static bool bao_ioeventfd_config_valid(struct bao_ioeventfd *config)
+{
+ if (WARN_ON_ONCE(!config))
+ return false;
+
+ if (config->flags & ~(BAO_IOEVENTFD_FLAG_DATAMATCH |
+ BAO_IOEVENTFD_FLAG_DEASSIGN))
+ return false;
+
+ if (config->reserved)
+ return false;
+
+ if (config->addr + config->len < config->addr)
+ return false;
+
+ if (!(config->len == 1 || config->len == 2 || config->len == 4 ||
+ config->len == 8))
+ return false;
+
+ return true;
+}
+
+/**
+ * bao_ioeventfd_is_conflict - Check if an ioeventfd conflicts with existing ones
+ * @dm: Bao device model
+ * @ioeventfd: Ioeventfd to check
+ *
+ * Return: True if an existing ioeventfd matches address, eventfd,
+ * and optionally data.
+ */
+static bool bao_ioeventfd_is_conflict(struct bao_dm *dm,
+ struct ioeventfd *ioeventfd)
+{
+ struct ioeventfd *p;
+
+ lockdep_assert_held(&dm->ioeventfds_lock);
+
+ if (WARN_ON_ONCE(!dm || !ioeventfd))
+ return true;
+
+ list_for_each_entry(p, &dm->ioeventfds, list) {
+ if (p->eventfd == ioeventfd->eventfd &&
+ p->addr == ioeventfd->addr &&
+ (p->wildcard || ioeventfd->wildcard ||
+ p->data == ioeventfd->data)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/**
+ * bao_ioeventfd_match - Find ioeventfd matching an I/O request
+ * @dm: Bao device model
+ * @addr: I/O request address
+ * @data: I/O request data
+ * @len: I/O request length
+ *
+ * Return: The matching ioeventfd, NULL if none matches.
+ */
+static struct ioeventfd *bao_ioeventfd_match(struct bao_dm *dm, u64 addr,
+ u64 data, int len)
+{
+ struct ioeventfd *p;
+
+ lockdep_assert_held(&dm->ioeventfds_lock);
+
+ if (WARN_ON_ONCE(!dm))
+ return NULL;
+
+ list_for_each_entry(p, &dm->ioeventfds, list) {
+ if (p->addr == addr && p->length >= len &&
+ (p->wildcard || p->data == data)) {
+ return p;
+ }
+ }
+
+ return NULL;
+}
+
+/**
+ * bao_ioeventfd_assign - Assign and create an eventfd for a DM
+ * @dm: Bao device model to assign the eventfd to
+ * @config: Configuration of the eventfd to create
+ *
+ * Creates a new ioeventfd associated with the given eventfd and
+ * adds it to the Bao DM. Validates the configuration, checks for
+ * conflicts with existing ioeventfds, and registers the corresponding
+ * I/O client address range. Supports optional data matching for
+ * virtio 1.0 notifications; if not set, wildcard matching is used.
+ *
+ * Return: 0 on success, a negative error code on failure
+ */
+static int bao_ioeventfd_assign(struct bao_dm *dm, struct bao_ioeventfd *config)
+{
+ struct eventfd_ctx *eventfd;
+ struct ioeventfd *new;
+ int rc = 0;
+
+ if (WARN_ON_ONCE(!dm || !config))
+ return -EINVAL;
+
+ eventfd = eventfd_ctx_fdget(config->fd);
+ if (IS_ERR(eventfd))
+ return PTR_ERR(eventfd);
+
+ new = kzalloc_obj(*new);
+ if (!new) {
+ rc = -ENOMEM;
+ goto err_put_eventfd;
+ }
+
+ INIT_LIST_HEAD(&new->list);
+ new->addr = config->addr;
+ new->length = config->len;
+ new->eventfd = eventfd;
+ new->wildcard = !(config->flags & BAO_IOEVENTFD_FLAG_DATAMATCH);
+ if (!new->wildcard)
+ new->data = config->data;
+
+ mutex_lock(&dm->ioeventfds_lock);
+
+ if (dm->nr_ioeventfds >= BAO_IOEVENTFD_MAX) {
+ rc = -ENOSPC;
+ goto err_unlock_free;
+ }
+
+ if (bao_ioeventfd_is_conflict(dm, new)) {
+ rc = -EEXIST;
+ goto err_unlock_free;
+ }
+
+ rc = bao_io_client_range_add(dm->ioeventfd_client, new->addr,
+ new->addr + new->length - 1);
+ if (rc < 0)
+ goto err_unlock_free;
+
+ list_add_tail(&new->list, &dm->ioeventfds);
+ dm->nr_ioeventfds++;
+ mutex_unlock(&dm->ioeventfds_lock);
+
+ return 0;
+
+err_unlock_free:
+ mutex_unlock(&dm->ioeventfds_lock);
+ kfree(new);
+err_put_eventfd:
+ eventfd_ctx_put(eventfd);
+ return rc;
+}
+
+/**
+ * bao_ioeventfd_deassign - Deassign and destroy an eventfd from a DM
+ * @dm: Bao device model to deassign the eventfd from
+ * @config: Configuration of the eventfd to remove
+ *
+ * Return: 0 on success, -ENOENT if no such ioeventfd is registered, another
+ * negative error code on failure
+ */
+static int bao_ioeventfd_deassign(struct bao_dm *dm,
+ struct bao_ioeventfd *config)
+{
+ bool wildcard = !(config->flags & BAO_IOEVENTFD_FLAG_DATAMATCH);
+ struct ioeventfd *p;
+ struct eventfd_ctx *eventfd;
+ int ret = -ENOENT;
+
+ if (WARN_ON_ONCE(!dm || !config))
+ return -EINVAL;
+
+ eventfd = eventfd_ctx_fdget(config->fd);
+ if (IS_ERR(eventfd))
+ return PTR_ERR(eventfd);
+
+ mutex_lock(&dm->ioeventfds_lock);
+
+ list_for_each_entry(p, &dm->ioeventfds, list) {
+ /*
+ * Match the full registration, data match included, so that
+ * removing one of several ioeventfds of an address that only
+ * differ by their data leaves the others in place.
+ */
+ if (p->eventfd != eventfd || p->addr != config->addr ||
+ p->length != config->len || p->wildcard != wildcard ||
+ (!wildcard && p->data != config->data))
+ continue;
+
+ bao_io_client_range_del(dm->ioeventfd_client, p->addr,
+ p->addr + p->length - 1);
+
+ bao_ioeventfd_shutdown(dm, p);
+ ret = 0;
+ break;
+ }
+
+ mutex_unlock(&dm->ioeventfds_lock);
+ eventfd_ctx_put(eventfd);
+
+ return ret;
+}
+
+/**
+ * bao_ioeventfd_handler - Handle an Ioeventfd client I/O request
+ * @client: Ioeventfd client associated with the request
+ * @req: I/O request to process
+ *
+ * Processes I/O requests from the Bao I/O client kernel thread
+ * (bao_io_client_kernel_thread). For READ operations, the value is
+ * ignored and set to 0 since virtio MMIO drivers only write to the
+ * `QueueNotify` field. WRITE operations are checked against the
+ * registered ioeventfds, and the corresponding eventfd is signaled
+ * if a match is found.
+ *
+ * Return: 0 on success, a negative error code on failure
+ */
+static int bao_ioeventfd_handler(struct bao_io_client *client,
+ struct bao_virtio_request *req)
+{
+ struct ioeventfd *p;
+
+ if (WARN_ON_ONCE(!client || !req))
+ return -EINVAL;
+
+ if (req->op == BAO_IO_READ) {
+ req->value = 0;
+ return 0;
+ }
+
+ mutex_lock(&client->dm->ioeventfds_lock);
+
+ p = bao_ioeventfd_match(client->dm, req->addr, req->value,
+ req->access_width);
+ if (p)
+ eventfd_signal(p->eventfd);
+
+ mutex_unlock(&client->dm->ioeventfds_lock);
+
+ return 0;
+}
+
+int bao_ioeventfd_client_config(struct bao_dm *dm, struct bao_ioeventfd *config)
+{
+ if (WARN_ON_ONCE(!dm || !config))
+ return -EINVAL;
+
+ if (!bao_ioeventfd_config_valid(config))
+ return -EINVAL;
+
+ if (config->flags & BAO_IOEVENTFD_FLAG_DEASSIGN)
+ return bao_ioeventfd_deassign(dm, config);
+
+ return bao_ioeventfd_assign(dm, config);
+}
+
+int bao_ioeventfd_client_init(struct bao_dm *dm)
+{
+ char name[BAO_NAME_MAX_LEN];
+
+ if (WARN_ON_ONCE(!dm))
+ return -EINVAL;
+
+ mutex_init(&dm->ioeventfds_lock);
+ INIT_LIST_HEAD(&dm->ioeventfds);
+
+ snprintf(name, sizeof(name), "bao-ioevfdc%u", dm->info.id);
+
+ dm->ioeventfd_client = bao_io_client_create(dm, bao_ioeventfd_handler,
+ NULL, false, name);
+ if (!dm->ioeventfd_client)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void bao_ioeventfd_client_destroy(struct bao_dm *dm)
+{
+ struct ioeventfd *p;
+ struct ioeventfd *next;
+
+ if (WARN_ON_ONCE(!dm))
+ return;
+
+ mutex_lock(&dm->ioeventfds_lock);
+ list_for_each_entry_safe(p, next, &dm->ioeventfds, list)
+ bao_ioeventfd_shutdown(dm, p);
+ mutex_unlock(&dm->ioeventfds_lock);
+}diff --git a/drivers/virt/bao/io-dispatcher/irqfd.c b/drivers/virt/bao/io-dispatcher/irqfd.c
new file mode 100644
index 000000000000..ef81b9623320
--- /dev/null
+++ b/drivers/virt/bao/io-dispatcher/irqfd.c
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Bao Hypervisor Irqfd Server
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ */
+
+#include <linux/eventfd.h>
+#include <linux/file.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include "bao_drv.h"
+
+/* Cleanup work has been queued; set via test_and_set_bit(). */
+#define BAO_IRQFD_SHUTDOWN 0
+
+/**
+ * struct irqfd - Properties of an IRQ eventfd
+ * @dm: Associated Bao device model
+ * @wait: Wait queue entry for blocking/waking
+ * @shutdown: Work struct for async shutdown
+ * @eventfd: Eventfd used to signal interrupts
+ * @list: List node within &bao_dm.irqfds
+ * @pt: Poll table for select/poll on the eventfd
+ * @flags: Internal lifecycle flags (BAO_IRQFD_*)
+ *
+ * Represents an IRQ eventfd registered to a Bao device model.
+ */
+struct irqfd {
+ struct bao_dm *dm;
+ wait_queue_entry_t wait;
+ struct work_struct shutdown;
+ struct eventfd_ctx *eventfd;
+ struct list_head list;
+ poll_table pt;
+ unsigned long flags;
+};
+
+/* Queue the cleanup work at most once. Safe from atomic context. */
+static void bao_irqfd_queue_shutdown(struct irqfd *irqfd)
+{
+ if (!test_and_set_bit(BAO_IRQFD_SHUTDOWN, &irqfd->flags))
+ queue_work(irqfd->dm->irqfd_server, &irqfd->shutdown);
+}
+
+/**
+ * bao_irqfd_inject - Inject a notify hypercall into the Bao hypervisor
+ * @id: Bao DM ID
+ *
+ * Return: 0 on success, -EIO if the hypercall fails.
+ */
+static int bao_irqfd_inject(u32 id)
+{
+ struct bao_remio_hypercall_ctx ctx = {
+ .dm_id = id,
+ .addr = 0,
+ .op = BAO_IO_NOTIFY,
+ .value = 0,
+ .access_width = 0,
+ .request_id = 0,
+ };
+
+ if (bao_remio_hypercall(&ctx))
+ return -EIO;
+
+ return 0;
+}
+
+/**
+ * bao_irqfd_wakeup - Custom wake-up handler for eventfd signaling
+ * @wait: Wait queue entry
+ * @mode: Mode flags
+ * @sync: Sync indicator
+ * @key: Poll bits (cast from void *)
+ *
+ * Called by the Linux kernel poll table when the underlying eventfd is signaled.
+ * Injects a Bao notify hypercall on EPOLLIN or schedules shutdown on EPOLLHUP.
+ *
+ * Return: 0, so the other waiters on the eventfd are woken up too.
+ */
+static int bao_irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode,
+ int sync, void *key)
+{
+ struct irqfd *irqfd = container_of(wait, struct irqfd, wait);
+ __poll_t flags = key_to_poll(key);
+
+ if (flags & EPOLLIN)
+ bao_irqfd_inject(irqfd->dm->info.id);
+
+ if (flags & EPOLLHUP)
+ /* Defer teardown to the cleanup work; can't sleep here. */
+ bao_irqfd_queue_shutdown(irqfd);
+
+ return 0;
+}
+
+/**
+ * bao_irqfd_poll_func - Register an IRQFD with a poll table
+ * @file: File to poll
+ * @wqh: Wait queue head
+ * @pt: Poll table
+ *
+ * Adds the irqfd's wait queue entry to the kernel wait queue for event monitoring.
+ */
+static void bao_irqfd_poll_func(struct file *file, wait_queue_head_t *wqh,
+ poll_table *pt)
+{
+ struct irqfd *irqfd;
+
+ if (WARN_ON_ONCE(!pt || !wqh))
+ return;
+
+ irqfd = container_of(pt, struct irqfd, pt);
+ add_wait_queue(wqh, &irqfd->wait);
+}
+
+/**
+ * irqfd_shutdown_work - Workqueue handler to shutdown an irqfd
+ * @work: Work struct for the shutdown operation
+ *
+ * Sole owner of @irqfd: unlinks it (if still linked), detaches its waitqueue
+ * entry, drops the eventfd reference and frees it.
+ */
+static void irqfd_shutdown_work(struct work_struct *work)
+{
+ struct irqfd *irqfd = container_of(work, struct irqfd, shutdown);
+ struct bao_dm *dm = irqfd->dm;
+ u64 cnt;
+
+ mutex_lock(&dm->irqfds_lock);
+ if (!list_empty(&irqfd->list)) {
+ list_del_init(&irqfd->list);
+ dm->nr_irqfds--;
+ }
+ mutex_unlock(&dm->irqfds_lock);
+
+ eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);
+ eventfd_ctx_put(irqfd->eventfd);
+ kfree(irqfd);
+}
+
+/**
+ * bao_irqfd_assign - Assign an eventfd to a DM and create an irqfd
+ * @dm: Bao device model to assign the eventfd
+ * @args: Configuration of the irqfd to assign
+ *
+ * Return: 0 on success, a negative error code on failure
+ */
+static int bao_irqfd_assign(struct bao_dm *dm, struct bao_irqfd *args)
+{
+ struct eventfd_ctx *eventfd = NULL;
+ struct irqfd *irqfd;
+ struct irqfd *tmp;
+ __poll_t events;
+ struct fd f;
+ int ret = 0;
+
+ if (WARN_ON_ONCE(!dm || !args))
+ return -EINVAL;
+
+ irqfd = kzalloc_obj(*irqfd);
+ if (!irqfd)
+ return -ENOMEM;
+
+ irqfd->dm = dm;
+ INIT_LIST_HEAD(&irqfd->list);
+ INIT_WORK(&irqfd->shutdown, irqfd_shutdown_work);
+
+ f = fdget(args->fd);
+ if (!fd_file(f)) {
+ ret = -EBADF;
+ goto out_free_irqfd;
+ }
+
+ eventfd = eventfd_ctx_fileget(fd_file(f));
+ if (IS_ERR(eventfd)) {
+ ret = PTR_ERR(eventfd);
+ goto out_fdput;
+ }
+ irqfd->eventfd = eventfd;
+
+ init_waitqueue_func_entry(&irqfd->wait, bao_irqfd_wakeup);
+ init_poll_funcptr(&irqfd->pt, bao_irqfd_poll_func);
+
+ /*
+ * Hold irqfds_lock across the waitqueue install (vfs_poll) and list_add
+ * so the irqfd is not visible to deassign/destroy before its waitqueue
+ * entry is in place, and any racing POLLHUP cleanup work blocks on
+ * irqfds_lock until publication completes.
+ */
+ mutex_lock(&dm->irqfds_lock);
+ if (dm->nr_irqfds >= BAO_IRQFD_MAX) {
+ ret = -ENOSPC;
+ mutex_unlock(&dm->irqfds_lock);
+ goto out_put_eventfd;
+ }
+
+ list_for_each_entry(tmp, &dm->irqfds, list) {
+ if (irqfd->eventfd == tmp->eventfd) {
+ ret = -EBUSY;
+ mutex_unlock(&dm->irqfds_lock);
+ goto out_put_eventfd;
+ }
+ }
+
+ events = vfs_poll(fd_file(f), &irqfd->pt);
+ list_add_tail(&irqfd->list, &dm->irqfds);
+ dm->nr_irqfds++;
+ if (events & EPOLLIN)
+ bao_irqfd_inject(dm->info.id);
+ mutex_unlock(&dm->irqfds_lock);
+
+ fdput(f);
+ return 0;
+
+out_put_eventfd:
+ eventfd_ctx_put(eventfd);
+out_fdput:
+ fdput(f);
+out_free_irqfd:
+ kfree(irqfd);
+ return ret;
+}
+
+/**
+ * bao_irqfd_deassign - Deassign an eventfd and destroy the associated irqfd
+ * @dm: Bao device model to remove the irqfd from
+ * @args: Configuration of the irqfd to deassign
+ *
+ * Return: 0 on success, a negative error code on failure
+ */
+static int bao_irqfd_deassign(struct bao_dm *dm, struct bao_irqfd *args)
+{
+ struct irqfd *irqfd;
+ struct irqfd *tmp;
+ struct eventfd_ctx *eventfd;
+
+ if (WARN_ON_ONCE(!dm || !args))
+ return -EINVAL;
+
+ eventfd = eventfd_ctx_fdget(args->fd);
+ if (IS_ERR(eventfd))
+ return PTR_ERR(eventfd);
+
+ mutex_lock(&dm->irqfds_lock);
+ list_for_each_entry_safe(irqfd, tmp, &dm->irqfds, list) {
+ if (irqfd->eventfd == eventfd) {
+ list_del_init(&irqfd->list);
+ dm->nr_irqfds--;
+ bao_irqfd_queue_shutdown(irqfd);
+ break;
+ }
+ }
+ mutex_unlock(&dm->irqfds_lock);
+
+ eventfd_ctx_put(eventfd);
+
+ /* Wait for cleanup work to finish so the eventfd is fully detached. */
+ flush_workqueue(dm->irqfd_server);
+
+ return 0;
+}
+
+int bao_irqfd_server_config(struct bao_dm *dm, struct bao_irqfd *config)
+{
+ if (WARN_ON_ONCE(!dm || !config))
+ return -EINVAL;
+
+ if (config->flags & ~BAO_IRQFD_FLAG_DEASSIGN)
+ return -EINVAL;
+
+ if (config->flags & BAO_IRQFD_FLAG_DEASSIGN)
+ return bao_irqfd_deassign(dm, config);
+
+ return bao_irqfd_assign(dm, config);
+}
+
+int bao_irqfd_server_init(struct bao_dm *dm)
+{
+ if (WARN_ON_ONCE(!dm))
+ return -EINVAL;
+
+ mutex_init(&dm->irqfds_lock);
+ INIT_LIST_HEAD(&dm->irqfds);
+
+ dm->irqfd_server = alloc_workqueue("bao-ioirqfds%u",
+ WQ_UNBOUND | WQ_HIGHPRI, 0,
+ dm->info.id);
+ if (!dm->irqfd_server)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void bao_irqfd_server_destroy(struct bao_dm *dm)
+{
+ struct irqfd *irqfd;
+ struct irqfd *next;
+
+ if (WARN_ON_ONCE(!dm))
+ return;
+
+ mutex_lock(&dm->irqfds_lock);
+ list_for_each_entry_safe(irqfd, next, &dm->irqfds, list) {
+ list_del_init(&irqfd->list);
+ dm->nr_irqfds--;
+ bao_irqfd_queue_shutdown(irqfd);
+ }
+ mutex_unlock(&dm->irqfds_lock);
+
+ /* Drain all cleanup work before tearing the workqueue down. */
+ if (dm->irqfd_server) {
+ flush_workqueue(dm->irqfd_server);
+ destroy_workqueue(dm->irqfd_server);
+ }
+}diff --git a/include/uapi/linux/bao.h b/include/uapi/linux/bao.h
new file mode 100644
index 000000000000..162880d0cbdf
--- /dev/null
+++ b/include/uapi/linux/bao.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Provides the Bao Hypervisor IOCTLs and global structures
+ *
+ * Copyright (c) Bao Project and Contributors. All rights reserved.
+ *
+ * Authors:
+ * João Peixoto <jpeixoto@osyx.tech>
+ * José Martins <jose@osyx.tech>
+ * David Cerdeira <davidmcerdeira@osyx.tech>
+ */
+
+#ifndef _UAPI_BAO_H
+#define _UAPI_BAO_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+/**
+ * struct bao_virtio_request - Parameters of a Bao VirtIO request
+ * @dm_id: Device model ID (set by the kernel; ignored on input, the DM is
+ * the one bound to the file descriptor)
+ * @addr: MMIO register address accessed
+ * @op: Operation type, BAO_IO_WRITE or BAO_IO_READ
+ * @value: Value to write or read
+ * @access_width: Access width (VirtIO MMIO supports 4-byte aligned accesses)
+ * @request_id: Request ID of the I/O request
+ *
+ * BAO_IOCTL_IO_CLIENT_ATTACH fills in a request; BAO_IOCTL_IO_REQUEST_COMPLETE
+ * completes the request with that @request_id, which must be a read or a write
+ * handed out on the same file descriptor, taking only @value from userspace.
+ */
+struct bao_virtio_request {
+ __u64 dm_id;
+ __u64 addr;
+ __u64 op;
+ __u64 value;
+ __u64 access_width;
+ __u64 request_id;
+};
+
+/* Values of bao_virtio_request.op: the frontend wrote to, or read from, @addr */
+#define BAO_IO_WRITE 0
+#define BAO_IO_READ 1
+
+/**
+ * struct bao_ioeventfd - Parameters of an ioeventfd request
+ * @fd: Eventfd file descriptor associated with the I/O request
+ * @flags: Logical OR of BAO_IOEVENTFD_FLAG_*
+ * @addr: Start address of the I/O range
+ * @len: Length of the I/O range
+ * @reserved: Reserved, must be 0
+ * @data: Data for matching (used if data matching is enabled)
+ */
+struct bao_ioeventfd {
+ __s32 fd;
+ __u32 flags;
+ __u64 addr;
+ __u32 len;
+ __u32 reserved;
+ __u64 data;
+};
+
+/* Only signal the eventfd when the written value equals bao_ioeventfd.data */
+#define BAO_IOEVENTFD_FLAG_DATAMATCH (1U << 1)
+/* Remove the ioeventfd instead of adding it */
+#define BAO_IOEVENTFD_FLAG_DEASSIGN (1U << 2)
+
+/**
+ * struct bao_irqfd - Parameters of an IRQFD request
+ * @fd: File descriptor of the eventfd
+ * @flags: Logical OR of BAO_IRQFD_FLAG_*
+ */
+struct bao_irqfd {
+ __s32 fd;
+ __u32 flags;
+};
+
+/* Remove the irqfd instead of adding it */
+#define BAO_IRQFD_FLAG_DEASSIGN (1U << 0)
+
+/**
+ * struct bao_dm_info - Parameters of a Bao device model
+ * @shmem_addr: Base address of the shared memory
+ * @shmem_size: Size of the shared memory
+ * @id: Virtual ID of the DM
+ * @irq: Hypervisor notification line, as a line of the system's root
+ * interrupt controller (e.g. a GIC SPI on arm64, a PLIC source on
+ * riscv), which the driver maps and uses as the backend I/O doorbell
+ *
+ * The layout is free of implicit padding so it is identical on every
+ * architecture.
+ */
+struct bao_dm_info {
+ __u64 shmem_addr;
+ __u64 shmem_size;
+ __u32 id;
+ __u32 irq;
+};
+
+/*
+ * The ioctl type for Bao, documented in
+ * Documentation/userspace-api/ioctl/ioctl-number.rst
+ */
+#define BAO_IOCTL_TYPE 0xA7
+
+/*
+ * Bao userspace IOCTL commands
+ * Follows Linux kernel convention, see Documentation/driver-api/ioctl.rst
+ */
+/*
+ * Issued on the /dev/bao control device by the userspace VMM to instantiate a
+ * device model from its own configuration. On success returns a new file
+ * descriptor bound to that DM; all other commands below operate on that fd.
+ */
+#define BAO_IOCTL_CREATE_DM _IOW(BAO_IOCTL_TYPE, 0x00, struct bao_dm_info)
+#define BAO_IOCTL_DM_GET_INFO _IOR(BAO_IOCTL_TYPE, 0x01, struct bao_dm_info)
+/* Waits for the next I/O request of the DM and returns it */
+#define BAO_IOCTL_IO_CLIENT_ATTACH \
+ _IOR(BAO_IOCTL_TYPE, 0x02, struct bao_virtio_request)
+#define BAO_IOCTL_IO_REQUEST_COMPLETE \
+ _IOW(BAO_IOCTL_TYPE, 0x03, struct bao_virtio_request)
+#define BAO_IOCTL_IOEVENTFD _IOW(BAO_IOCTL_TYPE, 0x04, struct bao_ioeventfd)
+#define BAO_IOCTL_IRQFD _IOW(BAO_IOCTL_TYPE, 0x05, struct bao_irqfd)
+
+#endif /* _UAPI_BAO_H */--
2.53.0