On Thu, Jul 05, 2018 at 08:05:03PM +0200, Peter Zijlstra wrote:

On Mon, Jul 02, 2018 at 01:30:19AM +0800, Guo Ren wrote:

quoted

+static irqreturn_t handle_ipi(int irq, void *dev)
+{
+	unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits;
+
+	while (true) {
+		unsigned long ops;
+
+		/* Order bit clearing and data access. */
+		mb();
+
+		ops = xchg(pending_ipis, 0);
+		if (ops == 0)
+			return IRQ_HANDLED;
+
+		if (ops & (1 << IPI_RESCHEDULE))
+			scheduler_ipi();
+
+		if (ops & (1 << IPI_CALL_FUNC))
+			generic_smp_call_function_interrupt();
+
+		BUG_ON((ops >> IPI_MAX) != 0);
+
+		/* Order data access and bit testing. */
+		mb();
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void (*send_arch_ipi)(const unsigned long *mask, unsigned long irq) = NULL;
+
+void __init set_send_ipi(void (*func)(const unsigned long *, unsigned long))
+{
+	if (send_arch_ipi)
+		return;
+
+	send_arch_ipi = func;
+}
+
+static void
+send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
+{
+	int i;
+
+	mb();
+	for_each_cpu(i, to_whom)
+		set_bit(operation, &ipi_data[i].bits);
+
+	mb();
+	send_arch_ipi(cpumask_bits(to_whom), IPI_IRQ);
+}

Please explain those mb()'s... I'm thinking you meant to use smp_mb().

Yes, smp_mb(). Current smp_mb()&mb() is the same: sync.is.

In next version patch, I'll seperate smp_mb() and mb() and use ld/st.barrier
instead of sync.is. Sync.is is expensive that it flush cpu's pipeline.

But then for handle_ipi(), the xchg() should already imply all those.

Yes, approve.

And the send_ipi_message() only needs the second.

Yes, approve.

 Guo Ren