[RFC PATCH for 4.21 13/16] cpu-opv/selftests: Provide percpu_op API
From: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Date: 2018-11-01 10:00:34
Also in:
linux-kselftest, lkml
Subsystem:
kernel selftest framework, the rest · Maintainers:
Shuah Khan, Shuah Khan, Linus Torvalds
Introduce percpu-op.h API. It uses rseq internally as fast-path if
invoked from the right CPU, else cpu_opv as slow-path if called
from the wrong CPU or if rseq fails.
This allows acting on per-cpu data from various CPUs transparently from
user-space: cpu_opv will take care of migrating the thread to the
requested CPU. Use-cases such as rebalancing memory across per-cpu
memory pools, or migrating tasks for a user-space scheduler, are thus
facilitated. This also handles debugger single-stepping.
The use from userspace is, e.g. for a counter increment:
int cpu, ret;
cpu = percpu_current_cpu();
ret = percpu_addv(&data->c[cpu].count, 1, cpu);
if (unlikely(ret)) {
perror("percpu_addv");
return -1;
}
return 0;
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
CC: Shuah Khan <shuah@kernel.org>
CC: Russell King <redacted>
CC: Catalin Marinas <catalin.marinas@arm.com>
CC: Will Deacon <redacted>
CC: Thomas Gleixner <redacted>
CC: Paul Turner <redacted>
CC: Peter Zijlstra <peterz@infradead.org>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Andi Kleen <redacted>
CC: Dave Watson <redacted>
CC: Chris Lameter <redacted>
CC: Ingo Molnar <mingo@redhat.com>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Ben Maurer <redacted>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: "Paul E. McKenney" <redacted>
CC: Josh Triplett <josh@joshtriplett.org>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Andrew Morton <akpm@linux-foundation.org>
CC: Boqun Feng <redacted>
CC: linux-kselftest@vger.kernel.org
CC: linux-api@vger.kernel.org
---
tools/testing/selftests/cpu-opv/percpu-op.h | 151 ++++++++++++++++++++++++++++
1 file changed, 151 insertions(+)
create mode 100644 tools/testing/selftests/cpu-opv/percpu-op.h
diff --git a/tools/testing/selftests/cpu-opv/percpu-op.h b/tools/testing/selftests/cpu-opv/percpu-op.h
new file mode 100644
index 000000000000..918171e585d7
--- /dev/null
+++ b/tools/testing/selftests/cpu-opv/percpu-op.h@@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * percpu-op.h + * + * (C) Copyright 2017-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#ifndef PERCPU_OP_H +#define PERCPU_OP_H + +#include <stdint.h> +#include <stdbool.h> +#include <errno.h> +#include <stdlib.h> +#include "rseq.h" +#include "cpu-op.h" + +static inline uint32_t percpu_current_cpu(void) +{ + return rseq_current_cpu(); +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, + int cpu) +{ + int ret; + + ret = rseq_cmpeqv_storev(v, expect, newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_storev(v, expect, newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, + off_t voffp, intptr_t *load, int cpu) +{ + int ret; + + ret = rseq_cmpnev_storeoffp_load(v, expectnot, voffp, load, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpnev_storeoffp_load(v, expectnot, voffp, + load, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_addv(intptr_t *v, intptr_t count, int cpu) +{ + if (rseq_unlikely(rseq_addv(v, count, cpu))) + return cpu_op_addv(v, count, cpu); + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_storev_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, + newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_storev_storev(v, expect, v2, newv2, + newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_storev_storev_release(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_trystorev_storev_release(v, expect, v2, newv2, + newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_storev_storev_release(v, expect, v2, newv2, + newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_cmpeqv_storev(v, expect, v2, expect2, newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_cmpeqv_storev(v, expect, v2, expect2, + newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, + newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_memcpy_storev(v, expect, dst, src, len, + newv, cpu); + } + return 0; +} + +static inline __attribute__((always_inline)) +int percpu_cmpeqv_memcpy_storev_release(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + int ret; + + ret = rseq_cmpeqv_trymemcpy_storev_release(v, expect, dst, src, len, + newv, cpu); + if (rseq_unlikely(ret)) { + if (ret > 0) + return ret; + return cpu_op_cmpeqv_memcpy_storev_release(v, expect, dst, src, + len, newv, cpu); + } + return 0; +} + +#endif /* PERCPU_OP_H_ */
--
2.11.0