Re: [dpdk-dev] [RFC PATCH v3 1/5] eal: add new definitions for wait scheme
From: Ananyev, Konstantin <hidden>
Date: 2021-10-07 16:19:22
quoted hunk ↗ jump to hunk
Introduce macros as generic interface for address monitoring. Signed-off-by: Feifei Wang <redacted> Reviewed-by: Ruifeng Wang <redacted> --- lib/eal/arm/include/rte_pause_64.h | 151 ++++++++++++++++++---------- lib/eal/include/generic/rte_pause.h | 78 ++++++++++++++ 2 files changed, 175 insertions(+), 54 deletions(-)diff --git a/lib/eal/arm/include/rte_pause_64.h b/lib/eal/arm/include/rte_pause_64.h index e87d10b8cc..205510e044 100644 --- a/lib/eal/arm/include/rte_pause_64.h +++ b/lib/eal/arm/include/rte_pause_64.h@@ -31,20 +31,12 @@ static inline void rte_pause(void) /* Put processor into low power WFE(Wait For Event) state. */ #define __WFE() { asm volatile("wfe" : : : "memory"); } -static __rte_always_inline void -rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, - int memorder) -{ - uint16_t value; - - assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); - - /* - * Atomic exclusive load from addr, it returns the 16-bit content of - * *addr while making it 'monitored',when it is written by someone - * else, the 'monitored' state is cleared and a event is generated - * implicitly to exit WFE. - */ +/* + * Atomic exclusive load from addr, it returns the 16-bit content of + * *addr while making it 'monitored', when it is written by someone + * else, the 'monitored' state is cleared and a event is generated + * implicitly to exit WFE. + */ #define __LOAD_EXC_16(src, dst, memorder) { \ if (memorder == __ATOMIC_RELAXED) { \ asm volatile("ldxrh %w[tmp], [%x[addr]]" \@@ -58,6 +50,52 @@ rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, : "memory"); \ } } +/* + * Atomic exclusive load from addr, it returns the 32-bit content of + * *addr while making it 'monitored', when it is written by someone + * else, the 'monitored' state is cleared and a event is generated + * implicitly to exit WFE. + */ +#define __LOAD_EXC_32(src, dst, memorder) { \ + if (memorder == __ATOMIC_RELAXED) { \ + asm volatile("ldxr %w[tmp], [%x[addr]]" \ + : [tmp] "=&r" (dst) \ + : [addr] "r"(src) \ + : "memory"); \ + } else { \ + asm volatile("ldaxr %w[tmp], [%x[addr]]" \ + : [tmp] "=&r" (dst) \ + : [addr] "r"(src) \ + : "memory"); \ + } } + +/* + * Atomic exclusive load from addr, it returns the 64-bit content of + * *addr while making it 'monitored', when it is written by someone + * else, the 'monitored' state is cleared and a event is generated + * implicitly to exit WFE. + */ +#define __LOAD_EXC_64(src, dst, memorder) { \ + if (memorder == __ATOMIC_RELAXED) { \ + asm volatile("ldxr %x[tmp], [%x[addr]]" \ + : [tmp] "=&r" (dst) \ + : [addr] "r"(src) \ + : "memory"); \ + } else { \ + asm volatile("ldaxr %x[tmp], [%x[addr]]" \ + : [tmp] "=&r" (dst) \ + : [addr] "r"(src) \ + : "memory"); \ + } } + +static __rte_always_inline void +rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, + int memorder) +{ + uint16_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + __LOAD_EXC_16(addr, value, memorder) if (value != expected) { __SEVL()@@ -66,7 +104,6 @@ rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, __LOAD_EXC_16(addr, value, memorder) } while (value != expected); } -#undef __LOAD_EXC_16 } static __rte_always_inline void@@ -77,25 +114,6 @@ rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); - /* - * Atomic exclusive load from addr, it returns the 32-bit content of - * *addr while making it 'monitored',when it is written by someone - * else, the 'monitored' state is cleared and a event is generated - * implicitly to exit WFE. - */ -#define __LOAD_EXC_32(src, dst, memorder) { \ - if (memorder == __ATOMIC_RELAXED) { \ - asm volatile("ldxr %w[tmp], [%x[addr]]" \ - : [tmp] "=&r" (dst) \ - : [addr] "r"(src) \ - : "memory"); \ - } else { \ - asm volatile("ldaxr %w[tmp], [%x[addr]]" \ - : [tmp] "=&r" (dst) \ - : [addr] "r"(src) \ - : "memory"); \ - } } - __LOAD_EXC_32(addr, value, memorder) if (value != expected) { __SEVL()@@ -104,7 +122,6 @@ rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, __LOAD_EXC_32(addr, value, memorder) } while (value != expected); } -#undef __LOAD_EXC_32 } static __rte_always_inline void@@ -115,25 +132,6 @@ rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); - /* - * Atomic exclusive load from addr, it returns the 64-bit content of - * *addr while making it 'monitored',when it is written by someone - * else, the 'monitored' state is cleared and a event is generated - * implicitly to exit WFE. - */ -#define __LOAD_EXC_64(src, dst, memorder) { \ - if (memorder == __ATOMIC_RELAXED) { \ - asm volatile("ldxr %x[tmp], [%x[addr]]" \ - : [tmp] "=&r" (dst) \ - : [addr] "r"(src) \ - : "memory"); \ - } else { \ - asm volatile("ldaxr %x[tmp], [%x[addr]]" \ - : [tmp] "=&r" (dst) \ - : [addr] "r"(src) \ - : "memory"); \ - } } - __LOAD_EXC_64(addr, value, memorder) if (value != expected) { __SEVL()@@ -143,6 +141,51 @@ rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, } while (value != expected); } } + +#define rte_wait_event_16(addr, mask, expected, cond, memorder) \ +do { \ + uint16_t value \ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \ + __LOAD_EXC_16(addr, value, memorder) \ + if ((value & mask) cond expected) { \ + __SEVL() \ + do { \ + __WFE() \ + __LOAD_EXC_16(addr, value, memorder) \ + } while ((value & mask) cond expected); \ + } \ +} while (0) + +#define rte_wait_event_32(addr, mask, expected, cond, memorder) \ +do { \ + uint32_t value \ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \ + __LOAD_EXC_32(addr, value, memorder) \ + if ((value & mask) op expected) { \ + __SEVL() \ + do { \ + __WFE() \ + __LOAD_EXC_32(addr, value, memorder) \ + } while ((value & mask) cond expected); \ + } \ +} while (0) + +#define rte_wait_event_64(addr, mask, expected, cond, memorder) \ +do { \ + uint64_t value \ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \ + __LOAD_EXC_64(addr, value, memorder) \ + if ((value & mask) cond expected) { \ + __SEVL() \ + do { \ + __WFE() \ + __LOAD_EXC_64(addr, value, memorder) \ + } while ((value & mask) cond expected); \ + } \ +} while (0) + +#undef __LOAD_EXC_16 +#undef __LOAD_EXC_32 #undef __LOAD_EXC_64 #undef __SEVLdiff --git a/lib/eal/include/generic/rte_pause.h b/lib/eal/include/generic/rte_pause.h index 668ee4a184..4e32107eca 100644 --- a/lib/eal/include/generic/rte_pause.h +++ b/lib/eal/include/generic/rte_pause.h@@ -111,6 +111,84 @@ rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, while (__atomic_load_n(addr, memorder) != expected) rte_pause(); } + +/* + * Wait until a 16-bit *addr breaks the condition, with a relaxed memory + * ordering model meaning the loads around this API can be reordered. + * + * @param addr + * A pointer to the memory location. + * @param mask + * A mask of value bits in interest + * @param expected + * A 16-bit expected value to be in the memory location. + * @param cond + * A symbol representing the condition (==, !=). + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */
Hmm, so now we have 2 APIs doing similar thing: rte_wait_until_equal_n() and rte_wait_event_n(). Can we probably unite them somehow? At least make rte_wait_until_equal_n() to use rte_wait_event_n() underneath.
+#define rte_wait_event_16(addr, mask, expected, cond, memorder) \
+do { \
+ assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \And why user is not allowed to use __ATOMIC_SEQ_CST here? BTW, if we expect memorder to always be a constant, might be better BUILD_BUG_ON()?
+ \ + while ((__atomic_load_n(addr, memorder) & mask) cond expected) \ + rte_pause(); \ +} while (0)
Two thoughts with these macros:
1. It is a goof practise to put () around macro parameters in the macro body.
Will save from a lot of unexpected troubles.
2. I think these 3 macros can be united into one.
Something like:
#define rte_wait_event(addr, mask, expected, cond, memorder) do {\
typeof (*(addr)) val = __atomic_load_n((addr), (memorder)); \
if ((val & (typeof(val))(mask)) cond (typeof(val))(expected)) \
break; \
rte_pause(); \
} while (1);
+
+/*
+ * Wait until a 32-bit *addr breaks the condition, with a relaxed memory
+ * ordering model meaning the loads around this API can be reordered.
+ *
+ * @param addr
+ * A pointer to the memory location.
+ * @param mask
+ * A mask of value bits in interest.
+ * @param expected
+ * A 32-bit expected value to be in the memory location.
+ * @param cond
+ * A symbol representing the condition (==, !=).
+ * @param memorder
+ * Two different memory orders that can be specified:
+ * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ * C++11 memory orders with the same names, see the C++11 standard or
+ * the GCC wiki on atomic synchronization for detailed definition.
+ */
+#define rte_wait_event_32(addr, mask, expected, cond, memorder) \
+do { \
+ assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \
+ \
+ while ((__atomic_load_n(addr, memorder) & mask) cond expected) \
+ rte_pause(); \
+} while (0)
+
+/*
+ * Wait until a 64-bit *addr breaks the condition, with a relaxed memory
+ * ordering model meaning the loads around this API can be reordered.
+ *
+ * @param addr
+ * A pointer to the memory location.
+ * @param mask
+ * A mask of value bits in interest
+ * @param expected
+ * A 64-bit expected value to be in the memory location.
+ * @param cond
+ * A symbol representing the condition (==, !=).
+ * @param memorder
+ * Two different memory orders that can be specified:
+ * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ * C++11 memory orders with the same names, see the C++11 standard or
+ * the GCC wiki on atomic synchronization for detailed definition.
+ */
+#define rte_wait_event_64(addr, mask, expected, cond, memorder) \
+do { \
+ assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); \
+ \
+ while ((__atomic_load_n(addr, memorder) & mask) cond expected) \
+ rte_pause(); \
+} while (0)
#endif
#endif /* _RTE_PAUSE_H_ */
--
2.25.1