diff --git a/arch/x86/include/asm/switch_to.h b/arch/x86/include/asm/switch_to.h
index fcc5cd387fd1..0c243dd569fe 100644
--- a/arch/x86/include/asm/switch_to.h
+++ b/arch/x86/include/asm/switch_to.h
@@ -30,7 +30,7 @@ static inline void prepare_switch_to(struct task_struct *prev,
*
* To minimize cache pollution, just follow the stack pointer.
*/
- READ_ONCE(*(unsigned char *)next->thread.sp);
+ (void)READ_ONCE(*(unsigned char *)next->thread.sp);
#endif
}
diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c
index 952286f4826c..1c10632a48bb 100644
--- a/fs/overlayfs/util.c
+++ b/fs/overlayfs/util.c
@@ -222,8 +222,8 @@ void ovl_dentry_update(struct dentry *dentry, struct dentry *upperdentry)
void ovl_inode_init(struct inode *inode, struct inode *realinode, bool is_upper)
{
- WRITE_ONCE(inode->i_private, (unsigned long) realinode |
- (is_upper ? OVL_ISUPPER_MASK : 0));
+ WRITE_ONCE(inode->i_private, (void *)((unsigned long) realinode |
+ (is_upper ? OVL_ISUPPER_MASK : 0)));
}
void ovl_inode_update(struct inode *inode, struct inode *upperinode)@@ -231,7 +231,7 @@ void ovl_inode_update(struct inode *inode, struct inode *upperinode)
WARN_ON(!upperinode);
WARN_ON(!inode_unhashed(inode));
WRITE_ONCE(inode->i_private,
- (unsigned long) upperinode | OVL_ISUPPER_MASK);
+ (void *)((unsigned long) upperinode | OVL_ISUPPER_MASK));
if (!S_ISDIR(upperinode->i_mode))
__insert_inode_hash(inode, (unsigned long) upperinode);
}
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 56b90897a459..b619f5853af8 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -288,6 +288,10 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
}
}
+#define __ALIGNED_WORD(x) \
+ ((sizeof(x) == 1 || sizeof(x) == 2 || sizeof(x) == 4 || \
+ sizeof(x) == sizeof(long)) && (sizeof(x) == __alignof__(x))) \
+
/*
* Prevent the compiler from merging or refetching reads or writes. The
* compiler is also forbidden from reordering successive instances of
@@ -309,8 +313,13 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
* mutilate accesses that either do not require ordering or that interact
* with an explicit memory barrier or atomic instruction that provides the
* required ordering.
+ *
+ * Unaligned data is particularly tricky here: if the type that gets
+ * passed in is not naturally aligned, we cast to a type of higher
+ * alignment, which is not well-defined in C. This is fine as long
+ * as the actual data is aligned, but otherwise might require a trap
+ * to satisfy the load.
*/
-
#define __READ_ONCE(x, check) \
({ \
union { typeof(x) __val; char __c[1]; } __u; \@@ -320,7 +329,32 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
__read_once_size_nocheck(&(x), __u.__c, sizeof(x)); \
__u.__val; \
})
-#define READ_ONCE(x) __READ_ONCE(x, 1)
+
+#define __WRITE_ONCE(x, val) \
+({ \
+ union { typeof(x) __val; char __c[1]; } __u = \
+ { .__val = (__force typeof(x)) (val) }; \
+ __write_once_size(&(x), __u.__c, sizeof(x)); \
+ __u.__val; \
+})
+
+
+/*
+ * the common case is simple: x is naturally aligned, not an array,
+ * and accessible with a single load, avoiding the need for local
+ * variables. With KASAN, this is important as any call to
+ *__write_once_size(),__read_once_size_nocheck() or __read_once_size()
+ * uses significant amounts of stack space for checking that we don't
+ * overflow the union.
+ */
+#define __READ_ONCE_SIMPLE(x) \
+ (typeof(x))(*(volatile typeof(&(x)))&(x))
+
+#define __WRITE_ONCE_SIMPLE(x, val) \
+ ({*(volatile typeof(&(x)))&(x) = (val); })
+
+#define READ_ONCE(x) __builtin_choose_expr(__ALIGNED_WORD(x), \
+ __READ_ONCE_SIMPLE(x), __READ_ONCE(x, 1))
/*
* Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need@@ -328,13 +362,8 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
*/
#define READ_ONCE_NOCHECK(x) __READ_ONCE(x, 0)
-#define WRITE_ONCE(x, val) \
-({ \
- union { typeof(x) __val; char __c[1]; } __u = \
- { .__val = (__force typeof(x)) (val) }; \
- __write_once_size(&(x), __u.__c, sizeof(x)); \
- __u.__val; \
-})
+#define WRITE_ONCE(x, val) do { __builtin_choose_expr(__ALIGNED_WORD(x), \
+ __WRITE_ONCE_SIMPLE(x, val), __WRITE_ONCE(x, val)); } while (0)
#endif /* __KERNEL__ */
--
2.9.0