diff --git a/include/linux/filter.h b/include/linux/filter.h
index 4f0cd67caccb..5c2f57ec3940 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1994,8 +1994,8 @@ int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len);
int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len);
void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len);
-void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
- void *buf, unsigned long len, bool flush);
+int bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
+ void *buf, unsigned long len, bool flush);
int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset,
const void *from, u32 len, u64 flags);
void *bpf_skb_meta_pointer(struct sk_buff *skb, u32 offset);
@@ -2029,9 +2029,10 @@ static inline void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
return NULL;
}
-static inline void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off, void *buf,
- unsigned long len, bool flush)
+static inline int bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
+ void *buf, unsigned long len, bool flush)
{
+ return -EOPNOTSUPP;
}
static inline int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset,diff --git a/include/net/xdp.h b/include/net/xdp.h
index 07231adfb5f8..16693cf9653e 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -136,6 +136,18 @@ static __always_inline void xdp_buff_set_frag_unreadable(struct xdp_buff *xdp)
xdp->flags |= XDP_FLAGS_FRAGS_UNREADABLE;
}
+static __always_inline void
+xdp_buff_clear_frag_unreadable(struct xdp_buff *xdp)
+{
+ xdp->flags &= ~XDP_FLAGS_FRAGS_UNREADABLE;
+}
+
+static __always_inline bool
+xdp_buff_is_frag_unreadable(const struct xdp_buff *xdp)
+{
+ return !!(xdp->flags & XDP_FLAGS_FRAGS_UNREADABLE);
+}
+
static __always_inline u32 xdp_buff_get_skb_flags(const struct xdp_buff *xdp)
{
return xdp->flags;diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 6788489a2608..8c9e4640bcfe 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -3140,7 +3140,9 @@ __bpf_kfunc void *bpf_dynptr_slice(const struct bpf_dynptr *p, u64 offset,
if (!buffer__nullable)
return NULL;
- bpf_xdp_copy_buf(ptr->data, ptr->offset + offset, buffer__nullable, len, false);
+ if (bpf_xdp_copy_buf(ptr->data, ptr->offset + offset,
+ buffer__nullable, len, false))
+ return NULL;
return buffer__nullable;
}
case BPF_DYNPTR_TYPE_SKB_META:
diff --git a/net/core/filter.c b/net/core/filter.c
index f42ecfb5841f..9a55f61fbcdf 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4269,8 +4269,8 @@ static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
.arg2_type = ARG_ANYTHING,
};
-void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
- void *buf, unsigned long len, bool flush)
+int bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
+ void *buf, unsigned long len, bool flush)
{
unsigned long ptr_len, ptr_off = 0;
skb_frag_t *next_frag, *end_frag;@@ -4282,9 +4282,12 @@ void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
src = flush ? buf : xdp->data + off;
dst = flush ? xdp->data + off : buf;
memcpy(dst, src, len);
- return;
+ return 0;
}
+ if (unlikely(xdp_buff_is_frag_unreadable(xdp)))
+ return -EFAULT;
+
sinfo = xdp_get_shared_info_from_buff(xdp);
end_frag = &sinfo->frags[sinfo->nr_frags];
next_frag = &sinfo->frags[0];
@@ -4314,6 +4317,8 @@ void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
ptr_len = skb_frag_size(next_frag);
next_frag++;
}
+
+ return 0;
}
void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
@@ -4332,6 +4337,9 @@ void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
if (likely(offset < size)) /* linear area */
goto out;
+ if (unlikely(xdp_buff_is_frag_unreadable(xdp)))
+ return ERR_PTR(-EFAULT);
+
sinfo = xdp_get_shared_info_from_buff(xdp);
offset -= size;
for (i = 0; i < sinfo->nr_frags; i++) { /* paged area */@@ -4358,10 +4366,9 @@ BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset,
return PTR_ERR(ptr);
if (!ptr)
- bpf_xdp_copy_buf(xdp, offset, buf, len, false);
- else
- memmove(buf, ptr, len);
+ return bpf_xdp_copy_buf(xdp, offset, buf, len, false);
+ memmove(buf, ptr, len);
return 0;
}
@@ -4390,11 +4397,9 @@ BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,
return PTR_ERR(ptr);
if (!ptr)
- bpf_xdp_copy_buf(xdp, offset, buf, len, true);
- else
- /* through a dynptr the source may be an overlapping slice */
- memmove(ptr, buf, len);
+ return bpf_xdp_copy_buf(xdp, offset, buf, len, true);
+ memmove(ptr, buf, len);
return 0;
}
@@ -4422,6 +4427,8 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset)
if (!rxq->frag_size || rxq->frag_size > xdp->frame_sz)
return -EOPNOTSUPP;
+ if (unlikely(xdp_buff_is_frag_unreadable(xdp)))
+ return -EFAULT;
tailroom = rxq->frag_size - skb_frag_size(frag) -
skb_frag_off(frag) % rxq->frag_size;
@@ -4503,6 +4510,7 @@ static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset)
if (unlikely(!sinfo->nr_frags)) {
xdp_buff_clear_frags_flag(xdp);
xdp_buff_clear_frag_pfmemalloc(xdp);
+ xdp_buff_clear_frag_unreadable(xdp);
xdp->data_end -= offset;
}
@@ -5355,8 +5363,7 @@ static unsigned long bpf_xdp_copy(void *dst, const void *ctx,
{
struct xdp_buff *xdp = (struct xdp_buff *)ctx;
- bpf_xdp_copy_buf(xdp, off, dst, len, false);
- return 0;
+ return bpf_xdp_copy_buf(xdp, off, dst, len, false) ? len : 0;
}
BPF_CALL_5(bpf_xdp_event_output, struct xdp_buff *, xdp, struct bpf_map *, map,@@ -5369,6 +5376,9 @@ BPF_CALL_5(bpf_xdp_event_output, struct xdp_buff *, xdp, struct bpf_map *, map,
if (unlikely(!xdp || xdp_size > xdp_get_buff_len(xdp)))
return -EFAULT;
+ if (unlikely(xdp_size > xdp->data_end - xdp->data &&
+ xdp_buff_is_frag_unreadable(xdp)))
+ return -EFAULT;
return bpf_event_output(map, flags, meta, meta_size, xdp,
xdp_size, bpf_xdp_copy);
@@ -12797,6 +12807,7 @@ __bpf_kfunc int bpf_sock_ops_enable_tx_tstamp(struct bpf_sock_ops_kern *skops,
* Return:
* * %0 - success
* * %-EINVAL - invalid len
+ * * %-EFAULT - the kernel cannot read the fragments
*/
__bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len)
{@@ -12812,6 +12823,8 @@ __bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len)
if (unlikely(len > xdp_get_buff_len(xdp)))
return -EINVAL;
+ if (unlikely(xdp_buff_is_frag_unreadable(xdp)))
+ return -EFAULT;
start = xdp_data_meta_unsupported(xdp) ? xdp->data : xdp->data_meta;
--
2.56.1