include/linux/bpf_verifier.h                  |  1 +
 kernel/bpf/verifier.c                         | 78 ++++++++++++++++++-
 tools/testing/selftests/bpf/progs/bpf_misc.h  |  4 +
 .../selftests/bpf/progs/verifier_and.c        |  8 +-
 .../selftests/bpf/progs/verifier_bounds.c     | 61 ++++++++++++---
 .../selftests/bpf/progs/verifier_movsx.c      | 16 +++-
 .../selftests/bpf/progs/verifier_unpriv.c     |  8 +-
 .../bpf/progs/verifier_value_ptr_arith.c      | 16 +++-
 .../selftests/bpf/verifier/dead_code.c        |  3 +-
 tools/testing/selftests/bpf/verifier/jmp32.c  | 33 +++-----
 tools/testing/selftests/bpf/verifier/jset.c   | 10 +--
 11 files changed, 184 insertions(+), 54 deletions(-)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index cebb67becdad..f1573e093120 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h

@@ -576,6 +576,7 @@ struct bpf_insn_aux_data {
        u64 map_key_state; /* constant (32 bit) key tracking for maps */
        int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
        u32 seen; /* this insn was processed by the verifier at env->pass_cnt */
+       bool nospec; /* do not execute this instruction speculatively */
        bool nospec_result; /* result is unsafe under speculation, nospec must follow */
        bool zext_dst; /* this insn zero extends dst reg */
        bool needs_zext; /* alu op needs to clear upper bits */

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 1e1eca85b7a6..db26b477dd45 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c

@@ -2014,6 +2014,18 @@ static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
        return 0;
 }

+static bool error_recoverable_with_nospec(int err)
+{
+       /* Should only return true for non-fatal errors that are allowed to
+        * occur during speculative verification. For these we can insert a
+        * nospec and the program might still be accepted. Do not include
+        * something like ENOMEM because it is likely to re-occur for the next
+        * architectural path once it has been recovered-from in all speculative
+        * paths.
+        */
+       return err == -EPERM || err == -EACCES || err == -EINVAL;
+}

+
 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
                                             int insn_idx, int prev_insn_idx,
                                             bool speculative)

@@ -11160,7 +11172,7 @@ static int check_get_func_ip(struct bpf_verifier_env *env)
        return -ENOTSUPP;
 }

-static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
+static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)
 {
        return &env->insn_aux_data[env->insn_idx];
 }

@@ -13997,7 +14009,9 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
                                    const struct bpf_insn *insn)
 {
-       return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;
+       return env->bypass_spec_v1 ||
+               BPF_SRC(insn->code) == BPF_K ||
+               cur_aux(env)->nospec;
 }

 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,

@@ -19687,10 +19701,41 @@ static int do_check(struct bpf_verifier_env *env)
                sanitize_mark_insn_seen(env);
                prev_insn_idx = env->insn_idx;

+               /* Reduce verification complexity by stopping speculative path
+                * verification when a nospec is encountered.
+                */
+               if (state->speculative && cur_aux(env)->nospec)
+                       goto process_bpf_exit;
+
                err = do_check_insn(env, insn, &do_print_state);
-               if (err < 0) {
+               if (state->speculative && error_recoverable_with_nospec(err)) {
+                       /* Prevent this speculative path from ever reaching the
+                        * insn that would have been unsafe to execute.
+                        */
+                       cur_aux(env)->nospec = true;
+                       /* If it was an ADD/SUB insn, potentially remove any
+                        * markings for alu sanitization.
+                        */
+                       cur_aux(env)->alu_state = 0;
+                       goto process_bpf_exit;
+               } else if (err < 0) {
                        return err;
                } else if (err == PROCESS_BPF_EXIT) {
+                       goto process_bpf_exit;
+               }
+               WARN_ON_ONCE(err);
+
+               if (state->speculative && cur_aux(env)->nospec_result) {
+                       /* If we are on a path that performed a jump-op, this
+                        * may skip a nospec patched-in after the jump. This can
+                        * currently never happen because nospec_result is only
+                        * used for the write-ops
+                        * `*(size*)(dst_reg+off)=src_reg|imm32` which must
+                        * never skip the following insn. Still, add a warning
+                        * to document this in case nospec_result is used
+                        * elsewhere in the future.
+                        */
+                       WARN_ON_ONCE(env->insn_idx != prev_insn_idx + 1);

 process_bpf_exit:
                        mark_verifier_state_scratched(env);
                        update_branch_counts(env, env->cur_state);

@@ -19709,7 +19754,6 @@ static int do_check(struct bpf_verifier_env *env)
                                continue;
                        }
                }
-               WARN_ON_ONCE(err);
        }

        return 0;

@@ -20838,6 +20882,29 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
                bpf_convert_ctx_access_t convert_ctx_access;
                u8 mode;

+               if (env->insn_aux_data[i + delta].nospec) {
+                       WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state);
+                       struct bpf_insn patch[] = {
+                               BPF_ST_NOSPEC(),
+                               *insn,
+                       };
+
+                       cnt = ARRAY_SIZE(patch);
+                       new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
+                       if (!new_prog)
+                               return -ENOMEM;
+
+                       delta    += cnt - 1;
+                       env->prog = new_prog;
+                       insn      = new_prog->insnsi + i + delta;
+                       /* This can not be easily merged with the
+                        * nospec_result-case, because an insn may require a
+                        * nospec before and after itself. Therefore also do not
+                        * 'continue' here but potentially apply further
+                        * patching to insn. *insn should equal patch[1] now.
+                        */
+               }
+
                if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
                    insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
                    insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||

@@ -20888,6 +20955,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)

                if (type == BPF_WRITE &&
                    env->insn_aux_data[i + delta].nospec_result) {
+                       /* nospec_result is only used to mitigate Spectre v4 and
+                        * to limit verification-time for Spectre v1.
+                        */
                        struct bpf_insn patch[] = {
                                *insn,
                                BPF_ST_NOSPEC(),

diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h
index 863df7c0fdd0..97c727ab81be 100644
--- a/tools/testing/selftests/bpf/progs/bpf_misc.h
+++ b/tools/testing/selftests/bpf/progs/bpf_misc.h

@@ -230,4 +230,8 @@
 #define CAN_USE_LOAD_ACQ_STORE_REL
 #endif

+#if defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)
+#define SPEC_V1
+#endif
+
 #endif

[...]