[RFC PATCH 1/2] tick/nohz: Add a policy dependency for full dynticks

@@ -111,9 +111,10 @@ enum tick_dep_bits {
 	TICK_DEP_BIT_SCHED		= 2,
 	TICK_DEP_BIT_CLOCK_UNSTABLE	= 3,
 	TICK_DEP_BIT_RCU		= 4,
-	TICK_DEP_BIT_RCU_EXP		= 5
+	TICK_DEP_BIT_RCU_EXP		= 5,
+	TICK_DEP_BIT_POLICY		= 6
 };
-#define TICK_DEP_BIT_MAX TICK_DEP_BIT_RCU_EXP
+#define TICK_DEP_BIT_MAX TICK_DEP_BIT_POLICY
 
 #define TICK_DEP_MASK_NONE		0
 #define TICK_DEP_MASK_POSIX_TIMER	(1 << TICK_DEP_BIT_POSIX_TIMER)

@@ -122,6 +123,7 @@ enum tick_dep_bits {
 #define TICK_DEP_MASK_CLOCK_UNSTABLE	(1 << TICK_DEP_BIT_CLOCK_UNSTABLE)
 #define TICK_DEP_MASK_RCU		(1 << TICK_DEP_BIT_RCU)
 #define TICK_DEP_MASK_RCU_EXP		(1 << TICK_DEP_BIT_RCU_EXP)
+#define TICK_DEP_MASK_POLICY		(1 << TICK_DEP_BIT_POLICY)
 
 #ifdef CONFIG_NO_HZ_COMMON
 extern bool tick_nohz_enabled;

@@ -205,6 +207,8 @@ extern void tick_nohz_dep_set_signal(struct task_struct *tsk,
 extern void tick_nohz_dep_clear_signal(struct signal_struct *signal,
 				       enum tick_dep_bits bit);
 extern bool tick_nohz_cpu_hotpluggable(unsigned int cpu);
+extern void tick_nohz_full_get_allowed(struct cpumask *mask);
+extern int tick_nohz_full_set_allowed(const struct cpumask *mask);
 
 /*
  * The below are tick_nohz_[set,clear]_dep() wrappers that optimize off-cases

@@ -435,7 +435,8 @@ TRACE_EVENT(itimer_expire,
 		tick_dep_name(SCHED)			\
 		tick_dep_name(CLOCK_UNSTABLE)		\
 		tick_dep_name(RCU)			\
-		tick_dep_name_end(RCU_EXP)
+		tick_dep_name(RCU_EXP)			\
+		tick_dep_name_end(POLICY)
 
 #undef tick_dep_name
 #undef tick_dep_mask_name

@@ -27,6 +27,7 @@
 #include <linux/posix-timers.h>
 #include <linux/context_tracking.h>
 #include <linux/mm.h>
+#include <linux/mutex.h>
 
 #include <asm/irq_regs.h>
 

@@ -337,6 +338,7 @@ EXPORT_SYMBOL_GPL(tick_nohz_full_mask);
 bool tick_nohz_full_running;
 EXPORT_SYMBOL_GPL(tick_nohz_full_running);
 static atomic_t tick_dep_mask;
+static DEFINE_MUTEX(tick_nohz_full_policy_mutex);
 
 static bool check_tick_dependency(atomic_t *dep)
 {

@@ -375,6 +377,11 @@ static bool check_tick_dependency(atomic_t *dep)
 		return true;
 	}
 
+	if (val & TICK_DEP_MASK_POLICY) {
+		trace_tick_stop(0, TICK_DEP_MASK_POLICY);
+		return true;
+	}
+
 	return false;
 }
 

@@ -556,6 +563,57 @@ void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
 }
 EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
 
+/* Read policy, not whether a CPU currently has its tick stopped. */
+void tick_nohz_full_get_allowed(struct cpumask *mask)
+{
+	int cpu;
+
+	cpumask_clear(mask);
+	if (!tick_nohz_full_enabled())
+		return;
+
+	/* Exclude the reset of tick_cpu_sched during CPU teardown. */
+	cpus_read_lock();
+	mutex_lock(&tick_nohz_full_policy_mutex);
+	for_each_cpu(cpu, tick_nohz_full_mask) {
+		struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
+
+		if (!(atomic_read(&ts->tick_dep_mask) & TICK_DEP_MASK_POLICY))
+			cpumask_set_cpu(cpu, mask);
+	}
+	mutex_unlock(&tick_nohz_full_policy_mutex);
+	cpus_read_unlock();
+}
+
+int tick_nohz_full_set_allowed(const struct cpumask *mask)
+{
+	int cpu;
+
+	if (!tick_nohz_full_enabled())
+		return -EOPNOTSUPP;
+	if (!cpumask_subset(mask, tick_nohz_full_mask))
+		return -EINVAL;
+
+	cpus_read_lock();
+	mutex_lock(&tick_nohz_full_policy_mutex);
+	for_each_cpu(cpu, tick_nohz_full_mask) {
+		if (cpumask_test_cpu(cpu, mask)) {
+			tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_POLICY);
+		} else if (cpu_online(cpu)) {
+			/* The existing kick requests an asynchronous restart. */
+			tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_POLICY);
+		} else {
+			struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
+
+			/* Record policy without queuing irq_work on an offline CPU. */
+			atomic_or(TICK_DEP_MASK_POLICY, &ts->tick_dep_mask);
+		}
+	}
+	mutex_unlock(&tick_nohz_full_policy_mutex);
+	cpus_read_unlock();
+	return 0;
+}
+
 /*
  * Set a per-task tick dependency. RCU needs this. Also posix CPU timers
  * in order to elapse per task timers.

@@ -1528,6 +1586,7 @@ void tick_sched_timer_dying(int cpu)
 {
 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 	unsigned long idle_calls, idle_sleeps;
+	int policy;
 
 	/* This must happen before hrtimers are migrated! */
 	if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))

@@ -1535,7 +1594,10 @@ void tick_sched_timer_dying(int cpu)
 
 	idle_calls = ts->idle_calls;
 	idle_sleeps = ts->idle_sleeps;
+	/* Preserve only the policy dependency across CPU offline/online. */
+	policy = atomic_read(&ts->tick_dep_mask) & TICK_DEP_MASK_POLICY;
 	memset(ts, 0, sizeof(*ts));
+	atomic_set(&ts->tick_dep_mask, policy);
 	ts->idle_calls = idle_calls;
 	ts->idle_sleeps = idle_sleeps;
 }