[PATCH] arm64: Support hard limit of cpu count by nr_cpus

Enable the hard limit of cpu count by nr_cpus on arm64.
The code is borrowed from MIPS.

Reported-by: Shiyuan Hu <redacted>
Signed-off-by: Kefeng Wang <redacted>
---
 arch/arm64/kernel/setup.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 536dce2..597b777 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c

@@ -224,6 +224,21 @@ static void __init request_standard_resources(void)
 
 u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
 
+static void __init prefill_possible_map(void)
+{
+	int i, possible = num_possible_cpus();
+
+	if (possible > nr_cpu_ids)
+		possible = nr_cpu_ids;
+
+	for (i = 0; i < possible; i++)
+		set_cpu_possible(i, true);
+	for (; i < NR_CPUS; i++)
+		set_cpu_possible(i, false);
+
+	nr_cpu_ids = possible;
+}
+
 void __init setup_arch(char **cmdline_p)
 {
 	pr_info("Boot CPU: AArch64 Processor [%08x]\n", read_cpuid_id());

@@ -285,6 +300,7 @@ void __init setup_arch(char **cmdline_p)
 	cpu_read_bootcpu_ops();
 	smp_init_cpus();
 	smp_build_mpidr_hash();
+	prefill_possible_map();
 
 #ifdef CONFIG_VT
 #if defined(CONFIG_VGA_CONSOLE)

-- 
1.7.12.4

On Fri, Aug 05, 2016 at 02:03:55PM +0800, Kefeng Wang wrote:

Shouldn't we just avoid marking those CPUs as possible in smp_cpu_setup,
rather than rewriting things later on?

Also, can you explain exactly what functionality is missing at the moment,
please? I assume it's the nr_cpus= option not working correctly?

Will

On 2016/8/5 16:32, Will Deacon wrote:

On Fri, Aug 05, 2016 at 02:03:55PM +0800, Kefeng Wang wrote:

quoted

Enable the hard limit of cpu count by nr_cpus on arm64.
The code is borrowed from MIPS.

Reported-by: Shiyuan Hu <redacted>
Signed-off-by: Kefeng Wang <redacted>
---
 arch/arm64/kernel/setup.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 536dce2..597b777 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c

@@ -224,6 +224,21 @@ static void __init request_standard_resources(void)
 
 u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
 
+static void __init prefill_possible_map(void)
+{
+	int i, possible = num_possible_cpus();
+
+	if (possible > nr_cpu_ids)
+		possible = nr_cpu_ids;
+
+	for (i = 0; i < possible; i++)
+		set_cpu_possible(i, true);
+	for (; i < NR_CPUS; i++)
+		set_cpu_possible(i, false);
+
+	nr_cpu_ids = possible;
+}

Shouldn't we just avoid marking those CPUs as possible in smp_cpu_setup,
rather than rewriting things later on?

How about this one,

From 0965eebecbc5aea0473df0cb45d32b2fccf72a9a Mon Sep 17 00:00:00 2001

From: Kefeng Wang <redacted>
Date: Fri, 5 Aug 2016 17:44:33 +0800
Subject: [PATCH v2] arm64: Support hard limit of cpu count by nr_cpus

Enable the hard limit of cpu count by set boot options nr_cpus=x
on arm64.

Signed-off-by: Kefeng Wang <redacted>
---
 arch/arm64/kernel/smp.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 76a6d92..0ac3ff5 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c

@@ -496,7 +496,10 @@ static int __init smp_cpu_setup(int cpu)
 	if (cpu_ops[cpu]->cpu_init(cpu))
 		return -ENODEV;

-	set_cpu_possible(cpu, true);
+	if (cpu < nr_cpu_ids)
+		set_cpu_possible(cpu, true);
+	else
+		return -EINVAL;

 	return 0;
 }

-- 
1.7.12.4


> 
> Also, can you explain exactly what functionality is missing at the moment,
> please? I assume it's the nr_cpus= option not working correctly?

Yes. they use same bootargs nr_cpus=1 on both arm64 and x86, but find it can't work in arm64.

> 
> Will
> 
> .
> 

On 05/08/16 10:58, Kefeng Wang wrote:

Or is this even better ?

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 490db85..dbd6a5e 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c

@@ -659,9 +659,9 @@ void __init smp_init_cpus(void)
                 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
                                       acpi_parse_gic_cpu_interface, 0);
  
-       if (cpu_count > NR_CPUS)
+       if (cpu_count > nr_cpu_ids)
                 pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
-                       cpu_count, NR_CPUS);
+                       cpu_count, nr_cpu_ids);
  
         if (!bootcpu_valid) {
                 pr_err("missing boot CPU MPIDR, not enabling secondaries\n");

@@ -675,7 +675,7 @@ void __init smp_init_cpus(void)
          * with entries in cpu_logical_map while initializing the cpus.
          * If the cpu set-up fails, invalidate the cpu_logical_map entry.
          */
-       for (i = 1; i < NR_CPUS; i++) {
+       for (i = 1; i < nr_cpu_ids; i++) {
                 if (cpu_logical_map(i) != INVALID_HWID) {
                         if (smp_cpu_setup(i))
                                 cpu_logical_map(i) = INVALID_HWID;

Cheers
Suzuki

On 2016/8/5 18:19, Suzuki K Poulose wrote:

On 05/08/16 10:58, Kefeng Wang wrote:

quoted

On 2016/8/5 16:32, Will Deacon wrote:

quoted

On Fri, Aug 05, 2016 at 02:03:55PM +0800, Kefeng Wang wrote:

quoted

Enable the hard limit of cpu count by nr_cpus on arm64.
The code is borrowed from MIPS.

[...]

Or is this even better ?

Agree, and let's show a better warning.

  change to 	  pr_warn("%d cores exceeds configured maximum of %d - clipping\n",

Thanks,
Kefeng

Enable the hard limit of cpu count by set boot options nr_cpus=x
on arm64, and show a better warning when cpu number exceeds the limit.

Reported-by: Shiyuan Hu <redacted>
Signed-off-by: Kefeng Wang <redacted>
---

Changed since v1:
- clip cpu number in smp_init_cpus suggested-by Will and Suzuki, and update
  the warning.

 arch/arm64/kernel/smp.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 76a6d92..cbfc31c 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c

@@ -661,9 +661,9 @@ void __init smp_init_cpus(void)
 		acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
 				      acpi_parse_gic_cpu_interface, 0);
 
-	if (cpu_count > NR_CPUS)
-		pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
-			cpu_count, NR_CPUS);
+	if (cpu_count > nr_cpu_ids)
+		pr_warn("%d cores exceeds configured maximum of %d - clipping\n",
+			cpu_count, nr_cpu_ids);
 
 	if (!bootcpu_valid) {
 		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");

@@ -677,7 +677,7 @@ void __init smp_init_cpus(void)
 	 * with entries in cpu_logical_map while initializing the cpus.
 	 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
 	 */
-	for (i = 1; i < NR_CPUS; i++) {
+	for (i = 1; i < nr_cpu_ids; i++) {
 		if (cpu_logical_map(i) != INVALID_HWID) {
 			if (smp_cpu_setup(i))
 				cpu_logical_map(i) = INVALID_HWID;

-- 
1.7.12.4

On 08/08/16 07:05, Kefeng Wang wrote:

I think we should leave the message as it was, and is better than what you have
changed it to.


With that change,

Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>

On 2016/8/8 18:50, Suzuki K Poulose wrote:

On 08/08/16 07:05, Kefeng Wang wrote:

quoted

Enable the hard limit of cpu count by set boot options nr_cpus=x
on arm64, and show a better warning when cpu number exceeds the limit.

Reported-by: Shiyuan Hu <redacted>
Signed-off-by: Kefeng Wang <redacted>
---

Changed since v1:
- clip cpu number in smp_init_cpus suggested-by Will and Suzuki, and update
  the warning.

 arch/arm64/kernel/smp.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 76a6d92..cbfc31c 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c

@@ -661,9 +661,9 @@ void __init smp_init_cpus(void)
         acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
                       acpi_parse_gic_cpu_interface, 0);

-    if (cpu_count > NR_CPUS)
-        pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
-            cpu_count, NR_CPUS);
+    if (cpu_count > nr_cpu_ids)
+        pr_warn("%d cores exceeds configured maximum of %d - clipping\n",
+            cpu_count, nr_cpu_ids);

I think we should leave the message as it was, and is better than what you have
changed it to.

The old message is OF only and the old logic shows message when the first cpu number greater than NR_CPUS.
But now the cpu_count is the total cpu counts(get from DT or ACPI), so I change the message.

With that change,

Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>


.

On 08/08/16 12:28, Kefeng Wang wrote:

On 2016/8/8 18:50, Suzuki K Poulose wrote:

quoted

On 08/08/16 07:05, Kefeng Wang wrote:

quoted

Enable the hard limit of cpu count by set boot options nr_cpus=x
on arm64, and show a better warning when cpu number exceeds the limit.

Reported-by: Shiyuan Hu <redacted>
Signed-off-by: Kefeng Wang <redacted>
---

Changed since v1:
- clip cpu number in smp_init_cpus suggested-by Will and Suzuki, and update
  the warning.

 arch/arm64/kernel/smp.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 76a6d92..cbfc31c 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c

@@ -661,9 +661,9 @@ void __init smp_init_cpus(void)
         acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
                       acpi_parse_gic_cpu_interface, 0);

-    if (cpu_count > NR_CPUS)
-        pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
-            cpu_count, NR_CPUS);
+    if (cpu_count > nr_cpu_ids)
+        pr_warn("%d cores exceeds configured maximum of %d - clipping\n",
+            cpu_count, nr_cpu_ids);

I think we should leave the message as it was, and is better than what you have
changed it to.

The old message is OF only and the old logic shows message when the first cpu number greater than NR_CPUS.

I think you misunderstood "of" (as in plain english) with OF (as in Open Firmware). To avoid
the confusion, how about making it :

"Number of cores (%d) exceeds configured maximum of %d - clipping\n" ?

Certainly a message like :

"5 cores exceeds configured maximum of %d..." doesn't look nice.


Suzuki

But now the cpu_count is the total cpu counts(get from DT or ACPI), so I change the message.

quoted

With that change,

Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>


.

On 2016/8/8 20:52, Suzuki K Poulose wrote:

On 08/08/16 12:28, Kefeng Wang wrote:

quoted

On 2016/8/8 18:50, Suzuki K Poulose wrote:

quoted

On 08/08/16 07:05, Kefeng Wang wrote:

quoted

Enable the hard limit of cpu count by set boot options nr_cpus=x
on arm64, and show a better warning when cpu number exceeds the limit.

Reported-by: Shiyuan Hu <redacted>
Signed-off-by: Kefeng Wang <redacted>
---

Changed since v1:
- clip cpu number in smp_init_cpus suggested-by Will and Suzuki, and update
  the warning.

 arch/arm64/kernel/smp.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 76a6d92..cbfc31c 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c

@@ -661,9 +661,9 @@ void __init smp_init_cpus(void)
         acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
                       acpi_parse_gic_cpu_interface, 0);

-    if (cpu_count > NR_CPUS)
-        pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
-            cpu_count, NR_CPUS);
+    if (cpu_count > nr_cpu_ids)
+        pr_warn("%d cores exceeds configured maximum of %d - clipping\n",
+            cpu_count, nr_cpu_ids);

I think we should leave the message as it was, and is better than what you have
changed it to.

The old message is OF only and the old logic shows message when the first cpu number greater than NR_CPUS.

I think you misunderstood "of" (as in plain english) with OF (as in Open Firmware). To avoid
the confusion, how about making it :

"Number of cores (%d) exceeds configured maximum of %d - clipping\n" ?

Er, I do misunderstood it, my bad, will use this one, thanks :)

Kefeng

Certainly a message like :

"5 cores exceeds configured maximum of %d..." doesn't look nice.


Suzuki

quoted

But now the cpu_count is the total cpu counts(get from DT or ACPI), so I change the message.

quoted

With that change,

Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>


.

.

Enable the hard limit of cpu count by set boot options nr_cpus=x
on arm64, and make a minor change about message when total number
of cpu exceeds the limit.

Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Reported-by: Shiyuan Hu <redacted>
Signed-off-by: Kefeng Wang <redacted>
---

Change since v2:
- change the warning message suggested-by Suzuki, and Suzuki's reviewed-by 
Change since v1:
- clip cpu number in smp_init_cpus suggested-by Will and Suzuki, and update
  the warning.

 arch/arm64/kernel/smp.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 76a6d92..d93d433 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c

@@ -661,9 +661,9 @@ void __init smp_init_cpus(void)
 		acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
 				      acpi_parse_gic_cpu_interface, 0);
 
-	if (cpu_count > NR_CPUS)
-		pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
-			cpu_count, NR_CPUS);
+	if (cpu_count > nr_cpu_ids)
+		pr_warn("Number of cores (%d) exceeds configured maximum of %d - clipping\n",
+			cpu_count, nr_cpu_ids);
 
 	if (!bootcpu_valid) {
 		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");

@@ -677,7 +677,7 @@ void __init smp_init_cpus(void)
 	 * with entries in cpu_logical_map while initializing the cpus.
 	 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
 	 */
-	for (i = 1; i < NR_CPUS; i++) {
+	for (i = 1; i < nr_cpu_ids; i++) {
 		if (cpu_logical_map(i) != INVALID_HWID) {
 			if (smp_cpu_setup(i))
 				cpu_logical_map(i) = INVALID_HWID;

-- 
1.7.12.4