On 10/29/2018 11:11 PM, John Fastabend wrote:

On 10/29/2018 02:19 PM, Shannon Nelson wrote:

quoted

This is an example of using XDP to redirect the processing of
particular vlan packets to specific CPUs.  This is in response
to comments received on a kernel patch put forth previously
to do something similar using RPS.
     https://www.spinics.net/lists/netdev/msg528210.html
     [PATCH net-next] net: enable RPS on vlan devices

This XDP application watches for inbound vlan-tagged packets
and redirects those packets to be processed on a specific CPU
as configured in a BPF map.  The BPF map can be modified by
this user program, which can also load and unload the kernel
XDP code.

One example use is for supporting VMs where we can't control the
OS being used: we'd like to separate the VM CPU processing from
the host's CPUs as a way to help mitigate L1TF related issues.
When running the VM's traffic on a vlan we can stick the host's
Rx processing on one set of CPUs separate from the VM's CPUs.

This example currently uses a vlan key and cpu value in the
BPF map, so only can do one CPU per vlan.  This could easily
be modified to use a bitpattern of CPUs rather than a CPU id
to allow multiple CPUs per vlan.

Great, so does this solve your use case then? At least on drivers
with XDP support?

quoted

Signed-off-by: Shannon Nelson <redacted>
---

Some really small and trivial nits below.

Acked-by: John Fastabend <john.fastabend@gmail.com>

[...]

quoted

+	if (install) {
+

new line probably not needed. 

quoted

+		/* check to see if already installed */
+		errno = 0;
+		access(pin_prog_name, R_OK);
+		if (errno != ENOENT) {
+			fprintf(stderr, "ERR: %s is already installed\n", argv[0]);
+			return -1;
+		}
+
+		/* load the XDP program and maps with the convenient library */
+		if (load_bpf_file(filename)) {
+			fprintf(stderr, "ERR: load_bpf_file(%s): \n%s",
+				filename, bpf_log_buf);
+			return -1;
+		}
+		if (!prog_fd[0]) {
+			fprintf(stderr, "ERR: load_bpf_file(%s): %d %s\n",
+				filename, errno, strerror(errno));
+			return -1;
+		}
+
+		/* pin the XDP program and maps */
+		if (bpf_obj_pin(prog_fd[0], pin_prog_name) < 0) {
+			fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
+				pin_prog_name, errno, strerror(errno));
+			if (errno == 2)
+				fprintf(stderr, "     (is the BPF fs mounted on /sys/fs/bpf?)\n");
+			return -1;
+		}
+		if (bpf_obj_pin(map_fd[0], pin_vlanmap_name) < 0) {
+			fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
+				pin_vlanmap_name, errno, strerror(errno));
+			return -1;
+		}
+		if (bpf_obj_pin(map_fd[2], pin_countermap_name) < 0) {
+			fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
+				pin_countermap_name, errno, strerror(errno));
+			return -1;
+		}
+
+		/* prep the vlan map with "not used" values */
+		c64 = UNDEF_CPU;
+		for (v64 = 0; v64 < 4096; v64++) {

maybe #define MAX_VLANS 4096 just to avoid constants.

quoted

+			if (bpf_map_update_elem(map_fd[0], &v64, &c64, 0)) {
+				fprintf(stderr, "ERR: preping vlan map failed on v=%llu: %d %s\n",
+					v64, errno, strerror(errno));
+				return -1;
+			}
+		}
+
+		/* prep the cpumap with queue sizes */
+		c64 = 128+64;  /* see note in xdp_redirect_cpu_user.c */
+		for (v64 = 0; v64 < MAX_CPUS; v64++) {
+			if (bpf_map_update_elem(map_fd[1], &v64, &c64, 0)) {
+				if (errno == ENODEV) {
+					/* Save the last CPU number attempted
+					 * into the counters map
+					 */
+					c64 = CPU_COUNT;
+					ret = bpf_map_update_elem(map_fd[2], &c64, &v64, 0);
+					break;
+				}
+
+				fprintf(stderr, "ERR: preping cpu map failed on v=%llu: %d %s\n",
+					v64, errno, strerror(errno));
+				return -1;
+			}
+		}
+
+		/* wire the XDP program to the device */
+		if (bpf_set_link_xdp_fd(ifindex, prog_fd[0], 0) < 0) {
+			fprintf(stderr, "ERR: bpf_set_link_xdp_fd(): %d %s\n",
+				errno, strerror(errno));
+			return -1;
+		}
+
+		return 0;
+	}
+
+	if (remove) {
+

Ditto

quoted

+		/* unlink the program from the device */
+		if (bpf_set_link_xdp_fd(ifindex, -1, 0) < 0)
+			fprintf(stderr, "ERR: bpf_set_link_xdp_fd(): %d %s\n",
+				errno, strerror(errno));
+
+		/* unlink pinned files */
+		if (unlink(pin_prog_name))
+			fprintf(stderr, "ERR: unlink(%s): %d %s\n",
+				pin_prog_name, errno, strerror(errno));
+		if (unlink(pin_vlanmap_name))
+			fprintf(stderr, "ERR: unlink(%s): %d %s\n",
+				pin_vlanmap_name, errno, strerror(errno));
+		if (unlink(pin_countermap_name))
+			fprintf(stderr, "ERR: unlink(%s): %d %s\n",
+				pin_countermap_name, errno, strerror(errno));
+
+		return 0;
+	}
+
+	if (vlan == 0) {
+		fprintf(stderr, "ERR: required option --vlan missing\n");
+		goto error;
+	}
+
+	if (cpu == MAX_CPUS && vlan > 0) {
+		fprintf(stderr, "ERR: required option --cpu missing\n");
+		goto error;
+	}
+
+	vfd = bpf_obj_get(pin_vlanmap_name);
+	if (vfd < 0) {
+		fprintf(stderr, "ERR: can't find pinned map %s: %d %s\n",
+			pin_vlanmap_name, errno, strerror(errno));
+		if (errno == ENOENT)
+			fprintf(stderr, "   (has %s been installed yet?)\n", argv[0]);
+		return -1;
+	}
+
+	/* decode the requested action */
+	if (vlan > 0) {
+		/* check cpu against the max value found */
+		cfd = bpf_obj_get(pin_countermap_name);
+		if (cfd < 0) {
+			fprintf(stderr, "ERR: can't find pinned map %s: %d %s\n",
+				pin_countermap_name, errno, strerror(errno));
+			return -1;
+		}
+		c64 = CPU_COUNT;
+		ret = bpf_map_lookup_elem(cfd, &c64, &v64);
+		if (cpu >= v64) {

Need to check ret code?

Yeah, I expect this would have another respin, thanks!

quoted

+			fprintf(stderr, "ERR: cpu %d greater than max %llu\n", cpu, v64);
+			return -1;
+		}
+
+		/* Note that the value and key pointers really do need to be
+		 * pointers to 64-bit values, else things get a bit muddled.
+		 */
+		v64 = vlan;
+		c64 = cpu;
+		ret = bpf_map_update_elem(vfd, &v64, &c64, 0);
+		if (ret) {
+			fprintf(stderr, "Adding vlan %d CPU %d failed: %d %s\n",
+				vlan, cpu, errno, strerror(errno));
+			return -1;
+		}
+
+	} else {
+		v64 = -vlan;
+		c64 = UNDEF_CPU;
+
+		/* We can't actually delete from a TYPE_ARRAY map, so we
+		 * simply set it to an undefined value.
+		 */
+		ret = bpf_map_update_elem(vfd, &v64, &c64, 0);
+		if (ret) {
+			fprintf(stderr, "Delete of vlan %llu failed: %d %s\n",
+				v64, errno, strerror(errno));
+			return -1;
+		}
+	}
+
+	return 0;
+
+error:
+	usage(argv);
+	return -1;
+}