Re: [RFC PATCH v4 4/4] test/vsock: vsock_perf utility
From: Stefano Garzarella <sgarzare@redhat.com>
Date: 2022-12-13 11:19:00
Also in:
lkml, virtualization
On Tue, Dec 06, 2022 at 08:54:28PM +0000, Arseniy Krasnov wrote:
This adds utility to check vsock rx/tx performance. Usage as sender: ./vsock_perf --sender <cid> --port <port> --bytes <bytes to send)
Little typo "<bytes to send>". I mean replace `)` with `>`
quoted hunk ↗ jump to hunk
Usage as receiver: ./vsock_perf --port <port> --rcvlowat <SO_RCVLOWAT> Signed-off-by: Arseniy Krasnov <redacted> --- tools/testing/vsock/Makefile | 3 +- tools/testing/vsock/README | 34 +++ tools/testing/vsock/vsock_perf.c | 434 +++++++++++++++++++++++++++++++ 3 files changed, 470 insertions(+), 1 deletion(-) create mode 100644 tools/testing/vsock/vsock_perf.cdiff --git a/tools/testing/vsock/Makefile b/tools/testing/vsock/Makefile index f8293c6910c9..43a254f0e14d 100644 --- a/tools/testing/vsock/Makefile +++ b/tools/testing/vsock/Makefile@@ -1,8 +1,9 @@# SPDX-License-Identifier: GPL-2.0-only -all: test +all: test vsock_perf test: vsock_test vsock_diag_test vsock_test: vsock_test.o timeout.o control.o util.o vsock_diag_test: vsock_diag_test.o timeout.o control.o util.o +vsock_perf: vsock_perf.o CFLAGS += -g -O2 -Werror -Wall -I. -I../../include -I../../../usr/include -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -D_GNU_SOURCE .PHONY: all test cleandiff --git a/tools/testing/vsock/README b/tools/testing/vsock/README index 4d5045e7d2c3..e6f6735bba05 100644 --- a/tools/testing/vsock/README +++ b/tools/testing/vsock/README@@ -35,3 +35,37 @@ Invoke test binaries in both directions as follows: --control-port=$GUEST_IP \ --control-port=1234 \ --peer-cid=3 + +vsock_perf utility +------------------- +'vsock_perf' is a simple tool to measure vsock performance. It works in +sender/receiver modes: sender connect to peer at the specified port and +starts data transmission to the receiver. After data processing is done, +it prints several metrics(see below). + +Usage: +# run as sender +# connect to CID 2, port 1234, send 1G of data, tx buf size is 1M +./vsock_perf --sender 2 --port 1234 --bytes 1G --buf-size 1M + +Output: +tx performance: A Gb/s + +Output explanation: +A is calculated as "number of bytes to send" / "time in tx loop" + +# run as receiver +# listen port 1234, rx buf size is 1M, socket buf size is 1G, SO_RCVLOWAT is 64K +./vsock_perf --port 1234 --buf-size 1M --vsk-size 1G --rcvlowat 64K + +Output: +rx performance: A Gb/s +total in 'read()': B sec +POLLIN wakeups: C +average in 'read()': D ns + +Output explanation: +A is calculated as "number of received bytes" / "time in rx loop". +B is time, spent in 'read()' system call(excluding 'poll()') +C is number of 'poll()' wake ups with POLLIN bit set. +D is B / C, e.g. average amount of time, spent in single 'read()'.diff --git a/tools/testing/vsock/vsock_perf.c b/tools/testing/vsock/vsock_perf.c new file mode 100644 index 000000000000..248fc3285d5c --- /dev/null +++ b/tools/testing/vsock/vsock_perf.c@@ -0,0 +1,434 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vsock_perf - benchmark utility for vsock. + * + * Copyright (C) 2022 SberDevices. + * + * Author: Arseniy Krasnov <AVKrasnov@sberdevices.ru> + */ +#include <getopt.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> +#include <stdint.h> +#include <poll.h> +#include <sys/socket.h> +#include <linux/vm_sockets.h> + +#define DEFAULT_BUF_SIZE_BYTES (128 * 1024) +#define DEFAULT_TO_SEND_BYTES (64 * 1024) +#define DEFAULT_VSOCK_BUF_BYTES (256 * 1024) +#define DEFAULT_RCVLOWAT_BYTES 1 +#define DEFAULT_PORT 1234 + +#define BYTES_PER_GB (1024 * 1024 * 1024ULL) +#define NSEC_PER_SEC (1000000000ULL) + +static unsigned int port = DEFAULT_PORT; +static unsigned long buf_size_bytes = DEFAULT_BUF_SIZE_BYTES; +static unsigned long vsock_buf_bytes = DEFAULT_VSOCK_BUF_BYTES; + +static inline time_t current_nsec(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_REALTIME, &ts)) { + perror("clock_gettime"); + exit(EXIT_FAILURE); + } + + return (ts.tv_sec * NSEC_PER_SEC) + ts.tv_nsec; +} + +/* From lib/cmdline.c. */ +static unsigned long memparse(const char *ptr) +{ + char *endptr; + + unsigned long long ret = strtoull(ptr, &endptr, 0); + + switch (*endptr) { + case 'E': + case 'e': + ret <<= 10; + case 'P': + case 'p': + ret <<= 10; + case 'T': + case 't': + ret <<= 10; + case 'G': + case 'g': + ret <<= 10; + case 'M': + case 'm': + ret <<= 10; + case 'K': + case 'k': + ret <<= 10; + endptr++; + default: + break; + } + + return ret; +} + +static void vsock_increase_buf_size(int fd) +{ + if (setsockopt(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_MAX_SIZE, + &vsock_buf_bytes, sizeof(vsock_buf_bytes))) { + perror("setsockopt(SO_VM_SOCKETS_BUFFER_MAX_SIZE)"); + exit(EXIT_FAILURE); + } + + if (setsockopt(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_SIZE, + &vsock_buf_bytes, sizeof(vsock_buf_bytes))) { + perror("setsockopt(SO_VM_SOCKETS_BUFFER_SIZE)"); + exit(EXIT_FAILURE); + } +} + +static int vsock_connect(unsigned int cid, unsigned int port) +{ + union { + struct sockaddr sa; + struct sockaddr_vm svm; + } addr = { + .svm = { + .svm_family = AF_VSOCK, + .svm_port = port, + .svm_cid = cid, + }, + }; + int fd; + + fd = socket(AF_VSOCK, SOCK_STREAM, 0); + + if (fd < 0) { + perror("socket"); + return -1; + } + + if (connect(fd, &addr.sa, sizeof(addr.svm)) < 0) { + perror("connect"); + close(fd); + return -1; + } + + return fd; +} + +static float get_gbps(unsigned long bytes, time_t ns_delta) +{ + return ((float)bytes / BYTES_PER_GB) / + ((float)ns_delta / NSEC_PER_SEC); +} + +static void run_receiver(unsigned long rcvlowat_bytes) +{ + unsigned int read_cnt; + time_t rx_begin_ns; + time_t in_read_ns; + size_t total_recv; + int client_fd; + char *data; + int fd; + union { + struct sockaddr sa; + struct sockaddr_vm svm; + } addr = { + .svm = { + .svm_family = AF_VSOCK, + .svm_port = port, + .svm_cid = VMADDR_CID_ANY, + }, + }; + union { + struct sockaddr sa; + struct sockaddr_vm svm; + } clientaddr; + + socklen_t clientaddr_len = sizeof(clientaddr.svm); + + printf("Run as receiver\n"); + printf("Listen port %u\n", port); + printf("RX buffer %lu bytes\n", buf_size_bytes); + printf("vsock buffer %lu bytes\n", vsock_buf_bytes); + printf("SO_RCVLOWAT %lu bytes\n", rcvlowat_bytes); + + fd = socket(AF_VSOCK, SOCK_STREAM, 0); + + if (fd < 0) { + perror("socket"); + exit(EXIT_FAILURE); + } + + if (bind(fd, &addr.sa, sizeof(addr.svm)) < 0) { + perror("bind"); + exit(EXIT_FAILURE); + } + + if (listen(fd, 1) < 0) { + perror("listen"); + exit(EXIT_FAILURE); + } + + client_fd = accept(fd, &clientaddr.sa, &clientaddr_len); + + if (client_fd < 0) { + perror("accept"); + exit(EXIT_FAILURE); + } + + vsock_increase_buf_size(client_fd); + + if (setsockopt(client_fd, SOL_SOCKET, SO_RCVLOWAT, + &rcvlowat_bytes, + sizeof(rcvlowat_bytes))) { + perror("setsockopt(SO_RCVLOWAT)"); + exit(EXIT_FAILURE); + } + + data = malloc(buf_size_bytes); + + if (!data) { + printf("malloc failed\n");
This is an error message, so we can use fprintf(stderr, ...)
+ exit(EXIT_FAILURE);
+ }
+
+ read_cnt = 0;
+ in_read_ns = 0;
+ total_recv = 0;
+ rx_begin_ns = current_nsec();
+
+ while (1) {
+ struct pollfd fds = { 0 };
+
+ fds.fd = client_fd;
+ fds.events = POLLIN | POLLERR |
+ POLLHUP | POLLRDHUP;
+
+ if (poll(&fds, 1, -1) < 0) {
+ perror("poll");
+ exit(EXIT_FAILURE);
+ }
+
+ if (fds.revents & POLLERR) {
+ printf("'poll()' error\n");Ditto.
+ exit(EXIT_FAILURE);
+ }
+
+ if (fds.revents & POLLIN) {
+ ssize_t bytes_read;
+ time_t t;
+
+ t = current_nsec();
+ bytes_read = read(fds.fd, data, buf_size_bytes);
+ in_read_ns += (current_nsec() - t);
+ read_cnt++;
+
+ if (!bytes_read)
+ break;
+
+ if (bytes_read < 0) {
+ perror("read");
+ exit(EXIT_FAILURE);
+ }
+
+ total_recv += bytes_read;
+ }
+
+ if (fds.revents & (POLLHUP | POLLRDHUP))
+ break;
+ }
+
+ printf("rx performance: %f Gb/s\n",
+ get_gbps(total_recv, current_nsec() - rx_begin_ns));
+ printf("total time in 'read()': %f sec\n", (float)in_read_ns / NSEC_PER_SEC);
+ printf("POLLIN wakeups: %i\n", read_cnt);
+ printf("average time in 'read()': %f ns\n", (float)in_read_ns / read_cnt);
+
+ free(data);
+ close(client_fd);
+ close(fd);
+}
+
+static void run_sender(int peer_cid, unsigned long to_send_bytes)
+{
+ time_t tx_begin_ns;
+ size_t total_send;
+ void *data;
+ int fd;
+
+ printf("Run as sender\n");
+ printf("Connect to %i:%u\n", peer_cid, port);
+ printf("Send %lu bytes\n", to_send_bytes);
+ printf("TX buffer %lu bytes\n", buf_size_bytes);
+
+ fd = vsock_connect(peer_cid, port);
+
+ if (fd < 0)
+ exit(EXIT_FAILURE);
+
+ data = malloc(buf_size_bytes);
+
+ if (!data) {
+ printf("malloc failed\n");Ditto.
+ exit(EXIT_FAILURE);
+ }
+
+ memset(data, 0, buf_size_bytes);
+ total_send = 0;
+ tx_begin_ns = current_nsec();
+
+ while (total_send < to_send_bytes) {
+ ssize_t sent;
+
+ sent = write(fd, data, buf_size_bytes);
+
+ if (sent <= 0) {
+ perror("write");
+ exit(EXIT_FAILURE);
+ }
+
+ total_send += sent;
+ }
I don't know if it would be useful, but maybe we can also print the
total number of bytes and the time spent (the same in the receiver as
well).
Here I would add
tx_end_ns = current_nsec();
Then use it later.
+
+ printf("tx performance: %f Gb/s\n",
+ get_gbps(total_send, current_nsec() - tx_begin_ns));
+Did you compared the performance with iperf-vsock and uperf? I see an order of magnitude difference but I did not have time to check all the parameters to make a fair comparison. It would be helpful if you can compare and report it. Thanks, Stefano
+ close(fd);
+ free(data);
+}
+
+static const char optstring[] = "";
+static const struct option longopts[] = {
+ {
+ .name = "help",
+ .has_arg = no_argument,
+ .val = 'H',
+ },
+ {
+ .name = "sender",
+ .has_arg = required_argument,
+ .val = 'S',
+ },
+ {
+ .name = "port",
+ .has_arg = required_argument,
+ .val = 'P',
+ },
+ {
+ .name = "bytes",
+ .has_arg = required_argument,
+ .val = 'M',
+ },
+ {
+ .name = "buf-size",
+ .has_arg = required_argument,
+ .val = 'B',
+ },
+ {
+ .name = "vsk-size",
+ .has_arg = required_argument,
+ .val = 'V',
+ },
+ {
+ .name = "rcvlowat",
+ .has_arg = required_argument,
+ .val = 'R',
+ },
+ {},
+};
+
+static void usage(void)
+{
+ printf("Usage: ./vsock_perf [--help] [options]\n"
+ "\n"
+ "This is benchmarking utility, to test vsock performance.\n"
+ "It runs in two modes: sender or receiver. In sender mode, it\n"
+ "connects to the specified CID and starts data transmission.\n"
+ "\n"
+ "Options:\n"
+ " --help This message\n"
+ " --sender <cid> Sender mode (receiver default)\n"
+ " <cid> of the receiver to connect to\n"
+ " --port <port> Port (default %d)\n"
+ " --bytes <bytes>KMG Bytes to send (default %d)\n"
+ " --buf-size <bytes>KMG Data buffer size (default %d). In sender mode\n"
+ " it is the buffer size, passed to 'write()'. In\n"
+ " receiver mode it is the buffer size passed to 'read()'.\n"
+ " --vsk-size <bytes>KMG Socket buffer size (default %d)\n"
+ " --rcvlowat <bytes>KMG SO_RCVLOWAT value (default %d)\n"
+ "\n", DEFAULT_PORT, DEFAULT_TO_SEND_BYTES,
+ DEFAULT_BUF_SIZE_BYTES, DEFAULT_VSOCK_BUF_BYTES,
+ DEFAULT_RCVLOWAT_BYTES);
+ exit(EXIT_FAILURE);
+}
+
+static long strtolx(const char *arg)
+{
+ long value;
+ char *end;
+
+ value = strtol(arg, &end, 10);
+
+ if (end != arg + strlen(arg))
+ usage();
+
+ return value;
+}
+
+int main(int argc, char **argv)
+{
+ unsigned long to_send_bytes = DEFAULT_TO_SEND_BYTES;
+ unsigned long rcvlowat_bytes = DEFAULT_RCVLOWAT_BYTES;
+ int peer_cid = -1;
+ bool sender = false;
+
+ while (1) {
+ int opt = getopt_long(argc, argv, optstring, longopts, NULL);
+
+ if (opt == -1)
+ break;
+
+ switch (opt) {
+ case 'V': /* Peer buffer size. */
+ vsock_buf_bytes = memparse(optarg);
+ break;
+ case 'R': /* SO_RCVLOWAT value. */
+ rcvlowat_bytes = memparse(optarg);
+ break;
+ case 'P': /* Port to connect to. */
+ port = strtolx(optarg);
+ break;
+ case 'M': /* Bytes to send. */
+ to_send_bytes = memparse(optarg);
+ break;
+ case 'B': /* Size of rx/tx buffer. */
+ buf_size_bytes = memparse(optarg);
+ break;
+ case 'S': /* Sender mode. CID to connect to. */
+ peer_cid = strtolx(optarg);
+ sender = true;
+ break;
+ case 'H': /* Help. */
+ usage();
+ break;
+ default:
+ usage();
+ }
+ }
+
+ if (!sender)
+ run_receiver(rcvlowat_bytes);
+ else
+ run_sender(peer_cid, to_send_bytes);
+
+ return 0;
+}
--
2.25.1