Thread (11 messages) 11 messages, 4 authors, 2025-11-15
STALE229d
Revisions (2)
  1. v1 current
  2. v2 [diff vs current]

[PATCH 1/2] t/unit-tests: add UTF-8 width tests for CJK chars

From: Jiang Xin <hidden>
Date: 2025-11-14 05:52:55
Subsystem: kernel build + files below scripts/ (unless maintained elsewhere), the rest · Maintainers: Nathan Chancellor, Nicolas Schier, Linus Torvalds

This commit adds a new test suite (u-utf8-width.c) to test the UTF-8
width functions in Git, particularly focusing on multi-byte characters
from East Asian languages like Chinese, Japanese, and Korean that
typically require 2 display columns per character.

The test suite includes:
- Tests for utf8_strnwidth with Chinese strings
- Tests for utf8_strwidth with Chinese strings
- Tests for Japanese and Korean characters
- Edge case tests with invalid UTF-8 sequences
- Proper test function naming following the Clar framework convention

Also updated the build configuration in Makefile and meson.build to
include the new test suite in the build process.

Co-developed-by: Claude <redacted>
Signed-off-by: Jiang Xin <redacted>
---
 Makefile                    |  1 +
 t/meson.build               |  1 +
 t/unit-tests/u-utf8-width.c | 85 +++++++++++++++++++++++++++++++++++++
 3 files changed, 87 insertions(+)
 create mode 100644 t/unit-tests/u-utf8-width.c
diff --git a/Makefile b/Makefile
index 7e0f77e298..2a67546154 100644
--- a/Makefile
+++ b/Makefile
@@ -1525,6 +1525,7 @@ CLAR_TEST_SUITES += u-string-list
 CLAR_TEST_SUITES += u-strvec
 CLAR_TEST_SUITES += u-trailer
 CLAR_TEST_SUITES += u-urlmatch-normalization
+CLAR_TEST_SUITES += u-utf8-width
 CLAR_TEST_PROG = $(UNIT_TEST_BIN)/unit-tests$(X)
 CLAR_TEST_OBJS = $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(CLAR_TEST_SUITES))
 CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/clar/clar.o
diff --git a/t/meson.build b/t/meson.build
index a5531df415..dc43d69636 100644
--- a/t/meson.build
+++ b/t/meson.build
@@ -24,6 +24,7 @@ clar_test_suites = [
   'unit-tests/u-strvec.c',
   'unit-tests/u-trailer.c',
   'unit-tests/u-urlmatch-normalization.c',
+  'unit-tests/u-utf8-width.c',
 ]
 
 clar_sources = [
diff --git a/t/unit-tests/u-utf8-width.c b/t/unit-tests/u-utf8-width.c
new file mode 100644
index 0000000000..455294ca90
--- /dev/null
+++ b/t/unit-tests/u-utf8-width.c
@@ -0,0 +1,85 @@
+#include "unit-test.h"
+#include "utf8.h"
+#include "strbuf.h"
+
+/*
+ * Test utf8_strnwidth with various Chinese strings
+ * Chinese characters typically have a width of 2 columns when displayed
+ */
+void test_utf8_width__strnwidth_chinese(void)
+{
+	const char *ansi_test;
+	const char *str;
+
+	/* Test basic ASCII - each character should have width 1 */
+	cl_assert_equal_i(5, utf8_strnwidth("hello", 5, 0));
+	cl_assert_equal_i(5, utf8_strnwidth("hello", 5, 1));  /* skip_ansi = 1 */
+
+	/* Test simple Chinese characters - each should have width 2 */
+	cl_assert_equal_i(4, utf8_strnwidth("你好", 6, 0));  /* "你好" is 6 bytes (3 bytes per char in UTF-8), 4 display columns */
+
+	/* Test mixed ASCII and Chinese - ASCII = 1 column, Chinese = 2 columns */
+	cl_assert_equal_i(6, utf8_strnwidth("hi你好", 8, 0));  /* "h"(1) + "i"(1) + "你"(2) + "好"(2) = 6 */
+
+	/* Test longer Chinese string */
+	cl_assert_equal_i(10, utf8_strnwidth("你好世界!", 15, 0));  /* 5 Chinese chars = 10 display columns */
+
+	/* Test with skip_ansi = 1 to make sure it works with escape sequences */
+	ansi_test = "\033[31m你好\033[0m";
+	cl_assert_equal_i(4, utf8_strnwidth(ansi_test, strlen(ansi_test), 1));  /* Skip escape sequences, just count "你好" which should be 4 columns */
+
+	/* Test individual Chinese character width */
+	cl_assert_equal_i(2, utf8_strnwidth("中", 3, 0));  /* Single Chinese char should be 2 columns */
+
+	/* Test empty string */
+	cl_assert_equal_i(0, utf8_strnwidth("", 0, 0));
+
+	/* Test length limiting */
+	str = "你好世界";
+	cl_assert_equal_i(2, utf8_strnwidth(str, 3, 0));  /* Only first char "你"(2 columns) within 3 bytes */
+	cl_assert_equal_i(4, utf8_strnwidth(str, 6, 0));  /* First two chars "你好"(4 columns) in 6 bytes */
+}
+
+/*
+ * Tests for utf8_strwidth (simpler version without length limit)
+ */
+void test_utf8_width__strwidth_chinese(void)
+{
+	/* Test basic ASCII */
+	cl_assert_equal_i(5, utf8_strwidth("hello"));
+
+	/* Test Chinese characters */
+	cl_assert_equal_i(4, utf8_strwidth("你好"));  /* 2 Chinese chars = 4 display columns */
+
+	/* Test mixed ASCII and Chinese */
+	cl_assert_equal_i(9, utf8_strwidth("hello世界"));  /* 5 ASCII (5 cols) + 2 Chinese (4 cols) = 9 */
+	cl_assert_equal_i(7, utf8_strwidth("hi世界!"));   /* 2 ASCII (2 cols) + 2 Chinese (4 cols) + 1 ASCII (1 col) = 7 */
+}
+
+/*
+ * Additional tests with other East Asian characters
+ */
+void test_utf8_width__strnwidth_japanese_korean(void)
+{
+	/* Japanese characters (should also be 2 columns each) */
+	cl_assert_equal_i(10, utf8_strnwidth("こんにちは", 15, 0));  /* 5 Japanese chars @ 2 cols each = 10 display columns */
+
+	/* Korean characters (should also be 2 columns each) */
+	cl_assert_equal_i(10, utf8_strnwidth("안녕하세요", 15, 0));  /* 5 Korean chars @ 2 cols each = 10 display columns */
+}
+
+/*
+ * Test edge cases with partial UTF-8 sequences
+ */
+void test_utf8_width__strnwidth_edge_cases(void)
+{
+	const char *invalid;
+	unsigned char truncated_bytes[] = {0xe4, 0xbd, 0x00};  /* First 2 bytes of "中" + null */
+
+	/* Test invalid UTF-8 - should fall back to byte count */
+	invalid = "\xff\xfe";  /* Invalid UTF-8 sequence */
+	cl_assert_equal_i(2, utf8_strnwidth(invalid, 2, 0));  /* Should return length if invalid UTF-8 */
+
+	/* Test partial UTF-8 character (truncated) */
+	cl_assert_equal_i(2, utf8_strnwidth((const char*)truncated_bytes, 2, 0));  /* Invalid UTF-8, returns byte count */
+}
-- 
2.52.0.rc2.5.g4c20a63325.dirty
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help