[PATCH 1/2] t/unit-tests: add UTF-8 width tests for CJK chars
From: Jiang Xin <hidden>
Date: 2025-11-14 05:52:55
Subsystem:
kernel build + files below scripts/ (unless maintained elsewhere), the rest · Maintainers:
Nathan Chancellor, Nicolas Schier, Linus Torvalds
This commit adds a new test suite (u-utf8-width.c) to test the UTF-8 width functions in Git, particularly focusing on multi-byte characters from East Asian languages like Chinese, Japanese, and Korean that typically require 2 display columns per character. The test suite includes: - Tests for utf8_strnwidth with Chinese strings - Tests for utf8_strwidth with Chinese strings - Tests for Japanese and Korean characters - Edge case tests with invalid UTF-8 sequences - Proper test function naming following the Clar framework convention Also updated the build configuration in Makefile and meson.build to include the new test suite in the build process. Co-developed-by: Claude <redacted> Signed-off-by: Jiang Xin <redacted> --- Makefile | 1 + t/meson.build | 1 + t/unit-tests/u-utf8-width.c | 85 +++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 t/unit-tests/u-utf8-width.c
diff --git a/Makefile b/Makefile
index 7e0f77e298..2a67546154 100644
--- a/Makefile
+++ b/Makefile@@ -1525,6 +1525,7 @@ CLAR_TEST_SUITES += u-string-list CLAR_TEST_SUITES += u-strvec CLAR_TEST_SUITES += u-trailer CLAR_TEST_SUITES += u-urlmatch-normalization +CLAR_TEST_SUITES += u-utf8-width CLAR_TEST_PROG = $(UNIT_TEST_BIN)/unit-tests$(X) CLAR_TEST_OBJS = $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(CLAR_TEST_SUITES)) CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/clar/clar.o
diff --git a/t/meson.build b/t/meson.build
index a5531df415..dc43d69636 100644
--- a/t/meson.build
+++ b/t/meson.build@@ -24,6 +24,7 @@ clar_test_suites = [ 'unit-tests/u-strvec.c', 'unit-tests/u-trailer.c', 'unit-tests/u-urlmatch-normalization.c', + 'unit-tests/u-utf8-width.c', ] clar_sources = [
diff --git a/t/unit-tests/u-utf8-width.c b/t/unit-tests/u-utf8-width.c
new file mode 100644
index 0000000000..455294ca90
--- /dev/null
+++ b/t/unit-tests/u-utf8-width.c@@ -0,0 +1,85 @@ +#include "unit-test.h" +#include "utf8.h" +#include "strbuf.h" + +/* + * Test utf8_strnwidth with various Chinese strings + * Chinese characters typically have a width of 2 columns when displayed + */ +void test_utf8_width__strnwidth_chinese(void) +{ + const char *ansi_test; + const char *str; + + /* Test basic ASCII - each character should have width 1 */ + cl_assert_equal_i(5, utf8_strnwidth("hello", 5, 0)); + cl_assert_equal_i(5, utf8_strnwidth("hello", 5, 1)); /* skip_ansi = 1 */ + + /* Test simple Chinese characters - each should have width 2 */ + cl_assert_equal_i(4, utf8_strnwidth("你好", 6, 0)); /* "你好" is 6 bytes (3 bytes per char in UTF-8), 4 display columns */ + + /* Test mixed ASCII and Chinese - ASCII = 1 column, Chinese = 2 columns */ + cl_assert_equal_i(6, utf8_strnwidth("hi你好", 8, 0)); /* "h"(1) + "i"(1) + "你"(2) + "好"(2) = 6 */ + + /* Test longer Chinese string */ + cl_assert_equal_i(10, utf8_strnwidth("你好世界!", 15, 0)); /* 5 Chinese chars = 10 display columns */ + + /* Test with skip_ansi = 1 to make sure it works with escape sequences */ + ansi_test = "\033[31m你好\033[0m"; + cl_assert_equal_i(4, utf8_strnwidth(ansi_test, strlen(ansi_test), 1)); /* Skip escape sequences, just count "你好" which should be 4 columns */ + + /* Test individual Chinese character width */ + cl_assert_equal_i(2, utf8_strnwidth("中", 3, 0)); /* Single Chinese char should be 2 columns */ + + /* Test empty string */ + cl_assert_equal_i(0, utf8_strnwidth("", 0, 0)); + + /* Test length limiting */ + str = "你好世界"; + cl_assert_equal_i(2, utf8_strnwidth(str, 3, 0)); /* Only first char "你"(2 columns) within 3 bytes */ + cl_assert_equal_i(4, utf8_strnwidth(str, 6, 0)); /* First two chars "你好"(4 columns) in 6 bytes */ +} + +/* + * Tests for utf8_strwidth (simpler version without length limit) + */ +void test_utf8_width__strwidth_chinese(void) +{ + /* Test basic ASCII */ + cl_assert_equal_i(5, utf8_strwidth("hello")); + + /* Test Chinese characters */ + cl_assert_equal_i(4, utf8_strwidth("你好")); /* 2 Chinese chars = 4 display columns */ + + /* Test mixed ASCII and Chinese */ + cl_assert_equal_i(9, utf8_strwidth("hello世界")); /* 5 ASCII (5 cols) + 2 Chinese (4 cols) = 9 */ + cl_assert_equal_i(7, utf8_strwidth("hi世界!")); /* 2 ASCII (2 cols) + 2 Chinese (4 cols) + 1 ASCII (1 col) = 7 */ +} + +/* + * Additional tests with other East Asian characters + */ +void test_utf8_width__strnwidth_japanese_korean(void) +{ + /* Japanese characters (should also be 2 columns each) */ + cl_assert_equal_i(10, utf8_strnwidth("こんにちは", 15, 0)); /* 5 Japanese chars @ 2 cols each = 10 display columns */ + + /* Korean characters (should also be 2 columns each) */ + cl_assert_equal_i(10, utf8_strnwidth("안녕하세요", 15, 0)); /* 5 Korean chars @ 2 cols each = 10 display columns */ +} + +/* + * Test edge cases with partial UTF-8 sequences + */ +void test_utf8_width__strnwidth_edge_cases(void) +{ + const char *invalid; + unsigned char truncated_bytes[] = {0xe4, 0xbd, 0x00}; /* First 2 bytes of "中" + null */ + + /* Test invalid UTF-8 - should fall back to byte count */ + invalid = "\xff\xfe"; /* Invalid UTF-8 sequence */ + cl_assert_equal_i(2, utf8_strnwidth(invalid, 2, 0)); /* Should return length if invalid UTF-8 */ + + /* Test partial UTF-8 character (truncated) */ + cl_assert_equal_i(2, utf8_strnwidth((const char*)truncated_bytes, 2, 0)); /* Invalid UTF-8, returns byte count */ +}
--
2.52.0.rc2.5.g4c20a63325.dirty