Re: [PATCH] Build in gettext poison feature unconditionally

2 messages, 2 authors, 2016-06-15 · open the first message on its own page

Re: [PATCH] Build in gettext poison feature unconditionally

From: Junio C Hamano <hidden>
Date: 2016-06-15 22:54:32

Jonathan Nieder [off-list ref] writes:
Nguyễn Thái Ngọc Duy wrote:
quoted
The runtime cost should be small. "gcc -O2" inlines _() and
use_gettext_poison(). But even if it does not, performance should not
be impacted as _() calls are usually not on critical path. If some of
them are, we better fix there as gettext() may or may not be cheap
anyway.
That seems reasonable to me.  The increase in code size of a commonly
inlined function and the extra "if" in a common if not
performance-critical codepath is annoying (and I'd prefer to keep
use_gettext_poison() un-inlined), but in any event the cost would go
away once the translation-based implementation of poison lands.
I would say it is not worse than just "annoying"; if the cost will
go away, I'd rather see this conversion postponed and is done as
part of (and preferrably at the end of) the "poison with a
poison-locale" series.
Yes, that would be nice (or perhaps a mode to run most tests in
the current locale and rerun test assertions that use a test_i18n*
helper or C_LOCALE_OUTPUT prerequisite in the C locale).
Yeah, I think that is the right direction to go; I suspect that
"poison with a poison-locale" would have to be ready to allow us to
go there, though.

[PATCH] Support generate poison .mo files for testing

From: Nguyễn Thái Ngọc Duy <hidden>
Date: 2016-06-15 22:54:32

test-poisongen does a similar job to gettext poison feature except
that it does it at build time. Gibberish .mo files are generated for
all supported langauges and put in po/build/poison-locale. Target
"poison-locale" is for this.

User can run the test with these .mo files by setting POISON_LOCALE
while running the test suite. User must also set LANG/LC_* correctly
(and the system is supposed to support that locale).

Signed-off-by: Nguyễn Thái Ngọc Duy <redacted>
---
 On Tue, Aug 21, 2012 at 11:37 PM, Junio C Hamano [off-list ref] wrote:
 > I would say it is not worse than just "annoying"; if the cost will
 > go away, I'd rather see this conversion postponed and is done as
 > part of (and preferrably at the end of) the "poison with a
 > poison-locale" series.

 OK let me redo step one. test-poisongen requires libgettextpo. I'm
 not sure if this library if gnu specific. We may need another flag
 for it instead of NO_GETTEXT. We don't need a fake language code with
 this approach.

 Makefile         |  19 ++++++++
 t/test-lib.sh    |  10 +++-
 test-poisongen.c | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 wrap-for-bin.sh  |   6 ++-
 4 files changed, 171 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 t/test-lib.sh
 create mode 100644 test-poisongen.c
 mode change 100644 => 100755 wrap-for-bin.sh
diff --git a/Makefile b/Makefile
index 6b0c961..6ea2665 100644
--- a/Makefile
+++ b/Makefile
@@ -496,6 +496,9 @@ TEST_PROGRAMS_NEED_X += test-mergesort
 TEST_PROGRAMS_NEED_X += test-mktemp
 TEST_PROGRAMS_NEED_X += test-parse-options
 TEST_PROGRAMS_NEED_X += test-path-utils
+ifndef NO_GETTEXT
+TEST_PROGRAMS_NEED_X += test-poisongen
+endif
 TEST_PROGRAMS_NEED_X += test-revision-walking
 TEST_PROGRAMS_NEED_X += test-run-command
 TEST_PROGRAMS_NEED_X += test-scrap-cache-tree
@@ -2428,6 +2431,19 @@ endif
 po/build/locale/%/LC_MESSAGES/git.mo: po/%.po
 	$(QUIET_MSGFMT)mkdir -p $(dir $@) && $(MSGFMT) -o $@ $<
 
+ifndef NO_GETTEXT
+POISON_MOFILES := $(patsubst po/%.po,po/build/poison-locale/%/LC_MESSAGES/git.mo,$(POFILES))
+
+po/build/poison-locale/%.po: po/%.po test-poisongen$X po/git.pot
+	$(QUIET_MSGFMT)mkdir -p $(dir $@) && \
+	./test-poisongen po/git.pot $@
+
+po/build/poison-locale/%/LC_MESSAGES/git.mo: po/build/poison-locale/%.po
+	$(QUIET_MSGFMT)mkdir -p $(dir $@) && $(MSGFMT) -o $@ $<
+
+poison-locale: $(POISON_MOFILES)
+endif
+
 FIND_SOURCE_FILES = ( git ls-files '*.[hcS]' 2>/dev/null || \
 			$(FIND) . \( -name .git -type d -prune \) \
 				-o \( -name '*.[hcS]' -type f -print \) )
@@ -2564,6 +2580,9 @@ test-svn-fe$X: vcs-svn/lib.a
 
 .PRECIOUS: $(TEST_OBJS)
 
+test-poisongen$X: test-poisongen.o GIT-LDFLAGS $(GITLIBS)
+	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS) -lgettextpo
+
 test-%$X: test-%.o GIT-LDFLAGS $(GITLIBS)
 	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS)
 
diff --git a/t/test-lib.sh b/t/test-lib.sh
old mode 100644
new mode 100755
index bb4f886..d4060e8
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -55,8 +55,10 @@ GIT_BUILD_DIR="$TEST_DIRECTORY"/..
 export PERL_PATH SHELL_PATH
 
 # For repeatability, reset the environment to known value.
-LANG=C
-LC_ALL=C
+if [ -z "$POISON_LOCALE" ]; then
+	LANG=C
+	LC_ALL=C
+fi
 PAGER=cat
 TZ=UTC
 TERM=dumb
@@ -92,6 +94,10 @@ export GIT_MERGE_VERBOSITY GIT_MERGE_AUTOEDIT
 export GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME
 export GIT_COMMITTER_EMAIL GIT_COMMITTER_NAME
 export EDITOR
+if test -n "$POISON_LOCALE"; then
+	GIT_POISON_LOCALE=yes
+	export GIT_POISON_LOCALE
+fi
 
 # Protect ourselves from common misconfiguration to export
 # CDPATH into the environment
diff --git a/test-poisongen.c b/test-poisongen.c
new file mode 100644
index 0000000..5905aa9
--- /dev/null
+++ b/test-poisongen.c
@@ -0,0 +1,139 @@
+#include <gettext-po.h>
+#include "cache.h"
+#include "strbuf.h"
+
+static void xerror(int severity,
+		   po_message_t message,
+		   const char *filename, size_t lineno, size_t column,
+		   int multiline_p, const char *message_text)
+{
+	die("%s:%d:%d %s", filename, lineno, column, message_text);
+}
+
+static void xerror2(int severity,
+		    po_message_t message1,
+		    const char *filename1, size_t lineno1, size_t column1,
+		    int multiline_p1, const char *message_text1,
+		    po_message_t message2,
+		    const char *filename2, size_t lineno2, size_t column2,
+		    int multiline_p2, const char *message_text2)
+{
+	die("%s:%d:%d %s (%s:%d:%d %s)",
+	    filename1, lineno1, column1, message_text1,
+	    filename2, lineno2, column2, message_text2);
+}
+
+static void translate(const char *msg, struct strbuf *buf)
+{
+	const char *end = msg + strlen(msg);
+	const char *text = "* GETTEXT POISON *";
+	int text_len = strlen(text);
+	int t = 0;
+
+	strbuf_reset(buf);
+	/* preserve \n and printf format specifiers because msgfmt
+	   barfs otherwise. */
+	while (msg < end) {
+		/* printf specifiers and shell variables, it's a quite
+		   relax check */
+		if ((*msg == '%' || *msg == '$') && msg+1 < end) {
+			strbuf_addch(buf, *msg++);
+			do
+			       strbuf_addch(buf, *msg);
+			while (msg < end && !isspace(*msg++));
+		} else if (*msg == '\n') {
+			/* we only need to preserve trailing newlines, doing
+			   more does not really harm */
+			strbuf_addch(buf, '\n');
+			msg++;
+		} else {
+			strbuf_addch(buf, text[t]);
+			t = (t + 1) % text_len;
+			msg++;
+		}
+	}
+}
+
+int main(int argc, char **argv)
+{
+	const char *project_header = "Project-Id-Version: ";
+	const char *charset_header = "Content-Type: text/plain; charset=";
+	const char *plural_header = "Plural-Forms: nplurals=";
+	struct po_xerror_handler handler = { xerror, xerror2 };
+	po_file_t src;
+	const char *file = argv[1];
+	po_message_iterator_t iter;
+	po_message_t msg;
+	struct strbuf buf = STRBUF_INIT;
+
+	if (argc != 3)
+		die("usage: test-poisongen <pot file> <poison po file>");
+
+	src = po_file_read(file, &handler);
+	if (src == NULL)
+		die("could not open %s\n", file);
+
+	iter = po_message_iterator(src, "messages");
+	while ((msg = po_next_message(iter)) != NULL) {
+		/* msgid "" is the header, special handling */
+		if (po_message_msgid(msg)[0] == '\0') {
+			const char *p;
+
+			/* no fuzzy, msgfmt does not like it */
+			po_message_set_fuzzy(msg, 0);
+
+			strbuf_reset(&buf);
+			strbuf_addstr(&buf, po_message_msgstr(msg));
+
+			if (!prefixcmp(buf.buf, project_header) &&
+			    !prefixcmp(buf.buf + strlen(project_header),
+				       "PACKAGE VERSION\n")) {
+				strbuf_splice(&buf, strlen(project_header),
+					      strlen("PACKAGE VERSION"),
+					      "git poison",
+					      strlen("git poison"));
+			}
+
+			/* Content-Type: text/plain; charset=UTF-8 */
+			if ((p = strstr(buf.buf, charset_header)) != NULL &&
+			    !prefixcmp(p + strlen(charset_header), "CHARSET\n")) {
+				p += strlen(charset_header);
+				strbuf_splice(&buf, p - buf.buf, strlen("CHARSET"),
+					      "UTF-8", strlen("UTF-8"));
+			}
+
+			/* Plural-Forms: nplurals=2; plural=1 */
+			if ((p = strstr(buf.buf, plural_header)) != NULL &&
+			    !prefixcmp(p + strlen(plural_header), "INTEGER; plural=EXPRESSION")) {
+				int offset;
+				p += strlen(plural_header);
+				offset = p - buf.buf;
+				strbuf_splice(&buf, offset, strlen("INTEGER"), "2", 1);
+				offset += 1;
+				assert(!prefixcmp(buf.buf + offset, "; plural=EXPRESSION"));
+				offset += strlen("; plural=");
+				strbuf_splice(&buf, offset, strlen("EXPRESSION"), "1", 1);
+			}
+
+			po_message_set_msgstr(msg, buf.buf);
+			continue;
+		}
+		if (po_message_msgid_plural(msg)) {
+			int index = 0;
+
+			translate(po_message_msgid(msg), &buf);
+			po_message_set_msgstr_plural(msg, index++, buf.buf);
+
+			while (po_message_msgstr_plural(msg, index)) {
+				translate(po_message_msgid_plural(msg), &buf);
+				po_message_set_msgstr_plural(msg, index++, buf.buf);
+			}
+		} else {
+			translate(po_message_msgid(msg), &buf);
+			po_message_set_msgstr(msg, buf.buf);
+		}
+	}
+
+	po_file_write(src, argv[2], &handler);
+	return 0;
+}
diff --git a/wrap-for-bin.sh b/wrap-for-bin.sh
old mode 100644
new mode 100755
index 53a8dd0..99bc816
--- a/wrap-for-bin.sh
+++ b/wrap-for-bin.sh
@@ -15,7 +15,11 @@ else
 	export GIT_TEMPLATE_DIR
 fi
 GITPERLLIB='@@BUILD_DIR@@/perl/blib/lib'
-GIT_TEXTDOMAINDIR='@@BUILD_DIR@@/po/build/locale'
+if test -n "$GIT_POISON_LOCALE"; then
+	GIT_TEXTDOMAINDIR='@@BUILD_DIR@@/po/build/poison-locale'
+else
+	GIT_TEXTDOMAINDIR='@@BUILD_DIR@@/po/build/locale'
+fi
 PATH='@@BUILD_DIR@@/bin-wrappers:'"$PATH"
 export GIT_EXEC_PATH GITPERLLIB PATH GIT_TEXTDOMAINDIR
 
-- 
1.7.12.rc2
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help