Inter-revision diff: patch 2

Comparing v17 (message) to v21 (message)

--- v17
+++ v21
@@ -2,49 +2,80 @@
 
 This is a modified version of an earlier patch by Andi Kleen.
 
-We expect architectures to describe the performance monitoring events
-for each CPU in a corresponding JSON file, which look like:
-
-	[
-	{
-	"EventCode": "0x00",
-	"UMask": "0x01",
-	"EventName": "INST_RETIRED.ANY",
-	"BriefDescription": "Instructions retired from execution.",
-	"PublicDescription": "Instructions retired from execution.",
-	"Counter": "Fixed counter 1",
-	"CounterHTOff": "Fixed counter 1",
-	"SampleAfterValue": "2000003",
-	"SampleAfterValue": "2000003",
-	"MSRIndex": "0",
-	"MSRValue": "0",
-	"TakenAlone": "0",
-	"CounterMask": "0",
-	"Invert": "0",
-	"AnyThread": "0",
-	"EdgeDetect": "0",
-	"PEBS": "0",
-	"PRECISE_STORE": "0",
-	"Errata": "null",
-	"Offcore": "0"
-	}
-	]
-
-We also expect the architectures to provide a mapping between individual
-CPUs to their JSON files. Eg:
-
-	GenuineIntel-6-1E,V1,/NHM-EP/NehalemEP_core_V1.json,core
+We expect architectures to create JSON files describing the performance
+monitoring (PMU) events that each CPU model/family of the architecture
+supports.
+
+Following is an example of the JSON file entry for an x86 event:
+
+    	[
+    	...
+    	{
+    	"EventCode": "0x00",
+    	"UMask": "0x01",
+    	"EventName": "INST_RETIRED.ANY",
+    	"BriefDescription": "Instructions retired from execution.",
+    	"PublicDescription": "Instructions retired from execution.",
+    	"Counter": "Fixed counter 1",
+    	"CounterHTOff": "Fixed counter 1",
+    	"SampleAfterValue": "2000003",
+    	"SampleAfterValue": "2000003",
+    	"MSRIndex": "0",
+    	"MSRValue": "0",
+    	"TakenAlone": "0",
+    	"CounterMask": "0",
+    	"Invert": "0",
+    	"AnyThread": "0",
+    	"EdgeDetect": "0",
+    	"PEBS": "0",
+    	"PRECISE_STORE": "0",
+    	"Errata": "null",
+    	"Offcore": "0"
+    	},
+    	...
+
+    	]
+
+All the PMU events supported by a CPU model/family must be grouped into
+"topics" such as "Piplelining", "Floating-point", "Virtual-memory" etc.
+
+All events belonging to a topic must be placed in a separate JSON file
+(eg: "Pipeling.json") and all the topic JSON files for a CPU model must
+be in a separate directory.
+
+	Eg: for the CPU model "Silvermont_core":
+
+    	$ ls tools/perf/pmu-events/arch/x86/Silvermont_core
+    	Floating-point.json
+    	Memory.json
+    	Other.json
+    	Pipelining.json
+    	Virtualmemory.json
+
+Finally, to allow multiple CPU models to share a single set of JSON files,
+architectures must provide a mapping between a model and its set of events:
+
+    	$ grep Silvermont tools/perf/pmu-events/arch/x86/mapfile.csv
+    	GenuineIntel-6-4D,V13,Silvermont_core,core
+    	GenuineIntel-6-4C,V13,Silvermont_core,core
 
 which maps each CPU, identified by [vendor, family, model, version, type]
-to a JSON file.
-
-Given these files, the program, jevents::
-	- locates all JSON files for the architecture,
-	- parses each JSON file and generates a C-style "PMU-events table"
-	  (pmu-events.c)
+to a directory of JSON files. Thus two (or more) CPU models support the
+set of PMU events listed in the directory.
+
+    	tools/perf/pmu-events/arch/x86/Silvermont_core/
+
+Given this organization of files, the program, jevents:
+
+	- locates all JSON files for each CPU-model of the architecture,
+
+	- parses all JSON files for the CPU-model and generates a C-style
+	  "PMU-events table" (pmu-events.c) for the model
+
 	- locates a mapfile for the architecture
-	- builds a global table, mapping each model of CPU to the
-	  corresponding PMU-events table.
+
+	- builds a global table, mapping each model of CPU to the corresponding
+	  PMU-events table.
 
 The 'pmu-events.c' is generated when building perf and added to libperf.a.
 The global table pmu_events_map[] table in this pmu-events.c will be used
@@ -68,10 +99,10 @@
 files as BSD licenced too. As part of perf they become GPLv2.
 
 Signed-off-by: Andi Kleen <ak@linux.intel.com>
+Signed-off-by: Jiri Olsa <jolsa@redhat.com>
 Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
-Acked-by: Jiri Olsa <jolsa@redhat.com>
+Acked-by: Ingo Molnar <mingo@kernel.org>
 ---
-
 v2: Address review feedback. Rename option to --event-files
 v3: Add JSON example
 v4: Update manpages.
@@ -109,36 +140,50 @@
 	  to recent perf/core.
 
 v16:	- Rebase to upstream; fix conflicts in tools/perf/Makefile.perf
+
+v18:	- Rebase to upstream; fix conflicts in tools/perf/Makefile.perf
+
+v20: 	- Rebase to upstream; rename a local variable to 'ldirname' to avoid
+	  collision with the dirname().
+
+v21:	- Breakup the large JSON files into separate topics like
+	  Pipelining.json, Cache.json etc (by Jiri Olsa).
+	- Ensure BriefDescription field is non-null before adding extra
+	  description for PEBS events.
 ---
- tools/perf/Makefile.perf           |  26 +-
+ tools/perf/Makefile.perf           |  28 +-
  tools/perf/pmu-events/Build        |  11 +
- tools/perf/pmu-events/jevents.c    | 686 +++++++++++++++++++++++++++++++++++++
+ tools/perf/pmu-events/jevents.c    | 766 +++++++++++++++++++++++++++++++++++++
  tools/perf/pmu-events/jevents.h    |  17 +
- tools/perf/pmu-events/json.h       |   3 +
- tools/perf/pmu-events/pmu-events.h |  35 ++
- 6 files changed, 774 insertions(+), 4 deletions(-)
+ tools/perf/pmu-events/json.h       |   6 +
+ tools/perf/pmu-events/pmu-events.h |  36 ++
+ 6 files changed, 860 insertions(+), 4 deletions(-)
  create mode 100644 tools/perf/pmu-events/Build
  create mode 100644 tools/perf/pmu-events/jevents.c
  create mode 100644 tools/perf/pmu-events/jevents.h
  create mode 100644 tools/perf/pmu-events/pmu-events.h
 
 diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf
-index 6c5c699..34b12a1 100644
+index 828cfd7..0abebcb 100644
 --- a/tools/perf/Makefile.perf
 +++ b/tools/perf/Makefile.perf
-@@ -297,14 +297,29 @@ strip: $(PROGRAMS) $(OUTPUT)perf
- PERF_IN := $(OUTPUT)perf-in.o
+@@ -347,6 +347,14 @@ PERF_IN := $(OUTPUT)perf-in.o
+ export srctree OUTPUT RM CC LD AR CFLAGS V BISON FLEX AWK
+ include $(srctree)/tools/build/Makefile.include
  
- export srctree OUTPUT RM CC LD AR CFLAGS V BISON FLEX AWK
 +JEVENTS       := $(OUTPUT)pmu-events/jevents
 +JEVENTS_IN    := $(OUTPUT)pmu-events/jevents-in.o
 +PMU_EVENTS_IN := $(OUTPUT)pmu-events/pmu-events-in.o
 +
 +export JEVENTS
 +
- build := -f $(srctree)/tools/build/Makefile.build dir=. obj
- 
- $(PERF_IN): $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)common-cmds.h FORCE
++build := -f $(srctree)/tools/build/Makefile.build dir=. obj
++
+ $(PERF_IN): prepare FORCE
+ 	@(test -f ../../include/uapi/linux/perf_event.h && ( \
+         (diff -B ../include/uapi/linux/perf_event.h ../../include/uapi/linux/perf_event.h >/dev/null) \
+@@ -434,9 +442,18 @@ $(PERF_IN): prepare FORCE
+ 	|| echo "Warning: tools/include/linux/coresight-pmu.h differs from kernel" >&2 )) || true
  	$(Q)$(MAKE) $(build)=perf
  
 -$(OUTPUT)perf: $(PERFLIBS) $(PERF_IN) $(LIBTRACEEVENT_DYNAMIC_LIST)
@@ -156,9 +201,9 @@
 -		$(PERF_IN) $(LIBS) -o $@
 +		$(PERF_IN) $(PMU_EVENTS_IN) $(LIBS) -o $@
  
- $(GTK_IN): FORCE
+ $(GTK_IN): fixdep FORCE
  	$(Q)$(MAKE) $(build)=gtk
-@@ -333,6 +348,8 @@ perf.spec $(SCRIPTS) \
+@@ -465,6 +482,8 @@ perf.spec $(SCRIPTS) \
  ifneq ($(OUTPUT),)
  %.o: $(OUTPUT)%.o
  	@echo "    # Redirected target $@ => $(OUTPUT)$@"
@@ -167,15 +212,16 @@
  util/%.o: $(OUTPUT)util/%.o
  	@echo "    # Redirected target $@ => $(OUTPUT)$@"
  bench/%.o: $(OUTPUT)bench/%.o
-@@ -571,9 +588,10 @@ clean: $(LIBTRACEEVENT)-clean $(LIBAPI)-clean config-clean
+@@ -720,10 +739,11 @@ clean:: $(LIBTRACEEVENT)-clean $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clea
  	$(call QUIET_CLEAN, core-objs)  $(RM) $(LIB_FILE) $(OUTPUT)perf-archive $(OUTPUT)perf-with-kcore $(LANG_BINDINGS)
- 	$(Q)find . -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete
+ 	$(Q)find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete
  	$(Q)$(RM) $(OUTPUT).config-detected
 -	$(call QUIET_CLEAN, core-progs) $(RM) $(ALL_PROGRAMS) perf perf-read-vdso32 perf-read-vdsox32
 +	$(call QUIET_CLEAN, core-progs) $(RM) $(ALL_PROGRAMS) perf perf-read-vdso32 perf-read-vdsox32 $(OUTPUT)pmu-events/jevents $(srctree)/tools/perf/pmu-events/pmu-events.c
  	$(call QUIET_CLEAN, core-gen)   $(RM)  *.spec *.pyc *.pyo */*.pyc */*.pyo $(OUTPUT)common-cmds.h TAGS tags cscope* $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)FEATURE-DUMP $(OUTPUT)util/*-bison* $(OUTPUT)util/*-flex* \
--		$(OUTPUT)util/intel-pt-decoder/inat-tables.c
-+		$(OUTPUT)util/intel-pt-decoder/inat-tables.c \
+ 		$(OUTPUT)util/intel-pt-decoder/inat-tables.c $(OUTPUT)fixdep \
+-		$(OUTPUT)tests/llvm-src-{base,kbuild,prologue,relocation}.c
++		$(OUTPUT)tests/llvm-src-{base,kbuild,prologue,relocation}.c \
 +		$(OUTPUT)pmu-events/pmu-events.c
  	$(QUIET_SUBDIR0)Documentation $(QUIET_SUBDIR1) clean
  	$(python-clean)
@@ -199,10 +245,10 @@
 +	$(Q)$(call echo-cmd,gen)$(JEVENTS) $(ARCH) pmu-events/arch $(OUTPUT)pmu-events/pmu-events.c $(V)
 diff --git a/tools/perf/pmu-events/jevents.c b/tools/perf/pmu-events/jevents.c
 new file mode 100644
-index 0000000..5f7603b
+index 0000000..a9ca86d
 --- /dev/null
 +++ b/tools/perf/pmu-events/jevents.c
-@@ -0,0 +1,686 @@
+@@ -0,0 +1,766 @@
 +#define  _XOPEN_SOURCE 500	/* needed for nftw() */
 +
 +/* Parse event JSON files */
@@ -256,7 +302,7 @@
 +#define __maybe_unused                  __attribute__((unused))
 +#endif
 +
-+int verbose = 0;
++int verbose;
 +char *prog;
 +
 +int eprintf(int level, int var, const char *fmt, ...)
@@ -285,7 +331,7 @@
 +static void addfield(char *map, char **dst, const char *sep,
 +		     const char *a, jsmntok_t *bt)
 +{
-+	unsigned len = strlen(a) + 1 + strlen(sep);
++	unsigned int len = strlen(a) + 1 + strlen(sep);
 +	int olen = *dst ? strlen(*dst) : 0;
 +	int blen = bt ? json_len(bt) : 0;
 +	char *out;
@@ -402,15 +448,83 @@
 +	goto out_free;						\
 +} } while (0)
 +
++#define TOPIC_DEPTH 256
++static char *topic_array[TOPIC_DEPTH];
++static int   topic_level;
++
++static char *get_topic(void)
++{
++	char *tp_old, *tp = NULL;
++	int i;
++
++	for (i = 0; i < topic_level + 1; i++) {
++		int n;
++
++		tp_old = tp;
++		n = asprintf(&tp, "%s%s", tp ?: "", topic_array[i]);
++		if (n < 0) {
++			pr_info("%s: asprintf() error %s\n", prog);
++			return NULL;
++		}
++		free(tp_old);
++	}
++
++	for (i = 0; i < (int) strlen(tp); i++) {
++		char c = tp[i];
++
++		if (c == '-')
++			tp[i] = ' ';
++		else if (c == '.') {
++			tp[i] = '\0';
++			break;
++		}
++	}
++
++	return tp;
++}
++
++static int add_topic(int level, char *bname)
++{
++	char *topic;
++
++	level -= 2;
++
++	if (level >= TOPIC_DEPTH)
++		return -EINVAL;
++
++	topic = strdup(bname);
++	if (!topic) {
++		pr_info("%s: strdup() error %s for file %s\n", prog,
++				strerror(errno), bname);
++		return -ENOMEM;
++	}
++
++	free(topic_array[topic_level]);
++	topic_array[topic_level] = topic;
++	topic_level              = level;
++	return 0;
++}
++
++struct perf_entry_data {
++	FILE *outfp;
++	char *topic;
++};
++
++static int close_table;
++
 +static void print_events_table_prefix(FILE *fp, const char *tblname)
 +{
 +	fprintf(fp, "struct pmu_event %s[] = {\n", tblname);
++	close_table = 1;
 +}
 +
 +static int print_events_table_entry(void *data, char *name, char *event,
 +				    char *desc)
 +{
-+	FILE *outfp = data;
++	struct perf_entry_data *pd = data;
++	FILE *outfp = pd->outfp;
++	char *topic = pd->topic;
++
 +	/*
 +	 * TODO: Remove formatting chars after debugging to reduce
 +	 *	 string lengths.
@@ -420,6 +534,7 @@
 +	fprintf(outfp, "\t.name = \"%s\",\n", name);
 +	fprintf(outfp, "\t.event = \"%s\",\n", event);
 +	fprintf(outfp, "\t.desc = \"%s\",\n", desc);
++	fprintf(outfp, "\t.topic = \"%s\",\n", topic);
 +
 +	fprintf(outfp, "},\n");
 +
@@ -436,6 +551,7 @@
 +
 +	fprintf(outfp, "},\n");
 +	fprintf(outfp, "};\n");
++	close_table = 0;
 +}
 +
 +/* Call func with each event in the json file */
@@ -501,7 +617,7 @@
 +			}
 +			/* ignore unknown fields */
 +		}
-+		if (precise && !strstr(desc, "(Precise Event)")) {
++		if (precise && desc && !strstr(desc, "(Precise Event)")) {
 +			if (json_streq(map, precise, "2"))
 +				addfield(map, &desc, " ", "(Must be precise)",
 +						NULL);
@@ -587,41 +703,6 @@
 +	fprintf(outfp, "};\n");
 +}
 +
-+/*
-+ * Process the JSON file @json_file and write a table of PMU events found in
-+ * the JSON file to the outfp.
-+ */
-+static int process_json(FILE *outfp, const char *json_file)
-+{
-+	char *tblname;
-+	int err;
-+
-+	/*
-+	 * Drop file name suffix. Replace hyphens with underscores.
-+	 * Fail if file name contains any alphanum characters besides
-+	 * underscores.
-+	 */
-+	tblname = file_name_to_table_name((char *)json_file);
-+	if (!tblname) {
-+		pr_info("%s: Error determining table name for %s\n", prog,
-+				json_file);
-+		return -1;
-+	}
-+
-+	print_events_table_prefix(outfp, tblname);
-+
-+	err = json_events(json_file, print_events_table_entry, outfp);
-+
-+	if (err) {
-+		pr_info("%s: Translation failed\n", prog);
-+		return -1;
-+	}
-+
-+	print_events_table_suffix(outfp);
-+
-+	return 0;
-+}
-+
 +static int process_mapfile(FILE *outfp, char *fpath)
 +{
 +	int n = 16384;
@@ -733,17 +814,46 @@
 + * nftw() doesn't let us pass an argument to the processing function,
 + * so use a global variables.
 + */
-+FILE *eventsfp;
-+char *mapfile;
++static FILE *eventsfp;
++static char *mapfile;
 +
 +static int process_one_file(const char *fpath, const struct stat *sb,
-+				int typeflag __maybe_unused,
-+				struct FTW *ftwbuf __maybe_unused)
-+{
-+	char *bname;
-+
-+	if (!S_ISREG(sb->st_mode))
++			    int typeflag, struct FTW *ftwbuf)
++{
++	char *tblname, *bname  = (char *) fpath + ftwbuf->base;
++	int is_dir  = typeflag == FTW_D;
++	int is_file = typeflag == FTW_F;
++	int level   = ftwbuf->level;
++	int err = 0;
++
++	pr_debug("%s %d %7jd %-20s %s\n",
++		 is_file ? "f" : is_dir ? "d" : "x",
++		 level, sb->st_size, bname, fpath);
++
++	/* base dir */
++	if (level == 0)
 +		return 0;
++
++	/* model directory, reset topic */
++	if (level == 1 && is_dir) {
++		if (close_table)
++			print_events_table_suffix(eventsfp);
++
++		/*
++		 * Drop file name suffix. Replace hyphens with underscores.
++		 * Fail if file name contains any alphanum characters besides
++		 * underscores.
++		 */
++		tblname = file_name_to_table_name(bname);
++		if (!tblname) {
++			pr_info("%s: Error determining table name for %s\n", prog,
++				bname);
++			return -1;
++		}
++
++		print_events_table_prefix(eventsfp, tblname);
++		return 0;
++	}
 +
 +	/*
 +	 * Save the mapfile name for now. We will process mapfile
@@ -752,14 +862,18 @@
 +	 *
 +	 * TODO: Allow for multiple mapfiles? Punt for now.
 +	 */
-+	bname = basename((char *)fpath);
-+	if (!strncmp(bname, "mapfile.csv", 11)) {
-+		if (mapfile) {
-+			pr_info("%s: Many mapfiles? Using %s, ignoring %s\n",
-+					prog, mapfile, fpath);
-+		} else {
-+			mapfile = strdup(fpath);
++	if (level == 1 && is_file) {
++		if (!strncmp(bname, "mapfile.csv", 11)) {
++			if (mapfile) {
++				pr_info("%s: Many mapfiles? Using %s, ignoring %s\n",
++						prog, mapfile, fpath);
++			} else {
++				mapfile = strdup(fpath);
++			}
++			return 0;
 +		}
++
++		pr_info("%s: Ignoring file %s\n", prog, fpath);
 +		return 0;
 +	}
 +
@@ -767,12 +881,18 @@
 +	 * If the file name does not have a .json extension,
 +	 * ignore it. It could be a readme.txt for instance.
 +	 */
-+	bname += strlen(bname) - 5;
-+	if (strncmp(bname, ".json", 5)) {
-+		pr_info("%s: Ignoring file without .json suffix %s\n", prog,
++	if (is_file) {
++		char *suffix = bname + strlen(bname) - 5;
++
++		if (strncmp(suffix, ".json", 5)) {
++			pr_info("%s: Ignoring file without .json suffix %s\n", prog,
 +				fpath);
-+		return 0;
-+	}
++			return 0;
++		}
++	}
++
++	if (level > 1 && add_topic(level, bname))
++		return -ENOMEM;
 +
 +	/*
 +	 * Assume all other files are JSON files.
@@ -786,13 +906,18 @@
 +	 * i.e. if JSON file name cannot be mapped to C-style table name,
 +	 * fail.
 +	 */
-+	if (process_json(eventsfp, fpath)) {
-+		pr_info("%s: Error processing JSON file %s, ignoring all\n",
-+				prog, fpath);
-+		return -1;
-+	}
-+
-+	return 0;
++	if (is_file) {
++		struct perf_entry_data data = {
++			.topic = get_topic(),
++			.outfp = eventsfp,
++		};
++
++		err = json_events(fpath, print_events_table_entry, &data);
++
++		free(data.topic);
++	}
++
++	return err;
 +}
 +
 +#ifndef PATH_MAX
@@ -818,9 +943,8 @@
 +int main(int argc, char *argv[])
 +{
 +	int rc;
-+	int flags;
 +	int maxfds;
-+	char dirname[PATH_MAX];
++	char ldirname[PATH_MAX];
 +
 +	const char *arch;
 +	const char *output_file;
@@ -849,7 +973,7 @@
 +	/* Include pmu-events.h first */
 +	fprintf(eventsfp, "#include \"../../pmu-events/pmu-events.h\"\n");
 +
-+	sprintf(dirname, "%s/%s", start_dirname, arch);
++	sprintf(ldirname, "%s/%s", start_dirname, arch);
 +
 +	/*
 +	 * The mapfile allows multiple CPUids to point to the same JSON file,
@@ -860,17 +984,19 @@
 +	 * separate tables for each symlink (presumably, each symlink refers
 +	 * to specific version of the CPU).
 +	 */
-+	flags = FTW_DEPTH;
 +
 +	maxfds = get_maxfds();
 +	mapfile = NULL;
-+	rc = nftw(dirname, process_one_file, maxfds, flags);
++	rc = nftw(ldirname, process_one_file, maxfds, 0);
 +	if (rc && verbose) {
-+		pr_info("%s: Error walking file tree %s\n", prog, dirname);
++		pr_info("%s: Error walking file tree %s\n", prog, ldirname);
 +		goto empty_map;
 +	} else if (rc) {
 +		goto empty_map;
 +	}
++
++	if (close_table)
++		print_events_table_suffix(eventsfp);
 +
 +	if (!mapfile) {
 +		pr_info("%s: No CPU->JSON mapping?\n", prog);
@@ -913,25 +1039,28 @@
 +
 +#endif
 diff --git a/tools/perf/pmu-events/json.h b/tools/perf/pmu-events/json.h
-index 6b8337e..5ac88ec 100644
+index 6b8337e..5339704 100644
 --- a/tools/perf/pmu-events/json.h
 +++ b/tools/perf/pmu-events/json.h
-@@ -24,6 +24,9 @@ extern int eprintf(int level, int var, const char *fmt, ...);
+@@ -24,6 +24,12 @@ extern int eprintf(int level, int var, const char *fmt, ...);
  #define pr_err(fmt, ...) \
  	eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__)
  
 +#define pr_info(fmt, ...) \
 +	eprintf(1, verbose, pr_fmt(fmt), ##__VA_ARGS__)
 +
++#define pr_debug(fmt, ...) \
++	eprintf(2, verbose, pr_fmt(fmt), ##__VA_ARGS__)
++
  #ifndef roundup
  #define roundup(x, y) (                                \
  {                                                      \
 diff --git a/tools/perf/pmu-events/pmu-events.h b/tools/perf/pmu-events/pmu-events.h
 new file mode 100644
-index 0000000..39fec04
+index 0000000..70d5479
 --- /dev/null
 +++ b/tools/perf/pmu-events/pmu-events.h
-@@ -0,0 +1,35 @@
+@@ -0,0 +1,36 @@
 +#ifndef PMU_EVENTS_H
 +#define PMU_EVENTS_H
 +
@@ -942,6 +1071,7 @@
 +	const char *name;
 +	const char *event;
 +	const char *desc;
++	const char *topic;
 +};
 +
 +/*
@@ -968,4 +1098,4 @@
 +
 +#endif
 -- 
-2.5.3
+1.8.3.1
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help