[PATCH] index-pack usage of mmap() is unacceptably slower on many OSes other than Linux
From: Nicolas Pitre <hidden>
Date: 2016-08-11 20:17:35
Subsystem:
the rest · Maintainer:
Linus Torvalds
Possibly related (same subject, not in this thread)
- 2016-08-11 · Re: [PATCH] index-pack usage of mmap() is unacceptably slower on many OSes other than Linux · Linus Torvalds <torvalds@osdl.org>
- 2016-08-11 · Re: [PATCH] index-pack usage of mmap() is unacceptably slower on many OSes other than Linux · Nikolai Weibull <hidden>
- 2016-08-11 · Re: [PATCH] index-pack usage of mmap() is unacceptably slower on many OSes other than Linux · Junio C Hamano <hidden>
- 2016-08-11 · Re: [PATCH] index-pack usage of mmap() is unacceptably slower on many OSes other than Linux · Linus Torvalds <torvalds@osdl.org>
- 2016-08-11 · Re: [PATCH] index-pack usage of mmap() is unacceptably slower on many OSes other than Linux · Nicolas Pitre <hidden>
It was reported by Randal L. Schwartz [off-list ref] that indexing the Linux repository ~150MB pack takes about an hour on OS x while it's a minute on Linux. It seems that the OS X mmap() implementation is more than 2 orders of magnitude slower than the Linux one. Linus proposed a patch replacing mmap() with pread() bringing index-pack performance on OS X in line with the Linux one. The performances on Linux also improved by a small margin. Signed-off-by: Nicolas Pitre <redacted> --- OK looks like this has been sorted out while I was away. Good! On Tue, 19 Dec 2006, Junio C Hamano wrote:
Johannes Schindelin [off-list ref] writes:quoted
Hi, in a very unscientific test, without your patch local cloning of the LilyPond repo takes 1m33s (user), and with your patch (pread() instead of mmap()) it takes 1m13s (user). The real times are somewhat bogus, but still in favour of pread(), but only by 8 seconds instead of 20. This is on Linux 2.4.32.Interesting. Anybody have numbers from 2.6?
My 37 seconds of yesterday dropped to 32. This is Linus's patch plus a few cosmetic changes.
diff --git a/index-pack.c b/index-pack.c
index 6d6c92b..e08a687 100644
--- a/index-pack.c
+++ b/index-pack.c@@ -1,3 +1,8 @@ +#define _XOPEN_SOURCE 500 +#include <unistd.h> +#include <sys/time.h> +#include <signal.h> + #include "cache.h" #include "delta.h" #include "pack.h"
@@ -6,8 +11,6 @@ #include "commit.h" #include "tag.h" #include "tree.h" -#include <sys/time.h> -#include <signal.h> static const char index_pack_usage[] = "git-index-pack [-v] [-o <index-file>] [{ ---keep | --keep=<msg> }] { <pack-file> | --stdin [--fix-thin] [<pack-file>] }";
@@ -87,7 +90,7 @@ static unsigned display_progress(unsigned n, unsigned total, unsigned last_pc) static unsigned char input_buffer[4096]; static unsigned long input_offset, input_len, consumed_bytes; static SHA_CTX input_ctx; -static int input_fd, output_fd, mmap_fd; +static int input_fd, output_fd, pack_fd; /* Discard current buffer used content. */ static void flush(void)
@@ -148,14 +151,14 @@ static const char *open_pack_file(const char *pack_name) output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600); if (output_fd < 0) die("unable to create %s: %s\n", pack_name, strerror(errno)); - mmap_fd = output_fd; + pack_fd = output_fd; } else { input_fd = open(pack_name, O_RDONLY); if (input_fd < 0) die("cannot open packfile '%s': %s", pack_name, strerror(errno)); output_fd = -1; - mmap_fd = input_fd; + pack_fd = input_fd; } SHA1_Init(&input_ctx); return pack_name;
@@ -279,27 +282,25 @@ static void *get_data_from_pack(struct object_entry *obj) { unsigned long from = obj[0].offset + obj[0].hdr_size; unsigned long len = obj[1].offset - from; - unsigned pg_offset = from % getpagesize(); - unsigned char *map, *data; + unsigned char *src, *data; z_stream stream; int st; - map = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE, - mmap_fd, from - pg_offset); - if (map == MAP_FAILED) - die("cannot mmap pack file: %s", strerror(errno)); + src = xmalloc(len); + if (pread(pack_fd, src, len, from) != len) + die("cannot pread pack file: %s", strerror(errno)); data = xmalloc(obj->size); memset(&stream, 0, sizeof(stream)); stream.next_out = data; stream.avail_out = obj->size; - stream.next_in = map + pg_offset; + stream.next_in = src; stream.avail_in = len; inflateInit(&stream); while ((st = inflate(&stream, Z_FINISH)) == Z_OK); inflateEnd(&stream); if (st != Z_STREAM_END || stream.total_out != obj->size) die("serious inflate inconsistency"); - munmap(map, len + pg_offset); + free(src); return data; }