diff --git a/.gitignore b/.gitignore
index 5087ce1..b92f122 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,6 +71,7 @@
/git-http-fetch
/git-http-push
/git-imap-send
+/git-index-helper
/git-index-pack
/git-init
/git-init-db
diff --git a/Documentation/git-index-helper.txt b/Documentation/git-index-helper.txt
new file mode 100644
index 0000000..61605e9
--- /dev/null
+++ b/Documentation/git-index-helper.txt
@@ -0,0 +1,46 @@
+git-index-helper(1)
+===================
+
+NAME
+----
+git-index-helper - A simple cache daemon for speeding up index file access
+
+SYNOPSIS
+--------
+[verse]
+'git index-helper' [options]
+
+DESCRIPTION
+-----------
+Keep the index file in memory for faster access. This daemon is per
+repository.
+
+OPTIONS
+-------
+
+--exit-after=<n>::
+ Exit if the cached index is not accessed for `<n>`
+ seconds. Specify 0 to wait forever. Default is 600.
+
+NOTES
+-----
+$GIT_DIR/index-helper.path is a symlink to a Unix domain socket in
+$TMPDIR that the daemon reads commands from. At least on Linux, shared
+memory objects are available via /dev/shm with the name pattern
+"git-<something>-<SHA1>". Normally the daemon will clean up shared
+memory objects when it exits. But if it crashes, some objects could
+remain there and they can be safely deleted with "rm" command. The
+following commands are used to control the daemon:
+
+"refresh"::
+ Reread the index.
+
+"poke":
+ Let the daemon know the index is to be read. It keeps the
+ daemon alive longer, unless `--exit-after=0` is used.
+
+All commands and replies are terminated by a 0 byte.
+
+GIT
+---
+Part of the linkgit:git[1] suite
diff --git a/Makefile b/Makefile
index 2742a69..24d6c0e 100644
--- a/Makefile
+++ b/Makefile
@@ -370,6 +370,8 @@ all::
# Define HAVE_BSD_SYSCTL if your platform has a BSD-compatible sysctl function.
#
# Define HAVE_GETDELIM if your system has the getdelim() function.
+#
+# Define HAVE_SHM if you platform support shm_* functions in librt.
GIT-VERSION-FILE: FORCE
@$(SHELL_PATH) ./GIT-VERSION-GEN
@@ -805,6 +807,7 @@ LIB_OBJS += sha1-lookup.o
LIB_OBJS += sha1_file.o
LIB_OBJS += sha1_name.o
LIB_OBJS += shallow.o
+LIB_OBJS += shm.o
LIB_OBJS += sideband.o
LIB_OBJS += sigchain.o
LIB_OBJS += split-index.o
@@ -1433,6 +1436,12 @@ ifdef HAVE_DEV_TTY
BASIC_CFLAGS += -DHAVE_DEV_TTY
endif
+ifdef HAVE_SHM
+ BASIC_CFLAGS += -DHAVE_SHM
+ EXTLIBS += -lrt
+ PROGRAM_OBJS += index-helper.o
+endif
+
ifdef DIR_HAS_BSD_GROUP_SEMANTICS
COMPAT_CFLAGS += -DDIR_HAS_BSD_GROUP_SEMANTICS
endif
@@ -2159,6 +2168,7 @@ GIT-BUILD-OPTIONS: FORCE
@echo NO_PERL=\''$(subst ','\'',$(subst ','\'',$(NO_PERL)))'\' >>$@+
@echo NO_PYTHON=\''$(subst ','\'',$(subst ','\'',$(NO_PYTHON)))'\' >>$@+
@echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@+
+ @echo HAVE_SHM=\''$(subst ','\'',$(subst ','\'',$(HAVE_SHM)))'\' >>$@+
ifdef TEST_OUTPUT_DIRECTORY
@echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+
endif
diff --git a/cache.h b/cache.h
index 4180e2b..43fb314 100644
--- a/cache.h
+++ b/cache.h
@@ -334,6 +334,8 @@ struct index_state {
struct cache_time timestamp;
unsigned name_hash_initialized : 1,
keep_mmap : 1,
+ from_shm : 1,
+ to_shm : 1,
initialized : 1;
struct hashmap name_hash;
struct hashmap dir_hash;diff --git a/config.mak.uname b/config.mak.uname
index 1139b44..d28d05d 100644
--- a/config.mak.uname
+++ b/config.mak.uname
@@ -38,6 +38,7 @@ ifeq ($(uname_S),Linux)
HAVE_CLOCK_MONOTONIC = YesPlease
HAVE_GETDELIM = YesPlease
SANE_TEXT_GREP=-a
+ HAVE_SHM = YesPlease
endif
ifeq ($(uname_S),GNU/kFreeBSD)
HAVE_ALLOCA_H = YesPlease
diff --git a/git-compat-util.h b/git-compat-util.h
index c07e0c1..8b878fe 100644
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -513,6 +513,7 @@ static inline int ends_with(const char *str, const char *suffix)
#define PROT_READ 1
#define PROT_WRITE 2
#define MAP_PRIVATE 1
+#define MAP_SHARED 2
#endif
#define mmap git_mmap
@@ -1045,4 +1046,21 @@ struct tm *git_gmtime_r(const time_t *, struct tm *);
#define getc_unlocked(fh) getc(fh)
#endif
+#ifdef __linux__
+#define UNIX_PATH_MAX 108
+#elif defined(__APPLE__) || defined(BSD)
+#define UNIX_PATH_MAX 104
+#else
+/*
+ * Quoth POSIX: The size of sun_path has intentionally been left
+ * undefined. This is because different implementations use different
+ * sizes. For example, 4.3 BSD uses a size of 108, and 4.4 BSD uses a
+ * size of 104. Since most implementations originate from BSD
+ * versions, the size is typically in the range 92 to 108.
+ *
+ * Thanks, POSIX! Super-helpful! Hope we don't overflow any buffers!
+ */
+#define UNIX_PATH_MAX 92
+#endif
+
#endif
diff --git a/index-helper.c b/index-helper.c
new file mode 100644
index 0000000..263b066
--- /dev/null
+++ b/index-helper.c
@@ -0,0 +1,284 @@
+#include "cache.h"
+#include "parse-options.h"
+#include "sigchain.h"
+#include "strbuf.h"
+#include "exec_cmd.h"
+#include "split-index.h"
+#include "shm.h"
+#include "lockfile.h"
+
+struct shm {
+ unsigned char sha1[20];
+ void *shm;
+ size_t size;
+};
+
+static struct shm shm_index;
+static struct shm shm_base_index;
+
+static void release_index_shm(struct shm *is)
+{
+ if (!is->shm)
+ return;
+ munmap(is->shm, is->size);
+ git_shm_unlink("git-index-%s", sha1_to_hex(is->sha1));
+ is->shm = NULL;
+}
+
+static void cleanup_shm(void)
+{
+ release_index_shm(&shm_index);
+ release_index_shm(&shm_base_index);
+}
+
+static void cleanup(void)
+{
+ unlink(git_path("index-helper.path"));
+ cleanup_shm();
+}
+
+static void cleanup_on_signal(int sig)
+{
+ /* We ignore sigpipes -- that's just a client being broken. */
+ if (sig == SIGPIPE)
+ return;
+ cleanup();
+ sigchain_pop(sig);
+ raise(sig);
+}
+
+static void share_index(struct index_state *istate, struct shm *is)
+{
+ void *new_mmap;
+ if (istate->mmap_size <= 20 ||
+ hashcmp(istate->sha1,
+ (unsigned char *)istate->mmap + istate->mmap_size - 20) ||
+ !hashcmp(istate->sha1, is->sha1) ||
+ git_shm_map(O_CREAT | O_EXCL | O_RDWR, 0700, istate->mmap_size,
+ &new_mmap, PROT_READ | PROT_WRITE, MAP_SHARED,
+ "git-index-%s", sha1_to_hex(istate->sha1)) < 0)
+ return;
+
+ release_index_shm(is);
+ is->size = istate->mmap_size;
+ is->shm = new_mmap;
+ hashcpy(is->sha1, istate->sha1);
+ memcpy(new_mmap, istate->mmap, istate->mmap_size - 20);
+
+ /*
+ * The trailing hash must be written last after everything is
+ * written. It's the indication that the shared memory is now
+ * ready.
+ * The memory barrier here matches read-cache.c:try_shm.
+ */
+ __sync_synchronize();
+
+ hashcpy((unsigned char *)new_mmap + istate->mmap_size - 20, is->sha1);
+}
+
+static void share_the_index(void)
+{
+ if (the_index.split_index && the_index.split_index->base)
+ share_index(the_index.split_index->base, &shm_base_index);
+ share_index(&the_index, &shm_index);
+ discard_index(&the_index);
+}
+
+static void refresh(void)
+{
+ the_index.keep_mmap = 1;
+ the_index.to_shm = 1;
+ if (read_cache() < 0)
+ die(_("could not read index"));
+ share_the_index();
+}
+
+static void set_socket_nonblocking(int fd)
+{
+ int flags;
+
+ flags = fcntl(fd, F_GETFL, NULL);
+
+ if (flags < 0)
+ die(_("fcntl failed"));
+
+ if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0)
+ die(_("fcntl failed"));
+}
+
+#ifdef HAVE_SHM
+
+static void loop(int fd, int idle_in_seconds)
+{
+ struct timeval timeout;
+ struct timeval *timeout_p;
+
+ while (1) {
+ fd_set readfds;
+ int result, client_fd;
+ struct strbuf command = STRBUF_INIT;
+
+ /* need to reset timer in case select() decremented it */
+ if (idle_in_seconds) {
+ timeout.tv_usec = 0;
+ timeout.tv_sec = idle_in_seconds;
+ timeout_p = &timeout;
+ } else {
+ timeout_p = NULL;
+ }
+
+ /* Wait for a request */
+ FD_ZERO(&readfds);
+ FD_SET(fd, &readfds);
+ result = select(fd + 1, &readfds, NULL, NULL, timeout_p);
+ if (result < 0)
+ die_errno(_("select() failed"));
+ if (result == 0)
+ /* timeout */
+ break;
+
+ client_fd = accept(fd, NULL, NULL);
+ if (client_fd < 0)
+ /*
+ * An error here is unlikely -- it probably
+ * indicates that the connecting process has
+ * already dropped the connection.
+ */
+ continue;
+
+ /*
+ * Our connection to the client is blocking since a client
+ * can always be killed by SIGINT or similar.
+ */
+ if (strbuf_getwholeline_fd(&command, client_fd, '\0') == 0) {
+ if (!strcmp(command.buf, "refresh")) {
+ refresh();
+ } else if (!strcmp(command.buf, "poke")) {
+ /*
+ * Just a poke to keep us
+ * alive, nothing to do.
+ */
+ } else {
+ warning("BUG: Bogus command %s", command.buf);
+ }
+ } else {
+ /*
+ * No command from client. Probably it's just
+ * a liveness check. Just close up.
+ */
+ }
+ close(client_fd);
+ }
+
+ close(fd);
+}
+
+#else
+
+static void loop(int fd, int idle_in_seconds)
+{
+ die(_("index-helper is not supported on this platform"));
+}
+
+#endif
+
+static int setup_socket(const char *socket_path)
+{
+ struct sockaddr_un address = {0};
+ int fd;
+ int len;
+
+ len = strlen(socket_path);
+ if (len > UNIX_PATH_MAX - 1)
+ die("path %s is too long for a socket", socket_path);
+
+ fd = socket(PF_UNIX, SOCK_STREAM, 0);
+ if (fd < 0)
+ return -1;
+
+ address.sun_family = AF_UNIX;
+ strncpy(address.sun_path, socket_path, UNIX_PATH_MAX);
+
+ if (bind(fd, (struct sockaddr *) &address, sizeof(address)))
+ die_errno(_("failed to bind to socket %s"), socket_path);
+
+ set_socket_nonblocking(fd);
+
+ if (listen(fd, 3))
+ die_errno(_("failed to listen on socket %s"), socket_path);
+
+ return fd;
+}
+
+static const char * const usage_text[] = {
+ N_("git index-helper [options]"),
+ NULL
+};
+
+static void make_socket_path(struct strbuf *path)
+{
+ const char *tmpdir;
+
+ tmpdir = getenv("TMPDIR");
+ if (!tmpdir)
+ tmpdir = "/tmp";
+
+ /*
+ * We need to make a dir that we can own, so that users other
+ * than us cannot poke the daemon, and so that, if the daemon
+ * dies, no other process can recreate the socket and trick us
+ * into talking to an imposter.
+ */
+ strbuf_addf(path, "%s/XXXXXX", tmpdir);
+ if (!mkdtemp(path->buf))
+ die(("failed to make temp dir for socket"));
+
+ /*
+ * Use a stupid filename because we want to minimize the path
+ * length since socket filenames must be short.
+ */
+ strbuf_addstr(path, "/s");
+}
+
+int main(int argc, char **argv)
+{
+ const char *prefix;
+ int idle_in_seconds = 600;
+ int fd;
+ struct strbuf socket_path = STRBUF_INIT;
+ struct option options[] = {
+ OPT_INTEGER(0, "exit-after", &idle_in_seconds,
+ N_("exit if not used after some seconds")),
+ OPT_END()
+ };
+
+ git_extract_argv0_path(argv[0]);
+ git_setup_gettext();
+
+ if (argc == 2 && !strcmp(argv[1], "-h"))
+ usage_with_options(usage_text, options);
+
+ prefix = setup_git_directory();
+ if (parse_options(argc, (const char **)argv, prefix,
+ options, usage_text, 0))
+ die(_("too many arguments"));
+
+ atexit(cleanup);
+ sigchain_push_common(cleanup_on_signal);
+
+ make_socket_path(&socket_path);
+ fd = setup_socket(socket_path.buf);
+ if (fd < 0)
+ die_errno(_("could not set up index-helper socket"));
+ /*
+ * Now that the socket is set up, we symlink it into
+ * GIT_DIR so clients can find it.
+ */
+ if (unlink(git_path("index-helper.path")) && errno != ENOENT)
+ die(_("failed to delete old index-helper.path"));
+ if (symlink(socket_path.buf, git_path("index-helper.path")))
+ die(_("failed to symlink socket path into index-helper.path"));
+ loop(fd, idle_in_seconds);
+
+ return 0;
+}diff --git a/read-cache.c b/read-cache.c
index 7e387e9..66f2874 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -18,6 +18,7 @@
#include "varint.h"
#include "split-index.h"
#include "utf8.h"
+#include "shm.h"
static struct cache_entry *refresh_cache_entry(struct cache_entry *ce,
unsigned int options);
@@ -1541,6 +1542,91 @@ static void post_read_index_from(struct index_state *istate)
tweak_untracked_cache(istate);
}
+static int poke_daemon(struct index_state *istate,
+ const struct stat *st, int refresh_cache)
+{
+ int fd;
+ int ret = 0;
+ struct sockaddr_un address = {0};
+
+ /* if this is from index-helper, do not poke itself (recursively) */
+ if (istate->to_shm)
+ return 0;
+
+ if (readlink(git_path("index-helper.path"), address.sun_path,
+ UNIX_PATH_MAX) < 0)
+ return -1;
+
+ fd = socket(PF_UNIX, SOCK_STREAM, 0);
+ if (fd < 0)
+ return -1;
+
+ address.sun_family = AF_UNIX;
+ if (connect(fd, (struct sockaddr *) &address, sizeof(address))) {
+ warning("Failed to connect to socket %s", address.sun_path);
+ close(fd);
+ return -1;
+ }
+
+ if (refresh_cache) {
+ ret = write_in_full(fd, "refresh", 8) != 8;
+ } else {
+ ret = write_in_full(fd, "poke", 5) != 5;
+ }
+
+ close(fd);
+ return ret;
+}
+
+static int is_main_index(struct index_state *istate)
+{
+ return istate == &the_index ||
+ (the_index.split_index &&
+ istate == the_index.split_index->base);
+}
+
+/*
+ * Try to open and verify a cached shm index if available. Return 0 if
+ * succeeds (istate->mmap and istate->mmap_size are updated). Return
+ * negative otherwise.
+ */
+static int try_shm(struct index_state *istate)
+{
+ void *new_mmap = NULL;
+ size_t old_size = istate->mmap_size;
+ ssize_t new_size;
+ const unsigned char *sha1;
+ struct stat st;
+
+ if (!is_main_index(istate) ||
+ old_size <= 20 ||
+ stat(git_path("index-helper.path"), &st))
+ return -1;
+ if (poke_daemon(istate, &st, 0))
+ return -1;
+ sha1 = (unsigned char *)istate->mmap + old_size - 20;
+ new_size = git_shm_map(O_RDONLY, 0700, -1, &new_mmap,
+ PROT_READ, MAP_SHARED,
+ "git-index-%s", sha1_to_hex(sha1));
+ if (new_size <= 20 ||
+ hashcmp((unsigned char *)istate->mmap + old_size - 20,
+ (unsigned char *)new_mmap + new_size - 20)) {
+ if (new_mmap)
+ munmap(new_mmap, new_size);
+ poke_daemon(istate, &st, 1);
+ return -1;
+ }
+
+ /* The memory barrier here matches index-helper.c:share_index. */
+ __sync_synchronize();
+
+ munmap(istate->mmap, istate->mmap_size);
+ istate->mmap = new_mmap;
+ istate->mmap_size = new_size;
+ istate->from_shm = 1;
+ return 0;
+}
+
/* remember to discard_cache() before reading a different cache! */
int do_read_index(struct index_state *istate, const char *path, int must_exist)
{@@ -1555,6 +1641,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
if (istate->initialized)
return istate->cache_nr;
+ istate->from_shm = 0;
istate->timestamp.sec = 0;
istate->timestamp.nsec = 0;
fd = open(path, O_RDONLY);
@@ -1574,15 +1661,17 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
mmap = xmmap(NULL, mmap_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (mmap == MAP_FAILED)
die_errno("unable to map index file");
- if (istate->keep_mmap) {
- istate->mmap = mmap;
- istate->mmap_size = mmap_size;
- }
close(fd);
- hdr = mmap;
- if (verify_hdr(hdr, mmap_size) < 0)
+ istate->mmap = mmap;
+ istate->mmap_size = mmap_size;
+ if (try_shm(istate) &&
+ verify_hdr(istate->mmap, istate->mmap_size) < 0)
goto unmap;
+ hdr = mmap = istate->mmap;
+ mmap_size = istate->mmap_size;
+ if (!istate->keep_mmap)
+ istate->mmap = NULL;
hashcpy(istate->sha1, (const unsigned char *)hdr + mmap_size - 20);
istate->version = ntohl(hdr->hdr_version);@@ -1662,6 +1751,8 @@ int read_index_from(struct index_state *istate, const char *path)
else
split_index->base = xcalloc(1, sizeof(*split_index->base));
split_index->base->keep_mmap = istate->keep_mmap;
+ split_index->base->to_shm = istate->to_shm;
+ split_index->base->from_shm = istate->from_shm;
ret = do_read_index(split_index->base,
git_path("sharedindex.%s",
sha1_to_hex(split_index->base_sha1)), 1);@@ -1712,6 +1803,8 @@ int discard_index(struct index_state *istate)
discard_split_index(istate);
free_untracked_cache(istate->untracked);
istate->untracked = NULL;
+ istate->from_shm = 0;
+ istate->to_shm = 0;
return 0;
}
@@ -2138,9 +2231,14 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l
return ret;
assert((flags & (COMMIT_LOCK | CLOSE_LOCK)) !=
(COMMIT_LOCK | CLOSE_LOCK));
- if (flags & COMMIT_LOCK)
- return commit_locked_index(lock);
- else if (flags & CLOSE_LOCK)
+ if (flags & COMMIT_LOCK) {
+ struct stat st;
+ ret = commit_locked_index(lock);
+ if (!ret && is_main_index(istate) &&
+ !stat(git_path("index-helper.path"), &st))
+ poke_daemon(istate, &st, 1);
+ return ret;
+ } else if (flags & CLOSE_LOCK)
return close_lock_file(lock);
else
return ret;diff --git a/shm.c b/shm.c
new file mode 100644
index 0000000..4ec1a00
--- /dev/null
+++ b/shm.c
@@ -0,0 +1,67 @@
+#include "git-compat-util.h"
+#include "shm.h"
+
+#ifdef HAVE_SHM
+
+#define SHM_PATH_LEN 72 /* we don't create very long paths.. */
+
+ssize_t git_shm_map(int oflag, int perm, ssize_t length, void **mmap,
+ int prot, int flags, const char *fmt, ...)
+{
+ va_list ap;
+ char path[SHM_PATH_LEN];
+ int fd;
+
+ path[0] = '/';
+ va_start(ap, fmt);
+ vsprintf(path + 1, fmt, ap);
+ va_end(ap);
+ fd = shm_open(path, oflag, perm);
+ if (fd < 0)
+ return -1;
+ if (length > 0 && ftruncate(fd, length)) {
+ shm_unlink(path);
+ close(fd);
+ return -1;
+ }
+ if (length < 0 && !(oflag & O_CREAT)) {
+ struct stat st;
+ if (fstat(fd, &st))
+ die_errno("unable to stat %s", path);
+ length = st.st_size;
+ }
+ *mmap = xmmap(NULL, length, prot, flags, fd, 0);
+ close(fd);
+ if (*mmap == MAP_FAILED) {
+ *mmap = NULL;
+ shm_unlink(path);
+ return -1;
+ }
+ return length;
+}
+
+void git_shm_unlink(const char *fmt, ...)
+{
+ va_list ap;
+ char path[SHM_PATH_LEN];
+
+ path[0] = '/';
+ va_start(ap, fmt);
+ vsprintf(path + 1, fmt, ap);
+ va_end(ap);
+ shm_unlink(path);
+}
+
+#else
+
+ssize_t git_shm_map(int oflag, int perm, ssize_t length, void **mmap,
+ int prot, int flags, const char *fmt, ...)
+{
+ return -1;
+}
+
+void git_shm_unlink(const char *fmt, ...)
+{
+}
+
+#endifdiff --git a/shm.h b/shm.h
new file mode 100644
index 0000000..798d3fd
--- /dev/null
+++ b/shm.h
@@ -0,0 +1,23 @@
+#ifndef SHM_H
+#define SHM_H
+
+/*
+ * Create or open a shared memory and mmap it. Return mmap size if
+ * successful, -1 otherwise. If successful mmap contains the mmap'd
+ * pointer. If oflag does not contain O_CREAT and length is negative,
+ * the mmap size is retrieved from existing shared memory object.
+ *
+ * The mmap could be freed by munmap, even on Windows. Note that on
+ * Windows, git_shm_unlink() is no-op, so the last unmap will destroy
+ * the shared memory.
+ */
+ssize_t git_shm_map(int oflag, int perm, ssize_t length, void **mmap,
+ int prot, int flags, const char *fmt, ...);
+
+/*
+ * Unlink a shared memory object. Only needed on POSIX platforms. On
+ * Windows this is no-op.
+ */
+void git_shm_unlink(const char *fmt, ...);
+
+#endif
diff --git a/t/t7900-index-helper.sh b/t/t7900-index-helper.sh
new file mode 100755
index 0000000..6a06094
--- /dev/null
+++ b/t/t7900-index-helper.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+#
+# Copyright (c) 2016, Twitter, Inc
+#
+
+test_description='git-index-helper
+
+Testing git index-helper
+'
+
+. ./test-lib.sh
+
+test -z "$HAVE_SHM" && {
+ skip_all='skipping index-helper tests: no SHM'
+ test_done
+}
+
+test_expect_success 'index-helper smoke test' '
+ git index-helper --exit-after 1 &&
+ test_path_is_missing .git/index-helper.path
+'
+
+test_done--
2.4.2.767.g62658d5-twtrsrc