[PATCH 1/3] Add support for external programs for handling native fetches

transport_get() can call transport_shim_init() to have list and
fetch-ref operations handled by running a separate program as:

 git shim-<something> <remote> [<url>]

This program then accepts, on its stdin, "list" and "fetch <hex>
<name>" commands; the former prints out a list of available refs and
either their hashes or what they are symreefs to, while the latter
fetches them into the local object database and prints a newline when done.

Signed-off-by: Daniel Barkalow <redacted>
---
This is similar to, but different from, my git-vcs-* stuff; this one 
expects the helper to be able to look up sha1s for refs efficiently, and 
to be able to fetch objects directly into the local object database 
without fast-import. On the otherhand, it should be sufficiently general 
to allow arbitrary methods for moving the data around.

 Documentation/git-shim.txt |   37 ++++++++++++
 Makefile                   |    1 +
 transport-shim.c           |  140 ++++++++++++++++++++++++++++++++++++++++++++
 transport.h                |    3 +
 4 files changed, 181 insertions(+), 0 deletions(-)
 create mode 100644 Documentation/git-shim.txt
 create mode 100644 transport-shim.c

diff --git a/Documentation/git-shim.txt b/Documentation/git-shim.txt
new file mode 100644
index 0000000..26a611f
--- /dev/null
+++ b/Documentation/git-shim.txt

@@ -0,0 +1,37 @@
+git-shim(1)
+============
+
+NAME
+----
+git-shim - Helper programs for interoperation with remote git
+
+SYNOPSIS
+--------
+'git shim-<transport>' <remote>
+
+DESCRIPTION
+-----------
+
+These programs are normally not used directly by end users, but are
+invoked by various git programs that interact with remote repositories
+when the repository they would operate on will be accessed using
+transport code not linked into the main git binary.
+
+COMMANDS
+--------
+
+Commands are given by the caller on the helper's standard input, one per line.
+
+'list'::
+	Outputs the names of refs followed by the hex of their values
+	or "@<name>" for symrefs, one per line. After the complete
+	list, outputs a blank line.
+
+'fetch' ref::
+	Fetches the given ref, writing the necessary objects to the
+	database. Outputs a blank line when the fetch is complete.
++
+If a fatal error occurs, the program writes the error message to
+stderr and exits. The caller should expect that a suitable error
+message has been printed if the child closes the connection without
+completing a valid response for the current command.

diff --git a/Makefile b/Makefile
index bde27ed..01efc73 100644
--- a/Makefile
+++ b/Makefile

@@ -549,6 +549,7 @@ LIB_OBJS += symlinks.o
 LIB_OBJS += tag.o
 LIB_OBJS += trace.o
 LIB_OBJS += transport.o
+LIB_OBJS += transport-shim.o
 LIB_OBJS += tree-diff.o
 LIB_OBJS += tree.o
 LIB_OBJS += tree-walk.o

diff --git a/transport-shim.c b/transport-shim.c
new file mode 100644
index 0000000..3fc6abf
--- /dev/null
+++ b/transport-shim.c

@@ -0,0 +1,140 @@
+#include "cache.h"
+#include "transport.h"
+
+#include "run-command.h"
+#include "commit.h"
+#include "diff.h"
+#include "revision.h"
+
+struct shim_data
+{
+	const char *name;
+	struct child_process *shim;
+};
+
+static struct child_process *get_shim(struct transport *transport)
+{
+	struct shim_data *data = transport->data;
+	if (!data->shim) {
+		struct strbuf buf = STRBUF_INIT;
+		struct child_process *shim = xcalloc(1, sizeof(*shim));
+		shim->in = -1;
+		shim->out = -1;
+		shim->err = 0;
+		shim->argv = xcalloc(4, sizeof(*shim->argv));
+		strbuf_addf(&buf, "shim-%s", data->name);
+		shim->argv[0] = buf.buf;
+		shim->argv[1] = transport->remote->name;
+		shim->argv[2] = transport->url;
+		shim->git_cmd = 1;
+		start_command(shim);
+		data->shim = shim;
+	}
+	return data->shim;
+}
+
+static int disconnect_shim(struct transport *transport)
+{
+	struct shim_data *data = transport->data;
+	if (data->shim) {
+		write(data->shim->in, "\n", 1);
+		close(data->shim->in);
+		finish_command(data->shim);
+		free(data->shim);
+		transport->data = NULL;
+	}
+	return 0;
+}
+
+static int fetch_refs_via_shim(struct transport *transport,
+			       int nr_heads, const struct ref **to_fetch)
+{
+	struct child_process *shim;
+	const struct ref *posn;
+	struct strbuf buf = STRBUF_INIT;
+	int i, count;
+	FILE *file;
+
+	count = 0;
+	for (i = 0; i < nr_heads; i++) {
+		posn = to_fetch[i];
+		if (posn->status & REF_STATUS_UPTODATE)
+			continue;
+		count++;
+	}
+
+	if (count) {
+		shim = get_shim(transport);
+		for (i = 0; i < nr_heads; i++) {
+			posn = to_fetch[i];
+			if (posn->status & REF_STATUS_UPTODATE)
+				continue;
+			write(shim->in, "fetch ", 6);
+			write(shim->in, sha1_to_hex(posn->old_sha1), 40);
+			write(shim->in, " ", 1);
+			write(shim->in, posn->name, strlen(posn->name));
+			write(shim->in, "\n", 1);
+		}
+		file = fdopen(shim->out, "r");
+		while (count) {
+			if (strbuf_getline(&buf, file, '\n') == EOF)
+				exit(128); // child died, message supplied already
+
+			count--;
+		}
+	}
+	return 0;
+}
+
+static struct ref *get_refs_via_shim(struct transport *transport, int for_push)
+{
+	struct child_process *shim;
+	struct ref *ret = NULL;
+	struct ref **end = &ret;
+	struct ref *posn;
+	struct strbuf buf = STRBUF_INIT;
+	FILE *file;
+
+	shim = get_shim(transport);
+	write(shim->in, "list\n", 5);
+
+	file = fdopen(shim->out, "r");
+	while (1) {
+		char *eon;
+		if (strbuf_getline(&buf, file, '\n') == EOF)
+			exit(128); // child died, message supplied already
+
+		if (!*buf.buf)
+			break;
+
+		eon = strchr(buf.buf, ' ');
+		if (eon)
+			*eon = '\0';
+		*end = alloc_ref(buf.buf);
+		if (eon) {
+			if (eon[1] == '@')
+				(*end)->symref = xstrdup(eon + 2);
+			else
+				get_sha1_hex(eon + 1, (*end)->old_sha1);
+		}
+		end = &((*end)->next);
+		strbuf_reset(&buf);
+	}
+	strbuf_release(&buf);
+
+	for (posn = ret; posn; posn = posn->next)
+		resolve_remote_symref(posn, ret);
+
+	return ret;
+}
+
+void transport_shim_init(struct transport *transport, const char *name)
+{
+	struct shim_data *data = xmalloc(sizeof(*data));
+	data->shim = NULL;
+	data->name = name;
+	transport->data = data;
+	transport->get_refs_list = get_refs_via_shim;
+	transport->fetch = fetch_refs_via_shim;
+	transport->disconnect = disconnect_shim;
+}

diff --git a/transport.h b/transport.h
index 51b5397..01f650d 100644
--- a/transport.h
+++ b/transport.h

@@ -77,4 +77,7 @@ void transport_unlock_pack(struct transport *transport);
 int transport_disconnect(struct transport *transport);
 char *transport_anonymize_url(const char *url);
 
+/* Transport methods defined outside transport.c */
+void transport_shim_init(struct transport *transport, const char *name);
+
 #endif

-- 
1.6.3.2.425.g3dedc

Hi,

On Sat, 25 Jul 2009, Daniel Barkalow wrote:

transport_get() can call transport_shim_init() to have list and
fetch-ref operations handled by running a separate program as:

 git shim-<something> <remote> [<url>]

This program then accepts, on its stdin, "list" and "fetch <hex>
<name>" commands; the former prints out a list of available refs and
either their hashes or what they are symreefs to, while the latter
fetches them into the local object database and prints a newline when done.

Signed-off-by: Daniel Barkalow <redacted>
---
This is similar to, but different from, my git-vcs-* stuff; this one 
expects the helper to be able to look up sha1s for refs efficiently, and 
to be able to fetch objects directly into the local object database 
without fast-import. On the otherhand, it should be sufficiently general 
to allow arbitrary methods for moving the data around.

I wonder if this cannot be integrated into the git-vcs stuff; after all, 
they do pretty much the same things, except that the vcs helpers are 
pretty dumb, and the shim helpers are not as dumb.

I could imagine that the name of the helper could reveal its capability to 
act as a shim helper: git-vcs-shim-http

Ciao,
Dscho

On Sat, 25 Jul 2009, Daniel Barkalow wrote:

This is similar to, but different from, my git-vcs-* stuff; this one 
expects the helper to be able to look up sha1s for refs efficiently, and 
to be able to fetch objects directly into the local object database 
without fast-import. On the otherhand, it should be sufficiently general 
to allow arbitrary methods for moving the data around.

Wonderful. 

I tested the series, and now there is (not surprisingly, but I made sure 
to test anyway) no difference what-so-ever between NO_CURL and default, 
and 'ldd' looks nice.

Plus it looks like that whole "shim" thing is a good idea in general, in 
that it allows a much more flexible model for fetching/pushing.

So a very big Acked-by: from me for the series. I didn't test that http: 
works with it, but I don't personally even care, so I'd ack it even 
without that ;)

Btw, some real timing numbers for 'time make -j64 test':

 - before:
	real	1m16.070s
	user	2m47.046s
	sys	2m34.698s

 - after:
	real	0m58.851s
	user	1m57.087s
	sys	1m44.671s

so that's actually a real-world example of the whole 'scripting 
performance'. Not an insignificant speedup on my machine (with an 
obligatory "nyaah, nyaah, I can do the git test-suite under a minute" just 
to rub peoples noses in the fact that my desktop computer is disgustingly 
fast).

That's an almost 30% performance improvement, despite the fact that parts 
of the test suite didn't actually change (shell costs are the same, the 
svn tests are quite perl-intensive etc).

			Linus

Daniel Barkalow [off-list ref] wrote:

+Commands are given by the caller on the helper's standard input, one per line.
+
+'list'::
+	Outputs the names of refs followed by the hex of their values
+	or "@<name>" for symrefs, one per line. After the complete
+	list, outputs a blank line.

I hate to be a nit, but why this format and not the one that is
produced by `git ls-remote` and the native protocol?  I know its
pretty arbitrary either way, but since we already have a history
of using "SHA1 ref\n" why not continue that tradition here?

-- 
Shawn.

On Sat, 25 Jul 2009, Shawn O. Pearce wrote:

Daniel Barkalow [off-list ref] wrote:

quoted

+Commands are given by the caller on the helper's standard input, one per line.
+
+'list'::
+	Outputs the names of refs followed by the hex of their values
+	or "@<name>" for symrefs, one per line. After the complete
+	list, outputs a blank line.

I hate to be a nit, but why this format and not the one that is
produced by `git ls-remote` and the native protocol?  I know its
pretty arbitrary either way, but since we already have a history
of using "SHA1 ref\n" why not continue that tradition here?

In the VCS protocol, the SHA1 is generally not known when simply listing 
refs, so I didn't have a useless initial field in that protocol; I just 
inherited having the line start with the name for this protocol.

In any case, I want to be able to support symrefs, so an extension to the 
tradition is required, but putting @<dest> before the name instead of 
after is easy enough.

	-Daniel
*This .sig left intentionally blank*

On Sat, 25 Jul 2009, Linus Torvalds wrote:

Btw, some real timing numbers for 'time make -j64 test':

 - before:
	real	1m16.070s
	user	2m47.046s
	sys	2m34.698s

 - after:
	real	0m58.851s
	user	1m57.087s
	sys	1m44.671s

so that's actually a real-world example of the whole 'scripting 
performance'. Not an insignificant speedup on my machine (with an 
obligatory "nyaah, nyaah, I can do the git test-suite under a minute" just 
to rub peoples noses in the fact that my desktop computer is disgustingly 
fast).

That's an almost 30% performance improvement, despite the fact that parts 
of the test suite didn't actually change (shell costs are the same, the 
svn tests are quite perl-intensive etc).

Just in case people wonder _why_, here is a profile from before and after. 
Note how big a deal the page faulting, unmapping (TLB flushes etc), and 
fork() is (copy_page_range()).

And notice how the biggest user space cost - even after the change - is 
do_lookup_x() in the dynamic loader. But before the change it was the very 
top entry, and you had things like strcmp and _dl_relocate_object pretty 
high too. Not to mention that you got just a lot _more_ munmap's and page 
faults:

 - before:

     4.51%               git  /lib64/ld-2.10.1.so        [.] do_lookup_x
     3.17%               git  [kernel]                   [k] unmap_vmas
     2.75%               git  [kernel]                   [k] page_fault
     1.48%               git  [kernel]                   [k] copy_page_c
     1.43%               git  /lib64/ld-2.10.1.so        [.] strcmp
     1.30%               git  [kernel]                   [k] _spin_lock
     1.12%               git  /lib64/ld-2.10.1.so        [.] _dl_relocate_object
     0.99%           git-svn  [kernel]                   [k] copy_page_range
     0.99%               git  [kernel]                   [k] kmem_cache_alloc
     0.97%               git  [kernel]                   [k] get_page_from_freelist
     0.92%               git  [kernel]                   [k] copy_page_range
     0.88%               git  [kernel]                   [k] clear_page_c
     0.80%               git  [kernel]                   [k] find_vma
     0.79%               git  /lib64/ld-2.10.1.so        [.] _dl_lookup_symbol_x
     0.68%               git  [kernel]                   [k] handle_mm_fault
     0.68%               git  /lib64/libc-2.10.1.so      [.] _int_malloc
     0.63%               git  /bin/bash                  0x00000000046e96
     0.57%               git  /lib64/libc-2.10.1.so      [.] __GI__dl_addr
     0.51%               git  [kernel]                   [k] release_pages

 - after:

     3.02%               git  [kernel]                   [k] unmap_vmas
     2.74%               git  [kernel]                   [k] page_fault
     1.32%               git  [kernel]                   [k] copy_page_c
     1.23%               git  [kernel]                   [k] _spin_lock
     1.17%           git-svn  [kernel]                   [k] copy_page_range
     1.06%               git  [kernel]                   [k] copy_page_range
     0.99%               git  /lib64/ld-2.10.1.so        [.] do_lookup_x
     0.95%               git  /lib64/libc-2.10.1.so      [.] _int_malloc
     0.83%               git  [kernel]                   [k] get_page_from_freelist
     0.83%               git  /lib64/libc-2.10.1.so      [.] __GI__dl_addr
     0.82%               git  [kernel]                   [k] clear_page_c
     0.70%               git  [kernel]                   [k] kmem_cache_alloc
     0.65%               git  [kernel]                   [k] handle_mm_fault
     0.62%               git  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so  [.] Perl_yyparse
     0.60%           git-svn  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so  [.] Perl_yyparse
     0.59%               git  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so  [.] Perl_yylex
     0.58%               git  [kernel]                   [k] release_pages
     0.58%               git  [kernel]                   [k] page_remove_rmap
     0.57%               git  /bin/bash                  0x0000000004c2df
     0.55%           git-svn  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so  [.] Perl_yylex
     0.52%               git  [kernel]                   [k] find_vma
     0.52%               git  [kernel]                   [k] strnlen_user
     0.52%           git-svn  /lib64/libc-2.10.1.so      [.] _int_malloc

Interesting to see how after the change, perl is now looking like a fairly 
big part.

The big picture (not per-function, but per-program split by code segment: 
kernel, executable, library) shows the same thing. git does have a high 
kernel component in general, but something like "make test" makes it even 
bigger, since most of the costs are really forking a _lot_ of git 
programs:

- before:

    33.23%               git  [kernel]                 
    11.93%               git  /lib64/ld-2.10.1.so      
     7.55%           git-svn  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so
     6.82%               git  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so
     4.83%               git  /lib64/libc-2.10.1.so    
     3.28%           git-svn  [kernel]                 
     1.82%                sh  [kernel]                 
     1.57%               git  /bin/bash                
     1.52%              perl  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so
     1.37%           git-svn  /lib64/libc-2.10.1.so    
     1.28%              tput  [kernel]                 
     1.26%   git-filter-bran  [kernel]                 
     0.98%                rm  [kernel]                 
     0.97%               sed  [kernel]                 
     0.82%   git-rebase--int  [kernel]                 
     0.71%        git-bisect  [kernel]                 
     0.64%               git  ./git                    
     0.62%              grep  [kernel]                 
     0.55%               cat  [kernel]                 

 - after:

    30.30%               git  [kernel]                 
    10.62%           git-svn  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so
     9.77%               git  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so
     6.31%               git  /lib64/libc-2.10.1.so    
     4.31%           git-svn  [kernel]                 
     3.49%               git  /lib64/ld-2.10.1.so      
     2.17%               git  /bin/bash                
     2.10%              perl  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so
     1.93%           git-svn  /lib64/libc-2.10.1.so    
     1.90%                sh  [kernel]                 
     1.40%   git-filter-bran  [kernel]                 
     1.24%              tput  [kernel]                 
     0.95%               sed  [kernel]                 
     0.91%                rm  [kernel]                 
     0.89%               git  ./git                    
     0.84%   git-rebase--int  [kernel]                 
     0.82%   git-filter-bran  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so
     0.75%                sh  /usr/lib64/perl5/5.10.0/x86_64-linux-thread-multi/CORE/libperl.so
     0.59%                sh  /lib64/libc-2.10.1.so    
     0.57%              grep  [kernel]                 
     0.57%        git-bisect  [kernel]                 
     0.55%               cat  [kernel]                 

Note how the biggest user-space component used to be the dynamic loader. 
Now it's down there way below the perl overhead.

And notice how while the dynamic loader was "just" 11% of all overhead 
(and is still 3.5% after the fix), the reason performance has improved by 
30% is that the dynamic loader has a _huge_ kernel overhead due to the 
whole mmap/munmap/mprotect/page-fault-to-COW/etc code. 

		Linus

On Sat, Jul 25, 2009 at 12:25 PM, Linus
Torvalds[off-list ref] wrote:

On Sat, 25 Jul 2009, Daniel Barkalow wrote:

quoted

This is similar to, but different from, my git-vcs-* stuff; this one
expects the helper to be able to look up sha1s for refs efficiently, and
to be able to fetch objects directly into the local object database
without fast-import. On the otherhand, it should be sufficiently general
to allow arbitrary methods for moving the data around.

Wonderful.

I tested the series, and now there is (not surprisingly, but I made sure
to test anyway) no difference what-so-ever between NO_CURL and default,
and 'ldd' looks nice.

Plus it looks like that whole "shim" thing is a good idea in general, in
that it allows a much more flexible model for fetching/pushing.

So a very big Acked-by: from me for the series. I didn't test that http:
works with it, but I don't personally even care, so I'd ack it even
without that ;)

Btw, some real timing numbers for 'time make -j64 test':

 - before:
       real    1m16.070s
       user    2m47.046s
       sys     2m34.698s

 - after:
       real    0m58.851s
       user    1m57.087s
       sys     1m44.671s

so that's actually a real-world example of the whole 'scripting
performance'. Not an insignificant speedup on my machine (with an
obligatory "nyaah, nyaah, I can do the git test-suite under a minute" just
to rub peoples noses in the fact that my desktop computer is disgustingly
fast).

But more importantly, how fast can your machine compile the kernel?
(without ccache or any "cheats" like that, of course)

That's an almost 30% performance improvement, despite the fact that parts
of the test suite didn't actually change (shell costs are the same, the
svn tests are quite perl-intensive etc).

                       Linus

On Sat, 25 Jul 2009, Steven Noonan wrote:

But more importantly, how fast can your machine compile the kernel?
(without ccache or any "cheats" like that, of course)

Without ccache? One minute. With ccache? 16 seconds.

		Linus