[PATCH] Add '--create-index' to git-unpack-objects

Subsystems: documentation, the rest

DORMANTno replies

9 messages, 3 authors, 2016-06-15 · open the first message on its own page

[PATCH] Add '--create-index' to git-unpack-objects

From: Johannes Schindelin <hidden>
Date: 2016-06-15 22:42:08

Add the option '--create-index' to git-unpack-objects, which makes it 
create an index file instead of expanding its contents. While at it, 
document the dry-run option '-n', and optionally take a pack file instead 
of stdin.

Signed-off-by: Johannes Schindelin <redacted>

---

	Is anyone interested in writing the index to stdout? It is easy to
	add this feature to sha1create...

 Documentation/git-unpack-objects.txt |   13 +++-
 t/t5300-pack-object.sh               |    5 ++
 unpack-objects.c                     |  108 +++++++++++++++++++++++++++++++---
 3 files changed, 114 insertions(+), 12 deletions(-)

applies-to: 6dfbbc44a77ad3300caed532814e7feb02ff794b
1bff150c89d2636c54fa45bf62e7b248ba2a7b8d
diff --git a/Documentation/git-unpack-objects.txt b/Documentation/git-unpack-objects.txt
index b716ba1..58120e4 100644
--- a/Documentation/git-unpack-objects.txt
+++ b/Documentation/git-unpack-objects.txt
@@ -8,14 +8,15 @@ git-unpack-objects - Unpack objects from
 
 SYNOPSIS
 --------
-'git-unpack-objects' [-q] <pack-file
+'git-unpack-objects' [-q] [-n] [--create-index index-file] [pack-file]
 
 
 DESCRIPTION
 -----------
-Reads a packed archive (.pack) from the standard input, and
+Reads a packed archive (.pack) from a file or stdin, and
 expands the objects contained in the pack into "one-file
-one-object" format in $GIT_OBJECT_DIRECTORY.
+one-object" format in $GIT_OBJECT_DIRECTORY, or alternatively,
+create an index for it.
 
 OPTIONS
 -------
@@ -23,6 +24,12 @@ OPTIONS
 	The command usually shows percentage progress.  This
 	flag suppresses it.
 
+-n::
+	Perform a dry run, i.e. do not write any files.
+
+--create-index <filename>::
+	Instead of unpacking the files, create an index for this
+	pack.
 
 Author
 ------
diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh
index bb62336..593bfc2 100755
--- a/t/t5300-pack-object.sh
+++ b/t/t5300-pack-object.sh
@@ -79,6 +79,11 @@ test_expect_success \
      git-unpack-objects -n <test-2-${packname_2}.pack &&
      git-unpack-objects <test-2-${packname_2}.pack'
 
+test_expect_success \
+     'create-index from pack' \
+     'git-unpack-objects --create-index index <test-2-${packname_2}.pack &&
+     cmp test-2-${packname_2}.idx index'
+
 unset GIT_OBJECT_DIRECTORY
 cd $TRASH/.git2
 test_expect_success \
diff --git a/unpack-objects.c b/unpack-objects.c
index 8ae1a1c..b6dad74 100644
--- a/unpack-objects.c
+++ b/unpack-objects.c
@@ -2,17 +2,29 @@
 #include "object.h"
 #include "delta.h"
 #include "pack.h"
+#include "csum-file.h"
 
 #include <sys/time.h>
 
-static int dry_run, quiet;
-static const char unpack_usage[] = "git-unpack-objects [-q] < pack-file";
+static int dry_run, quiet, create_index;
+static const char unpack_usage[] = "git-unpack-objects [-n] [-q] [--create-index filename] [pack-file]";
 
 /* We always read in 4kB chunks. */
+static int fd_in = 0;
 static unsigned char buffer[4096];
+static off_t buffer_offset;
 static unsigned long offset, len, eof;
 static SHA_CTX ctx;
 
+/* To recreate an index */
+static const char* index_filename;
+typedef struct {
+	unsigned int offset;
+	unsigned char sha1[20];
+} object_entry;
+static object_entry* objects;
+static unsigned char *current_sha1;
+
 /*
  * Make sure at least "min" bytes are available in the buffer, and
  * return the pointer to the buffer.
@@ -28,10 +40,11 @@ static void * fill(int min)
 	if (offset) {
 		SHA1_Update(&ctx, buffer, offset);
 		memcpy(buffer, buffer + offset, len);
+		buffer_offset += offset;
 		offset = 0;
 	}
 	do {
-		int ret = read(0, buffer + len, sizeof(buffer) - len);
+		int ret = read(fd_in, buffer + len, sizeof(buffer) - len);
 		if (ret <= 0) {
 			if (!ret)
 				die("early EOF");
@@ -83,6 +96,7 @@ struct delta_info {
 	unsigned char base_sha1[20];
 	unsigned long size;
 	void *delta;
+	unsigned char* sha1;
 	struct delta_info *next;
 };
 
@@ -95,6 +109,7 @@ static void add_delta_to_list(unsigned c
 	memcpy(info->base_sha1, base_sha1, 20);
 	info->size = size;
 	info->delta = delta;
+	info->sha1 = current_sha1;
 	info->next = delta_list;
 	delta_list = info;
 }
@@ -104,7 +119,15 @@ static void added_object(unsigned char *
 static void write_object(void *buf, unsigned long size, const char *type)
 {
 	unsigned char sha1[20];
-	if (write_sha1_file(buf, size, type, sha1) < 0)
+	if (create_index) {
+		char header[100];
+		SHA_CTX c;
+
+		SHA1_Init(&c);
+		SHA1_Update(&c, header, 1+sprintf(header, "%s %lu", type, size));
+		SHA1_Update(&c, buf, size);
+		SHA1_Final(current_sha1, &c);
+	} else if (write_sha1_file(buf, size, type, sha1) < 0)
 		die("failed to write object");
 	added_object(sha1, type, buf, size);
 }
@@ -136,6 +159,7 @@ static void added_object(unsigned char *
 		if (!memcmp(info->base_sha1, sha1, 20)) {
 			*p = info->next;
 			p = &delta_list;
+			current_sha1 = info->sha1;
 			resolve_delta(type, data, size, info->delta, info->size);
 			free(info);
 			continue;
@@ -156,8 +180,10 @@ static int unpack_non_delta_entry(enum o
 	case OBJ_TAG:    type = "tag"; break;
 	default: die("bad type %d", kind);
 	}
+
 	if (!dry_run)
 		write_object(buf, size, type);
+
 	free(buf);
 	return 0;
 }
@@ -174,7 +200,7 @@ static int unpack_delta_entry(unsigned l
 	use(20);
 
 	delta_data = get_data(delta_size);
-	if (dry_run) {
+	if (dry_run && !create_index) {
 		free(delta_data);
 		return 0;
 	}
@@ -239,6 +265,40 @@ static void unpack_one(unsigned nr, unsi
 	}
 }
 
+int compare_object_entries(const void* a, const void* b)
+{
+	const object_entry *first = a;
+	const object_entry *second = b;
+
+	return memcmp(first->sha1, second->sha1, 20);
+}
+
+void show_index(unsigned int nr_objects, char* pack_sha1)
+{
+	unsigned int n, i;
+	unsigned int top_index[256];
+	struct sha1file* index_file;
+	
+	/* sort by sha1 */
+	qsort(objects, nr_objects, sizeof(object_entry), compare_object_entries);
+
+	for (n = i = 0; i < 256; i++) {
+		while (n < nr_objects && objects[n].sha1[0] == i)
+			n++;
+		top_index[i] = htonl(n);
+	}
+	top_index[255] = nr_objects;
+
+	/* write to file */
+	index_file = sha1create(index_filename);
+	sha1write(index_file, top_index, sizeof(top_index));
+	sha1write(index_file, objects, sizeof(object_entry)*nr_objects);
+	sha1write(index_file, pack_sha1, 20);
+	sha1close(index_file, NULL, 1);
+
+	free(objects);
+}
+
 /*
  * We unpack from the end, older files first. Now, usually
  * there are deltas etc, so we'll not actually write the
@@ -251,17 +311,29 @@ static void unpack_all(void)
 	unsigned version = ntohl(hdr->hdr_version);
 	unsigned nr_objects = ntohl(hdr->hdr_entries);
 
+	if (create_index)
+		objects = xmalloc(sizeof(object_entry)*nr_objects);
+
 	if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
 		die("bad pack file");
 	if (version != PACK_VERSION)
 		die("unable to handle pack file version %d", version);
-	fprintf(stderr, "Unpacking %d objects\n", nr_objects);
+	if (!quiet)
+		fprintf(stderr, "Unpacking %d objects\n", nr_objects);
 
 	use(sizeof(struct pack_header));
-	for (i = 0; i < nr_objects; i++)
+	for (i = 0; i < nr_objects; i++) {
+		if (create_index) {
+			objects[i].offset = htonl(buffer_offset+offset);
+			current_sha1 = (unsigned char*)&objects[i].sha1;
+		}
 		unpack_one(i+1, nr_objects);
+	}
 	if (delta_list)
 		die("unresolved deltas left after unpacking");
+
+	if (create_index)
+		show_index(nr_objects, fill(20));
 }
 
 int main(int argc, char **argv)
@@ -281,11 +353,22 @@ int main(int argc, char **argv)
 				quiet = 1;
 				continue;
 			}
+			if (!strcmp(arg, "--create-index")) {
+				create_index = 1;
+				if (i >= argc-1)
+					usage(unpack_usage);
+				index_filename = argv[++i];
+				continue;
+			}
 			usage(unpack_usage);
 		}
 
-		/* We don't take any non-flag arguments now.. Maybe some day */
-		usage(unpack_usage);
+		if (i != argc-1)
+			usage(unpack_usage);
+
+		fd_in = open(argv[i], O_RDONLY);
+		if (fd_in < 0)
+			die("Could not open %s: %s\n", argv[i], strerror(errno));
 	}
 	SHA1_Init(&ctx);
 	unpack_all();
@@ -295,6 +378,9 @@ int main(int argc, char **argv)
 		die("final sha1 did not match");
 	use(20);
 
+	if (create_index)
+		return 0;
+
 	/* Write the last part of the buffer to stdout */
 	while (len) {
 		int ret = write(1, buffer + offset, len);
@@ -312,5 +398,9 @@ int main(int argc, char **argv)
 	/* All done */
 	if (!quiet)
 		fprintf(stderr, "\n");
+
+	if (fd_in)
+		close(fd_in);
+
 	return 0;
 }
---
0.99.8.GIT

Re: [PATCH] Add '--create-index' to git-unpack-objects

From: Sergey Vlasov <hidden>
Date: 2016-06-15 22:42:08

On Wed, 12 Oct 2005 13:02:36 +0200 (CEST) Johannes Schindelin wrote:
Add the option '--create-index' to git-unpack-objects, which makes it 
create an index file instead of expanding its contents. While at it, 
document the dry-run option '-n', and optionally take a pack file instead 
of stdin.
quoted hunk
@@ -104,7 +119,15 @@ static void added_object(unsigned char *
 static void write_object(void *buf, unsigned long size, const char *type)
 {
 	unsigned char sha1[20];
-	if (write_sha1_file(buf, size, type, sha1) < 0)
+	if (create_index) {
+		char header[100];
+		SHA_CTX c;
+
+		SHA1_Init(&c);
+		SHA1_Update(&c, header, 1+sprintf(header, "%s %lu", type, size));
+		SHA1_Update(&c, buf, size);
+		SHA1_Final(current_sha1, &c);
+	} else if (write_sha1_file(buf, size, type, sha1) < 0)
 		die("failed to write object");
Sorry, but this cannot work.  git-unpack-objects does a streaming
unpack, and it needs to be able to read back the objects it has written
out previously (in case a delta later in the stream references some
older object).  Saving unpacked objects in memory would obviously be
unacceptable.

However, if you need to create a pack index, you obviously have a pack
file with random access ability, and in this case it is possible to
build the index efficiently (in two passes over the pack file) without
storing unpacked objects in the filesystem.  I made a separate utility
to do this; will send a patch in some minutes.

[PATCH] Add git-index-pack utility

From: Sergey Vlasov <hidden>
Date: 2016-06-15 22:42:08

git-index-pack builds a pack index file for an existing packed
archive.  With this utility a packed archive which was transferred
without the corresponding pack index can be added to objects/pack/
without repacking.

Signed-off-by: Sergey Vlasov <redacted>


---

 Documentation/git-index-pack.txt |   44 ++++
 Documentation/git.txt            |    3 
 Makefile                         |    2 
 index-pack.c                     |  451 ++++++++++++++++++++++++++++++++++++++
 t/t5300-pack-object.sh           |   18 ++
 5 files changed, 517 insertions(+), 1 deletions(-)
 create mode 100644 Documentation/git-index-pack.txt
 create mode 100644 index-pack.c

230c0068103bd1ed87788a6464dda2d7de58c2e2
diff --git a/Documentation/git-index-pack.txt b/Documentation/git-index-pack.txt
new file mode 100644
--- /dev/null
+++ b/Documentation/git-index-pack.txt
@@ -0,0 +1,44 @@
+git-index-pack(1)
+=================
+
+NAME
+----
+git-index-pack - Build pack index file for an existing packed archive
+
+
+SYNOPSIS
+--------
+'git-index-pack' [-o <index-file>] <pack-file>
+
+
+DESCRIPTION
+-----------
+Reads a packed archive (.pack) from the specified file, and
+builds a pack index file (.idx) for it.  The packed archive
+together with the pack index can then be placed in the
+objects/pack/ directory of a git repository.
+
+
+OPTIONS
+-------
+-o <index-file>::
+	Write the generated pack index into the specified
+	file.  Without this option the name of pack index
+	file is constructed from the name of packed archive
+	file by replacing .pack with .idx (and the program
+	fails if the name of packed archive does not end
+	with .pack).
+
+
+Author
+------
+Written by Sergey Vlasov <vsu@altlinux.ru>
+
+Documentation
+-------------
+Documentation by Sergey Vlasov
+
+GIT
+---
+Part of the gitlink:git[7] suite
+
diff --git a/Documentation/git.txt b/Documentation/git.txt
--- a/Documentation/git.txt
+++ b/Documentation/git.txt
@@ -68,6 +68,9 @@ gitlink:git-commit-tree[1]::
 gitlink:git-hash-object[1]::
 	Computes the object ID from a file.
 
+gitlink:git-index-pack.html[1]::
+	Build pack index file for an existing packed archive.
+
 gitlink:git-init-db[1]::
 	Creates an empty git object database
 
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -110,7 +110,7 @@ PROGRAMS = \
 	git-convert-objects$X git-diff-files$X \
 	git-diff-index$X git-diff-stages$X \
 	git-diff-tree$X git-fetch-pack$X git-fsck-objects$X \
-	git-hash-object$X git-init-db$X \
+	git-hash-object$X git-index-pack$X git-init-db$X \
 	git-local-fetch$X git-ls-files$X git-ls-tree$X git-merge-base$X \
 	git-merge-index$X git-mktag$X git-pack-objects$X git-patch-id$X \
 	git-peek-remote$X git-prune-packed$X git-read-tree$X \
diff --git a/index-pack.c b/index-pack.c
new file mode 100644
--- /dev/null
+++ b/index-pack.c
@@ -0,0 +1,451 @@
+#include "cache.h"
+#include "delta.h"
+#include "pack.h"
+#include "csum-file.h"
+
+static const char index_pack_usage[] =
+"git-index-pack [-o index-file] pack-file";
+
+struct object_entry
+{
+	unsigned long offset;
+	enum object_type type;
+	enum object_type real_type;
+	unsigned char sha1[20];
+};
+
+struct delta_entry
+{
+	struct object_entry *obj;
+	unsigned char base_sha1[20];
+};
+
+static const char *pack_name;
+static unsigned char *pack_base;
+static unsigned long pack_size;
+static struct object_entry *objects;
+static struct delta_entry *deltas;
+static int nr_objects;
+static int nr_deltas;
+
+static void open_pack_file(void)
+{
+	int fd;
+	struct stat st;
+
+	fd = open(pack_name, O_RDONLY);
+	if (fd < 0)
+		die("cannot open packfile '%s': %s", pack_name,
+		    strerror(errno));
+	if (fstat(fd, &st)) {
+		int err = errno;
+		close(fd);
+		die("cannot fstat packfile '%s': %s", pack_name,
+		    strerror(err));
+	}
+	pack_size = st.st_size;
+	pack_base = mmap(NULL, pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (pack_base == MAP_FAILED) {
+		int err = errno;
+		close(fd);
+		die("cannot mmap packfile '%s': %s", pack_name,
+		    strerror(err));
+	}
+	close(fd);
+}
+
+static void parse_pack_header(void)
+{
+	const struct pack_header *hdr;
+	unsigned char sha1[20];
+	SHA_CTX ctx;
+
+	/* Ensure there are enough bytes for the header and final SHA1 */
+	if (pack_size < sizeof(struct pack_header) + 20)
+		die("packfile '%s' is too small", pack_name);
+
+	/* Header consistency check */
+	hdr = (void *)pack_base;
+	if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
+		die("packfile '%s' signature mismatch", pack_name);
+	if (hdr->hdr_version != htonl(PACK_VERSION))
+		die("packfile '%s' version %d different from ours %d",
+		    pack_name, ntohl(hdr->hdr_version), PACK_VERSION);
+
+	nr_objects = ntohl(hdr->hdr_entries);
+
+	/* Check packfile integrity */
+	SHA1_Init(&ctx);
+	SHA1_Update(&ctx, pack_base, pack_size - 20);
+	SHA1_Final(sha1, &ctx);
+	if (memcmp(sha1, pack_base + pack_size - 20, 20))
+		die("packfile '%s' SHA1 mismatch", pack_name);
+}
+
+static void bad_object(unsigned long offset, const char *format,
+		       ...) NORETURN __attribute__((format (printf, 2, 3)));
+
+static void bad_object(unsigned long offset, const char *format, ...)
+{
+	va_list params;
+	char buf[1024];
+
+	va_start(params, format);
+	vsnprintf(buf, sizeof(buf), format, params);
+	va_end(params);
+	die("packfile '%s': bad object at offset %lu: %s",
+	    pack_name, offset, buf);
+}
+
+static void *unpack_entry_data(unsigned long offset,
+			       unsigned long *current_pos, unsigned long size)
+{
+	unsigned long pack_limit = pack_size - 20;
+	unsigned long pos = *current_pos;
+	z_stream stream;
+	void *buf = xmalloc(size);
+
+	memset(&stream, 0, sizeof(stream));
+	stream.next_out = buf;
+	stream.avail_out = size;
+	stream.next_in = pack_base + pos;
+	stream.avail_in = pack_limit - pos;
+	inflateInit(&stream);
+
+	for (;;) {
+		int ret = inflate(&stream, 0);
+		if (ret == Z_STREAM_END)
+			break;
+		if (ret != Z_OK)
+			bad_object(offset, "inflate returned %d", ret);
+	}
+	inflateEnd(&stream);
+	if (stream.total_out != size)
+		bad_object(offset, "size mismatch (expected %lu, got %lu)",
+			   size, stream.total_out);
+	*current_pos = pack_limit - stream.avail_in;
+	return buf;
+}
+
+static void *unpack_raw_entry(unsigned long offset,
+			      enum object_type *obj_type,
+			      unsigned long *obj_size,
+			      unsigned char *delta_base,
+			      unsigned long *next_obj_offset)
+{
+	unsigned long pack_limit = pack_size - 20;
+	unsigned long pos = offset;
+	unsigned char c;
+	unsigned long size;
+	unsigned shift;
+	enum object_type type;
+	void *data;
+
+	c = pack_base[pos++];
+	type = (c >> 4) & 7;
+	size = (c & 15);
+	shift = 4;
+	while (c & 0x80) {
+		if (pos >= pack_limit)
+			bad_object(offset, "object extends past end of pack");
+		c = pack_base[pos++];
+		size += (c & 0x7fUL) << shift;
+		shift += 7;
+	}
+
+	switch (type) {
+	case OBJ_DELTA:
+		if (pos + 20 >= pack_limit)
+			bad_object(offset, "object extends past end of pack");
+		memcpy(delta_base, pack_base + pos, 20);
+		pos += 20;
+		/* fallthru */
+	case OBJ_COMMIT:
+	case OBJ_TREE:
+	case OBJ_BLOB:
+	case OBJ_TAG:
+		data = unpack_entry_data(offset, &pos, size);
+		break;
+	default:
+		bad_object(offset, "bad object type %d", type);
+	}
+
+	*obj_type = type;
+	*obj_size = size;
+	*next_obj_offset = pos;
+	return data;
+}
+
+static int find_delta(const unsigned char *base_sha1)
+{
+	int first = 0, last = nr_deltas;
+
+        while (first < last) {
+                int next = (first + last) / 2;
+                struct delta_entry *delta = &deltas[next];
+                int cmp;
+
+                cmp = memcmp(base_sha1, delta->base_sha1, 20);
+                if (!cmp)
+                        return next;
+                if (cmp < 0) {
+                        last = next;
+                        continue;
+                }
+                first = next+1;
+        }
+        return -first-1;
+}
+
+static int find_deltas_based_on_sha1(const unsigned char *base_sha1,
+				     int *first_index, int *last_index)
+{
+	int first = find_delta(base_sha1);
+	int last = first;
+	int end = nr_deltas - 1;
+
+	if (first < 0)
+		return -1;
+	while (first > 0 && !memcmp(deltas[first-1].base_sha1, base_sha1, 20))
+		--first;
+	while (last < end && !memcmp(deltas[last+1].base_sha1, base_sha1, 20))
+		++last;
+	*first_index = first;
+	*last_index = last;
+	return 0;
+}
+
+static void sha1_object(const void *data, unsigned long size,
+			enum object_type type, unsigned char *sha1)
+{
+	SHA_CTX ctx;
+	char header[50];
+	int header_size;
+	const char *type_str;
+
+	switch (type) {
+	case OBJ_COMMIT: type_str = "commit"; break;
+	case OBJ_TREE:   type_str = "tree"; break;
+	case OBJ_BLOB:   type_str = "blob"; break;
+	case OBJ_TAG:    type_str = "tag"; break;
+	default:
+		die("bad type %d", type);
+	}
+
+	header_size = sprintf(header, "%s %lu", type_str, size) + 1;
+
+	SHA1_Init(&ctx);
+	SHA1_Update(&ctx, header, header_size);
+	SHA1_Update(&ctx, data, size);
+	SHA1_Final(sha1, &ctx);
+}
+
+static void resolve_delta(struct delta_entry *delta, void *base_data,
+			  unsigned long base_size, enum object_type type)
+{
+	struct object_entry *obj = delta->obj;
+	void *delta_data;
+	unsigned long delta_size;
+	void *result;
+	unsigned long result_size;
+	enum object_type delta_type;
+	unsigned char base_sha1[20];
+	unsigned long next_obj_offset;
+	int j, first, last;
+
+	obj->real_type = type;
+	delta_data = unpack_raw_entry(obj->offset, &delta_type,
+				      &delta_size, base_sha1,
+				      &next_obj_offset);
+	result = patch_delta(base_data, base_size, delta_data, delta_size,
+			     &result_size);
+	free(delta_data);
+	if (!result)
+		bad_object(obj->offset, "failed to apply delta");
+	sha1_object(result, result_size, type, obj->sha1);
+	if (!find_deltas_based_on_sha1(obj->sha1, &first, &last)) {
+		for (j = first; j <= last; j++)
+			resolve_delta(&deltas[j], result, result_size, type);
+	}
+	free(result);
+}
+
+static int compare_delta_entry(const void *a, const void *b)
+{
+	const struct delta_entry *delta_a = a;
+	const struct delta_entry *delta_b = b;
+	return memcmp(delta_a->base_sha1, delta_b->base_sha1, 20);
+}
+
+static void parse_pack_objects(void)
+{
+	int i;
+	unsigned long offset = sizeof(struct pack_header);
+	unsigned char base_sha1[20];
+	void *data;
+	unsigned long data_size;
+
+	/*
+	 * First pass:
+	 * - find locations of all objects;
+	 * - calculate SHA1 of all non-delta objects;
+	 * - remember base SHA1 for all deltas.
+	 */
+	for (i = 0; i < nr_objects; i++) {
+		struct object_entry *obj = &objects[i];
+		obj->offset = offset;
+		data = unpack_raw_entry(offset, &obj->type, &data_size,
+					base_sha1, &offset);
+		obj->real_type = obj->type;
+		if (obj->type == OBJ_DELTA) {
+			struct delta_entry *delta = &deltas[nr_deltas++];
+			delta->obj = obj;
+			memcpy(delta->base_sha1, base_sha1, 20);
+		} else
+			sha1_object(data, data_size, obj->type, obj->sha1);
+		free(data);
+	}
+	if (offset != pack_size - 20)
+		die("packfile '%s' has junk at the end", pack_name);
+
+	/* Sort deltas by base SHA1 for fast searching */
+	qsort(deltas, nr_deltas, sizeof(struct delta_entry),
+	      compare_delta_entry);
+
+	/*
+	 * Second pass:
+	 * - for all non-delta objects, look if it is used as a base for
+	 *   deltas;
+	 * - if used as a base, uncompress the object and apply all deltas,
+	 *   recursively checking if the resulting object is used as a base
+	 *   for some more deltas.
+	 */
+	for (i = 0; i < nr_objects; i++) {
+		struct object_entry *obj = &objects[i];
+		int j, first, last;
+
+		if (obj->type == OBJ_DELTA)
+			continue;
+		if (find_deltas_based_on_sha1(obj->sha1, &first, &last))
+			continue;
+		data = unpack_raw_entry(obj->offset, &obj->type, &data_size,
+					base_sha1, &offset);
+		for (j = first; j <= last; j++)
+			resolve_delta(&deltas[j], data, data_size, obj->type);
+		free(data);
+	}
+
+	/* Check for unresolved deltas */
+	for (i = 0; i < nr_deltas; i++) {
+		if (deltas[i].obj->real_type == OBJ_DELTA)
+			die("packfile '%s' has unresolved deltas",  pack_name);
+	}
+}
+
+static int sha1_compare(const void *_a, const void *_b)
+{
+	struct object_entry *a = *(struct object_entry **)_a;
+	struct object_entry *b = *(struct object_entry **)_b;
+	return memcmp(a->sha1, b->sha1, 20);
+}
+
+static void write_index_file(const char *index_name)
+{
+	struct sha1file *f;
+	struct object_entry **sorted_by_sha =
+		xcalloc(nr_objects, sizeof(struct object_entry *));
+	struct object_entry **list = sorted_by_sha;
+	struct object_entry **last = sorted_by_sha + nr_objects;
+	unsigned int array[256];
+	int i;
+
+	for (i = 0; i < nr_objects; ++i)
+		sorted_by_sha[i] = &objects[i];
+	qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
+	      sha1_compare);
+
+	unlink(index_name);
+	f = sha1create("%s", index_name);
+
+	/*
+	 * Write the first-level table (the list is sorted,
+	 * but we use a 256-entry lookup to be able to avoid
+	 * having to do eight extra binary search iterations).
+	 */
+	for (i = 0; i < 256; i++) {
+		struct object_entry **next = list;
+		while (next < last) {
+			struct object_entry *obj = *next;
+			if (obj->sha1[0] != i)
+				break;
+			next++;
+		}
+		array[i] = htonl(next - sorted_by_sha);
+		list = next;
+	}
+	sha1write(f, array, 256 * sizeof(int));
+
+	/*
+	 * Write the actual SHA1 entries..
+	 */
+	list = sorted_by_sha;
+	for (i = 0; i < nr_objects; i++) {
+		struct object_entry *obj = *list++;
+		unsigned int offset = htonl(obj->offset);
+		sha1write(f, &offset, 4);
+		sha1write(f, obj->sha1, 20);
+	}
+	sha1write(f, pack_base + pack_size - 20, 20);
+	sha1close(f, NULL, 1);
+	free(sorted_by_sha);
+}
+
+int main(int argc, char **argv)
+{
+	int i;
+	char *index_name = NULL;
+	char *index_name_buf = NULL;
+
+	for (i = 1; i < argc; i++) {
+		const char *arg = argv[i];
+
+		if (*arg == '-') {
+			if (!strcmp(arg, "-o")) {
+				if (index_name || (i+1) >= argc)
+					usage(index_pack_usage);
+				index_name = argv[++i];
+			} else
+				usage(index_pack_usage);
+			continue;
+		}
+
+		if (pack_name)
+			usage(index_pack_usage);
+		pack_name = arg;
+	}
+
+	if (!pack_name)
+		usage(index_pack_usage);
+	if (!index_name) {
+		int len = strlen(pack_name);
+		if (len < 5 || strcmp(pack_name + len - 5, ".pack"))
+			die("packfile name '%s' does not end with '.pack'",
+			    pack_name);
+		index_name_buf = xmalloc(len - 1);
+		memcpy(index_name_buf, pack_name, len - 5);
+		strcpy(index_name_buf + len - 5, ".idx");
+		index_name = index_name_buf;
+	}
+
+	open_pack_file();
+	parse_pack_header();
+	objects = xcalloc(nr_objects, sizeof(struct object_entry));
+	deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
+	parse_pack_objects();
+	free(deltas);
+	write_index_file(index_name);
+	free(objects);
+	free(index_name_buf);
+
+	return 0;
+}
diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh
--- a/t/t5300-pack-object.sh
+++ b/t/t5300-pack-object.sh
@@ -165,4 +165,22 @@ test_expect_success \
 
      :'
 
+test_expect_success \
+    'build pack index for an existing pack' \
+    'cp test-1-${packname_1}.pack test-3.pack &&
+     git-index-pack -o tmp.idx test-3.pack &&
+     cmp tmp.idx test-1-${packname_1}.idx &&
+
+     git-index-pack test-3.pack &&
+     cmp test-3.idx test-1-${packname_1}.idx &&
+
+     cp test-2-${packname_2}.pack test-3.pack &&
+     git-index-pack -o tmp.idx test-2-${packname_2}.pack &&
+     cmp tmp.idx test-2-${packname_2}.idx &&
+
+     git-index-pack test-3.pack &&
+     cmp test-3.idx test-2-${packname_2}.idx &&
+
+     :'
+
 test_done

Re: [PATCH] Add '--create-index' to git-unpack-objects

From: Johannes Schindelin <hidden>
Date: 2016-06-15 22:42:08

Hi,

On Wed, 12 Oct 2005, Sergey Vlasov wrote:
On Wed, 12 Oct 2005 13:02:36 +0200 (CEST) Johannes Schindelin wrote:
quoted
 static void write_object(void *buf, unsigned long size, const char *type)
 {
 	unsigned char sha1[20];
-	if (write_sha1_file(buf, size, type, sha1) < 0)
+	if (create_index) {
+		char header[100];
+		SHA_CTX c;
+
+		SHA1_Init(&c);
+		SHA1_Update(&c, header, 1+sprintf(header, "%s %lu", type, size));
+		SHA1_Update(&c, buf, size);
+		SHA1_Final(current_sha1, &c);
+	} else if (write_sha1_file(buf, size, type, sha1) < 0)
 		die("failed to write object");
Sorry, but this cannot work.  git-unpack-objects does a streaming
unpack, and it needs to be able to read back the objects it has written
out previously (in case a delta later in the stream references some
older object).
Even worse, my code did not anticipate that the base objects could have 
been handled earlier (and thus the deltas would never be resolved).
Saving unpacked objects in memory would obviously be unacceptable.
Actually, this is what git-unpack-objects does. All unresolved deltas are 
stored in a linked list, and handled later.

Of course, it would be nicer to use a seekable file if you have one. But 
then, I am not at all sure that base objects should be allowed to come 
later in the file: since the delta chains must not be cyclic, the objects
can be sorted. Thus, it could be guaranteed that the base objects are 
already unpacked when unpacking the derived object.

Ciao,
Dscho

Re: [PATCH] Add git-index-pack utility

From: Johannes Schindelin <hidden>
Date: 2016-06-15 22:42:08

Hi,

you cheated! You use mmap(), not lseek()! Note that mmap() is more 
efficient only if the platform provides mmap()...

Ciao,
Dscho

Re: [PATCH] Add '--create-index' to git-unpack-objects

From: Sergey Vlasov <hidden>
Date: 2016-06-15 22:42:08

On Wed, Oct 12, 2005 at 04:25:22PM +0200, Johannes Schindelin wrote:
On Wed, 12 Oct 2005, Sergey Vlasov wrote:
quoted
Saving unpacked objects in memory would obviously be unacceptable.
Actually, this is what git-unpack-objects does. All unresolved deltas are 
stored in a linked list, and handled later.
Yes, this may be a problem if the pack is large and contains many
deltas.  But these stored deltas are thrown away immediately when the
base object is found; if you want to implement a streaming reindex,
you will need to store them until you reach the end of pack (or write
those objects to some temporary files).
Of course, it would be nicer to use a seekable file if you have one. But 
then, I am not at all sure that base objects should be allowed to come 
later in the file: since the delta chains must not be cyclic, the objects
can be sorted. Thus, it could be guaranteed that the base objects are 
already unpacked when unpacking the derived object.
Hmm, pack-objects.c:write_one() does exactly the opposite - it writes
the base object _after_ writing out the delta (but it does not ensure
that ordering completely, so references to base objects can be
pointing in both directions).  Why?

Re: [PATCH] Add git-index-pack utility

From: Sergey Vlasov <hidden>
Date: 2016-06-15 22:42:08

On Wed, Oct 12, 2005 at 04:33:17PM +0200, Johannes Schindelin wrote:
you cheated! You use mmap(), not lseek()! Note that mmap() is more 
efficient only if the platform provides mmap()...
Yes, I need to steal some code for handling the deflated streams from
unpack-objects.c...

Re: [PATCH] Add '--create-index' to git-unpack-objects

From: Johannes Schindelin <hidden>
Date: 2016-06-15 22:42:08

Hi,

On Wed, 12 Oct 2005, Sergey Vlasov wrote:
Hmm, pack-objects.c:write_one() does exactly the opposite - it writes
the base object _after_ writing out the delta (but it does not ensure
that ordering completely, so references to base objects can be
pointing in both directions).  Why?
Okay, I did not read that far. However, having quite a few packs out there 
in this format, there is no option to change the format now.

Ciao,
Dscho

Re: [PATCH] Add '--create-index' to git-unpack-objects

From: Linus Torvalds <torvalds@osdl.org>
Date: 2016-06-15 22:42:08


On Wed, 12 Oct 2005, Sergey Vlasov wrote:
Hmm, pack-objects.c:write_one() does exactly the opposite - it writes
the base object _after_ writing out the delta (but it does not ensure
that ordering completely, so references to base objects can be
pointing in both directions).  Why?
pack-objects.c is actually going to some trouble to make sure that the 
resulting pack is "optimal" in layout for the most recent case.

Not that I have actually verified optimality, but it was _meant_ to be 
that way. And my limited tests seemed to agree.

So it writes out all objects in "recency order", which is the order it 
gets them from git-rev-list: it's the same order as the objects are 
discovered when we traverse the history in time (except all commits come 
first, since most operations will traverse the commit history more than 
they will traverse the rest of the object links).

So the objects that are reachable in the most recent tree are all supposed 
to be at the beginning of the pack-file, just after the commits.

Now, think about what happens if such an object is a delta against 
something else...

In other words, if the most recent tree contains a delta against a much 
older object, we want not only the _delta_ to be early in the pack-file, 
we want the object that it is a delta _against_ to be there too (just 
_after_ the delta, to be exact: we obviously read the delta first, so it 
should come first in the pack).

The point being, that if you unpack the latest tree (ie "git checkout" or 
any of the normal "git diff" behaviour), the pack-file will basically be 
walked in a dense manner, and linearly starting roughly from the 
beginning. Which is the optimal IO pattern. Dense and ascending reads.

Now, if the object is reachable through some recent branch, but the delta 
is not, then that is not true. In that case, you want to write the recent 
base object early in the pack-file, but you do _not_ want to write the 
delta together with it, because that would be the wrong thing for the 
"recent head" case: it would add stuff to the beginning of the pack-file 
that isn't needed for recent objects.

So that's why it's an assymmetric thing. The preferred ordering of time 
breaks the symmetry.

			Linus
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help