Re: [PATCH] repack: rewrite the shell script in C.

5 messages, 3 authors, 2016-06-15 · open the first message on its own page

Re: [PATCH] repack: rewrite the shell script in C.

From: Matthieu Moy <hidden>
Date: 2016-06-15 22:58:28

Stefan Beller [off-list ref] writes:
All tests are constantly positive now.
Cool!
+/*
+ * Fills the filename list with all the files found in the pack directory
Detail: "filename list" could be "fname_list" to match the actual
argument below.
+ * ending with .pack, without that extension.
+ */
+void get_pack_filenames(char *packdir, struct string_list *fname_list)
+{
+	DIR *dir;
+	struct dirent *e;
+	char *path, *suffix, *fname;
+
+	path = mkpath("%s/pack", get_object_directory());
+	suffix = ".pack";
+
+	dir = opendir(path);
I think you should test and complain if dir is NULL ("cannot open pack
directory: ...")
+void remove_pack(char *path, char* sha1)
+{
+	char *exts[] = {".pack", ".idx", ".keep"};
+	int ext = 0;
+	for (ext = 0; ext < 3; ext++) {
+		char *fname;
+		fname = mkpath("%s/%s%s", path, sha1, exts[ext]);
+		unlink(fname);
Here also, the return value from unlink is not checked. Probably not
serious (mistakenly deleting a pack file would be very serious, but
keeping it around by mistake shouldn't harm), but a warning message may
be welcome.

These kinds of warnings are never shown in normal usage, but may be
welcome when something goes really wrong with the repo, as a diagnosis
tool for the user. The shell version had these warnings implicitly since
"rm" displays the message on stderr when it fails.
+	struct child_process cmd;
+	struct argv_array cmd_args = ARGV_ARRAY_INIT;
Since the command is going to be "pack-objects", you may call the
variables pack_cmd and pack_cmd_args or so.
+	if (local)
+		argv_array_push(&cmd_args,  "--local");
+	if (quiet)
+		argv_array_push(&cmd_args,  "--quiet");
+	if (delta_base_offset)
+		argv_array_push(&cmd_args,  "--delta-base-offset");
+
+	argv_array_push(&cmd_args, packtmp);
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.argv = cmd_args.argv;
+	cmd.git_cmd = 1;
+	cmd.out = -1;
+	cmd.no_stdin = 1;
+
+	if (start_command(&cmd))
+		return 1;
A warning message would be welcome in addition to returning 1.
+	if (!count_packs && !quiet)
+		printf("Nothing new to pack.\n");
+
+	int failed = 0;
Don't declare variables inside code, it's not C90.
+	for_each_string_list_item(item, &names) {
+		for (ext = 0; ext < 2; ext++) {
+			char *fname, *fname_old;
+			fname = mkpathdup("%s/%s%s", packdir, item->string, exts[ext]);
+			if (!file_exists(fname)) {
+				free(fname);
+				continue;
+			}
+
+			fname_old = mkpath("%s/old-%s%s", packdir, item->string, exts[ext]);
+			if (file_exists(fname_old))
+				unlink(fname_old);
Unchecked returned value.
+			if (rename(fname, fname_old)) {
+				failed = 1;
+				break;
+			}
+			string_list_append_nodup(&rollback, fname);
+			free(fname);
+		}
+		if (failed)
+			break;
+	}
I tend to dislike these "set a variable and break twice" to exit nested
loops. Using an auxiliary function, you could just do

int f()
{
	for_each {
		for () {
			...
			if ()
				return 1;
			...
		}
	}
	return 0;
}

(Matter of taste, though. Some people may disagree)

A good side effect would be to move some code out of cmd_repack, which
is rather long.
+	/* Now the ones with the same name are out of the way... */
+	for_each_string_list_item(item, &names) {
+		for (ext = 0; ext < 2; ext++) {
+			char *fname, *fname_old;
+			struct stat statbuffer;
+			fname = mkpathdup("%s/pack-%s%s", packdir, item->string, exts[ext]);
+			fname_old = mkpath("%s-%s%s", packtmp, item->string, exts[ext]);
+			if (!stat(fname_old, &statbuffer)) {
+				statbuffer.st_mode &= ~S_IWUSR | ~S_IWGRP | ~S_IWOTH;
+				chmod(fname_old, statbuffer.st_mode);
Unchecked return value.
+	/* Remove the "old-" files */
+	for_each_string_list_item(item, &names) {
+		char *fname;
+		fname = mkpath("%s/old-pack-%s.idx", packdir, item->string);
+		if (remove_path(fname))
+			die_errno(_("removing '%s' failed"), fname);
+
+		fname = mkpath("%s/old-pack-%s.pack", packdir, item->string);
+		if (remove_path(fname))
+			die_errno(_("removing '%s' failed"), fname);
Does this have to be a fatal error? If I read correctly, it wasn't fatal
in the shell version.

Any reason why you duplicate the code for .idx and .pack here, while you
iterate over an ext array in other places of the code?
+	if (delete_redundant) {
+		sort_string_list(&names);
+		for_each_string_list_item(item, &existing_packs) {
+			char *sha1;
+			size_t len = strlen(item->string);
+			if (len < 40)
+				continue;
+			sha1 = item->string + len - 40;
+			if (!string_list_has_string(&names, sha1))
+				remove_pack(packdir, item->string);
+		}
+		argv_array_clear(&cmd_args);
+		argv_array_push(&cmd_args, "prune-packed");
+		if (quiet)
+			argv_array_push(&cmd_args, "--quiet");
+
+		memset(&cmd, 0, sizeof(cmd));
+		cmd.argv = cmd_args.argv;
+		cmd.git_cmd = 1;
+		run_command(&cmd);
+	}
It's tempting to call prune_packed_objects() directly here, but it's
implemented in builtin/ so it would require a refactoring patch to be
moved to libgit.a before I guess.
+	if (!no_update_server_info) {
+		argv_array_clear(&cmd_args);
+		argv_array_push(&cmd_args, "update-server-info");
+
+		memset(&cmd, 0, sizeof(cmd));
+		cmd.argv = cmd_args.argv;
+		cmd.git_cmd = 1;
+		run_command(&cmd);
+	}
+	return 0;
+}
Any reason to fork a new process instead of just calling
update_server_info() directly?

Not that efficiency matters here, but the code would be a bit simpler.

-- 
Matthieu Moy
http://www-verimag.imag.fr/~moy/

Re: [PATCH] repack: rewrite the shell script in C.

From: Stefan Beller <hidden>
Date: 2016-06-15 22:58:28

On 08/21/2013 10:49 AM, Matthieu Moy wrote:
quoted
+	if (start_command(&cmd))
quoted
+		return 1;
A warning message would be welcome in addition to returning 1.
Johannes Sixt proposes to retain the return value of
the sub process, which I'd agree on.
However why do we need a warning message here?

I'd expect the pack-objects to bring up the warning as
the stderr is untouched in the command invocation.
And we already passed either --quiet or not, so pack-objects
should know how to behave on its own.
Also it is a builtin command, so we do not need to check
if it is found or not, so I'd strongly rely on the error
and warning reporting from the underlying process, no?

Thanks,
Stefan

Re: [PATCH] repack: rewrite the shell script in C.

From: Stefan Beller <hidden>
Date: 2016-06-15 22:58:28

On 08/21/2013 10:49 AM, Matthieu Moy wrote:
I tend to dislike these "set a variable and break twice" to exit nested
loops. Using an auxiliary function, you could just do

int f()
{
	for_each {
		for () {
			...
			if ()
				return 1;
			...
		}
	}
	return 0;
}

(Matter of taste, though. Some people may disagree)

A good side effect would be to move some code out of cmd_repack, which
is rather long.
Thank you very much for the review, it helps me very much to focus
on the small details.

I intend to have the C code in this patch as close to the
shell version as possible. This goes both for functionality as
well as style/organisation within the file.

All the additional changes, such as this one
(Or in the previous mail, retaining the error code of subprocesses)
I'd like to put in small follow up patches changing just one thing
at a time.

But as these follow up changes heavily rely on the very first patch
I will first try to get that right, meaning accepted into pu.
Then I can send patches with these proposals such as making more
functions.

Thanks,
Stefan

Re: [PATCH] repack: rewrite the shell script in C.

From: Johannes Sixt <hidden>
Date: 2016-06-15 22:58:29

Am 21.08.2013 10:49, schrieb Matthieu Moy:
Stefan Beller [off-list ref] writes:
quoted
+	for_each_string_list_item(item, &names) {
+		for (ext = 0; ext < 2; ext++) {
+			char *fname, *fname_old;
+			fname = mkpathdup("%s/%s%s", packdir, item->string, exts[ext]);
+			if (!file_exists(fname)) {
+				free(fname);
+				continue;
+			}
+
+			fname_old = mkpath("%s/old-%s%s", packdir, item->string, exts[ext]);
+			if (file_exists(fname_old))
+				unlink(fname_old);
Unchecked returned value.
Good catch! The original was 'rm -f ... && mv ... || failed=t'
quoted
+	/* Now the ones with the same name are out of the way... */
+	for_each_string_list_item(item, &names) {
+		for (ext = 0; ext < 2; ext++) {
+			char *fname, *fname_old;
+			struct stat statbuffer;
+			fname = mkpathdup("%s/pack-%s%s", packdir, item->string, exts[ext]);
+			fname_old = mkpath("%s-%s%s", packtmp, item->string, exts[ext]);
+			if (!stat(fname_old, &statbuffer)) {
+				statbuffer.st_mode &= ~S_IWUSR | ~S_IWGRP | ~S_IWOTH;
+				chmod(fname_old, statbuffer.st_mode);
Unchecked return value.
The original was an unchecked 'chmod a-w', so we don't care.

Of course, we could mimic the original better by issuing warnings.
quoted
+	/* Remove the "old-" files */
+	for_each_string_list_item(item, &names) {
+		char *fname;
+		fname = mkpath("%s/old-pack-%s.idx", packdir, item->string);
+		if (remove_path(fname))
+			die_errno(_("removing '%s' failed"), fname);
+
+		fname = mkpath("%s/old-pack-%s.pack", packdir, item->string);
+		if (remove_path(fname))
+			die_errno(_("removing '%s' failed"), fname);
Does this have to be a fatal error? If I read correctly, it wasn't fatal
in the shell version.
Good catch.

-- Hannes

[PATCH] repack: rewrite the shell script in C (squashing proposal)

From: Stefan Beller <hidden>
Date: 2016-06-15 22:58:29

This patch is meant to be squashed into bb4335a21441a0
(repack: rewrite the shell script in C), I'll do so when rerolling
the series. For reviewing I'll just send this patch.

* Remove comments, which likely get out of date (authorship is kept in
  git anyway)
* rename get_pack_filenames to get_non_kept_pack_filenames
* catch return value of unlink and fail as the shell version did
* beauty fixes to remove_temporary_files as Junio proposed
* install signal handling after static variables packdir, packtmp are set
* remove adding the empty string to the buffer.
* fix the rollback mechanism (wrong variable name)

Signed-off-by: Stefan Beller <redacted>
---
 builtin/repack.c | 78 ++++++++++++++++++++++++++------------------------------
 1 file changed, 36 insertions(+), 42 deletions(-)
diff --git a/builtin/repack.c b/builtin/repack.c
index 1f13e0d..e0d1f17 100644
--- a/builtin/repack.c
+++ b/builtin/repack.c
@@ -1,8 +1,3 @@
-/*
- * The shell version was written by Linus Torvalds (2005) and many others.
- * This is a translation into C by Stefan Beller (2013)
- */
-
 #include "builtin.h"
 #include "cache.h"
 #include "dir.h"
@@ -13,9 +8,8 @@
 #include "string-list.h"
 #include "argv-array.h"
 
-/* enabled by default since 22c79eab (2008-06-25) */
 static int delta_base_offset = 1;
-char *packdir;
+static char *packdir, *packtmp;
 
 static const char *const git_repack_usage[] = {
 	N_("git repack [options]"),
@@ -41,18 +35,16 @@ static void remove_temporary_files(void)
 	DIR *dir;
 	struct dirent *e;
 
-	/* .git/objects/pack */
-	strbuf_addstr(&buf, get_object_directory());
-	strbuf_addstr(&buf, "/pack");
-	dir = opendir(buf.buf);
-	if (!dir) {
-		strbuf_release(&buf);
+	dir = opendir(packdir);
+	if (!dir)
 		return;
-	}
 
-	/* .git/objects/pack/.tmp-$$-pack-* */
+	strbuf_addstr(&buf, packdir);
+
+	/* dirlen holds the length of the path before the file name */
 	dirlen = buf.len + 1;
-	strbuf_addf(&buf, "/.tmp-%d-pack-", (int)getpid());
+	strbuf_addf(&buf, "%s", packtmp);
+	/* prefixlen holds the length of the prefix */
 	prefixlen = buf.len - dirlen;
 
 	while ((e = readdir(dir))) {
@@ -73,11 +65,16 @@ static void remove_pack_on_signal(int signo)
 	raise(signo);
 }
 
-static void get_pack_filenames(struct string_list *fname_list)
+/*
+ * Adds all packs hex strings to the fname list, which do not
+ * have a corresponding .keep file.
+ */
+static void get_non_kept_pack_filenames(struct string_list *fname_list)
 {
 	DIR *dir;
 	struct dirent *e;
 	char *fname;
+	size_t len;
 
 	if (!(dir = opendir(packdir)))
 		return;
@@ -86,7 +83,7 @@ static void get_pack_filenames(struct string_list *fname_list)
 		if (suffixcmp(e->d_name, ".pack"))
 			continue;
 
-		size_t len = strlen(e->d_name) - strlen(".pack");
+		len = strlen(e->d_name) - strlen(".pack");
 		fname = xmemdupz(e->d_name, len);
 
 		if (!file_exists(mkpath("%s/%s.keep", packdir, fname)))
@@ -95,14 +92,14 @@ static void get_pack_filenames(struct string_list *fname_list)
 	closedir(dir);
 }
 
-static void remove_redundant_pack(const char *path, const char *sha1)
+static void remove_redundant_pack(const char *path_prefix, const char *hex)
 {
 	const char *exts[] = {".pack", ".idx", ".keep"};
 	int i;
 	struct strbuf buf = STRBUF_INIT;
 	size_t plen;
 
-	strbuf_addf(&buf, "%s/%s", path, sha1);
+	strbuf_addf(&buf, "%s/%s", path_prefix, hex);
 	plen = buf.len;
 
 	for (i = 0; i < ARRAY_SIZE(exts); i++) {
@@ -115,15 +112,14 @@ static void remove_redundant_pack(const char *path, const char *sha1)
 int cmd_repack(int argc, const char **argv, const char *prefix)
 {
 	const char *exts[2] = {".idx", ".pack"};
-	char *packtmp;
 	struct child_process cmd;
 	struct string_list_item *item;
 	struct argv_array cmd_args = ARGV_ARRAY_INIT;
 	struct string_list names = STRING_LIST_INIT_DUP;
-	struct string_list rollback = STRING_LIST_INIT_DUP;
+	struct string_list rollback = STRING_LIST_INIT_NODUP;
 	struct string_list existing_packs = STRING_LIST_INIT_DUP;
 	struct strbuf line = STRBUF_INIT;
-	int count_packs, ext, ret;
+	int nr_packs, ext, ret, failed;
 	FILE *out;
 
 	/* variables to be filled by option parsing */
@@ -173,11 +169,11 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
 	argc = parse_options(argc, argv, prefix, builtin_repack_options,
 				git_repack_usage, 0);
 
-	sigchain_push_common(remove_pack_on_signal);
-
 	packdir = mkpathdup("%s/pack", get_object_directory());
 	packtmp = mkpathdup("%s/.tmp-%d-pack", packdir, (int)getpid());
 
+	sigchain_push_common(remove_pack_on_signal);
+
 	argv_array_push(&cmd_args, "pack-objects");
 	argv_array_push(&cmd_args, "--keep-true-parents");
 	argv_array_push(&cmd_args, "--honor-pack-keep");
@@ -201,7 +197,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
 		argv_array_push(&cmd_args, "--unpacked");
 		argv_array_push(&cmd_args, "--incremental");
 	} else {
-		get_pack_filenames(&existing_packs);
+		get_non_kept_pack_filenames(&existing_packs);
 
 		if (existing_packs.nr && delete_redundant) {
 			if (unpack_unreachable)
@@ -233,14 +229,13 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
 	if (ret)
 		return ret;
 
-	count_packs = 0;
+	nr_packs = 0;
 	out = xfdopen(cmd.out, "r");
 	while (strbuf_getline(&line, out, '\n') != EOF) {
 		if (line.len != 40)
 			die("repack: Expecting 40 character sha1 lines only from pack-objects.");
-		strbuf_addstr(&line, "");
 		string_list_append(&names, line.buf);
-		count_packs++;
+		nr_packs++;
 	}
 	fclose(out);
 	ret = finish_command(&cmd);
@@ -248,10 +243,10 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
 		return ret;
 	argv_array_clear(&cmd_args);
 
-	if (!count_packs && !quiet)
+	if (!nr_packs && !quiet)
 		printf("Nothing new to pack.\n");
 
-	int failed = 0;
+	failed = 0;
 	for_each_string_list_item(item, &names) {
 		for (ext = 0; ext < 2; ext++) {
 			char *fname, *fname_old;
@@ -262,24 +257,23 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
 				continue;
 			}
 
-			fname_old = mkpathdup("%s/old-%s%s", packdir,
+			fname_old = mkpath("%s/old-%s%s", packdir,
 						item->string, exts[ext]);
 			if (file_exists(fname_old))
-				unlink(fname_old);
+				if (unlink(fname_old))
+					failed = 1;
 
-			if (rename(fname, fname_old)) {
+			if (!failed && rename(fname, fname_old)) {
 				failed = 1;
 				break;
 			}
-			string_list_append_nodup(&rollback, fname);
-			free(fname);
-			free(fname_old);
+			string_list_append(&rollback, fname);
 		}
 		if (failed)
 			break;
 	}
 	if (failed) {
-		struct string_list rollback_failure;
+		struct string_list rollback_failure = STRING_LIST_INIT_DUP;
 		for_each_string_list_item(item, &rollback) {
 			char *fname, *fname_old;
 			fname = mkpathdup("%s/%s", packdir, item->string);
@@ -289,7 +283,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
 			free(fname);
 		}
 
-		if (rollback.nr) {
+		if (rollback_failure.nr) {
 			int i;
 			fprintf(stderr,
 				"WARNING: Some packs in use have been renamed by\n"
@@ -299,10 +293,10 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
 				"WARNING: attempt to rename them back to their\n"
 				"WARNING: original names also failed.\n"
 				"WARNING: Please rename them in %s manually:\n", packdir);
-			for (i = 0; i < rollback.nr; i++)
+			for (i = 0; i < rollback_failure.nr; i++)
 				fprintf(stderr, "WARNING:   old-%s -> %s\n",
-					rollback.items[i].string,
-					rollback.items[i].string);
+					rollback_failure.items[i].string,
+					rollback_failure.items[i].string);
 		}
 		exit(1);
 	}
-- 
1.8.4.rc3.1.gc1ebd90
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help