[PATCH] prune-packed: new option --min-age=N

Subsystems: the rest

DORMANTno replies

5 messages, 2 authors, 2016-06-15 · open the first message on its own page

[PATCH] prune-packed: new option --min-age=N

From: Matthias Lederhofer <hidden>
Date: 2016-06-15 22:42:49

This option makes prune-packed ignore all packs younger than N seconds
(using mtime).

Signed-off-by: Matthias Lederhofer <redacted>
---
This option allows to remove the unpacked version of an object only if
it has been packed for a minimum time.  It is intended to work around
the problem of freshly packed objects which should not be deleted
because someone might still try to open the unpacked version.

if [ $(git count-objects | cut -d ' ' -f 3) -gt 10240 ]; then
    git repack
fi
git prune-packed --min-age=$((24*60*60))
---
 builtin-prune-packed.c |   48 +++++++++++++++++++++++++++++++++++++++++++-----
 builtin-prune.c        |    2 +-
 builtin.h              |    2 +-
 3 files changed, 45 insertions(+), 7 deletions(-)
diff --git a/builtin-prune-packed.c b/builtin-prune-packed.c
index a57b76d..4359a41 100644
--- a/builtin-prune-packed.c
+++ b/builtin-prune-packed.c
@@ -7,7 +7,8 @@ static const char prune_packed_usage[] =
 #define DRY_RUN 01
 #define VERBOSE 02
 
-static void prune_dir(int i, DIR *dir, char *pathname, int len, int opts)
+static void prune_dir(int i, DIR *dir, char *pathname, int len, int opts,
+		      char **ignore_packed)
 {
 	struct dirent *de;
 	char hex[40];
@@ -20,7 +21,7 @@ static void prune_dir(int i, DIR *dir, char *pathname, int len, int opts)
 		memcpy(hex+2, de->d_name, 38);
 		if (get_sha1_hex(hex, sha1))
 			continue;
-		if (!has_sha1_pack(sha1, NULL))
+		if (!has_sha1_pack(sha1, ignore_packed))
 			continue;
 		memcpy(pathname + len, de->d_name, 38);
 		if (opts & DRY_RUN)
@@ -32,12 +33,46 @@ static void prune_dir(int i, DIR *dir, char *pathname, int len, int opts)
 	rmdir(pathname);
 }
 
-void prune_packed_objects(int opts)
+static char **prune_ignore(int min_age, int opts)
+{
+	int ignore_max = 10;
+	int ignore_count = 0;
+	char **ignore_packed = xcalloc(ignore_max, sizeof(char*));
+	int now = time(NULL);
+	struct packed_git *p;
+
+	prepare_packed_git();
+	for (p = packed_git; p; p = p->next) {
+		struct stat s;
+		if (stat(p->pack_name, &s)) {
+			warn("unable to stat %s", p->pack_name);
+			continue;
+		}
+		if (now-s.st_mtime >= min_age)
+			continue;
+		if (opts & VERBOSE)
+			fprintf(stderr, "ignoring pack %s\n", p->pack_name);
+		if (ignore_count >= ignore_max-1) {
+			ignore_max += 10;
+			ignore_packed = xrealloc(ignore_packed,
+				ignore_max*sizeof(char*));
+		}
+		ignore_packed[ignore_count++] = p->pack_name;
+	}
+	ignore_packed[ignore_count] = NULL;
+	return ignore_packed;
+}
+
+void prune_packed_objects(int opts, int min_age)
 {
 	int i;
 	static char pathname[PATH_MAX];
 	const char *dir = get_object_directory();
 	int len = strlen(dir);
+	char **ignore_packed = NULL;
+
+	if (min_age > 0)
+		ignore_packed = prune_ignore(min_age, opts);
 
 	if (len > PATH_MAX - 42)
 		die("impossible object directory");
@@ -54,7 +89,7 @@ void prune_packed_objects(int opts)
 				((i+1) * 100) / 256);
 		if (!d)
 			continue;
-		prune_dir(i, d, pathname, len + 3, opts);
+		prune_dir(i, d, pathname, len + 3, opts, ignore_packed);
 		closedir(d);
 	}
 	if (opts == VERBOSE)
@@ -65,6 +100,7 @@ int cmd_prune_packed(int argc, const char **argv, const char *prefix)
 {
 	int i;
 	int opts = VERBOSE;
+	int min_age = 0;
 
 	for (i = 1; i < argc; i++) {
 		const char *arg = argv[i];
@@ -74,6 +110,8 @@ int cmd_prune_packed(int argc, const char **argv, const char *prefix)
 				opts |= DRY_RUN;
 			else if (!strcmp(arg, "-q"))
 				opts &= ~VERBOSE;
+			else if (!strncmp(arg, "--min-age=", 10))
+				min_age = atoi(arg+10);
 			else
 				usage(prune_packed_usage);
 			continue;
@@ -82,6 +120,6 @@ int cmd_prune_packed(int argc, const char **argv, const char *prefix)
 		usage(prune_packed_usage);
 	}
 	sync();
-	prune_packed_objects(opts);
+	prune_packed_objects(opts, min_age);
 	return 0;
 }
diff --git a/builtin-prune.c b/builtin-prune.c
index 6f0ba0d..46a0fe5 100644
--- a/builtin-prune.c
+++ b/builtin-prune.c
@@ -100,6 +100,6 @@ int cmd_prune(int argc, const char **argv, const char *prefix)
 	prune_object_dir(get_object_directory());
 
 	sync();
-	prune_packed_objects(show_only);
+	prune_packed_objects(show_only, 0);
 	return 0;
 }
diff --git a/builtin.h b/builtin.h
index 818c7bf..e9929da 100644
--- a/builtin.h
+++ b/builtin.h
@@ -11,7 +11,7 @@ extern int mailinfo(FILE *in, FILE *out, int ks, const char *encoding, const cha
 extern int split_mbox(const char **mbox, const char *dir, int allow_bare, int nr_prec, int skip);
 extern void stripspace(FILE *in, FILE *out);
 extern int write_tree(unsigned char *sha1, int missing_ok, const char *prefix);
-extern void prune_packed_objects(int);
+extern void prune_packed_objects(int, int);
 
 extern int cmd_add(int argc, const char **argv, const char *prefix);
 extern int cmd_annotate(int argc, const char **argv, const char *prefix);
-- 
1.4.4.4

Re: [PATCH] prune-packed: new option --min-age=N

From: Shawn O. Pearce <hidden>
Date: 2016-06-15 22:42:49

Matthias Lederhofer [off-list ref] wrote:
This option allows to remove the unpacked version of an object only if
it has been packed for a minimum time.  It is intended to work around
the problem of freshly packed objects which should not be deleted
because someone might still try to open the unpacked version.
Are we not rescanning for new packs if we fail to find an
object in the object directory?  I thought we were doing this in
read_sha1_file.  *looks at code* Indeed, we are...

Is this somehow not working?

-- 
Shawn.

Re: [PATCH] prune-packed: new option --min-age=N

From: Matthias Lederhofer <hidden>
Date: 2016-06-15 22:42:49

Shawn O. Pearce [off-list ref] wrote:
Matthias Lederhofer [off-list ref] wrote:
quoted
This option allows to remove the unpacked version of an object only if
it has been packed for a minimum time.  It is intended to work around
the problem of freshly packed objects which should not be deleted
because someone might still try to open the unpacked version.
Are we not rescanning for new packs if we fail to find an
object in the object directory?  I thought we were doing this in
read_sha1_file.  *looks at code* Indeed, we are...

Is this somehow not working?
I just remembered people were warning about automatic prune-packed
(e.g. cron), so this patch does not make much sense anymore.

Any comments about adding such an option to git prune for unpacked
objects?  This would allow to run git prune automatically on
repositories while new objects are created.

Re: [PATCH] prune-packed: new option --min-age=N

From: Shawn O. Pearce <hidden>
Date: 2016-06-15 22:42:49

Matthias Lederhofer [off-list ref] wrote:
I just remembered people were warning about automatic prune-packed
(e.g. cron), so this patch does not make much sense anymore.
I think we've resolved it.  There may be some hole we aren't aware
of, but I doubt one exists.  The last hole was packs could be
removed by repack while they were still being brought in during
push or 'fetch -k', but that was fixed with the .keep file.
 
Any comments about adding such an option to git prune for unpacked
objects?  This would allow to run git prune automatically on
repositories while new objects are created.
An age makes sense on just plain prune.  If the loose object was
last modified several hours ago and its not being referenced by
anything currently, its probably safe to remove.

If you are going to implement this I would suggest making the default
age 2 hours and allow the user to configure it from a gc.pruneexpire
configuration option, much like gc.reflogexpire.

-- 
Shawn.

[RFC] prune: --expire=seconds

From: Matthias Lederhofer <hidden>
Date: 2016-06-15 22:42:49

This option specifies the minimum age of an object before it
may be removed by prune.  The default value is 2 hours and
may be changed using gc.pruneexpire.

Signed-off-by: Matthias Lederhofer <redacted>
---
Shawn O. Pearce [off-list ref] wrote:
If you are going to implement this I would suggest making the default
age 2 hours and allow the user to configure it from a gc.pruneexpire
configuration option, much like gc.reflogexpire.
Here it is, I've set the default value to 2 hours as you suggested.
Any other comments if the default should be a value >0 or 0 to keep
the old behaviour?
---
 builtin-prune.c |   25 ++++++++++++++++++++++++-
 1 files changed, 24 insertions(+), 1 deletions(-)
diff --git a/builtin-prune.c b/builtin-prune.c
index 6f0ba0d..f46892d 100644
--- a/builtin-prune.c
+++ b/builtin-prune.c
@@ -5,8 +5,10 @@
 #include "builtin.h"
 #include "reachable.h"
 
-static const char prune_usage[] = "git-prune [-n]";
+static const char prune_usage[] = "git-prune [-n] [--expire=seconds]";
 static int show_only;
+static int prune_expire;
+static time_t now;
 
 static int prune_object(char *path, const char *filename, const unsigned char *sha1)
 {
@@ -38,6 +40,7 @@ static int prune_dir(int i, char *path)
 		char name[100];
 		unsigned char sha1[20];
 		int len = strlen(de->d_name);
+		struct stat st;
 
 		switch (len) {
 		case 2:
@@ -60,6 +63,11 @@ static int prune_dir(int i, char *path)
 			if (lookup_object(sha1))
 				continue;
 
+			if (prune_expire > 0 &&
+			    !stat(mkpath("%s/%s", path, de->d_name), &st) &&
+			    now-st.st_mtime < prune_expire)
+				continue;
+
 			prune_object(path, de->d_name, sha1);
 			continue;
 		}
@@ -79,10 +87,21 @@ static void prune_object_dir(const char *path)
 	}
 }
 
+static void git_prune_config(const char *var, const char *value)
+{
+	if (!strcmp(var, "gc.pruneexpire")) {
+		prune_expire = git_config_int(var, value);
+		return 0;
+	}
+	return git_default_config(var, value);
+}
+
 int cmd_prune(int argc, const char **argv, const char *prefix)
 {
 	int i;
 	struct rev_info revs;
+	prune_expire = 2*60*60;
+	now = time(NULL);
 
 	for (i = 1; i < argc; i++) {
 		const char *arg = argv[i];
@@ -90,6 +109,10 @@ int cmd_prune(int argc, const char **argv, const char *prefix)
 			show_only = 1;
 			continue;
 		}
+		if (!strncmp(arg, "--expire=", 9)) {
+			prune_expire = atoi(arg+9);
+			continue;
+		}
 		usage(prune_usage);
 	}
 
-- 
1.4.4.4
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help