[PATCH v2 0/7] More work supporting objects larger than 4GB on Windows
From: Johannes Schindelin via GitGitGadget <hidden>
Date: 2026-06-15 11:52:32
This patch series tries to address the problems pointed out by the expensive
tests that now run in CI: t5608 and t7508 verify various aspects about
objects larger than 4GB, which Git does not currently handle correctly when
run on a platform where size_t is 64-bit and unsigned long is 32-bit.
Changes vs v1:
* Rebased onto master, which merged ps/odb-source-loose (with which these
patches previously conflicted rather badly).
* Removed superfluous size_t s variables (thanks, Patrick!).
Johannes Schindelin (7):
compat/msvc: use _chsize_s for ftruncate
patch-delta: use size_t for sizes
pack-objects(check_pack_inflate()): use size_t instead of unsigned
long
packfile: widen unpack_entry()'s size out-parameter to size_t
pack-objects: use size_t for in-core object sizes
packfile,delta: drop the `cast_size_t_to_ulong()` wrappers
odb: use size_t for object_info.sizep and the size APIs
apply.c | 8 ++--
archive.c | 4 +-
attr.c | 2 +-
bisect.c | 2 +-
blame.c | 15 +++++--
builtin/cat-file.c | 61 ++++++++++++++---------------
builtin/difftool.c | 2 +-
builtin/fast-export.c | 7 +++-
builtin/fast-import.c | 29 ++++++++++----
builtin/fsck.c | 2 +-
builtin/grep.c | 12 +++---
builtin/index-pack.c | 10 ++---
builtin/log.c | 2 +-
builtin/ls-files.c | 2 +-
builtin/ls-tree.c | 4 +-
builtin/merge-tree.c | 6 +--
builtin/mktag.c | 2 +-
builtin/notes.c | 6 +--
builtin/pack-objects.c | 73 +++++++++++++++++++++--------------
builtin/repo.c | 4 +-
builtin/tag.c | 4 +-
builtin/unpack-file.c | 2 +-
builtin/unpack-objects.c | 8 ++--
bundle.c | 2 +-
combine-diff.c | 4 +-
commit.c | 10 ++---
compat/msvc-posix.h | 24 +++++++++++-
config.c | 2 +-
delta.h | 20 +++-------
diff.c | 5 ++-
dir.c | 2 +-
entry.c | 4 +-
fmt-merge-msg.c | 4 +-
fsck.c | 2 +-
grep.c | 4 +-
http-push.c | 2 +-
list-objects-filter.c | 2 +-
mailmap.c | 2 +-
match-trees.c | 4 +-
merge-blobs.c | 6 +--
merge-blobs.h | 2 +-
merge-ort.c | 2 +-
notes-cache.c | 2 +-
notes-merge.c | 2 +-
notes.c | 8 ++--
object-file.c | 6 +--
object.c | 2 +-
odb.c | 12 +++---
odb.h | 10 ++---
odb/source-loose.c | 12 +-----
odb/streaming.c | 13 +------
pack-bitmap.c | 4 +-
pack-check.c | 5 +--
pack-objects.h | 2 +-
packfile.c | 54 ++++++++++----------------
packfile.h | 5 ++-
patch-delta.c | 8 ++--
path-walk.c | 2 +-
protocol-caps.c | 5 ++-
read-cache.c | 6 +--
ref-filter.c | 2 +-
reflog.c | 2 +-
rerere.c | 2 +-
submodule-config.c | 2 +-
t/helper/test-delta.c | 10 +++--
t/helper/test-pack-deltas.c | 3 +-
t/helper/test-partial-clone.c | 2 +-
t/unit-tests/u-odb-inmemory.c | 2 +-
tag.c | 4 +-
tree-walk.c | 10 +++--
tree.c | 2 +-
xdiff-interface.c | 2 +-
72 files changed, 300 insertions(+), 271 deletions(-)
base-commit: ea97ad8d017de0c9037451a78008a0fd60abea0c
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-2137%2Fdscho%2Fobjects-larger-than-4gb-on-windows-pt2-v2
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-2137/dscho/objects-larger-than-4gb-on-windows-pt2-v2
Pull-Request: https://github.com/gitgitgadget/git/pull/2137
Range-diff vs v1:
1: de9fc5c455 = 1: 531bca775c compat/msvc: use _chsize_s for ftruncate
2: 1fd7646ca1 = 2: 66a642c39e patch-delta: use size_t for sizes
3: ddb75326cd = 3: 271a5299e3 pack-objects(check_pack_inflate()): use size_t instead of unsigned long
4: bdebc36f21 = 4: 5c329535df packfile: widen unpack_entry()'s size out-parameter to size_t
5: 68750ba2d1 = 5: 01b9209b26 pack-objects: use size_t for in-core object sizes
6: 460d733fee = 6: 12c142f8ab packfile,delta: drop the `cast_size_t_to_ulong()` wrappers
7: f3aeae983a ! 7: 37d030d867 odb: use size_t for object_info.sizep and the size APIs
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const
struct object_info oi = OBJECT_INFO_INIT;
unsigned flags = OBJECT_INFO_LOOKUP_REPLACE;
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
- if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG)) {
- size_t s = size;
- buf = replace_idents_using_mailmap(buf, &s);
+ if (odb_read_object_info_extended(the_repository->objects, &oid, &oi, flags) < 0)
+ die("git cat-file: could not get object info");
+
+- if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG)) {
+- size_t s = size;
+- buf = replace_idents_using_mailmap(buf, &s);
- size = cast_size_t_to_ulong(s);
-+ size = s;
- }
+- }
++ if (use_mailmap && (type == OBJ_COMMIT || type == OBJ_TAG))
++ buf = replace_idents_using_mailmap(buf, &size);
printf("%"PRIuMAX"\n", (uintmax_t)size);
+ ret = 0;
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
break;
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const
case 'p':
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
- if (use_mailmap) {
- size_t s = size;
- buf = replace_idents_using_mailmap(buf, &s);
+ if (!buf)
+ die("Cannot read object %s", obj_name);
+
+- if (use_mailmap) {
+- size_t s = size;
+- buf = replace_idents_using_mailmap(buf, &s);
- size = cast_size_t_to_ulong(s);
-+ size = s;
- }
+- }
++ if (use_mailmap)
++ buf = replace_idents_using_mailmap(buf, &size);
/* otherwise just spit out the data */
+ break;
@@ builtin/cat-file.c: static int cat_one_file(int opt, const char *exp_type, const char *obj_name)
- if (use_mailmap) {
- size_t s = size;
- buf = replace_idents_using_mailmap(buf, &s);
+ buf = odb_read_object_peeled(the_repository->objects, &oid,
+ exp_type_id, &size, NULL);
+
+- if (use_mailmap) {
+- size_t s = size;
+- buf = replace_idents_using_mailmap(buf, &s);
- size = cast_size_t_to_ulong(s);
-+ size = s;
- }
+- }
++ if (use_mailmap)
++ buf = replace_idents_using_mailmap(buf, &size);
break;
}
+ default:
@@ builtin/cat-file.c: cleanup:
struct expand_data {
struct object_id oid;
@@ builtin/cat-file.c: static void print_object_or_die(struct batch_options *opt, s
contents = odb_read_object(the_repository->objects, oid,
@@ builtin/cat-file.c: static void print_object_or_die(struct batch_options *opt, struct expand_data *d
- if (use_mailmap) {
- size_t s = size;
- contents = replace_idents_using_mailmap(contents, &s);
+ if (!contents)
+ die("object %s disappeared", oid_to_hex(oid));
+
+- if (use_mailmap) {
+- size_t s = size;
+- contents = replace_idents_using_mailmap(contents, &s);
- size = cast_size_t_to_ulong(s);
-+ size = s;
- }
+- }
++ if (use_mailmap)
++ contents = replace_idents_using_mailmap(contents, &size);
if (type != data->type)
+ die("object %s changed type!?", oid_to_hex(oid));
@@ builtin/cat-file.c: static void batch_object_write(const char *obj_name,
+ }
+
+ if (use_mailmap && (data->type == OBJ_COMMIT || data->type == OBJ_TAG)) {
+- size_t s = data->size;
+ char *buf = NULL;
+
+ buf = odb_read_object(the_repository->objects, &data->oid,
+ &data->type, &data->size);
if (!buf)
die(_("unable to read %s"), oid_to_hex(&data->oid));
- buf = replace_idents_using_mailmap(buf, &s);
+- buf = replace_idents_using_mailmap(buf, &s);
- data->size = cast_size_t_to_ulong(s);
-+ data->size = s;
++ buf = replace_idents_using_mailmap(buf, &data->size);
free(buf);
}
@@ builtin/log.c: static int show_blob_object(const struct object_id *oid, struct r
## builtin/ls-files.c ##
@@ builtin/ls-files.c: static void expand_objectsize(struct repository *repo, struct strbuf *line,
- const enum object_type type, unsigned int padded)
- {
+ size_t len;
+
if (type == OBJ_BLOB) {
- unsigned long size;
+ size_t size;
@@ builtin/ls-files.c: static void expand_objectsize(struct repository *repo, struc
## builtin/ls-tree.c ##
@@ builtin/ls-tree.c: static void expand_objectsize(struct strbuf *line, const struct object_id *oid,
- const enum object_type type, unsigned int padded)
- {
+ size_t len;
+
if (type == OBJ_BLOB) {
- unsigned long size;
+ size_t size;
@@ notes.c: static void format_note(struct notes_tree *t, const struct object_id *o
if (!t)
## object-file.c ##
-@@ object-file.c: static int parse_loose_header(const char *hdr, struct object_info *oi)
+@@ object-file.c: int parse_loose_header(const char *hdr, struct object_info *oi)
}
if (oi->sizep)
@@ object-file.c: static int parse_loose_header(const char *hdr, struct object_info
/*
* The length must be followed by a zero byte
-@@ object-file.c: static int read_object_info_from_path(struct odb_source *source,
- void *map = NULL;
- git_zstream stream, *stream_to_end = NULL;
- char hdr[MAX_HEADER_LEN];
-- unsigned long size_scratch;
-+ size_t size_scratch;
- enum object_type type_scratch;
- struct stat st;
-
@@ object-file.c: int force_object_loose(struct odb_source *source,
- {
+ struct odb_source_files *files = odb_source_files_downcast(source);
const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
void *buf;
- unsigned long len;
@@ object-file.c: int read_loose_object(struct repository *repo,
fd = git_open(path);
if (fd >= 0)
-@@ object-file.c: int odb_source_loose_read_object_stream(struct odb_read_stream **out,
- struct object_info oi = OBJECT_INFO_INIT;
- struct odb_loose_read_stream *st;
- unsigned long mapsize;
-- unsigned long size_ul;
- void *mapped;
-
- mapped = odb_source_loose_map_object(source, oid, &mapsize);
-@@ object-file.c: int odb_source_loose_read_object_stream(struct odb_read_stream **out,
- goto error;
- }
-
-- /*
-- * object_info.sizep is unsigned long* (32-bit on Windows), but
-- * st->base.size is size_t (64-bit). Use temporary variable.
-- * Note: loose objects >4GB would still truncate here, but such
-- * large loose objects are uncommon (they'd normally be packed).
-- */
-- oi.sizep = &size_ul;
-+ oi.sizep = &st->base.size;
- oi.typep = &st->base.type;
-
- if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
- goto error;
-- st->base.size = size_ul;
-
- st->mapped = mapped;
- st->mapsize = mapsize;
## object.c ##
@@ object.c: struct object *parse_object_with_flags(struct repository *r,
@@ odb.h: int odb_read_object_info_extended(struct object_database *odb,
enum odb_has_object_flags {
/* Retry packed storage after checking packed and loose storage */
+ ## odb/source-loose.c ##
+@@ odb/source-loose.c: static int read_object_info_from_path(struct odb_source_loose *loose,
+ void *map = NULL;
+ git_zstream stream, *stream_to_end = NULL;
+ char hdr[MAX_HEADER_LEN];
+- unsigned long size_scratch;
++ size_t size_scratch;
+ enum object_type type_scratch;
+ struct stat st;
+
+@@ odb/source-loose.c: static int odb_source_loose_read_object_stream(struct odb_read_stream **out,
+ struct object_info oi = OBJECT_INFO_INIT;
+ struct odb_loose_read_stream *st;
+ unsigned long mapsize;
+- unsigned long size_ul;
+ void *mapped;
+
+ mapped = odb_source_loose_map_object(loose, oid, &mapsize);
+@@ odb/source-loose.c: static int odb_source_loose_read_object_stream(struct odb_read_stream **out,
+ goto error;
+ }
+
+- /*
+- * object_info.sizep is unsigned long* (32-bit on Windows), but
+- * st->base.size is size_t (64-bit). Use temporary variable.
+- * Note: loose objects >4GB would still truncate here, but such
+- * large loose objects are uncommon (they'd normally be packed).
+- */
+- oi.sizep = &size_ul;
++ oi.sizep = &st->base.size;
+ oi.typep = &st->base.type;
+
+ if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
+ goto error;
+- st->base.size = size_ul;
+
+ st->mapped = mapped;
+ st->mapsize = mapsize;
+
## odb/streaming.c ##
@@ odb/streaming.c: static int open_istream_incore(struct odb_read_stream **out,
.base.read = read_istream_incore,
--
gitgitgadget