From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
After the initial posting here:
http://news.gmane.org/group/gmane.comp.version-control.git/thread=233061
This is a repost plus the basic read side working, at least to validate
the write side and the pack format itself. And many many bug fixes.
This can also be fetched here:
git://git.linaro.org/people/nico/git
I consider the actual pack format definition final as implemented
by this code.
TODO:
- index-pack support
- native tree walk support
- native commit graph walk support
- better heuristics when creating tree delta encoding
- integration with pack-objects
- transfer protocol backward compatibility
- thin pack completion
- figure out unexplained runtime performance issues
However, as I mentioned already, I've put more time on this project lately
than I actually had available. I really wanted to bring this project far
enough to be able to kick it out the door for others to take over, and
there we are.
I'm always available for design discussions and code review. But don't
expect much additional code from me at this point.
@junio: I'm hoping you can take this branch as is, and apply any ffurther
patches on top.
The diffstat goes like this:
Makefile | 3 +
cache.h | 11 +
hex.c | 11 +
pack-check.c | 4 +-
pack-revindex.c | 7 +-
pack-write.c | 6 +-
packv4-create.c | 1105 +++++++++++++++++++++++++++++++++++++++++++++++++
packv4-parse.c | 408 ++++++++++++++++++
packv4-parse.h | 9 +
sha1_file.c | 110 ++++-
10 files changed, 1648 insertions(+), 26 deletions(-)
Enjoy !
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Here's the code to dump a table into a pack. Table entries are written
according to the current sort order. This is important as objects use
this order to index into the table.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 49 insertions(+)
@@ -544,6 +544,55 @@ static int create_pack_dictionaries(struct packed_git *p,return0;}+staticunsignedlongwrite_dict_table(structsha1file*f,structdict_table*t)+{+unsignedcharbuffer[1024];+unsignedhdrlen;+unsignedlongsize,datalen;+z_streamstream;+inti,status;++/*+*Storeddicttableformat:uncompresseddatalengthfollowedby+*compressedcontent.+*/++datalen=t->ptr;+hdrlen=encode_varint(datalen,buffer);+sha1write(f,buffer,hdrlen);++memset(&stream,0,sizeof(stream));+deflateInit(&stream,pack_compression_level);++for(i=0;i<t->nb_entries;i++){+stream.next_in=t->data+t->entry[i].offset;+stream.avail_in=2+strlen((char*)t->data+t->entry[i].offset+2)+1;+do{+stream.next_out=buffer;+stream.avail_out=sizeof(buffer);+status=deflate(&stream,0);+size=stream.next_out-(unsignedchar*)buffer;+sha1write(f,buffer,size);+}while(status==Z_OK);+}+do{+stream.next_out=buffer;+stream.avail_out=sizeof(buffer);+status=deflate(&stream,Z_FINISH);+size=stream.next_out-(unsignedchar*)buffer;+sha1write(f,buffer,size);+}while(status==Z_OK);+if(status!=Z_STREAM_END)+die("unable to deflate dictionary table (%d)",status);+if(stream.total_in!=datalen)+die("dict data size mismatch (%ld vs %ld)",+stream.total_in,datalen);+datalen=stream.total_out;+deflateEnd(&stream);++returnhdrlen+datalen;+}+staticstructpacked_git*open_pack(constchar*path){chararg[PATH_MAX];
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
This goes as follows:
- Tree reference: either variable length encoding of the index
into the SHA1 table or the literal SHA1 prefixed by 0 (see
encode_sha1ref()).
- Parent count: variable length encoding of the number of parents.
This is normally going to occupy a single byte but doesn't have to.
- List of parent references: a list of encode_sha1ref() encoded
references, or nothing if the parent count was zero.
- Author reference: variable length encoding of an index into the author
identifier dictionary table which also covers the time zone. To make
the overall encoding efficient, the author table is sorted by usage
frequency so the most used names are first and require the shortest
index encoding.
- Author time stamp: variable length encoded. Year 2038 ready!
- Committer reference: same as author reference.
- Committer time stamp: same as author time stamp.
The remainder of the canonical commit object content is then zlib
compressed and appended to the above.
Rationale: The most important commit object data is densely encoded while
requiring no zlib inflate processing on access, and all SHA1 references
are most likely to be direct indices into the pack index file requiring
no SHA1 search into the pack index file.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 119 insertions(+)
@@ -274,6 +277,122 @@ static int encode_sha1ref(const unsigned char *sha1, unsigned char *buf)return1+20;}+/*+*Thisconvertsacanonicalcommitobjectbufferintoits+*tightlypackedrepresentationusingthealreadypopulated+*andsortedcommit_name_tabledictionary.Theparsingis+*strictsotoensurethecanonicalversionmayalwaysbe+*regeneratedandproducethesamehash.+*/+void*pv4_encode_commit(void*buffer,unsignedlong*sizep)+{+unsignedlongsize=*sizep;+char*in,*tail,*end;+unsignedchar*out;+unsignedcharsha1[20];+intnb_parents,index,tz_val;+unsignedlongtime;+z_streamstream;+intstatus;++/*+*Itisguaranteedthattheoutputisalwaysgoingtobesmaller+*thantheinput.Wecouldevendothisconversioninplace.+*/+in=buffer;+tail=in+size;+buffer=xmalloc(size);+out=buffer;++/* parse the "tree" line */+if(in+46>=tail||memcmp(in,"tree ",5)||in[45]!='\n')+gotobad_data;+if(get_sha1_lowhex(in+5,sha1)<0)+gotobad_data;+in+=46;+out+=encode_sha1ref(sha1,out);++/* count how many "parent" lines */+nb_parents=0;+while(in+48<tail&&!memcmp(in,"parent ",7)&&in[47]=='\n'){+nb_parents++;+in+=48;+}+out+=encode_varint(nb_parents,out);++/* rewind and parse the "parent" lines */+in-=48*nb_parents;+while(nb_parents--){+if(get_sha1_lowhex(in+7,sha1))+gotobad_data;+out+=encode_sha1ref(sha1,out);+in+=48;+}++/* parse the "author" line */+/* it must be at least "author x <x> 0 +0000\n" i.e. 21 chars */+if(in+21>=tail||memcmp(in,"author ",7))+gotobad_data;+in+=7;+end=get_nameend_and_tz(in,&tz_val);+if(!end)+gotobad_data;+index=dict_add_entry(commit_name_table,tz_val,in,end-in);+if(index<0)+gotobad_dict;+out+=encode_varint(index,out);+time=strtoul(end,&end,10);+if(!end||end[0]!=' '||end[6]!='\n')+gotobad_data;+out+=encode_varint(time,out);+in=end+7;++/* parse the "committer" line */+/* it must be at least "committer x <x> 0 +0000\n" i.e. 24 chars */+if(in+24>=tail||memcmp(in,"committer ",7))+gotobad_data;+in+=10;+end=get_nameend_and_tz(in,&tz_val);+if(!end)+gotobad_data;+index=dict_add_entry(commit_name_table,tz_val,in,end-in);+if(index<0)+gotobad_dict;+out+=encode_varint(index,out);+time=strtoul(end,&end,10);+if(!end||end[0]!=' '||end[6]!='\n')+gotobad_data;+out+=encode_varint(time,out);+in=end+7;++/* finally, deflate the remaining data */+memset(&stream,0,sizeof(stream));+deflateInit(&stream,pack_compression_level);+stream.next_in=(unsignedchar*)in;+stream.avail_in=tail-in;+stream.next_out=(unsignedchar*)out;+stream.avail_out=size-(out-(unsignedchar*)buffer);+status=deflate(&stream,Z_FINISH);+end=(char*)stream.next_out;+deflateEnd(&stream);+if(status!=Z_STREAM_END){+error("deflate error status %d",status);+gotobad;+}++*sizep=end-(char*)buffer;+returnbuffer;++bad_data:+error("bad commit data");+gotobad;+bad_dict:+error("bad dict entry");+bad:+free(buffer);+returnNULL;+}+staticstructpack_idx_entry*get_packed_object_list(structpacked_git*p){unsignedi,nr_objects=p->num_objects;
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
The name dictionary is loaded if not already done. We know it is
located right after the SHA1 table (20 bytes per object) which is
itself right after the 12-byte header.
Then the index is parsed from the input buffer and a pointer to the
corresponding entry is returned.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
cache.h | 3 +++
packv4-parse.c | 24 ++++++++++++++++++++++++
2 files changed, 27 insertions(+)
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Because there is no delta object cache for tree objects yet, walking
tree entries may result in a lot of recursion.
Let's add --min-tree-copy=N where N is the minimum number of copied
entries in a single copy sequence allowed for encoding tree deltas.
The default is 1. Specifying 0 disables tree deltas entirely.
This allows for experiments with the delta width and see the influence
on pack size vs runtime access cost.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 27 ++++++++++++++++++++-------
1 file changed, 20 insertions(+), 7 deletions(-)
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
If the path or name index is zero, this means the entry data is to be
found inline rather than being located in the dictionary table. This is
there to allow easy completion of thin packs without having to add new
table entries which would have required a full rewrite of the pack data.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 6 +++---
packv4-parse.c | 28 ++++++++++++++++++++++------
2 files changed, 25 insertions(+), 9 deletions(-)
@@ -125,11 +125,19 @@ const unsigned char *get_nameref(struct packed_git *p, const unsigned char **srcload_name_dict(p);index=decode_varint(srcp);-if(index>=p->name_dict->nb_entries){++if(!index){+/* the entry data is inline */+constunsignedchar*data=*srcp;+*srcp+=2+strlen((constchar*)*srcp+2)+1;+returndata;+}++if(index-1>=p->name_dict->nb_entries){error("%s: index overflow",__func__);returnNULL;}-returnp->name_dict->data+p->name_dict->offsets[index];+returnp->name_dict->data+p->name_dict->offsets[index-1];}staticvoidload_path_dict(structpacked_git*p)
@@ -151,16 +159,24 @@ static void load_path_dict(struct packed_git *p)p->path_dict=paths;}-constunsignedchar*get_pathref(structpacked_git*p,unsignedintindex)+constunsignedchar*get_pathref(structpacked_git*p,unsignedintindex,+constunsignedchar**srcp){if(!p->path_dict)load_path_dict(p);-if(index>=p->path_dict->nb_entries){+if(!index){+/* the entry data is inline */+constunsignedchar*data=*srcp;+*srcp+=2+strlen((constchar*)*srcp+2)+1;+returndata;+}++if(index-1>=p->path_dict->nb_entries){error("%s: index overflow",__func__);returnNULL;}-returnp->path_dict->data+p->path_dict->offsets[index];+returnp->path_dict->data+p->path_dict->offsets[index-1];}void*pv4_get_commit(structpacked_git*p,structpack_window**w_curs,
@@ -1687,20 +1687,12 @@ static off_t get_delta_base(struct packed_git *p,*isstupid,asthenaREF_DELTAwouldbesmallertostore.*/if(type==OBJ_OFS_DELTA){-unsignedused=0;-unsignedcharc=base_info[used++];-base_offset=c&127;-while(c&128){-base_offset+=1;-if(!base_offset||MSB(base_offset,7))-return0;/* overflow */-c=base_info[used++];-base_offset=(base_offset<<7)+(c&127);-}+constunsignedchar*cp=base_info;+base_offset=decode_varint(&cp);base_offset=delta_obj_offset-base_offset;if(base_offset<=0||base_offset>=delta_obj_offset)return0;/* out of bound */-*curpos+=used;+*curpos+=cp-base_info;}elseif(type==OBJ_REF_DELTA){/* The base entry _must_ be in the same pack */base_offset=find_pack_entry_one(base_info,p);
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
There is only one type of delta with pack v4. The base reference
encoding already handles either an offset (via the pack index) or a
literal SHA1.
We assume in the literal SHA1 case that the object lives in the same
pack, just like with previous pack versions.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
sha1_file.c | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
For now we recreate the whole tree object in its canonical form.
Eventually, the core code should grow some ability to walk packv4 tree
entries directly which would be way more efficient. Not only would that
avoid double tree entry parsing, but the pack v4 encoding allows for
getting at child objects without going through the SHA1 search.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-parse.c | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 131 insertions(+), 6 deletions(-)
@@ -240,3 +236,132 @@ void *pv4_get_commit(struct packed_git *p, struct pack_window **w_curs,returndst;}++staticintdecode_entries(structpacked_git*p,structpack_window**w_curs,+off_toffset,unsignedintstart,unsignedintcount,+unsignedchar**dstp,unsignedlong*sizep,inthdr)+{+unsignedlongavail;+unsignedintnb_entries;+constunsignedchar*src,*scp;+off_tcopy_objoffset=0;++src=use_pack(p,w_curs,offset,&avail);+scp=src;++if(hdr){+/* we need to skip over the object header */+while(*scp&128)+if(++scp-src>=avail-20)+return-1;+/* let's still make sure this is actually a tree */+if((*scp++&0xf)!=OBJ_TREE)+return-1;+}++nb_entries=decode_varint(&scp);+if(scp==src||start>nb_entries||count>nb_entries-start)+return-1;+offset+=scp-src;+avail-=scp-src;+src=scp;++while(count){+unsignedintwhat;++if(avail<20){+src=use_pack(p,w_curs,offset,&avail);+if(avail<20)+return-1;+}+scp=src;++what=decode_varint(&scp);+if(scp==src)+return-1;++if(!(what&1)&&start!=0){+/*+*Thisisasingleentryandwehavetoskipit.+*Thepathindexwasparsedandisin'what'.+*SkipovertheSHA1index.+*/+while(*scp++&128);+start--;+}elseif(!(what&1)&&start==0){+/*+*Thisisanactualtreeentrytorecreate.+*/+constunsignedchar*path,*sha1;+unsignedmode;+intlen;++path=get_pathref(p,what>>1);+sha1=get_sha1ref(p,&scp);+if(!path||!sha1)+return-1;+mode=(path[0]<<8)|path[1];+len=snprintf((char*)*dstp,*sizep,"%o %s%c",+mode,path+2,'\0');+if(len+20>*sizep)+return-1;+hashcpy(*dstp+len,sha1);+*dstp+=len+20;+*sizep-=len+20;+count--;+}elseif(what&1){+/*+*Copyfromanothertreeobject.+*/+unsignedintcopy_start,copy_count;++copy_start=what>>1;+copy_count=decode_varint(&scp);+if(!copy_count)+return-1;++/*+*TheLSBofcopy_countisaflagindicatingif+*athirdvalueisprovidedtospecifythesource+*object.Thismaybeomittedwhenitdoesn't+*change,buthastobespecifiedatleastforthe+*firstcopysequence.+*/+if(copy_count&1){+unsignedindex=decode_varint(&scp);+if(!index)/* thin pack */+return-1;+copy_objoffset=+nth_packed_object_offset(p,index-1);+}+if(!copy_objoffset)+return-1;+copy_count>>=1;++if(start>=copy_count){+start-=copy_count;+}else{+intret;+copy_count-=start;+copy_start+=start;+start=0;+if(copy_count>count)+copy_count=count;+count-=copy_count;+ret=decode_entries(p,w_curs,+copy_objoffset,copy_start,copy_count,+dstp,sizep,1);+if(ret)+returnret;+/* force pack window readjustment */+avail=scp-src;+}+}++offset+=scp-src;+avail-=scp-src;+src=scp;+}++return0;+}
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Because the path dictionary table is located right after the name
dictionary table, we currently need to load the later to find the
former.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
cache.h | 2 ++
packv4-parse.c | 36 ++++++++++++++++++++++++++++++++++++
2 files changed, 38 insertions(+)
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
In pack v4 the object size and type is encoded differently from pack v3.
The object size uses the same efficient variable length number encoding
already used elsewhere.
The object type has 4 bits allocated to it compared to 3 bits in pack v3.
This should be quite sufficient for the foreseeable future, especially
since pack v4 has only one type of delta object instead of two.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
@@ -635,6 +635,33 @@ static unsigned long packv4_write_tables(struct sha1file *f, unsigned nr_objectsreturnwritten;}+staticintwrite_object_header(structsha1file*f,enumobject_typetype,unsignedlongsize)+{+unsignedcharbuf[16];+uint64_tval;+intlen;++/*+*Wereallyhaveonlyonekindofdeltaobject.+*/+if(type==OBJ_OFS_DELTA)+type=OBJ_REF_DELTA;++/*+*Weallocate4bitsintheLSBfortheobjecttypewhichshould+*begoodforquiteawhile,giventhatweeffectivelyencodes+*only5objecttypes:commit,tree,blob,delta,tag.+*/+val=size;+if(MSB(val,4))+die("fixme: the code doesn't currently cope with big sizes");+val<<=4;+val|=type;+len=encode_varint(val,buf);+sha1write(f,buf,len);+returnlen;+}+staticstructpacked_git*open_pack(constchar*path){chararg[PATH_MAX];
@@ -504,7 +504,7 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)hdr=idx_map;if(hdr->idx_signature==htonl(PACK_IDX_SIGNATURE)){version=ntohl(hdr->idx_version);-if(version<2||version>2){+if(version<2||version>3){munmap(idx_map,idx_size);returnerror("index file %s is version %"PRIu32" and is not supported by this binary"
@@ -539,12 +539,13 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)munmap(idx_map,idx_size);returnerror("wrong index v1 file size in %s",path);}-}elseif(version==2){+}elseif(version==2||version==3){+unsignedlongmin_size,max_size;/**Minimumsize:*-8bytesofheader*-256indexentries4byteseach-*-20-bytesha1entry*nr+*-20-bytesha1entry*nr(version2only)*-4-bytecrcentry*nr*-4-byteoffsetentry*nr*-20-byteSHA1ofthepackfile
@@ -573,6 +576,36 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)}}+if(version>=3){+/* the SHA1 table is located in the main pack file */+void*pack_map;+structpack_header*pack_hdr;++fd=git_open_noatime(p->pack_name);+if(fd<0){+munmap(idx_map,idx_size);+returnerror("unable to open %s",p->pack_name);+}+if(fstat(fd,&st)!=0||xsize_t(st.st_size)<12+nr*20){+close(fd);+munmap(idx_map,idx_size);+returnerror("size of %s is wrong",p->pack_name);+}+pack_map=xmmap(NULL,12+nr*20,PROT_READ,MAP_PRIVATE,fd,0);+close(fd);+pack_hdr=pack_map;+if(pack_hdr->hdr_signature!=htonl(PACK_SIGNATURE)||+pack_hdr->hdr_version!=htonl(4)||+pack_hdr->hdr_entries!=htonl(nr)){+munmap(idx_map,idx_size);+munmap(pack_map,12+nr*20);+returnerror("packfile for %s doesn't match expectations",path);+}+p->sha1_table=pack_map;+p->sha1_table+=12;+}else+p->sha1_table=NULL;+p->index_version=version;p->index_data=idx_map;p->index_size=idx_size;
@@ -2281,6 +2323,8 @@ off_t find_pack_entry_one(const unsigned char *sha1,stride=24;index+=4;}+if(p->index_version>2)+index=p->sha1_table;if(debug_lookup)printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Usage of snprintf() is possibly not the most efficient approach.
For example we could simply copy the needed strings and generate
the SHA1 hex strings directly into the destination buffer. But
such optimizations may come later.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-parse.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 74 insertions(+)
@@ -0,0 +1,30 @@+/*+*Codetoparsepackv4objectencoding+*+*(C)NicolasPitre<nico@fluxnic.net>+*+*Thiscodeisfreesoftware;youcanredistributeitand/ormodify+*itunderthetermsoftheGNUGeneralPublicLicenseversion2as+*publishedbytheFreeSoftwareFoundation.+*/++#include"cache.h"+#include"varint.h"++constunsignedchar*get_sha1ref(structpacked_git*p,+constunsignedchar**bufp)+{+constunsignedchar*sha1;++if(!**bufp){+sha1=*bufp+1;+*bufp+=21;+}else{+unsignedintindex=decode_varint(bufp);+if(index<1||index-1>p->num_objects)+die("bad index in %s",__func__);+sha1=p->sha1_table+(index-1)*20;+}++returnsha1;+}
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
For this we need the pack version. However only open_packed_git_1() has
been audited for pack v4 so far, hence the version validation is not
added to pack_version_ok() just yet.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
cache.h | 1 +
sha1_file.c | 14 ++++++++++++--
2 files changed, 13 insertions(+), 2 deletions(-)
@@ -845,10 +846,11 @@ static int open_packed_git_1(struct packed_git *p)returnerror("file %s is far too short to be a packfile",p->pack_name);if(hdr.hdr_signature!=htonl(PACK_SIGNATURE))returnerror("file %s is not a GIT packfile",p->pack_name);-if(!pack_version_ok(hdr.hdr_version))+if(!pack_version_ok(hdr.hdr_version)&&hdr.hdr_version!=htonl(4))returnerror("packfile %s is version %"PRIu32" and not"" supported (try upgrading GIT to a newer version)",p->pack_name,ntohl(hdr.hdr_version));+p->version=ntohl(hdr.hdr_version);/* Verify the pack matches its index. */if(p->num_objects!=ntohl(hdr.hdr_entries))
@@ -1725,7 +1727,15 @@ int unpack_object_header(struct packed_git *p,*insane,soweknowwon'texceedwhatwehavebeengiven.*/base=use_pack(p,w_curs,*curpos,&left);-used=unpack_object_header_buffer(base,left,&type,sizep);+if(p->version<4){+used=unpack_object_header_buffer(base,left,&type,sizep);+}else{+constunsignedchar*cp=base;+uintmax_tval=decode_varint(&cp);+used=cp-base;+type=val&0xf;+*sizep=val>>4;+}if(!used){type=OBJ_BAD;}else
@@ -956,56 +956,46 @@ static off_t packv4_write_object(struct sha1file *f, struct packed_git *p,returnhdrlen+size;}-staticstructpacked_git*open_pack(constchar*path)+staticchar*normalize_pack_name(constchar*path){-chararg[PATH_MAX];+charbuf[PATH_MAX];intlen;-structpacked_git*p;-len=strlcpy(arg,path,PATH_MAX);-if(len>=PATH_MAX){-error("name too long: %s",path);-returnNULL;-}+len=strlcpy(buf,path,PATH_MAX);+if(len>=PATH_MAX-6)+die("name too long: %s",path);/**Inadditionto"foo.idx"weaccept"foo.pack"and"foo";-*normalizetheseformsto"foo.idx"foradd_packed_git().+*normalizetheseformsto"foo.pack".*/-if(has_extension(arg,".pack")){-strcpy(arg+len-5,".idx");-len--;-}elseif(!has_extension(arg,".idx")){-if(len+4>=PATH_MAX){-error("name too long: %s.idx",arg);-returnNULL;-}-strcpy(arg+len,".idx");-len+=4;+if(has_extension(buf,".idx")){+strcpy(buf+len-4,".pack");+len++;+}elseif(!has_extension(buf,".pack")){+strcpy(buf+len,".pack");+len+=5;}-/*-*add_packed_git()usesourbuffer(containing"foo.idx")to-*buildthepackfilename("foo.pack").Makesureitfits.-*/-if(len+1>=PATH_MAX){-arg[len-4]='\0';-error("name too long: %s.pack",arg);-returnNULL;-}+returnxstrdup(buf);+}-p=add_packed_git(arg,len,1);-if(!p){-error("packfile %s not found.",arg);-returnNULL;-}+staticstructpacked_git*open_pack(constchar*path)+{+char*packname=normalize_pack_name(path);+intlen=strlen(packname);+structpacked_git*p;++strcpy(packname+len-5,".idx");+p=add_packed_git(packname,len-1,1);+if(!p)+die("packfile %s not found.",packname);install_packed_git(p);-if(open_pack_index(p)){-error("packfile %s index not opened",p->pack_name);-returnNULL;-}+if(open_pack_index(p))+die("packfile %s index not opened",p->pack_name);+free(packname);returnp;}
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
This is a minor change over pack index v2. Since pack v4 already contains
the sorted SHA1 table, it is therefore ommitted from the index file.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
pack-write.c | 6 +++++-
packv4-create.c | 10 +++++++++-
2 files changed, 14 insertions(+), 2 deletions(-)
@@ -87,6 +87,8 @@ const char *write_idx_file(const char *index_name, struct pack_idx_entry **objec/* if last object's offset is >= 2^31 we should use index V2 */index_version=need_large_offset(last_obj_offset,opts)?2:opts->version;+if(index_version<opts->version)+index_version=opts->version;/* index versions 2 and above need a header */if(index_version>=2){
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
This goes as follows:
- Number of tree entries: variable length encoded.
Then for each tree entry:
- Path component reference: variable length encoded index into the path
dictionary table which also covers the entry mode. To make the overall
encoding efficient, the path table is already sorted by usage frequency
so the most used path names are first and require the shortest index
encoding.
- SHA1 reference: either variable length encoding of the index into the
SHA1 table or the literal SHA1 prefixed by 0 (see encode_sha1ref()).
Rationale: all the tree object data is densely encoded while requiring
no zlib inflate processing on access, and all SHA1 references are most
likely to be direct indices into the pack index file requiring no SHA1
search. Path filtering can be accomplished on the path index directly
without any string comparison during the tree traversal.
Still lacking is some kind of delta encoding for multiple tree objects
with only small differences between them. But that'll come later.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 66 insertions(+)
@@ -393,6 +393,72 @@ bad:returnNULL;}+/*+*Thisconvertsacanonicaltreeobjectbufferintoits+*tightlypackedrepresentationusingthealreadypopulated+*andsortedtree_path_tabledictionary.Theparsingis+*strictsotoensurethecanonicalversionmayalwaysbe+*regeneratedandproducethesamehash.+*/+void*pv4_encode_tree(void*_buffer,unsignedlong*sizep)+{+unsignedlongsize=*sizep;+unsignedchar*in,*out,*end,*buffer=_buffer;+structtree_descdesc;+structname_entryname_entry;+intnb_entries;++if(!size)+returnNULL;++/*+*Wecan'tmakesuretheresultwillalwaysbesmallerthanthe+*input.Thesmallestpossibleentryis"0 x\0<40 byte SHA1>"+*or44bytes.Theoutputentrymayhavearealisticpathindex+*encodingusingupto3bytes,andanonindexableSHA1meaning+*41bytes.Andtheoutputdataalreadyhasthenb_entries+*headers.Inpracticetheoutputsizewillbesignificantly+*smallerbutfornowlet'smakeitsimple.+*/+in=buffer;+out=xmalloc(size+48);+end=out+size+48;+buffer=out;++/* let's count how many entries there are */+init_tree_desc(&desc,in,size);+nb_entries=0;+while(tree_entry(&desc,&name_entry))+nb_entries++;+out+=encode_varint(nb_entries,out);++init_tree_desc(&desc,in,size);+while(tree_entry(&desc,&name_entry)){+intpathlen,index;++if(end-out<48){+unsignedlongsofar=out-buffer;+buffer=xrealloc(buffer,(sofar+48)*2);+end=buffer+(sofar+48)*2;+out=buffer+sofar;+}++pathlen=tree_entry_len(&name_entry);+index=dict_add_entry(tree_path_table,name_entry.mode,+name_entry.path,pathlen);+if(index<0){+error("missing tree dict entry");+free(buffer);+returnNULL;+}+out+=encode_varint(index,out);+out+=encode_sha1ref(name_entry.sha1,out);+}++*sizep=out-buffer;+returnbuffer;+}+staticstructpack_idx_entry*get_packed_object_list(structpacked_git*p){unsignedi,nr_objects=p->num_objects;
@@ -1030,6 +1037,7 @@ static void process_one_pack(char *src_pack, char *dst_pack)written+=packv4_write_tables(f,nr_objects,objs);/* Let's write objects out, updating the object index list in place */+progress_state=start_progress("Writing objects",nr_objects);all_objs=objs;all_objs_nr=nr_objects;for(i=0;i<nr_objects;i++){
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
At least commit af25e94d4dcfb9608846242fabdd4e6014e5c9f0 in the Linux
kernel repository has "author <> 1120285620 -0700"
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
@@ -158,12 +158,12 @@ static char *get_nameend_and_tz(char *from, int *tz_val)char*end,*tz;tz=strchr(from,'\n');-/* let's assume the smallest possible string to be "x <x> 0 +0000\n" */-if(!tz||tz-from<13)+/* let's assume the smallest possible string to be " <> 0 +0000\n" */+if(!tz||tz-from<11)returnNULL;tz-=4;end=tz-4;-while(end-from>5&&*end!=' ')+while(end-from>3&&*end!=' ')end--;if(end[-1]!='>'||end[0]!=' '||tz[-2]!=' ')returnNULL;
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Let's actually open the destination pack file and write the header and
the tables.
The header isn't much different from pack v3, except for the pack version
number of course.
The first table is the sorted SHA1 table normally found in the pack index
file. With pack v4 we write this table in the main pack file instead as
it is index referenced by subsequent objects in the pack. Doing so has
many advantages:
- The SHA1 references used to be duplicated on disk: once in the pack
index file, and then at least once or more within commit and tree
objects referencing them. The only SHA1 which is not being listed more
than once this way is the one for a branch tip commit object and those
are normally very few. Now all that SHA1 data is represented only once.
- The SHA1 references found in commit and tree objects can be obtained
on disk directly without having to deflate those objects first.
The SHA1 table size is obtained by multiplying the number of objects by 20.
And then the commit and path dictionary tables are written right after
the SHA1 table.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 55 insertions(+), 5 deletions(-)
@@ -593,6 +593,48 @@ static unsigned long write_dict_table(struct sha1file *f, struct dict_table *t)returnhdrlen+datalen;}+staticstructsha1file*packv4_open(char*path)+{+intfd;++fd=open(path,O_CREAT|O_EXCL|O_WRONLY,0600);+if(fd<0)+die_errno("unable to create '%s'",path);+returnsha1fd(fd,path);+}++staticunsignedintpackv4_write_header(structsha1file*f,unsignednr_objects)+{+structpack_headerhdr;++hdr.hdr_signature=htonl(PACK_SIGNATURE);+hdr.hdr_version=htonl(4);+hdr.hdr_entries=htonl(nr_objects);+sha1write(f,&hdr,sizeof(hdr));++returnsizeof(hdr);+}++staticunsignedlongpackv4_write_tables(structsha1file*f,unsignednr_objects,+structpack_idx_entry*objs)+{+unsignedi;+unsignedlongwritten=0;++/* The sorted list of object SHA1's is always first */+for(i=0;i<nr_objects;i++)+sha1write(f,objs[i].sha1,20);+written=20*nr_objects;++/* Then the commit dictionary table */+written+=write_dict_table(f,commit_name_table);++/* Followed by the path component dictionary table */+written+=write_dict_table(f,tree_path_table);++returnwritten;+}+staticstructpacked_git*open_pack(constchar*path){chararg[PATH_MAX];
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
The SHA1 of the base object is retrieved and the corresponding object
is loaded in memory for pv4_encode_tree() to look at. Simple but
effective. Obviously this relies on the delta matching already performed
during the pack v3 delta search. Some native delta search for pack v4
could be investigated eventually.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 60 insertions(+), 3 deletions(-)
@@ -820,18 +820,56 @@ static unsigned long copy_object_data(struct sha1file *f, struct packed_git *p,returnwritten;}+staticunsignedchar*get_delta_base(structpacked_git*p,off_toffset,+unsignedchar*sha1_buf)+{+structpack_window*w_curs=NULL;+enumobject_typetype;+unsignedlongavail,size;+inthdrlen;+unsignedchar*src;+constunsignedchar*base_sha1=NULL;;++src=use_pack(p,&w_curs,offset,&avail);+hdrlen=unpack_object_header_buffer(src,avail,&type,&size);++if(type==OBJ_OFS_DELTA){+constunsignedchar*cp=src+hdrlen;+off_tbase_offset=decode_varint(&cp);+base_offset=offset-base_offset;+if(base_offset<=0||base_offset>=offset){+error("delta offset out of bound");+}else{+structrevindex_entry*revidx;+revidx=find_pack_revindex(p,base_offset);+base_sha1=nth_packed_object_sha1(p,revidx->nr);+}+}elseif(type==OBJ_REF_DELTA){+base_sha1=src+hdrlen;+}else+error("expected to get a delta but got a %s",typename(type));++unuse_pack(&w_curs);++if(!base_sha1)+returnNULL;+hashcpy(sha1_buf,base_sha1);+returnsha1_buf;+}+staticoff_tpackv4_write_object(structsha1file*f,structpacked_git*p,structpack_idx_entry*obj){void*src,*result;structobject_infooi={};-enumobject_typetype;+enumobject_typetype,packed_type;unsignedlongsize;unsignedinthdrlen;oi.typep=&type;oi.sizep=&size;-if(packed_object_info(p,obj->offset,&oi)<0)+packed_type=packed_object_info(p,obj->offset,&oi);+if(packed_type<0)die("cannot get type of %s from %s",sha1_to_hex(obj->sha1),p->pack_name);
@@ -859,7 +897,26 @@ static off_t packv4_write_object(struct sha1file *f, struct packed_git *p,result=pv4_encode_commit(src,&size);break;caseOBJ_TREE:-result=pv4_encode_tree(src,&size,NULL,0,NULL);+if(packed_type!=OBJ_TREE){+unsignedcharsha1_buf[20],*ref_sha1;+void*ref;+enumobject_typeref_type;+unsignedlongref_size;++ref_sha1=get_delta_base(p,obj->offset,sha1_buf);+if(!ref_sha1)+die("unable to get delta base sha1 for %s",+sha1_to_hex(obj->sha1));+ref=read_sha1_file(ref_sha1,&ref_type,&ref_size);+if(!ref||ref_type!=OBJ_TREE)+die("cannot obtain delta base for %s",+sha1_to_hex(obj->sha1));+result=pv4_encode_tree(src,&size,+ref,ref_size,ref_sha1);+free(ref);+}else{+result=pv4_encode_tree(src,&size,NULL,0,NULL);+}break;default:die("unexpected object type %d",type);
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Make sure the copy sequence is smaller than the list of tree entries it
is meant to replace. We do so by encoding tree entries in parallel with
the delta entry comparison, and then comparing the length of both
sequences.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 65 +++++++++++++++++++++++++++++++++++++++------------------
1 file changed, 45 insertions(+), 20 deletions(-)
@@ -524,12 +524,31 @@ void *pv4_encode_tree(void *_buffer, unsigned long *sizep,*/copy_start=(copy_start<<1)|1;copy_count=(copy_count<<1)|first_delta;-out+=encode_varint(copy_start,out);-out+=encode_varint(copy_count,out);+cp+=encode_varint(copy_start,cp);+cp+=encode_varint(copy_count,cp);if(first_delta)-out+=encode_sha1ref(delta_sha1,out);+cp+=encode_sha1ref(delta_sha1,cp);copy_count=0;-first_delta=0;++/*+*Nowlet'smakesurethisisgoingtotakeless+*spacethanthecorrespondingdirectentrieswe've+*createdinparallel.Ifsowedumpthecopy+*sequenceoverthoseentriesintheoutputbuffer.+*/+if(cp-copy_buf<out-&buffer[copy_pos]){+out=buffer+copy_pos;+memcpy(out,copy_buf,cp-copy_buf);+out+=cp-copy_buf;+first_delta=0;+}+}++if(end-out<48){+unsignedlongsofar=out-buffer;+buffer=xrealloc(buffer,(sofar+48)*2);+end=buffer+(sofar+48)*2;+out=buffer+sofar;}pathlen=tree_entry_len(&name_entry);
@@ -545,13 +564,19 @@ void *pv4_encode_tree(void *_buffer, unsigned long *sizep,}if(copy_count){-/* flush the trailing copy */+/* process the trailing copy */+unsignedcharcopy_buf[48],*cp=copy_buf;copy_start=(copy_start<<1)|1;copy_count=(copy_count<<1)|first_delta;-out+=encode_varint(copy_start,out);-out+=encode_varint(copy_count,out);+cp+=encode_varint(copy_start,cp);+cp+=encode_varint(copy_count,cp);if(first_delta)-out+=encode_sha1ref(delta_sha1,out);+cp+=encode_sha1ref(delta_sha1,cp);+if(cp-copy_buf<out-&buffer[copy_pos]){+out=buffer+copy_pos;+memcpy(out,copy_buf,cp-copy_buf);+out+=cp-copy_buf;+}}*sizep=out-buffer;
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
This is like get_sha1_hex() but stricter in accepting lowercase letters
only.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
cache.h | 3 +++
hex.c | 11 +++++++++++
2 files changed, 14 insertions(+)
@@ -56,6 +56,17 @@ int get_sha1_hex(const char *hex, unsigned char *sha1)return0;}+intget_sha1_lowhex(constchar*hex,unsignedchar*sha1)+{+inti;++/* uppercase letters (as well as '\0') have bit 5 clear */+for(i=0;i<20;i++)+if(!(hex[i]&0x20))+return-1;+returnget_sha1_hex(hex,sha1);+}+char*sha1_to_hex(constunsignedchar*sha1){staticintbufno;
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Blob and tag objects have no particular changes except for their object
header.
Delta objects are also copied as is, except for their delta base reference
which is converted to the new way as used elsewhere in pack v4 encoding
i.e. an index into the SHA1 table or a literal SHA1 prefixed by 0 if not
found in the table (see encode_sha1ref). This is true for both REF_DELTA
as well as OFS_DELTA.
Object payload is validated against the recorded CRC32 in the source
pack index file when possible before being copied.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 60 insertions(+)
@@ -662,6 +663,65 @@ static int write_object_header(struct sha1file *f, enum object_type type, unsignreturnlen;}+staticunsignedlongcopy_object_data(structsha1file*f,structpacked_git*p,+off_toffset)+{+structpack_window*w_curs=NULL;+structrevindex_entry*revidx;+enumobject_typetype;+unsignedlongavail,size,datalen,written;+inthdrlen,reflen,idx_nr;+unsignedchar*src,buf[24];++revidx=find_pack_revindex(p,offset);+idx_nr=revidx->nr;+datalen=revidx[1].offset-offset;++src=use_pack(p,&w_curs,offset,&avail);+hdrlen=unpack_object_header_buffer(src,avail,&type,&size);++written=write_object_header(f,type,size);++if(type==OBJ_OFS_DELTA){+constunsignedchar*cp=src+hdrlen;+off_tbase_offset=decode_varint(&cp);+hdrlen=cp-src;+base_offset=offset-base_offset;+if(base_offset<=0||base_offset>=offset)+die("delta offset out of bound");+revidx=find_pack_revindex(p,base_offset);+reflen=encode_sha1ref(nth_packed_object_sha1(p,revidx->nr),buf);+sha1write(f,buf,reflen);+written+=reflen;+}elseif(type==OBJ_REF_DELTA){+reflen=encode_sha1ref(src+hdrlen,buf);+hdrlen+=20;+sha1write(f,buf,reflen);+written+=reflen;+}++if(p->index_version>1&&+check_pack_crc(p,&w_curs,offset,datalen,idx_nr))+die("bad CRC for object at offset %"PRIuMAX" in %s",+(uintmax_t)offset,p->pack_name);++offset+=hdrlen;+datalen-=hdrlen;++while(datalen){+src=use_pack(p,&w_curs,offset,&avail);+if(avail>datalen)+avail=datalen;+sha1write(f,src,avail);+written+=avail;+offset+=avail;+datalen-=avail;+}+unuse_pack(&w_curs);++returnwritten;+}+staticstructpacked_git*open_pack(constchar*path){chararg[PATH_MAX];
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Let's create a struct pack_idx_entry list with sorted sha1 which will
be useful later. The offset sorted list is now a separate indirect
list.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 72 +++++++++++++++++++++++++++++++++------------------------
1 file changed, 42 insertions(+), 30 deletions(-)
@@ -292,9 +300,9 @@ static int create_pack_dictionaries(struct packed_git *p,oi.typep=&type;oi.sizep=&size;-if(packed_object_info(p,objects[i].offset,&oi)<0)+if(packed_object_info(p,obj->offset,&oi)<0)die("cannot get type of %s from %s",-sha1_to_hex(objects[i].sha1),p->pack_name);+sha1_to_hex(obj->sha1),p->pack_name);switch(type){caseOBJ_COMMIT:
@@ -306,16 +314,16 @@ static int create_pack_dictionaries(struct packed_git *p,default:continue;}-data=unpack_entry(p,objects[i].offset,&type,&size);+data=unpack_entry(p,obj->offset,&type,&size);if(!data)die("cannot unpack %s from %s",-sha1_to_hex(objects[i].sha1),p->pack_name);-if(check_sha1_signature(objects[i].sha1,data,size,typename(type)))+sha1_to_hex(obj->sha1),p->pack_name);+if(check_sha1_signature(obj->sha1,data,size,typename(type)))die("packed %s from %s is corrupt",-sha1_to_hex(objects[i].sha1),p->pack_name);+sha1_to_hex(obj->sha1),p->pack_name);if(add_dict_entries(data,size)<0)die("can't process %s object %s",-typename(type),sha1_to_hex(objects[i].sha1));+typename(type),sha1_to_hex(obj->sha1));free(data);}
@@ -378,14 +386,18 @@ static struct packed_git *open_pack(const char *path)staticvoidprocess_one_pack(char*src_pack){structpacked_git*p;-structidx_entry*objs;+structpack_idx_entry*objs,**p_objs;+unsignednr_objects;p=open_pack(src_pack);if(!p)die("unable to open source pack");+nr_objects=p->num_objects;objs=get_packed_object_list(p);-create_pack_dictionaries(p,objs);+p_objs=sort_objs_by_offset(objs,nr_objects);++create_pack_dictionaries(p,p_objs);}intmain(intargc,char*argv[])
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
This adds the missing code to finally be able to produce a complete
pack file version 4. We trap commit and tree objects as those have
a completely new encoding. Other object types are copied almost
unchanged.
As we go the pack index entries are updated in place to store the new
object offsets once they're written to the destination file. This will
be needed later for writing the pack index file.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 71 insertions(+), 3 deletions(-)
@@ -722,6 +722,59 @@ static unsigned long copy_object_data(struct sha1file *f, struct packed_git *p,returnwritten;}+staticoff_tpackv4_write_object(structsha1file*f,structpacked_git*p,+structpack_idx_entry*obj)+{+void*src,*result;+structobject_infooi={};+enumobject_typetype;+unsignedlongsize;+unsignedinthdrlen;++oi.typep=&type;+oi.sizep=&size;+if(packed_object_info(p,obj->offset,&oi)<0)+die("cannot get type of %s from %s",+sha1_to_hex(obj->sha1),p->pack_name);++/* Some objects are copied without decompression */+switch(type){+caseOBJ_COMMIT:+caseOBJ_TREE:+break;+default:+returncopy_object_data(f,p,obj->offset);+}++/* The rest is converted into their new format */+src=unpack_entry(p,obj->offset,&type,&size);+if(!src)+die("cannot unpack %s from %s",+sha1_to_hex(obj->sha1),p->pack_name);+if(check_sha1_signature(obj->sha1,src,size,typename(type)))+die("packed %s from %s is corrupt",+sha1_to_hex(obj->sha1),p->pack_name);++hdrlen=write_object_header(f,type,size);+switch(type){+caseOBJ_COMMIT:+result=pv4_encode_commit(src,&size);+break;+caseOBJ_TREE:+result=pv4_encode_tree(src,&size);+break;+default:+die("unexpected object type %d",type);+}+free(src);+if(!result)+die("can't convert %s object %s",+typename(type),sha1_to_hex(obj->sha1));+sha1write(f,result,size);+free(result);+returnhdrlen+size;+}+staticstructpacked_git*open_pack(constchar*path){chararg[PATH_MAX];
@@ -791,12 +845,26 @@ static void process_one_pack(char *src_pack, char *dst_pack)p_objs=sort_objs_by_offset(objs,nr_objects);create_pack_dictionaries(p,p_objs);+sort_dict_entries_by_hits(commit_name_table);+sort_dict_entries_by_hits(tree_path_table);f=packv4_open(dst_pack);if(!f)die("unable to open destination pack");-packv4_write_header(f,nr_objects);-packv4_write_tables(f,nr_objects,objs);+written+=packv4_write_header(f,nr_objects);+written+=packv4_write_tables(f,nr_objects,objs);++/* Let's write objects out, updating the object index list in place */+all_objs=objs;+all_objs_nr=nr_objects;+for(i=0;i<nr_objects;i++){+off_tobj_pos=written;+structpack_idx_entry*obj=p_objs[i];+written+=packv4_write_object(f,p,obj);+obj->offset=obj_pos;+}++sha1close(f,NULL,CSUM_CLOSE|CSUM_FSYNC);}intmain(intargc,char*argv[])
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
In order to be able to quickly walk tree objects, let's encode their
"delta" as a range of entries into another tree object.
In order to discriminate between a copy sequence from a regular entry,
the entry index LSB is reserved to indicate a copy sequence. Therefore
the actual index of a path component is shifted left one bit.
The encoding allows for the base object to change so multiple base
objects can be borrowed from. The code doesn't try to exploit this
possibility at the moment though.
The code isn't optimal at the moment as it doesn't consider the case
where a copy sequence could be larger than the local sequence it
means to replace.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 103 insertions(+), 5 deletions(-)
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Augment dict entries with a 16-bit prefix in order to store the file
mode value of tree entries.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 56 ++++++++++++++++++++++++++++++++++++--------------------
1 file changed, 36 insertions(+), 20 deletions(-)
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
The SHA1 reference is either an index into a SHA1 table using the variable
length number encoding, or the literal 20 bytes SHA1 prefixed with a 0.
The index 0 discriminates between an actual index value or the literal
SHA1. Therefore when the index is used its value must be increased by 1.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Let's create another dictionary table to hold the author and committer
entries. We use the same table format used for tree entries where the
16 bit data prefix is conveniently used to store the timezone value.
In order to copy straight from a commit object buffer, dict_add_entry()
is modified to get the string length as the provided string pointer is
not always be null terminated.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 89 insertions(+), 9 deletions(-)
@@ -205,6 +280,7 @@ static int create_pack_dictionaries(struct packed_git *p)enumobject_typetype;unsignedlongsize;structobject_infooi={};+int(*add_dict_entries)(void*,unsignedlong);oi.typep=&type;oi.sizep=&size;
@@ -213,7 +289,11 @@ static int create_pack_dictionaries(struct packed_git *p)sha1_to_hex(objects[i].sha1),p->pack_name);switch(type){+caseOBJ_COMMIT:+add_dict_entries=add_commit_dict_entries;+break;caseOBJ_TREE:+add_dict_entries=add_tree_dict_entries;break;default:continue;
@@ -225,7 +305,7 @@ static int create_pack_dictionaries(struct packed_git *p)if(check_sha1_signature(objects[i].sha1,data,size,typename(type)))die("packed %s from %s is corrupt",sha1_to_hex(objects[i].sha1),p->pack_name);-if(add_tree_dict_entries(data,size)<0)+if(add_dict_entries(data,size)<0)die("can't process %s object %s",typename(type),sha1_to_hex(objects[i].sha1));free(data);
@@ -285,6 +365,6 @@ int main(int argc, char *argv[])exit(1);}process_one_pack(argv[1]);-dict_dump(tree_path_table);+dict_dump();return0;}
@@ -275,7 +275,15 @@ static int create_pack_dictionaries(struct packed_git *p)}qsort(objects,nr_objects,sizeof(*objects),sort_by_offset);-for(i=0;i<nr_objects;i++){+returnobjects;+}++staticintcreate_pack_dictionaries(structpacked_git*p,+structidx_entry*objects)+{+unsignedinti;++for(i=0;i<p->num_objects;i++){void*data;enumobject_typetype;unsignedlongsize;
@@ -310,20 +318,21 @@ static int create_pack_dictionaries(struct packed_git *p)typename(type),sha1_to_hex(objects[i].sha1));free(data);}-free(objects);return0;}-staticintprocess_one_pack(constchar*path)+staticstructpacked_git*open_pack(constchar*path){chararg[PATH_MAX];intlen;structpacked_git*p;len=strlcpy(arg,path,PATH_MAX);-if(len>=PATH_MAX)-returnerror("name too long: %s",path);+if(len>=PATH_MAX){+error("name too long: %s",path);+returnNULL;+}/**Inadditionto"foo.idx"weaccept"foo.pack"and"foo";
@@ -333,8 +342,10 @@ static int process_one_pack(const char *path)strcpy(arg+len-5,".idx");len--;}elseif(!has_extension(arg,".idx")){-if(len+4>=PATH_MAX)-returnerror("name too long: %s.idx",arg);+if(len+4>=PATH_MAX){+error("name too long: %s.idx",arg);+returnNULL;+}strcpy(arg+len,".idx");len+=4;}
@@ -345,17 +356,36 @@ static int process_one_pack(const char *path)*/if(len+1>=PATH_MAX){arg[len-4]='\0';-returnerror("name too long: %s.pack",arg);+error("name too long: %s.pack",arg);+returnNULL;}p=add_packed_git(arg,len,1);-if(!p)-returnerror("packfile %s not found.",arg);+if(!p){+error("packfile %s not found.",arg);+returnNULL;+}install_packed_git(p);-if(open_pack_index(p))-returnerror("packfile %s index not opened",p->pack_name);-returncreate_pack_dictionaries(p);+if(open_pack_index(p)){+error("packfile %s index not opened",p->pack_name);+returnNULL;+}++returnp;+}++staticvoidprocess_one_pack(char*src_pack)+{+structpacked_git*p;+structidx_entry*objs;++p=open_pack(src_pack);+if(!p)+die("unable to open source pack");++objs=get_packed_object_list(p);+create_pack_dictionaries(p,objs);}intmain(intargc,char*argv[])
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
Let's read a pack to feed our dictionary with all the path strings
contained in all the tree objects.
Dump the resulting dictionary sorted by frequency to stdout.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
Makefile | 1 +
packv4-create.c | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 138 insertions(+)
@@ -550,6 +550,7 @@ PROGRAM_OBJS += shell.oPROGRAM_OBJS+=show-index.oPROGRAM_OBJS+=upload-pack.oPROGRAM_OBJS+=remote-testsvn.o+PROGRAM_OBJS+=packv4-create.o# Binary suffix, set to .exe for Windows buildsX=
@@ -135,3 +153,122 @@ void dict_dump(struct dict_table *t)t->entry[i].hits,t->data+t->entry[i].offset);}++structidx_entry+{+off_toffset;+constunsignedchar*sha1;+};++staticintsort_by_offset(constvoid*e1,constvoid*e2)+{+conststructidx_entry*entry1=e1;+conststructidx_entry*entry2=e2;+if(entry1->offset<entry2->offset)+return-1;+if(entry1->offset>entry2->offset)+return1;+return0;+}+staticintcreate_pack_dictionaries(structpacked_git*p)+{+uint32_tnr_objects,i;+structidx_entry*objects;++nr_objects=p->num_objects;+objects=xmalloc((nr_objects+1)*sizeof(*objects));+objects[nr_objects].offset=p->index_size-40;+for(i=0;i<nr_objects;i++){+objects[i].sha1=nth_packed_object_sha1(p,i);+objects[i].offset=nth_packed_object_offset(p,i);+}+qsort(objects,nr_objects,sizeof(*objects),sort_by_offset);++for(i=0;i<nr_objects;i++){+void*data;+enumobject_typetype;+unsignedlongsize;+structobject_infooi={};++oi.typep=&type;+oi.sizep=&size;+if(packed_object_info(p,objects[i].offset,&oi)<0)+die("cannot get type of %s from %s",+sha1_to_hex(objects[i].sha1),p->pack_name);++switch(type){+caseOBJ_TREE:+break;+default:+continue;+}+data=unpack_entry(p,objects[i].offset,&type,&size);+if(!data)+die("cannot unpack %s from %s",+sha1_to_hex(objects[i].sha1),p->pack_name);+if(check_sha1_signature(objects[i].sha1,data,size,typename(type)))+die("packed %s from %s is corrupt",+sha1_to_hex(objects[i].sha1),p->pack_name);+if(add_tree_dict_entries(data,size)<0)+die("can't process %s object %s",+typename(type),sha1_to_hex(objects[i].sha1));+free(data);+}+free(objects);++return0;+}++staticintprocess_one_pack(constchar*path)+{+chararg[PATH_MAX];+intlen;+structpacked_git*p;++len=strlcpy(arg,path,PATH_MAX);+if(len>=PATH_MAX)+returnerror("name too long: %s",path);++/*+*Inadditionto"foo.idx"weaccept"foo.pack"and"foo";+*normalizetheseformsto"foo.idx"foradd_packed_git().+*/+if(has_extension(arg,".pack")){+strcpy(arg+len-5,".idx");+len--;+}elseif(!has_extension(arg,".idx")){+if(len+4>=PATH_MAX)+returnerror("name too long: %s.idx",arg);+strcpy(arg+len,".idx");+len+=4;+}++/*+*add_packed_git()usesourbuffer(containing"foo.idx")to+*buildthepackfilename("foo.pack").Makesureitfits.+*/+if(len+1>=PATH_MAX){+arg[len-4]='\0';+returnerror("name too long: %s.pack",arg);+}++p=add_packed_git(arg,len,1);+if(!p)+returnerror("packfile %s not found.",arg);++install_packed_git(p);+if(open_pack_index(p))+returnerror("packfile %s index not opened",p->pack_name);+returncreate_pack_dictionaries(p);+}++intmain(intargc,char*argv[])+{+if(argc!=2){+fprintf(stderr,"Usage: %s <packfile>\n",argv[0]);+exit(1);+}+process_one_pack(argv[1]);+dict_dump(tree_path_table);+return0;+}
@@ -1687,20 +1687,12 @@ static off_t get_delta_base(struct packed_git *p,*isstupid,asthenaREF_DELTAwouldbesmallertostore.*/if(type==OBJ_OFS_DELTA){-unsignedused=0;-unsignedcharc=base_info[used++];-base_offset=c&127;-while(c&128){-base_offset+=1;-if(!base_offset||MSB(base_offset,7))-return0;/* overflow */-c=base_info[used++];-base_offset=(base_offset<<7)+(c&127);-}+constunsignedchar*cp=base_info;+base_offset=decode_varint(&cp);base_offset=delta_obj_offset-base_offset;if(base_offset<=0||base_offset>=delta_obj_offset)return0;/* out of bound */-*curpos+=used;+*curpos+=cp-base_info;}elseif(type==OBJ_REF_DELTA){/* The base entry _must_ be in the same pack */base_offset=find_pack_entry_one(base_info,p);
--
1.8.4.38.g317e65b
This patch seems to be a cleanup independent from pack v4, it applies
cleanly on master and passes all tests in itself.
Best,
Gábor
From: SZEDER Gábor <hidden> Date: 2016-06-15 22:58:38
Hi,
On Thu, Sep 05, 2013 at 02:19:28AM -0400, Nicolas Pitre wrote:
quoted hunk
Let's create another dictionary table to hold the author and committer
entries. We use the same table format used for tree entries where the
16 bit data prefix is conveniently used to store the timezone value.
In order to copy straight from a commit object buffer, dict_add_entry()
is modified to get the string length as the provided string pointer is
not always be null terminated.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 89 insertions(+), 9 deletions(-)
We need a +1 here on the left side, i.e.
if (t->ptr + val_len + str_len + 1 > t->size) {
The str_len variable accounted for the terminating null character
before, but this patch removes str_len = strlen(str) + 1; above, and
callers specify the length of str without the terminating null in
str_len. Thus it can lead to memory corruption, when the new entry
happens to end at 't->ptr + val_len + str_len' and the line added in
the next hunk writes the terminating null beyond the end of the
buffer. I couldn't create a v4 pack from a current linux repo because
of this; either glibc detected something or 'git packv4-create'
crashed.
Sidenote: couldn't we call the 'ptr' field something else, like
end_offset or end_idx? It took me some headscratching to figure out
why is it OK to compare a pointer to an integer above, or use a
pointer without dereferencing as an index into an array below (because
ptr is, well, not a pointer after all).
t->nb_entries++;
if (t->hash_size * 3 <= t->nb_entries * 4)
@@ -135,8 +136,73 @@ static void sort_dict_entries_by_hits(struct dict_table *t) rehash_entries(t); }+static struct dict_table *commit_name_table; static struct dict_table *tree_path_table;+/*+ * Parse the author/committer line from a canonical commit object.+ * The 'from' argument points right after the "author " or "committer "+ * string. The time zone is parsed and stored in *tz_val. The returned+ * pointer is right after the end of the email address which is also just+ * before the time value, or NULL if a parsing error is encountered.+ */+static char *get_nameend_and_tz(char *from, int *tz_val)+{+ char *end, *tz;++ tz = strchr(from, '\n');+ /* let's assume the smallest possible string to be "x <x> 0 +0000\n" */+ if (!tz || tz - from < 13)+ return NULL;+ tz -= 4;+ end = tz - 4;+ while (end - from > 5 && *end != ' ')+ end--;+ if (end[-1] != '>' || end[0] != ' ' || tz[-2] != ' ')+ return NULL;+ *tz_val = (tz[0] - '0') * 1000 ++ (tz[1] - '0') * 100 ++ (tz[2] - '0') * 10 ++ (tz[3] - '0');+ switch (tz[-1]) {+ default: return NULL;+ case '+': break;+ case '-': *tz_val = -*tz_val;+ }+ return end;+}++static int add_commit_dict_entries(void *buf, unsigned long size)+{+ char *name, *end = NULL;+ int tz_val;++ if (!commit_name_table)+ commit_name_table = create_dict_table();++ /* parse and add author info */+ name = strstr(buf, "\nauthor ");+ if (name) {+ name += 8;+ end = get_nameend_and_tz(name, &tz_val);+ }+ if (!name || !end)+ return -1;+ dict_add_entry(commit_name_table, tz_val, name, end - name);++ /* parse and add committer info */+ name = strstr(end, "\ncommitter ");+ if (name) {+ name += 11;+ end = get_nameend_and_tz(name, &tz_val);+ }+ if (!name || !end)+ return -1;+ dict_add_entry(commit_name_table, tz_val, name, end - name);++ return 0;+}+ static int add_tree_dict_entries(void *buf, unsigned long size) { struct tree_desc desc;
@@ -146,13 +212,16 @@ static int add_tree_dict_entries(void *buf, unsigned long size) tree_path_table = create_dict_table(); init_tree_desc(&desc, buf, size);- while (tree_entry(&desc, &name_entry))+ while (tree_entry(&desc, &name_entry)) {+ int pathlen = tree_entry_len(&name_entry); dict_add_entry(tree_path_table, name_entry.mode,- name_entry.path);+ name_entry.path, pathlen);+ }+ return 0; }-void dict_dump(struct dict_table *t)+void dump_dict_table(struct dict_table *t) { int i;
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
On Thu, 5 Sep 2013, SZEDER Gábor wrote:
Hi,
On Thu, Sep 05, 2013 at 02:19:28AM -0400, Nicolas Pitre wrote:
quoted
Let's create another dictionary table to hold the author and committer
entries. We use the same table format used for tree entries where the
16 bit data prefix is conveniently used to store the timezone value.
In order to copy straight from a commit object buffer, dict_add_entry()
is modified to get the string length as the provided string pointer is
not always be null terminated.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
---
packv4-create.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 89 insertions(+), 9 deletions(-)
We need a +1 here on the left side, i.e.
if (t->ptr + val_len + str_len + 1 > t->size) {
Absolutely, good catch.
Sidenote: couldn't we call the 'ptr' field something else, like
end_offset or end_idx? It took me some headscratching to figure out
why is it OK to compare a pointer to an integer above, or use a
pointer without dereferencing as an index into an array below (because
ptr is, well, not a pointer after all).
Indeed. This is a remnant of an earlier implementation which didn't use
realloc() and therefore this used to be a real pointer.
Both issues now addressed in my tree.
Thanks
Nicolas
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:38
On Thu, 5 Sep 2013, Nicolas Pitre wrote:
If the path or name index is zero, this means the entry data is to be
found inline rather than being located in the dictionary table. This is
there to allow easy completion of thin packs without having to add new
table entries which would have required a full rewrite of the pack data.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
I'm now dropping this patch. Please also remove this from your
documentation patch.
I think that we've found a way to better support thin packs.
You said:
What if the sender prepares the sha-1 table to contain missing objects
in advance? The sender should know what base objects are missing. Then
we only need to append objects at the receiving end and verify that
all new objects are also present in the sha-1 table.
So the SHA1 table is covered.
Missing objects in a thin pack cannot themselves be deltas. We had
their undeltified form at the end of a pack for the pack to be complete.
Therefore those missing objects serve only as base objects for other
deltas.
Although this is possible to have deltified commit objects in pack v2, I
don't think this happens very often. There is no deltified commit
objects in pack v4.
Blob objects are the same in pack v2 and pack v4. No dictionary
references are needed.
That leaves only tree objects. And because we've also discussed the
need to have non transcoded object representations for those odd cases
such as zero padded file modes, we might as well simply use that for the
appended tree objects already needed to complete a thin pack. At least
the strings in tree entries will be compressed that way.
Problem solved, and one less special case in the code.
What do you think?
Nicolas
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:39
On Thu, 5 Sep 2013, Nicolas Pitre wrote:
On Thu, 5 Sep 2013, Nicolas Pitre wrote:
quoted
If the path or name index is zero, this means the entry data is to be
found inline rather than being located in the dictionary table. This is
there to allow easy completion of thin packs without having to add new
table entries which would have required a full rewrite of the pack data.
Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
I'm now dropping this patch. Please also remove this from your
documentation patch.
Well... I couldn't resist another little change that has been nagging me
for a while.
Both the author and committer time stamps are very closely related most
of the time. So the committer time stamp is now encoded as a difference
against the author time stamp with the LSB indicating a negative
difference.
On git.git this saves 0.3% on the pack size. Not much, but still
impressive for only a time stamp.
Nicolas
On Fri, Sep 6, 2013 at 2:02 AM, Nicolas Pitre [off-list ref] wrote:
I think that we've found a way to better support thin packs.
You said:
quoted
What if the sender prepares the sha-1 table to contain missing objects
in advance? The sender should know what base objects are missing. Then
we only need to append objects at the receiving end and verify that
all new objects are also present in the sha-1 table.
So the SHA1 table is covered.
Missing objects in a thin pack cannot themselves be deltas. We had
their undeltified form at the end of a pack for the pack to be complete.
Therefore those missing objects serve only as base objects for other
deltas.
Although this is possible to have deltified commit objects in pack v2, I
don't think this happens very often. There is no deltified commit
objects in pack v4.
Blob objects are the same in pack v2 and pack v4. No dictionary
references are needed.
That leaves only tree objects. And because we've also discussed the
need to have non transcoded object representations for those odd cases
such as zero padded file modes, we might as well simply use that for the
appended tree objects already needed to complete a thin pack. At least
the strings in tree entries will be compressed that way.
Problem solved, and one less special case in the code.
What do you think?
Agreed.
Please also remove this from your documentation patch.
This makes index-pack recognize pack v4. It still lacks:
- the ability to walk through multi-base trees
- thin pack support
The first is not easy to solve imo and but does not impact us in short
term because pack-objects probably will not learn to produce such
trees any time soon.
The second should be done after pack-objects can produce thin packs,
else it's hard to verify that the code works as expected.
This bases on Nico's tree, which does not really match the series this
post is replied to due to some format changes. I don't know, maybe we
could share more code with packv4-parse.c. Right now I just need
something that works and somewhat maintainable.
Nguyễn Thái Ngọc Duy (12):
pack v4: split pv4_create_dict() out of load_dict()
index-pack: split out varint decoding code
index-pack: do not allocate buffer for unpacking deltas in the first pass
index-pack: split inflate/digest code out of unpack_entry_data
index-pack: parse v4 header and dictionaries
index-pack: make sure all objects are registered in v4's SHA-1 table
index-pack: parse v4 commit format
index-pack: parse v4 tree format
index-pack: move delta base queuing code to unpack_raw_entry
index-pack: record all delta bases in v4 (tree and ref-delta)
index-pack: skip looking for ofs-deltas in v4 as they are not allowed
index-pack: resolve v4 one-base trees
builtin/index-pack.c | 679 ++++++++++++++++++++++++++++++++++++++++++++-------
packv4-parse.c | 63 ++---
packv4-parse.h | 8 +
3 files changed, 627 insertions(+), 123 deletions(-)
--
1.8.2.83.gc99314b
@@ -455,55 +477,41 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size,returnbuf==fixed_buf?NULL:buf;}+staticvoidread_typesize_v2(structobject_entry*obj)+{+unsignedcharc=*(char*)fill_and_use(1);+unsignedshift;++obj->type=(c>>4)&7;+obj->size=(c&15);+shift=4;+while(c&128){+c=*(char*)fill_and_use(1);+obj->size+=(c&0x7f)<<shift;+shift+=7;+}+}+staticvoid*unpack_raw_entry(structobject_entry*obj,uniondelta_base*delta_base,unsignedchar*sha1){-unsignedchar*p;-unsignedlongsize,c;-off_tbase_offset;-unsignedshift;void*data;+uintmax_tval;obj->idx.offset=consumed_bytes;input_crc32=crc32(0,NULL,0);-p=fill(1);-c=*p;-use(1);-obj->type=(c>>4)&7;-size=(c&15);-shift=4;-while(c&0x80){-p=fill(1);-c=*p;-use(1);-size+=(c&0x7f)<<shift;-shift+=7;-}-obj->size=size;+read_typesize_v2(obj);switch(obj->type){caseOBJ_REF_DELTA:-hashcpy(delta_base->sha1,fill(20));-use(20);+hashcpy(delta_base->sha1,fill_and_use(20));break;caseOBJ_OFS_DELTA:memset(delta_base,0,sizeof(*delta_base));-p=fill(1);-c=*p;-use(1);-base_offset=c&127;-while(c&128){-base_offset+=1;-if(!base_offset||MSB(base_offset,7))-bad_object(obj->idx.offset,_("offset value overflow for delta base object"));-p=fill(1);-c=*p;-use(1);-base_offset=(base_offset<<7)+(c&127);-}-delta_base->offset=obj->idx.offset-base_offset;+val=read_varint();+delta_base->offset=obj->idx.offset-val;if(delta_base->offset<=0||delta_base->offset>=obj->idx.offset)bad_object(obj->idx.offset,_("delta base offset is out of bound"));break;
We do need deltas until the second pass. Allocating a buffer for it
then freeing later is wasteful is unnecessary. Make it use fixed_buf
(aka large blob code path).
Signed-off-by: Nguyễn Thái Ngọc Duy <redacted>
---
builtin/index-pack.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
@@ -429,33 +429,19 @@ static int is_delta_type(enum object_type type)return(type==OBJ_REF_DELTA||type==OBJ_OFS_DELTA);}-staticvoid*unpack_entry_data(unsignedlongoffset,unsignedlongsize,-enumobject_typetype,unsignedchar*sha1)+staticvoidread_and_inflate(unsignedlongoffset,+void*buf,unsignedlongsize,+unsignedlongwraparound,+git_SHA_CTX*ctx,+unsignedchar*sha1){-staticcharfixed_buf[8192];-intstatus;git_zstreamstream;-void*buf;-git_SHA_CTXc;-charhdr[32];-inthdrlen;--if(!is_delta_type(type)){-hdrlen=sprintf(hdr,"%s %lu",typename(type),size)+1;-git_SHA1_Init(&c);-git_SHA1_Update(&c,hdr,hdrlen);-}else-sha1=NULL;-if(is_delta_type(type)||-(type==OBJ_BLOB&&size>big_file_threshold))-buf=fixed_buf;-else-buf=xmalloc(size);+intstatus;memset(&stream,0,sizeof(stream));git_inflate_init(&stream);stream.next_out=buf;-stream.avail_out=buf==fixed_buf?sizeof(fixed_buf):size;+stream.avail_out=wraparound?wraparound:size;do{unsignedchar*last_out=stream.next_out;
@@ -464,17 +450,43 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size,status=git_inflate(&stream,0);use(input_len-stream.avail_in);if(sha1)-git_SHA1_Update(&c,last_out,stream.next_out-last_out);-if(buf==fixed_buf){+git_SHA1_Update(ctx,last_out,stream.next_out-last_out);+if(wraparound){stream.next_out=buf;-stream.avail_out=sizeof(fixed_buf);+stream.avail_out=wraparound;}}while(status==Z_OK);if(stream.total_out!=size||status!=Z_STREAM_END)bad_object(offset,_("inflate returned %d"),status);git_inflate_end(&stream);if(sha1)-git_SHA1_Final(sha1,&c);+git_SHA1_Final(sha1,ctx);+}++staticvoid*unpack_entry_data(unsignedlongoffset,unsignedlongsize,+enumobject_typetype,unsignedchar*sha1)+{+staticcharfixed_buf[8192];+void*buf;+git_SHA_CTXc;+charhdr[32];+inthdrlen;++if(!is_delta_type(type)){+hdrlen=sprintf(hdr,"%s %lu",typename(type),size)+1;+git_SHA1_Init(&c);+git_SHA1_Update(&c,hdr,hdrlen);+}else+sha1=NULL;+if(is_delta_type(type)||+(type==OBJ_BLOB&&size>big_file_threshold))+buf=fixed_buf;+else+buf=xmalloc(size);++read_and_inflate(offset,buf,size,+buf==fixed_buf?sizeof(fixed_buf):0,+&c,sha1);returnbuf==fixed_buf?NULL:buf;}
@@ -742,6 +742,19 @@ static int check_collison(struct object_entry *entry)return0;}+staticvoidcheck_against_sha1table(structobject_entry*obj)+{+constunsignedchar*found;+if(!packv4)+return;++found=bsearch(obj->idx.sha1,sha1_table,nr_objects,20,+(int(*)(constvoid*,constvoid*))hashcmp);+if(!found)+die(_("object %s not found in SHA-1 table"),+sha1_to_hex(obj->idx.sha1));+}+staticvoidsha1_object(constvoid*data,structobject_entry*obj_entry,unsignedlongsize,enumobject_typetype,constunsignedchar*sha1)
@@ -304,6 +304,30 @@ static uintmax_t read_varint(void)returnval;}+staticconstunsignedchar*read_sha1ref(void)+{+unsignedintindex=read_varint();+if(!index){+staticunsignedcharsha1[20];+hashcpy(sha1,fill_and_use(20));+returnsha1;+}+index--;+if(index>=nr_objects)+bad_object(consumed_bytes,+_("bad index in read_sha1ref"));+returnsha1_table+index*20;+}++staticconstunsignedchar*read_dictref(structpackv4_dict*dict)+{+unsignedintindex=read_varint();+if(index>=dict->nb_entries)+bad_object(consumed_bytes,+_("bad index in read_dictref"));+returndict->data+dict->offsets[index];+}+staticvoid*read_data(intsize){constintmax=sizeof(input_buffer);
@@ -319,6 +319,21 @@ static const unsigned char *read_sha1ref(void)returnsha1_table+index*20;}+staticconstunsignedchar*read_sha1table_ref(void)+{+constunsignedchar*sha1=read_sha1ref();+if(sha1<sha1_table||sha1>=sha1_table+nr_objects*20){+unsignedchar*found;+found=bsearch(sha1,sha1_table,nr_objects,20,+(int(*)(constvoid*,constvoid*))hashcmp);+if(!found)+bad_object(consumed_bytes,+_("SHA-1 %s not found in SHA-1 table"),+sha1_to_hex(sha1));+}+returnsha1;+}+staticconstunsignedchar*read_dictref(structpackv4_dict*dict){unsignedintindex=read_varint();
@@ -561,17 +576,93 @@ static void *unpack_commit_v4(unsigned int offset,returndst.buf;}-staticvoid*unpack_entry_data(unsignedlongoffset,unsignedlongsize,-enumobject_typetype,unsignedchar*sha1)+/*+*v4treesareactuallykindofdeltasandwedon'tdodeltainthe+*firstpass.Thisfunctiononlywalksthroughatreeobjecttofind+*theendoffset,registerobjectdependenciesandperformslimited+*validation.+*/+staticvoid*unpack_tree_v4(structobject_entry*obj,+unsignedintoffset,unsignedlongsize,+unsignedchar*sha1)+{+unsignedintnr=read_varint();+constunsignedchar*last_base=NULL;+structstrbufsb=STRBUF_INIT;+while(nr){+unsignedintcopy_start_or_path=read_varint();+if(copy_start_or_path&1){/* copy_start */+unsignedintcopy_count=read_varint();+if(copy_count&1){/* first delta */+last_base=read_sha1table_ref();+}elseif(!last_base)+bad_object(offset,+_("bad copy count index in unpack_tree_v4"));+copy_count>>=1;+if(!copy_count)+bad_object(offset,+_("bad copy count index in unpack_tree_v4"));+nr-=copy_count;+}else{/* path */+unsignedintpath_idx=copy_start_or_path>>1;+constunsignedchar*entry_sha1;++if(path_idx>=path_dict->nb_entries)+bad_object(offset,+_("bad path index in unpack_tree_v4"));+entry_sha1=read_sha1ref();+nr--;++if(!last_base){+constunsignedchar*path;+unsignedmode;++path=path_dict->data+path_dict->offsets[path_idx];+mode=(path[0]<<8)|path[1];+strbuf_addf(&sb,"%o %s%c",mode,path+2,'\0');+strbuf_add(&sb,entry_sha1,20);+if(sb.len>size)+bad_object(offset,+_("tree larger than expected"));+}+}+}++if(last_base){+strbuf_release(&sb);+returnNULL;+}else{+git_SHA_CTXctx;+charhdr[32];+inthdrlen;++if(sb.len!=size)+bad_object(offset,_("tree size mismatch"));++hdrlen=sprintf(hdr,"tree %lu",size)+1;+git_SHA1_Init(&ctx);+git_SHA1_Update(&ctx,hdr,hdrlen);+git_SHA1_Update(&ctx,sb.buf,size);+git_SHA1_Final(sha1,&ctx);+returnstrbuf_detach(&sb,NULL);+}+}++staticvoid*unpack_entry_data(structobject_entry*obj,unsignedchar*sha1){staticcharfixed_buf[8192];void*buf;git_SHA_CTXc;charhdr[32];inthdrlen;+unsignedlongoffset=obj->idx.offset;+unsignedlongsize=obj->size;+enumobject_typetype=obj->type;if(type==OBJ_PV4_COMMIT)returnunpack_commit_v4(offset,size,sha1);+if(type==OBJ_PV4_TREE)+returnunpack_tree_v4(obj,offset,size,sha1);if(!is_delta_type(type)){hdrlen=sprintf(hdr,"%s %lu",typename(type),size)+1;
@@ -1186,6 +1280,8 @@ static void parse_pack_objects(unsigned char *sha1)nr_deltas++;delta->obj_no=i;delta++;+}elseif(!data&&obj->type==OBJ_PV4_TREE){+/* delay sha1_object() until second pass */}elseif(!data){/* large blobs, check later */obj->real_type=OBJ_BAD;
For v2, ofs-delta and ref-delta can only have queue one delta base at
a time. A v4 tree can have more than one delta base. Move the queuing
code up to unpack_raw_entry() and give unpack_tree_v4() more
flexibility to add its bases.
Signed-off-by: Nguyễn Thái Ngọc Duy <redacted>
---
builtin/index-pack.c | 46 ++++++++++++++++++++++++++++++----------------
1 file changed, 30 insertions(+), 16 deletions(-)
@@ -717,14 +735,14 @@ static void *unpack_raw_entry(struct object_entry *obj,switch(obj->type){caseOBJ_REF_DELTA:-hashcpy(delta_base->sha1,fill_and_use(20));+add_sha1_delta(obj,fill_and_use(20));break;caseOBJ_OFS_DELTA:-memset(delta_base,0,sizeof(*delta_base));-val=read_varint();-delta_base->offset=obj->idx.offset-val;-if(delta_base->offset<=0||delta_base->offset>=obj->idx.offset)-bad_object(obj->idx.offset,_("delta base offset is out of bound"));+offset=obj->idx.offset-read_varint();+if(offset<=0||offset>=obj->idx.offset)+bad_object(obj->idx.offset,+_("delta base offset is out of bound"));+add_ofs_delta(obj,offset);break;caseOBJ_COMMIT:caseOBJ_TREE:
@@ -1275,12 +1292,9 @@ static void parse_pack_objects(unsigned char *sha1)nr_objects);for(i=0;i<nr_objects;i++){structobject_entry*obj=&objects[i];-void*data=unpack_raw_entry(obj,&delta->base,obj->idx.sha1);-if(is_delta_type(obj->type)){-nr_deltas++;-delta->obj_no=i;-delta++;-}elseif(!data&&obj->type==OBJ_PV4_TREE){+void*data=unpack_raw_entry(obj,obj->idx.sha1);+if(is_delta_type(obj->type)||+(!data&&obj->type==OBJ_PV4_TREE)){/* delay sha1_object() until second pass */}elseif(!data){/* large blobs, check later */
@@ -24,6 +24,7 @@ struct object_entry {enumobject_typereal_type;unsigneddelta_depth;intbase_object_no;+intnr_bases;/* only valid for v4 trees */};uniondelta_base{
@@ -489,6 +490,11 @@ static int is_delta_type(enum object_type type)return(type==OBJ_REF_DELTA||type==OBJ_OFS_DELTA);}+staticintis_delta_tree(conststructobject_entry*obj)+{+returnobj->type==OBJ_PV4_TREE&&obj->nr_bases>0;+}+staticvoidread_and_inflate(unsignedlongoffset,void*buf,unsignedlongsize,unsignedlongwraparound,
@@ -608,12 +628,14 @@ static void *unpack_tree_v4(struct object_entry *obj,unsignedintnr=read_varint();constunsignedchar*last_base=NULL;structstrbufsb=STRBUF_INIT;+intdelta_start=nr_deltas;while(nr){unsignedintcopy_start_or_path=read_varint();if(copy_start_or_path&1){/* copy_start */unsignedintcopy_count=read_varint();if(copy_count&1){/* first delta */last_base=read_sha1table_ref();+add_tree_delta_base(obj,last_base,delta_start);}elseif(!last_base)bad_object(offset,_("bad copy count index in unpack_tree_v4"));
@@ -735,9 +757,15 @@ static void *unpack_raw_entry(struct object_entry *obj,switch(obj->type){caseOBJ_REF_DELTA:-add_sha1_delta(obj,fill_and_use(20));+if(packv4)+add_sha1_delta(obj,read_sha1table_ref());+else+add_sha1_delta(obj,fill_and_use(20));break;caseOBJ_OFS_DELTA:+if(packv4)+die(_("pack version 4 does not support ofs-delta type (offset %lu)"),+obj->idx.offset);offset=obj->idx.offset-read_varint();if(offset<=0||offset>=obj->idx.offset)bad_object(obj->idx.offset,
@@ -1293,8 +1321,7 @@ static void parse_pack_objects(unsigned char *sha1)for(i=0;i<nr_objects;i++){structobject_entry*obj=&objects[i];void*data=unpack_raw_entry(obj,obj->idx.sha1);-if(is_delta_type(obj->type)||-(!data&&obj->type==OBJ_PV4_TREE)){+if(is_delta_type(obj->type)||is_delta_tree(obj)){/* delay sha1_object() until second pass */}elseif(!data){/* large blobs, check later */
This is the most common case for delta trees. In fact it's the only
kind that's produced by packv4-create. It fits well in the way
index-pack resolves deltas and benefits from threading (the set of
objects depending on this base does not overlap with the set of
objects depending on another base)
Multi-base trees will be probably processed differently.
Signed-off-by: Nguyễn Thái Ngọc Duy <redacted>
---
builtin/index-pack.c | 194 ++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 178 insertions(+), 16 deletions(-)
@@ -670,6 +673,8 @@ static void *unpack_tree_v4(struct object_entry *obj,}if(last_base){+if(nr_deltas-delta_start>1)+die("sorry guys, multi-base trees are not supported yet");strbuf_release(&sb);returnNULL;}else{
@@ -794,6 +799,83 @@ static void *unpack_raw_entry(struct object_entry *obj,returndata;}+staticvoid*patch_one_base_tree(conststructobject_entry*src,+constunsignedchar*src_buf,+constunsignedchar*delta_buf,+unsignedlongdelta_size,+unsignedlong*dst_size)+{+unsignedintnr;+constunsignedchar*last_base=NULL;+structstrbufsb=STRBUF_INIT;+constunsignedchar*p=delta_buf;++nr=decode_varint(&p);+while(nr&&p<delta_buf+delta_size){+unsignedintcopy_start_or_path=decode_varint(&p);+if(copy_start_or_path&1){/* copy_start */+structtree_descdesc;+structname_entryentry;+unsignedintcopy_count=decode_varint(&p);+unsignedintcopy_start=copy_start_or_path>>1;+if(!src)+die("we are not supposed to copy from another tree!");+if(copy_count&1){/* first delta */+unsignedintid=decode_varint(&p);+if(!id){+last_base=p;+p+=20;+}else+last_base=sha1_table+(id-1)*20;+if(hashcmp(last_base,src->idx.sha1))+die(_("bad tree base in patch_one_base_tree"));+}elseif(!last_base)+die(_("bad copy count index in patch_one_base_tree"));+copy_count>>=1;+if(!copy_count)+die(_("bad copy count index in patch_one_base_tree"));+nr-=copy_count;++init_tree_desc(&desc,src_buf,src->size);+while(tree_entry(&desc,&entry)){+if(copy_start)+copy_start--;+elseif(copy_count){+strbuf_addf(&sb,"%o %s%c",entry.mode,entry.path,'\0');+strbuf_add(&sb,entry.sha1,20);+copy_count--;+}else+break;+}+}else{/* path */+unsignedintpath_idx=copy_start_or_path>>1;+constunsignedchar*path;+unsignedmode;+unsignedintid;+constunsignedchar*entry_sha1;++if(path_idx>=path_dict->nb_entries)+die(_("bad path index in unpack_tree_v4"));+id=decode_varint(&p);+if(!id){+entry_sha1=p;+p+=20;+}else+entry_sha1=sha1_table+(id-1)*20;+nr--;++path=path_dict->data+path_dict->offsets[path_idx];+mode=(path[0]<<8)|path[1];+strbuf_addf(&sb,"%o %s%c",mode,path+2,'\0');+strbuf_add(&sb,entry_sha1,20);+}+}+if(nr!=0||p!=delta_buf+delta_size)+die(_("bad delta tree"));+*dst_size=sb.len;+returnsb.buf;+}+staticvoid*unpack_data(structobject_entry*obj,int(*consume)(constunsignedchar*,unsignedlong,void*),void*cb_data)
@@ -855,8 +937,33 @@ static void *unpack_data(struct object_entry *obj,returndata;}+staticvoid*get_tree_v4_from_pack(structobject_entry*obj,+unsignedlong*len_p)+{+off_tfrom=obj[0].idx.offset+obj[0].hdr_size;+unsignedlonglen=obj[1].idx.offset-from;+unsignedchar*data;+ssize_tn;++data=xmalloc(len);+n=pread(pack_fd,data,len,from);+if(n<0)+die_errno(_("cannot pread pack file"));+if(!n)+die(Q_("premature end of pack file, %lu byte missing",+"premature end of pack file, %lu bytes missing",+len),+len);+if(len_p)+*len_p=len;+returndata;+}+staticvoid*get_data_from_pack(structobject_entry*obj){+if(obj->type==OBJ_PV4_COMMIT||obj->type==OBJ_PV4_TREE)+die("BUG: unsupported code path");+returnunpack_data(obj,NULL,NULL);}
@@ -1213,6 +1355,25 @@ static struct base_data *find_unresolved_deltas_1(struct base_data *base,returnresult;}+while(base->tree_first<=base->tree_last){+structobject_entry*child=objects+deltas[base->tree_first].obj_no;+structbase_data*result;++assert(child->type==OBJ_PV4_TREE);+if(child->nr_bases>1){+/* maybe resolved in the third pass or something */+base->tree_first++;+continue;+}+result=alloc_base_data();+resolve_delta(child,base,result);+if(base->tree_first==base->tree_last)+free_base_data(base);++base->tree_first++;+returnresult;+}+unlink_base_data(base);returnNULL;}
@@ -1677,6 +1727,15 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) free(objects); free(index_name_buf); free(keep_name_buf);+ free(sha1_table);+ if (name_dict) {+ free((void*)name_dict->data);+ free(name_dict);+ }+ if (path_dict) {+ free((void*)path_dict->data);+ free(path_dict);+ }
The freeing of dictionary tables should probably have its own function
in packv4-parse.c. and a call to it added in free_pack_by_name() as
well.
Nicolas
@@ -319,6 +319,21 @@ static const unsigned char *read_sha1ref(void)returnsha1_table+index*20;}+staticconstunsignedchar*read_sha1table_ref(void)+{+constunsignedchar*sha1=read_sha1ref();+if(sha1<sha1_table||sha1>=sha1_table+nr_objects*20){+unsignedchar*found;+found=bsearch(sha1,sha1_table,nr_objects,20,+(int(*)(constvoid*,constvoid*))hashcmp);+if(!found)+bad_object(consumed_bytes,+_("SHA-1 %s not found in SHA-1 table"),+sha1_to_hex(sha1));+}+returnsha1;+}+staticconstunsignedchar*read_dictref(structpackv4_dict*dict){unsignedintindex=read_varint();
@@ -561,17 +576,93 @@ static void *unpack_commit_v4(unsigned int offset,returndst.buf;}-staticvoid*unpack_entry_data(unsignedlongoffset,unsignedlongsize,-enumobject_typetype,unsignedchar*sha1)+/*+*v4treesareactuallykindofdeltasandwedon'tdodeltainthe+*firstpass.Thisfunctiononlywalksthroughatreeobjecttofind+*theendoffset,registerobjectdependenciesandperformslimited+*validation.+*/+staticvoid*unpack_tree_v4(structobject_entry*obj,+unsignedintoffset,unsignedlongsize,+unsignedchar*sha1)+{+unsignedintnr=read_varint();+constunsignedchar*last_base=NULL;+structstrbufsb=STRBUF_INIT;+while(nr){+unsignedintcopy_start_or_path=read_varint();+if(copy_start_or_path&1){/* copy_start */+unsignedintcopy_count=read_varint();+if(copy_count&1){/* first delta */+last_base=read_sha1table_ref();+}elseif(!last_base)+bad_object(offset,+_("bad copy count index in unpack_tree_v4"));
Here the error message could be a little more explicit i.e. "missing
delta base" or the like in order to distinguish from the next error.
+ copy_count >>= 1;
+ if (!copy_count)
+ bad_object(offset,
+ _("bad copy count index in unpack_tree_v4"));
+ nr -= copy_count;
Also make sure copy_count <= nr here.
+ } else { /* path */
+ unsigned int path_idx = copy_start_or_path >> 1;
+ const unsigned char *entry_sha1;
+
+ if (path_idx >= path_dict->nb_entries)
+ bad_object(offset,
+ _("bad path index in unpack_tree_v4"));
+ entry_sha1 = read_sha1ref();
+ nr--;
+
+ if (!last_base) {
I've been confused for a while here by the use of last_base in the non
delta path. A comment indicating why this used here might be helpful to
those unfamiliar with the format.
@@ -640,16 +731,19 @@ static void *unpack_raw_entry(struct object_entry *obj, case OBJ_BLOB: case OBJ_TAG: break;- case OBJ_PV4_COMMIT: obj->real_type = OBJ_COMMIT; break;+ case OBJ_PV4_TREE:+ obj->real_type = OBJ_TREE;+ break;+ default: bad_object(obj->idx.offset, _("unknown object type %d"), obj->type); } obj->hdr_size = consumed_bytes - obj->idx.offset;- data = unpack_entry_data(obj->idx.offset, obj->size, obj->type, sha1);+ data = unpack_entry_data(obj, sha1); obj->idx.crc32 = input_crc32; return data; }
@@ -1186,6 +1280,8 @@ static void parse_pack_objects(unsigned char *sha1) nr_deltas++; delta->obj_no = i; delta++;+ } else if (!data && obj->type == OBJ_PV4_TREE) {+ /* delay sha1_object() until second pass */ } else if (!data) { /* large blobs, check later */ obj->real_type = OBJ_BAD;
--
1.8.2.83.gc99314b
--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
From: Nicolas Pitre <nico@fluxnic.net> Date: 2016-06-15 22:58:40
On Sat, 7 Sep 2013, Nguyễn Thái Ngọc Duy wrote:
quoted hunk
This is the most common case for delta trees. In fact it's the only
kind that's produced by packv4-create. It fits well in the way
index-pack resolves deltas and benefits from threading (the set of
objects depending on this base does not overlap with the set of
objects depending on another base)
Multi-base trees will be probably processed differently.
Signed-off-by: Nguyễn Thái Ngọc Duy <redacted>
---
builtin/index-pack.c | 194 ++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 178 insertions(+), 16 deletions(-)
@@ -670,6 +673,8 @@ static void *unpack_tree_v4(struct object_entry *obj,}if(last_base){+if(nr_deltas-delta_start>1)+die("sorry guys, multi-base trees are not supported yet");strbuf_release(&sb);returnNULL;}else{
@@ -794,6 +799,83 @@ static void *unpack_raw_entry(struct object_entry *obj,returndata;}+staticvoid*patch_one_base_tree(conststructobject_entry*src,+constunsignedchar*src_buf,+constunsignedchar*delta_buf,+unsignedlongdelta_size,+unsignedlong*dst_size)+{+unsignedintnr;+constunsignedchar*last_base=NULL;+structstrbufsb=STRBUF_INIT;+constunsignedchar*p=delta_buf;++nr=decode_varint(&p);+while(nr&&p<delta_buf+delta_size){+unsignedintcopy_start_or_path=decode_varint(&p);+if(copy_start_or_path&1){/* copy_start */+structtree_descdesc;+structname_entryentry;+unsignedintcopy_count=decode_varint(&p);+unsignedintcopy_start=copy_start_or_path>>1;+if(!src)+die("we are not supposed to copy from another tree!");+if(copy_count&1){/* first delta */+unsignedintid=decode_varint(&p);+if(!id){+last_base=p;+p+=20;+}else+last_base=sha1_table+(id-1)*20;+if(hashcmp(last_base,src->idx.sha1))+die(_("bad tree base in patch_one_base_tree"));+}elseif(!last_base)+die(_("bad copy count index in patch_one_base_tree"));+copy_count>>=1;+if(!copy_count)+die(_("bad copy count index in patch_one_base_tree"));+nr-=copy_count;++init_tree_desc(&desc,src_buf,src->size);+while(tree_entry(&desc,&entry)){+if(copy_start)+copy_start--;+elseif(copy_count){+strbuf_addf(&sb,"%o %s%c",entry.mode,entry.path,'\0');+strbuf_add(&sb,entry.sha1,20);+copy_count--;+}else+break;+}+}else{/* path */+unsignedintpath_idx=copy_start_or_path>>1;+constunsignedchar*path;+unsignedmode;+unsignedintid;+constunsignedchar*entry_sha1;++if(path_idx>=path_dict->nb_entries)+die(_("bad path index in unpack_tree_v4"));+id=decode_varint(&p);+if(!id){+entry_sha1=p;+p+=20;+}else+entry_sha1=sha1_table+(id-1)*20;
You should verify that id doesn't overflow the sha1 table here.
Similarly in other places.
Nicolas
@@ -22,8 +22,8 @@ struct object_entry {structpack_idx_entryidx;unsignedlongsize;unsignedinthdr_size;-enumobject_typetype;-enumobject_typereal_type;+enumobject_typetype;/* type as written in pack */+enumobject_typereal_type;/* type after delta resolving */unsigneddelta_depth;intbase_object_no;intnr_bases;/* only valid for v4 trees */
@@ -194,8 +194,10 @@ static int mark_link(struct object *obj, int type, void *data)return0;}-/* The content of each linked object must have been checked-oritmustbealreadypresentintheobjectdatabase*/+/*+*Thecontentofeachlinkedobjectmusthavebeencheckedoritmust+*bealreadypresentintheobjectdatabase+*/staticunsignedcheck_object(structobject*obj){if(!obj)
@@ -289,6 +291,19 @@ static inline void *fill_and_use(int bytes)returnp;}+staticvoidcheck_against_sha1table(constunsignedchar*sha1)+{+constunsignedchar*found;+if(!packv4)+return;++found=bsearch(sha1,sha1_table,nr_objects,20,+(int(*)(constvoid*,constvoid*))hashcmp);+if(!found)+die(_("object %s not found in SHA-1 table"),+sha1_to_hex(sha1));+}+staticNORETURNvoidbad_object(unsignedlongoffset,constchar*format,...)__attribute__((format(printf,2,3)));
@@ -325,15 +340,8 @@ static const unsigned char *read_sha1ref(void)staticconstunsignedchar*read_sha1table_ref(void){constunsignedchar*sha1=read_sha1ref();-if(sha1<sha1_table||sha1>=sha1_table+nr_objects*20){-unsignedchar*found;-found=bsearch(sha1,sha1_table,nr_objects,20,-(int(*)(constvoid*,constvoid*))hashcmp);-if(!found)-bad_object(consumed_bytes,-_("SHA-1 %s not found in SHA-1 table"),-sha1_to_hex(sha1));-}+if(sha1<sha1_table||sha1>=sha1_table+nr_objects*20)+check_against_sha1table(sha1);returnsha1;}
@@ -641,9 +634,9 @@ static void *unpack_tree_v4(struct object_entry *obj,add_tree_delta_base(obj,last_base,delta_start);}elseif(!last_base)bad_object(offset,-_("bad copy count index in unpack_tree_v4"));+_("missing delta base unpack_tree_v4"));copy_count>>=1;-if(!copy_count)+if(!copy_count||copy_count>nr)bad_object(offset,_("bad copy count index in unpack_tree_v4"));nr-=copy_count;
@@ -829,11 +838,9 @@ static void *patch_one_base_tree(const struct object_entry *src,last_base=sha1_table+(id-1)*20;if(hashcmp(last_base,src->idx.sha1))die(_("bad tree base in patch_one_base_tree"));-}elseif(!last_base)-die(_("bad copy count index in patch_one_base_tree"));+}+copy_count>>=1;-if(!copy_count)-die(_("bad copy count index in patch_one_base_tree"));nr-=copy_count;init_tree_desc(&desc,src_buf,src->size);
@@ -1079,19 +1090,6 @@ static int check_collison(struct object_entry *entry)return0;}-staticvoidcheck_against_sha1table(structobject_entry*obj)-{-constunsignedchar*found;-if(!packv4)-return;--found=bsearch(obj->idx.sha1,sha1_table,nr_objects,20,-(int(*)(constvoid*,constvoid*))hashcmp);-if(!found)-die(_("object %s not found in SHA-1 table"),-sha1_to_hex(obj->idx.sha1));-}-staticvoidsha1_object(constvoid*data,structobject_entry*obj_entry,unsignedlongsize,enumobject_typetype,constunsignedchar*sha1)
@@ -1460,10 +1468,19 @@ static struct packv4_dict *read_dict(void)staticvoidparse_dictionaries(void){+inti;if(!packv4)return;-sha1_table=read_data(20*nr_objects);+sha1_table=xmalloc(20*nr_objects);+hashcpy(sha1_table,fill_and_use(20));+for(i=1;i<nr_objects;i++){+unsignedchar*p=sha1_table+i*20;+hashcpy(p,fill_and_use(20));+if(hashcmp(p-20,p)>=0)+die(_("wrong order in SHA-1 table at entry %d"),i);+}+name_dict=read_dict();path_dict=read_dict();}
@@ -1492,9 +1509,9 @@ static void parse_pack_objects(unsigned char *sha1)/* large blobs, check later */obj->real_type=OBJ_BAD;nr_delays++;-check_against_sha1table(obj);+check_against_sha1table(obj->idx.sha1);}else{-check_against_sha1table(obj);+check_against_sha1table(obj->idx.sha1);sha1_object(data,NULL,obj->size,obj->real_type,obj->idx.sha1);}
@@ -19,8 +19,8 @@ struct object_entry {structpack_idx_entryidx;unsignedlongsize;unsignedinthdr_size;-enumobject_typetype;-enumobject_typereal_type;+enumobject_typetype;/* type as written in pack */+enumobject_typereal_type;/* type after delta resolving */unsigneddelta_depth;intbase_object_no;};
@@ -187,8 +187,10 @@ static int mark_link(struct object *obj, int type, void *data)return0;}-/* The content of each linked object must have been checked-oritmustbealreadypresentintheobjectdatabase*/+/*+*Thecontentofeachlinkedobjectmusthavebeencheckedoritmust+*bealreadypresentintheobjectdatabase+*/staticunsignedcheck_object(structobject*obj){if(!obj)
@@ -407,6 +409,11 @@ static int is_delta_type(enum object_type type)return(type==OBJ_REF_DELTA||type==OBJ_OFS_DELTA);}+/*+*Unpackanentrydatainthestreamedpack,calculatetheobject+*SHA-1ifit'snotalargeblob.Otherwisejusttrytoinflatethe+*objectto/dev/nulltodeterminetheendoftheentryinthepack.+*/staticvoid*unpack_entry_data(unsignedlongoffset,unsignedlongsize,enumobject_typetype,unsignedchar*sha1){
@@ -462,55 +484,41 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size,returnbuf==fixed_buf?NULL:buf;}+staticvoidread_typesize_v2(structobject_entry*obj)+{+unsignedcharc=*(char*)fill_and_use(1);+unsignedshift;++obj->type=(c>>4)&7;+obj->size=(c&15);+shift=4;+while(c&128){+c=*(char*)fill_and_use(1);+obj->size+=(c&0x7f)<<shift;+shift+=7;+}+}+staticvoid*unpack_raw_entry(structobject_entry*obj,uniondelta_base*delta_base,unsignedchar*sha1){-unsignedchar*p;-unsignedlongsize,c;-off_tbase_offset;-unsignedshift;void*data;+uintmax_tval;obj->idx.offset=consumed_bytes;input_crc32=crc32(0,NULL,0);-p=fill(1);-c=*p;-use(1);-obj->type=(c>>4)&7;-size=(c&15);-shift=4;-while(c&0x80){-p=fill(1);-c=*p;-use(1);-size+=(c&0x7f)<<shift;-shift+=7;-}-obj->size=size;+read_typesize_v2(obj);switch(obj->type){caseOBJ_REF_DELTA:-hashcpy(delta_base->sha1,fill(20));-use(20);+hashcpy(delta_base->sha1,fill_and_use(20));break;caseOBJ_OFS_DELTA:memset(delta_base,0,sizeof(*delta_base));-p=fill(1);-c=*p;-use(1);-base_offset=c&127;-while(c&128){-base_offset+=1;-if(!base_offset||MSB(base_offset,7))-bad_object(obj->idx.offset,_("offset value overflow for delta base object"));-p=fill(1);-c=*p;-use(1);-base_offset=(base_offset<<7)+(c&127);-}-delta_base->offset=obj->idx.offset-base_offset;+val=read_varint();+delta_base->offset=obj->idx.offset-val;if(delta_base->offset<=0||delta_base->offset>=obj->idx.offset)bad_object(obj->idx.offset,_("delta base offset is out of bound"));break;
We do need deltas until the second pass. Allocating a buffer for it
then freeing later is wasteful is unnecessary. Make it use fixed_buf
(aka large blob code path).
---
builtin/index-pack.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
@@ -431,6 +431,40 @@ static int is_delta_type(enum object_type type)return(type==OBJ_REF_DELTA||type==OBJ_OFS_DELTA);}+staticvoidread_and_inflate(unsignedlongoffset,+void*buf,unsignedlongsize,+unsignedlongwraparound,+git_SHA_CTX*ctx,+unsignedchar*sha1)+{+git_zstreamstream;+intstatus;++memset(&stream,0,sizeof(stream));+git_inflate_init(&stream);+stream.next_out=buf;+stream.avail_out=wraparound?wraparound:size;++do{+unsignedchar*last_out=stream.next_out;+stream.next_in=fill(1);+stream.avail_in=input_len;+status=git_inflate(&stream,0);+use(input_len-stream.avail_in);+if(sha1)+git_SHA1_Update(ctx,last_out,stream.next_out-last_out);+if(wraparound){+stream.next_out=buf;+stream.avail_out=wraparound;+}+}while(status==Z_OK);+if(stream.total_out!=size||status!=Z_STREAM_END)+bad_object(offset,_("inflate returned %d"),status);+git_inflate_end(&stream);+if(sha1)+git_SHA1_Final(sha1,ctx);+}+/**Unpackanentrydatainthestreamedpack,calculatetheobject*SHA-1ifit'snotalargeblob.Otherwisejusttrytoinflatethe
@@ -440,8 +474,6 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size,enumobject_typetype,unsignedchar*sha1){staticcharfixed_buf[8192];-intstatus;-git_zstreamstream;void*buf;git_SHA_CTXc;charhdr[32];
@@ -459,29 +491,9 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size,elsebuf=xmalloc(size);-memset(&stream,0,sizeof(stream));-git_inflate_init(&stream);-stream.next_out=buf;-stream.avail_out=buf==fixed_buf?sizeof(fixed_buf):size;--do{-unsignedchar*last_out=stream.next_out;-stream.next_in=fill(1);-stream.avail_in=input_len;-status=git_inflate(&stream,0);-use(input_len-stream.avail_in);-if(sha1)-git_SHA1_Update(&c,last_out,stream.next_out-last_out);-if(buf==fixed_buf){-stream.next_out=buf;-stream.avail_out=sizeof(fixed_buf);-}-}while(status==Z_OK);-if(stream.total_out!=size||status!=Z_STREAM_END)-bad_object(offset,_("inflate returned %d"),status);-git_inflate_end(&stream);-if(sha1)-git_SHA1_Final(sha1,&c);+read_and_inflate(offset,buf,size,+buf==fixed_buf?sizeof(fixed_buf):0,+&c,sha1);returnbuf==fixed_buf?NULL:buf;}
@@ -1035,6 +1041,40 @@ static void *threaded_second_pass(void *data)}#endif+staticstructpackv4_dict*read_dict(void)+{+unsignedlongsize;+unsignedchar*data;+structpackv4_dict*dict;++size=read_varint();+data=xmallocz(size);+read_and_inflate(consumed_bytes,data,size,0,NULL,NULL);+dict=pv4_create_dict(data,size);+if(!dict)+die("unable to parse dictionary");+returndict;+}++staticvoidparse_dictionaries(void)+{+inti;+if(!packv4)+return;++sha1_table=xmalloc(20*nr_objects);+hashcpy(sha1_table,fill_and_use(20));+for(i=1;i<nr_objects;i++){+unsignedchar*p=sha1_table+i*20;+hashcpy(p,fill_and_use(20));+if(hashcmp(p-20,p)>=0)+die(_("wrong order in SHA-1 table at entry %d"),i);+}++name_dict=read_dict();+path_dict=read_dict();+}+/**Firstpass:*-findlocationsofallobjects;
@@ -288,6 +288,19 @@ static inline void *fill_and_use(int bytes)returnp;}+staticvoidcheck_against_sha1table(constunsignedchar*sha1)+{+constunsignedchar*found;+if(!packv4)+return;++found=bsearch(sha1,sha1_table,nr_objects,20,+(int(*)(constvoid*,constvoid*))hashcmp);+if(!found)+die(_("object %s not found in SHA-1 table"),+sha1_to_hex(sha1));+}+staticNORETURNvoidbad_object(unsignedlongoffset,constchar*format,...)__attribute__((format(printf,2,3)));
@@ -319,6 +319,30 @@ static uintmax_t read_varint(void)returnval;}+staticconstunsignedchar*read_sha1ref(void)+{+unsignedintindex=read_varint();+if(!index){+staticunsignedcharsha1[20];+hashcpy(sha1,fill_and_use(20));+returnsha1;+}+index--;+if(index>=nr_objects)+bad_object(consumed_bytes,+_("bad index in read_sha1ref"));+returnsha1_table+index*20;+}++staticconstunsignedchar*read_dictref(structpackv4_dict*dict)+{+unsignedintindex=read_varint();+if(index>=dict->nb_entries)+bad_object(consumed_bytes,+_("bad index in read_dictref"));+returndict->data+dict->offsets[index];+}+staticconstchar*open_pack_file(constchar*pack_name){if(from_stdin){
@@ -561,21 +569,105 @@ static void *unpack_commit_v4(unsigned int offset, unsigned long size,}/*+*v4treesareactuallykindofdeltasandwedon'tdodeltainthe+*firstpass.Thisfunctiononlywalksthroughatreeobjecttofind+*theendoffset,registerobjectdependenciesandperformslimited+*validation.Forv4treesthathavenodependencies,wedo+*uncompressandcalculatetheirSHA-1.+*/+staticvoid*unpack_tree_v4(structobject_entry*obj,+unsignedintoffset,unsignedlongsize,+unsignedchar*sha1)+{+unsignedintnr=read_varint();+constunsignedchar*last_base=NULL;+structstrbufsb=STRBUF_INIT;+while(nr){+unsignedintcopy_start_or_path=read_varint();+if(copy_start_or_path&1){/* copy_start */+unsignedintcopy_count=read_varint();+if(copy_count&1){/* first delta */+last_base=read_sha1table_ref();+}elseif(!last_base)+bad_object(offset,+_("missing delta base unpack_tree_v4"));+copy_count>>=1;+if(!copy_count||copy_count>nr)+bad_object(offset,+_("bad copy count index in unpack_tree_v4"));+nr-=copy_count;+}else{/* path */+unsignedintpath_idx=copy_start_or_path>>1;+constunsignedchar*entry_sha1;++if(path_idx>=path_dict->nb_entries)+bad_object(offset,+_("bad path index in unpack_tree_v4"));+entry_sha1=read_sha1ref();+nr--;++/*+*Attempttorebuildacanonical(base)tree.+*Iflast_baseisset,thistreedependson+*anothertree,whichwehavenoaccessatthis+*stage,soreconstructionmustbedelayeduntil+*thesecondpass.+*/+if(!last_base){+constunsignedchar*path;+unsignedmode;++path=path_dict->data+path_dict->offsets[path_idx];+mode=(path[0]<<8)|path[1];+strbuf_addf(&sb,"%o %s%c",mode,path+2,'\0');+strbuf_add(&sb,entry_sha1,20);+if(sb.len>size)+bad_object(offset,+_("tree larger than expected"));+}+}+}++if(last_base){+strbuf_release(&sb);+returnNULL;+}else{+git_SHA_CTXctx;+charhdr[32];+inthdrlen;++if(sb.len!=size)+bad_object(offset,_("tree size mismatch"));++hdrlen=sprintf(hdr,"tree %lu",size)+1;+git_SHA1_Init(&ctx);+git_SHA1_Update(&ctx,hdr,hdrlen);+git_SHA1_Update(&ctx,sb.buf,size);+git_SHA1_Final(sha1,&ctx);+returnstrbuf_detach(&sb,NULL);+}+}++/**Unpackanentrydatainthestreamedpack,calculatetheobject*SHA-1ifit'snotalargeblob.Otherwisejusttrytoinflatethe*objectto/dev/nulltodeterminetheendoftheentryinthepack.*/-staticvoid*unpack_entry_data(unsignedlongoffset,unsignedlongsize,-enumobject_typetype,unsignedchar*sha1)+staticvoid*unpack_entry_data(structobject_entry*obj,unsignedchar*sha1){staticcharfixed_buf[8192];void*buf;git_SHA_CTXc;charhdr[32];inthdrlen;+unsignedlongoffset=obj->idx.offset;+unsignedlongsize=obj->size;+enumobject_typetype=obj->type;if(type==OBJ_PV4_COMMIT)returnunpack_commit_v4(offset,size,sha1);+if(type==OBJ_PV4_TREE)+returnunpack_tree_v4(obj,offset,size,sha1);if(!is_delta_type(type)){hdrlen=sprintf(hdr,"%s %lu",typename(type),size)+1;
@@ -1201,6 +1296,8 @@ static void parse_pack_objects(unsigned char *sha1)nr_deltas++;delta->obj_no=i;delta++;+}elseif(!data&&obj->type==OBJ_PV4_TREE){+/* delay sha1_object() until second pass */}elseif(!data){/* large blobs, check later */obj->real_type=OBJ_BAD;
For v2, ofs-delta and ref-delta can only have queue one delta base at
a time. A v4 tree can have more than one delta base. Move the queuing
code up to unpack_raw_entry() and give unpack_tree_v4() more
flexibility to add its bases.
---
builtin/index-pack.c | 46 ++++++++++++++++++++++++++++++----------------
1 file changed, 30 insertions(+), 16 deletions(-)
@@ -722,14 +740,14 @@ static void *unpack_raw_entry(struct object_entry *obj,switch(obj->type){caseOBJ_REF_DELTA:-hashcpy(delta_base->sha1,fill_and_use(20));+add_sha1_delta(obj,fill_and_use(20));break;caseOBJ_OFS_DELTA:-memset(delta_base,0,sizeof(*delta_base));-val=read_varint();-delta_base->offset=obj->idx.offset-val;-if(delta_base->offset<=0||delta_base->offset>=obj->idx.offset)-bad_object(obj->idx.offset,_("delta base offset is out of bound"));+offset=obj->idx.offset-read_varint();+if(offset<=0||offset>=obj->idx.offset)+bad_object(obj->idx.offset,+_("delta base offset is out of bound"));+add_ofs_delta(obj,offset);break;caseOBJ_COMMIT:caseOBJ_TREE:
@@ -1291,12 +1308,9 @@ static void parse_pack_objects(unsigned char *sha1)nr_objects);for(i=0;i<nr_objects;i++){structobject_entry*obj=&objects[i];-void*data=unpack_raw_entry(obj,&delta->base,obj->idx.sha1);-if(is_delta_type(obj->type)){-nr_deltas++;-delta->obj_no=i;-delta++;-}elseif(!data&&obj->type==OBJ_PV4_TREE){+void*data=unpack_raw_entry(obj,obj->idx.sha1);+if(is_delta_type(obj->type)||+(!data&&obj->type==OBJ_PV4_TREE)){/* delay sha1_object() until second pass */}elseif(!data){/* large blobs, check later */
@@ -24,6 +24,7 @@ struct object_entry {enumobject_typereal_type;/* type after delta resolving */unsigneddelta_depth;intbase_object_no;+intnr_bases;/* only valid for v4 trees */};uniondelta_base{
@@ -482,6 +483,11 @@ static int is_delta_type(enum object_type type)return(type==OBJ_REF_DELTA||type==OBJ_OFS_DELTA);}+staticintis_delta_tree(conststructobject_entry*obj)+{+returnobj->type==OBJ_PV4_TREE&&obj->nr_bases>0;+}+staticvoidread_and_inflate(unsignedlongoffset,void*buf,unsignedlongsize,unsignedlongwraparound,
@@ -601,12 +621,14 @@ static void *unpack_tree_v4(struct object_entry *obj,unsignedintnr=read_varint();constunsignedchar*last_base=NULL;structstrbufsb=STRBUF_INIT;+intdelta_start=nr_deltas;while(nr){unsignedintcopy_start_or_path=read_varint();if(copy_start_or_path&1){/* copy_start */unsignedintcopy_count=read_varint();if(copy_count&1){/* first delta */last_base=read_sha1table_ref();+add_tree_delta_base(obj,last_base,delta_start);}elseif(!last_base)bad_object(offset,_("missing delta base unpack_tree_v4"));
@@ -740,9 +762,15 @@ static void *unpack_raw_entry(struct object_entry *obj,switch(obj->type){caseOBJ_REF_DELTA:-add_sha1_delta(obj,fill_and_use(20));+if(packv4)+add_sha1_delta(obj,read_sha1table_ref());+else+add_sha1_delta(obj,fill_and_use(20));break;caseOBJ_OFS_DELTA:+if(packv4)+die(_("pack version 4 does not support ofs-delta type (offset %lu)"),+obj->idx.offset);offset=obj->idx.offset-read_varint();if(offset<=0||offset>=obj->idx.offset)bad_object(obj->idx.offset,
@@ -1309,8 +1337,7 @@ static void parse_pack_objects(unsigned char *sha1)for(i=0;i<nr_objects;i++){structobject_entry*obj=&objects[i];void*data=unpack_raw_entry(obj,obj->idx.sha1);-if(is_delta_type(obj->type)||-(!data&&obj->type==OBJ_PV4_TREE)){+if(is_delta_type(obj->type)||is_delta_tree(obj)){/* delay sha1_object() until second pass */}elseif(!data){/* large blobs, check later */