From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-12 11:14:24
From: ZheNing Hu <redacted>
Use non-const ref_format in *_atom_parser(), which can help us
modify the members of ref_format in *_atom_parser().
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/tag.c | 2 +-
ref-filter.c | 44 ++++++++++++++++++++++----------------------
ref-filter.h | 4 ++--
3 files changed, 25 insertions(+), 25 deletions(-)
@@ -116,7 +116,7 @@ void ref_array_sort(struct ref_sorting *sort, struct ref_array *array);voidref_sorting_set_sort_flags_all(structref_sorting*sorting,unsignedintmask,inton);/* Based on the given format and quote_style, fill the strbuf */intformat_ref_array_item(structref_array_item*info,-conststructref_format*format,+structref_format*format,structstrbuf*final_buf,structstrbuf*error_buf);/* Parse a single sort specifier and add it to the list */
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-12 11:14:27
From: ZheNing Hu <redacted>
In order to let cat-file use ref-filter logic, the following
methods are used:
1. Add `cat_file_mode` member in struct `ref_format`, this can
help us reject atoms in verify_ref_format() which cat-file
cannot use, e.g. `%(refname)`, `%(push)`, `%(upstream)`...
2. Change the type of member `format` in struct `batch_options`
to `ref_format`, We can add format data in it.
3. Let `batch_objects()` add atoms to format, and use
`verify_ref_format()` to check atoms.
4. Use `has_object_file()` in `batch_one_object()` to check
whether the input object exists.
5. Use `format_ref_array_item()` in `batch_object_write()` to
get the formatted data corresponding to the object. If the
return value of `format_ref_array_item()` is equals to zero,
use `batch_write()` to print object data; else if the return
value less than zero, use `die()` to print the error message
and exit; else return value greater than zero, only print the
error message, but not exit.
6. Let get_object() return 1 and print "<oid> missing" instead
of returning -1 and printing "missing object <oid> for <refname>",
this can help `format_ref_array_item()` just report that the
object is missing without letting Git exit.
Most of the atoms in `for-each-ref --format` are now supported,
such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,
`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:
`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,
`%(flag)`, `%(HEAD)`, because our objects don't have refname.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-cat-file.txt | 6 +
builtin/cat-file.c | 250 +++++++-------------------------
ref-filter.c | 15 +-
ref-filter.h | 3 +-
t/t1006-cat-file.sh | 252 +++++++++++++++++++++++++++++++++
t/t6301-for-each-ref-errors.sh | 2 +-
6 files changed, 324 insertions(+), 204 deletions(-)
@@ -226,6 +226,12 @@ newline. The available atoms are: after that first run of whitespace (i.e., the "rest" of the line) are output in place of the `%(rest)` atom.+Note that most of the atoms in `for-each-ref --format` are now supported,+such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,+`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:+`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,+`%(flag)`, `%(HEAD)`. See linkgit:git-for-each-ref[1].+ If no format is specified, the default format is `%(objectname) %(objecttype) %(objectsize)`.
@@ -25,7 +26,7 @@ struct batch_options {intall_objects;intunordered;intcmdmode;/* may be 'w' or 'c' for --filters or --textconv */-constchar*format;+structref_formatformat;};staticconstchar*force_path;
@@ -195,99 +196,10 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name,structexpand_data{structobject_idoid;-enumobject_typetype;-unsignedlongsize;-off_tdisk_size;constchar*rest;-structobject_iddelta_base_oid;--/*-*Ifmark_queryistrue,wedonotexpandanything,butrather-*justmarktheobject_infowithitemswewishtoquery.-*/-intmark_query;--/*-*Whethertosplittheinputonwhitespacebeforefeedingitto-*get_sha1;thisisdecidedduringthemark_queryphasebasedon-*whetherwehavea%(rest)tokeninourformat.-*/intsplit_on_whitespace;--/*-*Afteramark_queryrun,thisobject_infoissetuptobe-*passedtooid_object_info_extended.Itwillpointtothedata-*elementsabove,soyoucanretrievetheresponsefromthere.-*/-structobject_infoinfo;--/*-*Thisflagwillbetrueiftherequestedbatchformatandoptions-*don'trequireustocalloid_object_info,whichcanthenbe-*optimizedout.-*/-unsignedskip_object_info:1;};-staticintis_atom(constchar*atom,constchar*s,intslen)-{-intalen=strlen(atom);-returnalen==slen&&!memcmp(atom,s,alen);-}--staticvoidexpand_atom(structstrbuf*sb,constchar*atom,intlen,-void*vdata)-{-structexpand_data*data=vdata;--if(is_atom("objectname",atom,len)){-if(!data->mark_query)-strbuf_addstr(sb,oid_to_hex(&data->oid));-}elseif(is_atom("objecttype",atom,len)){-if(data->mark_query)-data->info.typep=&data->type;-else-strbuf_addstr(sb,type_name(data->type));-}elseif(is_atom("objectsize",atom,len)){-if(data->mark_query)-data->info.sizep=&data->size;-else-strbuf_addf(sb,"%"PRIuMAX,(uintmax_t)data->size);-}elseif(is_atom("objectsize:disk",atom,len)){-if(data->mark_query)-data->info.disk_sizep=&data->disk_size;-else-strbuf_addf(sb,"%"PRIuMAX,(uintmax_t)data->disk_size);-}elseif(is_atom("rest",atom,len)){-if(data->mark_query)-data->split_on_whitespace=1;-elseif(data->rest)-strbuf_addstr(sb,data->rest);-}elseif(is_atom("deltabase",atom,len)){-if(data->mark_query)-data->info.delta_base_oid=&data->delta_base_oid;-else-strbuf_addstr(sb,-oid_to_hex(&data->delta_base_oid));-}else-die("unknown format element: %.*s",len,atom);-}--staticsize_texpand_format(structstrbuf*sb,constchar*start,void*data)-{-constchar*end;--if(*start!='(')-return0;-end=strchr(start+1,')');-if(!end)-die("format element '%s' does not end in ')'",start);--expand_atom(sb,start+1,end-start-1,data);--returnend-start+1;-}-staticvoidbatch_write(structbatch_options*opt,constvoid*data,intlen){if(opt->buffer_output){
@@ -297,86 +209,31 @@ static void batch_write(struct batch_options *opt, const void *data, int len)write_or_die(1,data,len);}-staticvoidprint_object_or_die(structbatch_options*opt,structexpand_data*data)-{-conststructobject_id*oid=&data->oid;--assert(data->info.typep);--if(data->type==OBJ_BLOB){-if(opt->buffer_output)-fflush(stdout);-if(opt->cmdmode){-char*contents;-unsignedlongsize;--if(!data->rest)-die("missing path for '%s'",oid_to_hex(oid));--if(opt->cmdmode=='w'){-if(filter_object(data->rest,0100644,oid,-&contents,&size))-die("could not convert '%s' %s",-oid_to_hex(oid),data->rest);-}elseif(opt->cmdmode=='c'){-enumobject_typetype;-if(!textconv_object(the_repository,-data->rest,0100644,oid,-1,&contents,&size))-contents=read_object_file(oid,-&type,-&size);-if(!contents)-die("could not convert '%s' %s",-oid_to_hex(oid),data->rest);-}else-BUG("invalid cmdmode: %c",opt->cmdmode);-batch_write(opt,contents,size);-free(contents);-}else{-stream_blob(oid);-}-}-else{-enumobject_typetype;-unsignedlongsize;-void*contents;--contents=read_object_file(oid,&type,&size);-if(!contents)-die("object %s disappeared",oid_to_hex(oid));-if(type!=data->type)-die("object %s changed type!?",oid_to_hex(oid));-if(data->info.sizep&&size!=data->size)-die("object %s changed size!?",oid_to_hex(oid));--batch_write(opt,contents,size);-free(contents);-}-}staticvoidbatch_object_write(constchar*obj_name,structstrbuf*scratch,structbatch_options*opt,structexpand_data*data){-if(!data->skip_object_info&&-oid_object_info_extended(the_repository,&data->oid,&data->info,-OBJECT_INFO_LOOKUP_REPLACE)<0){-printf("%s missing\n",-obj_name?obj_name:oid_to_hex(&data->oid));-fflush(stdout);-return;-}+intret=0;+structstrbuferr=STRBUF_INIT;+structref_array_itemitem={data->oid,data->rest};strbuf_reset(scratch);-strbuf_expand(scratch,opt->format,expand_format,data);-strbuf_addch(scratch,'\n');-batch_write(opt,scratch->buf,scratch->len);-if(opt->print_contents){-print_object_or_die(opt,data);-batch_write(opt,"\n",1);+ret=format_ref_array_item(&item,&opt->format,scratch,&err);+if(!ret){+strbuf_addch(scratch,'\n');+batch_write(opt,scratch->buf,scratch->len);+strbuf_release(&err);+}elseif(ret<0){+die("%s\n",err.buf);+strbuf_release(&err);+}else{+/* when ret > 0 , don't call die and print the err to stdout*/+printf("%s\n",err.buf);+fflush(stdout);+strbuf_release(&err);}}
@@ -713,7 +567,7 @@ int cmd_cat_file(int argc, const char **argv, const char *prefix)batch.buffer_output=batch.all_objects;if(batch.enabled)-returnbatch_objects(&batch);+returnbatch_objects(&batch,options);if(unknown_type&&opt!='t'&&opt!='s')die("git cat-file --allow-unknown-type: use with -s or -t");
@@ -1017,8 +1017,15 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);-if(used_atom[at].atom_type==ATOM_REST)-die("this command reject atom %%(%.*s)",(int)(ep-sp-2),sp+2);+if((!format->cat_file_mode&&used_atom[at].atom_type==ATOM_REST)||+(format->cat_file_mode&&(used_atom[at].atom_type==ATOM_FLAG||+used_atom[at].atom_type==ATOM_HEAD||+used_atom[at].atom_type==ATOM_PUSH||+used_atom[at].atom_type==ATOM_REFNAME||+used_atom[at].atom_type==ATOM_SYMREF||+used_atom[at].atom_type==ATOM_UPSTREAM||+used_atom[at].atom_type==ATOM_WORKTREEPATH)))+die(_("this command reject atom %%(%.*s)"),(int)(ep-sp-2),sp+2);if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&used_atom[at].u.raw_data.option==RAW_BARE)
@@ -1735,8 +1742,8 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj}if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,OBJECT_INFO_LOOKUP_REPLACE))-returnstrbuf_addf_ret(err,-1,_("missing object %s for %s"),-oid_to_hex(&oi->oid),ref->refname);+returnstrbuf_addf_ret(err,1,_("%s missing"),+oid_to_hex(&oi->oid));if(oi->info.disk_sizep&&oi->disk_size<0)BUG("Object size is less than zero.");
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-12 11:14:28
From: ZheNing Hu <redacted>
After cat-file reuses the ref-filter logic, we re-implement the
functions of --textconv and --filters options.
Add members `use_textconv` and `use_filters` in struct `ref_format`,
and use global variables `use_filters` and `use_textconv` in
`ref-filter.c`, so that we can filter the content of the object
in get_object(). Use `actual_oi` to record the real expand_data:
it may point to the original `oi` or the `act_oi` processed by
`textconv_object()` or `convert_to_working_tree()`. `grab_values()`
will grab the contents of `actual_oi` and `grab_common_values()`
to grab the contents of origin `oi`, this ensures that `%(objectsize)`
still uses the size of the unfiltered data.
In `get_object()`, we made an optimization: Firstly, get the size and
type of the object instead of directly getting the object data.
If using --textconv, after successfully obtaining the filtered object
data, an extra oid_object_info_extended() will be skipped, which can
reduce the cost of object data copy; If using --filter, the data of
the object first will be getted first, and then convert_to_working_tree()
will be used to get the filtered object data.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 5 ++++
ref-filter.c | 66 ++++++++++++++++++++++++++++++++++++++++++++--
ref-filter.h | 5 ++--
3 files changed, 72 insertions(+), 4 deletions(-)
@@ -1027,6 +1031,9 @@ int verify_ref_format(struct ref_format *format)used_atom[at].atom_type==ATOM_WORKTREEPATH)))die(_("this command reject atom %%(%.*s)"),(int)(ep-sp-2),sp+2);+use_filters=format->use_filters;+use_textconv=format->use_textconv;+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&used_atom[at].u.raw_data.option==RAW_BARE)die(_("--format=%.*s cannot be used with"
@@ -1735,10 +1742,41 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj{/* parse_object_buffer() will set eaten to 0 if free() will be needed */inteaten=1;+structexpand_data*actual_oi=oi;+structexpand_dataact_oi={0};+if(oi->info.contentp){/* We need to know that to use parse_object_buffer properly */+void**temp_contentp=oi->info.contentp;+oi->info.contentp=NULL;oi->info.sizep=&oi->size;oi->info.typep=&oi->type;++/* get the type and size */+if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,+OBJECT_INFO_LOOKUP_REPLACE))+returnstrbuf_addf_ret(err,1,_("%s missing"),+oid_to_hex(&oi->oid));++oi->info.sizep=NULL;+oi->info.typep=NULL;+oi->info.contentp=temp_contentp;++if(use_textconv){+act_oi=*oi;++if(!ref->rest)+returnstrbuf_addf_ret(err,-1,_("missing path for '%s'"),+oid_to_hex(&act_oi.oid));+if(act_oi.type==OBJ_BLOB){+if(textconv_object(the_repository,+ref->rest,0100644,&act_oi.oid,+1,(char**)(&act_oi.content),&act_oi.size)){+actual_oi=&act_oi;+gotosuccess;+}+}+}}if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,OBJECT_INFO_LOOKUP_REPLACE))
@@ -1748,19 +1786,43 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objBUG("Object size is less than zero.");if(oi->info.contentp){-*obj=parse_object_buffer(the_repository,&oi->oid,oi->type,oi->size,oi->content,&eaten);+if(use_filters){+if(!ref->rest)+returnstrbuf_addf_ret(err,-1,_("missing path for '%s'"),+oid_to_hex(&oi->oid));+if(oi->type==OBJ_BLOB){+structstrbufstrbuf=STRBUF_INIT;+structcheckout_metadatameta;+act_oi=*oi;++init_checkout_metadata(&meta,NULL,NULL,&act_oi.oid);+if(convert_to_working_tree(&the_index,ref->rest,act_oi.content,act_oi.size,&strbuf,&meta)){+act_oi.size=strbuf.len;+act_oi.content=strbuf_detach(&strbuf,NULL);+actual_oi=&act_oi;+}else{+die("could not convert '%s' %s",+oid_to_hex(&oi->oid),ref->rest);+}+}+}++success:+*obj=parse_object_buffer(the_repository,&actual_oi->oid,actual_oi->type,actual_oi->size,actual_oi->content,&eaten);if(!*obj){if(!eaten)free(oi->content);returnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi);+grab_values(ref->value,deref,*obj,actual_oi);}grab_common_values(ref->value,deref,oi);if(!eaten)free(oi->content);+if(actual_oi!=oi)+free(actual_oi->content);return0;}
@@ -559,6 +560,15 @@ static int if_atom_parser(struct ref_format *format, struct used_atom *atom,return0;}+staticintrest_atom_parser(structref_format*format,structused_atom*atom,+constchar*arg,structstrbuf*err)+{+if(arg)+returnstrbuf_addf_ret(err,-1,_("%%(rest) does not take arguments"));+format->use_rest=1;+return0;+}+staticinthead_atom_parser(structref_format*format,structused_atom*atom,constchar*arg,structstrbuf*unused_err){
@@ -1006,6 +1017,9 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);+if(used_atom[at].atom_type==ATOM_REST)+die("this command reject atom %%(%.*s)",(int)(ep-sp-2),sp+2);+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&used_atom[at].u.raw_data.option==RAW_BARE)die(_("--format=%.*s cannot be used with"
@@ -1998,6 +1998,10 @@ test_expect_success '--format should list tags as per format given' 'test_cmpexpectactual'+test_expect_success'git tag -l with --format="%(rest)" must failed''+test_must_failgittag-l--format="%(rest)""v1*"+'+ test_expect_success"set up color tests"'echo"<RED>v1.0<RESET>">expect.color&&echo"v1.0">expect.bare&&
@@ -194,6 +194,10 @@ test_expect_success GPG 'verifying tag with --format' 'test_cmpexpectactual'+test_expect_successGPG'verifying tag with --format="%(rest)" must failed''+test_must_failgitverify-tag--format="%(rest)""fourth-signed"+'+ test_expect_successGPG'verifying a forged tag with --format should fail silently''test_must_failgitverify-tag--format="tagname : %(tag)"$(catforged1.tag)>actual-forged&&test_must_be_emptyactual-forged
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-12 11:14:40
From: ZheNing Hu <redacted>
Add new formatting option `%(raw)`, which will print the raw
object data without any changes. It will help further to migrate
all cat-file formatting logic from cat-file to ref-filter.
The raw data of blob, tree objects may contain '\0', but most of
the logic in `ref-filter` depends on the output of the atom being
text (specifically, no embedded NULs in it).
E.g. `quote_formatting()` use `strbuf_addstr()` or `*._quote_buf()`
add the data to the buffer. The raw data of a tree object is
`100644 one\0...`, only the `100644 one` will be added to the buffer,
which is incorrect.
Therefore, add a new member in `struct atom_value`: `s_size`, which
can record raw object size, it can help us add raw object data to
the buffer or compare two buffers which contain raw object data.
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a variable
in the host language, the host language may not support arbitrary binary
data in the variables of its string type.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Felipe Contreras [off-list ref]
Helped-by: Phillip Wood [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Based-on-patch-by: Olga Telezhnaya [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-for-each-ref.txt | 9 ++
ref-filter.c | 139 +++++++++++++++----
t/t6300-for-each-ref.sh | 207 +++++++++++++++++++++++++++++
3 files changed, 328 insertions(+), 27 deletions(-)
@@ -235,6 +235,15 @@ and `date` to extract the named component. For email fields (`authoremail`, without angle brackets, and `:localpart` to get the part before the `@` symbol out of the trimmed email.+The raw data in an object is `raw`.++raw:size::+ The raw data size of the object.++Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,+`--perl` because the host language may not support arbitrary binary data in the+variables of its string type.+ The message in a commit or a tag object is `contents`, from which `contents:<part>` can be used to extract various parts out of:
@@ -620,12 +637,15 @@ struct ref_formatting_state {structatom_value{constchar*s;+size_ts_size;int(*handler)(structatom_value*atomv,structref_formatting_state*state,structstrbuf*err);uintmax_tvalue;/* used for sorting when not FIELD_STR */structused_atom*atom;};+#define ATOM_VALUE_S_SIZE_INIT (-1)+/**Usedtoparseformatstringandsortspecifiers*/
@@ -644,13 +664,6 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnstrbuf_addf_ret(err,-1,_("malformed field name: %.*s"),(int)(ep-atom),atom);-/* Do we have the atom already used elsewhere? */-for(i=0;i<used_atom_cnt;i++){-intlen=strlen(used_atom[i].name);-if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))-returni;-}-/**Iftheatomnamehasacolon,stripitandeverythingafter*itoff-itspecifiestheformatforthisentry,and
@@ -660,6 +673,13 @@ static int parse_ref_filter_atom(const struct ref_format *format,arg=memchr(sp,':',ep-sp);atom_len=(arg?arg:ep)-sp;+/* Do we have the atom already used elsewhere? */+for(i=0;i<used_atom_cnt;i++){+intlen=strlen(used_atom[i].name);+if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))+returni;+}+/* Is the atom a valid one? */for(i=0;i<ARRAY_SIZE(valid_atom);i++){intlen=strlen(valid_atom[i].name);
@@ -709,11 +729,14 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnat;}-staticvoidquote_formatting(structstrbuf*s,constchar*str,intquote_style)+staticvoidquote_formatting(structstrbuf*s,constchar*str,size_tlen,intquote_style){switch(quote_style){caseQUOTE_NONE:-strbuf_addstr(s,str);+if(len!=ATOM_VALUE_S_SIZE_INIT)+strbuf_add(s,str,len);+else+strbuf_addstr(s,str);break;caseQUOTE_SHELL:sq_quote_buf(s,str);
@@ -867,18 +895,22 @@ static int then_atom_handler(struct atom_value *atomv, struct ref_formatting_staif(if_then_else->else_atom_seen)returnstrbuf_addf_ret(err,-1,_("format: %%(then) atom used after %%(else)"));if_then_else->then_atom_seen=1;+if(if_then_else->str)+str_len=strlen(if_then_else->str);/**Ifthe'equals'or'notequals'attributeisusedthen*performtherequiredcomparison.Ifnot,onlynon-empty*stringssatisfythe'if'condition.*/if(if_then_else->cmp_status==COMPARE_EQUAL){-if(!strcmp(if_then_else->str,cur->output.buf))+if(str_len==cur->output.len&&+!memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;}elseif(if_then_else->cmp_status==COMPARE_UNEQUAL){-if(strcmp(if_then_else->str,cur->output.buf))+if(str_len!=cur->output.len||+memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;-}elseif(cur->output.len&&!is_empty(cur->output.buf))+}elseif(cur->output.len&&!is_empty(&cur->output))if_then_else->condition_satisfied=1;strbuf_reset(&cur->output);return0;
@@ -974,6 +1006,10 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&+used_atom[at].u.raw_data.option==RAW_BARE)+die(_("--format=%.*s cannot be used with"+"--python, --shell, --tcl, --perl"),(int)(ep-sp-2),sp+2);cp=ep+1;if(skip_prefix(used_atom[at].name,"color:",&color))
@@ -708,6 +737,15 @@ test_atom refs/tags/signed-long contents "subject line bodycontents$sig"+test_expect_successGPG'basic atom: refs/tags/signed-long raw''+gitcat-filetagrefs/tags/signed-long>expected&&+gitfor-each-ref--format="%(raw)"refs/tags/signed-long>actual&&+sanitize_pgp<expected>expected.clean&&+sanitize_pgp<actual>actual.clean&&+echo"">>expected.clean&&+test_cmpexpected.cleanactual.clean+'+ test_expect_success'set up refs pointing to tree and blob''gitupdate-refrefs/mytrees/firstrefs/heads/main^{tree}&&gitupdate-refrefs/myblobs/firstrefs/heads/main:one
@@ -727,6 +775,165 @@ test_atom refs/myblobs/first contents:body "" test_atomrefs/myblobs/firstcontents:signature"" test_atomrefs/myblobs/firstcontents""+test_expect_success'basic atom: refs/myblobs/first raw''+gitcat-fileblobrefs/myblobs/first>expected&&+echo"">>expected&&+gitfor-each-ref--format="%(raw)"refs/myblobs/first>actual&&+test_cmpexpectedactual&&+gitcat-file-srefs/myblobs/first>expected&&+gitfor-each-ref--format="%(raw:size)"refs/myblobs/first>actual&&+test_cmpexpectedactual+'++test_expect_success'set up refs pointing to binary blob''+printf"a\0b\0c">blob1&&+printf"a\0c\0b">blob2&&+printf"\0a\0b\0c">blob3&&+printf"abc">blob4&&+printf"\0 \0 \0 ">blob5&&+printf"\0 \0a\0 ">blob6&&+printf" ">blob7&&+>blob8&&+githash-objectblob1-w|xargsgitupdate-refrefs/myblobs/blob1&&+githash-objectblob2-w|xargsgitupdate-refrefs/myblobs/blob2&&+githash-objectblob3-w|xargsgitupdate-refrefs/myblobs/blob3&&+githash-objectblob4-w|xargsgitupdate-refrefs/myblobs/blob4&&+githash-objectblob5-w|xargsgitupdate-refrefs/myblobs/blob5&&+githash-objectblob6-w|xargsgitupdate-refrefs/myblobs/blob6&&+githash-objectblob7-w|xargsgitupdate-refrefs/myblobs/blob7&&+githash-objectblob8-w|xargsgitupdate-refrefs/myblobs/blob8+'++test_expect_success'Verify sorts with raw''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob3+refs/myblobs/blob7+refs/mytrees/first+refs/myblobs/first+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob4+refs/heads/main+EOF+gitfor-each-ref--format="%(refname)"--sort=raw\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'Verify sorts with raw:size''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/first+refs/myblobs/blob7+refs/heads/main+refs/myblobs/blob4+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+refs/myblobs/blob5+refs/myblobs/blob6+refs/mytrees/first+EOF+gitfor-each-ref--format="%(refname)"--sort=raw:size\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'validate raw atom with %(if:equals)''+cat>expected<<-EOF&&+notequals+notequals+notequals+notequals+notequals+notequals+refs/myblobs/blob4+notequals+notequals+notequals+notequals+notequals+EOF+gitfor-each-ref--format="%(if:equals=abc)%(raw)%(then)%(refname)%(else)not equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'+test_expect_success'validate raw atom with %(if:notequals)''+cat>expected<<-EOF&&+refs/heads/ambiguous+refs/heads/main+refs/heads/newtag+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+equals+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob7+refs/myblobs/blob8+refs/myblobs/first+EOF+gitfor-each-ref--format="%(if:notequals=abc)%(raw)%(then)%(refname)%(else)equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'++test_expect_success'empty raw refs with %(if)''+cat>expected<<-EOF&&+refs/myblobs/blob1notempty+refs/myblobs/blob2notempty+refs/myblobs/blob3notempty+refs/myblobs/blob4notempty+refs/myblobs/blob5notempty+refs/myblobs/blob6notempty+refs/myblobs/blob7empty+refs/myblobs/blob8empty+refs/myblobs/firstnotempty+EOF+gitfor-each-ref--format="%(refname) %(if)%(raw)%(then)not empty%(else)empty%(end)"\+refs/myblobs/>actual&&+test_cmpexpectedactual+'++test_expect_success'%(raw) with --python must failed''+test_must_failgitfor-each-ref--format="%(raw)"--python+'++test_expect_success'%(raw) with --tcl must failed''+test_must_failgitfor-each-ref--format="%(raw)"--tcl+'++test_expect_success'%(raw) with --perl must failed''+test_must_failgitfor-each-ref--format="%(raw)"--perl+'++test_expect_success'%(raw) with --shell must failed''+test_must_failgitfor-each-ref--format="%(raw)"--shell+'++test_expect_success'%(raw) with --shell and --sort=raw must failed''+test_must_failgitfor-each-ref--format="%(raw)"--sort=raw--shell+'++test_expect_success'%(raw:size) with --shell''+gitfor-each-ref--format="%(raw:size)"|whilereadline+do+echo"'\''$line'\''">>expect+done&&+gitfor-each-ref--format="%(raw:size)"--shell>actual&&+test_cmpexpectactual+'++test_expect_success'for-each-ref --format compare with cat-file --batch''+gitrev-parserefs/mytrees/first|gitcat-file--batch>expected&&+gitfor-each-ref--format="%(objectname) %(objecttype) %(objectsize)+%(raw)" refs/mytrees/first >actual &&+test_cmpexpectedactual+'+ test_expect_success'set up multiple-sort tags''forwhenin100000200000do
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-12 11:15:23
From: ZheNing Hu <redacted>
Only tag and commit objects use `grab_sub_body_contents()` to grab
object contents in the current codebase. We want to teach the
function to also handle blobs and trees to get their raw data,
without parsing a blob (whose contents looks like a commit or a tag)
incorrectly as a commit or a tag.
Skip the block of code that is specific to handling commits and tags
early when the given object is of a wrong type to help later
addition to handle other types of objects in this function.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
@@ -1678,7 +1684,7 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objreturnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi->content);+grab_values(ref->value,deref,*obj,oi);}grab_common_values(ref->value,deref,oi);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-12 11:15:25
From: ZheNing Hu <redacted>
Let `populate_value()`, `get_ref_atom_value()` and
`format_ref_array_item()` get the return value of `get_value()`
correctly. This can help us later let `cat-file --batch` get the
correct error message and return value of `get_value()`.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-12 11:15:41
From: ZheNing Hu <redacted>
Reuse the `err` buffer in batch_object_write(), as the
buffer `scratch` does. This will reduce the overhead
of multiple allocations of memory of the err buffer.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 24 ++++++++++++++----------
1 file changed, 14 insertions(+), 10 deletions(-)
@@ -212,33 +212,32 @@ static void batch_write(struct batch_options *opt, const void *data, int len)staticvoidbatch_object_write(constchar*obj_name,structstrbuf*scratch,+structstrbuf*err,structbatch_options*opt,structexpand_data*data){intret=0;-structstrbuferr=STRBUF_INIT;structref_array_itemitem={data->oid,data->rest};strbuf_reset(scratch);+strbuf_reset(err);-ret=format_ref_array_item(&item,&opt->format,scratch,&err);+ret=format_ref_array_item(&item,&opt->format,scratch,err);if(!ret){strbuf_addch(scratch,'\n');batch_write(opt,scratch->buf,scratch->len);-strbuf_release(&err);}elseif(ret<0){-die("%s\n",err.buf);-strbuf_release(&err);+die("%s\n",err->buf);}else{/* when ret > 0 , don't call die and print the err to stdout*/-printf("%s\n",err.buf);+printf("%s\n",err->buf);fflush(stdout);-strbuf_release(&err);}}staticvoidbatch_one_object(constchar*obj_name,structstrbuf*scratch,+structstrbuf*err,structbatch_options*opt,structexpand_data*data){
From: Christian Couder <hidden> Date: 2021-06-12 20:10:37
On Sat, Jun 12, 2021 at 1:14 PM ZheNing Hu via GitGitGadget
[off-list ref] wrote:
From: ZheNing Hu <redacted>
Let `populate_value()`, `get_ref_atom_value()` and
`format_ref_array_item()` get the return value of `get_value()`
correctly. This can help us later let `cat-file --batch` get the
correct error message and return value of `get_value()`.
From: ZheNing Hu <hidden> Date: 2021-06-13 10:14:53
Christian Couder [off-list ref] 于2021年6月13日周日 上午4:09写道：
On Sat, Jun 12, 2021 at 1:14 PM ZheNing Hu via GitGitGadget
[off-list ref] wrote:
quoted
From: ZheNing Hu <redacted>
Let `populate_value()`, `get_ref_atom_value()` and
`format_ref_array_item()` get the return value of `get_value()`
correctly. This can help us later let `cat-file --batch` get the
correct error message and return value of `get_value()`.
Is it get_object() or get_value()?
Oh, it's get_object().
Thanks for pointing out:)
--
ZheNing Hu
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-15 13:29:34
From: ZheNing Hu <redacted>
Only tag and commit objects use `grab_sub_body_contents()` to grab
object contents in the current codebase. We want to teach the
function to also handle blobs and trees to get their raw data,
without parsing a blob (whose contents looks like a commit or a tag)
incorrectly as a commit or a tag.
Skip the block of code that is specific to handling commits and tags
early when the given object is of a wrong type to help later
addition to handle other types of objects in this function.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
@@ -1678,7 +1684,7 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objreturnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi->content);+grab_values(ref->value,deref,*obj,oi);}grab_common_values(ref->value,deref,oi);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-15 13:29:38
From: ZheNing Hu <redacted>
Add new formatting option `%(raw)`, which will print the raw
object data without any changes. It will help further to migrate
all cat-file formatting logic from cat-file to ref-filter.
The raw data of blob, tree objects may contain '\0', but most of
the logic in `ref-filter` depends on the output of the atom being
text (specifically, no embedded NULs in it).
E.g. `quote_formatting()` use `strbuf_addstr()` or `*._quote_buf()`
add the data to the buffer. The raw data of a tree object is
`100644 one\0...`, only the `100644 one` will be added to the buffer,
which is incorrect.
Therefore, add a new member in `struct atom_value`: `s_size`, which
can record raw object size, it can help us add raw object data to
the buffer or compare two buffers which contain raw object data.
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a variable
in the host language, the host language may not support arbitrary binary
data in the variables of its string type.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Felipe Contreras [off-list ref]
Helped-by: Phillip Wood [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Based-on-patch-by: Olga Telezhnaya [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-for-each-ref.txt | 9 ++
ref-filter.c | 139 +++++++++++++++----
t/t6300-for-each-ref.sh | 207 +++++++++++++++++++++++++++++
3 files changed, 328 insertions(+), 27 deletions(-)
@@ -235,6 +235,15 @@ and `date` to extract the named component. For email fields (`authoremail`, without angle brackets, and `:localpart` to get the part before the `@` symbol out of the trimmed email.+The raw data in an object is `raw`.++raw:size::+ The raw data size of the object.++Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,+`--perl` because the host language may not support arbitrary binary data in the+variables of its string type.+ The message in a commit or a tag object is `contents`, from which `contents:<part>` can be used to extract various parts out of:
@@ -620,12 +637,15 @@ struct ref_formatting_state {structatom_value{constchar*s;+size_ts_size;int(*handler)(structatom_value*atomv,structref_formatting_state*state,structstrbuf*err);uintmax_tvalue;/* used for sorting when not FIELD_STR */structused_atom*atom;};+#define ATOM_VALUE_S_SIZE_INIT (-1)+/**Usedtoparseformatstringandsortspecifiers*/
@@ -644,13 +664,6 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnstrbuf_addf_ret(err,-1,_("malformed field name: %.*s"),(int)(ep-atom),atom);-/* Do we have the atom already used elsewhere? */-for(i=0;i<used_atom_cnt;i++){-intlen=strlen(used_atom[i].name);-if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))-returni;-}-/**Iftheatomnamehasacolon,stripitandeverythingafter*itoff-itspecifiestheformatforthisentry,and
@@ -660,6 +673,13 @@ static int parse_ref_filter_atom(const struct ref_format *format,arg=memchr(sp,':',ep-sp);atom_len=(arg?arg:ep)-sp;+/* Do we have the atom already used elsewhere? */+for(i=0;i<used_atom_cnt;i++){+intlen=strlen(used_atom[i].name);+if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))+returni;+}+/* Is the atom a valid one? */for(i=0;i<ARRAY_SIZE(valid_atom);i++){intlen=strlen(valid_atom[i].name);
@@ -709,11 +729,14 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnat;}-staticvoidquote_formatting(structstrbuf*s,constchar*str,intquote_style)+staticvoidquote_formatting(structstrbuf*s,constchar*str,size_tlen,intquote_style){switch(quote_style){caseQUOTE_NONE:-strbuf_addstr(s,str);+if(len!=ATOM_VALUE_S_SIZE_INIT)+strbuf_add(s,str,len);+else+strbuf_addstr(s,str);break;caseQUOTE_SHELL:sq_quote_buf(s,str);
@@ -867,18 +895,22 @@ static int then_atom_handler(struct atom_value *atomv, struct ref_formatting_staif(if_then_else->else_atom_seen)returnstrbuf_addf_ret(err,-1,_("format: %%(then) atom used after %%(else)"));if_then_else->then_atom_seen=1;+if(if_then_else->str)+str_len=strlen(if_then_else->str);/**Ifthe'equals'or'notequals'attributeisusedthen*performtherequiredcomparison.Ifnot,onlynon-empty*stringssatisfythe'if'condition.*/if(if_then_else->cmp_status==COMPARE_EQUAL){-if(!strcmp(if_then_else->str,cur->output.buf))+if(str_len==cur->output.len&&+!memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;}elseif(if_then_else->cmp_status==COMPARE_UNEQUAL){-if(strcmp(if_then_else->str,cur->output.buf))+if(str_len!=cur->output.len||+memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;-}elseif(cur->output.len&&!is_empty(cur->output.buf))+}elseif(cur->output.len&&!is_empty(&cur->output))if_then_else->condition_satisfied=1;strbuf_reset(&cur->output);return0;
@@ -974,6 +1006,10 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&+used_atom[at].u.raw_data.option==RAW_BARE)+die(_("--format=%.*s cannot be used with"+"--python, --shell, --tcl, --perl"),(int)(ep-sp-2),sp+2);cp=ep+1;if(skip_prefix(used_atom[at].name,"color:",&color))
@@ -708,6 +737,15 @@ test_atom refs/tags/signed-long contents "subject line bodycontents$sig"+test_expect_successGPG'basic atom: refs/tags/signed-long raw''+gitcat-filetagrefs/tags/signed-long>expected&&+gitfor-each-ref--format="%(raw)"refs/tags/signed-long>actual&&+sanitize_pgp<expected>expected.clean&&+sanitize_pgp<actual>actual.clean&&+echo"">>expected.clean&&+test_cmpexpected.cleanactual.clean+'+ test_expect_success'set up refs pointing to tree and blob''gitupdate-refrefs/mytrees/firstrefs/heads/main^{tree}&&gitupdate-refrefs/myblobs/firstrefs/heads/main:one
@@ -727,6 +775,165 @@ test_atom refs/myblobs/first contents:body "" test_atomrefs/myblobs/firstcontents:signature"" test_atomrefs/myblobs/firstcontents""+test_expect_success'basic atom: refs/myblobs/first raw''+gitcat-fileblobrefs/myblobs/first>expected&&+echo"">>expected&&+gitfor-each-ref--format="%(raw)"refs/myblobs/first>actual&&+test_cmpexpectedactual&&+gitcat-file-srefs/myblobs/first>expected&&+gitfor-each-ref--format="%(raw:size)"refs/myblobs/first>actual&&+test_cmpexpectedactual+'++test_expect_success'set up refs pointing to binary blob''+printf"a\0b\0c">blob1&&+printf"a\0c\0b">blob2&&+printf"\0a\0b\0c">blob3&&+printf"abc">blob4&&+printf"\0 \0 \0 ">blob5&&+printf"\0 \0a\0 ">blob6&&+printf" ">blob7&&+>blob8&&+githash-objectblob1-w|xargsgitupdate-refrefs/myblobs/blob1&&+githash-objectblob2-w|xargsgitupdate-refrefs/myblobs/blob2&&+githash-objectblob3-w|xargsgitupdate-refrefs/myblobs/blob3&&+githash-objectblob4-w|xargsgitupdate-refrefs/myblobs/blob4&&+githash-objectblob5-w|xargsgitupdate-refrefs/myblobs/blob5&&+githash-objectblob6-w|xargsgitupdate-refrefs/myblobs/blob6&&+githash-objectblob7-w|xargsgitupdate-refrefs/myblobs/blob7&&+githash-objectblob8-w|xargsgitupdate-refrefs/myblobs/blob8+'++test_expect_success'Verify sorts with raw''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob3+refs/myblobs/blob7+refs/mytrees/first+refs/myblobs/first+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob4+refs/heads/main+EOF+gitfor-each-ref--format="%(refname)"--sort=raw\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'Verify sorts with raw:size''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/first+refs/myblobs/blob7+refs/heads/main+refs/myblobs/blob4+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+refs/myblobs/blob5+refs/myblobs/blob6+refs/mytrees/first+EOF+gitfor-each-ref--format="%(refname)"--sort=raw:size\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'validate raw atom with %(if:equals)''+cat>expected<<-EOF&&+notequals+notequals+notequals+notequals+notequals+notequals+refs/myblobs/blob4+notequals+notequals+notequals+notequals+notequals+EOF+gitfor-each-ref--format="%(if:equals=abc)%(raw)%(then)%(refname)%(else)not equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'+test_expect_success'validate raw atom with %(if:notequals)''+cat>expected<<-EOF&&+refs/heads/ambiguous+refs/heads/main+refs/heads/newtag+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+equals+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob7+refs/myblobs/blob8+refs/myblobs/first+EOF+gitfor-each-ref--format="%(if:notequals=abc)%(raw)%(then)%(refname)%(else)equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'++test_expect_success'empty raw refs with %(if)''+cat>expected<<-EOF&&+refs/myblobs/blob1notempty+refs/myblobs/blob2notempty+refs/myblobs/blob3notempty+refs/myblobs/blob4notempty+refs/myblobs/blob5notempty+refs/myblobs/blob6notempty+refs/myblobs/blob7empty+refs/myblobs/blob8empty+refs/myblobs/firstnotempty+EOF+gitfor-each-ref--format="%(refname) %(if)%(raw)%(then)not empty%(else)empty%(end)"\+refs/myblobs/>actual&&+test_cmpexpectedactual+'++test_expect_success'%(raw) with --python must failed''+test_must_failgitfor-each-ref--format="%(raw)"--python+'++test_expect_success'%(raw) with --tcl must failed''+test_must_failgitfor-each-ref--format="%(raw)"--tcl+'++test_expect_success'%(raw) with --perl must failed''+test_must_failgitfor-each-ref--format="%(raw)"--perl+'++test_expect_success'%(raw) with --shell must failed''+test_must_failgitfor-each-ref--format="%(raw)"--shell+'++test_expect_success'%(raw) with --shell and --sort=raw must failed''+test_must_failgitfor-each-ref--format="%(raw)"--sort=raw--shell+'++test_expect_success'%(raw:size) with --shell''+gitfor-each-ref--format="%(raw:size)"|whilereadline+do+echo"'\''$line'\''">>expect+done&&+gitfor-each-ref--format="%(raw:size)"--shell>actual&&+test_cmpexpectactual+'++test_expect_success'for-each-ref --format compare with cat-file --batch''+gitrev-parserefs/mytrees/first|gitcat-file--batch>expected&&+gitfor-each-ref--format="%(objectname) %(objecttype) %(objectsize)+%(raw)" refs/mytrees/first >actual &&+test_cmpexpectedactual+'+ test_expect_success'set up multiple-sort tags''forwhenin100000200000do
@@ -559,6 +560,15 @@ static int if_atom_parser(struct ref_format *format, struct used_atom *atom,return0;}+staticintrest_atom_parser(structref_format*format,structused_atom*atom,+constchar*arg,structstrbuf*err)+{+if(arg)+returnstrbuf_addf_ret(err,-1,_("%%(rest) does not take arguments"));+format->use_rest=1;+return0;+}+staticinthead_atom_parser(structref_format*format,structused_atom*atom,constchar*arg,structstrbuf*unused_err){
@@ -1006,6 +1017,9 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);+if(used_atom[at].atom_type==ATOM_REST)+die("this command reject atom %%(%.*s)",(int)(ep-sp-2),sp+2);+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&used_atom[at].u.raw_data.option==RAW_BARE)die(_("--format=%.*s cannot be used with"
@@ -1998,6 +1998,10 @@ test_expect_success '--format should list tags as per format given' 'test_cmpexpectactual'+test_expect_success'git tag -l with --format="%(rest)" must failed''+test_must_failgittag-l--format="%(rest)""v1*"+'+ test_expect_success"set up color tests"'echo"<RED>v1.0<RESET>">expect.color&&echo"v1.0">expect.bare&&
@@ -194,6 +194,10 @@ test_expect_success GPG 'verifying tag with --format' 'test_cmpexpectactual'+test_expect_successGPG'verifying tag with --format="%(rest)" must failed''+test_must_failgitverify-tag--format="%(rest)""fourth-signed"+'+ test_expect_successGPG'verifying a forged tag with --format should fail silently''test_must_failgitverify-tag--format="tagname : %(tag)"$(catforged1.tag)>actual-forged&&test_must_be_emptyactual-forged
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-15 13:29:41
From: ZheNing Hu <redacted>
Use non-const ref_format in *_atom_parser(), which can help us
modify the members of ref_format in *_atom_parser().
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/tag.c | 2 +-
ref-filter.c | 44 ++++++++++++++++++++++----------------------
ref-filter.h | 4 ++--
3 files changed, 25 insertions(+), 25 deletions(-)
@@ -116,7 +116,7 @@ void ref_array_sort(struct ref_sorting *sort, struct ref_array *array);voidref_sorting_set_sort_flags_all(structref_sorting*sorting,unsignedintmask,inton);/* Based on the given format and quote_style, fill the strbuf */intformat_ref_array_item(structref_array_item*info,-conststructref_format*format,+structref_format*format,structstrbuf*final_buf,structstrbuf*error_buf);/* Parse a single sort specifier and add it to the list */
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-15 13:29:43
From: ZheNing Hu <redacted>
Introduce free_array_item_internal() for freeing ref_array_item value.
It will be called internally by free_array_item(), and it will help
`cat-file --batch` free ref_array_item's memory later.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 11 ++++++++---
ref-filter.h | 2 ++
2 files changed, 10 insertions(+), 3 deletions(-)
@@ -111,6 +111,8 @@ struct ref_format {intfilter_refs(structref_array*array,structref_filter*filter,unsignedinttype);/* Clear all memory allocated to ref_array */voidref_array_clear(structref_array*array);+/* Free array item's value */+voidfree_array_item_internal(structref_array_item*item);/* Used to verify if the given format is correct and to parse out the used atoms */intverify_ref_format(structref_format*format);/* Sort the given ref_array as per the ref_sorting provided */
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-15 13:29:44
From: ZheNing Hu <redacted>
Let `populate_value()`, `get_ref_atom_value()` and
`format_ref_array_item()` get the return value of `get_object()`
correctly. This can help us later let `cat-file --batch` get the
correct error message and return value of `get_object()`.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-15 13:29:47
From: ZheNing Hu <redacted>
In order to let cat-file use ref-filter logic, the following
methods are used:
1. Add `cat_file_mode` member in struct `ref_format`, this can
help us reject atoms in verify_ref_format() which cat-file
cannot use, e.g. `%(refname)`, `%(push)`, `%(upstream)`...
2. Change the type of member `format` in struct `batch_options`
to `ref_format`, We can add format data in it.
3. Let `batch_objects()` add atoms to format, and use
`verify_ref_format()` to check atoms.
4. Use `has_object_file()` in `batch_one_object()` to check
whether the input object exists.
5. Use `format_ref_array_item()` in `batch_object_write()` to
get the formatted data corresponding to the object. If the
return value of `format_ref_array_item()` is equals to zero,
use `batch_write()` to print object data; else if the return
value less than zero, use `die()` to print the error message
and exit; else return value greater than zero, only print the
error message, but not exit.
6. Let get_object() return 1 and print "<oid> missing" instead
of returning -1 and printing "missing object <oid> for <refname>",
this can help `format_ref_array_item()` just report that the
object is missing without letting Git exit.
Most of the atoms in `for-each-ref --format` are now supported,
such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,
`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:
`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,
`%(flag)`, `%(HEAD)`, because our objects don't have refname.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-cat-file.txt | 6 +
builtin/cat-file.c | 248 +++++++-------------------------
ref-filter.c | 15 +-
ref-filter.h | 3 +-
t/t1006-cat-file.sh | 252 +++++++++++++++++++++++++++++++++
t/t6301-for-each-ref-errors.sh | 2 +-
6 files changed, 323 insertions(+), 203 deletions(-)
@@ -226,6 +226,12 @@ newline. The available atoms are: after that first run of whitespace (i.e., the "rest" of the line) are output in place of the `%(rest)` atom.+Note that most of the atoms in `for-each-ref --format` are now supported,+such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,+`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:+`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,+`%(flag)`, `%(HEAD)`. See linkgit:git-for-each-ref[1].+ If no format is specified, the default format is `%(objectname) %(objecttype) %(objectsize)`.
@@ -25,7 +26,7 @@ struct batch_options {intall_objects;intunordered;intcmdmode;/* may be 'w' or 'c' for --filters or --textconv */-constchar*format;+structref_formatformat;};staticconstchar*force_path;
@@ -195,99 +196,10 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name,structexpand_data{structobject_idoid;-enumobject_typetype;-unsignedlongsize;-off_tdisk_size;constchar*rest;-structobject_iddelta_base_oid;--/*-*Ifmark_queryistrue,wedonotexpandanything,butrather-*justmarktheobject_infowithitemswewishtoquery.-*/-intmark_query;--/*-*Whethertosplittheinputonwhitespacebeforefeedingitto-*get_sha1;thisisdecidedduringthemark_queryphasebasedon-*whetherwehavea%(rest)tokeninourformat.-*/intsplit_on_whitespace;--/*-*Afteramark_queryrun,thisobject_infoissetuptobe-*passedtooid_object_info_extended.Itwillpointtothedata-*elementsabove,soyoucanretrievetheresponsefromthere.-*/-structobject_infoinfo;--/*-*Thisflagwillbetrueiftherequestedbatchformatandoptions-*don'trequireustocalloid_object_info,whichcanthenbe-*optimizedout.-*/-unsignedskip_object_info:1;};-staticintis_atom(constchar*atom,constchar*s,intslen)-{-intalen=strlen(atom);-returnalen==slen&&!memcmp(atom,s,alen);-}--staticvoidexpand_atom(structstrbuf*sb,constchar*atom,intlen,-void*vdata)-{-structexpand_data*data=vdata;--if(is_atom("objectname",atom,len)){-if(!data->mark_query)-strbuf_addstr(sb,oid_to_hex(&data->oid));-}elseif(is_atom("objecttype",atom,len)){-if(data->mark_query)-data->info.typep=&data->type;-else-strbuf_addstr(sb,type_name(data->type));-}elseif(is_atom("objectsize",atom,len)){-if(data->mark_query)-data->info.sizep=&data->size;-else-strbuf_addf(sb,"%"PRIuMAX,(uintmax_t)data->size);-}elseif(is_atom("objectsize:disk",atom,len)){-if(data->mark_query)-data->info.disk_sizep=&data->disk_size;-else-strbuf_addf(sb,"%"PRIuMAX,(uintmax_t)data->disk_size);-}elseif(is_atom("rest",atom,len)){-if(data->mark_query)-data->split_on_whitespace=1;-elseif(data->rest)-strbuf_addstr(sb,data->rest);-}elseif(is_atom("deltabase",atom,len)){-if(data->mark_query)-data->info.delta_base_oid=&data->delta_base_oid;-else-strbuf_addstr(sb,-oid_to_hex(&data->delta_base_oid));-}else-die("unknown format element: %.*s",len,atom);-}--staticsize_texpand_format(structstrbuf*sb,constchar*start,void*data)-{-constchar*end;--if(*start!='(')-return0;-end=strchr(start+1,')');-if(!end)-die("format element '%s' does not end in ')'",start);--expand_atom(sb,start+1,end-start-1,data);--returnend-start+1;-}-staticvoidbatch_write(structbatch_options*opt,constvoid*data,intlen){if(opt->buffer_output){
@@ -297,87 +209,31 @@ static void batch_write(struct batch_options *opt, const void *data, int len)write_or_die(1,data,len);}-staticvoidprint_object_or_die(structbatch_options*opt,structexpand_data*data)-{-conststructobject_id*oid=&data->oid;--assert(data->info.typep);--if(data->type==OBJ_BLOB){-if(opt->buffer_output)-fflush(stdout);-if(opt->cmdmode){-char*contents;-unsignedlongsize;--if(!data->rest)-die("missing path for '%s'",oid_to_hex(oid));--if(opt->cmdmode=='w'){-if(filter_object(data->rest,0100644,oid,-&contents,&size))-die("could not convert '%s' %s",-oid_to_hex(oid),data->rest);-}elseif(opt->cmdmode=='c'){-enumobject_typetype;-if(!textconv_object(the_repository,-data->rest,0100644,oid,-1,&contents,&size))-contents=read_object_file(oid,-&type,-&size);-if(!contents)-die("could not convert '%s' %s",-oid_to_hex(oid),data->rest);-}else-BUG("invalid cmdmode: %c",opt->cmdmode);-batch_write(opt,contents,size);-free(contents);-}else{-stream_blob(oid);-}-}-else{-enumobject_typetype;-unsignedlongsize;-void*contents;--contents=read_object_file(oid,&type,&size);-if(!contents)-die("object %s disappeared",oid_to_hex(oid));-if(type!=data->type)-die("object %s changed type!?",oid_to_hex(oid));-if(data->info.sizep&&size!=data->size)-die("object %s changed size!?",oid_to_hex(oid));--batch_write(opt,contents,size);-free(contents);-}-}staticvoidbatch_object_write(constchar*obj_name,structstrbuf*scratch,structbatch_options*opt,structexpand_data*data){-if(!data->skip_object_info&&-oid_object_info_extended(the_repository,&data->oid,&data->info,-OBJECT_INFO_LOOKUP_REPLACE)<0){-printf("%s missing\n",-obj_name?obj_name:oid_to_hex(&data->oid));-fflush(stdout);-return;-}+intret=0;+structstrbuferr=STRBUF_INIT;+structref_array_itemitem={data->oid,data->rest};strbuf_reset(scratch);-strbuf_expand(scratch,opt->format,expand_format,data);-strbuf_addch(scratch,'\n');-batch_write(opt,scratch->buf,scratch->len);-if(opt->print_contents){-print_object_or_die(opt,data);-batch_write(opt,"\n",1);+ret=format_ref_array_item(&item,&opt->format,scratch,&err);+if(!ret){+strbuf_addch(scratch,'\n');+batch_write(opt,scratch->buf,scratch->len);+}elseif(ret<0){+die("%s\n",err.buf);+}else{+/* when ret > 0 , don't call die and print the err to stdout*/+printf("%s\n",err.buf);+fflush(stdout);}+free_array_item_internal(&item);+strbuf_release(&err);}staticvoidbatch_one_object(constchar*obj_name,
@@ -713,7 +567,7 @@ int cmd_cat_file(int argc, const char **argv, const char *prefix)batch.buffer_output=batch.all_objects;if(batch.enabled)-returnbatch_objects(&batch);+returnbatch_objects(&batch,options);if(unknown_type&&opt!='t'&&opt!='s')die("git cat-file --allow-unknown-type: use with -s or -t");
@@ -1017,8 +1017,15 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);-if(used_atom[at].atom_type==ATOM_REST)-die("this command reject atom %%(%.*s)",(int)(ep-sp-2),sp+2);+if((!format->cat_file_mode&&used_atom[at].atom_type==ATOM_REST)||+(format->cat_file_mode&&(used_atom[at].atom_type==ATOM_FLAG||+used_atom[at].atom_type==ATOM_HEAD||+used_atom[at].atom_type==ATOM_PUSH||+used_atom[at].atom_type==ATOM_REFNAME||+used_atom[at].atom_type==ATOM_SYMREF||+used_atom[at].atom_type==ATOM_UPSTREAM||+used_atom[at].atom_type==ATOM_WORKTREEPATH)))+die(_("this command reject atom %%(%.*s)"),(int)(ep-sp-2),sp+2);if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&used_atom[at].u.raw_data.option==RAW_BARE)
@@ -1735,8 +1742,8 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj}if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,OBJECT_INFO_LOOKUP_REPLACE))-returnstrbuf_addf_ret(err,-1,_("missing object %s for %s"),-oid_to_hex(&oi->oid),ref->refname);+returnstrbuf_addf_ret(err,1,_("%s missing"),+oid_to_hex(&oi->oid));if(oi->info.disk_sizep&&oi->disk_size<0)BUG("Object size is less than zero.");
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-15 13:29:48
From: ZheNing Hu <redacted>
Reuse the `err` buffer in batch_object_write(), as the
buffer `scratch` does. This will reduce the overhead
of multiple allocations of memory of the err buffer.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 22 ++++++++++++++--------
1 file changed, 14 insertions(+), 8 deletions(-)
@@ -212,32 +212,33 @@ static void batch_write(struct batch_options *opt, const void *data, int len)staticvoidbatch_object_write(constchar*obj_name,structstrbuf*scratch,+structstrbuf*err,structbatch_options*opt,structexpand_data*data){intret=0;-structstrbuferr=STRBUF_INIT;structref_array_itemitem={data->oid,data->rest};strbuf_reset(scratch);+strbuf_reset(err);-ret=format_ref_array_item(&item,&opt->format,scratch,&err);+ret=format_ref_array_item(&item,&opt->format,scratch,err);if(!ret){strbuf_addch(scratch,'\n');batch_write(opt,scratch->buf,scratch->len);}elseif(ret<0){-die("%s\n",err.buf);+die("%s\n",err->buf);}else{/* when ret > 0 , don't call die and print the err to stdout*/-printf("%s\n",err.buf);+printf("%s\n",err->buf);fflush(stdout);}free_array_item_internal(&item);-strbuf_release(&err);}staticvoidbatch_one_object(constchar*obj_name,structstrbuf*scratch,+structstrbuf*err,structbatch_options*opt,structexpand_data*data){
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-15 13:29:52
From: ZheNing Hu <redacted>
After cat-file reuses the ref-filter logic, we re-implement the
functions of --textconv and --filters options.
Add members `use_textconv` and `use_filters` in struct `ref_format`,
and use global variables `use_filters` and `use_textconv` in
`ref-filter.c`, so that we can filter the content of the object
in get_object(). Use `actual_oi` to record the real expand_data:
it may point to the original `oi` or the `act_oi` processed by
`textconv_object()` or `convert_to_working_tree()`. `grab_values()`
will grab the contents of `actual_oi` and `grab_common_values()`
to grab the contents of origin `oi`, this ensures that `%(objectsize)`
still uses the size of the unfiltered data.
In `get_object()`, we made an optimization: Firstly, get the size and
type of the object instead of directly getting the object data.
If using --textconv, after successfully obtaining the filtered object
data, an extra oid_object_info_extended() will be skipped, which can
reduce the cost of object data copy; If using --filter, the data of
the object first will be getted first, and then convert_to_working_tree()
will be used to get the filtered object data.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 5 ++++
ref-filter.c | 66 ++++++++++++++++++++++++++++++++++++++++++++--
ref-filter.h | 4 ++-
3 files changed, 72 insertions(+), 3 deletions(-)
@@ -1027,6 +1031,9 @@ int verify_ref_format(struct ref_format *format)used_atom[at].atom_type==ATOM_WORKTREEPATH)))die(_("this command reject atom %%(%.*s)"),(int)(ep-sp-2),sp+2);+use_filters=format->use_filters;+use_textconv=format->use_textconv;+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&used_atom[at].u.raw_data.option==RAW_BARE)die(_("--format=%.*s cannot be used with"
@@ -1735,10 +1742,41 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj{/* parse_object_buffer() will set eaten to 0 if free() will be needed */inteaten=1;+structexpand_data*actual_oi=oi;+structexpand_dataact_oi={0};+if(oi->info.contentp){/* We need to know that to use parse_object_buffer properly */+void**temp_contentp=oi->info.contentp;+oi->info.contentp=NULL;oi->info.sizep=&oi->size;oi->info.typep=&oi->type;++/* get the type and size */+if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,+OBJECT_INFO_LOOKUP_REPLACE))+returnstrbuf_addf_ret(err,1,_("%s missing"),+oid_to_hex(&oi->oid));++oi->info.sizep=NULL;+oi->info.typep=NULL;+oi->info.contentp=temp_contentp;++if(use_textconv){+act_oi=*oi;++if(!ref->rest)+returnstrbuf_addf_ret(err,-1,_("missing path for '%s'"),+oid_to_hex(&act_oi.oid));+if(act_oi.type==OBJ_BLOB){+if(textconv_object(the_repository,+ref->rest,0100644,&act_oi.oid,+1,(char**)(&act_oi.content),&act_oi.size)){+actual_oi=&act_oi;+gotosuccess;+}+}+}}if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,OBJECT_INFO_LOOKUP_REPLACE))
@@ -1748,19 +1786,43 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objBUG("Object size is less than zero.");if(oi->info.contentp){-*obj=parse_object_buffer(the_repository,&oi->oid,oi->type,oi->size,oi->content,&eaten);+if(use_filters){+if(!ref->rest)+returnstrbuf_addf_ret(err,-1,_("missing path for '%s'"),+oid_to_hex(&oi->oid));+if(oi->type==OBJ_BLOB){+structstrbufstrbuf=STRBUF_INIT;+structcheckout_metadatameta;+act_oi=*oi;++init_checkout_metadata(&meta,NULL,NULL,&act_oi.oid);+if(convert_to_working_tree(&the_index,ref->rest,act_oi.content,act_oi.size,&strbuf,&meta)){+act_oi.size=strbuf.len;+act_oi.content=strbuf_detach(&strbuf,NULL);+actual_oi=&act_oi;+}else{+die("could not convert '%s' %s",+oid_to_hex(&oi->oid),ref->rest);+}+}+}++success:+*obj=parse_object_buffer(the_repository,&actual_oi->oid,actual_oi->type,actual_oi->size,actual_oi->content,&eaten);if(!*obj){if(!eaten)free(oi->content);returnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi);+grab_values(ref->value,deref,*obj,actual_oi);}grab_common_values(ref->value,deref,oi);if(!eaten)free(oi->content);+if(actual_oi!=oi)+free(actual_oi->content);return0;}
On Sat, Jun 12 2021, ZheNing Hu via GitGitGadget wrote:
quoted hunk
From: ZheNing Hu <redacted>
Only tag and commit objects use `grab_sub_body_contents()` to grab
object contents in the current codebase. We want to teach the
function to also handle blobs and trees to get their raw data,
without parsing a blob (whose contents looks like a commit or a tag)
incorrectly as a commit or a tag.
Skip the block of code that is specific to handling commits and tags
early when the given object is of a wrong type to help later
addition to handle other types of objects in this function.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
We have 4 "real" object types, commit, tree, blob, tag. Do you really
mean "not tag or commit" here, don't you mean "is tree or blob" instead?
I.e. do we really want to pass OBJ_NONE etc. here?
quoted hunk
continue;
if (!subpos)
find_subpos(buf,
@@ -1438,17 +1442,19 @@ static void fill_missing_values(struct atom_value *val) * pointed at by the ref itself; otherwise it is the object the * ref (which is a tag) refers to. */-static void grab_values(struct atom_value *val, int deref, struct object *obj, void *buf)+static void grab_values(struct atom_value *val, int deref, struct object *obj, struct expand_data *data) {+ void *buf = data->content;+ switch (obj->type) { case OBJ_TAG: grab_tag_values(val, deref, obj);- grab_sub_body_contents(val, deref, buf);+ grab_sub_body_contents(val, deref, data); grab_person("tagger", val, deref, buf); break; case OBJ_COMMIT: grab_commit_values(val, deref, obj);- grab_sub_body_contents(val, deref, buf);+ grab_sub_body_contents(val, deref, data); grab_person("author", val, deref, buf); grab_person("committer", val, deref, buf); break;
@@ -1678,7 +1684,7 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj return strbuf_addf_ret(err, -1, _("parse_object_buffer failed on %s for %s"), oid_to_hex(&oi->oid), ref->refname); }- grab_values(ref->value, deref, *obj, oi->content);+ grab_values(ref->value, deref, *obj, oi); } grab_common_values(ref->value, deref, oi);
On Sat, Jun 12 2021, ZheNing Hu via GitGitGadget wrote:
From: ZheNing Hu <redacted>
Add new formatting option `%(raw)`, which will print the raw
object data without any changes. It will help further to migrate
all cat-file formatting logic from cat-file to ref-filter.
Nice goal and feature to have.
The raw data of blob, tree objects may contain '\0', but most of
the logic in `ref-filter` depends on the output of the atom being
text (specifically, no embedded NULs in it).
E.g. `quote_formatting()` use `strbuf_addstr()` or `*._quote_buf()`
add the data to the buffer. The raw data of a tree object is
`100644 one\0...`, only the `100644 one` will be added to the buffer,
which is incorrect.
Therefore, add a new member in `struct atom_value`: `s_size`, which
can record raw object size, it can help us add raw object data to
the buffer or compare two buffers which contain raw object data.
Most of the functions that deal with this already use a strbuf in some
way, before we had a const char *, now there's a size_t to go along with
it, why not simply use a strbuf in the struct for the data? You'll then
get the size and \0 handling for free, and any functions to deal with
conversion can stick to the strbuf API, there seems to be a lot of back
and forth now.
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a variable
in the host language, the host language may not support arbitrary binary
data in the variables of its string type.
Perl at least deals with that just fine, and to the extent that it
doesn't any new problems here would have nothing to do with \0 being in
the data. Perl doesn't have a notion of "binary has \0 in it", it always
supports \0, it has a notion of "is it utf-8 or not?", so any encoding
problems wouldn't be new. I'd think that the same would be true of
Python, but I'm not sure.
Maybe change the if (A) { if (B) {} ) to if (A && B) {} ?
quoted hunk
}
if (oid_object_info_extended(the_repository, &oi->oid, &oi->info,
OBJECT_INFO_LOOKUP_REPLACE))
@@ -1748,19 +1786,43 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj BUG("Object size is less than zero."); if (oi->info.contentp) {- *obj = parse_object_buffer(the_repository, &oi->oid, oi->type, oi->size, oi->content, &eaten);+ if (use_filters) {+ if(!ref->rest)+ return strbuf_addf_ret(err, -1, _("missing path for '%s'"),+ oid_to_hex(&oi->oid));+ if (oi->type == OBJ_BLOB) {+ struct strbuf strbuf = STRBUF_INIT;+ struct checkout_metadata meta;+ act_oi = *oi;++ init_checkout_metadata(&meta, NULL, NULL, &act_oi.oid);+ if (convert_to_working_tree(&the_index, ref->rest, act_oi.content, act_oi.size, &strbuf, &meta)) {+ act_oi.size = strbuf.len;+ act_oi.content = strbuf_detach(&strbuf, NULL);+ actual_oi = &act_oi;+ } else {+ die("could not convert '%s' %s",+ oid_to_hex(&oi->oid), ref->rest);+ }
... but here instead of "if (!x) { bad } do stuff" we have if (x) {do
stuff} else { bad }". Better to get the die out of the way, and avoid
the indentation on the "do stuff" IMO.
Not a new problem, but an earlier cleanup to simply change this to
designated initializers would be welcome, see recent work of mine in
fsck.h for an example.
I.e. we keep churning on changing this *_INIT just to populate the one
-1 field at the end, can also be simply:
#define FOO_INIT { .that_field = 1 }
On Sat, Jun 12 2021, ZheNing Hu via GitGitGadget wrote:
quoted hunk
From: ZheNing Hu <redacted>
Let `populate_value()`, `get_ref_atom_value()` and
`format_ref_array_item()` get the return value of `get_value()`
correctly. This can help us later let `cat-file --batch` get the
correct error message and return value of `get_value()`.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
The usual style is to not bunch up variables based on type, but only if
they're related, i.e. we'd do:
if i, j; /* proceed to use i and j in two for-loops */
But:
int i; /* for the for-loop */
int ret = 0; /* for our return value */
(Without the comments)
Maybe more personal style, I'd just write this as:
ret = x();
if (!ret)
return ret;
Makes it easier to read and balance parens in your head for the common
case...
On Tue, Jun 15 2021, ZheNing Hu via GitGitGadget wrote:
This patch series make cat-file reuse ref-filter logic, which based on
5a5b5f78 ([GSOC] ref-filter: add %(rest) atom)
Change from last version:
1. Use free_array_item_internal() to solve the memory leak problem.
2. Change commit message of ([GSOC] ref-filter: teach get_object() return
useful value).
I left some comments, but saw after the fact that I'd replied to the v1
E-Mails by accident, but anyway, the comments were all on things that
are also in v2, so it worked out in the end. Sorry about the confusion.
The raw data of blob, tree objects may contain '\0', but most of
the logic in `ref-filter` depends on the output of the atom being
text (specifically, no embedded NULs in it).
E.g. `quote_formatting()` use `strbuf_addstr()` or `*._quote_buf()`
add the data to the buffer. The raw data of a tree object is
`100644 one\0...`, only the `100644 one` will be added to the buffer,
which is incorrect.
Therefore, add a new member in `struct atom_value`: `s_size`, which
can record raw object size, it can help us add raw object data to
the buffer or compare two buffers which contain raw object data.
Most of the functions that deal with this already use a strbuf in some
way, before we had a const char *, now there's a size_t to go along with
it, why not simply use a strbuf in the struct for the data? You'll then
get the size and \0 handling for free, and any functions to deal with
conversion can stick to the strbuf API, there seems to be a lot of back
and forth now.
Yes, strbuf is a suitable choice when using <str,len> pair.
But if replace v->s with strbuf, the possible changes will be larger.
quoted
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a variable
in the host language, the host language may not support arbitrary binary
data in the variables of its string type.
Perl at least deals with that just fine, and to the extent that it
doesn't any new problems here would have nothing to do with \0 being in
the data. Perl doesn't have a notion of "binary has \0 in it", it always
supports \0, it has a notion of "is it utf-8 or not?", so any encoding
problems wouldn't be new. I'd think that the same would be true of
Python, but I'm not sure.
Not python safe. See [1].
Regarding the perl language, I support Junio's point of view: it can be
re-supported in the future.
}
if (oid_object_info_extended(the_repository, &oi->oid, &oi->info,
OBJECT_INFO_LOOKUP_REPLACE))
@@ -1748,19 +1786,43 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj BUG("Object size is less than zero."); if (oi->info.contentp) {- *obj = parse_object_buffer(the_repository, &oi->oid, oi->type, oi->size, oi->content, &eaten);+ if (use_filters) {+ if(!ref->rest)+ return strbuf_addf_ret(err, -1, _("missing path for '%s'"),+ oid_to_hex(&oi->oid));+ if (oi->type == OBJ_BLOB) {+ struct strbuf strbuf = STRBUF_INIT;+ struct checkout_metadata meta;+ act_oi = *oi;++ init_checkout_metadata(&meta, NULL, NULL, &act_oi.oid);+ if (convert_to_working_tree(&the_index, ref->rest, act_oi.content, act_oi.size, &strbuf, &meta)) {+ act_oi.size = strbuf.len;+ act_oi.content = strbuf_detach(&strbuf, NULL);+ actual_oi = &act_oi;+ } else {+ die("could not convert '%s' %s",+ oid_to_hex(&oi->oid), ref->rest);+ }
... but here instead of "if (!x) { bad } do stuff" we have if (x) {do
stuff} else { bad }". Better to get the die out of the way, and avoid
the indentation on the "do stuff" IMO.
And this part:
@@ -1786,25 +1784,21 @@ static int get_object(struct ref_array_item
Not a new problem, but an earlier cleanup to simply change this to
designated initializers would be welcome, see recent work of mine in
fsck.h for an example.
I.e. we keep churning on changing this *_INIT just to populate the one
-1 field at the end, can also be simply:
#define FOO_INIT { .that_field = 1 }
I agree, this method is better. We don't need to care about which members
so many "0" point to.
Thanks.
--
ZheNing Hu
On Sat, Jun 12 2021, ZheNing Hu via GitGitGadget wrote:
quoted
From: ZheNing Hu <redacted>
Let `populate_value()`, `get_ref_atom_value()` and
`format_ref_array_item()` get the return value of `get_value()`
correctly. This can help us later let `cat-file --batch` get the
correct error message and return value of `get_value()`.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
The usual style is to not bunch up variables based on type, but only if
they're related, i.e. we'd do:
if i, j; /* proceed to use i and j in two for-loops */
But:
int i; /* for the for-loop */
int ret = 0; /* for our return value */
(Without the comments)
Maybe more personal style, I'd just write this as:
ret = x();
if (!ret)
return ret;
Makes it easier to read and balance parens in your head for the common
case...
On Tue, Jun 15 2021, ZheNing Hu via GitGitGadget wrote:
quoted
This patch series make cat-file reuse ref-filter logic, which based on
5a5b5f78 ([GSOC] ref-filter: add %(rest) atom)
Change from last version:
1. Use free_array_item_internal() to solve the memory leak problem.
2. Change commit message of ([GSOC] ref-filter: teach get_object() return
useful value).
I left some comments, but saw after the fact that I'd replied to the v1
E-Mails by accident, but anyway, the comments were all on things that
are also in v2, so it worked out in the end. Sorry about the confusion.
It's okay. Your comments are very useful.
Thanks.
--
ZheNing Hu
The raw data of blob, tree objects may contain '\0', but most of
the logic in `ref-filter` depends on the output of the atom being
text (specifically, no embedded NULs in it).
E.g. `quote_formatting()` use `strbuf_addstr()` or `*._quote_buf()`
add the data to the buffer. The raw data of a tree object is
`100644 one\0...`, only the `100644 one` will be added to the buffer,
which is incorrect.
Therefore, add a new member in `struct atom_value`: `s_size`, which
can record raw object size, it can help us add raw object data to
the buffer or compare two buffers which contain raw object data.
Most of the functions that deal with this already use a strbuf in some
way, before we had a const char *, now there's a size_t to go along with
it, why not simply use a strbuf in the struct for the data? You'll then
get the size and \0 handling for free, and any functions to deal with
conversion can stick to the strbuf API, there seems to be a lot of back
and forth now.
Yes, strbuf is a suitable choice when using <str,len> pair.
But if replace v->s with strbuf, the possible changes will be larger.
I for one would like to see it done that way, those changes are usually
easy to read. Also it seems a large part of 2/8 is extra new code
because we didn't do that, e.g. getting length differently if something
is a strbuf or not, passing char*/size_t pairs to new functions etc.
quoted
quoted
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a variable
in the host language, the host language may not support arbitrary binary
data in the variables of its string type.
Perl at least deals with that just fine, and to the extent that it
doesn't any new problems here would have nothing to do with \0 being in
the data. Perl doesn't have a notion of "binary has \0 in it", it always
supports \0, it has a notion of "is it utf-8 or not?", so any encoding
problems wouldn't be new. I'd think that the same would be true of
Python, but I'm not sure.
Not python safe. See [1].
Regarding the perl language, I support Junio's point of view: it can be
re-supported in the future.
Ah, I'd missed that. Anyway, if it's easy it seems you discovered that
Perl deals with it correctly, so we could just have it support this.
Just "echo" will do, ditto for the rest. Also odd to go back and forth
between populating expected.clean & actual.clean.
Are you saying that sanitize_pgp is not needed?
No that instead of:
echo "" >x
You can do:
echo >x
And also that going back and forth between populating different files is
confusing, i.e. this:
echo a >x
echo c >y
echo b >>x
is better as:
echo a >x
echo b >>x
echo c >y
Yes, strbuf is a suitable choice when using <str,len> pair.
But if replace v->s with strbuf, the possible changes will be larger.
I for one would like to see it done that way, those changes are usually
easy to read. Also it seems a large part of 2/8 is extra new code
because we didn't do that, e.g. getting length differently if something
is a strbuf or not, passing char*/size_t pairs to new functions etc.
After some refactoring, I found that there are two problems:
1. There are a lot of codes like this in ref-filter to fill v->s:
v->s = show_ref(...)
v->s = copy_email(...)
It is very difficult to modify here: We know that show_ref()
or copy_email() will allocate a block of memory to v->s, but
if v->s is a strbuf, what should we do? In copy_email(), we
can pass the v->s to copy_email() and use strbuf_add()/strbuf_addstr()
instead of xstrdup() and xmemdupz(). But show_ref() will call
external functions like shorten_unambiguous_ref(), we don’t know
whether it will return us NULL or a dynamically allocated memory.
If continue to pass v->s to the inner function, it is not a feasible
method. Or we can use strbuf_attach() + strlen(), I'm not sure
this is a good method.
2. See:
- for (i = 0; i < used_atom_cnt; i++) {
+ for (i = 0; i < used_atom_cnt; i++) {
struct atom_value *v = &ref->value[i];
- if (v->s == NULL && used_atom[i].source == SOURCE_NONE)
+ if (v->s.len == 0 && used_atom[i].source == SOURCE_NONE)
return strbuf_addf_ret(err, -1, _("missing
object %s for %s"),
oid_to_hex(&ref->objectname), ref->refname);
}
In the case of using strbuf, I don’t know how to distinguish between an empty
strbuf and NULL. It can be easily distinguished by using c-style "const char*".
quoted
Not python safe. See [1].
Regarding the perl language, I support Junio's point of view: it can be
re-supported in the future.
Ah, I'd missed that. Anyway, if it's easy it seems you discovered that
Perl deals with it correctly, so we could just have it support this.
Well, it's ok, support for perl will be put in a separate commit.
Just "echo" will do, ditto for the rest. Also odd to go back and forth
between populating expected.clean & actual.clean.
Are you saying that sanitize_pgp is not needed?
No that instead of:
echo "" >x
You can do:
echo >x
And also that going back and forth between populating different files is
confusing, i.e. this:
echo a >x
echo c >y
echo b >>x
is better as:
echo a >x
echo b >>x
echo c >y
Yes, strbuf is a suitable choice when using <str,len> pair.
But if replace v->s with strbuf, the possible changes will be larger.
I for one would like to see it done that way, those changes are usually
easy to read. Also it seems a large part of 2/8 is extra new code
because we didn't do that, e.g. getting length differently if something
is a strbuf or not, passing char*/size_t pairs to new functions etc.
After some refactoring, I found that there are two problems:
1. There are a lot of codes like this in ref-filter to fill v->s:
v->s = show_ref(...)
v->s = copy_email(...)
It is very difficult to modify here: We know that show_ref()
or copy_email() will allocate a block of memory to v->s, but
if v->s is a strbuf, what should we do? In copy_email(), we
can pass the v->s to copy_email() and use strbuf_add()/strbuf_addstr()
instead of xstrdup() and xmemdupz(). But show_ref() will call
external functions like shorten_unambiguous_ref(), we don’t know
whether it will return us NULL or a dynamically allocated memory.
If continue to pass v->s to the inner function, it is not a feasible
method. Or we can use strbuf_attach() + strlen(), I'm not sure
this is a good method.
2. See:
- for (i = 0; i < used_atom_cnt; i++) {
+ for (i = 0; i < used_atom_cnt; i++) {
struct atom_value *v = &ref->value[i];
- if (v->s == NULL && used_atom[i].source == SOURCE_NONE)
+ if (v->s.len == 0 && used_atom[i].source == SOURCE_NONE)
return strbuf_addf_ret(err, -1, _("missing
object %s for %s"),
oid_to_hex(&ref->objectname), ref->refname);
}
In the case of using strbuf, I don’t know how to distinguish between an empty
strbuf and NULL. It can be easily distinguished by using c-style "const char*".
Yes, sometimes it's just too much of a hassle, looking at
shorten_unambiguous_ref() which returns a xstrdup()'d value that could
indeed be strbuf_attach'd. I haven't tried the conversion myself,
perhaps it's too much hassle.
Just a suggestion from reading your patch in isolation.
quoted
quoted
Not python safe. See [1].
Regarding the perl language, I support Junio's point of view: it can be
re-supported in the future.
Ah, I'd missed that. Anyway, if it's easy it seems you discovered that
Perl deals with it correctly, so we could just have it support this.
Well, it's ok, support for perl will be put in a separate commit.
Just "echo" will do, ditto for the rest. Also odd to go back and forth
between populating expected.clean & actual.clean.
Are you saying that sanitize_pgp is not needed?
No that instead of:
echo "" >x
You can do:
echo >x
And also that going back and forth between populating different files is
confusing, i.e. this:
echo a >x
echo c >y
echo b >>x
is better as:
echo a >x
echo b >>x
echo c >y
From: Christian Couder <hidden> Date: 2021-06-18 13:47:19
On Fri, Jun 18, 2021 at 12:51 PM Ævar Arnfjörð Bjarmason
[off-list ref] wrote:
On Fri, Jun 18 2021, ZheNing Hu wrote:
quoted
After some refactoring, I found that there are two problems:
1. There are a lot of codes like this in ref-filter to fill v->s:
v->s = show_ref(...)
v->s = copy_email(...)
It is very difficult to modify here: We know that show_ref()
or copy_email() will allocate a block of memory to v->s, but
if v->s is a strbuf, what should we do? In copy_email(), we
can pass the v->s to copy_email() and use strbuf_add()/strbuf_addstr()
instead of xstrdup() and xmemdupz(). But show_ref() will call
external functions like shorten_unambiguous_ref(), we don’t know
whether it will return us NULL or a dynamically allocated memory.
If continue to pass v->s to the inner function, it is not a feasible
method. Or we can use strbuf_attach() + strlen(), I'm not sure
this is a good method.
If you resend this patch, it might be a good idea to add a short
version of the above explanations into the commit message.
[...]
quoted
In the case of using strbuf, I don’t know how to distinguish between an empty
strbuf and NULL. It can be easily distinguished by using c-style "const char*".
Maybe this could also be part of the explanation.
Yes, sometimes it's just too much of a hassle, looking at
shorten_unambiguous_ref() which returns a xstrdup()'d value that could
indeed be strbuf_attach'd. I haven't tried the conversion myself,
perhaps it's too much hassle.
Just a suggestion from reading your patch in isolation.
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:10
From: ZheNing Hu <redacted>
Only tag and commit objects use `grab_sub_body_contents()` to grab
object contents in the current codebase. We want to teach the
function to also handle blobs and trees to get their raw data,
without parsing a blob (whose contents looks like a commit or a tag)
incorrectly as a commit or a tag.
Skip the block of code that is specific to handling commits and tags
early when the given object is of a wrong type to help later
addition to handle other types of objects in this function.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
@@ -1678,7 +1684,7 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objreturnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi->content);+grab_values(ref->value,deref,*obj,oi);}grab_common_values(ref->value,deref,oi);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:13
From: ZheNing Hu <redacted>
Use non-const ref_format in *_atom_parser(), which can help us
modify the members of ref_format in *_atom_parser().
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/tag.c | 2 +-
ref-filter.c | 44 ++++++++++++++++++++++----------------------
ref-filter.h | 4 ++--
3 files changed, 25 insertions(+), 25 deletions(-)
@@ -116,7 +116,7 @@ void ref_array_sort(struct ref_sorting *sort, struct ref_array *array);voidref_sorting_set_sort_flags_all(structref_sorting*sorting,unsignedintmask,inton);/* Based on the given format and quote_style, fill the strbuf */intformat_ref_array_item(structref_array_item*info,-conststructref_format*format,+structref_format*format,structstrbuf*final_buf,structstrbuf*error_buf);/* Parse a single sort specifier and add it to the list */
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:14
From: ZheNing Hu <redacted>
Add new formatting option `%(raw)`, which will print the raw
object data without any changes. It will help further to migrate
all cat-file formatting logic from cat-file to ref-filter.
The raw data of blob, tree objects may contain '\0', but most of
the logic in `ref-filter` depends on the output of the atom being
text (specifically, no embedded NULs in it).
E.g. `quote_formatting()` use `strbuf_addstr()` or `*._quote_buf()`
add the data to the buffer. The raw data of a tree object is
`100644 one\0...`, only the `100644 one` will be added to the buffer,
which is incorrect.
Therefore, we need to find a way to record the length of the
atom_value's member `s`. Although strbuf can already record the
string and its length, if we want to replace the type of atom_value's
member `s` with strbuf, many places in ref-filter that are filled
with dynamically allocated mermory in `v->s` are not easy to replace.
At the same time, we need to check if `v->s == NULL` in
populate_value(), and strbuf cannot easily distinguish NULL and empty
strings, but c-style "const char *" can do it. So add a new member in
`struct atom_value`: `s_size`, which can record raw object size, it
can help us add raw object data to the buffer or compare two buffers
which contain raw object data.
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a
variable in the host language, the host language may not support
arbitrary binary data in the variables of its string type.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Helped-by: Felipe Contreras [off-list ref]
Helped-by: Phillip Wood [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Based-on-patch-by: Olga Telezhnaya [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-for-each-ref.txt | 9 ++
ref-filter.c | 139 +++++++++++++++----
t/t6300-for-each-ref.sh | 216 +++++++++++++++++++++++++++++
3 files changed, 337 insertions(+), 27 deletions(-)
@@ -235,6 +235,15 @@ and `date` to extract the named component. For email fields (`authoremail`, without angle brackets, and `:localpart` to get the part before the `@` symbol out of the trimmed email.+The raw data in an object is `raw`.++raw:size::+ The raw data size of the object.++Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,+`--perl` because the host language may not support arbitrary binary data in the+variables of its string type.+ The message in a commit or a tag object is `contents`, from which `contents:<part>` can be used to extract various parts out of:
@@ -620,12 +637,15 @@ struct ref_formatting_state {structatom_value{constchar*s;+size_ts_size;int(*handler)(structatom_value*atomv,structref_formatting_state*state,structstrbuf*err);uintmax_tvalue;/* used for sorting when not FIELD_STR */structused_atom*atom;};+#define ATOM_VALUE_S_SIZE_INIT (-1)+/**Usedtoparseformatstringandsortspecifiers*/
@@ -644,13 +664,6 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnstrbuf_addf_ret(err,-1,_("malformed field name: %.*s"),(int)(ep-atom),atom);-/* Do we have the atom already used elsewhere? */-for(i=0;i<used_atom_cnt;i++){-intlen=strlen(used_atom[i].name);-if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))-returni;-}-/**Iftheatomnamehasacolon,stripitandeverythingafter*itoff-itspecifiestheformatforthisentry,and
@@ -660,6 +673,13 @@ static int parse_ref_filter_atom(const struct ref_format *format,arg=memchr(sp,':',ep-sp);atom_len=(arg?arg:ep)-sp;+/* Do we have the atom already used elsewhere? */+for(i=0;i<used_atom_cnt;i++){+intlen=strlen(used_atom[i].name);+if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))+returni;+}+/* Is the atom a valid one? */for(i=0;i<ARRAY_SIZE(valid_atom);i++){intlen=strlen(valid_atom[i].name);
@@ -709,11 +729,14 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnat;}-staticvoidquote_formatting(structstrbuf*s,constchar*str,intquote_style)+staticvoidquote_formatting(structstrbuf*s,constchar*str,size_tlen,intquote_style){switch(quote_style){caseQUOTE_NONE:-strbuf_addstr(s,str);+if(len!=ATOM_VALUE_S_SIZE_INIT)+strbuf_add(s,str,len);+else+strbuf_addstr(s,str);break;caseQUOTE_SHELL:sq_quote_buf(s,str);
@@ -867,18 +895,22 @@ static int then_atom_handler(struct atom_value *atomv, struct ref_formatting_staif(if_then_else->else_atom_seen)returnstrbuf_addf_ret(err,-1,_("format: %%(then) atom used after %%(else)"));if_then_else->then_atom_seen=1;+if(if_then_else->str)+str_len=strlen(if_then_else->str);/**Ifthe'equals'or'notequals'attributeisusedthen*performtherequiredcomparison.Ifnot,onlynon-empty*stringssatisfythe'if'condition.*/if(if_then_else->cmp_status==COMPARE_EQUAL){-if(!strcmp(if_then_else->str,cur->output.buf))+if(str_len==cur->output.len&&+!memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;}elseif(if_then_else->cmp_status==COMPARE_UNEQUAL){-if(strcmp(if_then_else->str,cur->output.buf))+if(str_len!=cur->output.len||+memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;-}elseif(cur->output.len&&!is_empty(cur->output.buf))+}elseif(cur->output.len&&!is_empty(&cur->output))if_then_else->condition_satisfied=1;strbuf_reset(&cur->output);return0;
@@ -974,6 +1006,10 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&+used_atom[at].u.raw_data.option==RAW_BARE)+die(_("--format=%.*s cannot be used with"+"--python, --shell, --tcl, --perl"),(int)(ep-sp-2),sp+2);cp=ep+1;if(skip_prefix(used_atom[at].name,"color:",&color))
@@ -708,6 +737,15 @@ test_atom refs/tags/signed-long contents "subject line bodycontents$sig"+test_expect_successGPG'basic atom: refs/tags/signed-long raw''+gitcat-filetagrefs/tags/signed-long>expected&&+gitfor-each-ref--format="%(raw)"refs/tags/signed-long>actual&&+sanitize_pgp<expected>expected.clean&&+echo>>expected.clean&&+sanitize_pgp<actual>actual.clean&&+test_cmpexpected.cleanactual.clean+'+ test_expect_success'set up refs pointing to tree and blob''gitupdate-refrefs/mytrees/firstrefs/heads/main^{tree}&&gitupdate-refrefs/myblobs/firstrefs/heads/main:one
@@ -727,6 +775,174 @@ test_atom refs/myblobs/first contents:body "" test_atomrefs/myblobs/firstcontents:signature"" test_atomrefs/myblobs/firstcontents""+test_expect_success'basic atom: refs/myblobs/first raw''+gitcat-fileblobrefs/myblobs/first>expected&&+echo>>expected&&+gitfor-each-ref--format="%(raw)"refs/myblobs/first>actual&&+test_cmpexpectedactual&&+gitcat-file-srefs/myblobs/first>expected&&+gitfor-each-ref--format="%(raw:size)"refs/myblobs/first>actual&&+test_cmpexpectedactual+'++test_expect_success'set up refs pointing to binary blob''+printf"a\0b\0c">blob1&&+printf"a\0c\0b">blob2&&+printf"\0a\0b\0c">blob3&&+printf"abc">blob4&&+printf"\0 \0 \0 ">blob5&&+printf"\0 \0a\0 ">blob6&&+printf" ">blob7&&+>blob8&&+obj=$(githash-object-wblob1)&&+gitupdate-refrefs/myblobs/blob1"$obj"&&+obj=$(githash-object-wblob2)&&+gitupdate-refrefs/myblobs/blob2"$obj"&&+obj=$(githash-object-wblob3)&&+gitupdate-refrefs/myblobs/blob3"$obj"&&+obj=$(githash-object-wblob4)&&+gitupdate-refrefs/myblobs/blob4"$obj"&&+obj=$(githash-object-wblob5)&&+gitupdate-refrefs/myblobs/blob5"$obj"&&+obj=$(githash-object-wblob6)&&+gitupdate-refrefs/myblobs/blob6"$obj"&&+obj=$(githash-object-wblob7)&&+gitupdate-refrefs/myblobs/blob7"$obj"&&+obj=$(githash-object-wblob8)&&+gitupdate-refrefs/myblobs/blob8"$obj"+'++test_expect_success'Verify sorts with raw''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob3+refs/myblobs/blob7+refs/mytrees/first+refs/myblobs/first+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob4+refs/heads/main+EOF+gitfor-each-ref--format="%(refname)"--sort=raw\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'Verify sorts with raw:size''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/first+refs/myblobs/blob7+refs/heads/main+refs/myblobs/blob4+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+refs/myblobs/blob5+refs/myblobs/blob6+refs/mytrees/first+EOF+gitfor-each-ref--format="%(refname)"--sort=raw:size\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'validate raw atom with %(if:equals)''+cat>expected<<-EOF&&+notequals+notequals+notequals+notequals+notequals+notequals+refs/myblobs/blob4+notequals+notequals+notequals+notequals+notequals+EOF+gitfor-each-ref--format="%(if:equals=abc)%(raw)%(then)%(refname)%(else)not equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'++test_expect_success'validate raw atom with %(if:notequals)''+cat>expected<<-EOF&&+refs/heads/ambiguous+refs/heads/main+refs/heads/newtag+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+equals+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob7+refs/myblobs/blob8+refs/myblobs/first+EOF+gitfor-each-ref--format="%(if:notequals=abc)%(raw)%(then)%(refname)%(else)equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'++test_expect_success'empty raw refs with %(if)''+cat>expected<<-EOF&&+refs/myblobs/blob1notempty+refs/myblobs/blob2notempty+refs/myblobs/blob3notempty+refs/myblobs/blob4notempty+refs/myblobs/blob5notempty+refs/myblobs/blob6notempty+refs/myblobs/blob7empty+refs/myblobs/blob8empty+refs/myblobs/firstnotempty+EOF+gitfor-each-ref--format="%(refname) %(if)%(raw)%(then)not empty%(else)empty%(end)"\+refs/myblobs/>actual&&+test_cmpexpectedactual+'++test_expect_success'%(raw) with --python must fail''+test_must_failgitfor-each-ref--format="%(raw)"--python+'++test_expect_success'%(raw) with --tcl must fail''+test_must_failgitfor-each-ref--format="%(raw)"--tcl+'++test_expect_success'%(raw) with --perl must fail''+test_must_failgitfor-each-ref--format="%(raw)"--perl+'++test_expect_success'%(raw) with --shell must fail''+test_must_failgitfor-each-ref--format="%(raw)"--shell+'++test_expect_success'%(raw) with --shell and --sort=raw must fail''+test_must_failgitfor-each-ref--format="%(raw)"--sort=raw--shell+'++test_expect_success'%(raw:size) with --shell''+gitfor-each-ref--format="%(raw:size)"|whilereadline+do+echo"'\''$line'\''">>expect+done&&+gitfor-each-ref--format="%(raw:size)"--shell>actual&&+test_cmpexpectactual+'++test_expect_success'for-each-ref --format compare with cat-file --batch''+gitrev-parserefs/mytrees/first|gitcat-file--batch>expected&&+gitfor-each-ref--format="%(objectname) %(objecttype) %(objectsize)+%(raw)" refs/mytrees/first >actual &&+test_cmpexpectedactual+'+ test_expect_success'set up multiple-sort tags''forwhenin100000200000do
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:16
From: ZheNing Hu <redacted>
When we use ref_array_item which is not dynamically allocated and
want to free the space of its member "value" after the end of use,
free_array_item() does not meet our needs, because it tries to free
ref_array_item itself and its member "symref".
Introduce free_ref_array_item_value() for freeing ref_array_item value.
It will be called internally by free_array_item(), and it will help
`cat-file --batch` free ref_array_item's value memory later.
Helped-by: Junio C Hamano [off-list ref]
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 11 ++++++++---
ref-filter.h | 2 ++
2 files changed, 10 insertions(+), 3 deletions(-)
@@ -111,6 +111,8 @@ struct ref_format {intfilter_refs(structref_array*array,structref_filter*filter,unsignedinttype);/* Clear all memory allocated to ref_array */voidref_array_clear(structref_array*array);+/* Free ref_array_item's value */+voidfree_ref_array_item_value(structref_array_item*item);/* Used to verify if the given format is correct and to parse out the used atoms */intverify_ref_format(structref_format*format);/* Sort the given ref_array as per the ref_sorting provided */
@@ -559,6 +560,15 @@ static int if_atom_parser(struct ref_format *format, struct used_atom *atom,return0;}+staticintrest_atom_parser(structref_format*format,structused_atom*atom,+constchar*arg,structstrbuf*err)+{+if(arg)+returnstrbuf_addf_ret(err,-1,_("%%(rest) does not take arguments"));+format->use_rest=1;+return0;+}+staticinthead_atom_parser(structref_format*format,structused_atom*atom,constchar*arg,structstrbuf*unused_err){
@@ -1998,6 +1998,10 @@ test_expect_success '--format should list tags as per format given' 'test_cmpexpectactual'+test_expect_success'git tag -l with --format="%(rest)" must fail''+test_must_failgittag-l--format="%(rest)""v1*"+'+ test_expect_success"set up color tests"'echo"<RED>v1.0<RESET>">expect.color&&echo"v1.0">expect.bare&&
@@ -194,6 +194,10 @@ test_expect_success GPG 'verifying tag with --format' 'test_cmpexpectactual'+test_expect_successGPG'verifying tag with --format="%(rest)" must fail''+test_must_failgitverify-tag--format="%(rest)""fourth-signed"+'+ test_expect_successGPG'verifying a forged tag with --format should fail silently''test_must_failgitverify-tag--format="tagname : %(tag)"$(catforged1.tag)>actual-forged&&test_must_be_emptyactual-forged
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:19
From: ZheNing Hu <redacted>
Since in the refactor of `git cat-file --batch` later,
oid_object_info_extended() in get_object() will be used to obtain
the info of an object with it's oid. When the object cannot be
obtained in the git repository, `cat-file --batch` expects to output
"<oid> missing" and continue the next oid query instead of letting
Git exit. In other error conditions, Git should exit normally. So we
can achieve this function by passing the return value of get_object().
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:21
From: ZheNing Hu <redacted>
Reuse the `err` buffer in batch_object_write(), as the
buffer `scratch` does. This will reduce the overhead
of multiple allocations of memory of the err buffer.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 22 ++++++++++++++--------
1 file changed, 14 insertions(+), 8 deletions(-)
@@ -212,32 +212,33 @@ static void batch_write(struct batch_options *opt, const void *data, int len)staticvoidbatch_object_write(constchar*obj_name,structstrbuf*scratch,+structstrbuf*err,structbatch_options*opt,structexpand_data*data){intret=0;-structstrbuferr=STRBUF_INIT;structref_array_itemitem={data->oid,data->rest};strbuf_reset(scratch);+strbuf_reset(err);-ret=format_ref_array_item(&item,&opt->format,scratch,&err);+ret=format_ref_array_item(&item,&opt->format,scratch,err);if(!ret){strbuf_addch(scratch,'\n');batch_write(opt,scratch->buf,scratch->len);}elseif(ret<0){-die("%s\n",err.buf);+die("%s\n",err->buf);}else{/* when ret > 0 , don't call die and print the err to stdout*/-printf("%s\n",err.buf);+printf("%s\n",err->buf);fflush(stdout);}free_ref_array_item_value(&item);-strbuf_release(&err);}staticvoidbatch_one_object(constchar*obj_name,structstrbuf*scratch,+structstrbuf*err,structbatch_options*opt,structexpand_data*data){
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:31
From: ZheNing Hu <redacted>
In order to let cat-file use ref-filter logic, the following
methods are used:
1. Add `cat_file_mode` member in struct `ref_format`, this can
help us reject atoms in verify_ref_format() which cat-file
cannot use, e.g. `%(refname)`, `%(push)`, `%(upstream)`...
2. Change the type of member `format` in struct `batch_options`
to `ref_format`, We can add format data in it.
3. Let `batch_objects()` add atoms to format, and use
`verify_ref_format()` to check atoms.
4. Use `has_object_file()` in `batch_one_object()` to check
whether the input object exists.
5. Let get_object() return 1 and print "<oid> missing" instead
of returning -1 and printing "missing object <oid> for <refname>",
this can help `format_ref_array_item()` just report that the
object is missing without letting Git exit.
6. Use `format_ref_array_item()` in `batch_object_write()` to
get the formatted data corresponding to the object. If the
return value of `format_ref_array_item()` is equals to zero,
use `batch_write()` to print object data; else if the return
value less than zero, use `die()` to print the error message
and exit; else return value greater than zero, only print the
error message, but not exit.
7. Use free_ref_array_item_value() to free ref_array_item's
value.
Most of the atoms in `for-each-ref --format` are now supported,
such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,
`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:
`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,
`%(flag)`, `%(HEAD)`, because our objects don't have refname.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-cat-file.txt | 6 +
builtin/cat-file.c | 248 +++++++-------------------------
ref-filter.c | 15 +-
ref-filter.h | 3 +-
t/t1006-cat-file.sh | 252 +++++++++++++++++++++++++++++++++
t/t6301-for-each-ref-errors.sh | 2 +-
6 files changed, 323 insertions(+), 203 deletions(-)
@@ -226,6 +226,12 @@ newline. The available atoms are: after that first run of whitespace (i.e., the "rest" of the line) are output in place of the `%(rest)` atom.+Note that most of the atoms in `for-each-ref --format` are now supported,+such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,+`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:+`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,+`%(flag)`, `%(HEAD)`. See linkgit:git-for-each-ref[1].+ If no format is specified, the default format is `%(objectname) %(objecttype) %(objectsize)`.
@@ -25,7 +26,7 @@ struct batch_options {intall_objects;intunordered;intcmdmode;/* may be 'w' or 'c' for --filters or --textconv */-constchar*format;+structref_formatformat;};staticconstchar*force_path;
@@ -195,99 +196,10 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name,structexpand_data{structobject_idoid;-enumobject_typetype;-unsignedlongsize;-off_tdisk_size;constchar*rest;-structobject_iddelta_base_oid;--/*-*Ifmark_queryistrue,wedonotexpandanything,butrather-*justmarktheobject_infowithitemswewishtoquery.-*/-intmark_query;--/*-*Whethertosplittheinputonwhitespacebeforefeedingitto-*get_sha1;thisisdecidedduringthemark_queryphasebasedon-*whetherwehavea%(rest)tokeninourformat.-*/intsplit_on_whitespace;--/*-*Afteramark_queryrun,thisobject_infoissetuptobe-*passedtooid_object_info_extended.Itwillpointtothedata-*elementsabove,soyoucanretrievetheresponsefromthere.-*/-structobject_infoinfo;--/*-*Thisflagwillbetrueiftherequestedbatchformatandoptions-*don'trequireustocalloid_object_info,whichcanthenbe-*optimizedout.-*/-unsignedskip_object_info:1;};-staticintis_atom(constchar*atom,constchar*s,intslen)-{-intalen=strlen(atom);-returnalen==slen&&!memcmp(atom,s,alen);-}--staticvoidexpand_atom(structstrbuf*sb,constchar*atom,intlen,-void*vdata)-{-structexpand_data*data=vdata;--if(is_atom("objectname",atom,len)){-if(!data->mark_query)-strbuf_addstr(sb,oid_to_hex(&data->oid));-}elseif(is_atom("objecttype",atom,len)){-if(data->mark_query)-data->info.typep=&data->type;-else-strbuf_addstr(sb,type_name(data->type));-}elseif(is_atom("objectsize",atom,len)){-if(data->mark_query)-data->info.sizep=&data->size;-else-strbuf_addf(sb,"%"PRIuMAX,(uintmax_t)data->size);-}elseif(is_atom("objectsize:disk",atom,len)){-if(data->mark_query)-data->info.disk_sizep=&data->disk_size;-else-strbuf_addf(sb,"%"PRIuMAX,(uintmax_t)data->disk_size);-}elseif(is_atom("rest",atom,len)){-if(data->mark_query)-data->split_on_whitespace=1;-elseif(data->rest)-strbuf_addstr(sb,data->rest);-}elseif(is_atom("deltabase",atom,len)){-if(data->mark_query)-data->info.delta_base_oid=&data->delta_base_oid;-else-strbuf_addstr(sb,-oid_to_hex(&data->delta_base_oid));-}else-die("unknown format element: %.*s",len,atom);-}--staticsize_texpand_format(structstrbuf*sb,constchar*start,void*data)-{-constchar*end;--if(*start!='(')-return0;-end=strchr(start+1,')');-if(!end)-die("format element '%s' does not end in ')'",start);--expand_atom(sb,start+1,end-start-1,data);--returnend-start+1;-}-staticvoidbatch_write(structbatch_options*opt,constvoid*data,intlen){if(opt->buffer_output){
@@ -297,87 +209,31 @@ static void batch_write(struct batch_options *opt, const void *data, int len)write_or_die(1,data,len);}-staticvoidprint_object_or_die(structbatch_options*opt,structexpand_data*data)-{-conststructobject_id*oid=&data->oid;--assert(data->info.typep);--if(data->type==OBJ_BLOB){-if(opt->buffer_output)-fflush(stdout);-if(opt->cmdmode){-char*contents;-unsignedlongsize;--if(!data->rest)-die("missing path for '%s'",oid_to_hex(oid));--if(opt->cmdmode=='w'){-if(filter_object(data->rest,0100644,oid,-&contents,&size))-die("could not convert '%s' %s",-oid_to_hex(oid),data->rest);-}elseif(opt->cmdmode=='c'){-enumobject_typetype;-if(!textconv_object(the_repository,-data->rest,0100644,oid,-1,&contents,&size))-contents=read_object_file(oid,-&type,-&size);-if(!contents)-die("could not convert '%s' %s",-oid_to_hex(oid),data->rest);-}else-BUG("invalid cmdmode: %c",opt->cmdmode);-batch_write(opt,contents,size);-free(contents);-}else{-stream_blob(oid);-}-}-else{-enumobject_typetype;-unsignedlongsize;-void*contents;--contents=read_object_file(oid,&type,&size);-if(!contents)-die("object %s disappeared",oid_to_hex(oid));-if(type!=data->type)-die("object %s changed type!?",oid_to_hex(oid));-if(data->info.sizep&&size!=data->size)-die("object %s changed size!?",oid_to_hex(oid));--batch_write(opt,contents,size);-free(contents);-}-}staticvoidbatch_object_write(constchar*obj_name,structstrbuf*scratch,structbatch_options*opt,structexpand_data*data){-if(!data->skip_object_info&&-oid_object_info_extended(the_repository,&data->oid,&data->info,-OBJECT_INFO_LOOKUP_REPLACE)<0){-printf("%s missing\n",-obj_name?obj_name:oid_to_hex(&data->oid));-fflush(stdout);-return;-}+intret=0;+structstrbuferr=STRBUF_INIT;+structref_array_itemitem={data->oid,data->rest};strbuf_reset(scratch);-strbuf_expand(scratch,opt->format,expand_format,data);-strbuf_addch(scratch,'\n');-batch_write(opt,scratch->buf,scratch->len);-if(opt->print_contents){-print_object_or_die(opt,data);-batch_write(opt,"\n",1);+ret=format_ref_array_item(&item,&opt->format,scratch,&err);+if(!ret){+strbuf_addch(scratch,'\n');+batch_write(opt,scratch->buf,scratch->len);+}elseif(ret<0){+die("%s\n",err.buf);+}else{+/* when ret > 0 , don't call die and print the err to stdout*/+printf("%s\n",err.buf);+fflush(stdout);}+free_ref_array_item_value(&item);+strbuf_release(&err);}staticvoidbatch_one_object(constchar*obj_name,
@@ -713,7 +567,7 @@ int cmd_cat_file(int argc, const char **argv, const char *prefix)batch.buffer_output=batch.all_objects;if(batch.enabled)-returnbatch_objects(&batch);+returnbatch_objects(&batch,options);if(unknown_type&&opt!='t'&&opt!='s')die("git cat-file --allow-unknown-type: use with -s or -t");
@@ -1021,8 +1021,15 @@ int verify_ref_format(struct ref_format *format)if(at<0)die("%s",err.buf);-if(used_atom[at].atom_type==ATOM_REST)-die("this command reject atom %%(%.*s)",(int)(ep-sp-2),sp+2);+if((!format->cat_file_mode&&used_atom[at].atom_type==ATOM_REST)||+(format->cat_file_mode&&(used_atom[at].atom_type==ATOM_FLAG||+used_atom[at].atom_type==ATOM_HEAD||+used_atom[at].atom_type==ATOM_PUSH||+used_atom[at].atom_type==ATOM_REFNAME||+used_atom[at].atom_type==ATOM_SYMREF||+used_atom[at].atom_type==ATOM_UPSTREAM||+used_atom[at].atom_type==ATOM_WORKTREEPATH)))+die(_("this command reject atom %%(%.*s)"),(int)(ep-sp-2),sp+2);if((format->quote_style==QUOTE_PYTHON||format->quote_style==QUOTE_SHELL||
@@ -1742,8 +1749,8 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj}if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,OBJECT_INFO_LOOKUP_REPLACE))-returnstrbuf_addf_ret(err,-1,_("missing object %s for %s"),-oid_to_hex(&oi->oid),ref->refname);+returnstrbuf_addf_ret(err,1,_("%s missing"),+oid_to_hex(&oi->oid));if(oi->info.disk_sizep&&oi->disk_size<0)BUG("Object size is less than zero.");
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:33
From: ZheNing Hu <redacted>
Because the perl language can handle binary data correctly,
add the function perl_quote_buf_with_len(), which can specify
the length of the data and prevent the data from being truncated
at '\0' to help `--format="%(raw)"` re-support `--perl`.
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-for-each-ref.txt | 2 +-
quote.c | 17 +++++++++++++++++
quote.h | 1 +
ref-filter.c | 15 +++++++++++----
t/t6300-for-each-ref.sh | 19 +++++++++++++++++--
5 files changed, 47 insertions(+), 7 deletions(-)
@@ -241,7 +241,7 @@ raw:size:: The raw data size of the object. Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,-`--perl` because the host language may not support arbitrary binary data in the+because the host language may not support arbitrary binary data in the variables of its string type. The message in a commit or a tag object is `contents`, from which
@@ -94,6 +94,7 @@ char *quote_path(const char *in, const char *prefix, struct strbuf *out, unsigne/* quoting as a string literal for other languages */voidperl_quote_buf(structstrbuf*sb,constchar*src);+voidperl_quote_buf_with_len(structstrbuf*sb,constchar*src,size_tlen);voidpython_quote_buf(structstrbuf*sb,constchar*src);voidtcl_quote_buf(structstrbuf*sb,constchar*src);voidbasic_regex_quote_buf(structstrbuf*sb,constchar*src);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-19 07:03:33
From: ZheNing Hu <redacted>
After cat-file reuses the ref-filter logic, we re-implement the
functions of --textconv and --filters options.
Add members `use_textconv` and `use_filters` in struct `ref_format`,
and use global variables `use_filters` and `use_textconv` in
`ref-filter.c`, so that we can filter the content of the object
in get_object(). Use `actual_oi` to record the real expand_data:
it may point to the original `oi` or the `act_oi` processed by
`textconv_object()` or `convert_to_working_tree()`. `grab_values()`
will grab the contents of `actual_oi` and `grab_common_values()`
to grab the contents of origin `oi`, this ensures that `%(objectsize)`
still uses the size of the unfiltered data.
In `get_object()`, we made an optimization: Firstly, get the size and
type of the object instead of directly getting the object data.
If using --textconv, after successfully obtaining the filtered object
data, an extra oid_object_info_extended() will be skipped, which can
reduce the cost of object data copy; If using --filter, the data of
the object first will be getted first, and then convert_to_working_tree()
will be used to get the filtered object data.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 5 ++++
ref-filter.c | 59 ++++++++++++++++++++++++++++++++++++++++++++--
ref-filter.h | 4 +++-
3 files changed, 65 insertions(+), 3 deletions(-)
@@ -1031,6 +1035,9 @@ int verify_ref_format(struct ref_format *format)used_atom[at].atom_type==ATOM_WORKTREEPATH)))die(_("this command reject atom %%(%.*s)"),(int)(ep-sp-2),sp+2);+use_filters=format->use_filters;+use_textconv=format->use_textconv;+if((format->quote_style==QUOTE_PYTHON||format->quote_style==QUOTE_SHELL||format->quote_style==QUOTE_TCL)&&
@@ -1742,10 +1749,38 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj{/* parse_object_buffer() will set eaten to 0 if free() will be needed */inteaten=1;+structexpand_data*actual_oi=oi;+structexpand_dataact_oi={0};+if(oi->info.contentp){/* We need to know that to use parse_object_buffer properly */+void**temp_contentp=oi->info.contentp;+oi->info.contentp=NULL;oi->info.sizep=&oi->size;oi->info.typep=&oi->type;++/* get the type and size */+if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,+OBJECT_INFO_LOOKUP_REPLACE))+returnstrbuf_addf_ret(err,1,_("%s missing"),+oid_to_hex(&oi->oid));++oi->info.sizep=NULL;+oi->info.typep=NULL;+oi->info.contentp=temp_contentp;++if(use_textconv&&!ref->rest)+returnstrbuf_addf_ret(err,-1,_("missing path for '%s'"),+oid_to_hex(&act_oi.oid));+if(use_textconv&&oi->type==OBJ_BLOB){+act_oi=*oi;+if(textconv_object(the_repository,+ref->rest,0100644,&act_oi.oid,+1,(char**)(&act_oi.content),&act_oi.size)){+actual_oi=&act_oi;+gotosuccess;+}+}}if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,OBJECT_INFO_LOOKUP_REPLACE))
@@ -1755,19 +1790,39 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objBUG("Object size is less than zero.");if(oi->info.contentp){-*obj=parse_object_buffer(the_repository,&oi->oid,oi->type,oi->size,oi->content,&eaten);+if(use_filters&&!ref->rest)+returnstrbuf_addf_ret(err,-1,_("missing path for '%s'"),+oid_to_hex(&oi->oid));+if(use_filters&&oi->type==OBJ_BLOB){+structstrbufstrbuf=STRBUF_INIT;+structcheckout_metadatameta;+act_oi=*oi;++init_checkout_metadata(&meta,NULL,NULL,&act_oi.oid);+if(!convert_to_working_tree(&the_index,ref->rest,act_oi.content,act_oi.size,&strbuf,&meta))+die("could not convert '%s' %s",+oid_to_hex(&oi->oid),ref->rest);+act_oi.size=strbuf.len;+act_oi.content=strbuf_detach(&strbuf,NULL);+actual_oi=&act_oi;+}++success:+*obj=parse_object_buffer(the_repository,&actual_oi->oid,actual_oi->type,actual_oi->size,actual_oi->content,&eaten);if(!*obj){if(!eaten)free(oi->content);returnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi);+grab_values(ref->value,deref,*obj,actual_oi);}grab_common_values(ref->value,deref,oi);if(!eaten)free(oi->content);+if(actual_oi!=oi)+free(actual_oi->content);return0;}
From: Christian Couder <hidden> Date: 2021-06-21 05:55:57
On Sat, Jun 19, 2021 at 9:03 AM ZheNing Hu via GitGitGadget
[off-list ref] wrote:
From: ZheNing Hu <redacted>
In order to let cat-file use ref-filter logic, the following
methods are used:
Maybe: s/the following methods are used/let's do the following/
1. Add `cat_file_mode` member in struct `ref_format`, this can
help us reject atoms in verify_ref_format() which cat-file
cannot use, e.g. `%(refname)`, `%(push)`, `%(upstream)`...
2. Change the type of member `format` in struct `batch_options`
to `ref_format`, We can add format data in it.
Not sure what "We can add format data in it." means.
3. Let `batch_objects()` add atoms to format, and use
`verify_ref_format()` to check atoms.
4. Use `has_object_file()` in `batch_one_object()` to check
whether the input object exists.
5. Let get_object() return 1 and print "<oid> missing" instead
of returning -1 and printing "missing object <oid> for <refname>",
this can help `format_ref_array_item()` just report that the
object is missing without letting Git exit.
6. Use `format_ref_array_item()` in `batch_object_write()` to
get the formatted data corresponding to the object. If the
return value of `format_ref_array_item()` is equals to zero,
use `batch_write()` to print object data; else if the return
value less than zero, use `die()` to print the error message
and exit; else return value greater than zero, only print the
s/else return value greater/else if the return value is greater/
error message, but not exit.
s/not exit/don't exit/
7. Use free_ref_array_item_value() to free ref_array_item's
value.
That looks like a lot of changes in a single commit. I wonder if this
commit could be split.
Most of the atoms in `for-each-ref --format` are now supported,
such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,
`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:
`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,
`%(flag)`, `%(HEAD)`, because our objects don't have refname.
s/refname/a refname/
It might be worth talking a bit about possible performance changes.
[...]
+ ret = format_ref_array_item(&item, &opt->format, scratch, &err);
+ if (!ret) {
+ strbuf_addch(scratch, '\n');
+ batch_write(opt, scratch->buf, scratch->len);
+ } else if (ret < 0) {
+ die("%s\n", err.buf);
This if (ret < 0) could be checked first.
+ } else {
+ /* when ret > 0 , don't call die and print the err to stdout*/
I think it would be more helpful to tell what ret > 0 means, rather
than what we do below (which can easily be seen).
+ printf("%s\n", err.buf);
+ fflush(stdout);
}
For example:
if (ret < 0) {
die("%s\n", err.buf);
if (ret) {
/* ret > 0 means ... */
printf("%s\n", err.buf);
fflush(stdout);
} else {
strbuf_addch(scratch, '\n');
batch_write(opt, scratch->buf, scratch->len);
}
+static int batch_objects(struct batch_options *opt, const struct option *options)
It's unfortunate that one argument is called "opt" and the other one
"options". I wonder if the first one could be called "batch" as it
seems to be called this way somewhere else.
From: ZheNing Hu <hidden> Date: 2021-06-21 13:05:46
Christian Couder [off-list ref] 于2021年6月21日周一 下午1:55写道：
quoted
1. Add `cat_file_mode` member in struct `ref_format`, this can
help us reject atoms in verify_ref_format() which cat-file
cannot use, e.g. `%(refname)`, `%(push)`, `%(upstream)`...
2. Change the type of member `format` in struct `batch_options`
to `ref_format`, We can add format data in it.
Not sure what "We can add format data in it." means.
Well, there is something wrong with the expression here. What I want
to express is that we can fill its member "format" with the atoms like
"%(objectname) %(refname)", and then pass it to the ref-filter.
quoted
7. Use free_ref_array_item_value() to free ref_array_item's
value.
That looks like a lot of changes in a single commit. I wonder if this
commit could be split.
Yeah, But I don’t know if I should take it apart step by step, If taken apart,
those intermediate commits are likely to fail the test.
quoted
Most of the atoms in `for-each-ref --format` are now supported,
such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,
`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:
`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,
`%(flag)`, `%(HEAD)`, because our objects don't have refname.
s/refname/a refname/
It might be worth talking a bit about possible performance changes.
Makes sense. The performance has indeed deteriorated.
[...]
quoted
+ ret = format_ref_array_item(&item, &opt->format, scratch, &err);
+ if (!ret) {
+ strbuf_addch(scratch, '\n');
+ batch_write(opt, scratch->buf, scratch->len);
+ } else if (ret < 0) {
+ die("%s\n", err.buf);
This if (ret < 0) could be checked first.
Yes, it is better to put error checking first.
quoted
+ } else {
+ /* when ret > 0 , don't call die and print the err to stdout*/
I think it would be more helpful to tell what ret > 0 means, rather
than what we do below (which can easily be seen).
Ah, There is indeed only one situation for ret > 0 for the time being:
Show "<oid> missing" without exiting Git.
quoted
+ printf("%s\n", err.buf);
+ fflush(stdout);
}
For example:
if (ret < 0) {
die("%s\n", err.buf);
if (ret) {
/* ret > 0 means ... */
printf("%s\n", err.buf);
fflush(stdout);
} else {
strbuf_addch(scratch, '\n');
batch_write(opt, scratch->buf, scratch->len);
}
+static int batch_objects(struct batch_options *opt, const struct option *options)
It's unfortunate that one argument is called "opt" and the other one
"options". I wonder if the first one could be called "batch" as it
seems to be called this way somewhere else.
I wonder if this should be:
opt->format.format = strbuf_detach(&format, NULL);
No. here our opt->format.format will not be changed, it would
be better for us to use `strbuf_release(&format)` for resource
recovery. (strbuf_detach() will forced to let us free opt->format.format)
quoted
+ if (verify_ref_format(&opt->format))
+ usage_with_options(cat_file_usage, options);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:04
From: ZheNing Hu <redacted>
Only tag and commit objects use `grab_sub_body_contents()` to grab
object contents in the current codebase. We want to teach the
function to also handle blobs and trees to get their raw data,
without parsing a blob (whose contents looks like a commit or a tag)
incorrectly as a commit or a tag.
Skip the block of code that is specific to handling commits and tags
early when the given object is of a wrong type to help later
addition to handle other types of objects in this function.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
@@ -1678,7 +1684,7 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objreturnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi->content);+grab_values(ref->value,deref,*obj,oi);}grab_common_values(ref->value,deref,oi);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:04
From: ZheNing Hu <redacted>
Because the perl language can handle binary data correctly,
add the function perl_quote_buf_with_len(), which can specify
the length of the data and prevent the data from being truncated
at '\0' to help `--format="%(raw)"` re-support `--perl`.
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-for-each-ref.txt | 2 +-
quote.c | 17 +++++++++++++++++
quote.h | 1 +
ref-filter.c | 15 +++++++++++----
t/t6300-for-each-ref.sh | 19 +++++++++++++++++--
5 files changed, 47 insertions(+), 7 deletions(-)
@@ -241,7 +241,7 @@ raw:size:: The raw data size of the object. Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,-`--perl` because the host language may not support arbitrary binary data in the+because the host language may not support arbitrary binary data in the variables of its string type. The message in a commit or a tag object is `contents`, from which
@@ -94,6 +94,7 @@ char *quote_path(const char *in, const char *prefix, struct strbuf *out, unsigne/* quoting as a string literal for other languages */voidperl_quote_buf(structstrbuf*sb,constchar*src);+voidperl_quote_buf_with_len(structstrbuf*sb,constchar*src,size_tlen);voidpython_quote_buf(structstrbuf*sb,constchar*src);voidtcl_quote_buf(structstrbuf*sb,constchar*src);voidbasic_regex_quote_buf(structstrbuf*sb,constchar*src);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:04
From: ZheNing Hu <redacted>
Add new formatting option `%(raw)`, which will print the raw
object data without any changes. It will help further to migrate
all cat-file formatting logic from cat-file to ref-filter.
The raw data of blob, tree objects may contain '\0', but most of
the logic in `ref-filter` depends on the output of the atom being
text (specifically, no embedded NULs in it).
E.g. `quote_formatting()` use `strbuf_addstr()` or `*._quote_buf()`
add the data to the buffer. The raw data of a tree object is
`100644 one\0...`, only the `100644 one` will be added to the buffer,
which is incorrect.
Therefore, we need to find a way to record the length of the
atom_value's member `s`. Although strbuf can already record the
string and its length, if we want to replace the type of atom_value's
member `s` with strbuf, many places in ref-filter that are filled
with dynamically allocated mermory in `v->s` are not easy to replace.
At the same time, we need to check if `v->s == NULL` in
populate_value(), and strbuf cannot easily distinguish NULL and empty
strings, but c-style "const char *" can do it. So add a new member in
`struct atom_value`: `s_size`, which can record raw object size, it
can help us add raw object data to the buffer or compare two buffers
which contain raw object data.
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a
variable in the host language, the host language may not support
arbitrary binary data in the variables of its string type.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Helped-by: Felipe Contreras [off-list ref]
Helped-by: Phillip Wood [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Based-on-patch-by: Olga Telezhnaya [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-for-each-ref.txt | 9 ++
ref-filter.c | 139 +++++++++++++++----
t/t6300-for-each-ref.sh | 216 +++++++++++++++++++++++++++++
3 files changed, 337 insertions(+), 27 deletions(-)
@@ -235,6 +235,15 @@ and `date` to extract the named component. For email fields (`authoremail`, without angle brackets, and `:localpart` to get the part before the `@` symbol out of the trimmed email.+The raw data in an object is `raw`.++raw:size::+ The raw data size of the object.++Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,+`--perl` because the host language may not support arbitrary binary data in the+variables of its string type.+ The message in a commit or a tag object is `contents`, from which `contents:<part>` can be used to extract various parts out of:
@@ -620,12 +637,15 @@ struct ref_formatting_state {structatom_value{constchar*s;+size_ts_size;int(*handler)(structatom_value*atomv,structref_formatting_state*state,structstrbuf*err);uintmax_tvalue;/* used for sorting when not FIELD_STR */structused_atom*atom;};+#define ATOM_VALUE_S_SIZE_INIT (-1)+/**Usedtoparseformatstringandsortspecifiers*/
@@ -644,13 +664,6 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnstrbuf_addf_ret(err,-1,_("malformed field name: %.*s"),(int)(ep-atom),atom);-/* Do we have the atom already used elsewhere? */-for(i=0;i<used_atom_cnt;i++){-intlen=strlen(used_atom[i].name);-if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))-returni;-}-/**Iftheatomnamehasacolon,stripitandeverythingafter*itoff-itspecifiestheformatforthisentry,and
@@ -660,6 +673,13 @@ static int parse_ref_filter_atom(const struct ref_format *format,arg=memchr(sp,':',ep-sp);atom_len=(arg?arg:ep)-sp;+/* Do we have the atom already used elsewhere? */+for(i=0;i<used_atom_cnt;i++){+intlen=strlen(used_atom[i].name);+if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))+returni;+}+/* Is the atom a valid one? */for(i=0;i<ARRAY_SIZE(valid_atom);i++){intlen=strlen(valid_atom[i].name);
@@ -709,11 +729,14 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnat;}-staticvoidquote_formatting(structstrbuf*s,constchar*str,intquote_style)+staticvoidquote_formatting(structstrbuf*s,constchar*str,size_tlen,intquote_style){switch(quote_style){caseQUOTE_NONE:-strbuf_addstr(s,str);+if(len!=ATOM_VALUE_S_SIZE_INIT)+strbuf_add(s,str,len);+else+strbuf_addstr(s,str);break;caseQUOTE_SHELL:sq_quote_buf(s,str);
@@ -867,18 +895,22 @@ static int then_atom_handler(struct atom_value *atomv, struct ref_formatting_staif(if_then_else->else_atom_seen)returnstrbuf_addf_ret(err,-1,_("format: %%(then) atom used after %%(else)"));if_then_else->then_atom_seen=1;+if(if_then_else->str)+str_len=strlen(if_then_else->str);/**Ifthe'equals'or'notequals'attributeisusedthen*performtherequiredcomparison.Ifnot,onlynon-empty*stringssatisfythe'if'condition.*/if(if_then_else->cmp_status==COMPARE_EQUAL){-if(!strcmp(if_then_else->str,cur->output.buf))+if(str_len==cur->output.len&&+!memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;}elseif(if_then_else->cmp_status==COMPARE_UNEQUAL){-if(strcmp(if_then_else->str,cur->output.buf))+if(str_len!=cur->output.len||+memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;-}elseif(cur->output.len&&!is_empty(cur->output.buf))+}elseif(cur->output.len&&!is_empty(&cur->output))if_then_else->condition_satisfied=1;strbuf_reset(&cur->output);return0;
@@ -974,6 +1006,10 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&+used_atom[at].u.raw_data.option==RAW_BARE)+die(_("--format=%.*s cannot be used with"+"--python, --shell, --tcl, --perl"),(int)(ep-sp-2),sp+2);cp=ep+1;if(skip_prefix(used_atom[at].name,"color:",&color))
@@ -708,6 +737,15 @@ test_atom refs/tags/signed-long contents "subject line bodycontents$sig"+test_expect_successGPG'basic atom: refs/tags/signed-long raw''+gitcat-filetagrefs/tags/signed-long>expected&&+gitfor-each-ref--format="%(raw)"refs/tags/signed-long>actual&&+sanitize_pgp<expected>expected.clean&&+echo>>expected.clean&&+sanitize_pgp<actual>actual.clean&&+test_cmpexpected.cleanactual.clean+'+ test_expect_success'set up refs pointing to tree and blob''gitupdate-refrefs/mytrees/firstrefs/heads/main^{tree}&&gitupdate-refrefs/myblobs/firstrefs/heads/main:one
@@ -727,6 +775,174 @@ test_atom refs/myblobs/first contents:body "" test_atomrefs/myblobs/firstcontents:signature"" test_atomrefs/myblobs/firstcontents""+test_expect_success'basic atom: refs/myblobs/first raw''+gitcat-fileblobrefs/myblobs/first>expected&&+echo>>expected&&+gitfor-each-ref--format="%(raw)"refs/myblobs/first>actual&&+test_cmpexpectedactual&&+gitcat-file-srefs/myblobs/first>expected&&+gitfor-each-ref--format="%(raw:size)"refs/myblobs/first>actual&&+test_cmpexpectedactual+'++test_expect_success'set up refs pointing to binary blob''+printf"a\0b\0c">blob1&&+printf"a\0c\0b">blob2&&+printf"\0a\0b\0c">blob3&&+printf"abc">blob4&&+printf"\0 \0 \0 ">blob5&&+printf"\0 \0a\0 ">blob6&&+printf" ">blob7&&+>blob8&&+obj=$(githash-object-wblob1)&&+gitupdate-refrefs/myblobs/blob1"$obj"&&+obj=$(githash-object-wblob2)&&+gitupdate-refrefs/myblobs/blob2"$obj"&&+obj=$(githash-object-wblob3)&&+gitupdate-refrefs/myblobs/blob3"$obj"&&+obj=$(githash-object-wblob4)&&+gitupdate-refrefs/myblobs/blob4"$obj"&&+obj=$(githash-object-wblob5)&&+gitupdate-refrefs/myblobs/blob5"$obj"&&+obj=$(githash-object-wblob6)&&+gitupdate-refrefs/myblobs/blob6"$obj"&&+obj=$(githash-object-wblob7)&&+gitupdate-refrefs/myblobs/blob7"$obj"&&+obj=$(githash-object-wblob8)&&+gitupdate-refrefs/myblobs/blob8"$obj"+'++test_expect_success'Verify sorts with raw''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob3+refs/myblobs/blob7+refs/mytrees/first+refs/myblobs/first+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob4+refs/heads/main+EOF+gitfor-each-ref--format="%(refname)"--sort=raw\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'Verify sorts with raw:size''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/first+refs/myblobs/blob7+refs/heads/main+refs/myblobs/blob4+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+refs/myblobs/blob5+refs/myblobs/blob6+refs/mytrees/first+EOF+gitfor-each-ref--format="%(refname)"--sort=raw:size\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'validate raw atom with %(if:equals)''+cat>expected<<-EOF&&+notequals+notequals+notequals+notequals+notequals+notequals+refs/myblobs/blob4+notequals+notequals+notequals+notequals+notequals+EOF+gitfor-each-ref--format="%(if:equals=abc)%(raw)%(then)%(refname)%(else)not equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'++test_expect_success'validate raw atom with %(if:notequals)''+cat>expected<<-EOF&&+refs/heads/ambiguous+refs/heads/main+refs/heads/newtag+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+equals+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob7+refs/myblobs/blob8+refs/myblobs/first+EOF+gitfor-each-ref--format="%(if:notequals=abc)%(raw)%(then)%(refname)%(else)equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'++test_expect_success'empty raw refs with %(if)''+cat>expected<<-EOF&&+refs/myblobs/blob1notempty+refs/myblobs/blob2notempty+refs/myblobs/blob3notempty+refs/myblobs/blob4notempty+refs/myblobs/blob5notempty+refs/myblobs/blob6notempty+refs/myblobs/blob7empty+refs/myblobs/blob8empty+refs/myblobs/firstnotempty+EOF+gitfor-each-ref--format="%(refname) %(if)%(raw)%(then)not empty%(else)empty%(end)"\+refs/myblobs/>actual&&+test_cmpexpectedactual+'++test_expect_success'%(raw) with --python must fail''+test_must_failgitfor-each-ref--format="%(raw)"--python+'++test_expect_success'%(raw) with --tcl must fail''+test_must_failgitfor-each-ref--format="%(raw)"--tcl+'++test_expect_success'%(raw) with --perl must fail''+test_must_failgitfor-each-ref--format="%(raw)"--perl+'++test_expect_success'%(raw) with --shell must fail''+test_must_failgitfor-each-ref--format="%(raw)"--shell+'++test_expect_success'%(raw) with --shell and --sort=raw must fail''+test_must_failgitfor-each-ref--format="%(raw)"--sort=raw--shell+'++test_expect_success'%(raw:size) with --shell''+gitfor-each-ref--format="%(raw:size)"|whilereadline+do+echo"'\''$line'\''">>expect+done&&+gitfor-each-ref--format="%(raw:size)"--shell>actual&&+test_cmpexpectactual+'++test_expect_success'for-each-ref --format compare with cat-file --batch''+gitrev-parserefs/mytrees/first|gitcat-file--batch>expected&&+gitfor-each-ref--format="%(objectname) %(objecttype) %(objectsize)+%(raw)" refs/mytrees/first >actual &&+test_cmpexpectedactual+'+ test_expect_success'set up multiple-sort tags''forwhenin100000200000do
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:08
From: ZheNing Hu <redacted>
Use non-const ref_format in *_atom_parser(), which can help us
modify the members of ref_format in *_atom_parser().
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/tag.c | 2 +-
ref-filter.c | 44 ++++++++++++++++++++++----------------------
ref-filter.h | 4 ++--
3 files changed, 25 insertions(+), 25 deletions(-)
@@ -116,7 +116,7 @@ void ref_array_sort(struct ref_sorting *sort, struct ref_array *array);voidref_sorting_set_sort_flags_all(structref_sorting*sorting,unsignedintmask,inton);/* Based on the given format and quote_style, fill the strbuf */intformat_ref_array_item(structref_array_item*info,-conststructref_format*format,+structref_format*format,structstrbuf*final_buf,structstrbuf*error_buf);/* Parse a single sort specifier and add it to the list */
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:09
From: ZheNing Hu <redacted>
Since in the refactor of `git cat-file --batch` later,
oid_object_info_extended() in get_object() will be used to obtain
the info of an object with it's oid. When the object cannot be
obtained in the git repository, `cat-file --batch` expects to output
"<oid> missing" and continue the next oid query instead of letting
Git exit. In other error conditions, Git should exit normally. So we
can achieve this function by passing the return value of get_object().
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
@@ -559,6 +560,15 @@ static int if_atom_parser(struct ref_format *format, struct used_atom *atom,return0;}+staticintrest_atom_parser(structref_format*format,structused_atom*atom,+constchar*arg,structstrbuf*err)+{+if(arg)+returnstrbuf_addf_ret(err,-1,_("%%(rest) does not take arguments"));+format->use_rest=1;+return0;+}+staticinthead_atom_parser(structref_format*format,structused_atom*atom,constchar*arg,structstrbuf*unused_err){
@@ -1998,6 +1998,10 @@ test_expect_success '--format should list tags as per format given' 'test_cmpexpectactual'+test_expect_success'git tag -l with --format="%(rest)" must fail''+test_must_failgittag-l--format="%(rest)""v1*"+'+ test_expect_success"set up color tests"'echo"<RED>v1.0<RESET>">expect.color&&echo"v1.0">expect.bare&&
@@ -194,6 +194,10 @@ test_expect_success GPG 'verifying tag with --format' 'test_cmpexpectactual'+test_expect_successGPG'verifying tag with --format="%(rest)" must fail''+test_must_failgitverify-tag--format="%(rest)""fourth-signed"+'+ test_expect_successGPG'verifying a forged tag with --format should fail silently''test_must_failgitverify-tag--format="tagname : %(tag)"$(catforged1.tag)>actual-forged&&test_must_be_emptyactual-forged
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:12
From: ZheNing Hu <redacted>
Add `cat_file_mode` member in struct `ref_format`, when
`cat-file --batch` use ref-filter logic later, it can help
us reject atoms in verify_ref_format() which cat-file cannot
use, e.g. `%(refname)`, `%(push)`, `%(upstream)`...
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 11 +++++++++--
ref-filter.h | 1 +
2 files changed, 10 insertions(+), 2 deletions(-)
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:13
From: ZheNing Hu <redacted>
When we use ref_array_item which is not dynamically allocated and
want to free the space of its member "value" after the end of use,
free_array_item() does not meet our needs, because it tries to free
ref_array_item itself and its member "symref".
Introduce free_ref_array_item_value() for freeing ref_array_item value.
It will be called internally by free_array_item(), and it will help
`cat-file --batch` free ref_array_item's value memory later.
Helped-by: Junio C Hamano [off-list ref]
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 11 ++++++++---
ref-filter.h | 2 ++
2 files changed, 10 insertions(+), 3 deletions(-)
@@ -111,6 +111,8 @@ struct ref_format {intfilter_refs(structref_array*array,structref_filter*filter,unsignedinttype);/* Clear all memory allocated to ref_array */voidref_array_clear(structref_array*array);+/* Free ref_array_item's value */+voidfree_ref_array_item_value(structref_array_item*item);/* Used to verify if the given format is correct and to parse out the used atoms */intverify_ref_format(structref_format*format);/* Sort the given ref_array as per the ref_sorting provided */
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:14
From: ZheNing Hu <redacted>
Let get_object() return 1 and print "<oid> missing" instead
of returning -1 and printing "missing object <oid> for <refname>"
if oid_object_info_extended() unable to find the data corresponding
to oid. When `cat-file --batch` use ref-filter logic later it can
help `format_ref_array_item()` just report that the object is missing
without letting Git exit.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 4 ++--
t/t6301-for-each-ref-errors.sh | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
@@ -1749,8 +1749,8 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj}if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,OBJECT_INFO_LOOKUP_REPLACE))-returnstrbuf_addf_ret(err,-1,_("missing object %s for %s"),-oid_to_hex(&oi->oid),ref->refname);+returnstrbuf_addf_ret(err,1,_("%s missing"),+oid_to_hex(&oi->oid));if(oi->info.disk_sizep&&oi->disk_size<0)BUG("Object size is less than zero.");
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:17
From: ZheNing Hu <redacted>
Use `has_object_file()` in `batch_one_object()` to check
whether the input object exists. This can help us reject
the missing oid when we let `cat-file --batch` use ref-filter
logic later.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 7 +++++++
1 file changed, 7 insertions(+)
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:19
From: ZheNing Hu <redacted>
Since cat-file reuses ref-filter logic later will add the
formal parameter "const struct option *options" to
batch_objects(), the two synonymous parameters of "opt"
and "options" may confuse readers, so change batch_options
parameter of batch_objects() from "opt" to "batch".
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
@@ -513,13 +513,13 @@ static int batch_objects(struct batch_options *opt)*/memset(&data,0,sizeof(data));data.mark_query=1;-strbuf_expand(&output,opt->format,expand_format,&data);+strbuf_expand(&output,batch->format,expand_format,&data);data.mark_query=0;strbuf_release(&output);-if(opt->cmdmode)+if(batch->cmdmode)data.split_on_whitespace=1;-if(opt->all_objects){+if(batch->all_objects){structobject_infoempty=OBJECT_INFO_INIT;if(!memcmp(&data.info,&empty,sizeof(empty)))data.skip_object_info=1;
@@ -529,20 +529,20 @@ static int batch_objects(struct batch_options *opt)*Ifweareprintingouttheobject,thenalwaysfillinthetype,*sincewewillwanttodecidewhetherornottostream.*/-if(opt->print_contents)+if(batch->print_contents)data.info.typep=&data.type;-if(opt->all_objects){+if(batch->all_objects){structobject_cb_datacb;if(has_promisor_remote())warning("This repository uses promisor remotes. Some objects may not be loaded.");-cb.opt=opt;+cb.opt=batch;cb.expand=&data;cb.scratch=&output;-if(opt->unordered){+if(batch->unordered){structoidsetseen=OIDSET_INIT;cb.seen=&seen;
@@ -592,7 +592,7 @@ static int batch_objects(struct batch_options *opt)data.rest=p;}-batch_one_object(input.buf,&output,opt,&data);+batch_one_object(input.buf,&output,batch,&data);}strbuf_release(&input);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:22
From: ZheNing Hu <redacted>
In order to let cat-file use ref-filter logic, let's do the
following:
1. Change the type of member `format` in struct `batch_options`
to `ref_format`, we will pass it to ref-filter later.
2. Let `batch_objects()` add atoms to format, and use
`verify_ref_format()` to check atoms.
3. Use `format_ref_array_item()` in `batch_object_write()` to
get the formatted data corresponding to the object. If the
return value of `format_ref_array_item()` is equals to zero,
use `batch_write()` to print object data; else if the return
value is less than zero, use `die()` to print the error message
and exit; else if return value is greater than zero, only print
the error message, but don't exit.
4. Use free_ref_array_item_value() to free ref_array_item's
value.
Most of the atoms in `for-each-ref --format` are now supported,
such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,
`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:
`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,
`%(flag)`, `%(HEAD)`, because our objects don't have a refname.
The performance for `git cat-file --batch-all-objects
--batch-check` on the Git repository itself with performance
testing tool `hyperfine` changes from 669.4 ms ± 31.1 ms to
1.134 s ± 0.063 s.
The performance for `git cat-file --batch-all-objects --batch
/dev/null` on the Git repository itself with performance testing
tool `time` change from "27.37s user 0.29s system 98% cpu 28.089
total" to "33.69s user 1.54s system 87% cpu 40.258 total".
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-cat-file.txt | 6 +
builtin/cat-file.c | 244 ++++++-------------------------
t/t1006-cat-file.sh | 252 +++++++++++++++++++++++++++++++++
3 files changed, 305 insertions(+), 197 deletions(-)
@@ -226,6 +226,12 @@ newline. The available atoms are: after that first run of whitespace (i.e., the "rest" of the line) are output in place of the `%(rest)` atom.+Note that most of the atoms in `for-each-ref --format` are now supported,+such as `%(tree)`, `%(parent)`, `%(author)`, `%(tagger)`, `%(if)`,+`%(then)`, `%(else)`, `%(end)`. But these atoms will be rejected:+`%(refname)`, `%(symref)`, `%(upstream)`, `%(push)`, `%(worktreepath)`,+`%(flag)`, `%(HEAD)`. See linkgit:git-for-each-ref[1].+ If no format is specified, the default format is `%(objectname) %(objecttype) %(objectsize)`.
@@ -25,7 +26,7 @@ struct batch_options {intall_objects;intunordered;intcmdmode;/* may be 'w' or 'c' for --filters or --textconv */-constchar*format;+structref_formatformat;};staticconstchar*force_path;
@@ -195,99 +196,10 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name,structexpand_data{structobject_idoid;-enumobject_typetype;-unsignedlongsize;-off_tdisk_size;constchar*rest;-structobject_iddelta_base_oid;--/*-*Ifmark_queryistrue,wedonotexpandanything,butrather-*justmarktheobject_infowithitemswewishtoquery.-*/-intmark_query;--/*-*Whethertosplittheinputonwhitespacebeforefeedingitto-*get_sha1;thisisdecidedduringthemark_queryphasebasedon-*whetherwehavea%(rest)tokeninourformat.-*/intsplit_on_whitespace;--/*-*Afteramark_queryrun,thisobject_infoissetuptobe-*passedtooid_object_info_extended.Itwillpointtothedata-*elementsabove,soyoucanretrievetheresponsefromthere.-*/-structobject_infoinfo;--/*-*Thisflagwillbetrueiftherequestedbatchformatandoptions-*don'trequireustocalloid_object_info,whichcanthenbe-*optimizedout.-*/-unsignedskip_object_info:1;};-staticintis_atom(constchar*atom,constchar*s,intslen)-{-intalen=strlen(atom);-returnalen==slen&&!memcmp(atom,s,alen);-}--staticvoidexpand_atom(structstrbuf*sb,constchar*atom,intlen,-void*vdata)-{-structexpand_data*data=vdata;--if(is_atom("objectname",atom,len)){-if(!data->mark_query)-strbuf_addstr(sb,oid_to_hex(&data->oid));-}elseif(is_atom("objecttype",atom,len)){-if(data->mark_query)-data->info.typep=&data->type;-else-strbuf_addstr(sb,type_name(data->type));-}elseif(is_atom("objectsize",atom,len)){-if(data->mark_query)-data->info.sizep=&data->size;-else-strbuf_addf(sb,"%"PRIuMAX,(uintmax_t)data->size);-}elseif(is_atom("objectsize:disk",atom,len)){-if(data->mark_query)-data->info.disk_sizep=&data->disk_size;-else-strbuf_addf(sb,"%"PRIuMAX,(uintmax_t)data->disk_size);-}elseif(is_atom("rest",atom,len)){-if(data->mark_query)-data->split_on_whitespace=1;-elseif(data->rest)-strbuf_addstr(sb,data->rest);-}elseif(is_atom("deltabase",atom,len)){-if(data->mark_query)-data->info.delta_base_oid=&data->delta_base_oid;-else-strbuf_addstr(sb,-oid_to_hex(&data->delta_base_oid));-}else-die("unknown format element: %.*s",len,atom);-}--staticsize_texpand_format(structstrbuf*sb,constchar*start,void*data)-{-constchar*end;--if(*start!='(')-return0;-end=strchr(start+1,')');-if(!end)-die("format element '%s' does not end in ')'",start);--expand_atom(sb,start+1,end-start-1,data);--returnend-start+1;-}-staticvoidbatch_write(structbatch_options*opt,constvoid*data,intlen){if(opt->buffer_output){
@@ -297,87 +209,34 @@ static void batch_write(struct batch_options *opt, const void *data, int len)write_or_die(1,data,len);}-staticvoidprint_object_or_die(structbatch_options*opt,structexpand_data*data)-{-conststructobject_id*oid=&data->oid;--assert(data->info.typep);--if(data->type==OBJ_BLOB){-if(opt->buffer_output)-fflush(stdout);-if(opt->cmdmode){-char*contents;-unsignedlongsize;--if(!data->rest)-die("missing path for '%s'",oid_to_hex(oid));--if(opt->cmdmode=='w'){-if(filter_object(data->rest,0100644,oid,-&contents,&size))-die("could not convert '%s' %s",-oid_to_hex(oid),data->rest);-}elseif(opt->cmdmode=='c'){-enumobject_typetype;-if(!textconv_object(the_repository,-data->rest,0100644,oid,-1,&contents,&size))-contents=read_object_file(oid,-&type,-&size);-if(!contents)-die("could not convert '%s' %s",-oid_to_hex(oid),data->rest);-}else-BUG("invalid cmdmode: %c",opt->cmdmode);-batch_write(opt,contents,size);-free(contents);-}else{-stream_blob(oid);-}-}-else{-enumobject_typetype;-unsignedlongsize;-void*contents;--contents=read_object_file(oid,&type,&size);-if(!contents)-die("object %s disappeared",oid_to_hex(oid));-if(type!=data->type)-die("object %s changed type!?",oid_to_hex(oid));-if(data->info.sizep&&size!=data->size)-die("object %s changed size!?",oid_to_hex(oid));--batch_write(opt,contents,size);-free(contents);-}-}staticvoidbatch_object_write(constchar*obj_name,structstrbuf*scratch,structbatch_options*opt,structexpand_data*data){-if(!data->skip_object_info&&-oid_object_info_extended(the_repository,&data->oid,&data->info,-OBJECT_INFO_LOOKUP_REPLACE)<0){-printf("%s missing\n",-obj_name?obj_name:oid_to_hex(&data->oid));-fflush(stdout);-return;-}+intret=0;+structstrbuferr=STRBUF_INIT;+structref_array_itemitem={data->oid,data->rest};strbuf_reset(scratch);-strbuf_expand(scratch,opt->format,expand_format,data);-strbuf_addch(scratch,'\n');-batch_write(opt,scratch->buf,scratch->len);-if(opt->print_contents){-print_object_or_die(opt,data);-batch_write(opt,"\n",1);+ret=format_ref_array_item(&item,&opt->format,scratch,&err);+if(ret<0){+die("%s\n",err.buf);+}if(ret){+/* ret > 0 means when the object corresponding to oid+*cannotbefoundinformat_ref_array_item(),weonlyprint+*theerrormessage.+*/+printf("%s\n",err.buf);+fflush(stdout);+}else{+strbuf_addch(scratch,'\n');+batch_write(opt,scratch->buf,scratch->len);}+free_ref_array_item_value(&item);+strbuf_release(&err);}staticvoidbatch_one_object(constchar*obj_name,
@@ -720,7 +570,7 @@ int cmd_cat_file(int argc, const char **argv, const char *prefix)batch.buffer_output=batch.all_objects;if(batch.enabled)-returnbatch_objects(&batch);+returnbatch_objects(&batch,options);if(unknown_type&&opt!='t'&&opt!='s')die("git cat-file --allow-unknown-type: use with -s or -t");
@@ -586,4 +586,256 @@ test_expect_success 'cat-file --unordered works' 'test_cmpexpectactual'+."$TEST_DIRECTORY"/lib-gpg.sh+."$TEST_DIRECTORY"/lib-terminal.sh++test_expect_success'cat-file --batch|--batch-check setup''+echo1>blob1&&+printf"a\0b\0\c">blob2&&+gitaddblob1blob2&&+gitcommit-m"Commit Message"&&+gitbranch-Mmain&&+gittag-a-m"v0.0.0"testtag&&+gitupdate-refrefs/myblobs/blob1HEAD:blob1&&+gitupdate-refrefs/myblobs/blob2HEAD:blob2&&+gitupdate-refrefs/mytrees/tree1HEAD^{tree}+'++batch_test_atom(){+iftest"$3"="fail"+then+test_expect_${4:-success}$PREREQ"basic atom: $1$2 must fail""+test_must_failgitcat-file--batch-check='$2'>bad<<-EOF+$1+EOF+"+else+test_expect_${4:-success}$PREREQ"basic atom: $1$2""+gitfor-each-ref--format='$2'$1>expected&&+gitcat-file--batch-check='$2'>actual<<-EOF&&+$1+EOF+sanitize_pgp<actual>actual.clean&&+cmpexpectedactual.clean+"+fi+}++batch_test_atomrefs/heads/main'%(refname)'fail+batch_test_atomrefs/heads/main'%(refname:)'fail+batch_test_atomrefs/heads/main'%(refname:short)'fail+batch_test_atomrefs/heads/main'%(refname:lstrip=1)'fail+batch_test_atomrefs/heads/main'%(refname:lstrip=2)'fail+batch_test_atomrefs/heads/main'%(refname:lstrip=-1)'fail+batch_test_atomrefs/heads/main'%(refname:lstrip=-2)'fail+batch_test_atomrefs/heads/main'%(refname:rstrip=1)'fail+batch_test_atomrefs/heads/main'%(refname:rstrip=2)'fail+batch_test_atomrefs/heads/main'%(refname:rstrip=-1)'fail+batch_test_atomrefs/heads/main'%(refname:rstrip=-2)'fail+batch_test_atomrefs/heads/main'%(refname:strip=1)'fail+batch_test_atomrefs/heads/main'%(refname:strip=2)'fail+batch_test_atomrefs/heads/main'%(refname:strip=-1)'fail+batch_test_atomrefs/heads/main'%(refname:strip=-2)'fail+batch_test_atomrefs/heads/main'%(upstream)'fail+batch_test_atomrefs/heads/main'%(upstream:short)'fail+batch_test_atomrefs/heads/main'%(upstream:lstrip=2)'fail+batch_test_atomrefs/heads/main'%(upstream:lstrip=-2)'fail+batch_test_atomrefs/heads/main'%(upstream:rstrip=2)'fail+batch_test_atomrefs/heads/main'%(upstream:rstrip=-2)'fail+batch_test_atomrefs/heads/main'%(upstream:strip=2)'fail+batch_test_atomrefs/heads/main'%(upstream:strip=-2)'fail+batch_test_atomrefs/heads/main'%(push)'fail+batch_test_atomrefs/heads/main'%(push:short)'fail+batch_test_atomrefs/heads/main'%(push:lstrip=1)'fail+batch_test_atomrefs/heads/main'%(push:lstrip=-1)'fail+batch_test_atomrefs/heads/main'%(push:rstrip=1)'fail+batch_test_atomrefs/heads/main'%(push:rstrip=-1)'fail+batch_test_atomrefs/heads/main'%(push:strip=1)'fail+batch_test_atomrefs/heads/main'%(push:strip=-1)'fail+batch_test_atomrefs/heads/main'%(objecttype)'+batch_test_atomrefs/heads/main'%(objectsize)'+batch_test_atomrefs/heads/main'%(objectsize:disk)'+batch_test_atomrefs/heads/main'%(deltabase)'+batch_test_atomrefs/heads/main'%(objectname)'+batch_test_atomrefs/heads/main'%(objectname:short)'+batch_test_atomrefs/heads/main'%(objectname:short=1)'+batch_test_atomrefs/heads/main'%(objectname:short=10)'+batch_test_atomrefs/heads/main'%(tree)'+batch_test_atomrefs/heads/main'%(tree:short)'+batch_test_atomrefs/heads/main'%(tree:short=1)'+batch_test_atomrefs/heads/main'%(tree:short=10)'+batch_test_atomrefs/heads/main'%(parent)'+batch_test_atomrefs/heads/main'%(parent:short)'+batch_test_atomrefs/heads/main'%(parent:short=1)'+batch_test_atomrefs/heads/main'%(parent:short=10)'+batch_test_atomrefs/heads/main'%(numparent)'+batch_test_atomrefs/heads/main'%(object)'+batch_test_atomrefs/heads/main'%(type)'+batch_test_atomrefs/heads/main'%(raw)'+batch_test_atomrefs/heads/main'%(*objectname)'+batch_test_atomrefs/heads/main'%(*objecttype)'+batch_test_atomrefs/heads/main'%(author)'+batch_test_atomrefs/heads/main'%(authorname)'+batch_test_atomrefs/heads/main'%(authoremail)'+batch_test_atomrefs/heads/main'%(authoremail:trim)'+batch_test_atomrefs/heads/main'%(authoremail:localpart)'+batch_test_atomrefs/heads/main'%(authordate)'+batch_test_atomrefs/heads/main'%(committer)'+batch_test_atomrefs/heads/main'%(committername)'+batch_test_atomrefs/heads/main'%(committeremail)'+batch_test_atomrefs/heads/main'%(committeremail:trim)'+batch_test_atomrefs/heads/main'%(committeremail:localpart)'+batch_test_atomrefs/heads/main'%(committerdate)'+batch_test_atomrefs/heads/main'%(tag)'+batch_test_atomrefs/heads/main'%(tagger)'+batch_test_atomrefs/heads/main'%(taggername)'+batch_test_atomrefs/heads/main'%(taggeremail)'+batch_test_atomrefs/heads/main'%(taggeremail:trim)'+batch_test_atomrefs/heads/main'%(taggeremail:localpart)'+batch_test_atomrefs/heads/main'%(taggerdate)'+batch_test_atomrefs/heads/main'%(creator)'+batch_test_atomrefs/heads/main'%(creatordate)'+batch_test_atomrefs/heads/main'%(subject)'+batch_test_atomrefs/heads/main'%(subject:sanitize)'+batch_test_atomrefs/heads/main'%(contents:subject)'+batch_test_atomrefs/heads/main'%(body)'+batch_test_atomrefs/heads/main'%(contents:body)'+batch_test_atomrefs/heads/main'%(contents:signature)'+batch_test_atomrefs/heads/main'%(contents)'+batch_test_atomrefs/heads/main'%(HEAD)'fail+batch_test_atomrefs/heads/main'%(upstream:track)'fail+batch_test_atomrefs/heads/main'%(upstream:trackshort)'fail+batch_test_atomrefs/heads/main'%(upstream:track,nobracket)'fail+batch_test_atomrefs/heads/main'%(upstream:nobracket,track)'fail+batch_test_atomrefs/heads/main'%(push:track)'fail+batch_test_atomrefs/heads/main'%(push:trackshort)'fail+batch_test_atomrefs/heads/main'%(worktreepath)'fail+batch_test_atomrefs/heads/main'%(symref)'fail+batch_test_atomrefs/heads/main'%(flag)'fail++batch_test_atomrefs/tags/testtag'%(refname)'fail+batch_test_atomrefs/tags/testtag'%(refname:short)'fail+batch_test_atomrefs/tags/testtag'%(upstream)'fail+batch_test_atomrefs/tags/testtag'%(push)'fail+batch_test_atomrefs/tags/testtag'%(objecttype)'+batch_test_atomrefs/tags/testtag'%(objectsize)'+batch_test_atomrefs/tags/testtag'%(objectsize:disk)'+batch_test_atomrefs/tags/testtag'%(*objectsize:disk)'+batch_test_atomrefs/tags/testtag'%(deltabase)'+batch_test_atomrefs/tags/testtag'%(*deltabase)'+batch_test_atomrefs/tags/testtag'%(objectname)'+batch_test_atomrefs/tags/testtag'%(objectname:short)'+batch_test_atomrefs/tags/testtag'%(tree)'+batch_test_atomrefs/tags/testtag'%(tree:short)'+batch_test_atomrefs/tags/testtag'%(tree:short=1)'+batch_test_atomrefs/tags/testtag'%(tree:short=10)'+batch_test_atomrefs/tags/testtag'%(parent)'+batch_test_atomrefs/tags/testtag'%(parent:short)'+batch_test_atomrefs/tags/testtag'%(parent:short=1)'+batch_test_atomrefs/tags/testtag'%(parent:short=10)'+batch_test_atomrefs/tags/testtag'%(numparent)'+batch_test_atomrefs/tags/testtag'%(object)'+batch_test_atomrefs/tags/testtag'%(type)'+batch_test_atomrefs/tags/testtag'%(*objectname)'+batch_test_atomrefs/tags/testtag'%(*objecttype)'+batch_test_atomrefs/tags/testtag'%(author)'+batch_test_atomrefs/tags/testtag'%(authorname)'+batch_test_atomrefs/tags/testtag'%(authoremail)'+batch_test_atomrefs/tags/testtag'%(authoremail:trim)'+batch_test_atomrefs/tags/testtag'%(authoremail:localpart)'+batch_test_atomrefs/tags/testtag'%(authordate)'+batch_test_atomrefs/tags/testtag'%(committer)'+batch_test_atomrefs/tags/testtag'%(committername)'+batch_test_atomrefs/tags/testtag'%(committeremail)'+batch_test_atomrefs/tags/testtag'%(committeremail:trim)'+batch_test_atomrefs/tags/testtag'%(committeremail:localpart)'+batch_test_atomrefs/tags/testtag'%(committerdate)'+batch_test_atomrefs/tags/testtag'%(tag)'+batch_test_atomrefs/tags/testtag'%(tagger)'+batch_test_atomrefs/tags/testtag'%(taggername)'+batch_test_atomrefs/tags/testtag'%(taggeremail)'+batch_test_atomrefs/tags/testtag'%(taggeremail:trim)'+batch_test_atomrefs/tags/testtag'%(taggeremail:localpart)'+batch_test_atomrefs/tags/testtag'%(taggerdate)'+batch_test_atomrefs/tags/testtag'%(creator)'+batch_test_atomrefs/tags/testtag'%(creatordate)'+batch_test_atomrefs/tags/testtag'%(subject)'+batch_test_atomrefs/tags/testtag'%(subject:sanitize)'+batch_test_atomrefs/tags/testtag'%(contents:subject)'+batch_test_atomrefs/tags/testtag'%(body)'+batch_test_atomrefs/tags/testtag'%(contents:body)'+batch_test_atomrefs/tags/testtag'%(contents:signature)'+batch_test_atomrefs/tags/testtag'%(contents)'+batch_test_atomrefs/tags/testtag'%(HEAD)'fail++batch_test_atomrefs/myblobs/blob1'%(refname)'fail+batch_test_atomrefs/myblobs/blob1'%(upstream)'fail+batch_test_atomrefs/myblobs/blob1'%(push)'fail+batch_test_atomrefs/myblobs/blob1'%(HEAD)'fail++batch_test_atomrefs/myblobs/blob1'%(objectname)'+batch_test_atomrefs/myblobs/blob1'%(objecttype)'+batch_test_atomrefs/myblobs/blob1'%(objectsize)'+batch_test_atomrefs/myblobs/blob1'%(objectsize:disk)'+batch_test_atomrefs/myblobs/blob1'%(deltabase)'++batch_test_atomrefs/myblobs/blob1'%(contents)'+batch_test_atomrefs/myblobs/blob2'%(contents)'++batch_test_atomrefs/myblobs/blob1'%(raw)'+batch_test_atomrefs/myblobs/blob2'%(raw)'+batch_test_atomrefs/mytrees/tree1'%(raw)'++batch_test_atomrefs/myblobs/blob1'%(raw:size)'+batch_test_atomrefs/myblobs/blob2'%(raw:size)'+batch_test_atomrefs/mytrees/tree1'%(raw:size)'++batch_test_atomrefs/myblobs/blob1'%(if:equals=blob)%(objecttype)%(then)commit%(else)not commit%(end)'+batch_test_atomrefs/myblobs/blob2'%(if:equals=blob)%(objecttype)%(then)commit%(else)not commit%(end)'+batch_test_atomrefs/mytrees/tree1'%(if:equals=tree)%(objecttype)%(then)tree%(else)not tree%(end)'++batch_test_atomrefs/heads/main'%(align:60) objectname is %(objectname)%(end)|%(objectname)'+batch_test_atomrefs/heads/main'%(align:left,60) objectname is %(objectname)%(end)|%(objectname)'+batch_test_atomrefs/heads/main'%(align:middle,60) objectname is %(objectname)%(end)|%(objectname)'+batch_test_atomrefs/heads/main'%(align:60,right) objectname is %(objectname)%(end)|%(objectname)'++batch_test_atomrefs/heads/main'VALID'+batch_test_atomrefs/heads/main'%(INVALID)'fail+batch_test_atomrefs/heads/main'%(authordate:INVALID)'fail++test_expect_success'%(rest) works with both a branch and a tag''+cat>expected<<-EOF&&+123commit123+456tag456+EOF+gitcat-file--batch-check="%(rest) %(objecttype) %(rest)">actual<<-EOF&&+refs/heads/main123+refs/tags/testtag456+EOF+test_cmpexpectedactual+'++batch_test_atomrefs/heads/main'%(objectname)%(objecttype)%(objectsize)+%(raw)'+batch_test_atomrefs/tags/testtag'%(objectname)%(objecttype)%(objectsize)+%(raw)'+batch_test_atomrefs/myblobs/blob1'%(objectname)%(objecttype)%(objectsize)+%(raw)'+batch_test_atomrefs/myblobs/blob2'%(objectname)%(objecttype)%(objectsize)+%(raw)'+++test_expect_success'cat-file --batch equals to --batch-check with atoms''+gitcat-file--batch-check="%(objectname) %(objecttype) %(objectsize)+%(raw)" >expected <<-EOF &&+refs/heads/main+refs/tags/testtag+EOF+gitcat-file--batch>actual<<-EOF&&+refs/heads/main+refs/tags/testtag+EOF+cmpexpectedactual+'+ test_done
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:26
From: ZheNing Hu <redacted>
Reuse the `err` buffer in batch_object_write(), as the
buffer `scratch` does. This will reduce the overhead
of multiple allocations of memory of the err buffer.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 22 ++++++++++++++--------
1 file changed, 14 insertions(+), 8 deletions(-)
@@ -212,35 +212,36 @@ static void batch_write(struct batch_options *opt, const void *data, int len)staticvoidbatch_object_write(constchar*obj_name,structstrbuf*scratch,+structstrbuf*err,structbatch_options*opt,structexpand_data*data){intret=0;-structstrbuferr=STRBUF_INIT;structref_array_itemitem={data->oid,data->rest};strbuf_reset(scratch);+strbuf_reset(err);-ret=format_ref_array_item(&item,&opt->format,scratch,&err);+ret=format_ref_array_item(&item,&opt->format,scratch,err);if(ret<0){-die("%s\n",err.buf);+die("%s\n",err->buf);}if(ret){/* ret > 0 means when the object corresponding to oid*cannotbefoundinformat_ref_array_item(),weonlyprint*theerrormessage.*/-printf("%s\n",err.buf);+printf("%s\n",err->buf);fflush(stdout);}else{strbuf_addch(scratch,'\n');batch_write(opt,scratch->buf,scratch->len);}free_ref_array_item_value(&item);-strbuf_release(&err);}staticvoidbatch_one_object(constchar*obj_name,structstrbuf*scratch,+structstrbuf*err,structbatch_options*opt,structexpand_data*data){
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-22 03:21:28
From: ZheNing Hu <redacted>
After cat-file reuses the ref-filter logic, we re-implement the
functions of --textconv and --filters options.
Add members `use_textconv` and `use_filters` in struct `ref_format`,
and use global variables `use_filters` and `use_textconv` in
`ref-filter.c`, so that we can filter the content of the object
in get_object(). Use `actual_oi` to record the real expand_data:
it may point to the original `oi` or the `act_oi` processed by
`textconv_object()` or `convert_to_working_tree()`. `grab_values()`
will grab the contents of `actual_oi` and `grab_common_values()`
to grab the contents of origin `oi`, this ensures that `%(objectsize)`
still uses the size of the unfiltered data.
In `get_object()`, we made an optimization: Firstly, get the size and
type of the object instead of directly getting the object data.
If using --textconv, after successfully obtaining the filtered object
data, an extra oid_object_info_extended() will be skipped, which can
reduce the cost of object data copy; If using --filter, the data of
the object first will be getted first, and then convert_to_working_tree()
will be used to get the filtered object data.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/cat-file.c | 6 +++++
ref-filter.c | 59 ++++++++++++++++++++++++++++++++++++++++++++--
ref-filter.h | 2 ++
3 files changed, 65 insertions(+), 2 deletions(-)
@@ -1031,6 +1035,9 @@ int verify_ref_format(struct ref_format *format)used_atom[at].atom_type==ATOM_WORKTREEPATH)))die(_("this command reject atom %%(%.*s)"),(int)(ep-sp-2),sp+2);+use_filters=format->use_filters;+use_textconv=format->use_textconv;+if((format->quote_style==QUOTE_PYTHON||format->quote_style==QUOTE_SHELL||format->quote_style==QUOTE_TCL)&&
@@ -1742,10 +1749,38 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **obj{/* parse_object_buffer() will set eaten to 0 if free() will be needed */inteaten=1;+structexpand_data*actual_oi=oi;+structexpand_dataact_oi={0};+if(oi->info.contentp){/* We need to know that to use parse_object_buffer properly */+void**temp_contentp=oi->info.contentp;+oi->info.contentp=NULL;oi->info.sizep=&oi->size;oi->info.typep=&oi->type;++/* get the type and size */+if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,+OBJECT_INFO_LOOKUP_REPLACE))+returnstrbuf_addf_ret(err,1,_("%s missing"),+oid_to_hex(&oi->oid));++oi->info.sizep=NULL;+oi->info.typep=NULL;+oi->info.contentp=temp_contentp;++if(use_textconv&&!ref->rest)+returnstrbuf_addf_ret(err,-1,_("missing path for '%s'"),+oid_to_hex(&act_oi.oid));+if(use_textconv&&oi->type==OBJ_BLOB){+act_oi=*oi;+if(textconv_object(the_repository,+ref->rest,0100644,&act_oi.oid,+1,(char**)(&act_oi.content),&act_oi.size)){+actual_oi=&act_oi;+gotosuccess;+}+}}if(oid_object_info_extended(the_repository,&oi->oid,&oi->info,OBJECT_INFO_LOOKUP_REPLACE))
@@ -1755,19 +1790,39 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objBUG("Object size is less than zero.");if(oi->info.contentp){-*obj=parse_object_buffer(the_repository,&oi->oid,oi->type,oi->size,oi->content,&eaten);+if(use_filters&&!ref->rest)+returnstrbuf_addf_ret(err,-1,_("missing path for '%s'"),+oid_to_hex(&oi->oid));+if(use_filters&&oi->type==OBJ_BLOB){+structstrbufstrbuf=STRBUF_INIT;+structcheckout_metadatameta;+act_oi=*oi;++init_checkout_metadata(&meta,NULL,NULL,&act_oi.oid);+if(!convert_to_working_tree(&the_index,ref->rest,act_oi.content,act_oi.size,&strbuf,&meta))+die("could not convert '%s' %s",+oid_to_hex(&oi->oid),ref->rest);+act_oi.size=strbuf.len;+act_oi.content=strbuf_detach(&strbuf,NULL);+actual_oi=&act_oi;+}++success:+*obj=parse_object_buffer(the_repository,&actual_oi->oid,actual_oi->type,actual_oi->size,actual_oi->content,&eaten);if(!*obj){if(!eaten)free(oi->content);returnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi);+grab_values(ref->value,deref,*obj,actual_oi);}grab_common_values(ref->value,deref,oi);if(!eaten)free(oi->content);+if(actual_oi!=oi)+free(actual_oi->content);return0;}
On 22/06/21 10.20, ZheNing Hu via GitGitGadget wrote:
From: ZheNing Hu <redacted>
Since in the refactor of `git cat-file --batch` later,
oid_object_info_extended() in get_object() will be used to obtain
the info of an object with it's oid. When the object cannot be
obtained in the git repository, `cat-file --batch` expects to output
"<oid> missing" and continue the next oid query instead of letting
Git exit. In other error conditions, Git should exit normally. So we
can achieve this function by passing the return value of get_object().
s/Since/Because/
--
An old man doll... just what I always wanted! - Clara
On 22/06/21 10.20, ZheNing Hu via GitGitGadget wrote:
From: ZheNing Hu <redacted>
Since cat-file reuses ref-filter logic later will add the
formal parameter "const struct option *options" to
batch_objects(), the two synonymous parameters of "opt"
and "options" may confuse readers, so change batch_options
parameter of batch_objects() from "opt" to "batch".
Better say "Because later cat-file reuses ref-filter logic that will add
parameter ... ".
--
An old man doll... just what I always wanted! - Clara
On 22/06/21 10.20, ZheNing Hu via GitGitGadget wrote:
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a
variable in the host language, the host language may not support
arbitrary binary data in the variables of its string type.
Better say:
"Note that `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, and `--perl` because if the binary raw data is passed to a
variable in such languages, these may not support arbitrary binary data
in their string variable type."
--
An old man doll... just what I always wanted! - Clara
On 22/06/21 10.20, ZheNing Hu via GitGitGadget wrote:
quoted
Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, `--perl` because if our binary raw data is passed to a
variable in the host language, the host language may not support
arbitrary binary data in the variables of its string type.
Better say:
"Note that `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, and `--perl` because if the binary raw data is passed to a
variable in such languages, these may not support arbitrary binary data
in their string variable type."
Thanks, Bagas Sanjaya, I will change all of them.
--
An old man doll... just what I always wanted! - Clara
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-25 16:02:34
This patch series make cat-file reuse ref-filter logic.
Change from last version:
1. At the suggestion of Bagas Sanjaya, modified the expression of submitted
information.
2. Remove grab_oid() function in ref-filter to reduce repeated checks.
ZheNing Hu (15):
[GSOC] ref-filter: add obj-type check in grab contents
[GSOC] ref-filter: add %(raw) atom
[GSOC] ref-filter: --format=%(raw) re-support --perl
[GSOC] ref-filter: use non-const ref_format in *_atom_parser()
[GSOC] ref-filter: add %(rest) atom
[GSOC] ref-filter: pass get_object() return value to their callers
[GSOC] ref-filter: introduce free_ref_array_item_value() function
[GSOC] ref-filter: add cat_file_mode in struct ref_format
[GSOC] ref-filter: modify the error message and value in get_object
[GSOC] cat-file: add has_object_file() check
[GSOC] cat-file: change batch_objects parameter name
[GSOC] cat-file: reuse ref-filter logic
[GSOC] cat-file: reuse err buf in batch_object_write()
[GSOC] cat-file: re-implement --textconv, --filters options
[GSOC] ref-filter: remove grab_oid() function
Documentation/git-cat-file.txt | 6 +
Documentation/git-for-each-ref.txt | 9 +
builtin/cat-file.c | 277 ++++++----------------
builtin/tag.c | 2 +-
quote.c | 17 ++
quote.h | 1 +
ref-filter.c | 357 ++++++++++++++++++++++-------
ref-filter.h | 14 +-
t/t1006-cat-file.sh | 252 ++++++++++++++++++++
t/t3203-branch-output.sh | 4 +
t/t6300-for-each-ref.sh | 235 +++++++++++++++++++
t/t6301-for-each-ref-errors.sh | 2 +-
t/t7004-tag.sh | 4 +
t/t7030-verify-tag.sh | 4 +
14 files changed, 888 insertions(+), 296 deletions(-)
base-commit: 1197f1a46360d3ae96bd9c15908a3a6f8e562207
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-980%2Fadlternative%2Fcat-file-batch-refactor-v5
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-980/adlternative/cat-file-batch-refactor-v5
Pull-Request: https://github.com/gitgitgadget/git/pull/980
Range-diff vs v4:
1: f72ad9cc5e8 = 1: f72ad9cc5e8 [GSOC] ref-filter: add obj-type check in grab contents
2: ab497d66c11 ! 2: 4e473838b9d [GSOC] ref-filter: add %(raw) atom
@@ Commit message
can help us add raw object data to the buffer or compare two buffers
which contain raw object data.
- Beyond, `--format=%(raw)` cannot be used with `--python`, `--shell`,
- `--tcl`, `--perl` because if our binary raw data is passed to a
- variable in the host language, the host language may not support
- arbitrary binary data in the variables of its string type.
+ Note that `--format=%(raw)` cannot be used with `--python`, `--shell`,
+ `--tcl`, and `--perl` because if the binary raw data is passed to a
+ variable in such languages, these may not support arbitrary binary data
+ in their string variable type.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
3: b54dbc431e0 = 3: 765cf08a108 [GSOC] ref-filter: --format=%(raw) re-support --perl
4: 9fbbb3c492f = 4: d2aeafd0ef3 [GSOC] ref-filter: use non-const ref_format in *_atom_parser()
5: 08aa44e5e57 = 5: 1ca3a42f041 [GSOC] ref-filter: add %(rest) atom
6: 05682bccf9f ! 6: 67f1a3cca9a [GSOC] ref-filter: pass get_object() return value to their callers
@@ Metadata
## Commit message ##
[GSOC] ref-filter: pass get_object() return value to their callers
- Since in the refactor of `git cat-file --batch` later,
+ Because in the refactor of `git cat-file --batch` later,
oid_object_info_extended() in get_object() will be used to obtain
the info of an object with it's oid. When the object cannot be
obtained in the git repository, `cat-file --batch` expects to output
7: 06db6cd6f1f = 7: 2a48a48e81c [GSOC] ref-filter: introduce free_ref_array_item_value() function
8: b0d9e139935 = 8: be55005be75 [GSOC] ref-filter: add cat_file_mode in struct ref_format
9: db7dd8b042c = 9: 937f88b7837 [GSOC] ref-filter: modify the error message and value in get_object
10: 6b577969734 = 10: 45657499c55 [GSOC] cat-file: add has_object_file() check
11: 069aa203666 ! 11: bf5c0a017ad [GSOC] cat-file: change batch_objects parameter name
@@ Metadata
## Commit message ##
[GSOC] cat-file: change batch_objects parameter name
- Since cat-file reuses ref-filter logic later will add the
- formal parameter "const struct option *options" to
- batch_objects(), the two synonymous parameters of "opt"
- and "options" may confuse readers, so change batch_options
- parameter of batch_objects() from "opt" to "batch".
+ Because later cat-file reuses ref-filter logic that will add
+ parameter "const struct option *options" to batch_objects(),
+ the two synonymous parameters of "opt" and "options" may
+ confuse readers, so change batch_options parameter of
+ batch_objects() from "opt" to "batch".
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
12: 258ec0a46c5 = 12: 370101ba65f [GSOC] cat-file: reuse ref-filter logic
13: bda6aae9a6c = 13: 69eef47065d [GSOC] cat-file: reuse err buf in batch_object_write()
14: d1114a2bd74 = 14: a7ac037a946 [GSOC] cat-file: re-implement --textconv, --filters options
-: ----------- > 15: 843de8864a9 [GSOC] ref-filter: remove grab_oid() function
--
gitgitgadget
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-25 16:02:36
From: ZheNing Hu <redacted>
Add new formatting option `%(raw)`, which will print the raw
object data without any changes. It will help further to migrate
all cat-file formatting logic from cat-file to ref-filter.
The raw data of blob, tree objects may contain '\0', but most of
the logic in `ref-filter` depends on the output of the atom being
text (specifically, no embedded NULs in it).
E.g. `quote_formatting()` use `strbuf_addstr()` or `*._quote_buf()`
add the data to the buffer. The raw data of a tree object is
`100644 one\0...`, only the `100644 one` will be added to the buffer,
which is incorrect.
Therefore, we need to find a way to record the length of the
atom_value's member `s`. Although strbuf can already record the
string and its length, if we want to replace the type of atom_value's
member `s` with strbuf, many places in ref-filter that are filled
with dynamically allocated mermory in `v->s` are not easy to replace.
At the same time, we need to check if `v->s == NULL` in
populate_value(), and strbuf cannot easily distinguish NULL and empty
strings, but c-style "const char *" can do it. So add a new member in
`struct atom_value`: `s_size`, which can record raw object size, it
can help us add raw object data to the buffer or compare two buffers
which contain raw object data.
Note that `--format=%(raw)` cannot be used with `--python`, `--shell`,
`--tcl`, and `--perl` because if the binary raw data is passed to a
variable in such languages, these may not support arbitrary binary data
in their string variable type.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Helped-by: Felipe Contreras [off-list ref]
Helped-by: Phillip Wood [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Based-on-patch-by: Olga Telezhnaya [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-for-each-ref.txt | 9 ++
ref-filter.c | 139 +++++++++++++++----
t/t6300-for-each-ref.sh | 216 +++++++++++++++++++++++++++++
3 files changed, 337 insertions(+), 27 deletions(-)
@@ -235,6 +235,15 @@ and `date` to extract the named component. For email fields (`authoremail`, without angle brackets, and `:localpart` to get the part before the `@` symbol out of the trimmed email.+The raw data in an object is `raw`.++raw:size::+ The raw data size of the object.++Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,+`--perl` because the host language may not support arbitrary binary data in the+variables of its string type.+ The message in a commit or a tag object is `contents`, from which `contents:<part>` can be used to extract various parts out of:
@@ -620,12 +637,15 @@ struct ref_formatting_state {structatom_value{constchar*s;+size_ts_size;int(*handler)(structatom_value*atomv,structref_formatting_state*state,structstrbuf*err);uintmax_tvalue;/* used for sorting when not FIELD_STR */structused_atom*atom;};+#define ATOM_VALUE_S_SIZE_INIT (-1)+/**Usedtoparseformatstringandsortspecifiers*/
@@ -644,13 +664,6 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnstrbuf_addf_ret(err,-1,_("malformed field name: %.*s"),(int)(ep-atom),atom);-/* Do we have the atom already used elsewhere? */-for(i=0;i<used_atom_cnt;i++){-intlen=strlen(used_atom[i].name);-if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))-returni;-}-/**Iftheatomnamehasacolon,stripitandeverythingafter*itoff-itspecifiestheformatforthisentry,and
@@ -660,6 +673,13 @@ static int parse_ref_filter_atom(const struct ref_format *format,arg=memchr(sp,':',ep-sp);atom_len=(arg?arg:ep)-sp;+/* Do we have the atom already used elsewhere? */+for(i=0;i<used_atom_cnt;i++){+intlen=strlen(used_atom[i].name);+if(len==ep-atom&&!memcmp(used_atom[i].name,atom,len))+returni;+}+/* Is the atom a valid one? */for(i=0;i<ARRAY_SIZE(valid_atom);i++){intlen=strlen(valid_atom[i].name);
@@ -709,11 +729,14 @@ static int parse_ref_filter_atom(const struct ref_format *format,returnat;}-staticvoidquote_formatting(structstrbuf*s,constchar*str,intquote_style)+staticvoidquote_formatting(structstrbuf*s,constchar*str,size_tlen,intquote_style){switch(quote_style){caseQUOTE_NONE:-strbuf_addstr(s,str);+if(len!=ATOM_VALUE_S_SIZE_INIT)+strbuf_add(s,str,len);+else+strbuf_addstr(s,str);break;caseQUOTE_SHELL:sq_quote_buf(s,str);
@@ -867,18 +895,22 @@ static int then_atom_handler(struct atom_value *atomv, struct ref_formatting_staif(if_then_else->else_atom_seen)returnstrbuf_addf_ret(err,-1,_("format: %%(then) atom used after %%(else)"));if_then_else->then_atom_seen=1;+if(if_then_else->str)+str_len=strlen(if_then_else->str);/**Ifthe'equals'or'notequals'attributeisusedthen*performtherequiredcomparison.Ifnot,onlynon-empty*stringssatisfythe'if'condition.*/if(if_then_else->cmp_status==COMPARE_EQUAL){-if(!strcmp(if_then_else->str,cur->output.buf))+if(str_len==cur->output.len&&+!memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;}elseif(if_then_else->cmp_status==COMPARE_UNEQUAL){-if(strcmp(if_then_else->str,cur->output.buf))+if(str_len!=cur->output.len||+memcmp(if_then_else->str,cur->output.buf,cur->output.len))if_then_else->condition_satisfied=1;-}elseif(cur->output.len&&!is_empty(cur->output.buf))+}elseif(cur->output.len&&!is_empty(&cur->output))if_then_else->condition_satisfied=1;strbuf_reset(&cur->output);return0;
@@ -974,6 +1006,10 @@ int verify_ref_format(struct ref_format *format)at=parse_ref_filter_atom(format,sp+2,ep,&err);if(at<0)die("%s",err.buf);+if(format->quote_style&&used_atom[at].atom_type==ATOM_RAW&&+used_atom[at].u.raw_data.option==RAW_BARE)+die(_("--format=%.*s cannot be used with"+"--python, --shell, --tcl, --perl"),(int)(ep-sp-2),sp+2);cp=ep+1;if(skip_prefix(used_atom[at].name,"color:",&color))
@@ -708,6 +737,15 @@ test_atom refs/tags/signed-long contents "subject line bodycontents$sig"+test_expect_successGPG'basic atom: refs/tags/signed-long raw''+gitcat-filetagrefs/tags/signed-long>expected&&+gitfor-each-ref--format="%(raw)"refs/tags/signed-long>actual&&+sanitize_pgp<expected>expected.clean&&+echo>>expected.clean&&+sanitize_pgp<actual>actual.clean&&+test_cmpexpected.cleanactual.clean+'+ test_expect_success'set up refs pointing to tree and blob''gitupdate-refrefs/mytrees/firstrefs/heads/main^{tree}&&gitupdate-refrefs/myblobs/firstrefs/heads/main:one
@@ -727,6 +775,174 @@ test_atom refs/myblobs/first contents:body "" test_atomrefs/myblobs/firstcontents:signature"" test_atomrefs/myblobs/firstcontents""+test_expect_success'basic atom: refs/myblobs/first raw''+gitcat-fileblobrefs/myblobs/first>expected&&+echo>>expected&&+gitfor-each-ref--format="%(raw)"refs/myblobs/first>actual&&+test_cmpexpectedactual&&+gitcat-file-srefs/myblobs/first>expected&&+gitfor-each-ref--format="%(raw:size)"refs/myblobs/first>actual&&+test_cmpexpectedactual+'++test_expect_success'set up refs pointing to binary blob''+printf"a\0b\0c">blob1&&+printf"a\0c\0b">blob2&&+printf"\0a\0b\0c">blob3&&+printf"abc">blob4&&+printf"\0 \0 \0 ">blob5&&+printf"\0 \0a\0 ">blob6&&+printf" ">blob7&&+>blob8&&+obj=$(githash-object-wblob1)&&+gitupdate-refrefs/myblobs/blob1"$obj"&&+obj=$(githash-object-wblob2)&&+gitupdate-refrefs/myblobs/blob2"$obj"&&+obj=$(githash-object-wblob3)&&+gitupdate-refrefs/myblobs/blob3"$obj"&&+obj=$(githash-object-wblob4)&&+gitupdate-refrefs/myblobs/blob4"$obj"&&+obj=$(githash-object-wblob5)&&+gitupdate-refrefs/myblobs/blob5"$obj"&&+obj=$(githash-object-wblob6)&&+gitupdate-refrefs/myblobs/blob6"$obj"&&+obj=$(githash-object-wblob7)&&+gitupdate-refrefs/myblobs/blob7"$obj"&&+obj=$(githash-object-wblob8)&&+gitupdate-refrefs/myblobs/blob8"$obj"+'++test_expect_success'Verify sorts with raw''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob3+refs/myblobs/blob7+refs/mytrees/first+refs/myblobs/first+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob4+refs/heads/main+EOF+gitfor-each-ref--format="%(refname)"--sort=raw\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'Verify sorts with raw:size''+cat>expected<<-EOF&&+refs/myblobs/blob8+refs/myblobs/first+refs/myblobs/blob7+refs/heads/main+refs/myblobs/blob4+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+refs/myblobs/blob5+refs/myblobs/blob6+refs/mytrees/first+EOF+gitfor-each-ref--format="%(refname)"--sort=raw:size\+refs/heads/mainrefs/myblobs/refs/mytrees/first>actual&&+test_cmpexpectedactual+'++test_expect_success'validate raw atom with %(if:equals)''+cat>expected<<-EOF&&+notequals+notequals+notequals+notequals+notequals+notequals+refs/myblobs/blob4+notequals+notequals+notequals+notequals+notequals+EOF+gitfor-each-ref--format="%(if:equals=abc)%(raw)%(then)%(refname)%(else)not equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'++test_expect_success'validate raw atom with %(if:notequals)''+cat>expected<<-EOF&&+refs/heads/ambiguous+refs/heads/main+refs/heads/newtag+refs/myblobs/blob1+refs/myblobs/blob2+refs/myblobs/blob3+equals+refs/myblobs/blob5+refs/myblobs/blob6+refs/myblobs/blob7+refs/myblobs/blob8+refs/myblobs/first+EOF+gitfor-each-ref--format="%(if:notequals=abc)%(raw)%(then)%(refname)%(else)equals%(end)"\+refs/myblobs/refs/heads/>actual&&+test_cmpexpectedactual+'++test_expect_success'empty raw refs with %(if)''+cat>expected<<-EOF&&+refs/myblobs/blob1notempty+refs/myblobs/blob2notempty+refs/myblobs/blob3notempty+refs/myblobs/blob4notempty+refs/myblobs/blob5notempty+refs/myblobs/blob6notempty+refs/myblobs/blob7empty+refs/myblobs/blob8empty+refs/myblobs/firstnotempty+EOF+gitfor-each-ref--format="%(refname) %(if)%(raw)%(then)not empty%(else)empty%(end)"\+refs/myblobs/>actual&&+test_cmpexpectedactual+'++test_expect_success'%(raw) with --python must fail''+test_must_failgitfor-each-ref--format="%(raw)"--python+'++test_expect_success'%(raw) with --tcl must fail''+test_must_failgitfor-each-ref--format="%(raw)"--tcl+'++test_expect_success'%(raw) with --perl must fail''+test_must_failgitfor-each-ref--format="%(raw)"--perl+'++test_expect_success'%(raw) with --shell must fail''+test_must_failgitfor-each-ref--format="%(raw)"--shell+'++test_expect_success'%(raw) with --shell and --sort=raw must fail''+test_must_failgitfor-each-ref--format="%(raw)"--sort=raw--shell+'++test_expect_success'%(raw:size) with --shell''+gitfor-each-ref--format="%(raw:size)"|whilereadline+do+echo"'\''$line'\''">>expect+done&&+gitfor-each-ref--format="%(raw:size)"--shell>actual&&+test_cmpexpectactual+'++test_expect_success'for-each-ref --format compare with cat-file --batch''+gitrev-parserefs/mytrees/first|gitcat-file--batch>expected&&+gitfor-each-ref--format="%(objectname) %(objecttype) %(objectsize)+%(raw)" refs/mytrees/first >actual &&+test_cmpexpectedactual+'+ test_expect_success'set up multiple-sort tags''forwhenin100000200000do
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-25 16:02:38
From: ZheNing Hu <redacted>
Only tag and commit objects use `grab_sub_body_contents()` to grab
object contents in the current codebase. We want to teach the
function to also handle blobs and trees to get their raw data,
without parsing a blob (whose contents looks like a commit or a tag)
incorrectly as a commit or a tag.
Skip the block of code that is specific to handling commits and tags
early when the given object is of a wrong type to help later
addition to handle other types of objects in this function.
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Junio C Hamano [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
@@ -1678,7 +1684,7 @@ static int get_object(struct ref_array_item *ref, int deref, struct object **objreturnstrbuf_addf_ret(err,-1,_("parse_object_buffer failed on %s for %s"),oid_to_hex(&oi->oid),ref->refname);}-grab_values(ref->value,deref,*obj,oi->content);+grab_values(ref->value,deref,*obj,oi);}grab_common_values(ref->value,deref,oi);
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-25 16:02:39
From: ZheNing Hu <redacted>
Because the perl language can handle binary data correctly,
add the function perl_quote_buf_with_len(), which can specify
the length of the data and prevent the data from being truncated
at '\0' to help `--format="%(raw)"` re-support `--perl`.
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
Documentation/git-for-each-ref.txt | 2 +-
quote.c | 17 +++++++++++++++++
quote.h | 1 +
ref-filter.c | 15 +++++++++++----
t/t6300-for-each-ref.sh | 19 +++++++++++++++++--
5 files changed, 47 insertions(+), 7 deletions(-)
@@ -241,7 +241,7 @@ raw:size:: The raw data size of the object. Note that `--format=%(raw)` can not be used with `--python`, `--shell`, `--tcl`,-`--perl` because the host language may not support arbitrary binary data in the+because the host language may not support arbitrary binary data in the variables of its string type. The message in a commit or a tag object is `contents`, from which
@@ -94,6 +94,7 @@ char *quote_path(const char *in, const char *prefix, struct strbuf *out, unsigne/* quoting as a string literal for other languages */voidperl_quote_buf(structstrbuf*sb,constchar*src);+voidperl_quote_buf_with_len(structstrbuf*sb,constchar*src,size_tlen);voidpython_quote_buf(structstrbuf*sb,constchar*src);voidtcl_quote_buf(structstrbuf*sb,constchar*src);voidbasic_regex_quote_buf(structstrbuf*sb,constchar*src);
@@ -559,6 +560,15 @@ static int if_atom_parser(struct ref_format *format, struct used_atom *atom,return0;}+staticintrest_atom_parser(structref_format*format,structused_atom*atom,+constchar*arg,structstrbuf*err)+{+if(arg)+returnstrbuf_addf_ret(err,-1,_("%%(rest) does not take arguments"));+format->use_rest=1;+return0;+}+staticinthead_atom_parser(structref_format*format,structused_atom*atom,constchar*arg,structstrbuf*unused_err){
@@ -1998,6 +1998,10 @@ test_expect_success '--format should list tags as per format given' 'test_cmpexpectactual'+test_expect_success'git tag -l with --format="%(rest)" must fail''+test_must_failgittag-l--format="%(rest)""v1*"+'+ test_expect_success"set up color tests"'echo"<RED>v1.0<RESET>">expect.color&&echo"v1.0">expect.bare&&
@@ -194,6 +194,10 @@ test_expect_success GPG 'verifying tag with --format' 'test_cmpexpectactual'+test_expect_successGPG'verifying tag with --format="%(rest)" must fail''+test_must_failgitverify-tag--format="%(rest)""fourth-signed"+'+ test_expect_successGPG'verifying a forged tag with --format should fail silently''test_must_failgitverify-tag--format="tagname : %(tag)"$(catforged1.tag)>actual-forged&&test_must_be_emptyactual-forged
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-25 16:02:45
From: ZheNing Hu <redacted>
Use non-const ref_format in *_atom_parser(), which can help us
modify the members of ref_format in *_atom_parser().
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
builtin/tag.c | 2 +-
ref-filter.c | 44 ++++++++++++++++++++++----------------------
ref-filter.h | 4 ++--
3 files changed, 25 insertions(+), 25 deletions(-)
@@ -116,7 +116,7 @@ void ref_array_sort(struct ref_sorting *sort, struct ref_array *array);voidref_sorting_set_sort_flags_all(structref_sorting*sorting,unsignedintmask,inton);/* Based on the given format and quote_style, fill the strbuf */intformat_ref_array_item(structref_array_item*info,-conststructref_format*format,+structref_format*format,structstrbuf*final_buf,structstrbuf*error_buf);/* Parse a single sort specifier and add it to the list */
From: ZheNing Hu via GitGitGadget <hidden> Date: 2021-06-25 16:02:48
From: ZheNing Hu <redacted>
Because in the refactor of `git cat-file --batch` later,
oid_object_info_extended() in get_object() will be used to obtain
the info of an object with it's oid. When the object cannot be
obtained in the git repository, `cat-file --batch` expects to output
"<oid> missing" and continue the next oid query instead of letting
Git exit. In other error conditions, Git should exit normally. So we
can achieve this function by passing the return value of get_object().
Mentored-by: Christian Couder [off-list ref]
Mentored-by: Hariom Verma [off-list ref]
Helped-by: Ævar Arnfjörð Bjarmason [off-list ref]
Signed-off-by: ZheNing Hu <redacted>
---
ref-filter.c | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)