The `git-for-each-ref(1)` command is used to iterate over references
present in a repository. In large repositories with millions of
references, it would be optimal to paginate this output such that we
can start iteration from a given reference. This would avoid having to
iterate over all references from the beginning each time when paginating
through results.
The previous commit added 'seek' functionality to the reference
backends. Utilize this and expose a '--skip-until' option in
'git-for-each-ref(1)'. When used, the reference iteration seeks to the
first matching reference and iterates from there onward.
This enables efficient pagination workflows like:
git for-each-ref --count=100
git for-each-ref --count=100 --skip-until=refs/heads/branch-100
git for-each-ref --count=100 --skip-until=refs/heads/branch-200
Signed-off-by: Karthik Nayak <karthik.188@xxxxxxxxx>
---
Documentation/git-for-each-ref.adoc | 6 +-
builtin/for-each-ref.c | 5 +
ref-filter.c | 57 ++++++++----
ref-filter.h | 1 +
t/t6302-for-each-ref-filter.sh | 180 ++++++++++++++++++++++++++++++++++++
5 files changed, 230 insertions(+), 19 deletions(-)
diff --git a/Documentation/git-for-each-ref.adoc b/Documentation/git-for-each-ref.adoc
index 5ef89fc0fe..4bf7c66b8c 100644
--- a/Documentation/git-for-each-ref.adoc
+++ b/Documentation/git-for-each-ref.adoc
@@ -14,7 +14,7 @@ SYNOPSIS
[--points-at=<object>]
[--merged[=<object>]] [--no-merged[=<object>]]
[--contains[=<object>]] [--no-contains[=<object>]]
- [--exclude=<pattern> ...]
+ [--exclude=<pattern> ...] [--skip-until=<pattern>]
DESCRIPTION
-----------
@@ -108,6 +108,10 @@ TAB %(refname)`.
--include-root-refs::
List root refs (HEAD and pseudorefs) apart from regular refs.
+--skip-until::
+ Skip references up to the specified pattern. Cannot be used with
+ general pattern matching.
+
FIELD NAMES
-----------
diff --git a/builtin/for-each-ref.c b/builtin/for-each-ref.c
index 3d2207ec77..543013cd11 100644
--- a/builtin/for-each-ref.c
+++ b/builtin/for-each-ref.c
@@ -13,6 +13,7 @@ static char const * const for_each_ref_usage[] = {
N_("git for-each-ref [--points-at <object>]"),
N_("git for-each-ref [--merged [<commit>]] [--no-merged [<commit>]]"),
N_("git for-each-ref [--contains [<commit>]] [--no-contains [<commit>]]"),
+ N_("git for-each-ref [--skip-until <pattern>]"),
NULL
};
@@ -44,6 +45,7 @@ int cmd_for_each_ref(int argc,
OPT_GROUP(""),
OPT_INTEGER( 0 , "count", &format.array_opts.max_count, N_("show only <n> matched refs")),
OPT_STRING( 0 , "format", &format.format, N_("format"), N_("format to use for the output")),
+ OPT_STRING( 0 , "skip-until", &filter.seek, N_("skip-until"), N_("skip references until")),
OPT__COLOR(&format.use_color, N_("respect format colors")),
OPT_REF_FILTER_EXCLUDE(&filter),
OPT_REF_SORT(&sorting_options),
@@ -100,6 +102,9 @@ int cmd_for_each_ref(int argc,
filter.name_patterns = argv;
}
+ if (filter.seek && filter.name_patterns && filter.name_patterns[0])
+ die(_("cannot use --skip-until with patterns"));
+
if (include_root_refs)
flags |= FILTER_REFS_ROOT_REFS | FILTER_REFS_DETACHED_HEAD;
diff --git a/ref-filter.c b/ref-filter.c
index 7a274633cf..9d0255d5db 100644
--- a/ref-filter.c
+++ b/ref-filter.c
@@ -2692,10 +2692,13 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
each_ref_fn cb,
void *cb_data)
{
+ struct ref_iterator *iter;
+ int flags = 0, ret = 0;
+
if (filter->kind & FILTER_REFS_ROOT_REFS) {
/* In this case, we want to print all refs including root refs. */
- return refs_for_each_include_root_refs(get_main_ref_store(the_repository),
- cb, cb_data);
+ flags |= DO_FOR_EACH_INCLUDE_ROOT_REFS;
+ goto non_prefix_iter;
}
if (!filter->match_as_path) {
@@ -2704,8 +2707,7 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
* prefixes like "refs/heads/" etc. are stripped off,
* so we have to look at everything:
*/
- return refs_for_each_fullref_in(get_main_ref_store(the_repository),
- "", NULL, cb, cb_data);
+ goto non_prefix_iter;
}
if (filter->ignore_case) {
@@ -2714,20 +2716,28 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
* so just return everything and let the caller
* sort it out.
*/
- return refs_for_each_fullref_in(get_main_ref_store(the_repository),
- "", NULL, cb, cb_data);
+ goto non_prefix_iter;
}
if (!filter->name_patterns[0]) {
/* no patterns; we have to look at everything */
- return refs_for_each_fullref_in(get_main_ref_store(the_repository),
- "", filter->exclude.v, cb, cb_data);
+ goto non_prefix_iter;
}
return refs_for_each_fullref_in_prefixes(get_main_ref_store(the_repository),
NULL, filter->name_patterns,
filter->exclude.v,
cb, cb_data);
+
+non_prefix_iter:
+ iter = refs_ref_iterator_begin(get_main_ref_store(the_repository), "",
+ NULL, 0, flags);
+ if (filter->seek)
+ ret = ref_iterator_seek(iter, filter->seek, 0);
+ if (ret)
+ return ret;
+
+ return do_for_each_ref_iterator(iter, cb, cb_data);
}
/*
@@ -3200,6 +3210,8 @@ static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref
if (!filter->kind)
die("filter_refs: invalid type");
else {
+ const char *prefix = NULL;
+
/*
* For common cases where we need only branches or remotes or tags,
* we only iterate through those refs. If a mix of refs is needed,
@@ -3207,19 +3219,28 @@ static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref
* of filter_ref_kind().
*/
if (filter->kind == FILTER_REFS_BRANCHES)
- ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
- "refs/heads/", NULL,
- fn, cb_data);
+ prefix = "refs/heads/";
else if (filter->kind == FILTER_REFS_REMOTES)
- ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
- "refs/remotes/", NULL,
- fn, cb_data);
+ prefix = "refs/remotes/";
else if (filter->kind == FILTER_REFS_TAGS)
- ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
- "refs/tags/", NULL, fn,
- cb_data);
- else if (filter->kind & FILTER_REFS_REGULAR)
+ prefix = "refs/tags/";
+
+ if (prefix) {
+ struct ref_iterator *iter;
+
+ iter = refs_ref_iterator_begin(get_main_ref_store(the_repository),
+ "", NULL, 0, 0);
+
+ if (filter->seek)
+ ret = ref_iterator_seek(iter, filter->seek, 0);
+ else if (prefix)
+ ret = ref_iterator_seek(iter, prefix, 1);
+
+ if (!ret)
+ ret = do_for_each_ref_iterator(iter, fn, cb_data);
+ } else if (filter->kind & FILTER_REFS_REGULAR) {
ret = for_each_fullref_in_pattern(filter, fn, cb_data);
+ }
/*
* When printing all ref types, HEAD is already included,
diff --git a/ref-filter.h b/ref-filter.h
index c98c4fbd4c..9e97c65bc2 100644
--- a/ref-filter.h
+++ b/ref-filter.h
@@ -64,6 +64,7 @@ struct ref_array {
struct ref_filter {
const char **name_patterns;
+ const char *seek;
struct strvec exclude;
struct oid_array points_at;
struct commit_list *with_commit;
diff --git a/t/t6302-for-each-ref-filter.sh b/t/t6302-for-each-ref-filter.sh
index bb02b86c16..af2c60a2ce 100755
--- a/t/t6302-for-each-ref-filter.sh
+++ b/t/t6302-for-each-ref-filter.sh
@@ -541,4 +541,184 @@ test_expect_success 'validate worktree atom' '
test_cmp expect actual
'
+test_expect_success 'skip until with empty value' '
+ cat >expect <<-\EOF &&
+ refs/heads/main
+ refs/heads/main_worktree
+ refs/heads/side
+ refs/odd/spot
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until="" >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until to a specific reference' '
+ cat >expect <<-\EOF &&
+ refs/odd/spot
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until=refs/odd/spot >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until to a specific reference with partial match' '
+ cat >expect <<-\EOF &&
+ refs/odd/spot
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until=refs/odd/sp >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until just behind a specific reference' '
+ cat >expect <<-\EOF &&
+ refs/odd/spot
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until=refs/odd/parrot >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until to specific directory' '
+ cat >expect <<-\EOF &&
+ refs/odd/spot
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until=refs/odd >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until to specific directory with trailing slash' '
+ cat >expect <<-\EOF &&
+ refs/odd/spot
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until=refs/lost >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until just behind a specific directory' '
+ cat >expect <<-\EOF &&
+ refs/odd/spot
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until=refs/odd/ >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until overflow specific reference length' '
+ cat >expect <<-\EOF &&
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until=refs/odd/spotnew >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until overflow specific reference path' '
+ cat >expect <<-\EOF &&
+ refs/tags/annotated-tag
+ refs/tags/doubly-annotated-tag
+ refs/tags/doubly-signed-tag
+ refs/tags/foo1.10
+ refs/tags/foo1.3
+ refs/tags/foo1.6
+ refs/tags/four
+ refs/tags/one
+ refs/tags/signed-tag
+ refs/tags/three
+ refs/tags/two
+ EOF
+ git for-each-ref --format="%(refname)" --skip-until=refs/odd/spot/new >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'skip until used with a pattern' '
+ cat >expect <<-\EOF &&
+ fatal: cannot use --skip-until with patterns
+ EOF
+ test_must_fail git for-each-ref --format="%(refname)" --skip-until=refs/odd/spot refs/tags 2>actual &&
+ test_cmp expect actual
+'
+
test_done
--
2.49.0
