Hi John
On 07/02/2022 16:33, John Cai via GitGitGadget wrote:
From: John Cai <johncai86@xxxxxxxxx>
Add a new flag --batch-command that accepts commands and arguments
from stdin, similar to git-update-ref --stdin.
At GitLab, we use a pair of long running cat-file processes when
accessing object content. One for iterating over object metadata with
--batch-check, and the other to grab object contents with --batch.
However, if we had --batch-command, we wouldn't need to keep both
processes around, and instead just have one --batch-command process
where we can flip between getting object info, and getting object
contents. Since we have a pair of cat-file processes per repository,
this means we can get rid of roughly half of long lived git cat-file
processes. Given there are many repositories being accessed at any given
time, this can lead to huge savings since on a given server.
git cat-file --batch-command
will enter an interactive command mode whereby the user can enter in
commands and their arguments that get queued in memory:
<command1> [arg1] [arg2] NL
<command2> [arg1] [arg2] NL
When --buffer mode is used, commands will be queued in memory until a
flush command is issued that execute them:
flush NL
The reason for a flush command is that when a consumer process (A)
talks to a git cat-file process (B) and interactively writes to and
reads from it in --buffer mode, (A) needs to be able to control when
the buffer is flushed to stdout.
Currently, from (A)'s perspective, the only way is to either
1. kill (B)'s process
2. send an invalid object to stdin.
1. is not ideal from a performance perspective as it will require
spawning a new cat-file process each time, and 2. is hacky and not a
good long term solution.
With this mechanism of queueing up commands and letting (A) issue a
flush command, process (A) can control when the buffer is flushed and
can guarantee it will receive all of the output when in --buffer mode.
This patch adds the basic structure for adding command which can be
extended in the future to add more commands. It also adds the following
two commands (on top of the flush command):
contents <object> NL
info <object> NL
The contents command takes an <object> argument and prints out the object
contents.
The info command takes a <object> argument and prints out the object
metadata.
These can be used in the following way with --buffer:
contents <sha1> NL
object <sha1> NL
There is no object command
>[...]
diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh
index 145eee11df9..c57a35ea20a 100755
--- a/t/t1006-cat-file.sh
+++ b/t/t1006-cat-file.sh
@@ -177,6 +177,20 @@ $content"
test_cmp expect actual
'
+ test -z "$content" ||
+ test_expect_success "--batch-command output of $type content is correct" '
+ maybe_remove_timestamp "$batch_output" $no_ts >expect &&
+ maybe_remove_timestamp "$(test_write_lines "contents $sha1" \
+ | git cat-file --batch-command)" $no_ts >actual &&
+ test_cmp expect actual
+ '
+
+ test_expect_success "--batch-command output of $type info is correct" '
+ echo "$sha1 $type $size" >expect &&
+ test_write_lines "info $sha1" | git cat-file --batch-command >actual &&
+ test_cmp expect actual
+ '
+
test_expect_success "custom --batch-check format" '
echo "$type $sha1" >expect &&
echo $sha1 | git cat-file --batch-check="%(objecttype) %(objectname)" >actual &&
@@ -213,6 +227,64 @@ $content"
'
}
+run_buffer_test_flush () {
+ type=$1
+ sha1=$2
+ size=$3
+
+ mkfifo input &&
+ test_when_finished 'rm input; exec 8<&-' &&
+ mkfifo output &&
+ exec 9<>output &&
+ test_when_finished 'rm output; exec 9<&-'
+ (
+ git cat-file --buffer --batch-command <input 2>err &
+ echo $! &&
+ wait $!
+ ) >&9 &
+ sh_pid=$! &&
+ read cat_file_pid <&9 &&
+ test_when_finished "kill $cat_file_pid
+ kill $sh_pid; wait $sh_pid; :" &&
+ echo "$sha1 $type $size" >expect &&
+ test_write_lines "info $sha1" flush "info $sha1" >input
+ # TODO - consume all available input, not just one
+ # line (see above).
+ # check output is flushed on exit
This test seems to have lost some code above this comment so the comment
is no longer correct - we do not test if the output is flushed on exit
and looking at the implementation I don't think it is.
Best Wishes
Phillip
+ read actual <&9 &&
+ echo "$actual" >actual &&
+ test_cmp expect actual &&
+ test_must_be_empty err
+}
+
+run_buffer_test_no_flush () {
+ type=$1
+ sha1=$2
+ size=$3
+
+ touch output &&
+ test_when_finished 'rm output' &&
+ mkfifo input &&
+ test_when_finished 'rm input' &&
+ mkfifo pid &&
+ exec 9<>pid &&
+ test_when_finished 'rm pid; exec 9<&-'
+ (
+ git cat-file --buffer --batch-command <input >output &
+ echo $! &&
+ wait $!
+ echo $?
+ ) >&9 &
+ sh_pid=$! &&
+ read cat_file_pid <&9 &&
+ test_when_finished "kill $cat_file_pid
+ kill $sh_pid; wait $sh_pid; :" &&
+ test_write_lines "info $sha1" "info $sha1" &&
+ kill $cat_file_pid &&
+ read status <&9 &&
+ test_must_be_empty output
+}
+
hello_content="Hello World"
hello_size=$(strlen "$hello_content")
hello_sha1=$(echo_without_newline "$hello_content" | git hash-object --stdin)
@@ -224,6 +296,14 @@ test_expect_success "setup" '
run_tests 'blob' $hello_sha1 $hello_size "$hello_content" "$hello_content"
+test_expect_success PIPE '--batch-command --buffer with flush for blob info' '
+ run_buffer_test_flush blob $hello_sha1 $hello_size
+'
+
+test_expect_success PIPE '--batch-command --buffer without flush for blob info' '
+ run_buffer_test_no_flush blob $hello_sha1 $hello_size false
+'
+
test_expect_success '--batch-check without %(rest) considers whole line' '
echo "$hello_sha1 blob $hello_size" >expect &&
git update-index --add --cacheinfo 100644 $hello_sha1 "white space" &&
@@ -238,6 +318,14 @@ tree_pretty_content="100644 blob $hello_sha1 hello"
run_tests 'tree' $tree_sha1 $tree_size "" "$tree_pretty_content"
+test_expect_success PIPE '--batch-command --buffer with flush for tree info' '
+ run_buffer_test_flush tree $tree_sha1 $tree_size
+'
+
+test_expect_success PIPE '--batch-command --buffer without flush for tree info' '
+ run_buffer_test_no_flush tree $tree_sha1 $tree_size false
+'
+
commit_message="Initial commit"
commit_sha1=$(echo_without_newline "$commit_message" | git commit-tree $tree_sha1)
commit_size=$(($(test_oid hexsz) + 137))
@@ -249,6 +337,14 @@ $commit_message"
run_tests 'commit' $commit_sha1 $commit_size "$commit_content" "$commit_content" 1
+test_expect_success PIPE '--batch-command --buffer with flush for commit info' '
+ run_buffer_test_flush commit $commit_sha1 $commit_size
+'
+
+test_expect_success PIPE '--batch-command --buffer without flush for commit info' '
+ run_buffer_test_no_flush commit $commit_sha1 $commit_size false
+'
+
tag_header_without_timestamp="object $hello_sha1
type blob
tag hellotag
@@ -263,11 +359,19 @@ tag_size=$(strlen "$tag_content")
run_tests 'tag' $tag_sha1 $tag_size "$tag_content" "$tag_content" 1
+test_expect_success PIPE '--batch-command --buffer with flush for tag info' '
+ run_buffer_test_flush tag $tag_sha1 $tag_size
+'
+
+test_expect_success PIPE '--batch-command --buffer without flush for tag info' '
+ run_buffer_test_no_flush tag $tag_sha1 $tag_size false
+'
+
test_expect_success \
"Reach a blob from a tag pointing to it" \
"test '$hello_content' = \"\$(git cat-file blob $tag_sha1)\""
-for batch in batch batch-check
+for batch in batch batch-check batch-command
do
for opt in t s e p
do
@@ -373,6 +477,62 @@ test_expect_success "--batch-check with multiple sha1s gives correct format" '
"$(echo_without_newline "$batch_check_input" | git cat-file --batch-check)"
'
+batch_command_info_input="info $hello_sha1
+info $tree_sha1
+info $commit_sha1
+info $tag_sha1
+info deadbeef
+info
+flush
+"
+
+test_expect_success "--batch-command with multiple info calls gives correct format" '
+ test "$batch_check_output" = "$(echo_without_newline \
+ "$batch_command_info_input" | git cat-file --batch-command --buffer)"
+'
+
+batch_command_contents_input="contents $hello_sha1
+contents $commit_sha1
+contents $tag_sha1
+contents deadbeef
+contents
+flush
+"
+
+test_expect_success "--batch-command with multiple contents calls gives correct format" '
+ test "$(maybe_remove_timestamp "$batch_output" 1)" = \
+ "$(maybe_remove_timestamp "$(echo_without_newline "$batch_command_contents_input" | git cat-file --batch-command)" 1)"
+'
+
+batch_command_mixed_input="info $hello_sha1
+contents $hello_sha1
+info $commit_sha1
+contents $commit_sha1
+info $tag_sha1
+contents $tag_sha1
+contents deadbeef
+info
+flush
+"
+
+batch_command_mixed_output="$hello_sha1 blob $hello_size
+$hello_sha1 blob $hello_size
+$hello_content
+$commit_sha1 commit $commit_size
+$commit_sha1 commit $commit_size
+$commit_content
+$tag_sha1 tag $tag_size
+$tag_sha1 tag $tag_size
+$tag_content
+deadbeef missing
+ missing"
+
+test_expect_success "--batch-command with mixed calls gives correct format" '
+ test "$(maybe_remove_timestamp "$batch_command_mixed_output" 1)" = \
+ "$(maybe_remove_timestamp "$(echo_without_newline \
+ "$batch_command_mixed_input" | git cat-file --batch-command --buffer)" 1)"
+'
+
test_expect_success 'setup blobs which are likely to delta' '
test-tool genrandom foo 10240 >foo &&
{ cat foo && echo plus; } >foo-plus &&
@@ -963,5 +1123,40 @@ test_expect_success 'cat-file --batch-all-objects --batch-check ignores replace'
echo "$orig commit $orig_size" >expect &&
test_cmp expect actual
'
+test_expect_success 'batch-command empty command' '
+ echo "" >cmd &&
+ test_expect_code 128 git cat-file --batch-command <cmd 2>err &&
+ grep -E "^fatal:.*empty command in input.*" err
+'
+
+test_expect_success 'batch-command whitespace before command' '
+ echo " info deadbeef" >cmd &&
+ test_expect_code 128 git cat-file --batch-command <cmd 2>err &&
+ grep -E "^fatal:.*whitespace before command.*" err
+'
+
+test_expect_success 'batch-command unknown command' '
+ echo unknown_command >cmd &&
+ test_expect_code 128 git cat-file --batch-command <cmd 2>err &&
+ grep -E "^fatal:.*unknown command.*" err
+'
+
+test_expect_success 'batch-command flush with arguments' '
+ echo "flush arg" >cmd &&
+ test_expect_code 128 git cat-file --batch-command --buffer <cmd 2>err &&
+ grep -E "^fatal:.*flush takes no arguments.*" err
+'
+
+test_expect_success 'batch-command flush without --buffer' '
+ echo "flush arg" >cmd &&
+ test_expect_code 128 git cat-file --batch-command <cmd 2>err &&
+ grep -E "^fatal:.*flush is only for --buffer mode.*" err
+'
+
+test_expect_success 'batch-command flush empty queue' '
+ echo flush >cmd &&
+ test_expect_code 128 git cat-file --batch-command --buffer <cmd 2>err &&
+ grep -E "^fatal:.*nothing to flush.*" err
+'
test_done
