--pack-stream --stdin assumes stdin is a stream of packs and creates
a single pack containing all objects of the stream.
Signed-off-by: Nguyán ThÃi Ngác Duy <pclouds@xxxxxxxxx>
---
Don't know if it's useful for anybody else. Might be a cheaper way
than git-repack to reduce the number of packs.
builtin/index-pack.c | 119 ++++++++++++++++++++++++++++++++++++++++++--------
1 files changed, 101 insertions(+), 18 deletions(-)
diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index 8dc5c0b..4864913 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -59,6 +59,7 @@ static int nr_deltas;
static int nr_resolved_deltas;
static int from_stdin;
+static int pack_stream;
static int strict;
static int verbose;
@@ -68,7 +69,7 @@ static struct progress *progress;
static unsigned char input_buffer[4096];
static unsigned int input_offset, input_len;
static off_t consumed_bytes;
-static git_SHA_CTX input_ctx;
+static git_SHA_CTX input_ctx, pack_stream_ctx;
static uint32_t input_crc32;
static int input_fd, output_fd, pack_fd;
@@ -115,23 +116,35 @@ static void check_objects(void)
/* Discard current buffer used content. */
-static void flush(void)
+static void flush_1(int in_pack_stream)
{
if (input_offset) {
- if (output_fd >= 0)
- write_or_die(output_fd, input_buffer, input_offset);
+ if (output_fd >= 0 &&
+ (!pack_stream || (pack_stream && in_pack_stream)))
+ write_or_die(output_fd, input_buffer, input_offset);
+
git_SHA1_Update(&input_ctx, input_buffer, input_offset);
+ if (pack_stream && in_pack_stream)
+ git_SHA1_Update(&pack_stream_ctx, input_buffer, input_offset);
memmove(input_buffer, input_buffer + input_offset, input_len);
input_offset = 0;
}
}
+static void flush(void)
+{
+ flush_1(1);
+}
+
/*
* Make sure at least "min" bytes are available in the buffer, and
* return the pointer to the buffer.
*/
-static void *fill(int min)
+static void *fill_gently(int min, int *got_eof)
{
+ if (got_eof)
+ *got_eof = 0;
+
if (min <= input_len)
return input_buffer + input_offset;
if (min > sizeof(input_buffer))
@@ -141,9 +154,16 @@ static void *fill(int min)
ssize_t ret = xread(input_fd, input_buffer + input_len,
sizeof(input_buffer) - input_len);
if (ret <= 0) {
- if (!ret)
- die("early EOF");
- die_errno("read error on input");
+ if (!ret) {
+ if (!got_eof)
+ die("early EOF");
+ else {
+ *got_eof = 1;
+ return NULL;
+ }
+ }
+ else
+ die_errno("read error on input");
}
input_len += ret;
if (from_stdin)
@@ -152,6 +172,11 @@ static void *fill(int min)
return input_buffer;
}
+static void *fill(int min)
+{
+ return fill_gently(min, NULL);
+}
+
static void use(int bytes)
{
if (bytes > input_len)
@@ -188,6 +213,8 @@ static const char *open_pack_file(const char *pack_name)
pack_fd = input_fd;
}
git_SHA1_Init(&input_ctx);
+ if (pack_stream)
+ git_SHA1_Init(&pack_stream_ctx);
return pack_name;
}
@@ -202,7 +229,7 @@ static void parse_pack_header(void)
die("pack version %"PRIu32" unsupported",
ntohl(hdr->hdr_version));
- nr_objects = ntohl(hdr->hdr_entries);
+ nr_objects += ntohl(hdr->hdr_entries);
use(sizeof(struct pack_header));
}
@@ -590,10 +617,10 @@ static int compare_delta_entry(const void *a, const void *b)
}
/* Parse all objects and return the pack content SHA1 hash */
-static void parse_pack_objects(unsigned char *sha1)
+static void parse_pack_objects(unsigned char *sha1, int start_object)
{
int i;
- struct delta_entry *delta = deltas;
+ struct delta_entry *delta = deltas + nr_deltas;
struct stat st;
/*
@@ -606,7 +633,7 @@ static void parse_pack_objects(unsigned char *sha1)
progress = start_progress(
from_stdin ? "Receiving objects" : "Indexing objects",
nr_objects);
- for (i = 0; i < nr_objects; i++) {
+ for (i = start_object; i < nr_objects; i++) {
struct object_entry *obj = &objects[i];
void *data = unpack_raw_entry(obj, &delta->base);
obj->real_type = obj->type;
@@ -630,11 +657,13 @@ static void parse_pack_objects(unsigned char *sha1)
use(20);
/* If input_fd is a file, we should have reached its end now. */
- if (fstat(input_fd, &st))
- die_errno("cannot fstat packfile");
- if (S_ISREG(st.st_mode) &&
- lseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size)
- die("pack has junk at the end");
+ if (!pack_stream) {
+ if (fstat(input_fd, &st))
+ die_errno("cannot fstat packfile");
+ if (S_ISREG(st.st_mode) &&
+ lseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size)
+ die("pack has junk at the end");
+ }
if (!nr_deltas)
return;
@@ -871,6 +900,54 @@ static int git_index_pack_config(const char *k, const char *v, void *cb)
return git_default_config(k, v, cb);
}
+static void parse_following_packs(unsigned char *pack_sha1,
+ const char *curr_pack,
+ int fix_thin_pack)
+{
+ unsigned char sha1[20];
+
+ for (;;) {
+ int got_eof;
+ int last_nr_objects = nr_objects;
+
+ /* SHA1 trailer of the previous pack */
+ consumed_bytes -= 20;
+ flush_1(0);
+
+ fill_gently(sizeof(struct pack_header), &got_eof);
+ if (got_eof) {
+ /*
+ * pretend we also consume the trailer like in
+ * single pack case, a new trailer will be
+ * rewritten by fixup_pack_header_footer().
+ */
+ consumed_bytes += 20;
+ break;
+ }
+
+ git_SHA1_Init(&input_ctx);
+ parse_pack_header();
+ /* un-use() pack headers */
+ consumed_bytes -= sizeof(struct pack_header);
+ flush_1(0);
+
+ objects = xrealloc(objects, (nr_objects + 1) * sizeof(struct object_entry));
+ deltas = xrealloc(deltas, nr_objects * sizeof(struct delta_entry));
+ parse_pack_objects(sha1, last_nr_objects);
+ }
+ git_SHA1_Final(pack_sha1, &pack_stream_ctx);
+
+ if (!fix_thin_pack ||
+ (nr_deltas - nr_resolved_deltas) == 0) {
+ unsigned char read_sha1[20];
+
+ hashcpy(read_sha1, pack_sha1);
+ fixup_pack_header_footer(output_fd, pack_sha1,
+ curr_pack, nr_objects,
+ read_sha1, consumed_bytes-20);
+ }
+}
+
int cmd_index_pack(int argc, const char **argv, const char *prefix)
{
int i, fix_thin_pack = 0;
@@ -898,6 +975,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
from_stdin = 1;
} else if (!strcmp(arg, "--fix-thin")) {
fix_thin_pack = 1;
+ } else if (!strcmp(arg, "--pack-stream")) {
+ pack_stream = 1;
} else if (!strcmp(arg, "--strict")) {
strict = 1;
} else if (!strcmp(arg, "--keep")) {
@@ -946,6 +1025,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
usage(index_pack_usage);
if (fix_thin_pack && !from_stdin)
die("--fix-thin cannot be used without --stdin");
+ if (pack_stream && !from_stdin)
+ die("--pack-stream cannot be used without --stdin");
if (!index_name && pack_name) {
int len = strlen(pack_name);
if (!has_extension(pack_name, ".pack"))
@@ -971,7 +1052,9 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
parse_pack_header();
objects = xmalloc((nr_objects + 1) * sizeof(struct object_entry));
deltas = xmalloc(nr_objects * sizeof(struct delta_entry));
- parse_pack_objects(pack_sha1);
+ parse_pack_objects(pack_sha1, 0);
+ if (pack_stream)
+ parse_following_packs(pack_sha1, curr_pack, fix_thin_pack);
if (nr_deltas == nr_resolved_deltas) {
stop_progress(&progress);
/* Flush remaining pack final 20-byte SHA1. */
--
1.7.2.2
--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html