[RFC PATCH 10/18] midx: use existing midx when writing

[Derrick Stolee <stolee@xxxxxxxxx>]

When writing a new MIDX file, it is faster to use an existing MIDX file
to load the object list and pack offsets and to only inspect pack-indexes
for packs not already covered by the MIDX file.

Signed-off-by: Derrick Stolee <dstolee@xxxxxxxxxxxxx>
---
 builtin/midx.c | 34 +++++++++++++++++++++++++++++++---
 midx.c         | 23 +++++++++++++++++++++++
 midx.h         |  2 ++
 3 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/builtin/midx.c b/builtin/midx.c
index ee9234583d..aff2085771 100644
--- a/builtin/midx.c
+++ b/builtin/midx.c
@@ -73,7 +73,7 @@ static int midx_read(void)
 static int build_midx_from_packs(
 	const char *pack_dir,
 	const char **pack_names, uint32_t nr_packs,
-	const char **midx_id)
+	const char **midx_id, struct midxed_git *midx)
 {
 	struct packed_git **packs;
 	const char **installed_pack_names;
@@ -86,6 +86,9 @@ static int build_midx_from_packs(
 	struct strbuf pack_path = STRBUF_INIT;
 	int baselen;
 
+	if (midx)
+		nr_total_packs += midx->num_packs;
+
 	if (!nr_total_packs) {
 		*midx_id = NULL;
 		return 0;
@@ -94,6 +97,12 @@ static int build_midx_from_packs(
 	ALLOC_ARRAY(packs, nr_total_packs);
 	ALLOC_ARRAY(installed_pack_names, nr_total_packs);
 
+	if (midx) {
+		for (i = 0; i < midx->num_packs; i++)
+			installed_pack_names[nr_installed_packs++] = midx->pack_names[i];
+		pack_offset = midx->num_packs;
+	}
+
 	strbuf_addstr(&pack_path, pack_dir);
 	strbuf_addch(&pack_path, '/');
 	baselen = pack_path.len;
@@ -101,6 +110,9 @@ static int build_midx_from_packs(
 		strbuf_setlen(&pack_path, baselen);
 		strbuf_addstr(&pack_path, pack_names[i]);
 
+		if (midx && contains_pack(midx, pack_names[i]))
+			continue;
+
 		strbuf_strip_suffix(&pack_path, ".pack");
 		strbuf_addstr(&pack_path, ".idx");
 
@@ -120,13 +132,24 @@ static int build_midx_from_packs(
 	if (!nr_objects || !nr_installed_packs) {
 		FREE_AND_NULL(packs);
 		FREE_AND_NULL(installed_pack_names);
-		*midx_id = NULL;
+
+		if (opts.has_existing)
+			*midx_id = oid_to_hex(&opts.old_midx_oid);
+		else
+			*midx_id = NULL;
+
 		return 0;
 	}
 
+	if (midx)
+		nr_objects += midx->num_objects;
+
 	ALLOC_ARRAY(objects, nr_objects);
 	nr_objects = 0;
 
+	for (i = 0; midx && i < midx->num_objects; i++)
+		nth_midxed_object_entry(midx, i, &objects[nr_objects++]);
+
 	for (i = pack_offset; i < nr_installed_packs; i++) {
 		struct packed_git *p = packs[i];
 
@@ -184,6 +207,10 @@ static int midx_write(void)
 	const char *midx_id = 0;
 	DIR *dir;
 	struct dirent *de;
+	struct midxed_git *midx = NULL;
+
+	if (opts.has_existing)
+		midx = get_midxed_git(opts.pack_dir, &opts.old_midx_oid);
 
 	dir = opendir(opts.pack_dir);
 	if (!dir) {
@@ -212,7 +239,8 @@ static int midx_write(void)
 	if (!nr_packs)
 		goto cleanup;
 
-	if (build_midx_from_packs(opts.pack_dir, pack_names, nr_packs, &midx_id))
+	if (build_midx_from_packs(opts.pack_dir, pack_names,
+				  nr_packs, &midx_id, midx))
 		die("failed to build MIDX");
 
 	if (midx_id == NULL)
diff --git a/midx.c b/midx.c
index 4e0df0285a..53eb29dac3 100644
--- a/midx.c
+++ b/midx.c
@@ -257,6 +257,29 @@ const struct object_id *nth_midxed_object_oid(struct object_id *oid,
 	return oid;
 }
 
+int contains_pack(struct midxed_git *m, const char *pack_name)
+{
+	uint32_t first = 0, last = m->num_packs;
+
+	while (first < last) {
+		uint32_t mid = first + (last - first) / 2;
+		const char *current;
+		int cmp;
+
+		current = m->pack_names[mid];
+		cmp = strcmp(pack_name, current);
+		if (!cmp)
+			return 1;
+		if (cmp > 0) {
+			first = mid + 1;
+			continue;
+		}
+		last = mid;
+	}
+
+	return 0;
+}
+
 static int midx_sha1_compare(const void *_a, const void *_b)
 {
 	struct pack_midx_entry *a = *(struct pack_midx_entry **)_a;
diff --git a/midx.h b/midx.h
index 9255909ae8..1e7a94651c 100644
--- a/midx.h
+++ b/midx.h
@@ -100,6 +100,8 @@ extern const struct object_id *nth_midxed_object_oid(struct object_id *oid,
 						     struct midxed_git *m,
 						     uint32_t n);
 
+extern int contains_pack(struct midxed_git *m, const char *pack_name);
+
 /*
  * Write a single MIDX file storing the given entries for the
  * given list of packfiles. If midx_name is null, then a temp
-- 
2.15.0