On Fri, Mar 14, 2025 at 1:18 PM Taylor Blau <me@xxxxxxxxxxxx> wrote:
>
> Prepare the pack-bitmap machinery to work with incremental MIDXs by
> adding a new "base" field to keep track of the bitmap index associated
> with the previous MIDX layer.
>
> The changes in this commit are mostly boilerplate to open the correct
> bitmap(s), add them to the chain bitmap layers along the "base" pointer,
s/chain/chain of/ ?
> ensures that the correct packs and their reverse indexes are loaded
s/ensures/ensure/ ?
> across MIDX layers, etc.
>
> While we're at it, keep track of a base_nr field to indicate how many
> bitmap layers (including the current bitmap) exist. This will be used in
> a future commit to allocate an array of 'struct ewah_bitmap' pointers to
> collect all of the respective type bitmaps among all layers to
> initialize a multi-EWAH iterator.
>
> Subsequent commits will teach the functions within the pack-bitmap
> machinery how to interact with these new fields.
>
> Signed-off-by: Taylor Blau <me@xxxxxxxxxxxx>
> ---
> pack-bitmap.c | 62 +++++++++++++++++++++++++++++++++++++++------------
> 1 file changed, 48 insertions(+), 14 deletions(-)
>
> diff --git a/pack-bitmap.c b/pack-bitmap.c
> index c26d85b5db..72fb11d014 100644
> --- a/pack-bitmap.c
> +++ b/pack-bitmap.c
> @@ -54,6 +54,16 @@ struct bitmap_index {
> struct packed_git *pack;
> struct multi_pack_index *midx;
>
> + /*
> + * If using a multi-pack index chain, 'base' points to the
> + * bitmap index corresponding to this bitmap's midx->base_midx.
> + *
> + * base_nr indicates how many layers precede this one, and is
> + * zero when base is NULL.
> + */
> + struct bitmap_index *base;
> + uint32_t base_nr;
> +
> /* mmapped buffer of the whole bitmap index */
> unsigned char *map;
> size_t map_size; /* size of the mmaped buffer */
> @@ -386,8 +396,15 @@ static int load_bitmap_entries_v1(struct bitmap_index *index)
> char *midx_bitmap_filename(struct multi_pack_index *midx)
> {
> struct strbuf buf = STRBUF_INIT;
> - get_midx_filename_ext(midx->repo->hash_algo, &buf, midx->object_dir,
> - get_midx_checksum(midx), MIDX_EXT_BITMAP);
> + if (midx->has_chain)
> + get_split_midx_filename_ext(midx->repo->hash_algo, &buf,
> + midx->object_dir,
> + get_midx_checksum(midx),
> + MIDX_EXT_BITMAP);
> + else
> + get_midx_filename_ext(midx->repo->hash_algo, &buf,
> + midx->object_dir, get_midx_checksum(midx),
> + MIDX_EXT_BITMAP);
>
> return strbuf_detach(&buf, NULL);
> }
> @@ -454,16 +471,21 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git,
> goto cleanup;
> }
>
> - for (i = 0; i < bitmap_git->midx->num_packs; i++) {
> - if (prepare_midx_pack(bitmap_repo(bitmap_git),
> - bitmap_git->midx,
> - i)) {
> + for (i = 0; i < bitmap_git->midx->num_packs + bitmap_git->midx->num_packs_in_base; i++) {
> + if (prepare_midx_pack(bitmap_repo(bitmap_git), bitmap_git->midx, i)) {
> warning(_("could not open pack %s"),
> bitmap_git->midx->pack_names[i]);
> goto cleanup;
> }
> }
>
> + if (midx->base_midx) {
> + bitmap_git->base = prepare_midx_bitmap_git(midx->base_midx);
> + bitmap_git->base_nr = bitmap_git->base->base_nr + 1;
> + } else {
> + bitmap_git->base_nr = 0;
> + }
> +
> return 0;
>
> cleanup:
> @@ -515,6 +537,7 @@ static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git
> bitmap_git->map_size = xsize_t(st.st_size);
> bitmap_git->map = xmmap(NULL, bitmap_git->map_size, PROT_READ, MAP_PRIVATE, fd, 0);
> bitmap_git->map_pos = 0;
> + bitmap_git->base_nr = 0;
> close(fd);
>
> if (load_bitmap_header(bitmap_git) < 0) {
> @@ -534,8 +557,7 @@ static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git
> static int load_reverse_index(struct repository *r, struct bitmap_index *bitmap_git)
> {
> if (bitmap_is_midx(bitmap_git)) {
> - uint32_t i;
> - int ret;
> + struct multi_pack_index *m;
>
> /*
> * The multi-pack-index's .rev file is already loaded via
> @@ -544,10 +566,15 @@ static int load_reverse_index(struct repository *r, struct bitmap_index *bitmap_
> * But we still need to open the individual pack .rev files,
> * since we will need to make use of them in pack-objects.
> */
> - for (i = 0; i < bitmap_git->midx->num_packs; i++) {
> - ret = load_pack_revindex(r, bitmap_git->midx->packs[i]);
> - if (ret)
> - return ret;
> + for (m = bitmap_git->midx; m; m = m->base_midx) {
> + uint32_t i;
> + int ret;
> +
> + for (i = 0; i < m->num_packs; i++) {
> + ret = load_pack_revindex(r, m->packs[i]);
> + if (ret)
> + return ret;
> + }
> }
> return 0;
> }
> @@ -573,6 +600,13 @@ static int load_bitmap(struct repository *r, struct bitmap_index *bitmap_git)
> if (!bitmap_git->table_lookup && load_bitmap_entries_v1(bitmap_git) < 0)
> goto failed;
>
> + if (bitmap_git->base) {
> + if (!bitmap_is_midx(bitmap_git))
> + BUG("non-MIDX bitmap has non-NULL base bitmap index");
> + if (load_bitmap(r, bitmap_git->base) < 0)
> + goto failed;
> + }
> +
> return 0;
>
> failed:
> @@ -657,10 +691,9 @@ struct bitmap_index *prepare_bitmap_git(struct repository *r)
>
> struct bitmap_index *prepare_midx_bitmap_git(struct multi_pack_index *midx)
> {
> - struct repository *r = midx->repo;
> struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git));
>
> - if (!open_midx_bitmap_1(bitmap_git, midx) && !load_bitmap(r, bitmap_git))
> + if (!open_midx_bitmap_1(bitmap_git, midx))
> return bitmap_git;
>
> free_bitmap_index(bitmap_git);
> @@ -2899,6 +2932,7 @@ void free_bitmap_index(struct bitmap_index *b)
> close_midx_revindex(b->midx);
> }
> free_pseudo_merge_map(&b->pseudo_merges);
> + free_bitmap_index(b->base);
> free(b);
> }
>
> --
> 2.49.0.13.gd0d564685b
>
