On Fri, 1 Jun 2007, Geert Bosch wrote:
> This patch unifies the write_index_file functions in
> builtin-pack-objects.c and index-pack.c. As the name
> "index" is overloaded in git, move in the direction of
> using "idx" and "pack idx" when refering to the pack index.
> There should be no change in functionality.
>
> Signed-off-by: Geert Bosch <bosch@xxxxxxxx>
Acked-by: Nicolas Pitre <nico@xxxxxxx>
> ---
> builtin-pack-objects.c | 218 +++++++++++-------------------------------------
> index-pack.c | 208 ++++++++-------------------------------------
> pack-write.c | 142 +++++++++++++++++++++++++++++++
> pack.h | 14 +++
> 4 files changed, 243 insertions(+), 339 deletions(-)
>
> Now with corrected commit message.
>
> -Geert
>
> diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
> index e52332d..a247238 100644
> --- a/builtin-pack-objects.c
> +++ b/builtin-pack-objects.c
> @@ -23,10 +23,9 @@ git-pack-objects [{ -q | --progress | --all-progress }] [--max-pack-size=N] \n\
> [--stdout | base-name] [<ref-list | <object-list]";
>
> struct object_entry {
> - unsigned char sha1[20];
> - uint32_t crc32; /* crc of raw pack data for this object */
> - off_t offset; /* offset into the final pack file */
> + struct pack_idx_entry idx;
> unsigned long size; /* uncompressed size */
> +
> unsigned int hash; /* name hint hash */
> unsigned int depth; /* delta depth */
> struct packed_git *in_pack; /* already in pack */
> @@ -65,7 +64,6 @@ static int allow_ofs_delta;
> static const char *pack_tmp_name, *idx_tmp_name;
> static char tmpname[PATH_MAX];
> static const char *base_name;
> -static unsigned char pack_file_sha1[20];
> static int progress = 1;
> static int window = 10;
> static uint32_t pack_size_limit;
> @@ -241,11 +239,11 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
> {
> unsigned long othersize, delta_size;
> enum object_type type;
> - void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize);
> + void *otherbuf = read_sha1_file(entry->delta->idx.sha1, &type, &othersize);
> void *delta_buf;
>
> if (!otherbuf)
> - die("unable to read %s", sha1_to_hex(entry->delta->sha1));
> + die("unable to read %s", sha1_to_hex(entry->delta->idx.sha1));
> delta_buf = diff_delta(otherbuf, othersize,
> buf, size, &delta_size, 0);
> if (!delta_buf || delta_size != entry->delta_size)
> @@ -374,11 +372,11 @@ static unsigned long write_object(struct sha1file *f,
> /* yes if unlimited packfile */
> !pack_size_limit ? 1 :
> /* no if base written to previous pack */
> - entry->delta->offset == (off_t)-1 ? 0 :
> + entry->delta->idx.offset == (off_t)-1 ? 0 :
> /* otherwise double-check written to this
> * pack, like we do below
> */
> - entry->delta->offset ? 1 : 0;
> + entry->delta->idx.offset ? 1 : 0;
>
> if (!pack_to_stdout)
> crc32_begin(f);
> @@ -405,16 +403,16 @@ static unsigned long write_object(struct sha1file *f,
> z_stream stream;
> unsigned long maxsize;
> void *out;
> - buf = read_sha1_file(entry->sha1, &type, &size);
> + buf = read_sha1_file(entry->idx.sha1, &type, &size);
> if (!buf)
> - die("unable to read %s", sha1_to_hex(entry->sha1));
> + die("unable to read %s", sha1_to_hex(entry->idx.sha1));
> if (size != entry->size)
> die("object %s size inconsistency (%lu vs %lu)",
> - sha1_to_hex(entry->sha1), size, entry->size);
> + sha1_to_hex(entry->idx.sha1), size, entry->size);
> if (usable_delta) {
> buf = delta_against(buf, size, entry);
> size = entry->delta_size;
> - obj_type = (allow_ofs_delta && entry->delta->offset) ?
> + obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
> OBJ_OFS_DELTA : OBJ_REF_DELTA;
> } else {
> /*
> @@ -451,7 +449,7 @@ static unsigned long write_object(struct sha1file *f,
> * encoding of the relative offset for the delta
> * base from this object's position in the pack.
> */
> - off_t ofs = entry->offset - entry->delta->offset;
> + off_t ofs = entry->idx.offset - entry->delta->idx.offset;
> unsigned pos = sizeof(dheader) - 1;
> dheader[pos] = ofs & 127;
> while (ofs >>= 7)
> @@ -475,7 +473,7 @@ static unsigned long write_object(struct sha1file *f,
> return 0;
> }
> sha1write(f, header, hdrlen);
> - sha1write(f, entry->delta->sha1, 20);
> + sha1write(f, entry->delta->idx.sha1, 20);
> hdrlen += 20;
> } else {
> if (limit && hdrlen + datalen + 20 >= limit) {
> @@ -496,7 +494,7 @@ static unsigned long write_object(struct sha1file *f,
> off_t offset;
>
> if (entry->delta) {
> - obj_type = (allow_ofs_delta && entry->delta->offset) ?
> + obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
> OBJ_OFS_DELTA : OBJ_REF_DELTA;
> reused_delta++;
> }
> @@ -506,11 +504,11 @@ static unsigned long write_object(struct sha1file *f,
> datalen = revidx[1].offset - offset;
> if (!pack_to_stdout && p->index_version > 1 &&
> check_pack_crc(p, &w_curs, offset, datalen, revidx->nr))
> - die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
> + die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1));
> offset += entry->in_pack_header_size;
> datalen -= entry->in_pack_header_size;
> if (obj_type == OBJ_OFS_DELTA) {
> - off_t ofs = entry->offset - entry->delta->offset;
> + off_t ofs = entry->idx.offset - entry->delta->idx.offset;
> unsigned pos = sizeof(dheader) - 1;
> dheader[pos] = ofs & 127;
> while (ofs >>= 7)
> @@ -524,7 +522,7 @@ static unsigned long write_object(struct sha1file *f,
> if (limit && hdrlen + 20 + datalen + 20 >= limit)
> return 0;
> sha1write(f, header, hdrlen);
> - sha1write(f, entry->delta->sha1, 20);
> + sha1write(f, entry->delta->idx.sha1, 20);
> hdrlen += 20;
> } else {
> if (limit && hdrlen + datalen + 20 >= limit)
> @@ -534,7 +532,7 @@ static unsigned long write_object(struct sha1file *f,
>
> if (!pack_to_stdout && p->index_version == 1 &&
> check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
> - die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
> + die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1));
> copy_pack_data(f, p, &w_curs, offset, datalen);
> unuse_pack(&w_curs);
> reused++;
> @@ -543,7 +541,7 @@ static unsigned long write_object(struct sha1file *f,
> written_delta++;
> written++;
> if (!pack_to_stdout)
> - entry->crc32 = crc32_end(f);
> + entry->idx.crc32 = crc32_end(f);
> return hdrlen + datalen;
> }
>
> @@ -554,7 +552,7 @@ static off_t write_one(struct sha1file *f,
> unsigned long size;
>
> /* offset is non zero if object is written already. */
> - if (e->offset || e->preferred_base)
> + if (e->idx.offset || e->preferred_base)
> return offset;
>
> /* if we are deltified, write out base object first. */
> @@ -564,10 +562,10 @@ static off_t write_one(struct sha1file *f,
> return 0;
> }
>
> - e->offset = offset;
> + e->idx.offset = offset;
> size = write_object(f, e, offset);
> if (!size) {
> - e->offset = 0;
> + e->idx.offset = 0;
> return 0;
> }
> written_list[nr_written++] = e;
> @@ -584,8 +582,7 @@ static int open_object_dir_tmp(const char *path)
> return mkstemp(tmpname);
> }
>
> -/* forward declarations for write_pack_file */
> -static void write_index_file(off_t last_obj_offset, unsigned char *sha1);
> +/* forward declaration for write_pack_file */
> static int adjust_perm(const char *path, mode_t mode);
>
> static void write_pack_file(void)
> @@ -602,6 +599,8 @@ static void write_pack_file(void)
> written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
>
> do {
> + unsigned char sha1[20];
> +
> if (pack_to_stdout) {
> f = sha1fd(1, "<stdout>");
> } else {
> @@ -633,23 +632,23 @@ static void write_pack_file(void)
> * If so, rewrite it like in fast-import
> */
> if (pack_to_stdout || nr_written == nr_remaining) {
> - sha1close(f, pack_file_sha1, 1);
> + sha1close(f, sha1, 1);
> } else {
> - sha1close(f, pack_file_sha1, 0);
> - fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written);
> + sha1close(f, sha1, 0);
> + fixup_pack_header_footer(f->fd, sha1, pack_tmp_name, nr_written);
> close(f->fd);
> }
>
> if (!pack_to_stdout) {
> - unsigned char object_list_sha1[20];
> mode_t mode = umask(0);
>
> umask(mode);
> mode = 0444 & ~mode;
>
> - write_index_file(last_obj_offset, object_list_sha1);
> + idx_tmp_name = write_idx_file(NULL,
> + (struct pack_idx_entry **) written_list, nr_written, sha1);
> snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
> - base_name, sha1_to_hex(object_list_sha1));
> + base_name, sha1_to_hex(sha1));
> if (adjust_perm(pack_tmp_name, mode))
> die("unable to make temporary pack file readable: %s",
> strerror(errno));
> @@ -657,19 +656,19 @@ static void write_pack_file(void)
> die("unable to rename temporary pack file: %s",
> strerror(errno));
> snprintf(tmpname, sizeof(tmpname), "%s-%s.idx",
> - base_name, sha1_to_hex(object_list_sha1));
> + base_name, sha1_to_hex(sha1));
> if (adjust_perm(idx_tmp_name, mode))
> die("unable to make temporary index file readable: %s",
> strerror(errno));
> if (rename(idx_tmp_name, tmpname))
> die("unable to rename temporary index file: %s",
> strerror(errno));
> - puts(sha1_to_hex(object_list_sha1));
> + puts(sha1_to_hex(sha1));
> }
>
> /* mark written objects as written to previous pack */
> for (j = 0; j < nr_written; j++) {
> - written_list[j]->offset = (off_t)-1;
> + written_list[j]->idx.offset = (off_t)-1;
> }
> nr_remaining -= nr_written;
> } while (nr_remaining && i < nr_objects);
> @@ -687,129 +686,12 @@ static void write_pack_file(void)
> */
> for (j = 0; i < nr_objects; i++) {
> struct object_entry *e = objects + i;
> - j += !e->offset && !e->preferred_base;
> + j += !e->idx.offset && !e->preferred_base;
> }
> if (j)
> die("wrote %u objects as expected but %u unwritten", written, j);
> }
>
> -static int sha1_sort(const void *_a, const void *_b)
> -{
> - const struct object_entry *a = *(struct object_entry **)_a;
> - const struct object_entry *b = *(struct object_entry **)_b;
> - return hashcmp(a->sha1, b->sha1);
> -}
> -
> -static uint32_t index_default_version = 1;
> -static uint32_t index_off32_limit = 0x7fffffff;
> -
> -static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
> -{
> - struct sha1file *f;
> - struct object_entry **sorted_by_sha, **list, **last;
> - uint32_t array[256];
> - uint32_t i, index_version;
> - SHA_CTX ctx;
> -
> - int fd = open_object_dir_tmp("tmp_idx_XXXXXX");
> - if (fd < 0)
> - die("unable to create %s: %s\n", tmpname, strerror(errno));
> - idx_tmp_name = xstrdup(tmpname);
> - f = sha1fd(fd, idx_tmp_name);
> -
> - if (nr_written) {
> - sorted_by_sha = written_list;
> - qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
> - list = sorted_by_sha;
> - last = sorted_by_sha + nr_written;
> - } else
> - sorted_by_sha = list = last = NULL;
> -
> - /* if last object's offset is >= 2^31 we should use index V2 */
> - index_version = (last_obj_offset >> 31) ? 2 : index_default_version;
> -
> - /* index versions 2 and above need a header */
> - if (index_version >= 2) {
> - struct pack_idx_header hdr;
> - hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
> - hdr.idx_version = htonl(index_version);
> - sha1write(f, &hdr, sizeof(hdr));
> - }
> -
> - /*
> - * Write the first-level table (the list is sorted,
> - * but we use a 256-entry lookup to be able to avoid
> - * having to do eight extra binary search iterations).
> - */
> - for (i = 0; i < 256; i++) {
> - struct object_entry **next = list;
> - while (next < last) {
> - struct object_entry *entry = *next;
> - if (entry->sha1[0] != i)
> - break;
> - next++;
> - }
> - array[i] = htonl(next - sorted_by_sha);
> - list = next;
> - }
> - sha1write(f, array, 256 * 4);
> -
> - /* Compute the SHA1 hash of sorted object names. */
> - SHA1_Init(&ctx);
> -
> - /* Write the actual SHA1 entries. */
> - list = sorted_by_sha;
> - for (i = 0; i < nr_written; i++) {
> - struct object_entry *entry = *list++;
> - if (index_version < 2) {
> - uint32_t offset = htonl(entry->offset);
> - sha1write(f, &offset, 4);
> - }
> - sha1write(f, entry->sha1, 20);
> - SHA1_Update(&ctx, entry->sha1, 20);
> - }
> -
> - if (index_version >= 2) {
> - unsigned int nr_large_offset = 0;
> -
> - /* write the crc32 table */
> - list = sorted_by_sha;
> - for (i = 0; i < nr_written; i++) {
> - struct object_entry *entry = *list++;
> - uint32_t crc32_val = htonl(entry->crc32);
> - sha1write(f, &crc32_val, 4);
> - }
> -
> - /* write the 32-bit offset table */
> - list = sorted_by_sha;
> - for (i = 0; i < nr_written; i++) {
> - struct object_entry *entry = *list++;
> - uint32_t offset = (entry->offset <= index_off32_limit) ?
> - entry->offset : (0x80000000 | nr_large_offset++);
> - offset = htonl(offset);
> - sha1write(f, &offset, 4);
> - }
> -
> - /* write the large offset table */
> - list = sorted_by_sha;
> - while (nr_large_offset) {
> - struct object_entry *entry = *list++;
> - uint64_t offset = entry->offset;
> - if (offset > index_off32_limit) {
> - uint32_t split[2];
> - split[0] = htonl(offset >> 32);
> - split[1] = htonl(offset & 0xffffffff);
> - sha1write(f, split, 8);
> - nr_large_offset--;
> - }
> - }
> - }
> -
> - sha1write(f, pack_file_sha1, 20);
> - sha1close(f, NULL, 1);
> - SHA1_Final(sha1, &ctx);
> -}
> -
> static int locate_object_entry_hash(const unsigned char *sha1)
> {
> int i;
> @@ -817,7 +699,7 @@ static int locate_object_entry_hash(const unsigned char *sha1)
> memcpy(&ui, sha1, sizeof(unsigned int));
> i = ui % object_ix_hashsz;
> while (0 < object_ix[i]) {
> - if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1))
> + if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1))
> return i;
> if (++i == object_ix_hashsz)
> i = 0;
> @@ -849,7 +731,7 @@ static void rehash_objects(void)
> object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
> memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
> for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
> - int ix = locate_object_entry_hash(oe->sha1);
> + int ix = locate_object_entry_hash(oe->idx.sha1);
> if (0 <= ix)
> continue;
> ix = -1 - ix;
> @@ -943,7 +825,7 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
>
> entry = objects + nr_objects++;
> memset(entry, 0, sizeof(*entry));
> - hashcpy(entry->sha1, sha1);
> + hashcpy(entry->idx.sha1, sha1);
> entry->hash = hash;
> if (type)
> entry->type = type;
> @@ -1263,13 +1145,13 @@ static void check_object(struct object_entry *entry)
> ofs += 1;
> if (!ofs || MSB(ofs, 7))
> die("delta base offset overflow in pack for %s",
> - sha1_to_hex(entry->sha1));
> + sha1_to_hex(entry->idx.sha1));
> c = buf[used_0++];
> ofs = (ofs << 7) + (c & 127);
> }
> if (ofs >= entry->in_pack_offset)
> die("delta base offset out of bound for %s",
> - sha1_to_hex(entry->sha1));
> + sha1_to_hex(entry->idx.sha1));
> ofs = entry->in_pack_offset - ofs;
> if (!no_reuse_delta && !entry->preferred_base)
> base_ref = find_packed_object_name(p, ofs);
> @@ -1316,10 +1198,10 @@ static void check_object(struct object_entry *entry)
> unuse_pack(&w_curs);
> }
>
> - entry->type = sha1_object_info(entry->sha1, &entry->size);
> + entry->type = sha1_object_info(entry->idx.sha1, &entry->size);
> if (entry->type < 0)
> die("unable to get type of object %s",
> - sha1_to_hex(entry->sha1));
> + sha1_to_hex(entry->idx.sha1));
> }
>
> static int pack_offset_sort(const void *_a, const void *_b)
> @@ -1329,7 +1211,7 @@ static int pack_offset_sort(const void *_a, const void *_b)
>
> /* avoid filesystem trashing with loose objects */
> if (!a->in_pack && !b->in_pack)
> - return hashcmp(a->sha1, b->sha1);
> + return hashcmp(a->idx.sha1, b->idx.sha1);
>
> if (a->in_pack < b->in_pack)
> return -1;
> @@ -1441,16 +1323,16 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
>
> /* Load data if not already done */
> if (!trg->data) {
> - trg->data = read_sha1_file(trg_entry->sha1, &type, &sz);
> + trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
> if (sz != trg_size)
> die("object %s inconsistent object length (%lu vs %lu)",
> - sha1_to_hex(trg_entry->sha1), sz, trg_size);
> + sha1_to_hex(trg_entry->idx.sha1), sz, trg_size);
> }
> if (!src->data) {
> - src->data = read_sha1_file(src_entry->sha1, &type, &sz);
> + src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
> if (sz != src_size)
> die("object %s inconsistent object length (%lu vs %lu)",
> - sha1_to_hex(src_entry->sha1), sz, src_size);
> + sha1_to_hex(src_entry->idx.sha1), sz, src_size);
> }
> if (!src->index) {
> src->index = create_delta_index(src->data, src_size);
> @@ -1825,12 +1707,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
> }
> if (!prefixcmp(arg, "--index-version=")) {
> char *c;
> - index_default_version = strtoul(arg + 16, &c, 10);
> - if (index_default_version > 2)
> + pack_idx_default_version = strtoul(arg + 16, &c, 10);
> + if (pack_idx_default_version > 2)
> die("bad %s", arg);
> if (*c == ',')
> - index_off32_limit = strtoul(c+1, &c, 0);
> - if (*c || index_off32_limit & 0x80000000)
> + pack_idx_off32_limit = strtoul(c+1, &c, 0);
> + if (*c || pack_idx_off32_limit & 0x80000000)
> die("bad %s", arg);
> continue;
> }
> diff --git a/index-pack.c b/index-pack.c
> index 58c4a9c..82c8da3 100644
> --- a/index-pack.c
> +++ b/index-pack.c
> @@ -13,13 +13,11 @@ static const char index_pack_usage[] =
>
> struct object_entry
> {
> - off_t offset;
> + struct pack_idx_entry idx;
> unsigned long size;
> unsigned int hdr_size;
> - uint32_t crc32;
> enum object_type type;
> enum object_type real_type;
> - unsigned char sha1[20];
> };
>
> union delta_base {
> @@ -197,7 +195,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
> unsigned shift;
> void *data;
>
> - obj->offset = consumed_bytes;
> + obj->idx.offset = consumed_bytes;
> input_crc32 = crc32(0, Z_NULL, 0);
>
> p = fill(1);
> @@ -229,15 +227,15 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
> while (c & 128) {
> base_offset += 1;
> if (!base_offset || MSB(base_offset, 7))
> - bad_object(obj->offset, "offset value overflow for delta base object");
> + bad_object(obj->idx.offset, "offset value overflow for delta base object");
> p = fill(1);
> c = *p;
> use(1);
> base_offset = (base_offset << 7) + (c & 127);
> }
> - delta_base->offset = obj->offset - base_offset;
> - if (delta_base->offset >= obj->offset)
> - bad_object(obj->offset, "delta base offset is out of bound");
> + delta_base->offset = obj->idx.offset - base_offset;
> + if (delta_base->offset >= obj->idx.offset)
> + bad_object(obj->idx.offset, "delta base offset is out of bound");
> break;
> case OBJ_COMMIT:
> case OBJ_TREE:
> @@ -245,19 +243,19 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
> case OBJ_TAG:
> break;
> default:
> - bad_object(obj->offset, "unknown object type %d", obj->type);
> + bad_object(obj->idx.offset, "unknown object type %d", obj->type);
> }
> - obj->hdr_size = consumed_bytes - obj->offset;
> + obj->hdr_size = consumed_bytes - obj->idx.offset;
>
> - data = unpack_entry_data(obj->offset, obj->size);
> - obj->crc32 = input_crc32;
> + data = unpack_entry_data(obj->idx.offset, obj->size);
> + obj->idx.crc32 = input_crc32;
> return data;
> }
>
> static void *get_data_from_pack(struct object_entry *obj)
> {
> - unsigned long from = obj[0].offset + obj[0].hdr_size;
> - unsigned long len = obj[1].offset - from;
> + unsigned long from = obj[0].idx.offset + obj[0].hdr_size;
> + unsigned long len = obj[1].idx.offset - from;
> unsigned long rdy = 0;
> unsigned char *src, *data;
> z_stream stream;
> @@ -360,11 +358,11 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
> &result_size);
> free(delta_data);
> if (!result)
> - bad_object(delta_obj->offset, "failed to apply delta");
> - sha1_object(result, result_size, type, delta_obj->sha1);
> + bad_object(delta_obj->idx.offset, "failed to apply delta");
> + sha1_object(result, result_size, type, delta_obj->idx.sha1);
> nr_resolved_deltas++;
>
> - hashcpy(delta_base.sha1, delta_obj->sha1);
> + hashcpy(delta_base.sha1, delta_obj->idx.sha1);
> if (!find_delta_children(&delta_base, &first, &last)) {
> for (j = first; j <= last; j++) {
> struct object_entry *child = objects + deltas[j].obj_no;
> @@ -374,7 +372,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
> }
>
> memset(&delta_base, 0, sizeof(delta_base));
> - delta_base.offset = delta_obj->offset;
> + delta_base.offset = delta_obj->idx.offset;
> if (!find_delta_children(&delta_base, &first, &last)) {
> for (j = first; j <= last; j++) {
> struct object_entry *child = objects + deltas[j].obj_no;
> @@ -418,12 +416,12 @@ static void parse_pack_objects(unsigned char *sha1)
> delta->obj_no = i;
> delta++;
> } else
> - sha1_object(data, obj->size, obj->type, obj->sha1);
> + sha1_object(data, obj->size, obj->type, obj->idx.sha1);
> free(data);
> if (verbose)
> display_progress(&progress, i+1);
> }
> - objects[i].offset = consumed_bytes;
> + objects[i].idx.offset = consumed_bytes;
> if (verbose)
> stop_progress(&progress);
>
> @@ -465,10 +463,10 @@ static void parse_pack_objects(unsigned char *sha1)
>
> if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
> continue;
> - hashcpy(base.sha1, obj->sha1);
> + hashcpy(base.sha1, obj->idx.sha1);
> ref = !find_delta_children(&base, &ref_first, &ref_last);
> memset(&base, 0, sizeof(base));
> - base.offset = obj->offset;
> + base.offset = obj->idx.offset;
> ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
> if (!ref && !ofs)
> continue;
> @@ -535,11 +533,11 @@ static void append_obj_to_pack(const unsigned char *sha1, void *buf,
> }
> header[n++] = c;
> write_or_die(output_fd, header, n);
> - obj[0].crc32 = crc32(0, Z_NULL, 0);
> - obj[0].crc32 = crc32(obj[0].crc32, header, n);
> - obj[1].offset = obj[0].offset + n;
> - obj[1].offset += write_compressed(output_fd, buf, size, &obj[0].crc32);
> - hashcpy(obj->sha1, sha1);
> + obj[0].idx.crc32 = crc32(0, Z_NULL, 0);
> + obj[0].idx.crc32 = crc32(obj[0].idx.crc32, header, n);
> + obj[1].idx.offset = obj[0].idx.offset + n;
> + obj[1].idx.offset += write_compressed(output_fd, buf, size, &obj[0].idx.crc32);
> + hashcpy(obj->idx.sha1, sha1);
> }
>
> static int delta_pos_compare(const void *_a, const void *_b)
> @@ -602,145 +600,6 @@ static void fix_unresolved_deltas(int nr_unresolved)
> free(sorted_by_pos);
> }
>
> -static uint32_t index_default_version = 1;
> -static uint32_t index_off32_limit = 0x7fffffff;
> -
> -static int sha1_compare(const void *_a, const void *_b)
> -{
> - struct object_entry *a = *(struct object_entry **)_a;
> - struct object_entry *b = *(struct object_entry **)_b;
> - return hashcmp(a->sha1, b->sha1);
> -}
> -
> -/*
> - * On entry *sha1 contains the pack content SHA1 hash, on exit it is
> - * the SHA1 hash of sorted object names.
> - */
> -static const char *write_index_file(const char *index_name, unsigned char *sha1)
> -{
> - struct sha1file *f;
> - struct object_entry **sorted_by_sha, **list, **last;
> - uint32_t array[256];
> - int i, fd;
> - SHA_CTX ctx;
> - uint32_t index_version;
> -
> - if (nr_objects) {
> - sorted_by_sha =
> - xcalloc(nr_objects, sizeof(struct object_entry *));
> - list = sorted_by_sha;
> - last = sorted_by_sha + nr_objects;
> - for (i = 0; i < nr_objects; ++i)
> - sorted_by_sha[i] = &objects[i];
> - qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
> - sha1_compare);
> - }
> - else
> - sorted_by_sha = list = last = NULL;
> -
> - if (!index_name) {
> - static char tmpfile[PATH_MAX];
> - snprintf(tmpfile, sizeof(tmpfile),
> - "%s/tmp_idx_XXXXXX", get_object_directory());
> - fd = mkstemp(tmpfile);
> - index_name = xstrdup(tmpfile);
> - } else {
> - unlink(index_name);
> - fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
> - }
> - if (fd < 0)
> - die("unable to create %s: %s", index_name, strerror(errno));
> - f = sha1fd(fd, index_name);
> -
> - /* if last object's offset is >= 2^31 we should use index V2 */
> - index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version;
> -
> - /* index versions 2 and above need a header */
> - if (index_version >= 2) {
> - struct pack_idx_header hdr;
> - hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
> - hdr.idx_version = htonl(index_version);
> - sha1write(f, &hdr, sizeof(hdr));
> - }
> -
> - /*
> - * Write the first-level table (the list is sorted,
> - * but we use a 256-entry lookup to be able to avoid
> - * having to do eight extra binary search iterations).
> - */
> - for (i = 0; i < 256; i++) {
> - struct object_entry **next = list;
> - while (next < last) {
> - struct object_entry *obj = *next;
> - if (obj->sha1[0] != i)
> - break;
> - next++;
> - }
> - array[i] = htonl(next - sorted_by_sha);
> - list = next;
> - }
> - sha1write(f, array, 256 * 4);
> -
> - /* compute the SHA1 hash of sorted object names. */
> - SHA1_Init(&ctx);
> -
> - /*
> - * Write the actual SHA1 entries..
> - */
> - list = sorted_by_sha;
> - for (i = 0; i < nr_objects; i++) {
> - struct object_entry *obj = *list++;
> - if (index_version < 2) {
> - uint32_t offset = htonl(obj->offset);
> - sha1write(f, &offset, 4);
> - }
> - sha1write(f, obj->sha1, 20);
> - SHA1_Update(&ctx, obj->sha1, 20);
> - }
> -
> - if (index_version >= 2) {
> - unsigned int nr_large_offset = 0;
> -
> - /* write the crc32 table */
> - list = sorted_by_sha;
> - for (i = 0; i < nr_objects; i++) {
> - struct object_entry *obj = *list++;
> - uint32_t crc32_val = htonl(obj->crc32);
> - sha1write(f, &crc32_val, 4);
> - }
> -
> - /* write the 32-bit offset table */
> - list = sorted_by_sha;
> - for (i = 0; i < nr_objects; i++) {
> - struct object_entry *obj = *list++;
> - uint32_t offset = (obj->offset <= index_off32_limit) ?
> - obj->offset : (0x80000000 | nr_large_offset++);
> - offset = htonl(offset);
> - sha1write(f, &offset, 4);
> - }
> -
> - /* write the large offset table */
> - list = sorted_by_sha;
> - while (nr_large_offset) {
> - struct object_entry *obj = *list++;
> - uint64_t offset = obj->offset;
> - if (offset > index_off32_limit) {
> - uint32_t split[2];
> - split[0] = htonl(offset >> 32);
> - split[1] = htonl(offset & 0xffffffff);
> - sha1write(f, split, 8);
> - nr_large_offset--;
> - }
> - }
> - }
> -
> - sha1write(f, sha1, 20);
> - sha1close(f, NULL, 1);
> - free(sorted_by_sha);
> - SHA1_Final(sha1, &ctx);
> - return index_name;
> -}
> -
> static void final(const char *final_pack_name, const char *curr_pack_name,
> const char *final_index_name, const char *curr_index_name,
> const char *keep_name, const char *keep_msg,
> @@ -830,6 +689,7 @@ int main(int argc, char **argv)
> const char *curr_index, *index_name = NULL;
> const char *keep_name = NULL, *keep_msg = NULL;
> char *index_name_buf = NULL, *keep_name_buf = NULL;
> + struct pack_idx_entry **idx_objects;
> unsigned char sha1[20];
>
> for (i = 1; i < argc; i++) {
> @@ -865,12 +725,12 @@ int main(int argc, char **argv)
> index_name = argv[++i];
> } else if (!prefixcmp(arg, "--index-version=")) {
> char *c;
> - index_default_version = strtoul(arg + 16, &c, 10);
> - if (index_default_version > 2)
> + pack_idx_default_version = strtoul(arg + 16, &c, 10);
> + if (pack_idx_default_version > 2)
> die("bad %s", arg);
> if (*c == ',')
> - index_off32_limit = strtoul(c+1, &c, 0);
> - if (*c || index_off32_limit & 0x80000000)
> + pack_idx_off32_limit = strtoul(c+1, &c, 0);
> + if (*c || pack_idx_off32_limit & 0x80000000)
> die("bad %s", arg);
> } else
> usage(index_pack_usage);
> @@ -940,7 +800,13 @@ int main(int argc, char **argv)
> nr_deltas - nr_resolved_deltas);
> }
> free(deltas);
> - curr_index = write_index_file(index_name, sha1);
> +
> + idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *));
> + for (i = 0; i < nr_objects; i++)
> + idx_objects[i] = &objects[i].idx;
> + curr_index = write_idx_file(index_name, idx_objects, nr_objects, sha1);
> + free(idx_objects);
> +
> final(pack_name, curr_pack,
> index_name, curr_index,
> keep_name, keep_msg,
> diff --git a/pack-write.c b/pack-write.c
> index ae2e481..1cf5f7c 100644
> --- a/pack-write.c
> +++ b/pack-write.c
> @@ -1,5 +1,147 @@
> #include "cache.h"
> #include "pack.h"
> +#include "csum-file.h"
> +
> +uint32_t pack_idx_default_version = 1;
> +uint32_t pack_idx_off32_limit = 0x7fffffff;
> +
> +static int sha1_compare(const void *_a, const void *_b)
> +{
> + struct pack_idx_entry *a = *(struct pack_idx_entry **)_a;
> + struct pack_idx_entry *b = *(struct pack_idx_entry **)_b;
> + return hashcmp(a->sha1, b->sha1);
> +}
> +
> +/*
> + * On entry *sha1 contains the pack content SHA1 hash, on exit it is
> + * the SHA1 hash of sorted object names. The objects array passed in
> + * will be sorted by SHA1 on exit.
> + */
> +const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1)
> +{
> + struct sha1file *f;
> + struct pack_idx_entry **sorted_by_sha, **list, **last;
> + off_t last_obj_offset = 0;
> + uint32_t array[256];
> + int i, fd;
> + SHA_CTX ctx;
> + uint32_t index_version;
> +
> + if (nr_objects) {
> + sorted_by_sha = objects;
> + list = sorted_by_sha;
> + last = sorted_by_sha + nr_objects;
> + for (i = 0; i < nr_objects; ++i) {
> + if (objects[i]->offset > last_obj_offset)
> + last_obj_offset = objects[i]->offset;
> + }
> + qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
> + sha1_compare);
> + }
> + else
> + sorted_by_sha = list = last = NULL;
> +
> + if (!index_name) {
> + static char tmpfile[PATH_MAX];
> + snprintf(tmpfile, sizeof(tmpfile),
> + "%s/tmp_idx_XXXXXX", get_object_directory());
> + fd = mkstemp(tmpfile);
> + index_name = xstrdup(tmpfile);
> + } else {
> + unlink(index_name);
> + fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
> + }
> + if (fd < 0)
> + die("unable to create %s: %s", index_name, strerror(errno));
> + f = sha1fd(fd, index_name);
> +
> + /* if last object's offset is >= 2^31 we should use index V2 */
> + index_version = (last_obj_offset >> 31) ? 2 : pack_idx_default_version;
> +
> + /* index versions 2 and above need a header */
> + if (index_version >= 2) {
> + struct pack_idx_header hdr;
> + hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
> + hdr.idx_version = htonl(index_version);
> + sha1write(f, &hdr, sizeof(hdr));
> + }
> +
> + /*
> + * Write the first-level table (the list is sorted,
> + * but we use a 256-entry lookup to be able to avoid
> + * having to do eight extra binary search iterations).
> + */
> + for (i = 0; i < 256; i++) {
> + struct pack_idx_entry **next = list;
> + while (next < last) {
> + struct pack_idx_entry *obj = *next;
> + if (obj->sha1[0] != i)
> + break;
> + next++;
> + }
> + array[i] = htonl(next - sorted_by_sha);
> + list = next;
> + }
> + sha1write(f, array, 256 * 4);
> +
> + /* compute the SHA1 hash of sorted object names. */
> + SHA1_Init(&ctx);
> +
> + /*
> + * Write the actual SHA1 entries..
> + */
> + list = sorted_by_sha;
> + for (i = 0; i < nr_objects; i++) {
> + struct pack_idx_entry *obj = *list++;
> + if (index_version < 2) {
> + uint32_t offset = htonl(obj->offset);
> + sha1write(f, &offset, 4);
> + }
> + sha1write(f, obj->sha1, 20);
> + SHA1_Update(&ctx, obj->sha1, 20);
> + }
> +
> + if (index_version >= 2) {
> + unsigned int nr_large_offset = 0;
> +
> + /* write the crc32 table */
> + list = sorted_by_sha;
> + for (i = 0; i < nr_objects; i++) {
> + struct pack_idx_entry *obj = *list++;
> + uint32_t crc32_val = htonl(obj->crc32);
> + sha1write(f, &crc32_val, 4);
> + }
> +
> + /* write the 32-bit offset table */
> + list = sorted_by_sha;
> + for (i = 0; i < nr_objects; i++) {
> + struct pack_idx_entry *obj = *list++;
> + uint32_t offset = (obj->offset <= pack_idx_off32_limit) ?
> + obj->offset : (0x80000000 | nr_large_offset++);
> + offset = htonl(offset);
> + sha1write(f, &offset, 4);
> + }
> +
> + /* write the large offset table */
> + list = sorted_by_sha;
> + while (nr_large_offset) {
> + struct pack_idx_entry *obj = *list++;
> + uint64_t offset = obj->offset;
> + if (offset > pack_idx_off32_limit) {
> + uint32_t split[2];
> + split[0] = htonl(offset >> 32);
> + split[1] = htonl(offset & 0xffffffff);
> + sha1write(f, split, 8);
> + nr_large_offset--;
> + }
> + }
> + }
> +
> + sha1write(f, sha1, 20);
> + sha1close(f, NULL, 1);
> + SHA1_Final(sha1, &ctx);
> + return index_name;
> +}
>
> void fixup_pack_header_footer(int pack_fd,
> unsigned char *pack_file_sha1,
> diff --git a/pack.h b/pack.h
> index d667fb8..f357c9f 100644
> --- a/pack.h
> +++ b/pack.h
> @@ -34,6 +34,10 @@ struct pack_header {
> */
> #define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */
>
> +/* These may be overridden by command-line parameters */
> +extern uint32_t pack_idx_default_version;
> +extern uint32_t pack_idx_off32_limit;
> +
> /*
> * Packed object index header
> */
> @@ -42,6 +46,16 @@ struct pack_idx_header {
> uint32_t idx_version;
> };
>
> +/*
> + * Common part of object structure used for write_idx_file
> + */
> +struct pack_idx_entry {
> + unsigned char sha1[20];
> + uint32_t crc32;
> + off_t offset;
> +};
> +
> +extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
>
> extern int verify_pack(struct packed_git *, int);
> extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t);
> --
> 1.5.1
>
> -
> To unsubscribe from this list: send the line "unsubscribe git" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
Nicolas
-
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
