To store the individual mtimes of objects in a cruft pack, introduce a
new `.mtimes` format that can optionally accompany a single pack in the
repository.
The format is defined in Documentation/technical/pack-format.txt, and
stores a 4-byte network order timestamp for each object in name (index)
order.
This patch prepares for cruft packs by defining the `.mtimes` format,
and introducing a basic API that callers can use to read out individual
mtimes.
Signed-off-by: Taylor Blau <me@xxxxxxxxxxxx>
---
Documentation/technical/pack-format.txt | 22 ++++
Makefile | 1 +
builtin/repack.c | 1 +
object-store.h | 5 +-
pack-mtimes.c | 139 ++++++++++++++++++++++++
pack-mtimes.h | 16 +++
packfile.c | 18 ++-
packfile.h | 1 +
8 files changed, 200 insertions(+), 3 deletions(-)
create mode 100644 pack-mtimes.c
create mode 100644 pack-mtimes.h
diff --git a/Documentation/technical/pack-format.txt b/Documentation/technical/pack-format.txt
index 8d2f42f29e..61d8d960e7 100644
--- a/Documentation/technical/pack-format.txt
+++ b/Documentation/technical/pack-format.txt
@@ -294,6 +294,28 @@ Pack file entry: <+
All 4-byte numbers are in network order.
+== pack-*.mtimes files have the format:
+
+ - A 4-byte magic number '0x4d544d45' ('MTME').
+
+ - A 4-byte version identifier (= 1).
+
+ - A 4-byte hash function identifier (= 1 for SHA-1, 2 for SHA-256).
+
+ - A table of mtimes (one per packed object, num_objects in total, each
+ a 4-byte unsigned integer in network order), in the same order as
+ objects appear in the index file (e.g., the first entry in the mtime
+ table corresponds to the object with the lowest lexically-sorted
+ oid). The mtimes count standard epoch seconds.
+
+ - A trailer, containing a:
+
+ checksum of the corresponding packfile, and
+
+ a checksum of all of the above.
+
+All 4-byte numbers are in network order.
+
== multi-pack-index (MIDX) files have the following format:
The multi-pack-index files refer to multiple pack-files and loose objects.
diff --git a/Makefile b/Makefile
index 12be39ac49..efd5e00717 100644
--- a/Makefile
+++ b/Makefile
@@ -949,6 +949,7 @@ LIB_OBJS += oidtree.o
LIB_OBJS += pack-bitmap-write.o
LIB_OBJS += pack-bitmap.o
LIB_OBJS += pack-check.o
+LIB_OBJS += pack-mtimes.o
LIB_OBJS += pack-objects.o
LIB_OBJS += pack-revindex.o
LIB_OBJS += pack-write.o
diff --git a/builtin/repack.c b/builtin/repack.c
index 0b2d1e5d82..acbb7b8c3b 100644
--- a/builtin/repack.c
+++ b/builtin/repack.c
@@ -212,6 +212,7 @@ static struct {
} exts[] = {
{".pack"},
{".rev", 1},
+ {".mtimes", 1},
{".bitmap", 1},
{".promisor", 1},
{".idx"},
diff --git a/object-store.h b/object-store.h
index 952efb6a4b..d87481f101 100644
--- a/object-store.h
+++ b/object-store.h
@@ -89,12 +89,15 @@ struct packed_git {
freshened:1,
do_not_close:1,
pack_promisor:1,
- multi_pack_index:1;
+ multi_pack_index:1,
+ is_cruft:1;
unsigned char hash[GIT_MAX_RAWSZ];
struct revindex_entry *revindex;
const uint32_t *revindex_data;
const uint32_t *revindex_map;
size_t revindex_size;
+ const uint32_t *mtimes_map;
+ size_t mtimes_size;
/* something like ".git/objects/pack/xxxxx.pack" */
char pack_name[FLEX_ARRAY]; /* more */
};
diff --git a/pack-mtimes.c b/pack-mtimes.c
new file mode 100644
index 0000000000..4c7c00fa67
--- /dev/null
+++ b/pack-mtimes.c
@@ -0,0 +1,139 @@
+#include "pack-mtimes.h"
+#include "object-store.h"
+#include "packfile.h"
+
+static char *pack_mtimes_filename(struct packed_git *p)
+{
+ size_t len;
+ if (!strip_suffix(p->pack_name, ".pack", &len))
+ BUG("pack_name does not end in .pack");
+ /* NEEDSWORK: this could reuse code from pack-revindex.c. */
+ return xstrfmt("%.*s.mtimes", (int)len, p->pack_name);
+}
+
+int pack_has_mtimes(struct packed_git *p)
+{
+ struct stat st;
+ char *fname = pack_mtimes_filename(p);
+
+ if (stat(fname, &st) < 0) {
+ if (errno == ENOENT)
+ return 0;
+ die_errno(_("could not stat %s"), fname);
+ }
+
+ free(fname);
+ return 1;
+}
+
+#define MTIMES_HEADER_SIZE (12)
+#define MTIMES_MIN_SIZE (MTIMES_HEADER_SIZE + (2 * the_hash_algo->rawsz))
+
+struct mtimes_header {
+ uint32_t signature;
+ uint32_t version;
+ uint32_t hash_id;
+};
+
+static int load_pack_mtimes_file(char *mtimes_file,
+ uint32_t num_objects,
+ const uint32_t **data_p, size_t *len_p)
+{
+ int fd, ret = 0;
+ struct stat st;
+ void *data = NULL;
+ size_t mtimes_size;
+ uint32_t *hdr;
+
+ fd = git_open(mtimes_file);
+
+ if (fd < 0) {
+ ret = -1;
+ goto cleanup;
+ }
+ if (fstat(fd, &st)) {
+ ret = error_errno(_("failed to read %s"), mtimes_file);
+ goto cleanup;
+ }
+
+ mtimes_size = xsize_t(st.st_size);
+
+ if (mtimes_size < MTIMES_MIN_SIZE) {
+ ret = error(_("mtimes file %s is too small"), mtimes_file);
+ goto cleanup;
+ }
+
+ if (mtimes_size - MTIMES_MIN_SIZE != st_mult(sizeof(uint32_t), num_objects)) {
+ ret = error(_("mtimes file %s is corrupt"), mtimes_file);
+ goto cleanup;
+ }
+
+ data = hdr = xmmap(NULL, mtimes_size, PROT_READ, MAP_PRIVATE, fd, 0);
+
+ if (ntohl(*hdr) != MTIMES_SIGNATURE) {
+ ret = error(_("mtimes file %s has unknown signature"), mtimes_file);
+ goto cleanup;
+ }
+
+ if (ntohl(*++hdr) != 1) {
+ ret = error(_("mtimes file %s has unsupported version %"PRIu32),
+ mtimes_file, ntohl(*hdr));
+ goto cleanup;
+ }
+ hdr++;
+ if (!(ntohl(*hdr) == 1 || ntohl(*hdr) == 2)) {
+ ret = error(_("mtimes file %s has unsupported hash id %"PRIu32),
+ mtimes_file, ntohl(*hdr));
+ goto cleanup;
+ }
+
+cleanup:
+ if (ret) {
+ if (data)
+ munmap(data, mtimes_size);
+ } else {
+ *len_p = mtimes_size;
+ *data_p = (const uint32_t *)data;
+ }
+
+ close(fd);
+ return ret;
+}
+
+int load_pack_mtimes(struct packed_git *p)
+{
+ char *mtimes_name = NULL;
+ int ret = 0;
+
+ if (!p->is_cruft)
+ return ret; /* not a cruft pack */
+ if (p->mtimes_map)
+ return ret; /* already loaded */
+
+ ret = open_pack_index(p);
+ if (ret < 0)
+ goto cleanup;
+
+ mtimes_name = pack_mtimes_filename(p);
+ ret = load_pack_mtimes_file(mtimes_name,
+ p->num_objects,
+ &p->mtimes_map,
+ &p->mtimes_size);
+ if (ret)
+ goto cleanup;
+
+cleanup:
+ free(mtimes_name);
+ return ret;
+}
+
+uint32_t nth_packed_mtime(struct packed_git *p, uint32_t pos)
+{
+ if (!p->mtimes_map)
+ BUG("pack .mtimes file not loaded for %s", p->pack_name);
+ if (p->num_objects <= pos)
+ BUG("pack .mtimes out-of-bounds (%"PRIu32" vs %"PRIu32")",
+ pos, p->num_objects);
+
+ return get_be32(p->mtimes_map + pos + 3);
+}
diff --git a/pack-mtimes.h b/pack-mtimes.h
new file mode 100644
index 0000000000..ac4247bb5e
--- /dev/null
+++ b/pack-mtimes.h
@@ -0,0 +1,16 @@
+#ifndef PACK_MTIMES_H
+#define PACK_MTIMES_H
+
+#include "git-compat-util.h"
+
+#define MTIMES_SIGNATURE 0x4d544d45 /* "MTME" */
+#define MTIMES_VERSION 1
+
+struct packed_git;
+
+int pack_has_mtimes(struct packed_git *p);
+int load_pack_mtimes(struct packed_git *p);
+
+uint32_t nth_packed_mtime(struct packed_git *p, uint32_t pos);
+
+#endif
diff --git a/packfile.c b/packfile.c
index 89402cfc69..ae79ac644e 100644
--- a/packfile.c
+++ b/packfile.c
@@ -333,12 +333,21 @@ void close_pack_revindex(struct packed_git *p) {
p->revindex_data = NULL;
}
+void close_pack_mtimes(struct packed_git *p) {
+ if (!p->mtimes_map)
+ return;
+
+ munmap((void *)p->mtimes_map, p->mtimes_size);
+ p->mtimes_map = NULL;
+}
+
void close_pack(struct packed_git *p)
{
close_pack_windows(p);
close_pack_fd(p);
close_pack_index(p);
close_pack_revindex(p);
+ close_pack_mtimes(p);
oidset_clear(&p->bad_objects);
}
@@ -362,7 +371,7 @@ void close_object_store(struct raw_object_store *o)
void unlink_pack_path(const char *pack_name, int force_delete)
{
- static const char *exts[] = {".pack", ".idx", ".rev", ".keep", ".bitmap", ".promisor"};
+ static const char *exts[] = {".pack", ".idx", ".rev", ".keep", ".bitmap", ".promisor", ".mtimes"};
int i;
struct strbuf buf = STRBUF_INIT;
size_t plen;
@@ -717,6 +726,10 @@ struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
if (!access(p->pack_name, F_OK))
p->pack_promisor = 1;
+ xsnprintf(p->pack_name + path_len, alloc - path_len, ".mtimes");
+ if (!access(p->pack_name, F_OK))
+ p->is_cruft = 1;
+
xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
free(p);
@@ -868,7 +881,8 @@ static void prepare_pack(const char *full_name, size_t full_name_len,
ends_with(file_name, ".pack") ||
ends_with(file_name, ".bitmap") ||
ends_with(file_name, ".keep") ||
- ends_with(file_name, ".promisor"))
+ ends_with(file_name, ".promisor") ||
+ ends_with(file_name, ".mtimes"))
string_list_append(data->garbage, full_name);
else
report_garbage(PACKDIR_FILE_GARBAGE, full_name);
diff --git a/packfile.h b/packfile.h
index 186146779d..32201d8af7 100644
--- a/packfile.h
+++ b/packfile.h
@@ -91,6 +91,7 @@ uint32_t get_pack_fanout(struct packed_git *p, uint32_t value);
unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
void close_pack_windows(struct packed_git *);
void close_pack_revindex(struct packed_git *);
+void close_pack_mtimes(struct packed_git *p);
void close_pack(struct packed_git *);
void close_object_store(struct raw_object_store *o);
void unuse_pack(struct pack_window **);
--
2.34.1.25.gb3157a20e6
