From: Han Xin <hanxin.hx@xxxxxxxxxxxxxxx>
We used to call "get_data()" in "unpack_non_delta_entry()" to read the
entire contents of a blob object, no matter how big it is. This
implementation may consume all the memory and cause OOM.
This can be improved by feeding data to "write_loose_object()" in a
stream. The input stream is implemented as an interface.
In the first step, we add a new flag called "HASH_STREAM" and make a
simple implementation, feeding the entire buffer in the stream to
"write_loose_object()" as a refactor.
Helped-by: Ævar Arnfjörð Bjarmason <avarab@xxxxxxxxx>
Helped-by: Jiang Xin <zhiyou.jx@xxxxxxxxxxxxxxx>
Signed-off-by: Han Xin <hanxin.hx@xxxxxxxxxxxxxxx>
---
cache.h | 1 +
object-file.c | 7 ++++++-
object-store.h | 5 +++++
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/cache.h b/cache.h
index eba12487b9..51bd435dea 100644
--- a/cache.h
+++ b/cache.h
@@ -888,6 +888,7 @@ int ie_modified(struct index_state *, const struct cache_entry *, struct stat *,
#define HASH_FORMAT_CHECK 2
#define HASH_RENORMALIZE 4
#define HASH_SILENT 8
+#define HASH_STREAM 16
int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags);
int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags);
diff --git a/object-file.c b/object-file.c
index eb972cdccd..06375a90d6 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1898,7 +1898,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
the_hash_algo->update_fn(&c, hdr, hdrlen);
/* Then the data itself.. */
- stream.next_in = (void *)buf;
+ if (flags & HASH_STREAM) {
+ struct input_stream *in_stream = (struct input_stream *)buf;
+ stream.next_in = (void *)in_stream->read(in_stream, &len);
+ } else {
+ stream.next_in = (void *)buf;
+ }
stream.avail_in = len;
do {
unsigned char *in0 = stream.next_in;
diff --git a/object-store.h b/object-store.h
index 952efb6a4b..ccc1fc9c1a 100644
--- a/object-store.h
+++ b/object-store.h
@@ -34,6 +34,11 @@ struct object_directory {
char *path;
};
+struct input_stream {
+ const void *(*read)(struct input_stream *, unsigned long *len);
+ void *data;
+};
+
KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
struct object_directory *, 1, fspathhash, fspatheq)
--
2.34.0
