[gentoo-commits] proj/portage-utils:master commit in: libq/

[Fabian Groffen]

commit:     cea99ce1b9b6d0c9ebb496dcd2f1e77ab9ab1dcd
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Wed Dec 29 12:17:32 2021 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Wed Dec 29 12:17:32 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage-utils.git/commit/?id=cea99ce1

libq/tree: avoid nasty realloc magic on tree_pkg_meta

doing reallocs possibly (hight probability) invalidates previously
retrieved pointers, which is really nasty to work with, and error prone,
so instead allocate incremental slabs where necessary

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 libq/tree.c | 206 +++++++++++++++++++++++++++++++-----------------------------
 libq/tree.h |  11 +++-
 2 files changed, 116 insertions(+), 101 deletions(-)

diff --git a/libq/tree.c b/libq/tree.c
index 87df175..114541d 100644
--- a/libq/tree.c
+++ b/libq/tree.c
@@ -651,6 +651,21 @@ tree_pkg_vdb_eat(
 	return ret;
 }
 
+#define tree_meta_alloc_storage(M,SIZ) { \
+	struct tree_pkg_meta_ll *blk; \
+	size_t                   newlen; \
+\
+	/* calculate new block size, ensuring it covers whatever we \
+	 * need to write this iteration */ \
+	newlen     = ((((SIZ) + 1) / BUFSIZ) + 1) * BUFSIZ; \
+	blk        = xmalloc(sizeof(*blk) + newlen); \
+	memset(blk, 0, sizeof(*blk)); \
+	blk->next  = M->storage; \
+	blk->ptr   = (char *)blk + sizeof(*blk); \
+	blk->len   = newlen; \
+	M->storage = blk; \
+}
+
 static tree_pkg_meta *
 tree_read_file_pms(tree_pkg_ctx *pkg_ctx)
 {
@@ -668,8 +683,9 @@ tree_read_file_pms(tree_pkg_ctx *pkg_ctx)
 		goto err;
 
 	len = sizeof(*ret) + s.st_size + 1;
-	ret = xzalloc(len);
-	ptr = (char*)ret + sizeof(*ret);
+	ret = xmalloc(len);
+	memset(ret, 0, sizeof(*ret));
+	ptr = (char *)ret + sizeof(*ret);
 	if ((off_t)fread(ptr, 1, s.st_size, f) != s.st_size)
 		goto err;
 	ptr[s.st_size] = '\0';
@@ -954,15 +970,15 @@ static void
 tree_read_file_binpkg_xpak_cb(
 	void *ctx,
 	char *pathname,
-	int pathname_len,
-	int data_offset,
-	int data_len,
+	int   pathname_len,
+	int   data_offset,
+	int   data_len,
 	char *data)
 {
 	tree_pkg_meta *m = (tree_pkg_meta *)ctx;
-	char **key;
-	size_t pos;
-	size_t len;
+	char         **key;
+	size_t         pos;
+	size_t         len;
 
 #define match_path(K) \
 	else if (pathname_len == (sizeof(#K) - 1) && strcmp(pathname, #K) == 0) \
@@ -994,37 +1010,28 @@ tree_read_file_binpkg_xpak_cb(
 		return;
 #undef match_path
 
-	/* hijack unused members */
-	pos = (size_t)m->Q__eclasses_;
-	len = (size_t)m->Q__md5_;
+	/* get current storage block */
+	if (m->storage != NULL) {
+		pos = m->storage->pos;
+		len = m->storage->len;
+	} else {
+		pos = 0;
+		len = 0;
+	}
 
 	/* trim whitespace (mostly trailing newline) */
 	while (isspace((int)data[data_offset + data_len - 1]))
 		data_len--;
 
 	if (len - pos < (size_t)(data_len + 1)) {
-		char *old_data = m->Q__data;
-		len += (((data_len + 1 - (len - pos)) / BUFSIZ) + 1) * BUFSIZ;
-		m->Q__data = xrealloc(m->Q__data, len);
-
-		/* re-position existing keys */
-		if (old_data != NULL && m->Q__data != old_data) {
-			char **newdata = (char **)m;
-			int elems = sizeof(tree_pkg_meta) / sizeof(char *);
-			while (elems-- > 1)  /* skip Q__data itself */
-				if (newdata[elems] != NULL)
-					newdata[elems] = m->Q__data + (newdata[elems] - old_data);
-		}
-
-		/* set after repositioning! */
-		m->Q__md5_ = (char *)len;
-		m->Q__eclasses_ = (char *)pos;
+		tree_meta_alloc_storage(m, data_len + 1);
+		len = m->storage->len;
+		pos = m->storage->pos;
 	}
 
-	*key = m->Q__data + pos;
+	*key = m->storage->ptr + pos;
 	snprintf(*key, len - pos, "%.*s", data_len, data + data_offset);
-	pos += data_len + 1;
-	m->Q__eclasses_ = (char *)pos;
+	m->storage->pos += data_len + 1;
 }
 
 static tree_pkg_meta *
@@ -1042,33 +1049,23 @@ tree_read_file_binpkg(tree_pkg_ctx *pkg_ctx)
 	if (newfd != -1) {
 		size_t fsize;
 		size_t needlen = 40 + 1 + 19 + 1;
-		size_t pos = (size_t)m->Q__eclasses_;
-		size_t len = (size_t)m->Q__md5_;
+		size_t pos = 0;
+		size_t len = 0;
 
-		if (len - pos < needlen) {
-			char *old_data = m->Q__data;
-			len += (((needlen - (len - pos)) / BUFSIZ) + 1) * BUFSIZ;
-			m->Q__data = xrealloc(m->Q__data, len);
-
-			/* re-position existing keys */
-			if (old_data != NULL && m->Q__data != old_data) {
-				char **newdata = (char **)m;
-				int elems = sizeof(tree_pkg_meta) / sizeof(char *);
-				while (elems-- > 1)  /* skip Q__data itself */
-					if (newdata[elems] != NULL)
-						newdata[elems] =
-							m->Q__data + (newdata[elems] - old_data);
-			}
+		if (m->storage != NULL) {
+			pos = m->storage->pos;
+			len = m->storage->len;
+		}
 
-			/* set after repositioning! */
-			m->Q__md5_ = (char *)len;
-			m->Q__eclasses_ = (char *)pos;
+		if (len - pos < needlen) {
+			tree_meta_alloc_storage(m, needlen);
+			len = m->storage->len;
+			pos = m->storage->pos;
 		}
 
-		m->Q_SHA1 = m->Q__data + pos;
+		m->Q_SHA1 = m->storage->ptr + pos;
 		m->Q_SIZE = m->Q_SHA1 + 40 + 1;
-		pos += needlen;
-		m->Q__eclasses_ = (char *)pos;
+		m->storage->pos += needlen;
 
 		lseek(newfd, 0, SEEK_SET);  /* reposition at the whole file */
 		if (hash_multiple_file_fd(newfd, NULL, m->Q_SHA1, NULL, NULL,
@@ -1123,13 +1120,48 @@ tree_pkg_read(tree_pkg_ctx *pkg_ctx)
 	return ret;
 }
 
+static tree_pkg_meta *
+tree_clone_meta(tree_pkg_meta *m)
+{
+	tree_pkg_meta *ret;
+	size_t         pos = 0;
+	size_t         len = 0;
+	char         **ptr;
+	char          *p;
+
+	/* compute necessary space upfront */
+	len = sizeof(*ret);
+	for (ptr = &m->Q__data; ptr <= &m->Q__last; ptr++)
+		if (*ptr != NULL)
+			len += strlen(*ptr);
+
+	/* malloc and copy */
+	ret = xzalloc(len);
+	p = (char *)ret + sizeof(*ret);
+	for (ptr = &m->Q__data; ptr <= &m->Q__last; ptr++, pos++) {
+		if (*ptr == NULL)
+			continue;
+		*(&ret->Q__data + pos) = p;
+		len = strlen(*ptr) + 1;
+		memcpy(p, *ptr, len);
+		p += len;
+	}
+
+	return ret;
+}
+
 static void
 tree_close_meta(tree_pkg_meta *cache)
 {
+	struct tree_pkg_meta_ll *blk;
+
 	if (cache == NULL)
 		errf("Cache is empty !");
-	if (cache->Q__data != NULL)
-		free(cache->Q__data);
+	while (cache->storage != NULL) {
+		blk = cache->storage->next;
+		free(cache->storage);
+		cache->storage = blk;
+	}
 	free(cache);
 }
 
@@ -1165,37 +1197,26 @@ tree_pkg_meta_get_int(tree_pkg_ctx *pkg_ctx, size_t offset, const char *keyn)
 				return NULL;
 			}
 
-			/* hijack unused members */
-			pos = (size_t)m->Q__eclasses_;
-			len = (size_t)m->Q__md5_;
+			if (m->storage != NULL) {
+				pos = m->storage->pos;
+				len = m->storage->len;
+			} else {
+				pos = 0;
+				len = 0;
+			}
 
-			/* TODO: this is an exact copy from tree_read_file_binpkg_xpak_cb */
 			if (len - pos < (size_t)(s.st_size + 1)) {
-				p = m->Q__data;
-				len += (((s.st_size + 1 - (len - pos)) / BUFSIZ) + 1) * BUFSIZ;
-				m->Q__data = xrealloc(m->Q__data, len);
-
-				/* re-position existing keys */
-				if (p != NULL && m->Q__data != p) {
-					char **newdata = (char **)m;
-					int elems = sizeof(tree_pkg_meta) / sizeof(char *);
-					while (elems-- > 1)  /* skip Q__data itself */
-						if (newdata[elems] != NULL)
-							newdata[elems] = m->Q__data + (newdata[elems] - p);
-				}
-
-				/* set after repositioning! */
-				m->Q__md5_ = (char *)len;
-				m->Q__eclasses_ = (char *)pos;
+				tree_meta_alloc_storage(m, s.st_size + 1);
+				pos = m->storage->pos;
+				len = m->storage->len;
 			}
 
-			p = *key = m->Q__data + pos;
+			p = *key = m->storage->ptr + pos;
 			if (read(fd, p, s.st_size) == (ssize_t)s.st_size) {
 				p[s.st_size] = '\0';
 				while (s.st_size > 0 && isspace((int)p[s.st_size - 1]))
 					p[--s.st_size] = '\0';
-				pos += s.st_size + 1;
-				m->Q__eclasses_ = (char *)pos;
+				m->storage->pos += s.st_size + 1;
 			}
 			close(fd);
 		}
@@ -1692,29 +1713,14 @@ tree_match_atom_cache_populate_cb(tree_pkg_ctx *ctx, void *priv)
 	pkg->repo = tctx->repo != NULL ? xstrdup(tctx->repo) : NULL;
 	if (meta != NULL) {
 		pkg->fd = -2;  /* don't try to read, we fill it in here */
-		pkg->meta = xmalloc(sizeof(*pkg->meta));
-		memcpy(pkg->meta, meta, sizeof(*pkg->meta));
-		if (ctx->cat_ctx->ctx->cachetype == CACHE_PACKAGES) {
-			pkg->meta->Q__data = NULL;  /* avoid free here (just to be sure) */
-		} else {  /* CACHE_BINPKG */
-			char **newdata;
-			int    elems;
-			size_t datasize = (size_t)meta->Q__eclasses_;
-
-			pkg->meta->Q__data = xmalloc(sizeof(char) * datasize);
-			memcpy(pkg->meta->Q__data, meta->Q__data, datasize);
-
-			/* re-position keys */
-			newdata = (char **)pkg->meta;
-			elems   = sizeof(tree_pkg_meta) / sizeof(char *);
-			while (elems-- > 1)  /* skip Q__data itself */
-				if (newdata[elems] != NULL)
-					newdata[elems] = pkg->meta->Q__data +
-						(newdata[elems] - meta->Q__data);
-
-			/* drop garbage used for Q__data admin in original case */
-			pkg->meta->Q__eclasses_ = NULL;
-			pkg->meta->Q__md5_      = NULL;
+		if (tctx->cachetype == CACHE_PACKAGES) {
+			/* need to copy, source is based on temp space in foreach */
+			pkg->meta = tree_clone_meta(meta);
+		} else {
+			/* BINPKG case, this one is read/allocated separately from
+			 * xpak archive, so can just take it over */
+			pkg->meta = meta;
+			ctx->meta = NULL;  /* avoid double free */
 		}
 	} else {
 		pkg->meta = NULL;

diff --git a/libq/tree.h b/libq/tree.h
index 052715b..8279281 100644
--- a/libq/tree.h
+++ b/libq/tree.h
@@ -80,7 +80,7 @@ struct tree_pkg_ctx {
 
 /* Ebuild data */
 struct tree_pkg_meta {
-	char *Q__data;
+#define Q__data Q_DEPEND  /* ptr to first member of the struct */
 	char *Q_DEPEND;        /* line 1 */
 	char *Q_RDEPEND;
 	char *Q_SLOT;
@@ -111,6 +111,15 @@ struct tree_pkg_meta {
 	/* These are MD5-Cache only */
 	char *Q__eclasses_;
 	char *Q__md5_;
+#define Q__last Q__md5_  /* ptr to last data member in struct */
+
+	/* for memory allocations backing the pointers above */
+	struct tree_pkg_meta_ll {
+		char                    *ptr;
+		size_t                   len;
+		size_t                   pos;
+		struct tree_pkg_meta_ll *next;
+	}    *storage;
 };
 
 /* Metadata.xml */