[gentoo-commits] proj/portage-utils:master commit in: libq/, /

[Fabian Groffen]

commit:     2fc9b2a308a7d72edede919956a0d290a72a7781
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 28 15:19:07 2019 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Sun Apr 28 15:19:07 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage-utils.git/commit/?id=2fc9b2a3

libq/{vdb,cache}: unify sorted and unsorted traversal

Get vdb to know some bits for cache, such that cache can be a more
shallow wrapper around it.  Use the same code to traverse in sorted mode
by just keeping a temp list in the sorted case.

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 libq/cache.c | 137 ++++++++++++++++++-------------------------------
 libq/cache.h |  14 +----
 libq/vdb.c   | 164 ++++++++++++++++++++++++++++++++++-------------------------
 libq/vdb.h   |  46 +++++++++++++----
 qgrep.c      |   4 +-
 5 files changed, 186 insertions(+), 179 deletions(-)

diff --git a/libq/cache.c b/libq/cache.c
index 7724287..a00dd6b 100644
--- a/libq/cache.c
+++ b/libq/cache.c
@@ -53,46 +53,43 @@ static const char portrepo_name[]    = "profiles/repo_name";
 cache_ctx *
 cache_open(const char *sroot, const char *portdir)
 {
-	q_vdb_ctx *dir;
 	cache_ctx *ret;
 	char buf[_Q_PATH_MAX];
+	char *repo = NULL;
 	size_t repolen = 0;
 
-	ret = xzalloc(sizeof(cache_ctx));
-
 	snprintf(buf, sizeof(buf), "%s%s/%s", sroot, portdir, portrepo_name);
-	if (eat_file(buf, &ret->repo, &repolen)) {
-		(void)rmspace(ret->repo);
+	if (eat_file(buf, &repo, &repolen)) {
+		(void)rmspace(repo);
 	} else {
-		ret->repo = NULL;  /* ignore missing repo file */
+		repo = NULL;  /* ignore missing repo file */
 	}
 
 	snprintf(buf, sizeof(buf), "%s/%s", portdir, portcachedir_md5);
-	dir = q_vdb_open2(sroot, buf, true);
-	if (dir != NULL) {
-		ret->dir_ctx = dir;
+	ret = q_vdb_open2(sroot, buf, true);
+	if (ret != NULL) {
 		ret->cachetype = CACHE_METADATA_MD5;
+		ret->repo = repo;
 		return ret;
 	}
 
 	snprintf(buf, sizeof(buf), "%s/%s", portdir, portcachedir_pms);
-	dir = q_vdb_open2(sroot, buf, true);
-	if (dir != NULL) {
-		ret->dir_ctx = dir;
+	ret = q_vdb_open2(sroot, buf, true);
+	if (ret != NULL) {
 		ret->cachetype = CACHE_METADATA_PMS;
+		ret->repo = repo;
 		return ret;
 	}
 
-	dir = q_vdb_open2(sroot, portdir, true);
-	if (dir != NULL) {
-		ret->dir_ctx = dir;
+	ret = q_vdb_open2(sroot, portdir, true);
+	if (ret != NULL) {
 		ret->cachetype = CACHE_EBUILD;
+		ret->repo = repo;
 		return ret;
 	}
 
 	cache_close(ret);
-	warnf("could not open repository at %s (under root %s)",
-			portdir, sroot);
+	warnf("could not open repository at %s (under root %s)", portdir, sroot);
 
 	return NULL;
 }
@@ -100,49 +97,41 @@ cache_open(const char *sroot, const char *portdir)
 void
 cache_close(cache_ctx *ctx)
 {
-	if (ctx->dir_ctx != NULL)
-		q_vdb_close(ctx->dir_ctx);
 	if (ctx->repo != NULL)
 		free(ctx->repo);
-	free(ctx);
+	if (ctx->ebuilddir_ctx != NULL)
+		free(ctx->ebuilddir_ctx);
+	q_vdb_close(ctx);
 }
 
 cache_cat_ctx *
 cache_open_cat(cache_ctx *ctx, const char *name)
 {
-	cache_cat_ctx *ret = q_vdb_open_cat(ctx->dir_ctx, name);
-	if (ret != NULL)
-		ret->ctx = (q_vdb_ctx *)ctx;
-	return ret;
+	return q_vdb_open_cat(ctx, name);
 }
 
 cache_cat_ctx *
 cache_next_cat(cache_ctx *ctx)
 {
-	cache_cat_ctx *ret = q_vdb_next_cat(ctx->dir_ctx);
-	if (ret != NULL)
-		ret->ctx = (q_vdb_ctx *)ctx;
-	return ret;
+	return q_vdb_next_cat(ctx);
 }
 
 void
 cache_close_cat(cache_cat_ctx *cat_ctx)
 {
-	q_vdb_close_cat(cat_ctx);
+	return q_vdb_close_cat(cat_ctx);
 }
 
 cache_pkg_ctx *
 cache_open_pkg(cache_cat_ctx *cat_ctx, const char *name)
 {
-	cache_pkg_ctx *ret = q_vdb_open_pkg(cat_ctx, name);
-	ret->repo = ((cache_ctx *)cat_ctx->ctx)->repo;
-	return ret;
+	return q_vdb_open_pkg(cat_ctx, name);
 }
 
 cache_pkg_ctx *
 cache_next_pkg(cache_cat_ctx *cat_ctx)
 {
-	cache_ctx *ctx = (cache_ctx *)(cat_ctx->ctx);
+	cache_ctx *ctx = (cache_ctx *)cat_ctx->ctx;
 	cache_pkg_ctx *ret = NULL;
 
 	if (ctx->cachetype == CACHE_EBUILD) {
@@ -152,7 +141,10 @@ cache_next_pkg(cache_cat_ctx *cat_ctx)
 		 * to CAT/P like in VDB and metadata */
 		do {
 			if (ctx->ebuilddir_pkg_ctx == NULL) {
-				q_vdb_ctx *pkgdir = &ctx->ebuilddir_ctx;
+				q_vdb_ctx *pkgdir = ctx->ebuilddir_ctx;
+
+				if (pkgdir == NULL)
+					pkgdir = ctx->ebuilddir_ctx = xzalloc(sizeof(q_vdb_ctx));
 
 				if ((ctx->ebuilddir_pkg_ctx = q_vdb_next_pkg(cat_ctx)) == NULL)
 					return NULL;
@@ -492,14 +484,16 @@ void
 cache_close_pkg(cache_pkg_ctx *pkg_ctx)
 {
 	/* avoid free of cache_ctx' repo by q_vdb_close_pkg */
-	if (((cache_ctx *)pkg_ctx->cat_ctx->ctx)->repo == pkg_ctx->repo)
+	if (pkg_ctx->cat_ctx->ctx->repo == pkg_ctx->repo)
 		pkg_ctx->repo = NULL;
+
 	q_vdb_close_pkg(pkg_ctx);
 }
 
-int
-cache_foreach_pkg(const char *sroot, const char *portdir,
-		q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter)
+static int
+cache_foreach_pkg_int(const char *sroot, const char *portdir,
+		q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter,
+		bool sort, void *catsortfunc, void *pkgsortfunc)
 {
 	cache_ctx *ctx;
 	cache_cat_ctx *cat_ctx;
@@ -510,6 +504,12 @@ cache_foreach_pkg(const char *sroot, const char *portdir,
 	if (!ctx)
 		return EXIT_FAILURE;
 
+	ctx->do_sort = sort;
+	if (catsortfunc != NULL)
+		ctx->catsortfunc = catsortfunc;
+	if (pkgsortfunc != NULL)
+		ctx->pkgsortfunc = pkgsortfunc;
+
 	ret = 0;
 	while ((cat_ctx = cache_next_cat(ctx))) {
 		if (filter && !filter(cat_ctx, priv))
@@ -518,63 +518,26 @@ cache_foreach_pkg(const char *sroot, const char *portdir,
 			ret |= callback(pkg_ctx, priv);
 			cache_close_pkg(pkg_ctx);
 		}
+		cache_close_cat(cat_ctx);
 	}
+	cache_close(ctx);
 
 	return ret;
 }
 
+int
+cache_foreach_pkg(const char *sroot, const char *portdir,
+		q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter)
+{
+	return cache_foreach_pkg_int(sroot, portdir, callback, priv,
+			filter, false, NULL, NULL);
+}
+
 int
 cache_foreach_pkg_sorted(const char *sroot, const char *portdir,
 		q_vdb_pkg_cb callback, void *priv,
 		void *catsortfunc, void *pkgsortfunc)
 {
-	cache_ctx *ctx;
-	cache_cat_ctx *cat_ctx;
-	cache_pkg_ctx *pkg_ctx;
-	int ret = 0;
-	int c;
-	int p;
-	int cat_cnt;
-	int pkg_cnt;
-	struct dirent **cat_de;
-	struct dirent **pkg_de;
-
-	ctx = cache_open(sroot, portdir);
-	if (!ctx)
-		return EXIT_FAILURE;
-
-	if (catsortfunc == NULL)
-		catsortfunc = alphasort;
-	if (pkgsortfunc == NULL)
-		pkgsortfunc = alphasort;
-
-	cat_cnt = scandirat(ctx->dir_ctx->vdb_fd,
-			".", &cat_de, q_vdb_filter_cat, catsortfunc);
-	for (c = 0; c < cat_cnt; c++) {
-		cat_ctx = cache_open_cat(ctx, cat_de[c]->d_name);
-		if (!cat_ctx)
-			continue;
-
-		pkg_cnt = scandirat(ctx->dir_ctx->vdb_fd,
-				cat_de[c]->d_name, &pkg_de, q_vdb_filter_pkg, pkgsortfunc);
-		for (p = 0; p < pkg_cnt; p++) {
-			if (pkg_de[p]->d_name[0] == '-')
-				continue;
-
-			pkg_ctx = cache_open_pkg(cat_ctx, pkg_de[p]->d_name);
-			if (!pkg_ctx)
-				continue;
-
-			ret |= callback(pkg_ctx, priv);
-
-			cache_close_pkg(pkg_ctx);
-		}
-		scandir_free(pkg_de, pkg_cnt);
-
-		cache_close_cat(cat_ctx);
-	}
-	scandir_free(cat_de, cat_cnt);
-
-	cache_close(ctx);
-	return ret;
+	return cache_foreach_pkg_int(sroot, portdir, callback, priv,
+			NULL, true, catsortfunc, pkgsortfunc);
 }

diff --git a/libq/cache.h b/libq/cache.h
index 0157824..f9b1d43 100644
--- a/libq/cache.h
+++ b/libq/cache.h
@@ -13,19 +13,7 @@
 #include "atom.h"
 #include "vdb.h"
 
-typedef struct cache_ctx {
-	q_vdb_ctx *dir_ctx;
-	enum {
-		CACHE_UNSET = 0,
-		CACHE_METADATA_MD5,
-		CACHE_METADATA_PMS,
-		CACHE_EBUILD,
-	} cachetype;
-	q_vdb_pkg_ctx *ebuilddir_pkg_ctx;
-	q_vdb_cat_ctx *ebuilddir_cat_ctx;
-	q_vdb_ctx ebuilddir_ctx;
-	char *repo;
-} cache_ctx;
+#define cache_ctx     q_vdb_ctx
 #define cache_cat_ctx q_vdb_cat_ctx
 #define cache_pkg_ctx q_vdb_pkg_ctx
 

diff --git a/libq/vdb.c b/libq/vdb.c
index a32ba53..9c53b5b 100644
--- a/libq/vdb.c
+++ b/libq/vdb.c
@@ -45,6 +45,13 @@ q_vdb_open2(const char *sroot, const char *svdb, bool quiet)
 	if (ctx->dir == NULL)
 		goto cv_error;
 
+	ctx->do_sort = false;
+	ctx->cat_de = NULL;
+	ctx->catsortfunc = alphasort;
+	ctx->pkgsortfunc = alphasort;
+	ctx->repo = NULL;
+	ctx->ebuilddir_ctx = NULL;
+	ctx->ebuilddir_pkg_ctx = NULL;
 	return ctx;
 
  cv_error:
@@ -69,6 +76,8 @@ q_vdb_close(q_vdb_ctx *ctx)
 	/* closedir() above does this for us: */
 	/* close(ctx->vdb_fd); */
 	close(ctx->portroot_fd);
+	if (ctx->do_sort)
+		scandir_free(ctx->cat_de, ctx->cat_cnt);
 	free(ctx);
 }
 
@@ -136,6 +145,7 @@ q_vdb_open_cat(q_vdb_ctx *ctx, const char *name)
 	cat_ctx->fd = fd;
 	cat_ctx->dir = dir;
 	cat_ctx->ctx = ctx;
+	cat_ctx->pkg_de = NULL;
 	return cat_ctx;
 }
 
@@ -143,22 +153,39 @@ q_vdb_cat_ctx *
 q_vdb_next_cat(q_vdb_ctx *ctx)
 {
 	/* search for a category directory */
-	q_vdb_cat_ctx *cat_ctx;
-	const struct dirent *de;
+	q_vdb_cat_ctx *cat_ctx = NULL;
 
- next_cat:
-	de = readdir(ctx->dir);
-	if (!de) {
-		q_vdb_close(ctx);
-		return NULL;
-	}
+	if (ctx->do_sort) {
+		if (ctx->cat_de == NULL) {
+			ctx->cat_cnt = scandirat(ctx->vdb_fd,
+					".", &ctx->cat_de, q_vdb_filter_cat, ctx->catsortfunc);
+			ctx->cat_cur = 0;
+		}
+
+		while (ctx->cat_cur < ctx->cat_cnt) {
+			cat_ctx = q_vdb_open_cat(ctx, ctx->cat_de[ctx->cat_cur++]->d_name);
+			if (!cat_ctx)
+				continue;
+			break;
+		}
+	} else {
+		/* cheaper "streaming" variant */
+		const struct dirent *de;
+		do {
+			de = readdir(ctx->dir);
+			if (!de)
+				break;
 
-	if (q_vdb_filter_cat(de) == 0)
-		goto next_cat;
+			if (q_vdb_filter_cat(de) == 0)
+				continue;
+
+			cat_ctx = q_vdb_open_cat(ctx, de->d_name);
+			if (!cat_ctx)
+				continue;
 
-	cat_ctx = q_vdb_open_cat(ctx, de->d_name);
-	if (!cat_ctx)
-		goto next_cat;
+			break;
+		} while (1);
+	}
 
 	return cat_ctx;
 }
@@ -169,6 +196,8 @@ q_vdb_close_cat(q_vdb_cat_ctx *cat_ctx)
 	closedir(cat_ctx->dir);
 	/* closedir() above does this for us: */
 	/* close(ctx->fd); */
+	if (cat_ctx->ctx->do_sort)
+		scandir_free(cat_ctx->pkg_de, cat_ctx->pkg_cnt);
 	free(cat_ctx);
 }
 
@@ -194,7 +223,7 @@ q_vdb_open_pkg(q_vdb_cat_ctx *cat_ctx, const char *name)
 	q_vdb_pkg_ctx *pkg_ctx = xmalloc(sizeof(*pkg_ctx));
 	pkg_ctx->name = name;
 	pkg_ctx->slot = NULL;
-	pkg_ctx->repo = NULL;
+	pkg_ctx->repo = cat_ctx->ctx->repo;
 	pkg_ctx->fd = -1;
 	pkg_ctx->cat_ctx = cat_ctx;
 	return pkg_ctx;
@@ -203,22 +232,40 @@ q_vdb_open_pkg(q_vdb_cat_ctx *cat_ctx, const char *name)
 q_vdb_pkg_ctx *
 q_vdb_next_pkg(q_vdb_cat_ctx *cat_ctx)
 {
-	q_vdb_pkg_ctx *pkg_ctx;
-	const struct dirent *de;
+	q_vdb_pkg_ctx *pkg_ctx = NULL;
 
- next_pkg:
-	de = readdir(cat_ctx->dir);
-	if (!de) {
-		q_vdb_close_cat(cat_ctx);
-		return NULL;
-	}
+	if (cat_ctx->ctx->do_sort) {
+		if (cat_ctx->pkg_de == NULL) {
+			cat_ctx->pkg_cnt = scandirat(cat_ctx->fd, ".", &cat_ctx->pkg_de,
+					q_vdb_filter_pkg, cat_ctx->ctx->pkgsortfunc);
+			cat_ctx->pkg_cur = 0;
+		}
+
+		while (cat_ctx->pkg_cur < cat_ctx->pkg_cnt) {
+			pkg_ctx =
+				q_vdb_open_pkg(cat_ctx,
+						cat_ctx->pkg_de[cat_ctx->pkg_cur++]->d_name);
+			if (!pkg_ctx)
+				continue;
+			break;
+		}
+	} else {
+		const struct dirent *de;
+		do {
+			de = readdir(cat_ctx->dir);
+			if (!de)
+				break;
+
+			if (q_vdb_filter_pkg(de) == 0)
+				continue;
 
-	if (q_vdb_filter_pkg(de) == 0)
-		goto next_pkg;
+			pkg_ctx = q_vdb_open_pkg(cat_ctx, de->d_name);
+			if (!pkg_ctx)
+				continue;
 
-	pkg_ctx = q_vdb_open_pkg(cat_ctx, de->d_name);
-	if (!pkg_ctx)
-		goto next_pkg;
+			break;
+		} while (1);
+	}
 
 	return pkg_ctx;
 }
@@ -275,9 +322,10 @@ q_vdb_close_pkg(q_vdb_pkg_ctx *pkg_ctx)
 	free(pkg_ctx);
 }
 
-int
-q_vdb_foreach_pkg(const char *sroot, const char *svdb,
-		q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter)
+static int
+q_vdb_foreach_pkg_int(const char *sroot, const char *svdb,
+		q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter,
+		bool sort, void *catsortfunc, void *pkgsortfunc)
 {
 	q_vdb_ctx *ctx;
 	q_vdb_cat_ctx *cat_ctx;
@@ -288,6 +336,12 @@ q_vdb_foreach_pkg(const char *sroot, const char *svdb,
 	if (!ctx)
 		return EXIT_FAILURE;
 
+	ctx->do_sort = sort;
+	if (catsortfunc != NULL)
+		ctx->catsortfunc = catsortfunc;
+	if (pkgsortfunc != NULL)
+		ctx->pkgsortfunc = pkgsortfunc;
+
 	ret = 0;
 	while ((cat_ctx = q_vdb_next_cat(ctx))) {
 		if (filter && !filter(cat_ctx, priv))
@@ -296,53 +350,27 @@ q_vdb_foreach_pkg(const char *sroot, const char *svdb,
 			ret |= callback(pkg_ctx, priv);
 			q_vdb_close_pkg(pkg_ctx);
 		}
+		q_vdb_close_cat(cat_ctx);
 	}
+	q_vdb_close(ctx);
 
 	return ret;
 }
 
+int
+q_vdb_foreach_pkg(const char *sroot, const char *svdb,
+		q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter)
+{
+	return q_vdb_foreach_pkg_int(sroot, svdb, callback, priv,
+			filter, false, NULL, NULL);
+}
+
 int
 q_vdb_foreach_pkg_sorted(const char *sroot, const char *svdb,
 		q_vdb_pkg_cb callback, void *priv)
 {
-	q_vdb_ctx *ctx;
-	q_vdb_cat_ctx *cat_ctx;
-	q_vdb_pkg_ctx *pkg_ctx;
-	int ret = 0;
-	int c, p, cat_cnt, pkg_cnt;
-	struct dirent **cat_de, **pkg_de;
-
-	ctx = q_vdb_open(sroot, svdb);
-	if (!ctx)
-		return EXIT_FAILURE;
-
-	cat_cnt = scandirat(ctx->vdb_fd, ".", &cat_de, q_vdb_filter_cat, alphasort);
-	for (c = 0; c < cat_cnt; ++c) {
-		cat_ctx = q_vdb_open_cat(ctx, cat_de[c]->d_name);
-		if (!cat_ctx)
-			continue;
-
-		pkg_cnt = scandirat(ctx->vdb_fd, cat_de[c]->d_name, &pkg_de, q_vdb_filter_pkg, alphasort);
-		for (p = 0; p < pkg_cnt; ++p) {
-			if (pkg_de[p]->d_name[0] == '-')
-				continue;
-
-			pkg_ctx = q_vdb_open_pkg(cat_ctx, pkg_de[p]->d_name);
-			if (!pkg_ctx)
-				continue;
-
-			ret |= callback(pkg_ctx, priv);
-
-			q_vdb_close_pkg(pkg_ctx);
-		}
-		scandir_free(pkg_de, pkg_cnt);
-
-		q_vdb_close_cat(cat_ctx);
-	}
-	scandir_free(cat_de, cat_cnt);
-
-	q_vdb_close(ctx);
-	return ret;
+	return q_vdb_foreach_pkg_int(sroot, svdb, callback, priv,
+			NULL, true, NULL, NULL);
 }
 
 struct dirent *

diff --git a/libq/vdb.h b/libq/vdb.h
index 102f5a9..ee2ee69 100644
--- a/libq/vdb.h
+++ b/libq/vdb.h
@@ -7,31 +7,59 @@
 #define _VDB_H 1
 
 #include <dirent.h>
+#include <stdbool.h>
 
 #include "set.h"
 
+typedef struct q_vdb_ctx q_vdb_ctx;
+typedef struct q_vdb_cat_ctx q_vdb_cat_ctx;
+typedef struct q_vdb_pkg_ctx q_vdb_pkg_ctx;
+
 /* VDB context */
-typedef struct {
-	int portroot_fd, vdb_fd;
+struct q_vdb_ctx {
+	int portroot_fd;
+	int vdb_fd;
 	DIR *dir;
-} q_vdb_ctx;
+	struct dirent **cat_de;
+	size_t cat_cnt;
+	size_t cat_cur;
+	void *catsortfunc;
+	void *pkgsortfunc;
+	bool do_sort:1;
+	enum {
+		CACHE_UNSET = 0,
+		CACHE_METADATA_MD5,
+		CACHE_METADATA_PMS,
+		CACHE_EBUILD,
+		CACHE_VDB,
+	} cachetype:3;
+	q_vdb_pkg_ctx *ebuilddir_pkg_ctx;
+	q_vdb_cat_ctx *ebuilddir_cat_ctx;
+	q_vdb_ctx *ebuilddir_ctx;
+	char *repo;
+};
 
 /* Category context */
-typedef struct {
+struct q_vdb_cat_ctx {
 	const char *name;
 	int fd;
 	DIR *dir;
 	const q_vdb_ctx *ctx;
-} q_vdb_cat_ctx;
+	struct dirent **pkg_de;
+	size_t pkg_cnt;
+	size_t pkg_cur;
+};
 
 /* Package context */
-typedef struct {
+struct q_vdb_pkg_ctx {
 	const char *name;
-	char *slot, *repo;
-	size_t slot_len, repo_len;
+	char *slot;
+	char *repo;
+	size_t slot_len;
+	size_t repo_len;
 	int fd;
 	q_vdb_cat_ctx *cat_ctx;
-} q_vdb_pkg_ctx;
+};
 
 /* Global helpers */
 typedef int (q_vdb_pkg_cb)(q_vdb_pkg_ctx *, void *priv);

diff --git a/qgrep.c b/qgrep.c
index 9d78c18..6cb5697 100644
--- a/qgrep.c
+++ b/qgrep.c
@@ -413,9 +413,9 @@ qgrep_cache_cb(cache_pkg_ctx *pkg_ctx, void *priv)
 	/* need to construct path in portdir to ebuild, pass it to grep */
 	cctx = (cache_ctx *)(pkg_ctx->cat_ctx->ctx);
 	if (cctx->cachetype == CACHE_EBUILD) {
-		pfd = cctx->dir_ctx->vdb_fd;
+		pfd = cctx->vdb_fd;
 	} else {
-		pfd = openat(cctx->dir_ctx->vdb_fd, "../..", O_RDONLY|O_CLOEXEC);
+		pfd = openat(cctx->vdb_fd, "../..", O_RDONLY|O_CLOEXEC);
 	}
 
 	/* cat/pkg/pkg-ver.ebuild */