Gentoo Archives: gentoo-commits

From: Mike Frysinger <vapier@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/portage-utils:master commit in: /, libq/
Date: Sat, 28 Nov 2015 02:44:49
Message-Id: 1448664350.89d451252b18c0ae2f768f6248964638f8aadd9f.vapier@gentoo
1 commit: 89d451252b18c0ae2f768f6248964638f8aadd9f
2 Author: Mike Frysinger <vapier <AT> gentoo <DOT> org>
3 AuthorDate: Fri Nov 27 22:45:50 2015 +0000
4 Commit: Mike Frysinger <vapier <AT> gentoo <DOT> org>
5 CommitDate: Fri Nov 27 22:45:50 2015 +0000
6 URL: https://gitweb.gentoo.org/proj/portage-utils.git/commit/?id=89d45125
7
8 qcache: add multiple overlay support
9
10 We have to rework two things:
11 - turn the arch array into a set; the implementation is not optimal
12 - pass in the current overlay to the callback func so it can keep
13 track of when we switch between them
14
15 With that in place, walking all the overlays is pretty quick.
16
17 URL: https://bugs.gentoo.org/553260
18
19 libq/virtuals.c | 16 +++++++
20 qcache.c | 131 +++++++++++++++++++++++++++++++++-----------------------
21 2 files changed, 94 insertions(+), 53 deletions(-)
22
23 diff --git a/libq/virtuals.c b/libq/virtuals.c
24 index 536c622..1818281 100644
25 --- a/libq/virtuals.c
26 +++ b/libq/virtuals.c
27 @@ -46,6 +46,22 @@ add_set(const char *name, queue *q)
28 return append_set(q, ll);
29 }
30
31 +/* Performance here is terrible. Should use a hash at some point. */
32 +_q_static queue *
33 +add_set_unique(const char *name, queue *q, bool *ok)
34 +{
35 + queue *ll = q;
36 + while (ll) {
37 + if (!strcmp(ll->name, name)) {
38 + *ok = false;
39 + return q;
40 + }
41 + ll = ll->next;
42 + }
43 + *ok = true;
44 + return add_set(name, q);
45 +}
46 +
47 /* remove a set from a cache. matches ->name and frees name,item */
48 _q_static queue *
49 del_set(char *s, queue *q, int *ok)
50
51 diff --git a/qcache.c b/qcache.c
52 index 5cac394..d4b7884 100644
53 --- a/qcache.c
54 +++ b/qcache.c
55 @@ -51,9 +51,10 @@ static const char * const qcache_opts_help[] = {
56 /********************************************************************/
57
58 typedef struct {
59 - char *category;
60 - char *package;
61 - char *ebuild;
62 + const char *overlay;
63 + const char *category;
64 + const char *package;
65 + const char *ebuild;
66 portage_cache *cache_data;
67 unsigned char cur;
68 unsigned char num;
69 @@ -63,7 +64,7 @@ typedef struct {
70 /* Global Variables */
71 /********************************************************************/
72
73 -static char **archlist; /* Read from PORTDIR/profiles/arch.list in qcache_init() */
74 +static queue *arches;
75 static int archlist_count;
76 static size_t arch_longest_len;
77 const char status[3] = {'-', '~', '+'};
78 @@ -113,6 +114,7 @@ int decode_status(char c)
79 _q_static
80 int decode_arch(const char *arch)
81 {
82 + queue *q = arches;
83 int a;
84 const char *p;
85
86 @@ -120,9 +122,13 @@ int decode_arch(const char *arch)
87 if (*p == '~' || *p == '-')
88 p++;
89
90 - for (a = 0; a < archlist_count; ++a)
91 - if (strcmp(archlist[a], p) == 0)
92 + a = 0;
93 + while (q) {
94 + if (strcmp(q->name, p) == 0)
95 return a;
96 + ++a;
97 + q = q->next;
98 + }
99
100 return -1;
101 }
102 @@ -139,6 +145,7 @@ int decode_arch(const char *arch)
103 _q_static
104 void print_keywords(const char *category, const char *ebuild, int *keywords)
105 {
106 + queue *arch = arches;
107 int a;
108 char *package;
109
110 @@ -149,14 +156,13 @@ void print_keywords(const char *category, const char *ebuild, int *keywords)
111 for (a = 0; a < archlist_count; ++a) {
112 switch (keywords[a]) {
113 case stable:
114 - printf("%s%c%s%s ", GREEN, status[keywords[a]], archlist[a], NORM);
115 + printf("%s%c%s%s ", GREEN, status[keywords[a]], arch->name, NORM);
116 break;
117 case testing:
118 - printf("%s%c%s%s ", YELLOW, status[keywords[a]], archlist[a], NORM);
119 - break;
120 - default:
121 + printf("%s%c%s%s ", YELLOW, status[keywords[a]], arch->name, NORM);
122 break;
123 }
124 + arch = arch->next;
125 }
126
127 printf("\n");
128 @@ -386,6 +392,8 @@ int qcache_ebuild_select(const struct dirent *entry)
129 /* Traversal function */
130 /********************************************************************/
131
132 +_q_static void qcache_load_arches(const char *overlay);
133 +
134 /*
135 * int qcache_traverse(void (*func)(qcache_data*));
136 *
137 @@ -399,14 +407,16 @@ int qcache_ebuild_select(const struct dirent *entry)
138 * exit or return -1 on failure.
139 */
140 _q_static
141 -int qcache_traverse(void (*func)(qcache_data*))
142 +int qcache_traverse_overlay(void (*func)(qcache_data*), const char *overlay)
143 {
144 - qcache_data data;
145 + qcache_data data = {
146 + .overlay = overlay,
147 + };
148 char *catpath, *pkgpath, *ebuildpath, *cachepath;
149 int i, j, k, len, num_cat, num_pkg, num_ebuild;
150 struct dirent **categories, **packages, **ebuilds;
151
152 - xasprintf(&catpath, "%s/dep/%s", portedb, portdir);
153 + xasprintf(&catpath, "%s/dep/%s", portedb, overlay);
154
155 if (-1 == (num_cat = scandir(catpath, &categories, qcache_file_select, alphasort))) {
156 errp("%s", catpath);
157 @@ -418,7 +428,7 @@ int qcache_traverse(void (*func)(qcache_data*))
158
159 /* traverse categories */
160 for (i = 0; i < num_cat; i++) {
161 - xasprintf(&pkgpath, "%s/%s", portdir, categories[i]->d_name);
162 + xasprintf(&pkgpath, "%s/%s", overlay, categories[i]->d_name);
163
164 if (-1 == (num_pkg = scandir(pkgpath, &packages, qcache_file_select, alphasort))) {
165 if (errno != ENOENT)
166 @@ -439,7 +449,7 @@ int qcache_traverse(void (*func)(qcache_data*))
167
168 /* traverse packages */
169 for (j = 0; j < num_pkg; j++) {
170 - xasprintf(&ebuildpath, "%s/%s/%s", portdir, categories[i]->d_name, packages[j]->d_name);
171 + xasprintf(&ebuildpath, "%s/%s/%s", overlay, categories[i]->d_name, packages[j]->d_name);
172
173 if (-1 == (num_ebuild = scandir(ebuildpath, &ebuilds, qcache_ebuild_select, qcache_vercmp))) {
174 /* Do not complain about spurious files */
175 @@ -504,9 +514,28 @@ int qcache_traverse(void (*func)(qcache_data*))
176 free(categories);
177 free(catpath);
178
179 + return 0;
180 +}
181 +
182 +_q_static
183 +int qcache_traverse(void (*func)(qcache_data*))
184 +{
185 + int ret;
186 + size_t n;
187 + const char *overlay;
188 +
189 + /* Preload all the arches. Not entirely correctly (as arches are bound
190 + * to overlays if set), but oh well. */
191 + array_for_each(overlays, n, overlay)
192 + qcache_load_arches(overlay);
193 +
194 + ret = 0;
195 + array_for_each(overlays, n, overlay)
196 + ret |= qcache_traverse_overlay(func, overlay);
197 +
198 func(NULL);
199
200 - return 0;
201 + return ret;
202 }
203
204 /********************************************************************/
205 @@ -653,6 +682,8 @@ _q_static
206 void qcache_stats(qcache_data *data)
207 {
208 static time_t runtime;
209 + static queue *allcats;
210 + static const char *last_overlay;
211 static int numpkg = 0;
212 static int numebld = 0;
213 static int numcat;
214 @@ -686,12 +717,14 @@ void qcache_stats(qcache_data *data)
215 (int)arch_longest_len, "", RED, "only", NORM);
216 printf("+%.*s+\n", (int)(arch_longest_len + 46), border);
217
218 + queue *arch = arches;
219 for (a = 0; a < archlist_count; ++a) {
220 - printf("| %s%*s%s |", GREEN, (int)arch_longest_len, archlist[a], NORM);
221 + printf("| %s%*s%s |", GREEN, (int)arch_longest_len, arch->name, NORM);
222 printf("%s%8d%s |", BLUE, packages_stable[a], NORM);
223 printf("%s%8d%s |", BLUE, packages_testing[a], NORM);
224 printf("%s%8d%s |", BLUE, packages_testing[a]+packages_stable[a], NORM);
225 printf("%s%11.2f%s%% |\n", BLUE, (100.0*(packages_testing[a]+packages_stable[a]))/numpkg, NORM);
226 + arch = arch->next;
227 }
228
229 printf("+%.*s+\n\n", (int)(arch_longest_len + 46), border);
230 @@ -704,23 +737,33 @@ void qcache_stats(qcache_data *data)
231 free(packages_testing);
232 free(keywords);
233 free(current_package_keywords);
234 + free_sets(allcats);
235 return;
236 }
237
238 - if (!numpkg) {
239 - struct dirent **categories;
240 + if (last_overlay != data->overlay) {
241 + DIR *dir;
242 + struct dirent *de;
243 char *catpath;
244
245 - xasprintf(&catpath, "%s/dep/%s", portedb, portdir);
246 + runtime = time(NULL);
247
248 - if (-1 == (numcat = scandir(catpath, &categories, qcache_file_select, alphasort))) {
249 - errp("%s", catpath);
250 - free(catpath);
251 - }
252 - scandir_free(categories, numcat);
253 + xasprintf(&catpath, "%s/dep/%s", portedb, data->overlay);
254 + dir = opendir(catpath);
255 + while ((de = readdir(dir)))
256 + if (de->d_type == DT_DIR && de->d_name[0] != '.') {
257 + bool ok;
258 + allcats = add_set_unique(de->d_name, allcats, &ok);
259 + if (ok)
260 + ++numcat;
261 + }
262 + closedir(dir);
263 + free(catpath);
264
265 - runtime = time(NULL);
266 + last_overlay = data->overlay;
267 + }
268
269 + if (!numpkg) {
270 packages_stable = xcalloc(archlist_count, sizeof(*packages_stable));
271 packages_testing = xcalloc(archlist_count, sizeof(*packages_testing));
272 keywords = xcalloc(archlist_count, sizeof(*keywords));
273 @@ -812,26 +855,15 @@ void qcache_testing_only(qcache_data *data)
274 /* Misc functions */
275 /********************************************************************/
276
277 -/*
278 - * int qcache_init();
279 - *
280 - * Initialize variables (archlist, num_arches)
281 - *
282 - * OUT:
283 - * 0 is return on success.
284 - * ERR:
285 - * -1 is returned on error.
286 - */
287 _q_static
288 -bool qcache_init(void)
289 +void qcache_load_arches(const char *overlay)
290 {
291 - bool ret = false;
292 FILE *fp;
293 char *filename, *s;
294 size_t buflen, linelen;
295 char *buf;
296
297 - xasprintf(&filename, "%s/profiles/arch.list", portdir);
298 + xasprintf(&filename, "%s/profiles/arch.list", overlay);
299 fp = fopen(filename, "re");
300 if (!fp)
301 goto done;
302 @@ -847,18 +879,18 @@ bool qcache_init(void)
303 if (buf[0] == '\0')
304 continue;
305
306 - ++archlist_count;
307 - archlist = xrealloc_array(archlist, sizeof(*archlist), archlist_count);
308 - archlist[archlist_count - 1] = xstrdup(buf);
309 - arch_longest_len = MAX(arch_longest_len, strlen(buf));
310 + bool ok;
311 + arches = add_set_unique(buf, arches, &ok);
312 + if (ok) {
313 + ++archlist_count;
314 + arch_longest_len = MAX(arch_longest_len, strlen(buf));
315 + }
316 }
317 free(buf);
318
319 - ret = true;
320 fclose(fp);
321 done:
322 free(filename);
323 - return ret;
324 }
325
326 /*
327 @@ -869,11 +901,7 @@ bool qcache_init(void)
328 _q_static
329 void qcache_free(void)
330 {
331 - size_t a;
332 -
333 - for (a = 0; a < archlist_count; ++a)
334 - free(archlist[a]);
335 - free(archlist);
336 + free_sets(arches);
337 }
338
339 /********************************************************************/
340 @@ -906,9 +934,6 @@ int qcache_main(int argc, char **argv)
341 }
342 }
343
344 - if (!qcache_init())
345 - err("Could not initialize arch list");
346 -
347 if (optind < argc)
348 qcache_test_arch = decode_arch(argv[optind]);