Gentoo Archives: gentoo-commits

From: Magnus Granberg <zorry@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] dev/zorry:master commit in: gobs/bin/, ebuild/dev-python/gobs/, gobs/pym/
Date: Sun, 29 Apr 2012 17:43:36
Message-Id: 1335721388.95733adad208effbf474e7b34005a9c44ebc4798.zorry@gentoo
1 commit: 95733adad208effbf474e7b34005a9c44ebc4798
2 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
3 AuthorDate: Sun Apr 29 17:43:08 2012 +0000
4 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
5 CommitDate: Sun Apr 29 17:43:08 2012 +0000
6 URL: http://git.overlays.gentoo.org/gitweb/?p=dev/zorry.git;a=commit;h=95733ada
7
8 redo the guest bin
9
10 ---
11 ebuild/dev-python/gobs/gobs-9999.ebuild | 2 +-
12 gobs/bin/gobs_buildquerys | 42 ---------------
13 gobs/bin/{gobs_host_jobs => gobs_guest_jobs} | 71 +++++++++++++-------------
14 gobs/bin/gobs_host_jobs | 21 --------
15 gobs/pym/build_log.py | 7 ++-
16 gobs/pym/build_queru.py | 38 ++++++++------
17 6 files changed, 62 insertions(+), 119 deletions(-)
18
19 diff --git a/ebuild/dev-python/gobs/gobs-9999.ebuild b/ebuild/dev-python/gobs/gobs-9999.ebuild
20 index 808e491..c178bd6 100644
21 --- a/ebuild/dev-python/gobs/gobs-9999.ebuild
22 +++ b/ebuild/dev-python/gobs/gobs-9999.ebuild
23 @@ -35,7 +35,7 @@ src_install() {
24 insinto /etc/gobs
25 doins ${FILESDIR}/gobs.conf || die
26 dobin ${S}/gobs/sbin/gobs_host_jobs || die
27 - dosbin ${S}/gobs/sbin/gobs_buildquerys || die
28 + dosbin ${S}/gobs/sbin/gobs_guest_jobs || die
29 dodoc ${S}/gobs/sql/pgdump.sql.gz || die
30 dodoc ${S}/gobs/doc/Setup.txt || die
31
32
33 diff --git a/gobs/bin/gobs_buildquerys b/gobs/bin/gobs_buildquerys
34 deleted file mode 100755
35 index 868d01a..0000000
36 --- a/gobs/bin/gobs_buildquerys
37 +++ /dev/null
38 @@ -1,42 +0,0 @@
39 -#!/usr/bin/python
40 -
41 -# Get the options from the config file set in gobs.readconf
42 -from gobs.readconf import get_conf_settings
43 -reader=get_conf_settings()
44 -gobs_settings_dict=reader.read_gobs_settings_all()
45 -# make a CM
46 -from gobs.ConnectionManager import connectionManager
47 -
48 -from gobs.check_setup import check_configure_guest
49 -from gobs.sync import git_pull
50 -from gobs.build_queru import queruaction
51 -import portage
52 -import sys
53 -import os
54 -import time
55 -from multiprocessing import Process
56 -
57 -def main_loop(config_profile):
58 - repeat = True
59 - #get a connection from the pool
60 - init_queru = queruaction(config_profile)
61 - while repeat:
62 - git_pull()
63 - if check_configure_guest(config_profile) is not True:
64 - # time.sleep(60)
65 - continue # retunr to the start of the function
66 - else:
67 - p = Process(target=init_queru.procces_qureru)
68 - p.start()
69 - p.join()
70 - time.sleep(10)
71 -
72 -def main():
73 - # Main
74 - config_profile = gobs_settings_dict['gobs_config']
75 - #we provide the main_loop with the ConnectionManager so we can hand out connections from within the loop
76 - main_loop(config_profile)
77 - connectionManager.closeAllConnections()
78 -
79 -if __name__ == "__main__":
80 - main()
81
82 diff --git a/gobs/bin/gobs_host_jobs b/gobs/bin/gobs_guest_jobs
83 similarity index 60%
84 copy from gobs/bin/gobs_host_jobs
85 copy to gobs/bin/gobs_guest_jobs
86 index 721351e..2c497d6 100755
87 --- a/gobs/bin/gobs_host_jobs
88 +++ b/gobs/bin/gobs_guest_jobs
89 @@ -1,56 +1,48 @@
90 #!/usr/bin/python
91 -# Copyright 2006-2011 Gentoo Foundation
92 -# Distributed under the terms of the GNU General Public License v2
93 -
94 from __future__ import print_function
95
96 -from gobs.init_setup_profile import setup_profile_main
97 +# Get the options from the config file set in gobs.readconf
98 from gobs.readconf import get_conf_settings
99 -from gobs.updatedb import update_db_main
100 -from gobs.buildquerydb import add_buildquery_main, del_buildquery_main
101 -
102 -reader = get_conf_settings()
103 +reader=get_conf_settings()
104 gobs_settings_dict=reader.read_gobs_settings_all()
105
106 -# make a CM
107 from gobs.ConnectionManager import connectionManager
108 CM=connectionManager(gobs_settings_dict)
109 #selectively import the pgsql/mysql querys
110 if CM.getName()=='pgsql':
111 from gobs.pgsql import *
112
113 -import logging
114 +from gobs.check_setup import check_configure_guest
115 +from gobs.sync import git_pull
116 +from gobs.build_queru import queruaction
117 +from gobs.buildquerydb import add_buildquery_main, del_buildquery_main
118 +
119 +import portage
120 +import sys
121 +import os
122 import time
123 +from multiprocessing import Process
124 +import logging
125
126 -def main():
127 - # Main
128 - config_profile = gobs_settings_dict['gobs_config']
129 - # Logging
130 - logging.basicConfig(filename=gobs_settings_dict['gobs_logfile'], \
131 - format='%(levelname)s: %(asctime)s %(message)s', level=logging.INFO)
132 +def main_loop(config_profile):
133 repeat = True
134 - logging.info("Job deamon started.")
135 + logging.info("Job and build deamon started.")
136 while repeat:
137 conn = CM.getConnection()
138 job = check_job_list(conn, config_profile)
139 -
140 if job is None:
141 - time.sleep(60)
142 CM.putConnection(conn)
143 - continue
144 + if check_configure_guest(config_profile) is not True:
145 + continue
146 + else:
147 + init_queru = queruaction(config_profile)
148 + p = Process(target=init_queru.procces_qureru)
149 + p.start()
150 + p.join()
151 + continue
152 else:
153 logging.info("Job: %s Type: %s Config: %s", job[1], job[0], config_profile)
154 - if job[0] == "updatedb":
155 - update_job_list(conn, "Runing", job[1])
156 - logging.info("Job %s is runing. Config: %s", job[0], config_profile)
157 - result = update_db_main()
158 - if result is True:
159 - update_job_list(conn, "Done", job[1])
160 - logging.info("Job %s is done. Config: %s", job[0], config_profile)
161 - else:
162 - update_job_list(conn, "Fail", job[1])
163 - logging.info("Job %s did fail. Config: %s", job[0], config_profile)
164 - elif job[0] == "addbuildquery":
165 + if job[0] == "addbuildquery":
166 update_job_list(conn, "Runing", job[1])
167 logging.info("Job %s is runing. Config: %s", job[0], config_profile)
168 result = add_buildquery_main(config_profile)
169 @@ -70,10 +62,19 @@ def main():
170 else:
171 update_job_list(conn, "Fail", job[1])
172 logging.info("Job %s did fail. Config: %s", job[0], config_profile)
173 - repeat = False
174 + repeat = False
175 + time.sleep(10)
176 CM.putConnection(conn)
177 - time.sleep(60)
178 - CM.closeAllConnections()
179 + return
180 +
181 +def main():
182 + # Main
183 + logging.basicConfig(filename=gobs_settings_dict['gobs_logfile'], \
184 + format='%(levelname)s: %(asctime)s %(message)s', level=logging.INFO)
185 + config_profile = gobs_settings_dict['gobs_config']
186 + #we provide the main_loop with the ConnectionManager so we can hand out connections from within the loop
187 + main_loop(config_profile)
188 + CM.closeAllConnections()
189
190 if __name__ == "__main__":
191 - main()
192 \ No newline at end of file
193 + main()
194
195 diff --git a/gobs/bin/gobs_host_jobs b/gobs/bin/gobs_host_jobs
196 index 721351e..812b942 100755
197 --- a/gobs/bin/gobs_host_jobs
198 +++ b/gobs/bin/gobs_host_jobs
199 @@ -4,7 +4,6 @@
200
201 from __future__ import print_function
202
203 -from gobs.init_setup_profile import setup_profile_main
204 from gobs.readconf import get_conf_settings
205 from gobs.updatedb import update_db_main
206 from gobs.buildquerydb import add_buildquery_main, del_buildquery_main
207 @@ -50,26 +49,6 @@ def main():
208 else:
209 update_job_list(conn, "Fail", job[1])
210 logging.info("Job %s did fail. Config: %s", job[0], config_profile)
211 - elif job[0] == "addbuildquery":
212 - update_job_list(conn, "Runing", job[1])
213 - logging.info("Job %s is runing. Config: %s", job[0], config_profile)
214 - result = add_buildquery_main(config_profile)
215 - if result is True:
216 - update_job_list(conn, "Done", job[1])
217 - logging.info("Job %s is done. Config: %s", job[0], config_profile)
218 - else:
219 - update_job_list(conn, "Fail", job[1])
220 - logging.info("Job %s did fail. Config: %s", job[0], config_profile)
221 - elif job[0] == "delbuildquery":
222 - update_job_list(conn, "Runing", job[1])
223 - logging.info("Job %s is runing. Config: %s", job[0], config_profile)
224 - result = del_buildquery_main(config_profile)
225 - if result is True:
226 - update_job_list(conn, "Done", job[1])
227 - logging.info("Job %s is done. Config: %s", job[0], config_profile)
228 - else:
229 - update_job_list(conn, "Fail", job[1])
230 - logging.info("Job %s did fail. Config: %s", job[0], config_profile)
231 repeat = False
232 CM.putConnection(conn)
233 time.sleep(60)
234
235 diff --git a/gobs/pym/build_log.py b/gobs/pym/build_log.py
236 index 35724c4..6742943 100644
237 --- a/gobs/pym/build_log.py
238 +++ b/gobs/pym/build_log.py
239 @@ -2,6 +2,7 @@ from __future__ import print_function
240 import re
241 import os
242 import platform
243 +import logging
244 try:
245 from subprocess import getstatusoutput as subprocess_getstatusoutput
246 except ImportError:
247 @@ -46,7 +47,7 @@ class gobs_buildlog(object):
248 categories = cpvr_list[0]
249 package = cpvr_list[1]
250 ebuild_version = cpv_getversion(pkg.cpv)
251 - print('cpv: ' + pkg.cpv)
252 + logging.info('cpv: %s', pkg.cpv)
253 init_package = gobs_package(settings, myportdb)
254 package_id = have_package_db(conn, categories, package)
255 # print("package_id %s" % package_id, file=sys.stdout)
256 @@ -560,7 +561,7 @@ class gobs_buildlog(object):
257 self.write_msg_file(msg_line, emerge_info_logfilename)
258 os.chmod(settings.get("PORTAGE_LOG_FILE"), 0664)
259 os.chmod(emerge_info_logfilename, 0664)
260 - print("Package: ", pkg.cpv, "logged to db.")
261 + logging.info("Package: %s logged to db.", pkg.cpv)
262 else:
263 # FIXME Remove the log some way so
264 # mergetask._locate_failure_log(x) works in action_build()
265 @@ -568,5 +569,5 @@ class gobs_buildlog(object):
266 # os.remove(settings.get("PORTAGE_LOG_FILE"))
267 #except:
268 # pass
269 - print("Package: ", pkg.cpv, "NOT logged to db.")
270 + logging.info("Package %s NOT logged to db.", pkg.cpv)
271 CM.putConnection(conn)
272
273 diff --git a/gobs/pym/build_queru.py b/gobs/pym/build_queru.py
274 index 200c41d..57c26a2 100644
275 --- a/gobs/pym/build_queru.py
276 +++ b/gobs/pym/build_queru.py
277 @@ -15,6 +15,8 @@ import os
278 import re
279 import sys
280 import signal
281 +import logging
282 +
283 from gobs.manifest import gobs_manifest
284 from gobs.depclean import main_depclean
285 from gobs.flags import gobs_use_flags
286 @@ -56,6 +58,8 @@ class queruaction(object):
287 self._mysettings = portage.config(config_root = "/")
288 self._config_profile = config_profile
289 self._myportdb = portage.portdb
290 + logging.basicConfig(filename=gobs_settings_dict['gobs_logfile'], \
291 + format='%(levelname)s: %(asctime)s %(message)s', level=logging.INFO)
292
293 def log_fail_queru(self, build_dict, settings):
294 conn=CM.getConnection()
295 @@ -158,7 +162,7 @@ class queruaction(object):
296 for k, v in use_changes.iteritems():
297 k_package = portage.versions.cpv_getkey(k)
298 iteritems_packages[ k_package ] = v
299 - print('iteritems_packages', iteritems_packages)
300 + logging.info('iteritems_packages %s', iteritems_packages)
301 build_cpv_dict = iteritems_packages
302 if use_changes is not None:
303 for k, v in build_cpv_dict.iteritems():
304 @@ -168,13 +172,13 @@ class queruaction(object):
305 build_use_flags_list.append(x)
306 if y is False:
307 build_use_flags_list.append("-" + x)
308 - print(k, build_use_flags_list)
309 + logging.info("k: %s, build_use_flags_list: %s", k, build_use_flags_list)
310 if not build_use_flags_list == []:
311 build_use_flags = ""
312 for flags in build_use_flags_list:
313 build_use_flags = build_use_flags + flags + ' '
314 filetext = k + ' ' + build_use_flags
315 - print('filetext', filetext)
316 + logging.info('filetext %s', filetext)
317 with open("/etc/portage/package.use/gobs.use", "a") as f:
318 f.write(filetext)
319 f.write('\n')
320 @@ -208,8 +212,8 @@ class queruaction(object):
321 graph_copy.difference_update(removed_nodes)
322 if not graph_copy.hasallzeros(ignore_priority = \
323 DepPrioritySatisfiedRange.ignore_medium):
324 - print("\n!!! --buildpkgonly requires all dependencies to be merged.")
325 - print("!!! Cannot merge requested packages. Merge deps and try again.\n")
326 + logging.info("\n!!! --buildpkgonly requires all dependencies to be merged.")
327 + logging.info("!!! Cannot merge requested packages. Merge deps and try again.\n")
328 return 1, settings, trees, mtimedb
329
330 mydepgraph.saveNomergeFavorites()
331 @@ -222,7 +226,7 @@ class queruaction(object):
332 clear_caches(trees)
333
334 retval = mergetask.merge()
335 - print('retval', retval)
336 + logging.info('retval %s', retval)
337 if retval:
338 build_dict['type_fail'] = 'merge fail'
339 build_dict['check_fail'] = True
340 @@ -424,7 +428,7 @@ class queruaction(object):
341 for x in myfiles:
342 ext = os.path.splitext(x)[1]
343 if (ext == ".ebuild" or ext == ".tbz2") and os.path.exists(os.path.abspath(x)):
344 - print(colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n"))
345 + logging.info("BAD\n*** emerging by path is broken and may not always work!!!\n")
346 break
347
348 root_config = trees[settings["ROOT"]]["root_config"]
349 @@ -444,11 +448,11 @@ class queruaction(object):
350 # Need to handle empty sets specially, otherwise emerge will react
351 # with the help message for empty argument lists
352 if oldargs and not myfiles:
353 - print("emerge: no targets left after set expansion")
354 + logging.info("emerge: no targets left after set expansion")
355 return 0
356
357 if ("--tree" in myopts) and ("--columns" in myopts):
358 - print("emerge: can't specify both of \"--tree\" and \"--columns\".")
359 + logging.info("emerge: can't specify both of \"--tree\" and \"--columns\".")
360 return 1
361
362 if '--emptytree' in myopts and '--noreplace' in myopts:
363 @@ -625,13 +629,13 @@ class queruaction(object):
364 if ebuild_version_checksum_tree == build_dict['checksum']:
365 init_flags = gobs_use_flags(settings, portdb, cpv)
366 build_use_flags_list = init_flags.comper_useflags(build_dict)
367 - print("build_use_flags_list", build_use_flags_list)
368 + logging.info("build_use_flags_list %s", build_use_flags_list)
369 manifest_error = init_manifest.check_file_in_manifest(portdb, cpv, build_dict, build_use_flags_list)
370 if manifest_error is None:
371 build_dict['check_fail'] = False
372 build_cpv_dict = {}
373 build_cpv_dict[cpv] = build_use_flags_list
374 - print(build_cpv_dict)
375 + logging.info("build_cpv_dict: %s", build_cpv_dict)
376 return build_cpv_dict
377 else:
378 build_dict['type_fail'] = "Manifest error"
379 @@ -654,11 +658,11 @@ class queruaction(object):
380 for flags in build_use_flags_list:
381 build_use_flags = build_use_flags + flags + " "
382 filetext = '=' + k + ' ' + build_use_flags
383 - print('filetext', filetext)
384 + logging.info('filetext: %s', filetext)
385 with open("/etc/portage/package.use/gobs.use", "a") as f:
386 f.write(filetext)
387 f.write('\n')
388 - print('build_cpv_list', build_cpv_list)
389 + logging.info('build_cpv_list: %s', build_cpv_list)
390 argscmd = []
391 if not "nooneshort" in build_dict['post_message']:
392 argscmd.append("--oneshot")
393 @@ -666,11 +670,11 @@ class queruaction(object):
394 argscmd.append("--usepkg")
395 for build_cpv in build_cpv_list:
396 argscmd.append(build_cpv)
397 - print(argscmd)
398 + logging.info("argscmd: %s" argscmd)
399 # Call main_emerge to build the package in build_cpv_list
400 build_fail = self.emerge_main(argscmd, build_dict)
401 # Run depclean
402 - print('build_fail', build_fail)
403 + logging.info('build_fail: %s', build_fail)
404 if not "noclean" in build_dict['post_message']:
405 depclean_fail = main_depclean()
406 try:
407 @@ -690,10 +694,10 @@ class queruaction(object):
408 if build_dict is None:
409 CM.putConnection(conn)
410 return
411 - print("build_dict", build_dict)
412 + logging.info("build_dict: %s", build_dict)
413 if not build_dict['ebuild_id'] is None and build_dict['checksum'] is not None:
414 buildqueru_cpv_dict = self.make_build_list(build_dict, settings, portdb)
415 - print('buildqueru_cpv_dict', buildqueru_cpv_dict)
416 + logging.info('buildqueru_cpv_dict: %s', buildqueru_cpv_dict)
417 if buildqueru_cpv_dict is None:
418 CM.putConnection(conn)
419 return