Gentoo Archives: gentoo-commits

From: Magnus Granberg <zorry@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] dev/zorry:master commit in: gobs/pym/
Date: Sat, 01 Dec 2012 11:26:32
Message-Id: 1354361152.d88a4d2bd81ed9b47a26c12ea0d8d725ff3dcfb5.zorry@gentoo
1 commit: d88a4d2bd81ed9b47a26c12ea0d8d725ff3dcfb5
2 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
3 AuthorDate: Sat Dec 1 11:25:52 2012 +0000
4 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
5 CommitDate: Sat Dec 1 11:25:52 2012 +0000
6 URL: http://git.overlays.gentoo.org/gitweb/?p=dev/zorry.git;a=commit;h=d88a4d2b
7
8 fix linment and typos
9
10 ---
11 gobs/pym/arch.py | 25 ---
12 gobs/pym/check_setup.py | 2 +-
13 gobs/pym/package.py | 511 ++++++++++++++++++++++-----------------------
14 gobs/pym/pgsql_querys.py | 518 +++++++++++++++++++++++-----------------------
15 gobs/pym/sync.py | 2 +-
16 gobs/pym/updatedb.py | 79 ++++----
17 6 files changed, 554 insertions(+), 583 deletions(-)
18
19 diff --git a/gobs/pym/arch.py b/gobs/pym/arch.py
20 deleted file mode 100644
21 index ebd0017..0000000
22 --- a/gobs/pym/arch.py
23 +++ /dev/null
24 @@ -1,25 +0,0 @@
25 -import portage
26 -from gobs.readconf import get_conf_settings
27 -reader=get_conf_settings()
28 -gobs_settings_dict=reader.read_gobs_settings_all()
29 -# make a CM
30 -from gobs.ConnectionManager import connectionManager
31 -CM=connectionManager(gobs_settings_dict)
32 -#selectively import the pgsql/mysql querys
33 -if CM.getName()=='pgsql':
34 - from gobs.pgsql import *
35 -
36 -class gobs_arch(object):
37 -
38 - def update_arch_db(self):
39 - conn = CM.getConnection()
40 - # FIXME: check for new keyword
41 - # Add arch db (keywords)
42 - if get_arch_db(conn) is None:
43 - arch_list = portage.archlist
44 - for arch in arch_list:
45 - if arch[0] not in ["~","-"]:
46 - arch_list.append("-" + arch)
47 - arch_list.append("-*")
48 - add_new_arch_db(conn,arch_list)
49 - CM.putConnection(conn)
50 \ No newline at end of file
51
52 diff --git a/gobs/pym/check_setup.py b/gobs/pym/check_setup.py
53 index a89e892..37d0285 100644
54 --- a/gobs/pym/check_setup.py
55 +++ b/gobs/pym/check_setup.py
56 @@ -14,7 +14,7 @@ from gobs.ConnectionManager import connectionManager
57 CM=connectionManager(gobs_settings_dict)
58 #selectively import the pgsql/mysql querys
59 if CM.getName()=='pgsql':
60 - from gobs.pgsql import *
61 + from gobs.pgsql_querys import *
62
63 def check_make_conf():
64 # Get the config list
65
66 diff --git a/gobs/pym/package.py b/gobs/pym/package.py
67 index cb6a1f1..e123e8e 100644
68 --- a/gobs/pym/package.py
69 +++ b/gobs/pym/package.py
70 @@ -4,7 +4,6 @@ from gobs.flags import gobs_use_flags
71 from gobs.repoman_gobs import gobs_repoman
72 from gobs.manifest import gobs_manifest
73 from gobs.text import get_file_text, get_ebuild_text
74 -from gobs.old_cpv import gobs_old_cpv
75 from gobs.readconf import get_conf_settings
76 from gobs.flags import gobs_use_flags
77 reader=get_conf_settings()
78 @@ -27,114 +26,114 @@ class gobs_package(object):
79
80 def change_config(self, config_setup):
81 # Change config_root config_setup = table config
82 - my_new_setup = "/var/cache/gobs/" + gobs_settings_dict['gobs_gitreponame'] + "/" + config_setup + "/"
83 - mysettings_setup = portage.config(config_root = my_new_setup)
84 - return mysettings_setup
85 + my_new_setup = "/var/cache/gobs/" + gobs_settings_dict['gobs_gitreponame'] + "/" + config_setup + "/"
86 + mysettings_setup = portage.config(config_root = my_new_setup)
87 + return mysettings_setup
88
89 def config_match_ebuild(self, cp, config_id_list):
90 config_cpv_listDict ={}
91 - if config_id_list == []:
92 - return config_cpv_listDict
93 - conn=CM.getConnection()
94 - for config_id in config_id_list:
95 + if config_id_list == []:
96 + return config_cpv_listDict
97 + conn=CM.getConnection()
98 + for config_id in config_id_list:
99 # Change config/setup
100 for config_id in config_id_list:
101
102 - # Change config/setup
103 - config_setup = get_config_db(conn, config_id)
104 - mysettings_setup = self.change_config(config_setup)
105 - myportdb_setup = portage.portdbapi(mysettings=mysettings_setup)
106 -
107 - # Get the latest cpv from portage with the config that we can build
108 - build_cpv = myportdb_setup.xmatch('bestmatch-visible', cp)
109 -
110 - # Check if could get cpv from portage and add it to the config_cpv_listDict.
111 - if build_cpv != "":
112 -
113 - # Get the iuse and use flags for that config/setup and cpv
114 - init_useflags = gobs_use_flags(mysettings_setup, myportdb_setup, build_cpv)
115 - iuse_flags_list, final_use_list = init_useflags.get_flags()
116 - iuse_flags_list2 = []
117 - for iuse_line in iuse_flags_list:
118 - iuse_flags_list2.append( init_useflags.reduce_flag(iuse_line))
119 -
120 - # Dict the needed info
121 - attDict = {}
122 - attDict['cpv'] = build_cpv
123 - attDict['useflags'] = final_use_list
124 - attDict['iuse'] = iuse_flags_list2
125 - config_cpv_listDict[config_id] = attDict
126 -
127 - # Clean some cache
128 - myportdb_setup.close_caches()
129 - portage.portdbapi.portdbapi_instances.remove(myportdb_setup)
130 - CM.putConnection(conn)
131 - return config_cpv_listDict
132 + # Change config/setup
133 + config_setup = get_config_db(conn, config_id)
134 + mysettings_setup = self.change_config(config_setup)
135 + myportdb_setup = portage.portdbapi(mysettings=mysettings_setup)
136 +
137 + # Get the latest cpv from portage with the config that we can build
138 + build_cpv = myportdb_setup.xmatch('bestmatch-visible', cp)
139 +
140 + # Check if could get cpv from portage and add it to the config_cpv_listDict.
141 + if build_cpv != "":
142 +
143 + # Get the iuse and use flags for that config/setup and cpv
144 + init_useflags = gobs_use_flags(mysettings_setup, myportdb_setup, build_cpv)
145 + iuse_flags_list, final_use_list = init_useflags.get_flags()
146 + iuse_flags_list2 = []
147 + for iuse_line in iuse_flags_list:
148 + iuse_flags_list2.append( init_useflags.reduce_flag(iuse_line))
149 +
150 + # Dict the needed info
151 + attDict = {}
152 + attDict['cpv'] = build_cpv
153 + attDict['useflags'] = final_use_list
154 + attDict['iuse'] = iuse_flags_list2
155 + config_cpv_listDict[config_id] = attDict
156 +
157 + # Clean some cache
158 + myportdb_setup.close_caches()
159 + portage.portdbapi.portdbapi_instances.remove(myportdb_setup)
160 + CM.putConnection(conn)
161 + return config_cpv_listDict
162
163 def get_ebuild_metadata(self, cpv, repo):
164 - # Get the auxdbkeys infos for the ebuild
165 - try:
166 - ebuild_auxdb_list = self._myportdb.aux_get(cpv, portage.auxdbkeys, myrepo=repo)
167 - except:
168 - ebuild_auxdb_list = []
169 - else:
170 - for i in range(len(ebuild_auxdb_list)):
171 - if ebuild_auxdb_list[i] == '':
172 - ebuild_auxdb_list[i] = ''
173 - return ebuild_auxdb_list
174 + # Get the auxdbkeys infos for the ebuild
175 + try:
176 + ebuild_auxdb_list = self._myportdb.aux_get(cpv, portage.auxdbkeys, myrepo=repo)
177 + except:
178 + ebuild_auxdb_list = []
179 + else:
180 + for i in range(len(ebuild_auxdb_list)):
181 + if ebuild_auxdb_list[i] == '':
182 + ebuild_auxdb_list[i] = ''
183 + return ebuild_auxdb_list
184
185 def get_packageDict(self, pkgdir, cpv, repo, config_id):
186 - attDict = {}
187 - conn=CM.getConnection()
188 -
189 - #Get categories, package and version from cpv
190 - ebuild_version_tree = portage.versions.cpv_getversion(cpv)
191 - element = portage.versions.cpv_getkey(cpv).split('/')
192 - categories = element[0]
193 - package = element[1]
194 -
195 - # Make a checksum of the ebuild
196 - try:
197 - ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0]
198 - except:
199 - ebuild_version_checksum_tree = "0"
200 - log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
201 - add_gobs_logs(conn, log_msg, "info", config_profile)
202 - log_msg = "C %s:%s ... Fail." % (cpv, repo)
203 - add_gobs_logs(conn, log_msg, "info", config_profile)
204 - ebuild_version_text_tree = '0'
205 - else:
206 - ebuild_version_text_tree = get_ebuild_text(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")
207 -
208 - # run repoman on the ebuild
209 - #init_repoman = gobs_repoman(self._mysettings, self._myportdb)
210 - #repoman_error = init_repoman.check_repoman(pkgdir, cpv, config_id)
211 - #if repoman_error != []:
212 - # log_msg = "Repoman: %s have errors on repo %s" % (cpv, repo,)
213 - # add_gobs_logs(conn, log_msg, "info", config_profile)
214 - repoman_error = []
215 -
216 - # Get the ebuild metadata
217 - ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
218 - # if there some error to get the metadata we add rubish to the
219 - # ebuild_version_metadata_tree and set ebuild_version_checksum_tree to 0
220 - # so it can be updated next time we update the db
221 - if ebuild_version_metadata_tree == []:
222 - log_msg = " QA: %s have broken metadata on repo %s" % (cpv, repo)
223 - add_gobs_logs(conn, log_msg, "info", config_profile)
224 - ebuild_version_metadata_tree = ['','','','','','','','','','','','','','','','','','','','','','','','','']
225 - ebuild_version_checksum_tree = '0'
226 + attDict = {}
227 + conn=CM.getConnection()
228 +
229 + #Get categories, package and version from cpv
230 + ebuild_version_tree = portage.versions.cpv_getversion(cpv)
231 + element = portage.versions.cpv_getkey(cpv).split('/')
232 + categories = element[0]
233 + package = element[1]
234 +
235 + # Make a checksum of the ebuild
236 + try:
237 + ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0]
238 + except:
239 + ebuild_version_checksum_tree = "0"
240 + log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
241 + add_gobs_logs(conn, log_msg, "info", config_profile)
242 + log_msg = "C %s:%s ... Fail." % (cpv, repo)
243 + add_gobs_logs(conn, log_msg, "info", config_profile)
244 + ebuild_version_text_tree = '0'
245 + else:
246 + ebuild_version_text_tree = get_ebuild_text(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")
247 +
248 + # run repoman on the ebuild
249 + #init_repoman = gobs_repoman(self._mysettings, self._myportdb)
250 + #repoman_error = init_repoman.check_repoman(pkgdir, cpv, config_id)
251 + #if repoman_error != []:
252 + # log_msg = "Repoman: %s have errors on repo %s" % (cpv, repo,)
253 + # add_gobs_logs(conn, log_msg, "info", config_profile)
254 + repoman_error = []
255 +
256 + # Get the ebuild metadata
257 + ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
258 + # if there some error to get the metadata we add rubish to the
259 + # ebuild_version_metadata_tree and set ebuild_version_checksum_tree to 0
260 + # so it can be updated next time we update the db
261 + if ebuild_version_metadata_tree == []:
262 + log_msg = " QA: %s have broken metadata on repo %s" % (cpv, repo)
263 + add_gobs_logs(conn, log_msg, "info", config_profile)
264 + ebuild_version_metadata_tree = ['','','','','','','','','','','','','','','','','','','','','','','','','']
265 + ebuild_version_checksum_tree = '0'
266
267 # add the ebuild info to the dict packages
268 - attDict['repo'] = repo
269 - attDict['ebuild_version_tree'] = ebuild_version_tree
270 - attDict['ebuild_version_checksum_tree']= ebuild_version_checksum_tree
271 - attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
272 - #attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
273 - attDict['ebuild_version_revision_tree'] = ebuild_version_text_tree[1]
274 - attDict['ebuild_error'] = repoman_error
275 - CM.putConnection(conn)
276 - return attDict
277 + attDict['repo'] = repo
278 + attDict['ebuild_version_tree'] = ebuild_version_tree
279 + attDict['ebuild_version_checksum_tree']= ebuild_version_checksum_tree
280 + attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
281 + #attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
282 + attDict['ebuild_version_revision_tree'] = ebuild_version_text_tree[1]
283 + attDict['ebuild_error'] = repoman_error
284 + CM.putConnection(conn)
285 + return attDict
286
287 def add_new_ebuild_buildquery_db(self, ebuild_id_list, packageDict, config_cpv_listDict):
288 conn=CM.getConnection()
289 @@ -160,197 +159,193 @@ class gobs_package(object):
290 for k, v in packageDict.iteritems():
291 ebuild_id = ebuild_id_list[i]
292
293 - # Comper and add the cpv to buildqueue
294 - if build_cpv == k:
295 - add_new_package_buildqueue(conn, ebuild_id, config_id, use_flagsDict, messages)
296 + # Comper and add the cpv to buildqueue
297 + if build_cpv == k:
298 + add_new_package_buildqueue(conn, ebuild_id, config_id, use_flagsDict, messages)
299
300 - # B = Build cpv use-flags config
301 - config_setup = get_config_db(conn, config_id)
302 + # B = Build cpv use-flags config
303 + config_setup = get_config_db(conn, config_id)
304
305 - # FIXME log_msg need a fix to log the use flags corect.
306 - log_msg = "B %s:%s USE: %s %s" % \
307 - (k, v['repo'], use_enable, config_setup,)
308 - add_gobs_logs(conn, log_msg, "info", config_profile)
309 + # FIXME log_msg need a fix to log the use flags corect.
310 + log_msg = "B %s:%s USE: %s %s" % \
311 + (k, v['repo'], use_enable, config_setup,)
312 + add_gobs_logs(conn, log_msg, "info", config_profile)
313 i = i +1
314 CM.putConnection(conn)
315
316 def get_package_metadataDict(self, pkgdir, package):
317 - # Make package_metadataDict
318 - attDict = {}
319 - package_metadataDict = {}
320 - changelog_checksum_tree = portage.checksum.sha256hash(pkgdir + "/ChangeLog")
321 - changelog_text_tree = get_file_text(pkgdir + "/ChangeLog")
322 - metadata_xml_checksum_tree = portage.checksum.sha256hash(pkgdir + "/metadata.xml")
323 - metadata_xml_text_tree = get_file_text(pkgdir + "/metadata.xml")
324 - attDict['changelog_checksum'] = changelog_checksum_tree[0]
325 - attDict['changelog_text'] = changelog_text_tree
326 - attDict['metadata_xml_checksum'] = metadata_xml_checksum_tree[0]
327 - attDict['metadata_xml_text'] = metadata_xml_text_tree
328 - package_metadataDict[package] = attDict
329 - return package_metadataDict
330 + # Make package_metadataDict
331 + attDict = {}
332 + package_metadataDict = {}
333 + changelog_checksum_tree = portage.checksum.sha256hash(pkgdir + "/ChangeLog")
334 + changelog_text_tree = get_file_text(pkgdir + "/ChangeLog")
335 + metadata_xml_checksum_tree = portage.checksum.sha256hash(pkgdir + "/metadata.xml")
336 + metadata_xml_text_tree = get_file_text(pkgdir + "/metadata.xml")
337 + attDict['changelog_checksum'] = changelog_checksum_tree[0]
338 + attDict['changelog_text'] = changelog_text_tree
339 + attDict['metadata_xml_checksum'] = metadata_xml_checksum_tree[0]
340 + attDict['metadata_xml_text'] = metadata_xml_text_tree
341 + package_metadataDict[package] = attDict
342 + return package_metadataDict
343
344 def add_new_package_db(self, categories, package, repo):
345 conn=CM.getConnection()
346 # Add new categories package ebuild to tables package and ebuilds
347 - # C = Checking
348 - # N = New Package
349 - log_msg = "C %s/%s:%s" % (categories, package, repo)
350 - add_gobs_logs(conn, log_msg, "info", config_profile)
351 - log_msg = "N %s/%s:%s" % (categories, package, repo)
352 - add_gobs_logs(conn, log_msg, "info", config_profile)
353 - pkgdir = self._myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package # Get RepoDIR + cp
354 + # C = Checking
355 + # N = New Package
356 + log_msg = "C %s/%s:%s" % (categories, package, repo)
357 + add_gobs_logs(conn, log_msg, "info", config_profile)
358 + log_msg = "N %s/%s:%s" % (categories, package, repo)
359 + add_gobs_logs(conn, log_msg, "info", config_profile)
360 + pkgdir = self._myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package # Get RepoDIR + cp
361
362 # Get the cp manifest file checksum.
363 - try:
364 - manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
365 - except:
366 - manifest_checksum_tree = "0"
367 - log_msg = "QA: Can't checksum the Manifest file. %s/%s:%s" % (categories, package, repo,)
368 - add_gobs_logs(conn, log_msg, "info", config_profile)
369 - log_msg = "C %s/%s:%s ... Fail." % (categories, package, repo)
370 - add_gobs_logs(conn, log_msg, "info", config_profile)
371 - CM.putConnection(conn)
372 - return
373 - package_id = add_new_manifest_sql(conn, categories, package, repo, manifest_checksum_tree)
374 -
375 - # Get the ebuild list for cp
376 - mytree = []
377 - mytree.append(self._myportdb.getRepositoryPath(repo))
378 - ebuild_list_tree = self._myportdb.cp_list((categories + "/" + package), use_cache=1, mytree=mytree)
379 - if ebuild_list_tree == []:
380 - log_msg = "QA: Can't get the ebuilds list. %s/%s:%s" % (categories, package, repo,)
381 - add_gobs_logs(conn, log_msg, "info", config_profile)
382 - log_msg = "C %s/%s:%s ... Fail." % (categories, package, repo)
383 - add_gobs_logs(conn, log_msg, "info", config_profile)
384 - CM.putConnection(conn)
385 - return
386 + try:
387 + manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
388 + except:
389 + manifest_checksum_tree = "0"
390 + log_msg = "QA: Can't checksum the Manifest file. %s/%s:%s" % (categories, package, repo,)
391 + add_gobs_logs(conn, log_msg, "info", config_profile)
392 + log_msg = "C %s/%s:%s ... Fail." % (categories, package, repo)
393 + add_gobs_logs(conn, log_msg, "info", config_profile)
394 + CM.putConnection(conn)
395 + return
396 + package_id = add_new_manifest_sql(conn, categories, package, repo, manifest_checksum_tree)
397 +
398 + # Get the ebuild list for cp
399 + mytree = []
400 + mytree.append(self._myportdb.getRepositoryPath(repo))
401 + ebuild_list_tree = self._myportdb.cp_list((categories + "/" + package), use_cache=1, mytree=mytree)
402 + if ebuild_list_tree == []:
403 + log_msg = "QA: Can't get the ebuilds list. %s/%s:%s" % (categories, package, repo,)
404 + add_gobs_logs(conn, log_msg, "info", config_profile)
405 + log_msg = "C %s/%s:%s ... Fail." % (categories, package, repo)
406 + add_gobs_logs(conn, log_msg, "info", config_profile)
407 + CM.putConnection(conn)
408 + return
409
410 # set config to default config
411 - default_config = get_default_config(conn)
412 + default_config = get_default_config(conn)
413
414 - # Make the needed packageDict with ebuild infos so we can add it later to the db.
415 - packageDict ={}
416 - ebuild_id_list = []
417 - for cpv in sorted(ebuild_list_tree):
418 - packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo, default_config)
419 + # Make the needed packageDict with ebuild infos so we can add it later to the db.
420 + packageDict ={}
421 + ebuild_id_list = []
422 + for cpv in sorted(ebuild_list_tree):
423 + packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo, default_config)
424
425 - # Add new ebuilds to the db
426 - ebuild_id_list = add_new_ebuild_sql(conn, package_id, packageDict)
427 + # Add new ebuilds to the db
428 + ebuild_id_list = add_new_ebuild_sql(conn, package_id, packageDict)
429
430 - # Get the best cpv for the configs and add it to config_cpv_listDict
431 - configs_id_list = get_config_id_list(conn)
432 - config_cpv_listDict = self.config_match_ebuild(categories + "/" + package, configs_id_list)
433 + # Get the best cpv for the configs and add it to config_cpv_listDict
434 + configs_id_list = get_config_id_list(conn)
435 + config_cpv_listDict = self.config_match_ebuild(categories + "/" + package, configs_id_list)
436
437 - # Add the ebuild to the buildquery table if needed
438 - self.add_new_ebuild_buildquery_db(ebuild_id_list, packageDict, config_cpv_listDict)
439 + # Add the ebuild to the buildquery table if needed
440 + self.add_new_ebuild_buildquery_db(ebuild_id_list, packageDict, config_cpv_listDict)
441
442 - log_msg = "C %s/%s:%s ... Done." % (categories, package, repo)
443 - add_gobs_logs(conn, log_msg, "info", config_profile)
444 - print(categories, package, repo)
445 - CM.putConnection(conn)
446 + log_msg = "C %s/%s:%s ... Done." % (categories, package, repo)
447 + add_gobs_logs(conn, log_msg, "info", config_profile)
448 + print(categories, package, repo)
449 + CM.putConnection(conn)
450
451 def update_package_db(self, package_id):
452 conn=CM.getConnection()
453 # Update the categories and package with new info
454 - # C = Checking
455 - cp, repo = get_cp_repo_from_package_id(conn, package_id)
456 - element = cp.split('/')
457 - package = element[1]
458 - log_msg = "C %s:%s" % (cp, repo)
459 - add_gobs_logs(conn, log_msg, "info", config_profile)
460 - pkgdir = self._myportdb.getRepositoryPath(repo) + "/" + cp # Get RepoDIR + cp
461 -
462 - # Get the cp mainfest file checksum
463 - try:
464 - manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
465 - except:
466 - manifest_checksum_tree = "0"
467 - log_msg = "QA: Can't checksum the Manifest file. %s:%s" % (cp, repo,)
468 - add_gobs_logs(conn, log_msg, "info", config_profile)
469 - log_msg = "C %s:%s ... Fail." % (cp, repo)
470 - add_gobs_logs(conn, log_msg, "info", config_profile)
471 - CM.putConnection(conn)
472 - return
473 -
474 - # Get the checksum from the db in package table
475 - manifest_checksum_db = get_manifest_db(conn, package_id)
476 -
477 - # if we have the same checksum return else update the package
478 - if manifest_checksum_tree != manifest_checksum_db:
479 + # C = Checking
480 + cp, repo = get_cp_repo_from_package_id(conn, package_id)
481 + element = cp.split('/')
482 + package = element[1]
483 + log_msg = "C %s:%s" % (cp, repo)
484 + add_gobs_logs(conn, log_msg, "info", config_profile)
485 + pkgdir = self._myportdb.getRepositoryPath(repo) + "/" + cp # Get RepoDIR + cp
486 +
487 + # Get the cp mainfest file checksum
488 + try:
489 + manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
490 + except:
491 + manifest_checksum_tree = "0"
492 + log_msg = "QA: Can't checksum the Manifest file. %s:%s" % (cp, repo,)
493 + add_gobs_logs(conn, log_msg, "info", config_profile)
494 + log_msg = "C %s:%s ... Fail." % (cp, repo)
495 + add_gobs_logs(conn, log_msg, "info", config_profile)
496 + CM.putConnection(conn)
497 + return
498 +
499 + # if we NOT have the same checksum in the db update the package
500 + if manifest_checksum_tree != get_manifest_db(conn, package_id):
501
502 # U = Update
503 - log_msg = "U %s:%s" % (cp, repo)
504 - add_gobs_logs(conn, log_msg, "info", config_profile)
505 -
506 - # Get the ebuild list for cp
507 - mytree = []
508 - mytree.append(self._myportdb.getRepositoryPath(repo))
509 - ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
510 - if ebuild_list_tree == []:
511 - log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
512 - add_gobs_logs(conn, log_msg, "info", config_profile)
513 - log_msg = "C %s:%s ... Fail." % (cp, repo)
514 - add_gobs_logs(conn, log_msg, "info", config_profile)
515 - CM.putConnection(conn)
516 - return
517 - packageDict ={}
518 - for cpv in sorted(ebuild_list_tree):
519 - old_ebuild_list = []
520 -
521 - # split out ebuild version
522 - ebuild_version_tree = portage.versions.cpv_getversion(cpv)
523 -
524 - # Get the checksum of the ebuild in tree and db
525 - # Make a checksum of the ebuild
526 - try:
527 - ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0]
528 - except:
529 - ebuild_version_checksum_tree = '0'
530 - manifest_checksum_tree = '0'
531 - log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
532 - add_gobs_logs(conn, log_msg, "info", config_profile)
533 - log_msg = "C %s:%s ... Fail." % (cpv, repo)
534 - add_gobs_logs(conn, log_msg, "info", config_profile)
535 - ebuild_version_manifest_checksum_db = get_ebuild_checksum(conn, package_id, ebuild_version_tree)
536 -
537 -
538 - # Check if the checksum have change
539 - if ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
540 -
541 - # set config to default config
542 - default_config = get_default_config(conn)
543 -
544 - # Get packageDict for ebuild
545 - packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo, default_config)
546 - if ebuild_version_manifest_checksum_db is None:
547 - # N = New ebuild
548 - log_msg = "N %s:%s" % (cpv, repo,)
549 - add_gobs_logs(conn, log_msg, "info", config_profile)
550 - else:
551 - # U = Updated ebuild
552 - log_msg = "U %s:%s" % (cpv, repo,)
553 - add_gobs_logs(conn, log_msg, "info", config_profile)
554 -
555 - # Fix so we can use add_new_ebuild_sql() to update the ebuilds
556 - old_ebuild_list.append(ebuild_version_tree)
557 - add_old_ebuild(conn, package_id, old_ebuild_list)
558 - update_active_ebuild_to_fales(conn, package_id, ebuild_version_tree)
559 + log_msg = "U %s:%s" % (cp, repo)
560 + add_gobs_logs(conn, log_msg, "info", config_profile)
561 +
562 + # Get the ebuild list for cp
563 + mytree = []
564 + mytree.append(self._myportdb.getRepositoryPath(repo))
565 + ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
566 + if ebuild_list_tree == []:
567 + log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
568 + add_gobs_logs(conn, log_msg, "info", config_profile)
569 + log_msg = "C %s:%s ... Fail." % (cp, repo)
570 + add_gobs_logs(conn, log_msg, "info", config_profile)
571 + CM.putConnection(conn)
572 + return
573 + packageDict ={}
574 + for cpv in sorted(ebuild_list_tree):
575 + old_ebuild_list = []
576 +
577 + # split out ebuild version
578 + ebuild_version_tree = portage.versions.cpv_getversion(cpv)
579 +
580 + # Get the checksum of the ebuild in tree and db
581 + # Make a checksum of the ebuild
582 + try:
583 + ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0]
584 + except:
585 + ebuild_version_checksum_tree = '0'
586 + manifest_checksum_tree = '0'
587 + log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
588 + add_gobs_logs(conn, log_msg, "info", config_profile)
589 + log_msg = "C %s:%s ... Fail." % (cpv, repo)
590 + add_gobs_logs(conn, log_msg, "info", config_profile)
591 + ebuild_version_manifest_checksum_db = get_ebuild_checksum(conn, package_id, ebuild_version_tree)
592 +
593 + # Check if the checksum have change
594 + if ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
595 +
596 + # set config to default config
597 + default_config = get_default_config(conn)
598 +
599 + # Get packageDict for ebuild
600 + packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo, default_config)
601 + if ebuild_version_manifest_checksum_db is None:
602 + # N = New ebuild
603 + log_msg = "N %s:%s" % (cpv, repo,)
604 + add_gobs_logs(conn, log_msg, "info", config_profile)
605 + else:
606 + # U = Updated ebuild
607 + log_msg = "U %s:%s" % (cpv, repo,)
608 + add_gobs_logs(conn, log_msg, "info", config_profile)
609 +
610 + # Fix so we can use add_new_ebuild_sql() to update the ebuilds
611 + old_ebuild_list.append(ebuild_version_tree)
612 + add_old_ebuild(conn, package_id, old_ebuild_list)
613 + update_active_ebuild_to_fales(conn, package_id, ebuild_version_tree
614 # Use packageDictand to update the db
615 - # Add new ebuilds to the db
616 - ebuild_id_list = add_new_ebuild_sql(conn, package_id, packageDict)
617 + # Add new ebuilds to the db
618 + ebuild_id_list = add_new_ebuild_sql(conn, package_id, packageDict)
619
620 - # update the cp manifest checksum
621 - update_manifest_sql(conn, package_id, manifest_checksum_tree)
622 + # update the cp manifest checksum
623 + update_manifest_sql(conn, package_id, manifest_checksum_tree)
624
625 - # Get the best cpv for the configs and add it to config_cpv_listDict
626 - configs_id_list = get_config_id_list(conn)
627 - config_cpv_listDict = self.config_match_ebuild(cp, configs_id_list)
628 + # Get the best cpv for the configs and add it to config_cpv_listDict
629 + configs_id_list = get_config_id_list(conn)
630 + config_cpv_listDict = self.config_match_ebuild(cp, configs_id_list)
631
632 - # Add the ebuild to the buildqueru table if needed
633 - self.add_new_ebuild_buildquery_db(ebuild_id_list, packageDict, config_cpv_listDict)
634 + # Add the ebuild to the buildqueru table if needed
635 + self.add_new_ebuild_buildquery_db(ebuild_id_list, packageDict, config_cpv_listDict)
636
637 - log_msg = "C %s:%s ... Done." % (cp, repo)
638 - add_gobs_logs(conn, log_msg, "info", config_profile)
639 + log_msg = "C %s:%s ... Done." % (cp, repo)
640 + add_gobs_logs(conn, log_msg, "info", config_profile)
641 CM.putConnection(conn)
642
643 def update_ebuild_db(self, build_dict):
644
645 diff --git a/gobs/pym/pgsql_querys.py b/gobs/pym/pgsql_querys.py
646 index 9184a20..48dd825 100644
647 --- a/gobs/pym/pgsql_querys.py
648 +++ b/gobs/pym/pgsql_querys.py
649 @@ -7,302 +7,304 @@ def add_gobs_logs(connection, log_msg, log_type, config):
650 sqlQ = 'INSERT INTO logs (config_id, type, msg) VALUES ( (SELECT config_id FROM configs WHERE config = %s), %s, %s )'
651 cursor.execute(sqlQ, (config, log_type, log_msg))
652 connection.commit()
653 -
654 - # Queryes to handel the jobs table
655 - def get_jobs_id(connection, config_profile):
656 - cursor = connection.cursor()
657 - sqlQ = "SELECT job_id FROM jobs WHERE status = 'Waiting' AND config_id = (SELECT config_id FROM configs WHERE config = %s)"
658 - cursor.execute(sqlQ, (config_profile,))
659 - entries = cursor.fetchall()
660 +
661 +# Queryes to handel the jobs table
662 +def get_jobs_id(connection, config_profile):
663 + cursor = connection.cursor()
664 + sqlQ = "SELECT job_id FROM jobs WHERE status = 'Waiting' AND config_id = (SELECT config_id FROM configs WHERE config = %s)"
665 + cursor.execute(sqlQ, (config_profile,))
666 + entries = cursor.fetchall()
667 + if entries is None:
668 + return None
669 + jobs_id = []
670 + for job_id in entries:
671 + jobs_id.append(job_id[0])
672 + return sorted(jobs_id)
673 +
674 +def get_job(connection, job_id):
675 + cursor = connection.cursor()
676 + sqlQ ='SELECT job FROM jobs WHERE job_id = %s'
677 + cursor.execute(sqlQ, (job_id,))
678 + job = cursor.fetchone()
679 + return job[0]
680 +
681 +def update_job_list(connection, status, job_id):
682 + cursor = connection.cursor()
683 + sqlQ = 'UPDATE jobs SET status = %s WHERE job_id = %s'
684 + cursor.execute(sqlQ, (status, job_id,))
685 + connection.commit()
686 +
687 +# Queryes to handel the configs* tables
688 +def get_config_list_all(connection):
689 + cursor = connection.cursor()
690 + sqlQ = 'SELECT config FROM configs'
691 + cursor.execute(sqlQ)
692 + entries = cursor.fetchall()
693 + return entries
694 +
695 +def update_make_conf(connection, configsDict):
696 + cursor = connection.cursor()
697 + sqlQ1 = 'UPDATE configs_metadata SET checksum = %s, make_conf_text = %s, active = %s, config_error = %s WHERE config_id = (SELECT config_id FROM configs WHERE config = %s)'
698 + for k, v in configsDict.iteritems():
699 + params = [v['make_conf_checksum_tree'], v['make_conf_text'], v['active'], v['config_error'], k]
700 + cursor.execute(sqlQ1, params)
701 + connection.commit()
702 +
703 +def get_default_config(connection):
704 + cursor = connection.cursor()
705 + sqlQ = "SELECT config FROM configs WHERE default_config = 'True'"
706 + cursor.execute(sqlQ)
707 + entries = cursor.fetchone()
708 + return entries
709 +
710 +def update_repo_db(connection, repo_list):
711 + cursor = connection.cursor()
712 + sqlQ1 = 'SELECT repo_id FROM repos WHERE repo = %s'
713 + sqlQ2 = 'INSERT INTO repos (repo) VALUES ( %s )'
714 + for repo in repo_list:
715 + cursor.execute(sqlQ1, (repo,))
716 + entries = cursor.fetchone()
717 if entries is None:
718 - return None
719 - jobs_id = []
720 - for job_id in entries:
721 - jobs_id.append(job_id[0])
722 - return sorted(jobs_id)
723 -
724 - def get_job(connection, job_id):
725 - cursor = connection.cursor()
726 - sqlQ ='SELECT job FROM jobs WHERE job_id = %s'
727 - cursor.execute(sqlQ, (job_id,))
728 - job = cursor.fetchone()
729 - return job[0]
730 -
731 - def update_job_list(connection, status, job_id):
732 - cursor = connection.cursor()
733 - sqlQ = 'UPDATE jobs SET status = %s WHERE job_id = %s'
734 - cursor.execute(sqlQ, (status, job_id,))
735 - connection.commit()
736 -
737 - # Queryes to handel the configs* tables
738 - def get_config_list_all(connection):
739 - cursor = connection.cursor()
740 - sqlQ = 'SELECT config FROM configs'
741 - cursor.execute(sqlQ)
742 - entries = cursor.fetchall()
743 - return entries
744 - def update_make_conf(connection, configsDict):
745 - cursor = connection.cursor()
746 - sqlQ1 = 'UPDATE configs_metadata SET checksum = %s, make_conf_text = %s, active = %s, config_error = %s WHERE config_id = (SELECT config_id FROM configs WHERE config = %s)'
747 - for k, v in configsDict.iteritems():
748 - params = [v['make_conf_checksum_tree'], v['make_conf_text'], v['active'], v['config_error'], k]
749 - cursor.execute(sqlQ1, params)
750 - connection.commit()
751 -
752 - def get_default_config(connection):
753 - cursor = connection.cursor()
754 - sqlQ = "SELECT config FROM configs WHERE default_config = 'True'"
755 - cursor.execute(sqlQ)
756 - entries = cursor.fetchone()
757 - return entries
758 -
759 - def update_repo_db(connection, repo_list):
760 - cursor = connection.cursor()
761 - sqlQ1 = 'SELECT repo_id FROM repos WHERE repo = %s'
762 - sqlQ2 = 'INSERT INTO repos (repo) VALUES ( %s )'
763 - for repo in repo_list:
764 - cursor.execute(sqlQ1, (repo,))
765 - entries = cursor.fetchone()
766 - if entries is None:
767 - cursor.execute(sqlQ2, (repo,))
768 - connection.commit()
769 - return
770 + cursor.execute(sqlQ2, (repo,))
771 + connection.commit()
772 + return
773 +
774 def get_package_id(connection, categories, package, repo):
775 - cursor = connection.cursor()
776 - sqlQ ='SELECT package_id FROM packages WHERE category = %s AND package = %s AND repo_id = (SELECT repo_id FROM repos WHERE repo = %s)'
777 - params = categories, package, repo
778 - cursor.execute(sqlQ, params)
779 - entries = cursor.fetchone()
780 - if entries is None:
781 - return None
782 - return entries[0]
783 + cursor = connection.cursor()
784 + sqlQ ='SELECT package_id FROM packages WHERE category = %s AND package = %s AND repo_id = (SELECT repo_id FROM repos WHERE repo = %s)'
785 + params = categories, package, repo
786 + cursor.execute(sqlQ, params)
787 + entries = cursor.fetchone()
788 + if entries is None:
789 + return None
790 + return entries[0]
791
792 # Add new info to the packages table
793 -
794 def get_repo_id(connection, repo):
795 - cursor = connection.cursor()
796 - sqlQ ='SELECT repo_id FROM repos WHERE repo = %s'
797 - cursor.execute(sqlQ, (repo,))
798 - entries = cursor.fetchone()
799 - if entries is None:
800 - return None
801 - return entries[0]
802 + cursor = connection.cursor()
803 + sqlQ ='SELECT repo_id FROM repos WHERE repo = %s'
804 + cursor.execute(sqlQ, (repo,))
805 + entries = cursor.fetchone()
806 + if entries is None:
807 + return None
808 + return entries[0]
809
810 def add_new_manifest_sql(connection, categories, package, repo, manifest_checksum_tree):
811 - cursor = connection.cursor()
812 - sqlQ = "INSERT INTO packages (category, package, repo_id, checksum, active) VALUES (%s, %s, %s, %s, 'True') RETURNING package_id"
813 - repo_id = get_repo_id(connection, repo)
814 - cursor.execute(sqlQ, (categories, package, repo_id, manifest_checksum_tree,))
815 - package_id = cursor.fetchone()[0]
816 - connection.commit()
817 - return package_id
818 + cursor = connection.cursor()
819 + sqlQ = "INSERT INTO packages (category, package, repo_id, checksum, active) VALUES (%s, %s, %s, %s, 'True') RETURNING package_id"
820 + repo_id = get_repo_id(connection, repo)
821 + cursor.execute(sqlQ, (categories, package, repo_id, manifest_checksum_tree,))
822 + package_id = cursor.fetchone()[0]
823 + connection.commit()
824 + return package_id
825
826 def get_restriction_id(connection, restriction):
827 - cursor = connection.cursor()
828 - sqlQ ='SELECT restriction_id FROM restrictions WHERE restriction = %s'
829 - cursor.execute(sqlQ, (restriction,))
830 - entries = cursor.fetchone()
831 - if entries is None:
832 - return None
833 - return entries[0]
834 + cursor = connection.cursor()
835 + sqlQ ='SELECT restriction_id FROM restrictions WHERE restriction = %s'
836 + cursor.execute(sqlQ, (restriction,))
837 + entries = cursor.fetchone()
838 + if entries is None:
839 + return None
840 + return entries[0]
841
842 def get_use_id(connection, use_flag):
843 - cursor = connection.cursor()
844 - sqlQ ='SELECT use_id FROM uses WHERE flag = %s'
845 - cursor.execute(sqlQ, (use_flag,))
846 - entries = cursor.fetchone()
847 - if entries is None:
848 - return None
849 - return entries[0]
850 + cursor = connection.cursor()
851 + sqlQ ='SELECT use_id FROM uses WHERE flag = %s'
852 + cursor.execute(sqlQ, (use_flag,))
853 + entries = cursor.fetchone()
854 + if entries is None:
855 + return None
856 + return entries[0]
857
858 def get_keyword_id(connection, keyword):
859 - cursor = connection.cursor()
860 - sqlQ ='SELECT keyword_id FROM keywords WHERE keyword = %s'
861 - cursor.execute(sqlQ, (keyword,))
862 - entries = cursor.fetchone()
863 - if entries is None:
864 - return None
865 - return entries[0]
866 + cursor = connection.cursor()
867 + sqlQ ='SELECT keyword_id FROM keywords WHERE keyword = %s'
868 + cursor.execute(sqlQ, (keyword,))
869 + entries = cursor.fetchone()
870 + if entries is None:
871 + return None
872 + return entries[0]
873
874 def add_new_ebuild_metadata_sql(connection, ebuild_id, keywords, restrictions, iuse_list):
875 - cursor = connection.cursor()
876 - sqlQ1 = 'INSERT INTO keywords (keyword) VALUES ( %s ) RETURNING keyword_id'
877 - sqlQ3 = 'INSERT INTO restrictions (restriction) VALUES ( %s ) RETURNING restriction_id'
878 - sqlQ4 = 'INSERT INTO ebuilds_restrictions (ebuild_id, restriction_id) VALUES ( %s, %s )'
879 - sqlQ5 = 'INSERT INTO uses (flag) VALUES ( %s ) RETURNING use_id'
880 - sqlQ6 = 'INSERT INTO ebuilds_iuse (ebuild_id, use_id, status) VALUES ( %s, %s, %s)'
881 - sqlQ7 = 'INSERT INTO ebuilds_keywords (ebuild_id, keyword_id, status) VALUES ( %s, %s, %s)'
882 - # FIXME restriction need some filter as iuse and keyword have.
883 - for restriction in restrictions:
884 - restriction_id = get_restriction_id(connection, restriction)
885 - if restriction_id is None:
886 - cursor.execute(sqlQ3, (restriction,))
887 - restriction_id = cursor.fetchone()[0]
888 - cursor.execute(sqlQ4, (ebuild_id, restriction_id,))
889 - for iuse in iuse_list:
890 - set_iuse = 'disable'
891 - if iuse[0] in ["+"]:
892 - iuse = iuse[1:]
893 - set_iuse = 'enable'
894 - elif iuse[0] in ["-"]:
895 - iuse = iuse[1:]
896 - use_id = get_use_id(connection, iuse)
897 - if use_id is None:
898 - cursor.execute(sqlQ5, (iuse,))
899 - use_id = cursor.fetchone()[0]
900 - for keyword in keywords:
901 - set_keyword = 'stable'
902 - if keyword[0] in ["~"]:
903 - keyword = keyword[1:]
904 - set_keyword = 'unstable'
905 - elif keyword[0] in ["-"]:
906 - keyword = keyword[1:]
907 - set_keyword = 'testing'
908 - keyword_id = get_keyword_id(connection, keyword)
909 - if keyword_id is None:
910 - cursor.execute(sqlQ1, (keyword,))
911 - keyword_id = cursor.fetchone()[0]
912 - cursor.execute(sqlQ7, (ebuild_id, keyword_id, set_keyword,))
913 - connection.commit()cursor.execute(sqlQ6, (ebuild_id, use_id, set_iuse,))
914 + cursor = connection.cursor()
915 + sqlQ1 = 'INSERT INTO keywords (keyword) VALUES ( %s ) RETURNING keyword_id'
916 + sqlQ3 = 'INSERT INTO restrictions (restriction) VALUES ( %s ) RETURNING restriction_id'
917 + sqlQ4 = 'INSERT INTO ebuilds_restrictions (ebuild_id, restriction_id) VALUES ( %s, %s )'
918 + sqlQ5 = 'INSERT INTO uses (flag) VALUES ( %s ) RETURNING use_id'
919 + sqlQ6 = 'INSERT INTO ebuilds_iuse (ebuild_id, use_id, status) VALUES ( %s, %s, %s)'
920 + sqlQ7 = 'INSERT INTO ebuilds_keywords (ebuild_id, keyword_id, status) VALUES ( %s, %s, %s)'
921 + # FIXME restriction need some filter as iuse and keyword have.
922 + for restriction in restrictions:
923 + restriction_id = get_restriction_id(connection, restriction)
924 + if restriction_id is None:
925 + cursor.execute(sqlQ3, (restriction,))
926 + restriction_id = cursor.fetchone()[0]
927 + cursor.execute(sqlQ4, (ebuild_id, restriction_id,))
928 + for iuse in iuse_list:
929 + set_iuse = 'disable'
930 + if iuse[0] in ["+"]:
931 + iuse = iuse[1:]
932 + set_iuse = 'enable'
933 + elif iuse[0] in ["-"]:
934 + iuse = iuse[1:]
935 + use_id = get_use_id(connection, iuse)
936 + if use_id is None:
937 + cursor.execute(sqlQ5, (iuse,))
938 + use_id = cursor.fetchone()[0]
939 + cursor.execute(sqlQ6, (ebuild_id, use_id, set_iuse,))
940 + for keyword in keywords:
941 + set_keyword = 'stable'
942 + if keyword[0] in ["~"]:
943 + keyword = keyword[1:]
944 + set_keyword = 'unstable'
945 + elif keyword[0] in ["-"]:
946 + keyword = keyword[1:]
947 + set_keyword = 'testing'
948 + keyword_id = get_keyword_id(connection, keyword)
949 + if keyword_id is None:
950 + cursor.execute(sqlQ1, (keyword,))
951 + keyword_id = cursor.fetchone()[0]
952 + cursor.execute(sqlQ7, (ebuild_id, keyword_id, set_keyword,))
953 + connection.commit()
954
955 def add_new_ebuild_sql(connection, package_id, ebuildDict):
956 - cursor = connection.cursor()
957 - sqlQ1 = 'SELECT repo_id FROM packages WHERE package_id = %s'
958 - sqlQ2 = "INSERT INTO ebuilds (package_id, version, checksum, active) VALUES (%s, %s, %s, 'True') RETURNING ebuild_id"
959 - sqlQ4 = "INSERT INTO ebuilds_metadata (ebuild_id, revision) VALUES (%s, %s)"
960 - ebuild_id_list = []
961 - cursor.execute(sqlQ1, (package_id,))
962 - repo_id = cursor.fetchone()[0]
963 - for k, v in ebuildDict.iteritems():
964 - cursor.execute(sqlQ2, (package_id, v['ebuild_version_tree'], v['ebuild_version_checksum_tree'],))
965 - ebuild_id = cursor.fetchone()[0]
966 - cursor.execute(sqlQ4, (ebuild_id, v['ebuild_version_revision_tree'],))
967 - ebuild_id_list.append(ebuild_id)
968 - restrictions = []
969 - keywords = []
970 - iuse = []
971 - for i in v['ebuild_version_metadata_tree'][4].split():
972 - restrictions.append(i)
973 - for i in v['ebuild_version_metadata_tree'][8].split():
974 - keywords.append(i)
975 - for i in v['ebuild_version_metadata_tree'][10].split():
976 - iuse.append(i)
977 - add_new_ebuild_metadata_sql(connection, ebuild_id, keywords, restrictions, iuse)
978 - connection.commit()
979 - return ebuild_id_list
980 + cursor = connection.cursor()
981 + sqlQ1 = 'SELECT repo_id FROM packages WHERE package_id = %s'
982 + sqlQ2 = "INSERT INTO ebuilds (package_id, version, checksum, active) VALUES (%s, %s, %s, 'True') RETURNING ebuild_id"
983 + sqlQ4 = "INSERT INTO ebuilds_metadata (ebuild_id, revision) VALUES (%s, %s)"
984 + ebuild_id_list = []
985 + cursor.execute(sqlQ1, (package_id,))
986 + repo_id = cursor.fetchone()[0]
987 + for k, v in ebuildDict.iteritems():
988 + cursor.execute(sqlQ2, (package_id, v['ebuild_version_tree'], v['ebuild_version_checksum_tree'],))
989 + ebuild_id = cursor.fetchone()[0]
990 + cursor.execute(sqlQ4, (ebuild_id, v['ebuild_version_revision_tree'],))
991 + ebuild_id_list.append(ebuild_id)
992 + restrictions = []
993 + keywords = []
994 + iuse = []
995 + for i in v['ebuild_version_metadata_tree'][4].split():
996 + restrictions.append(i)
997 + for i in v['ebuild_version_metadata_tree'][8].split():
998 + keywords.append(i)
999 + for i in v['ebuild_version_metadata_tree'][10].split():
1000 + iuse.append(i)
1001 + add_new_ebuild_metadata_sql(connection, ebuild_id, keywords, restrictions, iuse)
1002 + connection.commit()
1003 + return ebuild_id_list
1004
1005 def get_config_id_list(connection):
1006 - cursor = connection.cursor()
1007 - sqlQ = "SELECT configs.config_id FROM configs, configs_metadata WHERE configs.default_config = 'False' AND configs_metadata.active = 'True' AND configs.config_id = configs_metadata.config_id"
1008 - cursor.execute(sqlQ)
1009 - entries = cursor.fetchall()
1010 - if entries == ():
1011 - return None
1012 - else:
1013 - config_id_list = []
1014 - for config_id in entries:
1015 - config_id_list.append(config_id[0])
1016 - return config_id_list
1017 + cursor = connection.cursor()
1018 + sqlQ = "SELECT configs.config_id FROM configs, configs_metadata WHERE configs.default_config = 'False' AND configs_metadata.active = 'True' AND configs.config_id = configs_metadata.config_id"
1019 + cursor.execute(sqlQ)
1020 + entries = cursor.fetchall()
1021 + if entries == ():
1022 + return None
1023 + else:
1024 + config_id_list = []
1025 + for config_id in entries:
1026 + config_id_list.append(config_id[0])
1027 + return config_id_list
1028
1029 def get_config_db(connection, config_id):
1030 - cursor = connection.cursor()
1031 - sqlQ = 'SELECT config FROM configs WHERE config_id = %s'
1032 - cursor.execute(sqlQ,(config_id,))
1033 - entries = cursor.fetchone()
1034 - if entries is None:
1035 - return None
1036 - return entries[0]
1037 + cursor = connection.cursor()
1038 + sqlQ = 'SELECT config FROM configs WHERE config_id = %s'
1039 + cursor.execute(sqlQ,(config_id,))
1040 + entries = cursor.fetchone()
1041 + if entries is None:
1042 + return None
1043 + return entries[0]
1044
1045 def add_new_package_buildqueue(connection, ebuild_id, config_id, use_flagsDict, messages):
1046 - cursor = connection.cursor()
1047 - sqlQ1 = 'INSERT INTO build_jobs (ebuild_id, config_id) VALUES (%s, %s) RETURNING build_job_id'
1048 - sqlQ3 = 'INSERT INTO build_jobs_use (build_job_id, use_id, status) VALUES (%s, (SELECT use_id FROM uses WHERE flag = %s), %s)'
1049 - cursor.execute(sqlQ1, (ebuild_id, config_id,))
1050 - build_job_id = cursor.fetchone()[0]
1051 - for k, v in use_flagsDict.iteritems():
1052 - cursor.execute(sqlQ3, (build_job_id, k, v,))
1053 - connection.commit()
1054 + cursor = connection.cursor()
1055 + sqlQ1 = 'INSERT INTO build_jobs (ebuild_id, config_id) VALUES (%s, %s) RETURNING build_job_id'
1056 + sqlQ3 = 'INSERT INTO build_jobs_use (build_job_id, use_id, status) VALUES (%s, (SELECT use_id FROM uses WHERE flag = %s), %s)'
1057 + cursor.execute(sqlQ1, (ebuild_id, config_id,))
1058 + build_job_id = cursor.fetchone()[0]
1059 + for k, v in use_flagsDict.iteritems():
1060 + cursor.execute(sqlQ3, (build_job_id, k, v,))
1061 + connection.commit()
1062
1063 def get_manifest_db(connection, package_id):
1064 - cursor = connection.cursor()
1065 - sqlQ = 'SELECT checksum FROM packages WHERE package_id = %s'
1066 - cursor.execute(sqlQ, (package_id,))
1067 - entries = cursor.fetchone()
1068 - if entries is None:
1069 - return None
1070 - # If entries is not None we need [0]
1071 - return entries[0]
1072 + cursor = connection.cursor()
1073 + sqlQ = 'SELECT checksum FROM packages WHERE package_id = %s'
1074 + cursor.execute(sqlQ, (package_id,))
1075 + entries = cursor.fetchone()
1076 + if entries is None:
1077 + return None
1078 + # If entries is not None we need [0]
1079 + return entries[0]
1080
1081 def get_cp_from_package_id(connection, package_id):
1082 - cursor = connection.cursor()
1083 - sqlQ = "SELECT ARRAY_TO_STRING(ARRAY[category, package] , '/') AS cp FROM packages WHERE package_id = %s"
1084 - cursor.execute(sqlQ, (package_id,))
1085 - return cursor.fetchone()
1086 + cursor = connection.cursor()
1087 + sqlQ = "SELECT ARRAY_TO_STRING(ARRAY[category, package] , '/') AS cp FROM packages WHERE package_id = %s"
1088 + cursor.execute(sqlQ, (package_id,))
1089 + return cursor.fetchone()
1090
1091 def get_cp_repo_from_package_id(connection, package_id):
1092 - cursor =connection.cursor()
1093 - sqlQ = 'SELECT repos.repo FROM repos, packages WHERE repos.repo_id = packages.repo_id AND packages.package_id = %s'
1094 - cp = get_cp_from_package_id(connection, package_id)
1095 - cursor.execute(sqlQ, (package_id,))
1096 - repo = cursor.fetchone()
1097 - return cp[0], repo[0]
1098 + cursor =connection.cursor()
1099 + sqlQ = 'SELECT repos.repo FROM repos, packages WHERE repos.repo_id = packages.repo_id AND packages.package_id = %s'
1100 + cp = get_cp_from_package_id(connection, package_id)
1101 + cursor.execute(sqlQ, (package_id,))
1102 + repo = cursor.fetchone()
1103 + return cp[0], repo[0]
1104
1105 def get_ebuild_checksum(connection, package_id, ebuild_version_tree):
1106 - cursor = connection.cursor()
1107 - sqlQ = "SELECT checksum FROM ebuilds WHERE package_id = %s AND version = %s AND active = 'True'"
1108 - cursor.execute(sqlQ, (package_id, ebuild_version_tree))
1109 - entries = cursor.fetchone()
1110 - if entries is None:
1111 - return None
1112 - # If entries is not None we need [0]
1113 - return entries[0]
1114 + cursor = connection.cursor()
1115 + sqlQ = "SELECT checksum FROM ebuilds WHERE package_id = %s AND version = %s AND active = 'True'"
1116 + cursor.execute(sqlQ, (package_id, ebuild_version_tree))
1117 + entries = cursor.fetchone()
1118 + if entries is None:
1119 + return None
1120 + # If entries is not None we need [0]
1121 + return entries[0]
1122
1123 def add_old_ebuild(connection, package_id, old_ebuild_list):
1124 - cursor = connection.cursor()
1125 - sqlQ1 = "UPDATE ebuilds SET active = 'False' WHERE package_id = %s AND version = %s"
1126 - sqlQ2 = "SELECT ebuild_id FROM ebuilds WHERE package_id = %s AND version = %s AND active = 'True'"
1127 - sqlQ3 = "SELECT build_job_id FROM build_jobs WHERE ebuild_id = %s"
1128 - sqlQ4 = 'DELETE FROM build_jobs_use WHERE build_job_id = %s'
1129 - sqlQ5 = 'DELETE FROM build_jobs WHERE build_job_id = %s'
1130 - for old_ebuild in old_ebuild_list:
1131 - cursor.execute(sqlQ2, (package_id, old_ebuild[0]))
1132 - ebuild_id_list = cursor.fetchall()
1133 - if ebuild_id_list is not None:
1134 - for ebuild_id in ebuild_id_list:
1135 - cursor.execute(sqlQ3, (ebuild_id))
1136 - build_job_id_list = cursor.fetchall()
1137 - if build_job_id_list is not None:
1138 - for build_job_id in build_job_id_list:
1139 - cursor.execute(sqlQ4, (build_job_id))
1140 - cursor.execute(sqlQ5, (build_job_id))
1141 - cursor.execute(sqlQ1, (package_id, old_ebuild[0]))
1142 - connection.commit()
1143 + cursor = connection.cursor()
1144 + sqlQ1 = "UPDATE ebuilds SET active = 'False' WHERE package_id = %s AND version = %s"
1145 + sqlQ2 = "SELECT ebuild_id FROM ebuilds WHERE package_id = %s AND version = %s AND active = 'True'"
1146 + sqlQ3 = "SELECT build_job_id FROM build_jobs WHERE ebuild_id = %s"
1147 + sqlQ4 = 'DELETE FROM build_jobs_use WHERE build_job_id = %s'
1148 + sqlQ5 = 'DELETE FROM build_jobs WHERE build_job_id = %s'
1149 + for old_ebuild in old_ebuild_list:
1150 + cursor.execute(sqlQ2, (package_id, old_ebuild[0]))
1151 + ebuild_id_list = cursor.fetchall()
1152 + if ebuild_id_list is not None:
1153 + for ebuild_id in ebuild_id_list:
1154 + cursor.execute(sqlQ3, (ebuild_id))
1155 + build_job_id_list = cursor.fetchall()
1156 + if build_job_id_list is not None:
1157 + for build_job_id in build_job_id_list:
1158 + cursor.execute(sqlQ4, (build_job_id))
1159 + cursor.execute(sqlQ5, (build_job_id))
1160 + cursor.execute(sqlQ1, (package_id, old_ebuild[0]))
1161 + connection.commit()
1162
1163 def update_active_ebuild_to_fales(connection, package_id, ebuild_version_tree):
1164 - cursor = connection.cursor()
1165 - sqlQ ="UPDATE ebuilds SET active = 'False' WHERE package_id = %s AND version = %s AND active = 'True'"
1166 - cursor.execute(sqlQ, (package_id, ebuild_version_tree))
1167 - connection.commit()
1168 + cursor = connection.cursor()
1169 + sqlQ ="UPDATE ebuilds SET active = 'False' WHERE package_id = %s AND version = %s AND active = 'True'"
1170 + cursor.execute(sqlQ, (package_id, ebuild_version_tree))
1171 + connection.commit()
1172
1173 def update_manifest_sql(connection, package_id, manifest_checksum_tree):
1174 - cursor = connection.cursor()
1175 - sqlQ = 'UPDATE packages SET checksum = %s WHERE package_id = %s'
1176 - cursor.execute(sqlQ, (manifest_checksum_tree, package_id,))
1177 - connection.commit()
1178 + cursor = connection.cursor()
1179 + sqlQ = 'UPDATE packages SET checksum = %s WHERE package_id = %s'
1180 + cursor.execute(sqlQ, (manifest_checksum_tree, package_id,))
1181 + connection.commit()
1182
1183 def get_build_jobs_id_list_config(connection, config_id):
1184 - cursor = connection.cursor()
1185 - sqlQ = 'SELECT build_job_id FROM build_jobs WHERE config_id = %s'
1186 - cursor.execute(sqlQ, (config_id,))
1187 - entries = cursor.fetchall()
1188 - return entries
1189 + cursor = connection.cursor()
1190 + sqlQ = 'SELECT build_job_id FROM build_jobs WHERE config_id = %s'
1191 + cursor.execute(sqlQ, (config_id,))
1192 + entries = cursor.fetchall()
1193 + return entries
1194
1195 def del_old_build_jobs(connection, queue_id):
1196 - cursor = connection.cursor()
1197 - sqlQ1 = 'DELETE FROM build_jobs_use WHERE build_job_id = %s'
1198 - sqlQ2 = 'DELETE FROM build_jobs_retest WHERE build_job_id = %s'
1199 - sqlQ3 = 'DELETE FROM build_jobs WHERE build_job_id = %s'
1200 - cursor.execute(sqlQ1, (build_job_id,))
1201 - cursor.execute(sqlQ2, (build_job_id,))
1202 - cursor.execute(sqlQ3, (build_job_id,))
1203 - connection.commit()
1204 + cursor = connection.cursor()
1205 + sqlQ1 = 'DELETE FROM build_jobs_use WHERE build_job_id = %s'
1206 + sqlQ2 = 'DELETE FROM build_jobs_retest WHERE build_job_id = %s'
1207 + sqlQ3 = 'DELETE FROM build_jobs WHERE build_job_id = %s'
1208 + cursor.execute(sqlQ1, (build_job_id,))
1209 + cursor.execute(sqlQ2, (build_job_id,))
1210 + cursor.execute(sqlQ3, (build_job_id,))
1211 + connection.commit()
1212
1213 diff --git a/gobs/pym/sync.py b/gobs/pym/sync.py
1214 index 87eb79e..bfff592 100644
1215 --- a/gobs/pym/sync.py
1216 +++ b/gobs/pym/sync.py
1217 @@ -13,7 +13,7 @@ from gobs.ConnectionManager import connectionManager
1218 CM=connectionManager(gobs_settings_dict)
1219 #selectively import the pgsql/mysql querys
1220 if CM.getName()=='pgsql':
1221 - from gobs.pgsql import *
1222 + from gobs.pgsql_querys import *
1223
1224 config_profile = gobs_settings_dict['gobs_config']
1225
1226
1227 diff --git a/gobs/pym/updatedb.py b/gobs/pym/updatedb.py
1228 index fb185d2..144b6bd 100755
1229 --- a/gobs/pym/updatedb.py
1230 +++ b/gobs/pym/updatedb.py
1231 @@ -76,35 +76,34 @@ def update_cpv_db():
1232 myportdb = portage.portdbapi(mysettings=mysettings)
1233 init_package = gobs_package(mysettings, myportdb)
1234 repo_list = ()
1235 - repos_trees_list = []
1236 -
1237 - # Use all cores when multiprocessing
1238 - pool_cores= multiprocessing.cpu_count()
1239 - pool = multiprocessing.Pool(processes=pool_cores)
1240 -
1241 - # Will run some update checks and update package if needed
1242 - # Get categories/package list from portage and repos
1243 -
1244 - # Get the repos and update the repos db
1245 - repo_list = myportdb.getRepositories()
1246 - update_repo_db(conn, repo_list)
1247 - CM.putConnection(conn)
1248 -
1249 + repos_trees_list = []
1250 +
1251 + # Use all cores when multiprocessing
1252 + pool_cores= multiprocessing.cpu_count()
1253 + pool = multiprocessing.Pool(processes=pool_cores)
1254 +
1255 + # Will run some update checks and update package if needed
1256 + # Get categories/package list from portage and repos
1257 + # Get the repos and update the repos db
1258 + repo_list = myportdb.getRepositories()
1259 + update_repo_db(conn, repo_list)
1260 + CM.putConnection(conn)
1261 +
1262 # Get the rootdirs for the repos
1263 - repo_trees_list = myportdb.porttrees
1264 - for repo_dir in repo_trees_list:
1265 - repo = myportdb.getRepositoryName(repo_dir)
1266 - repo_dir_list = []
1267 - repo_dir_list.append(repo_dir)
1268 -
1269 - # Get the package list from the repo
1270 - package_id_list_tree = []
1271 - package_list_tree = myportdb.cp_all(trees=repo_dir_list)
1272 -
1273 - # Run the update package for all package in the list and in a multiprocessing pool
1274 - for package_line in sorted(package_list_tree):
1275 - pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, init_package, package_line, repo,))
1276 - # update_cpv_db_pool(mysettings, myportdb, init_package, package_line, repo)
1277 + repo_trees_list = myportdb.porttrees
1278 + for repo_dir in repo_trees_list:
1279 + repo = myportdb.getRepositoryName(repo_dir)
1280 + repo_dir_list = []
1281 + repo_dir_list.append(repo_dir)
1282 +
1283 + # Get the package list from the repo
1284 + package_id_list_tree = []
1285 + package_list_tree = myportdb.cp_all(trees=repo_dir_list)
1286 +
1287 + # Run the update package for all package in the list and in a multiprocessing pool
1288 + for package_line in sorted(package_list_tree):
1289 + pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, init_package, package_line, repo,))
1290 + # update_cpv_db_pool(mysettings, myportdb, init_package, package_line, repo)
1291 pool.close()
1292 pool.join()
1293 conn=CM.getConnection()
1294 @@ -114,15 +113,15 @@ def update_cpv_db():
1295
1296 def update_db_main():
1297 # Main
1298 - conn=CM.getConnection()
1299 -
1300 - # Logging
1301 - log_msg = "Update db started."
1302 - add_gobs_logs(conn, log_msg, "info", config_profile)
1303 -
1304 - # Update the cpv db
1305 - update_cpv_db()
1306 - log_msg = "Update db ... Done."
1307 - add_gobs_logs(conn, log_msg, "info", config_profile)
1308 - CM.putConnection(conn)
1309 - return True
1310 + conn=CM.getConnection()
1311 +
1312 + # Logging
1313 + log_msg = "Update db started."
1314 + add_gobs_logs(conn, log_msg, "info", config_profile)
1315 +
1316 + # Update the cpv db
1317 + update_cpv_db()
1318 + log_msg = "Update db ... Done."
1319 + add_gobs_logs(conn, log_msg, "info", config_profile)
1320 + CM.putConnection(conn)
1321 + return True