| 1 |
commit: 7ba5f957c082545a9eb097839fdc7eba826a7d80 |
| 2 |
Author: Antanas Uršulis <antanas.ursulis <AT> gmail <DOT> com> |
| 3 |
AuthorDate: Mon Jul 29 22:38:23 2013 +0000 |
| 4 |
Commit: Antanas Ursulis <antanas.ursulis <AT> gmail <DOT> com> |
| 5 |
CommitDate: Mon Jul 29 22:38:23 2013 +0000 |
| 6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/log-analysis.git;a=commit;h=7ba5f957 |
| 7 |
|
| 8 |
Implement additional metadata storage for logs received from Portage |
| 9 |
|
| 10 |
Flags like 'pkg_failed', 'test_failed', etc. are stored once per group. |
| 11 |
This is a discussion item, as it might make sense to store some data |
| 12 |
once per file instead. |
| 13 |
|
| 14 |
--- |
| 15 |
database.py | 2 +- |
| 16 |
portage_database.py | 12 ++++++++++++ |
| 17 |
portage_processor.py | 18 +++++++++++------- |
| 18 |
schema_portage.sql | 10 ++++++++++ |
| 19 |
4 files changed, 34 insertions(+), 8 deletions(-) |
| 20 |
|
| 21 |
diff --git a/database.py b/database.py |
| 22 |
index 79ae693..9927627 100644 |
| 23 |
--- a/database.py |
| 24 |
+++ b/database.py |
| 25 |
@@ -1,7 +1,7 @@ |
| 26 |
from contextlib import closing |
| 27 |
import MySQLdb |
| 28 |
|
| 29 |
-class DatabaseConnection: |
| 30 |
+class DatabaseConnection(object): |
| 31 |
def __init__(self, conn): |
| 32 |
self.conn = conn |
| 33 |
|
| 34 |
|
| 35 |
diff --git a/portage_database.py b/portage_database.py |
| 36 |
new file mode 100644 |
| 37 |
index 0000000..ca7831c |
| 38 |
--- /dev/null |
| 39 |
+++ b/portage_database.py |
| 40 |
@@ -0,0 +1,12 @@ |
| 41 |
+from contextlib import closing |
| 42 |
+from database import DatabaseConnection |
| 43 |
+ |
| 44 |
+class PortageDatabaseConnection(DatabaseConnection): |
| 45 |
+ def __init__(self, db): |
| 46 |
+ super(PortageDatabaseConnection, self).__init__(db.conn) |
| 47 |
+ |
| 48 |
+ # TODO: consider passing these arguments around in a dictionary or kwargs |
| 49 |
+ def insert_group_extra(self, group_id, pkg_name, matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc): |
| 50 |
+ with closing(self.conn.cursor()) as c: |
| 51 |
+ c.execute("insert into `groups_portage` values (%s, %s, %s, %s, %s, %s, %s, %s)", (group_id, pkg_name, matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc)) |
| 52 |
+ self.conn.commit() |
| 53 |
|
| 54 |
diff --git a/portage_processor.py b/portage_processor.py |
| 55 |
index 32ca9c4..252209e 100644 |
| 56 |
--- a/portage_processor.py |
| 57 |
+++ b/portage_processor.py |
| 58 |
@@ -1,4 +1,5 @@ |
| 59 |
import os, re, StringIO, time |
| 60 |
+from portage_database import PortageDatabaseConnection |
| 61 |
|
| 62 |
class PortageProcessor: |
| 63 |
_r = { |
| 64 |
@@ -14,16 +15,17 @@ class PortageProcessor: |
| 65 |
self.storage = storage |
| 66 |
|
| 67 |
def process(self, request, source, db): |
| 68 |
+ db = PortageDatabaseConnection(db) |
| 69 |
group_id = db.insert_group(source, request.group_name, 'portage', int(time.time())) |
| 70 |
|
| 71 |
- for f in request.files: |
| 72 |
- matches = 0 |
| 73 |
- pkg_failed = False |
| 74 |
- test_failed = False |
| 75 |
- collision = False |
| 76 |
- bug_assignee = 'bug-wranglers@g.o' |
| 77 |
- bug_cc = '' |
| 78 |
+ matches = 0 |
| 79 |
+ pkg_failed = False |
| 80 |
+ test_failed = False |
| 81 |
+ collision = False |
| 82 |
+ bug_assignee = 'bug-wranglers@g.o' |
| 83 |
+ bug_cc = '' |
| 84 |
|
| 85 |
+ for f in request.files: |
| 86 |
# TODO: look at proper HTML generation methods: |
| 87 |
# (*) either XHTML via xml.etree |
| 88 |
# (*) or Jinja2 (is it possible to parse and generate in one pass?) |
| 89 |
@@ -74,3 +76,5 @@ class PortageProcessor: |
| 90 |
|
| 91 |
self.storage.save_file(source, f.filename, output.getvalue()) |
| 92 |
file_id = db.insert_file(os.path.join(source, f.filename), group_id) |
| 93 |
+ |
| 94 |
+ db.insert_group_extra(group_id, 'TODO', matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc) |
| 95 |
|
| 96 |
diff --git a/schema_portage.sql b/schema_portage.sql |
| 97 |
new file mode 100644 |
| 98 |
index 0000000..4b9c80d |
| 99 |
--- /dev/null |
| 100 |
+++ b/schema_portage.sql |
| 101 |
@@ -0,0 +1,10 @@ |
| 102 |
+create table if not exists `groups_portage` ( |
| 103 |
+ `id` int primary key, |
| 104 |
+ `pkg_name` varchar(255) not null, |
| 105 |
+ `matches` int not null, |
| 106 |
+ `pkg_failed` bool not null, |
| 107 |
+ `test_failed` bool not null, |
| 108 |
+ `collision` bool not null, |
| 109 |
+ `bug_assignee` text not null, |
| 110 |
+ `bug_cc` text not null |
| 111 |
+); |
Archives