[gentoo-commits] proj/log-analysis:master commit in: /

[Antanas Ursulis]

commit:     e5bdc5bc099096186e00e61604484fc6a1ecdeb2
Author:     Antanas Uršulis <antanas.ursulis <AT> gmail <DOT> com>
AuthorDate: Mon Jul 29 18:55:51 2013 +0000
Commit:     Antanas Ursulis <antanas.ursulis <AT> gmail <DOT> com>
CommitDate: Mon Jul 29 18:55:51 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/log-analysis.git;a=commit;h=e5bdc5bc

Database (SQL) class for functionality common across all processors

Currently uses MySQLdb. Schema included.

---
 database.py          | 22 ++++++++++++++++++++++
 flask_app.py         | 18 ++++++++++++++----
 portage_processor.py |  9 +++++----
 schema.sql           | 12 ++++++++++++
 simple_client.py     |  1 +
 submission.proto     |  3 ++-
 6 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/database.py b/database.py
new file mode 100644
index 0000000..5202876
--- /dev/null
+++ b/database.py
@@ -0,0 +1,22 @@
+from contextlib import closing
+import MySQLdb
+
+class DatabaseConnection:
+	def __init__(self, conn):
+		self.conn = conn
+
+	def insert_file(self, path, group_id):
+		with closing(self.conn.cursor()) as c:
+			c.execute("insert into `files` (`path`, `group_id`) values (%s, %s)", (path, group_id))
+			self.conn.commit()
+			return c.lastrowid
+
+	def insert_group(self, name, provider, date):
+		with closing(self.conn.cursor()) as c:
+			c.execute("insert into `groups` (`name`, `provider`, `date`) values (%s, %s, %s)", (name, provider, date))
+			self.conn.commit()
+			return c.lastrowid
+
+def get_connection(user, passwd, db):
+	conn = MySQLdb.connect(user=user, passwd=passwd, db=db)
+	return DatabaseConnection(conn)

diff --git a/flask_app.py b/flask_app.py
index 832702c..5356bc4 100644
--- a/flask_app.py
+++ b/flask_app.py
@@ -4,14 +4,24 @@ When run as a script, the Flask development server is started.
 """
 
 import os, socket
-import submission_pb2, storage
-from flask import Flask, request
+import submission_pb2, storage, database
+from flask import Flask, request, g
 
 from portage_processor import PortageProcessor
 
 app = Flask(__name__)
 store = storage.FilesystemStorage('logs/')
-processors = {'portage' : PortageProcessor(None, store)} # TODO: initialise from config file
+processors = {'portage' : PortageProcessor(store)} # TODO: initialise from config file
+
+@×××.before_request
+def before_request():
+	g.db = database.get_connection('gsoc', 'gsocpasswd', 'loganalysis')
+
+@×××.teardown_request
+def teardown_request(exception):
+	db = getattr(g, 'db', None)
+	if db is not None:
+		db.conn.close()
 
 @app.route('/')
 def index():
@@ -23,7 +33,7 @@ def submit():
 	submission.ParseFromString(request.data)
 	source = socket.getfqdn(request.remote_addr) # TODO: is this ok?
 
-	processors[submission.provider].process(submission, source)
+	processors[submission.provider].process(submission, source, g.db)
 	return ''
 
 if __name__ == '__main__':

diff --git a/portage_processor.py b/portage_processor.py
index 2403cdf..66fb970 100644
--- a/portage_processor.py
+++ b/portage_processor.py
@@ -1,4 +1,4 @@
-import re, StringIO
+import os, re, StringIO, time
 
 class PortageProcessor:
 	_r = {
@@ -10,11 +10,11 @@ class PortageProcessor:
 		'escapes' : re.compile(r"\x1b\[[^\x40-\x7e]*[\x40-\x7e]")
 	}
 
-	def __init__(self, db, storage):
-		self.db = db
+	def __init__(self, storage):
 		self.storage = storage
 
-	def process(self, request, source):
+	def process(self, request, source, db):
+		group_id = db.insert_group(request.group_name, 'portage', int(time.time()))
 		for f in request.files:
 			matches = 0
 			pkg_failed = False
@@ -72,3 +72,4 @@ class PortageProcessor:
 ''')
 
 			self.storage.save_file(source, f.filename, output.getvalue())
+			file_id = db.insert_file(os.path.join(source, f.filename), group_id)

diff --git a/schema.sql b/schema.sql
new file mode 100644
index 0000000..564385e
--- /dev/null
+++ b/schema.sql
@@ -0,0 +1,12 @@
+create table if not exists `files` (
+	`id` int primary key auto_increment,
+	`path` text not null,
+	`group_id` int not null
+);
+
+create table if not exists `groups` (
+	`id` int primary key auto_increment,
+	`name` text not null,
+	`provider` varchar(16) not null,
+	`date` int not null
+);

diff --git a/simple_client.py b/simple_client.py
index ab4bccf..c89b6c1 100644
--- a/simple_client.py
+++ b/simple_client.py
@@ -7,6 +7,7 @@ import submission_pb2, sys, urllib2, os
 def send_submission(filenames):
 	submission = submission_pb2.Submission()
 	submission.provider = "portage"
+	submission.group_name = "Manual submission"
 
 	for f in filenames:
 		new_file = submission.files.add()

diff --git a/submission.proto b/submission.proto
index 42cf97c..3cbf474 100644
--- a/submission.proto
+++ b/submission.proto
@@ -5,5 +5,6 @@ message Submission {
   }
 
   required string provider = 1;
-  repeated File files = 2;
+  optional string group_name = 2;
+  repeated File files = 3;
 }