[gentoo-commits] proj/grumpy:master commit in: backend/lib/

[Mart Raudsepp]

commit:     df4ddb601efbef157147fcfd6057afd01636acab
Author:     Mart Raudsepp <leio <AT> gentoo <DOT> org>
AuthorDate: Sun Dec  4 05:26:10 2016 +0000
Commit:     Mart Raudsepp <leio <AT> gentoo <DOT> org>
CommitDate: Sun Dec  4 05:26:10 2016 +0000
URL:        https://gitweb.gentoo.org/proj/grumpy.git/commit/?id=df4ddb60

sync: Initial projects syncing to DB without members

 backend/lib/sync.py | 35 ++++++++++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/backend/lib/sync.py b/backend/lib/sync.py
index fbc653a..6ed8e01 100644
--- a/backend/lib/sync.py
+++ b/backend/lib/sync.py
@@ -2,19 +2,19 @@ import xml.etree.ElementTree as ET
 from flask import json
 import requests
 from .. import app, db
-from .models import Category, Package, PackageVersion
+from .models import Category, Maintainer, Package, PackageVersion
 
 proj_url = "https://api.gentoo.org/metastructure/projects.xml"
 pkg_url_base = "https://packages.gentoo.org/"
 http_session = requests.session()
 
-def sync_projects():
+def get_project_data():
     data = http_session.get(proj_url)
     if not data:
         print("Failed retrieving projects.xml")
         return
     root = ET.fromstring(data.text)
-    projects = []
+    projects = {}
     # Parsing is based on http://www.gentoo.org/dtd/projects.dtd as of 2016-11-10
     if root.tag.lower() != 'projects':
         print("Downloaded projects.xml root tag isn't 'projects'")
@@ -53,12 +53,33 @@ def sync_projects():
             else:
                 print("Skipping unknown <project> subtag <%s>" % tag)
         if 'email' in proj:
-            projects.append(proj)
+            projects[proj['email']] = proj
         else:
             print("Skipping incomplete project data due to lack of required email identifier: %s" % (proj,))
-    from pprint import pprint
-    print("Found the following projects and data:")
-    pprint(projects)
+    return projects
+
+def sync_projects():
+    projects = get_project_data()
+    existing_maintainers = {}
+    # TODO: Use UPSERT instead (on_conflict_do_update) if we can rely on postgresql:9.5
+    for maintainer in Maintainer.query.all():
+        existing_maintainers[maintainer.email] = maintainer
+    for email, data in projects.items():
+        if email in existing_maintainers:
+            print ("Updating project %s" % email)
+            existing_maintainers[email].is_project = True
+            if 'description' in data:
+                existing_maintainers[email].description = data['description']
+            if 'name' in data:
+                existing_maintainers[email].name = data['name']
+            if 'url' in data:
+                existing_maintainers[email].url = data['url']
+        else:
+            print ("Adding project %s" % email)
+            new_maintainer = Maintainer(email=data['email'], is_project=True, description=data['description'], name=data['name'], url=data['url'])
+            db.session.add(new_maintainer)
+    db.session.commit()
+
 
 def sync_categories():
     url = pkg_url_base + "categories.json"