[gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/libs/, ...

[Slava Bacherikov]

commit:     8d1b2afae52cb0a59442f7026c44db08854c2e24
Author:     Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Thu May 31 13:33:58 2012 +0000
Commit:     Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Thu May 31 13:33:58 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=8d1b2afa

Add scaning utility

---
 .../packages/management/commands/scanpackages.py   |  110 +++++++++++++++++++-
 gpackages/apps/packages/models.py                  |   28 +++--
 gpackages/libs/porttree.py                         |   28 +++++-
 3 files changed, 151 insertions(+), 15 deletions(-)

diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 4bad968..ea6a3ef 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -1,9 +1,115 @@
 from django.core.management.base import BaseCommand, CommandError
+from packages import models
+import porttree
 
+import datetime
+import logging
+#l = logging.getLogger('django.db.backends')
+#l.setLevel(logging.DEBUG)
+#l.addHandler(logging.FileHandler('database.log'))
 
 class Command(BaseCommand):
     args = ''
     help = 'Will scan package tree and update info about it in database'
-
+    porttree = porttree.PortTree()
     def handle(self, *args, **options):
-        print("Not implement yet")
+        licenses_cache = {}
+        def get_licenses_objects(ebuild):
+            licenses = ebuild.licenses
+            license_objects = []
+            licenses_set = frozenset(licenses)
+            cached_licenses = set(licenses_cache.keys())
+            # Getting from cache
+            geted_licenses = set()
+            for license in (cached_licenses & licenses_set):
+                geted_licenses.add(license)
+                license_objects.append(licenses_cache[license])
+            # Getting from database
+            request_licenses = list(licenses_set - geted_licenses)
+            if not request_licenses:
+                return license_objects
+            request = models.LicensModel.objects.filter(name__in = request_licenses)
+            # Add to list with licenses
+            license_objects.extend(request)
+            # Update cache
+            for license in request:
+                licenses_cache[license.name] = license
+                geted_licenses.add(license.name)
+            # Create not existend licenses fast using bulk_create, works only in django 1.4 or gt
+            need_create = list(licenses_set - geted_licenses)
+            if not need_create:
+                return license_objects
+            creating_list = []
+            for license in need_create:
+                creating_list.append(models.LicensModel(name = license))
+            
+            models.LicensModel.objects.bulk_create(creating_list)
+            # Retriwing created licenses from database
+            request_created = models.LicensModel.objects.filter(name__in = need_create)
+            license_objects.extend(request_created)
+            # Update cache
+            for license in request_created:
+                licenses_cache[license.name] = license
+                geted_licenses.add(license.name)
+            return license_objects
+
+        uses_cache = {}
+        def get_uses_objects(ebuild):
+            uses = [ use.name for use in ebuild.iter_uses() ]
+            uses_objects = []
+            uses_set = frozenset(uses)
+            cached_uses = set(uses_cache.keys())
+            # Getting from cache
+            geted_uses = set()
+            for use in (cached_uses & uses_set):
+                geted_uses.add(use)
+                uses_objects.append(uses_cache[use])
+            # Getting from database
+            request_use = list(uses_set - geted_uses)
+            if not request_use:
+                return uses_objects
+            request = models.UseFlagModel.objects.filter(name__in = request_use)
+            # Add to list with licenses
+            uses_objects.extend(request)
+            # Update cache
+            for use in request:
+                uses_cache[use.name] = use
+                geted_uses.add(use.name)
+            # Create not existend licenses fast using bulk_create, works only in django 1.4 or gt
+            need_create = list(uses_set - geted_uses)
+            if not need_create:
+                return license_objects
+            creating_list = []
+            for use in need_create:
+                creating_list.append(models.UseFlagModel(name = use))
+            
+            models.UseFlagModel.objects.bulk_create(creating_list)
+            # Retriwing created licenses from database
+            request_created = models.UseFlagModel.objects.filter(name__in = need_create)
+            uses_objects.extend(request_created)
+            # Update cache 
+            for use in request_created:
+                uses_cache[use.name] = use
+                geted_uses.add(use.name)
+            return uses_objects
+
+
+            
+        st = datetime.datetime.now()
+        for category in self.porttree.iter_categories():
+            category_object, categor_created = models.CategoryModel.objects.get_or_create(category = category)
+            for package in category.iter_packages():
+                print package
+                package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object)
+                for ebuild in package.iter_ebuilds():
+                    ebuild_object = models.EbuildModel()
+                    ebuild_object.init_by_ebuild(ebuild)
+                    ebuild_object.package = package_object
+                    # To Add some related objects it should have pk
+                    ebuild_object.save(force_insert=True)
+                    # Add licenses
+                    ebuild_object.licenses.add(*get_licenses_objects(ebuild))
+                    ebuild_object.use_flags.add(*get_uses_objects(ebuild))
+
+
+        print (datetime.datetime.now() - st).total_seconds()

diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 0ef5e8c..765150a 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -49,7 +49,7 @@ class PackageModel(models.Model):
 
         if isinstance(package_object, Package):
             super(PackageModel, self).__init__()
-            self.init_by_package(package_object)
+            self.init_by_package(package_object, category = kwargs.get('category'))
         else:
             super(PackageModel, self).__init__(*args, **kwargs)
             
@@ -67,10 +67,13 @@ class PackageModel(models.Model):
     def __unicode__(self):
         return '%s/%s' % (self.category, self.name)
     
-    def init_by_package(self, package):
+    def init_by_package(self, package, category = None):
         self.name = package.name
         self.update_info(package)
-        self.category, created = CategoryModel.objects.get_or_create(category = package.category)
+        if category is None:
+            self.category, created = CategoryModel.objects.get_or_create(category = package.category)
+        elif isinstance(category, CategoryModel):
+            self.category = category
 
     def update_info(self, package):
         self.changelog_hash = package.changelog_sha1
@@ -80,14 +83,14 @@ class PackageModel(models.Model):
         unique_together = ('name', 'category')
 
 class UseFlagModel(models.Model):
-    name = models.CharField(unique = True, max_length = 32)
+    name = models.CharField(unique = True, max_length = 60)
     description = models.TextField(blank = True)
     
     def __unicode__(self):
         return self.name
 
 class LicensModel(models.Model):
-    name = models.CharField(unique = True, max_length = 40)
+    name = models.CharField(unique = True, max_length = 60)
     #description = TextField()
     
     def __unicode__(self):
@@ -107,13 +110,12 @@ class EbuildModel(models.Model):
     is_deleted = models.BooleanField(default = False)
     is_masked = models.BooleanField(default = False)
 
-    homepage = models.URLField(blank = True, null = True)
+    homepage = models.URLField(blank = True, null = True, max_length=255)
     description = models.TextField(blank = True, null = True)
 
     objects = managers.EbuildManager()
 
     def __init__(self, *args, **kwargs ):
-        # TODO: Bug fix
         ebuild = None
         if 'ebuild' in kwargs:
             ebuild = kwargs['ebuild']
@@ -123,8 +125,8 @@ class EbuildModel(models.Model):
             self.init_with_keywords(ebuild)
     
     def __unicode__(self):
-        return '%s-%s' % (self.package, self.version)
-
+        return '%s-%s%s' % (self.package, self.version, ('-'+ self.revision if self.revision else '')) 
+    
     def init_by_ebuild(self, ebuild):
         self.is_masked = ebuild.is_masked
         self.version = ebuild.version
@@ -133,7 +135,13 @@ class EbuildModel(models.Model):
         self.ebuild_hash = ebuild.sha1
         self.homepage = ebuild.homepage
         self.description = ebuild.description
-        self.package = PackageModel.objects.get_or_create(package = ebuild.package)[0]
+
+    def init_related(self, ebuild, package = None):
+        self.init_by_ebuild(ebuild)
+        if package is None:
+            self.package = PackageModel.objects.get_or_create(package = ebuild.package)[0]
+        elif isinstance(package, PackageModel):
+            self.package = package
         self.save()
         l = []
         for license in ebuild.licenses:

diff --git a/gpackages/libs/porttree.py b/gpackages/libs/porttree.py
index a54ef12..160aa6b 100644
--- a/gpackages/libs/porttree.py
+++ b/gpackages/libs/porttree.py
@@ -53,6 +53,16 @@ class Use(ToStrMixin):
     def __unicode__(self):
         return self.name
 
+    def __eq__(self, other):
+        self.name == other.name
+
+    def __ne__(self, other):
+        self.name != other.name
+
+    def __hash__(self):
+        return hash(self.name)
+        
+
 
 class Keyword(ToStrMixin):
     status_repr = ['','~','-']
@@ -180,9 +190,12 @@ class Ebuild(ToStrMixin):
             l.append(keyword)
         return l
 
+    def get_uses_names(self):
+        return self.package_object.environment("IUSE").split()
+    
+
     def iter_uses(self):
-        uses = self.package_object.environment("IUSE").split()
-        for use in uses:
+        for use in self.get_uses_names():
             yield Use(use)
 
     def get_uses(self):
@@ -211,11 +224,20 @@ class Ebuild(ToStrMixin):
     def ebuild_path(self):
         return self.package_object.ebuild_path()
 
-    homepage = property(_ebuild_environment('HOMEPAGE'))
+    homepage_val = property(_ebuild_environment('HOMEPAGE'))
     license = property(_ebuild_environment('LICENSE'))
     description = property(_ebuild_environment('DESCRIPTION'))
 
     @property
+    def homepages(self):
+        return self.homepage_val.split()
+
+    @property
+    def homepage(self):
+        return self.homepages[0] if len(self.homepages)>=1 else ''
+
+
+    @property
     def licenses(self):
         return filter(_license_filter, self.license.split())