[gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/libs/, ...

[Slava Bacherikov]

commit:     57baa89e8bd8da906c8a001c92c7354635715b93
Author:     Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Wed Jun  6 20:39:37 2012 +0000
Commit:     Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Wed Jun  6 20:39:37 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=57baa89e

Add support multiple repo

---
 .../packages/management/commands/scanpackages.py   |    3 +-
 gpackages/apps/packages/models.py                  |   14 +++--
 gpackages/apps/packages/scan.py                    |   35 +++++++----
 gpackages/libs/generic.py                          |   28 +++++++-
 gpackages/libs/porttree.py                         |   69 ++++++++++++++++---
 5 files changed, 117 insertions(+), 32 deletions(-)

diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 27cc782..35a117b 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -12,5 +12,6 @@ class Command(BaseCommand):
     help = 'Will scan package tree and update info about it in database'
     def handle(self, *args, **options):
         st = datetime.datetime.now()
-        scan.scanpackages()
+        #scan.scanpackages()
+        scan.scan_all_repos()
         print (datetime.datetime.now() - st).total_seconds()

diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 90665bd..80e4103 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -23,10 +23,13 @@ class ArchesModel(models.Model):
         return self.name
 
 class RepositoryModel(models.Model):
-    name = models.CharField(max_length = 60)
+    name = models.CharField(unique = True, max_length = 60)
     description = models.TextField(blank = True, null = True)
     # And other fields
 
+    def __unicode__(self):
+        return self.name
+
 class CategoryModel(models.Model):
     def __init__(self, *args, **kwargs):
         category_object = None
@@ -125,9 +128,10 @@ class PackageModel(AbstractDateTimeModel):
         
         if 'package' in kwargs:
             package_object = kwargs['package']
+            del kwargs['package']
 
         if isinstance(package_object, Package):
-            super(PackageModel, self).__init__()
+            super(PackageModel, self).__init__(*args, **kwargs)
             self.init_by_package(package_object, category = kwargs.get('category'))
         else:
             super(PackageModel, self).__init__(*args, **kwargs)
@@ -136,7 +140,7 @@ class PackageModel(AbstractDateTimeModel):
 
     name = models.CharField(max_length = 254)
     category = models.ForeignKey(CategoryModel)
-    changelog = models.TextField(blank = True)
+    changelog = models.TextField(blank = True, null = True)
     changelog_hash = models.CharField(max_length = 128)
     manifest_hash = models.CharField(max_length = 128)
     metadata_hash = models.CharField(max_length = 128)
@@ -148,6 +152,7 @@ class PackageModel(AbstractDateTimeModel):
     maintainers = models.ManyToManyField(MaintainerModel, blank = True)
 
     description = models.TextField(blank = True, null = True)
+    repository = models.ForeignKey(RepositoryModel)
     # Different versions can have different licenses, or homepages.
     
     objects = managers.PackageManager()
@@ -188,7 +193,7 @@ class PackageModel(AbstractDateTimeModel):
         self.description = package.description
 
     class Meta:
-        unique_together = ('name', 'category')
+        unique_together = ('name', 'category', 'repository')
 
 class UseFlagModel(models.Model):
     name = models.CharField(unique = True, max_length = 60)
@@ -221,7 +226,6 @@ class LicensModel(models.Model):
 
 class EbuildModel(AbstractDateTimeModel):
     package = models.ForeignKey(PackageModel)
-    #repository = models.ForeignKey(RepositoryModel)
     version = models.CharField(max_length = 26)
     revision = models.CharField(max_length = 12)
     use_flags = models.ManyToManyField(UseFlagModel)

diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index f739c69..add62f1 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -1,16 +1,19 @@
 from packages import models
+from django.db import IntegrityError
 from collections import defaultdict
+from generic import StrThatIgnoreCase
 import porttree
 import herds
 import use_info
 
-porttree = porttree.PortTree()
+portage = porttree.Portage()
 
 def _get_from_cache(cache, what):
     save_to = []
     geted_items = set()
     for item in what:
-        if item in cache:
+        if item.lower() in cache:
+            item = StrThatIgnoreCase(item)
             geted_items.add(item)
             save_to.append(cache[item])
     return save_to, geted_items
@@ -26,8 +29,9 @@ def _update_cache_by_queryset(cache, queryset, field_name):
     if queryset is None:
         return None
     for item in queryset:
-        cache[getattr(item, field_name)] = item
-        geted_items.add(getattr(item, field_name))
+        val = StrThatIgnoreCase(getattr(item, field_name))
+        cache[val] = item
+        geted_items.add(val)
     return geted_items
 
 def _get_from_database_and_update_cache(Model, field_name, request_items, cache):
@@ -159,6 +163,7 @@ def update_globals_uses_descriptions():
     
 
 def scan_uses_description():
+    # need changes for support many repos !!!
     uses_local = use_info.get_local_uses_info()
     existend_use_objects = models.UseFlagModel.objects.filter(name__in = uses_local.keys())
     existend_use_local_descr = models.UseFlagDescriptionModel.objects.all()
@@ -196,18 +201,21 @@ def scan_uses_description():
                     use_desc_obj.save(force_update = True)
         models.UseFlagDescriptionModel.objects.bulk_create(to_create)
             
-def scanpackages(delete = True, force_update = False):
-    licenses_cache = {}
+licenses_cache = {}
+uses_cache = {}
+arches_cache = {}
+homepages_cache = {}    
+herds_cache = {}
+maintainers_cache = {}
+def scanpackages(porttree, porttree_obj, delete = True, force_update = False):
     def get_licenses_objects(ebuild):
         licenses = ebuild.licenses
         return _get_items(licenses, models.LicensModel, 'name', licenses_cache)
 
-    uses_cache = {}
     def get_uses_objects(ebuild):
         uses = [ use.name for use in ebuild.iter_uses() ]
         return _get_items(uses, models.UseFlagModel, 'name', uses_cache)
 
-    arches_cache = {}
     def get_keywords_objects(ebuild, ebuild_object):
         keywords_list = []
         for keyword in ebuild.get_keywords():
@@ -226,13 +234,11 @@ def scanpackages(delete = True, force_update = False):
         models.Keyword.objects.bulk_create(keywords_list)
 
 
-    homepages_cache = {}    
     def get_homepages_objects(ebuild):
         homepages = ebuild.homepages
         return _get_items(homepages, models.HomepageModel, 'url', homepages_cache)
 
     
-    herds_cache, maintainers_cache = scan_herds()
     def get_maintainers_objects(package):
         maintainers = package.metadata.maintainers()
         objects = []
@@ -319,8 +325,8 @@ def scanpackages(delete = True, force_update = False):
         category_object, category_created = models.CategoryModel.objects.get_or_create(category = category)
         existend_categorys.append(category_object.pk)
         for package in category.iter_packages():
-            print package
-            package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object)
+            print('%s [%s]' % (package, porttree))
+            package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object, repository = porttree_obj)
             existend_packages.append(package_object.pk)
             if not package_created:
                 if package_object.check_or_need_update(package) or force_update:
@@ -341,3 +347,8 @@ def scanpackages(delete = True, force_update = False):
     #models.CategoryModel.objects.exclude(pk__in = existend_categorys).delete()
 
 
+def scan_all_repos():
+    herds_cache, maintainers_cache = scan_herds()
+    for repo in portage.iter_trees():
+        repo_obj, repo_created = models.RepositoryModel.objects.get_or_create(name = repo.name)
+        scanpackages(repo, repo_obj)

diff --git a/gpackages/libs/generic.py b/gpackages/libs/generic.py
index ed08d93..c3d8d2b 100644
--- a/gpackages/libs/generic.py
+++ b/gpackages/libs/generic.py
@@ -2,6 +2,21 @@ import os.path
 import hashlib
 from datetime import datetime
 
+class StrThatIgnoreCase(unicode):
+    
+    def __init__(self, value):
+        super(StrThatIgnoreCase, self).__init__(value)
+        self._forcmp = value.lower()
+
+    def __hash__(self):
+        return hash(self._forcmp)
+
+    def __eq__(self, other):
+        return self._forcmp == unicode(other).lower()
+
+    def __ne__(self, other):
+        return self._forcmp != unicode(other).lower()
+
 class ToStrMixin(object):
     """Abstract class for inheritence, allow add simple `__str__` and `__repr__`
     methods
@@ -13,12 +28,19 @@ class ToStrMixin(object):
         return '<%s %s>' % (type(self).__name__, self.__str__())
 
 
+def file_get_content(file_path):
+    if os.path.exists(file_path):
+        with open(file_path, 'r') as f:
+            content = f.read()
+        return content
+    else:
+        return None
+
 def file_sha1(file_path):
     sha1 = 'NULL'
     if os.path.exists(file_path):
-        f = open(file_path, 'r')
-        sha1 = hashlib.sha1(f.read()).hexdigest()
-        f.close()
+        with open(file_path, 'r') as f:
+            sha1 = hashlib.sha1(f.read()).hexdigest()
     return sha1
 
 def file_mtime(file_path):

diff --git a/gpackages/libs/porttree.py b/gpackages/libs/porttree.py
index 5a4da3a..5b1dba8 100644
--- a/gpackages/libs/porttree.py
+++ b/gpackages/libs/porttree.py
@@ -6,7 +6,8 @@ from portage.exception import PortageException, FileNotFound, InvalidAtom, \
 
 from gentoolkit.package import Package as PackageInfo
 from gentoolkit.metadata import MetaData
-from generic import ToStrMixin, file_sha1, file_mtime, cached_property
+from generic import ToStrMixin, file_sha1, file_mtime, cached_property, \
+                    file_get_content, StrThatIgnoreCase
 from use_info import get_uses_info, get_local_uses_info
 import os
 
@@ -41,6 +42,21 @@ def _get_info_by_func(func, path1, path2):
         except IOError:
             return None
 
+class FakeMetaData(ToStrMixin):
+
+    def herds(self):
+        return []
+
+    def maintainers(self):
+        return []
+
+    def descriptions(self):
+        return []
+    
+    def __unicode__(self):
+        return 'fake'
+
+
 class Use(ToStrMixin):
     "Represend Use flag as object"
     __slots__ = ('name',)
@@ -51,7 +67,7 @@ class Use(ToStrMixin):
         """
         if name.startswith('+') or name.startswith('-'):
             name = name[1:]
-        self.name = name
+        self.name = StrThatIgnoreCase(name)
 
     def __unicode__(self):
         return self.name
@@ -106,18 +122,28 @@ def _gen_all_use(func, iterator):
     return use_all_dict
 
 class Portage(object):
+
+    def __init__(self):
+        self.treemap = PORTDB.repositories.treemap
+        self.tree_order = PORTDB.repositories.prepos_order
+
+    def get_tree_by_name(self, tree_name):
+        if tree_name in self.treemap:
+            return PortTree(self.treemap[tree_name], tree_name)
+        else:
+            raise AttributeError
     
     def iter_trees(self):
-        tree_dict = PORTDB.repositories.treemap
-        for tree_name in PORTDB.repositories.prepos_order:
+        tree_dict = self.treemap
+        for tree_name in self.tree_order:
             yield PortTree(tree_dict[tree_name], tree_name)
 
-    def iter_packages():
+    def iter_packages(self):
         for tree in self.iter_trees():
             for package in tree.iter_package():
                 yield package
 
-    def iter_ebuilds():
+    def iter_ebuilds(self):
         for tree in self.iter_trees():
             for ebuild in tree.iter_ebuilds():
                 yield ebuild
@@ -147,6 +173,7 @@ class PortTree(ToStrMixin):
     def __init__(self, tree_path = '/usr/portage', name = 'main'):
         """Args:
             tree_path -- full path to portage tree as str
+            name -- repo name as str
         """
         self.porttree = tree_path
         self.name = name
@@ -220,6 +247,10 @@ class Category(ToStrMixin):
         "Full path to category"
         return os.path.join(self.porttree.porttree_path, self.category)
 
+    @property
+    def porttree_path(self):
+        return self.porttree.porttree
+
 
 class Package(ToStrMixin):
     "Represent package as object"
@@ -235,7 +266,12 @@ class Package(ToStrMixin):
         ebuilds = PORTDB.cp_list(self.package,
                                  mytree = self.category.porttree.porttree)
         for ebuild in ebuilds:
-            yield Ebuild(self ,ebuild)
+            try:
+                PORTDB.aux_get(ebuild, [], mytree = self.category.porttree_path)
+            except KeyError:
+                pass
+            else:
+                yield Ebuild(self ,ebuild)
 
     def __unicode__(self):
         return '%s' % self.package
@@ -246,7 +282,10 @@ class Package(ToStrMixin):
 
     @cached_property
     def metadata(self):
-        return MetaData( self.metadata_path)
+        try:
+            return MetaData( self.metadata_path)
+        except IOError:
+            return FakeMetaData()
 
     @property
     def cp(self):
@@ -287,7 +326,7 @@ class Package(ToStrMixin):
 
     @cached_property
     def changelog(self):
-        return open(self.changelog_path,'r').read()
+        return file_get_content(self.changelog_path)
 
 
 class Ebuild(ToStrMixin):
@@ -371,7 +410,11 @@ class Ebuild(ToStrMixin):
     @cached_property
     def homepages(self):
         "List of homepages"
-        return self.homepage_val.split()
+        ho_list = self.homepage_val.split()
+        ret_list = []
+        for ho in ho_list:
+            ret_list.append(StrThatIgnoreCase(ho))
+        return ret_list
 
     @cached_property
     def homepage(self):
@@ -382,7 +425,11 @@ class Ebuild(ToStrMixin):
     @cached_property
     def licenses(self):
         "List of licenses used in ebuild"
-        return filter(_license_filter, self.license.split())
+        license_list = filter(_license_filter, self.license.split())
+        ret_list = []
+        for lic in license_list:
+            ret_list.append(StrThatIgnoreCase(lic))
+        return ret_list
 
     sha1 = cached_property(_file_hash("ebuild_path"), name = 'sha1')
     mtime = cached_property(_file_mtime("ebuild_path"), name = 'mtime')