[gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/libs/package_info/package_backends/

[Slava Bacherikov]

commit:     e6a1805fa32a963d1728c1e95797a13874662200
Author:     Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Thu Jul  5 17:43:48 2012 +0000
Commit:     Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Thu Jul  5 17:43:48 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=e6a1805f

Add stats calculating

---
 gpackages/apps/packages/models.py                  |   92 ++++++++++++++++++--
 gpackages/apps/packages/stats.py                   |   31 +++++++
 .../libs/package_info/package_backends/pkgcore.py  |    6 ++
 .../libs/package_info/package_backends/portage.py  |    6 ++
 4 files changed, 127 insertions(+), 8 deletions(-)

diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 51b839e..efb4638 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -6,6 +6,7 @@ from package_info.generic import get_from_kwargs_and_del
 from package_info.repo_info import REPOS_TYPE
 # relative
 from .keywords import KeywordRepr
+from .stats import StatsMixin
 #from utils import get_link
 from package_info.validators import validate_url, validate_email, \
                                     validate_name
@@ -90,12 +91,17 @@ class ArchesModel(models.Model):
     def __unicode__(self):
         return self.name
 
-class RepositoryModel(AbstractDateTimeModel):
+class RepositoryModel(StatsMixin, AbstractDateTimeModel):
     QUALITY_CHOICES = ( (0, 'stable'),
                         (1, 'testing'),
                         (2, 'experimental'),
                       )
 
+    stats_params = (
+        ('packages_count', 'packagemodel'),
+        ('ebuilds_count', 'packagemodel__ebuildmodel'),
+    )
+
     def __init__(self, *args, **kwargs):
         repo = get_from_kwargs_and_del('repo', kwargs)
         super(RepositoryModel, self).__init__(*args, **kwargs)
@@ -113,6 +119,12 @@ class RepositoryModel(AbstractDateTimeModel):
     official = models.BooleanField(default = False)
     quality = models.PositiveSmallIntegerField(choices = QUALITY_CHOICES)
 
+    # Maybe in future auto generate this field by metaclass ?
+    # For fast stats
+    packages_count = models.PositiveIntegerField(default = 0)
+    ebuilds_count = models.PositiveIntegerField(default = 0)
+
+
     objects = managers.RepositoryManager()
 
     def init_by_repo(self, repo):
@@ -194,7 +206,7 @@ class RepositorySourceModel(models.Model):
     def __unicode__(self):
         return self.url
 
-class CategoryModel(models.Model):
+class CategoryModel(StatsMixin, models.Model):
     def __init__(self, *args, **kwargs):
         super(CategoryModel, self).__init__(*args, **kwargs)
 
@@ -202,6 +214,13 @@ class CategoryModel(models.Model):
         if isinstance(category, AbstractCategory):  
             self.update_by_category(category)
 
+    stats_params = (
+        ('virtual_packages_count', 'virtualpackagemodel'),
+        ('packages_count', 'virtualpackagemodel__packagemodel'),
+        ('repositories_count', 'virtualpackagemodel__packagemodel__repository'),
+        ('ebuilds_count', 'virtualpackagemodel__packagemodel__ebuildmodel'),
+    )
+
     def update_by_category(self, category):
         self.description = category.metadata.default_descr
         self.metadata_hash = category.metadata_sha1
@@ -212,21 +231,47 @@ class CategoryModel(models.Model):
     category = models.CharField(unique = True, max_length = 70, db_index = True)
     description = models.TextField(blank = True, null = True)
     metadata_hash = models.CharField(max_length = 128, null = True)
+
+    # Maybe in future auto generate this field by metaclass ?
+    # For fast stats
+    virtual_packages_count = models.PositiveIntegerField(default = 0)
+    packages_count = models.PositiveIntegerField(default = 0)
+    repositories_count = models.PositiveIntegerField(default = 0)
+    ebuilds_count =  models.PositiveIntegerField(default = 0)
     
     def __unicode__(self):
         return unicode(self.category)
 
-class MaintainerModel(AbstractDateTimeModel):
+class MaintainerModel(StatsMixin, AbstractDateTimeModel):
 
     def __init__(self, *args, **kwargs):
         maintainer = get_from_kwargs_and_del('maintainer', kwargs)
         super(MaintainerModel, self).__init__(*args, **kwargs)
         if maintainer is not None:
             self.init_by_maintainer(maintainer)
+
+    stats_params = (
+           ('packages_count', 'packagemodel'),
+           ('herds_count', 'herdsmodel'),
+           ('ebuilds_count', 'packagemodel__ebuildmodel'),
+           ('repositories_count', 'packagemodel__ebuildmodel'),
+           ('news_author_count', 'author_news_set'),
+           ('news_translator_count', 'translator_news_set')
+        )
         
     name = models.CharField(max_length = 255, blank = True, null = True)
     email = models.EmailField(unique = True, validators = [validate_email], db_index = True)
 
+    # For fast stats
+    # Maybe use django-composition ?
+    # Maybe in future auto generate this field by metaclass ?
+    herds_count = models.PositiveIntegerField(default = 0)
+    packages_count = models.PositiveIntegerField(default = 0)
+    ebuilds_count =  models.PositiveIntegerField(default = 0)
+    repositories_count =  models.PositiveIntegerField(default = 0)
+    news_author_count = models.PositiveIntegerField(default = 0)
+    news_translator_count = models.PositiveIntegerField(default = 0)
+
     objects = managers.MaintainerManager()
 
     def init_by_maintainer(self, maintainer):
@@ -246,7 +291,7 @@ class MaintainerModel(AbstractDateTimeModel):
     class Meta:
         ordering = ('name',)
 
-class HerdsModel(AbstractDateTimeModel):
+class HerdsModel(StatsMixin, AbstractDateTimeModel):
 
     def __init__(self, *args, **kwargs):
         herd = get_from_kwargs_and_del('herd', kwargs)
@@ -254,11 +299,26 @@ class HerdsModel(AbstractDateTimeModel):
         if herd is not None:
             self.init_by_herd(herd)
 
+    stats_params = (
+           ('packages_count', 'packagemodel'),
+           ('maintainers_count', 'maintainers'),
+           ('ebuilds_count', 'packagemodel__ebuildmodel'),
+           ('repositories_count', 'packagemodel__repository'),
+        )
+
     name = models.CharField(unique = True, max_length = 150, db_index = True)
     email = models.EmailField(validators = [validate_email])
     description = models.TextField(blank = True, null = True)
     maintainers = models.ManyToManyField(MaintainerModel, blank = True)
 
+    # For fast stats
+    # Maybe use django-composition ?
+    # Maybe in future auto generate this field by metaclass ?
+    maintainers_count = models.PositiveIntegerField(default = 0)
+    packages_count = models.PositiveIntegerField(default = 0)
+    ebuilds_count = models.PositiveIntegerField(default = 0)
+    repositories_count = models.PositiveIntegerField(default = 0)
+
     objects = managers.HerdsManager()
 
     def init_by_herd(self, herd):
@@ -304,7 +364,8 @@ class VirtualPackageModel(models.Model):
     class Meta:
         unique_together = ('name', 'category')
 
-class PackageModel(AbstractDateTimeModel):
+class PackageModel(StatsMixin, AbstractDateTimeModel):
+
     def __init__(self, *args, **kwargs):
         package_object, category = \
             get_from_kwargs_and_del(('package','category' ), kwargs)
@@ -313,6 +374,9 @@ class PackageModel(AbstractDateTimeModel):
         if isinstance(package_object, AbstarctPackage):
             self.init_by_package(package_object, category = category)
             
+    stats_params = (
+           ('ebuilds_count', 'ebuildmodel'),
+        )
         
     virtual_package = models.ForeignKey(VirtualPackageModel, db_index = True)
     changelog = models.TextField(blank = True, null = True)
@@ -329,6 +393,8 @@ class PackageModel(AbstractDateTimeModel):
     description = models.TextField(blank = True, null = True)
     repository = models.ForeignKey(RepositoryModel, db_index = True)
     # Different versions can have different licenses, or homepages.
+
+    ebuilds_count = models.PositiveIntegerField(default = 0)
     
     objects = managers.PackageManager()
 
@@ -407,10 +473,16 @@ class PackageModel(AbstractDateTimeModel):
         unique_together = ('virtual_package', 'repository')
         ordering = ('-updated_datetime',)
 
-class UseFlagModel(models.Model):
+class UseFlagModel(StatsMixin, models.Model):
+    stats_params = (
+           ('ebuilds_count', 'ebuildmodel'),
+        )
+
     name = models.CharField(unique = True, max_length = 60, db_index = True)
     description = models.TextField(blank = True)
     
+    ebuilds_count = models.PositiveIntegerField(default = 0)
+
     def __unicode__(self):
         return self.name
 
@@ -431,9 +503,14 @@ class UseFlagDescriptionModel(models.Model):
     class Meta:
         unique_together = ('use_flag', 'package')
 
-class LicenseModel(models.Model):
+class LicenseModel(StatsMixin, models.Model):
+    stats_params = (
+           ('ebuilds_count', 'ebuildmodel'),
+        )
+
     name = models.CharField(unique = True, max_length = 60, db_index = True)
     #description = TextField()
+    ebuilds_count = models.PositiveIntegerField(default = 0)
     
     def __unicode__(self):
         return self.name
@@ -648,4 +725,3 @@ class Keyword(models.Model):
     class Meta:
         unique_together = ('ebuild', 'arch')
 
-

diff --git a/gpackages/apps/packages/stats.py b/gpackages/apps/packages/stats.py
new file mode 100644
index 0000000..9bf3efa
--- /dev/null
+++ b/gpackages/apps/packages/stats.py
@@ -0,0 +1,31 @@
+from django.db.models import Count
+from itertools import starmap
+
+gen_prefix = lambda x: x + '_prefix'
+
+def gen_query_dict(params):
+    query_dict = {}
+    for key, value in starmap(lambda x,y: (gen_prefix(x), y), params):
+        query_dict[key] = Count(value, distinct = True)
+
+    return query_dict
+
+def make_query_for_stats(model, params):
+    query_dict = gen_query_dict(params)
+    return model.objects.annotate(**query_dict)
+
+def update_stats(model, params):
+    query = make_query_for_stats(model, params)
+    for obj in query:
+        for key, mykey in starmap(lambda  x,y: (x, gen_prefix(x)), params):
+            setattr(obj, key, getattr(obj, mykey))
+
+        obj.save(force_update = True)
+
+class StatsMixin(object):
+
+    stats_params = ((),)
+
+    @classmethod
+    def calc_stats(cls):
+        update_stats(cls, cls.stats_params)

diff --git a/gpackages/libs/package_info/package_backends/pkgcore.py b/gpackages/libs/package_info/package_backends/pkgcore.py
index 9a17208..2ad6b69 100644
--- a/gpackages/libs/package_info/package_backends/pkgcore.py
+++ b/gpackages/libs/package_info/package_backends/pkgcore.py
@@ -57,6 +57,9 @@ class Portage(PortageMixin):
 
     def __unicode__(self):
         return u'pkgcore'
+
+    def __len__(self):
+        return len(self.repo_list)
         
 class PortTree(PortTreeMixin):
 
@@ -86,6 +89,9 @@ class PortTree(PortTreeMixin):
     def _versions(self):
         return self._repo_obj.versions
 
+    def __len__(self):
+        return len(self.categories)
+
 class Category(CategoryMixin):
     
     __slots__ = ('_repo_obj', 'name')

diff --git a/gpackages/libs/package_info/package_backends/portage.py b/gpackages/libs/package_info/package_backends/portage.py
index 729e6e9..e3707d4 100644
--- a/gpackages/libs/package_info/package_backends/portage.py
+++ b/gpackages/libs/package_info/package_backends/portage.py
@@ -51,6 +51,9 @@ class Portage(PortageMixin):
     def __unicode__(self):
         return u'portage'
 
+    def __len__(self):
+        return len(self.tree_order)
+
 class PortTree(PortTreeMixin):
     "Represent portage tree as object"
 
@@ -72,6 +75,9 @@ class PortTree(PortTreeMixin):
         "Full path to portage tree"
         return self.porttree
 
+    def __len__(self):
+        return len(PORTDB.settings.categories)
+
 class Category(CategoryMixin):
     "Represent category of portage tree as object"