[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

[Magnus Granberg]

commit:     dfb70ebe084176e56f39b71545b9bd71196b1c93
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 25 19:51:56 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Sep 25 19:51:56 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=dfb70ebe

Add support for package.use and exclude package to build

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 10 +++------
 buildbot_gentoo_ci/db/model.py             | 10 ++++-----
 buildbot_gentoo_ci/db/projects.py          | 24 +++++++++++++++------
 buildbot_gentoo_ci/steps/builders.py       | 16 +++++++++++++-
 buildbot_gentoo_ci/steps/portage.py        | 34 ++++++++++++++++++------------
 5 files changed, 62 insertions(+), 32 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index d2e195c..41c3171 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -152,13 +152,9 @@ def run_build_request():
     f.addStep(builders.RunEmerge(step='match'))
     # Add the needed steps for build
     f.addStep(builders.RunBuild())
-    # clean up the worker
-    # look at the log to see if we need to do stuff
-    # run pre-depclean and depclean if set
-    f.addStep(builders.RunEmerge(step='pre-depclean'))
-    # run preserved-libs and depclean
-    f.addStep(builders.RunEmerge(step='preserved-libs'))
-    f.addStep(builders.RunEmerge(step='depclean'))
+    # run eclean pkg and dist
+    #f.addStep(builders.RunEclean(step='pkg')
+    #f.addStep(builders.RunEclean(step='dist')
     return f
 
 def parse_build_log():

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index e2cbc8d..12d5ee9 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -178,15 +178,15 @@ class Model(base.DBConnectorComponent):
     )
 
     # projects etc/portage/package.* settings
-    projects_portage_package = sautils.Table(
-        "projects_portage_package", metadata,
+    projects_portages_package = sautils.Table(
+        "projects_portages_package", metadata,
         sa.Column('id', sa.Integer, primary_key=True),
         sa.Column('project_uuid', sa.String(36),
                   sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
                   nullable=False),
-        sa.Column('directorys', sa.Enum('use', 'accept_keywords', 'env'), nullable=False),
-        sa.Column('value1', sa.String(255), nullable=False),
-        sa.Column('value2', sa.String(255), nullable=True),
+        sa.Column('directory', sa.Enum('use', 'accept_keywords', 'env', 'exclude'), nullable=False),
+        sa.Column('package', sa.String(255), nullable=False),
+        sa.Column('value', sa.String(255), nullable=True),
     )
 
     projects_emerge_options = sautils.Table(

diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 408450e..48d4288 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -152,6 +152,18 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
         res = yield self.db.pool.do(thd)
         return res
 
+    @defer.inlineCallbacks
+    def getProjectPortagePackageByUuidAndExclude(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.projects_portages_package
+            q = tbl.select()
+            q = q.where(tbl.c.project_uuid == uuid)
+            q = q.where(tbl.c.directory == 'exclude')
+            return [self._row2dict_projects_portages_package(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
     @defer.inlineCallbacks
     def getProjectEmergeOptionsByUuid(self, uuid):
         def thd(conn):
@@ -263,16 +275,16 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
             )
 
     def _row2dict_projects_portages_package(self, conn, row):
-        if row.value2 == '':
-            value2 = None
+        if row.value == '':
+            value = None
         else:
-            value2 = row.value2
+            value = row.value
         return dict(
             id=row.id,
             project_uuid=row.project_uuid,
-            directorys=row.directorys,
-            value1=row.value1,
-            value2=value2
+            directory=row.directory,
+            package=row.package,
+            value=value
             )
 
     def _row2dict_projects_emerge_options(self, conn, row):

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 7732f39..4f2b1b6 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -13,6 +13,7 @@ from twisted.python import log
 from buildbot.process.buildstep import BuildStep
 from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
 from buildbot.plugins import steps
 
 def PersOutputOfEmerge(rc, stdout, stderr):
@@ -451,9 +452,19 @@ class RunEmerge(BuildStep):
             aftersteps_list.append(CheckDepcleanLogs('depclean'))
 
         if self.step == 'match':
+            packages_excludes = yield self.gentooci.db.projects.getProjectPortagePackageByUuidAndExclude(self.getProperty('project_data')['uuid'])
             cpv = self.getProperty("cpv")
             c = yield catpkgsplit(cpv)[0]
             p = yield catpkgsplit(cpv)[1]
+            # Check if package is on the exclude list
+            if packages_excludes != []:
+                for package_exclude in packages_excludes:
+                    if '/' not in package_exclude:
+                        if package_exclude == p:
+                            return SKIPPED
+                    else:
+                        if package_exclude == c + '/' + p:
+                            return SKIPPED
             shell_commad_list.append('-pO')
             # don't use bin for match
             shell_commad_list.append('--usepkg=n')
@@ -900,11 +911,14 @@ class RunBuild(BuildStep):
         if not self.getProperty('cpv_build'):
             #FIXME:
             # trigger pars_build_log if we have any logs to check
-            return SUCCESS
+            return SKIPPED
         aftersteps_list = []
         aftersteps_list.append(RunEmergeInfo())
         aftersteps_list.append(RunEmerge(step='pre-build'))
         aftersteps_list.append(RunEmerge(step='build'))
+        aftersteps_list.append(RunEmerge(step='pre-depclean'))
+        aftersteps_list.append(RunEmerge(step='preserved-libs'))
+        aftersteps_list.append(RunEmerge(step='depclean'))
         self.setProperty('depclean', False, 'depclean')
         self.setProperty('preserved_libs', False, 'preserved-libs')
         yield self.build.addStepsAfterCurrentStep(aftersteps_list)

diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 03552b6..94323ba 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -252,20 +252,28 @@ class SetPackageDefault(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        project_data = self.getProperty('project_data')
-        aftersteps_list = []
-        packagedir_list = []
-        packagedir_list.append('env')
-        packagedir_list.append('use')
-        #FIXME:
-        # get list what dir we need to make from db
-        # create the dirs
-        for packagedir in packagedir_list:
-            aftersteps_list.append(steps.MakeDirectory(dir='package.' + packagedir,
-                                workdir='/etc/portage/'))
-        #FIXME:
+        separator1 = '\n'
+        separator2 = ' '
+        self.aftersteps_list = []
+        #FIXME: accept_keywords, env
         # add the needed package.* settings from db
-        yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+        package_conf_use_list = []
+        package_settings = yield self.gentooci.db.projects.getProjectPortagePackageByUuid(self.getProperty('project_data')['uuid'])
+        for package_setting in package_settings:
+            if package_setting['directory'] == 'use':
+                package_conf_use_list.append(separator2.join(package_setting['package'],package_setting['value']))
+        if package_conf_use_list != []:
+            package_conf_use_string = separator1.join(package_conf_use_list)
+            self.aftersteps_list.append(
+                        steps.StringDownload(package_conf_use_string + separator1,
+                            workerdest='default.conf',
+                            workdir='/etc/portage/package.use/'
+                            )
+                        )
+            # create the dir
+            aftersteps_list.append(steps.MakeDirectory(dir='package.use',
+                                workdir='/etc/portage/'))
+        yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
         return SUCCESS
 
 class SetEnvDefault(BuildStep):