[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/config/

[Magnus Granberg]

commit:     22ad0178cca549026ab51f9f26e3fb9ad28f6374
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 27 22:45:35 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 27 22:45:35 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=22ad0178

Add SetPackageDefault and SetEnvDefault to steps

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  6 +-
 buildbot_gentoo_ci/db/model.py             | 27 ++++++++-
 buildbot_gentoo_ci/db/portages.py          | 14 +++++
 buildbot_gentoo_ci/db/projects.py          | 44 ++++++++++++++
 buildbot_gentoo_ci/steps/portage.py        | 97 ++++++++++++++++++++++++++++++
 5 files changed, 185 insertions(+), 3 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index ac8bac1..acafed8 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -9,6 +9,7 @@ from buildbot_gentoo_ci.steps import category
 from buildbot_gentoo_ci.steps import package
 from buildbot_gentoo_ci.steps import version
 from buildbot_gentoo_ci.steps import builders
+from buildbot_gentoo_ci.steps import portage
 
 def update_db_check():
     f = util.BuildFactory()
@@ -114,9 +115,10 @@ def run_build_request():
     f.addStep(builders.UpdateRepos())
     # setup make.conf
     f.addStep(builders.SetMakeConf())
-    # setup package.*
-    #f.addStep(portages.SetPackageUse())
     # setup env
+    f.addStep(portage.SetEnvDefault())
+    # setup package.*
+    #f.addStep(portage.SetPackageDefault())
     # setup files in /etc if needed
     # run --regen if needed on repo
     # update packages before any tests

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 2153b79..b91f158 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -139,7 +139,6 @@ class Model(base.DBConnectorComponent):
         sa.Column('project_uuid', sa.String(36),
                   sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
                   nullable=False),
-        # FIXME: directorys should be moved to own table
         sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'), nullable=False),
         sa.Column('value', sa.String(255), nullable=False),
     )
@@ -162,6 +161,32 @@ class Model(base.DBConnectorComponent):
         sa.Column('value', sa.String(255), nullable=False),
     )
 
+    # projects etc/portage/env settings
+    projects_portages_env = sautils.Table(
+        "projects_portages_env", metadata,
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('project_uuid', sa.String(36),
+                  sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+                  nullable=False),
+        sa.Column('makeconf_id', sa.String(255),
+                  sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'),
+                  nullable=False),
+        sa.Column('name', sa.String(255), nullable=False),
+        sa.Column('value', sa.String(255), nullable=False),
+    )
+
+    # projects etc/portage/package.* settings
+    projects_portage_package = sautils.Table(
+        "projects_portage_package", metadata,
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('project_uuid', sa.String(36),
+                  sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+                  nullable=False),
+        sa.Column('directorys', sa.Enum('use', 'accept_keywords', 'env'), nullable=False),
+        sa.Column('value1', sa.String(255), nullable=False),
+        sa.Column('value2', sa.String(255), nullable=True),
+    )
+
     projects_emerge_options = sautils.Table(
         "projects_emerge_options", metadata,
         sa.Column('id', sa.Integer, primary_key=True),

diff --git a/buildbot_gentoo_ci/db/portages.py b/buildbot_gentoo_ci/db/portages.py
index 428fb84..86321e3 100644
--- a/buildbot_gentoo_ci/db/portages.py
+++ b/buildbot_gentoo_ci/db/portages.py
@@ -36,6 +36,20 @@ class PortagesConnectorComponent(base.DBConnectorComponent):
         res = yield self.db.pool.do(thd)
         return res
 
+    @defer.inlineCallbacks
+    def getVariableById(self, id):
+        def thd(conn):
+            tbl = self.db.model.portages_makeconf
+            q = tbl.select()
+            q = q.where(tbl.c.id == id)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
     def _row2dict(self, conn, row):
         return dict(
             id=row.id,

diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 37d401c..37ae2f1 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -130,6 +130,28 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
         res = yield self.db.pool.do(thd)
         return res
 
+    @defer.inlineCallbacks
+    def getProjectPortageEnvByUuid(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.projects_portages_env
+            q = tbl.select()
+            q = q.where(tbl.c.project_uuid == uuid)
+            return [self._row2dict_projects_portages_env(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getProjectPortagePackageByUuid(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.projects_portages_package
+            q = tbl.select()
+            q = q.where(tbl.c.project_uuid == uuid)
+            return [self._row2dict_projects_portages_package(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
     @defer.inlineCallbacks
     def getProjectEmergeOptionsByUuid(self, uuid):
         def thd(conn):
@@ -190,6 +212,28 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
             build_id=0
             )
 
+    def _row2dict_projects_portages_env(self, conn, row):
+        return dict(
+            id=row.id,
+            project_uuid=row.project_uuid,
+            makeconf_id=row.makeconf_id,
+            name=row.name,
+            value=row.value
+            )
+
+    def _row2dict_projects_portages_package(self, conn, row):
+        if row.value2 == '':
+            value2 = None
+        else:
+            value2 = row.value2
+        return dict(
+            id=row.id,
+            project_uuid=row.project_uuid,
+            directorys=row.directorys,
+            value1=row.value1,
+            value2=value2
+            )
+
     def _row2dict_projects_emerge_options(self, conn, row):
         return dict(
             id=row.id,

diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
new file mode 100644
index 0000000..52daf83
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -0,0 +1,97 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.plugins import steps
+
+class SetPackageDefault(BuildStep):
+
+    name = 'SetPackageDefault'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        project_data = self.getProperty('project_data')
+        aftersteps_list = []
+        packagedir_list = []
+        packagedir_list.append('env')
+        #FIXME:
+        # get list what dir we need to make from db
+        # create the dirs
+        for packagedir in packagedir_list:
+            aftersteps_list.append(steps.MakeDirectory(dir='package.' + packagedir,
+                                workdir='/etc/portage/'))
+        #FIXME:
+        # add the needed package.* settings from db
+        yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+        return SUCCESS
+
+class SetEnvDefault(BuildStep):
+
+    name = 'SetEnvDefault'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def getPortageEnv(self, portage_env_data, portage_env_dict = {}):
+        for project_portage_env in portage_env_data:
+            if not project_portage_env['name'] in portage_env_dict:
+                portage_env_dict[project_portage_env['name']] = {
+                    project_portage_env['makeconf_id'] : []
+                    }
+            if not project_portage_env['makeconf_id'] in portage_env_dict[project_portage_env['name']]:
+                portage_env_dict[project_portage_env['name']][project_portage_env['makeconf_id']] = []
+            portage_env_dict[project_portage_env['name']][project_portage_env['makeconf_id']].append(project_portage_env['value'])
+        return portage_env_dict
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        project_data = self.getProperty('project_data')
+        default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+        aftersteps_list = []
+        separator1 = '\n'
+        separator2 = ' '
+        # create the dir
+        aftersteps_list.append(steps.MakeDirectory(dir='env',
+                                workdir='/etc/portage/'))
+        #FIXME:
+        # add env settings from the db
+        default_project_portage_env_data = yield self.gentooci.db.projects.getProjectPortageEnvByUuid(default_project_data['uuid'])
+        project_portage_env_data = yield self.gentooci.db.projects.getProjectPortageEnvByUuid(project_data['uuid'])
+        project_portage_env_dict = yield self.getPortageEnv(default_project_portage_env_data, portage_env_dict = {})
+        project_portage_env_dict = yield self.getPortageEnv(project_portage_env_data, portage_env_dict = project_portage_env_dict)
+        print(project_portage_env_dict)
+        for k, v in project_portage_env_dict.items():
+            env_strings = []
+            for a, b in v.items():
+                variable_data = yield self.gentooci.db.portages.getVariableById(a)
+                env_variable_string = variable_data['variable'] + '="' + separator2.join(b) + '"'
+                env_strings.append(env_variable_string)
+            yield self.build.addStepsAfterCurrentStep([
+            steps.StringDownload(separator1.join(env_strings) + separator1,
+                                workerdest=k + '.conf',
+                                workdir='/etc/portage/env/')
+            ])
+        yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+        return SUCCESS