[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/

[Magnus Granberg]

commit:     c667f7b39ec20d30678e758dc74d669f6634797a
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 31 17:43:03 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 31 17:43:03 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c667f7b3

Add SetMakeConf

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  3 +
 buildbot_gentoo_ci/db/connector.py         |  2 +
 buildbot_gentoo_ci/db/model.py             | 18 ++++++
 buildbot_gentoo_ci/db/portages.py          | 43 ++++++++++++++
 buildbot_gentoo_ci/db/projects.py          | 21 +++++++
 buildbot_gentoo_ci/steps/builders.py       | 95 ++++++++++++++++++++++++++++++
 6 files changed, 182 insertions(+)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 3fad219..861a4e1 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,6 +88,7 @@ def build_request_check():
 def run_build_request():
     f = util.BuildFactory()
     # FIXME: 5
+    # Move the etc/portage stuff to is own file
     # set needed Propertys
     f.addStep(builders.SetupPropertys())
     # Clean and add new /etc/portage
@@ -105,4 +106,6 @@ def run_build_request():
     f.addStep(builders.SetReposConf())
     # update the repositorys listed in project_repository
     f.addStep(builders.UpdateRepos())
+    # setup make.conf
+    f.addStep(builders.SetMakeConf())
     return f

diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index 35febe6..7d218ab 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -36,6 +36,7 @@ from buildbot_gentoo_ci.db import categorys
 from buildbot_gentoo_ci.db import packages
 from buildbot_gentoo_ci.db import versions
 from buildbot_gentoo_ci.db import keywords
+from buildbot_gentoo_ci.db import portages
 
 upgrade_message = textwrap.dedent("""\
 
@@ -81,6 +82,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
         self.packages = packages.PackagesConnectorComponent(self)
         self.versions = versions.VersionsConnectorComponent(self)
         self.keywords = keywords.KeywordsConnectorComponent(self)
+        self.portages = portages.PortagesConnectorComponent(self)
 
     @defer.inlineCallbacks
     def setup(self, config, check_version=True, verbose=True):

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 596d04e..bd111be 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -143,6 +143,24 @@ class Model(base.DBConnectorComponent):
         sa.Column('value', sa.String(255), nullable=False),
     )
 
+    portages_makeconf = sautils.Table(
+        "portages_makeconf", metadata,
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('variable', sa.String(255), nullable=False),
+    )
+
+    projects_portages_makeconf = sautils.Table(
+        "projects_portages_makeconf", metadata,
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('project_uuid', sa.String(36),
+                  sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+                  nullable=False),
+        sa.Column('makeconf_id', sa.String(255),
+                  sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'),
+                  nullable=False),
+        sa.Column('value', sa.String(255), nullable=False),
+    )
+
     keywords = sautils.Table(
         "keywords", metadata,
         # unique uuid per keyword

diff --git a/buildbot_gentoo_ci/db/portages.py b/buildbot_gentoo_ci/db/portages.py
new file mode 100644
index 0000000..428fb84
--- /dev/null
+++ b/buildbot_gentoo_ci/db/portages.py
@@ -0,0 +1,43 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors. 
+# Buildbot is free software: you can redistribute it and/or modify it 
+# under the terms of the GNU General Public License as published by the 
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import uuid
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class PortagesConnectorComponent(base.DBConnectorComponent):
+
+    @defer.inlineCallbacks
+    def getVariables(self):
+        def thd(conn):
+            tbl = self.db.model.portages_makeconf
+            q = tbl.select()
+            return [self._row2dict(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
+    def _row2dict(self, conn, row):
+        return dict(
+            id=row.id,
+            variable=row.variable
+            )

diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 00e1569..1f19a00 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -118,6 +118,18 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
         res = yield self.db.pool.do(thd)
         return res
 
+    @defer.inlineCallbacks
+    def getProjectMakeConfById(self, uuid, id):
+        def thd(conn):
+            tbl = self.db.model.projects_portages_makeconf
+            q = tbl.select()
+            q = q.where(tbl.c.project_uuid == uuid)
+            q = q.where(tbl.c.makeconf_id == id)
+            return [self._row2dict_projects_portages_makeconf(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
     def _row2dict(self, conn, row):
         return dict(
             uuid=row.uuid,
@@ -149,3 +161,12 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
             directorys=row.directorys,
             value=row.value
             )
+
+    def _row2dict_projects_portages_makeconf(self, conn, row):
+        return dict(
+            id=row.id,
+            project_uuid=row.project_uuid,
+            makeconf_id=row.makeconf_id,
+            value=row.value,
+            build_id=0
+            )

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index d3b3607..085d096 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -219,3 +219,98 @@ class UpdateRepos(BuildStep):
                             workdir=os.path.join(repository_path, ''))
             ])
         return SUCCESS
+
+class SetMakeConf(BuildStep):
+
+    name = 'SetMakeConf'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        #FIXME: Make a dict before we pass it to the make.conf
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        project_data = self.getProperty('project_data')
+        makeconf_variables_data = yield self.gentooci.db.portages.getVariables()
+        separator1 = '\n'
+        separator2 = ' '
+        makeconf_list = []
+        for k in makeconf_variables_data:
+            makeconf_variables_values_data = yield self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
+            makeconf_variable_list = []
+            # we add some default values
+            #FIXME:
+            # we could set them in a config variables
+            # FEATURES
+            if k['variable'] == 'FEATURES':
+                makeconf_variable_list.append('xattr')
+                makeconf_variable_list.append('cgroup')
+                makeconf_variable_list.append('-news')
+                makeconf_variable_list.append('-collision-protect')
+            # EMERGE_DEFAULT_OPTS
+            if k['variable'] == 'EMERGE_DEFAULT_OPTS':
+                makeconf_variable_list.append('--buildpkg=y')
+                makeconf_variable_list.append('--rebuild-if-new-rev=y')
+                makeconf_variable_list.append('--rebuilt-binaries=y')
+                makeconf_variable_list.append('--usepkg=y')
+                makeconf_variable_list.append('--nospinner')
+                makeconf_variable_list.append('--color=n')
+                makeconf_variable_list.append('--ask=n')
+            # CFLAGS
+            if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
+                makeconf_variable_list.append('-O2')
+                makeconf_variable_list.append('-pipe')
+                makeconf_variable_list.append('-march=native')
+                makeconf_variable_list.append('-fno-diagnostics-color')
+                #FIXME:
+                # Depend on worker we may have to add a diffrent march
+            if k['variable'] == 'CXXFLAGS':
+                makeconf_variable_list.append('${CFLAGS}')
+            if k['variable'] == 'FFLAGS':
+                makeconf_variable_list.append('${FCFLAGS}')
+            if k['variable'] == 'ACCEPT_PROPERTIES':
+                makeconf_variable_list.append('-interactive')
+            if k['variable'] == 'ACCEPT_RESTRICT':
+                makeconf_variable_list.append('-fetch')
+            for v in makeconf_variables_values_data:
+                if v['build_id'] is 0:
+                    makeconf_variable_list.append(v['value'])
+            if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list != []:
+                makeconf_variable_list.append('ACCEPT_LICENSE="*"')
+            if makeconf_variable_list != []:
+                makeconf_variable_string = k['variable'] + '="' + separator2.join(makeconf_variable_list) + '"'
+                makeconf_list.append(makeconf_variable_string)
+        # add hardcoded variables and values
+        #FIXME:
+        # we could set them in a config variables
+        makeconf_list.append('LC_MESSAGES=C')
+        makeconf_list.append('NOCOLOR="true"')
+        makeconf_list.append('GCC_COLORS=""')
+        makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
+        makeconf_list.append('CLEAN_DELAY=0')
+        makeconf_list.append('NOCOLOR=true')
+        makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
+        makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
+        makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
+        makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
+        # add ACCEPT_KEYWORDS from the project_data info
+        keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
+        if project_data['status'] == 'unstable':
+            makeconf_keyword = '~' + keyword_data['name']
+        else:
+            makeconf_keyword = keyword_data['name']
+        makeconf_list.append('ACCEPT_KEYWORDS="' + makeconf_keyword + '"')
+        makeconf_string = separator1.join(makeconf_list)
+        print(makeconf_string)
+        yield self.build.addStepsAfterCurrentStep([
+            steps.StringDownload(makeconf_string + separator1,
+                                workerdest="make.conf",
+                                workdir='/etc/portage/')
+            ])
+        return SUCCESS