[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/

[Magnus Granberg]

commit:     396cbf6ec3c530541c278155828677a341fda248
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Sep  8 00:21:03 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Sep  8 00:21:03 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=396cbf6e

Support dynamically projects and worker in run_build_request

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/builders.py | 24 +++++++++++++++++++++---
 buildbot_gentoo_ci/db/model.py        | 20 ++++++++++++++++++++
 buildbot_gentoo_ci/db/projects.py     | 18 ++++++++++++++++++
 buildbot_gentoo_ci/steps/builders.py  | 28 +++++++++++++++++++---------
 4 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index 56cdde1..c28d016 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,16 +1,34 @@
 # Copyright 2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+from twisted.internet import defer
+
 from buildbot.plugins import util
 from buildbot_gentoo_ci.config import buildfactorys
 
-# FIXME: get workers from db or file
+# FIXME: get LocalWorkers and BuildWorkers from db or file
 LocalWorkers = []
 LocalWorkers.append('updatedb_1')
 LocalWorkers.append('updatedb_2')
 LocalWorkers.append('updatedb_3')
 LocalWorkers.append('updatedb_4')
 
+BuildWorkers = []
+BuildWorkers.append('a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd')
+
+@×××××.inlineCallbacks
+def CanWorkerBuildProject(builder, wfb, request):
+    gentooci = builder.master.namedServices['services'].namedServices['gentooci']
+    project_build_data = request.properties['project_build_data']
+    project_workers = yield gentooci.db.projects.getWorkersByProjectUuid(project_build_data['project_uuid'])
+    print(project_workers)
+    print(wfb)
+    for worker in project_workers:
+        if wfb.worker.workername == worker['worker_uuid']:
+            return True
+    print('no worker')
+    return False
+
 def gentoo_builders(b=[]):
     b.append(util.BuilderConfig(
         name='update_db_check',
@@ -57,11 +75,11 @@ def gentoo_builders(b=[]):
         factory=buildfactorys.build_request_check()
         )
     )
-    # FIXME: get workers from db or file
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='run_build_request',
-        workername='bot-test',
+        workernames=BuildWorkers,
+        canStartBuild=CanWorkerBuildProject,
         collapseRequests=False,
         factory=buildfactorys.run_build_request()
         )

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index ca9932a..d9a3972 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -232,6 +232,17 @@ class Model(base.DBConnectorComponent):
         sa.Column('search_type', sa.Enum('in', 'startswith', 'endswith', 'search'), default='in'),
     )
 
+    projects_workers = sautils.Table(
+        "projects_workers", metadata,
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('project_uuid', sa.String(36),
+                  sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+                  nullable=False),
+        sa.Column('worker_uuid', sa.String(36),
+                  sa.ForeignKey('workers.uuid', ondelete='CASCADE'),
+                  nullable=False),
+    )
+
     keywords = sautils.Table(
         "keywords", metadata,
         # unique uuid per keyword
@@ -289,6 +300,15 @@ class Model(base.DBConnectorComponent):
         sa.Column('status', sa.Enum('stable','unstable','negative','all'), nullable=False),
     )
 
+    workers = sautils.Table(
+        "workers", metadata,
+        # unique id per project
+        sa.Column('uuid', sa.String(36), primary_key=True,
+                  default=lambda: str(uuid.uuid4())),
+        sa.Column('type', sa.Enum('local','default','latent'), nullable=False),
+        sa.Column('enabled', sa.Boolean, default=False),
+    )
+
     # Tables related to users
     # -----------------------
 

diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 176be92..fbef435 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -189,6 +189,17 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
         res = yield self.db.pool.do(thd)
         return res
 
+    @defer.inlineCallbacks
+    def getWorkersByProjectUuid(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.projects_workers
+            q = tbl.select()
+            q = q.where(tbl.c.project_uuid == uuid)
+            return [self._row2dict_projects_workers(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
     def _row2dict(self, conn, row):
         return dict(
             uuid=row.uuid,
@@ -217,6 +228,13 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
             build=row.build
             )
 
+    def _row2dict_projects_workers(self, conn, row):
+        return dict(
+            id=row.id,
+            project_uuid=row.project_uuid,
+            worker_uuid=row.worker_uuid,
+            )
+
     def _row2dict_projects_portage(self, conn, row):
         return dict(
             id=row.id,

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 5b53018..9918938 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -186,7 +186,10 @@ class TriggerRunBuildRequest(BuildStep):
     haltOnFailure = True
     flunkOnFailure = True
 
-    def __init__(self, **kwargs):
+    def __init__(self, projectrepository_data, use_data, project_data, **kwargs):
+        self.projectrepository_data = projectrepository_data
+        self.use_data = use_data
+        self.project_data = project_data
         super().__init__(**kwargs)
 
     @defer.inlineCallbacks
@@ -194,7 +197,7 @@ class TriggerRunBuildRequest(BuildStep):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
         if self.getProperty('project_build_data') is None:
             project_build_data = {}
-            project_build_data['project_uuid'] = self.getProperty('project_data')['uuid']
+            project_build_data['project_uuid'] = self.project_data['uuid']
             project_build_data['version_uuid'] = self.getProperty("version_data")['uuid']
             project_build_data['status'] = 'waiting'
             project_build_data['requested'] = False
@@ -210,8 +213,8 @@ class TriggerRunBuildRequest(BuildStep):
                         set_properties={
                             'cpv' : self.getProperty("cpv"),
                             'version_data' : self.getProperty("version_data"),
-                            'projectrepository_data' : self.getProperty('projectrepository_data'),
-                            'use_data' : self.getProperty("use_data"),
+                            'projectrepository_data' : self.projectrepository_data,
+                            'use_data' : self.use_data,
                             'fullcheck' : self.getProperty("fullcheck"),
                             'project_build_data' : project_build_data
                         }
@@ -239,14 +242,17 @@ class GetProjectRepositoryData(BuildStep):
             return SUCCESS
         # for loop to get all the projects that have the repository
         for projectrepository_data in self.projectrepositorys_data:
+            print(projectrepository_data)
             # get project data
             project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
+            #FIXME: check if we have working workers
+            project_workers = yield self.gentooci.db.projects.getWorkersByProjectUuid(project_data['uuid'])
+            if project_workers == []:
+                print('No Workers on this profile')
+                continue
             # check if auto, enabled and not in config.project['project']
-            if project_data['auto'] is True and project_data['enabled'] is True and project_data['name'] != self.gentooci.config.project['project']:
+            if project_data['auto'] is True and project_data['enabled'] is True and project_data['name'] != self.gentooci.config.project['project']['update_db']:
                 # set Property projectrepository_data so we can use it in the trigger
-                self.setProperty('projectrepository_data', projectrepository_data, 'projectrepository_data')
-                self.setProperty('use_data', None, 'use_data')
-                self.setProperty('project_data', project_data, 'project_data')
                 # get name o project keyword
                 project_keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
                 # if not * (all keywords)
@@ -258,7 +264,11 @@ class GetProjectRepositoryData(BuildStep):
                             version_keywords_data = self.getProperty("version_keyword_dict")[project_keyword_data['name']]
                             # if match trigger BuildRequest on cpv
                             if project_data['status'] == version_keywords_data['status']:
-                                yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
+                                yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest(
+                                    projectrepository_data = projectrepository_data,
+                                    use_data = None,
+                                    project_data = project_data
+                                )])
         return SUCCESS
 
 class SetupPropertys(BuildStep):