[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

[Magnus Granberg]

commit:     bfa939daa5d6aad8f4a7b129bced7e9dba796828
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 17 17:24:48 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Apr 17 17:24:48 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=bfa939da

Add support for more LocalWorkers

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/builders.py      | 28 +++++++-------
 buildbot_gentoo_ci/config/buildfactorys.py |  8 ----
 buildbot_gentoo_ci/config/schedulers.py    | 10 ++---
 buildbot_gentoo_ci/config/workers.py       |  3 ++
 buildbot_gentoo_ci/steps/category.py       |  2 +-
 buildbot_gentoo_ci/steps/package.py        | 32 +++++++--------
 buildbot_gentoo_ci/steps/portage.py        |  3 +-
 buildbot_gentoo_ci/steps/update_db.py      | 62 ++++++++++--------------------
 buildbot_gentoo_ci/steps/version.py        | 46 +++++++++++-----------
 9 files changed, 85 insertions(+), 109 deletions(-)

diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index d022ac7..56cdde1 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -4,8 +4,14 @@
 from buildbot.plugins import util
 from buildbot_gentoo_ci.config import buildfactorys
 
+# FIXME: get workers from db or file
+LocalWorkers = []
+LocalWorkers.append('updatedb_1')
+LocalWorkers.append('updatedb_2')
+LocalWorkers.append('updatedb_3')
+LocalWorkers.append('updatedb_4')
+
 def gentoo_builders(b=[]):
-    # FIXME: get workers from db
     b.append(util.BuilderConfig(
         name='update_db_check',
         workername='updatedb_1',
@@ -14,47 +20,44 @@ def gentoo_builders(b=[]):
         factory=buildfactorys.update_db_check()
         )
     )
-    # FIXME: get workers from db
     b.append(util.BuilderConfig(
         name='update_repo_check',
-        workername='updatedb_1',
+        workername='updatedb_2',
         workerbuilddir='builds',
         collapseRequests=True,
         factory=buildfactorys.update_repo_check()
         )
     )
-    # FIXME: get workers from db
-    # Use multiplay workers depend on Property(cpv)
+    # Use multiplay workers depend on Property(cp)
     # if cp do not match next one, use diffrent worker then
     # or first cp have done its buildsteps.
+    # first LocalWorker need to be done before we can use mulitplay workers (git pull)
     b.append(util.BuilderConfig(
         name='update_cpv_data',
-        workername='updatedb_1',
+        workernames=LocalWorkers,
         workerbuilddir='builds',
         collapseRequests=False,
         factory=buildfactorys.update_db_cp()
         )
     )
-    # FIXME: get workers from db
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='update_v_data',
-        workername='updatedb_1',
+        workernames=LocalWorkers,
         workerbuilddir='builds',
         collapseRequests=False,
         factory=buildfactorys.update_db_v()
         )
     )
-    # FIXME: get workers from db
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='build_request_data',
-        workername='updatedb_1',
+        workernames=LocalWorkers,
         collapseRequests=False,
         factory=buildfactorys.build_request_check()
         )
     )
-    # FIXME: get workers from db
+    # FIXME: get workers from db or file
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='run_build_request',
@@ -63,11 +66,10 @@ def gentoo_builders(b=[]):
         factory=buildfactorys.run_build_request()
         )
     )
-    # FIXME: get workers from db
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='parse_build_log',
-        workername='updatedb_1',
+        workernames=LocalWorkers,
         collapseRequests=False,
         factory=buildfactorys.parse_build_log()
         )

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 962eeb0..e08127b 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -21,14 +21,6 @@ def update_db_check():
     f.addStep(update_db.GetDataGentooCiProject())
     # update the repos
     f.addStep(update_db.TriggerUpdateRepositorys())
-    # Check if needed path is there
-    f.addStep(portage.CheckPathLocal())
-    # setup the profile
-    f.addStep(portage.SetMakeProfileLocal())
-    # setup repos.conf dir
-    f.addStep(portage.SetReposConfLocal())
-    # setup make.conf
-    f.addStep(portage.SetMakeConfLocal())
     # Make a for loop and trigger new builders with cpv from git_changes
     #   return cpv, repository, project_data
     f.addStep(update_db.TriggerCheckForCPV())

diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index 107bb3c..e520076 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -12,7 +12,7 @@ def builderUpdateDbNames(props):
     return list(builders)
 
 @util.renderer
-def gitUpdateDb(props):
+def getGitChanges(props):
     k = props.changes[0]
     change_data = {}
     print(k)
@@ -23,7 +23,6 @@ def gitUpdateDb(props):
             p = v.split('/')[1]
             pv = v.split('/')[2][:-7]
             cpv = c + '/' + pv
-            print(cpv)
             change_data['cp'] = c + '/' + p
             change_data['cpvs'].append(cpv)
     if k['repository'].endswith('.git'):
@@ -35,16 +34,15 @@ def gitUpdateDb(props):
     change_data['comments'] = k['comments']
     change_data['revision'] = k['revision']
     change_data['timestamp'] =k['when_timestamp']
-    print(change_data)
     return change_data
 
 def gentoo_schedulers():
     scheduler_update_db = schedulers.SingleBranchScheduler(
         name='scheduler_update_db',
         treeStableTimer=0,
-        properties = {
-                        'git_change' : gitUpdateDb,
-                    },
+        properties = dict(
+                        change_data = getGitChanges
+                        ),
         builderNames = builderUpdateDbNames,
         change_filter=util.ChangeFilter(branch='master'),
     )

diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py
index a566b79..6141d89 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -6,5 +6,8 @@ from buildbot.plugins import worker
 def gentoo_workers(w=[]):
     # FIXME: Get workers from db
     w.append(worker.LocalWorker('updatedb_1'))
+    w.append(worker.LocalWorker('updatedb_2'))
+    w.append(worker.LocalWorker('updatedb_3'))
+    w.append(worker.LocalWorker('updatedb_4'))
     w.append(worker.Worker('bot-test', 'test1234'))
     return w

diff --git a/buildbot_gentoo_ci/steps/category.py b/buildbot_gentoo_ci/steps/category.py
index 81061e6..946dc63 100644
--- a/buildbot_gentoo_ci/steps/category.py
+++ b/buildbot_gentoo_ci/steps/category.py
@@ -49,7 +49,7 @@ class CheckC(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        self.category = yield self.getProperty("cpv").split('/')[0]
+        self.category = self.getProperty("change_data")['cp'].split('/')[0]
         print(self.category)
         self.category_data = yield self.gentooci.db.categorys.getCategoryByName(self.category)
         print(self.category_data)

diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
index b4e06b7..76c90cc 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -57,7 +57,7 @@ class CheckP(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        self.package = yield catpkgsplit(self.getProperty("cpv"))[1]
+        self.package = self.getProperty("change_data")['cp'].split('/')[1]
         print(self.package)
         self.package_data = yield self.gentooci.db.packages.getPackageByName(self.package,
                                                                             self.getProperty("category_data")['uuid'],
@@ -86,20 +86,22 @@ class TriggerCheckForV(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         addStepUpdateVData = []
-        addStepUpdateVData.append(
-                    steps.Trigger(
-                        schedulerNames=['update_v_data'],
-                        waitForFinish=False,
-                        updateSourceStamp=False,
-                        set_properties={
-                            'cpv' : self.getProperty("cpv"),
-                            'package_data' : self.getProperty("package_data"),
-                            'repository_data' : self.getProperty("repository_data"),
-                            'category_data' : self.getProperty("category_data"),
-                            'revision_data' : self.getProperty("revision_data"),
-                            'project_data' : self.getProperty("project_data"),
-                        }
-                    )
+        for cpv in self.getProperty("change_data")['cpvs']:
+            print(cpv)
+            addStepUpdateVData.append(
+                steps.Trigger(
+                    schedulerNames=['update_v_data'],
+                    waitForFinish=False,
+                    updateSourceStamp=False,
+                    set_properties={
+                        'cpv' : cpv,
+                        'package_data' : self.getProperty("package_data"),
+                        'repository_data' : self.getProperty("repository_data"),
+                        'category_data' : self.getProperty("category_data"),
+                        'change_data' : self.getProperty("change_data"),
+                        'project_data' : self.getProperty("project_data"),
+                    }
                 )
+            )
         yield self.build.addStepsAfterCurrentStep(addStepUpdateVData)
         return SUCCESS

diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index c7e82ce..37f7342 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -407,11 +407,12 @@ class SetMakeProfileLocal(BuildStep):
         if os.path.isfile(parent_path):
             return SKIPPED
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty('project_data')['profile_repository_uuid'])
         self.repository_basedir = self.gentooci.config.project['repository_basedir']
         makeprofiles_paths = []
         makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'make.profile')
         for makeprofile in makeprofiles_data:
-            makeprofile_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name'], 'profiles', makeprofile['value'], '')
+            makeprofile_path = yield os.path.join(self.repository_basedir, self.profile_repository_data['name'], 'profiles', makeprofile['value'], '')
             makeprofiles_paths.append('../../../' + makeprofile_path)
         yield WriteTextToFile(parent_path, makeprofiles_paths)
         return SUCCESS

diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index b19d0e7..8e7f8ee 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -25,11 +25,13 @@ class GetDataGentooCiProject(BuildStep):
             return FAILURE
         self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.project_data['profile_repository_uuid'])
         if self.profile_repository_data is None:
-            log.err('No data for repository in the database')
+            log.err('No data for profile repository in the database')
+            return FAILURE
+        if not isinstance(self.getProperty("change_data"), dict):
             return FAILURE
         print(self.project_data)
         print(self.profile_repository_data)
-        print(self.getProperty("git_change"))
+        print(self.getProperty("change_data"))
         print(self.getProperty("repository"))
         repository = False
         self.repository_data = False
@@ -39,6 +41,9 @@ class GetDataGentooCiProject(BuildStep):
                     repository = v[:-4]
         if repository:
             self.repository_data = yield self.gentooci.db.repositorys.getRepositoryByName(repository)
+        if self.getProperty("change_data")['repository'] != self.repository_data['name']:
+            log.msg("%s don't match" % self.getProperty("change_data")['repository'])
+            return FAILURE
         self.setProperty("project_data", self.project_data, 'project_data')
         self.setProperty("profile_repository_data", self.profile_repository_data, 'profile_repository_data')
         self.setProperty("repository_data", self.repository_data, 'repository_data')
@@ -65,7 +70,7 @@ class TriggerUpdateRepositorys(BuildStep):
                         set_properties={
                             'profile_repository_uuid' : self.getProperty("profile_repository_data")['uuid'],
                             'repository_uuid' : self.getProperty("repository_data")['uuid'],
-                            'commit_time' : self.getProperty("git_change")['timestamp'],
+                            'commit_time' : self.getProperty("change_data")['timestamp'],
                         }
             )
         ])
@@ -84,43 +89,16 @@ class TriggerCheckForCPV(BuildStep):
 
     @defer.inlineCallbacks
     def run(self):
-        change_data = self.getProperty("git_change")
-        # check if git_change is a dict
-        if not isinstance(change_data, dict):
-            return FAILURE
-        addStepUpdateCPVData = []
-        # make a trigger for all cpv in the list
-        for cpv in change_data['cpvs']:
-            self.success = True
-            if change_data['repository'] != self.getProperty("repository_data")['name']:
-                log.msg("%s don't match" % change_data['repository'])
-                self.success = False
-            # Trigger cpv builds and update db if we are working with ebuilds
-            # check that cpv is valied
-            if catpkgsplit(cpv) is None:
-                log.msg("%s is not vaild package name" % cpv)
-                self.success = False
-            if self.success:
-                revision_data = {}
-                revision_data['author'] = change_data['author']
-                revision_data['committer']  = change_data['committer']
-                revision_data['comments'] = change_data['comments']
-                revision_data['revision'] = change_data['revision']
-                addStepUpdateCPVData.append(
-                    steps.Trigger(
-                        schedulerNames=['update_cpv_data'],
-                        waitForFinish=False,
-                        updateSourceStamp=False,
-                        set_properties={
-                            'cpv' : cpv,
-                            'project_data' : self.getProperty("project_data"),
-                            'repository_data' : self.getProperty("repository_data"),
-                            'revision_data' : revision_data,
-                        }
-                    )
-                )
-        print(addStepUpdateCPVData)
-        yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
-        if self.success is False:
-            return FAILURE
+        yield self.build.addStepsAfterCurrentStep([
+            steps.Trigger(
+                schedulerNames=['update_cpv_data'],
+                waitForFinish=False,
+                updateSourceStamp=False,
+                set_properties={
+                    'project_data' : self.getProperty("project_data"),
+                    'repository_data' : self.getProperty("repository_data"),
+                    'change_data' : self.getProperty("change_data"),
+                }
+            )
+        ])
         return SUCCESS

diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index e6a927a..df97536 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -15,6 +15,7 @@ from twisted.python import log
 from buildbot.process.buildstep import BuildStep
 from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
+from buildbot.process.results import WARNINGS
 from buildbot.plugins import steps
 
 from buildbot_gentoo_ci.steps import portage as portage_steps
@@ -36,6 +37,7 @@ class GetVData(BuildStep):
         # set cwd to builddir
         yield os.chdir(self.getProperty("builddir"))
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        print(self.getProperty("cpv"))
         self.version = yield cpv_getversion(self.getProperty("cpv"))
         print(self.version)
         self.old_version_data = yield self.gentooci.db.versions.getVersionByName(self.version, self.getProperty("package_data")['uuid'])
@@ -90,8 +92,8 @@ class GetCommitdata(BuildStep):
 
     #@defer.inlineCallbacks
     def run(self):
-        print(self.getProperty("revision_data"))
-        self.setProperty('commit_id', self.getProperty("revision_data")['revision'], 'commit_id')
+        print(self.getProperty("change_data"))
+        self.setProperty('commit_id', self.getProperty("change_data")['revision'], 'commit_id')
         return SUCCESS
 
 class AddVersionKeyword(BuildStep):
@@ -243,32 +245,30 @@ class CheckV(BuildStep):
 
     @defer.inlineCallbacks
     def run(self):
-        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        self.old_version_data = self.getProperty("old_version_data")
-        self.ebuild_file = self.getProperty("ebuild_file")
         addStepVData = []
-        print(self.ebuild_file)
-        print(self.old_version_data)
+        print(self.getProperty("ebuild_file"))
+        print(self.getProperty("old_version_data"))
         print(self.getProperty("ebuild_file_hash"))
-        if self.getProperty("ebuild_file") is None and self.getProperty("old_version_data") is not None:
-            addStepVData.append(TriggerBuildCheck())
-            addStepVData.append(DeleteOldVersion())
-        if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None:
-            if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']:
-                addStepVData.append(GetCommitdata())
-                #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH
-                #addStepVData.append(portage_steps.SetEnvForEbuildSH())
-                addStepVData.append(portage_steps.GetAuxMetadata())
-                addStepVData.append(AddVersion())
-                addStepVData.append(AddVersionKeyword())
+        if self.getProperty("ebuild_file") is None:
+            if self.getProperty("old_version_data") is None:
+                return WARNINGS
+            else:
                 addStepVData.append(TriggerBuildCheck())
                 addStepVData.append(DeleteOldVersion())
-            else:
-                return SUCCESS
-        if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None:
+        else:
+            if self.getProperty("old_version_data") is not None:
+                if self.getProperty("ebuild_file_hash") == self.getProperty("old_version_data")['file_hash']:
+                    return WARNINGS
+                else:
+                    addStepVData.append(DeleteOldVersion())
+            # setup /etc/portage
+            addStepVData.append(portage_steps.CheckPathLocal())
+            addStepVData.append(portage_steps.SetMakeProfileLocal())
+            addStepVData.append(portage_steps.SetReposConfLocal())
+            addStepVData.append(portage_steps.SetMakeConfLocal())
+            # get commit data
             addStepVData.append(GetCommitdata())
-            #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH
-            #addStepVData.append(portage_steps.SetEnvForEbuildSH())
+            # get ebuild aux metadata
             addStepVData.append(portage_steps.GetAuxMetadata())
             addStepVData.append(AddVersion())
             addStepVData.append(AddVersionKeyword())