[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/

[Magnus Granberg]

commit:     7c12da2ec9470fcf82b37943e2a1cc523351aefe
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 28 01:09:38 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jan 28 01:09:38 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7c12da2e

Add SetReposConf and UpdateRepos

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py | 13 ++--
 buildbot_gentoo_ci/db/model.py             |  3 +-
 buildbot_gentoo_ci/db/projects.py          | 29 ++++++++-
 buildbot_gentoo_ci/steps/builders.py       | 98 ++++++++++++++++++++++++++----
 4 files changed, 123 insertions(+), 20 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index a6bafb5..3fad219 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,8 +88,8 @@ def build_request_check():
 def run_build_request():
     f = util.BuildFactory()
     # FIXME: 5
-    # update repo for the profile
-    f.addStep(builders.UpdateProfileRepo())
+    # set needed Propertys
+    f.addStep(builders.SetupPropertys())
     # Clean and add new /etc/portage
     f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
                                 workdir='/etc/'))
@@ -99,9 +99,10 @@ def run_build_request():
     f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
                                 workdir='/etc/portage/'))
     f.addStep(builders.SetMakeProfile())
-    # setup repo.conf dir
-    #f.addStep(buildbot_steps.MakeDirectory(dir="repo.conf",
+    # setup repos.conf dir
+    f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
                                 workdir='/etc/portage/'))
-    # check if we have all repository's in repos.conf listed in project_repository's
-    # update all repos listed in project_repository's
+    f.addStep(builders.SetReposConf())
+    # update the repositorys listed in project_repository
+    f.addStep(builders.UpdateRepos())
     return f

diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 0defb0c..596d04e 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -138,7 +138,8 @@ class Model(base.DBConnectorComponent):
         sa.Column('project_uuid', sa.String(36),
                   sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
                   nullable=False),
-        sa.Column('directorys', sa.Enum('make.profile'), nullable=False),
+        # FIXME: directorys should be moved to own table
+        sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'), nullable=False),
         sa.Column('value', sa.String(255), nullable=False),
     )
 

diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 5c3406a..00e1569 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -80,7 +80,19 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
         return res
 
     @defer.inlineCallbacks
-    def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+    def getRepositorysByProjectUuid(self, uuid, auto=True):
+        def thd(conn):
+            tbl = self.db.model.projects_repositorys
+            q = tbl.select()
+            q = q.where(tbl.c.project_uuid == uuid)
+            q = q.where(tbl.c.auto == auto)
+            return [self._row2dict_projects_repositorys(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getAllProjectPortageByUuidAndDirectory(self, uuid, directory):
         def thd(conn):
             tbl = self.db.model.projects_portage
             q = tbl.select()
@@ -91,6 +103,21 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
         res = yield self.db.pool.do(thd)
         return res
 
+    @defer.inlineCallbacks
+    def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+        def thd(conn):
+            tbl = self.db.model.projects_portage
+            q = tbl.select()
+            q = q.where(tbl.c.project_uuid == uuid)
+            q = q.where(tbl.c.directorys == directory)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict_projects_portage(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
     def _row2dict(self, conn, row):
         return dict(
             uuid=row.uuid,

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 1c8cbb0..d3b3607 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -80,9 +80,9 @@ class GetProjectRepositoryData(BuildStep):
                         yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
         return SUCCESS
 
-class UpdateProfileRepo(BuildStep):
+class SetupPropertys(BuildStep):
     
-    name = 'UpdateProfileRepo'
+    name = 'SetupPropertys'
     description = 'Running'
     descriptionDone = 'Ran'
     descriptionSuffix = None
@@ -101,17 +101,8 @@ class UpdateProfileRepo(BuildStep):
         self.setProperty('portage_repos_path', self.portage_repos_path, 'portage_repos_path')
         projectrepository_data = self.getProperty('projectrepository_data')
         print(projectrepository_data)
-        repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
         project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
         self.setProperty('project_data', project_data, 'project_data')
-        self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
-        profile_repository_path = yield os.path.join(self.portage_repos_path, self.profile_repository_data['name'])
-        yield self.build.addStepsAfterCurrentStep([
-            steps.Git(repourl=self.profile_repository_data['mirror_url'],
-                            mode='incremental',
-                            submodules=True,
-                            workdir=os.path.join(profile_repository_path, ''))
-            ])
         return SUCCESS
 
 class SetMakeProfile(BuildStep):
@@ -133,7 +124,7 @@ class SetMakeProfile(BuildStep):
         project_data = self.getProperty('project_data')
         profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
         makeprofiles_paths = []
-        makeprofiles_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
+        makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
         for makeprofile in makeprofiles_data:
             makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', makeprofile['value'], '')
             makeprofiles_paths.append('../../..' + makeprofile_path)
@@ -145,3 +136,86 @@ class SetMakeProfile(BuildStep):
                                 workdir='/etc/portage/')
             ])
         return SUCCESS
+
+class SetReposConf(BuildStep):
+
+    name = 'SetReposConf'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        portage_repos_path = self.getProperty('portage_repos_path')
+        project_data = self.getProperty('project_data')
+        # setup the default.conf
+        repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'repos.conf')
+        if repos_conf_data is None:
+            print('Default repo is not set in repos.conf')
+            return FAILURE
+        # check if repos_conf_data['value'] is vaild repo name
+        separator = '\n'
+        default_conf = []
+        default_conf.append('[DEFAULT]')
+        default_conf.append('main-repo = ' + repos_conf_data['value'])
+        default_conf.append('auto-sync = no')
+        default_conf_string = separator.join(default_conf)
+        yield self.build.addStepsAfterCurrentStep([
+            steps.StringDownload(default_conf_string + separator,
+                                workerdest="repos.conf/default.conf",
+                                workdir='/etc/portage/')
+            ])
+        # add all repos that project have in projects_repositorys to repos.conf/reponame.conf
+        projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+        for project_repository_data in projects_repositorys_data:
+            repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+            repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
+            repository_conf = []
+            repository_conf.append('[' + repository_data['name'] + ']')
+            repository_conf.append('location = ' + repository_path)
+            repository_conf.append('sync-uri = ' + repository_data['mirror_url'])
+            repository_conf.append('sync-type = git')
+            repository_conf.append('auto-sync = no')
+            repository_conf_string = separator.join(repository_conf)
+            yield self.build.addStepsAfterCurrentStep([
+                steps.StringDownload(repository_conf_string + separator,
+                                workerdest='repos.conf/' + repository_data['name'] + '.conf',
+                                workdir='/etc/portage/')
+                ])
+        return SUCCESS
+
+class UpdateRepos(BuildStep):
+
+    name = 'UpdateRepos'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        portage_repos_path = self.getProperty('portage_repos_path')
+        project_data = self.getProperty('project_data')
+        # update/add all repos that in project_repository for the project
+        projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+        for project_repository_data in projects_repositorys_data:
+            repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+            repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
+            yield self.build.addStepsAfterCurrentStep([
+            steps.Git(repourl=repository_data['mirror_url'],
+                            mode='incremental',
+                            submodules=True,
+                            workdir=os.path.join(repository_path, ''))
+            ])
+        return SUCCESS