[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

[Magnus Granberg]

commit:     0db09acc4ba75ea8343465083d8e16d897708b47
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 27 22:59:29 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 27 22:59:29 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=0db09acc

Move the steps that have with portage to do to portage

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |   6 +-
 buildbot_gentoo_ci/steps/builders.py       | 197 -----------------------------
 buildbot_gentoo_ci/steps/portage.py        | 197 +++++++++++++++++++++++++++++
 3 files changed, 200 insertions(+), 200 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index acafed8..b592541 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -106,15 +106,15 @@ def run_build_request():
     #NOTE: pkgcheck do not support it as a dir
     #f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
     #                            workdir='/etc/portage/'))
-    f.addStep(builders.SetMakeProfile())
+    f.addStep(portage.SetMakeProfile())
     # setup repos.conf dir
     f.addStep(buildbot_steps.MakeDirectory(dir="repos.conf",
                                 workdir='/etc/portage/'))
-    f.addStep(builders.SetReposConf())
+    f.addStep(portage.SetReposConf())
     # update the repositorys listed in project_repository
     f.addStep(builders.UpdateRepos())
     # setup make.conf
-    f.addStep(builders.SetMakeConf())
+    f.addStep(portage.SetMakeConf())
     # setup env
     f.addStep(portage.SetEnvDefault())
     # setup package.*

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 3c19345..81d6e95 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -196,106 +196,6 @@ class SetupPropertys(BuildStep):
         self.setProperty('cpv_build', False, 'cpv_build')
         return SUCCESS
 
-class SetMakeProfile(BuildStep):
-
-    name = 'SetMakeProfile'
-    description = 'Running'
-    descriptionDone = 'Ran'
-    descriptionSuffix = None
-    haltOnFailure = True
-    flunkOnFailure = True
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    @defer.inlineCallbacks
-    def run(self):
-        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        portage_repos_path = self.getProperty('portage_repos_path')
-        project_data = self.getProperty('project_data')
-        profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
-        makeprofiles_paths = []
-        #NOTE: pkgcheck don't support make.profile as a dir
-        # we only support one line in db
-        makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
-        for makeprofile in makeprofiles_data:
-            makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', makeprofile['value'], '')
-        #    makeprofiles_paths.append('../../..' + makeprofile_path)
-        #separator = '\n'
-        #makeprofile_path_string = separator.join(makeprofiles_paths)
-        # yield self.build.addStepsAfterCurrentStep([
-        #    steps.StringDownload(makeprofile_path_string + separator,
-        #                        workerdest="make.profile/parent",
-        #                        workdir='/etc/portage/')
-        #    ])
-        #NOTE: pkgcheck profile link
-        shell_commad_list = [
-                    'ln',
-                    '-s'
-                    ]
-        shell_commad_list.append(makeprofile_path)
-        shell_commad_list.append('/etc/portage/make.profile')
-        yield self.build.addStepsAfterCurrentStep([
-            steps.ShellCommandNewStyle(
-                        command=shell_commad_list,
-                        workdir='/'
-                )
-            ])
-        return SUCCESS
-
-class SetReposConf(BuildStep):
-
-    name = 'SetReposConf'
-    description = 'Running'
-    descriptionDone = 'Ran'
-    descriptionSuffix = None
-    haltOnFailure = True
-    flunkOnFailure = True
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    @defer.inlineCallbacks
-    def run(self):
-        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        portage_repos_path = self.getProperty('portage_repos_path')
-        project_data = self.getProperty('project_data')
-        # setup the default.conf
-        repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'repos.conf')
-        if repos_conf_data is None:
-            print('Default repo is not set in repos.conf')
-            return FAILURE
-        # check if repos_conf_data['value'] is vaild repo name
-        separator = '\n'
-        default_conf = []
-        default_conf.append('[DEFAULT]')
-        default_conf.append('main-repo = ' + repos_conf_data['value'])
-        default_conf.append('auto-sync = no')
-        default_conf_string = separator.join(default_conf)
-        yield self.build.addStepsAfterCurrentStep([
-            steps.StringDownload(default_conf_string + separator,
-                                workerdest="repos.conf/default.conf",
-                                workdir='/etc/portage/')
-            ])
-        # add all repos that project have in projects_repositorys to repos.conf/reponame.conf
-        projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
-        for project_repository_data in projects_repositorys_data:
-            repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
-            repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
-            repository_conf = []
-            repository_conf.append('[' + repository_data['name'] + ']')
-            repository_conf.append('location = ' + repository_path)
-            repository_conf.append('sync-uri = ' + repository_data['mirror_url'])
-            repository_conf.append('sync-type = git')
-            repository_conf.append('auto-sync = no')
-            repository_conf_string = separator.join(repository_conf)
-            yield self.build.addStepsAfterCurrentStep([
-                steps.StringDownload(repository_conf_string + separator,
-                                workerdest='repos.conf/' + repository_data['name'] + '.conf',
-                                workdir='/etc/portage/')
-                ])
-        return SUCCESS
-
 class UpdateRepos(BuildStep):
 
     name = 'UpdateRepos'
@@ -326,103 +226,6 @@ class UpdateRepos(BuildStep):
             ])
         return SUCCESS
 
-class SetMakeConf(BuildStep):
-
-    name = 'SetMakeConf'
-    description = 'Running'
-    descriptionDone = 'Ran'
-    descriptionSuffix = None
-    haltOnFailure = True
-    flunkOnFailure = True
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    @defer.inlineCallbacks
-    def run(self):
-        #FIXME: Make a dict before we pass it to the make.conf
-        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        project_data = self.getProperty('project_data')
-        makeconf_variables_data = yield self.gentooci.db.portages.getVariables()
-        separator1 = '\n'
-        separator2 = ' '
-        makeconf_list = []
-        for k in makeconf_variables_data:
-            makeconf_variables_values_data = yield self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
-            makeconf_variable_list = []
-            # we add some default values
-            #FIXME:
-            # we could set them in a config variables
-            # FEATURES
-            if k['variable'] == 'FEATURES':
-                makeconf_variable_list.append('xattr')
-                makeconf_variable_list.append('cgroup')
-                makeconf_variable_list.append('-news')
-                makeconf_variable_list.append('-collision-protect')
-            # EMERGE_DEFAULT_OPTS
-            if k['variable'] == 'EMERGE_DEFAULT_OPTS':
-                makeconf_variable_list.append('--buildpkg=y')
-                makeconf_variable_list.append('--rebuild-if-new-rev=y')
-                makeconf_variable_list.append('--rebuilt-binaries=y')
-                makeconf_variable_list.append('--usepkg=y')
-                makeconf_variable_list.append('--binpkg-respect-use=y')
-                makeconf_variable_list.append('--binpkg-changed-deps=y')
-                makeconf_variable_list.append('--nospinner')
-                makeconf_variable_list.append('--color=n')
-                makeconf_variable_list.append('--ask=n')
-            # CFLAGS
-            if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
-                makeconf_variable_list.append('-O2')
-                makeconf_variable_list.append('-pipe')
-                makeconf_variable_list.append('-march=native')
-                makeconf_variable_list.append('-fno-diagnostics-color')
-                #FIXME:
-                # Depend on worker we may have to add a diffrent march
-            if k['variable'] == 'CXXFLAGS':
-                makeconf_variable_list.append('${CFLAGS}')
-            if k['variable'] == 'FFLAGS':
-                makeconf_variable_list.append('${FCFLAGS}')
-            if k['variable'] == 'ACCEPT_PROPERTIES':
-                makeconf_variable_list.append('-interactive')
-            if k['variable'] == 'ACCEPT_RESTRICT':
-                makeconf_variable_list.append('-fetch')
-            for v in makeconf_variables_values_data:
-                if v['build_id'] == 0:
-                    makeconf_variable_list.append(v['value'])
-            if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list != []:
-                makeconf_variable_list.append('ACCEPT_LICENSE="*"')
-            if makeconf_variable_list != []:
-                makeconf_variable_string = k['variable'] + '="' + separator2.join(makeconf_variable_list) + '"'
-                makeconf_list.append(makeconf_variable_string)
-        # add hardcoded variables and values
-        #FIXME:
-        # we could set them in a config variables
-        makeconf_list.append('LC_MESSAGES=C')
-        makeconf_list.append('NOCOLOR="true"')
-        makeconf_list.append('GCC_COLORS=""')
-        makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
-        makeconf_list.append('CLEAN_DELAY=0')
-        makeconf_list.append('NOCOLOR=true')
-        makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
-        makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
-        makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
-        makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
-        # add ACCEPT_KEYWORDS from the project_data info
-        keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
-        if project_data['status'] == 'unstable':
-            makeconf_keyword = '~' + keyword_data['name']
-        else:
-            makeconf_keyword = keyword_data['name']
-        makeconf_list.append('ACCEPT_KEYWORDS="' + makeconf_keyword + '"')
-        makeconf_string = separator1.join(makeconf_list)
-        print(makeconf_string)
-        yield self.build.addStepsAfterCurrentStep([
-            steps.StringDownload(makeconf_string + separator1,
-                                workerdest="make.conf",
-                                workdir='/etc/portage/')
-            ])
-        return SUCCESS
-
 class RunEmerge(BuildStep):
 
     name = 'RunEmerge'

diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 52daf83..30361e6 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -11,6 +11,203 @@ from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
 from buildbot.plugins import steps
 
+class SetMakeProfile(BuildStep):
+
+    name = 'SetMakeProfile'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        portage_repos_path = self.getProperty('portage_repos_path')
+        project_data = self.getProperty('project_data')
+        profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
+        makeprofiles_paths = []
+        #NOTE: pkgcheck don't support make.profile as a dir
+        # we only support one line in db
+        makeprofiles_data = yield self.gentooci.db.projects.getAllProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
+        for makeprofile in makeprofiles_data:
+            makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', makeprofile['value'], '')
+        #    makeprofiles_paths.append('../../..' + makeprofile_path)
+        #separator = '\n'
+        #makeprofile_path_string = separator.join(makeprofiles_paths)
+        # yield self.build.addStepsAfterCurrentStep([
+        #    steps.StringDownload(makeprofile_path_string + separator,
+        #                        workerdest="make.profile/parent",
+        #                        workdir='/etc/portage/')
+        #    ])
+        #NOTE: pkgcheck profile link
+        shell_commad_list = [
+                    'ln',
+                    '-s'
+                    ]
+        shell_commad_list.append(makeprofile_path)
+        shell_commad_list.append('/etc/portage/make.profile')
+        yield self.build.addStepsAfterCurrentStep([
+            steps.ShellCommandNewStyle(
+                        command=shell_commad_list,
+                        workdir='/'
+                )
+            ])
+        return SUCCESS
+
+class SetReposConf(BuildStep):
+
+    name = 'SetReposConf'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        portage_repos_path = self.getProperty('portage_repos_path')
+        project_data = self.getProperty('project_data')
+        # setup the default.conf
+        repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'repos.conf')
+        if repos_conf_data is None:
+            print('Default repo is not set in repos.conf')
+            return FAILURE
+        # check if repos_conf_data['value'] is vaild repo name
+        separator = '\n'
+        default_conf = []
+        default_conf.append('[DEFAULT]')
+        default_conf.append('main-repo = ' + repos_conf_data['value'])
+        default_conf.append('auto-sync = no')
+        default_conf_string = separator.join(default_conf)
+        yield self.build.addStepsAfterCurrentStep([
+            steps.StringDownload(default_conf_string + separator,
+                                workerdest="repos.conf/default.conf",
+                                workdir='/etc/portage/')
+            ])
+        # add all repos that project have in projects_repositorys to repos.conf/reponame.conf
+        projects_repositorys_data = yield self.gentooci.db.projects.getRepositorysByProjectUuid(project_data['uuid'])
+        for project_repository_data in projects_repositorys_data:
+            repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
+            repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
+            repository_conf = []
+            repository_conf.append('[' + repository_data['name'] + ']')
+            repository_conf.append('location = ' + repository_path)
+            repository_conf.append('sync-uri = ' + repository_data['mirror_url'])
+            repository_conf.append('sync-type = git')
+            repository_conf.append('auto-sync = no')
+            repository_conf_string = separator.join(repository_conf)
+            yield self.build.addStepsAfterCurrentStep([
+                steps.StringDownload(repository_conf_string + separator,
+                                workerdest='repos.conf/' + repository_data['name'] + '.conf',
+                                workdir='/etc/portage/')
+                ])
+        return SUCCESS
+
+class SetMakeConf(BuildStep):
+
+    name = 'SetMakeConf'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        #FIXME: Make a dict before we pass it to the make.conf
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        project_data = self.getProperty('project_data')
+        makeconf_variables_data = yield self.gentooci.db.portages.getVariables()
+        separator1 = '\n'
+        separator2 = ' '
+        makeconf_list = []
+        for k in makeconf_variables_data:
+            makeconf_variables_values_data = yield self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
+            makeconf_variable_list = []
+            # we add some default values
+            #FIXME:
+            # we could set them in a config variables
+            # FEATURES
+            if k['variable'] == 'FEATURES':
+                makeconf_variable_list.append('xattr')
+                makeconf_variable_list.append('cgroup')
+                makeconf_variable_list.append('-news')
+                makeconf_variable_list.append('-collision-protect')
+            # EMERGE_DEFAULT_OPTS
+            if k['variable'] == 'EMERGE_DEFAULT_OPTS':
+                makeconf_variable_list.append('--buildpkg=y')
+                makeconf_variable_list.append('--rebuild-if-new-rev=y')
+                makeconf_variable_list.append('--rebuilt-binaries=y')
+                makeconf_variable_list.append('--usepkg=y')
+                makeconf_variable_list.append('--binpkg-respect-use=y')
+                makeconf_variable_list.append('--binpkg-changed-deps=y')
+                makeconf_variable_list.append('--nospinner')
+                makeconf_variable_list.append('--color=n')
+                makeconf_variable_list.append('--ask=n')
+            # CFLAGS
+            if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
+                makeconf_variable_list.append('-O2')
+                makeconf_variable_list.append('-pipe')
+                makeconf_variable_list.append('-march=native')
+                makeconf_variable_list.append('-fno-diagnostics-color')
+                #FIXME:
+                # Depend on worker we may have to add a diffrent march
+            if k['variable'] == 'CXXFLAGS':
+                makeconf_variable_list.append('${CFLAGS}')
+            if k['variable'] == 'FFLAGS':
+                makeconf_variable_list.append('${FCFLAGS}')
+            if k['variable'] == 'ACCEPT_PROPERTIES':
+                makeconf_variable_list.append('-interactive')
+            if k['variable'] == 'ACCEPT_RESTRICT':
+                makeconf_variable_list.append('-fetch')
+            for v in makeconf_variables_values_data:
+                if v['build_id'] == 0:
+                    makeconf_variable_list.append(v['value'])
+            if k['variable'] == 'ACCEPT_LICENSE' and makeconf_variable_list != []:
+                makeconf_variable_list.append('ACCEPT_LICENSE="*"')
+            if makeconf_variable_list != []:
+                makeconf_variable_string = k['variable'] + '="' + separator2.join(makeconf_variable_list) + '"'
+                makeconf_list.append(makeconf_variable_string)
+        # add hardcoded variables and values
+        #FIXME:
+        # we could set them in a config variables
+        makeconf_list.append('LC_MESSAGES=C')
+        makeconf_list.append('NOCOLOR="true"')
+        makeconf_list.append('GCC_COLORS=""')
+        makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
+        makeconf_list.append('CLEAN_DELAY=0')
+        makeconf_list.append('NOCOLOR=true')
+        makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
+        makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
+        makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
+        makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
+        # add ACCEPT_KEYWORDS from the project_data info
+        keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
+        if project_data['status'] == 'unstable':
+            makeconf_keyword = '~' + keyword_data['name']
+        else:
+            makeconf_keyword = keyword_data['name']
+        makeconf_list.append('ACCEPT_KEYWORDS="' + makeconf_keyword + '"')
+        makeconf_string = separator1.join(makeconf_list)
+        print(makeconf_string)
+        yield self.build.addStepsAfterCurrentStep([
+            steps.StringDownload(makeconf_string + separator1,
+                                workerdest="make.conf",
+                                workdir='/etc/portage/')
+            ])
+        return SUCCESS
+
 class SetPackageDefault(BuildStep):
 
     name = 'SetPackageDefault'