[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/

[Magnus Granberg]

commit:     b3edf64aaff42659b3593dfe24a7e0a68e1881a0
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 24 00:33:19 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Dec 24 00:33:19 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b3edf64a

put emerge and package info to a file insted

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  4 ++
 buildbot_gentoo_ci/steps/builders.py       | 74 +++++++++++++++++++++++------
 buildbot_gentoo_ci/steps/logs.py           | 75 +++++++++++++++++++++++++++++-
 3 files changed, 137 insertions(+), 16 deletions(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 27f0076..6203a1f 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -172,8 +172,12 @@ def parse_build_log():
     f.addStep(logs.MakeIssue())
     # add sum log to buildbot log
     f.addStep(logs.setBuildbotLog())
+    # pers the emerge info
+    f.addStep(logs.SetupParserEmergeInfoLog())
     # add emerge info to log and db
     f.addStep(logs.setEmergeInfoLog())
+    # add package info to log and db
+    f.addStep(logs.setPackageInfoLog())
     # set BuildStatus
     f.addStep(logs.setBuildStatus())
     # setup things for the irc bot

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 7247abd..e76fe3d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -590,14 +590,35 @@ class CheckEmergeLogs(BuildStep):
             workdir=workdir
         ))
 
-    @defer.inlineCallbacks
-    def getLogFile(self, cpv, log_dict):
-        masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")) ,log_dict[cpv]['full_logname'])
+    def addFileUploade(self, sourcefile, destfile):
         self.aftersteps_list.append(steps.FileUpload(
-            workersrc=log_dict[cpv]['log_path'],
-            masterdest=masterdest
+            workersrc=sourcefile,
+            masterdest=destfile
         ))
 
+    @defer.inlineCallbacks
+    def getLogFile(self, cpv, log_dict):
+        destfile = yield os.path.join(self.masterdest, log_dict[cpv]['full_logname'])
+        sourcefile = log_dict[cpv]['log_path']
+        self.addFileUploade(sourcefile, destfile)
+
+    @defer.inlineCallbacks
+    def getEmergeFiles(self, faild_version_data):
+        # get emerge info
+        destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
+        sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
+        self.addFileUploade(sourcefile, destfile)
+        #FIXME:
+        # if faild_version_data:
+        # get emerge.log
+        # get elogs
+        # world file
+
+    def getBuildWorkdirFiles(self):
+        #FIXME:
+        # get files from the build workdir
+        pass
+
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
@@ -610,6 +631,8 @@ class CheckEmergeLogs(BuildStep):
                     ]
         package_dict = emerge_output['packages']
 
+        self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
+
         #FIXME: Prosees the logs and do stuff
         # preserved-libs
         if emerge_output['preserved_libs'] and projects_emerge_options['preserved_libs']:
@@ -757,12 +780,12 @@ class CheckEmergeLogs(BuildStep):
                 print(log_dict)
                 cpv = self.getProperty('cpv')
                 faild_cpv = emerge_output['failed']
+                faild_version_data = False
                 if cpv in log_dict or faild_cpv in log_dict:
+                    yield self.createDistDir()
                     if cpv in log_dict:
                         self.log_data[cpv] = log_dict[cpv]
-                        yield self.createDistDir()
                         yield self.getLogFile(cpv, log_dict)
-                        faild_version_data = False
                     if faild_cpv:
                         # failed and build requested cpv
                         if cpv == faild_cpv:
@@ -772,6 +795,8 @@ class CheckEmergeLogs(BuildStep):
                             self.log_data[faild_cpv] = log_dict[faild_cpv]
                             yield self.getLogFile(faild_cpv, log_dict)
                             faild_version_data = yield self.getVersionData(faild_cpv)
+                        self.getBuildWorkdirFiles()
+                    self.getEmergeFiles(faild_version_data)
                     self.aftersteps_list.append(steps.Trigger(
                         schedulerNames=['parse_build_log'],
                         waitForFinish=False,
@@ -785,7 +810,6 @@ class CheckEmergeLogs(BuildStep):
                             'repository_data' : self.getProperty('repository_data'),
                             'faild_cpv' : faild_cpv,
                             'step' : self.step,
-                            'emerge_info' : self.getProperty('emerge_info_output')['emerge_info'],
                             'build_workername' : self.getProperty('workername')
                         }
                     ))
@@ -920,17 +944,37 @@ class RunEmergeInfo(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         aftersteps_list = []
+        # add emerge info
         shell_commad_list = [
                     'emerge',
                     ]
         shell_commad_list.append('--info')
+        shell_commad_list.append('>')
+        shell_commad_list.append('/tmp/emerge_info.txt')
         aftersteps_list.append(
-                steps.SetPropertyFromCommand(
-                        command=shell_commad_list,
-                        strip=True,
-                        extract_fn=PersOutputOfEmergeInfo,
-                        workdir='/',
-                        timeout=None
+                steps.ShellCommand(
+                        # the list need to be joined to pipe to a file
+                        command=' '.join(shell_commad_list),
+                        workdir='/'
+                ))
+        # add package info
+        cpv = self.getProperty("cpv")
+        c = yield catpkgsplit(cpv)[0]
+        p = yield catpkgsplit(cpv)[1]
+        shell_commad_list = [
+                    'emerge',
+                    ]
+        shell_commad_list.append('-qpvO')
+        shell_commad_list.append('=' + self.getProperty('cpv'))
+        shell_commad_list.append('--usepkg-exclude')
+        shell_commad_list.append(c + '/' + p)
+        shell_commad_list.append('>>')
+        shell_commad_list.append('/tmp/emerge_info.txt')
+        aftersteps_list.append(
+                steps.ShellCommand(
+                        # the list need to be joined to pipe to a file
+                        command=' '.join(shell_commad_list),
+                        workdir='/'
                 ))
         yield self.build.addStepsAfterCurrentStep(aftersteps_list)
         return SUCCESS
@@ -954,8 +998,8 @@ class RunBuild(BuildStep):
             # trigger pars_build_log if we have any logs to check
             return SKIPPED
         aftersteps_list = []
-        aftersteps_list.append(RunEmergeInfo())
         aftersteps_list.append(RunEmerge(step='pre-build'))
+        aftersteps_list.append(RunEmergeInfo())
         aftersteps_list.append(RunEmerge(step='build'))
         aftersteps_list.append(RunEmerge(step='pre-depclean'))
         aftersteps_list.append(RunEmerge(step='preserved-libs'))

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index dc5d3a1..d6ee705 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -45,6 +45,23 @@ def PersOutputOfLogParser(rc, stdout, stderr):
         'build_summery_output' : build_summery_output
         }
 
+def PersOutputOfEmergeInfo(rc, stdout, stderr):
+    #FIXME: line for package info
+    emerge_info_output = {}
+    emerge_info_output['rc'] = rc
+    emerge_info_list = []
+    emerge_package_info = []
+    for line in stdout.split('\n'):
+        if line.startswith('['):
+            emerge_package_info.append(line)
+        else:
+            emerge_info_list.append(line)
+    emerge_info_output['emerge_info'] = emerge_info_list
+    emerge_info_output['emerge_package_info'] = emerge_package_info
+    return {
+        'emerge_info_output' : emerge_info_output
+        }
+
 class SetupPropertys(BuildStep):
     
     name = 'SetupPropertys'
@@ -387,6 +404,38 @@ class setBuildbotLog(BuildStep):
             yield log.addStdout(line + '\n')
         return SUCCESS
 
+class SetupParserEmergeInfoLog(BuildStep):
+
+    name = 'SetupParserEmergeInfoLog'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
+        command = []
+        command.append('cat')
+        command.append('emerge_info.txt')
+        self.aftersteps_list = []
+        self.aftersteps_list.append(master_steps.MasterSetPropertyFromCommand(
+                                                            name = 'RunEmergeInfoLogParser',
+                                                            haltOnFailure = True,
+                                                            flunkOnFailure = True,
+                                                            command=command,
+                                                            workdir=workdir,
+                                                            strip=False,
+                                                            extract_fn=PersOutputOfEmergeInfo
+                                                            ))
+        yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+        return SUCCESS
+
 class setEmergeInfoLog(BuildStep):
 
     name = 'setEmergeInfoLog'
@@ -406,10 +455,34 @@ class setEmergeInfoLog(BuildStep):
         log = yield self.addLog('emerge_info')
         #FIXME: add emerge info to db
         # add line for line
-        for line in self.getProperty('emerge_info'):
+        for line in self.getProperty('emerge_info_output')['emerge_info']:
             yield log.addStdout(line + '\n')
         return SUCCESS
 
+class setPackageInfoLog(BuildStep):
+
+    name = 'setPackageInfoLog'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        #setup the log
+        log = yield self.addLog('package_info')
+        #FIXME: add package info to db
+        # add line for line
+        for line in self.getProperty('emerge_info_output')['emerge_package_info']:
+            yield log.addStdout(line + '\n')
+        return SUCCESS
+
+
 class Upload(BuildStep):
 
     name = 'Upload'