[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/

[Magnus Granberg]

commit:     1fc28f8dc77b5ef0c9d2d6eac054d91b455d9048
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 18 19:16:44 2022 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 18 19:16:44 2022 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1fc28f8d

Add support for find logfiles

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/steps/builders.py | 110 ++++++++++++++++++++++++++++++-----
 1 file changed, 97 insertions(+), 13 deletions(-)

diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index d8314e4..dd2fbc2 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -181,6 +181,19 @@ def PersOutputOfElogLs(rc, stdout, stderr):
         'elog_ls_output' : elog_ls_output
         }
 
+def PersOutputOfBuildWorkdir(rc, stdout, stderr):
+    build_workdir_find_output = {}
+    build_workdir_find_output['rc'] = rc
+    build_workdir_find_list = []
+    for line in stdout.split('\n'):
+        find_line = line.replace('./', '')
+        if find_line != '':
+            build_workdir_find_list.append(find_line)
+    build_workdir_find_output['build_workdir_find'] = build_workdir_find_list
+    return {
+        'build_workdir_find_output' : build_workdir_find_output
+        }
+
 class TriggerRunBuildRequest(BuildStep):
     
     name = 'TriggerRunBuildRequest'
@@ -317,6 +330,9 @@ class SetupPropertys(BuildStep):
                                                     project_build_data['buildbot_build_id'])
         self.setProperty('project_build_data', project_build_data, 'project_build_data')
         print(self.getProperty("project_build_data"))
+        self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
+        self.setProperty('masterdest', self.masterdest, 'masterdest')
+
         return SUCCESS
 
 class UpdateRepos(BuildStep):
@@ -578,6 +594,7 @@ class CheckElogLogs(BuildStep):
 
     def addFileUploade(self, sourcefile, destfile):
         self.aftersteps_list.append(steps.FileUpload(
+            mode = 0o644,
             workersrc=sourcefile,
             masterdest=destfile
         ))
@@ -600,6 +617,51 @@ class CheckElogLogs(BuildStep):
             yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
         return SUCCESS
 
+class CheckBuildWorkDirs(BuildStep):
+
+    name = 'CheckBuildWorkdir'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.aftersteps_list = []
+
+    @defer.inlineCallbacks
+    def run(self):
+        cpv = self.getProperty('faild_cpv')
+        cpv_build_dir = yield os.path.join('/', 'var', 'tmp', 'portage', self.getProperty('cpv_build_dir'))
+        compressed_log_file = cpv.replace('/', '_') + '.' + str(self.getProperty("buildnumber")) + '.logs.tar.bz2'
+        masterdest_file = yield os.path.join(self.getProperty('masterdest'), compressed_log_file)
+        # cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work')
+        if self.getProperty('build_workdir_find_output')['build_workdir_find'] != []:
+            shell_commad_list = []
+            shell_commad_list.append('tar')
+            shell_commad_list.append('-cjpf')
+            shell_commad_list.append(compressed_log_file)
+            for find_line in sorted(self.getProperty('build_workdir_find_output')['build_workdir_find']):
+                print(find_line)
+                filename = yield os.path.join('work', find_line)
+                shell_commad_list.append(filename)
+            self.aftersteps_list.append(
+                steps.ShellCommand(
+                        name = 'Tarlogs',
+                        command = shell_commad_list,
+                        workdir = cpv_build_dir
+            ))
+            self.aftersteps_list.append(steps.FileUpload(
+                name = 'UploadFindlogs',
+                mode = 0o644,
+                workersrc = compressed_log_file,
+                masterdest = masterdest_file,
+                workdir = cpv_build_dir
+            ))
+        if self.aftersteps_list != []:
+            yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+        return SUCCESS
+
 class CheckEmergeLogs(BuildStep):
 
     name = 'CheckEmergeLogs'
@@ -640,13 +702,14 @@ class CheckEmergeLogs(BuildStep):
 
     def addFileUploade(self, sourcefile, destfile):
         self.aftersteps_list.append(steps.FileUpload(
+            mode = 0o644,
             workersrc=sourcefile,
             masterdest=destfile
         ))
 
     @defer.inlineCallbacks
     def getLogFile(self, cpv, log_dict):
-        destfile = yield os.path.join(self.masterdest, log_dict[cpv]['full_logname'])
+        destfile = yield os.path.join(self.getProperty('masterdest'), log_dict[cpv]['full_logname'])
         sourcefile = log_dict[cpv]['log_path']
         self.addFileUploade(sourcefile, destfile)
 
@@ -670,42 +733,62 @@ class CheckEmergeLogs(BuildStep):
     @defer.inlineCallbacks
     def getEmergeFiles(self, cpv):
         # get emerge info
-        destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
+        destfile = yield os.path.join(self.getProperty('masterdest'), 'emerge_info.txt')
         sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
         self.addFileUploade(sourcefile, destfile)
         # get emerge.log
-        destfile2 = yield os.path.join(self.masterdest, 'emerge.log')
+        destfile2 = yield os.path.join(self.getProperty('masterdest'), 'emerge.log')
         sourcefile2 = yield os.path.join('/', 'var', 'log', 'emerge.log')
         self.addFileUploade(sourcefile2, destfile2)
         # world file
-        destfile3 = yield os.path.join(self.masterdest, 'world')
+        destfile3 = yield os.path.join(self.getProperty('masterdest'), 'world')
         sourcefile3 = yield os.path.join('/', 'var', 'lib', 'portage', 'world')
         self.addFileUploade(sourcefile3, destfile3)
         # get elogs
         self.getElogFiles(cpv)
 
-    def getBuildWorkdirFiles(self):
+    @defer.inlineCallbacks
+    def getBuildWorkDirs(self, cpv):
         #FIXME:
         # get files from the build workdir
-        pass
+        cpv_build_dir = yield os.path.join('/', 'var', 'tmp', 'portage', cpv)
+        print(cpv_build_dir)
+        self.setProperty('cpv_build_dir', cpv_build_dir, 'cpv_build_dir')
+        cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work')
+        #FIXME: take find pattern from db or config
+        find_pattern_list = ['meson-log.txt', 'CMakeCache.txt']
+        shell_commad_list = []
+        # we have *.log as default
+        shell_commad_list.append('find')
+        shell_commad_list.append('-name')
+        shell_commad_list.append('*.log')
+        for find_pattern in find_pattern_list:
+            shell_commad_list.append('-o')
+            shell_commad_list.append('-name')
+            shell_commad_list.append(find_pattern)
+        self.aftersteps_list.append(
+                steps.SetPropertyFromCommand(
+                        name = 'FindLogs',
+                        command = shell_commad_list,
+                        strip = True,
+                        extract_fn = PersOutputOfBuildWorkdir,
+                        workdir = cpv_build_work_dir
+                ))
+        self.aftersteps_list.append(CheckBuildWorkDirs())
 
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
         project_data = self.getProperty('project_data')
         projects_emerge_options = yield self.gentooci.db.projects.getProjectEmergeOptionsByUuid(project_data['uuid'])
-        emerge_output = self.getProperty('emerge_output')
-        self.faild_cpv = emerge_output['failed']
-        self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
         shell_commad_list = [
                     'emerge',
                     '-v'
                     ]
+        emerge_output = self.getProperty('emerge_output')
+        self.faild_cpv = emerge_output['failed']
         package_dict = emerge_output['packages']
 
-        self.masterdest = yield os.path.join(self.master.basedir, 'workers', self.getProperty('workername'), str(self.getProperty("buildnumber")))
-        self.setProperty('masterdest', self.masterdest, 'masterdest')
-
         #FIXME: Prosees the logs and do stuff
         # preserved-libs
         if emerge_output['preserved_libs'] and projects_emerge_options['preserved_libs']:
@@ -849,8 +932,9 @@ class CheckEmergeLogs(BuildStep):
                             self.log_data[self.faild_cpv] = log_dict[self.faild_cpv]
                             yield self.getLogFile(self.faild_cpv, log_dict)
                             faild_version_data = yield self.getVersionData(self.faild_cpv)
+                        self.setProperty('faild_cpv', self.faild_cpv, 'faild_cpv')
                         self.getEmergeFiles(self.faild_cpv)
-                        self.getBuildWorkdirFiles()
+                        self.getBuildWorkDirs(self.faild_cpv)
                     else:
                         self.getEmergeFiles(cpv)
                     self.aftersteps_list.append(steps.Trigger(