[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/config/, buildbot_gentoo_ci/steps/

[Magnus Granberg]

commit:     cdbe9295f945bd4f40db2333e016c075e2c64784
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue May 18 09:13:59 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue May 18 09:13:59 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=cdbe9295

Add support to push log to Minio

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/config/buildfactorys.py |  2 +-
 buildbot_gentoo_ci/steps/logs.py           | 28 ++++++++++++++
 buildbot_gentoo_ci/steps/minio.py          | 60 ++++++++++++++++++++++++++++++
 3 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 417e385..d2e195c 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -171,7 +171,7 @@ def parse_build_log():
     # pers the log from pkg check
     #f.addStep(logs.ParserPkgCheckLog())
     # Upload the log to the cloud and remove the log
-    #f.addStep(logs.Upload())
+    f.addStep(logs.Upload())
     # check the sum log if we need to make a issue/bug/pr report
     # set it SUCCESS/FAILURE/WARNINGS
     f.addStep(logs.MakeIssue())

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 3c0ef8d..d69c447 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -18,6 +18,8 @@ from buildbot.process.results import FAILURE
 from buildbot.process.results import WARNINGS
 from buildbot.plugins import steps
 
+from buildbot_gentoo_ci.steps import minio
+
 class SetupPropertys(BuildStep):
     
     name = 'SetupPropertys'
@@ -277,6 +279,32 @@ class setEmergeInfoLog(BuildStep):
             yield log.addStdout(line + '\n')
         return SUCCESS
 
+class Upload(BuildStep):
+
+    name = 'Upload'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        if self.getProperty('faild_cpv'):
+            log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
+        else:
+            log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')]
+        bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs'
+        file_path = yield os.path.join(self.master.basedir, 'cpv_logs', log_cpv['full_logname'])
+        aftersteps_list = []
+        aftersteps_list.append(minio.putFileToMinio(file_path, log_cpv['full_logname'], bucket))
+        yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+        return SUCCESS
+
 class setBuildStatus(BuildStep):
 
     name = 'setBuildStatus'

diff --git a/buildbot_gentoo_ci/steps/minio.py b/buildbot_gentoo_ci/steps/minio.py
new file mode 100644
index 0000000..ac9ac3d
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/minio.py
@@ -0,0 +1,60 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from minio import Minio
+from minio.error import ResponseError, BucketAlreadyOwnedByYou, BucketAlreadyExists
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+
+#FIXME:
+# get url, user from config
+# get password from secret
+url = ''
+user = ''
+password = ''
+
+class putFileToMinio(BuildStep):
+
+    name = 'putFileToMinio'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = False
+    flunkOnFailure = True
+    warnOnWarnings = True
+
+    def __init__(self, filename, target, bucket, **kwargs):
+        self.filename = filename
+        self.bucket = bucket
+        self.target = target
+        super().__init__(**kwargs)
+
+    def getMinioConnect(self, url, user, password):
+        minioclient = Minio(
+            url,
+            access_key = user,
+            secret_key = password,
+            secure = False
+            )
+        return minioclient
+
+    @defer.inlineCallbacks
+    def pushFileToMinio(self):
+        try:
+            yield self.minio_connect.fput_object(self.bucket, self.target, self.filename)
+        except ResponseError as err:
+            print(err)
+            return FAILURE
+        return True
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        self.minio_connect = yield self.getMinioConnect(url, user, password)
+        success = yield self.pushFileToMinio()
+        return SUCCESS