[gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/

[Magnus Granberg]

commit:     b2f1c115420e05c27fb7a57cf45734cb7f4a7731
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 26 20:26:04 2021 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Apr 26 20:26:04 2021 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b2f1c115

Add more checks and move * and >>> check to last

Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>

 buildbot_gentoo_ci/steps/logs.py | 121 ++++++++++++++++++++++-----------------
 1 file changed, 70 insertions(+), 51 deletions(-)

diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 178d71e..0960b9b 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -57,39 +57,46 @@ class ParserBuildLog(BuildStep):
         self.summery_dict = {}
         self.index = 1
         self.log_search_pattern_list = []
-        self.max_text_lines = self.index -1
+        self.max_text_lines = 0
         super().__init__(**kwargs)
 
+    #FIXME: ansifilter
+    def ansiFilter(self, text):
+        return text
+
     @defer.inlineCallbacks
     def get_log_search_pattern(self):
         # get pattern from the projects
         # add that to log_search_pattern_list
         for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
-            self.log_search_pattern_list.append(project_pattern)
-        # get the default profile pattern
+            # check if the search pattern is vaild
+            try:
+                re.compile(project_pattern['search'])
+            except re.error:
+                print("Non valid regex pattern")
+                print(project_pattern)
+            else:
+                self.log_search_pattern_list.append(project_pattern)
+        # get the default project pattern
         # add if not pattern is in project ignore
+        self.project_pattern_ignore = yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('project_data')['uuid'])
         for project_pattern in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])):
-            match = True
-            for project_pattern_ignore in (yield self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('default_project_data')['uuid'])):
-                if project_pattern['search'] == project_pattern_ignore['search']:
-                    match = False
-            if match:
-                self.log_search_pattern_list.append(project_pattern)
+            if not project_pattern['search'] in self.project_pattern_ignore:
+                # check if the search pattern is vaild
+                try:
+                    re.compile(project_pattern['search'])
+                except re.error:
+                    print("Non valid regex pattern")
+                    print(project_pattern)
+                else:
+                    self.log_search_pattern_list.append(project_pattern)
 
     def search_buildlog(self, tmp_index):
         # get text line to search
-        text_line = self.logfile_text_dict[tmp_index]
+        text_line = self.ansiFilter(self.logfile_text_dict[tmp_index])
         # loop true the pattern list for match
         for search_pattern in self.log_search_pattern_list:
             search_hit = False
-            # we add all line that start with ' * ' as info
-            # we add all line that start with '>>>' but not '>>> /' as info
-            if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
-                self.summery_dict[tmp_index] = {}
-                self.summery_dict[tmp_index]['text'] = text_line
-                self.summery_dict[tmp_index]['type'] = 'info'
-                self.summery_dict[tmp_index]['status'] = 'info'
-                self.summery_dict[tmp_index]['search_pattern_id'] = 0
             if search_pattern['search_type'] == 'in':
                 if search_pattern['search'] in text_line:
                     search_hit = True
@@ -100,63 +107,74 @@ class ParserBuildLog(BuildStep):
                 if text_line.endswith(search_pattern['search']):
                     search_hit = True
             if search_pattern['search_type'] == 'search':
-                if search_pattern['search'] in text_line:
+                if re.search(search_pattern['search'], text_line):
                     search_hit = True
+            # add the line if the pattern match
             if search_hit:
                 print(text_line)
-                print(search_pattern['search'])
+                print(search_pattern)
+                print(tmp_index)
                 self.summery_dict[tmp_index] = {}
                 self.summery_dict[tmp_index]['text'] = text_line
                 self.summery_dict[tmp_index]['type'] = search_pattern['type']
                 self.summery_dict[tmp_index]['status'] = search_pattern['status']
                 self.summery_dict[tmp_index]['search_pattern_id'] = search_pattern['id']
                 # add upper text lines if requested
-                # max 10
-            if search_pattern['start'] != 0 and search_hit:
-                i = tmp_index
-                i_start = i - search_pattern['start']
-                match = True
-                while match:
-                    i = i - 1
-                    if i < 0 or i < i_start:
-                        match = False
-                    else:
-                        self.summery_dict[i] = {}
-                        self.summery_dict[i]['text'] = self.logfile_text_dict[i]
-                        self.summery_dict[i]['type'] = search_pattern['type']
-                        self.summery_dict[i]['status'] = 'info'
+                # max 5
+                if search_pattern['start'] != 0:
+                    i = tmp_index - search_pattern['start'] - 1
+                    match = True
+                    while match:
+                        i = i + 1
+                        if i < (tmp_index - 9) or i == tmp_index:
+                            match = False
+                        else:
+                            if not i in self.summery_dict:
+                                self.summery_dict[i] = {}
+                                self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i])
+                                self.summery_dict[i]['type'] = 'info'
+                                self.summery_dict[i]['status'] = 'info'
                 # add lower text lines if requested
-                # max 10
-            if search_pattern['end'] != 0 and search_hit:
-                i = tmp_index
-                i_end = i + search_pattern['end']
-                match = True
-                while match:
-                    i = i + 1
-                    if i > self.max_text_lines or i > i_end:
-                        match = False
-                    else:
-                        self.summery_dict[i] = {}
-                        self.summery_dict[i]['text'] = self.logfile_text_dict[i]
-                        self.summery_dict[i]['type'] = search_pattern['type']
-                        self.summery_dict[i]['status'] = 'info'
+                # max 5
+                if search_pattern['end'] != 0:
+                    i = tmp_index
+                    end = tmp_index + search_pattern['end']
+                    match = True
+                    while match:
+                        i = i + 1
+                        if i > self.max_text_lines or i > end:
+                            match = False
+                        else:
+                            if not i in self.summery_dict:
+                                self.summery_dict[i] = {}
+                                self.summery_dict[i]['text'] = self.ansiFilter(self.logfile_text_dict[i])
+                                self.summery_dict[i]['type'] = 'info'
+                                self.summery_dict[i]['status'] = 'info'
+            else:
+                # we add all line that start with ' * ' as info
+                # we add all line that start with '>>>' but not '>>> /' as info
+                if text_line.startswith(' * ') or (text_line.startswith('>>>') and not text_line.startswith('>>> /')):
+                    if not tmp_index in self.summery_dict:
+                        self.summery_dict[tmp_index] = {}
+                        self.summery_dict[tmp_index]['text'] = text_line
+                        self.summery_dict[tmp_index]['type'] = 'info'
+                        self.summery_dict[tmp_index]['status'] = 'info'
 
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
-        #FIXME:
-        # get the log parser pattern from db
         yield self.get_log_search_pattern()
         # open the log file
         # read it to a buffer
         # make a dict of the buffer
-        # maby use mulitiprocces to speed up the search
+        # maybe use mulitiprocces to speed up the search
         print(self.getProperty('log_build_data'))
         if self.getProperty('faild_cpv'):
             log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
         else:
             log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')]
         file_path = yield os.path.join(self.master.basedir, 'cpv_logs', log_cpv['full_logname'])
+        #FIXME: decode it to utf-8
         with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f:
             for text_line in f:
                 self.logfile_text_dict[self.index] = text_line.strip('\n')
@@ -168,6 +186,7 @@ class ParserBuildLog(BuildStep):
                 if self.index >= 20:
                     del self.logfile_text_dict[self.index - 19]
                 self.index = self.index + 1
+                self.max_text_lines = self.index
             f.close()
         # check last 10 lines in logfile_text_dict
         yield self.search_buildlog(self.index - 10)