[gentoo-commits] proj/qa-scripts:master commit in: pkgcheck2html/

[Michał Górny]

commit:     cce1dfea834ae526ebbe8506fdad19cc03287730
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sat Dec  3 20:56:23 2016 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sun Dec  4 08:49:47 2016 +0000
URL:        https://gitweb.gentoo.org/proj/qa-scripts.git/commit/?id=cce1dfea

Add pkgcheck XML output to HTML formatter scripts

 pkgcheck2html/jinja2htmlcompress.py   | 150 +++++++++++++++++++++++++++
 pkgcheck2html/output.css              | 184 ++++++++++++++++++++++++++++++++++
 pkgcheck2html/output.html.jinja       |  67 +++++++++++++
 pkgcheck2html/pkgcheck2html.conf.json |  26 +++++
 pkgcheck2html/pkgcheck2html.py        | 139 +++++++++++++++++++++++++
 5 files changed, 566 insertions(+)

diff --git a/pkgcheck2html/jinja2htmlcompress.py b/pkgcheck2html/jinja2htmlcompress.py
new file mode 100644
index 0000000..5dfb211
--- /dev/null
+++ b/pkgcheck2html/jinja2htmlcompress.py
@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2htmlcompress
+    ~~~~~~~~~~~~~~~~~~
+
+    A Jinja2 extension that eliminates useless whitespace at template
+    compilation time without extra overhead.
+
+    :copyright: (c) 2011 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
+import re
+from jinja2.ext import Extension
+from jinja2.lexer import Token, describe_token
+from jinja2 import TemplateSyntaxError
+
+
+_tag_re = re.compile(r'(?:<(/?)([a-zA-Z0-9_-]+)\s*|(>\s*))(?s)')
+_ws_normalize_re = re.compile(r'[ \t\r\n]+')
+
+
+class StreamProcessContext(object):
+
+    def __init__(self, stream):
+        self.stream = stream
+        self.token = None
+        self.stack = []
+
+    def fail(self, message):
+        raise TemplateSyntaxError(message, self.token.lineno,
+                                  self.stream.name, self.stream.filename)
+
+
+def _make_dict_from_listing(listing):
+    rv = {}
+    for keys, value in listing:
+        for key in keys:
+            rv[key] = value
+    return rv
+
+
+class HTMLCompress(Extension):
+    isolated_elements = set(['script', 'style', 'noscript', 'textarea'])
+    void_elements = set(['br', 'img', 'area', 'hr', 'param', 'input',
+                         'embed', 'col'])
+    block_elements = set(['div', 'p', 'form', 'ul', 'ol', 'li', 'table', 'tr',
+                          'tbody', 'thead', 'tfoot', 'tr', 'td', 'th', 'dl',
+                          'dt', 'dd', 'blockquote', 'h1', 'h2', 'h3', 'h4',
+                          'h5', 'h6', 'pre'])
+    breaking_rules = _make_dict_from_listing([
+        (['p'], set(['#block'])),
+        (['li'], set(['li'])),
+        (['td', 'th'], set(['td', 'th', 'tr', 'tbody', 'thead', 'tfoot'])),
+        (['tr'], set(['tr', 'tbody', 'thead', 'tfoot'])),
+        (['thead', 'tbody', 'tfoot'], set(['thead', 'tbody', 'tfoot'])),
+        (['dd', 'dt'], set(['dl', 'dt', 'dd']))
+    ])
+
+    def is_isolated(self, stack):
+        for tag in reversed(stack):
+            if tag in self.isolated_elements:
+                return True
+        return False
+
+    def is_breaking(self, tag, other_tag):
+        breaking = self.breaking_rules.get(other_tag)
+        return breaking and (tag in breaking or
+            ('#block' in breaking and tag in self.block_elements))
+
+    def enter_tag(self, tag, ctx):
+        while ctx.stack and self.is_breaking(tag, ctx.stack[-1]):
+            self.leave_tag(ctx.stack[-1], ctx)
+        if tag not in self.void_elements:
+            ctx.stack.append(tag)
+
+    def leave_tag(self, tag, ctx):
+        if not ctx.stack:
+            ctx.fail('Tried to leave "%s" but something closed '
+                     'it already' % tag)
+        if tag == ctx.stack[-1]:
+            ctx.stack.pop()
+            return
+        for idx, other_tag in enumerate(reversed(ctx.stack)):
+            if other_tag == tag:
+                for num in xrange(idx + 1):
+                    ctx.stack.pop()
+            elif not self.breaking_rules.get(other_tag):
+                break
+
+    def normalize(self, ctx):
+        pos = 0
+        buffer = []
+        def write_data(value):
+            if not self.is_isolated(ctx.stack):
+                value = _ws_normalize_re.sub(' ', value.strip())
+            buffer.append(value)
+
+        for match in _tag_re.finditer(ctx.token.value):
+            closes, tag, sole = match.groups()
+            preamble = ctx.token.value[pos:match.start()]
+            write_data(preamble)
+            if sole:
+                write_data(sole)
+            else:
+                buffer.append(match.group())
+                (closes and self.leave_tag or self.enter_tag)(tag, ctx)
+            pos = match.end()
+
+        write_data(ctx.token.value[pos:])
+        return u''.join(buffer)
+
+    def filter_stream(self, stream):
+        ctx = StreamProcessContext(stream)
+        for token in stream:
+            if token.type != 'data':
+                yield token
+                continue
+            ctx.token = token
+            value = self.normalize(ctx)
+            yield Token(token.lineno, 'data', value)
+
+
+class SelectiveHTMLCompress(HTMLCompress):
+
+    def filter_stream(self, stream):
+        ctx = StreamProcessContext(stream)
+        strip_depth = 0
+        while 1:
+            if stream.current.type == 'block_begin':
+                if stream.look().test('name:strip') or \
+                   stream.look().test('name:endstrip'):
+                    stream.skip()
+                    if stream.current.value == 'strip':
+                        strip_depth += 1
+                    else:
+                        strip_depth -= 1
+                        if strip_depth < 0:
+                            ctx.fail('Unexpected tag endstrip')
+                    stream.skip()
+                    if stream.current.type != 'block_end':
+                        ctx.fail('expected end of block, got %s' %
+                                 describe_token(stream.current))
+                    stream.skip()
+            if strip_depth > 0 and stream.current.type == 'data':
+                ctx.token = stream.current
+                value = self.normalize(ctx)
+                yield Token(stream.current.lineno, 'data', value)
+            else:
+                yield stream.current
+            stream.next()

diff --git a/pkgcheck2html/output.css b/pkgcheck2html/output.css
new file mode 100644
index 0000000..6888102
--- /dev/null
+++ b/pkgcheck2html/output.css
@@ -0,0 +1,184 @@
+/* (c) 2016 Michał Górny, Patrice Clement */
+/* 2-clause BSD license */
+
+*
+{
+	box-sizing: border-box;
+}
+
+body
+{
+	margin: 0;
+	background-color: #463C65;
+	font-family: sans-serif;
+	font-size: 14px;
+}
+
+address
+{
+	color: white;
+	text-align: center;
+	margin: 1em;
+}
+
+.nav
+{
+	width: 20%;
+	position: absolute;
+	top: 0;
+}
+
+.nav ul
+{
+	list-style: none;
+	margin: 0;
+	padding: 1%;
+}
+
+.nav li
+{
+	padding: 0 1em;
+	border-radius: 4px;
+	margin-bottom: .3em;
+	background-color: #62548F;
+}
+
+.nav li a
+{
+	display: block;
+	width: 100%;
+}
+
+.nav h2
+{
+	color: white;
+	text-align: center;
+	font-size: 300%;
+	font-weight: bold;
+	text-transform: uppercase;
+	font-family: serif;
+}
+
+ul.nav li.header
+{
+	background-color: #463C65;
+}
+
+.content, h1
+{
+	padding: 2%;
+	margin: 0 0 0 20%;
+	background-color: #DDDAEC;
+}
+
+h1 {
+	font-family: serif;
+	color: #23457F;
+	font-size: 400%;
+	line-height: 2em;
+	font-weight: bold;
+	text-transform: uppercase;
+	text-align: center;
+	letter-spacing: .15em;
+}
+
+th
+{
+	text-align: left;
+	padding: 1em 0;
+	color: #23457F;
+}
+
+th.h2
+{
+	font-size: 120%;
+}
+
+th.h3
+{
+	padding-left: 1em;
+	font-size: 110%;
+}
+
+th:target
+{
+	background-color: #dfd;
+}
+
+th small
+{
+	padding-left: .5em;
+	visibility: hidden;
+}
+
+th:hover small
+{
+	visibility: visible;
+}
+
+td
+{
+	background-color: white;
+	line-height: 2em;
+	font-size: 120%;
+	padding-left: .5em;
+	white-space: pre-wrap;
+}
+
+td:hover
+{
+	background-color: #eee;
+}
+
+tr.err td
+{
+	background-color: #7E0202;
+	color: white;
+}
+
+tr.err td:hover
+{
+	background-color: #DA0404;
+}
+
+tr.warn td
+{
+	background-color: orange;
+}
+
+tr.warn td:hover
+{
+	background-color: #FFBB3E;
+}
+
+.nav a
+{
+	font-size: 150%;
+	line-height: 1.5em;
+	text-decoration: none;
+	white-space: pre;
+	overflow: hidden;
+	text-overflow: ellipsis;
+}
+
+.warn a
+{
+	color: orange;
+}
+
+.err a
+{
+	color: #F06F74;
+}
+
+.nav li:hover
+{
+	min-width: 100%;
+	width: -moz-max-content;
+	width: max-content;
+}
+
+.nav a:hover
+{
+	color: white;
+}

diff --git a/pkgcheck2html/output.html.jinja b/pkgcheck2html/output.html.jinja
new file mode 100644
index 0000000..2e44619
--- /dev/null
+++ b/pkgcheck2html/output.html.jinja
@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="utf-8"/>
+		<title>Gentoo CI - QA check results</title>
+		<link rel="stylesheet" type="text/css" href="output.css" />
+	</head>
+
+	<body>
+		<h1>QA check results</h1>
+
+		{% if errors or warnings %}
+			<div class="nav">
+				<h2>issues</h2>
+
+				<ul>
+					{% for g in errors %}
+						<li class="err"><a href="#{{ g|join('/') }}">{{ g|join('/') }}</a></li>
+					{% endfor %}
+					{% for g in warnings %}
+						<li class="warn"><a href="#{{ g|join('/') }}">{{ g|join('/') }}</a></li>
+					{% endfor %}
+				</ul>
+			</div>
+		{% endif %}
+
+		<div class="content">
+			<table>
+				{% for g, r in results %}
+					{% set h2_id = g[0] if g else "global" %}
+					<tr><th colspan="3" class="h2" id="{{ h2_id }}">
+						{{ g[0] if g else "Global-scope results" }}
+						<small><a href="#{{ h2_id }}">¶</a></small>
+					</th></tr>
+
+					{% for g, r in r %}
+						{% if g[0] %}
+							{% set h3_id = g[0] + "/" + g[1] if g[1] else "_cat" %}
+							<tr><th colspan="3" class="h3" id="{{ h3_id }}">
+								{{ g[1] if g[1] else "Category results" }}
+								<small><a href="#{{ h3_id }}">¶</a></small>
+							</th></tr>
+						{% endif %}
+
+						{% for g, r in r %}
+							{% for rx in r %}
+								{% set class_str = "" %}
+								{% if rx.css_class %}
+									{% set class_str = ' class="' + rx.css_class + '"' %}
+								{% endif %}
+								<tr{{ class_str }}>
+									<td>{{ g[2] if loop.index == 1 else "" }}</td>
+									<td>{{ rx.class }}</td>
+									<td>{{ rx.msg|escape }}</td>
+								</tr>
+							{% endfor %}
+						{% endfor %}
+					{% endfor %}
+				{% endfor %}
+			</table>
+		</div>
+
+		<address>Generated based on results from: {{ ts.strftime("%F %T UTC") }}</address>
+	</body>
+</html>
+
+<!-- vim:se ft=jinja : -->

diff --git a/pkgcheck2html/pkgcheck2html.conf.json b/pkgcheck2html/pkgcheck2html.conf.json
new file mode 100644
index 0000000..f9c597e
--- /dev/null
+++ b/pkgcheck2html/pkgcheck2html.conf.json
@@ -0,0 +1,26 @@
+{
+    "CatMetadataXmlInvalidPkgRef": "err",
+    "VisibleVcsPkg": "err",
+    "MissingUri": "warn",
+    "CatBadlyFormedXml": "err",
+    "Glep31Violation": "err",
+    "PkgBadlyFormedXml": "err",
+    "CatInvalidXml": "err",
+    "CatMetadataXmlInvalidCatRef": "err",
+    "PkgMetadataXmlInvalidCatRef": "err",
+    "ConflictingChksums": "err",
+    "MissingChksum": "warn",
+    "MissingManifest": "err",
+    "CrappyDescription": "warn",
+    "PkgMetadataXmlInvalidPkgRef": "err",
+    "PkgMetadataXmlInvalidProjectError": "err",
+    "PkgInvalidXml": "err",
+    "NonsolvableDeps": "err",
+    "UnusedLocalFlags": "err",
+    "MetadataLoadError": "err",
+    "UnknownManifest": "err",
+    "NoFinalNewline": "err",
+    "UnstatedIUSE": "err",
+    "MetadataError": "err",
+    "WrongIndentFound": "err"
+}

diff --git a/pkgcheck2html/pkgcheck2html.py b/pkgcheck2html/pkgcheck2html.py
new file mode 100755
index 0000000..466d8c1
--- /dev/null
+++ b/pkgcheck2html/pkgcheck2html.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# vim:se fileencoding=utf8 :
+# (c) 2015-2016 Michał Górny
+# 2-clause BSD license
+
+import argparse
+import datetime
+import io
+import json
+import os
+import os.path
+import sys
+import xml.etree.ElementTree
+
+import jinja2
+
+
+class Result(object):
+    def __init__(self, el, class_mapping):
+        self._el = el
+        self._class_mapping = class_mapping
+
+    def __getattr__(self, key):
+        return self._el.findtext(key) or ''
+
+    @property
+    def css_class(self):
+        return self._class_mapping.get(getattr(self, 'class'), '')
+
+
+def result_sort_key(r):
+    return (r.category, r.package, r.version, getattr(r, 'class'), r.msg)
+
+
+def get_results(input_paths, class_mapping):
+    for input_path in input_paths:
+        checks = xml.etree.ElementTree.parse(input_path).getroot()
+        for r in checks:
+            yield Result(r, class_mapping)
+
+
+def split_result_group(it):
+    for r in it:
+        if not r.category:
+            yield ((), r)
+        elif not r.package:
+            yield ((r.category,), r)
+        elif not r.version:
+            yield ((r.category, r.package), r)
+        else:
+            yield ((r.category, r.package, r.version), r)
+
+
+def group_results(it, level = 3):
+    prev_group = ()
+    prev_l = []
+
+    for g, r in split_result_group(it):
+        if g[:level] != prev_group:
+            if prev_l:
+                yield (prev_group, prev_l)
+            prev_group = g[:level]
+            prev_l = []
+        prev_l.append(r)
+    yield (prev_group, prev_l)
+
+
+def deep_group(it, level = 1):
+    for g, r in group_results(it, level):
+        if level > 3:
+            for x in r:
+                yield x
+        else:
+            yield (g, deep_group(r, level+1))
+
+
+def find_of_class(it, cls, level = 2):
+    for g, r in group_results(it, level):
+        for x in r:
+            if x.css_class == cls:
+                yield g
+                break
+
+
+def get_result_timestamp(paths):
+    for p in paths:
+        st = os.stat(p)
+        return datetime.datetime.utcfromtimestamp(st.st_mtime)
+
+
+def main(*args):
+    p = argparse.ArgumentParser()
+    p.add_argument('-o', '--output', default='-',
+            help='Output HTML file ("-" for stdout)')
+    p.add_argument('-t', '--timestamp', default=None,
+            help='Timestamp for results (git ISO8601-like UTC)')
+    p.add_argument('files', nargs='+',
+            help='Input XML files')
+    args = p.parse_args(args)
+
+    conf_path = os.path.join(os.path.dirname(__file__), 'pkgcheck2html.conf.json')
+    with io.open(conf_path, 'r', encoding='utf8') as f:
+        class_mapping = json.load(f)
+
+    jenv = jinja2.Environment(
+            loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
+            extensions=['jinja2htmlcompress.HTMLCompress'])
+    t = jenv.get_template('output.html.jinja')
+
+    results = sorted(get_results(args.files, class_mapping), key=result_sort_key)
+
+    types = {}
+    for r in results:
+        cl = getattr(r, 'class')
+        if cl not in types:
+            types[cl] = 0
+        types[cl] += 1
+
+    if args.timestamp is not None:
+        ts = datetime.datetime.strptime(args.timestamp, '%Y-%m-%d %H:%M:%S')
+    else:
+        ts = get_result_timestamp(args.files)
+
+    out = t.render(
+        results = deep_group(results),
+        warnings = list(find_of_class(results, 'warn')),
+        errors = list(find_of_class(results, 'err')),
+        ts = ts,
+    )
+
+    if args.output == '-':
+        sys.stdout.write(out)
+    else:
+        with io.open(args.output, 'w', encoding='utf8') as f:
+            f.write(out)
+
+
+if __name__ == '__main__':
+    sys.exit(main(*sys.argv[1:]))