[gentoo-commits] proj/pkgcore/pkgdev:main commit in: data/share/bash-completion/completions/, src/pkgdev/scripts/, tests/scripts/

[Arthur Zamarin]

commit:     6819d87b2e1a65aa57f959f07b8d226578dda634
Author:     Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  8 20:17:13 2023 +0000
Commit:     Arthur Zamarin <arthurzam <AT> gentoo <DOT> org>
CommitDate: Wed Mar  1 19:18:47 2023 +0000
URL:        https://gitweb.gentoo.org/proj/pkgcore/pkgdev.git/commit/?id=6819d87b

pkgdev bugs: new tool for filing stable bugs

This new tool isn't complete, and any usage should be manually monitored
for failures or incorrect results. This tool will be improved in the
future, but for now it's a good start.

Resolves: https://github.com/pkgcore/pkgdev/issues/113
Signed-off-by: Arthur Zamarin <arthurzam <AT> gentoo.org>

 data/share/bash-completion/completions/pkgdev |  22 ++
 src/pkgdev/scripts/pkgdev_bugs.py             | 415 ++++++++++++++++++++++++++
 tests/scripts/test_pkgdev_bugs.py             | 104 +++++++
 3 files changed, 541 insertions(+)

diff --git a/data/share/bash-completion/completions/pkgdev b/data/share/bash-completion/completions/pkgdev
index 37e7a4b..223a7d9 100644
--- a/data/share/bash-completion/completions/pkgdev
+++ b/data/share/bash-completion/completions/pkgdev
@@ -7,6 +7,7 @@ _pkgdev() {
     _init_completion || return
 
     local subcommands="
+        bugs
         commit
         manifest
         mask
@@ -229,6 +230,27 @@ _pkgdev() {
                     ;;
             esac
             ;;
+        bugs)
+            subcmd_options="
+                --api-key
+                --auto-cc-arches
+                --dot
+                -s --stablereq
+                -k --keywording
+            "
+
+            case "${prev}" in
+                --api-key | --auto-cc-arches)
+                    COMPREPLY=()
+                    ;;
+                --dot)
+                    COMPREPLY=($(compgen -f -- "${cur}"))
+                    ;;
+                *)
+                    COMPREPLY+=($(compgen -W "${subcmd_options}" -- "${cur}"))
+                    ;;
+            esac
+            ;;
     esac
 }
 complete -F _pkgdev pkgdev

diff --git a/src/pkgdev/scripts/pkgdev_bugs.py b/src/pkgdev/scripts/pkgdev_bugs.py
new file mode 100644
index 0000000..9e8938f
--- /dev/null
+++ b/src/pkgdev/scripts/pkgdev_bugs.py
@@ -0,0 +1,415 @@
+"""Automatic bugs filler"""
+
+import json
+import urllib.request as urllib
+from collections import defaultdict
+from functools import partial
+from itertools import chain
+
+from pkgcheck import const as pkgcheck_const
+from pkgcheck.addons import ArchesAddon, init_addon
+from pkgcheck.addons.profiles import ProfileAddon
+from pkgcheck.checks import visibility
+from pkgcheck.scripts import argparse_actions
+from pkgcore.ebuild.atom import atom
+from pkgcore.ebuild.ebuild_src import package
+from pkgcore.ebuild.misc import sort_keywords
+from pkgcore.repository import multiplex
+from pkgcore.restrictions import boolean, packages, values
+from pkgcore.test.misc import FakePkg
+from pkgcore.util import commandline
+from snakeoil.cli import arghparse
+from snakeoil.cli.input import userquery
+from snakeoil.formatters import Formatter
+
+from ..cli import ArgumentParser
+from .argparsers import _determine_cwd_repo, cwd_repo_argparser
+
+bugs = ArgumentParser(
+    prog="pkgdev bugs", description=__doc__, verbose=False, quiet=False,
+    parents=(cwd_repo_argparser, )
+)
+bugs.add_argument(
+    "--api-key",
+    metavar="KEY",
+    help="Bugzilla API key",
+    docs="""
+        The Bugzilla API key to use for authentication. Used mainly to overcome
+        rate limiting done by bugzilla server. This tool doesn't perform any
+        bug editing, just fetching info for the bug.
+    """,
+)
+bugs.add_argument(
+    "targets", metavar="target", nargs="+",
+    action=commandline.StoreTarget,
+    help="extended atom matching of packages",
+)
+bugs.add_argument(
+    "--dot",
+    help="path file where to save the graph in dot format",
+)
+bugs.add_argument(
+    "--auto-cc-arches",
+    action=arghparse.CommaSeparatedNegationsAppend,
+    default=([], []),
+    help="automatically add CC-ARCHES for the listed email addresses",
+    docs="""
+        Comma separated list of email addresses, for which automatically add
+        CC-ARCHES if one of the maintainers matches the email address. If the
+        package is maintainer-needed, always add CC-ARCHES.
+    """,
+)
+
+bugs.add_argument(
+    "--cache",
+    action=argparse_actions.CacheNegations,
+    help=arghparse.SUPPRESS,
+)
+bugs.add_argument(
+    "--cache-dir",
+    type=arghparse.create_dir,
+    default=pkgcheck_const.USER_CACHE_DIR,
+    help=arghparse.SUPPRESS,
+)
+bugs_state = bugs.add_mutually_exclusive_group()
+bugs_state.add_argument(
+    "-s",
+    "--stablereq",
+    dest="keywording",
+    default=None,
+    action="store_false",
+    help="File stable request bugs",
+)
+bugs_state.add_argument(
+    "-k",
+    "--keywording",
+    dest="keywording",
+    default=None,
+    action="store_true",
+    help="File rekeywording bugs",
+)
+
+ArchesAddon.mangle_argparser(bugs)
+ProfileAddon.mangle_argparser(bugs)
+
+
+@××××.bind_delayed_default(1500, "target_repo")
+def _validate_args(namespace, attr):
+    _determine_cwd_repo(bugs, namespace)
+    setattr(namespace, attr, namespace.repo)
+    setattr(namespace, "verbosity", 1)
+    setattr(namespace, "search_repo", multiplex.tree(*namespace.repo.trees))
+    setattr(namespace, "query_caching_freq", "package")
+
+
+@××××.bind_final_check
+def _validate_args(parser, namespace):
+    if namespace.keywording:
+        parser.error("keywording is not implemented yet, sorry")
+
+def _get_suggested_keywords(repo, pkg: package):
+    match_keywords = {
+        x
+        for pkgver in repo.match(pkg.unversioned_atom)
+        for x in pkgver.keywords
+        if x[0] not in '-~'
+    }
+
+    # limit stablereq to whatever is ~arch right now
+    match_keywords.intersection_update(x.lstrip('~') for x in pkg.keywords if x[0] == '~')
+
+    return frozenset({x for x in match_keywords if '-' not in x})
+
+
+class GraphNode:
+    __slots__ = ("pkgs", "edges", "bugno")
+
+    def __init__(self, pkgs: tuple[tuple[package, set[str]], ...], bugno=None):
+        self.pkgs = pkgs
+        self.edges: set[GraphNode] = set()
+        self.bugno = bugno
+
+    def __eq__(self, __o: object):
+        return self is __o
+
+    def __hash__(self):
+        return hash(id(self))
+
+    def __str__(self):
+        return ", ".join(str(pkg.versioned_atom) for pkg, _ in self.pkgs)
+
+    def __repr__(self):
+        return str(self)
+
+    def lines(self):
+        for pkg, keywords in self.pkgs:
+            yield f"{pkg.versioned_atom} {' '.join(sort_keywords(keywords))}"
+
+    @property
+    def dot_edge(self):
+        return f'"{self.pkgs[0][0].versioned_atom}"'
+
+    def cleanup_keywords(self, repo):
+        previous = frozenset()
+        for pkg, keywords in self.pkgs:
+            if keywords == previous:
+                keywords.clear()
+                keywords.add("^")
+            else:
+                previous = frozenset(keywords)
+
+        for pkg, keywords in self.pkgs:
+            suggested = _get_suggested_keywords(repo, pkg)
+            if keywords == set(suggested):
+                keywords.clear()
+                keywords.add("*")
+
+    def file_bug(self, api_key: str, auto_cc_arches: frozenset[str], observer=None) -> int:
+        if self.bugno is not None:
+            return self.bugno
+        for dep in self.edges:
+            if dep.bugno is None:
+                dep.file_bug(api_key, auto_cc_arches, observer)
+        maintainers = dict.fromkeys(
+            maintainer.email
+            for pkg, _ in self.pkgs
+            for maintainer in pkg.maintainers
+        )
+        if not maintainers or "*" in auto_cc_arches or auto_cc_arches.intersection(maintainers):
+            keywords = ["CC-ARCHES"]
+        else:
+            keywords = []
+        maintainers = tuple(maintainers) or ("maintainer-needed@g.o", )
+
+        request_data = dict(
+            Bugzilla_api_key=api_key,
+            product="Gentoo Linux",
+            component="Stabilization",
+            severity="enhancement",
+            version="unspecified",
+            summary=f"{', '.join(pkg.versioned_atom.cpvstr for pkg, _ in self.pkgs)}: stablereq",
+            description="Please stabilize",
+            keywords=keywords,
+            cf_stabilisation_atoms="\n".join(self.lines()),
+            assigned_to=maintainers[0],
+            cc=maintainers[1:],
+            depends_on=list({dep.bugno for dep in self.edges}),
+        )
+        request = urllib.Request(
+            url='https://bugs.gentoo.org/rest/bug',
+            data=json.dumps(request_data).encode('utf-8'),
+            method='POST',
+            headers={
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+            },
+        )
+        with urllib.urlopen(request, timeout=30) as response:
+            reply = json.loads(response.read().decode('utf-8'))
+        self.bugno = int(reply['id'])
+        if observer is not None:
+            observer(self)
+        return self.bugno
+
+class DependencyGraph:
+    def __init__(self, out: Formatter, err: Formatter, options):
+        self.out = out
+        self.err = err
+        self.options = options
+        self.profile_addon: ProfileAddon = init_addon(ProfileAddon, options)
+
+        self.nodes: set[GraphNode] = set()
+        self.starting_nodes: set[GraphNode] = set()
+
+    def mk_fake_pkg(self, pkg: package, keywords: set[str]):
+        return FakePkg(
+            cpv=pkg.cpvstr,
+            eapi=str(pkg.eapi),
+            iuse=pkg.iuse,
+            repo=pkg.repo,
+            keywords=tuple(keywords),
+            data={
+                attr: str(getattr(pkg, attr.lower()))
+                for attr in pkg.eapi.dep_keys
+            },
+        )
+
+    def find_best_match(self, restrict, pkgset: list[package]) -> package:
+        restrict = boolean.AndRestriction(restrict, packages.PackageRestriction(
+            "properties", values.ContainmentMatch("live", negate=True)
+        ))
+        # prefer using already selected packages in graph
+        all_pkgs = (pkg for node in self.nodes for pkg, _ in node.pkgs)
+        if intersect := tuple(filter(restrict.match, all_pkgs)):
+            return max(intersect)
+        matches = sorted(filter(restrict.match, pkgset), reverse=True)
+        for match in matches:
+            if not all(keyword.startswith("~") for keyword in match.keywords):
+                return match
+        return matches[0]
+
+    def _find_dependencies(self, pkg: package, keywords: set[str]):
+        check = visibility.VisibilityCheck(self.options, profile_addon=self.profile_addon)
+
+        issues: dict[str, dict[str, set[atom]]] = defaultdict(partial(defaultdict, set))
+        for res in check.feed(self.mk_fake_pkg(pkg, keywords)):
+            if isinstance(res, visibility.NonsolvableDeps):
+                for dep in res.deps:
+                    dep = atom(dep).no_usedeps
+                    issues[dep.key][res.keyword.lstrip('~')].add(dep)
+
+        for pkgname, problems in issues.items():
+            pkgset: list[package] = self.options.repo.match(atom(pkgname))
+            try:
+                combined = boolean.AndRestriction(*set().union(*problems.values()))
+                match = self.find_best_match(combined, pkgset)
+                yield match, set(problems.keys())
+            except ValueError:
+                results: dict[package, set[str]] = defaultdict(set)
+                for keyword, deps in problems.items():
+                    match = self.find_best_match(deps, pkgset)
+                    results[match].add(keyword)
+                yield from results.items()
+
+    def build_full_graph(self, targets: list[package]):
+        check_nodes = [(pkg, set()) for pkg in targets]
+
+        vertices: dict[package, GraphNode] = {}
+        edges = []
+        while len(check_nodes):
+            pkg, keywords = check_nodes.pop(0)
+            if pkg in vertices:
+                vertices[pkg].pkgs[0][1].update(keywords)
+                continue
+
+            keywords.update(_get_suggested_keywords(self.options.repo, pkg))
+            assert keywords
+            self.nodes.add(new_node := GraphNode(((pkg, keywords), )))
+            vertices[pkg] = new_node
+            self.out.write(f"Checking {pkg.versioned_atom} on {' '.join(sort_keywords(keywords))!r}")
+            self.out.flush()
+
+            for dep, keywords in self._find_dependencies(pkg, keywords):
+                edges.append((pkg, dep))
+                check_nodes.append((dep, keywords))
+
+        for src, dst in edges:
+            vertices[src].edges.add(vertices[dst])
+        self.starting_nodes = {vertices[starting_node] for starting_node in targets}
+
+    def output_dot(self, dot_file):
+        with open(dot_file, "w") as dot:
+            dot.write("digraph {\n")
+            dot.write("\trankdir=LR;\n")
+            for node in self.nodes:
+                node_text = "\\n".join(node.lines())
+                dot.write(f'\t{node.dot_edge}[label="{node_text}"];\n')
+                for other in node.edges:
+                    dot.write(f"\t{node.dot_edge} -> {other.dot_edge};\n")
+            dot.write("}\n")
+            dot.close()
+
+    def merge_nodes(self, nodes: tuple[GraphNode, ...]) -> GraphNode:
+        self.nodes.difference_update(nodes)
+        self.starting_nodes.difference_update(nodes)
+        new_node = GraphNode(list(chain.from_iterable(n.pkgs for n in nodes)))
+
+        for node in nodes:
+            new_node.edges.update(node.edges.difference(nodes))
+
+        for node in self.nodes:
+            if node.edges.intersection(nodes):
+                node.edges.difference_update(nodes)
+                node.edges.add(new_node)
+
+        self.nodes.add(new_node)
+        return new_node
+
+    @staticmethod
+    def _find_cycles(nodes: tuple[GraphNode, ...], stack: list[GraphNode]) -> tuple[GraphNode, ...]:
+        node = stack[-1]
+        for edge in node.edges:
+            if edge in stack:
+                return tuple(stack[stack.index(edge):])
+            stack.append(edge)
+            if cycle := DependencyGraph._find_cycles(nodes, stack):
+                return cycle
+            stack.pop()
+        return ()
+
+    def merge_cycles(self):
+        new_starts = set()
+        while self.starting_nodes:
+            starting_node = self.starting_nodes.pop()
+            assert starting_node in self.nodes
+            while cycle := self._find_cycles(tuple(self.nodes), [starting_node]):
+                print("Found cycle:", " -> ".join(str(n) for n in cycle))
+                new_node = self.merge_nodes(cycle)
+                if starting_node not in self.nodes:
+                    starting_node = new_node
+            new_starts.add(starting_node)
+        self.starting_nodes.update(new_starts)
+
+    def merge_new_keywords_children(self):
+        repo = self.options.search_repo
+        found_someone = True
+        while found_someone:
+            reverse_edges: dict[GraphNode, set[GraphNode]] = defaultdict(set)
+            for node in self.nodes:
+                for dep in node.edges:
+                    reverse_edges[dep].add(node)
+            found_someone = False
+            for node, origs in reverse_edges.items():
+                if len(origs) != 1:
+                    continue
+                existing_keywords = frozenset().union(*(
+                    pkgver.keywords
+                    for pkg in node.pkgs
+                    for pkgver in repo.match(pkg[0].unversioned_atom)
+                ))
+                if existing_keywords & frozenset().union(*(pkg[1] for pkg in node.pkgs)):
+                    continue # not fully new keywords
+                orig = next(iter(origs))
+                print(f"Merging {node} into {orig}")
+                self.merge_nodes((orig, node))
+                found_someone = True
+                break
+
+    def file_bugs(self, api_key: str, auto_cc_arches: frozenset[str]):
+        def observe(node: GraphNode):
+            self.out.write(
+                f"https://bugs.gentoo.org/{node.bugno} ",
+                " | ".join(node.lines()),
+                " depends on bugs ", {dep.bugno for dep in node.edges}
+            )
+            self.out.flush()
+
+        for node in self.starting_nodes:
+            node.file_bug(api_key, auto_cc_arches, observe)
+
+
+@××××.bind_main_func
+def main(options, out: Formatter, err: Formatter):
+    search_repo = options.search_repo
+    targets = [max(search_repo.itermatch(target)) for _, target in options.targets]
+    d = DependencyGraph(out, err, options)
+    d.build_full_graph(targets)
+    d.merge_cycles()
+    d.merge_new_keywords_children()
+
+    for node in d.nodes:
+        node.cleanup_keywords(search_repo)
+
+    if options.dot is not None:
+        d.output_dot(options.dot)
+        out.write(out.fg("green"), f"Dot file written to {options.dot}", out.reset)
+
+    if not userquery(f'Continue and create {len(d.nodes)} stablereq bugs?', out, err, default_answer=False):
+        return 1
+
+    if options.api_key is None:
+        err.write(out.fg("red"), "No API key provided, exiting", out.reset)
+        return 1
+
+    disabled, enabled = options.auto_cc_arches
+    d.file_bugs(options.api_key, frozenset(enabled).difference(disabled))

diff --git a/tests/scripts/test_pkgdev_bugs.py b/tests/scripts/test_pkgdev_bugs.py
new file mode 100644
index 0000000..f23051e
--- /dev/null
+++ b/tests/scripts/test_pkgdev_bugs.py
@@ -0,0 +1,104 @@
+import itertools
+import os
+import sys
+import json
+import textwrap
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+from pkgcore.ebuild.atom import atom
+from pkgcore.test.misc import FakePkg
+from pkgdev.scripts import pkgdev_bugs as bugs
+from snakeoil.formatters import PlainTextFormatter
+from snakeoil.osutils import pjoin
+
+def mk_pkg(repo, cpvstr, maintainers, **kwargs):
+    kwargs.setdefault("KEYWORDS", ["~amd64"])
+    pkgdir = os.path.dirname(repo.create_ebuild(cpvstr, **kwargs))
+    # stub metadata
+    with open(pjoin(pkgdir, 'metadata.xml'), 'w') as f:
+        f.write(textwrap.dedent(f"""\
+            <?xml version="1.0" encoding="UTF-8"?>
+            <!DOCTYPE pkgmetadata SYSTEM "https://www.gentoo.org/dtd/metadata.dtd">
+            <pkgmetadata>
+                <maintainer type="person">
+                    {' '.join(f'<email>{maintainer}@gentoo.org</email>' for maintainer in maintainers)}
+                </maintainer>
+            </pkgmetadata>
+        """))
+
+
+def mk_repo(repo):
+    mk_pkg(repo, 'cat/u-0', ['dev1'])
+    mk_pkg(repo, 'cat/z-0', [], RDEPEND=['cat/u', 'cat/x'])
+    mk_pkg(repo, 'cat/v-0', ['dev2'], RDEPEND='cat/x')
+    mk_pkg(repo, 'cat/y-0', ['dev1'], RDEPEND=['cat/z', 'cat/v'])
+    mk_pkg(repo, 'cat/x-0', ['dev3'], RDEPEND='cat/y')
+    mk_pkg(repo, 'cat/w-0', ['dev3'], RDEPEND='cat/x')
+
+
+class BugsSession:
+    def __init__(self):
+        self.counter = iter(itertools.count(1))
+        self.calls = []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        ...
+
+    def read(self):
+        return json.dumps({'id': next(self.counter)}).encode('utf-8')
+
+    def __call__(self, request, *_args, **_kwargs):
+        self.calls.append(json.loads(request.data))
+        return self
+
+
+class TestBugFiling:
+    def test_bug_filing(self, repo):
+        mk_repo(repo)
+        session = BugsSession()
+        pkg = max(repo.itermatch(atom('=cat/u-0')))
+        with patch('pkgdev.scripts.pkgdev_bugs.urllib.urlopen', session):
+            bugs.GraphNode(((pkg, {'*'}), )).file_bug("API", frozenset())
+        assert len(session.calls) == 1
+        call = session.calls[0]
+        assert call['Bugzilla_api_key'] == 'API'
+        assert call['summary'] == 'cat/u-0: stablereq'
+        assert call['assigned_to'] == 'dev1@g.o'
+        assert not call['cc']
+        assert call['cf_stabilisation_atoms'] == '=cat/u-0 *'
+        assert not call['depends_on']
+
+    def test_bug_filing_maintainer_needed(self, repo):
+        mk_repo(repo)
+        session = BugsSession()
+        pkg = max(repo.itermatch(atom('=cat/z-0')))
+        with patch('pkgdev.scripts.pkgdev_bugs.urllib.urlopen', session):
+            bugs.GraphNode(((pkg, {'*'}), )).file_bug("API", frozenset())
+        assert len(session.calls) == 1
+        call = session.calls[0]
+        assert call['assigned_to'] == 'maintainer-needed@g.o'
+        assert not call['cc']
+
+    def test_bug_filing_multiple_pkgs(self, repo):
+        mk_repo(repo)
+        session = BugsSession()
+        pkgX = max(repo.itermatch(atom('=cat/x-0')))
+        pkgY = max(repo.itermatch(atom('=cat/y-0')))
+        pkgZ = max(repo.itermatch(atom('=cat/z-0')))
+        dep = bugs.GraphNode((), 2)
+        node = bugs.GraphNode(((pkgX, {'*'}), (pkgY, {'*'}), (pkgZ, {'*'})))
+        node.edges.add(dep)
+        with patch('pkgdev.scripts.pkgdev_bugs.urllib.urlopen', session):
+            node.file_bug("API", frozenset())
+        assert len(session.calls) == 1
+        call = session.calls[0]
+        assert call['summary'] == 'cat/x-0, cat/y-0, cat/z-0: stablereq'
+        assert call['assigned_to'] == 'dev3@g.o'
+        assert call['cc'] == ['dev1@g.o']
+        assert call['cf_stabilisation_atoms'] == '=cat/x-0 *\n=cat/y-0 *\n=cat/z-0 *'
+        assert call['depends_on'] == [2]