contrib/crbug_crawler.py

#!/usr/bin/python

# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
This script crawls crbug. Sort-of.
Invocation:
    Get all bugs with labels, strings (in summary and/or comments):
        crbug_crawler.py --labels 'one two three'
                         --queries '"first query" "second query"'

    Get baddest open bugs of all time:
        crbug_crawler.py --reap

Tips:
    - Label based queries will return faster than text queries.
    - contrib/crbug_shell.py is a wrapper that allows you to incrementally
        filter search results using this script.
"""

import argparse
import cmd
import logging
import sys
import shlex

import common
from autotest_lib.client.common_lib import global_config
from autotest_lib.server.cros.dynamic_suite import reporting


def _parse_args(args):
    if not args:
        import crbug_crawler
        logging.error('Improper usage of crbug_crawler: %s',
                crbug_crawler.__doc__)
        sys.exit(1)

    description = ('Usage: crbug_crawler.py --reap')
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('--quiet', help=('Turn off logging noise.'),
            action='store_true', default=False)
    parser.add_argument('--num', help='Number of issues to output.', default=10,
            type=int)
    parser.add_argument('--queries',
                        help=('Search query. Eg: --queries "%s %s"' %
                              ('build_Root', 'login')),
                        default='')
    parser.add_argument('--labels',
                        help=('Search labels. Eg: --labels "%s %s"' %
                              ('autofiled', 'Pri-1')), default=None)
    parser.add_argument('--reap', help=('Top autofiled bugs ordered by count.'),
            action='store_true', default=False)
    return parser.parse_args(args)


class Update(object):
    """Class encapsulating fields of an update to a bug.
    """
    open_statuses = ['Unconfirmed', 'Untriaged', 'Available', 'Assigned',
                     'Started', 'ExternalDependency']
    closed_statuses = ['Fixed', 'Verified', 'Duplicate', 'WontFix', 'Archived']

    def __init__(self, comment='', labels='', status=''):
        self.comment = comment
        self.labels = labels if labels else []
        self.status = status


    def __str__(self):
        msg = 'status: %s' % self.status
        if self.labels:
            msg = '%s labels: %s' % (msg, self.labels)
        if self.comment:
            msg = '%s comment: %s' % (msg, self.comment)
        return msg


class UpdateManager(object):
    """Update manager that allows you to revert status updates.

    This class keeps track of the last update applied and is capable
    of reverting it.
    """

    def __init__(self, autocommit=False):
        """Initialize update manager.

        @param autocommit: If False just print out the update instead
            of committing it.
        """
        self.history = {}
        self.present = {}
        self.reporter = reporting.Reporter()
        self.phapi_lib = self.reporter.get_bug_tracker_client()
        self.autocommit = autocommit


    def revert(self):
        """Only manages status reverts as of now.
        """
        for issue_id, update in self.history.iteritems():
            logging.warning('You will have to manually update %s and %s on %s',
                    self.present[issue_id].labels,
                    self.present[issue_id].comment, issue_id)
            # Create a new update with just the status.
            self.update(issue_id, Update(status=update.status))


    def update(self, old_issue, update):
        """Record the state of an issue before updating it.

        @param old_issue: The issue to update. If an id is specified an
            issue is constructed. If an issue object (as defined in phapi_lib
            Issue)is passed in, it is used directly.
        @param update: The Update object to apply to the issue.
        """
        if type(old_issue) == int:
            old_issue = self.phapi_lib.get_tracker_issue_by_id(old_issue)
        old_update = Update(
                labels=old_issue.labels, status=old_issue.status)

        if not update.status:
            update.status = old_update.status
        elif (update.status not in Update.open_statuses and
              update.status not in Update.closed_statuses):
            raise ValueError('Unknown status %s' % update.status)

        if not self.autocommit:
            logging.warning('Would have applied the following update: '
                    '%s -> %s', old_update, update)
            return

        self.history[old_issue.id] = old_update
        self.reporter.modify_bug_report(
                issue_id=old_issue.id, comment=update.comment,
                label_update=update.labels,
                status=update.status)
        self.present[old_issue.id] = update


class Crawler(object):
    """Class capable of crawling crbug.

    This class applies filters to issues it crawls and caches them locally.
    """

    # The limit at which we ask for confirmation to proceed with the crawl.
    PROMPT_LIMIT = 2000

    def __init__(self):
        self.reporter = reporting.Reporter()
        self.phapi_client = self.reporter.get_bug_tracker_client()
        self.issues = None
        self.all_autofiled_query = 'ANCHOR  TestFailure'
        self.all_autofiled_label = 'autofiled'
        self.prompted = False


    def fuzzy_search(self, query='', label='', fast=True):
        """Returns all issues using one query and/or one label.

        @param query: A string representing the query.
        @param label: A string representing the label.
        @param fast: If true, don't bother fetching comments.

        @return: A list of issues matching the query. If fast is
            specified the issues won't have comments.
        """
        if not query and not label:
            raise ValueError('Require query or labels to make a tracker query, '
                    'try query = "%s" or one of the predefined labels %s' %
                    (self.fuzzy_search_anchor(),
                     self.reporter._PREDEFINED_LABELS))
        if type(label) != str:
            raise ValueError('The crawler only supports one label per query, '
                    'and it must be a string. you supplied %s' % label)
        return self.phapi_client.get_tracker_issues_by_text(
                query, label=label, full_text=not fast)


    @staticmethod
    def _get_autofiled_count(issue):
        """Return the autofiled count.

        @param issue: An issue object that has labels.

        @return: An integer representing the autofiled count.
        """
        for label in issue.labels:
            if 'autofiled-count-' in label:
                return int(label.replace('autofiled-count-', ''))

        # Force bugs without autofiled-count to sink
        return 0


    def _prompt_crawl(self, new_issues, start_index):
        """Warn the user that a crawl is getting large.

        This method prompts for a y/n answer in case the user wants to abort the
        crawl and specify another set of labels/queries.

        @param new_issues: A list of issues used with the start_index to
            determine the number of issues already processed.
        @param start_index: The start index of the next crawl iteration.
        """
        logging.warning('Found %s issues, Crawling issues starting from %s',
                len(new_issues), start_index)
        if start_index > self.PROMPT_LIMIT and not self.prompted:
            logging.warning('Already crawled %s issues, it is possible that'
                    'you\'ve specified a very general label. If this is the '
                    'case consider re-rodering the labels so they start with '
                    'the rarest. Continue crawling [y/n]?',
                    start_index + len(new_issues))
            self.prompted = raw_input() == 'y'
            if not self.prompted:
                sys.exit(0)


    def exhaustive_crawl(self, query='', label='', fast=True):
        """Perform an exhaustive crawl using one label and query string.

        @param query: A string representing one query.
        @param lable: A string representing one label.

        @return A list of issues sorted by descending autofiled count.
        """
        start_index = 0
        self.phapi_client.set_max_results(200)
        logging.warning('Performing an exhaustive crawl with label %s query %s',
                label, query)
        vague_issues = []
        new_issues = self.fuzzy_search(query=query, label=label, fast=fast)
        while new_issues:
            vague_issues += new_issues
            start_index += len(new_issues) + 1
            self.phapi_client.set_start_index(start_index)
            new_issues = self.fuzzy_search(query=query, label=label,
                    fast=fast)
            self._prompt_crawl(new_issues, start_index)

        # Subsequent calls will clear the issues cache with new results.
        self.phapi_client.set_start_index(1)
        return sorted(vague_issues, reverse=True,
                      key=lambda issue: self._get_autofiled_count(issue))


    @staticmethod
    def filter_labels(issues, labels):
        """Takes a list of labels and returns matching issues.

        @param issues: A list of issues to parse for labels.
        @param labels: A list of labels to match.

        @return: A list of matching issues. The issues must contain
            all the labels specified.
        """
        if not labels:
            return issues
        matching_issues = set([])
        labels = set(labels)
        for issue in issues:
            issue_labels = set(issue.labels)
            if issue_labels.issuperset(labels):
                matching_issues.add(issue)
        return matching_issues


    @classmethod
    def does_query_match(cls, issue, query):
        """Check if a query matches the given issue.

        @param issue: The issue to check.
        @param query: The query to check against.

        @return: True if the query matches, false otherwise.
        """
        if query in issue.title or query in issue.summary:
            return True
        # We can only search comments if the issue is a complete issue
        # i.e as defined in phapi_lib.Issue.
        try:
            if any(query in comment for comment in issue.comments):
                return True
        except (AttributeError, TypeError):
            pass
        return False


    @classmethod
    def filter_queries(cls, issues, queries):
        """Take a list of queries and returns matching issues.

        @param issues: A list of issues to parse. If the issues contain
            comments and a query is not in the issues title or summmary,
            the comments are parsed for a substring match.
        @param queries: A list of queries to parse the issues for.
            This method looks for an exact substring match within each issue.

        @return: A list of matching issues.
        """
        if not queries:
            return issues
        matching_issues = set([])
        for issue in issues:
            # For each query, check if it's in the title, description or
            # comments. If a query isn't in any of these, discard the issue.
            for query in queries:
                if cls.does_query_match(issue, query):
                    matching_issues.add(issue)
                else:
                    if issue in matching_issues:
                        logging.warning('%s: %s\n \tPassed a subset of the '
                                'queries but failed query %s',
                                issue.id, issue.title, query)
                        matching_issues.remove(issue)
                    break
        return matching_issues


    def filter_issues(self, queries='', labels=None, fast=True):
        """Run the queries, labels filters by crawling crbug.

        @param queries: A space seperated string of queries, usually passed
            through the command line.
        @param labels: A space seperated string of labels, usually passed
            through the command line.
        @param fast: If specified, skip creating comments for issues since this
            can be a slow process. This value is only a suggestion, since it is
            ignored if multiple queries are specified.
        """
        queries = shlex.split(queries)
        labels = shlex.split(labels) if labels else None

        # We'll need comments to filter multiple queries.
        if len(queries) > 1:
            fast = False
        matching_issues = self.exhaustive_crawl(
                query=queries.pop(0) if queries else '',
                label=labels.pop(0) if labels else '', fast=fast)
        matching_issues = self.filter_labels(matching_issues, labels)
        matching_issues = self.filter_queries(matching_issues, queries)
        self.issues = list(matching_issues)


    def dump_issues(self, limit=None):
        """Print issues.
        """
        if limit and limit < len(self.issues):
            issues = self.issues[:limit]
        else:
            issues = self.issues
        #TODO: Modify formatting, include some paging etc.
        for issue in issues:
            try:
                print ('[%s] %s crbug.com/%s %s' %
                       (self._get_autofiled_count(issue),
                        issue.status, issue.id, issue.title))
            except UnicodeEncodeError as e:
                print "Unicdoe error decoding issue id %s" % issue.id
                continue


def _update_test(args):
    """A simple update test, to record usage.
    """
    updater = UpdateManager(autocommit=True)
    for issue in issues:
        updater.update(issue,
                       Update(comment='this is bogus', labels=['bogus'],
                              status='Assigned'))
    updater.revert()


def configure_logging(quiet=False):
    """Configure logging.

    @param quiet: True to turn off warning messages.
    """
    logging.basicConfig()
    logger = logging.getLogger()
    level = logging.WARNING
    if quiet:
        level = logging.ERROR
    logger.setLevel(level)


def main(args):
    crawler = Crawler()
    if args.reap:
        if args.queries or args.labels:
            logging.error('Query based ranking of bugs not supported yet.')
            return
        queries = ''
        labels = crawler.all_autofiled_label
    else:
        queries = args.queries
        labels = args.labels
    crawler.filter_issues(queries=queries, labels=labels,
            fast=False if queries else True)
    crawler.dump_issues(int(args.num))
    logging.warning('\nThis is a truncated list of %s results, use --num %s '
            'to get them all. If you want more informative results/better '
            'querying capabilities try crbug_shell.py.',
            args.num, len(crawler.issues))


if __name__ == '__main__':
    args = _parse_args(sys.argv[1:])
    configure_logging(args.quiet)
    main(args)