troll-o-matic.py

#!/usr/bin/python3

from exceptions import GerritFetchError
from gerrit import Gerrit, GerritRevision, GerritMessage
from reviewer import Reviewer

from trollconfig import TrollConfig
from trollreview import ReviewType
from trollreviewer import ChangeReviewer
from trollreviewerfromgit import FromgitChangeReviewer
from trollreviewerupstream import UpstreamChangeReviewer
from trollreviewerfromlist import FromlistChangeReviewer
from trollreviewerchromium import ChromiumChangeReviewer
from trollstats import TrollStats

import argparse
import datetime
import json
import logging
from logging import handlers
import re
import requests
import sys
import time

logger = logging.getLogger('rom')
logger.setLevel(logging.DEBUG) # leave this to handlers

class Troll(object):
  RETRY_REVIEW_KEY='retry-bot-review'

  def __init__(self, config):
    self.config = config
    self.gerrit = Gerrit(config.gerrit_url, netrc=config.netrc)
    self.gerrit_admin = Gerrit(config.gerrit_url, netrc=config.netrc_admin)
    self.tag = 'autogenerated:review-o-matic'
    self.ignore_list = {}
    self.stats = TrollStats('{}'.format(self.config.stats_file))

  def do_review(self, project, change, review):
    logger.info('Review for change: {}'.format(change.url()))
    logger.info('  Issues: {}, Feedback: {}, Vote:{}, Notify:{}'.format(
        review.issues.keys(), review.feedback.keys(), review.vote,
        review.notify))

    if review.dry_run:
      print(review.generate_review_message(self.RETRY_REVIEW_KEY))
      if review.inline_comments:
        print('')
        print('-- Inline comments:')
        for f,comments in review.inline_comments.items():
          for c in comments:
            print('{}:{}'.format(f, c['line']))
            print(c['message'])

      print('------')
      return

    self.stats.update_for_review(project, review)

    self.gerrit.review(change, self.tag,
                       review.generate_review_message(self.RETRY_REVIEW_KEY),
                       review.notify, vote_code_review=review.vote,
                       inline_comments=review.inline_comments)

    if self.config.results_file:
      with open(self.config.results_file, 'a+') as f:
        f.write('{}: Issues: {}, Feedback: {}, Vote:{}, Notify:{}\n'.format(
          change.url(), review.issues.keys(), review.feedback.keys(),
          review.vote, review.notify))

  def get_changes(self, project, prefix):
    message = '{}:'.format(prefix)
    after = datetime.date.today() - datetime.timedelta(days=5)
    changes = self.gerrit.query_changes(status='open', message=message,
                    after=after, project=project.gerrit_project,
                    branches=project.monitor_branches)
    return changes

  def add_change_to_ignore_list(self, change):
    self.ignore_list[change.number] = change.current_revision.number

  def is_change_in_ignore_list(self, change):
    return self.ignore_list.get(change.number) == change.current_revision.number

  def process_change(self, project, rev, c):
    if self.config.chatty:
      logger.debug('Processing change {}'.format(c.url()))

    force_review = self.config.force_cl or self.config.force_all

    # Look for a retry request in the topic
    retry_request = False
    topic_list = c.topic.split() if c.topic else None
    if topic_list and self.RETRY_REVIEW_KEY in topic_list:
      retry_request = True
      force_review = True
      logger.error('Received retry request on change {} (topic={})'.format(
                   c.url(), c.topic))

    # Look for prior reviews and retry requests
    last_review = None
    for m in c.get_messages():
      if not m.revision_num == c.current_revision.number:
        continue
      if m.tag == self.tag:
        last_review = m

    age_days = None
    if not force_review and last_review:
      age_days = (datetime.datetime.utcnow() - last_review.date).days

    if age_days != None and self.config.chatty:
      logger.debug('    Reviewed {} days ago'.format(age_days))

    # Find a reviewer or ignore if not found
    reviewer = None
    if not ChangeReviewer.can_review_change(project, c, age_days):
      # Some patches are blanket unreviewable, check these first
      reviewer = None
    elif FromlistChangeReviewer.can_review_change(project, c, age_days):
      reviewer = FromlistChangeReviewer(project, rev, c,
                                        self.config.gerrit_msg_limit,
                                        self.config.dry_run)
    elif FromgitChangeReviewer.can_review_change(project, c, age_days):
      reviewer = FromgitChangeReviewer(project, rev, c,
                                       self.config.gerrit_msg_limit,
                                       self.config.dry_run, age_days)
    elif UpstreamChangeReviewer.can_review_change(project, c, age_days):
      reviewer = UpstreamChangeReviewer(project, rev, c,
                                        self.config.gerrit_msg_limit,
                                        self.config.dry_run)
    elif ChromiumChangeReviewer.can_review_change(project, c, age_days):
      reviewer = ChromiumChangeReviewer(project, rev, c,
                                        self.config.gerrit_msg_limit,
                                        self.config.dry_run,
                                        self.config.verbose)

    # Clear the retry request from the topic
    if retry_request:
      topic_list.remove(self.RETRY_REVIEW_KEY)
      c.topic = ' '.join(topic_list)
      if not self.gerrit_admin.set_topic(c):
        logger.error('ERROR: Failed to clear retry request from change')
        return None

    if not reviewer:
      self.add_change_to_ignore_list(c)
      return None

    if not force_review and self.is_change_in_ignore_list(c):
      return None

    return reviewer.review_patch()

  def process_changes(self, project, changes):
    rev = Reviewer(git_dir=project.local_repo, verbose=self.config.verbose,
                   chatty=self.config.chatty)
    ret = 0
    for c in changes:
      ignore = False
      for b in project.ignore_branches:
        if re.match(b, c.branch):
          ignore = True
          break
      if ignore:
        if self.config.chatty:
          logger.debug('Ignoring change {}'.format(c))
        self.add_change_to_ignore_list(c)
        continue

      try:
        result = self.process_change(project, rev, c)
        if result:
          self.do_review(project, c, result)
          ret += 1
        self.add_change_to_ignore_list(c)
      except GerritFetchError as e:
        logger.error('Gerrit fetch failed, will retry, {}'.format(c.url()))
        logger.exception('Exception: {}'.format(e))
        # Don't add change to ignore list, we want to retry next time
      except Exception as e:
        logger.error('Exception processing change {}'.format(c.url()))
        logger.exception('Exception: {}'.format(e))
        self.add_change_to_ignore_list(c)

    return ret

  def run(self):
    if self.config.force_cl:
      c = self.gerrit.get_change(self.config.force_cl, self.config.force_rev)
      logger.info('Force reviewing change  {}'.format(c))
      project = self.config.get_project(c.project)
      if not project:
        raise ValueError('Could not find project!')
      self.process_changes(project, [c])
      return

    while True:
      try:
        did_review = 0
        for project in self.config.projects.values():
          if (self.config.force_project and
              project.name != self.config.force_project):
            continue
          if self.config.chatty:
            logger.debug('Running for project {}'.format(project.name))
          for p in project.prefixes:
            changes = self.get_changes(project, p)
            if self.config.chatty:
              logger.debug('{} changes for prefix {}'.format(len(changes), p))
            did_review += self.process_changes(project, changes)

        if did_review > 0:
          self.stats.summarize(logging.INFO)
          if not self.config.dry_run:
            self.stats.save()

        if not self.config.daemon:
          return
        if self.config.chatty:
          logger.debug('Finished! Going to sleep until next run')

      except (requests.exceptions.HTTPError, OSError) as e:
        logger.error('Error getting changes: ({})'.format(str(e)))
        logger.exception('Exception getting changes: {}'.format(e))
        time.sleep(60)

      time.sleep(120)


def setup_logging(config):
  info_handler = logging.StreamHandler(sys.stdout)
  info_handler.setFormatter(logging.Formatter('%(levelname)6s - %(name)s - %(message)s'))
  if config.verbose:
    info_handler.setLevel(logging.DEBUG)
  else:
    info_handler.setLevel(logging.INFO)
  logger.addHandler(info_handler)

  if not config.log_file or config.dry_run:
    return

  err_handler = logging.handlers.RotatingFileHandler(config.log_file,
                                                     maxBytes=10000000,
                                                     backupCount=20)
  err_handler.setLevel(logging.WARNING)
  f = logging.Formatter(
        '%(asctime)s %(levelname)8s - %(name)s.%(funcName)s:%(lineno)d - ' +
        '%(message)s')
  err_handler.setFormatter(f)
  logger.addHandler(err_handler)


def main():
  config = TrollConfig()

  setup_logging(config)

  troll = Troll(config)
  troll.run()

if __name__ == '__main__':
  sys.exit(main())