#!/usr/bin/env python3 # # ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==# # # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ==-------------------------------------------------------------------------==# import argparse from git import Repo # type: ignore import github import os import re import requests import sys import time from typing import List, Optional class IssueSubscriber: @property def team_name(self) -> str: return self._team_name def __init__(self, token:str, repo:str, issue_number:int, label_name:str): self.repo = github.Github(token).get_repo(repo) self.org = github.Github(token).get_organization(self.repo.organization.login) self.issue = self.repo.get_issue(issue_number) self._team_name = 'issue-subscribers-{}'.format(label_name).lower() def run(self) -> bool: for team in self.org.get_teams(): if self.team_name != team.name.lower(): continue comment = '@llvm/{}'.format(team.slug) self.issue.create_comment(comment) return True return False def setup_llvmbot_git(git_dir = '.'): """ Configure the git repo in `git_dir` with the llvmbot account so commits are attributed to llvmbot. """ repo = Repo(git_dir) with repo.config_writer() as config: config.set_value('user', 'name', 'llvmbot') config.set_value('user', 'email', 'llvmbot@llvm.org') def phab_api_call(phab_token:str, url:str, args:dict) -> dict: """ Make an API call to the Phabricator web service and return a dictionary containing the json response. """ data = { "api.token" : phab_token } data.update(args) response = requests.post(url, data = data) return response.json() def phab_login_to_github_login(phab_token:str, repo:github.Repository.Repository, phab_login:str) -> Optional[str]: """ Tries to translate a Phabricator login to a github login by finding a commit made in Phabricator's Differential. The commit's SHA1 is then looked up in the github repo and the committer's login associated with that commit is returned. :param str phab_token: The Conduit API token to use for communication with Pabricator :param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential :param str phab_login: The Phabricator login to be translated. """ args = { "constraints[authors][0]" : phab_login, # PHID for "LLVM Github Monorepo" repository "constraints[repositories][0]" : "PHID-REPO-f4scjekhnkmh7qilxlcy", "limit" : 1 } # API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/ r = phab_api_call(phab_token, "https://reviews.llvm.org/api/diffusion.commit.search", args) data = r['result']['data'] if len(data) == 0: # Can't find any commits associated with this user return None commit_sha = data[0]['fields']['identifier'] committer = repo.get_commit(commit_sha).committer if not committer: # This committer had an email address GitHub could not recognize, so # it can't link the user to a GitHub account. print(f"Warning: Can't find github account for {phab_login}") return None return committer.login def phab_get_commit_approvers(phab_token:str, commit:github.Commit.Commit) -> list: args = { "corpus" : commit.commit.message } # API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/ r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.parsecommitmessage", args) review_id = r['result']['revisionIDFieldInfo']['value'] if not review_id: # No Phabricator revision for this commit return [] args = { 'constraints[ids][0]' : review_id, 'attachments[reviewers]' : True } # API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/ r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.revision.search", args) reviewers = r['result']['data'][0]['attachments']['reviewers']['reviewers'] accepted = [] for reviewer in reviewers: if reviewer['status'] != 'accepted': continue phid = reviewer['reviewerPHID'] args = { 'constraints[phids][0]' : phid } # API documentation: https://reviews.llvm.org/conduit/method/user.search/ r = phab_api_call(phab_token, "https://reviews.llvm.org/api/user.search", args) accepted.append(r['result']['data'][0]['fields']['username']) return accepted class ReleaseWorkflow: CHERRY_PICK_FAILED_LABEL = 'release:cherry-pick-failed' """ This class implements the sub-commands for the release-workflow command. The current sub-commands are: * create-branch * create-pull-request The execute_command method will automatically choose the correct sub-command based on the text in stdin. """ def __init__(self, token:str, repo:str, issue_number:int, branch_repo_name:str, branch_repo_token:str, llvm_project_dir:str, phab_token:str) -> None: self._token = token self._repo_name = repo self._issue_number = issue_number self._branch_repo_name = branch_repo_name if branch_repo_token: self._branch_repo_token = branch_repo_token else: self._branch_repo_token = self.token self._llvm_project_dir = llvm_project_dir self._phab_token = phab_token @property def token(self) -> str: return self._token @property def repo_name(self) -> str: return self._repo_name @property def issue_number(self) -> int: return self._issue_number @property def branch_repo_name(self) -> str: return self._branch_repo_name @property def branch_repo_token(self) -> str: return self._branch_repo_token @property def llvm_project_dir(self) -> str: return self._llvm_project_dir @property def phab_token(self) -> str: return self._phab_token @property def repo(self) -> github.Repository.Repository: return github.Github(self.token).get_repo(self.repo_name) @property def issue(self) -> github.Issue.Issue: return self.repo.get_issue(self.issue_number) @property def push_url(self) -> str: return 'https://{}@github.com/{}'.format(self.branch_repo_token, self.branch_repo_name) @property def branch_name(self) -> str: return 'issue{}'.format(self.issue_number) @property def release_branch_for_issue(self) -> Optional[str]: issue = self.issue milestone = issue.milestone if milestone is None: return None m = re.search('branch: (.+)',milestone.description) if m: return m.group(1) return None def print_release_branch(self) -> None: print(self.release_branch_for_issue) def issue_notify_branch(self) -> None: self.issue.create_comment('/branch {}/{}'.format(self.branch_repo_name, self.branch_name)) def issue_notify_pull_request(self, pull:github.PullRequest.PullRequest) -> None: self.issue.create_comment('/pull-request {}#{}'.format(self.branch_repo_name, pull.number)) def make_ignore_comment(self, comment: str) -> str: """ Returns the comment string with a prefix that will cause a Github workflow to skip parsing this comment. :param str comment: The comment to ignore """ return "\n"+comment def issue_notify_no_milestone(self, comment:List[str]) -> None: message = "{}\n\nError: Command failed due to missing milestone.".format(''.join(['>' + line for line in comment])) self.issue.create_comment(self.make_ignore_comment(message)) @property def action_url(self) -> str: if os.getenv('CI'): return 'https://github.com/{}/actions/runs/{}'.format(os.getenv('GITHUB_REPOSITORY'), os.getenv('GITHUB_RUN_ID')) return "" def issue_notify_cherry_pick_failure(self, commit:str) -> github.IssueComment.IssueComment: message = self.make_ignore_comment("Failed to cherry-pick: {}\n\n".format(commit)) action_url = self.action_url if action_url: message += action_url + "\n\n" message += "Please manually backport the fix and push it to your github fork. Once this is done, please add a comment like this:\n\n`/branch //`" issue = self.issue comment = issue.create_comment(message) issue.add_to_labels(self.CHERRY_PICK_FAILED_LABEL) return comment def issue_notify_pull_request_failure(self, branch:str) -> github.IssueComment.IssueComment: message = "Failed to create pull request for {} ".format(branch) message += self.action_url return self.issue.create_comment(message) def issue_remove_cherry_pick_failed_label(self): if self.CHERRY_PICK_FAILED_LABEL in [l.name for l in self.issue.labels]: self.issue.remove_from_labels(self.CHERRY_PICK_FAILED_LABEL) def pr_request_review(self, pr:github.PullRequest.PullRequest): """ This function will try to find the best reviewers for `commits` and then add a comment requesting review of the backport and assign the pull request to the selected reviewers. The reviewers selected are those users who approved the patch in Phabricator. """ reviewers = [] for commit in pr.get_commits(): approvers = phab_get_commit_approvers(self.phab_token, commit) for a in approvers: login = phab_login_to_github_login(self.phab_token, self.repo, a) if not login: continue reviewers.append(login) if len(reviewers): message = "{} What do you think about merging this PR to the release branch?".format( " ".join(["@" + r for r in reviewers])) pr.create_issue_comment(message) pr.add_to_assignees(*reviewers) def create_branch(self, commits:List[str]) -> bool: """ This function attempts to backport `commits` into the branch associated with `self.issue_number`. If this is successful, then the branch is pushed to `self.branch_repo_name`, if not, a comment is added to the issue saying that the cherry-pick failed. :param list commits: List of commits to cherry-pick. """ print('cherry-picking', commits) branch_name = self.branch_name local_repo = Repo(self.llvm_project_dir) local_repo.git.checkout(self.release_branch_for_issue) for c in commits: try: local_repo.git.cherry_pick('-x', c) except Exception as e: self.issue_notify_cherry_pick_failure(c) raise e push_url = self.push_url print('Pushing to {} {}'.format(push_url, branch_name)) local_repo.git.push(push_url, 'HEAD:{}'.format(branch_name), force=True) self.issue_notify_branch() self.issue_remove_cherry_pick_failed_label() return True def check_if_pull_request_exists(self, repo:github.Repository.Repository, head:str) -> bool: pulls = repo.get_pulls(head=head) return pulls.totalCount != 0 def create_pull_request(self, owner:str, repo_name:str, branch:str) -> bool: """ reate a pull request in `self.branch_repo_name`. The base branch of the pull request will be chosen based on the the milestone attached to the issue represented by `self.issue_number` For example if the milestone is Release 13.0.1, then the base branch will be release/13.x. `branch` will be used as the compare branch. https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch """ repo = github.Github(self.token).get_repo(self.branch_repo_name) issue_ref = '{}#{}'.format(self.repo_name, self.issue_number) pull = None release_branch_for_issue = self.release_branch_for_issue if release_branch_for_issue is None: return False head_branch = branch if not repo.fork: # If the target repo is not a fork of llvm-project, we need to copy # the branch into the target repo. GitHub only supports cross-repo pull # requests on forked repos. head_branch = f'{owner}-{branch}' local_repo = Repo(self.llvm_project_dir) push_done = False for _ in range(0,5): try: local_repo.git.fetch(f'https://github.com/{owner}/{repo_name}', f'{branch}:{branch}') local_repo.git.push(self.push_url, f'{branch}:{head_branch}', force=True) push_done = True break except Exception as e: print(e) time.sleep(30) continue if not push_done: raise Exception("Failed to mirror branch into {}".format(self.push_url)) owner = repo.owner.login head = f"{owner}:{head_branch}" if self.check_if_pull_request_exists(repo, head): print("PR already exists...") return True try: pull = repo.create_pull(title=f"PR for {issue_ref}", body='resolves {}'.format(issue_ref), base=release_branch_for_issue, head=head, maintainer_can_modify=False) try: if self.phab_token: self.pr_request_review(pull) except Exception as e: print("error: Failed while searching for reviewers", e) except Exception as e: self.issue_notify_pull_request_failure(branch) raise e if pull is None: return False self.issue_notify_pull_request(pull) self.issue_remove_cherry_pick_failed_label() # TODO(tstellar): Do you really want to always return True? return True def execute_command(self) -> bool: """ This function reads lines from STDIN and executes the first command that it finds. The 2 supported commands are: /cherry-pick commit0 <...> /branch // """ for line in sys.stdin: line.rstrip() m = re.search(r"/([a-z-]+)\s(.+)", line) if not m: continue command = m.group(1) args = m.group(2) if command == 'cherry-pick': return self.create_branch(args.split()) if command == 'branch': m = re.match('([^/]+)/([^/]+)/(.+)', args) if m: owner = m.group(1) repo = m.group(2) branch = m.group(3) return self.create_pull_request(owner, repo, branch) print("Do not understand input:") print(sys.stdin.readlines()) return False parser = argparse.ArgumentParser() parser.add_argument('--token', type=str, required=True, help='GitHub authentiation token') parser.add_argument('--repo', type=str, default=os.getenv('GITHUB_REPOSITORY', 'llvm/llvm-project'), help='The GitHub repository that we are working with in the form of / (e.g. llvm/llvm-project)') subparsers = parser.add_subparsers(dest='command') issue_subscriber_parser = subparsers.add_parser('issue-subscriber') issue_subscriber_parser.add_argument('--label-name', type=str, required=True) issue_subscriber_parser.add_argument('--issue-number', type=int, required=True) release_workflow_parser = subparsers.add_parser('release-workflow') release_workflow_parser.add_argument('--llvm-project-dir', type=str, default='.', help='directory containing the llvm-project checout') release_workflow_parser.add_argument('--issue-number', type=int, required=True, help='The issue number to update') release_workflow_parser.add_argument('--phab-token', type=str, help='Phabricator conduit API token. See https://reviews.llvm.org/settings/user//page/apitokens/') release_workflow_parser.add_argument('--branch-repo-token', type=str, help='GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.') release_workflow_parser.add_argument('--branch-repo', type=str, default='llvm/llvm-project-release-prs', help='The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)') release_workflow_parser.add_argument('sub_command', type=str, choices=['print-release-branch', 'auto'], help='Print to stdout the name of the release branch ISSUE_NUMBER should be backported to') llvmbot_git_config_parser = subparsers.add_parser('setup-llvmbot-git', help='Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot') args = parser.parse_args() if args.command == 'issue-subscriber': issue_subscriber = IssueSubscriber(args.token, args.repo, args.issue_number, args.label_name) issue_subscriber.run() elif args.command == 'release-workflow': release_workflow = ReleaseWorkflow(args.token, args.repo, args.issue_number, args.branch_repo, args.branch_repo_token, args.llvm_project_dir, args.phab_token) if not release_workflow.release_branch_for_issue: release_workflow.issue_notify_no_milestone(sys.stdin.readlines()) sys.exit(1) if args.sub_command == 'print-release-branch': release_workflow.print_release_branch() else: if not release_workflow.execute_command(): sys.exit(1) elif args.command == 'setup-llvmbot-git': setup_llvmbot_git()