diff --git a/llvm/utils/git/github-project.py b/llvm/utils/git/github-project.py new file mode 100755 --- /dev/null +++ b/llvm/utils/git/github-project.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python + +import requests +import sys +import json +from pprint import pprint +from typing import Optional, Dict, Union, Tuple, Any, List +import argparse + +# GraphQL to find all the ID's we need for other operations +# This is because things like Status in the board can't be +# referred to by name. +PROJECT_FIELDS = """ +query($org: String!, $number: Int!) { + organization(login: $org) { + projectV2(number: $number) { + id + fields(first:20) { + nodes { + ... on ProjectV2Field { + id + name + } + ... on ProjectV2SingleSelectField { + id + name + options { + id + name + } + } + } + } + } + } +} +""" + +# Map Issue Number -> Issue Id which is needed +ISSUE_ID = """ +query($org: String!, $repo: String!, $number: Int!) { + organization(login: $org) { + repository(name: $repo) { + issue(number: $number) { + id + } + } + } +} +""" + +# Make a search query - this is similar to the +# search feature on the website and can take +# the same query. +SEARCH_ISSUES = """ +query($searchQuery: String!) { + search( + query: $searchQuery + type: ISSUE + first: 100 + ) { + edges { + node { + ... on Issue { + id + number + title + url + } + } + } + } +} +""" + +# GraphQL to add a issue to a project +ADD_ISSUE_TO_PROJECT = """ +mutation AddToProject($project: ID!, $issue: ID!) { + addProjectV2ItemById(input: { projectId: $project, contentId: $issue }) { + item { + id + } + } +} +""" + +# GraphQL to query information from project +ALL_ISSUES_IN_PROJECT = """ +query($org: String!, $number: Int!) { + organization(login: $org) { + projectV2(number: $number) { + items(last: 100) { + nodes { + id + + fieldValues(first: 8) { + nodes { + ... on ProjectV2ItemFieldSingleSelectValue { + name + field { + ... on ProjectV2FieldCommon { + name + } + } + } + ... on ProjectV2ItemFieldTextValue { + text + field { + ... on ProjectV2FieldCommon { + name + } + } + } + ... on ProjectV2ItemFieldPullRequestValue { + pullRequests(last: 10) { + nodes { + url + number + } + } + } + } + } + + content { + ...on Issue { + title + number + url + state + milestone { + number + } + } + } + } + } + } + } +} +""" + +# GraphQL set the status on issues in the project board +SET_PROJECT_STATUS = """ +mutation UpdateProject($project: ID!, $item: ID!, $status_field: ID!, $status_value: String!) { + updateProjectV2ItemFieldValue(input: { projectId: $project, itemId: $item, fieldId: $status_field, value: { singleSelectOptionId: $status_value } }) { + projectV2Item { + id + } + } +} +""" + +session = requests.session() + +# Helper function to send the GraphQL request +def graphql(query: str, variables: Dict[str, Union[int, str]]) -> Dict[Any, Any]: + req = session.post("https://api.github.com/graphql", json={"query": query, "variables": variables}) + req.raise_for_status() + jsondata = req.json() + + if "errors" in jsondata: + print("Something went wrong:") + pprint(jsondata["errors"]) + raise RuntimeError + + if not "data" in jsondata: + print("No data in return - instead we got:") + pprint(jsondata) + raise RuntimeError + + return jsondata + +# Helper function to get data from a deeply nested dict +# gv(dict, "status.item.id") -> dict["status"]["item"]["id"] +# plus that it handles if something is not there. +def gv(data_dict: Dict, value_key: str, def_value:Any = None) -> Any: + keys = value_key.split(".") + + cd = data_dict + for key in keys: + if isinstance(cd, str) and cd != "null": + cd = json.loads(cd) + if key in cd: + cd = cd[key] + else: + return def_value + return cd + +# Get Project ID and all the fields of it. +def project_id_and_fields(project_id: int) -> Tuple[str, Dict]: + data = graphql(PROJECT_FIELDS, {"org": "llvm", "number": project_id}) + fields = {} + for node in gv(data, "data.organization.projectV2.fields.nodes", []): + fields[node["name"]] = { + "id": node["id"], + "options": {n['name']:n['id'] for n in gv(node, "options", [])} + } + return (gv(data, "data.organization.projectV2.id", 0), fields) + +# Get the issue id by the issue number +def issue_id(issue_id: int) -> str: + data = graphql(ISSUE_ID, {"org": "llvm", "repo": "llvm-project", "number": issue_id}) + return gv(data, "data.organization.repository.issue.id", 0) + +# Run a GitHub search +def search_issues(query: str) -> List[Dict[str, Any]]: + data = graphql(SEARCH_ISSUES, {"searchQuery": query}) + issues = [] + for node in gv(data, "data.search.edges", []): + node = node["node"] + issues.append({ + "id": node["id"], + "number": node["number"], + "title": node["title"], + "url": node["url"] + }) + return issues + +# Add the issue to the project +def add_issue_to_project(project_id:str, issue_id:str) -> str: + data = graphql(ADD_ISSUE_TO_PROJECT, {"project": project_id, "issue": issue_id}) + return gv(data, "data.addProjectNextItem.projectNextItem.id", None) + +# Get all issues from a project with all the extra data +def get_issues_from_project(org: str, project_id: int) -> List[Dict[str, Any]]: + data = graphql(ALL_ISSUES_IN_PROJECT, {"org": org, "number": project_id}) + issues = [] + for node in gv(data, "data.organization.projectV2.items.nodes", []): + idata = { + "milestone": gv(node, "content.milestone.number", 0), + "number": gv(node, "content.number", 0), + "title": gv(node, "content.title", ""), + "state": gv(node, "content.state", ""), + "id": gv(node, "id", None), + "pr_url": [], + "pr": [], + "url": gv(node, "content.url", None), + "status": None, + } + + for field in gv(node, "fieldValues.nodes", []): + if "pullRequests" in field: + for pr in gv(field, "pullRequests.nodes", []): + idata["pr_url"].append(pr["url"]) + idata["pr"].append(str(pr["number"])) + + if "field" in field and gv(field, "field.name") == "Status": + idata["status"] = field["name"] + + issues.append(idata) + + return issues + +# Set the status of a item in the project +def set_project_status(project_id:str, issue_id:str, status_id:str, new_status:str) -> str: + # $project: ID!, $item: ID!, $status_field: ID!, $status_value: String! + data = graphql(SET_PROJECT_STATUS, {"project": project_id, "item": issue_id, "status_field": status_id, "status_value": new_status}) + return gv(data, "data.updateProjectNxetItemField.projectNextItem.id", "") + +def list_command(args: Any): + project_id = None + status_to_set = None + status_id = None + if args.set_status: + project_id, fields = project_id_and_fields(args.projectnum) + status_id = gv(fields, "Status.id", None) + possible_status = gv(fields, "Status.options") + if not args.set_status in possible_status: + print(f"Can't set status to {args.set_status} - possible values are: {', '.join(possible_status.keys())}") + sys.exit(1) + + status_to_set = possible_status[args.set_status] + + issues = get_issues_from_project(args.org, args.projectnum) + + filtered_issues = issues + if args.milestone or args.status or args.has_pr: + if args.milestone: + filtered_issues = [iss for iss in filtered_issues if iss['milestone'] == args.milestone] + if args.status: + filtered_issues = [iss for iss in filtered_issues if iss['status'] == args.status] + if args.state: + filtered_issues = [iss for iss in filtered_issues if iss['state'] == args.state] + if args.has_pr: + filtered_issues = [iss for iss in filtered_issues if len(iss['pr']) > 0] + + for iss in filtered_issues: + if args.output: + stuff = args.output.split(",") + output = [] + for s in stuff: + val = iss[s] + if isinstance(val, list): + output.append("|".join(val)) + else: + output.append(str(val)) + + print(";".join(output)) + else: + pr = ', '.join([p for p in iss["pr_url"]]) + if pr: + pr = f" PR: {pr}\n" + print(f"{iss['number']}: {iss['title']}\n State: {iss['state']}\n URL: {iss['url']}\n milestone: {iss['milestone']}\n{pr} Status: {iss['status']}") + + if project_id and status_to_set and status_id: + print(f"Setting status to {args.set_status} ...\n") + set_project_status(project_id, iss['id'], status_id, status_to_set) + +def search_command(args: Any): + issues = search_issues(args.query) + project_id = None + for iss in issues: + if args.add_to_project: + if not project_id: + project_id, _ = project_id_and_fields(args.projectnum) + print(f"Adding {iss['number']} to project {args.projectnum}...") + add_issue_to_project(project_id, iss["id"]) + else: + print(f"{iss['number']}: {iss['title']}\n URL: {iss['url']}") + +def search_add_command(args: Any): + issues = search_issues(args.query) + project_id, _ = project_id_and_fields(args.projectnum) + for iss in issues: + print(f"Addding {iss['number']} to {args.projectnum} ({project_id})") + add_issue_to_project(project_id, iss['id']) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument("--token", required=True, help="GitHub token") + parser.add_argument("--org", default="llvm", help="Organiziation to query") + parser.add_argument("--projectnum", default=3, help="The GitHub project id") + + commands = parser.add_subparsers(dest="command") + + listparser = commands.add_parser("list", help="list issues from a project") + listparser.add_argument("--milestone", default=None, type=int, help="List issues in milestone") + listparser.add_argument("--status", default=None, type=str, help="List issues with project status") + listparser.add_argument("--output", default=None, type=str, help="Output these fields, can be title, url, pr, pr_url, id") + listparser.add_argument("--has-pr", default=False, action="store_true", help="List issues with a linked PR") + listparser.add_argument("--state", default=None, type=str, help="List issues that has this state, can be OPEN or CLOSED") + listparser.add_argument("--set-status", default=None, type=str, help="Set the project status of issues matched by this list.") + + searchparser = commands.add_parser("search", help="Run a GitHub search") + searchparser.add_argument("--query", required=True, help="GitHub search query - uses the same syntax as the website") + searchparser.add_argument("--add-to-project", default=False, action="store_true", help="Add all the issues matched to the project defined with --projectnum") + + args = parser.parse_args() + + session.headers["Authorization"] = "Bearer " + args.token + + if args.command == "list": + list_command(args) + elif args.command == "search": + search_command(args) diff --git a/llvm/utils/git/sync-release-repo.sh b/llvm/utils/git/sync-release-repo.sh new file mode 100755 --- /dev/null +++ b/llvm/utils/git/sync-release-repo.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e +set -x + +BRANCH="release/15.x" +REPO="https://github.com/llvm/llvm-project" + +git config remote.upstream.url >&- || git remote add upstream $REPO + +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# Make sure we are up to date on all our repos first +git fetch --all + +# Start by resetting to the upstream release branch - this is the most important +# branch always +git switch -c sync-repos upstream/$BRANCH +git merge --ff-only origin/$BRANCH + +if ! git diff-index --quiet origin/$BRANCH; then + echo "Changes in origin - pushing to upstream" + git push origin sync-repos:$BRANCH +fi + +# Then we need to update again +git fetch --all + +# And merge all the new data to the current branch +git merge --ff-only upstream/$BRANCH + +# If anything changed let's merge it +if ! git diff-index --quiet upstream/$BRANCH; then + echo "Changes in upstream - pushing to origin" + git push upstream sync-repos:$BRANCH +fi + +# Done - let's clean up +git switch $CURRENT_BRANCH + +if git diff-index --quiet HEAD; then + git reset --hard @{u} +fi + +git branch -d sync-repos