On Wed, Aug 30, 2017 at 1:58 PM, Sage Weil <sweil@xxxxxxxxxx> wrote: > I got tired of doing this by hand and finally wrote a script to do it > instead: > > https://github.com/ceph/ceph/pull/17382 > > Go tag your PRs with wip-whatever-testing, then > > git checkout master > git pull > ../src/script/build-integration-branch wip-whatever-testing > make && ctest -j12 && git push ci $(git rev-parse --abbrev-ref HEAD) I figure I'll share my own integration branch script (attached) which is based on some work John Spray did. It's probably too specific to my own workflow but pieces can be pulled out to suit others' needs. How it's used: $ python2 ptl_tool.py pr#1 pr#2 [...] which does: 1) Builds a branch based on refs/upstream/heads/master named wip-$NAME-testing-$DATE. 2) Fetches comments/reviews from GitHub for each PR, looking for review approvals for Reviewed-bys. 3) Merges refs/upstream/pull/pr#/head with a commit message including each commit's title and "Reviewed-by". 4) Labels each merged PR with "wip-$NAME-testing". 5) Tags the integration branch locally for posterity. You can also then merge into master or a release branch using: $ python2 ptl_tool.py master pr#1 [...] and the commit message is adjusted accordingly. (Also, no testing label is applied as it's not an integration branch.) Some of the quirks to my setup: o I don't have a master branch in my local repository as I don't find it useful. I use detached HEADs when merging stuff into master and then push to upstream. o I pull all of the upstream references: [remote "upstream"] url = git@xxxxxxxxxx:ceph/ceph.git fetch = +refs/*:refs/remotes/upstream/* Hope this is useful to someone. -- Patrick Donnelly
# TODO # Look for check failures? # redmine issue update: http://www.redmine.org/projects/redmine/wiki/Rest_Issues import json import re import requests import sys import git import datetime import logging from os.path import expanduser log = logging.getLogger(__name__) log.addHandler(logging.StreamHandler()) log.setLevel(logging.INFO) BASE = "refs/remotes/upstream/heads/%s" USER = "pdonnell" with open(expanduser("~/.github.key")) as f: PASSWORD = f.read().strip() BRANCH_PREFIX = "wip-%s-testing-" % USER TESTING_LABEL = ["wip-%s-testing" % USER] SPECIAL_BRANCHES = ('master', 'luminous', 'jewel', 'HEAD') INDICATIONS = [ re.compile("(Reviewed-by: .+ <[\w@.-]+>)", re.IGNORECASE), re.compile("(Acked-by: .+ <[\w@.-]+>)", re.IGNORECASE), re.compile("(Tested-by: .+ <[\w@.-]+>)", re.IGNORECASE), ] APPROVAL_TO_REVIEWED_BY = { "ajarr": "Reviewed-by: Ramana Raja <rraja@xxxxxxxxxx>", "batrick": "Reviewed-by: Patrick Donnelly <pdonnell@xxxxxxxxxx>", "fullerdj": "Reviewed-by: Douglas Fuller <dfuller@xxxxxxxxxx>", "gregsfortytwo": "Reviewed-by: Gregory Farnum <gfarnum@xxxxxxxxxx>", "jcsp": "Reviewed-by: John Spray <john.spray@xxxxxxxxxx>", "jlayton": "Reviewed-by: Jeff Layton <jlayton@xxxxxxxxxx>", "joscollin": "Reviewed-by: Jos Collin <jcollin@xxxxxxxxxx>", "liewegas": "Reviewed-by: Sage Weil <sage@xxxxxxxxxx>", "renhwztetecs": "Reviewed-by: huanwen ren <ren.huanwen@xxxxxxxxxx>", "smithfarm": "Reviewed-by: Nathan Cutler <ncutler@xxxxxxxx>", "tchaikov": "Reviewed-by: Kefu Chai <kchai@xxxxxxxxxx>", "theanalyst": "Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@xxxxxxxxx>", "ukernel": "Reviewed-by: Zheng Yan <zyan@xxxxxxxxxx>", } def build_branch(branch_name, pull_requests): repo = git.Repo(".") repo.remotes.upstream.fetch() # First get the latest base branch from upstream if branch_name == 'HEAD': log.info("Branch base is HEAD; not checking out!") else: if branch_name in SPECIAL_BRANCHES: base = BASE % branch_name else: base = BASE % "master" log.info("Branch base on {}".format(base)) base = filter(lambda r: r.path == base, repo.refs)[0] # So we know that we're not on an old test branch, detach HEAD onto ref: base.checkout() # If the branch is master, leave HEAD detached (but use "master" for commit message) created = False if branch_name not in SPECIAL_BRANCHES: # Delete test branch if it already existed try: getattr(repo.branches, branch_name).delete( repo, getattr(repo.branches, branch_name), force=True) log.info("Deleted old test branch %s" % branch_name) except AttributeError: pass log.info("Creating branch {branch_name}".format(branch_name=branch_name)) base.checkout(b=branch_name) created = True for pr in pull_requests: log.info("Merging PR {pr}".format(pr=pr)) pr = int(pr) r = filter(lambda r: r.path == "refs/remotes/upstream/pull/%d/head" % pr, repo.refs)[0] message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, branch_name, r.path) for commit in repo.iter_commits(rev="HEAD.."+r.path): message = message + ("\t%s\n" % commit.message.split('\n', 1)[0]) message = message + "\n" comments = requests.get("https://api.github.com/repos/ceph/ceph/issues/{pr}/comments".format(pr=pr), auth=(USER, PASSWORD)) if comments.status_code != 200: log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments)) return reviews = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}/reviews".format(pr=pr), auth=(USER, PASSWORD)) if reviews.status_code != 200: log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments)) return review_comments = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}/comments".format(pr=pr), auth=(USER, PASSWORD)) if review_comments.status_code != 200: log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments)) return indications = set() for comment in comments.json()+review_comments.json(): for indication in INDICATIONS: for cap in indication.findall(comment["body"]): indications.add(cap) for review in reviews.json(): if review["state"] == "APPROVED": indications.add(APPROVAL_TO_REVIEWED_BY[review["user"]["login"]]) for indication in indications: message = message + indication + "\n" repo.git.merge(r, '--no-ff', m=message) if branch_name not in SPECIAL_BRANCHES: req = requests.post("https://api.github.com/repos/ceph/ceph/issues/{pr}/labels".format(pr=pr), data=json.dumps(TESTING_LABEL), auth=(USER, PASSWORD)) if req.status_code != 200: log.error("PR #%d could not be labeled %s: %s" % (pr, wip, req)) return # If we created a branch, tag it for future reference. if created: name = "testing/%s" % branch_name log.info("Creating tag %s" % name) git.refs.tag.Tag.create(repo, name, force=True) if __name__ == "__main__": if sys.argv[1] in SPECIAL_BRANCHES: branch_name = sys.argv[1] pull_requests = sys.argv[2:] else: branch_name = BRANCH_PREFIX + datetime.datetime.now().strftime("%Y%m%d") pull_requests = sys.argv[1:] build_branch(branch_name, pull_requests)
