From 61327de90dffc54018ad0eaa1536f4eed305b5f5 Mon Sep 17 00:00:00 2001 From: "David V. Lu" Date: Thu, 1 Feb 2018 15:19:48 -0500 Subject: [PATCH 1/6] Guess gitlab path, use distro as track --- bloom/commands/release.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/bloom/commands/release.py b/bloom/commands/release.py index d3f4fe7..7b3ce9a 100644 --- a/bloom/commands/release.py +++ b/bloom/commands/release.py @@ -44,6 +44,7 @@ import os import pkg_resources import platform +import requests import shutil import subprocess import sys @@ -295,6 +296,31 @@ def validate_github_url(url, url_type): return True +GIT_TOKENS_PATH = os.path.expanduser('~/.git-tokens') +LOCUS_GITLAB = 'http://gitlab-ci-token:{}@gitlab.locusbots.io/locusrobotics/{}.git' +_git_tokens = None + + +def get_git_tokens(): + global _git_tokens + if _git_tokens: + return _git_tokens + if not os.path.exists(GIT_TOKENS_PATH): + return + _git_tokens = yaml.load(open(GIT_TOKENS_PATH)) + return _git_tokens + + +def guess_locus_release_repo(repository): + git_tokens = get_git_tokens() + if not git_tokens or 'gitlab' not in git_tokens: + return + url = LOCUS_GITLAB.format(git_tokens['gitlab'], repository + '-release') + r = requests.get(url) + if r.status_code == requests.codes.ok: + return url + + def get_repo_uri(repository, distro): url = None # Fetch the distro file @@ -308,6 +334,9 @@ def get_repo_uri(repository, distro): matches = difflib.get_close_matches(repository, distribution_file.repositories) if matches: info(fmt("@{yf}Did you mean one of these: '" + "', '".join([m for m in matches]) + "'?")) + locus_url = guess_locus_release_repo(repository) + if locus_url: + return locus_url if url is None: info("Could not determine release repository url for repository '{0}' of distro '{1}'" .format(repository, distro)) @@ -1258,7 +1287,7 @@ def get_argument_parser(): add('repository', help="repository to run bloom on") add('--list-tracks', '-l', action='store_true', default=False, help="list available tracks for repository") - add('--track', '-t', required=True, help="track to run") + add('--track', '-t', required=False, help="track to run") add('--non-interactive', '-y', action='store_true', default=False) add('--ros-distro', '--rosdistro', '-r', required=True, help="determines the ROS distro file used") @@ -1285,6 +1314,8 @@ def main(sysargs=None): parser = get_argument_parser() parser = add_global_arguments(parser) args = parser.parse_args(sysargs) + if args.track is None: + args.track = args.ros_distro handle_global_arguments(args) if args.list_tracks: From ba2e38319d134b8b7c0c0f3daefa602b22a191b6 Mon Sep 17 00:00:00 2001 From: "David V. Lu" Date: Thu, 1 Feb 2018 16:16:22 -0500 Subject: [PATCH 2/6] Make gitlab pull request --- bloom/commands/release.py | 266 ++++++++++++++++++++++++-------------- 1 file changed, 172 insertions(+), 94 deletions(-) diff --git a/bloom/commands/release.py b/bloom/commands/release.py index 7b3ce9a..4937c01 100644 --- a/bloom/commands/release.py +++ b/bloom/commands/release.py @@ -663,7 +663,20 @@ def get_gh_info(url): return url_paths[1], url_paths[2], url_paths[3], '/'.join(url_paths[4:]) +def get_gl_info(url): + # returns base_org, base_repo, base_branch, base_path + o = urlparse(url) + if 'gitlab' not in o.netloc: + return None, None, None, None, None + server = '{}://{}'.format(o.scheme, o.netloc) + url_paths = o.path.split('/') + if len(url_paths) < 6: + return None, None, None, None, None + return server, url_paths[1], url_paths[2], url_paths[4], '/'.join(url_paths[5:]) + + _gh = None +_gl = None def get_github_interface(quiet=False): @@ -736,6 +749,24 @@ def get_github_interface(quiet=False): return gh +def get_gitlab_interface(server, quiet=False): + global _gl + if _gl is not None: + return _gl + # Make sure we can import gitlab + try: + import gitlab + except: + return None + + # Grab the token + git_tokens = get_git_tokens() + if not git_tokens or 'gitlab' not in git_tokens: + return + _gl = gitlab.Gitlab(server, private_token=git_tokens['gitlab'], api_version=4) + return _gl + + def get_changelog_summary(release_tag): summary = u"" packages = dict([(p.name, p) for p in get_packages().values()]) @@ -785,60 +816,23 @@ def open_pull_request(track, repository, distro, interactive, override_release_r return None version = updated_distribution_file.repositories[repository].release_repository.version updated_distro_file_yaml = yaml_from_distribution_file(updated_distribution_file) - # Determine if the distro file is hosted on github... - base_org, base_repo, base_branch, base_path = get_gh_info(get_distribution_file_url(distro)) - if None in [base_org, base_repo, base_branch, base_path]: - warning("Automated pull request only available via github.com") - return + + # Determine where the distro file is hosted... + distro_url = get_distribution_file_url(distro) + base_org, base_repo, base_branch, base_path = get_gh_info(distro_url) + if None not in [base_org, base_repo, base_branch, base_path]: + server = 'http://github.com' + else: + server, base_org, base_repo, base_branch, base_path = get_gl_info(distro_url) + if None in [server, base_org, base_repo, base_branch, base_path]: + warning("Automated pull request only available via github.com or gitlab") + return + # If we did replace the branch in the url with a commit, restore that now if _rosdistro_index_original_branch is not None: base_branch = _rosdistro_index_original_branch - # Get the github interface - gh = get_github_interface() - if gh is None: - return None - # Determine the head org/repo for the pull request - head_org = gh.username # The head org will always be gh user - head_repo = None - # Check if the github user and the base org are the same - if gh.username == base_org: - # If it is, then a fork is not necessary - head_repo = gh.get_repo(base_org, base_repo) - else: - info(fmt("@{bf}@!==> @|@!Checking on GitHub for a fork to make the pull request from...")) - # It is not, so a fork will be required - # Check if a fork already exists on the user's account - try: - repo_forks = gh.list_forks(base_org, base_repo) - user_forks = [r for r in repo_forks if r.get('owner', {}).get('login', '') == gh.username] - # github allows only 1 fork per org as far as I know. We just take the first one. - head_repo = user_forks[0] if user_forks else None - - except GithubException as exc: - debug("Received GithubException while checking for fork: {exc}".format(**locals())) - pass # 404 or unauthorized, but unauthorized should have been caught above - - # If not head_repo still, a fork does not exist and must be created - if head_repo is None: - warning("Could not find a fork of {base_org}/{base_repo} on the {gh.username} GitHub account." - .format(**locals())) - warning("Would you like to create one now?") - if not maybe_continue(): - warning("Skipping the pull request...") - return - # Create a fork - try: - head_repo = gh.create_fork(base_org, base_repo) # Will raise if not successful - except GithubException as exc: - error("Aborting pull request: {0}".format(exc)) - return - head_repo = head_repo.get('name', '') - info(fmt("@{bf}@!==> @|@!" + - "Using this fork to make a pull request from: {head_org}/{head_repo}".format(**locals()))) - # Clone the fork - info(fmt("@{bf}@!==> @|@!" + "Cloning {0}/{1}...".format(head_org, head_repo))) - new_branch = None + # Create content for PR title = "{0}: {1} in '{2}' [bloom]".format(repository, version, base_path) track_dict = get_tracks_dict_raw()['tracks'][track] body = u"""\ @@ -859,50 +853,134 @@ def open_pull_request(track, repository, distro, interactive, override_release_r release_repo=updated_distribution_file.repositories[repository].release_repository.url, ) body += get_changelog_summary(generate_release_tag(distro)) - with temporary_directory() as temp_dir: - def _my_run(cmd, msg=None): - if msg: - info(fmt("@{bf}@!==> @|@!" + sanitize(msg))) - else: - info(fmt("@{bf}@!==> @|@!" + sanitize(str(cmd)))) - from subprocess import check_call - check_call(cmd, shell=True) - # Use the oauth token to clone - rosdistro_url = 'https://{gh.token}:x-oauth-basic@github.com/{base_org}/{base_repo}.git'.format(**locals()) - rosdistro_fork_url = 'https://{gh.token}:x-oauth-basic@github.com/{head_org}/{head_repo}.git'.format(**locals()) - _my_run('mkdir -p {base_repo}'.format(**locals())) - with change_directory(base_repo): - _my_run('git init') - branches = [x['name'] for x in gh.list_branches(head_org, head_repo)] - new_branch = 'bloom-{repository}-{count}' - count = 0 - while new_branch.format(repository=repository, count=count) in branches: - count += 1 - new_branch = new_branch.format(repository=repository, count=count) - # Final check - info(fmt("@{cf}Pull Request Title: @{yf}" + sanitize(title))) - info(fmt("@{cf}Pull Request Body : \n@{yf}" + sanitize(body))) - msg = fmt("@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}" + - "{head_org}/{head_repo}:{new_branch}".format(**locals()) + - "@|@!' @!@{kf}into@| @!'@|@!@{bf}" + - "{base_org}/{base_repo}:{base_branch}".format(**locals()) + - "@|@!'?") - info(msg) - if interactive and not maybe_continue(): - warning("Skipping the pull request...") - return - _my_run('git checkout -b {new_branch}'.format(**locals())) - _my_run('git pull {rosdistro_url} {base_branch}'.format(**locals()), "Pulling latest rosdistro branch") - if _rosdistro_index_commit is not None: - _my_run('git reset --hard {_rosdistro_index_commit}'.format(**globals())) - with open('{0}'.format(base_path), 'w') as f: - info(fmt("@{bf}@!==> @|@!Writing new distribution file: ") + str(base_path)) - f.write(updated_distro_file_yaml) - _my_run('git add {0}'.format(base_path)) - _my_run('git commit -m "{0}"'.format(title)) - _my_run('git push {rosdistro_fork_url} {new_branch}'.format(**locals()), "Pushing changes to fork") - # Open the pull request - return gh.create_pull_request(base_org, base_repo, base_branch, head_org, new_branch, title, body) + + if server == 'http://github.com': + # Get the github interface + gh = get_github_interface() + if gh is None: + return None + # Determine the head org/repo for the pull request + head_org = gh.username # The head org will always be gh user + head_repo = None + # Check if the github user and the base org are the same + if gh.username == base_org: + # If it is, then a fork is not necessary + head_repo = gh.get_repo(base_org, base_repo) + else: + info(fmt("@{bf}@!==> @|@!Checking on GitHub for a fork to make the pull request from...")) + # It is not, so a fork will be required + # Check if a fork already exists on the user's account + + try: + repo_forks = gh.list_forks(base_org, base_repo) + user_forks = [r for r in repo_forks if r.get('owner', {}).get('login', '') == gh.username] + # github allows only 1 fork per org as far as I know. We just take the first one. + head_repo = user_forks[0] if user_forks else None + + except GithubException as exc: + debug("Received GithubException while checking for fork: {exc}".format(**locals())) + pass # 404 or unauthorized, but unauthorized should have been caught above + + # If not head_repo still, a fork does not exist and must be created + if head_repo is None: + warning("Could not find a fork of {base_org}/{base_repo} on the {gh.username} GitHub account." + .format(**locals())) + warning("Would you like to create one now?") + if not maybe_continue(): + warning("Skipping the pull request...") + return + # Create a fork + try: + head_repo = gh.create_fork(base_org, base_repo) # Will raise if not successful + except GithubException as exc: + error("Aborting pull request: {0}".format(exc)) + return + head_repo = head_repo.get('name', '') + info(fmt("@{bf}@!==> @|@!" + + "Using this fork to make a pull request from: {head_org}/{head_repo}".format(**locals()))) + # Clone the fork + info(fmt("@{bf}@!==> @|@!" + "Cloning {0}/{1}...".format(head_org, head_repo))) + new_branch = None + + with temporary_directory() as temp_dir: + def _my_run(cmd, msg=None): + if msg: + info(fmt("@{bf}@!==> @|@!" + sanitize(msg))) + else: + info(fmt("@{bf}@!==> @|@!" + sanitize(str(cmd)))) + from subprocess import check_call + check_call(cmd, shell=True) + # Use the oauth token to clone + rosdistro_url = 'https://{gh.token}:x-oauth-basic@github.com/{base_org}/{base_repo}.git'.format(**locals()) + rosdistro_fork_url = 'https://{gh.token}:x-oauth-basic@github.com/{head_org}/{head_repo}.git'.format(**locals()) + _my_run('mkdir -p {base_repo}'.format(**locals())) + with change_directory(base_repo): + _my_run('git init') + branches = [x['name'] for x in gh.list_branches(head_org, head_repo)] + new_branch = 'bloom-{repository}-{count}' + count = 0 + while new_branch.format(repository=repository, count=count) in branches: + count += 1 + new_branch = new_branch.format(repository=repository, count=count) + # Final check + info(fmt("@{cf}Pull Request Title: @{yf}" + sanitize(title))) + info(fmt("@{cf}Pull Request Body : \n@{yf}" + sanitize(body))) + msg = fmt("@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}" + + "{head_org}/{head_repo}:{new_branch}".format(**locals()) + + "@|@!' @!@{kf}into@| @!'@|@!@{bf}" + + "{base_org}/{base_repo}:{base_branch}".format(**locals()) + + "@|@!'?") + info(msg) + if interactive and not maybe_continue(): + warning("Skipping the pull request...") + return + _my_run('git checkout -b {new_branch}'.format(**locals())) + _my_run('git pull {rosdistro_url} {base_branch}'.format(**locals()), "Pulling latest rosdistro branch") + if _rosdistro_index_commit is not None: + _my_run('git reset --hard {_rosdistro_index_commit}'.format(**globals())) + with open('{0}'.format(base_path), 'w') as f: + info(fmt("@{bf}@!==> @|@!Writing new distribution file: ") + str(base_path)) + f.write(updated_distro_file_yaml) + _my_run('git add {0}'.format(base_path)) + _my_run('git commit -m "{0}"'.format(title)) + _my_run('git push {rosdistro_fork_url} {new_branch}'.format(**locals()), "Pushing changes to fork") + # Open the pull request + return gh.create_pull_request(base_org, base_repo, base_branch, head_org, new_branch, title, body) + else: + gl = get_gitlab_interface(server) + if gl is None: + return None + + project = gl.projects.get('{}/{}'.format(base_org, base_repo)) + + # Determine New Branch Name + branches = [x.name for x in project.branches.list()] + new_branch = 'bloom-{repository}-{count}' + count = 0 + while new_branch.format(repository=repository, count=count) in branches: + count += 1 + new_branch = new_branch.format(repository=repository, count=count) + + branch = project.branches.create({'branch': new_branch, 'ref': base_branch}) + data = { + 'branch': new_branch, # v4 + 'commit_message': title, + 'actions': [ + { + 'action': 'update', + 'file_path': base_path, + 'content': updated_distro_file_yaml + } + ] + } + + commit = project.commits.create(data) + mr = project.mergerequests.create({'source_branch': new_branch, + 'target_branch': base_branch, + 'title': title, + 'description': body}) + return mr.web_url + _original_version = None From 194ef2936795a6102fcae90f4ff72241e1b141c7 Mon Sep 17 00:00:00 2001 From: "David V. Lu" Date: Fri, 2 Feb 2018 09:59:21 -0500 Subject: [PATCH 3/6] Move gitlab import --- bloom/commands/release.py | 6 +----- setup.py | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bloom/commands/release.py b/bloom/commands/release.py index 4937c01..8388531 100644 --- a/bloom/commands/release.py +++ b/bloom/commands/release.py @@ -39,6 +39,7 @@ import base64 import datetime import difflib +import gitlab import getpass import json import os @@ -753,11 +754,6 @@ def get_gitlab_interface(server, quiet=False): global _gl if _gl is not None: return _gl - # Make sure we can import gitlab - try: - import gitlab - except: - return None # Grab the token git_tokens = get_git_tokens() diff --git a/setup.py b/setup.py index c7a6a2a..3b3f239 100755 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ 'setuptools', 'empy', 'python-dateutil', + 'python-gitlab', 'PyYAML', 'rosdep >= 0.10.25', 'rosdistro >= 0.4.0', From faf82e52e6829eeaffa94d832b6732079b31ccdb Mon Sep 17 00:00:00 2001 From: "David V. Lu" Date: Fri, 2 Feb 2018 12:58:16 -0500 Subject: [PATCH 4/6] Check .config/bloom for token --- bloom/commands/release.py | 69 +++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/bloom/commands/release.py b/bloom/commands/release.py index 8388531..19f59e1 100644 --- a/bloom/commands/release.py +++ b/bloom/commands/release.py @@ -680,20 +680,28 @@ def get_gl_info(url): _gl = None -def get_github_interface(quiet=False): - global _gh - if _gh is not None: - return _gh - # First check to see if the oauth token is stored +def get_bloom_config_and_path(): oauth_config_path = os.path.join(os.path.expanduser('~'), '.config', 'bloom') config = {} if os.path.exists(oauth_config_path): with open(oauth_config_path, 'r') as f: config = json.loads(f.read()) - token = config.get('oauth_token', None) - username = config.get('github_user', None) - if token and username: - return Github(username, auth=auth_header_from_oauth_token(token), token=token) + return config, oauth_config_path + + + +def get_github_interface(quiet=False): + global _gh + if _gh is not None: + return _gh + # First check to see if the oauth token is stored + config, oauth_config_path = get_bloom_config_and_path() + token = config.get('oauth_token', None) + username = config.get('github_user', None) + + if token and username: + return Github(username, auth=auth_header_from_oauth_token(token), token=token) + if not os.path.isdir(os.path.dirname(oauth_config_path)): os.makedirs(os.path.dirname(oauth_config_path)) if quiet: @@ -755,11 +763,44 @@ def get_gitlab_interface(server, quiet=False): if _gl is not None: return _gl - # Grab the token - git_tokens = get_git_tokens() - if not git_tokens or 'gitlab' not in git_tokens: - return - _gl = gitlab.Gitlab(server, private_token=git_tokens['gitlab'], api_version=4) + config, oauth_config_path = get_bloom_config_and_path() + if 'gitlab' in config: + _gl = gitlab.Gitlab(server, private_token=config['gitlab'], api_version=4) + return _gl + + if quiet: + return None + + info("") + warning("Looks like bloom doesn't have a gitlab token for you yet.") + warning("Go to http://{}/profile/personal_access_tokens to create one.".format(server)) + warning("Make sure you give it API access.") + warning("The token will be stored in `~/.config/bloom`.") + warning("You can delete the token from that file to have a new token generated.") + warning("Guard this token like a password, because it allows someone/something to act on your behalf.") + info("") + if not maybe_continue('y', "Would you like to input a token now"): + return None + token = None + while token is None: + try: + token = safe_input("Gitlab Token: ") + except (KeyboardInterrupt, EOFError): + return None + try: + gl = gitlab.Gitlab(server, private_token=token, api_version=4) + gl.auth() + with open(oauth_config_path, 'a') as f: + config.update({'gitlab': token}) + f.write(json.dumps(config)) + info("The token was stored in the bloom config file") + _gl = gl + break + except gitlab.exceptions.GitlabAuthenticationError: + error("Failed to authenticate your token.") + if not maybe_continue(): + return None + return _gl From 57050d687daacc8b2f9e49da54213f685969e02c Mon Sep 17 00:00:00 2001 From: "David V. Lu" Date: Fri, 2 Feb 2018 13:07:46 -0500 Subject: [PATCH 5/6] Use BLOOM_RELEASE_REPO_BASE --- bloom/commands/release.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/bloom/commands/release.py b/bloom/commands/release.py index 19f59e1..6dd0f33 100644 --- a/bloom/commands/release.py +++ b/bloom/commands/release.py @@ -297,26 +297,11 @@ def validate_github_url(url, url_type): return True -GIT_TOKENS_PATH = os.path.expanduser('~/.git-tokens') -LOCUS_GITLAB = 'http://gitlab-ci-token:{}@gitlab.locusbots.io/locusrobotics/{}.git' -_git_tokens = None - - -def get_git_tokens(): - global _git_tokens - if _git_tokens: - return _git_tokens - if not os.path.exists(GIT_TOKENS_PATH): - return - _git_tokens = yaml.load(open(GIT_TOKENS_PATH)) - return _git_tokens - - -def guess_locus_release_repo(repository): - git_tokens = get_git_tokens() - if not git_tokens or 'gitlab' not in git_tokens: - return - url = LOCUS_GITLAB.format(git_tokens['gitlab'], repository + '-release') +def infer_release_repo_from_env(repository): + base = os.environ.get('BLOOM_RELEASE_REPO_BASE', None) + if base is None: + return None + url = base + repository + '-release.git' r = requests.get(url) if r.status_code == requests.codes.ok: return url @@ -335,9 +320,8 @@ def get_repo_uri(repository, distro): matches = difflib.get_close_matches(repository, distribution_file.repositories) if matches: info(fmt("@{yf}Did you mean one of these: '" + "', '".join([m for m in matches]) + "'?")) - locus_url = guess_locus_release_repo(repository) - if locus_url: - return locus_url + if url is None: + url = infer_release_repo_from_env(repository) if url is None: info("Could not determine release repository url for repository '{0}' of distro '{1}'" .format(repository, distro)) From 1ba4a82bde067ed710dbb5116bd5a710b1790999 Mon Sep 17 00:00:00 2001 From: "David V. Lu" Date: Mon, 5 Feb 2018 10:45:52 -0500 Subject: [PATCH 6/6] Remove lime break --- bloom/commands/release.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bloom/commands/release.py b/bloom/commands/release.py index 6dd0f33..053c947 100644 --- a/bloom/commands/release.py +++ b/bloom/commands/release.py @@ -673,7 +673,6 @@ def get_bloom_config_and_path(): return config, oauth_config_path - def get_github_interface(quiet=False): global _gh if _gh is not None: