diff --git a/README.md b/README.md index bb32844f..912582ec 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Supported Platforms: -------------------- * Gentoo * OpenEmbedded + * Nix Installation: ============= diff --git a/setup.py b/setup.py index 0dc34dcd..0e61a3f9 100755 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ def append_local_version_label(public_version): 'console_scripts': [ 'superflore-gen-ebuilds = superflore.generators.ebuild:main', 'superflore-gen-oe-recipes = superflore.generators.bitbake:main', + 'superflore-gen-nix = superflore.generators.nix:main', 'superflore-check-ebuilds = superflore.test_integration.gentoo:main', ] } diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index 88c9c4cd..7ba12a59 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -150,11 +150,12 @@ def main(): delta = "Regenerated: '%s'\n" % args.only overlay.add_generated_files(args.ros_distro) commit_msg = '\n'.join([get_pr_text( - title + '\n' + pr_comment.replace( - '**superflore**', 'superflore'), markup=''), delta]) + comment=title + '\n' + pr_comment.replace( + '**superflore**', 'superflore'), + markup=''), delta]) overlay.commit_changes(args.ros_distro, commit_msg) if args.dry_run: - save_pr(overlay, args.only, '', pr_comment, title=title) + save_pr(overlay, delta, '', pr_comment, title=title) sys.exit(0) file_pr(overlay, delta, '', pr_comment, distro=args.ros_distro, title=title) @@ -216,7 +217,8 @@ def main(): args.ros_distro, now) commit_msg = '\n'.join([get_pr_text( - title + '\n' + pr_comment.replace('**superflore**', 'superflore'), + comment=title + '\n' + + pr_comment.replace('**superflore**', 'superflore'), markup=''), delta]) overlay.commit_changes(args.ros_distro, commit_msg) delta = gen_delta_msg(total_changes) diff --git a/superflore/generators/ebuild/run.py b/superflore/generators/ebuild/run.py index 866d4c52..2d22fdee 100644 --- a/superflore/generators/ebuild/run.py +++ b/superflore/generators/ebuild/run.py @@ -150,15 +150,15 @@ def main(): regen_dict[args.ros_distro] = to_commit overlay.regenerate_manifests(regen_dict) overlay.commit_changes(args.ros_distro) + delta = "Regenerated: '%s'\n" % args.only if args.dry_run: save_pr( overlay, - args.only, + delta, missing_deps=gen_missing_deps_msg(missing_depends), comment=pr_comment ) sys.exit(0) - delta = "Regenerated: '%s'\n" % args.only file_pr( overlay, delta, diff --git a/superflore/generators/nix/__init__.py b/superflore/generators/nix/__init__.py new file mode 100644 index 00000000..5e40da76 --- /dev/null +++ b/superflore/generators/nix/__init__.py @@ -0,0 +1,3 @@ +from superflore.generators.nix.run import main +if __name__ == '__main__': + main() diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py new file mode 100644 index 00000000..06eb4a68 --- /dev/null +++ b/superflore/generators/nix/gen_packages.py @@ -0,0 +1,110 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import re +from typing import Dict, Iterable + +from rosdistro import DistributionFile +from rosinstall_generator.distro import get_package_names +from superflore.exceptions import UnresolvedDependency +from superflore.generators.nix.nix_package import NixPackage +from superflore.generators.nix.nix_package_set import NixPackageSet +from superflore.utils import err +from superflore.utils import make_dir +from superflore.utils import ok +from superflore.utils import warn + +org = "Open Source Robotics Foundation" +org_license = "BSD" + +_version_regex = re.compile(r"version\s*=\s*\"([^\"]*)\"") + + +def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, + preserve_existing: bool, tar_dir: str, + sha256_cache: Dict[str, str]): + all_pkgs = set(get_package_names(distro)[0]) + + if pkg not in all_pkgs: + raise RuntimeError("Unknown package '{}'".format(pkg)) + + normalized_pkg = NixPackage.normalize_name(pkg) + + package_dir = os.path.join(overlay.repo.repo_dir, 'distros', distro.name, + normalized_pkg) + package_file = os.path.join(package_dir, 'default.nix') + make_dir(package_dir) + + # check for an existing package + existing = os.path.exists(package_file) + previous_version = None + + if preserve_existing and existing: + ok("derivation for package '{}' up to date, skipping...".format(pkg)) + return None, [], None + + if existing: + with open(package_file, 'r') as f: + existing_derivation = f.read() + version_match = _version_regex.search(existing_derivation) + if version_match: + try: + previous_version = version_match.group(1) + except IndexError: + pass + if not previous_version: + warn("Failed to extract previous package version") + + try: + current = NixPackage(pkg, distro, tar_dir, sha256_cache, all_pkgs) + except Exception as e: + err('Failed to generate derivation for package {}!'.format(pkg)) + raise e + + try: + derivation_text = current.derivation.get_text(org, org_license) + except UnresolvedDependency: + err("'Failed to resolve required dependencies for package {}!" + .format(pkg)) + unresolved = current.unresolved_dependencies + for dep in unresolved: + err(" unresolved: \"{}\"".format(dep)) + return None, unresolved, None + except Exception as e: + err('Failed to generate derivation for package {}!'.format(pkg)) + raise e + + ok("Successfully generated derivation for package '{}'.".format(pkg)) + try: + with open('{0}'.format(package_file), "w") as recipe_file: + recipe_file.write(derivation_text) + except Exception as e: + err("Failed to write derivation to disk!") + raise e + return current, previous_version, normalized_pkg + + +def regenerate_pkg_set(overlay, distro_name: str, pkg_names: Iterable[str]): + distro_dir = os.path.join(overlay.repo.repo_dir, 'distros', distro_name) + overlay_file = os.path.join(distro_dir, 'generated.nix') + make_dir(distro_dir) + + package_set = NixPackageSet(pkg_names) + + try: + with open(overlay_file, "w") as recipe_file: + recipe_file.write(package_set.get_text(org, org_license)) + except Exception as e: + err("Failed to write derivation to disk!") + raise e diff --git a/superflore/generators/nix/nix_expression.py b/superflore/generators/nix/nix_expression.py new file mode 100644 index 00000000..e096e6ec --- /dev/null +++ b/superflore/generators/nix/nix_expression.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2016 David Bensoussan, Synapticon GmbH +# Copyright (c) 2019 Open Source Robotics Foundation, Inc. +# +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from operator import attrgetter +import os +from textwrap import dedent +from time import gmtime, strftime +from typing import Iterable, Set +import urllib.parse + +from superflore.utils import get_license + + +def _escape_nix_string(string: str): + return '"{}"'.format(string.replace("\\", "\\\\") + .replace("${", r"\${") + .replace('"', r"\"")) + + +class NixLicense: + """ + Converts a ROS license to the correct Nix license attribute. + """ + + _LICENSE_MAP = { + 'Apache-2.0': 'asl20', + 'ASL 2.0': 'asl20', + 'Boost-1.0': 'boost', + 'BSD-2': 'bsd2', + 'BSD-3-Clause': 'bsd3', + 'BSD': 'bsdOriginal', + 'CC-BY-NC-SA-4.0': 'cc-by-nc-sa-40', + 'GPL-1': 'gpl1', + 'GPL-2': 'gpl2', + 'GPL-3.0-only': 'gpl3Only', + 'GPL-3': 'gpl3', + 'LGPL-2.1': 'lgpl21', + 'LGPL-2': 'lgpl2', + 'LGPL-3.0-only': 'lgpl3Only', + 'LGPL-3': 'lgpl3', + 'MIT': 'mit', + 'MPL-1.0': 'mpl10', + 'MPL-1.1': 'mpl11', + 'MPL-2.0': 'mpl20', + 'PD': 'publicDomain', + } + + def __init__(self, name): + try: + name = get_license(name) + self.name = self._LICENSE_MAP[name] + self.custom = False + except KeyError: + self.name = name + self.custom = True + + @property + def nix_code(self) -> str: + if self.custom: + return _escape_nix_string(self.name) + else: + return self.name + + +class NixExpression: + def __init__(self, name: str, version: str, + src_url: str, src_sha256: str, + description: str, licenses: Iterable[NixLicense], + distro_name: str, + build_type: str, + build_inputs: Set[str] = set(), + propagated_build_inputs: Set[str] = set(), + check_inputs: Set[str] = set(), + native_build_inputs: Set[str] = set(), + propagated_native_build_inputs: Set[str] = set() + ) -> None: + self.name = name + self.version = version + self.src_url = src_url + self.src_sha256 = src_sha256 + # fetchurl's naming logic cannot account for URL parameters + self.src_name = os.path.basename( + urllib.parse.urlparse(self.src_url).path) + + self.description = description + self.licenses = licenses + self.distro_name = distro_name + self.build_type = build_type + + self.build_inputs = build_inputs + self.propagated_build_inputs = propagated_build_inputs + self.check_inputs = check_inputs + self.native_build_inputs = native_build_inputs + self.propagated_native_build_inputs = \ + propagated_native_build_inputs + + @staticmethod + def _to_nix_list(it: Iterable[str]) -> str: + return '[ ' + ' '.join(it) + ' ]' + + @staticmethod + def _to_nix_parameter(dep: str) -> str: + return dep.split('.')[0] + + def get_text(self, distributor: str, license_name: str) -> str: + """ + Generate the Nix expression, given the distributor line + and the license text. + """ + + ret = [] + ret += dedent(''' + # Copyright {} {} + # Distributed under the terms of the {} license + + ''').format( + strftime("%Y", gmtime()), distributor, + license_name) + + ret += '{ lib, buildRosPackage, fetchurl, ' + \ + ', '.join(sorted(set(map(self._to_nix_parameter, + self.build_inputs | + self.propagated_build_inputs | + self.check_inputs | + self.native_build_inputs | + self.propagated_native_build_inputs))) + ) + ' }:' + + ret += dedent(''' + buildRosPackage {{ + pname = "ros-{distro_name}-{name}"; + version = "{version}"; + + src = fetchurl {{ + url = "{src_url}"; + name = "{src_name}"; + sha256 = "{src_sha256}"; + }}; + + buildType = "{build_type}"; + ''').format( + distro_name=self.distro_name, + name=self.name, + version=self.version, + src_url=self.src_url, + src_name=self.src_name, + src_sha256=self.src_sha256, + build_type=self.build_type) + + if self.build_inputs: + ret += " buildInputs = {};\n" \ + .format(self._to_nix_list(sorted(self.build_inputs))) + + if self.check_inputs: + ret += " checkInputs = {};\n" \ + .format(self._to_nix_list(sorted(self.check_inputs))) + + if self.propagated_build_inputs: + ret += " propagatedBuildInputs = {};\n" \ + .format(self._to_nix_list(sorted( + self.propagated_build_inputs))) + + if self.native_build_inputs: + ret += " nativeBuildInputs = {};\n" \ + .format(self._to_nix_list(sorted(self.native_build_inputs))) + + if self.propagated_native_build_inputs: + ret += " propagatedNativeBuildInputs = {};\n".format( + self._to_nix_list(sorted(self.propagated_native_build_inputs))) + + ret += dedent(''' + meta = {{ + description = {}; + license = with lib.licenses; {}; + }}; + }} + ''').format(_escape_nix_string(self.description), + self._to_nix_list(map(attrgetter('nix_code'), + self.licenses))) + + return ''.join(ret) diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py new file mode 100644 index 00000000..95df620b --- /dev/null +++ b/superflore/generators/nix/nix_package.py @@ -0,0 +1,176 @@ +import hashlib +import itertools +import os +import re +import tarfile +from typing import Dict, Iterable, Set + +from rosdistro import DistributionFile +from rosdistro.dependency_walker import DependencyWalker +from rosdistro.rosdistro import RosPackage +from rosinstall_generator.distro import _generate_rosinstall +from superflore.exceptions import UnresolvedDependency +from superflore.generators.nix.nix_expression import NixExpression, NixLicense +from superflore.PackageMetadata import PackageMetadata +from superflore.utils import (download_file, get_distro_condition_context, + get_distros, get_pkg_version, info, resolve_dep, + retry_on_exception, warn) + + +class NixPackage: + """ + Retrieves the required metadata to define a Nix package derivation. + """ + + def __init__(self, name: str, distro: DistributionFile, tar_dir: str, + sha256_cache: Dict[str, str], all_pkgs: Set[str]) -> None: + self.distro = distro + self._all_pkgs = all_pkgs + + pkg = distro.release_packages[name] + repo = distro.repositories[pkg.repository_name].release_repository + ros_pkg = RosPackage(name, repo) + + rosinstall = _generate_rosinstall(name, repo.url, + repo.get_release_tag(name), True) + + normalized_name = NixPackage.normalize_name(name) + version = get_pkg_version(distro, name) + src_uri = rosinstall[0]['tar']['uri'] + + archive_path = os.path.join(tar_dir, '{}-{}-{}.tar.gz' + .format(self.normalize_name(name), + version, distro.name)) + + downloaded_archive = False + if os.path.exists(archive_path): + info("using cached archive for package '{}'...".format(name)) + else: + info("downloading archive version for package '{}'..." + .format(name)) + retry_on_exception(download_file, src_uri, archive_path, + retry_msg="network error downloading '{}'" + .format(src_uri), + error_msg="failed to download archive for '{}'" + .format(name)) + downloaded_archive = True + + if downloaded_archive or archive_path not in sha256_cache: + sha256_cache[archive_path] = hashlib.sha256( + open(archive_path, 'rb').read()).hexdigest() + src_sha256 = sha256_cache[archive_path] + + # We already have the archive, so try to extract package.xml from it. + # This is much faster than downloading it from GitHub. + package_xml_regex = re.compile(r'^[^/]+/package\.xml$') + package_xml = None + archive = tarfile.open(archive_path, 'r|*') + for file in archive: + if package_xml_regex.match(file.name): + package_xml = archive.extractfile(file).read() + break + # Fallback to the standard method of fetching package.xml + if package_xml is None: + warn("failed to extract package.xml from archive") + package_xml = retry_on_exception(ros_pkg.get_package_xml, + distro.name) + + metadata = PackageMetadata( + package_xml, NixPackage._get_condition_context(distro.name)) + + dep_walker = DependencyWalker(distro, + get_distro_condition_context( + distro.name)) + + buildtool_deps = dep_walker.get_depends(pkg.name, "buildtool") + buildtool_export_deps = dep_walker.get_depends( + pkg.name, "buildtool_export") + build_deps = dep_walker.get_depends(pkg.name, "build") + build_export_deps = dep_walker.get_depends(pkg.name, "build_export") + exec_deps = dep_walker.get_depends(pkg.name, "exec") + test_deps = dep_walker.get_depends(pkg.name, "test") + + self.unresolved_dependencies = set() + + # buildtool_depends are added to buildInputs and nativeBuildInputs. + # Some (such as CMake) have binaries that need to run at build time + # (and therefore need to be in nativeBuildInputs. Others (such as + # ament_cmake_*) need to be added to CMAKE_PREFIX_PATH and therefore + # need to be in buildInputs. There is no easy way to distinguish these + # two cases, so they are added to both, which generally works fine. + build_inputs = set(self._resolve_dependencies( + build_deps | buildtool_deps)) + propagated_build_inputs = self._resolve_dependencies( + exec_deps | build_export_deps | buildtool_export_deps) + build_inputs -= propagated_build_inputs + + check_inputs = self._resolve_dependencies(test_deps) + check_inputs -= build_inputs + + native_build_inputs = self._resolve_dependencies( + buildtool_deps | buildtool_export_deps) + + self._derivation = NixExpression( + name=normalized_name, + version=version, + src_url=src_uri, + src_sha256=src_sha256, + description=metadata.description, + licenses=map(NixLicense, metadata.upstream_license), + distro_name=distro.name, + build_type=metadata.build_type, + build_inputs=build_inputs, + propagated_build_inputs=propagated_build_inputs, + check_inputs=check_inputs, + native_build_inputs=native_build_inputs) + + def _resolve_dependencies(self, deps: Iterable[str]) -> Set[str]: + return set(itertools.chain.from_iterable( + map(self._resolve_dependency, deps))) + + def _resolve_dependency(self, d: str) -> Iterable[str]: + try: + return (self.normalize_name(d),) \ + if d in self._all_pkgs \ + else resolve_dep(d, 'nix')[0] + except UnresolvedDependency: + self.unresolved_dependencies.add(d) + return tuple() + + @staticmethod + def _get_ros_version(distro): + distros = get_distros() + return 2 if distro not in distros \ + else int(distros[distro]['distribution_type'][len('ros'):]) + + @staticmethod + def _get_ros_python_version(distro): + return 2 if distro in ['melodic'] else 3 + + @staticmethod + def _get_condition_context(distro): + context = dict() + context["ROS_OS_OVERRIDE"] = "nixos" + context["ROS_DISTRO"] = distro + context["ROS_VERSION"] = str(NixPackage._get_ros_version(distro)) + context["ROS_PYTHON_VERSION"] = str( + NixPackage._get_ros_python_version(distro)) + return context + + @staticmethod + def normalize_name(name: str) -> str: + """ + Convert underscores to dashes to match normal Nix package naming + conventions. + + :param name: original package name + :return: normalized package name + """ + return name.replace('_', '-') + + @property + def derivation(self): + if self.unresolved_dependencies: + raise UnresolvedDependency("failed to resolve dependencies!") + + return self._derivation diff --git a/superflore/generators/nix/nix_package_set.py b/superflore/generators/nix/nix_package_set.py new file mode 100644 index 00000000..f89093c7 --- /dev/null +++ b/superflore/generators/nix/nix_package_set.py @@ -0,0 +1,31 @@ +from textwrap import dedent +from time import gmtime, strftime +from typing import Iterable + +from superflore.generators.nix.nix_package import NixPackage + + +class NixPackageSet: + """ + Code generator for Nix overlay package set. Each package in the set must + be defined as a function in a default.nix file within a directory named + after the package. The package functions are called with callPackage. + """ + + def __init__(self, pkg_names: Iterable[str]): + self.pkg_names = pkg_names + + def get_text(self, distributor: str, license_name: str) -> str: + ret = [] + ret += dedent(''' + # Copyright {} {} + # Distributed under the terms of the {} license + + self: super: {{ + + ''').format(strftime("%Y", gmtime()), distributor, license_name) + ret.extend((" {0} = self.callPackage ./{0} {{}};\n\n" + .format(NixPackage.normalize_name(pkg_name)) + for pkg_name in self.pkg_names)) + ret += "}\n" + return ''.join(ret) diff --git a/superflore/generators/nix/nix_ros_overlay.py b/superflore/generators/nix/nix_ros_overlay.py new file mode 100644 index 00000000..19f3abeb --- /dev/null +++ b/superflore/generators/nix/nix_ros_overlay.py @@ -0,0 +1,64 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from superflore.repo_instance import RepoInstance +from superflore.utils import info +from superflore.utils import rand_ascii_str + + +class NixRosOverlay(object): + def __init__(self, repo_dir, do_clone, org='lopsided98', + repo='nix-ros-overlay', from_branch='', new_branch=True): + self.repo = RepoInstance(org, repo, repo_dir, do_clone, + from_branch=from_branch) + if new_branch: + self.branch_name = 'nix-bot-%s' % rand_ascii_str() + info('Creating new branch {0}...'.format(self.branch_name)) + self.repo.create_branch(self.branch_name) + else: + self.branch_name = None + + def commit_changes(self, distro): + info('Adding changes...') + if distro == 'all': + commit_msg = 'regenerate all distros, {0}' + self.repo.git.add('distros/*/*/default.nix') + self.repo.git.add('distros/*/generated.nix') + else: + commit_msg = 'regenerate rosPackages.{1}, {0}' + self.repo.git.add('distros/' + distro) + if self.repo.git.status('--porcelain') == '': + info('Nothing changed; no commit done') + else: + timestamp = os.getenv( + 'SUPERFLORE_GENERATION_DATETIME', + time.ctime()) + commit_msg = commit_msg.format(timestamp, distro) + if self.branch_name: + info('Committing to branch {0}...'.format(self.branch_name)) + else: + info('Committing to current branch') + self.repo.git.commit(m=commit_msg) + + def pull_request(self, message, distro=None, title=''): + if not title: + timestamp = os.getenv( + 'SUPERFLORE_GENERATION_DATETIME', + time.ctime()) + title = 'rosdistro sync, {0}'.format(timestamp) + self.repo.pull_request(message, title, branch=self.branch_name, + fork=False) diff --git a/superflore/generators/nix/run.py b/superflore/generators/nix/run.py new file mode 100644 index 00000000..6fea4bb0 --- /dev/null +++ b/superflore/generators/nix/run.py @@ -0,0 +1,206 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from rosinstall_generator.distro import get_distro +from superflore.CacheManager import CacheManager +from superflore.exceptions import NoGitHubAuthToken +from superflore.generate_installers import generate_installers +from superflore.generators.nix.gen_packages import regenerate_pkg, \ + regenerate_pkg_set +from superflore.generators.nix.nix_ros_overlay import NixRosOverlay +from superflore.parser import get_parser +from superflore.repo_instance import RepoInstance +from superflore.TempfileManager import TempfileManager +from superflore.utils import clean_up, get_distros_by_status +from superflore.utils import err +from superflore.utils import file_pr +from superflore.utils import gen_delta_msg +from superflore.utils import gen_missing_deps_msg +from superflore.utils import info +from superflore.utils import load_pr +from superflore.utils import ok +from superflore.utils import save_pr +from superflore.utils import url_to_repo_org +from superflore.utils import warn + + +def main(): + preserve_existing = True + parser = get_parser('Deploy ROS packages using Nix') + parser.add_argument( + '--tar-archive-dir', + help='location to store archived packages', + type=str + ) + args = parser.parse_args(sys.argv[1:]) + pr_comment = args.pr_comment + skip_keys = args.skip_keys or [] + selected_targets = None + if not args.dry_run: + if 'SUPERFLORE_GITHUB_TOKEN' not in os.environ: + raise NoGitHubAuthToken() + if args.pr_only: + if args.dry_run: + parser.error('Invalid args! cannot dry-run and file PR') + if not args.output_repository_path: + parser.error('Invalid args! no repository specified') + try: + prev_overlay = RepoInstance(args.output_repository_path, False) + msg, title = load_pr() + prev_overlay.pull_request(msg, title=title) + clean_up() + sys.exit(0) + except Exception as e: + err('Failed to file PR!') + err('reason: {0}'.format(e)) + sys.exit(1) + elif args.all: + warn('"All" mode detected... This may take a while!') + preserve_existing = False + elif args.ros_distro: + warn('"{0}" distro detected...'.format(args.ros_distro)) + selected_targets = [args.ros_distro] + preserve_existing = False + elif args.only: + parser.error('Invalid args! --only requires specifying --ros-distro') + if not selected_targets: + selected_targets = get_distros_by_status('active') + \ + get_distros_by_status('rolling') + repo_org = 'lopsided98' + repo_name = 'nix-ros-overlay' + if args.upstream_repo: + repo_org, repo_name = url_to_repo_org(args.upstream_repo) + with TempfileManager(args.output_repository_path) as _repo: + if not args.output_repository_path: + # give our group write permissions to the temp dir + os.chmod(_repo, 17407) + # clone if args.output_repository_path is None + overlay = NixRosOverlay( + _repo, + not args.output_repository_path, + org=repo_org, + repo=repo_name, + from_branch=args.upstream_branch, + new_branch=(not args.no_branch) + ) + if not preserve_existing and not args.only: + pr_comment = pr_comment or ( + 'Superflore Nix generator began regeneration of all packages ' + ' from ROS distro %s from nix-ros-overlay commit %s.' % ( + selected_targets, + overlay.repo.get_last_hash() + ) + ) + elif not args.only: + pr_comment = pr_comment or ( + 'Superflore Nix generator ran update from nix-ros-overlay ' + + 'commit %s.' % (overlay.repo.get_last_hash()) + ) + # generate installers + total_installers = dict() + total_broken = set() + total_changes = dict() + if args.tar_archive_dir: + sha256_filename = '%s/sha256_cache.pickle' % args.tar_archive_dir + else: + sha256_filename = None + with TempfileManager(args.tar_archive_dir) as tar_dir, \ + CacheManager(sha256_filename) as sha256_cache: + if args.only: + for pkg in args.only: + if pkg in skip_keys: + warn("Package '%s' is in skip-keys list, skipping..." + % pkg) + continue + info("Regenerating package '%s'..." % pkg) + try: + regenerate_pkg( + overlay, + pkg, + get_distro(args.ros_distro), + preserve_existing, + tar_dir, + sha256_cache + ) + except KeyError: + err("No package to satisfy key '%s'" % pkg) + sys.exit(1) + # Commit changes and file pull request + regen_dict = dict() + regen_dict[args.ros_distro] = args.only + overlay.commit_changes(args.ros_distro) + delta = "Regenerated: '%s'\n" % args.only + if args.dry_run: + save_pr(overlay, delta, '', pr_comment) + sys.exit(0) + file_pr(overlay, delta, '', pr_comment, distro=args.ros_distro) + ok('Successfully synchronized repositories!') + sys.exit(0) + + for distro in selected_targets: + distro_installers, distro_broken, distro_changes = \ + generate_installers( + get_distro(distro), + overlay, + regenerate_pkg, + preserve_existing, + tar_dir, + sha256_cache, + skip_keys=skip_keys, + ) + for key in distro_broken.keys(): + for pkg in distro_broken[key]: + total_broken.add(pkg) + + total_changes[distro] = distro_changes + total_installers[distro] = distro_installers + + # If we are just updating a few packages using --only, then + # leave the package set alone. This means that new packages + # will not be added, but it is still useful for updates. + if not preserve_existing: + regenerate_pkg_set(overlay, distro, distro_installers) + ok('Generated package set for distro \'{}\'' + .format(distro)) + + num_changes = 0 + for distro_name in total_changes: + num_changes += len(total_changes[distro_name]) + + if num_changes == 0: + info('ROS distro is up to date.') + info('Exiting...') + clean_up() + sys.exit(0) + + # remove duplicates + delta = gen_delta_msg(total_changes) + missing_deps = gen_missing_deps_msg(total_broken) + + # Commit changes and file pull request + overlay.commit_changes('all' if args.all else args.ros_distro) + + if args.dry_run: + info('Running in dry mode, not filing PR') + save_pr( + overlay, delta, missing_deps=missing_deps, comment=pr_comment + ) + sys.exit(0) + file_pr(overlay, delta, missing_deps, comment=pr_comment) + + clean_up() + ok('Successfully synchronized repositories!') diff --git a/superflore/repo_instance.py b/superflore/repo_instance.py index 1f732410..74e91d34 100644 --- a/superflore/repo_instance.py +++ b/superflore/repo_instance.py @@ -92,30 +92,31 @@ def rebase(self, target): """ self.git.rebase(i=target) - def pull_request(self, message, title, branch='master', remote='origin'): - info('Forking repository if a fork does not exist...') + def pull_request(self, message, title, branch='master', fork=True): self.github = Github(os.environ['SUPERFLORE_GITHUB_TOKEN']) self.gh_user = self.github.get_user() self.gh_upstream = self.github.get_repo( - '%s/%s' % ( - self.repo_owner, self.repo_name - ) + '{}/{}'.format(self.repo_owner, self.repo_name) ) - # TODO(allenh1): Don't fork if you're authorized for repo - forked_repo = self.gh_user.create_fork(self.gh_upstream) - info('Pushing changes to fork...') - self.git.remote('add', 'github', forked_repo.html_url) + if fork: + info('Forking repository if a fork does not exist...') + pr_repo = self.gh_user.create_fork(self.gh_upstream) + pr_head = '{}:{}'.format(self.gh_user.login, self.branch or branch) + else: + pr_repo = self.gh_upstream + pr_head = self.branch or branch + info('Pushing changes to repo...') + self.git.remote('add', 'github', pr_repo.html_url) retry_on_exception( self.git.push, '-u', 'github', self.branch or branch, retry_msg='Could not push', error_msg='Error during push', sleep_secs=0.0, ) info('Filing pull-request...') - pr_head = '%s:%s' % (self.gh_user.login, self.branch) pr = self.gh_upstream.create_pull( title=title, body=message, - base=self.from_branch or branch, + base=self.from_branch, head=pr_head ) ok('Successfully filed a pull request.') diff --git a/superflore/rosdep_support.py b/superflore/rosdep_support.py index 5ab8f1e2..ff0bab00 100644 --- a/superflore/rosdep_support.py +++ b/superflore/rosdep_support.py @@ -53,6 +53,9 @@ def get_view(os_name, os_version, ros_distro): return view_cache[key] +_installer_ctx = create_default_installer_context() + + def resolve_more_for_os(rosdep_key, view, installer, os_name, os_version): """ Resolve rosdep key to dependencies and installer key. @@ -64,9 +67,8 @@ def resolve_more_for_os(rosdep_key, view, installer, os_name, os_version): :raises: :exc:`rosdep2.ResolutionError` """ d = view.lookup(rosdep_key) - ctx = create_default_installer_context() - os_installers = ctx.get_os_installer_keys(os_name) - default_os_installer = ctx.get_default_os_installer_key(os_name) + os_installers = _installer_ctx.get_os_installer_keys(os_name) + default_os_installer = _installer_ctx.get_default_os_installer_key(os_name) inst_key, rule = d.get_rule_for_platform(os_name, os_version, os_installers, default_os_installer) @@ -82,15 +84,14 @@ def resolve_rosdep_key( ignored=None ): ignored = ignored or [] - ctx = create_default_installer_context() try: - installer_key = ctx.get_default_os_installer_key(os_name) + installer_key = _installer_ctx.get_default_os_installer_key(os_name) except KeyError: raise UnresolvedDependency( "could not resolve package {} for os {}." .format(key, os_name) ) - installer = ctx.get_installer(installer_key) + installer = _installer_ctx.get_installer(installer_key) ros_distro = ros_distro or DEFAULT_ROS_DISTRO view = get_view(os_name, os_version, ros_distro) try: diff --git a/superflore/test_integration/nix/__init__.py b/superflore/test_integration/nix/__init__.py new file mode 100644 index 00000000..b95c42b8 --- /dev/null +++ b/superflore/test_integration/nix/__init__.py @@ -0,0 +1,4 @@ +from superflore.test_integration.nix.main import main + +if __name__ == '__main__': + main() diff --git a/superflore/test_integration/nix/build_base.py b/superflore/test_integration/nix/build_base.py new file mode 100644 index 00000000..b94dba79 --- /dev/null +++ b/superflore/test_integration/nix/build_base.py @@ -0,0 +1,52 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from docker.errors import ContainerError +from superflore.docker import Docker +from superflore.generators.nix.nix_package import NixPackage +from superflore.utils import err +from superflore.utils import info +from superflore.utils import ok + + +class NixBuilder: + def __init__(self, image_owner='nixos', image_name='nix'): + self.container = Docker() + self.container.pull(image_owner, image_name) + self.package_list = dict() + + def add_target(self, ros_distro, pkg): + pkg = NixPackage.normalize_name(pkg) + self.package_list[ + 'rosPackages.{}.{}'.format(ros_distro, pkg)] = 'unknown' + + def run(self, verbose=True, log_file=None): + info('testing Nix package integrity') + + nix_ros_overlay_url = 'https://github.com/lopsided98/nix-ros-overlay' \ + '/archive/master.tar.gz' + for pkg in sorted(self.package_list.keys()): + self.container.add_sh_command( + 'nix-build {} -A {}'.format(nix_ros_overlay_url, pkg)) + try: + self.container.run(rm=True, show_cmd=True, log_file=log_file) + self.package_list[pkg] = 'building' + ok(" '%s': building" % pkg) + except ContainerError: + self.package_list[pkg] = 'failing' + err(" '%s': failing" % pkg) + if verbose: + print(self.container.log) + self.container.clear_commands() + return self.package_list diff --git a/superflore/test_integration/nix/main.py b/superflore/test_integration/nix/main.py new file mode 100644 index 00000000..186012da --- /dev/null +++ b/superflore/test_integration/nix/main.py @@ -0,0 +1,85 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +from superflore.test_integration.nix.build_base import NixBuilder +from superflore.utils import get_distros_by_status +import yaml + + +def main(): + """ + Test Nix builds in a Docker container. Each package build is run in a new + container, so everything is built from scratch each time. + """ + + tester = NixBuilder() + parser = argparse.ArgumentParser( + description='Check if ROS packages are building with Nix' + ) + parser.add_argument( + '--ros-distro', + help='distro(s) to check', + type=str, + nargs="+", + default=get_distros_by_status('active') + ) + parser.add_argument( + '--pkgs', + help='packages to build', + type=str, + nargs='+' + ) + parser.add_argument( + '-f', + help='build packages specified by the input file', + type=str + ) + parser.add_argument( + '-v', + '--verbose', + help='show output from docker', + action="store_true" + ) + parser.add_argument( + '--log-file', + help='location to store the log file', + type=str + ) + args = parser.parse_args(sys.argv[1:]) + + if args.f: + # load the yaml file holding the test files + with open(args.f, 'r') as test_file: + test_dict = yaml.load(test_file) + for distro, pkg_list in test_dict.items(): + for pkg in pkg_list: + tester.add_target(distro, pkg) + elif args.pkgs: + # use passed-in arguments to test + for distro in args.ros_distro: + for pkg in args.pkgs: + tester.add_target(distro, pkg) + else: + parser.error('Invalid args! You must supply a package list.') + sys.exit(1) + results = tester.run(args.verbose, args.log_file) + failures = 0 + for test_case in results.keys(): + if results[test_case] == 'failing': + failures = failures + 1 + # set exit status to the number of failures + sys.exit(failures) diff --git a/superflore/utils.py b/superflore/utils.py index 8fb5cc71..65ad499a 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -17,9 +17,12 @@ import os import random import re +import shutil import string import sys import time +from typing import Dict +import urllib.request from pkg_resources import DistributionNotFound, get_distribution from superflore.exceptions import UnknownPlatform @@ -43,24 +46,27 @@ def info(string): # pragma: no cover print(colored('>>>> {0}'.format(string), 'cyan')) -def get_pr_text(comment=None, markup='```'): +def get_pr_text(delta=None, missing_deps=None, comment=None, markup='```'): msg = '' if comment: - msg += '%s\n' % comment + msg += comment + '\n' msg += 'This pull request was generated by running the following command:' msg += '\n\n' args = sys.argv args[0] = args[0].split('/')[-1] msg += '{1}\n{0}\n{1}\n'.format(' '.join(args), markup) + if delta: + msg += '\n' + delta + '\n' + if missing_deps: + msg += missing_deps + '\n' return msg -def save_pr(overlay, delta, missing_deps, comment, - title='rosdistro sync, {0}\n'.format(time.ctime())): +def save_pr(overlay, delta, missing_deps, comment, title=''): with open('.pr-title.tmp', 'w') as title_file: title_file.write(title) with open('.pr-message.tmp', 'w') as pr_msg_file: - pr_msg_file.write('%s\n' % get_pr_text(comment)) + pr_msg_file.write(get_pr_text(delta, missing_deps, comment)) def load_pr(): @@ -83,9 +89,8 @@ def load_pr(): def file_pr(overlay, delta, missing_deps, comment, distro=None, title=''): try: - msg = get_pr_text(comment) - overlay.pull_request('{}\n{}\n{}'.format(msg, delta, missing_deps), - distro, title=title) + msg = get_pr_text(delta, missing_deps, comment) + overlay.pull_request(msg, distro, title=title) except Exception as e: err( 'Failed to file PR with the %s/%s repo!' % ( @@ -700,6 +705,8 @@ def resolve_dep(pkg, os, distro=None): return resolve_rosdep_key(pkg, 'openembedded', '', distro) elif os == 'gentoo': return resolve_rosdep_key(pkg, 'gentoo', '2.4.0') + elif os == 'nix': + return resolve_rosdep_key(pkg, 'nixos', '') else: msg = "Unknown target platform '{0}'".format(os) raise UnknownPlatform(msg) @@ -715,6 +722,30 @@ def get_distros_by_status(status='active'): if t[1].get('distribution_status') == status] +def get_distro_condition_context(distro_name: str) -> Dict[str, str]: + """ + Get the condition context for a particular ROS distro. This context is used + to evaluate conditions in package.xml format 3. + + This allows superflore to support packages that are designed to work with + both ROS 1 and 2, which have conditional dependencies on catkin and ament. + + :param distro_name: name of the ROS distro + :return: dictionary containing context keys and their values + """ + index = get_cached_index() + distro = index.distributions[distro_name] + context = { + 'ROS_DISTRO': distro_name, + 'ROS_PYTHON_VERSION': distro['python_version'] + } + if distro['distribution_type'] == 'ros1': + context['ROS_VERSION'] = '1' + elif distro['distribution_type'] == 'ros2': + context['ROS_VERSION'] = '2' + return context + + def gen_delta_msg(total_changes, markup='*'): """Return string of changes for the PR message.""" delta = '' @@ -760,6 +791,23 @@ def url_to_repo_org(url): return url[0], url[1] +def download_file(url, filename): + # GitLab returns 403 when using the default urllib User-Agent + request = urllib.request.Request(url, headers={ + 'User-Agent': 'superflore/{}'.format(get_superflore_version()) + }) + try: + with urllib.request.urlopen(request) as response, \ + open(filename, 'wb') as file: + shutil.copyfileobj(response, file) + except Exception as e: + try: + os.remove(filename) + except Exception as re: + warn('failed to remove: {}: {}'.format(filename, re)) + raise e + + def retry_on_exception(callback, *args, max_retries=5, num_retry=0, retry_msg='', error_msg='', sleep_secs=0.125): try: diff --git a/tests/test_nix.py b/tests/test_nix.py new file mode 100644 index 00000000..c4dabea0 --- /dev/null +++ b/tests/test_nix.py @@ -0,0 +1,39 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from superflore.generators.nix.nix_expression import NixLicense + +class TestNixLicense(unittest.TestCase): + + def test_known_license(self): + l = NixLicense('GPL 3') + self.assertEqual(l.nix_code, 'gpl3') + + def test_unknown_license(self): + l = NixLicense("some license") + self.assertEqual(l.nix_code, '"some-license"') + + def test_public_domain(self): + l = NixLicense("Public Domain") + self.assertEqual(l.nix_code, 'publicDomain') + + def test_escape_quote(self): + l = NixLicense(r'license with "quotes" and \backslash" '); + self.assertEqual(l.nix_code, r'"license-with-\"quotes\"-and-\\backslash"') + + def test_escape_quote(self): + l = NixLicense('some license with the "${" sequence'); + self.assertEqual(l.nix_code, r'"some-license-with-the-\"\${\"-sequence"') diff --git a/tests/test_utils.py b/tests/test_utils.py index 74a3c7ab..f5f3eb1a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -183,7 +183,7 @@ def test_get_pr_text(self): expected = 'sample\n'\ 'This pull request was generated by running the following command:\n\n'\ '```\na b c d\n```\n' - self.assertEqual(expected, get_pr_text('sample')) + self.assertEqual(expected, get_pr_text(comment='sample')) def test_cleanup(self): """Test PR dry run cleanup"""