diff --git a/.travis.yml b/.travis.yml index 62c2a15..133600d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,19 @@ language: python install: pip install tox -env: - - TOX_ENV=docs - - TOX_ENV=py27-flake8 - - TOX_ENV=py27-virtualenv-150x - - TOX_ENV=py27-virtualenv-151x - - TOX_ENV=py27-virtualenv-152x +matrix: + include: + - python: 2.7 + env: TOX_ENV=docs + - python: 2.7 + env: TOX_ENV=py27-flake8 + - python: 2.7 + env: TOX_ENV=py27-virtualenv-150x + - python: 2.7 + env: TOX_ENV=py27-virtualenv-151x + - python: 3.6 + env: TOX_ENV=py27-virtualenv-150x + - python: 3.6 + env: TOX_ENV=py27-virtualenv-151x script: tox -e $TOX_ENV notifications: email: diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 7ce4dad..4892722 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -66,6 +66,15 @@ The following options are only available if ``gcloud`` is installed. e.g. ``S3_BUCKET``, ``GCS_BUCKET`` instead of being passed in as a parameter. +Using Python 3 inside virtualenv +================================ + +Use ``-p`` argument to choose Python executable installed in virtualenv: + +.. code-block:: shell-session + + $ terrarium -p python3.6 --target env --storage-dir path/to/environments install requirements.txt + Tips #### diff --git a/terrarium/terrarium.py b/terrarium/terrarium.py index 9e9fd94..1d3c365 100755 --- a/terrarium/terrarium.py +++ b/terrarium/terrarium.py @@ -8,6 +8,7 @@ import tempfile import shutil import logging +import subprocess from logging import getLogger, StreamHandler @@ -162,6 +163,17 @@ def get_backup_location(self, target=None): target = self.get_target_location() return ''.join([target, self.args.backup_suffix]) + def get_wheel_search_dir(self): + import virtualenv + path = os.path.join(os.path.dirname(virtualenv.__file__), + 'virtualenv_support') + if not os.path.exists(path): + raise RuntimeError( + ('{0}: virtualenv wheel dir not found, ' + 'check your virtualenv installation' + .format(path))) + return path + def environment_exists(self, env): return os.path.exists(os.path.join( env, @@ -209,12 +221,24 @@ def install(self): # Run the bootstrap script which pip installs everything that has # been defined as a requirement - call_subprocess([ + args = [ sys.executable, bootstrap, '--prompt=(%s)' % prompt, new_target - ]) + ] + if self.args.python_executable: + # Pass explicit python version to bootstrap script and + # set wheel path from 2.7 virtualenv. Wheels bundled + # with virtualenv are universal and work with any + # version of Python. The benefit of doing it is that + # user doesn't have to install python3-virtualenv just + # for wheel files. + args += [ + '-p', self.args.python_executable, + '--extra-search-dir', self.get_wheel_search_dir(), + ] + call_subprocess(args) # Do we want to copy the bootstrap into the environment for future # use? @@ -285,6 +309,8 @@ def replace_all_in_directory( for name in os.listdir(location): full_path = os.path.join(location, name) data = None + if not os.path.isfile(full_path): + continue with open(full_path) as f: header = f.read(len(MAGIC_NUM['ELF'])) # Skip binary files @@ -482,9 +508,26 @@ def download(self, target): os.unlink(archive) return True + def get_version_tuple_from_executable(self, executable): + cmd = [executable, '--version'] + # py3 prints to stdout, py2 to stderr + pipe = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + out, _ = pipe.communicate() + if pipe.returncode != 0: + raise OSError('Command {0} failed with error code {1}, output: {2}' + .format(cmd, pipe.returncode, out)) + # out: Python 3.6.3 + version = out.strip().split()[1] + return version.split('.') + def make_remote_key(self): import platform - major, minor, patch = platform.python_version_tuple() + if self.args.python_executable: + major, minor, patch = self.get_version_tuple_from_executable( + self.args.python_executable) + else: + major, minor, patch = platform.python_version_tuple() context = { 'digest': self.digest, 'python_vmajor': major, @@ -604,9 +647,6 @@ def after_install(options, base): import os import tempfile - # Debug logging for virtualenv - logger.consumers = [({virtualenv_log_level}, sys.stdout)] - home_dir, lib_dir, inc_dir, bin_dir = path_locations(base) # Update prefix and executable to point to the virtualenv @@ -627,7 +667,7 @@ def after_install(options, base): # Activate the virtualenv activate_this = join(bin_dir, 'activate_this.py') - execfile(activate_this, dict(__file__=activate_this)) + exec(open(activate_this).read(), dict(__file__=activate_this)) import pip try: @@ -635,35 +675,43 @@ def after_install(options, base): except ImportError: # Pip >= 10 from pip._internal.commands.install import InstallCommand - # Debug logging for pip - if hasattr(pip, '_internal'): - pip._internal.logger.consumers = [({virtualenv_log_level}, sys.stdout)] - else: - pip.logger.consumers = [({virtualenv_log_level}, sys.stdout)] - # If we are on a version of pip before 1.2, load version control modules # for installing 'editables' if hasattr(pip, 'version_control'): pip.version_control() + fd, file_path = tempfile.mkstemp() + with os.fdopen(fd, 'w') as f: + f.write('\n'.join(REQUIREMENTS)) + # Run pip install + c = None + options = None try: c = InstallCommand() except TypeError: try: from pip.baseparser import create_main_parser except ImportError: # Pip >= 10 - from pip._internal.baseparser import create_main_parser - main_parser = create_main_parser() - c = InstallCommand(main_parser) - - fd, file_path = tempfile.mkstemp() - with os.fdopen(fd, 'w') as f: - f.write('\n'.join(REQUIREMENTS)) - options, args = c.parser.parse_args(['-r', file_path]) - options.require_venv = True - options.ignore_installed = True - requirementSet = c.run(options, args) + try: + from pip._internal.baseparser import create_main_parser + except ImportError: # Pip >= 19.3 + from pip._internal.commands import create_command + from pip._internal.cli.main_parser import parse_command + _, options = parse_command(['install', '-r', file_path, '--ignore-installed']) + c = create_command('install') + if not c: + main_parser = create_main_parser() + c = InstallCommand(main_parser) + + if not options: + options, args = c.parser.parse_args(['-r', file_path]) + options.require_venv = True + options.ignore_installed = True + if hasattr(c, 'main'): + c.main(options) + else: + c.run(options, args) os.unlink(file_path) @@ -710,6 +758,11 @@ def parse_args(): VIRTUAL_ENV. ''', ) + ap.add_argument( + '-p', '--python-executable', + default='', + help='Python executable to use in virtualenv, eg. python3.6', + ) ap.add_argument( '--pip-log-level', default=25, diff --git a/tests/tests.py b/tests/tests.py index cbb068b..9b85b33 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -9,6 +9,7 @@ import platform import copy import sys +import virtualenv class TerrariumTester(unittest.TestCase): @@ -60,6 +61,19 @@ def requirements(self): def opts(self): return self.config['opts'] + @property + def virtualenv_version_tuple(self): + parts = virtualenv.virtualenv_version.split('.') + return tuple([int(v) for v in parts]) + + def is_python_version_available(self, executable): + try: + result = subprocess.call( + [executable, '--version'], stdout=subprocess.PIPE) + return result == 0 + except OSError: + return False + def config_pop(self): return self.configs.pop() @@ -175,7 +189,6 @@ def _add_test_requirement(self): self._add_requirements(test_requirement) def _add_terrarium_requirement(self): - import virtualenv self._add_requirements( self._get_path_terrarium(), 'virtualenv==%s' % virtualenv.virtualenv_version @@ -622,3 +635,25 @@ def test_require_download(self): 'Refusing to build a new bundle.\n', ) self.assertNotExists(self.python) + + def test_install_with_python3_executable(self): + skip = None + if not self.is_python_version_available('python3.6'): + skip = 'python3.6 not installed' + if self.virtualenv_version_tuple < (15, 0): + skip = 'py3 not supported with this virtualenv version' + + if skip: + if hasattr(unittest, 'SkipTest'): + raise unittest.SkipTest(skip) + else: + # SkipTest not supported in Python 2.6 + print(skip) + return + + self.assertInstall( + storage_dir=self.storage_dir, python_executable='python3.6') + self.assertTrue(os.path.join(self.target, 'bin', 'python3.6')) + self.assertEqual( + os.readlink(os.path.join(self.target, 'bin', 'python')), + 'python3.6')