Support Python 3 in virtual envs

.travis.yml

@@ -1,11 +1,19 @@
 language: python
 install: pip install tox
-env:
-  - TOX_ENV=docs
-  - TOX_ENV=py27-flake8
-  - TOX_ENV=py27-virtualenv-150x
-  - TOX_ENV=py27-virtualenv-151x
-  - TOX_ENV=py27-virtualenv-152x
+matrix:
+  include:
+    - python: 2.7
+      env: TOX_ENV=docs
+    - python: 2.7
+      env: TOX_ENV=py27-flake8
+    - python: 2.7
+      env: TOX_ENV=py27-virtualenv-150x
+    - python: 2.7
+      env: TOX_ENV=py27-virtualenv-151x
+    - python: 3.6
+      env: TOX_ENV=py27-virtualenv-150x
+    - python: 3.6
+      env: TOX_ENV=py27-virtualenv-151x
 script: tox -e $TOX_ENV
 notifications:
   email:

docs/user_guide.rst

@@ -66,6 +66,15 @@ The following options are only available if ``gcloud`` is installed.
     e.g. ``S3_BUCKET``, ``GCS_BUCKET``
     instead of being passed in as a parameter.
 
+Using Python 3 inside virtualenv
+================================
+
+Use ``-p`` argument to choose Python executable installed in virtualenv:
+
+.. code-block:: shell-session
+
+    $ terrarium -p python3.6 --target env --storage-dir path/to/environments install requirements.txt
+
 Tips
 ####
 

terrarium/terrarium.py

@@ -8,6 +8,7 @@
 import tempfile
 import shutil
 import logging
+import subprocess
 
 from logging import getLogger, StreamHandler
 
@@ -162,6 +163,17 @@ def get_backup_location(self, target=None):
             target = self.get_target_location()
         return ''.join([target, self.args.backup_suffix])
 
+    def get_wheel_search_dir(self):
+        import virtualenv
+        path = os.path.join(os.path.dirname(virtualenv.__file__),
+                            'virtualenv_support')
+        if not os.path.exists(path):
+            raise RuntimeError(
+                ('{0}: virtualenv wheel dir not found, '
+                 'check your virtualenv installation'
+                 .format(path)))
+        return path
+
     def environment_exists(self, env):
         return os.path.exists(os.path.join(
             env,
@@ -209,12 +221,24 @@ def install(self):
 
             # Run the bootstrap script which pip installs everything that has
             # been defined as a requirement
-            call_subprocess([
+            args = [
                 sys.executable,
                 bootstrap,
                 '--prompt=(%s)' % prompt,
                 new_target
-            ])
+            ]
+            if self.args.python_executable:
+                # Pass explicit python version to bootstrap script and
+                # set wheel path from 2.7 virtualenv. Wheels bundled
+                # with virtualenv are universal and work with any
+                # version of Python. The benefit of doing it is that
+                # user doesn't have to install python3-virtualenv just
+                # for wheel files.
+                args += [
+                    '-p', self.args.python_executable,
+                    '--extra-search-dir', self.get_wheel_search_dir(),
+                ]
+            call_subprocess(args)
 
             # Do we want to copy the bootstrap into the environment for future
             # use?
@@ -285,6 +309,8 @@ def replace_all_in_directory(
         for name in os.listdir(location):
             full_path = os.path.join(location, name)
             data = None
+            if not os.path.isfile(full_path):
+                continue
             with open(full_path) as f:
                 header = f.read(len(MAGIC_NUM['ELF']))
                 # Skip binary files
@@ -482,9 +508,26 @@ def download(self, target):
                     os.unlink(archive)
                     return True
 
+    def get_version_tuple_from_executable(self, executable):
+        cmd = [executable, '--version']
+        # py3 prints to stdout, py2 to stderr
+        pipe = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        out, _ = pipe.communicate()
+        if pipe.returncode != 0:
+            raise OSError('Command {0} failed with error code {1}, output: {2}'
+                          .format(cmd, pipe.returncode, out))
+        # out: Python 3.6.3
+        version = out.strip().split()[1]
+        return version.split('.')
+
     def make_remote_key(self):
         import platform
-        major, minor, patch = platform.python_version_tuple()
+        if self.args.python_executable:
+            major, minor, patch = self.get_version_tuple_from_executable(
+                self.args.python_executable)
+        else:
+            major, minor, patch = platform.python_version_tuple()
         context = {
             'digest': self.digest,
             'python_vmajor': major,
@@ -604,9 +647,6 @@ def after_install(options, base):
     import os
     import tempfile
 
-    # Debug logging for virtualenv
-    logger.consumers = [({virtualenv_log_level}, sys.stdout)]
-
     home_dir, lib_dir, inc_dir, bin_dir = path_locations(base)
 
     # Update prefix and executable to point to the virtualenv
@@ -627,43 +667,51 @@ def after_install(options, base):
 
     # Activate the virtualenv
     activate_this = join(bin_dir, 'activate_this.py')
-    execfile(activate_this, dict(__file__=activate_this))
+    exec(open(activate_this).read(), dict(__file__=activate_this))
 
     import pip
     try:
         from pip.commands.install import InstallCommand
     except ImportError:  # Pip >= 10
         from pip._internal.commands.install import InstallCommand
 
-    # Debug logging for pip
-    if hasattr(pip, '_internal'):
-        pip._internal.logger.consumers = [({virtualenv_log_level}, sys.stdout)]
-    else:
-        pip.logger.consumers = [({virtualenv_log_level}, sys.stdout)]
-
     # If we are on a version of pip before 1.2, load version control modules
     # for installing 'editables'
     if hasattr(pip, 'version_control'):
         pip.version_control()
 
+    fd, file_path = tempfile.mkstemp()
+    with os.fdopen(fd, 'w') as f:
+        f.write('\n'.join(REQUIREMENTS))
+
     # Run pip install
+    c = None
+    options = None
     try:
         c = InstallCommand()
     except TypeError:
         try:
             from pip.baseparser import create_main_parser
         except ImportError:  # Pip >= 10
-            from pip._internal.baseparser import create_main_parser
-        main_parser = create_main_parser()
-        c = InstallCommand(main_parser)
-
-    fd, file_path = tempfile.mkstemp()
-    with os.fdopen(fd, 'w') as f:
-        f.write('\n'.join(REQUIREMENTS))
-    options, args = c.parser.parse_args(['-r', file_path])
-    options.require_venv = True
-    options.ignore_installed = True
-    requirementSet = c.run(options, args)
+            try:
+                from pip._internal.baseparser import create_main_parser
+            except ImportError:  # Pip >= 19.3
+                from pip._internal.commands import create_command
+                from pip._internal.cli.main_parser import parse_command
+                _, options = parse_command(['install', '-r', file_path, '--ignore-installed'])
+                c = create_command('install')
+        if not c:
+            main_parser = create_main_parser()
+            c = InstallCommand(main_parser)
+
+    if not options:
+        options, args = c.parser.parse_args(['-r', file_path])
+        options.require_venv = True
+        options.ignore_installed = True
+    if hasattr(c, 'main'):
+        c.main(options)
+    else:
+        c.run(options, args)
 
     os.unlink(file_path)
 
@@ -710,6 +758,11 @@ def parse_args():
             VIRTUAL_ENV.
         ''',
     )
+    ap.add_argument(
+        '-p', '--python-executable',
+        default='',
+        help='Python executable to use in virtualenv, eg. python3.6',
+    )
     ap.add_argument(
         '--pip-log-level',
         default=25,

tests/tests.py

@@ -9,6 +9,7 @@
 import platform
 import copy
 import sys
+import virtualenv
 
 
 class TerrariumTester(unittest.TestCase):
@@ -60,6 +61,19 @@ def requirements(self):
     def opts(self):
         return self.config['opts']
 
+    @property
+    def virtualenv_version_tuple(self):
+        parts = virtualenv.virtualenv_version.split('.')
+        return tuple([int(v) for v in parts])
+
+    def is_python_version_available(self, executable):
+        try:
+            result = subprocess.call(
+                [executable, '--version'], stdout=subprocess.PIPE)
+            return result == 0
+        except OSError:
+            return False
+
     def config_pop(self):
         return self.configs.pop()
 
@@ -175,7 +189,6 @@ def _add_test_requirement(self):
         self._add_requirements(test_requirement)
 
     def _add_terrarium_requirement(self):
-        import virtualenv
         self._add_requirements(
             self._get_path_terrarium(),
             'virtualenv==%s' % virtualenv.virtualenv_version
@@ -622,3 +635,25 @@ def test_require_download(self):
             'Refusing to build a new bundle.\n',
         )
         self.assertNotExists(self.python)
+
+    def test_install_with_python3_executable(self):
+        skip = None
+        if not self.is_python_version_available('python3.6'):
+            skip = 'python3.6 not installed'
+        if self.virtualenv_version_tuple < (15, 0):
+            skip = 'py3 not supported with this virtualenv version'
+
+        if skip:
+            if hasattr(unittest, 'SkipTest'):
+                raise unittest.SkipTest(skip)
+            else:
+                # SkipTest not supported in Python 2.6
+                print(skip)
+                return
+
+        self.assertInstall(
+            storage_dir=self.storage_dir, python_executable='python3.6')
+        self.assertTrue(os.path.join(self.target, 'bin', 'python3.6'))
+        self.assertEqual(
+            os.readlink(os.path.join(self.target, 'bin', 'python')),
+            'python3.6')