Allows anonymous download of S3 buckets

docs/_quickstart.rst

@@ -4,6 +4,13 @@
 
 See ``terrarium --help`` for a complete list of options and commands.
 
+.. note::
+
+    The following documentation assumes you are familiar with `virtualenv
+    <http://www.virtualenv.org/en/latest/>`_. If not, run through `this
+    tutorial <http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_. We'll
+    wait.
+
 Creating a new environment
 ##########################
 
@@ -12,7 +19,7 @@ includes the packages defined in ``requirements.txt``
 
 .. code-block:: shell-session
 
-    $ terrarium --target env install requirements.txt
+    $ terrarium --target path/to/env install requirements.txt
 
 Replacing an existing environment
 #################################
@@ -22,8 +29,8 @@ existing activated virtual environment with a different set of packages.
 
 .. code-block:: shell-session
 
-    $ terrarium --target env install requirements.txt
-    $ source env/bin/activate
+    $ terrarium --target path/to/env install requirements.txt
+    $ source path/to/env/bin/activate
     $ terrarium install other_requirements.txt
 
 .. note::

requirements/testing.txt

@@ -1 +1,2 @@
 nose
+requests==1.0.0

setup.py

@@ -43,6 +43,7 @@ def main():
     python_version = sys.version_info[:2]
     install_requires = [
         'virtualenv>=1.7.2,<=1.9.1',
+        'requests>=1.0.0',
     ]
     if python_version < (2, 7) or (3, 0) <= python_version <= (3, 1):
         install_requires += ['argparse']

terrarium/terrarium.py

@@ -8,6 +8,7 @@
 import tempfile
 import shutil
 import logging
+import requests
 
 from logging import getLogger, StreamHandler
 
@@ -393,6 +394,50 @@ def _get_s3_bucket(self):
             pass
         return boto.s3.bucket.Bucket(conn, name=self.args.s3_bucket)
 
+    def _attempt_s3_download(self, target):
+        # attempt to download from public S3 url
+        base_s3_url = "https://%(bucket_name)s.s3.amazonaws.com/%(key)s"
+        remote_key = self.make_remote_key()
+        bucket_name = self.args.s3_bucket
+        s3_url = base_s3_url % {"bucket_name": bucket_name, "key": remote_key}
+        r = requests.get(s3_url, stream=True)
+        if r.ok:
+            fd, archive = tempfile.mkstemp()
+            with open(archive, 'wb') as f:
+                for chunk in r.iter_content(1024):
+                    f.write(chunk)
+
+            self.extract(archive, target)
+            os.close(fd)
+            os.unlink(archive)
+            return True
+        # Check if permissions denied
+        elif not self.args.s3_access_key and 'AccessDenied' in r.content:
+            print (
+                'Access was denied to s3 bucket hash %s '
+                'and no key was provided. '
+                'Make sure the s3 bucket hash is public '
+                'readable.' % (remote_key)
+            )
+
+        if not boto:
+            return False
+        bucket = self._get_s3_bucket()
+        if bucket:
+            key = bucket.get_key(remote_key)
+            if key:
+                logger.info(
+                    'Downloading %s/%s from S3 '
+                    '(this may take time) ...'
+                    % (self.args.s3_bucket, remote_key)
+                )
+                fd, archive = tempfile.mkstemp()
+                key.get_contents_to_filename(archive)
+                self.extract(archive, target)
+                os.close(fd)
+                os.unlink(archive)
+                return True
+
     def download(self, target):
         if self.args.storage_dir:
             remote_archive = os.path.join(
@@ -413,23 +458,10 @@ def download(self, target):
                 os.unlink(local_archive)
                 return True
             logger.error('Download archive failed')
-        if boto and self.args.s3_bucket:
-            bucket = self._get_s3_bucket()
-            if bucket:
-                remote_key = self.make_remote_key()
-                key = bucket.get_key(remote_key)
-                if key:
-                    logger.info(
-                        'Downloading %s/%s from S3 '
-                        '(this may take time) ...'
-                        % (self.args.s3_bucket, remote_key)
-                    )
-                    fd, archive = tempfile.mkstemp()
-                    key.get_contents_to_filename(archive)
-                    self.extract(archive, target)
-                    os.close(fd)
-                    os.unlink(archive)
-                    return True
+
+        if self.args.s3_bucket:
+            download_successful = self._attempt_s3_download(target)
+            return download_successful
 
     def make_remote_key(self):
         import platform
@@ -493,8 +525,14 @@ def upload(self, target):
                 target,
                 self.args.storage_dir,
             )
-        if boto and self.args.s3_bucket:
-            self.upload_to_s3(target)
+        if self.args.s3_bucket:
+            if boto:
+                self.upload_to_s3(target)
+            else:
+                logger.critical(
+                    "--upload combined with --s3-bucket requires that you "
+                    "have boto installed, which does not appear to be the case"
+                )
 
     def create_bootstrap(self, dest):
         extra_text = (
@@ -784,12 +822,6 @@ def parse_args():
     args = ap.parse_args()
     args.add_sensitive_arguments(*sensitive_arguments)
 
-    if not boto and args.s3_bucket is not None:
-        ap.error(
-            '--s3-bucket requires that you have boto installed, '
-            'which does not appear to be the case'
-        )
-
     return args
 
 

tests/tests.py

@@ -500,18 +500,13 @@ def test_logging_output(self):
             'Terrarium is finished\n'
         ))
 
-    def test_boto_required_to_use_s3_bucket(self):
+    def test_boto_not_required_to_use_s3_bucket(self):
         self._add_test_requirement()
 
-        output = self.assertInstall(
-            return_code=2,
+        self.assertInstall(
+            return_code=0,
             s3_bucket='bucket',
         )
-        self.assertTrue(
-            'error: --s3-bucket requires that you have boto installed, '
-            'which does not appear to be the case'
-            in output[1]
-        )
 
     def test_sensitive_arguments_are_sensitive(self):
         command = 'hash %s' % (