From 542f727d40c60a32e7b3b250a6824b55e4d81679 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 22 Apr 2014 09:02:48 -0400 Subject: [PATCH 1/9] Updated docs for creating new environments --- docs/_quickstart.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/_quickstart.rst b/docs/_quickstart.rst index 9d990e9..ddb9883 100644 --- a/docs/_quickstart.rst +++ b/docs/_quickstart.rst @@ -4,6 +4,13 @@ See ``terrarium --help`` for a complete list of options and commands. +.. note:: + + The following documentation assumes you are familiar with `virtualenv + `_. If not, run through `this + tutorial `_. We'll + wait. + Creating a new environment ########################## @@ -12,7 +19,7 @@ includes the packages defined in ``requirements.txt`` .. code-block:: shell-session - $ terrarium --target env install requirements.txt + $ terrarium --target path/to/env install requirements.txt Replacing an existing environment ################################# @@ -22,8 +29,8 @@ existing activated virtual environment with a different set of packages. .. code-block:: shell-session - $ terrarium --target env install requirements.txt - $ source env/bin/activate + $ terrarium --target path/to/env install requirements.txt + $ source path/to/env/bin/activate $ terrarium install other_requirements.txt .. note:: From 9027d0d021d12347214a5729dd0b2c4c1622e4e0 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 22 Apr 2014 09:36:39 -0400 Subject: [PATCH 2/9] Pulled attempt to download from S3 bucket into its own method --- terrarium/terrarium.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/terrarium/terrarium.py b/terrarium/terrarium.py index 269c6f2..4b43569 100755 --- a/terrarium/terrarium.py +++ b/terrarium/terrarium.py @@ -393,6 +393,26 @@ def _get_s3_bucket(self): pass return boto.s3.bucket.Bucket(conn, name=self.args.s3_bucket) + def _attempt_s3_download(self, target): + if not boto: + return False + bucket = self._get_s3_bucket() + if bucket: + remote_key = self.make_remote_key() + key = bucket.get_key(remote_key) + if key: + logger.info( + 'Downloading %s/%s from S3 ' + '(this may take time) ...' + % (self.args.s3_bucket, remote_key) + ) + fd, archive = tempfile.mkstemp() + key.get_contents_to_filename(archive) + self.extract(archive, target) + os.close(fd) + os.unlink(archive) + return True + def download(self, target): if self.args.storage_dir: remote_archive = os.path.join( @@ -413,23 +433,9 @@ def download(self, target): os.unlink(local_archive) return True logger.error('Download archive failed') - if boto and self.args.s3_bucket: - bucket = self._get_s3_bucket() - if bucket: - remote_key = self.make_remote_key() - key = bucket.get_key(remote_key) - if key: - logger.info( - 'Downloading %s/%s from S3 ' - '(this may take time) ...' - % (self.args.s3_bucket, remote_key) - ) - fd, archive = tempfile.mkstemp() - key.get_contents_to_filename(archive) - self.extract(archive, target) - os.close(fd) - os.unlink(archive) - return True + + if self.args.s3_bucket: + self._attempt_s3_download(target) def make_remote_key(self): import platform From f4a1cdb2311577853ef1a1b4579cf0bd46b63909 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 22 Apr 2014 10:05:57 -0400 Subject: [PATCH 3/9] Refs #24. First crack at HTTP S3 download attempt. --- terrarium/terrarium.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/terrarium/terrarium.py b/terrarium/terrarium.py index 4b43569..c3c6e03 100755 --- a/terrarium/terrarium.py +++ b/terrarium/terrarium.py @@ -8,6 +8,7 @@ import tempfile import shutil import logging +import requests from logging import getLogger, StreamHandler @@ -394,11 +395,27 @@ def _get_s3_bucket(self): return boto.s3.bucket.Bucket(conn, name=self.args.s3_bucket) def _attempt_s3_download(self, target): + # attempt to download from public S3 url + base_s3_url = "https://%(bucket_name)s.s3.amazonaws.com/%(key)s" + remote_key = self.make_remote_key() + bucket_name = self.args.s3_bucket + s3_url = base_s3_url % {"bucket_name": bucket_name, "key": remote_key} + r = requests.get(s3_url, stream=True) + if r.ok: + fd, archive = tempfile.mkstemp() + with open(archive, 'wb') as f: + for chunk in r.iter_content(1024): + f.write(chunk) + + self.extract(archive, target) + os.close(fd) + os.unlink(archive) + return True + if not boto: return False bucket = self._get_s3_bucket() if bucket: - remote_key = self.make_remote_key() key = bucket.get_key(remote_key) if key: logger.info( @@ -435,7 +452,8 @@ def download(self, target): logger.error('Download archive failed') if self.args.s3_bucket: - self._attempt_s3_download(target) + download_successful = self._attempt_s3_download(target) + return download_successful def make_remote_key(self): import platform From a6fcd826835dcba814df2c17a8f92f23d532e5a4 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 22 Apr 2014 10:26:17 -0400 Subject: [PATCH 4/9] Refs #24. Added requests to the dependencies. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e872013..8a45ae4 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def main(): python_version = sys.version_info[:2] install_requires = [ 'virtualenv>=1.7.2,<=1.9.1', + 'requests>=1.0.0', ] if python_version < (2, 7) or (3, 0) <= python_version <= (3, 1): install_requires += ['argparse'] From 5016a21fa51fe8153e358084123085b556162c86 Mon Sep 17 00:00:00 2001 From: winhamwr Date: Tue, 22 Apr 2014 10:32:41 -0400 Subject: [PATCH 5/9] Refs #24. Don't immediately bail if boto isn't installed. We can use the S3 bucket via HTTP --- terrarium/terrarium.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/terrarium/terrarium.py b/terrarium/terrarium.py index c3c6e03..3291399 100755 --- a/terrarium/terrarium.py +++ b/terrarium/terrarium.py @@ -517,8 +517,14 @@ def upload(self, target): target, self.args.storage_dir, ) - if boto and self.args.s3_bucket: - self.upload_to_s3(target) + if self.args.s3_bucket: + if boto: + self.upload_to_s3(target) + else: + logger.critical( + "--upload combined with --s3-bucket requires that you " + "have boto installed, which does not appear to be the case" + ) def create_bootstrap(self, dest): extra_text = ( @@ -808,12 +814,6 @@ def parse_args(): args = ap.parse_args() args.add_sensitive_arguments(*sensitive_arguments) - if not boto and args.s3_bucket is not None: - ap.error( - '--s3-bucket requires that you have boto installed, ' - 'which does not appear to be the case' - ) - return args From 8e2a1d7434c299b359d28d7c7c05eb143c86905e Mon Sep 17 00:00:00 2001 From: Charles Shinaver Date: Wed, 25 Jun 2014 12:04:43 -0400 Subject: [PATCH 6/9] Refs #24 Fixed failing test to not immediately bail if boto not installed --- tests/tests.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index f1ba95e..a0b0f02 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -500,18 +500,13 @@ def test_logging_output(self): 'Terrarium is finished\n' )) - def test_boto_required_to_use_s3_bucket(self): + def test_boto_not_required_to_use_s3_bucket(self): self._add_test_requirement() - output = self.assertInstall( - return_code=2, + self.assertInstall( + return_code=0, s3_bucket='bucket', ) - self.assertTrue( - 'error: --s3-bucket requires that you have boto installed, ' - 'which does not appear to be the case' - in output[1] - ) def test_sensitive_arguments_are_sensitive(self): command = 'hash %s' % ( From 95b4a96f6880eae9b2c134cb734d75b4b985b60b Mon Sep 17 00:00:00 2001 From: Charles Shinaver Date: Sat, 5 Jul 2014 13:27:08 -0400 Subject: [PATCH 7/9] Added requests==1.0.0 requirement which was causing test errors --- requirements/testing.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/testing.txt b/requirements/testing.txt index f3c7e8e..87a4388 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1 +1,2 @@ nose +requests==1.0.0 From 3f78eab8a0cffed13703449f635f1b27a2bf8c99 Mon Sep 17 00:00:00 2001 From: Charles Shinaver Date: Sat, 5 Jul 2014 13:27:08 -0400 Subject: [PATCH 8/9] Refs #24 Added requests==1.0.0 requirement which was causing test errors --- requirements/testing.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/testing.txt b/requirements/testing.txt index f3c7e8e..87a4388 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1 +1,2 @@ nose +requests==1.0.0 From e5f042d273f6383ba070a17dbe2baecb9a8f4909 Mon Sep 17 00:00:00 2001 From: Charles Shinaver Date: Fri, 8 Aug 2014 15:25:17 -0400 Subject: [PATCH 9/9] Refs #24 Now displays helpful error when s3 bucket hash is not public readable --- terrarium/terrarium.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/terrarium/terrarium.py b/terrarium/terrarium.py index 3291399..d355d0b 100755 --- a/terrarium/terrarium.py +++ b/terrarium/terrarium.py @@ -411,6 +411,14 @@ def _attempt_s3_download(self, target): os.close(fd) os.unlink(archive) return True + # Check if permissions denied + elif not self.args.s3_access_key and 'AccessDenied' in r.content: + print ( + 'Access was denied to s3 bucket hash %s ' + 'and no key was provided. ' + 'Make sure the s3 bucket hash is public ' + 'readable.' % (remote_key) + ) if not boto: return False