diff --git a/object_storage/client.py b/object_storage/client.py index 0bfbd17..86e1f80 100755 --- a/object_storage/client.py +++ b/object_storage/client.py @@ -11,6 +11,7 @@ from object_storage import errors +import six import logging logger = logging.getLogger(__name__) @@ -20,7 +21,7 @@ def __init__(self, controller, headers={}): _headers = {} # Lowercase headers - for key, value in headers.iteritems(): + for key, value in headers.items(): _key = key.lower() _headers[_key] = value self.headers = _headers @@ -42,7 +43,7 @@ def __init__(self, controller, headers={}): _properties['url'] = controller.url meta = {} - for key, value in self.headers.iteritems(): + for key, value in self.headers.items(): if key.startswith('meta_'): meta[key[5:]] = value elif key.startswith('x-account-meta-'): @@ -53,6 +54,12 @@ def __init__(self, controller, headers={}): self.properties = _properties self.data = self.properties + def __len__(self): + return len(self.properties) + + def __iter__(self): + return iter(self.properties) + class Client(object): """ @@ -160,7 +167,7 @@ def search(self, q=None, options=None, **kwargs): params = {} options = options or {} options.update(kwargs) - for key, val in options.iteritems(): + for key, val in options.items(): if key.startswith('q_'): params["q.%s" % key[2:]] = val else: @@ -176,7 +183,7 @@ def search(self, q=None, options=None, **kwargs): def _formatter(response): """ Formats search results. """ headers = response.headers - items = json.loads(response.content) + items = json.loads(response.content if isinstance(response.content, six.string_types) else response.content.decode('utf8')) objs = [] for item in items: if 'type' not in item or item['type'] == 'container': @@ -234,9 +241,9 @@ def set_metadata(self, meta, headers={}): @raises ResponseError """ meta_headers = {} - for k, v in headers.iteritems(): + for k, v in headers.items(): meta_headers[k] = v - for k, v in meta.iteritems(): + for k, v in meta.items(): meta_headers["x-account-meta-%s" % (k, )] = v self.make_request('POST', headers=meta_headers) @@ -262,7 +269,7 @@ def delete_container(self, name, recursive=False): try: return self.make_request('DELETE', [name], params=params, formatter=lambda r: True) - except errors.ResponseError, ex: + except errors.ResponseError as ex: if ex.status == 409: raise errors.ContainerNotEmpty(ex.status, "ContainerNotEmpty Error") @@ -282,7 +289,7 @@ def containers(self, marker=None, headers=None): def _formatter(res): containers = [] if res.content: - items = json.loads(res.content) + items = json.loads(res.content if isinstance(res.content, six.string_types) else res.content.decode('utf8')) for item in items: name = item.get('name', None) containers.append(self.container(name, item)) diff --git a/object_storage/container.py b/object_storage/container.py index 0ac5146..ac8b24d 100755 --- a/object_storage/container.py +++ b/object_storage/container.py @@ -4,6 +4,8 @@ See COPYING for license information """ import os +import six + from object_storage.utils import json, Model from object_storage import errors from object_storage.storage_object import StorageObject @@ -16,7 +18,7 @@ def __init__(self, controller, name, headers={}): _headers = {} # Lowercase headers - for key, value in headers.iteritems(): + for key, value in headers.items(): _key = key.lower() _headers[_key] = value self.headers = _headers @@ -43,7 +45,7 @@ def __init__(self, controller, name, headers={}): meta = {} - for key, value in self.headers.iteritems(): + for key, value in self.headers.items(): if key.startswith('meta_'): meta[key[5:]] = value elif key.startswith('x-container-meta-'): @@ -54,6 +56,12 @@ def __init__(self, controller, name, headers={}): self.properties = _properties self.data = self.properties + def __len__(self): + return len(self.properties) + + def __iter__(self): + return iter(self.properties) + class Container: """ Container class. Encapsulates Storage containers. """ @@ -150,7 +158,7 @@ def set_metadata(self, meta): @raises ResponseError """ meta_headers = {} - for k, v in meta.iteritems(): + for k, v in meta.items(): meta_headers["x-container-meta-%s" % (k, )] = v return self.make_request('POST', headers=meta_headers) @@ -227,7 +235,7 @@ def objects(self, limit=None, marker=None, base_only=False, headers=None): def _formatter(res): objects = {} if res.content: - items = json.loads(res.content) + items = json.loads(res.content if isinstance(res.content, six.string_types) else res.content.decode('utf8')) for item in items: if 'name' in item: objects[item['name']] = self.storage_object( diff --git a/object_storage/storage_object.py b/object_storage/storage_object.py index 94070e9..c7d678b 100755 --- a/object_storage/storage_object.py +++ b/object_storage/storage_object.py @@ -6,8 +6,9 @@ from object_storage.utils import json, Model import mimetypes import os +import six import logging - +from io import IOBase try: import StringIO except ImportError: @@ -30,7 +31,7 @@ def __init__(self, controller, container, name, headers={}): _headers = {} # Lowercase headers - for key, value in headers.iteritems(): + for key, value in headers.items(): _key = key.lower() _headers[_key] = value self.headers = _headers @@ -59,7 +60,7 @@ def __init__(self, controller, container, name, headers={}): _properties['url'] = controller.url meta = {} - for key, value in self.headers.iteritems(): + for key, value in self.headers.items(): if key.startswith('meta_'): meta[key[5:]] = value elif key.startswith('x-object-meta-'): @@ -70,6 +71,11 @@ def __init__(self, controller, container, name, headers={}): self.properties = _properties self.data = self.properties + def __len__(self): + return len(self.properties) + + def __iter__(self): + return iter(self.properties) class StorageObject: """ @@ -185,7 +191,7 @@ def list(self, limit=None, marker=None, base_only=False): def _formatter(res): objects = {} if res.content: - items = json.loads(res.content) + items = json.loads(res.content if isinstance(res.content, six.string_types) else res.content.decode('utf8')) for item in items: if 'name' in item: objects[item['name']] = self.client.storage_object( @@ -222,7 +228,7 @@ def set_metadata(self, meta): @raises ResponseError """ meta_headers = {} - for k, v in meta.iteritems(): + for k, v in meta.items(): meta_headers["X-Object-Meta-%s" % (k, )] = v return self.make_request('POST', headers=meta_headers) @@ -327,7 +333,7 @@ def send(self, data, check_md5=True): @return: StorageObject, self """ size = None - if isinstance(data, file): + if isinstance(data, IOBase): try: data.flush() except IOError: @@ -337,7 +343,7 @@ def send(self, data, check_md5=True): if hasattr(data, '__len__'): size = len(data) - if isinstance(data, basestring): + if isinstance(data, six.binary_type): data = StringIO.StringIO(data) headers = {} @@ -364,7 +370,7 @@ def send(self, data, check_md5=True): res = conn.finish() if check_md5: - assert checksum.hexdigest() == res.headers['etag'], \ + assert checksum.hexdigest() == res.headers.get('etag'), \ 'md5 hashes do not match' res.headers['content-length'] = transfered self.model = StorageObjectModel( @@ -480,6 +486,9 @@ def make_request(self, method, path=None, *args, **kwargs): def fileno(self): return 1 + def close(self): + pass + def __len__(self): if not self.model: self.load() diff --git a/object_storage/transport/__init__.py b/object_storage/transport/__init__.py index 14481cd..15f0804 100644 --- a/object_storage/transport/__init__.py +++ b/object_storage/transport/__init__.py @@ -3,12 +3,18 @@ See COPYING for license information """ -import httplib from socket import timeout -from urlparse import urlparse -import urllib2 +import sys import re +if sys.version_info.major == 2: + from urllib2 import Request, urlopen + from urlparse import urlparse + from httplib import HTTPConnection, HTTPSConnection +else: + from urllib.request import Request, urlopen + from urllib.parse import urlparse + from http.client import HTTPConnection, HTTPSConnection from object_storage.errors import ResponseError, NotFound from object_storage import consts @@ -45,8 +51,7 @@ def _authenticate(self): def get_headers(self): """ Get default headers for this connection """ - return dict([('User-Agent', consts.USER_AGENT)] + - self.auth_headers.items()) + return dict([('User-Agent', consts.USER_AGENT)] + list(self.auth_headers.items())) def chunk_upload(self, method, url, size=None, headers=None): """ Returns new ChunkedConnection """ @@ -58,10 +63,10 @@ def chunk_upload(self, method, url, size=None, headers=None): def chunk_download(self, url, chunk_size=10 * 1024): """ Returns new ChunkedConnection """ headers = self.get_headers() - req = urllib2.Request(url) - for k, v in headers.iteritems(): + req = Request(url) + for k, v in headers.items(): req.add_header(k, v) - r = urllib2.urlopen(req) + r = urlopen(req) while True: buff = r.read(chunk_size) if not buff: @@ -158,16 +163,16 @@ def __init__(self, conn, method, url, size=None, headers=None): port = int(port) if scheme == 'https': - self.req = httplib.HTTPSConnection(host, port) + self.req = HTTPSConnection(host, port) else: - self.req = httplib.HTTPConnection(host, port) + self.req = HTTPConnection(host, port) try: self.req.putrequest('PUT', path) - for key, value in headers.iteritems(): + for key, value in headers.items(): self.req.putheader(key, value) self.req.endheaders() - except Exception: - raise ResponseError(0, 'Disconnected') + except Exception as e: + raise ResponseError(0, 'Disconnected: %s' % e) def send(self, chunk): """ Sends a chunk of data. """ @@ -178,7 +183,7 @@ def send(self, chunk): self.req.send("\r\n") else: self.req.send(chunk) - except timeout, err: + except timeout as err: raise err except: raise ResponseError(0, 'Disconnected') @@ -188,7 +193,7 @@ def finish(self): try: if self._chunked_encoding: self.req.send("0\r\n\r\n") - except timeout, err: + except timeout as err: raise err except: raise ResponseError(0, 'Disconnected') @@ -199,7 +204,7 @@ def finish(self): r = Response() r.status_code = res.status r.version = res.version - r.headers = dict(res.getheaders()) + r.headers = dict([(k.lower(), v) for k,v in res.getheaders()]) r.content = content r.raise_for_status() return r diff --git a/object_storage/transport/httplib2conn.py b/object_storage/transport/httplib2conn.py index 16b6cb6..7763c2f 100755 --- a/object_storage/transport/httplib2conn.py +++ b/object_storage/transport/httplib2conn.py @@ -3,13 +3,13 @@ See COPYING for license information """ -import urllib +import six from object_storage import errors from object_storage.transport import BaseAuthentication, \ BaseAuthenticatedConnection, Response import httplib2 -from object_storage.utils import json +from object_storage.utils import json, unicode_urlencode import logging logger = logging.getLogger(__name__) @@ -39,7 +39,7 @@ def make_request(self, method, url=None, headers=None, formatter=None, headers.update(self.get_headers()) if params: - url = "%s?%s" % (url, urllib.urlencode(params)) + url = "%s?%s" % (url, unicode_urlencode(params)) def _make_request(headers): logger.debug("%s %s %s" % (method, url, headers)) @@ -100,7 +100,7 @@ def authenticate(self): raise errors.AuthenticationError('Invalid Credentials') response.raise_for_status() try: - storage_options = json.loads(response.content)['storage'] + storage_options = json.loads(response.content if isinstance(response.content, six.string_types) else response.content.decode('utf8'))['storage'] except ValueError: raise errors.StorageURLNotFound("Could not parse services JSON.") diff --git a/object_storage/transport/requestsconn.py b/object_storage/transport/requestsconn.py index adc623a..3f9f849 100755 --- a/object_storage/transport/requestsconn.py +++ b/object_storage/transport/requestsconn.py @@ -4,13 +4,14 @@ See COPYING for license information """ import requests +import six from object_storage.transport import BaseAuthentication, \ BaseAuthenticatedConnection from object_storage import errors from object_storage.utils import json import logging -logger = logging.getLogger(__name__) +logger = logging.getLogger('softlayer.transport.requests') class AuthenticatedConnection(BaseAuthenticatedConnection): @@ -24,6 +25,7 @@ def __init__(self, auth, **kwargs): self.auth = auth self.auth.authenticate() self._authenticate() + self.session = requests.Session() def make_request(self, method, url=None, *args, **kwargs): """ Makes a request """ @@ -34,15 +36,14 @@ def make_request(self, method, url=None, *args, **kwargs): kwargs['headers'] = headers if 'verify' not in kwargs: - kwargs['verify'] = False + kwargs['verify'] = True formatter = None if 'formatter' in kwargs: formatter = kwargs.get('formatter') del kwargs['formatter'] - print method, url, args, kwargs - res = requests.request(method, url, *args, **kwargs) + res = self.session.request(method, url, *args, **kwargs) if kwargs.get('return_response', True): res = self._check_success(res) if res.status_code == 404: @@ -64,10 +65,10 @@ def _check_success(self, res): if res.status_code == 401: # Authenticate and try again with a (hopefully) new token + self.auth.authenticate() self._authenticate() res.request.headers.update(self.auth_headers) - res.request.send(anyway=True) - res = res.request.response + res = self.session.send(res.request) return res @@ -92,7 +93,7 @@ def authenticate(self): headers = {'X-Storage-User': self.username, 'X-Storage-Pass': self.api_key, 'Content-Length': '0'} - response = requests.get(self.auth_url, headers=headers, verify=False) + response = requests.get(self.auth_url, headers=headers, verify=True) if response.status_code == 401: raise errors.AuthenticationError('Invalid Credentials') @@ -100,7 +101,7 @@ def authenticate(self): response.raise_for_status() try: - storage_options = json.loads(response.content)['storage'] + storage_options = json.loads(response.content if isinstance(response.content, six.string_types) else response.content.decode('utf8'))['storage'] except ValueError: raise errors.StorageURLNotFound("Could not parse services JSON.") diff --git a/object_storage/transport/twist.py b/object_storage/transport/twist.py index 2cc72b6..f48a2f5 100755 --- a/object_storage/transport/twist.py +++ b/object_storage/transport/twist.py @@ -20,9 +20,9 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH import urlparse -import urllib +import six -from object_storage.utils import json +from object_storage.utils import json, unicode_urlencode def complete_request(resp, callback=None, load_body=True): @@ -98,7 +98,7 @@ def _nothing(result): params = kwargs.get('params', None) if params: - params = urllib.urlencode(params) + params = unicode_urlencode(params) url = _full_url(url, params) body = kwargs.get('data') @@ -173,7 +173,7 @@ def _authenticate(self, response): response.raise_for_status() try: - storage_options = json.loads(response.content)['storage'] + storage_options = json.loads(response.content if isinstance(response.content, six.string_types) else response.content.decode('utf8'))['storage'] except ValueError: raise errors.StorageURLNotFound("Could not parse services JSON.") diff --git a/object_storage/utils.py b/object_storage/utils.py index 0b0a5fd..fd040f6 100755 --- a/object_storage/utils.py +++ b/object_storage/utils.py @@ -41,9 +41,13 @@ def keys(self): return self.properties.keys() if sys.version_info >= (3,): + from urllib.parse import quote, urlencode + def unicode_quote(s): - from urllib.parse import quote return quote(s) + + def unicode_urlencode(params): + return urlencode(params) else: def unicode_quote(s): """ Solves an issue with url-quoting unicode strings""" @@ -52,6 +56,8 @@ def unicode_quote(s): else: return urllib.quote(str(s)) + def unicode_urlencode(params): + return urllib.urlencode(params) def get_path(parts=None): """ diff --git a/tests/test_authentication.py b/tests/test_authentication.py index 760668d..a68b79d 100644 --- a/tests/test_authentication.py +++ b/tests/test_authentication.py @@ -8,15 +8,15 @@ class BaseAuthenticationTest(unittest.TestCase): def test_instance_setup(self): - self.assert_(self.auth.storage_url is None, + self.assertTrue(self.auth.storage_url is None, "Storage url is set correctly") - self.assert_(self.auth.auth_token is None, "auth_token set correctly") + self.assertTrue(self.auth.auth_token is None, "auth_token set correctly") def test_authenticate(self): self.auth.authenticate() - self.assert_(self.auth.storage_url == 'STORAGE_URL', + self.assertTrue(self.auth.storage_url == 'STORAGE_URL', "storage_url set correctly") - self.assert_(self.auth.auth_token == 'AUTH_TOKEN', + self.assertTrue(self.auth.auth_token == 'AUTH_TOKEN', "auth_token set correctly") def setUp(self): diff --git a/tests/test_client.py b/tests/test_client.py index 18de453..f8ac3a7 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -9,19 +9,19 @@ class ClientTest(unittest.TestCase): def test_instance_setup(self): - self.assert_(self.client.username == 'username', "username set") - self.assert_(self.client.api_key == 'api_key', "api_key set") - self.assert_(self.client.container_class == self.container_class, + self.assertTrue(self.client.username == 'username', "username set") + self.assertTrue(self.client.api_key == 'api_key', "api_key set") + self.assertTrue(self.client.container_class == self.container_class, "container_class set") - self.assert_(self.client.object_class == self.object_class, + self.assertTrue(self.client.object_class == self.object_class, "object_class set") - self.assert_(self.client.conn == self.connection, "connection set") - self.assert_(self.client.delimiter == '/', "default delimiter set") + self.assertTrue(self.client.conn == self.connection, "connection set") + self.assertTrue(self.client.delimiter == '/', "default delimiter set") def test_set_delimiter(self): delimiter = Mock() self.client.set_delimiter(delimiter) - self.assert_(self.client.delimiter == delimiter, + self.assertTrue(self.client.delimiter == delimiter, "set_delimiter sets the delimiter") def test_container(self): @@ -41,7 +41,7 @@ def test_get_container(self): loaded_item = Mock() self.container_class().load.return_value = loaded_item result = self.client.get_container('container_name') - self.assert_(loaded_item == result) + self.assertTrue(loaded_item == result) self.container_class.assert_called_with('container_name', client=self.client, headers=None) @@ -50,7 +50,7 @@ def test_create_container(self): created_item = Mock() self.container_class().create.return_value = created_item result = self.client.create_container('container_name') - self.assert_(created_item == result, 'returns the container itself') + self.assertTrue(created_item == result, 'returns the container itself') self.container_class.assert_called_with('container_name', client=self.client, headers=None) @@ -95,7 +95,7 @@ def test_get_object(self): loaded_item = Mock() self.object_class().load.return_value = loaded_item result = self.client.get_object('object_name', 'container_name') - self.assert_(loaded_item == result, "Returns the correct object") + self.assertTrue(loaded_item == result, "Returns the correct object") self.object_class.assert_called_with('object_name', 'container_name', client=self.client, headers=None) @@ -112,34 +112,34 @@ def test_make_request_listpath(self): 'METHOD', 'storage_url/PATH/PATH2') def test_is_dir(self): - self.assert_(self.client.is_dir() is True, + self.assertTrue(self.client.is_dir() is True, 'Client itself is a directory') def test_path(self): - self.assert_(self.client.path == '', + self.assertTrue(self.client.path == '', "Path returns an empty string for Client") def test_get_url(self): self.connection.storage_url = 'storage_url' url = self.client.get_url(['path']) - self.assert_(url == 'storage_url/path', + self.assertTrue(url == 'storage_url/path', "URL Returns correctly with one-item list path") self.connection.storage_url = 'storage_url' url = self.client.get_url(['path', 'path2']) - self.assert_(url == 'storage_url/path/path2', + self.assertTrue(url == 'storage_url/path/path2', "URL Returns correctly with two-item list path") self.connection.storage_url = 'storage_url' self.storage_url = None url = self.client.get_url(['path']) - self.assert_(url == 'storage_url/path', + self.assertTrue(url == 'storage_url/path', "The storage_url is retreived from connection object " "when it isn't set") self.connection.storage_url = 'storage_url' url = self.client.get_url(['path']) - self.assert_(url == 'storage_url/path', + self.assertTrue(url == 'storage_url/path', 'URL is returned correctly with a string path') def test_chunk_upload(self): @@ -151,7 +151,7 @@ def test_chunk_upload(self): self.connection.chunk_upload.return_value = _chunkable chunkable = self.client.chunk_upload('path', headers=_headers, size=_size) - self.assert_(chunkable == _chunkable, + self.assertTrue(chunkable == _chunkable, 'Chunkable returns from conn.get_chunkable') self.connection.chunk_upload.assert_called_once_with('PUT', _url, headers=_headers, @@ -161,7 +161,7 @@ def test_getitem(self): _container = Mock() self.client.container = Mock(return_value=_container) container = self.client['CONTAINER'] - self.assert_(container == _container, + self.assertTrue(container == _container, 'Container returns from client.container()') self.client.container.assert_called_once_with('CONTAINER') diff --git a/tests/test_container.py b/tests/test_container.py index ed01a69..ab85750 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -9,8 +9,8 @@ class ContainerTest(unittest.TestCase): def test_instance_setup(self): - self.assert_(self.client == self.container.client, "client is set") - self.assert_(self.container.name == 'CONTAINER', "name is set") + self.assertTrue(self.client == self.container.client, "client is set") + self.assertTrue(self.container.name == 'CONTAINER', "name is set") def test_create(self): self.container.make_request = Mock() @@ -84,7 +84,7 @@ def test_url(self): def test_is_dir(self): result = self.container.is_dir() - self.assert_(result is True) + self.assertTrue(result is True) def test_make_request(self): self.container.make_request('METHOD', 1, 2, a1=1, a2=2) @@ -96,7 +96,7 @@ def test_getitem(self): _obj = Mock() self.container.storage_object = Mock(return_value=_obj) obj = self.container['OBJECT'] - self.assert_(obj == _obj, "Object returns from container.object()") + self.assertTrue(obj == _obj, "Object returns from container.object()") self.container.storage_object.assert_called_once_with('OBJECT') def setUp(self): diff --git a/tests/test_storage_object.py b/tests/test_storage_object.py index 21850f8..d884e32 100644 --- a/tests/test_storage_object.py +++ b/tests/test_storage_object.py @@ -8,9 +8,9 @@ class ClientTest(unittest.TestCase): def test_instance_setup(self): - self.assert_(self.client == self.obj.client, "client is set") - self.assert_(self.obj.container == 'CONTAINER', "container is set") - self.assert_(self.obj.name == 'NAME', "name is set") + self.assertTrue(self.client == self.obj.client, "client is set") + self.assertTrue(self.obj.container == 'CONTAINER', "container is set") + self.assertTrue(self.obj.name == 'NAME', "name is set") def test_create(self): # no content_type and no ext diff --git a/tox.ini b/tox.ini index f33d3f3..05a30a7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pep8,py25,py26,py27,pypy +envlist = pep8,py25,py26,py27,py34,pypy [testenv] commands = {envbindir}/nosetests []