From 20a43a1e38d48af0cc38c706b956831c0c1f21ea Mon Sep 17 00:00:00 2001 From: Romain Meson Date: Thu, 17 Oct 2019 18:55:22 +0200 Subject: [PATCH 1/7] Modify cursor.__iter__ to use a local copy of skip, so the original value set by the user is not lost. Also, fix a bug where a second iteration on the cursor would results in no content at all, because of self.retrieved not being reset. This is not an useful information outside of the iteration algo so self.retrieved was made local to reduce impact. Adding .vscode in .gitignore --- .gitignore | 3 +++ appnexus/cursor.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 2f1f405..0531d74 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,6 @@ ENV/ # Rope project settings .ropeproject + +# Visual Studio code +.vscode/ diff --git a/appnexus/cursor.py b/appnexus/cursor.py index 6ad9048..15a7c26 100644 --- a/appnexus/cursor.py +++ b/appnexus/cursor.py @@ -23,7 +23,6 @@ def __init__(self, client, service_name, representation, **specs): self.service_name = service_name self.representation = representation self.specs = specs - self.retrieved = 0 self._skip = 0 self._limit = float('inf') @@ -39,21 +38,22 @@ def __getitem__(self, idx): def __iter__(self): """Iterate over all AppNexus objects matching the specifications""" + retrieved = 0 + skip = self._skip for page in self.iter_pages(): data = self.extract_data(page) - if self._skip >= len(data): - self._skip -= len(data) + if skip >= len(data): + skip -= len(data) continue - elif self._skip: - self._skip = 0 - data = data[self._skip:] - lasting = self._limit - self.retrieved + elif skip: + data = data[skip:] + lasting = self._limit - retrieved if not lasting: break elif lasting < len(data): data = data[:lasting] for entity in data: - self.retrieved += 1 + retrieved += 1 yield entity def extract_data(self, page): From 5915e131dcb4a27b69f3df0dfd6f777aaea6525b Mon Sep 17 00:00:00 2001 From: Romain Meson Date: Fri, 18 Oct 2019 10:54:43 +0200 Subject: [PATCH 2/7] Fix the computation of start_element in gen_random_collection() in tests/helpers.py. The last page generated was not getting the right start_element and causing a StopIteration when the generated collection was assigned as a side_effect on a cursor. --- tests/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers.py b/tests/helpers.py index 7013a9a..573eb96 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -31,6 +31,7 @@ def gen_random_collection(count=None, object_type="campaigns"): start_element=i * 100) result.append(random_page) if count % 100 != 0: + i = i + 1 if len(result) else 0 random_page = gen_random_page(count=count, object_type=object_type, start_element=i * 100, num_elements=count % 100) From ae6993df13d0bf3b8f8583a6691a587c7993bf18 Mon Sep 17 00:00:00 2001 From: Romain Meson Date: Mon, 21 Oct 2019 16:01:58 +0200 Subject: [PATCH 3/7] Revamp the algorithm on the cursor on how it determines which query to made. The logic handling skip (if defined) and limit (if defined) has been transfered from __iter__ to iter_pages to avoid unecessary round-trips with AppNexus API. Add unit tests around that logic and revamp of the helpers that generate collections when mocking client.get results. --- appnexus/cursor.py | 23 +++----- tests/cursor.py | 144 ++++++++++++++++++++++++++++++++++++++++++++- tests/helpers.py | 54 ++++++++++------- 3 files changed, 182 insertions(+), 39 deletions(-) diff --git a/appnexus/cursor.py b/appnexus/cursor.py index 0da2252..dc07d50 100644 --- a/appnexus/cursor.py +++ b/appnexus/cursor.py @@ -39,19 +39,8 @@ def __getitem__(self, idx): def __iter__(self): """Iterate over all AppNexus objects matching the specifications""" retrieved = 0 - skip = self._skip for page in self.iter_pages(): data = self.extract_data(page) - if skip >= len(data): - skip -= len(data) - continue - elif skip: - data = data[skip:] - lasting = self._limit - retrieved - if not lasting: - break - elif lasting < len(data): - data = data[:lasting] for entity in data: retrieved += 1 yield entity @@ -86,15 +75,17 @@ def get_page(self, start_element=0, num_elements=None): specs.update(start_element=start_element, num_elements=num_elements) return self.client.get(self.service_name, **specs) - def iter_pages(self, skip_elements=0): + def iter_pages(self): """Iterate as much as needed to get all available pages""" - start_element = skip_elements + start_element = self._skip + num_elements = min(self._limit, self.batch_size) count = -1 while start_element < count or count == -1: - page = self.get_page(start_element) + page = self.get_page(start_element, num_elements) yield page - start_element = page["start_element"] + page["num_elements"] - count = page["count"] + start_element = start_element + page["num_elements"] + num_elements = min(page["count"] - num_elements, self.batch_size) + count = min(page["count"], self._skip + self._limit) def count(self): """Returns the number of elements matching the specifications""" diff --git a/tests/cursor.py b/tests/cursor.py index 2ad3bda..79a9f59 100644 --- a/tests/cursor.py +++ b/tests/cursor.py @@ -4,7 +4,10 @@ from appnexus.client import AppNexusClient from appnexus.cursor import Cursor -from .helpers import gen_random_collection +from .helpers import * + + +COLLECTION_DEFAULT_SIZE = 324 @pytest.fixture @@ -55,7 +58,12 @@ def response_dict2(): @pytest.fixture def random_response_dict(): - return gen_random_collection(count=324) + return gen_random_collection(count=COLLECTION_DEFAULT_SIZE) + + +@pytest.fixture +def ordered_response_dict(): + return gen_ordered_collection(start_element=0, count=COLLECTION_DEFAULT_SIZE) @pytest.fixture @@ -74,6 +82,31 @@ def random_cursor(mocker, random_response_dict): return Cursor(client, "campaign", representations.raw) +@pytest.fixture +def ordered_cursor(mocker, ordered_response_dict): + client = AppNexusClient("test", "test") + mocker.patch.object(client, "get") + client.get.side_effect = ordered_response_dict + return Cursor(client, "campaign", representations.raw) + + +def mock_ordered_cursor(mocker, start=0, count=COLLECTION_DEFAULT_SIZE, factor=1): + client = AppNexusClient("test", "test") + mocker.patch.object(client, "get") + client.get.side_effect = gen_ordered_collection(start, count) * factor + cursor = Cursor(client, "campaign", representations.raw) + mocker.patch.object(cursor, "get_page", wraps=cursor.get_page) + return cursor + + +@pytest.fixture +def double_ordered_cursor(mocker, ordered_response_dict): + client = AppNexusClient("test", "test") + mocker.patch.object(client, "get") + client.get.side_effect = ordered_response_dict * 2 + return Cursor(client, "campaign", representations.raw) + + def test_cursor_count(cursor, response_dict): assert cursor.count() == response_dict["count"] @@ -164,3 +197,110 @@ def test_uncallable_representation(): def test_requests_volume_on_iteration(cursor): _ = [r for r in cursor] assert cursor.client.get.call_count == 1 + + +def test_skip_none(mocker): + cursor = mock_ordered_cursor(mocker, start=0, count=COLLECTION_DEFAULT_SIZE) + results = [r for r in cursor] + assert len(results) == COLLECTION_DEFAULT_SIZE + assert results[0]['id'] == 0 + assert results[-1]['id'] == COLLECTION_DEFAULT_SIZE - 1 + assert cursor.get_page.call_count == 4 + + +def test_skip_ten(mocker): + skip = 10 + cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_DEFAULT_SIZE) + cursor.skip(skip) + results = [r for r in cursor] + assert len(results) == COLLECTION_DEFAULT_SIZE - skip + assert results[0]['id'] == skip + assert results[-1]['id'] == COLLECTION_DEFAULT_SIZE - 1 + assert cursor.get_page.call_count == 4 + + +def test_skip_hundred_ten(mocker): + skip = 110 + cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_DEFAULT_SIZE) + cursor.skip(skip) + results = [r for r in cursor] + assert len(results) == COLLECTION_DEFAULT_SIZE - skip + assert results[0]['id'] == skip + assert results[-1]['id'] == COLLECTION_DEFAULT_SIZE - 1 + assert cursor.get_page.call_count == 3 + + +def test_skip_twice(mocker): + skip = 10 + cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_DEFAULT_SIZE, factor=2) + cursor.skip(skip) + results = [r for r in cursor] + assert len(results) == COLLECTION_DEFAULT_SIZE - skip + assert results[0]['id'] == skip + assert cursor.get_page.call_count == 4 + results = [r for r in cursor] + assert len(results) == COLLECTION_DEFAULT_SIZE - skip + assert results[0]['id'] == skip + assert cursor.get_page.call_count == 8 + + +def test_limit_ten(mocker): + limit = 10 + cursor = mock_ordered_cursor(mocker, start=0, count=limit) + cursor.limit(limit) + results = [r for r in cursor] + assert len(results) == limit + assert results[0]['id'] == 0 + assert results[-1]['id'] == limit - 1 + assert cursor.get_page.call_count == 1 + + +def test_limit_hundred_ten(mocker): + limit = 110 + cursor = mock_ordered_cursor(mocker, start=0, count=limit) + cursor.limit(limit) + results = [r for r in cursor] + assert len(results) == limit + assert results[0]['id'] == 0 + assert results[-1]['id'] == limit - 1 + assert cursor.get_page.call_count == 2 + + +def test_limit_thousand(mocker): + limit = 1000 + cursor = mock_ordered_cursor(mocker, start=0, count=COLLECTION_DEFAULT_SIZE) + cursor.limit(limit) + results = [r for r in cursor] + assert len(results) == COLLECTION_DEFAULT_SIZE + assert results[0]['id'] == 0 + assert results[-1]['id'] == COLLECTION_DEFAULT_SIZE - 1 + assert cursor.get_page.call_count == 4 + + +def test_limit_twice(mocker): + limit = 50 + cursor = mock_ordered_cursor(mocker, start=0, count=limit, factor=2) + cursor.limit(limit) + results = [r for r in cursor] + assert len(results) == limit + assert results[0]['id'] == 0 + assert results[-1]['id'] == limit - 1 + assert cursor.get_page.call_count == 1 + results = [r for r in cursor] + assert len(results) == limit + assert results[0]['id'] == 0 + assert results[-1]['id'] == limit - 1 + assert cursor.get_page.call_count == 2 + + +def test_skip_and_limit(mocker): + skip = 10 + limit = 150 + cursor = mock_ordered_cursor(mocker, start=skip, count=skip+limit) + cursor.skip(skip) + cursor.limit(limit) + results = [r for r in cursor] + assert len(results) == limit + assert results[0]['id'] == skip + assert results[-1]['id'] == limit + skip - 1 + assert cursor.get_page.call_count == 2 diff --git a/tests/helpers.py b/tests/helpers.py index 573eb96..a1cd982 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,12 +1,29 @@ import random -def gen_random_object(): - return {"id": random.randrange(1000000)} +def gen_collection(object_generator_func, start_element=0, count=None, object_type="campaigns"): + if count is None: + random.randrange(10000) + result = [] + i = 0 + volume = count - start_element + for i in range(volume // 100): + page = gen_page(object_generator_func, count=count, object_type=object_type, + start_element=start_element + i * 100) + result.append(page) + if volume % 100 != 0: + i = i + 1 if len(result) else 0 + page = gen_page(object_generator_func, count=count, object_type=object_type, + start_element=start_element + i * 100, + num_elements=volume % 100) + result.append(page) + return result -def gen_random_page(num_elements=None, start_element=0, count=None, - object_type="campaigns"): +def gen_page(object_generator_func, start_element=0, num_elements=None, + count=None, object_type="campaigns"): + if not object_generator_func or not callable(object_generator_func): + raise ValueError("object_generator_func has to be set and callable") if count is None: count = random.randrange(10000) if num_elements is None: @@ -17,23 +34,18 @@ def gen_random_page(num_elements=None, start_element=0, count=None, "start_element": start_element, "num_elements": num_elements, "count": count, - object_type: [gen_random_object() for _ in range(num_elements)] + object_type: [object_generator_func(start_element + index) \ + for index in range(num_elements)] } -def gen_random_collection(count=None, object_type="campaigns"): - if count is None: - count = random.randrange(10000) - result = [] - i = 0 - for i in range(count // 100): - random_page = gen_random_page(count=count, object_type=object_type, - start_element=i * 100) - result.append(random_page) - if count % 100 != 0: - i = i + 1 if len(result) else 0 - random_page = gen_random_page(count=count, object_type=object_type, - start_element=i * 100, - num_elements=count % 100) - result.append(random_page) - return result +def gen_random_collection(start_element=0, count=None, object_type="campaigns"): + return gen_collection( + object_generator_func=lambda index: {"id": random.randrange(1000000)}, + start_element=start_element, count=count, object_type=object_type) + + +def gen_ordered_collection(start_element, count, object_type="campaigns"): + return gen_collection( + object_generator_func=lambda index: {"id": index}, + start_element=start_element, count=count, object_type=object_type) From 18dd233495dbb2d2b3f80b1f507ea16b0d0332be Mon Sep 17 00:00:00 2001 From: Romain Meson Date: Mon, 21 Oct 2019 17:57:12 +0200 Subject: [PATCH 4/7] Address flake8 issues in CI --- tests/cursor.py | 44 ++++++++++++++++++++++---------------------- tests/helpers.py | 24 ++++++++++++++---------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/tests/cursor.py b/tests/cursor.py index 79a9f59..9e22122 100644 --- a/tests/cursor.py +++ b/tests/cursor.py @@ -4,10 +4,9 @@ from appnexus.client import AppNexusClient from appnexus.cursor import Cursor -from .helpers import * +from .helpers import gen_ordered_collection, gen_random_collection - -COLLECTION_DEFAULT_SIZE = 324 +COLLECTION_SIZE = 324 @pytest.fixture @@ -58,12 +57,12 @@ def response_dict2(): @pytest.fixture def random_response_dict(): - return gen_random_collection(count=COLLECTION_DEFAULT_SIZE) + return gen_random_collection(count=COLLECTION_SIZE) @pytest.fixture def ordered_response_dict(): - return gen_ordered_collection(start_element=0, count=COLLECTION_DEFAULT_SIZE) + return gen_ordered_collection(start_element=0, count=COLLECTION_SIZE) @pytest.fixture @@ -90,7 +89,7 @@ def ordered_cursor(mocker, ordered_response_dict): return Cursor(client, "campaign", representations.raw) -def mock_ordered_cursor(mocker, start=0, count=COLLECTION_DEFAULT_SIZE, factor=1): +def mock_ordered_cursor(mocker, start=0, count=COLLECTION_SIZE, factor=1): client = AppNexusClient("test", "test") mocker.patch.object(client, "get") client.get.side_effect = gen_ordered_collection(start, count) * factor @@ -200,46 +199,47 @@ def test_requests_volume_on_iteration(cursor): def test_skip_none(mocker): - cursor = mock_ordered_cursor(mocker, start=0, count=COLLECTION_DEFAULT_SIZE) + cursor = mock_ordered_cursor(mocker, start=0, count=COLLECTION_SIZE) results = [r for r in cursor] - assert len(results) == COLLECTION_DEFAULT_SIZE + assert len(results) == COLLECTION_SIZE assert results[0]['id'] == 0 - assert results[-1]['id'] == COLLECTION_DEFAULT_SIZE - 1 + assert results[-1]['id'] == COLLECTION_SIZE - 1 assert cursor.get_page.call_count == 4 def test_skip_ten(mocker): skip = 10 - cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_DEFAULT_SIZE) + cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_SIZE) cursor.skip(skip) results = [r for r in cursor] - assert len(results) == COLLECTION_DEFAULT_SIZE - skip + assert len(results) == COLLECTION_SIZE - skip assert results[0]['id'] == skip - assert results[-1]['id'] == COLLECTION_DEFAULT_SIZE - 1 + assert results[-1]['id'] == COLLECTION_SIZE - 1 assert cursor.get_page.call_count == 4 def test_skip_hundred_ten(mocker): skip = 110 - cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_DEFAULT_SIZE) + cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_SIZE) cursor.skip(skip) results = [r for r in cursor] - assert len(results) == COLLECTION_DEFAULT_SIZE - skip + assert len(results) == COLLECTION_SIZE - skip assert results[0]['id'] == skip - assert results[-1]['id'] == COLLECTION_DEFAULT_SIZE - 1 + assert results[-1]['id'] == COLLECTION_SIZE - 1 assert cursor.get_page.call_count == 3 def test_skip_twice(mocker): skip = 10 - cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_DEFAULT_SIZE, factor=2) + cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_SIZE, + factor=2) cursor.skip(skip) results = [r for r in cursor] - assert len(results) == COLLECTION_DEFAULT_SIZE - skip + assert len(results) == COLLECTION_SIZE - skip assert results[0]['id'] == skip assert cursor.get_page.call_count == 4 results = [r for r in cursor] - assert len(results) == COLLECTION_DEFAULT_SIZE - skip + assert len(results) == COLLECTION_SIZE - skip assert results[0]['id'] == skip assert cursor.get_page.call_count == 8 @@ -268,12 +268,12 @@ def test_limit_hundred_ten(mocker): def test_limit_thousand(mocker): limit = 1000 - cursor = mock_ordered_cursor(mocker, start=0, count=COLLECTION_DEFAULT_SIZE) + cursor = mock_ordered_cursor(mocker, start=0, count=COLLECTION_SIZE) cursor.limit(limit) results = [r for r in cursor] - assert len(results) == COLLECTION_DEFAULT_SIZE + assert len(results) == COLLECTION_SIZE assert results[0]['id'] == 0 - assert results[-1]['id'] == COLLECTION_DEFAULT_SIZE - 1 + assert results[-1]['id'] == COLLECTION_SIZE - 1 assert cursor.get_page.call_count == 4 @@ -296,7 +296,7 @@ def test_limit_twice(mocker): def test_skip_and_limit(mocker): skip = 10 limit = 150 - cursor = mock_ordered_cursor(mocker, start=skip, count=skip+limit) + cursor = mock_ordered_cursor(mocker, start=skip, count=skip + limit) cursor.skip(skip) cursor.limit(limit) results = [r for r in cursor] diff --git a/tests/helpers.py b/tests/helpers.py index a1cd982..e7335b1 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,27 +1,30 @@ import random -def gen_collection(object_generator_func, start_element=0, count=None, object_type="campaigns"): +def gen_collection(object_generator_func, start_element=0, count=None, + object_type="campaigns"): if count is None: random.randrange(10000) result = [] i = 0 volume = count - start_element for i in range(volume // 100): - page = gen_page(object_generator_func, count=count, object_type=object_type, - start_element=start_element + i * 100) + page = gen_page(object_generator_func, count=count, + object_type=object_type, + start_element=start_element + i * 100) result.append(page) if volume % 100 != 0: i = i + 1 if len(result) else 0 - page = gen_page(object_generator_func, count=count, object_type=object_type, - start_element=start_element + i * 100, - num_elements=volume % 100) + page = gen_page(object_generator_func, count=count, + object_type=object_type, + start_element=start_element + i * 100, + num_elements=volume % 100) result.append(page) return result def gen_page(object_generator_func, start_element=0, num_elements=None, - count=None, object_type="campaigns"): + count=None, object_type="campaigns"): if not object_generator_func or not callable(object_generator_func): raise ValueError("object_generator_func has to be set and callable") if count is None: @@ -34,12 +37,13 @@ def gen_page(object_generator_func, start_element=0, num_elements=None, "start_element": start_element, "num_elements": num_elements, "count": count, - object_type: [object_generator_func(start_element + index) \ - for index in range(num_elements)] + object_type: [object_generator_func(start_element + index) + for index in range(num_elements)] } -def gen_random_collection(start_element=0, count=None, object_type="campaigns"): +def gen_random_collection(start_element=0, count=None, + object_type="campaigns"): return gen_collection( object_generator_func=lambda index: {"id": random.randrange(1000000)}, start_element=start_element, count=count, object_type=object_type) From 60b82b73617ef81dacd6066d298037668a37a49e Mon Sep 17 00:00:00 2001 From: Romain Meson Date: Sun, 27 Sep 2020 16:33:56 +0200 Subject: [PATCH 5/7] Remove unused variable retrieved in cursor.__iter__ --- appnexus/cursor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/appnexus/cursor.py b/appnexus/cursor.py index dc07d50..6dafd89 100644 --- a/appnexus/cursor.py +++ b/appnexus/cursor.py @@ -38,11 +38,9 @@ def __getitem__(self, idx): def __iter__(self): """Iterate over all AppNexus objects matching the specifications""" - retrieved = 0 for page in self.iter_pages(): data = self.extract_data(page) for entity in data: - retrieved += 1 yield entity def extract_data(self, page): From 8c671cbfe1f0e9fd1bafad9a1dbe6baaea4d0260 Mon Sep 17 00:00:00 2001 From: Romain Meson Date: Sun, 27 Sep 2020 18:03:17 +0200 Subject: [PATCH 6/7] In tests, generalize gen_collection signature to get ride of overloading helpers --- tests/cursor.py | 8 +++--- tests/helpers.py | 71 ++++++++++++++++++++++++++---------------------- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/tests/cursor.py b/tests/cursor.py index 9e22122..96d522c 100644 --- a/tests/cursor.py +++ b/tests/cursor.py @@ -4,7 +4,7 @@ from appnexus.client import AppNexusClient from appnexus.cursor import Cursor -from .helpers import gen_ordered_collection, gen_random_collection +from .helpers import gen_collection COLLECTION_SIZE = 324 @@ -57,12 +57,12 @@ def response_dict2(): @pytest.fixture def random_response_dict(): - return gen_random_collection(count=COLLECTION_SIZE) + return gen_collection(count=COLLECTION_SIZE, randomize=True) @pytest.fixture def ordered_response_dict(): - return gen_ordered_collection(start_element=0, count=COLLECTION_SIZE) + return gen_collection(count=COLLECTION_SIZE, randomize=False) @pytest.fixture @@ -92,7 +92,7 @@ def ordered_cursor(mocker, ordered_response_dict): def mock_ordered_cursor(mocker, start=0, count=COLLECTION_SIZE, factor=1): client = AppNexusClient("test", "test") mocker.patch.object(client, "get") - client.get.side_effect = gen_ordered_collection(start, count) * factor + client.get.side_effect = gen_collection(start, count) * factor cursor = Cursor(client, "campaign", representations.raw) mocker.patch.object(cursor, "get_page", wraps=cursor.get_page) return cursor diff --git a/tests/helpers.py b/tests/helpers.py index e7335b1..032e7f8 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,30 +1,16 @@ import random -def gen_collection(object_generator_func, start_element=0, count=None, - object_type="campaigns"): - if count is None: - random.randrange(10000) - result = [] - i = 0 - volume = count - start_element - for i in range(volume // 100): - page = gen_page(object_generator_func, count=count, - object_type=object_type, - start_element=start_element + i * 100) - result.append(page) - if volume % 100 != 0: - i = i + 1 if len(result) else 0 - page = gen_page(object_generator_func, count=count, - object_type=object_type, - start_element=start_element + i * 100, - num_elements=volume % 100) - result.append(page) - return result +def default_object_generator_func(index, randomize=False): + if randomize: + return {"id": random.randrange(1000000)} + return {"id": index} -def gen_page(object_generator_func, start_element=0, num_elements=None, - count=None, object_type="campaigns"): +def gen_page(start_element=0, count=None, num_elements=None, + object_type="campaigns", + object_generator_func=default_object_generator_func, + randomize=False): if not object_generator_func or not callable(object_generator_func): raise ValueError("object_generator_func has to be set and callable") if count is None: @@ -37,19 +23,40 @@ def gen_page(object_generator_func, start_element=0, num_elements=None, "start_element": start_element, "num_elements": num_elements, "count": count, - object_type: [object_generator_func(start_element + index) + object_type: [object_generator_func(start_element + index, randomize) for index in range(num_elements)] } -def gen_random_collection(start_element=0, count=None, - object_type="campaigns"): - return gen_collection( - object_generator_func=lambda index: {"id": random.randrange(1000000)}, - start_element=start_element, count=count, object_type=object_type) +def gen_collection(start_element=0, count=None, object_type="campaigns", + object_generator_func=default_object_generator_func, + randomize=False): + if count is None: + random.randrange(10000) + result = [] + i = 0 + volume = count - start_element + for i in range(volume // 100): + page = gen_page( + start_element=start_element + i * 100, + count=count, + object_type=object_type, + object_generator_func=object_generator_func, + randomize=randomize + ) + result.append(page) -def gen_ordered_collection(start_element, count, object_type="campaigns"): - return gen_collection( - object_generator_func=lambda index: {"id": index}, - start_element=start_element, count=count, object_type=object_type) + if volume % 100 != 0: + i = i + 1 if len(result) else 0 + page = gen_page( + start_element=start_element + i * 100, + count=count, + num_elements=volume % 100, + object_type=object_type, + object_generator_func=object_generator_func, + randomize=randomize + ) + result.append(page) + + return result From 2deb232f1c80f1568b7cd9c41ce88b8a194648cc Mon Sep 17 00:00:00 2001 From: Romain Meson Date: Sun, 27 Sep 2020 19:22:33 +0200 Subject: [PATCH 7/7] Remove unused fixture and parameters with default values when calling mock_ordered_cursor --- tests/cursor.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/tests/cursor.py b/tests/cursor.py index 96d522c..631021a 100644 --- a/tests/cursor.py +++ b/tests/cursor.py @@ -78,7 +78,9 @@ def random_cursor(mocker, random_response_dict): client = AppNexusClient("test", "test") mocker.patch.object(client, "get") client.get.side_effect = random_response_dict - return Cursor(client, "campaign", representations.raw) + cursor = Cursor(client, "campaign", representations.raw) + mocker.patch.object(cursor, "get_page", wraps=cursor.get_page) + return cursor @pytest.fixture @@ -86,7 +88,9 @@ def ordered_cursor(mocker, ordered_response_dict): client = AppNexusClient("test", "test") mocker.patch.object(client, "get") client.get.side_effect = ordered_response_dict - return Cursor(client, "campaign", representations.raw) + cursor = Cursor(client, "campaign", representations.raw) + mocker.patch.object(cursor, "get_page", wraps=cursor.get_page) + return cursor def mock_ordered_cursor(mocker, start=0, count=COLLECTION_SIZE, factor=1): @@ -98,14 +102,6 @@ def mock_ordered_cursor(mocker, start=0, count=COLLECTION_SIZE, factor=1): return cursor -@pytest.fixture -def double_ordered_cursor(mocker, ordered_response_dict): - client = AppNexusClient("test", "test") - mocker.patch.object(client, "get") - client.get.side_effect = ordered_response_dict * 2 - return Cursor(client, "campaign", representations.raw) - - def test_cursor_count(cursor, response_dict): assert cursor.count() == response_dict["count"] @@ -198,18 +194,17 @@ def test_requests_volume_on_iteration(cursor): assert cursor.client.get.call_count == 1 -def test_skip_none(mocker): - cursor = mock_ordered_cursor(mocker, start=0, count=COLLECTION_SIZE) - results = [r for r in cursor] +def test_skip_none(ordered_cursor): + results = [r for r in ordered_cursor] assert len(results) == COLLECTION_SIZE assert results[0]['id'] == 0 assert results[-1]['id'] == COLLECTION_SIZE - 1 - assert cursor.get_page.call_count == 4 + assert ordered_cursor.get_page.call_count == 4 def test_skip_ten(mocker): skip = 10 - cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_SIZE) + cursor = mock_ordered_cursor(mocker, start=skip) cursor.skip(skip) results = [r for r in cursor] assert len(results) == COLLECTION_SIZE - skip @@ -220,7 +215,7 @@ def test_skip_ten(mocker): def test_skip_hundred_ten(mocker): skip = 110 - cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_SIZE) + cursor = mock_ordered_cursor(mocker, start=skip) cursor.skip(skip) results = [r for r in cursor] assert len(results) == COLLECTION_SIZE - skip @@ -231,8 +226,7 @@ def test_skip_hundred_ten(mocker): def test_skip_twice(mocker): skip = 10 - cursor = mock_ordered_cursor(mocker, start=skip, count=COLLECTION_SIZE, - factor=2) + cursor = mock_ordered_cursor(mocker, start=skip, factor=2) cursor.skip(skip) results = [r for r in cursor] assert len(results) == COLLECTION_SIZE - skip @@ -246,7 +240,7 @@ def test_skip_twice(mocker): def test_limit_ten(mocker): limit = 10 - cursor = mock_ordered_cursor(mocker, start=0, count=limit) + cursor = mock_ordered_cursor(mocker, count=limit) cursor.limit(limit) results = [r for r in cursor] assert len(results) == limit @@ -257,7 +251,7 @@ def test_limit_ten(mocker): def test_limit_hundred_ten(mocker): limit = 110 - cursor = mock_ordered_cursor(mocker, start=0, count=limit) + cursor = mock_ordered_cursor(mocker, count=limit) cursor.limit(limit) results = [r for r in cursor] assert len(results) == limit @@ -268,7 +262,7 @@ def test_limit_hundred_ten(mocker): def test_limit_thousand(mocker): limit = 1000 - cursor = mock_ordered_cursor(mocker, start=0, count=COLLECTION_SIZE) + cursor = mock_ordered_cursor(mocker) cursor.limit(limit) results = [r for r in cursor] assert len(results) == COLLECTION_SIZE @@ -279,7 +273,7 @@ def test_limit_thousand(mocker): def test_limit_twice(mocker): limit = 50 - cursor = mock_ordered_cursor(mocker, start=0, count=limit, factor=2) + cursor = mock_ordered_cursor(mocker, count=limit, factor=2) cursor.limit(limit) results = [r for r in cursor] assert len(results) == limit