From 34c9085be6680f8b398c61aa475c61d97c279338 Mon Sep 17 00:00:00 2001 From: John Myers Date: Wed, 4 Mar 2026 13:07:59 -0800 Subject: [PATCH] feat(e2e): parallelize e2e tests with pytest-xdist (default -n 5) Closes #101 Add pytest-xdist for parallel e2e test execution with configurable concurrency. Default to 5 workers; override via E2E_PARALLEL env var (accepts a number or 'auto' for CPU-count matching). Make session-scoped mock inference route fixtures worker-safe by incorporating the xdist worker_id into route names and routing hints. --- build/test.toml | 4 ++-- e2e/python/conftest.py | 33 ++++++++++++++++++++-------- e2e/python/test_inference_routing.py | 16 +++++++++----- pyproject.toml | 1 + uv.lock | 24 ++++++++++++++++++++ 5 files changed, 62 insertions(+), 16 deletions(-) diff --git a/build/test.toml b/build/test.toml index cbc0f13b..499f5658 100644 --- a/build/test.toml +++ b/build/test.toml @@ -18,10 +18,10 @@ env = { UV_NO_SYNC = "1" } run = "uv run pytest python/" ["test:e2e:sandbox"] -description = "Run sandbox end-to-end tests" +description = "Run sandbox end-to-end tests (E2E_PARALLEL=N or 'auto'; default 5)" depends = ["python:proto", "cluster:deploy"] env = { UV_NO_SYNC = "1", PYTHONPATH = "python" } -run = "uv run pytest -o python_files='test_*.py' e2e/python" +run = "uv run pytest -o python_files='test_*.py' -n ${E2E_PARALLEL:-5} e2e/python" ["test:e2e:port-forward"] description = "Run port-forward integration test" diff --git a/e2e/python/conftest.py b/e2e/python/conftest.py index 1157a4f3..b4672788 100644 --- a/e2e/python/conftest.py +++ b/e2e/python/conftest.py @@ -72,12 +72,25 @@ def inference_client(sandbox_client: SandboxClient) -> InferenceRouteClient: return InferenceRouteClient.from_sandbox_client(sandbox_client) +@pytest.fixture(scope="session") +def _worker_suffix(worker_id: str) -> str: + """Return a suffix for worker-unique resource names. + + Uses the built-in ``worker_id`` fixture from pytest-xdist which returns + ``"gw0"``, ``"gw1"``, etc. for workers, or ``"master"`` for non-xdist runs. + """ + if worker_id == "master": + return "" + return f"-{worker_id}" + + @pytest.fixture(scope="session") def mock_inference_route( inference_client: InferenceRouteClient, + _worker_suffix: str, ) -> Iterator[str]: - name = "e2e-mock-local" - routing_hint = "e2e_mock_local" + name = f"e2e-mock-local{_worker_suffix}" + routing_hint = f"e2e_mock_local{_worker_suffix}" # Clean up any leftover route from a previous run. try: inference_client.delete(name) @@ -93,7 +106,7 @@ def mock_inference_route( model_id="mock/test-model", enabled=True, ) - yield name + yield routing_hint try: inference_client.delete(name) except grpc.RpcError: @@ -103,9 +116,10 @@ def mock_inference_route( @pytest.fixture(scope="session") def mock_anthropic_route( inference_client: InferenceRouteClient, + _worker_suffix: str, ) -> Iterator[str]: - name = "e2e-mock-anthropic" - routing_hint = "e2e_mock_anthropic" + name = f"e2e-mock-anthropic{_worker_suffix}" + routing_hint = f"e2e_mock_anthropic{_worker_suffix}" try: inference_client.delete(name) except grpc.RpcError: @@ -120,7 +134,7 @@ def mock_anthropic_route( model_id="mock/claude-test", enabled=True, ) - yield name + yield routing_hint try: inference_client.delete(name) except grpc.RpcError: @@ -130,10 +144,11 @@ def mock_anthropic_route( @pytest.fixture(scope="session") def mock_disallowed_route( inference_client: InferenceRouteClient, + _worker_suffix: str, ) -> Iterator[str]: """Route that exists but is NOT in any sandbox's allowed_routes.""" - name = "e2e-mock-disallowed" - routing_hint = "e2e_mock_disallowed" + name = f"e2e-mock-disallowed{_worker_suffix}" + routing_hint = f"e2e_mock_disallowed{_worker_suffix}" try: inference_client.delete(name) except grpc.RpcError: @@ -148,7 +163,7 @@ def mock_disallowed_route( model_id="mock/disallowed-model", enabled=True, ) - yield name + yield routing_hint try: inference_client.delete(name) except grpc.RpcError: diff --git a/e2e/python/test_inference_routing.py b/e2e/python/test_inference_routing.py index ff9c53b2..932f100c 100644 --- a/e2e/python/test_inference_routing.py +++ b/e2e/python/test_inference_routing.py @@ -177,7 +177,9 @@ def test_inference_call_routed_to_backend( 4. Forward locally via sandbox router to the policy-allowed backend 5. Return the mock response from the configured route """ - spec = datamodel_pb2.SandboxSpec(policy=_inference_routing_policy()) + spec = datamodel_pb2.SandboxSpec( + policy=_inference_routing_policy(mock_inference_route) + ) def call_chat_completions() -> str: import json @@ -226,7 +228,9 @@ def test_non_inference_request_denied( undeclared endpoint should be denied with 403 when inference routing is configured — only recognized inference API patterns are routed. """ - spec = datamodel_pb2.SandboxSpec(policy=_inference_routing_policy()) + spec = datamodel_pb2.SandboxSpec( + policy=_inference_routing_policy(mock_inference_route) + ) def make_non_inference_request() -> str: import ssl @@ -265,7 +269,7 @@ def test_inference_anthropic_messages_protocol( policy = sandbox_pb2.SandboxPolicy( version=1, inference=sandbox_pb2.InferencePolicy( - allowed_routes=["e2e_mock_anthropic"], + allowed_routes=[mock_anthropic_route], ), filesystem=_BASE_FILESYSTEM, landlock=_BASE_LANDLOCK, @@ -323,8 +327,10 @@ def test_inference_route_filtering_by_allowed_routes( allowed route should succeed, while inference requests that can't match any allowed route get an error from the sandbox router. """ - # Policy only allows e2e_mock_local, NOT e2e_mock_disallowed - spec = datamodel_pb2.SandboxSpec(policy=_inference_routing_policy()) + # Policy only allows the mock_inference_route, NOT mock_disallowed_route + spec = datamodel_pb2.SandboxSpec( + policy=_inference_routing_policy(mock_inference_route) + ) def call_allowed_route() -> str: import json diff --git a/pyproject.toml b/pyproject.toml index 40783f02..39d76d7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dev = [ "pytest>=8.0", "pytest-asyncio>=0.23", "pytest-cov>=4.0", + "pytest-xdist>=3.0", "ruff>=0.4", "ty>=0.0.1a6", "maturin>=1.5,<2.0", diff --git a/uv.lock b/uv.lock index dc7ab75a..15d7aacc 100644 --- a/uv.lock +++ b/uv.lock @@ -94,6 +94,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/db/d291e30fdf7ea617a335531e72294e0c723356d7fdde8fba00610a76bda9/coverage-7.13.2-py3-none-any.whl", hash = "sha256:40ce1ea1e25125556d8e76bd0b61500839a07944cc287ac21d5626f3e620cad5", size = 210943, upload-time = "2026-01-25T13:00:02.388Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "grpcio" version = "1.78.0" @@ -224,6 +233,7 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "pytest-xdist" }, { name = "ruff" }, { name = "setuptools-scm" }, { name = "ty" }, @@ -243,6 +253,7 @@ dev = [ { name = "pytest", specifier = ">=8.0" }, { name = "pytest-asyncio", specifier = ">=0.23" }, { name = "pytest-cov", specifier = ">=4.0" }, + { name = "pytest-xdist", specifier = ">=3.0" }, { name = "ruff", specifier = ">=0.4" }, { name = "setuptools-scm", specifier = ">=8" }, { name = "ty", specifier = ">=0.0.1a6" }, @@ -333,6 +344,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "ruff" version = "0.14.14"