merge duplicate extra_hosts kwarg (#18192)

Author: zzstoatzz

src/integrations/prefect-docker/prefect_docker/worker.py

@@ -320,7 +320,7 @@ def get_network_mode(self) -> Optional[str]:
         # Default to unset
         return None
 
-    def get_extra_hosts(self, docker_client) -> Optional[dict[str, str]]:
+    def get_extra_hosts(self, docker_client: DockerClient) -> Optional[dict[str, str]]:
         """
         A host.docker.internal -> host-gateway mapping is necessary for communicating
         with the API on Linux machines. Docker Desktop on macOS will automatically
@@ -665,9 +665,29 @@ def _build_container_settings(
         container_create_kwargs = {
             k: v
             for k, v in container_create_kwargs.items()
-            if k not in configuration.model_fields.keys()
+            if k not in configuration.__class__.model_fields.keys()
         }
 
+        # Get extra_hosts from configuration
+        extra_hosts = configuration.get_extra_hosts(docker_client)
+
+        # If user provided extra_hosts in container_create_kwargs, merge them
+        if "extra_hosts" in container_create_kwargs:
+            user_extra_hosts = container_create_kwargs.pop("extra_hosts")
+            if extra_hosts:
+                # Merge user's extra_hosts with the auto-generated ones
+                # Convert list format to dict if necessary
+                if isinstance(user_extra_hosts, list):
+                    for host_entry in user_extra_hosts:
+                        if ":" in host_entry:
+                            host, ip = host_entry.split(":", 1)
+                            extra_hosts[host] = ip
+                elif isinstance(user_extra_hosts, dict):
+                    extra_hosts.update(user_extra_hosts)
+            else:
+                # No auto-generated extra_hosts, use user's directly
+                extra_hosts = user_extra_hosts
+
         return dict(
             image=configuration.image,
             network=configuration.networks[0] if configuration.networks else None,
@@ -676,7 +696,7 @@ def _build_container_settings(
             environment=configuration.env,
             auto_remove=configuration.auto_remove,
             labels=configuration.labels,
-            extra_hosts=configuration.get_extra_hosts(docker_client),
+            extra_hosts=extra_hosts,
             name=configuration.name,
             volumes=configuration.volumes,
             mem_limit=configuration.mem_limit,

src/integrations/prefect-docker/tests/test_worker.py

@@ -655,6 +655,35 @@ async def test_adds_docker_host_gateway_on_linux(
     assert call_extra_hosts == {"host.docker.internal": "host-gateway"}
 
 
+async def test_user_provided_extra_hosts_merge_with_auto_generated(
+    mock_docker_client: MagicMock,
+    monkeypatch: pytest.MonkeyPatch,
+    flow_run: FlowRun,
+    default_docker_worker_job_configuration: DockerWorkerJobConfiguration,
+):
+    """Test that user-provided extra_hosts are merged with auto-generated ones without error.
+
+    this is a regression test for https://github.com/PrefectHQ/prefect/issues/18187
+    """
+    monkeypatch.setattr("sys.platform", "linux")
+
+    default_docker_worker_job_configuration.container_create_kwargs = {
+        "extra_hosts": ["host.docker.internal:host-gateway"]
+    }
+    default_docker_worker_job_configuration.prepare_for_flow_run(flow_run=flow_run)
+
+    async with DockerWorker(work_pool_name="test") as worker:
+        await worker.run(
+            flow_run=flow_run, configuration=default_docker_worker_job_configuration
+        )
+
+    mock_docker_client.containers.create.assert_called_once()
+    call_extra_hosts = mock_docker_client.containers.create.call_args[1].get(
+        "extra_hosts"
+    )
+    assert call_extra_hosts == {"host.docker.internal": "host-gateway"}
+
+
 async def test_default_image_pull_policy_pulls_image_with_latest_tag(
     mock_docker_client, flow_run, default_docker_worker_job_configuration
 ):