Pre v2.4.0rc8 cleanups (#1053)

* Pre-release cleanups

* Add listener pool test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add multi listener test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

benchmark/_proxy.py

@@ -23,8 +23,9 @@
             open_file_limit=65536,
             enable_web_server=True,
             disable_proxy_server=True,
-            num_acceptors=10,
-            local_executor=1,
+            num_acceptors=4,
+            num_workers=10,
+            local_executor=0,
             log_file='/dev/null',
     ) as _:
         while True:

examples/task.py

@@ -110,6 +110,9 @@ def start_remote_pool(flags: argparse.Namespace) -> None:
             pass
 
 
+# TODO: TaskWork, LocalTaskExecutor, RemoteTaskExecutor
+# should not be needed, abstract those pieces out in the core
+# for stateless tasks.
 if __name__ == '__main__':
     flags = FlagParser.initialize(
         ['--disable-http-proxy'],

proxy/core/listener/pool.py

@@ -15,7 +15,7 @@
 from .unix import UnixSocketListener
 
 
-if TYPE_CHECKING:
+if TYPE_CHECKING:   # pragma: no cover
     from .base import BaseListener
 
 

proxy/core/work/fd/fd.py

@@ -45,7 +45,7 @@ def work(self, *args: Any) -> None:
             self.works[fileno].initialize()
             self._total += 1
         except Exception as e:
-            logger.exception(
+            logger.exception(   # pragma: no cover
                 'Exception occurred during initialization',
                 exc_info=e,
             )

proxy/http/server/reverse.py

@@ -25,7 +25,7 @@
 )
 
 
-if TYPE_CHECKING:
+if TYPE_CHECKING:   # pragma: no cover
     from .plugin import ReverseProxyBasePlugin
 
 
@@ -92,7 +92,7 @@ def handle_request(self, request: HttpParser) -> None:
             # )
             self.upstream.queue(memoryview(request.build()))
         except ConnectionRefusedError:
-            raise HttpProtocolException(
+            raise HttpProtocolException(    # pragma: no cover
                 'Connection refused by upstream server {0}:{1}'.format(
                     text_(self.choice.hostname), port,
                 ),

proxy/plugin/__init__.py

@@ -26,14 +26,12 @@
 from .custom_dns_resolver import CustomDnsResolverPlugin
 from .filter_by_client_ip import FilterByClientIpPlugin
 from .filter_by_url_regex import FilterByURLRegexPlugin
-from .cache_by_content_type import CacheByContentTypePlugin
 from .modify_chunk_response import ModifyChunkResponsePlugin
 from .redirect_to_custom_server import RedirectToCustomServerPlugin
 
 
 __all__ = [
     'CacheResponsesPlugin',
-    'CacheByContentTypePlugin',
     'BaseCacheResponsesPlugin',
     'FilterByUpstreamHostPlugin',
     'ManInTheMiddlePlugin',

proxy/plugin/cache_by_content_type.py

@@ -32,7 +32,7 @@
 )
 
 
-if TYPE_CHECKING:
+if TYPE_CHECKING:   # pragma: no cover
     from .core.listener import TcpSocketListener
 
 

proxy/proxy.py

@@ -22,7 +22,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
 
     @staticmethod
     def create(*args: Any) -> SocksClientConnection:
-        return SocksClientConnection(*args)
+        return SocksClientConnection(*args)     # pragma: no cover
 
     def handle_data(self, data: memoryview) -> Optional[bool]:
-        return super().handle_data(data)
+        return super().handle_data(data)    # pragma: no cover

proxy/socks/handler.py

@@ -27,35 +27,31 @@ class TestListener(unittest.TestCase):
     def test_setup_and_teardown(self, mock_socket: mock.Mock) -> None:
         sock = mock_socket.return_value
         flags = FlagParser.initialize(port=0)
-        listener = TcpSocketListener(flags=flags)
-        listener.setup()
-        mock_socket.assert_called_with(
-            socket.AF_INET6 if flags.hostname.version == 6 else socket.AF_INET,
-            socket.SOCK_STREAM,
-        )
-        self.assertEqual(sock.setsockopt.call_count, 2)
-        self.assertEqual(
-            sock.setsockopt.call_args_list[0][0],
-            (socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
-        )
-        self.assertEqual(
-            sock.setsockopt.call_args_list[1][0],
-            (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
-        )
-        sock.bind.assert_called_with(
-            (str(flags.hostname), 0),
-        )
-        sock.listen.assert_called_with(flags.backlog)
-        sock.setblocking.assert_called_with(False)
-
-        listener.shutdown()
+        with TcpSocketListener(flags=flags) as listener:
+            mock_socket.assert_called_with(
+                socket.AF_INET6 if flags.hostname.version == 6 else socket.AF_INET,
+                socket.SOCK_STREAM,
+            )
+            self.assertEqual(sock.setsockopt.call_count, 2)
+            self.assertEqual(
+                sock.setsockopt.call_args_list[0][0],
+                (socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
+            )
+            self.assertEqual(
+                sock.setsockopt.call_args_list[1][0],
+                (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),
+            )
+            sock.bind.assert_called_with(
+                (str(flags.hostname), 0),
+            )
+            sock.listen.assert_called_with(flags.backlog)
+            sock.setblocking.assert_called_with(False)
+            self.assertEqual(
+                listener.fileno(),
+                mock_socket.return_value.fileno.return_value,
+            )
         sock.close.assert_called_once()
 
-    # FIXME: Ignore is necessary for as long as pytest hasn't figured out
-    # FIXME: typing for their fixtures.
-    # Refs:
-    # * https://github.com/pytest-dev/pytest/issues/7469#issuecomment-918345196
-    # * https://github.com/pytest-dev/pytest/issues/3342
     @pytest.mark.skipif(
         IS_WINDOWS,
         reason='AF_UNIX not available on Windows',