[CacheByContentTypePlugin] Prepare for content type parsing (#1038)

* Move default cache directory within `proxy.py` instance data directory

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add `CacheByContentTypePlugin` skeleton and remove unnecessary mixin

* Fix startup issues after removal of mixin

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* .

* Fix broken mock

* doc fix

* doc

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

helper/benchmark.sh

@@ -145,6 +145,9 @@ def _env_threadless_compliant() -> bool:
 DEFAULT_DEVTOOLS_LOADER_ID = secrets.token_hex(8)
 
 DEFAULT_DATA_DIRECTORY_PATH = os.path.join(str(pathlib.Path.home()), '.proxy')
+DEFAULT_CACHE_DIRECTORY_PATH = os.path.join(
+    DEFAULT_DATA_DIRECTORY_PATH, 'cache',
+)
 
 # Cor plugins enabled by default or via flags
 DEFAULT_ABC_PLUGINS = [

proxy/common/constants.py

@@ -395,6 +395,11 @@ def initialize(
             )
         os.makedirs(args.ca_cert_dir, exist_ok=True)
 
+        # FIXME: Necessary here until flags framework provides a way
+        # for flag owners to initialize
+        os.makedirs(args.cache_dir, exist_ok=True)
+        os.makedirs(os.path.join(args.cache_dir, 'response'), exist_ok=True)
+
         return args
 
     @staticmethod

proxy/common/flag.py

@@ -16,6 +16,7 @@
 import sys
 import socket
 import logging
+import argparse
 import functools
 import ipaddress
 import contextlib
@@ -34,6 +35,13 @@
 logger = logging.getLogger(__name__)
 
 
+def tls_interception_enabled(flags: argparse.Namespace) -> bool:
+    return flags.ca_key_file is not None and \
+        flags.ca_cert_dir is not None and \
+        flags.ca_signing_key_file is not None and \
+        flags.ca_cert_file is not None
+
+
 def is_threadless(threadless: bool, threaded: bool) -> bool:
     # if default is threadless then return true unless
     # user has overridden mode using threaded flag.

proxy/common/utils.py

@@ -8,8 +8,6 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-from typing import Any
-
 from ..common.types import Readables, Writables, Descriptors
 
 
@@ -19,10 +17,6 @@ class DescriptorsHandlerMixin:
     include web and proxy plugins.  By using DescriptorsHandlerMixin, class
     becomes complaint with core event loop."""
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        # FIXME: Required for multi-level inheritance to work
-        super().__init__(*args, **kwargs)   # type: ignore
-
     # @abstractmethod
     async def get_descriptors(self) -> Descriptors:
         """Implementations must return a list of descriptions that they wish to

proxy/http/descriptors.py

@@ -13,11 +13,11 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, List, Union, Optional
 
-from .mixins import TlsInterceptionPropertyMixin
 from .parser import HttpParser
 from .connection import HttpClientConnection
 from ..core.event import EventQueue
 from .descriptors import DescriptorsHandlerMixin
+from ..common.utils import tls_interception_enabled
 
 
 if TYPE_CHECKING:   # pragma: no cover
@@ -26,7 +26,6 @@
 
 class HttpProtocolHandlerPlugin(
         DescriptorsHandlerMixin,
-        TlsInterceptionPropertyMixin,
         ABC,
 ):
     """Base HttpProtocolHandler Plugin class.
@@ -59,7 +58,6 @@ def __init__(
             event_queue: Optional[EventQueue] = None,
             upstream_conn_pool: Optional['UpstreamConnectionPool'] = None,
     ):
-        super().__init__(uid, flags, client, event_queue, upstream_conn_pool)
         self.uid: str = uid
         self.flags: argparse.Namespace = flags
         self.client: HttpClientConnection = client
@@ -95,3 +93,7 @@ def on_client_connection_close(self) -> None:
         perform any cleanup work here.
         """
         pass  # pragma: no cover
+
+    @property
+    def tls_interception_enabled(self) -> bool:
+        return tls_interception_enabled(self.flags)

proxy/http/mixins.py

@@ -12,11 +12,11 @@
 from abc import ABC
 from typing import TYPE_CHECKING, Any, Dict, Tuple, Optional
 
-from ..mixins import TlsInterceptionPropertyMixin
 from ..parser import HttpParser
 from ..connection import HttpClientConnection
 from ...core.event import EventQueue
 from ..descriptors import DescriptorsHandlerMixin
+from ...common.utils import tls_interception_enabled
 
 
 if TYPE_CHECKING:   # pragma: no cover
@@ -25,7 +25,6 @@
 
 class HttpProxyBasePlugin(
         DescriptorsHandlerMixin,
-        TlsInterceptionPropertyMixin,
         ABC,
 ):
     """Base HttpProxyPlugin Plugin class.
@@ -40,7 +39,6 @@ def __init__(
             event_queue: EventQueue,
             upstream_conn_pool: Optional['UpstreamConnectionPool'] = None,
     ) -> None:
-        super().__init__(uid, flags, client, event_queue, upstream_conn_pool)
         self.uid = uid                  # pragma: no cover
         self.flags = flags              # pragma: no cover
         self.client = client            # pragma: no cover
@@ -170,4 +168,4 @@ def do_intercept(self, _request: HttpParser) -> bool:
         flags BUT only conditionally enable interception for
         certain requests.
         """
-        return self.tls_interception_enabled
+        return tls_interception_enabled(self.flags)

proxy/http/plugin.py

@@ -26,12 +26,14 @@
 from .custom_dns_resolver import CustomDnsResolverPlugin
 from .filter_by_client_ip import FilterByClientIpPlugin
 from .filter_by_url_regex import FilterByURLRegexPlugin
+from .cache_by_content_type import CacheByContentTypePlugin
 from .modify_chunk_response import ModifyChunkResponsePlugin
 from .redirect_to_custom_server import RedirectToCustomServerPlugin
 
 
 __all__ = [
     'CacheResponsesPlugin',
+    'CacheByContentTypePlugin',
     'BaseCacheResponsesPlugin',
     'FilterByUpstreamHostPlugin',
     'ManInTheMiddlePlugin',

proxy/http/proxy/plugin.py

@@ -8,8 +8,9 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+import os
 import multiprocessing
-from typing import Any
+from typing import Any, Dict, Optional
 
 from .base import BaseCacheResponsesPlugin
 from .store.disk import OnDiskCacheStore
@@ -24,6 +25,16 @@ class CacheResponsesPlugin(BaseCacheResponsesPlugin):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.disk_store = OnDiskCacheStore(
-            uid=self.uid, cache_dir=self.flags.cache_dir,
+            uid=self.uid,
+            cache_dir=os.path.join(
+                self.flags.cache_dir,
+                'responses',
+            ),
         )
         self.set_store(self.disk_store)
+
+    def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        context.update({
+            'cache_file_path': self.disk_store.cache_file_path,
+        })
+        return super().on_access_log(context)