Add supported lexer validation to paste service and default the paste_url (#182)

Author: ChrisLovering

docs/changelog.rst

@@ -3,6 +3,11 @@
 
 Changelog
 =========
+- :release:`9.9.0 <18th June 2023>`
+- :feature:`182` Default pastebin url to https://paste.pythondiscord.com
+- :feature:`182` Add supported lexer validation to paste service.
+
+
 - :release:`9.8.0 <13th June 2023>`
 - :support:`181` Bump Discord.py to :literal-url:`2.3.0 <https://github.com/Rapptz/discord.py/releases/tag/v2.3.0>`.
 

pydis_core/utils/paste_service.py

@@ -1,3 +1,4 @@
+import json
 from typing import TypedDict
 
 from aiohttp import ClientConnectorError, ClientSession
@@ -6,10 +7,14 @@
 
 log = logging.get_logger(__name__)
 
+DEFAULT_PASTEBIN = "https://paste.pythondiscord.com"
 FAILED_REQUEST_ATTEMPTS = 3
-MAX_PASTE_SIZE = 128 * 1024  # 128kB
+MAX_PASTE_SIZE = 512 * 1024  # 512kB
 """The maximum allows size of a paste, in bytes."""
 
+# A dict where the keys are paste services and the keys are the lists of lexers that paste service supports.
+_lexers_supported_by_pastebin: dict[str,list[str]] = {}
+
 
 class PasteResponse(TypedDict):
     """
@@ -27,6 +32,9 @@ class PasteResponse(TypedDict):
 class PasteUploadError(Exception):
     """Raised when an error is encountered uploading to the paste service."""
 
+class PasteUnsupportedLexerError(Exception):
+    """Raised when an unsupported lexer is used."""
+
 
 class PasteTooLongError(Exception):
     """Raised when content is too large to upload to the paste service."""
@@ -35,21 +43,21 @@ class PasteTooLongError(Exception):
 async def send_to_paste_service(
     *,
     contents: str,
-    paste_url: str,
     http_session: ClientSession,
     file_name: str = "",
     lexer: str = "python",
+    paste_url: str = DEFAULT_PASTEBIN,
     max_size: int = MAX_PASTE_SIZE,
 ) -> PasteResponse:
     """
     Upload some contents to the paste service.
 
     Args:
         contents: The content to upload to the paste service.
-        paste_url: The base url to the paste service.
         http_session (aiohttp.ClientSession): The session to use when POSTing the content to the paste service.
         file_name: The name of the file to save to the paste service.
         lexer: The lexer to save the content with.
+        paste_url: The base url to the paste service.
         max_size: The max number of bytes to be allowed. Anything larger than :obj:`MAX_PASTE_SIZE` will be rejected.
 
     Raises:
@@ -63,6 +71,18 @@ async def send_to_paste_service(
     if max_size > MAX_PASTE_SIZE:
         raise ValueError(f"`max_length` must not be greater than {MAX_PASTE_SIZE}")
 
+    if paste_url not in _lexers_supported_by_pastebin:
+        try:
+            async with http_session.get(f"{paste_url}/api/v1/lexer") as response:
+                response_json = await response.json()  # Supported lexers are the keys.
+        except Exception:
+            raise PasteUploadError("Could not fetch supported lexers from selected paste_url.")
+
+        _lexers_supported_by_pastebin[paste_url] = list(response_json)
+
+    if lexer not in _lexers_supported_by_pastebin[paste_url]:
+        raise PasteUnsupportedLexerError(f"Lexer '{lexer}' not supported by pastebin.")
+
     contents_size = len(contents.encode())
     if contents_size > max_size:
         log.info("Contents too large to send to paste service.")
@@ -79,7 +99,7 @@ async def send_to_paste_service(
     for attempt in range(1, FAILED_REQUEST_ATTEMPTS + 1):
         try:
             async with http_session.post(f"{paste_url}/api/v1/paste", json=payload) as response:
-                response_json = await response.json()
+                response_text = await response.text()
         except ClientConnectorError:
             log.warning(
                 f"Failed to connect to paste service at url {paste_url}, "
@@ -93,6 +113,13 @@ async def send_to_paste_service(
             )
             continue
 
+        if "Text exceeds size limit" in response_text:
+            log.info("Contents too large to send to paste service after lexing.")
+            raise PasteTooLongError(
+                f"Contents of size {contents_size} greater than maximum size {max_size} after lexing"
+            )
+
+        response_json = json.loads(response_text)
         if response.status == 400:
             log.warning(
                 f"Paste service returned error {response_json['message']} with status code {response.status}, "

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pydis_core"
-version = "9.8.0"
+version = "9.9.0"
 description = "PyDis core provides core functionality and utility to the bots of the Python Discord community."
 authors = ["Python Discord <info@pythondiscord.com>"]
 license = "MIT"