Merge branch 'refs/heads/main' into ruff-updates

# Conflicts:
#	adafruit_requests.py
#	examples/cpython/requests_cpython_simpletest.py
#	examples/esp32spi/requests_esp32spi_simpletest.py
#	examples/fona/requests_fona_simpletest.py
#	examples/wifi/expanded/requests_wifi_api_fitbit.py
#	examples/wifi/expanded/requests_wifi_api_openskynetwork_private.py
#	examples/wifi/expanded/requests_wifi_api_openskynetwork_private_area.py
#	examples/wifi/expanded/requests_wifi_api_openskynetwork_public.py
#	examples/wifi/expanded/requests_wifi_api_twitter.py
#	examples/wifi/expanded/requests_wifi_api_youtube.py
#	examples/wifi/requests_wifi_simpletest.py
#	examples/wiznet5k/requests_wiznet5k_simpletest.py

Author: FoamyGuy

adafruit_requests.py

@@ -41,13 +41,16 @@
 
 import errno
 import json as json_module
+import os
 import sys
 
 from adafruit_connection_manager import get_connection_manager
 
+SEEK_END = 2
+
 if not sys.implementation.name == "circuitpython":
     from types import TracebackType
-    from typing import Any, Dict, Optional, Type
+    from typing import IO, Any, Dict, Optional, Type
 
     from circuitpython_typing.socket import (
         SocketpoolModuleType,
@@ -83,6 +86,21 @@ class Response:
     """The response from a request, contains all the headers/content"""
 
     encoding = None
+    socket: SocketType
+    """The underlying socket object (CircuitPython extension, not in standard requests)
+
+    Under the following circumstances, calling code may directly access the underlying
+    socket object:
+
+     * The request was made with ``stream=True``
+     * The request headers included ``{'connection': 'close'}``
+     * No methods or properties on the Response object that access the response content
+       may be used
+
+    Methods and properties that access response headers may be accessed.
+
+    It is still necessary to ``close`` the response object for correct management of
+    sockets, including doing so implicitly via ``with requests.get(...) as response``."""
 
     def __init__(self, sock: SocketType, session: "Session") -> None:
         self.socket = sock
@@ -245,7 +263,8 @@ def _parse_headers(self) -> None:
             header = self._readto(b"\r\n")
             if not header:
                 break
-            title, content = bytes(header).split(b": ", 1)
+            title, content = bytes(header).split(b":", 1)
+            content = content.strip()
             if title and content:
                 # enforce that all headers are lowercase
                 title = str(title, "utf-8").lower()
@@ -318,7 +337,7 @@ def json(self) -> Any:
         obj = json_module.load(self._raw)
         if not self._cached:
             self._cached = obj
-        self.close()
+
         return obj
 
     def iter_content(self, chunk_size: int = 1, decode_unicode: bool = False) -> bytes:
@@ -354,18 +373,81 @@ def __init__(
         self._session_id = session_id
         self._last_response = None
 
+    def _build_boundary_data(self, files: dict):  # pylint: disable=too-many-locals
+        boundary_string = self._build_boundary_string()
+        content_length = 0
+        boundary_objects = []
+
+        for field_name, field_values in files.items():
+            file_name = field_values[0]
+            file_handle = field_values[1]
+
+            boundary_objects.append(
+                f'--{boundary_string}\r\nContent-Disposition: form-data; name="{field_name}"'
+            )
+            if file_name is not None:
+                boundary_objects.append(f'; filename="{file_name}"')
+            boundary_objects.append("\r\n")
+            if len(field_values) >= 3:
+                file_content_type = field_values[2]
+                boundary_objects.append(f"Content-Type: {file_content_type}\r\n")
+            if len(field_values) >= 4:
+                file_headers = field_values[3]
+                for file_header_key, file_header_value in file_headers.items():
+                    boundary_objects.append(
+                        f"{file_header_key}: {file_header_value}\r\n"
+                    )
+            boundary_objects.append("\r\n")
+
+            if hasattr(file_handle, "read"):
+                content_length += self._get_file_length(file_handle)
+
+            boundary_objects.append(file_handle)
+            boundary_objects.append("\r\n")
+
+        boundary_objects.append(f"--{boundary_string}--\r\n")
+
+        for boundary_object in boundary_objects:
+            if isinstance(boundary_object, str):
+                content_length += len(boundary_object)
+
+        return boundary_string, content_length, boundary_objects
+
+    @staticmethod
+    def _build_boundary_string():
+        return os.urandom(16).hex()
+
     @staticmethod
     def _check_headers(headers: Dict[str, str]):
         if not isinstance(headers, dict):
-            raise AttributeError("headers must be in dict format")
+            raise TypeError("Headers must be in dict format")
 
         for key, value in headers.items():
             if isinstance(value, (str, bytes)) or value is None:
                 continue
-            raise AttributeError(
+            raise TypeError(
                 f"Header part ({value}) from {key} must be of type str or bytes, not {type(value)}"
             )
 
+    @staticmethod
+    def _get_file_length(file_handle: IO):
+        is_binary = False
+        try:
+            file_handle.seek(0)
+            # read at least 4 bytes incase we are reading a b64 stream
+            content = file_handle.read(4)
+            is_binary = isinstance(content, bytes)
+        except UnicodeError:
+            is_binary = False
+
+        if not is_binary:
+            raise ValueError("Files must be opened in binary mode")
+
+        file_handle.seek(0, SEEK_END)
+        content_length = file_handle.tell()
+        file_handle.seek(0)
+        return content_length
+
     @staticmethod
     def _send(socket: SocketType, data: bytes):
         total_sent = 0
@@ -391,6 +473,22 @@ def _send(socket: SocketType, data: bytes):
     def _send_as_bytes(self, socket: SocketType, data: str):
         return self._send(socket, bytes(data, "utf-8"))
 
+    def _send_boundary_objects(self, socket: SocketType, boundary_objects: Any):
+        for boundary_object in boundary_objects:
+            if isinstance(boundary_object, str):
+                self._send_as_bytes(socket, boundary_object)
+            else:
+                self._send_file(socket, boundary_object)
+
+    def _send_file(self, socket: SocketType, file_handle: IO):
+        chunk_size = 36
+        b = bytearray(chunk_size)
+        while True:
+            size = file_handle.readinto(b)
+            if size == 0:
+                break
+            self._send(socket, b[:size])
+
     def _send_header(self, socket, header, value):
         if value is None:
             return
@@ -411,6 +509,7 @@ def _send_request(  # noqa: PLR0913 Too many arguments in function definition
         headers: Dict[str, str],
         data: Any,
         json: Any,
+        files: Optional[Dict[str, tuple]],
     ):
         # Check headers
         self._check_headers(headers)
@@ -421,11 +520,13 @@ def _send_request(  # noqa: PLR0913 Too many arguments in function definition
         # If json is sent, set content type header and convert to string
         if json is not None:
             assert data is None
+            assert files is None
             content_type_header = "application/json"
             data = json_module.dumps(json)
 
         # If data is sent and it's a dict, set content type header and convert to string
         if data and isinstance(data, dict):
+            assert files is None
             content_type_header = "application/x-www-form-urlencoded"
             _post_data = ""
             for k in data:
@@ -437,6 +538,23 @@ def _send_request(  # noqa: PLR0913 Too many arguments in function definition
         if data and isinstance(data, str):
             data = bytes(data, "utf-8")
 
+        # If files are send, build data to send and calculate length
+        content_length = 0
+        data_is_file = False
+        boundary_objects = None
+        if files and isinstance(files, dict):
+            boundary_string, content_length, boundary_objects = (
+                self._build_boundary_data(files)
+            )
+            content_type_header = f"multipart/form-data; boundary={boundary_string}"
+        elif data and hasattr(data, "read"):
+            data_is_file = True
+            content_length = self._get_file_length(data)
+        else:
+            if data is None:
+                data = b""
+            content_length = len(data)
+
         self._send_as_bytes(socket, method)
         self._send(socket, b" /")
         self._send_as_bytes(socket, path)
@@ -452,16 +570,20 @@ def _send_request(  # noqa: PLR0913 Too many arguments in function definition
             self._send_header(socket, "User-Agent", "Adafruit CircuitPython")
         if content_type_header and not "content-type" in supplied_headers:
             self._send_header(socket, "Content-Type", content_type_header)
-        if data and not "content-length" in supplied_headers:
-            self._send_header(socket, "Content-Length", str(len(data)))
+        if (data or files) and not "content-length" in supplied_headers:
+            self._send_header(socket, "Content-Length", str(content_length))
         # Iterate over keys to avoid tuple alloc
         for header in headers:
             self._send_header(socket, header, headers[header])
         self._send(socket, b"\r\n")
 
         # Send data
-        if data:
+        if data_is_file:
+            self._send_file(socket, data)
+        elif data:
             self._send(socket, bytes(data))
+        elif boundary_objects:
+            self._send_boundary_objects(socket, boundary_objects)
 
     def request(  # noqa: PLR0912,PLR0913,PLR0915 Too many branches,Too many arguments in function definition,Too many statements
         self,
@@ -473,6 +595,7 @@ def request(  # noqa: PLR0912,PLR0913,PLR0915 Too many branches,Too many argumen
         stream: bool = False,
         timeout: float = 60,
         allow_redirects: bool = True,
+        files: Optional[Dict[str, tuple]] = None,
     ) -> Response:
         """Perform an HTTP request to the given url which we will parse to determine
         whether to use SSL ('https://') or not. We can also send some provided 'data'
@@ -521,7 +644,9 @@ def request(  # noqa: PLR0912,PLR0913,PLR0915 Too many branches,Too many argumen
             )
             ok = True
             try:
-                self._send_request(socket, host, method, path, headers, data, json)
+                self._send_request(
+                    socket, host, method, path, headers, data, json, files
+                )
             except OSError as exc:
                 last_exc = exc
                 ok = False

examples/cpython/requests_cpython_advanced.py

@@ -15,17 +15,13 @@
 headers = {"user-agent": "blinka/1.0.0"}
 
 print("Fetching JSON data from %s..." % JSON_GET_URL)
-response = requests.get(JSON_GET_URL, headers=headers)
-print("-" * 60)
-
-json_data = response.json()
-headers = json_data["headers"]
-print("Response's Custom User-Agent Header: {0}".format(headers["User-Agent"]))
-print("-" * 60)
-
-# Read Response's HTTP status code
-print("Response HTTP Status Code: ", response.status_code)
-print("-" * 60)
-
-# Close, delete and collect the response data
-response.close()
+with requests.get(JSON_GET_URL, headers=headers) as response:
+    print("-" * 60)
+    json_data = response.json()
+    headers = json_data["headers"]
+    print("Response's Custom User-Agent Header: {0}".format(headers["User-Agent"]))
+    print("-" * 60)
+
+    # Read Response's HTTP status code
+    print("Response HTTP Status Code: ", response.status_code)
+    print("-" * 60)

examples/cpython/requests_cpython_simpletest.py

@@ -14,39 +14,31 @@
 JSON_POST_URL = "https://httpbin.org/post"
 
 print("Fetching text from %s" % TEXT_URL)
-response = requests.get(TEXT_URL)
-print("-" * 40)
-
-print("Text Response: ", response.text)
-print("-" * 40)
-response.close()
+with requests.get(TEXT_URL) as response:
+    print("-" * 40)
+    print("Text Response: ", response.text)
+    print("-" * 40)
 
 print("Fetching JSON data from %s" % JSON_GET_URL)
-response = requests.get(JSON_GET_URL)
-print("-" * 40)
-
-print("JSON Response: ", response.json())
-print("-" * 40)
-response.close()
+with requests.get(JSON_GET_URL) as response:
+    print("-" * 40)
+    print("JSON Response: ", response.json())
+    print("-" * 40)
 
 data = "31F"
 print(f"POSTing data to {JSON_POST_URL}: {data}")
-response = requests.post(JSON_POST_URL, data=data)
-print("-" * 40)
-
-json_resp = response.json()
-# Parse out the 'data' key from json_resp dict.
-print("Data received from server:", json_resp["data"])
-print("-" * 40)
-response.close()
+with requests.post(JSON_POST_URL, data=data) as response:
+    print("-" * 40)
+    json_resp = response.json()
+    # Parse out the 'data' key from json_resp dict.
+    print("Data received from server:", json_resp["data"])
+    print("-" * 40)
 
 json_data = {"Date": "July 25, 2019"}
 print(f"POSTing data to {JSON_POST_URL}: {json_data}")
-response = requests.post(JSON_POST_URL, json=json_data)
-print("-" * 40)
-
-json_resp = response.json()
-# Parse out the 'json' key from json_resp dict.
-print("JSON Data received from server:", json_resp["json"])
-print("-" * 40)
-response.close()
+with requests.post(JSON_POST_URL, json=json_data) as response:
+    print("-" * 40)
+    json_resp = response.json()
+    # Parse out the 'json' key from json_resp dict.
+    print("JSON Data received from server:", json_resp["json"])
+    print("-" * 40)

examples/esp32spi/requests_esp32spi_advanced.py

@@ -53,17 +53,14 @@
 headers = {"user-agent": "blinka/1.0.0"}
 
 print("Fetching JSON data from %s..." % JSON_GET_URL)
-response = requests.get(JSON_GET_URL, headers=headers)
-print("-" * 60)
+with requests.get(JSON_GET_URL, headers=headers) as response:
+    print("-" * 60)
 
-json_data = response.json()
-headers = json_data["headers"]
-print("Response's Custom User-Agent Header: {0}".format(headers["User-Agent"]))
-print("-" * 60)
+    json_data = response.json()
+    headers = json_data["headers"]
+    print("Response's Custom User-Agent Header: {0}".format(headers["User-Agent"]))
+    print("-" * 60)
 
-# Read Response's HTTP status code
-print("Response HTTP Status Code: ", response.status_code)
-print("-" * 60)
-
-# Close, delete and collect the response data
-response.close()
+    # Read Response's HTTP status code
+    print("Response HTTP Status Code: ", response.status_code)
+    print("-" * 60)