[pull] master from robbrad:master

CHANGELOG.md

@@ -1,4 +1,56 @@
 =======
+## 0.151.0 (2025-04-27)
+
+### Feat
+
+- version bump
+
+### Fix
+
+- more robust brent date handling
+- input.json requires web_driver
+- Rugby fix
+- simplified blackburn
+
+## 0.150.0 (2025-04-27)
+
+### Feat
+
+- added melton
+
+### Fix
+
+- added melton
+- processed all bins for Moray
+
+## 0.149.0 (2025-04-27)
+
+### Feat
+
+- added pembrokeshire
+
+## 0.148.6 (2025-04-27)
+
+### Fix
+
+- updated input.json
+- cloudflare fix - switch to selenium method
+
+## 0.148.5 (2025-04-27)
+
+### Fix
+
+- output check
+- parsed bin info
+- selenium navigation
+- input.json changes
+
+## 0.148.4 (2025-04-27)
+
+### Fix
+
+- used canonical 'nice name'
+
 ## 0.148.3 (2025-04-25)
 
 ### Fix

custom_components/uk_bin_collection/config_flow.py

@@ -253,7 +253,7 @@ async def async_step_reconfigure_confirm(
 
     async def get_councils_json(self) -> Dict[str, Any]:
         """Fetch and return the supported councils data, including aliases and sorted alphabetically."""
-        url = "https://raw.githubusercontent.com/robbrad/UKBinCollectionData/0.148.3/uk_bin_collection/tests/input.json"
+        url = "https://raw.githubusercontent.com/robbrad/UKBinCollectionData/0.151.0/uk_bin_collection/tests/input.json"
         try:
             async with aiohttp.ClientSession() as session:
                 async with session.get(url) as response:

custom_components/uk_bin_collection/manifest.json

@@ -9,7 +9,7 @@
     "integration_type": "service",
     "iot_class": "cloud_polling",
     "issue_tracker": "https://github.com/robbrad/UKBinCollectionData/issues",
-    "requirements": ["uk-bin-collection>=0.148.3"],
-    "version": "0.148.3",
+    "requirements": ["uk-bin-collection>=0.151.0"],
+    "version": "0.151.0",
     "zeroconf": []
 }

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "uk_bin_collection"
-version = "0.148.3"
+version = "0.151.0"
 description = "Python Lib to collect UK Bin Data"
 readme = "README.md"
 authors = ["Robert Bradley <robbrad182@gmail.com>"]

uk_bin_collection/uk_bin_collection/councils/ArgyllandButeCouncil.py

@@ -8,8 +8,6 @@
 
 
 # import the wonderful Beautiful Soup and the URL grabber
-
-
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the

uk_bin_collection/uk_bin_collection/councils/BrentCouncil.py

@@ -74,42 +74,56 @@ def parse_data(self, page: str, **kwargs) -> dict:
                     )
 
                     if service_details:
-
-                        # Extract next collection date
+                        # Extract next collection date only
                         next_collection_row = service_details.find(
                             "dt", string="Next collection"
                         )
-                        next_collection = (
-                            next_collection_row.find_next_sibling("dd").get_text(
-                                strip=True
-                            )
-                            if next_collection_row
-                            else "Unknown"
-                        )
-
-                        # Parse dates into standard dd/mm/yyyy format
-                        next_collection_date = datetime.strptime(
-                            remove_ordinal_indicator_from_date_string(next_collection),
-                            "%A, %d %B",
-                        )
-
-                        if (datetime.now().month == 12) and (
-                            next_collection.month == 1
-                        ):
-                            next_collection_date = next_collection_date.replace(
-                                year=next_year
+                        if next_collection_row:
+                            next_collection = next_collection_row.find_next_sibling(
+                                "dd"
+                            ).get_text(strip=True)
+
+                            # Remove the adjusted collection time message
+                            if (
+                                "(this collection has been adjusted from its usual time)"
+                                in next_collection
+                            ):
+                                next_collection = next_collection.replace(
+                                    "(this collection has been adjusted from its usual time)",
+                                    "",
+                                ).strip()
+
+                            # Parse date from format like "Wednesday, 7th May"
+                            next_collection = remove_ordinal_indicator_from_date_string(
+                                next_collection
                             )
-                        else:
-                            next_collection_date = next_collection_date.replace(
-                                year=current_year
-                            )
-
-                        dict_data = {
-                            "type": collection_type.strip(),
-                            "collectionDate": next_collection_date.strftime(
-                                date_format
-                            ),
-                        }
-                        data["bins"].append(dict_data)
+                            try:
+                                next_collection_date = datetime.strptime(
+                                    next_collection, "%A, %d %B"
+                                )
+
+                                # Handle year rollover
+                                if (
+                                    datetime.now().month == 12
+                                    and next_collection_date.month == 1
+                                ):
+                                    next_collection_date = next_collection_date.replace(
+                                        year=next_year
+                                    )
+                                else:
+                                    next_collection_date = next_collection_date.replace(
+                                        year=current_year
+                                    )
+
+                                dict_data = {
+                                    "type": collection_type.strip(),
+                                    "collectionDate": next_collection_date.strftime(
+                                        date_format
+                                    ),
+                                }
+                                data["bins"].append(dict_data)
+                                print(dict_data)
+                            except ValueError as e:
+                                print(f"Error parsing date {next_collection}: {e}")
 
         return data

uk_bin_collection/uk_bin_collection/councils/MaidstoneBoroughCouncil.py

@@ -0,0 +1,115 @@
+import time
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.wait import WebDriverWait
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+# import the wonderful Beautiful Soup and the URL grabber
+
+
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        driver = None
+        try:
+            page = "https://my.maidstone.gov.uk/service/Find-your-bin-day"
+            bin_data = {"bins": []}
+            user_paon = kwargs.get("paon")
+            user_postcode = kwargs.get("postcode")
+            web_driver = kwargs.get("web_driver")
+            headless = kwargs.get("headless")
+            check_postcode(user_postcode)
+
+            # Create Selenium webdriver
+            driver = create_webdriver(web_driver, headless, None, __name__)
+            driver.get(page)
+
+            iframe_presense = WebDriverWait(driver, 30).until(
+                EC.presence_of_element_located((By.ID, "fillform-frame-1"))
+            )
+            driver.switch_to.frame(iframe_presense)
+
+            wait = WebDriverWait(driver, 60)
+
+            # Postal code input
+            inputElement_postcodesearch = wait.until(
+                EC.element_to_be_clickable((By.NAME, "postcode"))
+            )
+            inputElement_postcodesearch.send_keys(user_postcode)
+
+            # Wait for the 'Select address' dropdown to be updated
+            dropdown_select = wait.until(
+                EC.presence_of_element_located((By.XPATH, "//span[contains(text(), 'Select...')]"))
+            )
+            dropdown_select.click()
+
+            dropdown = wait.until(
+                EC.element_to_be_clickable((By.XPATH, f"//div[contains(text(), ' {user_paon}')]"))
+            )
+            dropdown.click()
+
+            # Wait for 'Searching for...' to be added to page
+            WebDriverWait(driver, timeout=15).until(
+                EC.text_to_be_present_in_element(
+                    (By.CSS_SELECTOR, "span[data-name=html1]"), "Searching"
+                )
+            )
+
+            # Wait for 'Searching for...' to be removed from page
+            WebDriverWait(driver, timeout=15).until(
+                EC.none_of(
+                    EC.text_to_be_present_in_element(
+                        (By.CSS_SELECTOR, "span[data-name=html1]"), "Searching"
+                    )
+                )
+            )
+
+            # Even then it can still be adding data to the page...
+            time.sleep(5)
+
+            soup = BeautifulSoup(driver.page_source, features="html.parser")
+
+            # This is ugly but there is literally no consistency to the HTML
+            def is_a_collection_date(t):
+                return any("Next collection" in c for c in t.children)
+
+            for next_collection in soup.find_all(is_a_collection_date):
+                bin_info = list(
+                    next_collection.parent.select_one("div:nth-child(1)").children
+                )
+                if not bin_info:
+                    continue
+                bin = bin_info[0].get_text()
+                date = next_collection.select_one("strong").get_text(strip=True)
+                bin_date = datetime.strptime(date, "%d %b %Y")
+                dict_data = {
+                    "type": bin,
+                    "collectionDate": bin_date.strftime(date_format),
+                }
+                bin_data["bins"].append(dict_data)
+
+            bin_data["bins"].sort(
+                key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
+            )
+
+        except Exception as e:
+            # Here you can log the exception if needed
+            print(f"An error occurred: {e}")
+            # Optionally, re-raise the exception if you want it to propagate
+            raise
+        finally:
+            # This block ensures that the driver is closed regardless of an exception
+            if driver:
+                driver.quit()
+        return bin_data

uk_bin_collection/uk_bin_collection/councils/MeltonBoroughCouncil.py

@@ -0,0 +1,82 @@
+import json
+from datetime import datetime, timedelta
+
+import requests
+from bs4 import BeautifulSoup
+
+from uk_bin_collection.uk_bin_collection.common import *
+from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
+
+
+# import the wonderful Beautiful Soup and the URL grabber
+class CouncilClass(AbstractGetBinDataClass):
+    """
+    Concrete classes have to implement all abstract operations of the
+    base class. They can also override some operations with a default
+    implementation.
+    """
+
+    def extract_dates(self, date_string: str) -> list:
+        """
+        Extract dates from strings like "01/05/2025, and then 15/05/2025"
+        Returns list of datetime objects
+        """
+        date_string = date_string.replace("and then", ",")
+        date_parts = [part.strip() for part in date_string.split(",") if part.strip()]
+
+        dates = []
+        for part in date_parts:
+            try:
+                date_obj = datetime.strptime(part, "%d/%m/%Y")
+                dates.append(date_obj)
+            except ValueError:
+                continue
+
+        return dates
+
+    def parse_data(self, page: str, **kwargs) -> dict:
+        user_uprn = kwargs.get("uprn")
+        check_uprn(user_uprn)
+
+        url = f"https://my.melton.gov.uk/set-location?id={user_uprn}&redirect=collections&rememberloc="
+        response = requests.get(url)
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        collections = []
+
+        # Find all box items
+        box_items = soup.find_all("li", class_=lambda x: x and x.startswith("box-item"))
+
+        for box in box_items:
+            bin_type = box.find("h2")
+            if (
+                bin_type and "Missed bin" not in bin_type.text
+            ):  # Skip the "Missed bin" section
+                bin_name = bin_type.text.strip()
+
+                # Find the strong tag containing dates
+                dates_element = box.find("strong")
+                if dates_element:
+                    dates_text = dates_element.text.strip()
+                    # Use self.extract_dates instead of extract_dates
+                    collection_dates = self.extract_dates(dates_text)
+
+                    # Add each date for this bin type to collections
+                    for date in collection_dates:
+                        collections.append((bin_name, date))
+
+        # Sort the collections by date
+        ordered_data = sorted(collections, key=lambda x: x[1])
+
+        # Format the data as required
+        data = {"bins": []}
+        for item in ordered_data:
+            dict_data = {
+                "type": item[0],
+                "collectionDate": item[1].strftime(date_format),
+            }
+            data["bins"].append(dict_data)
+
+        print(json.dumps(data, indent=2))
+
+        return data