Merge pull request #1743 from seleniumbase/multithreaded-downloads

Make improvements to multithreaded downloads, and more

Author: mdmintz

examples/coffee_cart_tests.py

@@ -5,7 +5,7 @@
 
 class CoffeeCartTests(BaseCase):
     def test_1_verify_nav_link_to_coffee_cart(self):
-        self.open("https://seleniumbase.io/")
+        self.open("https://seleniumbase.io/help_docs/customizing_test_runs/")
         self.js_click('nav a:contains("Coffee Cart")')
         self.assert_title("Coffee Cart")
         self.assert_element('h4:contains("Espresso")')

examples/test_canvas.py

@@ -32,6 +32,8 @@ def test_canvas_click_from_center(self):
 
     def test_click_with_offset(self):
         self.open("https://seleniumbase.io/canvas/")
+        if self.undetectable:
+            self.skip("Skip this test in undetectable mode.")
         self.assert_title_contains("Canvas")
         self.highlight("canvas")
         rgb = self.get_pixel_colors()

examples/test_download_files.py

@@ -42,6 +42,11 @@ def test_download_files_from_pypi(self):
             or self.browser == "ie"
             or (self.is_chromium() and self.guest_mode and not self.headless)
             or (self.undetectable and (self.headless or self.headless2))
+            or (
+                self.is_chromium()
+                and int(self.get_chromium_version()) >= 110
+                and self.headless
+            )
         ):
             whl_href = self.get_attribute(whl_selector, "href")
             tar_href = self.get_attribute(tar_selector, "href")

help_docs/method_summary.md

@@ -485,8 +485,12 @@ self.is_chromium()
 
 self.get_chrome_version()
 
+self.get_chromium_version()
+
 self.get_chromedriver_version()
 
+self.get_chromium_driver_version()
+
 self.get_mfa_code(totp_key=None)
 # Duplicates: self.get_totp_code(totp_key=None)
 #             self.get_google_auth_password(totp_key=None)

requirements.txt

@@ -67,16 +67,14 @@ pytest-ordering==0.6
 pytest-rerunfailures==10.3;python_version<"3.7"
 pytest-rerunfailures==11.0;python_version>="3.7"
 pytest-xdist==2.5.0;python_version<"3.7"
-pytest-xdist==3.1.0;python_version>="3.7"
+pytest-xdist==3.2.0;python_version>="3.7"
 parameterized==0.8.1
 sbvirtualdisplay==1.2.0
 behave==1.2.6
-parse==1.19.0
-parse-type==0.6.0
 soupsieve==2.3.2.post1
 beautifulsoup4==4.11.2
 cryptography==36.0.2;python_version<"3.7"
-cryptography==39.0.0;python_version>="3.7"
+cryptography==39.0.1;python_version>="3.7"
 pygments==2.14.0
 pyreadline3==3.4.1;platform_system=="Windows"
 tabcompleter==1.1.0

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.12.7"
+__version__ = "4.12.8"

seleniumbase/common/exceptions.py

@@ -1,6 +1,7 @@
 """ SeleniumBase Exceptions
     NoSuchFileException => Called when self.assert_downloaded_file(...) fails.
     NotUsingChromeException => Used by Chrome-only methods if not using Chrome.
+    NotUsingChromiumException => Used by Chromium-only methods if not Chromium.
     OutOfScopeException => Used by BaseCase methods when setUp() is skipped.
     TextNotVisibleException => Called when expected text fails to appear.
     TimeLimitExceededException => Called when exceeding "--time-limit=SECONDS".
@@ -16,6 +17,10 @@ class NotUsingChromeException(Exception):
     pass
 
 
+class NotUsingChromiumException(Exception):
+    pass
+
+
 class OutOfScopeException(Exception):
     pass
 

seleniumbase/core/browser_launcher.py

@@ -9,6 +9,7 @@
 import time
 import urllib3
 import warnings
+import zipfile
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service as ChromeService
 from selenium.webdriver.edge.service import Service as EdgeService
@@ -351,8 +352,6 @@ def _unzip_to_new_folder(zip_file, folder):
     proxy_dir_lock = fasteners.InterProcessLock(PROXY_DIR_LOCK)
     with proxy_dir_lock:
         if not os.path.exists(folder):
-            import zipfile
-
             zip_ref = zipfile.ZipFile(zip_file, "r")
             os.makedirs(folder)
             zip_ref.extractall(folder)

seleniumbase/fixtures/base_case.py

@@ -61,6 +61,12 @@ def test_anything(self):
 from seleniumbase import config as sb_config
 from seleniumbase.__version__ import __version__
 from seleniumbase.common import decorators
+from seleniumbase.common.exceptions import (
+    NotUsingChromeException,
+    NotUsingChromiumException,
+    OutOfScopeException,
+    VisualException,
+)
 from seleniumbase.config import settings
 from seleniumbase.core import download_helper
 from seleniumbase.core import log_helper
@@ -6427,11 +6433,15 @@ def save_element_as_image_file(
     def download_file(self, file_url, destination_folder=None):
         """Downloads the file from the url to the destination folder.
         If no destination folder is specified, the default one is used.
-        (The default [Downloads Folder] = "./downloaded_files")"""
-        if not destination_folder:
-            destination_folder = constants.Files.DOWNLOADS_FOLDER
-        if not os.path.exists(destination_folder):
-            os.makedirs(destination_folder)
+        (The default folder for downloads is "./downloaded_files")"""
+        download_file_lock = fasteners.InterProcessLock(
+            constants.MultiBrowser.DOWNLOAD_FILE_LOCK
+        )
+        with download_file_lock:
+            if not destination_folder:
+                destination_folder = constants.Files.DOWNLOADS_FOLDER
+            if not os.path.exists(destination_folder):
+                os.makedirs(destination_folder)
         page_utils._download_file_to(file_url, destination_folder)
         if self.recorder_mode:
             url = self.get_current_url()
@@ -6489,6 +6499,12 @@ def get_browser_downloads_folder(self):
             and self.headless
         ):
             return os.path.join(os.path.expanduser("~"), "downloads")
+        elif (
+            self.driver.capabilities["browserName"].lower() == "chrome"
+            and int(self.get_chromedriver_version().split(".")[0]) >= 110
+            and self.headless
+        ):
+            return os.path.abspath(".")
         else:
             return download_helper.get_downloads_folder()
         return os.path.join(os.path.expanduser("~"), "downloads")
@@ -7088,17 +7104,27 @@ def __fail_if_not_using_chrome(self, method):
         if browser_name.lower() == "chrome":
             chrome = True
         if not chrome:
-            from seleniumbase.common.exceptions import NotUsingChromeException
-
             message = (
-                'Error: "%s" should only be called '
-                'by tests running with self.browser == "chrome"! '
+                'Error: "%s" should only be called by tests '
+                'running with "--browser=chrome" / "--chrome"! '
                 'You should add an "if" statement to your code before calling '
                 "this method if using browsers that are Not Chrome! "
                 'The browser detected was: "%s".' % (method, browser_name)
             )
             raise NotUsingChromeException(message)
 
+    def __fail_if_not_using_chromium(self, method):
+        browser_name = self.driver.capabilities["browserName"]
+        if not self.is_chromium():
+            message = (
+                'Error: "%s" should only be called by tests '
+                'running with a Chromium browser! (Chrome or Edge) '
+                'You should add an "if" statement to your code before calling '
+                "this method if using browsers that are Not Chromium! "
+                'The browser detected was: "%s".' % (method, browser_name)
+            )
+            raise NotUsingChromiumException(message)
+
     def get_chrome_version(self):
         self.__check_scope()
         self.__fail_if_not_using_chrome("get_chrome_version()")
@@ -7109,6 +7135,11 @@ def get_chrome_version(self):
             chrome_version = driver_capabilities["browserVersion"]
         return chrome_version
 
+    def get_chromium_version(self):
+        self.__check_scope()
+        self.__fail_if_not_using_chromium("get_chromium_version()")
+        return self.__get_major_browser_version()
+
     def get_chromedriver_version(self):
         self.__check_scope()
         self.__fail_if_not_using_chrome("get_chromedriver_version()")
@@ -7117,14 +7148,19 @@ def get_chromedriver_version(self):
         chromedriver_version = chromedriver_version.split(" ")[0]
         return chromedriver_version
 
-    def is_chromedriver_too_old(self):
-        """Before chromedriver 73, there was no version check, which
-        means it's possible to run a new Chrome with old drivers."""
+    def get_chromium_driver_version(self):
         self.__check_scope()
-        self.__fail_if_not_using_chrome("is_chromedriver_too_old()")
-        if int(self.get_chromedriver_version().split(".")[0]) < 73:
-            return True  # chromedriver is too old! Please upgrade!
-        return False
+        self.__fail_if_not_using_chromium("get_chromium_version()")
+        driver_version = None
+        if "chrome" in self.driver.capabilities:
+            chrome_dict = self.driver.capabilities["chrome"]
+            driver_version = chrome_dict["chromedriverVersion"]
+            driver_version = driver_version.split(" ")[0]
+        elif "msedge" in self.driver.capabilities:
+            edge_dict = self.driver.capabilities["msedge"]
+            driver_version = edge_dict["msedgedriverVersion"]
+            driver_version = driver_version.split(" ")[0]
+        return driver_version
 
     def get_mfa_code(self, totp_key=None):
         """Same as get_totp_code() and get_google_auth_password().
@@ -9303,8 +9339,6 @@ def __assert_eq(self, *args, **kwargs):
                 elif line.strip().startswith("*"):
                     minified_exception += line + "\n"
         if minified_exception:
-            from seleniumbase.common.exceptions import VisualException
-
             raise VisualException(minified_exception)
 
     def _process_visual_baseline_logs(self):
@@ -9684,8 +9718,6 @@ def __check_scope(self):
         if hasattr(self, "browser"):  # self.browser stores the type of browser
             return  # All good: setUp() already initialized variables in "self"
         else:
-            from seleniumbase.common.exceptions import OutOfScopeException
-
             message = (
                 "\n It looks like you are trying to call a SeleniumBase method"
                 "\n from outside the scope of your test class's `self` object,"
@@ -12730,9 +12762,18 @@ def __disable_beforeunload_as_needed(self):
 
     ############
 
+    @decorators.deprecated("The Driver Manager prevents old drivers.")
+    def is_chromedriver_too_old(self):
+        """Before chromedriver 73, there was no version check, which
+        means it's possible to run a new Chrome with old drivers."""
+        self.__fail_if_not_using_chrome("is_chromedriver_too_old()")
+        if int(self.get_chromedriver_version().split(".")[0]) < 73:
+            return True  # chromedriver is too old! Please upgrade!
+        return False
+
     @decorators.deprecated("You should use re.escape() instead.")
     def jq_format(self, code):
-        # DEPRECATED - re.escape() already performs the intended action.
+        # DEPRECATED - re.escape() already performs this action.
         return js_utils._jq_format(code)
 
     ############

seleniumbase/fixtures/constants.py

@@ -124,6 +124,7 @@ class MultiBrowser:
     DRIVER_FIXING_LOCK = Files.DOWNLOADS_FOLDER + "/driver_fixing.lock"
     DRIVER_REPAIRED = Files.DOWNLOADS_FOLDER + "/driver_fixed.lock"
     CERT_FIXING_LOCK = Files.DOWNLOADS_FOLDER + "/cert_fixing.lock"
+    DOWNLOAD_FILE_LOCK = Files.DOWNLOADS_FOLDER + "/download_file.lock"
 
 
 class SavedCookies: