Merge pull request #1493 from seleniumbase/undetected-chromedriver-updates

Undetected-Chromedriver Mode updates

Author: mdmintz

examples/verify_undetected.py

@@ -10,8 +10,8 @@ class UndetectedTest(BaseCase):
     def test_browser_is_undetected(self):
         self.open("https://nowsecure.nl")
         try:
-            self.assert_text("OH YEAH, you passed!", "h1", timeout=6.6)
-            self.post_message("Browser wasn't detected!", duration=1.6)
-            self._print("\n Success! Website did not detect Selenium!")
+            self.assert_text("OH YEAH, you passed!", "h1", timeout=6.75)
+            self.post_message("Selenium wasn't detected!", duration=1.6)
+            self._print("\n Success! Website did not detect Selenium! ")
         except Exception:
-            self.fail('Browser was detected! Try using: "pytest --uc"')
+            self.fail('Selenium was detected! Try using: "pytest --uc"')

mkdocs_build/requirements.txt

@@ -2,20 +2,19 @@
 # Minimum Python version: 3.7
 
 regex>=2022.8.17
-tqdm>=4.64.0
 docutils==0.19
 python-dateutil==2.8.2
 livereload==2.6.3
 joblib==1.1.0
 Markdown==3.3.7
 MarkupSafe==2.1.1
-keyring==23.9.0
+keyring==23.9.1
 pkginfo==1.8.3
 Jinja2==3.1.2
 click==8.1.3
 zipp==3.8.1
 ghp-import==2.1.0
-readme-renderer==37.0
+readme-renderer==37.1
 pymdown-extensions==9.5
 importlib-metadata==4.12.0
 bleach==5.0.1

requirements.txt

@@ -8,6 +8,7 @@ setuptools>=59.6.0;python_version>="3.6" and python_version<"3.7"
 setuptools>=65.3.0;python_version>="3.7"
 tomli>=1.2.3;python_version>="3.6" and python_version<"3.7"
 tomli>=2.0.1;python_version>="3.7"
+tqdm>=4.64.1
 wheel>=0.37.1
 attrs>=21.4.0;python_version<"3.6"
 attrs>=22.1.0;python_version>="3.6"
@@ -68,7 +69,7 @@ py==1.8.1;python_version<"3.6"
 py==1.11.0;python_version>="3.6"
 pytest==4.6.11;python_version<"3.6"
 pytest==7.0.1;python_version>="3.6" and python_version<"3.7"
-pytest==7.1.2;python_version>="3.7"
+pytest==7.1.3;python_version>="3.7"
 pytest-forked==1.3.0;python_version<"3.6"
 pytest-forked==1.4.0;python_version>="3.6"
 pytest-html==1.22.1;python_version<"3.6"
@@ -96,7 +97,7 @@ cryptography==37.0.4;python_version>="3.7"
 pygments==2.5.2;python_version<"3.6"
 pygments==2.13.0;python_version>="3.6"
 prompt-toolkit==1.0.18;python_version<"3.6"
-prompt-toolkit==3.0.30;python_version>="3.6"
+prompt-toolkit==3.0.31;python_version>="3.6"
 decorator==4.4.2;python_version<"3.6"
 decorator==5.1.1;python_version>="3.6"
 ipython==5.10.0;python_version<"3.6"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.3.1"
+__version__ = "4.3.2"

seleniumbase/core/browser_launcher.py

@@ -227,7 +227,7 @@ def _add_chrome_proxy_extension(
             chrome_options = add_chrome_ext_dir(chrome_options, PROXY_DIR_PATH)
 
     else:
-        # Pytest multi-threaded test
+        # Pytest multithreaded test
         if zip_it:
             proxy_zip_lock = fasteners.InterProcessLock(PROXY_ZIP_LOCK)
             with proxy_zip_lock:
@@ -2007,6 +2007,8 @@ def get_local_driver(
                 )
                 driver = Edge(service=service, options=edge_options)
             except Exception as e:
+                if not hasattr(e, "msg"):
+                    raise
                 auto_upgrade_edgedriver = False
                 edge_version = None
                 if (
@@ -2063,6 +2065,8 @@ def get_local_driver(
                     capabilities=capabilities,
                 )
             except Exception as e:
+                if not hasattr(e, "msg"):
+                    raise
                 auto_upgrade_edgedriver = False
                 edge_version = None
                 if (
@@ -2114,7 +2118,7 @@ def get_local_driver(
         arg_join = " ".join(sys.argv)
         if ("-n" in sys.argv) or (" -n=" in arg_join) or (arg_join == "-c"):
             # Skip if multithreaded
-            raise Exception("Can't run Safari tests in multi-threaded mode!")
+            raise Exception("Can't run Safari tests in multithreaded mode!")
         warnings.simplefilter("ignore", category=DeprecationWarning)
         return webdriver.safari.webdriver.WebDriver(quiet=False)
     elif browser_name == constants.Browser.OPERA:
@@ -2343,21 +2347,36 @@ def get_local_driver(
                         or is_using_uc(undetectable, browser_name)
                     ):
                         if selenium4_or_newer:
+                            if headless and "linux" not in PLATFORM:
+                                undetectable = False  # No support for headless
                             if undetectable:
                                 from seleniumbase import undetected
                                 from urllib.error import URLError
 
                                 if "linux" in PLATFORM:
-                                    chrome_options.headless = False  # Use xvfb
+                                    chrome_options.headless = False  # Use Xvfb
+                                    if "--headless" in (
+                                        chrome_options.arguments
+                                    ):
+                                        chrome_options.arguments.remove(
+                                            "--headless"
+                                        )
                                 cert = "unable to get local issuer certificate"
+                                uc_chrome_version = None
+                                if (
+                                    use_version.isnumeric
+                                    and int(use_version) >= 72
+                                ):
+                                    uc_chrome_version = int(use_version)
                                 uc_lock = fasteners.InterProcessLock(
                                     constants.MultiBrowser.DRIVER_FIXING_LOCK
                                 )
-                                with uc_lock:
+                                with uc_lock:  # No UC multithreaded tests
                                     try:
                                         driver = undetected.Chrome(
                                             options=chrome_options,
-                                            headless=headless,
+                                            headless=False,  # Xvfb needed
+                                            version_main=uc_chrome_version,
                                         )
                                     except URLError as e:
                                         if (
@@ -2371,7 +2390,8 @@ def get_local_driver(
                                             )
                                             driver = undetected.Chrome(
                                                 options=chrome_options,
-                                                headless=headless,
+                                                headless=False,  # Xvfb needed
+                                                version_main=uc_chrome_version,
                                             )
                                         else:
                                             raise
@@ -2402,6 +2422,8 @@ def get_local_driver(
                                 service_log_path=os.devnull,
                             )
                 except Exception as e:
+                    if not hasattr(e, "msg"):
+                        raise
                     auto_upgrade_chromedriver = False
                     if "This version of ChromeDriver only supports" in e.msg:
                         auto_upgrade_chromedriver = True
@@ -2513,6 +2535,8 @@ def get_local_driver(
                 try:
                     return webdriver.Chrome(options=chrome_options)
                 except Exception as e:
+                    if not hasattr(e, "msg"):
+                        raise
                     auto_upgrade_chromedriver = False
                     if "This version of ChromeDriver only supports" in e.msg:
                         auto_upgrade_chromedriver = True

seleniumbase/undetected/__init__.py

@@ -117,9 +117,8 @@ def __init__(
         suppress_welcome: (default: True)
             Suppress the Chrome welcome screen that appears on first-time runs.
 
-        use_subprocess:
-            False (default) Give Chrome its own process,
-                            rather than subprocessing chromedriver or python.
+        use_subprocess: (default: False)
+            Subprocess chromedriver/python: Don't make Chrome a parent process.
         """
         self.debug = debug
         patcher = Patcher(
@@ -238,7 +237,10 @@ def __init__(
                 mode="r+",
             ) as fs:
                 config = json.load(fs)
-                if config["profile"]["exit_type"] is not None:
+                if (
+                    "exit_type" not in config["profile"].keys()
+                    or config["profile"]["exit_type"] is not None
+                ):
                     config["profile"]["exit_type"] = None
                 fs.seek(0, 0)
                 json.dump(config, fs)

seleniumbase/undetected/options.py

@@ -26,6 +26,17 @@ def _undot_key(key, value):
             value = ChromeOptions._undot_key(rest, value)
         return {key: value}
 
+    @staticmethod
+    def _merge_nested(a, b):
+        """Merges b into a, overwriting duplicate leaf values using b."""
+        for key in b:
+            if key in a:
+                if isinstance(a[key], dict) and isinstance(b[key], dict):
+                    ChromeOptions._merge_nested(a[key], b[key])
+                    continue
+            a[key] = b[key]
+        return a
+
     def handle_prefs(self, user_data_dir):
         prefs = self.experimental_options.get("prefs")
         if prefs:
@@ -34,11 +45,13 @@ def handle_prefs(self, user_data_dir):
             os.makedirs(default_path, exist_ok=True)
             undot_prefs = {}
             for key, value in prefs.items():
-                undot_prefs.update(self._undot_key(key, value))
+                undot_prefs = self._merge_nested(
+                    undot_prefs, self._undot_key(key, value)
+                )
             prefs_file = os.path.join(default_path, "Preferences")
             if os.path.exists(prefs_file):
                 with open(prefs_file, encoding="latin1", mode="r") as f:
-                    undot_prefs.update(json.load(f))
+                    undot_prefs = self._merge_nested(json.load(f), undot_prefs)
             with open(prefs_file, encoding="latin1", mode="w") as f:
                 json.dump(undot_prefs, f)
             # Remove experimental_options to avoid errors

setup.py

@@ -51,7 +51,7 @@
         print("\n*** Installing twine: *** (Required for PyPI uploads)\n")
         os.system("python -m pip install --upgrade 'twine>=4.0.1'")
         print("\n*** Installing tqdm: *** (Required for PyPI uploads)\n")
-        os.system("python -m pip install --upgrade 'tqdm>=4.64.0'")
+        os.system("python -m pip install --upgrade tqdm")
         print("\n*** Rebuilding distribution packages: ***\n")
         os.system("python -m build")  # Create new tar/wheel
         print("\n*** Publishing The Release to PyPI: ***\n")
@@ -134,6 +134,7 @@
         'setuptools>=65.3.0;python_version>="3.7"',
         'tomli>=1.2.3;python_version>="3.6" and python_version<"3.7"',
         'tomli>=2.0.1;python_version>="3.7"',
+        "tqdm>=4.64.1",
         "wheel>=0.37.1",
         'attrs>=21.4.0;python_version<"3.6"',
         'attrs>=22.1.0;python_version>="3.6"',
@@ -193,7 +194,7 @@
         'py==1.11.0;python_version>="3.6"',
         'pytest==4.6.11;python_version<"3.6"',
         'pytest==7.0.1;python_version>="3.6" and python_version<"3.7"',
-        'pytest==7.1.2;python_version>="3.7"',
+        'pytest==7.1.3;python_version>="3.7"',
         'pytest-forked==1.3.0;python_version<"3.6"',
         'pytest-forked==1.4.0;python_version>="3.6"',
         'pytest-html==1.22.1;python_version<"3.6"',
@@ -221,7 +222,7 @@
         'pygments==2.5.2;python_version<"3.6"',
         'pygments==2.13.0;python_version>="3.6"',
         'prompt-toolkit==1.0.18;python_version<"3.6"',
-        'prompt-toolkit==3.0.30;python_version>="3.6"',
+        'prompt-toolkit==3.0.31;python_version>="3.6"',
         'decorator==4.4.2;python_version<"3.6"',
         'decorator==5.1.1;python_version>="3.6"',
         'ipython==5.10.0;python_version<"3.6"',