Config fixes required to support multiple browsers. (issue #60) (#67)

Author: raycardillo

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Multiple Browsers can be created without one affecting the other @raycardillo
+
 ### Added
 
 ### Changed

tests/bot_detection/test_browserscan.py

@@ -4,6 +4,9 @@
 async def test_browserscan(browser: zd.Browser):
     page = await browser.get("https://www.browserscan.net/bot-detection")
 
+    # wait for the page to fully load
+    await page.wait_for_ready_state("complete")
+
     # give the javascript some time to finish executing
     await page.wait(2)
 

tests/conftest.py

@@ -43,6 +43,7 @@ async def browser(request: pytest.FixtureRequest) -> AsyncGenerator[zd.Browser,
     )
     browser_pid = browser._process_pid
     assert browser_pid is not None and browser_pid > 0
+    await browser.wait(0)
     yield browser
     if PAUSE_AFTER_TEST:
         logger.info(

tests/core/test_multiple_browsers.py

@@ -0,0 +1,57 @@
+import zendriver as zd
+
+
+# browser fixture not used here because this is a special case
+
+
+async def test_multiple_browsers_diff_userdata():
+    config = zd.Config(
+        headless=True,
+        browser_args=[
+            "--enable-features=UseOzonePlatform",
+            "--ozone-platform=wayland",
+            "--window-size=800,800",
+            "--wait-for-debugger-webui",
+        ],
+    )
+    config.browser_connection_timeout = 1
+    config.browser_connection_max_tries = 15
+
+    browser1 = await zd.start(config)
+    browser2 = await zd.start(config)
+    browser3 = await zd.start(config)
+
+    assert not browser1.config.uses_custom_data_dir
+    assert not browser2.config.uses_custom_data_dir
+    assert not browser3.config.uses_custom_data_dir
+
+    # make sure ports are unique
+    ports = {browser1.config.port, browser2.config.port, browser3.config.port}
+    assert len(ports) == 3
+
+    # make sure user data dirs are unique
+    udds = {
+        browser1.config.user_data_dir,
+        browser2.config.user_data_dir,
+        browser3.config.user_data_dir,
+    }
+    assert len(udds) == 3
+
+    page1 = await browser1.get("https://example.com/one")
+    await page1
+    assert page1.target
+    assert page1.target.title == "Example Domain"
+
+    page2 = await browser2.get("https://example.com/two")
+    await page2
+    assert page2.target
+    assert page2.target.title == "Example Domain"
+
+    page3 = await browser3.get("https://example.com/three")
+    await page3
+    assert page3.target
+    assert page3.target.title == "Example Domain"
+
+    await browser1.stop()
+    await browser2.stop()
+    await browser3.stop()

zendriver/core/browser.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import copy
 import http
 import http.cookiejar
 import json
@@ -101,7 +102,7 @@ async def browser_atexit() -> None:
 
         return instance
 
-    def __init__(self, config: Config, **kwargs):
+    def __init__(self, config: Config):
         """
         constructor. to create a instance, use :py:meth:`Browser.create(...)`
 
@@ -117,15 +118,17 @@ def __init__(self, config: Config, **kwargs):
                 )
             )
         # weakref.finalize(self, self._quit, self)
-        self.config = config
+
+        # each instance gets it's own copy so this class gets a copy that it can
+        # use to help manage the browser instance data (needed for multiple browsers)
+        self.config = copy.deepcopy(config)
 
         self.targets: List = []
         """current targets (all types)"""
         self.info: ContraDict | None = None
         self._target = None
         self._process = None
         self._process_pid = None
-        self._keep_user_data_dir = None
         self._is_updating = asyncio.Event()
         self.connection = None
         logger.debug("Session object initialized: %s" % vars(self))
@@ -254,7 +257,7 @@ async def get(
             raise RuntimeError("Browser not yet started. use await browser.start()")
 
         if new_tab or new_window:
-            # creat new target using the browser session
+            # create new target using the browser session
             target_id = await self.connection.send(
                 cdp.target.create_target(
                     url, new_window=new_window, enable_begin_frame_control=True
@@ -572,21 +575,29 @@ async def stop(self):
             logger.debug("closed the connection")
 
         if self._process:
-            self._process.terminate()
-            logger.debug("gracefully stopping browser process")
-            # wait 3 seconds for the browser to stop
-            for _ in range(12):
-                if self._process.returncode is not None:
-                    break
-                await asyncio.sleep(0.25)
-            else:
-                logger.debug("browser process did not stop. killing it")
-                self._process.kill()
-                logger.debug("killed browser process")
+            try:
+                self._process.terminate()
+                logger.debug("gracefully stopping browser process")
+                # wait 3 seconds for the browser to stop
+                for _ in range(12):
+                    if self._process.returncode is not None:
+                        break
+                    await asyncio.sleep(0.25)
+                else:
+                    logger.debug("browser process did not stop. killing it")
+                    self._process.kill()
+                    logger.debug("killed browser process")
+
+                await self._process.wait()
+
+            except ProcessLookupError:
+                # ignore this well known race condition because it only means that
+                # the process was not found while trying to terminate or kill it
+                pass
 
-            await self._process.wait()
             self._process = None
             self._process_pid = None
+
         await self._cleanup_temporary_profile()
 
     async def _cleanup_temporary_profile(self) -> None:

zendriver/core/config.py

@@ -53,7 +53,7 @@ def __init__(
 
         Instances of this class are usually not instantiated by end users.
 
-        :param user_data_dir: the data directory to use
+        :param user_data_dir: the data directory to use (must be unique if using multiple browsers)
         :param headless: set to True for headless mode
         :param browser_executable_path: specify browser executable, instead of using autodetect
         :param browser_args: forwarded to browser executable. eg : ["--some-chromeparam=somevalue", "some-other-param=someval"]
@@ -78,11 +78,12 @@ def __init__(
         if not browser_args:
             browser_args = []
 
-        if not user_data_dir:
-            self._user_data_dir = temp_profile_dir()
-            self._custom_data_dir = False
-        else:
-            self.user_data_dir = user_data_dir
+        # defer creating a temp user data dir until the browser requests it so
+        # config can be used/reused as a template for multiple browser instances
+        self._user_data_dir = None
+        self._custom_data_dir = False
+        if user_data_dir:
+            self.user_data_dir = str(user_data_dir)
 
         if not browser_executable_path:
             browser_executable_path = find_chrome_executable()
@@ -96,10 +97,11 @@ def __init__(
         self.port = port
         self.expert = expert
         self._extensions: list[PathLike] = []
+
         # when using posix-ish operating system and running as root
         # you must use no_sandbox = True, which in case is corrected here
         if is_posix and is_root() and sandbox:
-            logger.info("detected root usage, autoo disabling sandbox mode")
+            logger.info("detected root usage, auto disabling sandbox mode")
             self.sandbox = False
 
         self.autodiscover_targets = True
@@ -136,13 +138,27 @@ def browser_args(self):
         return sorted(self._default_browser_args + self._browser_args)
 
     @property
-    def user_data_dir(self):
+    def user_data_dir(self) -> str:
+        """
+        Get the user data dir or lazily create a new one if unset.
+
+        Returns:
+            str: User data directory (used for Chrome profile)
+        """
+        if not self._user_data_dir:
+            self._user_data_dir = temp_profile_dir()
+            self._custom_data_dir = False
+
         return self._user_data_dir
 
     @user_data_dir.setter
     def user_data_dir(self, path: PathLike):
-        self._user_data_dir = str(path)
-        self._custom_data_dir = True
+        if path:
+            self._user_data_dir = str(path)
+            self._custom_data_dir = True
+        else:
+            self._user_data_dir = None
+            self._custom_data_dir = False
 
     @property
     def uses_custom_data_dir(self) -> bool: