Merge pull request #1572 from cryptidv/browser-agnostic

Make browsing with Selenium Browser Agnostic

.env.template

@@ -11,6 +11,9 @@ BROWSE_SUMMARY_MAX_TOKEN=300
 # USER_AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"
 # AI_SETTINGS_FILE - Specifies which AI Settings file to use (defaults to ai_settings.yaml)
 AI_SETTINGS_FILE=ai_settings.yaml
+# USE_WEB_BROWSER - Sets the web-browser drivers to use with selenium (defaults to chrome).
+# Note: set this to either 'chrome', 'firefox', or 'safari' depending on your current browser
+# USE_WEB_BROWSER=chrome
 
 ################################################################################
 ### LLM PROVIDER

autogpt/__main__.py

@@ -1,5 +1,6 @@
 """Main script for the autogpt package."""
 import logging
+from colorama import Fore
 from autogpt.agent.agent import Agent
 from autogpt.args import parse_arguments
 
@@ -33,7 +34,8 @@ def main() -> None:
     # Initialize memory and make sure it is empty.
     # this is particularly important for indexing and referencing pinecone memory
     memory = get_memory(cfg, init=True)
-    print(f"Using memory of type: {memory.__class__.__name__}")
+    logger.typewriter_log(f"Using memory of type:", Fore.GREEN, f"{memory.__class__.__name__}")
+    logger.typewriter_log(f"Using Browser:", Fore.GREEN, cfg.selenium_web_browser)
     agent = Agent(
         ai_name=ai_name,
         memory=memory,

autogpt/args.py

@@ -50,6 +50,12 @@ def parse_arguments() -> None:
         action="store_true",
         help="Skips the re-prompting messages at the beginning of the script",
     )
+    parser.add_argument(
+        "--use-browser",
+        "-b",
+        dest="browser_name",
+        help="Specifies which web-browser to use when using selenium to scrape the web."
+    )
     parser.add_argument(
         "--ai-settings",
         "-C",
@@ -126,3 +132,6 @@ def parse_arguments() -> None:
         logger.typewriter_log("Using AI Settings File:", Fore.GREEN, file)
         CFG.ai_settings_file = file
         CFG.skip_reprompt = True
+
+    if args.browser_name:
+        CFG.selenium_web_browser = args.browser_name

autogpt/commands/web_selenium.py

@@ -7,7 +7,10 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from webdriver_manager.chrome import ChromeDriverManager
-from selenium.webdriver.chrome.options import Options
+from webdriver_manager.firefox import GeckoDriverManager
+from selenium.webdriver.chrome.options import Options as ChromeOptions
+from selenium.webdriver.firefox.options import Options as FirefoxOptions
+from selenium.webdriver.safari.options import Options as SafariOptions
 import logging
 from pathlib import Path
 from autogpt.config import Config
@@ -49,14 +52,25 @@ def scrape_text_with_selenium(url: str) -> tuple[WebDriver, str]:
     """
     logging.getLogger("selenium").setLevel(logging.CRITICAL)
 
-    options = Options()
+    options_available = {'chrome': ChromeOptions, 'safari': SafariOptions, 'firefox': FirefoxOptions}
+
+    options = options_available[CFG.selenium_web_browser]()
     options.add_argument(
-        "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
-        " (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36"
-    )
-    driver = webdriver.Chrome(
-        executable_path=ChromeDriverManager().install(), options=options
+        "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36"
     )
+
+    if CFG.selenium_web_browser == "firefox":
+        driver = webdriver.Firefox(
+            executable_path=GeckoDriverManager().install(), options=options
+        )
+    elif CFG.selenium_web_browser == "safari":
+        # Requires a bit more setup on the users end
+        # See https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari
+        driver = webdriver.Safari(options=options)
+    else:
+        driver = webdriver.Chrome(
+            executable_path=ChromeDriverManager().install(), options=options
+        )
     driver.get(url)
 
     WebDriverWait(driver, 10).until(

autogpt/config/config.py

@@ -25,6 +25,7 @@ class Config(metaclass=Singleton):
         self.speak_mode = False
         self.skip_reprompt = False
 
+        self.selenium_web_browser = os.getenv("USE_WEB_BROWSER", "chrome")
         self.ai_settings_file = os.getenv("AI_SETTINGS_FILE", "ai_settings.yaml")
         self.fast_llm_model = os.getenv("FAST_LLM_MODEL", "gpt-3.5-turbo")
         self.smart_llm_model = os.getenv("SMART_LLM_MODEL", "gpt-4")