Significant-Gravitas
/
AutoGPT
mirror of https://github.com/Significant-Gravitas/AutoGPT.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #275 from master312/master 

Added fake user-agent headers to browser request.
pull/358/head
Toran Bruce Richards 2023-04-06 09:53:59 +01:00 committed by GitHub
parent 73b81210df 62854afe68
commit ba615fb883
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/browse.py
View File

@ -6,7 +6,7 @@ from llm_utils import create_chat_completion
cfg = Config()
def scrape_text(url):
response = requests.get(url)
response = requests.get(url, headers=cfg.user_agent_header)
# Check if the response contains an HTTP error
if response.status_code >= 400:
@ -40,7 +40,7 @@ def format_hyperlinks(hyperlinks):
def scrape_links(url):
response = requests.get(url)
response = requests.get(url, headers=cfg.user_agent_header)
# Check if the response contains an HTTP error
if response.status_code >= 400:

5
scripts/config.py
View File

@ -39,6 +39,11 @@ class Config(metaclass=Singleton):
self.google_api_key = os.getenv("GOOGLE_API_KEY")
self.custom_search_engine_id = os.getenv("CUSTOM_SEARCH_ENGINE_ID")
# User agent headers to use when browsing web
# Some websites might just completely deny request with an error code if no user agent was found.
self.user_agent_header = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"}
# Initialize the OpenAI API client
openai.api_key = self.openai_api_key