From 14c49fa7ead4ab83dffcc759e726c6208f2d3478 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Mon, 31 Jul 2023 12:54:27 +0100 Subject: [PATCH] handling helicone errors --- agbenchmark/reports/reports.py | 2 +- .../{ => utils}/get_data_from_helicone.py | 34 +++++++++++++------ .../Auto-GPT/folder3_07-31-11-25/report.json | 14 -------- .../BabyAGI/folder4_07-31-11-28/report.json | 13 ------- .../beebot/folder4_07-31-11-25/report.json | 13 ------- .../folder3_07-31-11-24/report.json | 14 -------- .../mini-agi/folder5_07-31-11-25/report.json | 13 ------- .../folder4_07-31-11-24/report.json | 14 -------- 8 files changed, 24 insertions(+), 93 deletions(-) rename agbenchmark/{ => utils}/get_data_from_helicone.py (58%) delete mode 100644 reports/Auto-GPT/folder3_07-31-11-25/report.json delete mode 100644 reports/BabyAGI/folder4_07-31-11-28/report.json delete mode 100644 reports/beebot/folder4_07-31-11-25/report.json delete mode 100644 reports/gpt-engineer/folder3_07-31-11-24/report.json delete mode 100644 reports/mini-agi/folder5_07-31-11-25/report.json delete mode 100644 reports/smol-developer/folder4_07-31-11-24/report.json diff --git a/agbenchmark/reports/reports.py b/agbenchmark/reports/reports.py index 25abc1f78..9abe5ea37 100644 --- a/agbenchmark/reports/reports.py +++ b/agbenchmark/reports/reports.py @@ -7,7 +7,7 @@ from typing import Any, Callable import pytest from agbenchmark.agent_interface import MOCK_FLAG -from agbenchmark.get_data_from_helicone import get_data_from_helicone +from agbenchmark.utils.get_data_from_helicone import get_data_from_helicone from agbenchmark.reports.ReportManager import ReportManager from agbenchmark.start_benchmark import CONFIG_PATH, REGRESSION_TESTS_PATH, REPORTS_PATH from agbenchmark.utils.data_types import DIFFICULTY_MAP, DifficultyLevel, SuiteConfig diff --git a/agbenchmark/get_data_from_helicone.py b/agbenchmark/utils/get_data_from_helicone.py similarity index 58% rename from agbenchmark/get_data_from_helicone.py rename to agbenchmark/utils/get_data_from_helicone.py index 1a58565ae..db823a588 100644 --- a/agbenchmark/get_data_from_helicone.py +++ b/agbenchmark/utils/get_data_from_helicone.py @@ -15,9 +15,9 @@ def get_data_from_helicone(challenge: str) -> Optional[float]: # Define the query, variables, and operation name query = """ -query ExampleQuery { - aggregatedHeliconeRequest { - cost +query ExampleQuery($properties: [PropertyFilter!]){ + aggregatedHeliconeRequest(properties: $properties) { + costUSD } } """ @@ -43,11 +43,23 @@ query ExampleQuery { operation_name = "ExampleQuery" # Make the request - response = requests.post( - url, - headers=headers, - json={"query": query, "variables": variables, "operationName": operation_name}, - ) - data = response.json() - - return data.get("data", {}).get("aggregatedHeliconeRequest", {}).get("cost", None) + try: + response = requests.post( + url, + headers=headers, + json={ + "query": query, + "variables": variables, + "operationName": operation_name, + }, + ) + response.raise_for_status() # Raises a HTTPError if the response was an unsuccessful status code + data = response.json() + except requests.HTTPError as http_err: + print(f"HTTP error occurred: {http_err}") + except Exception as err: + print(f"Other error occurred: {err}") + else: + return ( + data.get("data", {}).get("aggregatedHeliconeRequest", {}).get("cost", None) + ) diff --git a/reports/Auto-GPT/folder3_07-31-11-25/report.json b/reports/Auto-GPT/folder3_07-31-11-25/report.json deleted file mode 100644 index fc34b420b..000000000 --- a/reports/Auto-GPT/folder3_07-31-11-25/report.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "command": "agbenchmark start", - "completion_time": "2023-07-31-11:26", - "benchmark_start_time": "2023-07-31-11:25", - "metrics": { - "run_time": "42.59 seconds", - "highest_difficulty": "No successful tests" - }, - "tests": {}, - "config": { - "workspace": "auto_gpt_workspace", - "entry_path": "agbenchmark.benchmarks" - } -} \ No newline at end of file diff --git a/reports/BabyAGI/folder4_07-31-11-28/report.json b/reports/BabyAGI/folder4_07-31-11-28/report.json deleted file mode 100644 index 5d58bbb32..000000000 --- a/reports/BabyAGI/folder4_07-31-11-28/report.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "command": "agbenchmark start", - "completion_time": "2023-07-31-11:29", - "benchmark_start_time": "2023-07-31-11:28", - "metrics": { - "run_time": "61.6 seconds", - "highest_difficulty": "No successful tests" - }, - "tests": {}, - "config": { - "workspace": "babycoder/playground" - } -} \ No newline at end of file diff --git a/reports/beebot/folder4_07-31-11-25/report.json b/reports/beebot/folder4_07-31-11-25/report.json deleted file mode 100644 index 723b530c9..000000000 --- a/reports/beebot/folder4_07-31-11-25/report.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "command": "agbenchmark start", - "completion_time": "2023-07-31-11:26", - "benchmark_start_time": "2023-07-31-11:25", - "metrics": { - "run_time": "61.01 seconds", - "highest_difficulty": "No successful tests" - }, - "tests": {}, - "config": { - "workspace": "workspace" - } -} \ No newline at end of file diff --git a/reports/gpt-engineer/folder3_07-31-11-24/report.json b/reports/gpt-engineer/folder3_07-31-11-24/report.json deleted file mode 100644 index f7053b496..000000000 --- a/reports/gpt-engineer/folder3_07-31-11-24/report.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "command": "agbenchmark start", - "completion_time": "2023-07-31-11:25", - "benchmark_start_time": "2023-07-31-11:24", - "metrics": { - "run_time": "60.99 seconds", - "highest_difficulty": "No successful tests" - }, - "tests": {}, - "config": { - "workspace": "projects/my-new-project/workspace", - "entry_path": "agbenchmark.benchmarks" - } -} \ No newline at end of file diff --git a/reports/mini-agi/folder5_07-31-11-25/report.json b/reports/mini-agi/folder5_07-31-11-25/report.json deleted file mode 100644 index d77dfda0a..000000000 --- a/reports/mini-agi/folder5_07-31-11-25/report.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "command": "agbenchmark start", - "completion_time": "2023-07-31-11:25", - "benchmark_start_time": "2023-07-31-11:25", - "metrics": { - "run_time": "18.86 seconds", - "highest_difficulty": "No successful tests" - }, - "tests": {}, - "config": { - "workspace": "${os.path.join(Path.home(), 'miniagi')}" - } -} \ No newline at end of file diff --git a/reports/smol-developer/folder4_07-31-11-24/report.json b/reports/smol-developer/folder4_07-31-11-24/report.json deleted file mode 100644 index c6a1bacc8..000000000 --- a/reports/smol-developer/folder4_07-31-11-24/report.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "command": "agbenchmark start", - "completion_time": "2023-07-31-11:24", - "benchmark_start_time": "2023-07-31-11:24", - "metrics": { - "run_time": "7.39 seconds", - "highest_difficulty": "No successful tests" - }, - "tests": {}, - "config": { - "workspace": "generated", - "entry_path": "agbenchmark.benchmarks" - } -} \ No newline at end of file