handling helicone errors

agbenchmark/reports/reports.py

@@ -7,7 +7,7 @@ from typing import Any, Callable
 import pytest
 
 from agbenchmark.agent_interface import MOCK_FLAG
-from agbenchmark.get_data_from_helicone import get_data_from_helicone
+from agbenchmark.utils.get_data_from_helicone import get_data_from_helicone
 from agbenchmark.reports.ReportManager import ReportManager
 from agbenchmark.start_benchmark import CONFIG_PATH, REGRESSION_TESTS_PATH, REPORTS_PATH
 from agbenchmark.utils.data_types import DIFFICULTY_MAP, DifficultyLevel, SuiteConfig

agbenchmark/utils/get_data_from_helicone.py

@@ -15,9 +15,9 @@ def get_data_from_helicone(challenge: str) -> Optional[float]:
 
     # Define the query, variables, and operation name
     query = """
-query ExampleQuery {
+query ExampleQuery($properties: [PropertyFilter!]){
-  aggregatedHeliconeRequest {
+  aggregatedHeliconeRequest(properties: $properties) {
-    cost
+    costUSD
   }
 }
 """
@@ -43,11 +43,23 @@ query ExampleQuery {
     operation_name = "ExampleQuery"
 
     # Make the request
-    response = requests.post(
+    try:
-        url,
+        response = requests.post(
-        headers=headers,
+            url,
-        json={"query": query, "variables": variables, "operationName": operation_name},
+            headers=headers,
-    )
+            json={
-    data = response.json()
+                "query": query,
-
+                "variables": variables,
-    return data.get("data", {}).get("aggregatedHeliconeRequest", {}).get("cost", None)
+                "operationName": operation_name,
+            },
+        )
+        response.raise_for_status()  # Raises a HTTPError if the response was an unsuccessful status code
+        data = response.json()
+    except requests.HTTPError as http_err:
+        print(f"HTTP error occurred: {http_err}")
+    except Exception as err:
+        print(f"Other error occurred: {err}")
+    else:
+        return (
+            data.get("data", {}).get("aggregatedHeliconeRequest", {}).get("cost", None)
+        )

reports/Auto-GPT/folder3_07-31-11-25/report.json

@@ -1,14 +0,0 @@
-{
-    "command": "agbenchmark start",
-    "completion_time": "2023-07-31-11:26",
-    "benchmark_start_time": "2023-07-31-11:25",
-    "metrics": {
-        "run_time": "42.59 seconds",
-        "highest_difficulty": "No successful tests"
-    },
-    "tests": {},
-    "config": {
-        "workspace": "auto_gpt_workspace",
-        "entry_path": "agbenchmark.benchmarks"
-    }
-}

reports/BabyAGI/folder4_07-31-11-28/report.json

@@ -1,13 +0,0 @@
-{
-    "command": "agbenchmark start",
-    "completion_time": "2023-07-31-11:29",
-    "benchmark_start_time": "2023-07-31-11:28",
-    "metrics": {
-        "run_time": "61.6 seconds",
-        "highest_difficulty": "No successful tests"
-    },
-    "tests": {},
-    "config": {
-        "workspace": "babycoder/playground"
-    }
-}

reports/beebot/folder4_07-31-11-25/report.json

@@ -1,13 +0,0 @@
-{
-    "command": "agbenchmark start",
-    "completion_time": "2023-07-31-11:26",
-    "benchmark_start_time": "2023-07-31-11:25",
-    "metrics": {
-        "run_time": "61.01 seconds",
-        "highest_difficulty": "No successful tests"
-    },
-    "tests": {},
-    "config": {
-        "workspace": "workspace"
-    }
-}

reports/gpt-engineer/folder3_07-31-11-24/report.json

@@ -1,14 +0,0 @@
-{
-    "command": "agbenchmark start",
-    "completion_time": "2023-07-31-11:25",
-    "benchmark_start_time": "2023-07-31-11:24",
-    "metrics": {
-        "run_time": "60.99 seconds",
-        "highest_difficulty": "No successful tests"
-    },
-    "tests": {},
-    "config": {
-        "workspace": "projects/my-new-project/workspace",
-        "entry_path": "agbenchmark.benchmarks"
-    }
-}

reports/mini-agi/folder5_07-31-11-25/report.json

@@ -1,13 +0,0 @@
-{
-    "command": "agbenchmark start",
-    "completion_time": "2023-07-31-11:25",
-    "benchmark_start_time": "2023-07-31-11:25",
-    "metrics": {
-        "run_time": "18.86 seconds",
-        "highest_difficulty": "No successful tests"
-    },
-    "tests": {},
-    "config": {
-        "workspace": "${os.path.join(Path.home(), 'miniagi')}"
-    }
-}

reports/smol-developer/folder4_07-31-11-24/report.json

@@ -1,14 +0,0 @@
-{
-    "command": "agbenchmark start",
-    "completion_time": "2023-07-31-11:24",
-    "benchmark_start_time": "2023-07-31-11:24",
-    "metrics": {
-        "run_time": "7.39 seconds",
-        "highest_difficulty": "No successful tests"
-    },
-    "tests": {},
-    "config": {
-        "workspace": "generated",
-        "entry_path": "agbenchmark.benchmarks"
-    }
-}