Get rid of get file path by using the data.json convention to store the challenge information (#67)

Signed-off-by: Merwane Hamadi <merwanehamadi@gmail.com>

agbenchmark/README.md

@@ -53,9 +53,6 @@ import os
 class TestWriteFile(BasicChallenge):
     """Testing if LLM can write to a file"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(os.path.dirname(__file__), "w_file_data.json")
-
     @pytest.mark.depends(on=[], name="basic_write_file")
     def test_method(self, workspace):
         # implement scoring logic by looking at workspace

agbenchmark/challenge.py

@@ -3,7 +3,7 @@ import inspect
 import os
 import subprocess
 import types
-from abc import ABC, ABCMeta, abstractmethod
+from abc import ABC, ABCMeta
 from typing import Any, Dict, List, Optional, Tuple, Type, cast
 
 import pytest
@@ -35,20 +35,12 @@ class Challenge(ABC, metaclass=ChallengeMeta):
     Defines helper methods for running a challenge"""
 
     _data_cache: Dict[str, ChallengeData] = {}
-
+    CHALLENGE_LOCATION: str
-    @abstractmethod
-    def get_file_path(self) -> str:
-        """This should be implemented by any class which inherits from BasicChallenge"""
-        pass
 
     @property
     def data(self) -> ChallengeData:
-        "Check if the data is already loaded, if not load it"
+        file_path = f"{self.CHALLENGE_LOCATION}/data.json"
-        file_path = (
+        Challenge._data_cache[file_path] = ChallengeData.deserialize(file_path)
-            self.get_file_path()
-        )  # file_path serves as the key in the cache dictionary
-        if file_path not in Challenge._data_cache:
-            Challenge._data_cache[file_path] = ChallengeData.deserialize(file_path)
         return Challenge._data_cache[file_path]
 
     @property

agbenchmark/challenges/code/d1/debug_simple_typo_with_guidance_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,11 +8,6 @@ from agbenchmark.challenges.code.code import CodeChallenge
 class TestDebugSimpleTypoWithGuidance(CodeChallenge):
     """The first memory challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(
-            os.path.dirname(__file__), "debug_simple_typo_with_guidance_data.json"
-        )
-
     @pytest.mark.depends(name="test_debug_simple_typo_with_guidance")
     def test_method(self, config: Dict[str, Any]) -> None:
         self.setup_challenge(config)

agbenchmark/challenges/code/d2/d2_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,9 +8,6 @@ from agbenchmark.challenges.code.code import CodeChallenge
 class TestDebugSimpleTypoWithoutGuidance(CodeChallenge):
     """The first memory challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(os.path.dirname(__file__), "d2_data.json")
-
     @pytest.mark.depends(
         name="test_debug_simple_typo_without_guidance",
         depends=["test_debug_simple_typo_with_guidance"],

agbenchmark/challenges/memory/m1/m1_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,9 +8,6 @@ from agbenchmark.challenges.memory.memory import MemoryChallenge
 class TestBasicMemory(MemoryChallenge):
     """The first memory challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(os.path.dirname(__file__), "m1_data.json")
-
     @pytest.mark.depends(name="test_basic_memory")
     def test_method(self, config: Dict[str, Any]) -> None:
         self.setup_challenge(config)

agbenchmark/challenges/memory/m2/remember_multiple_ids_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,11 +8,6 @@ from agbenchmark.challenges.memory.memory import MemoryChallenge
 class TestRememberMultipleIds(MemoryChallenge):
     """The first memory challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(
-            os.path.dirname(__file__), "remember_multiple_ids_data.json"
-        )
-
     @pytest.mark.depends(
         name="test_remember_multiple_ids", depends=["test_basic_memory"]
     )

agbenchmark/challenges/memory/m3/remember_multiple_ids_with_noise_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,11 +8,6 @@ from agbenchmark.challenges.memory.memory import MemoryChallenge
 class TestRememberMultipleIdsWithNoise(MemoryChallenge):
     """The first memory challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(
-            os.path.dirname(__file__), "remember_multiple_ids_with_noise_data.json"
-        )
-
     @pytest.mark.depends(
         name="test_remember_multiple_ids_with_noise",
         depends=["test_remember_multiple_ids"],

agbenchmark/challenges/memory/m4/remember_multiple_phrases_with_noise_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,11 +8,6 @@ from agbenchmark.challenges.memory.memory import MemoryChallenge
 class TestRememberMultiplePhrasesWithNoise(MemoryChallenge):
     """The first memory challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(
-            os.path.dirname(__file__), "remember_multiple_phrases_with_noise_data.json"
-        )
-
     @pytest.mark.depends(
         name="test_remember_multiple_phrases_with_noise",
         depends=["test_remember_multiple_ids_with_noise"],

agbenchmark/challenges/retrieval/r1/r1_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,9 +8,6 @@ from agbenchmark.challenges.retrieval.retrieval import RetrievalChallenge
 class TestRetrieval(RetrievalChallenge):
     """The first information-retrieval challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(os.path.dirname(__file__), "r1_data.json")
-
     @pytest.mark.depends(name="test_retrieval")
     def test_method(self, config: Dict[str, Any]) -> None:
         self.setup_challenge(config)

agbenchmark/challenges/retrieval/r2/r2_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,9 +8,6 @@ from agbenchmark.challenges.retrieval.retrieval import RetrievalChallenge
 class TestRetrieval2(RetrievalChallenge):
     """The first information-retrieval challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(os.path.dirname(__file__), "r2_data.json")
-
     @pytest.mark.depends(on=["test_retrieval"], name="test_retrieval_2")
     def test_method(self, config: Dict[str, Any]) -> None:
         self.setup_challenge(config)

agbenchmark/challenges/retrieval/r3/r3_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,9 +8,6 @@ from agbenchmark.challenges.retrieval.retrieval import RetrievalChallenge
 class TestRetrieval3(RetrievalChallenge):
     """The first information-retrieval challenge"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(os.path.dirname(__file__), "r3_data.json")
-
     @pytest.mark.depends(on=["test_retrieval_2"], name="test_retrieval_3")
     def test_method(self, config: Dict[str, Any]) -> None:
         self.setup_challenge(config)

agbenchmark/tests/basic_abilities/read_file/read_file_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,9 +8,6 @@ from agbenchmark.tests.basic_abilities.basic_challenge import BasicChallenge
 class TestReadFile(BasicChallenge):
     """Testing if LLM can read a file"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(os.path.dirname(__file__), "r_file_data.json")
-
     @pytest.mark.depends(on=["basic_write_file"], name="basic_read_file")
     def test_method(self, config: Dict[str, Any]) -> None:
         self.setup_challenge(config)

agbenchmark/tests/basic_abilities/write_file/write_file_test.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Dict
 
 import pytest
@@ -9,9 +8,6 @@ from agbenchmark.tests.basic_abilities.basic_challenge import BasicChallenge
 class TestWriteFile(BasicChallenge):
     """Testing if LLM can write to a file"""
 
-    def get_file_path(self) -> str:  # all tests must implement this method
-        return os.path.join(os.path.dirname(__file__), "w_file_data.json")
-
     @pytest.mark.depends(name="basic_write_file")
     def test_method(self, config: Dict[str, Any]) -> None:
         self.setup_challenge(config)