From 7e21123a5d11ac7ffe30f2434bea47d76a4c1f48 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 12 Apr 2023 14:50:45 -0700 Subject: [PATCH 1/4] Add README.md with instructions on running tests --- tests/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..54cc11170 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,17 @@ +For now there aren't many tests. +So just run: + +``` +python tests/unit/json_tests.py +python tests/integration/memory_tests.py +``` + +paid test: +``` +python tests/integration/test_commands.py +``` +This test costs 0.004$ per run with GPT-3.5. We will setup a pipeline in github action to allow people to run these tests for free. + +The pipeline will be be triggered for now. + +TODO: when we setup pytest, replace lines above by: pytest commands From 36091853e0a649e2b268a21f3fa6d43cf837eb23 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 12 Apr 2023 14:50:45 -0700 Subject: [PATCH 2/4] Add integration test for write_file command --- tests/integration/test_commands.py | 68 ++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/integration/test_commands.py diff --git a/tests/integration/test_commands.py b/tests/integration/test_commands.py new file mode 100644 index 000000000..0a053eef3 --- /dev/null +++ b/tests/integration/test_commands.py @@ -0,0 +1,68 @@ +import contextlib +import os +import subprocess +import sys +import time +from pathlib import Path +import unittest +import platform + +# Import necessary modules for testing, file operations, and handling system paths. + +root_path = Path(__file__).resolve().parent.parent.parent +sys.path.append(str(root_path / 'scripts')) +from file_operations import read_file +from file_operations import delete_file + +env_vars = { + 'MEMORY_BACKEND': 'no_memory', + 'TEMPERATURE': "0" +} + +# Set environment variables for the test. + +class TestCommands(unittest.TestCase): + def test_write_file(self): + # Test case to check if the write_file command can successfully write 'Hello World' to a file named 'hello_world.txt'. + + # Read the current ai_settings.yaml file and store its content. + with open('ai_settings.yaml', 'r') as f: + ai_settings = f.read() + + try: + with contextlib.suppress(Exception): + # Clean up any existing 'hello_world.txt' file before testing. + delete_file('hello_world.txt') + + # Remove ai_settings.yaml file to avoid continuing from the previous session. + os.remove('ai_settings.yaml') + + # Prepare input data for the test. + input_data = '''write_file-GPT +an AI designed to use the write_file command to write 'Hello World' into a file named "hello_world.txt" and then use the task_complete command to complete the task. +Use the write_file command to write 'Hello World' into a file named "hello_world.txt". +Use the task_complete command to complete the task. +Do not use any other commands. + +y -5 +EOF''' + script_path = root_path / 'scripts' / 'main.py' + command = f'{sys.executable} {script_path}' + + # Execute the script with the input data. + process = subprocess.Popen(command, stdin=subprocess.PIPE, shell=True, env={**os.environ, **env_vars}) + process.communicate(input_data.encode()) + + # Read the content of the 'hello_world.txt' file created during the test. + content = read_file('hello_world.txt') + finally: + # Restore the original ai_settings.yaml file. + with open('ai_settings.yaml', 'w') as f: + f.write(ai_settings) + + # Check if the content of the 'hello_world.txt' file is equal to 'Hello World'. + self.assertEqual(content, 'Hello World', f"Expected 'Hello World', got {content}") + +# Run the test case. +if __name__ == '__main__': + unittest.main() From ca5a52f48ab10f2ec3ab698c222c796fbde143b0 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 12 Apr 2023 14:50:45 -0700 Subject: [PATCH 3/4] Update sys.path to use pathlib in json_tests.py --- tests/integration/test_commands.py | 19 ++++++++++++------- tests/unit/json_tests.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_commands.py b/tests/integration/test_commands.py index 0a053eef3..26413437f 100644 --- a/tests/integration/test_commands.py +++ b/tests/integration/test_commands.py @@ -19,15 +19,18 @@ env_vars = { 'TEMPERATURE': "0" } -# Set environment variables for the test. class TestCommands(unittest.TestCase): + def test_write_file(self): - # Test case to check if the write_file command can successfully write 'Hello World' to a file named 'hello_world.txt'. + # Test case to check if the write_file command can successfully write 'Hello World' to a file + # named 'hello_world.txt'. # Read the current ai_settings.yaml file and store its content. - with open('ai_settings.yaml', 'r') as f: - ai_settings = f.read() + ai_settings = None + with contextlib.suppress(Exception): + with open('ai_settings.yaml', 'r') as f: + ai_settings = f.read() try: with contextlib.suppress(Exception): @@ -56,13 +59,15 @@ EOF''' # Read the content of the 'hello_world.txt' file created during the test. content = read_file('hello_world.txt') finally: - # Restore the original ai_settings.yaml file. - with open('ai_settings.yaml', 'w') as f: - f.write(ai_settings) + if ai_settings: + # Restore the original ai_settings.yaml file. + with open('ai_settings.yaml', 'w') as f: + f.write(ai_settings) # Check if the content of the 'hello_world.txt' file is equal to 'Hello World'. self.assertEqual(content, 'Hello World', f"Expected 'Hello World', got {content}") + # Run the test case. if __name__ == '__main__': unittest.main() diff --git a/tests/unit/json_tests.py b/tests/unit/json_tests.py index 561b8a38a..4899f265f 100644 --- a/tests/unit/json_tests.py +++ b/tests/unit/json_tests.py @@ -1,6 +1,16 @@ import unittest +<<<<<<< HEAD from autogpt.json_parser import fix_and_parse_json +======= +import os +import sys +from pathlib import Path + +# Probably a better way: +sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'scripts')) +from json_parser import fix_and_parse_json +>>>>>>> 27e0703d (Update sys.path to use pathlib in json_tests.py) class TestParseJson(unittest.TestCase): From dc4094b2646ad4330bef23331f75fd15f4cdb9a0 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Fri, 14 Apr 2023 16:47:14 -0700 Subject: [PATCH 4/4] added smoke test --- .gitignore | 5 +++- tests/README.md | 17 -------------- .../test_commands.py => smoke_test.py} | 23 ++++--------------- tests/unit/json_tests.py | 10 -------- 4 files changed, 9 insertions(+), 46 deletions(-) delete mode 100644 tests/README.md rename tests/{integration/test_commands.py => smoke_test.py} (73%) diff --git a/.gitignore b/.gitignore index 9874d425c..91d932d8c 100644 --- a/.gitignore +++ b/.gitignore @@ -151,4 +151,7 @@ dmypy.json # Pyre type checker .pyre/ llama-* -vicuna-* \ No newline at end of file +vicuna-* + +# mac +.DS_Store diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 54cc11170..000000000 --- a/tests/README.md +++ /dev/null @@ -1,17 +0,0 @@ -For now there aren't many tests. -So just run: - -``` -python tests/unit/json_tests.py -python tests/integration/memory_tests.py -``` - -paid test: -``` -python tests/integration/test_commands.py -``` -This test costs 0.004$ per run with GPT-3.5. We will setup a pipeline in github action to allow people to run these tests for free. - -The pipeline will be be triggered for now. - -TODO: when we setup pytest, replace lines above by: pytest commands diff --git a/tests/integration/test_commands.py b/tests/smoke_test.py similarity index 73% rename from tests/integration/test_commands.py rename to tests/smoke_test.py index 26413437f..1f0f24f90 100644 --- a/tests/integration/test_commands.py +++ b/tests/smoke_test.py @@ -1,18 +1,9 @@ -import contextlib import os import subprocess import sys -import time -from pathlib import Path import unittest -import platform -# Import necessary modules for testing, file operations, and handling system paths. - -root_path = Path(__file__).resolve().parent.parent.parent -sys.path.append(str(root_path / 'scripts')) -from file_operations import read_file -from file_operations import delete_file +from autogpt.file_operations import delete_file, read_file env_vars = { 'MEMORY_BACKEND': 'no_memory', @@ -28,18 +19,15 @@ class TestCommands(unittest.TestCase): # Read the current ai_settings.yaml file and store its content. ai_settings = None - with contextlib.suppress(Exception): + if os.path.exists('ai_settings.yaml'): with open('ai_settings.yaml', 'r') as f: ai_settings = f.read() + os.remove('ai_settings.yaml') try: - with contextlib.suppress(Exception): + if os.path.exists('hello_world.txt'): # Clean up any existing 'hello_world.txt' file before testing. delete_file('hello_world.txt') - - # Remove ai_settings.yaml file to avoid continuing from the previous session. - os.remove('ai_settings.yaml') - # Prepare input data for the test. input_data = '''write_file-GPT an AI designed to use the write_file command to write 'Hello World' into a file named "hello_world.txt" and then use the task_complete command to complete the task. @@ -49,8 +37,7 @@ Do not use any other commands. y -5 EOF''' - script_path = root_path / 'scripts' / 'main.py' - command = f'{sys.executable} {script_path}' + command = f'{sys.executable} -m autogpt' # Execute the script with the input data. process = subprocess.Popen(command, stdin=subprocess.PIPE, shell=True, env={**os.environ, **env_vars}) diff --git a/tests/unit/json_tests.py b/tests/unit/json_tests.py index 4899f265f..561b8a38a 100644 --- a/tests/unit/json_tests.py +++ b/tests/unit/json_tests.py @@ -1,16 +1,6 @@ import unittest -<<<<<<< HEAD from autogpt.json_parser import fix_and_parse_json -======= -import os -import sys -from pathlib import Path - -# Probably a better way: -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'scripts')) -from json_parser import fix_and_parse_json ->>>>>>> 27e0703d (Update sys.path to use pathlib in json_tests.py) class TestParseJson(unittest.TestCase):