Self feedback Improvement (#3680)

* Improved `Self-Feedback`

* minor tweak

* Test: Updated `test_get_self_feedback.py`
pull/3764/head^2
AbTrax 2023-05-06 02:09:21 +10:00 committed by GitHub
parent f2bef76368
commit e12438de41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 12 deletions

View File

@ -161,7 +161,7 @@ class Agent:
)
logger.info(
"Enter 'y' to authorise command, 'y -N' to run N continuous commands, 's' to run self-feedback commands"
"Enter 'y' to authorise command, 'y -N' to run N continuous commands, 's' to run self-feedback commands or "
"'n' to exit program, or enter feedback for "
f"{self.ai_name}..."
)
@ -190,10 +190,8 @@ class Agent:
Fore.YELLOW,
"",
)
if self_feedback_resp[0].lower().strip() == cfg.authorise_key:
user_input = "GENERATE NEXT COMMAND JSON"
else:
user_input = self_feedback_resp
user_input = self_feedback_resp
command_name = "self_feedback"
break
elif console_input.lower().strip() == "":
logger.warn("Invalid input format.")
@ -244,6 +242,8 @@ class Agent:
)
elif command_name == "human_feedback":
result = f"Human feedback: {user_input}"
elif command_name == "self_feedback":
result = f"Self feedback: {user_input}"
else:
for plugin in cfg.plugins:
if not plugin.can_handle_pre_command():
@ -314,12 +314,11 @@ class Agent:
"""
ai_role = self.config.ai_role
feedback_prompt = f"Below is a message from an AI agent with the role of {ai_role}. Please review the provided Thought, Reasoning, Plan, and Criticism. If these elements accurately contribute to the successful execution of the assumed role, respond with the letter 'Y' followed by a space, and then explain why it is effective. If the provided information is not suitable for achieving the role's objectives, please provide one or more sentences addressing the issue and suggesting a resolution."
feedback_prompt = f"Below is a message from me, an AI Agent, assuming the role of {ai_role}. whilst keeping knowledge of my slight limitations as an AI Agent Please evaluate my thought process, reasoning, and plan, and provide a concise paragraph outlining potential improvements. Consider adding or removing ideas that do not align with my role and explaining why, prioritizing thoughts based on their significance, or simply refining my overall thought process."
reasoning = thoughts.get("reasoning", "")
plan = thoughts.get("plan", "")
thought = thoughts.get("thoughts", "")
criticism = thoughts.get("criticism", "")
feedback_thoughts = thought + reasoning + plan + criticism
feedback_thoughts = thought + reasoning + plan
return create_chat_completion(
[{"role": "user", "content": feedback_prompt + feedback_thoughts}],
llm_model,

View File

@ -9,12 +9,14 @@ def test_get_self_feedback(mocker):
"reasoning": "Sample reasoning.",
"plan": "Sample plan.",
"thoughts": "Sample thoughts.",
"criticism": "Sample criticism.",
}
# Define a fake response for the create_chat_completion function
fake_response = (
"Y The provided information is suitable for achieving the role's objectives."
"The AI Agent has demonstrated a reasonable thought process, but there is room for improvement. "
"For example, the reasoning could be elaborated to better justify the plan, and the plan itself "
"could be more detailed to ensure its effectiveness. In addition, the AI Agent should focus more "
"on its core role and prioritize thoughts that align with that role."
)
# Mock the create_chat_completion function
@ -36,5 +38,9 @@ def test_get_self_feedback(mocker):
"gpt-3.5-turbo",
)
# Check if the response is correct
assert feedback == fake_response
# Check if the response is a non-empty string
assert isinstance(feedback, str) and len(feedback) > 0
# Check if certain keywords from input thoughts are present in the feedback response
for keyword in ["reasoning", "plan", "thoughts"]:
assert keyword in feedback