Self feedback Improvement (#3680)
* Improved `Self-Feedback` * minor tweak * Test: Updated `test_get_self_feedback.py`pull/3764/head^2
parent
f2bef76368
commit
e12438de41
|
@ -161,7 +161,7 @@ class Agent:
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Enter 'y' to authorise command, 'y -N' to run N continuous commands, 's' to run self-feedback commands"
|
"Enter 'y' to authorise command, 'y -N' to run N continuous commands, 's' to run self-feedback commands or "
|
||||||
"'n' to exit program, or enter feedback for "
|
"'n' to exit program, or enter feedback for "
|
||||||
f"{self.ai_name}..."
|
f"{self.ai_name}..."
|
||||||
)
|
)
|
||||||
|
@ -190,10 +190,8 @@ class Agent:
|
||||||
Fore.YELLOW,
|
Fore.YELLOW,
|
||||||
"",
|
"",
|
||||||
)
|
)
|
||||||
if self_feedback_resp[0].lower().strip() == cfg.authorise_key:
|
user_input = self_feedback_resp
|
||||||
user_input = "GENERATE NEXT COMMAND JSON"
|
command_name = "self_feedback"
|
||||||
else:
|
|
||||||
user_input = self_feedback_resp
|
|
||||||
break
|
break
|
||||||
elif console_input.lower().strip() == "":
|
elif console_input.lower().strip() == "":
|
||||||
logger.warn("Invalid input format.")
|
logger.warn("Invalid input format.")
|
||||||
|
@ -244,6 +242,8 @@ class Agent:
|
||||||
)
|
)
|
||||||
elif command_name == "human_feedback":
|
elif command_name == "human_feedback":
|
||||||
result = f"Human feedback: {user_input}"
|
result = f"Human feedback: {user_input}"
|
||||||
|
elif command_name == "self_feedback":
|
||||||
|
result = f"Self feedback: {user_input}"
|
||||||
else:
|
else:
|
||||||
for plugin in cfg.plugins:
|
for plugin in cfg.plugins:
|
||||||
if not plugin.can_handle_pre_command():
|
if not plugin.can_handle_pre_command():
|
||||||
|
@ -314,12 +314,11 @@ class Agent:
|
||||||
"""
|
"""
|
||||||
ai_role = self.config.ai_role
|
ai_role = self.config.ai_role
|
||||||
|
|
||||||
feedback_prompt = f"Below is a message from an AI agent with the role of {ai_role}. Please review the provided Thought, Reasoning, Plan, and Criticism. If these elements accurately contribute to the successful execution of the assumed role, respond with the letter 'Y' followed by a space, and then explain why it is effective. If the provided information is not suitable for achieving the role's objectives, please provide one or more sentences addressing the issue and suggesting a resolution."
|
feedback_prompt = f"Below is a message from me, an AI Agent, assuming the role of {ai_role}. whilst keeping knowledge of my slight limitations as an AI Agent Please evaluate my thought process, reasoning, and plan, and provide a concise paragraph outlining potential improvements. Consider adding or removing ideas that do not align with my role and explaining why, prioritizing thoughts based on their significance, or simply refining my overall thought process."
|
||||||
reasoning = thoughts.get("reasoning", "")
|
reasoning = thoughts.get("reasoning", "")
|
||||||
plan = thoughts.get("plan", "")
|
plan = thoughts.get("plan", "")
|
||||||
thought = thoughts.get("thoughts", "")
|
thought = thoughts.get("thoughts", "")
|
||||||
criticism = thoughts.get("criticism", "")
|
feedback_thoughts = thought + reasoning + plan
|
||||||
feedback_thoughts = thought + reasoning + plan + criticism
|
|
||||||
return create_chat_completion(
|
return create_chat_completion(
|
||||||
[{"role": "user", "content": feedback_prompt + feedback_thoughts}],
|
[{"role": "user", "content": feedback_prompt + feedback_thoughts}],
|
||||||
llm_model,
|
llm_model,
|
||||||
|
|
|
@ -9,12 +9,14 @@ def test_get_self_feedback(mocker):
|
||||||
"reasoning": "Sample reasoning.",
|
"reasoning": "Sample reasoning.",
|
||||||
"plan": "Sample plan.",
|
"plan": "Sample plan.",
|
||||||
"thoughts": "Sample thoughts.",
|
"thoughts": "Sample thoughts.",
|
||||||
"criticism": "Sample criticism.",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Define a fake response for the create_chat_completion function
|
# Define a fake response for the create_chat_completion function
|
||||||
fake_response = (
|
fake_response = (
|
||||||
"Y The provided information is suitable for achieving the role's objectives."
|
"The AI Agent has demonstrated a reasonable thought process, but there is room for improvement. "
|
||||||
|
"For example, the reasoning could be elaborated to better justify the plan, and the plan itself "
|
||||||
|
"could be more detailed to ensure its effectiveness. In addition, the AI Agent should focus more "
|
||||||
|
"on its core role and prioritize thoughts that align with that role."
|
||||||
)
|
)
|
||||||
|
|
||||||
# Mock the create_chat_completion function
|
# Mock the create_chat_completion function
|
||||||
|
@ -36,5 +38,9 @@ def test_get_self_feedback(mocker):
|
||||||
"gpt-3.5-turbo",
|
"gpt-3.5-turbo",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Check if the response is correct
|
# Check if the response is a non-empty string
|
||||||
assert feedback == fake_response
|
assert isinstance(feedback, str) and len(feedback) > 0
|
||||||
|
|
||||||
|
# Check if certain keywords from input thoughts are present in the feedback response
|
||||||
|
for keyword in ["reasoning", "plan", "thoughts"]:
|
||||||
|
assert keyword in feedback
|
||||||
|
|
Loading…
Reference in New Issue