From 3525a4b6dbed72408f72dc2c595570713492ddb4 Mon Sep 17 00:00:00 2001 From: merwanehamadi Date: Thu, 15 Jun 2023 09:09:59 -0700 Subject: [PATCH] Count tokens with tiktoken (#4704) * Update OpenAI model info and remove duplicate modelsinfo.py (#4700) * Update OpenAI model info and remove duplicate modelsinfo.py * Fix max_tokens for gpt-4-0613 Signed-off-by: Merwane Hamadi Co-authored-by: Merwane Hamadi * Update count_message_tokens to support new OpenAI models Signed-off-by: Merwane Hamadi Co-authored-by: Merwane Hamadi * Fix error message in count_message_tokens --------- Signed-off-by: Merwane Hamadi Co-authored-by: Erik Peterson Co-authored-by: Reinier van der Leer --- .gitignore | 1 + autogpt/llm/api_manager.py | 4 +++- autogpt/llm/utils/token_counter.py | 26 +++++++++++--------------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 307a67237..29a0285a8 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ build/ develop-eggs/ dist/ plugins/ +plugins_config.yaml downloads/ eggs/ .eggs/ diff --git a/autogpt/llm/api_manager.py b/autogpt/llm/api_manager.py index 7a384562c..454b4f22b 100644 --- a/autogpt/llm/api_manager.py +++ b/autogpt/llm/api_manager.py @@ -89,7 +89,9 @@ class ApiManager(metaclass=Singleton): self.total_completion_tokens += completion_tokens self.total_cost += prompt_tokens * model_info.prompt_token_cost / 1000 if issubclass(type(model_info), CompletionModelInfo): - self.total_cost += completion_tokens * model_info.completion_token_cost / 1000 + self.total_cost += ( + completion_tokens * model_info.completion_token_cost / 1000 + ) logger.debug(f"Total running cost: ${self.total_cost:.3f}") diff --git a/autogpt/llm/utils/token_counter.py b/autogpt/llm/utils/token_counter.py index bd1dcf1b3..e34dbd1cd 100644 --- a/autogpt/llm/utils/token_counter.py +++ b/autogpt/llm/utils/token_counter.py @@ -24,32 +24,28 @@ def count_message_tokens( Returns: int: The number of tokens used by the list of messages. """ - try: - encoding = tiktoken.encoding_for_model(model) - except KeyError: - logger.warn("Warning: model not found. Using cl100k_base encoding.") - encoding = tiktoken.get_encoding("cl100k_base") - if model == "gpt-3.5-turbo": - # !Note: gpt-3.5-turbo may change over time. - # Returning num tokens assuming gpt-3.5-turbo-0301.") - return count_message_tokens(messages, model="gpt-3.5-turbo-0301") - elif model == "gpt-4": - # !Note: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.") - return count_message_tokens(messages, model="gpt-4-0314") - elif model == "gpt-3.5-turbo-0301": + if model.startswith("gpt-3.5-turbo"): tokens_per_message = ( 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n ) tokens_per_name = -1 # if there's a name, the role is omitted - elif model == "gpt-4-0314": + encoding_model = "gpt-3.5-turbo" + elif model.startswith("gpt-4"): tokens_per_message = 3 tokens_per_name = 1 + encoding_model = "gpt-4" else: raise NotImplementedError( - f"num_tokens_from_messages() is not implemented for model {model}.\n" + f"count_message_tokens() is not implemented for model {model}.\n" " See https://github.com/openai/openai-python/blob/main/chatml.md for" " information on how messages are converted to tokens." ) + try: + encoding = tiktoken.encoding_for_model(encoding_model) + except KeyError: + logger.warn("Warning: model not found. Using cl100k_base encoding.") + encoding = tiktoken.get_encoding("cl100k_base") + num_tokens = 0 for message in messages: num_tokens += tokens_per_message