Count tokens with tiktoken (#4704)

* Update OpenAI model info and remove duplicate modelsinfo.py (#4700)

* Update OpenAI model info and remove duplicate modelsinfo.py

* Fix max_tokens for gpt-4-0613

Signed-off-by: Merwane Hamadi <merwanehamadi@gmail.com>
Co-authored-by: Merwane Hamadi <merwanehamadi@gmail.com>

* Update count_message_tokens to support new OpenAI models

Signed-off-by: Merwane Hamadi <merwanehamadi@gmail.com>
Co-authored-by: Merwane Hamadi <merwanehamadi@gmail.com>

* Fix error message in count_message_tokens

---------

Signed-off-by: Merwane Hamadi <merwanehamadi@gmail.com>
Co-authored-by: Erik Peterson <e@eriklp.com>
Co-authored-by: Reinier van der Leer <github@pwuts.nl>

.gitignore

@@ -32,6 +32,7 @@ build/
 develop-eggs/
 dist/
 plugins/
+plugins_config.yaml
 downloads/
 eggs/
 .eggs/

autogpt/llm/api_manager.py

@@ -89,7 +89,9 @@ class ApiManager(metaclass=Singleton):
         self.total_completion_tokens += completion_tokens
         self.total_cost += prompt_tokens * model_info.prompt_token_cost / 1000
         if issubclass(type(model_info), CompletionModelInfo):
-            self.total_cost += completion_tokens * model_info.completion_token_cost / 1000
+            self.total_cost += (
+                completion_tokens * model_info.completion_token_cost / 1000
+            )
 
         logger.debug(f"Total running cost: ${self.total_cost:.3f}")
 

autogpt/llm/utils/token_counter.py

@@ -24,32 +24,28 @@ def count_message_tokens(
     Returns:
         int: The number of tokens used by the list of messages.
     """
-    try:
-        encoding = tiktoken.encoding_for_model(model)
-    except KeyError:
-        logger.warn("Warning: model not found. Using cl100k_base encoding.")
-        encoding = tiktoken.get_encoding("cl100k_base")
-    if model == "gpt-3.5-turbo":
-        # !Note: gpt-3.5-turbo may change over time.
-        # Returning num tokens assuming gpt-3.5-turbo-0301.")
-        return count_message_tokens(messages, model="gpt-3.5-turbo-0301")
-    elif model == "gpt-4":
-        # !Note: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.")
-        return count_message_tokens(messages, model="gpt-4-0314")
-    elif model == "gpt-3.5-turbo-0301":
+    if model.startswith("gpt-3.5-turbo"):
         tokens_per_message = (
             4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
         )
         tokens_per_name = -1  # if there's a name, the role is omitted
-    elif model == "gpt-4-0314":
+        encoding_model = "gpt-3.5-turbo"
+    elif model.startswith("gpt-4"):
         tokens_per_message = 3
         tokens_per_name = 1
+        encoding_model = "gpt-4"
     else:
         raise NotImplementedError(
-            f"num_tokens_from_messages() is not implemented for model {model}.\n"
+            f"count_message_tokens() is not implemented for model {model}.\n"
             " See https://github.com/openai/openai-python/blob/main/chatml.md for"
             " information on how messages are converted to tokens."
         )
+    try:
+        encoding = tiktoken.encoding_for_model(encoding_model)
+    except KeyError:
+        logger.warn("Warning: model not found. Using cl100k_base encoding.")
+        encoding = tiktoken.get_encoding("cl100k_base")
+
     num_tokens = 0
     for message in messages:
         num_tokens += tokens_per_message