Add an option to set the chunk size using the

configoration - BROWSE_CHUNK_MAX_LENGTH=4000 This way, we can avoid errors of exceeding chunk size when using gpt-3.5
2023-04-18 08:56:00 +03:00 · 2023-04-18 08:56:00 +03:00 · e34ede79b9
parent 67846bad21
commit e34ede79b9
1 changed files with 1 additions and 1 deletions
--- a/autogpt/processing/text.py
+++ b/autogpt/processing/text.py
@ -62,7 +62,7 @@ def summarize_text(
    print(f"Text length: {text_length} characters")

    summaries = []
-    chunks = list(split_text(text))
+    chunks = list(split_text(text, CFG.browse_chunk_max_length))
    scroll_ratio = 1 / len(chunks)

    for i, chunk in enumerate(chunks):