This commit is contained in:
2024-01-16 21:37:43 +01:00
parent 21d4ad64f3
commit 11e76ff837
3 changed files with 32 additions and 9 deletions

View File

@@ -76,6 +76,8 @@ def text_to_tokens(string_or_messages: str | list[str | dict | list] | Conversat
messages = []
if isinstance(string_or_messages, str):
messages = [{"role": "user", "content": string_or_messages}]
elif isinstance(string_or_messages, Conversation):
messages = string_or_messages.messages
else:
messages = string_or_messages
@@ -99,4 +101,4 @@ def text_to_tokens(string_or_messages: str | list[str | dict | list] | Conversat
num_tokens += text_to_tokens(message["content"])
num_tokens += 2 # every reply is primed with <im_start>assistant
return num_tokens
return num_tokens