feat: v2
This commit is contained in:
@@ -76,6 +76,8 @@ def text_to_tokens(string_or_messages: str | list[str | dict | list] | Conversat
|
||||
messages = []
|
||||
if isinstance(string_or_messages, str):
|
||||
messages = [{"role": "user", "content": string_or_messages}]
|
||||
elif isinstance(string_or_messages, Conversation):
|
||||
messages = string_or_messages.messages
|
||||
else:
|
||||
messages = string_or_messages
|
||||
|
||||
@@ -99,4 +101,4 @@ def text_to_tokens(string_or_messages: str | list[str | dict | list] | Conversat
|
||||
num_tokens += text_to_tokens(message["content"])
|
||||
num_tokens += 2 # every reply is primed with <im_start>assistant
|
||||
|
||||
return num_tokens
|
||||
return num_tokens
|
||||
|
Reference in New Issue
Block a user