@@ -38,7 +38,20 @@ def num_tokens_from_text(
38
38
return_tokens_per_name_and_message : bool = False ,
39
39
custom_token_count_function : Callable = None ,
40
40
) -> Union [int , Tuple [int , int , int ]]:
41
- """Return the number of tokens used by a text."""
41
+ """Return the number of tokens used by a text.
42
+
43
+ Args:
44
+ text (str): The text to count tokens for.
45
+ model (Optional, str): The model to use for tokenization. Default is "gpt-3.5-turbo-0613".
46
+ return_tokens_per_name_and_message (Optional, bool): Whether to return the number of tokens per name and per
47
+ message. Default is False.
48
+ custom_token_count_function (Optional, Callable): A custom function to count tokens. Default is None.
49
+
50
+ Returns:
51
+ int: The number of tokens used by the text.
52
+ int: The number of tokens per message. Only returned if return_tokens_per_name_and_message is True.
53
+ int: The number of tokens per name. Only returned if return_tokens_per_name_and_message is True.
54
+ """
42
55
if isinstance (custom_token_count_function , Callable ):
43
56
token_count , tokens_per_message , tokens_per_name = custom_token_count_function (text )
44
57
else :
@@ -89,7 +102,7 @@ def num_tokens_from_messages(
89
102
if key == "name" :
90
103
num_tokens += tokens_per_name
91
104
num_tokens += tokens_per_message
92
- num_tokens += custom_prime_count # ChatGPT every reply is primed with <|start|>assistant<|message|>
105
+ num_tokens += custom_prime_count # With ChatGPT, every reply is primed with <|start|>assistant<|message|>
93
106
return num_tokens
94
107
95
108
0 commit comments