From 8ded044901e7671a181b57a62f19f6bf13443e8c Mon Sep 17 00:00:00 2001 From: faisal-alvi Date: Tue, 24 Oct 2023 15:09:55 +0530 Subject: [PATCH] @hook classifai_openai_tokens_per_word --- .../Classifai/Providers/OpenAI/Tokenizer.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/includes/Classifai/Providers/OpenAI/Tokenizer.php b/includes/Classifai/Providers/OpenAI/Tokenizer.php index b588b3cc7..06242510d 100644 --- a/includes/Classifai/Providers/OpenAI/Tokenizer.php +++ b/includes/Classifai/Providers/OpenAI/Tokenizer.php @@ -43,10 +43,24 @@ public function __construct( $max_tokens ) { * @hook classifai_openai_characters_in_token * * @param int $characters_in_token How many characters in one token (roughly) + * @param int $max_tokens Maximum tokens the model supports. * * @return int */ - $this->characters_in_token = apply_filters( 'classifai_openai_characters_in_token', $this->characters_in_token ); + $this->characters_in_token = apply_filters( 'classifai_openai_characters_in_token', $this->characters_in_token, $max_tokens ); + + /** + * How many tokens a word will take (roughly) + * + * @since 0.1.0 + * @hook classifai_openai_tokens_per_word + * + * @param int $tokens_per_word How many tokens a word will take (roughly) + * @param int $max_tokens Maximum tokens the model supports. + * + * @return int + */ + $this->tokens_per_word = apply_filters( 'classifai_openai_tokens_per_word', $this->tokens_per_word, $max_tokens ); } /**