Skip to content

Commit

Permalink
Removed debug output
Browse files Browse the repository at this point in the history
  • Loading branch information
slyalin committed Dec 18, 2023
1 parent aa89069 commit dca5fc9
Showing 1 changed file with 0 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -249,8 +249,6 @@ def test_sentencepiece_model_detokenizer(sentencepice_tokenizers, test_string):

token_ids = hf_tokenizer(test_string, return_tensors="np").input_ids
hf_output = hf_tokenizer.batch_decode(token_ids, skip_special_tokens=True)
print('--- HERE 1 ---')
print(token_ids.shape)
ov_output = unpack_strings(ov_detokenizer(token_ids.astype("int32"))["string_output"])

assert ov_output == hf_output
Expand Down

0 comments on commit dca5fc9

Please sign in to comment.