riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 18:22:34 +06:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Add warning for stop string edge case (#33169)

Browse Source 
* Add warning for edge case

* make fixup
This commit is contained in:
Matt 2024-08-30 16:26:26 +01:00 committed by GitHub
parent e259d6d1e0
commit fbff27623a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/transformers/generation/stopping_criteria.py
View File

@ -348,7 +348,14 @@ class StopStringCriteria(StoppingCriteria):
# we need a fallback to handle this case
max_valid_positions = max(all_valid_positions) if all_valid_positions else 1
# There should always be at least one valid end_len, however, so no fallback needed here
max_valid_end_lens = max(len(val) for positions in token_end_overlaps.values() for val in positions.values())
valid_end_lens = [len(val) for positions in token_end_overlaps.values() for val in positions.values()]
if not valid_end_lens:
raise ValueError(
"Stop string preprocessing was unable to identify tokens matching one or more of the "
"supplied stop string(s). This is most often caused by the stop "
"strings containing unusual characters that are not in the tokenizer vocabulary."
)
max_valid_end_lens = max(valid_end_lens)
vec_size = len(stop_strings) * (max_valid_positions + max_valid_end_lens) + 1
gather_vec = np.full((len(token_list), vec_size), dtype=np.int32, fill_value=-1)