theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change AVOID_PENALTY to \n only

Browse Source
This commit is contained in:
josc146 2024-02-22 01:04:05 +09:00
parent fafd9f7f6e
commit 94fe71b49c
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend-python/utils/rwkv.py
View File

@ -379,7 +379,9 @@ class TextRWKV(AbstractRWKV):
assert len(dd) == 1 assert len(dd) == 1
self.AVOID_REPEAT_TOKENS.add(dd[0]) self.AVOID_REPEAT_TOKENS.add(dd[0])
self.AVOID_PENALTY_TOKENS = set() self.AVOID_PENALTY_TOKENS = set()
AVOID_PENALTY = "\n,.:，。：<>[]{}()/\\|;；" # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789 AVOID_PENALTY = (
"\n" # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789
)
for i in AVOID_PENALTY: for i in AVOID_PENALTY:
dd = self.pipeline.encode(i) dd = self.pipeline.encode(i)
assert len(dd) == 1 assert len(dd) == 1