theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore

Browse Source
This commit is contained in:
josc146
2024-02-06 12:19:12 +08:00
parent a1ae71d221
commit 3e336830a3
3 changed files with 2228 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend-python/utils/rwkv.py
View File

@@ -372,18 +372,18 @@ class TextRWKV(AbstractRWKV):
self.bot = "Assistant"
self.END_OF_LINE = 11
self.AVOID_REPEAT_TOKENS = []
self.AVOID_REPEAT_TOKENS = set()
AVOID_REPEAT = "，：？！"
for i in AVOID_REPEAT:
dd = self.pipeline.encode(i)
assert len(dd) == 1
self.AVOID_REPEAT_TOKENS += dd
self.AVOID_PENALTY_TOKENS = []
self.AVOID_REPEAT_TOKENS.add(dd[0])
self.AVOID_PENALTY_TOKENS = set()
AVOID_PENALTY = "\n,.:，。：<>[]{}()/\\|;；" # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789
for i in AVOID_PENALTY:
dd = self.pipeline.encode(i)
assert len(dd) == 1
self.AVOID_PENALTY_TOKENS += dd
self.AVOID_PENALTY_TOKENS.add(dd[0])
self.__preload()