theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore

Browse Source
This commit is contained in:
josc146 2024-02-06 12:19:12 +08:00
parent a1ae71d221
commit 3e336830a3
3 changed files with 2228 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -19,7 +19,6 @@ __pycache__
/cmd-helper.bat /cmd-helper.bat
/install-py-dep.bat /install-py-dep.bat
/backend-python/wkv_cuda /backend-python/wkv_cuda
/backend-python/rwkv*
*.exe *.exe
*.old *.old
.DS_Store .DS_Store

2224
backend-python/rwkv_pip/tokenizer-midipiano.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff

8
backend-python/utils/rwkv.py
View File

@ -372,18 +372,18 @@ class TextRWKV(AbstractRWKV):
self.bot = "Assistant" self.bot = "Assistant"
self.END_OF_LINE = 11 self.END_OF_LINE = 11
self.AVOID_REPEAT_TOKENS = [] self.AVOID_REPEAT_TOKENS = set()
AVOID_REPEAT = "，：？！" AVOID_REPEAT = "，：？！"
for i in AVOID_REPEAT: for i in AVOID_REPEAT:
dd = self.pipeline.encode(i) dd = self.pipeline.encode(i)
assert len(dd) == 1 assert len(dd) == 1
self.AVOID_REPEAT_TOKENS += dd self.AVOID_REPEAT_TOKENS.add(dd[0])
self.AVOID_PENALTY_TOKENS = [] self.AVOID_PENALTY_TOKENS = set()
AVOID_PENALTY = "\n,.:，。：<>[]{}()/\\|;；" # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789 AVOID_PENALTY = "\n,.:，。：<>[]{}()/\\|;；" # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789
for i in AVOID_PENALTY: for i in AVOID_PENALTY:
dd = self.pipeline.encode(i) dd = self.pipeline.encode(i)
assert len(dd) == 1 assert len(dd) == 1
self.AVOID_PENALTY_TOKENS += dd self.AVOID_PENALTY_TOKENS.add(dd[0])
self.__preload() self.__preload()