theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

global penalty

Browse Source
This commit is contained in:
josc146 2023-07-31 22:02:28 +08:00
parent 64fa939c19
commit d12a173f39
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend-python/utils/rwkv.py
View File

@ -375,6 +375,16 @@ class TextRWKV(AbstractRWKV):
+ occurrence[n] * self.penalty_alpha_frequency + occurrence[n] * self.penalty_alpha_frequency
) )
if i == 0:
for token in self.model_tokens:
token = int(token)
for xxx in occurrence:
occurrence[xxx] *= 0.996
if token not in occurrence:
occurrence[token] = 1
else:
occurrence[token] += 1
# Model only saw '\n\n' as [187, 187] before, but the tokenizer outputs [535] for it at the end # Model only saw '\n\n' as [187, 187] before, but the tokenizer outputs [535] for it at the end
def fix_tokens(self, tokens) -> List[int]: def fix_tokens(self, tokens) -> List[int]:
if "world" in self.name.lower(): if "world" in self.name.lower():