From d12a173f3957cbc4626a1c2e376963451c5e5fe7 Mon Sep 17 00:00:00 2001 From: josc146 Date: Mon, 31 Jul 2023 22:02:28 +0800 Subject: [PATCH] global penalty --- backend-python/utils/rwkv.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend-python/utils/rwkv.py b/backend-python/utils/rwkv.py index 8e4ccae..21ea4b4 100644 --- a/backend-python/utils/rwkv.py +++ b/backend-python/utils/rwkv.py @@ -375,6 +375,16 @@ class TextRWKV(AbstractRWKV): + occurrence[n] * self.penalty_alpha_frequency ) + if i == 0: + for token in self.model_tokens: + token = int(token) + for xxx in occurrence: + occurrence[xxx] *= 0.996 + if token not in occurrence: + occurrence[token] = 1 + else: + occurrence[token] += 1 + # Model only saw '\n\n' as [187, 187] before, but the tokenizer outputs [535] for it at the end def fix_tokens(self, tokens) -> List[int]: if "world" in self.name.lower():