From 94fe71b49c3a4ff7b13813f63524afdfef91ba57 Mon Sep 17 00:00:00 2001 From: josc146 Date: Thu, 22 Feb 2024 01:04:05 +0900 Subject: [PATCH] change AVOID_PENALTY to \n only --- backend-python/utils/rwkv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend-python/utils/rwkv.py b/backend-python/utils/rwkv.py index 197148d..b3defc8 100644 --- a/backend-python/utils/rwkv.py +++ b/backend-python/utils/rwkv.py @@ -379,7 +379,9 @@ class TextRWKV(AbstractRWKV): assert len(dd) == 1 self.AVOID_REPEAT_TOKENS.add(dd[0]) self.AVOID_PENALTY_TOKENS = set() - AVOID_PENALTY = "\n,.:，。：<>[]{}()/\\|;；" # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789 + AVOID_PENALTY = ( + "\n" # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789 + ) for i in AVOID_PENALTY: dd = self.pipeline.encode(i) assert len(dd) == 1