change AVOID_PENALTY to \n only

2024-02-22 01:04:05 +09:00 · 2024-02-22 01:04:05 +09:00 · 94fe71b49c
commit 94fe71b49c
parent fafd9f7f6e
1 changed files with 3 additions and 1 deletions
--- a/backend-python/utils/rwkv.py
+++ b/backend-python/utils/rwkv.py
@ -379,7 +379,9 @@ class TextRWKV(AbstractRWKV):
            assert len(dd) == 1
            self.AVOID_REPEAT_TOKENS.add(dd[0])
        self.AVOID_PENALTY_TOKENS = set()
-        AVOID_PENALTY = "\n,.:，。：<>[]{}()/\\|;；"  # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789
+        AVOID_PENALTY = (
+            "\n"  # \n,.:?!，。：？！"“”<>[]{}/\\|;；~`@#$%^&*()_+-=0123456789
+        )
        for i in AVOID_PENALTY:
            dd = self.pipeline.encode(i)
            assert len(dd) == 1