add support for python3.8 3.9

2023-06-12 12:09:23 +08:00
parent face4c97e8
commit fa0fcc2c89
3 changed files with 4 additions and 4 deletions
--- a/backend-python/rwkv_pip/rwkv_tokenizer.py
+++ b/backend-python/rwkv_pip/rwkv_tokenizer.py
@@ -72,9 +72,9 @@ class TRIE_TOKENIZER:
        for t, i in self.token2idx.items():
            _ = self.root.add(t, val=(t, i))

-    def encodeBytes(self, src: bytes) -> list[int]:
+    def encodeBytes(self, src: bytes):
        idx: int = 0
-        tokens: list[int] = []
+        tokens = []
        while idx < len(src):
            _idx: int = idx
            idx, _, values = self.root.find_longest(src, idx)