sync https://github.com/JL-er/RWKV-PEFT

2024-05-28 22:35:47 +08:00
parent 3488d22d22
commit f05a4acb04
138 changed files with 29047 additions and 334 deletions
--- a/finetune/lora/v6/src/binidx.py
+++ b/finetune/lora/v6/src/binidx.py
@@ -270,8 +270,10 @@ class MMapIndexedDataset(torch.utils.data.Dataset):
            np_array = np.append(np_array, np_array0)
        return np_array

-    def only(self, idx):
+    def only(self, idx, length=None):
        ptr, size = self._index[idx]
+        if length < size:
+            size = length
        np_array = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=size, offset=ptr
        )