sync https://github.com/JL-er/RWKV-PEFT

2024-05-28 22:35:47 +08:00
parent 3488d22d22
commit f05a4acb04
138 changed files with 29047 additions and 334 deletions
--- a/finetune/lora/v6/fla/init.py
+++ b/finetune/lora/v6/fla/init.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+from fla.layers import (ABCAttention, BasedLinearAttention, DeltaNet,
+                        GatedLinearAttention, HGRN2Attention, LinearAttention,
+                        MultiScaleRetention, ReBasedLinearAttention)
+from fla.models import (ABCForCausalLM, ABCModel, DeltaNetForCausalLM,
+                        DeltaNetModel, GLAForCausalLM, GLAModel,
+                        HGRN2ForCausalLM, HGRN2Model, HGRNForCausalLM,
+                        HGRNModel, LinearAttentionForCausalLM,
+                        LinearAttentionModel, RetNetForCausalLM, RetNetModel,
+                        RWKV6ForCausalLM, RWKV6Model, TransformerForCausalLM,
+                        TransformerModel)
+from fla.ops import (chunk_gla, chunk_retention, fused_chunk_based,
+                     fused_chunk_gla, fused_chunk_retention)
+
+__all__ = [
+    'ABCAttention',
+    'BasedLinearAttention',
+    'DeltaNet',
+    'HGRN2Attention',
+    'GatedLinearAttention',
+    'LinearAttention',
+    'MultiScaleRetention',
+    'ReBasedLinearAttention',
+    'ABCForCausalLM',
+    'ABCModel',
+    'DeltaNetForCausalLM',
+    'DeltaNetModel',
+    'HGRNForCausalLM',
+    'HGRNModel',
+    'HGRN2ForCausalLM',
+    'HGRN2Model',
+    'GLAForCausalLM',
+    'GLAModel',
+    'LinearAttentionForCausalLM',
+    'LinearAttentionModel',
+    'RetNetForCausalLM',
+    'RetNetModel',
+    'RWKV6ForCausalLM',
+    'RWKV6Model',
+    'TransformerForCausalLM',
+    'TransformerModel',
+    'chunk_gla',
+    'chunk_retention',
+    'fused_chunk_based',
+    'fused_chunk_gla',
+    'fused_chunk_retention'
+]
+
+__version__ = '0.1'