theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve fine-tune performance

Browse Source
This commit is contained in:
josc146 2024-02-04 19:33:32 +08:00
parent e3e075e432
commit 0da92ec7bf
2 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
finetune/install-wsl-dep-and-train.sh
View File

@ -50,8 +50,9 @@ echo "loading $loadModel"
modelInfo=$(python3 ./finetune/get_layer_and_embd.py $loadModel 5.2) modelInfo=$(python3 ./finetune/get_layer_and_embd.py $loadModel 5.2)
echo $modelInfo echo $modelInfo
if [[ $modelInfo =~ "--n_layer" ]]; then if [[ $modelInfo =~ "--n_layer" ]]; then
sudo rm -rf /root/.cache/torch_extensions
python3 ./finetune/lora/$modelInfo $@ --proj_dir lora-models --data_type binidx --lora \ python3 ./finetune/lora/$modelInfo $@ --proj_dir lora-models --data_type binidx --lora \
--lora_parts=att,ffn,time,ln --strategy deepspeed_stage_2 --accelerator gpu --lora_parts=att,ffn,time,ln --strategy deepspeed_stage_2 --accelerator gpu --ds_bucket_mb 2
else else
echo "modelInfo is invalid" echo "modelInfo is invalid"
exit 1 exit 1

4
finetune/requirements.txt
View File

@ -1,3 +1,3 @@
torch==1.13.1 torch==2.1.2
pytorch_lightning==1.9.5 pytorch_lightning==1.9.5
deepspeed==0.11.2 deepspeed==0.12.6