theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve finetune compatibility

Browse Source
This commit is contained in:
josc146 2023-07-03 22:19:20 +08:00
parent 525fb132d6
commit 511652b71c
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
finetune/json2binidx_tool/tools/preprocess_data.py vendored
View File

@ -17,11 +17,14 @@
"""Processing data for pretraining.""" """Processing data for pretraining."""
import argparse
import multiprocessing
import os import os
import sys import sys
sys.path.append(os.path.dirname(os.path.realpath(__file__)))
import argparse
import multiprocessing
import lm_dataformat as lmd import lm_dataformat as lmd
import numpy as np import numpy as np

1
main.go
View File

@ -37,6 +37,7 @@ func main() {
backend.CopyEmbed(finetune) backend.CopyEmbed(finetune)
os.Mkdir("models", os.ModePerm) os.Mkdir("models", os.ModePerm)
os.Mkdir("lora-models", os.ModePerm) os.Mkdir("lora-models", os.ModePerm)
os.Mkdir("finetune/json2binidx_tool/data", os.ModePerm)
} }
f, err := os.Create("lora-models/train_log.txt") f, err := os.Create("lora-models/train_log.txt")