improve finetune compatibility

finetune/json2binidx_tool/tools/preprocess_data.py

@@ -17,11 +17,14 @@
 
 """Processing data for pretraining."""
 
-import argparse
-import multiprocessing
 import os
 import sys
 
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+
+import argparse
+import multiprocessing
+
 import lm_dataformat as lmd
 import numpy as np
 

main.go

@@ -37,6 +37,7 @@ func main() {
 		backend.CopyEmbed(finetune)
 		os.Mkdir("models", os.ModePerm)
 		os.Mkdir("lora-models", os.ModePerm)
+		os.Mkdir("finetune/json2binidx_tool/data", os.ModePerm)
 	}
 
 	f, err := os.Create("lora-models/train_log.txt")