From 511652b71cf935534c09c6a40f189b32f185671a Mon Sep 17 00:00:00 2001
From: josc146 <josStorer@outlook.com>
Date: Mon, 3 Jul 2023 22:19:20 +0800
Subject: [PATCH] improve finetune compatibility

---
 finetune/json2binidx_tool/tools/preprocess_data.py | 7 +++++--
 main.go                                            | 1 +
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/finetune/json2binidx_tool/tools/preprocess_data.py b/finetune/json2binidx_tool/tools/preprocess_data.py
index e6296e2..b276494 100644
--- a/finetune/json2binidx_tool/tools/preprocess_data.py
+++ b/finetune/json2binidx_tool/tools/preprocess_data.py
@@ -17,11 +17,14 @@
 
 """Processing data for pretraining."""
 
-import argparse
-import multiprocessing
 import os
 import sys
 
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+
+import argparse
+import multiprocessing
+
 import lm_dataformat as lmd
 import numpy as np
 
diff --git a/main.go b/main.go
index abdc8bd..7b4a0f5 100644
--- a/main.go
+++ b/main.go
@@ -37,6 +37,7 @@ func main() {
 		backend.CopyEmbed(finetune)
 		os.Mkdir("models", os.ModePerm)
 		os.Mkdir("lora-models", os.ModePerm)
+		os.Mkdir("finetune/json2binidx_tool/data", os.ModePerm)
 	}
 
 	f, err := os.Create("lora-models/train_log.txt")