support using directory as training data

backend-golang/rwkv.go

@@ -43,6 +43,52 @@ func (a *App) ConvertData(python string, input string, outputPrefix string, voca
 	if strings.Contains(vocab, "rwkv_vocab_v20230424") {
 		tokenizerType = "RWKVTokenizer"
 	}
+
+	input = strings.TrimSuffix(input, "/")
+	if fi, err := os.Stat(input); err == nil && fi.IsDir() {
+		files, err := os.ReadDir(input)
+		if err != nil {
+			return "", err
+		}
+		jsonlFile, err := os.Create(outputPrefix + ".jsonl")
+		if err != nil {
+			return "", err
+		}
+		defer jsonlFile.Close()
+		for _, file := range files {
+			if file.IsDir() || !strings.HasSuffix(file.Name(), ".txt") {
+				continue
+			}
+			txtFile, err := os.Open(input + "/" + file.Name())
+			if err != nil {
+				return "", err
+			}
+			defer txtFile.Close()
+			jsonlFile.WriteString("{\"text\": \"")
+			buf := make([]byte, 1024)
+			for {
+				n, err := txtFile.Read(buf)
+				if err != nil {
+					break
+				}
+				// regex replace \r\n \n \r with \\n
+				jsonlFile.WriteString(
+					strings.ReplaceAll(
+						strings.ReplaceAll(
+							strings.ReplaceAll(
+								strings.ReplaceAll(string(buf[:n]),
+									"\r\n", "\\n"),
+								"\n", "\\n"),
+							"\r", "\\n"),
+						"\n\n", "\\n"))
+			}
+			jsonlFile.WriteString("\"}\n")
+		}
+		input = outputPrefix + ".jsonl"
+	} else if err != nil {
+		return "", err
+	}
+
 	return Cmd(python, "./finetune/json2binidx_tool/tools/preprocess_data.py", "--input", input, "--output-prefix", outputPrefix, "--vocab", vocab,
 		"--tokenizer-type", tokenizerType, "--dataset-impl", "mmap", "--append-eod")
 }

frontend/src/_locales/zh-hans/main.json

@@ -231,5 +231,6 @@
   "You are using WSL 1 for training, please upgrade to WSL 2. e.g. Run \"wsl --set-version Ubuntu-22.04 2\"": "你正在使用WSL 1进行训练，请升级到WSL 2。例如，运行\"wsl --set-version Ubuntu-22.04 2\"",
   "Matched CUDA is not installed": "未安装匹配的CUDA",
   "Failed to convert data": "数据转换失败",
-  "Failed to merge model": "合并模型失败"
+  "Failed to merge model": "合并模型失败",
+  "The data path should be a directory or a file in jsonl format (more formats will be supported in the future).\n\nWhen you provide a directory path, all the txt files within that directory will be automatically converted into training data. This is commonly used for large-scale training in writing, code generation, or knowledge bases.\n\nThe jsonl format file can be referenced at https://github.com/Abel2076/json2binidx_tool/blob/main/sample.jsonl.\nYou can also write it similar to OpenAI's playground format, as shown in https://platform.openai.com/playground/p/default-chat.\nEven for multi-turn conversations, they must be written in a single line using `\\n` to indicate line breaks. If they are different dialogues or topics, they should be written in separate lines.": "数据路径必须是一个文件夹，或者jsonl格式文件 (未来会支持更多格式)\n\n当你填写的路径是一个文件夹时，该文件夹内的所有txt文件会被自动转换为训练数据，通常这用于大批量训练写作，代码生成或知识库\n\njsonl文件的格式参考 https://github.com/Abel2076/json2binidx_tool/blob/main/sample.jsonl\n你也可以仿照openai的playground编写，参考 https://platform.openai.com/playground/p/default-chat\n即使是多轮对话也必须写在一行，用`\\n`表示换行，如果是不同对话或主题，则另起一行"
 }

frontend/src/components/DialogButton.tsx

@@ -11,6 +11,7 @@ import {
 } from '@fluentui/react-components';
 import { ToolTipButton } from './ToolTipButton';
 import { useTranslation } from 'react-i18next';
+import MarkdownRender from './MarkdownRender';
 
 export const DialogButton: FC<{
   text?: string | null
@@ -19,12 +20,13 @@ export const DialogButton: FC<{
   className?: string,
   title: string,
   contentText: string,
-  onConfirm: () => void,
+  markdown?: boolean,
+  onConfirm?: () => void,
   size?: 'small' | 'medium' | 'large',
   shape?: 'rounded' | 'circular' | 'square',
   appearance?: 'secondary' | 'primary' | 'outline' | 'subtle' | 'transparent',
 }> = ({
-  text, icon, tooltip, className, title, contentText,
+  text, icon, tooltip, className, title, contentText, markdown,
   onConfirm, size, shape, appearance
 }) => {
   const { t } = useTranslation();
@@ -41,7 +43,11 @@ export const DialogButton: FC<{
       <DialogBody>
         <DialogTitle>{title}</DialogTitle>
         <DialogContent>
-          {contentText}
+          {
+            markdown ?
+              <MarkdownRender>{contentText}</MarkdownRender> :
+              contentText
+          }
         </DialogContent>
         <DialogActions>
           <DialogTrigger disableButtonEnhancement>

frontend/src/pages/Train.tsx

@@ -39,6 +39,7 @@ import { Line } from 'react-chartjs-2';
 import { ChartJSOrUndefined } from 'react-chartjs-2/dist/types';
 import { WindowShow } from '../../wailsjs/runtime';
 import { t } from 'i18next';
+import { DialogButton } from '../components/DialogButton';
 
 ChartJS.register(
   CategoryScale,
@@ -400,29 +401,36 @@ const LoraFinetune: FC = observer(() => {
           title={t('Data Process')}
           content={
             <div className="flex flex-col gap-2">
-              <Labeled flex label={t('Data Path')}
+              <div className="flex gap-2 items-center">
-                content={
+                {t('Data Path')}
-                  <div className="grow flex gap-2">
+                <Input className="grow" style={{ minWidth: 0 }} value={dataParams.dataPath}
-                    <Input className="grow ml-2" value={dataParams.dataPath}
                   onChange={(e, data) => {
                     setDataParams({ dataPath: data.value });
                   }} />
+                <DialogButton text={t('Help')} title={t('Help')} markdown
+                  contentText={t('The data path should be a directory or a file in jsonl format (more formats will be supported in the future).\n\n' +
+                    'When you provide a directory path, all the txt files within that directory will be automatically converted into training data. ' +
+                    'This is commonly used for large-scale training in writing, code generation, or knowledge bases.\n\n' +
+                    'The jsonl format file can be referenced at https://github.com/Abel2076/json2binidx_tool/blob/main/sample.jsonl.\n' +
+                    'You can also write it similar to OpenAI\'s playground format, as shown in https://platform.openai.com/playground/p/default-chat.\n' +
+                    'Even for multi-turn conversations, they must be written in a single line using `\\n` to indicate line breaks. ' +
+                    'If they are different dialogues or topics, they should be written in separate lines.')} />
                 <ToolTipButton desc={t('Open Folder')} icon={<Folder20Regular />} onClick={() => {
                   OpenFileFolder(dataParams.dataPath, false);
                 }} />
               </div>
-                } />
               <div className="flex gap-2 items-center">
                 {t('Vocab Path')}
                 <Input className="grow" style={{ minWidth: 0 }} value={dataParams.vocabPath}
                   onChange={(e, data) => {
                     setDataParams({ vocabPath: data.value });
                   }} />
-                <Button appearance="secondary" size="large" onClick={async () => {
+                <Button appearance="secondary" onClick={async () => {
                   const ok = await checkDependencies(navigate);
                   if (!ok)
                     return;
-                  const outputPrefix = './finetune/json2binidx_tool/data/' + dataParams.dataPath.split(/[\/\\]/).pop()!.split('.')[0];
+                  const outputPrefix = './finetune/json2binidx_tool/data/' +
+                    dataParams.dataPath.replace(/[\/\\]$/, '').split(/[\/\\]/).pop()!.split('.')[0];
                   ConvertData(commonStore.settings.customPythonPath, dataParams.dataPath, outputPrefix, dataParams.vocabPath).then(async () => {
                     if (!await FileExists(outputPrefix + '_text_document.idx')) {
                       toast(t('Failed to convert data') + ' - ' + await GetPyError(), { type: 'error' });