improve lora finetune process (need to be refactored)

backend-golang/wsl.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package backend_golang
 
 import (
@@ -8,7 +10,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"time"
 
@@ -37,10 +38,6 @@ func isWslRunning() (bool, error) {
 }
 
 func (a *App) WslStart() error {
-	if runtime.GOOS != "windows" {
-		return errors.New("wsl not supported")
-	}
-
 	running, err := isWslRunning()
 	if err != nil {
 		return err
@@ -100,10 +97,6 @@ func (a *App) WslStart() error {
 }
 
 func (a *App) WslCommand(command string) error {
-	if runtime.GOOS != "windows" {
-		return errors.New("wsl not supported")
-	}
-
 	running, err := isWslRunning()
 	if err != nil {
 		return err
@@ -119,10 +112,6 @@ func (a *App) WslCommand(command string) error {
 }
 
 func (a *App) WslStop() error {
-	if runtime.GOOS != "windows" {
-		return errors.New("wsl not supported")
-	}
-
 	running, err := isWslRunning()
 	if err != nil {
 		return err
@@ -142,10 +131,6 @@ func (a *App) WslStop() error {
 }
 
 func (a *App) WslIsEnabled() error {
-	if runtime.GOOS != "windows" {
-		return errors.New("wsl not supported")
-	}
-
 	ex, err := os.Executable()
 	if err != nil {
 		return err
@@ -177,10 +162,6 @@ func (a *App) WslIsEnabled() error {
 }
 
 func (a *App) WslEnable(forceMode bool) error {
-	if runtime.GOOS != "windows" {
-		return errors.New("wsl not supported")
-	}
-
 	cmd := `/online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux`
 	_, err := su.ShellExecute(su.RUNAS, "dism", cmd, `C:\`)
 	if err != nil {
@@ -193,10 +174,6 @@ func (a *App) WslEnable(forceMode bool) error {
 }
 
 func (a *App) WslInstallUbuntu() error {
-	if runtime.GOOS != "windows" {
+	_, err := Cmd("ms-windows-store://pdp/?ProductId=9PN20MSR04DW")
-		return errors.New("wsl not supported")
+	return err
-	}
-
-	exec.Command("start", "ms-windows-store://pdp/?ProductId=9PN20MSR04DW").Start()
-	return nil
 }

backend-golang/wsl_not_windows.go

@@ -0,0 +1,31 @@
+//go:build darwin || linux
+
+package backend_golang
+
+import (
+	"errors"
+)
+
+func (a *App) WslStart() error {
+	return errors.New("wsl not supported")
+}
+
+func (a *App) WslCommand(command string) error {
+	return errors.New("wsl not supported")
+}
+
+func (a *App) WslStop() error {
+	return errors.New("wsl not supported")
+}
+
+func (a *App) WslIsEnabled() error {
+	return errors.New("wsl not supported")
+}
+
+func (a *App) WslEnable(forceMode bool) error {
+	return errors.New("wsl not supported")
+}
+
+func (a *App) WslInstallUbuntu() error {
+	return errors.New("wsl not supported")
+}

finetune/install-wsl-dep-and-train.sh

@@ -11,13 +11,13 @@ fi
 if dpkg -s "python3-pip" >/dev/null 2>&1; then
   echo "pip installed"
 else
-  sudo apt install python3-pip
+  sudo apt -y install python3-pip
 fi
 
 if dpkg -s "ninja-build" >/dev/null 2>&1; then
   echo "ninja installed"
 else
-  sudo apt install ninja-build
+  sudo apt -y install ninja-build
 fi
 
 if dpkg -s "cuda" >/dev/null 2>&1; then

frontend/src/_locales/zh-hans/main.json

@@ -195,7 +195,7 @@
   "Please convert data first.": "请先转换数据",
   "Ubuntu is not installed, do you want to install it?": "Ubuntu未安装，是否安装？",
   "Install Ubuntu": "安装Ubuntu",
-  "Please install Ubuntu using Microsoft Store": "请用Microsoft Store安装Ubuntu",
+  "Please install Ubuntu using Microsoft Store, after installation click the Open button in Microsoft Store and then click the Train button": "请用Microsoft Store安装Ubuntu，安装完成后，点击Microsoft Store界面的“打开”按钮，然后点击“训练”按钮",
   "WSL is not enabled, do you want to enable it?": "WSL未启用，是否启用？",
   "Enable WSL": "启用WSL",
   "After installation, please restart your computer to enable WSL": "安装完成后，请重启电脑以启用WSL",
@@ -221,5 +221,7 @@
   "Pre-FFN": "前馈网络预处理",
   "None": "空",
   "Merge model successfully": "合并模型成功",
-  "Convert Data successfully": "数据转换成功"
+  "Convert Data successfully": "数据转换成功",
+  "Please select a LoRA model": "请选择一个LoRA模型",
+  "You are using sample data for training. For formal training, please make sure to create your own jsonl file.": "你正在使用示例数据训练，对于正式训练场合，请务必创建你自己的jsonl训练数据"
 }

frontend/src/components/RunButton.tsx

@@ -1,23 +1,16 @@
 import React, { FC, MouseEventHandler, ReactElement } from 'react';
 import commonStore, { ModelStatus } from '../stores/commonStore';
-import {
+import { AddToDownloadList, CopyFile, FileExists, StartServer } from '../../wailsjs/go/backend_golang/App';
-  AddToDownloadList,
-  CopyFile,
-  DepCheck,
-  FileExists,
-  InstallPyDep,
-  StartServer
-} from '../../wailsjs/go/backend_golang/App';
 import { Button } from '@fluentui/react-components';
 import { observer } from 'mobx-react-lite';
 import { exit, getStatus, readRoot, switchModel, updateConfig } from '../apis';
 import { toast } from 'react-toastify';
-import { getStrategy, getSupportedCustomCudaFile, toastWithButton } from '../utils';
+import { checkDependencies, getStrategy, getSupportedCustomCudaFile, toastWithButton } from '../utils';
 import { useTranslation } from 'react-i18next';
 import { ToolTipButton } from './ToolTipButton';
 import { Play16Regular, Stop16Regular } from '@fluentui/react-icons';
 import { useNavigate } from 'react-router';
-import { BrowserOpenURL, WindowShow } from '../../wailsjs/runtime/runtime';
+import { WindowShow } from '../../wailsjs/runtime/runtime';
 
 const mainButtonText = {
   [ModelStatus.Offline]: 'Run',
@@ -57,52 +50,9 @@ export const RunButton: FC<{ onClickRun?: MouseEventHandler, iconMode?: boolean
         return;
       }
 
-      if (!commonStore.depComplete) {
+      const ok = await checkDependencies(navigate);
-        let depErrorMsg = '';
+      if (!ok)
-        await DepCheck(commonStore.settings.customPythonPath).catch((e) => {
+        return;
-          depErrorMsg = e.message || e;
-          WindowShow();
-          if (depErrorMsg === 'python zip not found') {
-            toastWithButton(t('Python target not found, would you like to download it?'), t('Download'), () => {
-              toastWithButton(`${t('Downloading')} Python`, t('Check'), () => {
-                navigate({ pathname: '/downloads' });
-              }, { autoClose: 3000 });
-              AddToDownloadList('python-3.10.11-embed-amd64.zip', 'https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip');
-            });
-          } else if (depErrorMsg.includes('DepCheck Error')) {
-            if (depErrorMsg.includes('vc_redist')) {
-              toastWithButton(t('Microsoft Visual C++ Redistributable is not installed, would you like to download it?'), t('Download'), () => {
-                BrowserOpenURL('https://aka.ms/vs/16/release/vc_redist.x64.exe');
-              });
-            } else {
-              toast(depErrorMsg, { type: 'info', position: 'bottom-left' });
-              if (commonStore.platform != 'linux')
-                toastWithButton(t('Python dependencies are incomplete, would you like to install them?'), t('Install'), () => {
-                  InstallPyDep(commonStore.settings.customPythonPath, commonStore.settings.cnMirror).catch((e) => {
-                    const errMsg = e.message || e;
-                    toast(t('Error') + ' - ' + errMsg, { type: 'error' });
-                  });
-                  setTimeout(WindowShow, 1000);
-                }, {
-                  autoClose: 8000
-                });
-              else
-                toastWithButton(t('On Linux system, you must manually install python dependencies.'), t('Check'), () => {
-                  BrowserOpenURL('https://github.com/josStorer/RWKV-Runner/blob/master/build/linux/Readme_Install.txt');
-                });
-            }
-          } else {
-            toast(depErrorMsg, { type: 'error' });
-          }
-        });
-        if (depErrorMsg) {
-          commonStore.setStatus({ status: ModelStatus.Offline });
-          return;
-        }
-        commonStore.setDepComplete(true);
-        if (commonStore.platform === 'windows')
-          CopyFile('./backend-python/wkv_cuda_utils/wkv_cuda_model.py', './py310/Lib/site-packages/rwkv/model.py');
-      }
 
       const currentModelSource = commonStore.modelSourceList.find(item => item.name === modelName);
 

frontend/src/pages/Train.tsx

@@ -17,7 +17,7 @@ import { toast } from 'react-toastify';
 import commonStore from '../stores/commonStore';
 import { observer } from 'mobx-react-lite';
 import { SelectTabEventHandler } from '@fluentui/react-tabs';
-import { refreshLocalModels, toastWithButton } from '../utils';
+import { checkDependencies, refreshLocalModels, toastWithButton } from '../utils';
 import { Section } from '../components/Section';
 import { Labeled } from '../components/Labeled';
 import { ToolTipButton } from '../components/ToolTipButton';
@@ -36,6 +36,7 @@ import {
 } from 'chart.js';
 import { Line } from 'react-chartjs-2';
 import { ChartJSOrUndefined } from 'react-chartjs-2/dist/types';
+import { WindowShow } from '../../wailsjs/runtime';
 
 ChartJS.register(
   CategoryScale,
@@ -187,10 +188,10 @@ const Terminal: FC = observer(() => {
       WslStart().then(() => {
         addWslMessage('WSL> ' + input);
         setInput('');
-        WslCommand(input).catch((e) => {
+        WslCommand(input).catch((e: any) => {
           toast((e.message || e), { type: 'error' });
         });
-      }).catch((e) => {
+      }).catch((e: any) => {
         toast((e.message || e), { type: 'error' });
       });
     }
@@ -207,7 +208,7 @@ const Terminal: FC = observer(() => {
         <Button onClick={() => {
           WslStop().then(() => {
             toast(t('Command Stopped'), { type: 'success' });
-          }).catch((e) => {
+          }).catch((e: any) => {
             toast((e.message || e), { type: 'error' });
           });
         }}>
@@ -250,13 +251,28 @@ const LoraFinetune: FC = observer(() => {
       });
   }, []);
 
-  const StartLoraFinetune = () => {
+  const StartLoraFinetune = async () => {
+    const ok = await checkDependencies(navigate);
+    if (!ok)
+      return;
+
+    const convertedDataPath = `./finetune/json2binidx_tool/data/${dataParams.dataPath.split('/').pop()!.split('.')[0]}_text_document`;
+    if (!await FileExists(convertedDataPath + '.idx')) {
+      toast(t('Please convert data first.'), { type: 'error' });
+      return;
+    }
+
     WslIsEnabled().then(() => {
-      WslStart().then(async () => {
+      WslStart().then(() => {
-        const convertedDataPath = `./finetune/json2binidx_tool/data/${dataParams.dataPath.split('/').pop()!.split('.')[0]}_text_document`;
+        setTimeout(WindowShow, 1000);
-        if (!await FileExists(convertedDataPath + '.idx')) {
+
-          toast(t('Please convert data first.'), { type: 'error' });
+        let ctxLen = loraParams.ctxLen;
-          return;
+        if (dataParams.dataPath === 'finetune/data/sample.jsonl') {
+          ctxLen = 150;
+          toast(t('You are using sample data for training. For formal training, please make sure to create your own jsonl file.'), {
+            type: 'info',
+            autoClose: 6000
+          });
         }
 
         commonStore.setChartData({
@@ -277,7 +293,7 @@ const LoraFinetune: FC = observer(() => {
           (loraParams.loraLoad ? `--lora_load lora-models/${loraParams.loraLoad} ` : '') +
           `--data_file ${convertedDataPath} ` +
           `--vocab_size ${loraParams.baseModel.toLowerCase().includes('world') ? '65536' : '50277'} ` +
-          `--ctx_len ${loraParams.ctxLen} --epoch_steps ${loraParams.epochSteps} --epoch_count ${loraParams.epochCount} ` +
+          `--ctx_len ${ctxLen} --epoch_steps ${loraParams.epochSteps} --epoch_count ${loraParams.epochCount} ` +
           `--epoch_begin ${loraParams.epochBegin} --epoch_save ${loraParams.epochSave} ` +
           `--micro_bsz ${loraParams.microBsz} --accumulate_grad_batches ${loraParams.accumGradBatches} ` +
           `--pre_ffn ${loraParams.preFfn ? '1' : '0'} --head_qk ${loraParams.headQk ? '1' : '0'} --lr_init ${loraParams.lrInit} --lr_final ${loraParams.lrFinal} ` +
@@ -285,15 +301,20 @@ const LoraFinetune: FC = observer(() => {
           `--beta1 ${loraParams.beta1} --beta2 ${loraParams.beta2} --adam_eps ${loraParams.adamEps} ` +
           `--devices ${loraParams.devices} --precision ${loraParams.precision} ` +
           `--grad_cp ${loraParams.gradCp ? '1' : '0'} ` +
-          `--lora_r ${loraParams.loraR} --lora_alpha ${loraParams.loraAlpha} --lora_dropout ${loraParams.loraDropout}`).catch((e) => {
+          `--lora_r ${loraParams.loraR} --lora_alpha ${loraParams.loraAlpha} --lora_dropout ${loraParams.loraDropout}`).catch((e: any) => {
           toast((e.message || e), { type: 'error' });
         });
       }).catch(e => {
         const msg = e.message || e;
         if (msg === 'ubuntu not found') {
+          WindowShow();
           toastWithButton(t('Ubuntu is not installed, do you want to install it?'), t('Install Ubuntu'), () => {
             WslInstallUbuntu().then(() => {
-              toast(t('Please install Ubuntu using Microsoft Store'), { type: 'info', autoClose: 6000 });
+              WindowShow();
+              toast(t('Please install Ubuntu using Microsoft Store, after installation click the Open button in Microsoft Store and then click the Train button'), {
+                type: 'info',
+                autoClose: 10000
+              });
             });
           });
         }
@@ -302,8 +323,10 @@ const LoraFinetune: FC = observer(() => {
       const msg = e.message || e;
 
       const enableWsl = (forceMode: boolean) => {
+        WindowShow();
         toastWithButton(t('WSL is not enabled, do you want to enable it?'), t('Enable WSL'), () => {
           WslEnable(forceMode).then(() => {
+            WindowShow();
             toast(t('After installation, please restart your computer to enable WSL'), {
               type: 'info',
               autoClose: false
@@ -380,7 +403,7 @@ const LoraFinetune: FC = observer(() => {
                     './finetune/json2binidx_tool/data/' + dataParams.dataPath.split('/').pop()!.split('.')[0],
                     dataParams.vocabPath).then(() => {
                     toast(t('Convert Data successfully'), { type: 'success' });
-                  }).catch((e) => {
+                  }).catch((e: any) => {
                     toast((e.message || e), { type: 'error' });
                   });
                 }}>{t('Convert')}</Button>
@@ -424,15 +447,22 @@ const LoraFinetune: FC = observer(() => {
                   <option key={index} value={name}>{name}</option>
                 )}
               </Select>
-              <Button onClick={() => {
+              <Button onClick={async () => {
-                MergeLora(commonStore.settings.customPythonPath, true, loraParams.loraAlpha,
+                const ok = await checkDependencies(navigate);
-                  'models/' + loraParams.baseModel, 'lora-models/' + loraParams.loraLoad,
+                if (!ok)
-                  `models/${loraParams.baseModel}-LoRA-${loraParams.loraLoad}`).then(() => {
+                  return;
-                  toast(t('Merge model successfully'), { type: 'success' });
+                if (loraParams.loraLoad) {
-                  refreshLocalModels({ models: commonStore.modelSourceList }, false);
+                  MergeLora(commonStore.settings.customPythonPath, true, loraParams.loraAlpha,
-                }).catch((e) => {
+                    'models/' + loraParams.baseModel, 'lora-models/' + loraParams.loraLoad,
-                  toast((e.message || e), { type: 'error' });
+                    `models/${loraParams.baseModel}-LoRA-${loraParams.loraLoad}`).then(() => {
-                });
+                    toast(t('Merge model successfully'), { type: 'success' });
+                    refreshLocalModels({ models: commonStore.modelSourceList }, false);
+                  }).catch((e: any) => {
+                    toast((e.message || e), { type: 'error' });
+                  });
+                } else {
+                  toast(t('Please select a LoRA model'), { type: 'info' });
+                }
               }}>{t('Merge Model')}</Button>
             </div>
             {
@@ -491,7 +521,7 @@ const LoraFinetune: FC = observer(() => {
         <Button appearance="secondary" size="large" onClick={() => {
           WslStop().then(() => {
             toast(t('Command Stopped'), { type: 'success' });
-          }).catch((e) => {
+          }).catch((e: any) => {
             toast((e.message || e), { type: 'error' });
           });
         }}>{t('Stop')}</Button>

frontend/src/startup.ts

@@ -59,6 +59,9 @@ async function initConfig() {
     if (configData.dataProcessParams)
       commonStore.setDataProcessParams(configData.dataProcessParams, false);
 
+    if (configData.loraFinetuneParams)
+      commonStore.setLoraFinetuneParameters(configData.loraFinetuneParams, false);
+
     if (configData.modelConfigs && Array.isArray(configData.modelConfigs))
       commonStore.setModelConfigs(configData.modelConfigs, false);
     else throw new Error('Invalid config.json');

frontend/src/utils/index.tsx

@@ -1,6 +1,9 @@
 import {
   AddToDownloadList,
+  CopyFile,
   DeleteFile,
+  DepCheck,
+  InstallPyDep,
   ListDirFiles,
   ReadFileInfo,
   ReadJson,
@@ -8,7 +11,7 @@ import {
   UpdateApp
 } from '../../wailsjs/go/backend_golang/App';
 import manifest from '../../../manifest.json';
-import commonStore from '../stores/commonStore';
+import commonStore, { ModelStatus } from '../stores/commonStore';
 import { toast } from 'react-toastify';
 import { t } from 'i18next';
 import { ToastOptions } from 'react-toastify/dist/types';
@@ -18,6 +21,8 @@ import { ModelSourceItem } from '../pages/Models';
 import { ModelConfig, ModelParameters } from '../pages/Configs';
 import { DownloadStatus } from '../pages/Downloads';
 import { DataProcessParameters, LoraFinetuneParameters } from '../pages/Train';
+import { BrowserOpenURL, WindowShow } from '../../wailsjs/runtime';
+import { NavigateFunction } from 'react-router';
 
 export type Cache = {
   version: string
@@ -347,6 +352,56 @@ export async function checkUpdate(notifyEvenLatest: boolean = false) {
   });
 }
 
+export const checkDependencies = async (navigate: NavigateFunction) => {
+  if (!commonStore.depComplete) {
+    let depErrorMsg = '';
+    await DepCheck(commonStore.settings.customPythonPath).catch((e) => {
+      depErrorMsg = e.message || e;
+      WindowShow();
+      if (depErrorMsg === 'python zip not found') {
+        toastWithButton(t('Python target not found, would you like to download it?'), t('Download'), () => {
+          toastWithButton(`${t('Downloading')} Python`, t('Check'), () => {
+            navigate({ pathname: '/downloads' });
+          }, { autoClose: 3000 });
+          AddToDownloadList('python-3.10.11-embed-amd64.zip', 'https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip');
+        });
+      } else if (depErrorMsg.includes('DepCheck Error')) {
+        if (depErrorMsg.includes('vc_redist')) {
+          toastWithButton(t('Microsoft Visual C++ Redistributable is not installed, would you like to download it?'), t('Download'), () => {
+            BrowserOpenURL('https://aka.ms/vs/16/release/vc_redist.x64.exe');
+          });
+        } else {
+          toast(depErrorMsg, { type: 'info', position: 'bottom-left' });
+          if (commonStore.platform != 'linux')
+            toastWithButton(t('Python dependencies are incomplete, would you like to install them?'), t('Install'), () => {
+              InstallPyDep(commonStore.settings.customPythonPath, commonStore.settings.cnMirror).catch((e) => {
+                const errMsg = e.message || e;
+                toast(t('Error') + ' - ' + errMsg, { type: 'error' });
+              });
+              setTimeout(WindowShow, 1000);
+            }, {
+              autoClose: 8000
+            });
+          else
+            toastWithButton(t('On Linux system, you must manually install python dependencies.'), t('Check'), () => {
+              BrowserOpenURL('https://github.com/josStorer/RWKV-Runner/blob/master/build/linux/Readme_Install.txt');
+            });
+        }
+      } else {
+        toast(depErrorMsg, { type: 'error' });
+      }
+    });
+    if (depErrorMsg) {
+      commonStore.setStatus({ status: ModelStatus.Offline });
+      return false;
+    }
+    commonStore.setDepComplete(true);
+    if (commonStore.platform === 'windows')
+      CopyFile('./backend-python/wkv_cuda_utils/wkv_cuda_model.py', './py310/Lib/site-packages/rwkv/model.py');
+  }
+  return true;
+};
+
 export function toastWithButton(text: string, buttonText: string, onClickButton: () => void, options?: ToastOptions) {
   let triggered = false;
   const id = toast(