theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve lora finetune process (need to be refactored)

Browse Source
This commit is contained in:
josc146 2023-07-03 21:40:16 +08:00
parent 134b2884e6
commit 76761ee453
8 changed files with 160 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
backend-golang/wsl.go
View File

@ -1,3 +1,5 @@
//go:build windows
package backend_golang
import (
@ -8,7 +10,6 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
@ -37,10 +38,6 @@ func isWslRunning() (bool, error) {
}
func (a *App) WslStart() error {
if runtime.GOOS != "windows" {
return errors.New("wsl not supported")
}
running, err := isWslRunning()
if err != nil {
return err
@ -100,10 +97,6 @@ func (a *App) WslStart() error {
}
func (a *App) WslCommand(command string) error {
if runtime.GOOS != "windows" {
return errors.New("wsl not supported")
}
running, err := isWslRunning()
if err != nil {
return err
@ -119,10 +112,6 @@ func (a *App) WslCommand(command string) error {
}
func (a *App) WslStop() error {
if runtime.GOOS != "windows" {
return errors.New("wsl not supported")
}
running, err := isWslRunning()
if err != nil {
return err
@ -142,10 +131,6 @@ func (a *App) WslStop() error {
}
func (a *App) WslIsEnabled() error {
if runtime.GOOS != "windows" {
return errors.New("wsl not supported")
}
ex, err := os.Executable()
if err != nil {
return err
@ -177,10 +162,6 @@ func (a *App) WslIsEnabled() error {
}
func (a *App) WslEnable(forceMode bool) error {
if runtime.GOOS != "windows" {
return errors.New("wsl not supported")
}
cmd := `/online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux`
_, err := su.ShellExecute(su.RUNAS, "dism", cmd, `C:\`)
if err != nil {
@ -193,10 +174,6 @@ func (a *App) WslEnable(forceMode bool) error {
}
func (a *App) WslInstallUbuntu() error {
if runtime.GOOS != "windows" {
return errors.New("wsl not supported")
}
exec.Command("start", "ms-windows-store://pdp/?ProductId=9PN20MSR04DW").Start()
return nil
_, err := Cmd("ms-windows-store://pdp/?ProductId=9PN20MSR04DW")
return err
}

31
backend-golang/wsl_not_windows.go Normal file
View File

@ -0,0 +1,31 @@
//go:build darwin || linux
package backend_golang
import (
"errors"
)
func (a *App) WslStart() error {
return errors.New("wsl not supported")
}
func (a *App) WslCommand(command string) error {
return errors.New("wsl not supported")
}
func (a *App) WslStop() error {
return errors.New("wsl not supported")
}
func (a *App) WslIsEnabled() error {
return errors.New("wsl not supported")
}
func (a *App) WslEnable(forceMode bool) error {
return errors.New("wsl not supported")
}
func (a *App) WslInstallUbuntu() error {
return errors.New("wsl not supported")
}

4
finetune/install-wsl-dep-and-train.sh
View File

@ -11,13 +11,13 @@ fi
if dpkg -s "python3-pip" >/dev/null 2>&1; then
echo "pip installed"
else
sudo apt install python3-pip
sudo apt -y install python3-pip
fi
if dpkg -s "ninja-build" >/dev/null 2>&1; then
echo "ninja installed"
else
sudo apt install ninja-build
sudo apt -y install ninja-build
fi
if dpkg -s "cuda" >/dev/null 2>&1; then

6
frontend/src/_locales/zh-hans/main.json
View File

@ -195,7 +195,7 @@
"Please convert data first.": "请先转换数据",
"Ubuntu is not installed, do you want to install it?": "Ubuntu未安装，是否安装？",
"Install Ubuntu": "安装Ubuntu",
"Please install Ubuntu using Microsoft Store": "请用Microsoft Store安装Ubuntu",
"Please install Ubuntu using Microsoft Store, after installation click the Open button in Microsoft Store and then click the Train button": "请用Microsoft Store安装Ubuntu，安装完成后，点击Microsoft Store界面的“打开”按钮，然后点击“训练”按钮",
"WSL is not enabled, do you want to enable it?": "WSL未启用，是否启用？",
"Enable WSL": "启用WSL",
"After installation, please restart your computer to enable WSL": "安装完成后，请重启电脑以启用WSL",
@ -221,5 +221,7 @@
"Pre-FFN": "前馈网络预处理",
"None": "空",
"Merge model successfully": "合并模型成功",
"Convert Data successfully": "数据转换成功"
"Convert Data successfully": "数据转换成功",
"Please select a LoRA model": "请选择一个LoRA模型",
"You are using sample data for training. For formal training, please make sure to create your own jsonl file.": "你正在使用示例数据训练，对于正式训练场合，请务必创建你自己的jsonl训练数据"
}

62
frontend/src/components/RunButton.tsx
View File

@ -1,23 +1,16 @@
import React, { FC, MouseEventHandler, ReactElement } from 'react';
import commonStore, { ModelStatus } from '../stores/commonStore';
import {
AddToDownloadList,
CopyFile,
DepCheck,
FileExists,
InstallPyDep,
StartServer
} from '../../wailsjs/go/backend_golang/App';
import { AddToDownloadList, CopyFile, FileExists, StartServer } from '../../wailsjs/go/backend_golang/App';
import { Button } from '@fluentui/react-components';
import { observer } from 'mobx-react-lite';
import { exit, getStatus, readRoot, switchModel, updateConfig } from '../apis';
import { toast } from 'react-toastify';
import { getStrategy, getSupportedCustomCudaFile, toastWithButton } from '../utils';
import { checkDependencies, getStrategy, getSupportedCustomCudaFile, toastWithButton } from '../utils';
import { useTranslation } from 'react-i18next';
import { ToolTipButton } from './ToolTipButton';
import { Play16Regular, Stop16Regular } from '@fluentui/react-icons';
import { useNavigate } from 'react-router';
import { BrowserOpenURL, WindowShow } from '../../wailsjs/runtime/runtime';
import { WindowShow } from '../../wailsjs/runtime/runtime';
const mainButtonText = {
[ModelStatus.Offline]: 'Run',
@ -57,52 +50,9 @@ export const RunButton: FC<{ onClickRun?: MouseEventHandler, iconMode?: boolean
return;
}
if (!commonStore.depComplete) {
let depErrorMsg = '';
await DepCheck(commonStore.settings.customPythonPath).catch((e) => {
depErrorMsg = e.message || e;
WindowShow();
if (depErrorMsg === 'python zip not found') {
toastWithButton(t('Python target not found, would you like to download it?'), t('Download'), () => {
toastWithButton(`${t('Downloading')} Python`, t('Check'), () => {
navigate({ pathname: '/downloads' });
}, { autoClose: 3000 });
AddToDownloadList('python-3.10.11-embed-amd64.zip', 'https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip');
});
} else if (depErrorMsg.includes('DepCheck Error')) {
if (depErrorMsg.includes('vc_redist')) {
toastWithButton(t('Microsoft Visual C++ Redistributable is not installed, would you like to download it?'), t('Download'), () => {
BrowserOpenURL('https://aka.ms/vs/16/release/vc_redist.x64.exe');
});
} else {
toast(depErrorMsg, { type: 'info', position: 'bottom-left' });
if (commonStore.platform != 'linux')
toastWithButton(t('Python dependencies are incomplete, would you like to install them?'), t('Install'), () => {
InstallPyDep(commonStore.settings.customPythonPath, commonStore.settings.cnMirror).catch((e) => {
const errMsg = e.message || e;
toast(t('Error') + ' - ' + errMsg, { type: 'error' });
});
setTimeout(WindowShow, 1000);
}, {
autoClose: 8000
});
else
toastWithButton(t('On Linux system, you must manually install python dependencies.'), t('Check'), () => {
BrowserOpenURL('https://github.com/josStorer/RWKV-Runner/blob/master/build/linux/Readme_Install.txt');
});
}
} else {
toast(depErrorMsg, { type: 'error' });
}
});
if (depErrorMsg) {
commonStore.setStatus({ status: ModelStatus.Offline });
return;
}
commonStore.setDepComplete(true);
if (commonStore.platform === 'windows')
CopyFile('./backend-python/wkv_cuda_utils/wkv_cuda_model.py', './py310/Lib/site-packages/rwkv/model.py');
}
const ok = await checkDependencies(navigate);
if (!ok)
return;
const currentModelSource = commonStore.modelSourceList.find(item => item.name === modelName);

78
frontend/src/pages/Train.tsx
View File

@ -17,7 +17,7 @@ import { toast } from 'react-toastify';
import commonStore from '../stores/commonStore';
import { observer } from 'mobx-react-lite';
import { SelectTabEventHandler } from '@fluentui/react-tabs';
import { refreshLocalModels, toastWithButton } from '../utils';
import { checkDependencies, refreshLocalModels, toastWithButton } from '../utils';
import { Section } from '../components/Section';
import { Labeled } from '../components/Labeled';
import { ToolTipButton } from '../components/ToolTipButton';
@ -36,6 +36,7 @@ import {
} from 'chart.js';
import { Line } from 'react-chartjs-2';
import { ChartJSOrUndefined } from 'react-chartjs-2/dist/types';
import { WindowShow } from '../../wailsjs/runtime';
ChartJS.register(
CategoryScale,
@ -187,10 +188,10 @@ const Terminal: FC = observer(() => {
WslStart().then(() => {
addWslMessage('WSL> ' + input);
setInput('');
WslCommand(input).catch((e) => {
WslCommand(input).catch((e: any) => {
toast((e.message || e), { type: 'error' });
});
}).catch((e) => {
}).catch((e: any) => {
toast((e.message || e), { type: 'error' });
});
}
@ -207,7 +208,7 @@ const Terminal: FC = observer(() => {
<Button onClick={() => {
WslStop().then(() => {
toast(t('Command Stopped'), { type: 'success' });
}).catch((e) => {
}).catch((e: any) => {
toast((e.message || e), { type: 'error' });
});
}}>
@ -250,13 +251,28 @@ const LoraFinetune: FC = observer(() => {
});
}, []);
const StartLoraFinetune = () => {
const StartLoraFinetune = async () => {
const ok = await checkDependencies(navigate);
if (!ok)
return;
const convertedDataPath = `./finetune/json2binidx_tool/data/${dataParams.dataPath.split('/').pop()!.split('.')[0]}_text_document`;
if (!await FileExists(convertedDataPath + '.idx')) {
toast(t('Please convert data first.'), { type: 'error' });
return;
}
WslIsEnabled().then(() => {
WslStart().then(async () => {
const convertedDataPath = `./finetune/json2binidx_tool/data/${dataParams.dataPath.split('/').pop()!.split('.')[0]}_text_document`;
if (!await FileExists(convertedDataPath + '.idx')) {
toast(t('Please convert data first.'), { type: 'error' });
return;
WslStart().then(() => {
setTimeout(WindowShow, 1000);
let ctxLen = loraParams.ctxLen;
if (dataParams.dataPath === 'finetune/data/sample.jsonl') {
ctxLen = 150;
toast(t('You are using sample data for training. For formal training, please make sure to create your own jsonl file.'), {
type: 'info',
autoClose: 6000
});
}
commonStore.setChartData({
@ -277,7 +293,7 @@ const LoraFinetune: FC = observer(() => {
(loraParams.loraLoad ? `--lora_load lora-models/${loraParams.loraLoad} ` : '') +
`--data_file ${convertedDataPath} ` +
`--vocab_size ${loraParams.baseModel.toLowerCase().includes('world') ? '65536' : '50277'} ` +
`--ctx_len ${loraParams.ctxLen} --epoch_steps ${loraParams.epochSteps} --epoch_count ${loraParams.epochCount} ` +
`--ctx_len ${ctxLen} --epoch_steps ${loraParams.epochSteps} --epoch_count ${loraParams.epochCount} ` +
`--epoch_begin ${loraParams.epochBegin} --epoch_save ${loraParams.epochSave} ` +
`--micro_bsz ${loraParams.microBsz} --accumulate_grad_batches ${loraParams.accumGradBatches} ` +
`--pre_ffn ${loraParams.preFfn ? '1' : '0'} --head_qk ${loraParams.headQk ? '1' : '0'} --lr_init ${loraParams.lrInit} --lr_final ${loraParams.lrFinal} ` +
@ -285,15 +301,20 @@ const LoraFinetune: FC = observer(() => {
`--beta1 ${loraParams.beta1} --beta2 ${loraParams.beta2} --adam_eps ${loraParams.adamEps} ` +
`--devices ${loraParams.devices} --precision ${loraParams.precision} ` +
`--grad_cp ${loraParams.gradCp ? '1' : '0'} ` +
`--lora_r ${loraParams.loraR} --lora_alpha ${loraParams.loraAlpha} --lora_dropout ${loraParams.loraDropout}`).catch((e) => {
`--lora_r ${loraParams.loraR} --lora_alpha ${loraParams.loraAlpha} --lora_dropout ${loraParams.loraDropout}`).catch((e: any) => {
toast((e.message || e), { type: 'error' });
});
}).catch(e => {
const msg = e.message || e;
if (msg === 'ubuntu not found') {
WindowShow();
toastWithButton(t('Ubuntu is not installed, do you want to install it?'), t('Install Ubuntu'), () => {
WslInstallUbuntu().then(() => {
toast(t('Please install Ubuntu using Microsoft Store'), { type: 'info', autoClose: 6000 });
WindowShow();
toast(t('Please install Ubuntu using Microsoft Store, after installation click the Open button in Microsoft Store and then click the Train button'), {
type: 'info',
autoClose: 10000
});
});
});
}
@ -302,8 +323,10 @@ const LoraFinetune: FC = observer(() => {
const msg = e.message || e;
const enableWsl = (forceMode: boolean) => {
WindowShow();
toastWithButton(t('WSL is not enabled, do you want to enable it?'), t('Enable WSL'), () => {
WslEnable(forceMode).then(() => {
WindowShow();
toast(t('After installation, please restart your computer to enable WSL'), {
type: 'info',
autoClose: false
@ -380,7 +403,7 @@ const LoraFinetune: FC = observer(() => {
'./finetune/json2binidx_tool/data/' + dataParams.dataPath.split('/').pop()!.split('.')[0],
dataParams.vocabPath).then(() => {
toast(t('Convert Data successfully'), { type: 'success' });
}).catch((e) => {
}).catch((e: any) => {
toast((e.message || e), { type: 'error' });
});
}}>{t('Convert')}</Button>
@ -424,15 +447,22 @@ const LoraFinetune: FC = observer(() => {
<option key={index} value={name}>{name}</option>
)}
</Select>
<Button onClick={() => {
MergeLora(commonStore.settings.customPythonPath, true, loraParams.loraAlpha,
'models/' + loraParams.baseModel, 'lora-models/' + loraParams.loraLoad,
`models/${loraParams.baseModel}-LoRA-${loraParams.loraLoad}`).then(() => {
toast(t('Merge model successfully'), { type: 'success' });
refreshLocalModels({ models: commonStore.modelSourceList }, false);
}).catch((e) => {
toast((e.message || e), { type: 'error' });
});
<Button onClick={async () => {
const ok = await checkDependencies(navigate);
if (!ok)
return;
if (loraParams.loraLoad) {
MergeLora(commonStore.settings.customPythonPath, true, loraParams.loraAlpha,
'models/' + loraParams.baseModel, 'lora-models/' + loraParams.loraLoad,
`models/${loraParams.baseModel}-LoRA-${loraParams.loraLoad}`).then(() => {
toast(t('Merge model successfully'), { type: 'success' });
refreshLocalModels({ models: commonStore.modelSourceList }, false);
}).catch((e: any) => {
toast((e.message || e), { type: 'error' });
});
} else {
toast(t('Please select a LoRA model'), { type: 'info' });
}
}}>{t('Merge Model')}</Button>
</div>
{
@ -491,7 +521,7 @@ const LoraFinetune: FC = observer(() => {
<Button appearance="secondary" size="large" onClick={() => {
WslStop().then(() => {
toast(t('Command Stopped'), { type: 'success' });
}).catch((e) => {
}).catch((e: any) => {
toast((e.message || e), { type: 'error' });
});
}}>{t('Stop')}</Button>

3
frontend/src/startup.ts
View File

@ -59,6 +59,9 @@ async function initConfig() {
if (configData.dataProcessParams)
commonStore.setDataProcessParams(configData.dataProcessParams, false);
if (configData.loraFinetuneParams)
commonStore.setLoraFinetuneParameters(configData.loraFinetuneParams, false);
if (configData.modelConfigs && Array.isArray(configData.modelConfigs))
commonStore.setModelConfigs(configData.modelConfigs, false);
else throw new Error('Invalid config.json');

57
frontend/src/utils/index.tsx
View File

@ -1,6 +1,9 @@
import {
AddToDownloadList,
CopyFile,
DeleteFile,
DepCheck,
InstallPyDep,
ListDirFiles,
ReadFileInfo,
ReadJson,
@ -8,7 +11,7 @@ import {
UpdateApp
} from '../../wailsjs/go/backend_golang/App';
import manifest from '../../../manifest.json';
import commonStore from '../stores/commonStore';
import commonStore, { ModelStatus } from '../stores/commonStore';
import { toast } from 'react-toastify';
import { t } from 'i18next';
import { ToastOptions } from 'react-toastify/dist/types';
@ -18,6 +21,8 @@ import { ModelSourceItem } from '../pages/Models';
import { ModelConfig, ModelParameters } from '../pages/Configs';
import { DownloadStatus } from '../pages/Downloads';
import { DataProcessParameters, LoraFinetuneParameters } from '../pages/Train';
import { BrowserOpenURL, WindowShow } from '../../wailsjs/runtime';
import { NavigateFunction } from 'react-router';
export type Cache = {
version: string
@ -347,6 +352,56 @@ export async function checkUpdate(notifyEvenLatest: boolean = false) {
});
}
export const checkDependencies = async (navigate: NavigateFunction) => {
if (!commonStore.depComplete) {
let depErrorMsg = '';
await DepCheck(commonStore.settings.customPythonPath).catch((e) => {
depErrorMsg = e.message || e;
WindowShow();
if (depErrorMsg === 'python zip not found') {
toastWithButton(t('Python target not found, would you like to download it?'), t('Download'), () => {
toastWithButton(`${t('Downloading')} Python`, t('Check'), () => {
navigate({ pathname: '/downloads' });
}, { autoClose: 3000 });
AddToDownloadList('python-3.10.11-embed-amd64.zip', 'https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip');
});
} else if (depErrorMsg.includes('DepCheck Error')) {
if (depErrorMsg.includes('vc_redist')) {
toastWithButton(t('Microsoft Visual C++ Redistributable is not installed, would you like to download it?'), t('Download'), () => {
BrowserOpenURL('https://aka.ms/vs/16/release/vc_redist.x64.exe');
});
} else {
toast(depErrorMsg, { type: 'info', position: 'bottom-left' });
if (commonStore.platform != 'linux')
toastWithButton(t('Python dependencies are incomplete, would you like to install them?'), t('Install'), () => {
InstallPyDep(commonStore.settings.customPythonPath, commonStore.settings.cnMirror).catch((e) => {
const errMsg = e.message || e;
toast(t('Error') + ' - ' + errMsg, { type: 'error' });
});
setTimeout(WindowShow, 1000);
}, {
autoClose: 8000
});
else
toastWithButton(t('On Linux system, you must manually install python dependencies.'), t('Check'), () => {
BrowserOpenURL('https://github.com/josStorer/RWKV-Runner/blob/master/build/linux/Readme_Install.txt');
});
}
} else {
toast(depErrorMsg, { type: 'error' });
}
});
if (depErrorMsg) {
commonStore.setStatus({ status: ModelStatus.Offline });
return false;
}
commonStore.setDepComplete(true);
if (commonStore.platform === 'windows')
CopyFile('./backend-python/wkv_cuda_utils/wkv_cuda_model.py', './py310/Lib/site-packages/rwkv/model.py');
}
return true;
};
export function toastWithButton(text: string, buttonText: string, onClickButton: () => void, options?: ToastOptions) {
let triggered = false;
const id = toast(