diff --git a/.gitignore b/.gitignore index ca4aef3..b432207 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,9 @@ frontend/dist __pycache__ .idea .vs -package.json.md5 -cache.json -stats.html *.pth -*.bin \ No newline at end of file +*.bin +/config.json +/cache.json +/frontend/stats.html +/frontend/package.json.md5 diff --git a/backend-python/routes/completion.py b/backend-python/routes/completion.py index 767ced4..f89f9cd 100644 --- a/backend-python/routes/completion.py +++ b/backend-python/routes/completion.py @@ -45,14 +45,14 @@ async def completions(body: CompletionBody, request: Request): async def eval_rwkv(): if body.stream: - for response, delta in rwkv_generate(model, completion_text): + for response, delta in rwkv_generate(model, completion_text, stop="Bob:"): if await request.is_disconnected(): break yield json.dumps({"response": response, "choices": [{"delta": {"content": delta}}], "model": "rwkv"}) yield "[DONE]" else: response = None - for response, delta in rwkv_generate(model, completion_text): + for response, delta in rwkv_generate(model, completion_text, stop="Bob:"): pass yield json.dumps({"response": response, "model": "rwkv"}) # torch_gc() diff --git a/backend-python/utils/rwkv.py b/backend-python/utils/rwkv.py index 2bd0cb7..246476d 100644 --- a/backend-python/utils/rwkv.py +++ b/backend-python/utils/rwkv.py @@ -2,7 +2,7 @@ from typing import Dict from langchain.llms import RWKV -def rwkv_generate(model: RWKV, prompt: str): +def rwkv_generate(model: RWKV, prompt: str, stop: str = None): model.model_state = None model.model_tokens = [] logits = model.run_rnn(model.tokenizer.encode(prompt).ids) @@ -34,6 +34,11 @@ def rwkv_generate(model: RWKV, prompt: str): delta: str = model.tokenizer.decode(model.model_tokens[out_last:]) if "\ufffd" not in delta: # avoid utf-8 display issues response += delta + if stop is not None: + if stop in response: + response = response.split(stop)[0] + yield response, "" + break yield response, delta out_last = begin + i + 1 if i >= model.max_tokens_per_generation - 100: diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 95f1872..c9ce9f7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -28,6 +28,7 @@ import {FC, useEffect, useState} from 'react'; import {Route, Routes, useLocation, useNavigate} from 'react-router'; import {pages} from './pages'; import {useMediaQuery} from 'usehooks-ts'; +import {ToastContainer} from 'react-toastify'; const App: FC = () => { const navigate = useNavigate(); @@ -80,6 +81,19 @@ const App: FC = () => { + ); }; diff --git a/frontend/src/components/NumberInput.tsx b/frontend/src/components/NumberInput.tsx new file mode 100644 index 0000000..3a76772 --- /dev/null +++ b/frontend/src/components/NumberInput.tsx @@ -0,0 +1,32 @@ +import React, * as React_2 from 'react'; +import {CSSProperties, FC} from 'react'; +import {Input} from '@fluentui/react-components'; +import {SliderOnChangeData} from '@fluentui/react-slider'; + +export const NumberInput: FC<{ + value: number, + min: number, + max: number, + step?: number, + onChange?: (ev: React_2.ChangeEvent, data: SliderOnChangeData) => void + style?: CSSProperties +}> = ({value, min, max, step, onChange, style}) => { + return ( + { + onChange?.(e, {value: Number(data.value)}); + }} + onBlur={(e) => { + if (onChange) { + if (step) { + const offset = (min > 0 ? min : 0) - (max < 0 ? max : 0); + value = Number((( + Math.round((value - offset) / step) * step) + + offset) + .toFixed(2)); // avoid precision issues + } + onChange(e, {value: Math.max(Math.min(value, max), min)}); + } + }}/> + ); +}; diff --git a/frontend/src/components/Page.tsx b/frontend/src/components/Page.tsx new file mode 100644 index 0000000..2e47b8e --- /dev/null +++ b/frontend/src/components/Page.tsx @@ -0,0 +1,12 @@ +import React, {FC, ReactElement} from 'react'; +import {Divider, Text} from '@fluentui/react-components'; + +export const Page: FC<{ title: string; content: ReactElement }> = ({title, content = true}) => { + return ( +
+ {title} + + {content} +
+ ); +}; diff --git a/frontend/src/components/ValuedSlider.tsx b/frontend/src/components/ValuedSlider.tsx new file mode 100644 index 0000000..71c0654 --- /dev/null +++ b/frontend/src/components/ValuedSlider.tsx @@ -0,0 +1,33 @@ +import React, * as React_2 from 'react'; +import {FC, useEffect, useRef} from 'react'; +import {Slider, Text} from '@fluentui/react-components'; +import {SliderOnChangeData} from '@fluentui/react-slider'; +import {NumberInput} from './NumberInput'; + +export const ValuedSlider: FC<{ + value: number, + min: number, + max: number, + step?: number, + input?: boolean + onChange?: (ev: React_2.ChangeEvent, data: SliderOnChangeData) => void +}> = ({value, min, max, step, input, onChange}) => { + const sliderRef = useRef(null); + useEffect(() => { + if (step && sliderRef.current && sliderRef.current.parentElement) { + if ((max - min) / step > 10) + sliderRef.current.parentElement.style.removeProperty('--fui-Slider--steps-percent'); + } + }, []); + + return ( +
+ + {input + ? + : {value}} +
+ ); +}; diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 39eb4ab..2232394 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -1,6 +1,7 @@ import React from 'react'; import {createRoot} from 'react-dom/client'; import './style.css'; +import 'react-toastify/dist/ReactToastify.css'; import App from './App'; import {HashRouter} from 'react-router-dom'; import {startup} from './startup'; diff --git a/frontend/src/pages/Configs.tsx b/frontend/src/pages/Configs.tsx index 1b1cc8e..e40b3b7 100644 --- a/frontend/src/pages/Configs.tsx +++ b/frontend/src/pages/Configs.tsx @@ -1,88 +1,184 @@ -import {Button, Divider, Dropdown, Input, Option, Slider, Switch, Text} from '@fluentui/react-components'; +import {Button, Dropdown, Input, Label, Option, Slider, Switch} from '@fluentui/react-components'; import {AddCircle20Regular, DataUsageSettings20Regular, Delete20Regular, Save20Regular} from '@fluentui/react-icons'; import React, {FC} from 'react'; import {Section} from '../components/Section'; import {Labeled} from '../components/Labeled'; import {ToolTipButton} from '../components/ToolTipButton'; +import commonStore, {ApiParameters, ModelParameters} from '../stores/commonStore'; +import {observer} from 'mobx-react-lite'; +import {toast} from 'react-toastify'; +import {ValuedSlider} from '../components/ValuedSlider'; +import {NumberInput} from '../components/NumberInput'; +import {Page} from '../components/Page'; + +export const Configs: FC = observer(() => { + const [selectedIndex, setSelectedIndex] = React.useState(commonStore.currentModelConfigIndex); + const [selectedConfig, setSelectedConfig] = React.useState(commonStore.modelConfigs[selectedIndex]); + + const updateSelectedIndex = (newIndex: number) => { + setSelectedIndex(newIndex); + setSelectedConfig(commonStore.modelConfigs[newIndex]); + + // if you don't want to update the config used by the current startup in real time, comment out this line + commonStore.setCurrentConfigIndex(newIndex); + }; + + const setSelectedConfigName = (newName: string) => { + setSelectedConfig({...selectedConfig, name: newName}); + }; + + const setSelectedConfigApiParams = (newParams: Partial) => { + setSelectedConfig({ + ...selectedConfig, apiParameters: { + ...selectedConfig.apiParameters, + ...newParams + } + }); + }; + + const setSelectedConfigModelParams = (newParams: Partial) => { + setSelectedConfig({ + ...selectedConfig, modelParameters: { + ...selectedConfig.modelParameters, + ...newParams + } + }); + }; -export const Configs: FC = () => { return ( -
- Configs - -
- - }/> - }/> - }/> + +
+ { + if (data.optionValue) { + updateSelectedIndex(Number(data.optionValue)); + } + }}> + {commonStore.modelConfigs.map((config, index) => + + )} + + } onClick={() => { + commonStore.createModelConfig(); + updateSelectedIndex(commonStore.modelConfigs.length - 1); + }}/> + } onClick={() => { + commonStore.deleteModelConfig(selectedIndex); + updateSelectedIndex(Math.min(selectedIndex, commonStore.modelConfigs.length - 1)); + }}/> + } onClick={() => { + commonStore.setModelConfig(selectedIndex, selectedConfig); + toast('Config Saved', {hideProgressBar: true, autoClose: 300, position: 'top-center'}); + }}/> +
+
+ + { + setSelectedConfigName(data.value); + }}/> +
+
+
+ { + setSelectedConfigApiParams({ + apiPort: data.value + }); + }}/> + }/> + { + setSelectedConfigApiParams({ + maxResponseToken: data.value + }); + }}/> + }/> + { + setSelectedConfigApiParams({ + temperature: data.value + }); + }}/> + }/> + { + setSelectedConfigApiParams({ + topP: data.value + }); + }}/> + }/> + { + setSelectedConfigApiParams({ + presencePenalty: data.value + }); + }}/> + }/> + { + setSelectedConfigApiParams({ + countPenalty: data.value + }); + }}/> + }/> +
+ } + /> +
+ + + {commonStore.modelSourceList.map((modelItem, index) => + + )} + + }/> +
+ }/> + + + + + + }/> + + + + + + }/> + + }/> + + }/> +
+ } + /> + +
+ +
-
-
- - }/> - - - 1000 -
- }/> - - }/> - - }/> - - }/> - - }/> - - } - /> -
- - - }/> - - }/> - - - - - - }/> - - - - - - }/> - - }/> - - }/> - - } - /> - -
- -
- + }/> ); -}; +}); diff --git a/frontend/src/pages/Home.tsx b/frontend/src/pages/Home.tsx index 082c5aa..837d19d 100644 --- a/frontend/src/pages/Home.tsx +++ b/frontend/src/pages/Home.tsx @@ -65,7 +65,7 @@ export const Home: FC = observer(() => { const onClickMainButton = async () => { if (commonStore.modelStatus === ModelStatus.Offline) { commonStore.setModelStatus(ModelStatus.Starting); - StartServer('cuda fp16i8', 'E:\\RWKV-4-Raven-3B-v10-Eng49%-Chn50%-Other1%-20230419-ctx4096.pth'); + StartServer('cuda fp16', 'models\\RWKV-4-Raven-1B5-v8-Eng-20230408-ctx4096.pth'); let timeoutCount = 5; let loading = false; diff --git a/frontend/src/pages/Models.tsx b/frontend/src/pages/Models.tsx index 9c93917..8390031 100644 --- a/frontend/src/pages/Models.tsx +++ b/frontend/src/pages/Models.tsx @@ -19,6 +19,8 @@ import commonStore, {ModelSourceItem} from '../stores/commonStore'; import {BrowserOpenURL} from '../../wailsjs/runtime'; import {DownloadFile, OpenFileFolder} from '../../wailsjs/go/backend_golang/App'; import manifest from '../../../manifest.json'; +import {toast} from 'react-toastify'; +import {Page} from '../components/Page'; const columns: TableColumnDefinition[] = [ createTableColumn({ @@ -109,6 +111,7 @@ const columns: TableColumnDefinition[] = [ } {item.downloadUrl && !item.isLocal && } onClick={() => { + toast(`Downloading ${item.name}`); DownloadFile(`./${manifest.localModelPath}/${item.name}`, item.downloadUrl!); }}/>} {item.url && } onClick={() => { @@ -123,47 +126,50 @@ const columns: TableColumnDefinition[] = [ export const Models: FC = observer(() => { return ( -
- Models -
-
- Model Source Manifest List - }/> -
- Provide JSON file URLs for the models manifest. Separate URLs with semicolons. The "models" - field in JSON files will be parsed into the following table. -