update

.gitignore

@@ -4,8 +4,9 @@ frontend/dist
 __pycache__
 .idea
 .vs
-package.json.md5
-cache.json
-stats.html
 *.pth
 *.bin
+/config.json
+/cache.json
+/frontend/stats.html
+/frontend/package.json.md5

backend-python/routes/completion.py

@@ -45,14 +45,14 @@ async def completions(body: CompletionBody, request: Request):
 
     async def eval_rwkv():
         if body.stream:
-            for response, delta in rwkv_generate(model, completion_text):
+            for response, delta in rwkv_generate(model, completion_text, stop="Bob:"):
                 if await request.is_disconnected():
                     break
                 yield json.dumps({"response": response, "choices": [{"delta": {"content": delta}}], "model": "rwkv"})
             yield "[DONE]"
         else:
             response = None
-            for response, delta in rwkv_generate(model, completion_text):
+            for response, delta in rwkv_generate(model, completion_text, stop="Bob:"):
                 pass
             yield json.dumps({"response": response, "model": "rwkv"})
         # torch_gc()

backend-python/utils/rwkv.py

@@ -2,7 +2,7 @@ from typing import Dict
 from langchain.llms import RWKV
 
 
-def rwkv_generate(model: RWKV, prompt: str):
+def rwkv_generate(model: RWKV, prompt: str, stop: str = None):
     model.model_state = None
     model.model_tokens = []
     logits = model.run_rnn(model.tokenizer.encode(prompt).ids)
@@ -34,6 +34,11 @@ def rwkv_generate(model: RWKV, prompt: str):
         delta: str = model.tokenizer.decode(model.model_tokens[out_last:])
         if "\ufffd" not in delta:  # avoid utf-8 display issues
             response += delta
+            if stop is not None:
+                if stop in response:
+                    response = response.split(stop)[0]
+                    yield response, ""
+                    break
             yield response, delta
             out_last = begin + i + 1
             if i >= model.max_tokens_per_generation - 100:

frontend/src/App.tsx

@@ -28,6 +28,7 @@ import {FC, useEffect, useState} from 'react';
 import {Route, Routes, useLocation, useNavigate} from 'react-router';
 import {pages} from './pages';
 import {useMediaQuery} from 'usehooks-ts';
+import {ToastContainer} from 'react-toastify';
 
 const App: FC = () => {
   const navigate = useNavigate();
@@ -80,6 +81,19 @@ const App: FC = () => {
           </Routes>
         </div>
       </div>
+      <ToastContainer
+        style={{
+          width: '250px'
+        }}
+        position="top-right"
+        autoClose={4000}
+        newestOnTop={true}
+        closeOnClick={false}
+        rtl={false}
+        pauseOnFocusLoss={false}
+        draggable={false}
+        theme={'dark'}
+      />
     </FluentProvider>
   );
 };

frontend/src/components/NumberInput.tsx

@@ -0,0 +1,32 @@
+import React, * as React_2 from 'react';
+import {CSSProperties, FC} from 'react';
+import {Input} from '@fluentui/react-components';
+import {SliderOnChangeData} from '@fluentui/react-slider';
+
+export const NumberInput: FC<{
+  value: number,
+  min: number,
+  max: number,
+  step?: number,
+  onChange?: (ev: React_2.ChangeEvent<HTMLInputElement>, data: SliderOnChangeData) => void
+  style?: CSSProperties
+}> = ({value, min, max, step, onChange, style}) => {
+  return (
+    <Input type="number" style={style} value={value.toString()} min={min} max={max} step={step}
+           onChange={(e, data) => {
+             onChange?.(e, {value: Number(data.value)});
+           }}
+           onBlur={(e) => {
+             if (onChange) {
+               if (step) {
+                 const offset = (min > 0 ? min : 0) - (max < 0 ? max : 0);
+                 value = Number(((
+                     Math.round((value - offset) / step) * step)
+                   + offset)
+                   .toFixed(2)); // avoid precision issues
+               }
+               onChange(e, {value: Math.max(Math.min(value, max), min)});
+             }
+           }}/>
+  );
+};

frontend/src/components/Page.tsx

@@ -0,0 +1,12 @@
+import React, {FC, ReactElement} from 'react';
+import {Divider, Text} from '@fluentui/react-components';
+
+export const Page: FC<{ title: string; content: ReactElement }> = ({title, content = true}) => {
+  return (
+    <div className="flex flex-col gap-2 p-2 h-full">
+      <Text size={600}>{title}</Text>
+      <Divider style={{flexGrow: 0}}/>
+      {content}
+    </div>
+  );
+};

frontend/src/components/ValuedSlider.tsx

@@ -0,0 +1,33 @@
+import React, * as React_2 from 'react';
+import {FC, useEffect, useRef} from 'react';
+import {Slider, Text} from '@fluentui/react-components';
+import {SliderOnChangeData} from '@fluentui/react-slider';
+import {NumberInput} from './NumberInput';
+
+export const ValuedSlider: FC<{
+  value: number,
+  min: number,
+  max: number,
+  step?: number,
+  input?: boolean
+  onChange?: (ev: React_2.ChangeEvent<HTMLInputElement>, data: SliderOnChangeData) => void
+}> = ({value, min, max, step, input, onChange}) => {
+  const sliderRef = useRef<HTMLInputElement>(null);
+  useEffect(() => {
+    if (step && sliderRef.current && sliderRef.current.parentElement) {
+      if ((max - min) / step > 10)
+        sliderRef.current.parentElement.style.removeProperty('--fui-Slider--steps-percent');
+    }
+  }, []);
+
+  return (
+    <div className="flex items-center">
+      <Slider ref={sliderRef} className="grow" style={{minWidth: '50%'}} value={value} min={min}
+              max={max} step={step}
+              onChange={onChange}/>
+      {input
+        ? <NumberInput style={{minWidth: 0}} value={value} min={min} max={max} step={step} onChange={onChange}/>
+        : <Text>{value}</Text>}
+    </div>
+  );
+};

frontend/src/main.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
 import {createRoot} from 'react-dom/client';
 import './style.css';
+import 'react-toastify/dist/ReactToastify.css';
 import App from './App';
 import {HashRouter} from 'react-router-dom';
 import {startup} from './startup';

frontend/src/pages/Configs.tsx

@@ -1,47 +1,138 @@
-import {Button, Divider, Dropdown, Input, Option, Slider, Switch, Text} from '@fluentui/react-components';
+import {Button, Dropdown, Input, Label, Option, Slider, Switch} from '@fluentui/react-components';
 import {AddCircle20Regular, DataUsageSettings20Regular, Delete20Regular, Save20Regular} from '@fluentui/react-icons';
 import React, {FC} from 'react';
 import {Section} from '../components/Section';
 import {Labeled} from '../components/Labeled';
 import {ToolTipButton} from '../components/ToolTipButton';
+import commonStore, {ApiParameters, ModelParameters} from '../stores/commonStore';
+import {observer} from 'mobx-react-lite';
+import {toast} from 'react-toastify';
+import {ValuedSlider} from '../components/ValuedSlider';
+import {NumberInput} from '../components/NumberInput';
+import {Page} from '../components/Page';
+
+export const Configs: FC = observer(() => {
+  const [selectedIndex, setSelectedIndex] = React.useState(commonStore.currentModelConfigIndex);
+  const [selectedConfig, setSelectedConfig] = React.useState(commonStore.modelConfigs[selectedIndex]);
+
+  const updateSelectedIndex = (newIndex: number) => {
+    setSelectedIndex(newIndex);
+    setSelectedConfig(commonStore.modelConfigs[newIndex]);
+
+    // if you don't want to update the config used by the current startup in real time, comment out this line
+    commonStore.setCurrentConfigIndex(newIndex);
+  };
+
+  const setSelectedConfigName = (newName: string) => {
+    setSelectedConfig({...selectedConfig, name: newName});
+  };
+
+  const setSelectedConfigApiParams = (newParams: Partial<ApiParameters>) => {
+    setSelectedConfig({
+      ...selectedConfig, apiParameters: {
+        ...selectedConfig.apiParameters,
+        ...newParams
+      }
+    });
+  };
+
+  const setSelectedConfigModelParams = (newParams: Partial<ModelParameters>) => {
+    setSelectedConfig({
+      ...selectedConfig, modelParameters: {
+        ...selectedConfig.modelParameters,
+        ...newParams
+      }
+    });
+  };
 
-export const Configs: FC = () => {
   return (
-    <div className="flex flex-col gap-2 p-2 h-full">
+    <Page title="Configs" content={
-      <Text size={600}>Configs</Text>
+      <div className="flex flex-col gap-2 overflow-hidden">
-      <Divider/>
+        <div className="flex gap-2 items-center">
-      <div className="flex gap-2 items-center w-full">
+          <Dropdown style={{minWidth: 0}} className="grow" value={commonStore.modelConfigs[selectedIndex].name}
-        <Dropdown style={{minWidth: 0}} className="grow"/>
+                    selectedOptions={[selectedIndex.toString()]}
-        <ToolTipButton desc="New Config" icon={<AddCircle20Regular/>}/>
+                    onOptionSelect={(_, data) => {
-        <ToolTipButton desc="Delete Config" icon={<Delete20Regular/>}/>
+                      if (data.optionValue) {
-        <ToolTipButton desc="Save Config" icon={<Save20Regular/>}/>
+                        updateSelectedIndex(Number(data.optionValue));
+                      }
+                    }}>
+            {commonStore.modelConfigs.map((config, index) =>
+              <Option key={index} value={index.toString()}>{config.name}</Option>
+            )}
+          </Dropdown>
+          <ToolTipButton desc="New Config" icon={<AddCircle20Regular/>} onClick={() => {
+            commonStore.createModelConfig();
+            updateSelectedIndex(commonStore.modelConfigs.length - 1);
+          }}/>
+          <ToolTipButton desc="Delete Config" icon={<Delete20Regular/>} onClick={() => {
+            commonStore.deleteModelConfig(selectedIndex);
+            updateSelectedIndex(Math.min(selectedIndex, commonStore.modelConfigs.length - 1));
+          }}/>
+          <ToolTipButton desc="Save Config" icon={<Save20Regular/>} onClick={() => {
+            commonStore.setModelConfig(selectedIndex, selectedConfig);
+            toast('Config Saved', {hideProgressBar: true, autoClose: 300, position: 'top-center'});
+          }}/>
         </div>
-      <div className="flex flex-col h-full gap-2 overflow-y-hidden">
+        <div className="flex items-center gap-4">
+          <Label>Config Name</Label>
+          <Input className="grow" value={selectedConfig.name} onChange={(e, data) => {
+            setSelectedConfigName(data.value);
+          }}/>
+        </div>
+        <div className="flex flex-col gap-2 overflow-y-hidden">
           <Section
             title="Default API Parameters"
             desc="Hover your mouse over the text to view a detailed description. Settings marked with * will take effect immediately after being saved."
             content={
               <div className="grid grid-cols-1 sm:grid-cols-2 gap-2">
                 <Labeled label="API Port" desc="127.0.0.1:8000" content={
-                <Input type="number" min={1} max={65535} step={1}/>
+                  <NumberInput value={selectedConfig.apiParameters.apiPort} min={1} max={65535} step={1}
+                               onChange={(e, data) => {
+                                 setSelectedConfigApiParams({
+                                   apiPort: data.value
+                                 });
+                               }}/>
                 }/>
                 <Labeled label="Max Response Token *" content={
-                <div className="flex items-center grow">
+                  <ValuedSlider value={selectedConfig.apiParameters.maxResponseToken} min={100} max={8100} step={400}
-                  <Slider style={{minWidth: 0}} className="grow" step={400} min={100} max={8100}/>
+                                input
-                  <Text>1000</Text>
+                                onChange={(e, data) => {
-                </div>
+                                  setSelectedConfigApiParams({
+                                    maxResponseToken: data.value
+                                  });
+                                }}/>
                 }/>
                 <Labeled label="Temperature *" content={
-                <Slider style={{minWidth: 0}} className="grow"/>
+                  <ValuedSlider value={selectedConfig.apiParameters.temperature} min={0} max={2} step={0.1} input
+                                onChange={(e, data) => {
+                                  setSelectedConfigApiParams({
+                                    temperature: data.value
+                                  });
+                                }}/>
                 }/>
                 <Labeled label="Top_P *" content={
-                <Slider style={{minWidth: 0}} className="grow"/>
+                  <ValuedSlider value={selectedConfig.apiParameters.topP} min={0} max={1} step={0.1} input
+                                onChange={(e, data) => {
+                                  setSelectedConfigApiParams({
+                                    topP: data.value
+                                  });
+                                }}/>
                 }/>
                 <Labeled label="Presence Penalty *" content={
-                <Slider style={{minWidth: 0}} className="grow"/>
+                  <ValuedSlider value={selectedConfig.apiParameters.presencePenalty} min={-2} max={2} step={0.1} input
+                                onChange={(e, data) => {
+                                  setSelectedConfigApiParams({
+                                    presencePenalty: data.value
+                                  });
+                                }}/>
                 }/>
                 <Labeled label="Count Penalty *" content={
-                <Slider style={{minWidth: 0}} className="grow"/>
+                  <ValuedSlider value={selectedConfig.apiParameters.countPenalty} min={-2} max={2} step={0.1} input
+                                onChange={(e, data) => {
+                                  setSelectedConfigApiParams({
+                                    countPenalty: data.value
+                                  });
+                                }}/>
                 }/>
               </div>
             }
@@ -52,7 +143,11 @@ export const Configs: FC = () => {
               <div className="grid grid-cols-1 sm:grid-cols-2 gap-2">
                 <Labeled label="Model" content={
                   <div className="flex gap-2 grow">
-                  <Dropdown style={{minWidth: 0}} className="grow"/>
+                    <Dropdown style={{minWidth: 0}} className="grow">
+                      {commonStore.modelSourceList.map((modelItem, index) =>
+                        <Option key={index} value={index.toString()}>{modelItem.name}</Option>
+                      )}
+                    </Dropdown>
                     <ToolTipButton desc="Manage Models" icon={<DataUsageSettings20Regular/>}/>
                   </div>
                 }/>
@@ -84,5 +179,6 @@ export const Configs: FC = () => {
           <Button appearance="primary" size="large">Run</Button>
         </div>
       </div>
+    }/>
   );
-};
+});

frontend/src/pages/Home.tsx

@@ -65,7 +65,7 @@ export const Home: FC = observer(() => {
   const onClickMainButton = async () => {
     if (commonStore.modelStatus === ModelStatus.Offline) {
       commonStore.setModelStatus(ModelStatus.Starting);
-      StartServer('cuda fp16i8', 'E:\\RWKV-4-Raven-3B-v10-Eng49%-Chn50%-Other1%-20230419-ctx4096.pth');
+      StartServer('cuda fp16', 'models\\RWKV-4-Raven-1B5-v8-Eng-20230408-ctx4096.pth');
 
       let timeoutCount = 5;
       let loading = false;

frontend/src/pages/Models.tsx

@@ -19,6 +19,8 @@ import commonStore, {ModelSourceItem} from '../stores/commonStore';
 import {BrowserOpenURL} from '../../wailsjs/runtime';
 import {DownloadFile, OpenFileFolder} from '../../wailsjs/go/backend_golang/App';
 import manifest from '../../../manifest.json';
+import {toast} from 'react-toastify';
+import {Page} from '../components/Page';
 
 const columns: TableColumnDefinition<ModelSourceItem>[] = [
   createTableColumn<ModelSourceItem>({
@@ -109,6 +111,7 @@ const columns: TableColumnDefinition<ModelSourceItem>[] = [
             }
             {item.downloadUrl && !item.isLocal &&
               <ToolTipButton desc="Download" icon={<ArrowDownload20Regular/>} onClick={() => {
+                toast(`Downloading ${item.name}`);
                 DownloadFile(`./${manifest.localModelPath}/${item.name}`, item.downloadUrl!);
               }}/>}
             {item.url && <ToolTipButton desc="Open Url" icon={<Open20Regular/>} onClick={() => {
@@ -123,17 +126,19 @@ const columns: TableColumnDefinition<ModelSourceItem>[] = [
 
 export const Models: FC = observer(() => {
   return (
-    <div className="flex flex-col gap-2 p-2 h-full">
+    <Page title="Models" content={
-      <Text size={600}>Models</Text>
+      <div className="flex flex-col gap-2 overflow-hidden">
         <div className="flex flex-col gap-1">
           <div className="flex justify-between">
             <Text weight="medium">Model Source Manifest List</Text>
             <ToolTipButton desc="Refresh" icon={<ArrowClockwise20Regular/>}/>
           </div>
-        <Text size={100}>Provide JSON file URLs for the models manifest. Separate URLs with semicolons. The "models"
+          <Text size={100}>
-          field in JSON files will be parsed into the following table.</Text>
+            Provide JSON file URLs for the models manifest. Separate URLs with semicolons. The "models"
+            field in JSON files will be parsed into the following table.
+          </Text>
           <Textarea size="large" resize="vertical"
-                  defaultValue={commonStore.modelSourceManifestList}
+                    value={commonStore.modelSourceManifestList}
                     onChange={(e, data) => commonStore.setModelSourceManifestList(data.value)}/>
         </div>
         <div className="flex grow overflow-hidden">
@@ -142,7 +147,7 @@ export const Models: FC = observer(() => {
             columns={columns}
             sortable={true}
             style={{display: 'flex'}}
-          className="flex-col"
+            className="flex-col w-full"
           >
             <DataGridHeader>
               <DataGridRow>
@@ -165,5 +170,6 @@ export const Models: FC = observer(() => {
           </DataGrid>
         </div>
       </div>
+    }/>
   );
 });

frontend/src/startup.ts

@@ -1,9 +1,10 @@
-import commonStore, {ModelSourceItem} from './stores/commonStore';
+import commonStore, {defaultModelConfigs, ModelSourceItem} from './stores/commonStore';
 import {ListDirFiles, ReadJson, SaveJson} from '../wailsjs/go/backend_golang/App';
 import manifest from '../../manifest.json';
 
 export async function startup() {
-  initConfig();
+  initCache();
+  await initConfig();
 }
 
 type Cache = {
@@ -11,6 +12,21 @@ type Cache = {
 }
 
 async function initConfig() {
+  await ReadJson('config.json').then((configData) => {
+    if (configData.modelSourceManifestList)
+      commonStore.setModelSourceManifestList(configData.modelSourceManifestList);
+    if (configData.modelConfigs && Array.isArray(configData.modelConfigs))
+      commonStore.setModelConfigs(configData.modelConfigs, false);
+    else throw new Error('Invalid config.json');
+    if (configData.currentModelConfigIndex &&
+      configData.currentModelConfigIndex >= 0 && configData.currentModelConfigIndex < configData.modelConfigs.length)
+      commonStore.setCurrentConfigIndex(configData.currentModelConfigIndex, false);
+  }).catch(() => {
+    commonStore.setModelConfigs(defaultModelConfigs, true);
+  });
+}
+
+async function initCache() {
   let cache: Cache = {models: []};
   await ReadJson('cache.json').then((cacheData: Cache) => {
     cache = cacheData;

frontend/src/stores/commonStore.ts

@@ -1,4 +1,5 @@
 import {makeAutoObservable} from 'mobx';
+import {SaveJson} from '../../wailsjs/go/backend_golang/App';
 
 export enum ModelStatus {
   Offline,
@@ -38,17 +39,17 @@ export type ModelParameters = {
 }
 
 export type ModelConfig = {
-  configName: string;
+  name: string;
   apiParameters: ApiParameters
   modelParameters: ModelParameters
 }
 
-const defaultModelConfigs: ModelConfig[] = [
+export const defaultModelConfigs: ModelConfig[] = [
   {
-    configName: 'Default',
+    name: 'Default',
     apiParameters: {
       apiPort: 8000,
-      maxResponseToken: 1000,
+      maxResponseToken: 4100,
       temperature: 1,
       topP: 1,
       presencePenalty: 0,
@@ -71,28 +72,61 @@ class CommonStore {
 
   modelStatus: ModelStatus = ModelStatus.Offline;
   currentModelConfigIndex: number = 0;
-  modelConfigs: ModelConfig[] = defaultModelConfigs;
+  modelConfigs: ModelConfig[] = [];
   modelSourceManifestList: string = 'https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner/manifest.json;';
   modelSourceList: ModelSourceItem[] = [];
 
+  async saveConfigs() {
+    await SaveJson('config.json', {
+      modelSourceManifestList: this.modelSourceManifestList,
+      currentModelConfigIndex: this.currentModelConfigIndex,
+      modelConfigs: this.modelConfigs
+    });
+  }
+
   setModelStatus = (status: ModelStatus) => {
     this.modelStatus = status;
   };
 
-  setCurrentConfigIndex = (index: number) => {
+  setCurrentConfigIndex = (index: number, saveConfig: boolean = true) => {
     this.currentModelConfigIndex = index;
+    if (saveConfig)
+      this.saveConfigs();
   };
 
-  setModelConfig = (index: number, config: ModelConfig) => {
+  setModelConfig = (index: number, config: ModelConfig, saveConfig: boolean = true) => {
     this.modelConfigs[index] = config;
+    if (saveConfig)
+      this.saveConfigs();
   };
 
-  createModelConfig = (config: ModelConfig = defaultModelConfigs[0]) => {
+  setModelConfigs = (configs: ModelConfig[], saveConfig: boolean = true) => {
+    this.modelConfigs = configs;
+    if (saveConfig)
+      this.saveConfigs();
+  };
+
+  createModelConfig = (config: ModelConfig = defaultModelConfigs[0], saveConfig: boolean = true) => {
+    if (config.name === defaultModelConfigs[0].name)
+      config.name = new Date().toLocaleString();
     this.modelConfigs.push(config);
+    if (saveConfig)
+      this.saveConfigs();
   };
 
-  deleteModelConfig = (index: number) => {
+  deleteModelConfig = (index: number, saveConfig: boolean = true) => {
     this.modelConfigs.splice(index, 1);
+    if (index < this.currentModelConfigIndex) {
+      this.setCurrentConfigIndex(this.currentModelConfigIndex - 1);
+    }
+    if (this.modelConfigs.length === 0) {
+      this.createModelConfig();
+    }
+    if (this.currentModelConfigIndex >= this.modelConfigs.length) {
+      this.setCurrentConfigIndex(this.modelConfigs.length - 1);
+    }
+    if (saveConfig)
+      this.saveConfigs();
   };
 
   setModelSourceManifestList = (value: string) => {