preliminary usable features

frontend/src/components/RunButton.tsx

@@ -0,0 +1,61 @@
+import React, {FC} from 'react';
+import commonStore, {ModelStatus} from '../stores/commonStore';
+import {StartServer} from '../../wailsjs/go/backend_golang/App';
+import {Button} from '@fluentui/react-components';
+import {observer} from 'mobx-react-lite';
+
+const mainButtonText = {
+  [ModelStatus.Offline]: 'Run',
+  [ModelStatus.Starting]: 'Starting',
+  [ModelStatus.Loading]: 'Loading',
+  [ModelStatus.Working]: 'Stop'
+};
+
+const onClickMainButton = async () => {
+  if (commonStore.modelStatus === ModelStatus.Offline) {
+    commonStore.setModelStatus(ModelStatus.Starting);
+    StartServer(commonStore.getStrategy(), `models\\${commonStore.getCurrentModelConfig().modelParameters.modelName}`);
+
+    let timeoutCount = 5;
+    let loading = false;
+    const intervalId = setInterval(() => {
+      fetch('http://127.0.0.1:8000')
+        .then(r => {
+          if (r.ok && !loading) {
+            clearInterval(intervalId);
+            commonStore.setModelStatus(ModelStatus.Loading);
+            loading = true;
+            fetch('http://127.0.0.1:8000/update-config', {
+              method: 'POST',
+              headers: {
+                'Content-Type': 'application/json'
+              },
+              body: JSON.stringify({})
+            }).then(async (r) => {
+              if (r.ok)
+                commonStore.setModelStatus(ModelStatus.Working);
+            });
+          }
+        }).catch(() => {
+        if (timeoutCount <= 0) {
+          clearInterval(intervalId);
+          commonStore.setModelStatus(ModelStatus.Offline);
+        }
+      });
+
+      timeoutCount--;
+    }, 1000);
+  } else {
+    commonStore.setModelStatus(ModelStatus.Offline);
+    fetch('http://127.0.0.1:8000/exit', {method: 'POST'});
+  }
+};
+
+export const RunButton: FC = observer(() => {
+  return (
+    <Button disabled={commonStore.modelStatus === ModelStatus.Starting} appearance="primary" size="large"
+            onClick={onClickMainButton}>
+      {mainButtonText[commonStore.modelStatus]}
+    </Button>
+  );
+});

frontend/src/pages/Configs.tsx

@@ -1,16 +1,17 @@
-import {Button, Dropdown, Input, Label, Option, Select, Slider, Switch} from '@fluentui/react-components';
+import {Dropdown, Input, Label, Option, Select, Switch} from '@fluentui/react-components';
 import {AddCircle20Regular, DataUsageSettings20Regular, Delete20Regular, Save20Regular} from '@fluentui/react-icons';
 import React, {FC} from 'react';
 import {Section} from '../components/Section';
 import {Labeled} from '../components/Labeled';
 import {ToolTipButton} from '../components/ToolTipButton';
-import commonStore, {ApiParameters, ModelParameters} from '../stores/commonStore';
+import commonStore, {ApiParameters, Device, ModelParameters, Precision} from '../stores/commonStore';
 import {observer} from 'mobx-react-lite';
 import {toast} from 'react-toastify';
 import {ValuedSlider} from '../components/ValuedSlider';
 import {NumberInput} from '../components/NumberInput';
 import {Page} from '../components/Page';
 import {useNavigate} from 'react-router';
+import {RunButton} from '../components/RunButton';
 
 export const Configs: FC = observer(() => {
   const [selectedIndex, setSelectedIndex] = React.useState(commonStore.currentModelConfigIndex);
@@ -164,30 +165,57 @@ export const Configs: FC = observer(() => {
                 }/>
                 <ToolTipButton text="Convert" desc="Convert model with these configs"/>
                 <Labeled label="Device" content={
-                  <Dropdown style={{minWidth: 0}} className="grow">
+                  <Dropdown style={{minWidth: 0}} className="grow" value={selectedConfig.modelParameters.device}
+                            selectedOptions={[selectedConfig.modelParameters.device]}
+                            onOptionSelect={(_, data) => {
+                              if (data.optionText) {
+                                setSelectedConfigModelParams({
+                                  device: data.optionText as Device
+                                });
+                              }
+                            }}>
                     <Option>CPU</Option>
                     <Option>CUDA</Option>
                   </Dropdown>
                 }/>
                 <Labeled label="Precision" content={
-                  <Dropdown style={{minWidth: 0}} className="grow">
+                  <Dropdown style={{minWidth: 0}} className="grow" value={selectedConfig.modelParameters.precision}
+                            selectedOptions={[selectedConfig.modelParameters.precision]}
+                            onOptionSelect={(_, data) => {
+                              if (data.optionText) {
+                                setSelectedConfigModelParams({
+                                  precision: data.optionText as Precision
+                                });
+                              }
+                            }}>
                     <Option>fp16</Option>
                     <Option>int8</Option>
                     <Option>fp32</Option>
                   </Dropdown>
                 }/>
                 <Labeled label="Streamed Layers" content={
-                  <Slider style={{minWidth: 0}} className="grow"/>
+                  <ValuedSlider value={selectedConfig.modelParameters.streamedLayers} min={0}
+                                max={selectedConfig.modelParameters.maxStreamedLayers} step={1} input
+                                onChange={(e, data) => {
+                                  setSelectedConfigModelParams({
+                                    streamedLayers: data.value
+                                  });
+                                }}/>
                 }/>
                 <Labeled label="Enable High Precision For Last Layer" content={
-                  <Switch/>
+                  <Switch checked={selectedConfig.modelParameters.enableHighPrecisionForLastLayer}
+                          onChange={(e, data) => {
+                            setSelectedConfigModelParams({
+                              enableHighPrecisionForLastLayer: data.checked
+                            });
+                          }}/>
                 }/>
               </div>
             }
           />
         </div>
         <div className="flex flex-row-reverse sm:fixed bottom-2 right-2">
-          <Button appearance="primary" size="large">Run</Button>
+          <RunButton/>
         </div>
       </div>
     }/>

frontend/src/pages/Home.tsx

@@ -1,4 +1,4 @@
-import {Button, CompoundButton, Dropdown, Link, Option, Text} from '@fluentui/react-components';
+import {CompoundButton, Dropdown, Link, Option, Text} from '@fluentui/react-components';
 import React, {FC, ReactElement} from 'react';
 import banner from '../assets/images/banner.jpg';
 import {
@@ -8,9 +8,9 @@ import {
   Storage20Regular
 } from '@fluentui/react-icons';
 import {useNavigate} from 'react-router';
-import commonStore, {ModelStatus} from '../stores/commonStore';
+import commonStore from '../stores/commonStore';
 import {observer} from 'mobx-react-lite';
-import {StartServer} from '../../wailsjs/go/backend_golang/App';
+import {RunButton} from '../components/RunButton';
 
 type NavCard = {
   label: string;
@@ -19,7 +19,7 @@ type NavCard = {
   icon: ReactElement;
 };
 
-export const navCards: NavCard[] = [
+const navCards: NavCard[] = [
   {
     label: 'Chat',
     desc: 'Go to chat page',
@@ -46,62 +46,13 @@ export const navCards: NavCard[] = [
   }
 ];
 
-const mainButtonText = {
-  [ModelStatus.Offline]: 'Run',
-  [ModelStatus.Starting]: 'Starting',
-  [ModelStatus.Loading]: 'Loading',
-  [ModelStatus.Working]: 'Stop'
-};
-
 export const Home: FC = observer(() => {
-  const [selectedConfig, setSelectedConfig] = React.useState('RWKV-3B-4G MEM');
-
   const navigate = useNavigate();
 
   const onClickNavCard = (path: string) => {
     navigate({pathname: path});
   };
 
-  const onClickMainButton = async () => {
-    if (commonStore.modelStatus === ModelStatus.Offline) {
-      commonStore.setModelStatus(ModelStatus.Starting);
-      StartServer('cuda fp16', 'models\\RWKV-4-Raven-1B5-v8-Eng-20230408-ctx4096.pth');
-
-      let timeoutCount = 5;
-      let loading = false;
-      const intervalId = setInterval(() => {
-        fetch('http://127.0.0.1:8000')
-          .then(r => {
-            if (r.ok && !loading) {
-              clearInterval(intervalId);
-              commonStore.setModelStatus(ModelStatus.Loading);
-              loading = true;
-              fetch('http://127.0.0.1:8000/update-config', {
-                method: 'POST',
-                headers: {
-                  'Content-Type': 'application/json'
-                },
-                body: JSON.stringify({})
-              }).then(async (r) => {
-                if (r.ok)
-                  commonStore.setModelStatus(ModelStatus.Working);
-              });
-            }
-          }).catch(() => {
-          if (timeoutCount <= 0) {
-            clearInterval(intervalId);
-            commonStore.setModelStatus(ModelStatus.Offline);
-          }
-        });
-
-        timeoutCount--;
-      }, 1000);
-    } else {
-      commonStore.setModelStatus(ModelStatus.Offline);
-      fetch('http://127.0.0.1:8000/exit', {method: 'POST'});
-    }
-  };
-
   return (
     <div className="flex flex-col justify-between h-full">
       <img className="rounded-xl select-none hidden sm:block" src={banner}/>
@@ -128,30 +79,18 @@ export const Home: FC = observer(() => {
       <div className="flex flex-col gap-2">
         <div className="flex flex-row-reverse sm:fixed bottom-2 right-2">
           <div className="flex gap-3">
-            <Dropdown style={{minWidth: 0}}
+            <Dropdown style={{minWidth: 0}} listbox={{style: {minWidth: 0}}}
-                      placeholder="Config"
+                      value={commonStore.getCurrentModelConfig().name}
-                      value={selectedConfig}
+                      selectedOptions={[commonStore.currentModelConfigIndex.toString()]}
                       onOptionSelect={(_, data) => {
                         if (data.optionValue)
-                          setSelectedConfig(data.optionValue);
+                          commonStore.setCurrentConfigIndex(Number(data.optionValue));
                       }}>
-              <Option id="item-1" key="item-1">
+              {commonStore.modelConfigs.map((config, index) =>
-                RWKV-3B-4G MEM
+                <Option key={index} value={index.toString()}>{config.name}</Option>
-              </Option>
+              )}
-              <Option id="item-2" key="item-2">
-                Item 2
-              </Option>
-              <Option id="item-3" key="item-3">
-                Item 3
-              </Option>
-              <Option id="item-4" key="item-4">
-                Item 4
-              </Option>
             </Dropdown>
-            <Button disabled={commonStore.modelStatus === ModelStatus.Starting} appearance="primary" size="large"
+            <RunButton/>
-                    onClick={onClickMainButton}>
-              {mainButtonText[commonStore.modelStatus]}
-            </Button>
           </div>
         </div>
         <div className="flex gap-4 items-end">

frontend/src/stores/commonStore.ts

@@ -30,12 +30,16 @@ export type ApiParameters = {
   countPenalty: number;
 }
 
+export type Device = 'CPU' | 'CUDA';
+export type Precision = 'fp16' | 'int8' | 'fp32';
+
 export type ModelParameters = {
   // different models can not have the same name
   modelName: string;
-  device: string;
+  device: Device;
-  precision: string;
+  precision: Precision;
   streamedLayers: number;
+  maxStreamedLayers: number;
   enableHighPrecisionForLastLayer: boolean;
 }
 
@@ -58,10 +62,11 @@ export const defaultModelConfigs: ModelConfig[] = [
       countPenalty: 0
     },
     modelParameters: {
-      modelName: '124M',
+      modelName: 'RWKV-4-Raven-1B5-v11-Eng99%-Other1%-20230425-ctx4096.pth',
-      device: 'CPU',
+      device: 'CUDA',
-      precision: 'fp32',
+      precision: 'fp16',
-      streamedLayers: 1,
+      streamedLayers: 25,
+      maxStreamedLayers: 25,
       enableHighPrecisionForLastLayer: false
     }
   }
@@ -86,6 +91,24 @@ class CommonStore {
     });
   }
 
+  getStrategy(modelConfig: ModelConfig | undefined = undefined) {
+    let params: ModelParameters;
+    if (modelConfig) params = modelConfig.modelParameters;
+    else params = this.getCurrentModelConfig().modelParameters;
+    let strategy = '';
+    strategy += (params.device === 'CPU' ? 'cpu' : 'cuda') + ' ';
+    strategy += (params.precision === 'fp16' ? 'fp16' : params.precision === 'int8' ? 'fp16i8' : 'fp32');
+    if (params.streamedLayers < params.maxStreamedLayers)
+      strategy += ` *${params.streamedLayers}+`;
+    if (params.enableHighPrecisionForLastLayer)
+      strategy += ' -> cpu fp32 *1';
+    return strategy;
+  }
+
+  getCurrentModelConfig = () => {
+    return this.modelConfigs[this.currentModelConfigIndex];
+  };
+
   setModelStatus = (status: ModelStatus) => {
     this.modelStatus = status;
   };