theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add macOS MPS configs

Browse Source
This commit is contained in:
josc146 2023-06-06 22:42:38 +08:00
parent 1ecb0b444b
commit 88a5d11e15
6 changed files with 739 additions and 567 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
frontend/src/components/ResetConfigsButton.tsx
View File

@ -3,14 +3,15 @@ import { DialogButton } from './DialogButton';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { ArrowReset20Regular } from '@fluentui/react-icons'; import { ArrowReset20Regular } from '@fluentui/react-icons';
import commonStore from '../stores/commonStore'; import commonStore from '../stores/commonStore';
import { defaultModelConfigs } from '../pages/Configs';
import { defaultModelConfigs, defaultModelConfigsMac } from '../pages/defaultModelConfigs';
export const ResetConfigsButton: FC<{ afterConfirm?: () => void }> = ({ afterConfirm }) => { export const ResetConfigsButton: FC<{ afterConfirm?: () => void }> = ({ afterConfirm }) => {
const { t } = useTranslation(); const { t } = useTranslation();
return <DialogButton icon={<ArrowReset20Regular />} tooltip={t('Reset All Configs')} title={t('Reset All Configs')} return <DialogButton icon={<ArrowReset20Regular />} tooltip={t('Reset All Configs')} title={t('Reset All Configs')}
contentText={t('Are you sure you want to reset all configs? This will obtain the latest preset configs, but will override your custom configs and cannot be undone.')} contentText={t('Are you sure you want to reset all configs? This will obtain the latest preset configs, but will override your custom configs and cannot be undone.')}
onConfirm={() => { onConfirm={() => {
commonStore.setModelConfigs(defaultModelConfigs, false); commonStore.setModelConfigs(commonStore.platform != 'darwin' ? defaultModelConfigs : defaultModelConfigsMac, false);
commonStore.setCurrentConfigIndex(0, true); commonStore.setCurrentConfigIndex(0, true);
afterConfirm?.(); afterConfirm?.();
}} />; }} />;

566
frontend/src/pages/Configs.tsx
View File

@ -30,7 +30,7 @@ export type ApiParameters = {
frequencyPenalty: number; frequencyPenalty: number;
} }
export type Device = 'CPU' | 'CUDA' | 'Custom'; export type Device = 'CPU' | 'CUDA' | 'MPS' | 'Custom';
export type Precision = 'fp16' | 'int8' | 'fp32'; export type Precision = 'fp16' | 'int8' | 'fp32';
export type ModelParameters = { export type ModelParameters = {
@ -52,564 +52,6 @@ export type ModelConfig = {
modelParameters: ModelParameters modelParameters: ModelParameters
} }
export const defaultModelConfigs: ModelConfig[] = [
{
name: 'GPU-2G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 4,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-4G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-4G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 24,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-4G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 24,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-4G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 8,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-4G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 8,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-6G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-6G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-6G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-6G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 18,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-6G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 18,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-8G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-8G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-8G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 27,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-8G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 27,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-10G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-10G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-12G-14B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-14B-v12-Eng98%-Other2%-20230523-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 24,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-16G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-16G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-16G-14B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-14B-v12-Eng98%-Other2%-20230523-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 37,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-18G-14B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-14B-v12-Eng98%-Other2%-20230523-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-32G-14B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-14B-v12-Eng98%-Other2%-20230523-ctx8192.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'CPU-6G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-12G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-12G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-28G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-28G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
}
];
export const Configs: FC = observer(() => { export const Configs: FC = observer(() => {
const { t } = useTranslation(); const { t } = useTranslation();
const [selectedIndex, setSelectedIndex] = React.useState(commonStore.currentModelConfigIndex); const [selectedIndex, setSelectedIndex] = React.useState(commonStore.currentModelConfigIndex);
@ -836,6 +278,7 @@ export const Configs: FC = observer(() => {
} }
}}> }}>
<Option value="CPU">CPU</Option> <Option value="CPU">CPU</Option>
{commonStore.platform === 'darwin' && <Option value="MPS">MPS</Option>}
<Option value="CUDA">CUDA</Option> <Option value="CUDA">CUDA</Option>
<Option value="Custom">{t('Custom')!}</Option> <Option value="Custom">{t('Custom')!}</Option>
</Dropdown> </Dropdown>
@ -903,7 +346,8 @@ export const Configs: FC = observer(() => {
onMouseEnter={() => setDisplayStrategyImg(true)} onMouseEnter={() => setDisplayStrategyImg(true)}
onMouseLeave={() => setDisplayStrategyImg(false)} onMouseLeave={() => setDisplayStrategyImg(false)}
content={ content={
<Input className="grow" placeholder="cuda:0 fp16 *20 -> cuda:1 fp16" <Input className="grow"
placeholder={commonStore.platform != 'darwin' ? 'cuda:0 fp16 *20 -> cuda:1 fp16' : 'mps fp32'}
value={selectedConfig.modelParameters.customStrategy} value={selectedConfig.modelParameters.customStrategy}
onChange={(e, data) => { onChange={(e, data) => {
setSelectedConfigModelParams({ setSelectedConfigModelParams({
@ -914,7 +358,7 @@ export const Configs: FC = observer(() => {
} }
{selectedConfig.modelParameters.device == 'Custom' && <div />} {selectedConfig.modelParameters.device == 'Custom' && <div />}
{ {
selectedConfig.modelParameters.device != 'CPU' && selectedConfig.modelParameters.device != 'CPU' && selectedConfig.modelParameters.device != 'MPS' &&
<Labeled label={t('Use Custom CUDA kernel to Accelerate')} <Labeled label={t('Use Custom CUDA kernel to Accelerate')}
desc={t('Enabling this option can greatly improve inference speed, but there may be compatibility issues. If it fails to start, please turn off this option.')} desc={t('Enabling this option can greatly improve inference speed, but there may be compatibility issues. If it fails to start, please turn off this option.')}
content={ content={

720
frontend/src/pages/defaultModelConfigs.ts Normal file
View File

@ -0,0 +1,720 @@
import { ModelConfig } from './Configs';
export const defaultModelConfigsMac: ModelConfig[] = [
{
name: 'MAC-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'MPS',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: false,
customStrategy: 'mps fp32'
}
},
{
name: 'MAC-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'MPS',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: false,
customStrategy: 'mps fp32'
}
},
{
name: 'MAC-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'MPS',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: false,
customStrategy: 'mps fp32'
}
},
{
name: 'CPU-6G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-12G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-12G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-28G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-28G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
}
];
export const defaultModelConfigs: ModelConfig[] = [
{
name: 'GPU-2G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 4,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-4G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-4G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 24,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-4G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 24,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-4G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 8,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-4G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 8,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-6G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-6G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-6G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-6G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 18,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-6G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 18,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: true,
useCustomCuda: true
}
},
{
name: 'GPU-8G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-8G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-8G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 27,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-8G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 27,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-10G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-10G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-12G-14B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-14B-v12-Eng98%-Other2%-20230523-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 24,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-16G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-16G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-16G-14B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-14B-v12-Eng98%-Other2%-20230523-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 37,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-18G-14B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-14B-v12-Eng98%-Other2%-20230523-ctx8192.pth',
device: 'CUDA',
precision: 'int8',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'GPU-32G-14B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-14B-v12-Eng98%-Other2%-20230523-ctx8192.pth',
device: 'CUDA',
precision: 'fp16',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false,
useCustomCuda: true
}
},
{
name: 'CPU-6G-1B5-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-1B5-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-12G-3B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng98%-Other2%-20230520-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-12G-3B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-3B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230527-ctx4096.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-28G-7B-EN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng98%-Other2%-20230521-ctx8192.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
},
{
name: 'CPU-28G-7B-CN',
apiParameters: {
apiPort: 8000,
maxResponseToken: 4100,
temperature: 1.2,
topP: 0.5,
presencePenalty: 0.4,
frequencyPenalty: 0.4
},
modelParameters: {
modelName: 'RWKV-4-Raven-7B-v12-Eng49%-Chn49%-Jpn1%-Other1%-20230530-ctx8192.pth',
device: 'CPU',
precision: 'fp32',
storedLayers: 41,
maxStoredLayers: 41,
enableHighPrecisionForLastLayer: false
}
}
];

4
frontend/src/startup.ts
View File

@ -3,8 +3,8 @@ import { GetPlatform, ReadJson } from '../wailsjs/go/backend_golang/App';
import { Cache, checkUpdate, downloadProgramFiles, LocalConfig, refreshModels } from './utils'; import { Cache, checkUpdate, downloadProgramFiles, LocalConfig, refreshModels } from './utils';
import { getStatus } from './apis'; import { getStatus } from './apis';
import { EventsOn } from '../wailsjs/runtime'; import { EventsOn } from '../wailsjs/runtime';
import { defaultModelConfigs } from './pages/Configs';
import manifest from '../../manifest.json'; import manifest from '../../manifest.json';
import { defaultModelConfigs, defaultModelConfigsMac } from './pages/defaultModelConfigs';
export async function startup() { export async function startup() {
downloadProgramFiles(); downloadProgramFiles();
@ -54,7 +54,7 @@ async function initConfig() {
configData.currentModelConfigIndex >= 0 && configData.currentModelConfigIndex < configData.modelConfigs.length) configData.currentModelConfigIndex >= 0 && configData.currentModelConfigIndex < configData.modelConfigs.length)
commonStore.setCurrentConfigIndex(configData.currentModelConfigIndex, false); commonStore.setCurrentConfigIndex(configData.currentModelConfigIndex, false);
}).catch(() => { }).catch(() => {
commonStore.setModelConfigs(defaultModelConfigs, true); commonStore.setModelConfigs(commonStore.platform != 'darwin' ? defaultModelConfigs : defaultModelConfigsMac, true);
}); });
} }

7
frontend/src/stores/commonStore.ts
View File

@ -2,7 +2,7 @@ import { makeAutoObservable } from 'mobx';
import { getUserLanguage, isSystemLightMode, saveConfigs } from '../utils'; import { getUserLanguage, isSystemLightMode, saveConfigs } from '../utils';
import { WindowSetDarkTheme, WindowSetLightTheme } from '../../wailsjs/runtime'; import { WindowSetDarkTheme, WindowSetLightTheme } from '../../wailsjs/runtime';
import manifest from '../../../manifest.json'; import manifest from '../../../manifest.json';
import { defaultModelConfigs, ModelConfig } from '../pages/Configs'; import { ModelConfig } from '../pages/Configs';
import { Conversations } from '../pages/Chat'; import { Conversations } from '../pages/Chat';
import { ModelSourceItem } from '../pages/Models'; import { ModelSourceItem } from '../pages/Models';
import { DownloadStatus } from '../pages/Downloads'; import { DownloadStatus } from '../pages/Downloads';
@ -11,6 +11,8 @@ import { IntroductionContent } from '../pages/Home';
import { AboutContent } from '../pages/About'; import { AboutContent } from '../pages/About';
import i18n from 'i18next'; import i18n from 'i18next';
import { CompletionPreset } from '../pages/Completion'; import { CompletionPreset } from '../pages/Completion';
import { defaultModelConfigs, defaultModelConfigsMac } from '../pages/defaultModelConfigs';
import commonStore from './commonStore';
export enum ModelStatus { export enum ModelStatus {
Offline, Offline,
@ -98,7 +100,8 @@ class CommonStore {
createModelConfig = (config: ModelConfig = defaultModelConfigs[0], saveConfig: boolean = true) => { createModelConfig = (config: ModelConfig = defaultModelConfigs[0], saveConfig: boolean = true) => {
if (config.name === defaultModelConfigs[0].name) { if (config.name === defaultModelConfigs[0].name) {
config = JSON.parse(JSON.stringify(config)); // deep copy // deep copy
config = JSON.parse(JSON.stringify(commonStore.platform != 'darwin' ? defaultModelConfigs[0] : defaultModelConfigsMac[0]));
config.name = new Date().toLocaleString(); config.name = new Date().toLocaleString();
} }
this.modelConfigs.push(config); this.modelConfigs.push(config);

4
frontend/src/utils/index.tsx
View File

@ -141,6 +141,10 @@ export const getStrategy = (modelConfig: ModelConfig | undefined = undefined) =>
if (params.enableHighPrecisionForLastLayer) if (params.enableHighPrecisionForLastLayer)
strategy += ' -> cpu fp32 *1'; strategy += ' -> cpu fp32 *1';
break; break;
case 'MPS':
strategy += 'mps ';
strategy += params.precision === 'fp16' ? 'fp16' : params.precision === 'int8' ? 'fp16i8' : 'fp32';
break;
case 'Custom': case 'Custom':
strategy = params.customStrategy || ''; strategy = params.customStrategy || '';
break; break;