release v1.7.7

CURRENT_CHANGE.md

@@ -1,8 +1,7 @@
 ## Changes
 
-### Fixes
-
-- fix v6 lora (https://github.com/JL-er/RWKV-LORA/commit/c03cdbbdafa498a7d65da37bf54a4228eff79132)
+- avoid program lag caused by frequent triggering of read/write operations due to Linux file system notification
+- improve styles
 
 ## Install
 

README.md

@@ -94,7 +94,8 @@ English | [简体中文](README_ZH.md) | [日本語](README_JA.md)
 - Built-in model conversion tool.
 - Built-in download management and remote model inspection.
 - Built-in one-click LoRA Finetune. (Windows Only)
-- Can also be used as an OpenAI ChatGPT and GPT-Playground client. (Fill in the API URL and API Key in Settings page)
+- Can also be used as an OpenAI ChatGPT, GPT-Playground, Ollama and more clients. (Fill in the API URL and API Key in
+  Settings page)
 - Multilingual localization.
 - Theme switching.
 - Automatic updates.

README_JA.md

@@ -89,8 +89,8 @@
 - 内蔵モデル変換ツール
 - ダウンロード管理とリモートモデル検査機能内蔵
 - 内蔵のLoRA微調整機能を搭載しています (Windowsのみ)
-- このプログラムは、OpenAI ChatGPTとGPT Playgroundのクライアントとしても使用できます（設定ページで `API URL` と `API Key`
-  を入力してください）
+- このプログラムは、OpenAI ChatGPT、GPT Playground、Ollama などのクライアントとしても使用できます（設定ページで `API URL`
+  と `API Key` を入力してください）
 - 多言語ローカライズ
 - テーマ切り替え
 - 自動アップデート

README_ZH.md

@@ -83,7 +83,7 @@ API兼容的接口，这意味着一切ChatGPT客户端都是RWKV客户端。
 - 内置模型转换工具
 - 内置下载管理和远程模型检视
 - 内置一键LoRA微调 (仅限Windows)
-- 也可用作 OpenAI ChatGPT 和 GPT Playground 客户端 (在设置内填写API URL和API Key)
+- 也可用作 OpenAI ChatGPT, GPT Playground, Ollama 等服务的客户端 (在设置内填写API URL和API Key)
 - 多语言本地化
 - 主题切换
 - 自动更新

backend-golang/app.go

@@ -7,7 +7,11 @@ import (
 	"context"
 	"errors"
 	"io"
+	"log"
+	"net"
 	"net/http"
+	"net/http/httputil"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -27,6 +31,7 @@ type App struct {
 	HasConfigData bool
 	ConfigData    map[string]any
 	Dev           bool
+	proxyPort     int
 	exDir         string
 	cmdPrefix     string
 }
@@ -36,6 +41,63 @@ func NewApp() *App {
 	return &App{}
 }
 
+func (a *App) newFetchProxy() {
+	go func() {
+		handler := func(w http.ResponseWriter, r *http.Request) {
+			if r.Method == "OPTIONS" {
+				w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+				w.Header().Set("Access-Control-Allow-Headers", "*")
+				w.Header().Set("Access-Control-Allow-Origin", "*")
+				return
+			}
+			proxy := &httputil.ReverseProxy{
+				ModifyResponse: func(res *http.Response) error {
+					res.Header.Set("Access-Control-Allow-Origin", "*")
+					return nil
+				},
+				Director: func(req *http.Request) {
+					realTarget := req.Header.Get("Real-Target")
+					if realTarget != "" {
+						realTarget, err := url.PathUnescape(realTarget)
+						if err != nil {
+							log.Printf("Error decoding target URL: %v\n", err)
+							return
+						}
+						target, err := url.Parse(realTarget)
+						if err != nil {
+							log.Printf("Error parsing target URL: %v\n", err)
+							return
+						}
+						req.Header.Set("Accept", "*/*")
+						req.Header.Del("Origin")
+						req.Header.Del("Referer")
+						req.Header.Del("Real-Target")
+						req.Header.Del("Sec-Fetch-Dest")
+						req.Header.Del("Sec-Fetch-Mode")
+						req.Header.Del("Sec-Fetch-Site")
+						req.URL.Scheme = target.Scheme
+						req.URL.Host = target.Host
+						req.URL.Path = target.Path
+						req.URL.RawQuery = url.PathEscape(target.RawQuery)
+						log.Println("Proxying to", realTarget)
+					} else {
+						log.Println("Real-Target header is missing")
+					}
+				},
+			}
+			proxy.ServeHTTP(w, r)
+		}
+		http.HandleFunc("/", handler)
+		listener, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			return
+		}
+		a.proxyPort = listener.Addr().(*net.TCPAddr).Port
+
+		http.Serve(listener, nil)
+	}()
+}
+
 // startup is called when the app starts. The context is saved
 // so we can call the runtime methods
 func (a *App) OnStartup(ctx context.Context) {
@@ -76,6 +138,7 @@ func (a *App) OnStartup(ctx context.Context) {
 	a.midiLoop()
 	a.watchFs()
 	a.monitorHardware()
+	a.newFetchProxy()
 }
 
 func (a *App) OnBeforeClose(ctx context.Context) bool {
@@ -239,3 +302,7 @@ func (a *App) RestartApp() error {
 func (a *App) GetPlatform() string {
 	return runtime.GOOS
 }
+
+func (a *App) GetProxyPort() int {
+	return a.proxyPort
+}

backend-python/routes/completion.py

@@ -53,6 +53,9 @@ class ChatCompletionBody(ModelConfigBody):
     assistant_name: Union[str, None] = Field(
         None, description="Internal assistant name", min_length=1
     )
+    system_name: Union[str, None] = Field(
+        None, description="Internal system name", min_length=1
+    )
     presystem: bool = Field(
         True, description="Whether to insert default system prompt at the beginning"
     )
@@ -68,6 +71,7 @@ class ChatCompletionBody(ModelConfigBody):
                 "stop": None,
                 "user_name": None,
                 "assistant_name": None,
+                "system_name": None,
                 "presystem": True,
                 "max_tokens": 1000,
                 "temperature": 1,
@@ -252,20 +256,9 @@ async def eval_rwkv(
                 }
 
 
-@router.post("/v1/chat/completions", tags=["Completions"])
-@router.post("/chat/completions", tags=["Completions"])
-async def chat_completions(body: ChatCompletionBody, request: Request):
-    model: TextRWKV = global_var.get(global_var.Model)
-    if model is None:
-        raise HTTPException(status.HTTP_400_BAD_REQUEST, "model not loaded")
-
-    if body.messages is None or body.messages == []:
-        raise HTTPException(status.HTTP_400_BAD_REQUEST, "messages not found")
-
-    interface = model.interface
-    user = model.user if body.user_name is None else body.user_name
-    bot = model.bot if body.assistant_name is None else body.assistant_name
-
+def chat_template_old(
+    model: TextRWKV, body: ChatCompletionBody, interface: str, user: str, bot: str
+):
     is_raven = model.rwkv_type == RWKVType.Raven
 
     completion_text: str = ""
@@ -334,6 +327,53 @@ The following is a coherent verbose detailed conversation between a girl named {
         completion_text += append_message + "\n\n"
     completion_text += f"{bot}{interface}"
 
+    return completion_text
+
+
+def chat_template(
+    model: TextRWKV, body: ChatCompletionBody, interface: str, user: str, bot: str
+):
+    completion_text: str = ""
+    if body.presystem:
+        completion_text = (
+            f"{user}{interface} hi\n\n{bot}{interface} Hi. "
+            + "I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.\n\n"
+        )
+
+    system = "System" if body.system_name is None else body.system_name
+    for message in body.messages:
+        append_message: str = ""
+        if message.role == Role.User:
+            append_message = f"{user}{interface} " + message.content
+        elif message.role == Role.Assistant:
+            append_message = f"{bot}{interface} " + message.content
+        elif message.role == Role.System:
+            append_message = f"{system}{interface} " + message.content
+        completion_text += append_message + "\n\n"
+    completion_text += f"{bot}{interface}"
+
+    return completion_text
+
+
+@router.post("/v1/chat/completions", tags=["Completions"])
+@router.post("/chat/completions", tags=["Completions"])
+async def chat_completions(body: ChatCompletionBody, request: Request):
+    model: TextRWKV = global_var.get(global_var.Model)
+    if model is None:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, "model not loaded")
+
+    if body.messages is None or body.messages == []:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, "messages not found")
+
+    interface = model.interface
+    user = model.user if body.user_name is None else body.user_name
+    bot = model.bot if body.assistant_name is None else body.assistant_name
+
+    if model.version < 5:
+        completion_text = chat_template_old(model, body, interface, user, bot)
+    else:
+        completion_text = chat_template(model, body, interface, user, bot)
+
     user_code = model.pipeline.decode([model.pipeline.encode(user)[0]])
     bot_code = model.pipeline.decode([model.pipeline.encode(bot)[0]])
     if type(body.stop) == str:
@@ -343,8 +383,8 @@ The following is a coherent verbose detailed conversation between a girl named {
         body.stop.append(f"\n\n{bot_code}")
     elif body.stop is None:
         body.stop = default_stop
-    if not body.presystem:
-        body.stop.append("\n\n")
+    # if not body.presystem:
+    #     body.stop.append("\n\n")
 
     if body.stream:
         return EventSourceResponse(

backend-python/rwkv_pip/cpp/model.py

@@ -9,6 +9,9 @@ class RWKV:
         self.model = rwkv_cpp_model.RWKVModel(self.library, model_path)
         self.w = {}  # fake weight
         self.w["emb.weight"] = [0] * self.model.n_vocab
+        self.version = (
+            self.model.arch_version_major + self.model.arch_version_minor / 10
+        )
 
     def forward(self, tokens: List[int], state: Union[Any, None] = None):
         return self.model.eval_sequence_in_chunks(tokens, state, use_numpy=True)

backend-python/rwkv_pip/cpp/rwkv_cpp_model.py

@@ -52,9 +52,14 @@ class RWKVModel:
         if 'gpu_layers_count' in kwargs:
             gpu_layer_count = kwargs['gpu_layers_count']
 
-        assert os.path.isfile(model_path), f'{model_path} is not a file'
-        assert thread_count > 0, 'Thread count must be > 0'
-        assert gpu_layer_count >= 0, 'GPU layer count must be >= 0'
+        if not os.path.isfile(model_path):
+            raise ValueError(f'{model_path} is not a file')
+        
+        if not (thread_count > 0):
+            raise ValueError('Thread count must be > 0')  
+
+        if not (gpu_layer_count >= 0):
+            raise ValueError('GPU layer count must be >= 0')
 
         self._library: rwkv_cpp_shared_library.RWKVSharedLibrary = shared_library
 
@@ -84,10 +89,19 @@ class RWKVModel:
             Count of layers to offload onto the GPU, must be >= 0.
         """
 
-        assert layer_count >= 0, 'Layer count must be >= 0'
+        if not (layer_count >= 0):
+            raise ValueError('Layer count must be >= 0')
 
         return self._library.rwkv_gpu_offload_layers(self._ctx, layer_count)
 
+    @property
+    def arch_version_major(self) -> int:
+        return self._library.rwkv_get_arch_version_major(self._ctx)
+
+    @property
+    def arch_version_minor(self) -> int:
+        return self._library.rwkv_get_arch_version_minor(self._ctx)
+
     @property
     def n_vocab(self) -> int:
         return self._library.rwkv_get_n_vocab(self._ctx)
@@ -133,7 +147,8 @@ class RWKVModel:
             Logits vector of shape (n_vocab); state for the next step.
         """
 
-        assert self._valid, 'Model was freed'
+        if not self._valid:
+            raise ValueError('Model was freed')
 
         use_numpy = self._detect_numpy_usage([state_in, state_out, logits_out], use_numpy)
 
@@ -207,7 +222,8 @@ class RWKVModel:
             Logits vector of shape (n_vocab); state for the next step.
         """
 
-        assert self._valid, 'Model was freed'
+        if not self._valid:
+            raise ValueError('Model was freed')
 
         use_numpy = self._detect_numpy_usage([state_in, state_out, logits_out], use_numpy)
 
@@ -281,7 +297,8 @@ class RWKVModel:
             Logits vector of shape (n_vocab); state for the next step.
         """
 
-        assert self._valid, 'Model was freed'
+        if not self._valid:
+            raise ValueError('Model was freed')
 
         use_numpy = self._detect_numpy_usage([state_in, state_out, logits_out], use_numpy)
 
@@ -320,7 +337,8 @@ class RWKVModel:
         The object must not be used anymore after calling this method.
         """
 
-        assert self._valid, 'Already freed'
+        if not self._valid:
+            raise ValueError('Already freed')
 
         self._valid = False
 
@@ -344,16 +362,25 @@ class RWKVModel:
     def _validate_tensor(self, tensor: NumpyArrayOrPyTorchTensor, name: str, size: int) -> None:
         if self._is_pytorch_tensor(tensor):
             tensor: torch.Tensor = tensor
-            assert tensor.device == torch.device('cpu'), f'{name} is not on CPU'
-            assert tensor.dtype == torch.float32, f'{name} is not of type float32'
-            assert tensor.shape == (size,), f'{name} has invalid shape {tensor.shape}, expected ({size})'
-            assert tensor.is_contiguous(), f'{name} is not contiguous'
+            
+            if tensor.device != torch.device('cpu'):
+                raise ValueError(f'{name} is not on CPU')
+            if tensor.dtype != torch.float32:
+                raise ValueError(f'{name} is not of type float32')
+            if tensor.shape != (size,):
+                raise ValueError(f'{name} has invalid shape {tensor.shape}, expected ({size})')
+            if not tensor.is_contiguous():
+                raise ValueError(f'{name} is not contiguous')
         else:
             import numpy as np
             tensor: np.ndarray = tensor
-            assert tensor.dtype == np.float32, f'{name} is not of type float32'
-            assert tensor.shape == (size,), f'{name} has invalid shape {tensor.shape}, expected ({size})'
-            assert tensor.data.contiguous, f'{name} is not contiguous'
+
+            if tensor.dtype != np.float32:
+                raise ValueError(f'{name} is not of type float32')
+            if tensor.shape != (size,):
+                raise ValueError(f'{name} has invalid shape {tensor.shape}, expected ({size})')
+            if not tensor.data.contiguous:
+                raise ValueError(f'{name} is not contiguous')
 
     def _get_data_ptr(self, tensor: NumpyArrayOrPyTorchTensor):
         if self._is_pytorch_tensor(tensor):

backend-python/rwkv_pip/cpp/rwkv_cpp_shared_library.py

@@ -6,21 +6,22 @@ import platform
 from typing import Optional, List, Tuple, Callable
 
 QUANTIZED_FORMAT_NAMES: Tuple[str, str, str, str, str] = (
-    'Q4_0',
-    'Q4_1',
-    'Q5_0',
-    'Q5_1',
-    'Q8_0'
+    "Q4_0",
+    "Q4_1",
+    "Q5_0",
+    "Q5_1",
+    "Q8_0",
 )
 
 P_FLOAT = ctypes.POINTER(ctypes.c_float)
 P_INT = ctypes.POINTER(ctypes.c_int32)
 
-class RWKVContext:
 
+class RWKVContext:
     def __init__(self, ptr: ctypes.pointer) -> None:
         self.ptr: ctypes.pointer = ptr
 
+
 class RWKVSharedLibrary:
     """
     Python wrapper around rwkv.cpp shared library.
@@ -39,7 +40,7 @@ class RWKVSharedLibrary:
         #  When Python is greater than 3.8, we need to reprocess the custom dll
         #  according to the documentation to prevent loading failure errors.
         #  https://docs.python.org/3/whatsnew/3.8.html#ctypes
-        if platform.system().lower() == 'windows':
+        if platform.system().lower() == "windows":
             self.library = ctypes.CDLL(shared_library_path, winmode=0)
         else:
             self.library = ctypes.cdll.LoadLibrary(shared_library_path)
@@ -47,39 +48,48 @@ class RWKVSharedLibrary:
         self.library.rwkv_init_from_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32]
         self.library.rwkv_init_from_file.restype = ctypes.c_void_p
 
-        self.library.rwkv_gpu_offload_layers.argtypes = [ctypes.c_void_p, ctypes.c_uint32]
+        self.library.rwkv_gpu_offload_layers.argtypes = [
+            ctypes.c_void_p,
+            ctypes.c_uint32,
+        ]
         self.library.rwkv_gpu_offload_layers.restype = ctypes.c_bool
 
         self.library.rwkv_eval.argtypes = [
-            ctypes.c_void_p, # ctx
-            ctypes.c_int32, # token
-            P_FLOAT, # state_in
-            P_FLOAT, # state_out
-            P_FLOAT  # logits_out
+            ctypes.c_void_p,  # ctx
+            ctypes.c_int32,  # token
+            P_FLOAT,  # state_in
+            P_FLOAT,  # state_out
+            P_FLOAT,  # logits_out
         ]
         self.library.rwkv_eval.restype = ctypes.c_bool
 
         self.library.rwkv_eval_sequence.argtypes = [
-            ctypes.c_void_p, # ctx
-            P_INT, # tokens
-            ctypes.c_size_t, # token count
-            P_FLOAT, # state_in
-            P_FLOAT, # state_out
-            P_FLOAT  # logits_out
+            ctypes.c_void_p,  # ctx
+            P_INT,  # tokens
+            ctypes.c_size_t,  # token count
+            P_FLOAT,  # state_in
+            P_FLOAT,  # state_out
+            P_FLOAT,  # logits_out
         ]
         self.library.rwkv_eval_sequence.restype = ctypes.c_bool
 
         self.library.rwkv_eval_sequence_in_chunks.argtypes = [
-            ctypes.c_void_p, # ctx
-            P_INT, # tokens
-            ctypes.c_size_t, # token count
-            ctypes.c_size_t, # chunk size
-            P_FLOAT, # state_in
-            P_FLOAT, # state_out
-            P_FLOAT  # logits_out
+            ctypes.c_void_p,  # ctx
+            P_INT,  # tokens
+            ctypes.c_size_t,  # token count
+            ctypes.c_size_t,  # chunk size
+            P_FLOAT,  # state_in
+            P_FLOAT,  # state_out
+            P_FLOAT,  # logits_out
         ]
         self.library.rwkv_eval_sequence_in_chunks.restype = ctypes.c_bool
 
+        self.library.rwkv_get_arch_version_major.argtypes = [ctypes.c_void_p]
+        self.library.rwkv_get_arch_version_major.restype = ctypes.c_uint32
+
+        self.library.rwkv_get_arch_version_minor.argtypes = [ctypes.c_void_p]
+        self.library.rwkv_get_arch_version_minor.restype = ctypes.c_uint32
+
         self.library.rwkv_get_n_vocab.argtypes = [ctypes.c_void_p]
         self.library.rwkv_get_n_vocab.restype = ctypes.c_size_t
 
@@ -101,7 +111,11 @@ class RWKVSharedLibrary:
         self.library.rwkv_free.argtypes = [ctypes.c_void_p]
         self.library.rwkv_free.restype = None
 
-        self.library.rwkv_quantize_model_file.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p]
+        self.library.rwkv_quantize_model_file.argtypes = [
+            ctypes.c_char_p,
+            ctypes.c_char_p,
+            ctypes.c_char_p,
+        ]
         self.library.rwkv_quantize_model_file.restype = ctypes.c_bool
 
         self.library.rwkv_get_system_info_string.argtypes = []
@@ -109,7 +123,9 @@ class RWKVSharedLibrary:
 
         self.nullptr = ctypes.cast(0, ctypes.c_void_p)
 
-    def rwkv_init_from_file(self, model_file_path: str, thread_count: int) -> RWKVContext:
+    def rwkv_init_from_file(
+        self, model_file_path: str, thread_count: int
+    ) -> RWKVContext:
         """
         Loads the model from a file and prepares it for inference.
         Throws an exception in case of any error. Error messages would be printed to stderr.
@@ -122,9 +138,12 @@ class RWKVSharedLibrary:
             Count of threads to use, must be positive.
         """
 
-        ptr = self.library.rwkv_init_from_file(model_file_path.encode('utf-8'), ctypes.c_uint32(thread_count))
+        ptr = self.library.rwkv_init_from_file(
+            model_file_path.encode("utf-8"), ctypes.c_uint32(thread_count)
+        )
 
-        assert ptr is not None, 'rwkv_init_from_file failed, check stderr'
+        if ptr is None:
+            raise ValueError("rwkv_init_from_file failed, check stderr")
 
         return RWKVContext(ptr)
 
@@ -145,17 +164,20 @@ class RWKVSharedLibrary:
             Count of layers to offload onto the GPU, must be >= 0.
         """
 
-        assert layer_count >= 0, 'Layer count must be >= 0'
+        if not (layer_count >= 0):
+            raise ValueError("Layer count must be >= 0")
 
-        return self.library.rwkv_gpu_offload_layers(ctx.ptr, ctypes.c_uint32(layer_count))
+        return self.library.rwkv_gpu_offload_layers(
+            ctx.ptr, ctypes.c_uint32(layer_count)
+        )
 
     def rwkv_eval(
-            self,
-            ctx: RWKVContext,
-            token: int,
-            state_in_address: Optional[int],
-            state_out_address: int,
-            logits_out_address: int
+        self,
+        ctx: RWKVContext,
+        token: int,
+        state_in_address: Optional[int],
+        state_out_address: int,
+        logits_out_address: int,
     ) -> None:
         """
         Evaluates the model for a single token.
@@ -176,21 +198,22 @@ class RWKVSharedLibrary:
             Address of the first element of a FP32 buffer of size rwkv_get_logits_buffer_element_count. This buffer will be written to.
         """
 
-        assert self.library.rwkv_eval(
+        if not self.library.rwkv_eval(
             ctx.ptr,
             ctypes.c_int32(token),
             ctypes.cast(0 if state_in_address is None else state_in_address, P_FLOAT),
             ctypes.cast(state_out_address, P_FLOAT),
-            ctypes.cast(logits_out_address, P_FLOAT)
-        ), 'rwkv_eval failed, check stderr'
+            ctypes.cast(logits_out_address, P_FLOAT),
+        ):
+            raise ValueError("rwkv_eval failed, check stderr")
 
     def rwkv_eval_sequence(
-            self,
-            ctx: RWKVContext,
-            tokens: List[int],
-            state_in_address: Optional[int],
-            state_out_address: int,
-            logits_out_address: int
+        self,
+        ctx: RWKVContext,
+        tokens: List[int],
+        state_in_address: Optional[int],
+        state_out_address: int,
+        logits_out_address: int,
     ) -> None:
         """
         Evaluates the model for a sequence of tokens.
@@ -223,23 +246,24 @@ class RWKVSharedLibrary:
             Address of the first element of a FP32 buffer of size rwkv_get_logits_buffer_element_count. This buffer will be written to.
         """
 
-        assert self.library.rwkv_eval_sequence(
+        if not self.library.rwkv_eval_sequence(
             ctx.ptr,
             ctypes.cast((ctypes.c_int32 * len(tokens))(*tokens), P_INT),
             ctypes.c_size_t(len(tokens)),
             ctypes.cast(0 if state_in_address is None else state_in_address, P_FLOAT),
             ctypes.cast(state_out_address, P_FLOAT),
-            ctypes.cast(logits_out_address, P_FLOAT)
-        ), 'rwkv_eval_sequence failed, check stderr'
+            ctypes.cast(logits_out_address, P_FLOAT),
+        ):
+            raise ValueError("rwkv_eval_sequence failed, check stderr")
 
     def rwkv_eval_sequence_in_chunks(
-            self,
-            ctx: RWKVContext,
-            tokens: List[int],
-            chunk_size: int,
-            state_in_address: Optional[int],
-            state_out_address: int,
-            logits_out_address: int
+        self,
+        ctx: RWKVContext,
+        tokens: List[int],
+        chunk_size: int,
+        state_in_address: Optional[int],
+        state_out_address: int,
+        logits_out_address: int,
     ) -> None:
         """
         Evaluates the model for a sequence of tokens using `rwkv_eval_sequence`, splitting a potentially long sequence into fixed-length chunks.
@@ -269,15 +293,40 @@ class RWKVSharedLibrary:
             Address of the first element of a FP32 buffer of size rwkv_get_logits_buffer_element_count. This buffer will be written to.
         """
 
-        assert self.library.rwkv_eval_sequence_in_chunks(
+        if not self.library.rwkv_eval_sequence_in_chunks(
             ctx.ptr,
             ctypes.cast((ctypes.c_int32 * len(tokens))(*tokens), P_INT),
             ctypes.c_size_t(len(tokens)),
             ctypes.c_size_t(chunk_size),
             ctypes.cast(0 if state_in_address is None else state_in_address, P_FLOAT),
             ctypes.cast(state_out_address, P_FLOAT),
-            ctypes.cast(logits_out_address, P_FLOAT)
-        ), 'rwkv_eval_sequence_in_chunks failed, check stderr'
+            ctypes.cast(logits_out_address, P_FLOAT),
+        ):
+            raise ValueError("rwkv_eval_sequence_in_chunks failed, check stderr")
+
+    def rwkv_get_arch_version_major(self, ctx: RWKVContext) -> int:
+        """
+        Returns the major version used by the given model.
+
+        Parameters
+        ----------
+        ctx : RWKVContext
+            RWKV context obtained from rwkv_init_from_file.
+        """
+
+        return self.library.rwkv_get_arch_version_major(ctx.ptr)
+
+    def rwkv_get_arch_version_minor(self, ctx: RWKVContext) -> int:
+        """
+        Returns the minor version used by the given model.
+
+        Parameters
+        ----------
+        ctx : RWKVContext
+            RWKV context obtained from rwkv_init_from_file.
+        """
+
+        return self.library.rwkv_get_arch_version_minor(ctx.ptr)
 
     def rwkv_get_n_vocab(self, ctx: RWKVContext) -> int:
         """
@@ -358,7 +407,9 @@ class RWKVSharedLibrary:
 
         ctx.ptr = self.nullptr
 
-    def rwkv_quantize_model_file(self, model_file_path_in: str, model_file_path_out: str, format_name: str) -> None:
+    def rwkv_quantize_model_file(
+        self, model_file_path_in: str, model_file_path_out: str, format_name: str
+    ) -> None:
         """
         Quantizes FP32 or FP16 model to one of INT4 formats.
         Throws an exception in case of any error. Error messages would be printed to stderr.
@@ -373,20 +424,25 @@ class RWKVSharedLibrary:
             One of QUANTIZED_FORMAT_NAMES.
         """
 
-        assert format_name in QUANTIZED_FORMAT_NAMES, f'Unknown format name {format_name}, use one of {QUANTIZED_FORMAT_NAMES}'
+        if format_name not in QUANTIZED_FORMAT_NAMES:
+            raise ValueError(
+                f"Unknown format name {format_name}, use one of {QUANTIZED_FORMAT_NAMES}"
+            )
 
-        assert self.library.rwkv_quantize_model_file(
-            model_file_path_in.encode('utf-8'),
-            model_file_path_out.encode('utf-8'),
-            format_name.encode('utf-8')
-        ), 'rwkv_quantize_model_file failed, check stderr'
+        if not self.library.rwkv_quantize_model_file(
+            model_file_path_in.encode("utf-8"),
+            model_file_path_out.encode("utf-8"),
+            format_name.encode("utf-8"),
+        ):
+            raise ValueError("rwkv_quantize_model_file failed, check stderr")
 
     def rwkv_get_system_info_string(self) -> str:
         """
         Returns system information string.
         """
 
-        return self.library.rwkv_get_system_info_string().decode('utf-8')
+        return self.library.rwkv_get_system_info_string().decode("utf-8")
+
 
 def load_rwkv_shared_library() -> RWKVSharedLibrary:
     """
@@ -396,27 +452,27 @@ def load_rwkv_shared_library() -> RWKVSharedLibrary:
 
     file_name: str
 
-    if 'win32' in sys.platform or 'cygwin' in sys.platform:
-        file_name = 'rwkv.dll'
-    elif 'darwin' in sys.platform:
-        file_name = 'librwkv.dylib'
+    if "win32" in sys.platform or "cygwin" in sys.platform:
+        file_name = "rwkv.dll"
+    elif "darwin" in sys.platform:
+        file_name = "librwkv.dylib"
     else:
-        file_name = 'librwkv.so'
+        file_name = "librwkv.so"
 
     # Possible sub-paths to the library relative to the repo dir.
     child_paths: List[Callable[[pathlib.Path], pathlib.Path]] = [
         # No lookup for Debug config here.
         # I assume that if a user wants to debug the library,
         # they will be able to find the library and set the exact path explicitly.
-        lambda p: p / 'backend-python' / 'rwkv_pip' / 'cpp' / file_name,
-        lambda p: p / 'bin' / 'Release' / file_name,
-        lambda p: p / 'bin' / file_name,
+        lambda p: p / "backend-python" / "rwkv_pip" / "cpp" / file_name,
+        lambda p: p / "bin" / "Release" / file_name,
+        lambda p: p / "bin" / file_name,
         # Some people prefer to build in the "build" subdirectory.
-        lambda p: p / 'build' / 'bin' / 'Release' / file_name,
-        lambda p: p / 'build' / 'bin' / file_name,
-        lambda p: p / 'build' / file_name,
+        lambda p: p / "build" / "bin" / "Release" / file_name,
+        lambda p: p / "build" / "bin" / file_name,
+        lambda p: p / "build" / file_name,
         # Fallback.
-        lambda p: p / file_name
+        lambda p: p / file_name,
     ]
 
     working_dir: pathlib.Path = pathlib.Path(os.path.abspath(os.getcwd()))
@@ -430,7 +486,7 @@ def load_rwkv_shared_library() -> RWKVSharedLibrary:
         # .
         working_dir,
         # Repo dir relative to this Python file.
-        pathlib.Path(os.path.abspath(__file__)).parent.parent.parent
+        pathlib.Path(os.path.abspath(__file__)).parent.parent.parent,
     ]
 
     for parent_path in parent_paths:
@@ -440,5 +496,7 @@ def load_rwkv_shared_library() -> RWKVSharedLibrary:
             if os.path.isfile(full_path):
                 return RWKVSharedLibrary(str(full_path))
 
-    assert False, (f'Failed to find {file_name} automatically; '
-                   f'you need to find the library and create RWKVSharedLibrary specifying the path to it')
+    raise ValueError(
+        f"Failed to find {file_name} automatically; "
+        f"you need to find the library and create RWKVSharedLibrary specifying the path to it"
+    )

backend-python/rwkv_pip/webgpu/model.py

@@ -18,6 +18,7 @@ class RWKV:
         self.w["emb.weight"] = [0] * self.info.num_vocab
         self.version = str(self.info.version).lower()
         self.wrp = getattr(wrp, self.version)
+        self.version = float(self.version.replace("v", ""))
 
         layer = (
             int(s.lstrip("layer"))

backend-python/utils/rwkv.py

@@ -26,6 +26,7 @@ class AbstractRWKV(ABC):
         self.EOS_ID = 0
 
         self.name = "rwkv"
+        self.version = 4
         self.model = model
         self.pipeline = pipeline
         self.model_state = None
@@ -665,6 +666,7 @@ def RWKV(model: str, strategy: str, tokenizer: Union[str, None]) -> AbstractRWKV
         else:
             rwkv = TextRWKV(model, pipeline)
     rwkv.name = filename
+    rwkv.version = model.version
 
     return rwkv
 

finetune/lora/v6/src/model.py

@@ -29,7 +29,6 @@ LORA_CONFIG = {
 
 
 class LoraLinear(nn.Module):
-
     def __init__(self, in_features: int, out_features: int, bias: bool):
         super().__init__()
 
@@ -356,8 +355,8 @@ class RWKV_TimeMix_RWKV5(MyModule):
         self.key = make_linear_att(args.n_embd, args.dim_att, bias=False)
         self.value = make_linear_att(args.n_embd, args.dim_att, bias=False)
 
-        self.output = nn.Linear(args.dim_att, args.n_embd, bias=False)
-        self.gate = nn.Linear(args.n_embd, args.dim_att, bias=False)
+        self.output = make_linear_att(args.dim_att, args.n_embd, bias=False)
+        self.gate = make_linear_att(args.n_embd, args.dim_att, bias=False)
         self.ln_x = nn.GroupNorm(self.n_head, args.dim_att)
 
     @MyFunction
@@ -465,8 +464,8 @@ class RWKV_Tmix_x060(MyModule):
         self.receptance = make_linear_att(args.n_embd, args.dim_att, bias=False)
         self.key = make_linear_att(args.n_embd, args.dim_att, bias=False)
         self.value = make_linear_att(args.n_embd, args.dim_att, bias=False)
-        self.output = nn.Linear(args.dim_att, args.n_embd, bias=False)
-        self.gate = nn.Linear(args.n_embd, args.dim_att, bias=False)
+        self.output = make_linear_att(args.dim_att, args.n_embd, bias=False)
+        self.gate = make_linear_att(args.n_embd, args.dim_att, bias=False)
         self.ln_x = nn.GroupNorm(
             self.n_head, args.dim_att, eps=(1e-5) * (args.head_size_divisor**2)
         )

frontend/package.json

@@ -9,7 +9,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@fluentui/react-components": "^9.20.0",
+    "@fluentui/react-components": "^9.47.2",
     "@fluentui/react-icons": "^2.0.201",
     "@magenta/music": "^1.23.1",
     "@microsoft/fetch-event-source": "^2.0.1",

frontend/src/App.tsx

@@ -83,7 +83,7 @@ const App: FC = observer(() => {
             ))}
           </TabList>
         </div>
-        <div className="h-full w-full p-2 box-border overflow-y-hidden">
+        <div className="h-full w-full py-2 pr-2 box-border overflow-y-hidden">
           <Routes>
             {pages.map(({ path, element }, index) => (
               <Route key={`${path}-${index}`} path={path} element={<LazyImportComponent lazyChildren={element} />} />

frontend/src/_locales/ja/main.json

@@ -351,5 +351,8 @@
   "Create a new user or AI message content. You can prepare a chat record with AI here, and fill in the responses you want to get from AI in the tone of AI. When you use this preset, the chat record will be processed, and at this point, AI will better understand what you want it to do or what role to play.": "新しいユーザーまたはAIメッセージコンテンツを作成します。ここでAIとのチャット記録を準備し、AIから得たい応答をAIのトーンで記入することができます。このプリセットを使用すると、チャット記録が処理され、この時点でAIはあなたが望むことやどのような役割を果たすかをよりよく理解することができます。",
   "The name used internally by the model when processing user message, changing this value helps improve the role-playing effect.": "ユーザーメッセージを処理する際にモデルが内部で使用する名前、この値を変更することで、役割演技の効果を向上させることができます。",
   "The name used internally by the model when processing AI message, changing this value helps improve the role-playing effect.": "AIメッセージを処理する際にモデルが内部で使用する名前、この値を変更することで、役割演技の効果を向上させることができます。",
-  "Inside the model, there is a default prompt to improve the model's handling of common issues, but it may degrade the role-playing effect. You can disable this option to achieve a better role-playing effect.": "モデル内部には、一般的な問題の処理を改善するためのデフォルトのプロンプトがありますが、役割演技の効果を低下させる可能性があります。このオプションを無効にすることで、より良い役割演技効果を得ることができます。"
+  "Inside the model, there is a default prompt to improve the model's handling of common issues, but it may degrade the role-playing effect. You can disable this option to achieve a better role-playing effect.": "モデル内部には、一般的な問題の処理を改善するためのデフォルトのプロンプトがありますが、役割演技の効果を低下させる可能性があります。このオプションを無効にすることで、より良い役割演技効果を得ることができます。",
+  "Exit without saving": "保存せずに終了",
+  "Content has been changed, are you sure you want to exit without saving?": "コンテンツが変更されています、保存せずに終了してもよろしいですか？",
+  "Don't forget to correctly fill in your Ollama API Chat Model Name.": "Ollama APIチャットモデル名を正しく記入するのを忘れないでください。"
 }

frontend/src/_locales/zh-hans/main.json

@@ -351,5 +351,8 @@
   "Create a new user or AI message content. You can prepare a chat record with AI here, and fill in the responses you want to get from AI in the tone of AI. When you use this preset, the chat record will be processed, and at this point, AI will better understand what you want it to do or what role to play.": "新建一个 用户 或 AI 的发言内容. 你可以在这里准备好一段你与 AI 的聊天记录, 并用 AI 的口吻正确填写你想得到的 AI 的回复, 这样你在使用这个预设时, 这些聊天记录也会被处理, 并且此时 AI 能更好地理解你希望它做什么 / 扮演什么样的角色",
   "The name used internally by the model when processing user message, changing this value helps improve the role-playing effect.": "模型内部处理用户发言时使用的名称, 更改此值有助于改善角色扮演效果",
   "The name used internally by the model when processing AI message, changing this value helps improve the role-playing effect.": "模型内部处理AI发言时使用的名称, 更改此值有助于改善角色扮演效果",
-  "Inside the model, there is a default prompt to improve the model's handling of common issues, but it may degrade the role-playing effect. You can disable this option to achieve a better role-playing effect.": "模型内部有一个默认提示来改善模型处理常规问题的效果, 但它可能会让角色扮演的效果变差, 你可以关闭此选项来获得更好的角色扮演效果"
+  "Inside the model, there is a default prompt to improve the model's handling of common issues, but it may degrade the role-playing effect. You can disable this option to achieve a better role-playing effect.": "模型内部有一个默认提示来改善模型处理常规问题的效果, 但它可能会让角色扮演的效果变差, 你可以关闭此选项来获得更好的角色扮演效果",
+  "Exit without saving": "退出而不保存",
+  "Content has been changed, are you sure you want to exit without saving?": "内容已经被修改, 你确定要退出而不保存吗?",
+  "Don't forget to correctly fill in your Ollama API Chat Model Name.": "不要忘记正确填写你的Ollama API 聊天模型名"
 }

frontend/src/components/DialogButton.tsx

@@ -16,20 +16,28 @@ import { LazyImportComponent } from './LazyImportComponent';
 const MarkdownRender = React.lazy(() => import('./MarkdownRender'));
 
 export const DialogButton: FC<{
-  text?: string | null
+  text?: string | null,
   icon?: ReactElement,
   tooltip?: string | null,
   className?: string,
   title: string,
-  contentText: string,
+  content?: string | ReactElement | null
   markdown?: boolean,
   onConfirm?: () => void,
   size?: 'small' | 'medium' | 'large',
   shape?: 'rounded' | 'circular' | 'square',
   appearance?: 'secondary' | 'primary' | 'outline' | 'subtle' | 'transparent',
+  cancelButton?: boolean,
+  confirmButton?: boolean,
+  cancelButtonText?: string,
+  confirmButtonText?: string,
 }> = ({
-  text, icon, tooltip, className, title, contentText, markdown,
-  onConfirm, size, shape, appearance
+  text, icon, tooltip, className, title, content, markdown,
+  onConfirm, size, shape, appearance,
+  cancelButton = true,
+  confirmButton = true,
+  cancelButtonText = 'Cancel',
+  confirmButtonText = 'Confirm'
 }) => {
   const { t } = useTranslation();
 
@@ -41,26 +49,31 @@ export const DialogButton: FC<{
         <Button className={className} icon={icon} size={size} shape={shape} appearance={appearance}>{text}</Button>
       }
     </DialogTrigger>
-    <DialogSurface>
+    <DialogSurface style={{ transform: 'unset' }}>
       <DialogBody>
         <DialogTitle>{title}</DialogTitle>
         <DialogContent>
           {
             markdown ?
               <LazyImportComponent lazyChildren={MarkdownRender}>
-                {contentText}
+                {content}
               </LazyImportComponent> :
-              contentText
+              content
           }
         </DialogContent>
         <DialogActions>
-          <DialogTrigger disableButtonEnhancement>
-            <Button appearance="secondary">{t('Cancel')}</Button>
-          </DialogTrigger>
-          <DialogTrigger disableButtonEnhancement>
-            <Button appearance="primary" onClick={onConfirm}>{t('Confirm')}
-            </Button>
-          </DialogTrigger>
+          {cancelButton && (
+            <DialogTrigger disableButtonEnhancement>
+              <Button appearance="secondary">{t(cancelButtonText)}</Button>
+            </DialogTrigger>
+          )}
+          {confirmButton && (
+            <DialogTrigger disableButtonEnhancement>
+              <Button appearance="primary" onClick={onConfirm}>
+                {t(confirmButtonText)}
+              </Button>
+            </DialogTrigger>
+          )}
         </DialogActions>
       </DialogBody>
     </DialogSurface>

frontend/src/components/MarkdownRender.tsx

@@ -28,10 +28,10 @@ const MarkdownRender: FC<ReactMarkdownOptions & { disabled?: boolean }> = (props
   return (
     <div dir="auto" className="prose markdown-body" style={{ maxWidth: '100%' }}>
       {props.disabled ?
-        <div style={{ whiteSpace: 'pre-wrap' }}>
+        <div style={{ whiteSpace: 'pre-wrap' }} className={props.className}>
           {props.children}
         </div> :
-        <ReactMarkdown
+        <ReactMarkdown className={props.className}
           allowedElements={[
             'div',
             'p',

frontend/src/components/ResetConfigsButton.tsx

@@ -9,7 +9,7 @@ import { defaultModelConfigs, defaultModelConfigsMac } from '../pages/defaultCon
 export const ResetConfigsButton: FC<{ afterConfirm?: () => void }> = ({ afterConfirm }) => {
   const { t } = useTranslation();
   return <DialogButton icon={<ArrowReset20Regular />} tooltip={t('Reset All Configs')} title={t('Reset All Configs')}
-    contentText={t('Are you sure you want to reset all configs? This will obtain the latest preset configs, but will override your custom configs and cannot be undone.')}
+    content={t('Are you sure you want to reset all configs? This will obtain the latest preset configs, but will override your custom configs and cannot be undone.')}
     onConfirm={() => {
       commonStore.setModelConfigs(commonStore.platform !== 'darwin' ? defaultModelConfigs : defaultModelConfigsMac, false);
       commonStore.setCurrentConfigIndex(0, true);

frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx

@@ -28,7 +28,7 @@ export const AudiotrackButton: FC<{
         {t('Open MIDI Input Audio Tracks')}
       </Button>
     </DialogTrigger>
-    <DialogSurface style={{ paddingTop: 0, maxWidth: '90vw', width: 'fit-content' }}>
+    <DialogSurface style={{ paddingTop: 0, maxWidth: '90vw', width: 'fit-content', transform: 'unset' }}>
       <DialogBody>
         <DialogContent className="overflow-hidden">
           <CustomToastContainer />

frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx

@@ -32,7 +32,7 @@ import {
   flushMidiRecordingContent,
   getMidiRawContentMainInstrument,
   getMidiRawContentTime,
-  getServerRoot,
+  getReqUrl,
   OpenFileDialog,
   refreshTracksTotalTime
 } from '../../utils';
@@ -474,8 +474,13 @@ const AudiotrackEditor: FC<{ setPrompt: (prompt: string) => void }> = observer((
                 OpenFileDialog('*.mid').then(async blob => {
                   const bodyForm = new FormData();
                   bodyForm.append('file_data', blob);
-                  fetch(getServerRoot(commonStore.getCurrentModelConfig().apiParameters.apiPort) + '/midi-to-text', {
+                  const {
+                    url,
+                    headers
+                  } = await getReqUrl(commonStore.getCurrentModelConfig().apiParameters.apiPort, '/midi-to-text');
+                  fetch(url, {
                     method: 'POST',
+                    headers,
                     body: bodyForm
                   }).then(async r => {
                       if (r.status === 200) {
@@ -494,7 +499,7 @@ const AudiotrackEditor: FC<{ setPrompt: (prompt: string) => void }> = observer((
                         commonStore.setTracks(tracks);
                         refreshTracksTotalTime();
                       } else {
-                        toast(r.statusText + '\n' + (await r.text()), {
+                        toast('Failed to fetch - ' + r.status + ' - ' + r.statusText + ' - ' + (await r.text()), {
                           type: 'error'
                         });
                       }

frontend/src/pages/Chat.tsx

@@ -41,19 +41,20 @@ import { OpenFileFolder, OpenOpenFileDialog, OpenSaveFileDialog } from '../../wa
 import {
   absPathAsset,
   bytesToReadable,
-  getServerRoot,
+  getReqUrl,
   newChatConversation,
   OpenFileDialog,
   setActivePreset,
   toastWithButton
 } from '../utils';
 import { useMediaQuery } from 'usehooks-ts';
-import { botName, ConversationMessage, MessageType, Role, userName, welcomeUuid } from '../types/chat';
+import { botName, ConversationMessage, MessageType, Role, systemName, userName, welcomeUuid } from '../types/chat';
 import { Labeled } from '../components/Labeled';
 import { ValuedSlider } from '../components/ValuedSlider';
 import { PresetsButton } from './PresetsManager/PresetsButton';
 import { webOpenOpenFileDialog } from '../utils/web-file-operations';
 import { defaultPenaltyDecay } from './defaultConfigs';
+import { AvatarProps } from '@fluentui/react-avatar';
 
 let chatSseControllers: {
   [id: string]: AbortController
@@ -92,6 +93,18 @@ const MoreUtilsButton: FC<{
   </Menu>;
 });
 
+const HiddenAvatar: FC<AvatarProps> = ({ children, hidden, ...props }) => {
+  if (hidden) {
+    return <div>{children}</div>;
+  } else {
+    return (
+      <Avatar {...props}>
+        {children}
+      </Avatar>
+    );
+  }
+};
+
 const ChatMessageItem: FC<{
   uuid: string,
   onSubmit: (message: string | null, answerId: string | null,
@@ -129,7 +142,9 @@ const ChatMessageItem: FC<{
   return <div
     className={classnames(
       'flex gap-2 mb-2 overflow-hidden',
-      messageItem.side === 'left' ? 'flex-row' : 'flex-row-reverse'
+      messageItem.side === 'left' && 'flex-row',
+      messageItem.side === 'right' && 'flex-row-reverse',
+      messageItem.side === 'center' && 'flex-row justify-center'
     )}
     onMouseEnter={() => {
       const utils = document.getElementById('utils-' + uuid);
@@ -140,22 +155,26 @@ const ChatMessageItem: FC<{
       if (utils) utils.classList.add('invisible');
     }}
   >
-    <Avatar
+    <HiddenAvatar
       color={messageItem.color}
       name={messageItem.sender}
       image={avatarImg ? { src: avatarImg } : undefined}
+      hidden={messageItem.side === 'center'}
     />
     <div
       className={classnames(
-        'flex p-2 rounded-lg overflow-hidden',
+        'flex rounded-lg overflow-hidden',
         editing ? 'grow' : '',
-        commonStore.settings.darkMode ? 'bg-neutral-800 border-neutral-600 border-[1px]' : (messageItem.side === 'left' ? 'bg-gray-200' : 'bg-blue-500'),
-        commonStore.settings.darkMode ? 'text-white' : (messageItem.side === 'left' ? 'text-gray-600' : 'text-white')
+        messageItem.side === 'center' ? 'p-1' : 'p-2',
+        commonStore.settings.darkMode ? 'bg-neutral-800 border-neutral-600 border-[1px]' : (messageItem.side === 'right' ? 'bg-blue-500' : 'bg-gray-200'),
+        commonStore.settings.darkMode ? 'text-white' : (messageItem.side === 'right' ? 'text-white' : 'text-gray-600'),
+        messageItem.side === 'center' && 'text-opacity-60 bg-opacity-30 border-opacity-30'
       )}
     >
       {!editing ?
         <div className="flex flex-col">
-          <MarkdownRender disabled={!commonStore.chatParams.markdown}>{messageItem.content}</MarkdownRender>
+          <MarkdownRender className={classnames(messageItem.side === 'center' && 'text-xs')}
+            disabled={!commonStore.chatParams.markdown || messageItem.side === 'center'}>{messageItem.content}</MarkdownRender>
           {uuid in commonStore.attachments &&
             <div className="flex grow">
               <div className="grow" />
@@ -375,7 +394,11 @@ const SidePanel: FC = observer(() => {
             return;
           const messageItem = commonStore.conversation[uuid];
           if (messageItem.type !== MessageType.Error) {
-            savedContent += `${messageItem.sender === userName ? user : bot}: ${messageItem.content}\n\n`;
+            if (messageItem.sender === userName) {
+              savedContent += `${user}: ${messageItem.content}\n\n`;
+            } else if (messageItem.sender === botName) {
+              savedContent += `${bot}: ${messageItem.content}\n\n`;
+            }
           }
         });
 
@@ -418,7 +441,7 @@ const ChatPanel: FC = observer(() => {
         [welcomeUuid]: {
           sender: botName,
           type: MessageType.Normal,
-          color: 'colorful',
+          color: 'neutral',
           avatarImg: logo,
           time: new Date().toISOString(),
           content: commonStore.platform === 'web' ? t('Hello, what can I do for you?') : t('Hello! I\'m RWKV, an open-source and commercially usable large language model.'),
@@ -452,7 +475,7 @@ const ChatPanel: FC = observer(() => {
   // if answerId is not null, override the answer with new response;
   // if startUuid is null, start generating api body messages from first message;
   // if endUuid is null, generate api body messages until last message;
-  const onSubmit = useCallback((message: string | null = null, answerId: string | null = null,
+  const onSubmit = useCallback(async (message: string | null = null, answerId: string | null = null,
     startUuid: string | null = null, endUuid: string | null = null, includeEndUuid: boolean = false) => {
     if (message) {
       const newId = uuid();
@@ -499,6 +522,8 @@ const ChatPanel: FC = observer(() => {
         messages.push({ role: 'user', content: messageItem.content });
       } else if (messageItem.done && messageItem.type === MessageType.Normal && messageItem.sender === botName) {
         messages.push({ role: 'assistant', content: messageItem.content });
+      } else if (messageItem.done && messageItem.type === MessageType.Normal && messageItem.sender === systemName) {
+        messages.push({ role: 'system', content: messageItem.content });
       }
     });
 
@@ -509,7 +534,7 @@ const ChatPanel: FC = observer(() => {
     commonStore.conversation[answerId] = {
       sender: botName,
       type: MessageType.Normal,
-      color: 'colorful',
+      color: 'neutral',
       avatarImg: logo,
       time: new Date().toISOString(),
       content: '',
@@ -520,15 +545,27 @@ const ChatPanel: FC = observer(() => {
     commonStore.setConversationOrder(commonStore.conversationOrder);
     setTimeout(scrollToBottom);
     let answer = '';
+    let finished = false;
+    const finish = () => {
+      finished = true;
+      if (answerId! in chatSseControllers)
+        delete chatSseControllers[answerId!];
+      commonStore.conversation[answerId!].done = true;
+      commonStore.conversation[answerId!].content = commonStore.conversation[answerId!].content.trim();
+      commonStore.setConversation(commonStore.conversation);
+      commonStore.setConversationOrder([...commonStore.conversationOrder]);
+    };
     const chatSseController = new AbortController();
     chatSseControllers[answerId] = chatSseController;
+    const { url, headers } = await getReqUrl(port, '/v1/chat/completions', true);
     fetchEventSource( // https://api.openai.com/v1/chat/completions || http://127.0.0.1:${port}/v1/chat/completions
-      getServerRoot(port, true) + '/v1/chat/completions',
+      url,
       {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
-          Authorization: `Bearer ${commonStore.settings.apiKey}`
+          Authorization: `Bearer ${commonStore.settings.apiKey}`,
+          ...headers
         },
         body: JSON.stringify({
           messages: messages.slice(-commonStore.chatParams.historyN),
@@ -547,13 +584,8 @@ const ChatPanel: FC = observer(() => {
         signal: chatSseController?.signal,
         onmessage(e) {
           scrollToBottom();
-          if (e.data.trim() === '[DONE]') {
-            if (answerId! in chatSseControllers)
-              delete chatSseControllers[answerId!];
-            commonStore.conversation[answerId!].done = true;
-            commonStore.conversation[answerId!].content = commonStore.conversation[answerId!].content.trim();
-            commonStore.setConversation(commonStore.conversation);
-            commonStore.setConversationOrder([...commonStore.conversationOrder]);
+          if (!finished && e.data.trim() === '[DONE]') {
+            finish();
             return;
           }
           let data;
@@ -566,6 +598,10 @@ const ChatPanel: FC = observer(() => {
           if (data.model)
             commonStore.setLastModelName(data.model);
           if (data.choices && Array.isArray(data.choices) && data.choices.length > 0) {
+            if (!finished && data.choices[0]?.finish_reason) {
+              finish();
+              return;
+            }
             answer += data.choices[0]?.delta?.content || '';
             commonStore.conversation[answerId!].content = answer;
             commonStore.setConversation(commonStore.conversation);
@@ -574,7 +610,12 @@ const ChatPanel: FC = observer(() => {
         },
         async onopen(response) {
           if (response.status !== 200) {
-            commonStore.conversation[answerId!].content += '\n[ERROR]\n```\n' + response.statusText + '\n' + (await response.text()) + '\n```';
+            let errText = await response.text();
+            try {
+              errText = JSON.stringify(JSON.parse(errText), null, 2);
+            } catch (e) {
+            }
+            commonStore.conversation[answerId!].content += '\n[ERROR]\n```\n' + response.status + ' - ' + response.statusText + '\n' + errText + '\n```';
             commonStore.setConversation(commonStore.conversation);
             commonStore.setConversationOrder([...commonStore.conversationOrder]);
             setTimeout(scrollToBottom);
@@ -617,7 +658,7 @@ const ChatPanel: FC = observer(() => {
           <DialogButton tooltip={t('Clear')}
             icon={<Delete28Regular />}
             size={mq ? 'large' : 'small'} shape="circular" appearance="subtle" title={t('Clear')}
-            contentText={t('Are you sure you want to clear the conversation? It cannot be undone.')}
+            content={t('Are you sure you want to clear the conversation? It cannot be undone.')}
             onConfirm={() => {
               if (generating) {
                 for (const id in chatSseControllers) {
@@ -625,7 +666,7 @@ const ChatPanel: FC = observer(() => {
                 }
                 chatSseControllers = {};
               }
-              setActivePreset(commonStore.activePreset);
+              setActivePreset(commonStore.activePreset, commonStore.activePresetIndex);
             }} />
           <div className="relative flex grow">
             <Textarea
@@ -688,8 +729,10 @@ const ChatPanel: FC = observer(() => {
                         const urlPath = `/file-to-text?file_name=${attachmentName}`;
                         const bodyForm = new FormData();
                         bodyForm.append('file_data', blob, attachmentName);
-                        fetch(getServerRoot(port) + urlPath, {
+                        const { url, headers } = await getReqUrl(port, urlPath);
+                        fetch(url, {
                           method: 'POST',
+                          headers,
                           body: bodyForm
                         }).then(async r => {
                             if (r.status === 200) {
@@ -712,7 +755,7 @@ const ChatPanel: FC = observer(() => {
                                   autoClose: 1000
                                 });
                             } else {
-                              toast(r.statusText + '\n' + (await r.text()), {
+                              toast('Failed to fetch - ' + r.status + ' - ' + r.statusText + ' - ' + (await r.text()), {
                                 type: 'error'
                               });
                             }

frontend/src/pages/Completion.tsx

@@ -14,7 +14,7 @@ import { ToolTipButton } from '../components/ToolTipButton';
 import { ArrowSync20Regular } from '@fluentui/react-icons';
 import { defaultPresets } from './defaultConfigs';
 import { CompletionParams, CompletionPreset } from '../types/completion';
-import { getServerRoot } from '../utils';
+import { getReqUrl } from '../utils';
 
 let completionSseController: AbortController | null = null;
 
@@ -68,7 +68,7 @@ const CompletionPanel: FC = observer(() => {
     });
   };
 
-  const onSubmit = (prompt: string) => {
+  const onSubmit = async (prompt: string) => {
     commonStore.setCompletionSubmittedPrompt(prompt);
 
     if (commonStore.status.status === ModelStatus.Offline && !commonStore.settings.apiUrl && commonStore.platform !== 'web') {
@@ -80,14 +80,21 @@ const CompletionPanel: FC = observer(() => {
     prompt += params.injectStart.replaceAll('\\n', '\n');
 
     let answer = '';
+    let finished = false;
+    const finish = () => {
+      finished = true;
+      commonStore.setCompletionGenerating(false);
+    };
     completionSseController = new AbortController();
+    const { url, headers } = await getReqUrl(port, '/v1/completions', true);
     fetchEventSource( // https://api.openai.com/v1/completions || http://127.0.0.1:${port}/v1/completions
-      getServerRoot(port, true) + '/v1/completions',
+      url,
       {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
-          Authorization: `Bearer ${commonStore.settings.apiKey}`
+          Authorization: `Bearer ${commonStore.settings.apiKey}`,
+          ...headers
         },
         body: JSON.stringify({
           prompt,
@@ -103,8 +110,8 @@ const CompletionPanel: FC = observer(() => {
         signal: completionSseController?.signal,
         onmessage(e) {
           scrollToBottom();
-          if (e.data.trim() === '[DONE]') {
-            commonStore.setCompletionGenerating(false);
+          if (!finished && e.data.trim() === '[DONE]') {
+            finish();
             return;
           }
           let data;
@@ -117,13 +124,17 @@ const CompletionPanel: FC = observer(() => {
           if (data.model)
             commonStore.setLastModelName(data.model);
           if (data.choices && Array.isArray(data.choices) && data.choices.length > 0) {
+            if (!finished && data.choices[0]?.finish_reason) {
+              finish();
+              return;
+            }
             answer += data.choices[0]?.text || data.choices[0]?.delta?.content || '';
             setPrompt(prompt + answer.replace(/\s+$/, '') + params.injectEnd.replaceAll('\\n', '\n'));
           }
         },
         async onopen(response) {
           if (response.status !== 200) {
-            toast(response.statusText + '\n' + (await response.text()), {
+            toast(response.status + ' - ' + response.statusText + ' - ' + (await response.text()), {
               type: 'error'
             });
           }
@@ -275,7 +286,7 @@ const CompletionPanel: FC = observer(() => {
             onSubmit(commonStore.completionSubmittedPrompt);
           }} />
           <DialogButton className="grow" text={t('Reset')} title={t('Reset')}
-            contentText={t('Are you sure you want to reset this page? It cannot be undone.')}
+            content={t('Are you sure you want to reset this page? It cannot be undone.')}
             onConfirm={() => {
               setPreset(defaultPresets.find((preset) => preset.name === name)!);
             }} />

frontend/src/pages/Composition.tsx

@@ -26,7 +26,7 @@ import {
   SaveFile,
   StartFile
 } from '../../wailsjs/go/backend_golang/App';
-import { getServerRoot, getSoundFont, toastWithButton } from '../utils';
+import { getReqUrl, getSoundFont, toastWithButton } from '../utils';
 import { CompositionParams } from '../types/composition';
 import { useMediaQuery } from 'usehooks-ts';
 import { AudiotrackButton } from './AudiotrackManager/AudiotrackButton';
@@ -135,7 +135,7 @@ const CompositionPanel: FC = observer(() => {
     }
   }, [commonStore.midiPorts]);
 
-  const generateNs = (autoPlay: boolean) => {
+  const generateNs = async (autoPlay: boolean) => {
     if (commonStore.getCurrentModelConfig().modelParameters.modelName.toLowerCase().includes('abc')) {
       import('abcjs').then(ABCJS => {
         ABCJS.renderAbc('abc-paper', commonStore.compositionParams.prompt, { responsive: 'resize' });
@@ -143,10 +143,12 @@ const CompositionPanel: FC = observer(() => {
       return;
     }
 
-    fetch(getServerRoot(port) + '/text-to-midi', {
+    const { url, headers } = await getReqUrl(port, '/text-to-midi');
+    fetch(url, {
       method: 'POST',
       headers: {
-        'Content-Type': 'application/json'
+        'Content-Type': 'application/json',
+        ...headers
       },
       body: JSON.stringify({
         'text': commonStore.compositionParams.prompt.replaceAll(/<pad>|<start>|<end>/g, '').replaceAll('  ', ' ').trim()
@@ -175,7 +177,7 @@ const CompositionPanel: FC = observer(() => {
     });
   };
 
-  const onSubmit = (prompt: string) => {
+  const onSubmit = async (prompt: string) => {
     commonStore.setCompositionSubmittedPrompt(prompt);
 
     if (commonStore.status.status === ModelStatus.Offline && !commonStore.settings.apiUrl && commonStore.platform !== 'web') {
@@ -185,14 +187,22 @@ const CompositionPanel: FC = observer(() => {
     }
 
     let answer = '';
+    let finished = false;
+    const finish = () => {
+      finished = true;
+      commonStore.setCompositionGenerating(false);
+      generateNs(commonStore.compositionParams.autoPlay);
+    };
     compositionSseController = new AbortController();
+    const { url, headers } = await getReqUrl(port, '/v1/completions', true);
     fetchEventSource( // https://api.openai.com/v1/completions || http://127.0.0.1:${port}/v1/completions
-      getServerRoot(port, true) + '/v1/completions',
+      url,
       {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
-          Authorization: `Bearer ${commonStore.settings.apiKey}`
+          Authorization: `Bearer ${commonStore.settings.apiKey}`,
+          ...headers
         },
         body: JSON.stringify({
           prompt,
@@ -205,9 +215,8 @@ const CompositionPanel: FC = observer(() => {
         signal: compositionSseController?.signal,
         onmessage(e) {
           scrollToBottom();
-          if (e.data.trim() === '[DONE]') {
-            commonStore.setCompositionGenerating(false);
-            generateNs(commonStore.compositionParams.autoPlay);
+          if (!finished && e.data.trim() === '[DONE]') {
+            finish();
             return;
           }
           let data;
@@ -220,13 +229,17 @@ const CompositionPanel: FC = observer(() => {
           if (data.model)
             commonStore.setLastModelName(data.model);
           if (data.choices && Array.isArray(data.choices) && data.choices.length > 0) {
+            if (!finished && data.choices[0]?.finish_reason) {
+              finish();
+              return;
+            }
             answer += data.choices[0]?.text || data.choices[0]?.delta?.content || '';
             setPrompt(prompt + answer.replace(/\s+$/, ''));
           }
         },
         async onopen(response) {
           if (response.status !== 200) {
-            toast(response.statusText + '\n' + (await response.text()), {
+            toast(response.status + ' - ' + response.statusText + ' - ' + (await response.text()), {
               type: 'error'
             });
           }
@@ -378,7 +391,7 @@ const CompositionPanel: FC = observer(() => {
               onSubmit(commonStore.compositionSubmittedPrompt);
             }} />
             <DialogButton className="grow" text={t('Reset')} title={t('Reset')}
-              contentText={t('Are you sure you want to reset this page? It cannot be undone.')}
+              content={t('Are you sure you want to reset this page? It cannot be undone.')}
               onConfirm={() => {
                 const isABC = commonStore.getCurrentModelConfig().modelParameters.modelName.toLowerCase().includes('abc');
                 const defaultPrompt = isABC ? defaultCompositionABCPrompt : defaultCompositionPrompt;

frontend/src/pages/PresetsManager/MessagesEditor.tsx

@@ -126,7 +126,7 @@ const MessagesEditor: FC = observer(() => {
                             }}>
                             <Option value="user">{t('user')!}</Option>
                             <Option value="assistant">{t('assistant')!}</Option>
-                            {/* TODO <Option value="system">{t('system')!}</Option>*/}
+                            <Option value="system">{t('system')!}</Option>
                           </Dropdown>
                           <Textarea resize="vertical" className="grow" value={item.content}
                             style={{ minWidth: 0, borderRadius: 0 }}

frontend/src/pages/PresetsManager/PresetsButton.tsx

@@ -4,6 +4,7 @@ import React, { FC, lazy, PropsWithChildren, ReactElement, useState } from 'reac
 import {
   Button,
   Dialog,
+  DialogActions,
   DialogBody,
   DialogContent,
   DialogSurface,
@@ -60,10 +61,15 @@ const MessagesEditor = lazy(() => import('./MessagesEditor'));
 
 const PresetCardFrame: FC<PropsWithChildren & {
   onClick?: React.MouseEventHandler<HTMLButtonElement>
+  highlight?: boolean
 }> = (props) => {
   return <Button
     className="flex flex-col gap-1 w-32 h-56 break-all"
-    style={{ minWidth: 0, borderRadius: '0.75rem', justifyContent: 'unset' }}
+    style={{
+      minWidth: 0,
+      borderRadius: '0.75rem',
+      justifyContent: 'unset', ...(props.highlight ? { borderColor: '#115ea3', borderWidth: '2px' } : {})
+    }}
     onClick={props.onClick}
   >
     {props.children}
@@ -71,31 +77,28 @@ const PresetCardFrame: FC<PropsWithChildren & {
 };
 
 const PresetCard: FC<{
-  avatarImg: string,
-  name: string,
-  desc: string,
-  tag: string,
   editable: boolean,
+  preset: Preset,
   presetIndex: number,
-  onClick?: () => void
 }> = observer(({
-  avatarImg, name, desc, tag, editable, presetIndex, onClick
+  editable, preset, presetIndex
 }) => {
   const { t } = useTranslation();
 
-  return <PresetCardFrame onClick={(e) => {
-    if (onClick && e.currentTarget.contains(e.target as Node))
-      onClick();
-  }}>
-    <img src={absPathAsset(avatarImg)} className="rounded-xl select-none ml-auto mr-auto h-28" />
-    <Text size={400}>{name}</Text>
+  return <PresetCardFrame highlight={commonStore.activePresetIndex === presetIndex}
+    onClick={(e) => {
+      if (e.currentTarget.contains(e.target as Node))
+        setActivePreset(presetIndex === -1 ? defaultPreset : preset, presetIndex);
+    }}>
+    <img src={absPathAsset(preset.avatarImg)} className="rounded-xl select-none ml-auto mr-auto h-28" />
+    <Text size={400}>{preset.name}</Text>
     <Text size={200} style={{
       overflow: 'hidden', textOverflow: 'ellipsis',
       display: '-webkit-box', WebkitLineClamp: 3, WebkitBoxOrient: 'vertical'
-    }}>{desc}</Text>
+    }}>{preset.desc}</Text>
     <div className="grow" />
     <div className="flex justify-between w-full items-end">
-      <div className="text-xs font-thin text-gray-500">{t(tag)}</div>
+      <div className="text-xs font-thin text-gray-500">{t(preset.tag)}</div>
       {editable ?
         <ChatPresetEditor presetIndex={presetIndex} triggerButton={
           <ToolTipButton size="small" appearance="transparent" desc={t('Edit')} icon={<Edit20Regular />}
@@ -115,9 +118,11 @@ const ChatPresetEditor: FC<{
   presetIndex: number
 }> = observer(({ triggerButton, presetIndex }) => {
   const { t } = useTranslation();
+  const [open, setOpen] = React.useState(false);
+  const [showExitConfirm, setShowExitConfirm] = React.useState(false);
   const [editingMessages, setEditingMessages] = useState(false);
 
-  if (!commonStore.editingPreset)
+  if (open && !commonStore.editingPreset)
     commonStore.setEditingPreset({ ...defaultPreset });
   const editingPreset = commonStore.editingPreset!;
 
@@ -168,26 +173,46 @@ const ChatPresetEditor: FC<{
   };
 
   const savePreset = () => {
+    setOpen(false);
+    setShowExitConfirm(false);
     if (presetIndex === -1) {
       commonStore.setPresets([...commonStore.presets, { ...editingPreset }]);
-      setEditingPreset(defaultPreset);
     } else {
       commonStore.presets[presetIndex] = editingPreset;
       commonStore.setPresets(commonStore.presets);
     }
+    commonStore.setEditingPreset(null);
   };
 
   const activatePreset = () => {
     savePreset();
-    setActivePreset(editingPreset);
+    setActivePreset(editingPreset, presetIndex === -1 ? commonStore.presets.length - 1 : presetIndex);
   };
 
   const deletePreset = () => {
+    if (commonStore.activePresetIndex === presetIndex) {
+      setActivePreset(defaultPreset, -1);
+    }
     commonStore.presets.splice(presetIndex, 1);
     commonStore.setPresets(commonStore.presets);
+    setOpen(false);
+    setShowExitConfirm(false);
+    commonStore.setEditingPreset(null);
   };
 
-  return <Dialog>
+  return <Dialog open={open} onOpenChange={(e, data) => {
+    if (data.open) {
+      setOpen(true);
+    } else if (!commonStore.editingPreset ||
+      (presetIndex === -1 && JSON.stringify(editingPreset) === JSON.stringify(defaultPreset)) ||
+      (presetIndex !== -1 && JSON.stringify(editingPreset) === JSON.stringify(commonStore.presets[presetIndex]))) {
+      setOpen(false);
+      setShowExitConfirm(false);
+      commonStore.setEditingPreset(null);
+    } else {
+      setShowExitConfirm(true);
+    }
+  }}>
     <DialogTrigger disableButtonEnhancement>
       {triggerButton}
     </DialogTrigger>
@@ -196,17 +221,35 @@ const ChatPresetEditor: FC<{
       maxWidth: '80vw',
       maxHeight: '80vh',
       width: '500px',
-      height: '100%'
+      height: '100%',
+      transform: 'unset' // override the style for the new version of @fluentui/react-components to avoid conflicts with react-beautiful-dnd
     }}>
       <DialogBody style={{ height: '100%', overflow: 'hidden' }}>
-        <DialogContent className="flex flex-col gap-1 overflow-hidden">
+        {editingPreset && <DialogContent className="flex flex-col gap-1 overflow-hidden">
           <CustomToastContainer />
+          <Dialog open={showExitConfirm}>
+            <DialogSurface style={{ transform: 'unset' }}>
+              <DialogBody>
+                <DialogContent>
+                  {t('Content has been changed, are you sure you want to exit without saving?')}
+                </DialogContent>
+                <DialogActions>
+                  <Button appearance="secondary" onClick={() => {
+                    setShowExitConfirm(false);
+                  }}>{t('Cancel')}</Button>
+                  <Button appearance="primary" onClick={() => {
+                    setOpen(false);
+                    setShowExitConfirm(false);
+                    commonStore.setEditingPreset(null);
+                  }}>{t('Exit without saving')}</Button>
+                </DialogActions>
+              </DialogBody>
+            </DialogSurface>
+          </Dialog>
           <div className="flex justify-between">{
             presetIndex === -1
               ? <div />
-              : <DialogTrigger disableButtonEnhancement>
-                <Button appearance="subtle" icon={<Delete20Regular />} onClick={deletePreset} />
-              </DialogTrigger>
+              : <Button appearance="subtle" icon={<Delete20Regular />} onClick={deletePreset} />
           }
             <DialogTrigger disableButtonEnhancement>
               <Button appearance="subtle" icon={<Dismiss20Regular />} />
@@ -309,14 +352,10 @@ const ChatPresetEditor: FC<{
             <Button onClick={copyPreset}>{t('Copy')}</Button>
           </div>
           <div className="flex justify-between">
-            <DialogTrigger disableButtonEnhancement>
-              <Button appearance="primary" onClick={savePreset}>{t('Save')}</Button>
-            </DialogTrigger>
-            <DialogTrigger disableButtonEnhancement>
-              <Button appearance="primary" onClick={activatePreset}>{t('Activate')}</Button>
-            </DialogTrigger>
+            <Button appearance="primary" onClick={savePreset}>{t('Save')}</Button>
+            <Button appearance="primary" onClick={activatePreset}>{t('Activate')}</Button>
           </div>
-        </DialogContent>
+        </DialogContent>}
       </DialogBody>
     </DialogSurface>
   </Dialog>;
@@ -339,20 +378,16 @@ const ChatPresets: FC = observer(() => {
     {/*  </div>*/}
     {/*</PresetCardFrame>*/}
     <PresetCard
+      preset={defaultPreset}
       presetIndex={-1}
       editable={false}
-      onClick={() => {
-        setActivePreset(defaultPreset);
-      }} avatarImg={defaultPreset.avatarImg} name={defaultPreset.name} desc={defaultPreset.desc} tag={defaultPreset.tag}
     />
     {commonStore.presets.map((preset, index) => {
       return <PresetCard
+        key={index}
+        preset={preset}
         presetIndex={index}
         editable={true}
-        onClick={() => {
-          setActivePreset(preset);
-        }}
-        key={index} avatarImg={preset.avatarImg} name={preset.name} desc={preset.desc} tag={preset.tag}
       />;
     })}
   </div>;
@@ -421,7 +456,7 @@ export const PresetsButton: FC<{
       <ToolTipButton desc={t('Presets')} size={size} shape={shape} appearance={appearance}
         icon={<Accessibility28Regular />} />
     </DialogTrigger>
-    <DialogSurface style={{ paddingTop: 0, maxWidth: '90vw', width: 'fit-content' }}>
+    <DialogSurface style={{ paddingTop: 0, maxWidth: '90vw', width: 'fit-content', transform: 'unset' }}>
       <DialogBody>
         <DialogContent>
           <CustomToastContainer />

frontend/src/pages/Settings.tsx

@@ -17,6 +17,7 @@ import { useTranslation } from 'react-i18next';
 import { checkUpdate, toastWithButton } from '../utils';
 import { RestartApp } from '../../wailsjs/go/backend_golang/App';
 import { Language, Languages } from '../types/settings';
+import { toast } from 'react-toastify';
 
 export const GeneralSettings: FC = observer(() => {
   const { t } = useTranslation();
@@ -113,11 +114,15 @@ export const AdvancedGeneralSettings: FC = observer(() => {
                   commonStore.setSettings({
                     apiCompletionModelName: 'rwkv'
                   });
+              } else if (data.optionText === 'Ollama') {
+                toast(t('Don\'t forget to correctly fill in your Ollama API Chat Model Name.'),
+                  { type: 'info' });
               }
             }}>
             <Option value="">{t('Localhost')!}</Option>
             <Option value="https://rwkv.ai-creator.net/chntuned">RWKV</Option>
             <Option value="https://api.openai.com">OpenAI</Option>
+            <Option value="http://localhost:11434">Ollama</Option>
           </Dropdown>
         </div>
       } />

frontend/src/pages/Train.tsx

@@ -342,7 +342,7 @@ const LoraFinetune: FC = observer(() => {
       };
 
       if (msg === 'wsl is not enabled') {
-        enableWsl(false);
+        enableWsl(true);
       } else if (msg.includes('wsl.state: The system cannot find the file')) {
         enableWsl(true);
       } else {
@@ -391,7 +391,7 @@ const LoraFinetune: FC = observer(() => {
                     setDataParams({ dataPath: data.value });
                   }} />
                 <DialogButton text={t('Help')} title={t('Help')} markdown
-                  contentText={t('The data path should be a directory or a file in jsonl format (more formats will be supported in the future).\n\n' +
+                  content={t('The data path should be a directory or a file in jsonl format (more formats will be supported in the future).\n\n' +
                     'When you provide a directory path, all the txt files within that directory will be automatically converted into training data. ' +
                     'This is commonly used for large-scale training in writing, code generation, or knowledge bases.\n\n' +
                     'The jsonl format file can be referenced at https://github.com/josStorer/RWKV-Runner/blob/master/finetune/data/sample.jsonl.\n' +

frontend/src/startup.ts

@@ -9,6 +9,7 @@ import { t } from 'i18next';
 import { Preset } from './types/presets';
 import { toast } from 'react-toastify';
 import { MidiMessage, MidiPort } from './types/composition';
+import { throttle } from 'lodash-es';
 
 export async function startup() {
   initPresets();
@@ -16,6 +17,7 @@ export async function startup() {
   await GetPlatform().then(p => commonStore.setPlatform(p as Platform));
 
   if (commonStore.platform !== 'web') {
+    document.body.style.setProperty('overflow', 'hidden');
     downloadProgramFiles();
     EventsOn('downloadList', (data) => {
       if (data)
@@ -102,7 +104,7 @@ async function initPresets() {
 }
 
 async function initLoraModels() {
-  const refreshLoraModels = () => {
+  const refreshLoraModels = throttle(() => {
     ListDirFiles('lora-models').then((data) => {
       if (!data) return;
       const loraModels = [];
@@ -113,7 +115,7 @@ async function initLoraModels() {
       }
       commonStore.setLoraModels(loraModels);
     });
-  };
+  }, 2000);
 
   refreshLoraModels();
   EventsOn('fsnotify', (data: string) => {
@@ -123,9 +125,12 @@ async function initLoraModels() {
 }
 
 async function initLocalModelsNotify() {
+  const throttleRefreshLocalModels = throttle(() => {
+    refreshLocalModels({ models: commonStore.modelSourceList }, false); //TODO fix bug that only add models
+  }, 2000);
   EventsOn('fsnotify', (data: string) => {
     if (data.includes('models') && !data.includes('lora-models'))
-      refreshLocalModels({ models: commonStore.modelSourceList }, false); //TODO fix bug that only add models
+      throttleRefreshLocalModels();
   });
 }
 

frontend/src/stores/commonStore.ts

@@ -63,6 +63,7 @@ class CommonStore {
   monitorData: MonitorData | null = null;
   depComplete: boolean = false;
   platform: Platform = 'windows';
+  proxyPort: number = 0;
   lastModelName: string = '';
   // presets manager
   editingPreset: Preset | null = null;
@@ -74,6 +75,7 @@ class CommonStore {
   conversation: Conversation = {};
   conversationOrder: string[] = [];
   activePreset: Preset | null = null;
+  activePresetIndex: number = -1;
   attachmentUploading: boolean = false;
   attachments: {
     [uuid: string]: Attachment[]
@@ -322,6 +324,10 @@ class CommonStore {
     this.platform = value;
   }
 
+  setProxyPort(value: number) {
+    this.proxyPort = value;
+  }
+
   setCurrentInput(value: string) {
     this.currentInput = value;
   }
@@ -342,7 +348,7 @@ class CommonStore {
     this.lastUnfinishedModelDownloads = value;
   }
 
-  setEditingPreset(value: Preset) {
+  setEditingPreset(value: Preset | null) {
     this.editingPreset = value;
   }
 
@@ -356,6 +362,10 @@ class CommonStore {
     this.activePreset = value;
   }
 
+  setActivePresetIndex(value: number) {
+    this.activePresetIndex = value;
+  }
+
   setCompletionSubmittedPrompt(value: string) {
     this.completionSubmittedPrompt = value;
   }

frontend/src/types/chat.ts

@@ -2,6 +2,7 @@ import { ApiParameters } from './configs';
 
 export const userName = 'M E';
 export const botName = 'A I';
+export const systemName = 'System';
 export const welcomeUuid = 'welcome';
 
 export enum MessageType {
@@ -9,7 +10,7 @@ export enum MessageType {
   Error
 }
 
-export type Side = 'left' | 'right'
+export type Side = 'left' | 'right' | 'center'
 export type Color = 'neutral' | 'brand' | 'colorful'
 export type MessageItem = {
   sender: string,

frontend/src/utils/index.tsx

@@ -2,6 +2,7 @@ import {
   AddToDownloadList,
   DeleteFile,
   DepCheck,
+  GetProxyPort,
   InstallPyDep,
   ListDirFiles,
   OpenOpenFileDialog,
@@ -26,9 +27,9 @@ import { DataProcessParameters, LoraFinetuneParameters } from '../types/train';
 import { InstrumentTypeNameMap, MidiMessage, tracksMinimalTotalTime } from '../types/composition';
 import logo from '../assets/images/logo.png';
 import { Preset } from '../types/presets';
-import { botName, Conversation, MessageType, Role, userName } from '../types/chat';
+import { botName, Conversation, MessageType, Role, systemName, userName } from '../types/chat';
 import { v4 as uuid } from 'uuid';
-import { findLastIndex } from 'lodash-es';
+import { findLastIndex, throttle } from 'lodash-es';
 
 export type Cache = {
   version: string
@@ -60,8 +61,7 @@ export async function refreshBuiltInModels(readCache: boolean = false) {
   else cache.models = manifest.models.slice();
 
   commonStore.setModelSourceList(cache.models);
-  await saveCache().catch(() => {
-  });
+  saveCache();
   return cache;
 }
 
@@ -119,8 +119,7 @@ export async function refreshLocalModels(cache: {
   commonStore.setModelSourceList(cache.models);
   if (initUnfinishedModels)
     initLastUnfinishedModelDownloads();
-  await saveCache().catch(() => {
-  });
+  saveCache();
 }
 
 function initLastUnfinishedModelDownloads() {
@@ -223,26 +222,34 @@ export const getStrategy = (modelConfig: ModelConfig | undefined = undefined) =>
   return strategy;
 };
 
-export const saveConfigs = async () => {
-  const data: LocalConfig = {
-    modelSourceManifestList: commonStore.modelSourceManifestList,
-    currentModelConfigIndex: commonStore.currentModelConfigIndex,
-    modelConfigs: commonStore.modelConfigs,
-    settings: commonStore.settings,
-    dataProcessParams: commonStore.dataProcessParams,
-    loraFinetuneParams: commonStore.loraFinetuneParams
-  };
-  return SaveJson('config.json', data);
-};
+export const saveConfigs = throttle(async () => {
+    const data: LocalConfig = {
+      modelSourceManifestList: commonStore.modelSourceManifestList,
+      currentModelConfigIndex: commonStore.currentModelConfigIndex,
+      modelConfigs: commonStore.modelConfigs,
+      settings: commonStore.settings,
+      dataProcessParams: commonStore.dataProcessParams,
+      loraFinetuneParams: commonStore.loraFinetuneParams
+    };
+    return SaveJson('config.json', data);
+  }, 500,
+  {
+    leading: true,
+    trailing: true
+  });
 
-export const saveCache = async () => {
-  const data: Cache = {
-    version: manifest.version,
-    models: commonStore.modelSourceList,
-    depComplete: commonStore.depComplete
-  };
-  return SaveJson('cache.json', data);
-};
+export const saveCache = throttle(async () => {
+    const data: Cache = {
+      version: manifest.version,
+      models: commonStore.modelSourceList,
+      depComplete: commonStore.depComplete
+    };
+    return SaveJson('cache.json', data);
+  }, 1000,
+  {
+    leading: true,
+    trailing: true
+  });
 
 export const savePresets = async () => {
   return SaveJson('presets.json', commonStore.presets);
@@ -310,6 +317,24 @@ export function bytesToReadable(size: number) {
   else return bytesToGb(size) + ' GB';
 }
 
+export async function getReqUrl(port: number, path: string, isCore: boolean = false): Promise<{
+  url: string,
+  headers: { [key: string]: string }
+}> {
+  const realUrl = getServerRoot(port, isCore) + path;
+  if (commonStore.platform === 'web')
+    return {
+      url: realUrl,
+      headers: {}
+    };
+  if (!commonStore.proxyPort)
+    await GetProxyPort().then(p => commonStore.setProxyPort(p));
+  return {
+    url: `http://127.0.0.1:${commonStore.proxyPort}`,
+    headers: { 'Real-Target': encodeURIComponent(realUrl) }
+  };
+}
+
 export function getServerRoot(defaultLocalPort: number, isCore: boolean = false) {
   const coreCustomApiUrl = commonStore.settings.coreApiUrl.trim().replace(/\/$/, '');
   if (isCore && coreCustomApiUrl)
@@ -581,8 +606,9 @@ export async function getSoundFont() {
   return soundUrl;
 }
 
-export const setActivePreset = (preset: Preset | null) => {
+export const setActivePreset = (preset: Preset | null, index: number) => {
   commonStore.setActivePreset(preset);
+  commonStore.setActivePresetIndex(index);
   //TODO if (preset.displayPresetMessages) {
   const { pushMessage, saveConversation } = newChatConversation();
   if (preset)
@@ -635,13 +661,13 @@ export function newChatConversation() {
     const newUuid = uuid();
     conversationOrder.push(newUuid);
     conversation[newUuid] = {
-      sender: role === 'user' ? userName : botName,
+      sender: role === 'user' ? userName : role === 'assistant' ? botName : systemName,
       type: MessageType.Normal,
-      color: role === 'user' ? 'brand' : 'colorful',
+      color: role === 'user' ? 'brand' : 'neutral',
       avatarImg: role === 'user' ? undefined : logo,
       time: new Date().toISOString(),
       content: content,
-      side: role === 'user' ? 'right' : 'left',
+      side: role === 'user' ? 'right' : role === 'assistant' ? 'left' : 'center',
       done: true
     };
   };

frontend/src/webWails.js

@@ -115,6 +115,12 @@ if (!window.go) {
   defineApp('ListDirFiles', async () => {
     return []
   })
+  defineApp('GetAbsPath', async (path) => {
+    return path
+  })
+  defineApp('GetProxyPort', async () => {
+    return 0
+  })
   defineApp('OpenOpenFileDialog', webOpenOpenFileDialog)
   defineApp('OpenSaveFileDialog', async (filterPattern, defaultFileName, savedContent) => {
     const saver = await import('file-saver')

frontend/wailsjs/go/backend_golang/App.d.ts

@@ -32,6 +32,8 @@ export function GetAbsPath(arg1:string):Promise<string>;
 
 export function GetPlatform():Promise<string>;
 
+export function GetProxyPort():Promise<number>;
+
 export function GetPyError():Promise<string>;
 
 export function InstallPyDep(arg1:string,arg2:boolean):Promise<string>;

frontend/wailsjs/go/backend_golang/App.js

@@ -62,6 +62,10 @@ export function GetPlatform() {
   return window['go']['backend_golang']['App']['GetPlatform']();
 }
 
+export function GetProxyPort() {
+  return window['go']['backend_golang']['App']['GetProxyPort']();
+}
+
 export function GetPyError() {
   return window['go']['backend_golang']['App']['GetPyError']();
 }

manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.7.4",
+  "version": "1.7.6",
   "introduction": {
     "en": "RWKV is an open-source, commercially usable large language model with high flexibility and great potential for development.\n### About This Tool\nThis tool aims to lower the barrier of entry for using large language models, making it accessible to everyone. It provides fully automated dependency and model management. You simply need to click and run, following the instructions, to deploy a local large language model. The tool itself is very compact and only requires a single executable file for one-click deployment.\nAdditionally, this tool offers an interface that is fully compatible with the OpenAI API. This means you can use any ChatGPT client as a client for RWKV, enabling capability expansion beyond just chat functionality.\n### Preset Configuration Rules at the Bottom\nThis tool comes with a series of preset configurations to reduce complexity. The naming rules for each configuration represent the following in order: device - required VRAM/memory - model size - model language.\nFor example, \"GPU-8G-3B-EN\" indicates that this configuration is for a graphics card with 8GB of VRAM, a model size of 3 billion parameters, and it uses an English language model.\nLarger model sizes have higher performance and VRAM requirements. Among configurations with the same model size, those with higher VRAM usage will have faster runtime.\nFor example, if you have 12GB of VRAM but running the \"GPU-12G-7B-EN\" configuration is slow, you can downgrade to \"GPU-8G-3B-EN\" for a significant speed improvement.\n### About RWKV\nRWKV is an RNN with Transformer-level LLM performance, which can also be directly trained like a GPT transformer (parallelizable). And it's 100% attention-free. You only need the hidden state at position t to compute the state at position t+1. You can use the \"GPT\" mode to quickly compute the hidden state for the \"RNN\" mode.<br/>So it's combining the best of RNN and transformer - great performance, fast inference, saves VRAM, fast training, \"infinite\" ctx_len, and free sentence embedding (using the final hidden state).",
     "zh": "RWKV是一个开源且允许商用的大语言模型，灵活性很高且极具发展潜力。\n### 关于本工具\n本工具旨在降低大语言模型的使用门槛，做到人人可用，本工具提供了全自动化的依赖和模型管理，你只需要直接点击运行，跟随引导，即可完成本地大语言模型的部署，工具本身体积极小，只需要一个exe即可完成一键部署。\n此外，本工具提供了与OpenAI API完全兼容的接口，这意味着你可以把任意ChatGPT客户端用作RWKV的客户端，实现能力拓展，而不局限于聊天。\n### 底部的预设配置规则\n本工具内置了一系列预设配置，以降低使用难度，每个配置名的规则，依次代表着：设备-所需显存/内存-模型规模-模型语言。\n例如，GPU-8G-3B-CN，表示该配置用于显卡，需要8G显存，模型规模为30亿参数，使用的是中文模型。\n模型规模越大，性能要求越高，显存要求也越高，而同样模型规模的配置中，显存占用越高的，运行速度越快。\n例如当你有12G显存，但运行GPU-12G-7B-CN配置速度比较慢，可降级成GPU-8G-3B-CN，将会大幅提速。\n### 关于RWKV\nRWKV是具有Transformer级别LLM性能的RNN，也可以像GPT Transformer一样直接进行训练（可并行化）。而且它是100% attention-free的。你只需在位置t处获得隐藏状态即可计算位置t + 1处的状态。你可以使用“GPT”模式快速计算用于“RNN”模式的隐藏状态。\n因此，它将RNN和Transformer的优点结合起来 - 高性能、快速推理、节省显存、快速训练、“无限”上下文长度以及免费的语句嵌入（使用最终隐藏状态）。"