lora finetune (need to be refactored)

2023-07-03 17:41:47 +08:00
parent c54d10795f
commit 987854fe49
42 changed files with 4825 additions and 158 deletions
--- a/backend-golang/app.go
+++ b/backend-golang/app.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"runtime"

+	"github.com/fsnotify/fsnotify"
 	"github.com/minio/selfupdate"
 	wruntime "github.com/wailsapp/wails/v2/pkg/runtime"
 )
@@ -41,6 +42,27 @@ func (a *App) OnStartup(ctx context.Context) {
 	}

 	a.downloadLoop()
+
+	watcher, err := fsnotify.NewWatcher()
+	if err == nil {
+		watcher.Add("./lora-models")
+		watcher.Add("./models")
+		go func() {
+			for {
+				select {
+				case event, ok := <-watcher.Events:
+					if !ok {
+						return
+					}
+					wruntime.EventsEmit(ctx, "fsnotify", event.Name)
+				case _, ok := <-watcher.Errors:
+					if !ok {
+						return
+					}
+				}
+			}
+		}()
+	}
 }

 func (a *App) UpdateApp(url string) (broken bool, err error) {
--- a/backend-golang/rwkv.go
+++ b/backend-golang/rwkv.go
@@ -31,6 +31,38 @@ func (a *App) ConvertModel(python string, modelPath string, strategy string, out
 	return Cmd(python, "./backend-python/convert_model.py", "--in", modelPath, "--out", outPath, "--strategy", strategy)
 }

+func (a *App) ConvertData(python string, input string, outputPrefix string, vocab string) (string, error) {
+	var err error
+	if python == "" {
+		python, err = GetPython()
+	}
+	if err != nil {
+		return "", err
+	}
+	tokenizerType := "HFTokenizer"
+	if strings.Contains(vocab, "rwkv_vocab_v20230424") {
+		tokenizerType = "RWKVTokenizer"
+	}
+	return Cmd(python, "./finetune/json2binidx_tool/tools/preprocess_data.py", "--input", input, "--output-prefix", outputPrefix, "--vocab", vocab,
+		"--tokenizer-type", tokenizerType, "--dataset-impl", "mmap", "--append-eod")
+}
+
+func (a *App) MergeLora(python string, useGpu bool, loraAlpha int, baseModel string, loraPath string, outputPath string) (string, error) {
+	var err error
+	if python == "" {
+		python, err = GetPython()
+	}
+	if err != nil {
+		return "", err
+	}
+	args := []string{python, "./finetune/lora/merge_lora.py"}
+	if useGpu {
+		args = append(args, "--use-gpu")
+	}
+	args = append(args, strconv.Itoa(loraAlpha), baseModel, loraPath, outputPath)
+	return Cmd(args...)
+}
+
 func (a *App) DepCheck(python string) error {
 	var err error
 	if python == "" {
--- a/backend-golang/wsl.go
+++ b/backend-golang/wsl.go
@@ -0,0 +1,202 @@
+package backend_golang
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	su "github.com/nyaosorg/go-windows-su"
+	wsl "github.com/ubuntu/gowsl"
+	wruntime "github.com/wailsapp/wails/v2/pkg/runtime"
+)
+
+var distro *wsl.Distro
+var stdin io.WriteCloser
+var cmd *exec.Cmd
+
+func isWslRunning() (bool, error) {
+	if distro == nil {
+		return false, nil
+	}
+	state, err := distro.State()
+	if err != nil {
+		return false, err
+	}
+	if state != wsl.Running {
+		distro = nil
+		return false, nil
+	}
+	return true, nil
+}
+
+func (a *App) WslStart() error {
+	if runtime.GOOS != "windows" {
+		return errors.New("wsl not supported")
+	}
+
+	running, err := isWslRunning()
+	if err != nil {
+		return err
+	}
+	if running {
+		return nil
+	}
+	distros, err := wsl.RegisteredDistros(context.Background())
+	if err != nil {
+		return err
+	}
+	for _, d := range distros {
+		if strings.Contains(d.Name(), "Ubuntu") {
+			distro = &d
+			break
+		}
+	}
+	if distro == nil {
+		return errors.New("ubuntu not found")
+	}
+
+	cmd = exec.Command("wsl", "-d", distro.Name(), "-u", "root")
+
+	stdin, err = cmd.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	stdout, err := cmd.StdoutPipe()
+	cmd.Stderr = cmd.Stdout
+	if err != nil {
+		// stdin.Close()
+		stdin = nil
+		return err
+	}
+
+	go func() {
+		reader := bufio.NewReader(stdout)
+		for {
+			if stdin == nil {
+				break
+			}
+			line, _, err := reader.ReadLine()
+			if err != nil {
+				wruntime.EventsEmit(a.ctx, "wslerr", err.Error())
+				break
+			}
+			wruntime.EventsEmit(a.ctx, "wsl", string(line))
+		}
+		// stdout.Close()
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *App) WslCommand(command string) error {
+	if runtime.GOOS != "windows" {
+		return errors.New("wsl not supported")
+	}
+
+	running, err := isWslRunning()
+	if err != nil {
+		return err
+	}
+	if !running {
+		return errors.New("wsl not running")
+	}
+	_, err = stdin.Write([]byte(command + "\n"))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *App) WslStop() error {
+	if runtime.GOOS != "windows" {
+		return errors.New("wsl not supported")
+	}
+
+	running, err := isWslRunning()
+	if err != nil {
+		return err
+	}
+	if !running {
+		return errors.New("wsl not running")
+	}
+	err = cmd.Process.Kill()
+	cmd = nil
+	// stdin.Close()
+	stdin = nil
+	distro = nil
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *App) WslIsEnabled() error {
+	if runtime.GOOS != "windows" {
+		return errors.New("wsl not supported")
+	}
+
+	ex, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	exDir := filepath.Dir(ex)
+
+	data, err := os.ReadFile(exDir + "/wsl.state")
+	if err == nil {
+		if strings.Contains(string(data), "Enabled") {
+			return nil
+		}
+	}
+
+	cmd := `-Command (Get-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux).State | Out-File -Encoding utf8 -FilePath ` + exDir + "/wsl.state"
+	_, err = su.ShellExecute(su.RUNAS, "powershell", cmd, exDir)
+	if err != nil {
+		return err
+	}
+	time.Sleep(2 * time.Second)
+	data, err = os.ReadFile(exDir + "/wsl.state")
+	if err != nil {
+		return err
+	}
+	if strings.Contains(string(data), "Enabled") {
+		return nil
+	} else {
+		return errors.New("wsl is not enabled")
+	}
+}
+
+func (a *App) WslEnable(forceMode bool) error {
+	if runtime.GOOS != "windows" {
+		return errors.New("wsl not supported")
+	}
+
+	cmd := `/online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux`
+	_, err := su.ShellExecute(su.RUNAS, "dism", cmd, `C:\`)
+	if err != nil {
+		return err
+	}
+	if forceMode {
+		os.WriteFile("./wsl.state", []byte("Enabled"), 0644)
+	}
+	return nil
+}
+
+func (a *App) WslInstallUbuntu() error {
+	if runtime.GOOS != "windows" {
+		return errors.New("wsl not supported")
+	}
+
+	exec.Command("start", "ms-windows-store://pdp/?ProductId=9PN20MSR04DW").Start()
+	return nil
+}