From 6fc5a335fb5a1ea8757ff1f6687e3429240ac50d Mon Sep 17 00:00:00 2001 From: josc146 Date: Mon, 29 May 2023 09:39:16 +0800 Subject: [PATCH] embed dependencies --- .gitignore | 1 + backend-golang/utils.go | 30 ++++++++ ...20B_tokenizer.json => .20B_tokenizer.json} | 0 ...20230424.txt => .rwkv_vocab_v20230424.txt} | 0 frontend/src/startup.ts | 20 +----- frontend/src/utils/index.tsx | 5 +- main.go | 13 ++++ manifest.json | 72 +------------------ 8 files changed, 52 insertions(+), 89 deletions(-) rename backend-python/rwkv_pip/{20B_tokenizer.json => .20B_tokenizer.json} (100%) rename backend-python/rwkv_pip/{rwkv_vocab_v20230424.txt => .rwkv_vocab_v20230424.txt} (100%) diff --git a/.gitignore b/.gitignore index 2c84165..752b824 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ __pycache__ /frontend/stats.html /frontend/package.json.md5 /backend-python/get-pip.py +/backend-python/.get-pip.py /py310 *.zip /cmd-helper.bat diff --git a/backend-golang/utils.go b/backend-golang/utils.go index 087cc3a..7adedff 100644 --- a/backend-golang/utils.go +++ b/backend-golang/utils.go @@ -3,8 +3,10 @@ package backend_golang import ( "archive/zip" "bufio" + "embed" "errors" "io" + "io/fs" "os" "os/exec" "path/filepath" @@ -41,6 +43,34 @@ func Cmd(args ...string) (string, error) { } } +func CopyEmbed(efs embed.FS) error { + err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + if err != nil { + return err + } + content, err := efs.ReadFile(path) + if err != nil { + return err + } + + err = os.MkdirAll(path[:strings.LastIndex(path, "/")], 0755) + if err != nil { + return err + } + + err = os.WriteFile(path, content, 0644) + if err != nil { + return err + } + + return nil + }) + return err +} + func GetPython() (string, error) { switch platform := runtime.GOOS; platform { case "windows": diff --git a/backend-python/rwkv_pip/20B_tokenizer.json b/backend-python/rwkv_pip/.20B_tokenizer.json similarity index 100% rename from backend-python/rwkv_pip/20B_tokenizer.json rename to backend-python/rwkv_pip/.20B_tokenizer.json diff --git a/backend-python/rwkv_pip/rwkv_vocab_v20230424.txt b/backend-python/rwkv_pip/.rwkv_vocab_v20230424.txt similarity index 100% rename from backend-python/rwkv_pip/rwkv_vocab_v20230424.txt rename to backend-python/rwkv_pip/.rwkv_vocab_v20230424.txt diff --git a/frontend/src/startup.ts b/frontend/src/startup.ts index fd810b2..50b5e29 100644 --- a/frontend/src/startup.ts +++ b/frontend/src/startup.ts @@ -1,26 +1,12 @@ import commonStore from './stores/commonStore'; -import { FileExists, ReadJson } from '../wailsjs/go/backend_golang/App'; -import { - Cache, - checkUpdate, - deleteDynamicProgramFiles, - downloadProgramFiles, - LocalConfig, - refreshModels, - saveCache -} from './utils'; +import { ReadJson } from '../wailsjs/go/backend_golang/App'; +import { Cache, checkUpdate, downloadProgramFiles, LocalConfig, refreshModels, saveCache } from './utils'; import { getStatus } from './apis'; import { EventsOn } from '../wailsjs/runtime'; import { defaultModelConfigs } from './pages/Configs'; export async function startup() { - FileExists('cache.json').then((exists) => { - if (exists) - downloadProgramFiles(); - else { - deleteDynamicProgramFiles().then(downloadProgramFiles); - } - }); + downloadProgramFiles(); EventsOn('downloadList', (data) => { if (data) commonStore.setDownloadList(data); diff --git a/frontend/src/utils/index.tsx b/frontend/src/utils/index.tsx index 2ed1b08..db5f01b 100644 --- a/frontend/src/utils/index.tsx +++ b/frontend/src/utils/index.tsx @@ -176,7 +176,7 @@ export function isSystemLightMode() { export function downloadProgramFiles() { manifest.programFiles.forEach(({ url, path }) => { FileExists(path).then(exists => { - if (!exists) + if (!exists && url) AddToDownloadList(path, url.replace('@master', '@v' + manifest.version)); }); }); @@ -184,7 +184,8 @@ export function downloadProgramFiles() { export function forceDownloadProgramFiles() { manifest.programFiles.forEach(({ url, path }) => { - AddToDownloadList(path, url.replace('@master', '@v' + manifest.version)); + if (url) + AddToDownloadList(path, url.replace('@master', '@v' + manifest.version)); }); } diff --git a/main.go b/main.go index 8c9da7f..7450ee1 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,20 @@ import ( //go:embed all:frontend/dist var assets embed.FS +//go:embed all:py310/Lib/site-packages/cyac +var cyac embed.FS + +//go:embed all:py310/Lib/site-packages/cyac-1.7.dist-info +var cyacInfo embed.FS + +//go:embed backend-python +var py embed.FS + func main() { + go backend.CopyEmbed(cyac) + go backend.CopyEmbed(cyacInfo) + go backend.CopyEmbed(py) + // Create an instance of the app structure app := backend.NewApp() diff --git a/manifest.json b/manifest.json index ce709df..c6f049d 100644 --- a/manifest.json +++ b/manifest.json @@ -11,79 +11,11 @@ "localModelDir": "models", "programFiles": [ { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/requirements.txt", - "path": "backend-python/requirements.txt" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/requirements_versions.txt", - "path": "backend-python/requirements_versions.txt" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/main.py", - "path": "backend-python/main.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/global_var.py", - "path": "backend-python/global_var.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/convert_model.py", - "path": "backend-python/convert_model.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/dep_check.py", - "path": "backend-python/dep_check.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/routes/completion.py", - "path": "backend-python/routes/completion.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/routes/config.py", - "path": "backend-python/routes/config.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/routes/state_cache.py", - "path": "backend-python/routes/state_cache.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/utils/ngrok.py", - "path": "backend-python/utils/ngrok.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/utils/rwkv.py", - "path": "backend-python/utils/rwkv.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/utils/torch.py", - "path": "backend-python/utils/torch.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/rwkv_tokenizer.py", - "path": "backend-python/rwkv_pip/rwkv_tokenizer.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/utils.py", - "path": "backend-python/rwkv_pip/utils.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/wkv_cuda_utils/wkv_cuda10_30.pyd", - "path": "backend-python/wkv_cuda_utils/wkv_cuda10_30.pyd" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/wkv_cuda_utils/wkv_cuda40.pyd", - "path": "backend-python/wkv_cuda_utils/wkv_cuda40.pyd" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/wkv_cuda_utils/wkv_cuda_model.py", - "path": "backend-python/wkv_cuda_utils/wkv_cuda_model.py" - }, - { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/rwkv_vocab_v20230424.txt", + "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/.rwkv_vocab_v20230424.txt", "path": "backend-python/rwkv_pip/rwkv_vocab_v20230424.txt" }, { - "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/20B_tokenizer.json", + "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/.20B_tokenizer.json", "path": "backend-python/rwkv_pip/20B_tokenizer.json" }, {