From e48f39375ee4a2510c3b7b068974a6c9dfdcc757 Mon Sep 17 00:00:00 2001
From: josc146 <josStorer@outlook.com>
Date: Sun, 10 Dec 2023 23:08:44 +0800
Subject: [PATCH] add midi tracks to webUI

---
 .../AudiotrackManager/AudiotrackEditor.tsx    | 15 +++++-
 frontend/src/pages/Chat.tsx                   |  2 +-
 frontend/src/pages/Composition.tsx            | 50 ++++++++++---------
 frontend/src/utils/web-file-operations.ts     |  9 +++-
 4 files changed, 48 insertions(+), 28 deletions(-)

diff --git a/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx b/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx
index 57029f1..5247489 100644
--- a/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx
+++ b/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx
@@ -18,7 +18,7 @@ import {
 } from '@fluentui/react-icons';
 import { Button, Card, DialogTrigger, Slider, Text, Tooltip } from '@fluentui/react-components';
 import { useWindowSize } from 'usehooks-ts';
-import commonStore from '../../stores/commonStore';
+import commonStore, { ModelStatus } from '../../stores/commonStore';
 import classnames from 'classnames';
 import {
   InstrumentType,
@@ -420,6 +420,7 @@ const AudiotrackEditor: FC<{ setPrompt: (prompt: string) => void }> = observer((
           <div key={track.id} className="flex gap-2 pb-1 border-b">
             <div className="flex gap-1 border-r h-7">
               <ToolTipButton desc={commonStore.recordingTrackId === track.id ? t('Stop') : t('Record')}
+                disabled={commonStore.platform === 'web'}
                 icon={commonStore.recordingTrackId === track.id ? <Stop16Filled /> : <Record16Regular />}
                 size="small" shape="circular" appearance="subtle"
                 onClick={() => {
@@ -481,6 +482,7 @@ const AudiotrackEditor: FC<{ setPrompt: (prompt: string) => void }> = observer((
           <div className="flex gap-1">
             <Button icon={<Add16Regular />} size="small" shape="circular"
               appearance="subtle"
+              disabled={commonStore.platform === 'web'}
               onClick={() => {
                 commonStore.setTracks([...commonStore.tracks, {
                   id: uuid(),
@@ -496,11 +498,20 @@ const AudiotrackEditor: FC<{ setPrompt: (prompt: string) => void }> = observer((
             <Button icon={<ArrowUpload16Regular />} size="small" shape="circular"
               appearance="subtle"
               onClick={() => {
+                if (commonStore.status.status === ModelStatus.Offline && !commonStore.settings.apiUrl && commonStore.platform !== 'web') {
+                  toast(t('Please click the button in the top right corner to start the model'), { type: 'warning' });
+                  return;
+                }
+
                 OpenOpenFileDialog('*.mid').then(async filePath => {
                   if (!filePath)
                     return;
 
-                  const blob = await fetch(absPathAsset(filePath)).then(r => r.blob());
+                  let blob: Blob;
+                  if (commonStore.platform === 'web')
+                    blob = (filePath as unknown as { blob: Blob }).blob;
+                  else
+                    blob = await fetch(absPathAsset(filePath)).then(r => r.blob());
                   const bodyForm = new FormData();
                   bodyForm.append('file_data', blob);
                   fetch(getServerRoot(commonStore.getCurrentModelConfig().apiParameters.apiPort) + '/midi-to-text', {
diff --git a/frontend/src/pages/Chat.tsx b/frontend/src/pages/Chat.tsx
index 32ecfd8..cd42740 100644
--- a/frontend/src/pages/Chat.tsx
+++ b/frontend/src/pages/Chat.tsx
@@ -567,7 +567,7 @@ const ChatPanel: FC = observer(() => {
                     const setUploading = () => commonStore.setAttachmentUploading(true);
                     // actually, status of web platform is always Offline
                     if (commonStore.platform === 'web' || commonStore.status.status === ModelStatus.Offline || currentConfig.modelParameters.device === 'WebGPU') {
-                      webOpenOpenFileDialog({ filterPattern, fnStartLoading: setUploading }).then(webReturn => {
+                      webOpenOpenFileDialog(filterPattern, setUploading).then(webReturn => {
                         if (webReturn.content)
                           commonStore.setCurrentTempAttachment(
                             {
diff --git a/frontend/src/pages/Composition.tsx b/frontend/src/pages/Composition.tsx
index e16e60c..fd54ecf 100644
--- a/frontend/src/pages/Composition.tsx
+++ b/frontend/src/pages/Composition.tsx
@@ -250,32 +250,36 @@ const CompositionPanel: FC = observer(() => {
                   }} />
               } />
             <div className="grow" />
-            <Checkbox className="select-none"
-              size="large" label={t('Use Local Sound Font')} checked={params.useLocalSoundFont}
-              onChange={async (_, data) => {
-                if (data.checked) {
-                  if (!await FileExists('assets/sound-font/accordion/instrument.json')) {
-                    toast(t('Failed to load local sound font, please check if the files exist - assets/sound-font'),
-                      { type: 'warning' });
-                    return;
+            {
+              commonStore.platform !== 'web' &&
+              <Checkbox className="select-none"
+                size="large" label={t('Use Local Sound Font')} checked={params.useLocalSoundFont}
+                onChange={async (_, data) => {
+                  if (data.checked) {
+                    if (!await FileExists('assets/sound-font/accordion/instrument.json')) {
+                      toast(t('Failed to load local sound font, please check if the files exist - assets/sound-font'),
+                        { type: 'warning' });
+                      return;
+                    }
                   }
-                }
-                setParams({
-                  useLocalSoundFont: data.checked as boolean
-                });
-                setSoundFont();
-              }} />
+                  setParams({
+                    useLocalSoundFont: data.checked as boolean
+                  });
+                  setSoundFont();
+                }} />
+            }
             <Checkbox className="select-none"
               size="large" label={t('Auto Play At The End')} checked={params.autoPlay} onChange={(_, data) => {
               setParams({
                 autoPlay: data.checked as boolean
               });
             }} />
-            {commonStore.platform !== 'web' &&
-              <Labeled flex breakline label={t('MIDI Input')}
-                desc={t('Select the MIDI input device to be used.')}
-                content={
-                  <div className="flex flex-col gap-1">
+            <Labeled flex breakline label={t('MIDI Input')}
+              desc={t('Select the MIDI input device to be used.')}
+              content={
+                <div className="flex flex-col gap-1">
+                  {
+                    commonStore.platform !== 'web' &&
                     <Dropdown style={{ minWidth: 0 }}
                       value={(commonStore.activeMidiDeviceIndex === -1 || !(commonStore.activeMidiDeviceIndex in commonStore.midiPorts))
                         ? t('None')!
@@ -299,10 +303,10 @@ const CompositionPanel: FC = observer(() => {
                         <Option key={i} value={i.toString()}>{p.name}</Option>)
                       }
                     </Dropdown>
-                    <AudiotrackButton setPrompt={setPrompt} />
-                  </div>
-                } />
-            }
+                  }
+                  <AudiotrackButton setPrompt={setPrompt} />
+                </div>
+              } />
           </div>
           <div className="flex justify-between gap-2">
             <ToolTipButton desc={t('Regenerate')} icon={<ArrowSync20Regular />} onClick={() => {
diff --git a/frontend/src/utils/web-file-operations.ts b/frontend/src/utils/web-file-operations.ts
index 872305b..1d6f224 100644
--- a/frontend/src/utils/web-file-operations.ts
+++ b/frontend/src/utils/web-file-operations.ts
@@ -1,12 +1,17 @@
 import { getDocument, GlobalWorkerOptions, PDFDocumentProxy } from 'pdfjs-dist';
 import { TextItem } from 'pdfjs-dist/types/src/display/api';
 
-export function webOpenOpenFileDialog({ filterPattern, fnStartLoading }: { filterPattern: string, fnStartLoading: Function | null }): Promise<{ blob: Blob, content?: string }> {
+export function webOpenOpenFileDialog(filterPattern: string, fnStartLoading: Function | undefined): Promise<{
+  blob: Blob,
+  content?: string
+}> {
   return new Promise((resolve, reject) => {
     const input = document.createElement('input');
     input.type = 'file';
     input.accept = filterPattern
     .replaceAll('*.txt', 'text/plain')
+    .replace('*.midi', 'audio/midi')
+    .replace('*.mid', 'audio/midi')
     .replaceAll('*.', 'application/')
     .replaceAll(';', ',');
 
@@ -15,7 +20,7 @@ export function webOpenOpenFileDialog({ filterPattern, fnStartLoading }: { filte
       const file: Blob = e.target?.files[0];
       if (fnStartLoading && typeof fnStartLoading === 'function')
         fnStartLoading();
-      if (!GlobalWorkerOptions.workerSrc)
+      if (!GlobalWorkerOptions.workerSrc && file.type === 'application/pdf')
         // @ts-ignore
         GlobalWorkerOptions.workerSrc = await import('pdfjs-dist/build/pdf.worker.min.mjs');
       if (file.type === 'text/plain') {