feat: save MIDI tracks to generation area; playing tracks and audio preview are still under development

backend-golang/midi.go

@@ -23,6 +23,7 @@ type MIDIMessage struct {
 
 var ports []Port
 var input rtmidi.MIDIIn
+var out rtmidi.MIDIOut
 var activeIndex int = -1
 var lastNoteTime time.Time
 
@@ -33,6 +34,14 @@ func (a *App) midiLoop() {
 		runtime.EventsEmit(a.ctx, "midiError", err.Error())
 		return
 	}
+	out, err = rtmidi.NewMIDIOutDefault()
+	if err != nil {
+		runtime.EventsEmit(a.ctx, "midiError", err.Error())
+	}
+	err = out.OpenPort(0, "")
+	if err != nil {
+		runtime.EventsEmit(a.ctx, "midiError", err.Error())
+	}
 	ticker := time.NewTicker(500 * time.Millisecond)
 	go func() {
 		for {
@@ -55,7 +64,7 @@ func (a *App) midiLoop() {
 
 func (a *App) OpenMidiPort(index int) error {
 	if input == nil {
-		return errors.New("failed to initialize MIDI")
+		return errors.New("failed to initialize MIDI input")
 	}
 	if activeIndex == index {
 		return nil
@@ -126,7 +135,7 @@ func (a *App) OpenMidiPort(index int) error {
 
 func (a *App) CloseMidiPort() error {
 	if input == nil {
-		return errors.New("failed to initialize MIDI")
+		return errors.New("failed to initialize MIDI input")
 	}
 	if activeIndex == -1 {
 		return nil
@@ -140,3 +149,16 @@ func (a *App) CloseMidiPort() error {
 	}
 	return nil
 }
+
+func (a *App) PlayNote(msg MIDIMessage) error {
+	if out == nil {
+		return errors.New("failed to initialize MIDI output")
+	}
+	channelByte := byte(msg.Channel)
+	if msg.MessageType == "NoteOn" {
+		out.SendMessage([]byte{0x90 | channelByte, byte(msg.Note), byte(msg.Velocity)})
+	} else if msg.MessageType == "NoteOff" {
+		out.SendMessage([]byte{0x80 | channelByte, byte(msg.Note), byte(msg.Velocity)})
+	}
+	return nil
+}

frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx

@@ -12,13 +12,15 @@ export const AudiotrackButton: FC<{
   size?: 'small' | 'medium' | 'large',
   shape?: 'rounded' | 'circular' | 'square';
   appearance?: 'secondary' | 'primary' | 'outline' | 'subtle' | 'transparent';
-}> = ({ size, shape, appearance }) => {
+  setPrompt: (prompt: string) => void;
+}> = ({ size, shape, appearance, setPrompt }) => {
   const { t } = useTranslation();
 
   return <Dialog onOpenChange={(e, data) => {
     if (!data.open) {
       flushMidiRecordingContent();
       commonStore.setRecordingTrackId('');
+      commonStore.setPlayingTrackId('');
     }
   }}>
     <DialogTrigger disableButtonEnhancement>
@@ -30,7 +32,7 @@ export const AudiotrackButton: FC<{
       <DialogBody>
         <DialogContent className="overflow-hidden">
           <CustomToastContainer />
-          <LazyImportComponent lazyChildren={AudiotrackEditor} />
+          <LazyImportComponent lazyChildren={AudiotrackEditor} lazyProps={{ setPrompt }} />
         </DialogContent>
       </DialogBody>
     </DialogSurface>

frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx

@@ -9,11 +9,13 @@ import {
   ArrowAutofitWidth20Regular,
   Delete16Regular,
   MusicNote220Regular,
+  Pause16Regular,
+  Play16Filled,
   Play16Regular,
   Record16Regular,
   Stop16Filled
 } from '@fluentui/react-icons';
-import { Button, Card, Slider, Text, Tooltip } from '@fluentui/react-components';
+import { Button, Card, DialogTrigger, Slider, Text, Tooltip } from '@fluentui/react-components';
 import { useWindowSize } from 'usehooks-ts';
 import commonStore from '../../stores/commonStore';
 import classnames from 'classnames';
@@ -26,6 +28,7 @@ import {
 import { toast } from 'react-toastify';
 import { ToastOptions } from 'react-toastify/dist/types';
 import { flushMidiRecordingContent, refreshTracksTotalTime } from '../../utils';
+import { PlayNote } from '../../../wailsjs/go/backend_golang/App';
 
 const snapValue = 25;
 const minimalMoveTime = 8; // 1000/125=8ms wait_events=125
@@ -109,9 +112,7 @@ const midiMessageToToken = (msg: MidiMessage) => {
 
 let dropRecordingTime = false;
 
-export const midiMessageHandler = (data: MidiMessage) => {
+export const midiMessageHandler = async (data: MidiMessage) => {
-  if (data.messageType === 'NoteOff')
-    return;
   if (data.messageType === 'ControlChange') {
     commonStore.setInstrumentType(Math.round(data.value / 127 * (InstrumentTypeNameMap.length - 1)));
     displayCurrentInstrumentType();
@@ -122,8 +123,15 @@ export const midiMessageHandler = (data: MidiMessage) => {
       dropRecordingTime = false;
       return;
     }
+    data = {
+      ...data,
+      instrument: commonStore.instrumentType
+    };
     commonStore.setRecordingRawContent([...commonStore.recordingRawContent, data]);
     commonStore.setRecordingContent(commonStore.recordingContent + midiMessageToToken(data));
+
+    //TODO data.channel = data.instrument;
+    PlayNote(data);
   }
 };
 
@@ -180,7 +188,7 @@ const Track: React.FC<TrackProps> = observer(({
   );
 });
 
-const AudiotrackEditor: FC = observer(() => {
+const AudiotrackEditor: FC<{ setPrompt: (prompt: string) => void }> = observer(({ setPrompt }) => {
   const { t } = useTranslation();
 
   const viewControlsContainerRef = useRef<HTMLDivElement>(null);
@@ -243,7 +251,7 @@ const AudiotrackEditor: FC = observer(() => {
       </div>
       <div className="flex pb-2 border-b" ref={toolbarRef}>
         <div className="flex gap-2" ref={toolbarButtonRef}>
-          <ToolTipButton desc={t('Play All')} icon={<Play16Regular />} />
+          <ToolTipButton disabled desc={t('Play All') + ' (Unavailable)'} icon={<Play16Regular />} />
           <ToolTipButton desc={t('Clear All')} icon={<Delete16Regular />} onClick={() => {
             commonStore.setTracks([]);
             commonStore.setTrackScale(1);
@@ -348,23 +356,38 @@ const AudiotrackEditor: FC = observer(() => {
             <div className="flex gap-1 border-r h-7">
               <ToolTipButton desc={commonStore.recordingTrackId === track.id ? t('Stop') : t('Record')}
                 icon={commonStore.recordingTrackId === track.id ? <Stop16Filled /> : <Record16Regular />}
-                size="small" shape="circular"
+                size="small" shape="circular" appearance="subtle"
-                appearance="subtle" onClick={() => {
+                onClick={() => {
-                flushMidiRecordingContent();
+                  flushMidiRecordingContent();
+                  commonStore.setPlayingTrackId('');
 
-                if (commonStore.recordingTrackId === track.id) {
+                  if (commonStore.recordingTrackId === track.id) {
+                    commonStore.setRecordingTrackId('');
+                  } else {
+                    dropRecordingTime = true;
+                    setSelectedTrackId(track.id);
+
+                    commonStore.setRecordingTrackId(track.id);
+                    commonStore.setRecordingContent(track.content);
+                    commonStore.setRecordingRawContent(track.rawContent.slice());
+                  }
+                }} />
+              <ToolTipButton disabled
+                desc={commonStore.playingTrackId === track.id ? t('Stop') : t('Play') + ' (Unavailable)'}
+                icon={commonStore.playingTrackId === track.id ? <Pause16Regular /> : <Play16Filled />}
+                size="small" shape="circular" appearance="subtle"
+                onClick={() => {
+                  flushMidiRecordingContent();
                   commonStore.setRecordingTrackId('');
-                } else {
-                  dropRecordingTime = true;
-                  setSelectedTrackId(track.id);
 
-                  commonStore.setRecordingTrackId(track.id);
+                  if (commonStore.playingTrackId === track.id) {
-                  commonStore.setRecordingContent(track.content);
+                    commonStore.setPlayingTrackId('');
-                  commonStore.setRecordingRawContent(track.rawContent.slice());
+                  } else {
-                }
+                    setSelectedTrackId(track.id);
-              }} />
+
-              <ToolTipButton desc={t('Play')} icon={<Play16Regular />} size="small" shape="circular"
+                    commonStore.setPlayingTrackId(track.id);
-                appearance="subtle" />
+                  }
+                }} />
               <ToolTipButton desc={t('Delete')} icon={<Delete16Regular />} size="small" shape="circular"
                 appearance="subtle" onClick={() => {
                 const tracks = commonStore.tracks.slice().filter(t => t.id !== track.id);
@@ -416,9 +439,52 @@ const AudiotrackEditor: FC = observer(() => {
           </div>
         </Card>
       }
-      <Button icon={<MusicNote220Regular />} style={{ minHeight: '32px' }}>
+      <DialogTrigger disableButtonEnhancement>
-        {t('Save to generation area')}
+        <Button icon={<MusicNote220Regular />} style={{ minHeight: '32px' }} onClick={() => {
-      </Button>
+          flushMidiRecordingContent();
+          commonStore.setRecordingTrackId('');
+          commonStore.setPlayingTrackId('');
+
+          const timestamp = [];
+          const sortedTracks = commonStore.tracks.slice().sort((a, b) => a.offsetTime - b.offsetTime);
+          for (const track of sortedTracks) {
+            timestamp.push(track.offsetTime);
+            let accContentTime = 0;
+            for (const msg of track.rawContent) {
+              if (msg.messageType === 'ElapsedTime') {
+                accContentTime += msg.value;
+                timestamp.push(track.offsetTime + accContentTime);
+              }
+            }
+          }
+          const sortedTimestamp = timestamp.slice().sort((a, b) => a - b);
+          const globalMessages: MidiMessage[] = sortedTimestamp.reduce((messages, current, i) =>
+              [...messages, {
+                messageType: 'ElapsedTime',
+                value: current - (i === 0 ? 0 : sortedTimestamp[i - 1])
+              } as MidiMessage]
+            , [] as MidiMessage[]);
+          for (const track of sortedTracks) {
+            let currentTime = track.offsetTime;
+            let accContentTime = 0;
+            for (const msg of track.rawContent) {
+              if (msg.messageType === 'ElapsedTime') {
+                accContentTime += msg.value;
+                currentTime = track.offsetTime + accContentTime;
+              } else if (msg.messageType === 'NoteOn') {
+                const insertIndex = sortedTimestamp.findIndex(t => t >= currentTime);
+                globalMessages.splice(insertIndex + 1, 0, msg);
+                sortedTimestamp.splice(insertIndex + 1, 0, 0); // placeholder
+              }
+            }
+          }
+          const result = globalMessages.map(m => midiMessageToToken(m)).join('');
+          commonStore.setCompositionSubmittedPrompt(result);
+          setPrompt(result);
+        }}>
+          {t('Save to generation area')}
+        </Button>
+      </DialogTrigger>
     </div>
   );
 });

frontend/src/pages/Composition.tsx

@@ -23,7 +23,7 @@ import {
   OpenMidiPort,
   OpenSaveFileDialogBytes
 } from '../../wailsjs/go/backend_golang/App';
-import { getServerRoot, toastWithButton } from '../utils';
+import { getServerRoot, getSoundFont, toastWithButton } from '../utils';
 import { CompositionParams } from '../types/composition';
 import { useMediaQuery } from 'usehooks-ts';
 import { AudiotrackButton } from './AudiotrackManager/AudiotrackButton';
@@ -71,20 +71,8 @@ const CompositionPanel: FC = observer(() => {
   };
 
   const setSoundFont = async () => {
-    let soundUrl: string;
-    if (commonStore.compositionParams.useLocalSoundFont)
-      soundUrl = 'assets/sound-font';
-    else
-      soundUrl = !commonStore.settings.giteeUpdatesSource ?
-        `https://raw.githubusercontent.com/josStorer/sgm_plus/master` :
-        `https://gitee.com/josc146/sgm_plus/raw/master`;
-    const fallbackUrl = 'https://cdn.jsdelivr.net/gh/josstorer/sgm_plus';
-    await fetch(soundUrl + '/soundfont.json').then(r => {
-      if (!r.ok)
-        soundUrl = fallbackUrl;
-    }).catch(() => soundUrl = fallbackUrl);
     if (playerRef.current) {
-      playerRef.current.soundFont = soundUrl;
+      playerRef.current.soundFont = await getSoundFont();
     }
   };
 
@@ -302,7 +290,7 @@ const CompositionPanel: FC = observer(() => {
                         <Option key={i} value={i.toString()}>{p.name}</Option>)
                       }
                     </Dropdown>
-                    <AudiotrackButton />
+                    <AudiotrackButton setPrompt={setPrompt} />
                   </div>
                 } />
             }

frontend/src/startup.ts

@@ -146,6 +146,6 @@ async function initMidi() {
     commonStore.setMidiPorts(data);
   });
   EventsOn('midiMessage', async (data: MidiMessage) => {
-    (await import('./pages/AudiotrackManager/AudiotrackEditor')).midiMessageHandler(data);
+    await (await import('./pages/AudiotrackManager/AudiotrackEditor')).midiMessageHandler(data);
   });
 }

frontend/src/stores/commonStore.ts

@@ -107,8 +107,9 @@ class CommonStore {
   trackTotalTime: number = tracksMinimalTotalTime;
   trackCurrentTime: number = 0;
   trackPlayStartTime: number = 0;
+  playingTrackId: string = '';
   recordingTrackId: string = '';
-  recordingContent: string = ''; // used to improve performance, and I'm too lazy to maintain an ID dictionary for this
+  recordingContent: string = ''; // used to improve performance of midiMessageHandler, and I'm too lazy to maintain an ID dictionary for this (although that would be better for realtime effects)
   recordingRawContent: MidiMessage[] = [];
   // configs
   currentModelConfigIndex: number = 0;
@@ -444,6 +445,10 @@ class CommonStore {
   setRecordingRawContent(value: MidiMessage[]) {
     this.recordingRawContent = value;
   }
+
+  setPlayingTrackId(value: string) {
+    this.playingTrackId = value;
+  }
 }
 
 export default new CommonStore();

frontend/src/types/composition.ts

@@ -32,6 +32,7 @@ export type MidiMessage = {
   velocity: number;
   control: number;
   value: number;
+  instrument: InstrumentType;
 }
 
 export enum InstrumentType {

frontend/src/utils/index.tsx

@@ -511,6 +511,22 @@ export function flushMidiRecordingContent() {
   commonStore.setRecordingRawContent([]);
 }
 
+export async function getSoundFont() {
+  let soundUrl: string;
+  if (commonStore.compositionParams.useLocalSoundFont)
+    soundUrl = 'assets/sound-font';
+  else
+    soundUrl = !commonStore.settings.giteeUpdatesSource ?
+      `https://raw.githubusercontent.com/josStorer/sgm_plus/master` :
+      `https://gitee.com/josc146/sgm_plus/raw/master`;
+  const fallbackUrl = 'https://cdn.jsdelivr.net/gh/josstorer/sgm_plus';
+  await fetch(soundUrl + '/soundfont.json').then(r => {
+    if (!r.ok)
+      soundUrl = fallbackUrl;
+  }).catch(() => soundUrl = fallbackUrl);
+  return soundUrl;
+}
+
 export function getSupportedCustomCudaFile(isBeta: boolean) {
   if ([' 10', ' 16', ' 20', ' 30', 'MX', 'Tesla P', 'Quadro P', 'NVIDIA P', 'TITAN X', 'TITAN RTX', 'RTX A',
     'Quadro RTX 4000', 'Quadro RTX 5000', 'Tesla T4', 'NVIDIA A10', 'NVIDIA A40'].some(v => commonStore.status.device_name.includes(v)))

frontend/wailsjs/go/backend_golang/App.d.ts

@@ -48,6 +48,8 @@ export function OpenSaveFileDialogBytes(arg1:string,arg2:string,arg3:Array<numbe
 
 export function PauseDownload(arg1:string):Promise<void>;
 
+export function PlayNote(arg1:backend_golang.MIDIMessage):Promise<void>;
+
 export function ReadFileInfo(arg1:string):Promise<backend_golang.FileInfo>;
 
 export function ReadJson(arg1:string):Promise<any>;

frontend/wailsjs/go/backend_golang/App.js

@@ -94,6 +94,10 @@ export function PauseDownload(arg1) {
   return window['go']['backend_golang']['App']['PauseDownload'](arg1);
 }
 
+export function PlayNote(arg1) {
+  return window['go']['backend_golang']['App']['PlayNote'](arg1);
+}
+
 export function ReadFileInfo(arg1) {
   return window['go']['backend_golang']['App']['ReadFileInfo'](arg1);
 }

frontend/wailsjs/go/models.ts

@@ -18,6 +18,28 @@ export namespace backend_golang {
 	        this.modTime = source["modTime"];
 	    }
 	}
+	export class MIDIMessage {
+	    messageType: string;
+	    channel: number;
+	    note: number;
+	    velocity: number;
+	    control: number;
+	    value: number;
+	
+	    static createFrom(source: any = {}) {
+	        return new MIDIMessage(source);
+	    }
+	
+	    constructor(source: any = {}) {
+	        if ('string' === typeof source) source = JSON.parse(source);
+	        this.messageType = source["messageType"];
+	        this.channel = source["channel"];
+	        this.note = source["note"];
+	        this.velocity = source["velocity"];
+	        this.control = source["control"];
+	        this.value = source["value"];
+	    }
+	}
 
 }