MIDI Recording and details improvement

backend-golang/app.go

@@ -55,6 +55,7 @@ func (a *App) OnStartup(ctx context.Context) {
 	}
 
 	a.downloadLoop()
+	a.midiLoop()
 	a.watchFs()
 	a.monitorHardware()
 }

backend-golang/midi.go

@@ -0,0 +1,142 @@
+package backend_golang
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mattrtaylor/go-rtmidi"
+	"github.com/wailsapp/wails/v2/pkg/runtime"
+)
+
+type Port struct {
+	Name string `json:"name"`
+}
+type MIDIMessage struct {
+	MessageType string `json:"messageType"`
+	Channel     int    `json:"channel"`
+	Note        int    `json:"note"`
+	Velocity    int    `json:"velocity"`
+	Control     int    `json:"control"`
+	Value       int    `json:"value"`
+}
+
+var ports []Port
+var input rtmidi.MIDIIn
+var activeIndex int = -1
+var lastNoteTime time.Time
+
+func (a *App) midiLoop() {
+	var err error
+	input, err = rtmidi.NewMIDIInDefault()
+	if err != nil {
+		runtime.EventsEmit(a.ctx, "midiError", err.Error())
+		return
+	}
+	ticker := time.NewTicker(500 * time.Millisecond)
+	go func() {
+		for {
+			<-ticker.C
+			count, err := input.PortCount()
+			if err != nil {
+				continue
+			}
+			ports = make([]Port, count)
+			for i := 0; i < count; i++ {
+				name, err := input.PortName(i)
+				if err == nil {
+					ports[i].Name = name
+				}
+			}
+			runtime.EventsEmit(a.ctx, "midiPorts", &ports)
+		}
+	}()
+}
+
+func (a *App) OpenMidiPort(index int) error {
+	if input == nil {
+		return errors.New("failed to initialize MIDI")
+	}
+	if activeIndex == index {
+		return nil
+	}
+	input.Destroy()
+	var err error
+	input, err = rtmidi.NewMIDIInDefault()
+	if err != nil {
+		return err
+	}
+	err = input.SetCallback(func(msg rtmidi.MIDIIn, bytes []byte, t float64) {
+		// https://www.midi.org/specifications-old/item/table-1-summary-of-midi-message
+		// https://www.rfc-editor.org/rfc/rfc6295.html
+		//
+		// msgType channel
+		//  1001     0000
+		//
+		msgType := bytes[0] >> 4
+		channel := bytes[0] & 0x0f
+		switch msgType {
+		case 0x8:
+			note := bytes[1]
+			runtime.EventsEmit(a.ctx, "midiMessage", &MIDIMessage{
+				MessageType: "NoteOff",
+				Channel:     int(channel),
+				Note:        int(note),
+			})
+		case 0x9:
+			elapsed := time.Since(lastNoteTime)
+			lastNoteTime = time.Now()
+			runtime.EventsEmit(a.ctx, "midiMessage", &MIDIMessage{
+				MessageType: "ElapsedTime",
+				Value:       int(elapsed.Milliseconds()),
+			})
+			note := bytes[1]
+			velocity := bytes[2]
+			runtime.EventsEmit(a.ctx, "midiMessage", &MIDIMessage{
+				MessageType: "NoteOn",
+				Channel:     int(channel),
+				Note:        int(note),
+				Velocity:    int(velocity),
+			})
+		case 0xb:
+			// control 12 => K1 knob, control 13 => K2 knob
+			control := bytes[1]
+			value := bytes[2]
+			runtime.EventsEmit(a.ctx, "midiMessage", &MIDIMessage{
+				MessageType: "ControlChange",
+				Channel:     int(channel),
+				Control:     int(control),
+				Value:       int(value),
+			})
+		default:
+			fmt.Printf("Unknown midi message: %v\n", bytes)
+		}
+	})
+	if err != nil {
+		return err
+	}
+	err = input.OpenPort(index, "")
+	if err != nil {
+		return err
+	}
+	activeIndex = index
+	lastNoteTime = time.Now()
+	return nil
+}
+
+func (a *App) CloseMidiPort() error {
+	if input == nil {
+		return errors.New("failed to initialize MIDI")
+	}
+	if activeIndex == -1 {
+		return nil
+	}
+	activeIndex = -1
+	input.Destroy()
+	var err error
+	input, err = rtmidi.NewMIDIInDefault()
+	if err != nil {
+		return err
+	}
+	return nil
+}

frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx

@@ -3,6 +3,8 @@ import { useTranslation } from 'react-i18next';
 import { Button, Dialog, DialogBody, DialogContent, DialogSurface, DialogTrigger } from '@fluentui/react-components';
 import { CustomToastContainer } from '../../components/CustomToastContainer';
 import { LazyImportComponent } from '../../components/LazyImportComponent';
+import { flushMidiRecordingContent } from '../../utils';
+import commonStore from '../../stores/commonStore';
 
 const AudiotrackEditor = lazy(() => import('./AudiotrackEditor'));
 
@@ -13,7 +15,12 @@ export const AudiotrackButton: FC<{
 }> = ({ size, shape, appearance }) => {
   const { t } = useTranslation();
 
-  return <Dialog>
+  return <Dialog onOpenChange={(e, data) => {
+    if (!data.open) {
+      flushMidiRecordingContent();
+      commonStore.setRecordingTrackId('');
+    }
+  }}>
     <DialogTrigger disableButtonEnhancement>
       <Button size={size} shape={shape} appearance={appearance}>
         {t('Open MIDI Input Audio Tracks')}

frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx

@@ -10,19 +10,39 @@ import {
   Delete16Regular,
   MusicNote220Regular,
   Play16Regular,
-  Record16Regular
+  Record16Regular,
+  Stop16Filled
 } from '@fluentui/react-icons';
 import { Button, Card, Slider, Text, Tooltip } from '@fluentui/react-components';
 import { useWindowSize } from 'usehooks-ts';
 import commonStore from '../../stores/commonStore';
 import classnames from 'classnames';
+import {
+  InstrumentTypeNameMap,
+  InstrumentTypeTokenMap,
+  MidiMessage,
+  tracksMinimalTotalTime
+} from '../../types/composition';
+import { toast } from 'react-toastify';
+import { ToastOptions } from 'react-toastify/dist/types';
+import { flushMidiRecordingContent, refreshTracksTotalTime } from '../../utils';
 
 const snapValue = 25;
-const minimalMoveTime = 8; // 1000/125=8ms
+const minimalMoveTime = 8; // 1000/125=8ms wait_events=125
-const scaleMin = 0.2;
+const scaleMin = 0.05;
 const scaleMax = 3;
 const baseMoveTime = Math.round(minimalMoveTime / scaleMin);
 
+const velocityEvents = 128;
+const velocityBins = 12;
+const velocityExp = 0.5;
+
+const minimalTrackWidth = 80;
+const trackInitOffsetPx = 10;
+const pixelFix = 0.5;
+const topToArrowIcon = 19;
+const arrowIconToTracks = 23;
+
 type TrackProps = {
   id: string;
   right: number;
@@ -31,6 +51,82 @@ type TrackProps = {
   onSelect: (id: string) => void;
 };
 
+const displayCurrentInstrumentType = () => {
+  const displayPanelId = 'instrument_panel_id';
+  const content: React.ReactNode =
+    <div className="flex gap-2 items-center">
+      {InstrumentTypeNameMap.map((name, i) =>
+        <Text key={name} style={{ whiteSpace: 'nowrap' }}
+          className={commonStore.instrumentType === i ? 'text-blue-600' : ''}
+          weight={commonStore.instrumentType === i ? 'bold' : 'regular'}
+          size={commonStore.instrumentType === i ? 300 : 100}
+        >{name}</Text>)}
+    </div>;
+  const options: ToastOptions = {
+    type: 'default',
+    autoClose: 2000,
+    toastId: displayPanelId,
+    position: 'top-left',
+    style: {
+      width: 'fit-content'
+    }
+  };
+  if (toast.isActive(displayPanelId))
+    toast.update(displayPanelId, {
+      render: content,
+      ...options
+    });
+  else
+    toast(content, options);
+};
+
+const velocityToBin = (velocity: number) => {
+  velocity = Math.max(0, Math.min(velocity, velocityEvents - 1));
+  const binsize = velocityEvents / (velocityBins - 1);
+  return Math.ceil((velocityEvents * ((Math.pow(velocityExp, (velocity / velocityEvents)) - 1.0) / (velocityExp - 1.0))) / binsize);
+};
+
+const midiMessageToToken = (msg: MidiMessage) => {
+  if (msg.messageType === 'NoteOn') {
+    const instrument = InstrumentTypeTokenMap[commonStore.instrumentType];
+    const note = msg.note.toString(16);
+    const velocity = velocityToBin(msg.velocity).toString(16);
+    return `${instrument}:${note}:${velocity} `;
+  } else if (msg.messageType === 'ElapsedTime') {
+    let time = Math.round(msg.value / minimalMoveTime);
+    const num = Math.floor(time / 125); // wait_events=125
+    time -= num * 125;
+    let ret = '';
+    for (let i = 0; i < num; i++) {
+      ret += 't125 ';
+    }
+    if (time > 0)
+      ret += `t${time} `;
+    return ret;
+  } else
+    return '';
+};
+
+let dropRecordingTime = false;
+
+export const midiMessageHandler = (data: MidiMessage) => {
+  if (data.messageType === 'NoteOff')
+    return;
+  if (data.messageType === 'ControlChange') {
+    commonStore.setInstrumentType(Math.round(data.value / 127 * (InstrumentTypeNameMap.length - 1)));
+    displayCurrentInstrumentType();
+    return;
+  }
+  if (commonStore.recordingTrackId) {
+    if (dropRecordingTime && data.messageType === 'ElapsedTime') {
+      dropRecordingTime = false;
+      return;
+    }
+    commonStore.setRecordingRawContent([...commonStore.recordingRawContent, data]);
+    commonStore.setRecordingContent(commonStore.recordingContent + midiMessageToToken(data));
+  }
+};
+
 const Track: React.FC<TrackProps> = observer(({
   id,
   right,
@@ -66,18 +162,19 @@ const Track: React.FC<TrackProps> = observer(({
         const tracks = commonStore.tracks.slice();
         tracks[trackIndex].offsetTime += offsetTime;
         commonStore.setTracks(tracks);
+        refreshTracksTotalTime();
       }}
     >
       <div
         className={`p-1 cursor-move rounded whitespace-nowrap overflow-hidden ${trackClass}`}
         style={{
-          width: `${Math.max(80,
+          width: `${Math.max(minimalTrackWidth,
             track.contentTime / (baseMoveTime * scale) * snapValue
           )}px`
         }}
         onClick={() => onSelect(id)}
       >
-        <span className="text-white">{t('Track') + ' ' + id}</span>
+        <span className="text-white">{t('Track') + ' ' + (track.content || id)}</span>
       </div>
     </Draggable>
   );
@@ -86,12 +183,15 @@ const Track: React.FC<TrackProps> = observer(({
 const AudiotrackEditor: FC = observer(() => {
   const { t } = useTranslation();
 
+  const viewControlsContainerRef = useRef<HTMLDivElement>(null);
   const currentTimeControlRef = useRef<HTMLDivElement>(null);
   const playStartTimeControlRef = useRef<HTMLDivElement>(null);
+  const tracksEndLineRef = useRef<HTMLDivElement>(null);
   const tracksRef = useRef<HTMLDivElement>(null);
   const toolbarRef = useRef<HTMLDivElement>(null);
   const toolbarButtonRef = useRef<HTMLDivElement>(null);
   const toolbarSliderRef = useRef<HTMLInputElement>(null);
+  const contentPreviewRef = useRef<HTMLDivElement>(null);
 
   const [refreshRef, setRefreshRef] = useState(false);
 
@@ -102,6 +202,20 @@ const AudiotrackEditor: FC = observer(() => {
   const playStartTimeControlX = useRef(0);
   const selectedTrack = selectedTrackId ? commonStore.tracks.find(t => t.id === selectedTrackId) : undefined;
 
+  useEffect(() => {
+    if (toolbarSliderRef.current && toolbarSliderRef.current.parentElement)
+      toolbarSliderRef.current.parentElement.style.removeProperty('--fui-Slider--steps-percent');
+  }, []);
+
+  const scrollContentToBottom = () => {
+    if (contentPreviewRef.current)
+      contentPreviewRef.current.scrollTop = contentPreviewRef.current.scrollHeight;
+  };
+
+  useEffect(() => {
+    scrollContentToBottom();
+  }, [commonStore.recordingContent]);
+
   useEffect(() => {
     setRefreshRef(!refreshRef);
   }, [windowSize, commonStore.tracks]);
@@ -115,10 +229,12 @@ const AudiotrackEditor: FC = observer(() => {
   const currentTimeControlWidth = (timeOfTracksWidth < commonStore.trackTotalTime)
     ? timeOfTracksWidth / commonStore.trackTotalTime * viewControlsContainerWidth
     : 0;
-  const playStartTimeControlPosition = {
+  const playStartTimeControlPosition = (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue;
-    x: (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue,
+  const tracksEndPosition = (commonStore.trackTotalTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue;
-    y: 0
+  const moveableTracksWidth = (tracksEndLineRef.current && viewControlsContainerRef.current &&
-  };
+    ((tracksEndLineRef.current.getBoundingClientRect().left - (viewControlsContainerRef.current.getBoundingClientRect().left + trackInitOffsetPx)) > 0))
+    ? tracksEndLineRef.current.getBoundingClientRect().left - (viewControlsContainerRef.current.getBoundingClientRect().left + trackInitOffsetPx)
+    : Infinity;
 
   return (
     <div className="flex flex-col gap-2 overflow-hidden" style={{ width: '80vw', height: '80vh' }}>
@@ -130,10 +246,28 @@ const AudiotrackEditor: FC = observer(() => {
           <ToolTipButton desc={t('Play All')} icon={<Play16Regular />} />
           <ToolTipButton desc={t('Clear All')} icon={<Delete16Regular />} onClick={() => {
             commonStore.setTracks([]);
+            commonStore.setTrackScale(1);
+            commonStore.setTrackTotalTime(tracksMinimalTotalTime);
+            commonStore.setTrackCurrentTime(0);
+            commonStore.setTrackPlayStartTime(0);
           }} />
         </div>
         <div className="grow">
-          <div className="flex flex-col ml-2 mr-2">
+          <div className="flex flex-col ml-2 mr-2" ref={viewControlsContainerRef}>
+            <div className="relative">
+              <Tooltip content={`${commonStore.trackTotalTime} ms`} showDelay={0} hideDelay={0}
+                relationship="description">
+                <div className="border-l absolute"
+                  ref={tracksEndLineRef}
+                  style={{
+                    height: (tracksRef.current && commonStore.tracks.length > 0)
+                      ? tracksRef.current.clientHeight - arrowIconToTracks
+                      : 0,
+                    top: `${topToArrowIcon + arrowIconToTracks}px`,
+                    left: `${tracksEndPosition + trackInitOffsetPx - pixelFix}px`
+                  }} />
+              </Tooltip>
+            </div>
             <Draggable axis="x" bounds={{
               left: 0,
               right: viewControlsContainerWidth - currentTimeControlWidth
@@ -160,17 +294,17 @@ const AudiotrackEditor: FC = observer(() => {
             </Draggable>
             <div className={classnames(
               'flex',
-              (playStartTimeControlPosition.x < 0 || playStartTimeControlPosition.x > viewControlsContainerWidth)
+              (playStartTimeControlPosition < 0 || playStartTimeControlPosition > viewControlsContainerWidth)
               && 'hidden'
             )}>
               <Draggable axis="x" bounds={{
                 left: 0,
                 right: (playStartTimeControlRef.current)
-                  ? viewControlsContainerWidth - playStartTimeControlRef.current.clientWidth
+                  ? Math.min(viewControlsContainerWidth - playStartTimeControlRef.current.clientWidth, moveableTracksWidth)
                   : 0
               }}
                 grid={[snapValue, snapValue]}
-                position={playStartTimeControlPosition}
+                position={{ x: playStartTimeControlPosition, y: 0 }}
                 onStart={(e, data) => {
                   playStartTimeControlX.current = data.lastX;
                 }}
@@ -192,14 +326,15 @@ const AudiotrackEditor: FC = observer(() => {
                         ? tracksRef.current.clientHeight
                         : 0,
                       top: '50%',
-                      left: 'calc(50% - 0.5px)'
+                      left: `calc(50% - ${pixelFix}px)`
                     }} />
                 </div>
               </Draggable>
             </div>
           </div>
         </div>
-        <Tooltip content={t('Scale View')!} showDelay={0} hideDelay={0} relationship="label">
+        <Tooltip content={t('Scale View')! + ': ' + commonStore.trackScale} showDelay={0} hideDelay={0}
+          relationship="description">
           <Slider ref={toolbarSliderRef} value={commonStore.trackScale} step={scaleMin} max={scaleMax} min={scaleMin}
             onChange={(e, data) => {
               commonStore.setTrackScale(data.value);
@@ -211,8 +346,23 @@ const AudiotrackEditor: FC = observer(() => {
         {commonStore.tracks.map(track =>
           <div key={track.id} className="flex gap-2 pb-1 border-b">
             <div className="flex gap-1 border-r h-7">
-              <ToolTipButton desc={t('Record')} icon={<Record16Regular />} size="small" shape="circular"
+              <ToolTipButton desc={commonStore.recordingTrackId === track.id ? t('Stop') : t('Record')}
-                appearance="subtle" />
+                icon={commonStore.recordingTrackId === track.id ? <Stop16Filled /> : <Record16Regular />}
+                size="small" shape="circular"
+                appearance="subtle" onClick={() => {
+                flushMidiRecordingContent();
+
+                if (commonStore.recordingTrackId === track.id) {
+                  commonStore.setRecordingTrackId('');
+                } else {
+                  dropRecordingTime = true;
+                  setSelectedTrackId(track.id);
+
+                  commonStore.setRecordingTrackId(track.id);
+                  commonStore.setRecordingContent(track.content);
+                  commonStore.setRecordingRawContent(track.rawContent.slice());
+                }
+              }} />
               <ToolTipButton desc={t('Play')} icon={<Play16Regular />} size="small" shape="circular"
                 appearance="subtle" />
               <ToolTipButton desc={t('Delete')} icon={<Delete16Regular />} size="small" shape="circular"
@@ -226,7 +376,7 @@ const AudiotrackEditor: FC = observer(() => {
                 <Track
                   id={track.id}
                   scale={scale}
-                  right={tracksWidth}
+                  right={Math.min(tracksWidth, moveableTracksWidth)}
                   isSelected={selectedTrackId === track.id}
                   onSelect={setSelectedTrackId}
                 />
@@ -240,6 +390,7 @@ const AudiotrackEditor: FC = observer(() => {
               commonStore.setTracks([...commonStore.tracks, {
                 id: uuid(),
                 content: '',
+                rawContent: [],
                 offsetTime: 0,
                 contentTime: 0
               }]);
@@ -253,12 +404,14 @@ const AudiotrackEditor: FC = observer(() => {
       </div>
       <div className="grow"></div>
       {selectedTrack &&
-        <Card size="small" appearance="outline" style={{ minHeight: '150px' }}>
+        <Card size="small" appearance="outline" style={{ minHeight: '150px', maxHeight: '200px' }}>
           <div className="flex flex-col gap-1 overflow-hidden">
             <Text size={100}>{`${t('Start Time')}: ${selectedTrack.offsetTime} ms`}</Text>
             <Text size={100}>{`${t('Content Time')}: ${selectedTrack.contentTime} ms`}</Text>
-            <div className="overflow-y-auto overflow-x-hidden">
+            <div className="overflow-y-auto overflow-x-hidden" ref={contentPreviewRef}>
-              {selectedTrack.content}
+              {selectedTrackId === commonStore.recordingTrackId
+                ? commonStore.recordingContent
+                : selectedTrack.content}
             </div>
           </div>
         </Card>

frontend/src/pages/Composition.tsx

@@ -16,7 +16,13 @@ import { PlayerElement, VisualizerElement } from 'html-midi-player';
 import * as mm from '@magenta/music/esm/core.js';
 import { NoteSequence } from '@magenta/music/esm/protobuf.js';
 import { defaultCompositionPrompt } from './defaultConfigs';
-import { FileExists, OpenFileFolder, OpenSaveFileDialogBytes } from '../../wailsjs/go/backend_golang/App';
+import {
+  CloseMidiPort,
+  FileExists,
+  OpenFileFolder,
+  OpenMidiPort,
+  OpenSaveFileDialogBytes
+} from '../../wailsjs/go/backend_golang/App';
 import { getServerRoot, toastWithButton } from '../utils';
 import { CompositionParams } from '../types/composition';
 import { useMediaQuery } from 'usehooks-ts';
@@ -273,8 +279,26 @@ const CompositionPanel: FC = observer(() => {
                 desc={t('Select the MIDI input device to be used.')}
                 content={
                   <div className="flex flex-col gap-1">
-                    <Dropdown style={{ minWidth: 0 }}>
+                    <Dropdown style={{ minWidth: 0 }}
-                      <Option>{t('None')!}</Option>
+                      value={commonStore.activeMidiDeviceIndex === -1 ? t('None')! : commonStore.midiPorts[commonStore.activeMidiDeviceIndex].name}
+                      selectedOptions={[commonStore.activeMidiDeviceIndex.toString()]}
+                      onOptionSelect={(_, data) => {
+                        if (data.optionValue) {
+                          const index = Number(data.optionValue);
+                          let action = (index === -1)
+                            ? () => CloseMidiPort()
+                            : () => OpenMidiPort(index);
+                          action().then(() => {
+                            commonStore.setActiveMidiDeviceIndex(index);
+                          }).catch((e) => {
+                            toast(t('Error') + ' - ' + (e.message || e), { type: 'error', autoClose: 2500 });
+                          });
+                        }
+                      }}>
+                      <Option value={'-1'}>{t('None')!}</Option>
+                      {commonStore.midiPorts.map((p, i) =>
+                        <Option key={i} value={i.toString()}>{p.name}</Option>)
+                      }
                     </Dropdown>
                     <AudiotrackButton />
                   </div>

frontend/src/startup.ts

@@ -7,6 +7,8 @@ import manifest from '../../manifest.json';
 import { defaultModelConfigs, defaultModelConfigsMac } from './pages/defaultConfigs';
 import { t } from 'i18next';
 import { Preset } from './types/presets';
+import { toast } from 'react-toastify';
+import { MidiMessage, MidiPort } from './types/composition';
 
 export async function startup() {
   initPresets();
@@ -26,6 +28,7 @@ export async function startup() {
     initLocalModelsNotify();
     initLoraModels();
     initHardwareMonitor();
+    initMidi();
   }
 
   await initConfig();
@@ -134,3 +137,15 @@ async function initHardwareMonitor() {
     }
   });
 }
+
+async function initMidi() {
+  EventsOn('midiError', (data: string) => {
+    toast('MIDI Error: ' + data, { type: 'error' });
+  });
+  EventsOn('midiPorts', (data: MidiPort[]) => {
+    commonStore.setMidiPorts(data);
+  });
+  EventsOn('midiMessage', async (data: MidiMessage) => {
+    (await import('./pages/AudiotrackManager/AudiotrackEditor')).midiMessageHandler(data);
+  });
+}

frontend/src/stores/commonStore.ts

@@ -9,7 +9,14 @@ import { Preset } from '../types/presets';
 import { AboutContent } from '../types/about';
 import { Attachment, ChatParams, Conversation } from '../types/chat';
 import { CompletionPreset } from '../types/completion';
-import { CompositionParams, Track } from '../types/composition';
+import {
+  CompositionParams,
+  InstrumentType,
+  MidiMessage,
+  MidiPort,
+  Track,
+  tracksMinimalTotalTime
+} from '../types/composition';
 import { ModelConfig } from '../types/configs';
 import { DownloadStatus } from '../types/downloads';
 import { IntroductionContent } from '../types/home';
@@ -90,11 +97,19 @@ class CommonStore {
   };
   compositionGenerating: boolean = false;
   compositionSubmittedPrompt: string = defaultCompositionPrompt;
+  // composition midi device
+  midiPorts: MidiPort[] = [];
+  activeMidiDeviceIndex: number = -1;
+  instrumentType: InstrumentType = InstrumentType.Piano;
+  // composition tracks
   tracks: Track[] = [];
   trackScale: number = 1;
-  trackTotalTime: number = 5000;
+  trackTotalTime: number = tracksMinimalTotalTime;
   trackCurrentTime: number = 0;
   trackPlayStartTime: number = 0;
+  recordingTrackId: string = '';
+  recordingContent: string = ''; // used to improve performance, and I'm too lazy to maintain an ID dictionary for this
+  recordingRawContent: MidiMessage[] = [];
   // configs
   currentModelConfigIndex: number = 0;
   modelConfigs: ModelConfig[] = [];
@@ -405,6 +420,30 @@ class CommonStore {
   setTrackPlayStartTime(value: number) {
     this.trackPlayStartTime = value;
   }
+
+  setMidiPorts(value: MidiPort[]) {
+    this.midiPorts = value;
+  }
+
+  setInstrumentType(value: InstrumentType) {
+    this.instrumentType = value;
+  }
+
+  setRecordingTrackId(value: string) {
+    this.recordingTrackId = value;
+  }
+
+  setActiveMidiDeviceIndex(value: number) {
+    this.activeMidiDeviceIndex = value;
+  }
+
+  setRecordingContent(value: string) {
+    this.recordingContent = value;
+  }
+
+  setRecordingRawContent(value: MidiMessage[]) {
+    this.recordingRawContent = value;
+  }
 }
 
 export default new CommonStore();

frontend/src/types/composition.ts

@@ -1,5 +1,7 @@
 import { NoteSequence } from '@magenta/music/esm/protobuf';
 
+export const tracksMinimalTotalTime = 5000;
+
 export type CompositionParams = {
   prompt: string,
   maxResponseToken: number,
@@ -13,6 +15,69 @@ export type CompositionParams = {
 export type Track = {
   id: string;
   content: string;
+  rawContent: MidiMessage[];
   offsetTime: number;
   contentTime: number;
-};
+};
+export type MidiPort = {
+  name: string;
+}
+
+export type MessageType = 'NoteOff' | 'NoteOn' | 'ElapsedTime' | 'ControlChange';
+
+export type MidiMessage = {
+  messageType: MessageType;
+  channel: number;
+  note: number;
+  velocity: number;
+  control: number;
+  value: number;
+}
+
+export enum InstrumentType {
+  Piano,
+  Percussion,
+  Drum,
+  Tuba,
+  Marimba,
+  Bass,
+  Guitar,
+  Violin,
+  Trumpet,
+  Sax,
+  Flute,
+  Lead,
+  Pad,
+}
+
+export const InstrumentTypeNameMap = [
+  'Piano',
+  'Percussion',
+  'Drum',
+  'Tuba',
+  'Marimba',
+  'Bass',
+  'Guitar',
+  'Violin',
+  'Trumpet',
+  'Sax',
+  'Flute',
+  'Lead',
+  'Pad'
+];
+
+export const InstrumentTypeTokenMap = [
+  'pi',
+  'p',
+  'd',
+  't',
+  'm',
+  'b',
+  'g',
+  'v',
+  'tr',
+  's',
+  'f',
+  'l',
+  'pa'
+];

frontend/src/utils/index.tsx

@@ -22,6 +22,7 @@ import { DownloadStatus } from '../types/downloads';
 import { ModelSourceItem } from '../types/models';
 import { Language, Languages, SettingsType } from '../types/settings';
 import { DataProcessParameters, LoraFinetuneParameters } from '../types/train';
+import { tracksMinimalTotalTime } from '../types/composition';
 
 export type Cache = {
   version: string
@@ -480,6 +481,36 @@ export function getHfDownloadUrl(url: string) {
   return url;
 }
 
+export function refreshTracksTotalTime() {
+  const endTimes = commonStore.tracks.map(t => t.offsetTime + t.contentTime);
+  const totalTime = Math.max(...endTimes) + tracksMinimalTotalTime;
+  if (commonStore.trackPlayStartTime > totalTime)
+    commonStore.setTrackPlayStartTime(totalTime);
+  commonStore.setTrackTotalTime(totalTime);
+}
+
+export function flushMidiRecordingContent() {
+  const recordingTrackIndex = commonStore.tracks.findIndex(t => t.id === commonStore.recordingTrackId);
+  if (recordingTrackIndex >= 0) {
+    const recordingTrack = commonStore.tracks[recordingTrackIndex];
+    const tracks = commonStore.tracks.slice();
+    const contentTime = commonStore.recordingRawContent
+    .reduce((sum, current) =>
+        sum + (current.messageType === 'ElapsedTime' ? current.value : 0)
+      , 0);
+    tracks[recordingTrackIndex] = {
+      ...recordingTrack,
+      content: commonStore.recordingContent,
+      rawContent: commonStore.recordingRawContent,
+      contentTime: contentTime
+    };
+    commonStore.setTracks(tracks);
+    refreshTracksTotalTime();
+  }
+  commonStore.setRecordingContent('');
+  commonStore.setRecordingRawContent([]);
+}
+
 export function getSupportedCustomCudaFile(isBeta: boolean) {
   if ([' 10', ' 16', ' 20', ' 30', 'MX', 'Tesla P', 'Quadro P', 'NVIDIA P', 'TITAN X', 'TITAN RTX', 'RTX A',
     'Quadro RTX 4000', 'Quadro RTX 5000', 'Tesla T4', 'NVIDIA A10', 'NVIDIA A40'].some(v => commonStore.status.device_name.includes(v)))

frontend/wailsjs/go/backend_golang/App.d.ts

@@ -4,6 +4,8 @@ import {backend_golang} from '../models';
 
 export function AddToDownloadList(arg1:string,arg2:string):Promise<void>;
 
+export function CloseMidiPort():Promise<void>;
+
 export function ContinueDownload(arg1:string):Promise<void>;
 
 export function ConvertData(arg1:string,arg2:string,arg3:string,arg4:string):Promise<string>;
@@ -36,6 +38,8 @@ export function MergeLora(arg1:string,arg2:boolean,arg3:number,arg4:string,arg5:
 
 export function OpenFileFolder(arg1:string,arg2:boolean):Promise<void>;
 
+export function OpenMidiPort(arg1:number):Promise<void>;
+
 export function OpenOpenFileDialog(arg1:string):Promise<string>;
 
 export function OpenSaveFileDialog(arg1:string,arg2:string,arg3:string):Promise<string>;

frontend/wailsjs/go/backend_golang/App.js

@@ -6,6 +6,10 @@ export function AddToDownloadList(arg1, arg2) {
   return window['go']['backend_golang']['App']['AddToDownloadList'](arg1, arg2);
 }
 
+export function CloseMidiPort() {
+  return window['go']['backend_golang']['App']['CloseMidiPort']();
+}
+
 export function ContinueDownload(arg1) {
   return window['go']['backend_golang']['App']['ContinueDownload'](arg1);
 }
@@ -70,6 +74,10 @@ export function OpenFileFolder(arg1, arg2) {
   return window['go']['backend_golang']['App']['OpenFileFolder'](arg1, arg2);
 }
 
+export function OpenMidiPort(arg1) {
+  return window['go']['backend_golang']['App']['OpenMidiPort'](arg1);
+}
+
 export function OpenOpenFileDialog(arg1) {
   return window['go']['backend_golang']['App']['OpenOpenFileDialog'](arg1);
 }

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/fsnotify/fsnotify v1.6.0
+	github.com/mattrtaylor/go-rtmidi v0.0.0-20220428034745-af795b1c1a79
 	github.com/minio/selfupdate v0.6.0
 	github.com/nyaosorg/go-windows-su v0.2.1
 	github.com/ubuntu/gowsl v0.0.0-20230615094051-94945650cc1e

go.sum

@@ -38,6 +38,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattrtaylor/go-rtmidi v0.0.0-20220428034745-af795b1c1a79 h1:CA1UHN3RuY70DlC0RlvgtB1e8h3kYzmvK7s8CFe+Ohw=
+github.com/mattrtaylor/go-rtmidi v0.0.0-20220428034745-af795b1c1a79/go.mod h1:oBuZjmjlKSj9CZKrNhcx/adNhHiiE0hZknECjIP8Z0Q=
 github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
 github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
 github.com/nyaosorg/go-windows-su v0.2.1 h1:5V0XavLyjOqPUp7psxxCvBISaneU4XmFPSMlejSl5sc=