MIDI Recording and details improvement
This commit is contained in:
josc146
parent
14a13d5768
commit
b625b8a6d1
@ -55,6 +55,7 @@ func (a *App) OnStartup(ctx context.Context) {
|
||||
}
|
||||
|
||||
a.downloadLoop()
|
||||
a.midiLoop()
|
||||
a.watchFs()
|
||||
a.monitorHardware()
|
||||
}
|
||||
|
||||
142
backend-golang/midi.go
Normal file
142
backend-golang/midi.go
Normal file
@ -0,0 +1,142 @@
|
||||
package backend_golang
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/mattrtaylor/go-rtmidi"
|
||||
"github.com/wailsapp/wails/v2/pkg/runtime"
|
||||
)
|
||||
|
||||
type Port struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
type MIDIMessage struct {
|
||||
MessageType string `json:"messageType"`
|
||||
Channel int `json:"channel"`
|
||||
Note int `json:"note"`
|
||||
Velocity int `json:"velocity"`
|
||||
Control int `json:"control"`
|
||||
Value int `json:"value"`
|
||||
}
|
||||
|
||||
var ports []Port
|
||||
var input rtmidi.MIDIIn
|
||||
var activeIndex int = -1
|
||||
var lastNoteTime time.Time
|
||||
|
||||
func (a *App) midiLoop() {
|
||||
var err error
|
||||
input, err = rtmidi.NewMIDIInDefault()
|
||||
if err != nil {
|
||||
runtime.EventsEmit(a.ctx, "midiError", err.Error())
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
go func() {
|
||||
for {
|
||||
<-ticker.C
|
||||
count, err := input.PortCount()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ports = make([]Port, count)
|
||||
for i := 0; i < count; i++ {
|
||||
name, err := input.PortName(i)
|
||||
if err == nil {
|
||||
ports[i].Name = name
|
||||
}
|
||||
}
|
||||
runtime.EventsEmit(a.ctx, "midiPorts", &ports)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (a *App) OpenMidiPort(index int) error {
|
||||
if input == nil {
|
||||
return errors.New("failed to initialize MIDI")
|
||||
}
|
||||
if activeIndex == index {
|
||||
return nil
|
||||
}
|
||||
input.Destroy()
|
||||
var err error
|
||||
input, err = rtmidi.NewMIDIInDefault()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = input.SetCallback(func(msg rtmidi.MIDIIn, bytes []byte, t float64) {
|
||||
// https://www.midi.org/specifications-old/item/table-1-summary-of-midi-message
|
||||
// https://www.rfc-editor.org/rfc/rfc6295.html
|
||||
//
|
||||
// msgType channel
|
||||
// 1001 0000
|
||||
//
|
||||
msgType := bytes[0] >> 4
|
||||
channel := bytes[0] & 0x0f
|
||||
switch msgType {
|
||||
case 0x8:
|
||||
note := bytes[1]
|
||||
runtime.EventsEmit(a.ctx, "midiMessage", &MIDIMessage{
|
||||
MessageType: "NoteOff",
|
||||
Channel: int(channel),
|
||||
Note: int(note),
|
||||
})
|
||||
case 0x9:
|
||||
elapsed := time.Since(lastNoteTime)
|
||||
lastNoteTime = time.Now()
|
||||
runtime.EventsEmit(a.ctx, "midiMessage", &MIDIMessage{
|
||||
MessageType: "ElapsedTime",
|
||||
Value: int(elapsed.Milliseconds()),
|
||||
})
|
||||
note := bytes[1]
|
||||
velocity := bytes[2]
|
||||
runtime.EventsEmit(a.ctx, "midiMessage", &MIDIMessage{
|
||||
MessageType: "NoteOn",
|
||||
Channel: int(channel),
|
||||
Note: int(note),
|
||||
Velocity: int(velocity),
|
||||
})
|
||||
case 0xb:
|
||||
// control 12 => K1 knob, control 13 => K2 knob
|
||||
control := bytes[1]
|
||||
value := bytes[2]
|
||||
runtime.EventsEmit(a.ctx, "midiMessage", &MIDIMessage{
|
||||
MessageType: "ControlChange",
|
||||
Channel: int(channel),
|
||||
Control: int(control),
|
||||
Value: int(value),
|
||||
})
|
||||
default:
|
||||
fmt.Printf("Unknown midi message: %v\n", bytes)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = input.OpenPort(index, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
activeIndex = index
|
||||
lastNoteTime = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) CloseMidiPort() error {
|
||||
if input == nil {
|
||||
return errors.New("failed to initialize MIDI")
|
||||
}
|
||||
if activeIndex == -1 {
|
||||
return nil
|
||||
}
|
||||
activeIndex = -1
|
||||
input.Destroy()
|
||||
var err error
|
||||
input, err = rtmidi.NewMIDIInDefault()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -3,6 +3,8 @@ import { useTranslation } from 'react-i18next';
|
||||
import { Button, Dialog, DialogBody, DialogContent, DialogSurface, DialogTrigger } from '@fluentui/react-components';
|
||||
import { CustomToastContainer } from '../../components/CustomToastContainer';
|
||||
import { LazyImportComponent } from '../../components/LazyImportComponent';
|
||||
import { flushMidiRecordingContent } from '../../utils';
|
||||
import commonStore from '../../stores/commonStore';
|
||||
|
||||
const AudiotrackEditor = lazy(() => import('./AudiotrackEditor'));
|
||||
|
||||
@ -13,7 +15,12 @@ export const AudiotrackButton: FC<{
|
||||
}> = ({ size, shape, appearance }) => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
return <Dialog>
|
||||
return <Dialog onOpenChange={(e, data) => {
|
||||
if (!data.open) {
|
||||
flushMidiRecordingContent();
|
||||
commonStore.setRecordingTrackId('');
|
||||
}
|
||||
}}>
|
||||
<DialogTrigger disableButtonEnhancement>
|
||||
<Button size={size} shape={shape} appearance={appearance}>
|
||||
{t('Open MIDI Input Audio Tracks')}
|
||||
|
||||
@ -10,19 +10,39 @@ import {
|
||||
Delete16Regular,
|
||||
MusicNote220Regular,
|
||||
Play16Regular,
|
||||
Record16Regular
|
||||
Record16Regular,
|
||||
Stop16Filled
|
||||
} from '@fluentui/react-icons';
|
||||
import { Button, Card, Slider, Text, Tooltip } from '@fluentui/react-components';
|
||||
import { useWindowSize } from 'usehooks-ts';
|
||||
import commonStore from '../../stores/commonStore';
|
||||
import classnames from 'classnames';
|
||||
import {
|
||||
InstrumentTypeNameMap,
|
||||
InstrumentTypeTokenMap,
|
||||
MidiMessage,
|
||||
tracksMinimalTotalTime
|
||||
} from '../../types/composition';
|
||||
import { toast } from 'react-toastify';
|
||||
import { ToastOptions } from 'react-toastify/dist/types';
|
||||
import { flushMidiRecordingContent, refreshTracksTotalTime } from '../../utils';
|
||||
|
||||
const snapValue = 25;
|
||||
const minimalMoveTime = 8; // 1000/125=8ms
|
||||
const scaleMin = 0.2;
|
||||
const minimalMoveTime = 8; // 1000/125=8ms wait_events=125
|
||||
const scaleMin = 0.05;
|
||||
const scaleMax = 3;
|
||||
const baseMoveTime = Math.round(minimalMoveTime / scaleMin);
|
||||
|
||||
const velocityEvents = 128;
|
||||
const velocityBins = 12;
|
||||
const velocityExp = 0.5;
|
||||
|
||||
const minimalTrackWidth = 80;
|
||||
const trackInitOffsetPx = 10;
|
||||
const pixelFix = 0.5;
|
||||
const topToArrowIcon = 19;
|
||||
const arrowIconToTracks = 23;
|
||||
|
||||
type TrackProps = {
|
||||
id: string;
|
||||
right: number;
|
||||
@ -31,6 +51,82 @@ type TrackProps = {
|
||||
onSelect: (id: string) => void;
|
||||
};
|
||||
|
||||
const displayCurrentInstrumentType = () => {
|
||||
const displayPanelId = 'instrument_panel_id';
|
||||
const content: React.ReactNode =
|
||||
<div className="flex gap-2 items-center">
|
||||
{InstrumentTypeNameMap.map((name, i) =>
|
||||
<Text key={name} style={{ whiteSpace: 'nowrap' }}
|
||||
className={commonStore.instrumentType === i ? 'text-blue-600' : ''}
|
||||
weight={commonStore.instrumentType === i ? 'bold' : 'regular'}
|
||||
size={commonStore.instrumentType === i ? 300 : 100}
|
||||
>{name}</Text>)}
|
||||
</div>;
|
||||
const options: ToastOptions = {
|
||||
type: 'default',
|
||||
autoClose: 2000,
|
||||
toastId: displayPanelId,
|
||||
position: 'top-left',
|
||||
style: {
|
||||
width: 'fit-content'
|
||||
}
|
||||
};
|
||||
if (toast.isActive(displayPanelId))
|
||||
toast.update(displayPanelId, {
|
||||
render: content,
|
||||
...options
|
||||
});
|
||||
else
|
||||
toast(content, options);
|
||||
};
|
||||
|
||||
const velocityToBin = (velocity: number) => {
|
||||
velocity = Math.max(0, Math.min(velocity, velocityEvents - 1));
|
||||
const binsize = velocityEvents / (velocityBins - 1);
|
||||
return Math.ceil((velocityEvents * ((Math.pow(velocityExp, (velocity / velocityEvents)) - 1.0) / (velocityExp - 1.0))) / binsize);
|
||||
};
|
||||
|
||||
const midiMessageToToken = (msg: MidiMessage) => {
|
||||
if (msg.messageType === 'NoteOn') {
|
||||
const instrument = InstrumentTypeTokenMap[commonStore.instrumentType];
|
||||
const note = msg.note.toString(16);
|
||||
const velocity = velocityToBin(msg.velocity).toString(16);
|
||||
return `${instrument}:${note}:${velocity} `;
|
||||
} else if (msg.messageType === 'ElapsedTime') {
|
||||
let time = Math.round(msg.value / minimalMoveTime);
|
||||
const num = Math.floor(time / 125); // wait_events=125
|
||||
time -= num * 125;
|
||||
let ret = '';
|
||||
for (let i = 0; i < num; i++) {
|
||||
ret += 't125 ';
|
||||
}
|
||||
if (time > 0)
|
||||
ret += `t${time} `;
|
||||
return ret;
|
||||
} else
|
||||
return '';
|
||||
};
|
||||
|
||||
let dropRecordingTime = false;
|
||||
|
||||
export const midiMessageHandler = (data: MidiMessage) => {
|
||||
if (data.messageType === 'NoteOff')
|
||||
return;
|
||||
if (data.messageType === 'ControlChange') {
|
||||
commonStore.setInstrumentType(Math.round(data.value / 127 * (InstrumentTypeNameMap.length - 1)));
|
||||
displayCurrentInstrumentType();
|
||||
return;
|
||||
}
|
||||
if (commonStore.recordingTrackId) {
|
||||
if (dropRecordingTime && data.messageType === 'ElapsedTime') {
|
||||
dropRecordingTime = false;
|
||||
return;
|
||||
}
|
||||
commonStore.setRecordingRawContent([...commonStore.recordingRawContent, data]);
|
||||
commonStore.setRecordingContent(commonStore.recordingContent + midiMessageToToken(data));
|
||||
}
|
||||
};
|
||||
|
||||
const Track: React.FC<TrackProps> = observer(({
|
||||
id,
|
||||
right,
|
||||
@ -66,18 +162,19 @@ const Track: React.FC<TrackProps> = observer(({
|
||||
const tracks = commonStore.tracks.slice();
|
||||
tracks[trackIndex].offsetTime += offsetTime;
|
||||
commonStore.setTracks(tracks);
|
||||
refreshTracksTotalTime();
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className={`p-1 cursor-move rounded whitespace-nowrap overflow-hidden ${trackClass}`}
|
||||
style={{
|
||||
width: `${Math.max(80,
|
||||
width: `${Math.max(minimalTrackWidth,
|
||||
track.contentTime / (baseMoveTime * scale) * snapValue
|
||||
)}px`
|
||||
}}
|
||||
onClick={() => onSelect(id)}
|
||||
>
|
||||
<span className="text-white">{t('Track') + ' ' + id}</span>
|
||||
<span className="text-white">{t('Track') + ' ' + (track.content || id)}</span>
|
||||
</div>
|
||||
</Draggable>
|
||||
);
|
||||
@ -86,12 +183,15 @@ const Track: React.FC<TrackProps> = observer(({
|
||||
const AudiotrackEditor: FC = observer(() => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
const viewControlsContainerRef = useRef<HTMLDivElement>(null);
|
||||
const currentTimeControlRef = useRef<HTMLDivElement>(null);
|
||||
const playStartTimeControlRef = useRef<HTMLDivElement>(null);
|
||||
const tracksEndLineRef = useRef<HTMLDivElement>(null);
|
||||
const tracksRef = useRef<HTMLDivElement>(null);
|
||||
const toolbarRef = useRef<HTMLDivElement>(null);
|
||||
const toolbarButtonRef = useRef<HTMLDivElement>(null);
|
||||
const toolbarSliderRef = useRef<HTMLInputElement>(null);
|
||||
const contentPreviewRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const [refreshRef, setRefreshRef] = useState(false);
|
||||
|
||||
@ -102,6 +202,20 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
const playStartTimeControlX = useRef(0);
|
||||
const selectedTrack = selectedTrackId ? commonStore.tracks.find(t => t.id === selectedTrackId) : undefined;
|
||||
|
||||
useEffect(() => {
|
||||
if (toolbarSliderRef.current && toolbarSliderRef.current.parentElement)
|
||||
toolbarSliderRef.current.parentElement.style.removeProperty('--fui-Slider--steps-percent');
|
||||
}, []);
|
||||
|
||||
const scrollContentToBottom = () => {
|
||||
if (contentPreviewRef.current)
|
||||
contentPreviewRef.current.scrollTop = contentPreviewRef.current.scrollHeight;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
scrollContentToBottom();
|
||||
}, [commonStore.recordingContent]);
|
||||
|
||||
useEffect(() => {
|
||||
setRefreshRef(!refreshRef);
|
||||
}, [windowSize, commonStore.tracks]);
|
||||
@ -115,10 +229,12 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
const currentTimeControlWidth = (timeOfTracksWidth < commonStore.trackTotalTime)
|
||||
? timeOfTracksWidth / commonStore.trackTotalTime * viewControlsContainerWidth
|
||||
: 0;
|
||||
const playStartTimeControlPosition = {
|
||||
x: (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue,
|
||||
y: 0
|
||||
};
|
||||
const playStartTimeControlPosition = (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue;
|
||||
const tracksEndPosition = (commonStore.trackTotalTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue;
|
||||
const moveableTracksWidth = (tracksEndLineRef.current && viewControlsContainerRef.current &&
|
||||
((tracksEndLineRef.current.getBoundingClientRect().left - (viewControlsContainerRef.current.getBoundingClientRect().left + trackInitOffsetPx)) > 0))
|
||||
? tracksEndLineRef.current.getBoundingClientRect().left - (viewControlsContainerRef.current.getBoundingClientRect().left + trackInitOffsetPx)
|
||||
: Infinity;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-2 overflow-hidden" style={{ width: '80vw', height: '80vh' }}>
|
||||
@ -130,10 +246,28 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
<ToolTipButton desc={t('Play All')} icon={<Play16Regular />} />
|
||||
<ToolTipButton desc={t('Clear All')} icon={<Delete16Regular />} onClick={() => {
|
||||
commonStore.setTracks([]);
|
||||
commonStore.setTrackScale(1);
|
||||
commonStore.setTrackTotalTime(tracksMinimalTotalTime);
|
||||
commonStore.setTrackCurrentTime(0);
|
||||
commonStore.setTrackPlayStartTime(0);
|
||||
}} />
|
||||
</div>
|
||||
<div className="grow">
|
||||
<div className="flex flex-col ml-2 mr-2">
|
||||
<div className="flex flex-col ml-2 mr-2" ref={viewControlsContainerRef}>
|
||||
<div className="relative">
|
||||
<Tooltip content={`${commonStore.trackTotalTime} ms`} showDelay={0} hideDelay={0}
|
||||
relationship="description">
|
||||
<div className="border-l absolute"
|
||||
ref={tracksEndLineRef}
|
||||
style={{
|
||||
height: (tracksRef.current && commonStore.tracks.length > 0)
|
||||
? tracksRef.current.clientHeight - arrowIconToTracks
|
||||
: 0,
|
||||
top: `${topToArrowIcon + arrowIconToTracks}px`,
|
||||
left: `${tracksEndPosition + trackInitOffsetPx - pixelFix}px`
|
||||
}} />
|
||||
</Tooltip>
|
||||
</div>
|
||||
<Draggable axis="x" bounds={{
|
||||
left: 0,
|
||||
right: viewControlsContainerWidth - currentTimeControlWidth
|
||||
@ -160,17 +294,17 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
</Draggable>
|
||||
<div className={classnames(
|
||||
'flex',
|
||||
(playStartTimeControlPosition.x < 0 || playStartTimeControlPosition.x > viewControlsContainerWidth)
|
||||
(playStartTimeControlPosition < 0 || playStartTimeControlPosition > viewControlsContainerWidth)
|
||||
&& 'hidden'
|
||||
)}>
|
||||
<Draggable axis="x" bounds={{
|
||||
left: 0,
|
||||
right: (playStartTimeControlRef.current)
|
||||
? viewControlsContainerWidth - playStartTimeControlRef.current.clientWidth
|
||||
? Math.min(viewControlsContainerWidth - playStartTimeControlRef.current.clientWidth, moveableTracksWidth)
|
||||
: 0
|
||||
}}
|
||||
grid={[snapValue, snapValue]}
|
||||
position={playStartTimeControlPosition}
|
||||
position={{ x: playStartTimeControlPosition, y: 0 }}
|
||||
onStart={(e, data) => {
|
||||
playStartTimeControlX.current = data.lastX;
|
||||
}}
|
||||
@ -192,14 +326,15 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
? tracksRef.current.clientHeight
|
||||
: 0,
|
||||
top: '50%',
|
||||
left: 'calc(50% - 0.5px)'
|
||||
left: `calc(50% - ${pixelFix}px)`
|
||||
}} />
|
||||
</div>
|
||||
</Draggable>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<Tooltip content={t('Scale View')!} showDelay={0} hideDelay={0} relationship="label">
|
||||
<Tooltip content={t('Scale View')! + ': ' + commonStore.trackScale} showDelay={0} hideDelay={0}
|
||||
relationship="description">
|
||||
<Slider ref={toolbarSliderRef} value={commonStore.trackScale} step={scaleMin} max={scaleMax} min={scaleMin}
|
||||
onChange={(e, data) => {
|
||||
commonStore.setTrackScale(data.value);
|
||||
@ -211,8 +346,23 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
{commonStore.tracks.map(track =>
|
||||
<div key={track.id} className="flex gap-2 pb-1 border-b">
|
||||
<div className="flex gap-1 border-r h-7">
|
||||
<ToolTipButton desc={t('Record')} icon={<Record16Regular />} size="small" shape="circular"
|
||||
appearance="subtle" />
|
||||
<ToolTipButton desc={commonStore.recordingTrackId === track.id ? t('Stop') : t('Record')}
|
||||
icon={commonStore.recordingTrackId === track.id ? <Stop16Filled /> : <Record16Regular />}
|
||||
size="small" shape="circular"
|
||||
appearance="subtle" onClick={() => {
|
||||
flushMidiRecordingContent();
|
||||
|
||||
if (commonStore.recordingTrackId === track.id) {
|
||||
commonStore.setRecordingTrackId('');
|
||||
} else {
|
||||
dropRecordingTime = true;
|
||||
setSelectedTrackId(track.id);
|
||||
|
||||
commonStore.setRecordingTrackId(track.id);
|
||||
commonStore.setRecordingContent(track.content);
|
||||
commonStore.setRecordingRawContent(track.rawContent.slice());
|
||||
}
|
||||
}} />
|
||||
<ToolTipButton desc={t('Play')} icon={<Play16Regular />} size="small" shape="circular"
|
||||
appearance="subtle" />
|
||||
<ToolTipButton desc={t('Delete')} icon={<Delete16Regular />} size="small" shape="circular"
|
||||
@ -226,7 +376,7 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
<Track
|
||||
id={track.id}
|
||||
scale={scale}
|
||||
right={tracksWidth}
|
||||
right={Math.min(tracksWidth, moveableTracksWidth)}
|
||||
isSelected={selectedTrackId === track.id}
|
||||
onSelect={setSelectedTrackId}
|
||||
/>
|
||||
@ -240,6 +390,7 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
commonStore.setTracks([...commonStore.tracks, {
|
||||
id: uuid(),
|
||||
content: '',
|
||||
rawContent: [],
|
||||
offsetTime: 0,
|
||||
contentTime: 0
|
||||
}]);
|
||||
@ -253,12 +404,14 @@ const AudiotrackEditor: FC = observer(() => {
|
||||
</div>
|
||||
<div className="grow"></div>
|
||||
{selectedTrack &&
|
||||
<Card size="small" appearance="outline" style={{ minHeight: '150px' }}>
|
||||
<Card size="small" appearance="outline" style={{ minHeight: '150px', maxHeight: '200px' }}>
|
||||
<div className="flex flex-col gap-1 overflow-hidden">
|
||||
<Text size={100}>{`${t('Start Time')}: ${selectedTrack.offsetTime} ms`}</Text>
|
||||
<Text size={100}>{`${t('Content Time')}: ${selectedTrack.contentTime} ms`}</Text>
|
||||
<div className="overflow-y-auto overflow-x-hidden">
|
||||
{selectedTrack.content}
|
||||
<div className="overflow-y-auto overflow-x-hidden" ref={contentPreviewRef}>
|
||||
{selectedTrackId === commonStore.recordingTrackId
|
||||
? commonStore.recordingContent
|
||||
: selectedTrack.content}
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
@ -16,7 +16,13 @@ import { PlayerElement, VisualizerElement } from 'html-midi-player';
|
||||
import * as mm from '@magenta/music/esm/core.js';
|
||||
import { NoteSequence } from '@magenta/music/esm/protobuf.js';
|
||||
import { defaultCompositionPrompt } from './defaultConfigs';
|
||||
import { FileExists, OpenFileFolder, OpenSaveFileDialogBytes } from '../../wailsjs/go/backend_golang/App';
|
||||
import {
|
||||
CloseMidiPort,
|
||||
FileExists,
|
||||
OpenFileFolder,
|
||||
OpenMidiPort,
|
||||
OpenSaveFileDialogBytes
|
||||
} from '../../wailsjs/go/backend_golang/App';
|
||||
import { getServerRoot, toastWithButton } from '../utils';
|
||||
import { CompositionParams } from '../types/composition';
|
||||
import { useMediaQuery } from 'usehooks-ts';
|
||||
@ -273,8 +279,26 @@ const CompositionPanel: FC = observer(() => {
|
||||
desc={t('Select the MIDI input device to be used.')}
|
||||
content={
|
||||
<div className="flex flex-col gap-1">
|
||||
<Dropdown style={{ minWidth: 0 }}>
|
||||
<Option>{t('None')!}</Option>
|
||||
<Dropdown style={{ minWidth: 0 }}
|
||||
value={commonStore.activeMidiDeviceIndex === -1 ? t('None')! : commonStore.midiPorts[commonStore.activeMidiDeviceIndex].name}
|
||||
selectedOptions={[commonStore.activeMidiDeviceIndex.toString()]}
|
||||
onOptionSelect={(_, data) => {
|
||||
if (data.optionValue) {
|
||||
const index = Number(data.optionValue);
|
||||
let action = (index === -1)
|
||||
? () => CloseMidiPort()
|
||||
: () => OpenMidiPort(index);
|
||||
action().then(() => {
|
||||
commonStore.setActiveMidiDeviceIndex(index);
|
||||
}).catch((e) => {
|
||||
toast(t('Error') + ' - ' + (e.message || e), { type: 'error', autoClose: 2500 });
|
||||
});
|
||||
}
|
||||
}}>
|
||||
<Option value={'-1'}>{t('None')!}</Option>
|
||||
{commonStore.midiPorts.map((p, i) =>
|
||||
<Option key={i} value={i.toString()}>{p.name}</Option>)
|
||||
}
|
||||
</Dropdown>
|
||||
<AudiotrackButton />
|
||||
</div>
|
||||
|
||||
@ -7,6 +7,8 @@ import manifest from '../../manifest.json';
|
||||
import { defaultModelConfigs, defaultModelConfigsMac } from './pages/defaultConfigs';
|
||||
import { t } from 'i18next';
|
||||
import { Preset } from './types/presets';
|
||||
import { toast } from 'react-toastify';
|
||||
import { MidiMessage, MidiPort } from './types/composition';
|
||||
|
||||
export async function startup() {
|
||||
initPresets();
|
||||
@ -26,6 +28,7 @@ export async function startup() {
|
||||
initLocalModelsNotify();
|
||||
initLoraModels();
|
||||
initHardwareMonitor();
|
||||
initMidi();
|
||||
}
|
||||
|
||||
await initConfig();
|
||||
@ -134,3 +137,15 @@ async function initHardwareMonitor() {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function initMidi() {
|
||||
EventsOn('midiError', (data: string) => {
|
||||
toast('MIDI Error: ' + data, { type: 'error' });
|
||||
});
|
||||
EventsOn('midiPorts', (data: MidiPort[]) => {
|
||||
commonStore.setMidiPorts(data);
|
||||
});
|
||||
EventsOn('midiMessage', async (data: MidiMessage) => {
|
||||
(await import('./pages/AudiotrackManager/AudiotrackEditor')).midiMessageHandler(data);
|
||||
});
|
||||
}
|
||||
|
||||
@ -9,7 +9,14 @@ import { Preset } from '../types/presets';
|
||||
import { AboutContent } from '../types/about';
|
||||
import { Attachment, ChatParams, Conversation } from '../types/chat';
|
||||
import { CompletionPreset } from '../types/completion';
|
||||
import { CompositionParams, Track } from '../types/composition';
|
||||
import {
|
||||
CompositionParams,
|
||||
InstrumentType,
|
||||
MidiMessage,
|
||||
MidiPort,
|
||||
Track,
|
||||
tracksMinimalTotalTime
|
||||
} from '../types/composition';
|
||||
import { ModelConfig } from '../types/configs';
|
||||
import { DownloadStatus } from '../types/downloads';
|
||||
import { IntroductionContent } from '../types/home';
|
||||
@ -90,11 +97,19 @@ class CommonStore {
|
||||
};
|
||||
compositionGenerating: boolean = false;
|
||||
compositionSubmittedPrompt: string = defaultCompositionPrompt;
|
||||
// composition midi device
|
||||
midiPorts: MidiPort[] = [];
|
||||
activeMidiDeviceIndex: number = -1;
|
||||
instrumentType: InstrumentType = InstrumentType.Piano;
|
||||
// composition tracks
|
||||
tracks: Track[] = [];
|
||||
trackScale: number = 1;
|
||||
trackTotalTime: number = 5000;
|
||||
trackTotalTime: number = tracksMinimalTotalTime;
|
||||
trackCurrentTime: number = 0;
|
||||
trackPlayStartTime: number = 0;
|
||||
recordingTrackId: string = '';
|
||||
recordingContent: string = ''; // used to improve performance, and I'm too lazy to maintain an ID dictionary for this
|
||||
recordingRawContent: MidiMessage[] = [];
|
||||
// configs
|
||||
currentModelConfigIndex: number = 0;
|
||||
modelConfigs: ModelConfig[] = [];
|
||||
@ -405,6 +420,30 @@ class CommonStore {
|
||||
setTrackPlayStartTime(value: number) {
|
||||
this.trackPlayStartTime = value;
|
||||
}
|
||||
|
||||
setMidiPorts(value: MidiPort[]) {
|
||||
this.midiPorts = value;
|
||||
}
|
||||
|
||||
setInstrumentType(value: InstrumentType) {
|
||||
this.instrumentType = value;
|
||||
}
|
||||
|
||||
setRecordingTrackId(value: string) {
|
||||
this.recordingTrackId = value;
|
||||
}
|
||||
|
||||
setActiveMidiDeviceIndex(value: number) {
|
||||
this.activeMidiDeviceIndex = value;
|
||||
}
|
||||
|
||||
setRecordingContent(value: string) {
|
||||
this.recordingContent = value;
|
||||
}
|
||||
|
||||
setRecordingRawContent(value: MidiMessage[]) {
|
||||
this.recordingRawContent = value;
|
||||
}
|
||||
}
|
||||
|
||||
export default new CommonStore();
|
||||
@ -1,5 +1,7 @@
|
||||
import { NoteSequence } from '@magenta/music/esm/protobuf';
|
||||
|
||||
export const tracksMinimalTotalTime = 5000;
|
||||
|
||||
export type CompositionParams = {
|
||||
prompt: string,
|
||||
maxResponseToken: number,
|
||||
@ -13,6 +15,69 @@ export type CompositionParams = {
|
||||
export type Track = {
|
||||
id: string;
|
||||
content: string;
|
||||
rawContent: MidiMessage[];
|
||||
offsetTime: number;
|
||||
contentTime: number;
|
||||
};
|
||||
};
|
||||
export type MidiPort = {
|
||||
name: string;
|
||||
}
|
||||
|
||||
export type MessageType = 'NoteOff' | 'NoteOn' | 'ElapsedTime' | 'ControlChange';
|
||||
|
||||
export type MidiMessage = {
|
||||
messageType: MessageType;
|
||||
channel: number;
|
||||
note: number;
|
||||
velocity: number;
|
||||
control: number;
|
||||
value: number;
|
||||
}
|
||||
|
||||
export enum InstrumentType {
|
||||
Piano,
|
||||
Percussion,
|
||||
Drum,
|
||||
Tuba,
|
||||
Marimba,
|
||||
Bass,
|
||||
Guitar,
|
||||
Violin,
|
||||
Trumpet,
|
||||
Sax,
|
||||
Flute,
|
||||
Lead,
|
||||
Pad,
|
||||
}
|
||||
|
||||
export const InstrumentTypeNameMap = [
|
||||
'Piano',
|
||||
'Percussion',
|
||||
'Drum',
|
||||
'Tuba',
|
||||
'Marimba',
|
||||
'Bass',
|
||||
'Guitar',
|
||||
'Violin',
|
||||
'Trumpet',
|
||||
'Sax',
|
||||
'Flute',
|
||||
'Lead',
|
||||
'Pad'
|
||||
];
|
||||
|
||||
export const InstrumentTypeTokenMap = [
|
||||
'pi',
|
||||
'p',
|
||||
'd',
|
||||
't',
|
||||
'm',
|
||||
'b',
|
||||
'g',
|
||||
'v',
|
||||
'tr',
|
||||
's',
|
||||
'f',
|
||||
'l',
|
||||
'pa'
|
||||
];
|
||||
|
||||
@ -22,6 +22,7 @@ import { DownloadStatus } from '../types/downloads';
|
||||
import { ModelSourceItem } from '../types/models';
|
||||
import { Language, Languages, SettingsType } from '../types/settings';
|
||||
import { DataProcessParameters, LoraFinetuneParameters } from '../types/train';
|
||||
import { tracksMinimalTotalTime } from '../types/composition';
|
||||
|
||||
export type Cache = {
|
||||
version: string
|
||||
@ -480,6 +481,36 @@ export function getHfDownloadUrl(url: string) {
|
||||
return url;
|
||||
}
|
||||
|
||||
export function refreshTracksTotalTime() {
|
||||
const endTimes = commonStore.tracks.map(t => t.offsetTime + t.contentTime);
|
||||
const totalTime = Math.max(...endTimes) + tracksMinimalTotalTime;
|
||||
if (commonStore.trackPlayStartTime > totalTime)
|
||||
commonStore.setTrackPlayStartTime(totalTime);
|
||||
commonStore.setTrackTotalTime(totalTime);
|
||||
}
|
||||
|
||||
export function flushMidiRecordingContent() {
|
||||
const recordingTrackIndex = commonStore.tracks.findIndex(t => t.id === commonStore.recordingTrackId);
|
||||
if (recordingTrackIndex >= 0) {
|
||||
const recordingTrack = commonStore.tracks[recordingTrackIndex];
|
||||
const tracks = commonStore.tracks.slice();
|
||||
const contentTime = commonStore.recordingRawContent
|
||||
.reduce((sum, current) =>
|
||||
sum + (current.messageType === 'ElapsedTime' ? current.value : 0)
|
||||
, 0);
|
||||
tracks[recordingTrackIndex] = {
|
||||
...recordingTrack,
|
||||
content: commonStore.recordingContent,
|
||||
rawContent: commonStore.recordingRawContent,
|
||||
contentTime: contentTime
|
||||
};
|
||||
commonStore.setTracks(tracks);
|
||||
refreshTracksTotalTime();
|
||||
}
|
||||
commonStore.setRecordingContent('');
|
||||
commonStore.setRecordingRawContent([]);
|
||||
}
|
||||
|
||||
export function getSupportedCustomCudaFile(isBeta: boolean) {
|
||||
if ([' 10', ' 16', ' 20', ' 30', 'MX', 'Tesla P', 'Quadro P', 'NVIDIA P', 'TITAN X', 'TITAN RTX', 'RTX A',
|
||||
'Quadro RTX 4000', 'Quadro RTX 5000', 'Tesla T4', 'NVIDIA A10', 'NVIDIA A40'].some(v => commonStore.status.device_name.includes(v)))
|
||||
|
||||
4
frontend/wailsjs/go/backend_golang/App.d.ts
generated
vendored
4
frontend/wailsjs/go/backend_golang/App.d.ts
generated
vendored
@ -4,6 +4,8 @@ import {backend_golang} from '../models';
|
||||
|
||||
export function AddToDownloadList(arg1:string,arg2:string):Promise<void>;
|
||||
|
||||
export function CloseMidiPort():Promise<void>;
|
||||
|
||||
export function ContinueDownload(arg1:string):Promise<void>;
|
||||
|
||||
export function ConvertData(arg1:string,arg2:string,arg3:string,arg4:string):Promise<string>;
|
||||
@ -36,6 +38,8 @@ export function MergeLora(arg1:string,arg2:boolean,arg3:number,arg4:string,arg5:
|
||||
|
||||
export function OpenFileFolder(arg1:string,arg2:boolean):Promise<void>;
|
||||
|
||||
export function OpenMidiPort(arg1:number):Promise<void>;
|
||||
|
||||
export function OpenOpenFileDialog(arg1:string):Promise<string>;
|
||||
|
||||
export function OpenSaveFileDialog(arg1:string,arg2:string,arg3:string):Promise<string>;
|
||||
|
||||
8
frontend/wailsjs/go/backend_golang/App.js
generated
8
frontend/wailsjs/go/backend_golang/App.js
generated
@ -6,6 +6,10 @@ export function AddToDownloadList(arg1, arg2) {
|
||||
return window['go']['backend_golang']['App']['AddToDownloadList'](arg1, arg2);
|
||||
}
|
||||
|
||||
export function CloseMidiPort() {
|
||||
return window['go']['backend_golang']['App']['CloseMidiPort']();
|
||||
}
|
||||
|
||||
export function ContinueDownload(arg1) {
|
||||
return window['go']['backend_golang']['App']['ContinueDownload'](arg1);
|
||||
}
|
||||
@ -70,6 +74,10 @@ export function OpenFileFolder(arg1, arg2) {
|
||||
return window['go']['backend_golang']['App']['OpenFileFolder'](arg1, arg2);
|
||||
}
|
||||
|
||||
export function OpenMidiPort(arg1) {
|
||||
return window['go']['backend_golang']['App']['OpenMidiPort'](arg1);
|
||||
}
|
||||
|
||||
export function OpenOpenFileDialog(arg1) {
|
||||
return window['go']['backend_golang']['App']['OpenOpenFileDialog'](arg1);
|
||||
}
|
||||
|
||||
1
go.mod
1
go.mod
@ -5,6 +5,7 @@ go 1.20
|
||||
require (
|
||||
github.com/cavaliergopher/grab/v3 v3.0.1
|
||||
github.com/fsnotify/fsnotify v1.6.0
|
||||
github.com/mattrtaylor/go-rtmidi v0.0.0-20220428034745-af795b1c1a79
|
||||
github.com/minio/selfupdate v0.6.0
|
||||
github.com/nyaosorg/go-windows-su v0.2.1
|
||||
github.com/ubuntu/gowsl v0.0.0-20230615094051-94945650cc1e
|
||||
|
||||
2
go.sum
2
go.sum
@ -38,6 +38,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattrtaylor/go-rtmidi v0.0.0-20220428034745-af795b1c1a79 h1:CA1UHN3RuY70DlC0RlvgtB1e8h3kYzmvK7s8CFe+Ohw=
|
||||
github.com/mattrtaylor/go-rtmidi v0.0.0-20220428034745-af795b1c1a79/go.mod h1:oBuZjmjlKSj9CZKrNhcx/adNhHiiE0hZknECjIP8Z0Q=
|
||||
github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
|
||||
github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
|
||||
github.com/nyaosorg/go-windows-su v0.2.1 h1:5V0XavLyjOqPUp7psxxCvBISaneU4XmFPSMlejSl5sc=
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user