theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: save MIDI tracks to generation area; playing tracks and audio preview are still under development

Browse Source
This commit is contained in:
josc146 2023-11-29 19:04:41 +08:00
parent 34112c79c7
commit a2062ae9cc
11 changed files with 172 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
backend-golang/midi.go
View File

@ -23,6 +23,7 @@ type MIDIMessage struct {
var ports []Port
var input rtmidi.MIDIIn
var out rtmidi.MIDIOut
var activeIndex int = -1
var lastNoteTime time.Time
@ -33,6 +34,14 @@ func (a *App) midiLoop() {
runtime.EventsEmit(a.ctx, "midiError", err.Error())
return
}
out, err = rtmidi.NewMIDIOutDefault()
if err != nil {
runtime.EventsEmit(a.ctx, "midiError", err.Error())
}
err = out.OpenPort(0, "")
if err != nil {
runtime.EventsEmit(a.ctx, "midiError", err.Error())
}
ticker := time.NewTicker(500 * time.Millisecond)
go func() {
for {
@ -55,7 +64,7 @@ func (a *App) midiLoop() {
func (a *App) OpenMidiPort(index int) error {
if input == nil {
return errors.New("failed to initialize MIDI")
return errors.New("failed to initialize MIDI input")
}
if activeIndex == index {
return nil
@ -126,7 +135,7 @@ func (a *App) OpenMidiPort(index int) error {
func (a *App) CloseMidiPort() error {
if input == nil {
return errors.New("failed to initialize MIDI")
return errors.New("failed to initialize MIDI input")
}
if activeIndex == -1 {
return nil
@ -140,3 +149,16 @@ func (a *App) CloseMidiPort() error {
}
return nil
}
func (a *App) PlayNote(msg MIDIMessage) error {
if out == nil {
return errors.New("failed to initialize MIDI output")
}
channelByte := byte(msg.Channel)
if msg.MessageType == "NoteOn" {
out.SendMessage([]byte{0x90 | channelByte, byte(msg.Note), byte(msg.Velocity)})
} else if msg.MessageType == "NoteOff" {
out.SendMessage([]byte{0x80 | channelByte, byte(msg.Note), byte(msg.Velocity)})
}
return nil
}

6
frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx
View File

@ -12,13 +12,15 @@ export const AudiotrackButton: FC<{
size?: 'small' | 'medium' | 'large',
shape?: 'rounded' | 'circular' | 'square';
appearance?: 'secondary' | 'primary' | 'outline' | 'subtle' | 'transparent';
}> = ({ size, shape, appearance }) => {
setPrompt: (prompt: string) => void;
}> = ({ size, shape, appearance, setPrompt }) => {
const { t } = useTranslation();
return <Dialog onOpenChange={(e, data) => {
if (!data.open) {
flushMidiRecordingContent();
commonStore.setRecordingTrackId('');
commonStore.setPlayingTrackId('');
}
}}>
<DialogTrigger disableButtonEnhancement>
@ -30,7 +32,7 @@ export const AudiotrackButton: FC<{
<DialogBody>
<DialogContent className="overflow-hidden">
<CustomToastContainer />
<LazyImportComponent lazyChildren={AudiotrackEditor} />
<LazyImportComponent lazyChildren={AudiotrackEditor} lazyProps={{ setPrompt }} />
</DialogContent>
</DialogBody>
</DialogSurface>

112
frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx
View File

@ -9,11 +9,13 @@ import {
ArrowAutofitWidth20Regular,
Delete16Regular,
MusicNote220Regular,
Pause16Regular,
Play16Filled,
Play16Regular,
Record16Regular,
Stop16Filled
} from '@fluentui/react-icons';
import { Button, Card, Slider, Text, Tooltip } from '@fluentui/react-components';
import { Button, Card, DialogTrigger, Slider, Text, Tooltip } from '@fluentui/react-components';
import { useWindowSize } from 'usehooks-ts';
import commonStore from '../../stores/commonStore';
import classnames from 'classnames';
@ -26,6 +28,7 @@ import {
import { toast } from 'react-toastify';
import { ToastOptions } from 'react-toastify/dist/types';
import { flushMidiRecordingContent, refreshTracksTotalTime } from '../../utils';
import { PlayNote } from '../../../wailsjs/go/backend_golang/App';
const snapValue = 25;
const minimalMoveTime = 8; // 1000/125=8ms wait_events=125
@ -109,9 +112,7 @@ const midiMessageToToken = (msg: MidiMessage) => {
let dropRecordingTime = false;
export const midiMessageHandler = (data: MidiMessage) => {
if (data.messageType === 'NoteOff')
return;
export const midiMessageHandler = async (data: MidiMessage) => {
if (data.messageType === 'ControlChange') {
commonStore.setInstrumentType(Math.round(data.value / 127 * (InstrumentTypeNameMap.length - 1)));
displayCurrentInstrumentType();
@ -122,8 +123,15 @@ export const midiMessageHandler = (data: MidiMessage) => {
dropRecordingTime = false;
return;
}
data = {
...data,
instrument: commonStore.instrumentType
};
commonStore.setRecordingRawContent([...commonStore.recordingRawContent, data]);
commonStore.setRecordingContent(commonStore.recordingContent + midiMessageToToken(data));
//TODO data.channel = data.instrument;
PlayNote(data);
}
};
@ -180,7 +188,7 @@ const Track: React.FC<TrackProps> = observer(({
);
});
const AudiotrackEditor: FC = observer(() => {
const AudiotrackEditor: FC<{ setPrompt: (prompt: string) => void }> = observer(({ setPrompt }) => {
const { t } = useTranslation();
const viewControlsContainerRef = useRef<HTMLDivElement>(null);
@ -243,7 +251,7 @@ const AudiotrackEditor: FC = observer(() => {
</div>
<div className="flex pb-2 border-b" ref={toolbarRef}>
<div className="flex gap-2" ref={toolbarButtonRef}>
<ToolTipButton desc={t('Play All')} icon={<Play16Regular />} />
<ToolTipButton disabled desc={t('Play All') + ' (Unavailable)'} icon={<Play16Regular />} />
<ToolTipButton desc={t('Clear All')} icon={<Delete16Regular />} onClick={() => {
commonStore.setTracks([]);
commonStore.setTrackScale(1);
@ -348,23 +356,38 @@ const AudiotrackEditor: FC = observer(() => {
<div className="flex gap-1 border-r h-7">
<ToolTipButton desc={commonStore.recordingTrackId === track.id ? t('Stop') : t('Record')}
icon={commonStore.recordingTrackId === track.id ? <Stop16Filled /> : <Record16Regular />}
size="small" shape="circular"
appearance="subtle" onClick={() => {
flushMidiRecordingContent();
size="small" shape="circular" appearance="subtle"
onClick={() => {
flushMidiRecordingContent();
commonStore.setPlayingTrackId('');
if (commonStore.recordingTrackId === track.id) {
if (commonStore.recordingTrackId === track.id) {
commonStore.setRecordingTrackId('');
} else {
dropRecordingTime = true;
setSelectedTrackId(track.id);
commonStore.setRecordingTrackId(track.id);
commonStore.setRecordingContent(track.content);
commonStore.setRecordingRawContent(track.rawContent.slice());
}
}} />
<ToolTipButton disabled
desc={commonStore.playingTrackId === track.id ? t('Stop') : t('Play') + ' (Unavailable)'}
icon={commonStore.playingTrackId === track.id ? <Pause16Regular /> : <Play16Filled />}
size="small" shape="circular" appearance="subtle"
onClick={() => {
flushMidiRecordingContent();
commonStore.setRecordingTrackId('');
} else {
dropRecordingTime = true;
setSelectedTrackId(track.id);
commonStore.setRecordingTrackId(track.id);
commonStore.setRecordingContent(track.content);
commonStore.setRecordingRawContent(track.rawContent.slice());
}
}} />
<ToolTipButton desc={t('Play')} icon={<Play16Regular />} size="small" shape="circular"
appearance="subtle" />
if (commonStore.playingTrackId === track.id) {
commonStore.setPlayingTrackId('');
} else {
setSelectedTrackId(track.id);
commonStore.setPlayingTrackId(track.id);
}
}} />
<ToolTipButton desc={t('Delete')} icon={<Delete16Regular />} size="small" shape="circular"
appearance="subtle" onClick={() => {
const tracks = commonStore.tracks.slice().filter(t => t.id !== track.id);
@ -416,9 +439,52 @@ const AudiotrackEditor: FC = observer(() => {
</div>
</Card>
}
<Button icon={<MusicNote220Regular />} style={{ minHeight: '32px' }}>
{t('Save to generation area')}
</Button>
<DialogTrigger disableButtonEnhancement>
<Button icon={<MusicNote220Regular />} style={{ minHeight: '32px' }} onClick={() => {
flushMidiRecordingContent();
commonStore.setRecordingTrackId('');
commonStore.setPlayingTrackId('');
const timestamp = [];
const sortedTracks = commonStore.tracks.slice().sort((a, b) => a.offsetTime - b.offsetTime);
for (const track of sortedTracks) {
timestamp.push(track.offsetTime);
let accContentTime = 0;
for (const msg of track.rawContent) {
if (msg.messageType === 'ElapsedTime') {
accContentTime += msg.value;
timestamp.push(track.offsetTime + accContentTime);
}
}
}
const sortedTimestamp = timestamp.slice().sort((a, b) => a - b);
const globalMessages: MidiMessage[] = sortedTimestamp.reduce((messages, current, i) =>
[...messages, {
messageType: 'ElapsedTime',
value: current - (i === 0 ? 0 : sortedTimestamp[i - 1])
} as MidiMessage]
, [] as MidiMessage[]);
for (const track of sortedTracks) {
let currentTime = track.offsetTime;
let accContentTime = 0;
for (const msg of track.rawContent) {
if (msg.messageType === 'ElapsedTime') {
accContentTime += msg.value;
currentTime = track.offsetTime + accContentTime;
} else if (msg.messageType === 'NoteOn') {
const insertIndex = sortedTimestamp.findIndex(t => t >= currentTime);
globalMessages.splice(insertIndex + 1, 0, msg);
sortedTimestamp.splice(insertIndex + 1, 0, 0); // placeholder
}
}
}
const result = globalMessages.map(m => midiMessageToToken(m)).join('');
commonStore.setCompositionSubmittedPrompt(result);
setPrompt(result);
}}>
{t('Save to generation area')}
</Button>
</DialogTrigger>
</div>
);
});

18
frontend/src/pages/Composition.tsx
View File

@ -23,7 +23,7 @@ import {
OpenMidiPort,
OpenSaveFileDialogBytes
} from '../../wailsjs/go/backend_golang/App';
import { getServerRoot, toastWithButton } from '../utils';
import { getServerRoot, getSoundFont, toastWithButton } from '../utils';
import { CompositionParams } from '../types/composition';
import { useMediaQuery } from 'usehooks-ts';
import { AudiotrackButton } from './AudiotrackManager/AudiotrackButton';
@ -71,20 +71,8 @@ const CompositionPanel: FC = observer(() => {
};
const setSoundFont = async () => {
let soundUrl: string;
if (commonStore.compositionParams.useLocalSoundFont)
soundUrl = 'assets/sound-font';
else
soundUrl = !commonStore.settings.giteeUpdatesSource ?
`https://raw.githubusercontent.com/josStorer/sgm_plus/master` :
`https://gitee.com/josc146/sgm_plus/raw/master`;
const fallbackUrl = 'https://cdn.jsdelivr.net/gh/josstorer/sgm_plus';
await fetch(soundUrl + '/soundfont.json').then(r => {
if (!r.ok)
soundUrl = fallbackUrl;
}).catch(() => soundUrl = fallbackUrl);
if (playerRef.current) {
playerRef.current.soundFont = soundUrl;
playerRef.current.soundFont = await getSoundFont();
}
};
@ -302,7 +290,7 @@ const CompositionPanel: FC = observer(() => {
<Option key={i} value={i.toString()}>{p.name}</Option>)
}
</Dropdown>
<AudiotrackButton />
<AudiotrackButton setPrompt={setPrompt} />
</div>
} />
}

2
frontend/src/startup.ts
View File

@ -146,6 +146,6 @@ async function initMidi() {
commonStore.setMidiPorts(data);
});
EventsOn('midiMessage', async (data: MidiMessage) => {
(await import('./pages/AudiotrackManager/AudiotrackEditor')).midiMessageHandler(data);
await (await import('./pages/AudiotrackManager/AudiotrackEditor')).midiMessageHandler(data);
});
}

7
frontend/src/stores/commonStore.ts
View File

@ -107,8 +107,9 @@ class CommonStore {
trackTotalTime: number = tracksMinimalTotalTime;
trackCurrentTime: number = 0;
trackPlayStartTime: number = 0;
playingTrackId: string = '';
recordingTrackId: string = '';
recordingContent: string = ''; // used to improve performance, and I'm too lazy to maintain an ID dictionary for this
recordingContent: string = ''; // used to improve performance of midiMessageHandler, and I'm too lazy to maintain an ID dictionary for this (although that would be better for realtime effects)
recordingRawContent: MidiMessage[] = [];
// configs
currentModelConfigIndex: number = 0;
@ -444,6 +445,10 @@ class CommonStore {
setRecordingRawContent(value: MidiMessage[]) {
this.recordingRawContent = value;
}
setPlayingTrackId(value: string) {
this.playingTrackId = value;
}
}
export default new CommonStore();

1
frontend/src/types/composition.ts
View File

@ -32,6 +32,7 @@ export type MidiMessage = {
velocity: number;
control: number;
value: number;
instrument: InstrumentType;
}
export enum InstrumentType {

16
frontend/src/utils/index.tsx
View File

@ -511,6 +511,22 @@ export function flushMidiRecordingContent() {
commonStore.setRecordingRawContent([]);
}
export async function getSoundFont() {
let soundUrl: string;
if (commonStore.compositionParams.useLocalSoundFont)
soundUrl = 'assets/sound-font';
else
soundUrl = !commonStore.settings.giteeUpdatesSource ?
`https://raw.githubusercontent.com/josStorer/sgm_plus/master` :
`https://gitee.com/josc146/sgm_plus/raw/master`;
const fallbackUrl = 'https://cdn.jsdelivr.net/gh/josstorer/sgm_plus';
await fetch(soundUrl + '/soundfont.json').then(r => {
if (!r.ok)
soundUrl = fallbackUrl;
}).catch(() => soundUrl = fallbackUrl);
return soundUrl;
}
export function getSupportedCustomCudaFile(isBeta: boolean) {
if ([' 10', ' 16', ' 20', ' 30', 'MX', 'Tesla P', 'Quadro P', 'NVIDIA P', 'TITAN X', 'TITAN RTX', 'RTX A',
'Quadro RTX 4000', 'Quadro RTX 5000', 'Tesla T4', 'NVIDIA A10', 'NVIDIA A40'].some(v => commonStore.status.device_name.includes(v)))

2
frontend/wailsjs/go/backend_golang/App.d.ts generated vendored
View File

@ -48,6 +48,8 @@ export function OpenSaveFileDialogBytes(arg1:string,arg2:string,arg3:Array<numbe
export function PauseDownload(arg1:string):Promise<void>;
export function PlayNote(arg1:backend_golang.MIDIMessage):Promise<void>;
export function ReadFileInfo(arg1:string):Promise<backend_golang.FileInfo>;
export function ReadJson(arg1:string):Promise<any>;

4
frontend/wailsjs/go/backend_golang/App.js generated
View File

@ -94,6 +94,10 @@ export function PauseDownload(arg1) {
return window['go']['backend_golang']['App']['PauseDownload'](arg1);
}
export function PlayNote(arg1) {
return window['go']['backend_golang']['App']['PlayNote'](arg1);
}
export function ReadFileInfo(arg1) {
return window['go']['backend_golang']['App']['ReadFileInfo'](arg1);
}

22
frontend/wailsjs/go/models.ts generated
View File

@ -18,6 +18,28 @@ export namespace backend_golang {
this.modTime = source["modTime"];
}
}
export class MIDIMessage {
messageType: string;
channel: number;
note: number;
velocity: number;
control: number;
value: number;
static createFrom(source: any = {}) {
return new MIDIMessage(source);
}
constructor(source: any = {}) {
if ('string' === typeof source) source = JSON.parse(source);
this.messageType = source["messageType"];
this.channel = source["channel"];
this.note = source["note"];
this.velocity = source["velocity"];
this.control = source["control"];
this.value = source["value"];
}
}
}