import React, { FC, useEffect, useRef, useState } from 'react'; import { observer } from 'mobx-react-lite'; import { useTranslation } from 'react-i18next'; import Draggable from 'react-draggable'; import { ToolTipButton } from '../../components/ToolTipButton'; import { v4 as uuid } from 'uuid'; import { Add16Regular, ArrowAutofitWidth20Regular, ArrowUpload16Regular, Delete16Regular, MusicNote220Regular, Pause16Regular, Play16Filled, Play16Regular, Record16Regular, Stop16Filled } from '@fluentui/react-icons'; import { Button, Card, DialogTrigger, Slider, Text, Tooltip } from '@fluentui/react-components'; import { useWindowSize } from 'usehooks-ts'; import commonStore, { ModelStatus } from '../../stores/commonStore'; import classnames from 'classnames'; import { InstrumentType, InstrumentTypeNameMap, InstrumentTypeTokenMap, MidiMessage, tracksMinimalTotalTime } from '../../types/composition'; import { toast } from 'react-toastify'; import { absPathAsset, flushMidiRecordingContent, getMidiRawContentMainInstrument, getMidiRawContentTime, getServerRoot, refreshTracksTotalTime } from '../../utils'; import { OpenOpenFileDialog, PlayNote } from '../../../wailsjs/go/backend_golang/App'; const snapValue = 25; const minimalMoveTime = 8; // 1000/125=8ms wait_events=125 const scaleMin = 0.05; const scaleMax = 3; const baseMoveTime = Math.round(minimalMoveTime / scaleMin); const velocityEvents = 128; const velocityBins = 12; const velocityExp = 0.5; const minimalTrackWidth = 80; const trackInitOffsetPx = 10; const pixelFix = 0.5; const topToArrowIcon = 19; const arrowIconToTracks = 23; const velocityToBin = (velocity: number) => { velocity = Math.max(0, Math.min(velocity, velocityEvents - 1)); const binsize = velocityEvents / (velocityBins - 1); return Math.ceil((velocityEvents * ((Math.pow(velocityExp, (velocity / velocityEvents)) - 1.0) / (velocityExp - 1.0))) / binsize); }; const binToVelocity = (bin: number) => { const binsize = velocityEvents / (velocityBins - 1); return Math.max(0, Math.ceil(velocityEvents * (Math.log(((velocityExp - 1) * binsize * bin) / velocityEvents + 1) / Math.log(velocityExp)) - 1)); }; const tokenToMidiMessage = (token: string): MidiMessage | null => { if (token.startsWith('<')) return null; if (token.startsWith('t') && !token.startsWith('t:')) { return { messageType: 'ElapsedTime', value: parseInt(token.substring(1)) * minimalMoveTime, channel: 0, note: 0, velocity: 0, control: 0, instrument: 0 }; } const instrument: InstrumentType = InstrumentTypeTokenMap.findIndex(t => token.startsWith(t + ':')); if (instrument >= 0) { const parts = token.split(':'); if (parts.length !== 3) return null; const note = parseInt(parts[1], 16); const velocity = parseInt(parts[2], 16); if (velocity < 0 || velocity > 127) return null; if (velocity === 0) return { messageType: 'NoteOff', note: note, instrument: instrument, channel: 0, velocity: 0, control: 0, value: 0 }; return { messageType: 'NoteOn', note: note, velocity: binToVelocity(velocity), instrument: instrument, channel: 0, control: 0, value: 0 } as MidiMessage; } return null; }; const midiMessageToToken = (msg: MidiMessage) => { if (msg.messageType === 'NoteOn' || msg.messageType === 'NoteOff') { const instrument = InstrumentTypeTokenMap[msg.instrument]; const note = msg.note.toString(16); const velocity = velocityToBin(msg.velocity).toString(16); return `${instrument}:${note}:${velocity} `; } else if (msg.messageType === 'ElapsedTime') { let time = Math.round(msg.value / minimalMoveTime); const num = Math.floor(time / 125); // wait_events=125 time -= num * 125; let ret = ''; for (let i = 0; i < num; i++) { ret += 't125 '; } if (time > 0) ret += `t${time} `; return ret; } else return ''; }; let dropRecordingTime = false; export const midiMessageHandler = async (data: MidiMessage) => { if (data.messageType === 'ControlChange') { commonStore.setInstrumentType(Math.round(data.value / 127 * (InstrumentTypeNameMap.length - 1))); return; } if (commonStore.recordingTrackId) { if (dropRecordingTime && data.messageType === 'ElapsedTime') { dropRecordingTime = false; return; } data = { ...data, instrument: commonStore.instrumentType }; commonStore.setRecordingRawContent([...commonStore.recordingRawContent, data]); commonStore.setRecordingContent(commonStore.recordingContent + midiMessageToToken(data)); //TODO data.channel = data.instrument; PlayNote(data); } }; type TrackProps = { id: string; right: number; scale: number; isSelected: boolean; onSelect: (id: string) => void; }; const Track: React.FC = observer(({ id, right, scale, isSelected, onSelect }) => { const { t } = useTranslation(); const trackIndex = commonStore.tracks.findIndex(t => t.id === id)!; const track = commonStore.tracks[trackIndex]; const trackClass = isSelected ? 'bg-blue-600' : (commonStore.settings.darkMode ? 'bg-blue-900' : 'bg-gray-700'); const controlX = useRef(0); let trackName = t('Track') + ' ' + id; if (track.mainInstrument) trackName = t('Track') + ' - ' + t('Piano is the main instrument')!.replace(t('Piano')!, t(track.mainInstrument)) + (track.content && (' - ' + track.content)); else if (track.content) trackName = t('Track') + ' - ' + track.content; return ( { controlX.current = data.lastX; }} onStop={(e, data) => { const delta = data.lastX - controlX.current; let offsetTime = Math.round(Math.round(delta / snapValue * baseMoveTime * scale) / minimalMoveTime) * minimalMoveTime; offsetTime = Math.min(Math.max( offsetTime, -track.offsetTime), commonStore.trackTotalTime - track.offsetTime); const tracks = commonStore.tracks.slice(); tracks[trackIndex].offsetTime += offsetTime; commonStore.setTracks(tracks); refreshTracksTotalTime(); }} >
onSelect(id)} > {trackName}
); }); const AudiotrackEditor: FC<{ setPrompt: (prompt: string) => void }> = observer(({ setPrompt }) => { const { t } = useTranslation(); const viewControlsContainerRef = useRef(null); const currentTimeControlRef = useRef(null); const playStartTimeControlRef = useRef(null); const tracksEndLineRef = useRef(null); const tracksRef = useRef(null); const toolbarRef = useRef(null); const toolbarButtonRef = useRef(null); const toolbarSliderRef = useRef(null); const contentPreviewRef = useRef(null); const [refreshRef, setRefreshRef] = useState(false); const windowSize = useWindowSize(); const scale = (scaleMin + scaleMax) - commonStore.trackScale; const [selectedTrackId, setSelectedTrackId] = useState(''); const playStartTimeControlX = useRef(0); const selectedTrack = selectedTrackId ? commonStore.tracks.find(t => t.id === selectedTrackId) : undefined; useEffect(() => { if (toolbarSliderRef.current && toolbarSliderRef.current.parentElement) toolbarSliderRef.current.parentElement.style.removeProperty('--fui-Slider--steps-percent'); }, []); const scrollContentToBottom = () => { if (contentPreviewRef.current) contentPreviewRef.current.scrollTop = contentPreviewRef.current.scrollHeight; }; useEffect(() => { scrollContentToBottom(); }, [commonStore.recordingContent]); useEffect(() => { setRefreshRef(!refreshRef); }, [windowSize, commonStore.tracks]); const viewControlsContainerWidth = (toolbarRef.current && toolbarButtonRef.current && toolbarSliderRef.current) ? toolbarRef.current.clientWidth - toolbarButtonRef.current.clientWidth - toolbarSliderRef.current.clientWidth - 16 // 16 = ml-2 mr-2 : 0; const tracksWidth = viewControlsContainerWidth; const timeOfTracksWidth = Math.floor(tracksWidth / snapValue) // number of moves * baseMoveTime * scale; const currentTimeControlWidth = (timeOfTracksWidth < commonStore.trackTotalTime) ? timeOfTracksWidth / commonStore.trackTotalTime * viewControlsContainerWidth : 0; const playStartTimeControlPosition = (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue; const tracksEndPosition = (commonStore.trackTotalTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue; const moveableTracksWidth = (tracksEndLineRef.current && viewControlsContainerRef.current && ((tracksEndLineRef.current.getBoundingClientRect().left - (viewControlsContainerRef.current.getBoundingClientRect().left + trackInitOffsetPx)) > 0)) ? tracksEndLineRef.current.getBoundingClientRect().left - (viewControlsContainerRef.current.getBoundingClientRect().left + trackInitOffsetPx) : Infinity; return (
{`${commonStore.trackPlayStartTime} ms / ${commonStore.trackTotalTime} ms`}
} /> } onClick={() => { commonStore.setTracks([]); commonStore.setTrackScale(1); commonStore.setTrackTotalTime(tracksMinimalTotalTime); commonStore.setTrackCurrentTime(0); commonStore.setTrackPlayStartTime(0); }} />
0) ? tracksRef.current.clientHeight - arrowIconToTracks : 0, top: `${topToArrowIcon + arrowIconToTracks}px`, left: `${tracksEndPosition + trackInitOffsetPx - pixelFix}px` }} />
{ setTimeout(() => { let offset = 0; if (currentTimeControlRef.current) { const match = currentTimeControlRef.current.style.transform.match(/translate\((.+)px,/); if (match) offset = parseFloat(match[1]); } const offsetTime = commonStore.trackTotalTime / viewControlsContainerWidth * offset; commonStore.setTrackCurrentTime(offsetTime); }, 1); }} >
viewControlsContainerWidth) && 'hidden' )}> { playStartTimeControlX.current = data.lastX; }} onStop={(e, data) => { const delta = data.lastX - playStartTimeControlX.current; let offsetTime = Math.round(Math.round(delta / snapValue * baseMoveTime * scale) / minimalMoveTime) * minimalMoveTime; offsetTime = Math.min(Math.max( offsetTime, -commonStore.trackPlayStartTime), commonStore.trackTotalTime - commonStore.trackPlayStartTime); commonStore.setTrackPlayStartTime(commonStore.trackPlayStartTime + offsetTime); }} >
0) ? tracksRef.current.clientHeight : 0, top: '50%', left: `calc(50% - ${pixelFix}px)` }} />
{ commonStore.setTrackScale(data.value); }} />
{commonStore.tracks.map(track =>
: } size="small" shape="circular" appearance="subtle" onClick={() => { flushMidiRecordingContent(); commonStore.setPlayingTrackId(''); if (commonStore.recordingTrackId === track.id) { commonStore.setRecordingTrackId(''); } else { if (commonStore.activeMidiDeviceIndex === -1) { toast(t('Please select a MIDI device first'), { type: 'warning' }); return; } dropRecordingTime = true; setSelectedTrackId(track.id); commonStore.setRecordingTrackId(track.id); commonStore.setRecordingContent(track.content); commonStore.setRecordingRawContent(track.rawContent.slice()); } }} /> : } size="small" shape="circular" appearance="subtle" onClick={() => { flushMidiRecordingContent(); commonStore.setRecordingTrackId(''); if (commonStore.playingTrackId === track.id) { commonStore.setPlayingTrackId(''); } else { setSelectedTrackId(track.id); commonStore.setPlayingTrackId(track.id); } }} /> } size="small" shape="circular" appearance="subtle" onClick={() => { const tracks = commonStore.tracks.slice().filter(t => t.id !== track.id); commonStore.setTracks(tracks); refreshTracksTotalTime(); }} />
)}
{t('Select a track to preview the content')}
{selectedTrack &&
{`${t('Start Time')}: ${selectedTrack.offsetTime} ms`} {`${t('Content Duration')}: ${selectedTrack.contentTime} ms`}
{selectedTrackId === commonStore.recordingTrackId ? commonStore.recordingContent : selectedTrack.content}
} { commonStore.platform !== 'web' &&
{t('Current Instrument') + ':'} {InstrumentTypeNameMap.map((name, i) => {t(name)})}
}
); }); export default AudiotrackEditor;