basic MIDI Input Audio Tracks
This commit is contained in:
josc146
parent
7ce464ecda
commit
14a13d5768
14
frontend/package-lock.json
generated
14
frontend/package-lock.json
generated
@ -25,6 +25,7 @@
|
||||
"react-beautiful-dnd": "^13.1.1",
|
||||
"react-chartjs-2": "^5.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-draggable": "^4.4.6",
|
||||
"react-i18next": "^12.2.2",
|
||||
"react-markdown": "^8.0.7",
|
||||
"react-router": "^6.11.1",
|
||||
@ -5410,6 +5411,19 @@
|
||||
"loose-envify": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/react-draggable": {
|
||||
"version": "4.4.6",
|
||||
"resolved": "https://registry.npmjs.org/react-draggable/-/react-draggable-4.4.6.tgz",
|
||||
"integrity": "sha512-LtY5Xw1zTPqHkVmtM3X8MUOxNDOUhv/khTgBgrUvwaS064bwVvxT+q5El0uUFNx5IEPKXuRejr7UqLwBIg5pdw==",
|
||||
"dependencies": {
|
||||
"clsx": "^1.1.1",
|
||||
"prop-types": "^15.8.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">= 16.3.0",
|
||||
"react-dom": ">= 16.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/react-i18next": {
|
||||
"version": "12.2.2",
|
||||
"resolved": "https://registry.npmmirror.com/react-i18next/-/react-i18next-12.2.2.tgz",
|
||||
|
||||
@ -26,6 +26,7 @@
|
||||
"react-beautiful-dnd": "^13.1.1",
|
||||
"react-chartjs-2": "^5.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-draggable": "^4.4.6",
|
||||
"react-i18next": "^12.2.2",
|
||||
"react-markdown": "^8.0.7",
|
||||
"react-router": "^6.11.1",
|
||||
|
||||
31
frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx
Normal file
31
frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx
Normal file
@ -0,0 +1,31 @@
|
||||
import React, { FC, lazy } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Button, Dialog, DialogBody, DialogContent, DialogSurface, DialogTrigger } from '@fluentui/react-components';
|
||||
import { CustomToastContainer } from '../../components/CustomToastContainer';
|
||||
import { LazyImportComponent } from '../../components/LazyImportComponent';
|
||||
|
||||
const AudiotrackEditor = lazy(() => import('./AudiotrackEditor'));
|
||||
|
||||
export const AudiotrackButton: FC<{
|
||||
size?: 'small' | 'medium' | 'large',
|
||||
shape?: 'rounded' | 'circular' | 'square';
|
||||
appearance?: 'secondary' | 'primary' | 'outline' | 'subtle' | 'transparent';
|
||||
}> = ({ size, shape, appearance }) => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
return <Dialog>
|
||||
<DialogTrigger disableButtonEnhancement>
|
||||
<Button size={size} shape={shape} appearance={appearance}>
|
||||
{t('Open MIDI Input Audio Tracks')}
|
||||
</Button>
|
||||
</DialogTrigger>
|
||||
<DialogSurface style={{ paddingTop: 0, maxWidth: '90vw', width: 'fit-content' }}>
|
||||
<DialogBody>
|
||||
<DialogContent className="overflow-hidden">
|
||||
<CustomToastContainer />
|
||||
<LazyImportComponent lazyChildren={AudiotrackEditor} />
|
||||
</DialogContent>
|
||||
</DialogBody>
|
||||
</DialogSurface>
|
||||
</Dialog>;
|
||||
};
|
||||
273
frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx
Normal file
273
frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx
Normal file
@ -0,0 +1,273 @@
|
||||
import React, { FC, useEffect, useRef, useState } from 'react';
|
||||
import { observer } from 'mobx-react-lite';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import Draggable from 'react-draggable';
|
||||
import { ToolTipButton } from '../../components/ToolTipButton';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import {
|
||||
Add16Regular,
|
||||
ArrowAutofitWidth20Regular,
|
||||
Delete16Regular,
|
||||
MusicNote220Regular,
|
||||
Play16Regular,
|
||||
Record16Regular
|
||||
} from '@fluentui/react-icons';
|
||||
import { Button, Card, Slider, Text, Tooltip } from '@fluentui/react-components';
|
||||
import { useWindowSize } from 'usehooks-ts';
|
||||
import commonStore from '../../stores/commonStore';
|
||||
import classnames from 'classnames';
|
||||
|
||||
const snapValue = 25;
|
||||
const minimalMoveTime = 8; // 1000/125=8ms
|
||||
const scaleMin = 0.2;
|
||||
const scaleMax = 3;
|
||||
const baseMoveTime = Math.round(minimalMoveTime / scaleMin);
|
||||
|
||||
type TrackProps = {
|
||||
id: string;
|
||||
right: number;
|
||||
scale: number;
|
||||
isSelected: boolean;
|
||||
onSelect: (id: string) => void;
|
||||
};
|
||||
|
||||
const Track: React.FC<TrackProps> = observer(({
|
||||
id,
|
||||
right,
|
||||
scale,
|
||||
isSelected,
|
||||
onSelect
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const trackIndex = commonStore.tracks.findIndex(t => t.id === id)!;
|
||||
const track = commonStore.tracks[trackIndex];
|
||||
const trackClass = isSelected ? 'bg-blue-600' : 'bg-gray-700';
|
||||
const controlX = useRef(0);
|
||||
|
||||
return (
|
||||
<Draggable
|
||||
axis="x"
|
||||
bounds={{ left: 0, right }}
|
||||
grid={[snapValue, snapValue]}
|
||||
position={{
|
||||
x: (track.offsetTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue,
|
||||
y: 0
|
||||
}}
|
||||
onStart={(e, data) => {
|
||||
controlX.current = data.lastX;
|
||||
}}
|
||||
onStop={(e, data) => {
|
||||
const delta = data.lastX - controlX.current;
|
||||
let offsetTime = Math.round(Math.round(delta / snapValue * baseMoveTime * scale) / minimalMoveTime) * minimalMoveTime;
|
||||
offsetTime = Math.min(Math.max(
|
||||
offsetTime,
|
||||
-track.offsetTime), commonStore.trackTotalTime - track.offsetTime);
|
||||
|
||||
const tracks = commonStore.tracks.slice();
|
||||
tracks[trackIndex].offsetTime += offsetTime;
|
||||
commonStore.setTracks(tracks);
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className={`p-1 cursor-move rounded whitespace-nowrap overflow-hidden ${trackClass}`}
|
||||
style={{
|
||||
width: `${Math.max(80,
|
||||
track.contentTime / (baseMoveTime * scale) * snapValue
|
||||
)}px`
|
||||
}}
|
||||
onClick={() => onSelect(id)}
|
||||
>
|
||||
<span className="text-white">{t('Track') + ' ' + id}</span>
|
||||
</div>
|
||||
</Draggable>
|
||||
);
|
||||
});
|
||||
|
||||
const AudiotrackEditor: FC = observer(() => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
const currentTimeControlRef = useRef<HTMLDivElement>(null);
|
||||
const playStartTimeControlRef = useRef<HTMLDivElement>(null);
|
||||
const tracksRef = useRef<HTMLDivElement>(null);
|
||||
const toolbarRef = useRef<HTMLDivElement>(null);
|
||||
const toolbarButtonRef = useRef<HTMLDivElement>(null);
|
||||
const toolbarSliderRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const [refreshRef, setRefreshRef] = useState(false);
|
||||
|
||||
const windowSize = useWindowSize();
|
||||
const scale = (scaleMin + scaleMax) - commonStore.trackScale;
|
||||
|
||||
const [selectedTrackId, setSelectedTrackId] = useState<string>('');
|
||||
const playStartTimeControlX = useRef(0);
|
||||
const selectedTrack = selectedTrackId ? commonStore.tracks.find(t => t.id === selectedTrackId) : undefined;
|
||||
|
||||
useEffect(() => {
|
||||
setRefreshRef(!refreshRef);
|
||||
}, [windowSize, commonStore.tracks]);
|
||||
|
||||
const viewControlsContainerWidth = (toolbarRef.current && toolbarButtonRef.current && toolbarSliderRef.current) ?
|
||||
toolbarRef.current.clientWidth - toolbarButtonRef.current.clientWidth - toolbarSliderRef.current.clientWidth - 16 // 16 = ml-2 mr-2
|
||||
: 0;
|
||||
const tracksWidth = viewControlsContainerWidth;
|
||||
const timeOfTracksWidth = Math.floor(tracksWidth / snapValue) // number of moves
|
||||
* baseMoveTime * scale;
|
||||
const currentTimeControlWidth = (timeOfTracksWidth < commonStore.trackTotalTime)
|
||||
? timeOfTracksWidth / commonStore.trackTotalTime * viewControlsContainerWidth
|
||||
: 0;
|
||||
const playStartTimeControlPosition = {
|
||||
x: (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue,
|
||||
y: 0
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-2 overflow-hidden" style={{ width: '80vw', height: '80vh' }}>
|
||||
<div className="mx-auto">
|
||||
<Text size={100}>{`${commonStore.trackPlayStartTime} ms / ${commonStore.trackTotalTime} ms`}</Text>
|
||||
</div>
|
||||
<div className="flex pb-2 border-b" ref={toolbarRef}>
|
||||
<div className="flex gap-2" ref={toolbarButtonRef}>
|
||||
<ToolTipButton desc={t('Play All')} icon={<Play16Regular />} />
|
||||
<ToolTipButton desc={t('Clear All')} icon={<Delete16Regular />} onClick={() => {
|
||||
commonStore.setTracks([]);
|
||||
}} />
|
||||
</div>
|
||||
<div className="grow">
|
||||
<div className="flex flex-col ml-2 mr-2">
|
||||
<Draggable axis="x" bounds={{
|
||||
left: 0,
|
||||
right: viewControlsContainerWidth - currentTimeControlWidth
|
||||
}}
|
||||
position={{
|
||||
x: commonStore.trackCurrentTime / commonStore.trackTotalTime * viewControlsContainerWidth,
|
||||
y: 0
|
||||
}}
|
||||
onDrag={(e, data) => {
|
||||
setTimeout(() => {
|
||||
let offset = 0;
|
||||
if (currentTimeControlRef.current) {
|
||||
const match = currentTimeControlRef.current.style.transform.match(/translate\((.+)px,/);
|
||||
if (match)
|
||||
offset = parseFloat(match[1]);
|
||||
}
|
||||
const offsetTime = commonStore.trackTotalTime / viewControlsContainerWidth * offset;
|
||||
commonStore.setTrackCurrentTime(offsetTime);
|
||||
}, 1);
|
||||
}}
|
||||
>
|
||||
<div ref={currentTimeControlRef} className="h-2 bg-gray-700 cursor-move rounded"
|
||||
style={{ width: currentTimeControlWidth }} />
|
||||
</Draggable>
|
||||
<div className={classnames(
|
||||
'flex',
|
||||
(playStartTimeControlPosition.x < 0 || playStartTimeControlPosition.x > viewControlsContainerWidth)
|
||||
&& 'hidden'
|
||||
)}>
|
||||
<Draggable axis="x" bounds={{
|
||||
left: 0,
|
||||
right: (playStartTimeControlRef.current)
|
||||
? viewControlsContainerWidth - playStartTimeControlRef.current.clientWidth
|
||||
: 0
|
||||
}}
|
||||
grid={[snapValue, snapValue]}
|
||||
position={playStartTimeControlPosition}
|
||||
onStart={(e, data) => {
|
||||
playStartTimeControlX.current = data.lastX;
|
||||
}}
|
||||
onStop={(e, data) => {
|
||||
const delta = data.lastX - playStartTimeControlX.current;
|
||||
let offsetTime = Math.round(Math.round(delta / snapValue * baseMoveTime * scale) / minimalMoveTime) * minimalMoveTime;
|
||||
offsetTime = Math.min(Math.max(
|
||||
offsetTime,
|
||||
-commonStore.trackPlayStartTime), commonStore.trackTotalTime - commonStore.trackPlayStartTime);
|
||||
commonStore.setTrackPlayStartTime(commonStore.trackPlayStartTime + offsetTime);
|
||||
}}
|
||||
>
|
||||
<div className="relative cursor-move"
|
||||
ref={playStartTimeControlRef}>
|
||||
<ArrowAutofitWidth20Regular />
|
||||
<div className="border-l absolute border-gray-700"
|
||||
style={{
|
||||
height: (tracksRef.current && commonStore.tracks.length > 0)
|
||||
? tracksRef.current.clientHeight
|
||||
: 0,
|
||||
top: '50%',
|
||||
left: 'calc(50% - 0.5px)'
|
||||
}} />
|
||||
</div>
|
||||
</Draggable>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<Tooltip content={t('Scale View')!} showDelay={0} hideDelay={0} relationship="label">
|
||||
<Slider ref={toolbarSliderRef} value={commonStore.trackScale} step={scaleMin} max={scaleMax} min={scaleMin}
|
||||
onChange={(e, data) => {
|
||||
commonStore.setTrackScale(data.value);
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<div className="flex flex-col overflow-y-auto gap-1" ref={tracksRef}>
|
||||
{commonStore.tracks.map(track =>
|
||||
<div key={track.id} className="flex gap-2 pb-1 border-b">
|
||||
<div className="flex gap-1 border-r h-7">
|
||||
<ToolTipButton desc={t('Record')} icon={<Record16Regular />} size="small" shape="circular"
|
||||
appearance="subtle" />
|
||||
<ToolTipButton desc={t('Play')} icon={<Play16Regular />} size="small" shape="circular"
|
||||
appearance="subtle" />
|
||||
<ToolTipButton desc={t('Delete')} icon={<Delete16Regular />} size="small" shape="circular"
|
||||
appearance="subtle" onClick={() => {
|
||||
const tracks = commonStore.tracks.slice().filter(t => t.id !== track.id);
|
||||
commonStore.setTracks(tracks);
|
||||
}} />
|
||||
</div>
|
||||
<div className="relative grow overflow-hidden">
|
||||
<div className="absolute" style={{ left: -0 }}>
|
||||
<Track
|
||||
id={track.id}
|
||||
scale={scale}
|
||||
right={tracksWidth}
|
||||
isSelected={selectedTrackId === track.id}
|
||||
onSelect={setSelectedTrackId}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>)}
|
||||
<div className="flex justify-between items-center">
|
||||
<Button icon={<Add16Regular />} size="small" shape="circular"
|
||||
appearance="subtle"
|
||||
onClick={() => {
|
||||
commonStore.setTracks([...commonStore.tracks, {
|
||||
id: uuid(),
|
||||
content: '',
|
||||
offsetTime: 0,
|
||||
contentTime: 0
|
||||
}]);
|
||||
}}>
|
||||
{t('New Track')}
|
||||
</Button>
|
||||
<Text size={100}>
|
||||
{t('Select a track to preview the content')}
|
||||
</Text>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grow"></div>
|
||||
{selectedTrack &&
|
||||
<Card size="small" appearance="outline" style={{ minHeight: '150px' }}>
|
||||
<div className="flex flex-col gap-1 overflow-hidden">
|
||||
<Text size={100}>{`${t('Start Time')}: ${selectedTrack.offsetTime} ms`}</Text>
|
||||
<Text size={100}>{`${t('Content Time')}: ${selectedTrack.contentTime} ms`}</Text>
|
||||
<div className="overflow-y-auto overflow-x-hidden">
|
||||
{selectedTrack.content}
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
}
|
||||
<Button icon={<MusicNote220Regular />} style={{ minHeight: '32px' }}>
|
||||
{t('Save to generation area')}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
|
||||
export default AudiotrackEditor;
|
||||
@ -2,7 +2,7 @@ import 'html-midi-player';
|
||||
import React, { FC, useEffect, useRef } from 'react';
|
||||
import { observer } from 'mobx-react-lite';
|
||||
import { WorkHeader } from '../components/WorkHeader';
|
||||
import { Button, Checkbox, Textarea } from '@fluentui/react-components';
|
||||
import { Button, Checkbox, Dropdown, Option, Textarea } from '@fluentui/react-components';
|
||||
import { Labeled } from '../components/Labeled';
|
||||
import { ValuedSlider } from '../components/ValuedSlider';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
@ -20,6 +20,7 @@ import { FileExists, OpenFileFolder, OpenSaveFileDialogBytes } from '../../wails
|
||||
import { getServerRoot, toastWithButton } from '../utils';
|
||||
import { CompositionParams } from '../types/composition';
|
||||
import { useMediaQuery } from 'usehooks-ts';
|
||||
import { AudiotrackButton } from './AudiotrackManager/AudiotrackButton';
|
||||
|
||||
let compositionSseController: AbortController | null = null;
|
||||
|
||||
@ -267,6 +268,18 @@ const CompositionPanel: FC = observer(() => {
|
||||
autoPlay: data.checked as boolean
|
||||
});
|
||||
}} />
|
||||
{commonStore.platform !== 'web' &&
|
||||
<Labeled flex breakline label={t('MIDI Input')}
|
||||
desc={t('Select the MIDI input device to be used.')}
|
||||
content={
|
||||
<div className="flex flex-col gap-1">
|
||||
<Dropdown style={{ minWidth: 0 }}>
|
||||
<Option>{t('None')!}</Option>
|
||||
</Dropdown>
|
||||
<AudiotrackButton />
|
||||
</div>
|
||||
} />
|
||||
}
|
||||
</div>
|
||||
<div className="flex justify-between gap-2">
|
||||
<ToolTipButton desc={t('Regenerate')} icon={<ArrowSync20Regular />} onClick={() => {
|
||||
|
||||
@ -9,7 +9,7 @@ import { Preset } from '../types/presets';
|
||||
import { AboutContent } from '../types/about';
|
||||
import { Attachment, ChatParams, Conversation } from '../types/chat';
|
||||
import { CompletionPreset } from '../types/completion';
|
||||
import { CompositionParams } from '../types/composition';
|
||||
import { CompositionParams, Track } from '../types/composition';
|
||||
import { ModelConfig } from '../types/configs';
|
||||
import { DownloadStatus } from '../types/downloads';
|
||||
import { IntroductionContent } from '../types/home';
|
||||
@ -90,6 +90,11 @@ class CommonStore {
|
||||
};
|
||||
compositionGenerating: boolean = false;
|
||||
compositionSubmittedPrompt: string = defaultCompositionPrompt;
|
||||
tracks: Track[] = [];
|
||||
trackScale: number = 1;
|
||||
trackTotalTime: number = 5000;
|
||||
trackCurrentTime: number = 0;
|
||||
trackPlayStartTime: number = 0;
|
||||
// configs
|
||||
currentModelConfigIndex: number = 0;
|
||||
modelConfigs: ModelConfig[] = [];
|
||||
@ -380,6 +385,26 @@ class CommonStore {
|
||||
setSidePanelCollapsed(value: boolean | 'auto') {
|
||||
this.sidePanelCollapsed = value;
|
||||
}
|
||||
|
||||
setTracks(value: Track[]) {
|
||||
this.tracks = value;
|
||||
}
|
||||
|
||||
setTrackScale(value: number) {
|
||||
this.trackScale = value;
|
||||
}
|
||||
|
||||
setTrackTotalTime(value: number) {
|
||||
this.trackTotalTime = value;
|
||||
}
|
||||
|
||||
setTrackCurrentTime(value: number) {
|
||||
this.trackCurrentTime = value;
|
||||
}
|
||||
|
||||
setTrackPlayStartTime(value: number) {
|
||||
this.trackPlayStartTime = value;
|
||||
}
|
||||
}
|
||||
|
||||
export default new CommonStore();
|
||||
@ -10,3 +10,9 @@ export type CompositionParams = {
|
||||
midi: ArrayBuffer | null,
|
||||
ns: NoteSequence | null
|
||||
}
|
||||
export type Track = {
|
||||
id: string;
|
||||
content: string;
|
||||
offsetTime: number;
|
||||
contentTime: number;
|
||||
};
|
||||
@ -21,6 +21,7 @@ const embedded = [
|
||||
|
||||
// dependencies that exist in single component
|
||||
'react-beautiful-dnd',
|
||||
'react-draggable',
|
||||
'@magenta/music', 'html-midi-player',
|
||||
'react-markdown', 'rehype-highlight', 'rehype-raw', 'remark-breaks', 'remark-gfm'
|
||||
];
|
||||
|
||||
Loading…
Reference in New Issue
Block a user