diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6ece1db..f6ac9a0 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -25,6 +25,7 @@ "react-beautiful-dnd": "^13.1.1", "react-chartjs-2": "^5.2.0", "react-dom": "^18.2.0", + "react-draggable": "^4.4.6", "react-i18next": "^12.2.2", "react-markdown": "^8.0.7", "react-router": "^6.11.1", @@ -5410,6 +5411,19 @@ "loose-envify": "^1.1.0" } }, + "node_modules/react-draggable": { + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/react-draggable/-/react-draggable-4.4.6.tgz", + "integrity": "sha512-LtY5Xw1zTPqHkVmtM3X8MUOxNDOUhv/khTgBgrUvwaS064bwVvxT+q5El0uUFNx5IEPKXuRejr7UqLwBIg5pdw==", + "dependencies": { + "clsx": "^1.1.1", + "prop-types": "^15.8.1" + }, + "peerDependencies": { + "react": ">= 16.3.0", + "react-dom": ">= 16.3.0" + } + }, "node_modules/react-i18next": { "version": "12.2.2", "resolved": "https://registry.npmmirror.com/react-i18next/-/react-i18next-12.2.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index dbf0566..c0ff6de 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -26,6 +26,7 @@ "react-beautiful-dnd": "^13.1.1", "react-chartjs-2": "^5.2.0", "react-dom": "^18.2.0", + "react-draggable": "^4.4.6", "react-i18next": "^12.2.2", "react-markdown": "^8.0.7", "react-router": "^6.11.1", diff --git a/frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx b/frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx new file mode 100644 index 0000000..510fb90 --- /dev/null +++ b/frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx @@ -0,0 +1,31 @@ +import React, { FC, lazy } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Button, Dialog, DialogBody, DialogContent, DialogSurface, DialogTrigger } from '@fluentui/react-components'; +import { CustomToastContainer } from '../../components/CustomToastContainer'; +import { LazyImportComponent } from '../../components/LazyImportComponent'; + +const AudiotrackEditor = lazy(() => import('./AudiotrackEditor')); + +export const AudiotrackButton: FC<{ + size?: 'small' | 'medium' | 'large', + shape?: 'rounded' | 'circular' | 'square'; + appearance?: 'secondary' | 'primary' | 'outline' | 'subtle' | 'transparent'; +}> = ({ size, shape, appearance }) => { + const { t } = useTranslation(); + + return + + + + + + + + + + + + ; +}; \ No newline at end of file diff --git a/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx b/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx new file mode 100644 index 0000000..7625d64 --- /dev/null +++ b/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx @@ -0,0 +1,273 @@ +import React, { FC, useEffect, useRef, useState } from 'react'; +import { observer } from 'mobx-react-lite'; +import { useTranslation } from 'react-i18next'; +import Draggable from 'react-draggable'; +import { ToolTipButton } from '../../components/ToolTipButton'; +import { v4 as uuid } from 'uuid'; +import { + Add16Regular, + ArrowAutofitWidth20Regular, + Delete16Regular, + MusicNote220Regular, + Play16Regular, + Record16Regular +} from '@fluentui/react-icons'; +import { Button, Card, Slider, Text, Tooltip } from '@fluentui/react-components'; +import { useWindowSize } from 'usehooks-ts'; +import commonStore from '../../stores/commonStore'; +import classnames from 'classnames'; + +const snapValue = 25; +const minimalMoveTime = 8; // 1000/125=8ms +const scaleMin = 0.2; +const scaleMax = 3; +const baseMoveTime = Math.round(minimalMoveTime / scaleMin); + +type TrackProps = { + id: string; + right: number; + scale: number; + isSelected: boolean; + onSelect: (id: string) => void; +}; + +const Track: React.FC = observer(({ + id, + right, + scale, + isSelected, + onSelect +}) => { + const { t } = useTranslation(); + const trackIndex = commonStore.tracks.findIndex(t => t.id === id)!; + const track = commonStore.tracks[trackIndex]; + const trackClass = isSelected ? 'bg-blue-600' : 'bg-gray-700'; + const controlX = useRef(0); + + return ( + { + controlX.current = data.lastX; + }} + onStop={(e, data) => { + const delta = data.lastX - controlX.current; + let offsetTime = Math.round(Math.round(delta / snapValue * baseMoveTime * scale) / minimalMoveTime) * minimalMoveTime; + offsetTime = Math.min(Math.max( + offsetTime, + -track.offsetTime), commonStore.trackTotalTime - track.offsetTime); + + const tracks = commonStore.tracks.slice(); + tracks[trackIndex].offsetTime += offsetTime; + commonStore.setTracks(tracks); + }} + > +
onSelect(id)} + > + {t('Track') + ' ' + id} +
+ + ); +}); + +const AudiotrackEditor: FC = observer(() => { + const { t } = useTranslation(); + + const currentTimeControlRef = useRef(null); + const playStartTimeControlRef = useRef(null); + const tracksRef = useRef(null); + const toolbarRef = useRef(null); + const toolbarButtonRef = useRef(null); + const toolbarSliderRef = useRef(null); + + const [refreshRef, setRefreshRef] = useState(false); + + const windowSize = useWindowSize(); + const scale = (scaleMin + scaleMax) - commonStore.trackScale; + + const [selectedTrackId, setSelectedTrackId] = useState(''); + const playStartTimeControlX = useRef(0); + const selectedTrack = selectedTrackId ? commonStore.tracks.find(t => t.id === selectedTrackId) : undefined; + + useEffect(() => { + setRefreshRef(!refreshRef); + }, [windowSize, commonStore.tracks]); + + const viewControlsContainerWidth = (toolbarRef.current && toolbarButtonRef.current && toolbarSliderRef.current) ? + toolbarRef.current.clientWidth - toolbarButtonRef.current.clientWidth - toolbarSliderRef.current.clientWidth - 16 // 16 = ml-2 mr-2 + : 0; + const tracksWidth = viewControlsContainerWidth; + const timeOfTracksWidth = Math.floor(tracksWidth / snapValue) // number of moves + * baseMoveTime * scale; + const currentTimeControlWidth = (timeOfTracksWidth < commonStore.trackTotalTime) + ? timeOfTracksWidth / commonStore.trackTotalTime * viewControlsContainerWidth + : 0; + const playStartTimeControlPosition = { + x: (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue, + y: 0 + }; + + return ( +
+
+ {`${commonStore.trackPlayStartTime} ms / ${commonStore.trackTotalTime} ms`} +
+
+
+ } /> + } onClick={() => { + commonStore.setTracks([]); + }} /> +
+
+
+ { + setTimeout(() => { + let offset = 0; + if (currentTimeControlRef.current) { + const match = currentTimeControlRef.current.style.transform.match(/translate\((.+)px,/); + if (match) + offset = parseFloat(match[1]); + } + const offsetTime = commonStore.trackTotalTime / viewControlsContainerWidth * offset; + commonStore.setTrackCurrentTime(offsetTime); + }, 1); + }} + > +
+ +
viewControlsContainerWidth) + && 'hidden' + )}> + { + playStartTimeControlX.current = data.lastX; + }} + onStop={(e, data) => { + const delta = data.lastX - playStartTimeControlX.current; + let offsetTime = Math.round(Math.round(delta / snapValue * baseMoveTime * scale) / minimalMoveTime) * minimalMoveTime; + offsetTime = Math.min(Math.max( + offsetTime, + -commonStore.trackPlayStartTime), commonStore.trackTotalTime - commonStore.trackPlayStartTime); + commonStore.setTrackPlayStartTime(commonStore.trackPlayStartTime + offsetTime); + }} + > +
+ +
0) + ? tracksRef.current.clientHeight + : 0, + top: '50%', + left: 'calc(50% - 0.5px)' + }} /> +
+ +
+
+
+ + { + commonStore.setTrackScale(data.value); + }} + /> + +
+
+ {commonStore.tracks.map(track => +
+
+ } size="small" shape="circular" + appearance="subtle" /> + } size="small" shape="circular" + appearance="subtle" /> + } size="small" shape="circular" + appearance="subtle" onClick={() => { + const tracks = commonStore.tracks.slice().filter(t => t.id !== track.id); + commonStore.setTracks(tracks); + }} /> +
+
+
+ +
+
+
)} +
+ + + {t('Select a track to preview the content')} + +
+
+
+ {selectedTrack && + +
+ {`${t('Start Time')}: ${selectedTrack.offsetTime} ms`} + {`${t('Content Time')}: ${selectedTrack.contentTime} ms`} +
+ {selectedTrack.content} +
+
+ + } + +
+ ); +}); + +export default AudiotrackEditor; diff --git a/frontend/src/pages/Composition.tsx b/frontend/src/pages/Composition.tsx index 28eee7a..150336e 100644 --- a/frontend/src/pages/Composition.tsx +++ b/frontend/src/pages/Composition.tsx @@ -2,7 +2,7 @@ import 'html-midi-player'; import React, { FC, useEffect, useRef } from 'react'; import { observer } from 'mobx-react-lite'; import { WorkHeader } from '../components/WorkHeader'; -import { Button, Checkbox, Textarea } from '@fluentui/react-components'; +import { Button, Checkbox, Dropdown, Option, Textarea } from '@fluentui/react-components'; import { Labeled } from '../components/Labeled'; import { ValuedSlider } from '../components/ValuedSlider'; import { useTranslation } from 'react-i18next'; @@ -20,6 +20,7 @@ import { FileExists, OpenFileFolder, OpenSaveFileDialogBytes } from '../../wails import { getServerRoot, toastWithButton } from '../utils'; import { CompositionParams } from '../types/composition'; import { useMediaQuery } from 'usehooks-ts'; +import { AudiotrackButton } from './AudiotrackManager/AudiotrackButton'; let compositionSseController: AbortController | null = null; @@ -267,6 +268,18 @@ const CompositionPanel: FC = observer(() => { autoPlay: data.checked as boolean }); }} /> + {commonStore.platform !== 'web' && + + + + + +
+ } /> + }
} onClick={() => { diff --git a/frontend/src/stores/commonStore.ts b/frontend/src/stores/commonStore.ts index 229d0b8..c7f6983 100644 --- a/frontend/src/stores/commonStore.ts +++ b/frontend/src/stores/commonStore.ts @@ -9,7 +9,7 @@ import { Preset } from '../types/presets'; import { AboutContent } from '../types/about'; import { Attachment, ChatParams, Conversation } from '../types/chat'; import { CompletionPreset } from '../types/completion'; -import { CompositionParams } from '../types/composition'; +import { CompositionParams, Track } from '../types/composition'; import { ModelConfig } from '../types/configs'; import { DownloadStatus } from '../types/downloads'; import { IntroductionContent } from '../types/home'; @@ -90,6 +90,11 @@ class CommonStore { }; compositionGenerating: boolean = false; compositionSubmittedPrompt: string = defaultCompositionPrompt; + tracks: Track[] = []; + trackScale: number = 1; + trackTotalTime: number = 5000; + trackCurrentTime: number = 0; + trackPlayStartTime: number = 0; // configs currentModelConfigIndex: number = 0; modelConfigs: ModelConfig[] = []; @@ -380,6 +385,26 @@ class CommonStore { setSidePanelCollapsed(value: boolean | 'auto') { this.sidePanelCollapsed = value; } + + setTracks(value: Track[]) { + this.tracks = value; + } + + setTrackScale(value: number) { + this.trackScale = value; + } + + setTrackTotalTime(value: number) { + this.trackTotalTime = value; + } + + setTrackCurrentTime(value: number) { + this.trackCurrentTime = value; + } + + setTrackPlayStartTime(value: number) { + this.trackPlayStartTime = value; + } } export default new CommonStore(); \ No newline at end of file diff --git a/frontend/src/types/composition.ts b/frontend/src/types/composition.ts index dd3beca..45bf7d0 100644 --- a/frontend/src/types/composition.ts +++ b/frontend/src/types/composition.ts @@ -9,4 +9,10 @@ export type CompositionParams = { useLocalSoundFont: boolean, midi: ArrayBuffer | null, ns: NoteSequence | null -} \ No newline at end of file +} +export type Track = { + id: string; + content: string; + offsetTime: number; + contentTime: number; +}; \ No newline at end of file diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 8be9b1a..fbbffcd 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -21,6 +21,7 @@ const embedded = [ // dependencies that exist in single component 'react-beautiful-dnd', + 'react-draggable', '@magenta/music', 'html-midi-player', 'react-markdown', 'rehype-highlight', 'rehype-raw', 'remark-breaks', 'remark-gfm' ];