MIDI Recording and details improvement

2023-11-29 14:05:58 +08:00
parent 14a13d5768
commit b625b8a6d1
13 changed files with 520 additions and 28 deletions
--- a/frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx
+++ b/frontend/src/pages/AudiotrackManager/AudiotrackButton.tsx
@@ -3,6 +3,8 @@ import { useTranslation } from 'react-i18next';
 import { Button, Dialog, DialogBody, DialogContent, DialogSurface, DialogTrigger } from '@fluentui/react-components';
 import { CustomToastContainer } from '../../components/CustomToastContainer';
 import { LazyImportComponent } from '../../components/LazyImportComponent';
+import { flushMidiRecordingContent } from '../../utils';
+import commonStore from '../../stores/commonStore';

 const AudiotrackEditor = lazy(() => import('./AudiotrackEditor'));

@@ -13,7 +15,12 @@ export const AudiotrackButton: FC<{
 }> = ({ size, shape, appearance }) => {
  const { t } = useTranslation();

-  return <Dialog>
+  return <Dialog onOpenChange={(e, data) => {
+    if (!data.open) {
+      flushMidiRecordingContent();
+      commonStore.setRecordingTrackId('');
+    }
+  }}>
    <DialogTrigger disableButtonEnhancement>
      <Button size={size} shape={shape} appearance={appearance}>
        {t('Open MIDI Input Audio Tracks')}
--- a/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx
+++ b/frontend/src/pages/AudiotrackManager/AudiotrackEditor.tsx
@@ -10,19 +10,39 @@ import {
  Delete16Regular,
  MusicNote220Regular,
  Play16Regular,
-  Record16Regular
+  Record16Regular,
+  Stop16Filled
 } from '@fluentui/react-icons';
 import { Button, Card, Slider, Text, Tooltip } from '@fluentui/react-components';
 import { useWindowSize } from 'usehooks-ts';
 import commonStore from '../../stores/commonStore';
 import classnames from 'classnames';
+import {
+  InstrumentTypeNameMap,
+  InstrumentTypeTokenMap,
+  MidiMessage,
+  tracksMinimalTotalTime
+} from '../../types/composition';
+import { toast } from 'react-toastify';
+import { ToastOptions } from 'react-toastify/dist/types';
+import { flushMidiRecordingContent, refreshTracksTotalTime } from '../../utils';

 const snapValue = 25;
-const minimalMoveTime = 8; // 1000/125=8ms
-const scaleMin = 0.2;
+const minimalMoveTime = 8; // 1000/125=8ms wait_events=125
+const scaleMin = 0.05;
 const scaleMax = 3;
 const baseMoveTime = Math.round(minimalMoveTime / scaleMin);

+const velocityEvents = 128;
+const velocityBins = 12;
+const velocityExp = 0.5;
+
+const minimalTrackWidth = 80;
+const trackInitOffsetPx = 10;
+const pixelFix = 0.5;
+const topToArrowIcon = 19;
+const arrowIconToTracks = 23;
+
 type TrackProps = {
  id: string;
  right: number;
@@ -31,6 +51,82 @@ type TrackProps = {
  onSelect: (id: string) => void;
 };

+const displayCurrentInstrumentType = () => {
+  const displayPanelId = 'instrument_panel_id';
+  const content: React.ReactNode =
+    <div className="flex gap-2 items-center">
+      {InstrumentTypeNameMap.map((name, i) =>
+        <Text key={name} style={{ whiteSpace: 'nowrap' }}
+          className={commonStore.instrumentType === i ? 'text-blue-600' : ''}
+          weight={commonStore.instrumentType === i ? 'bold' : 'regular'}
+          size={commonStore.instrumentType === i ? 300 : 100}
+        >{name}</Text>)}
+    </div>;
+  const options: ToastOptions = {
+    type: 'default',
+    autoClose: 2000,
+    toastId: displayPanelId,
+    position: 'top-left',
+    style: {
+      width: 'fit-content'
+    }
+  };
+  if (toast.isActive(displayPanelId))
+    toast.update(displayPanelId, {
+      render: content,
+      ...options
+    });
+  else
+    toast(content, options);
+};
+
+const velocityToBin = (velocity: number) => {
+  velocity = Math.max(0, Math.min(velocity, velocityEvents - 1));
+  const binsize = velocityEvents / (velocityBins - 1);
+  return Math.ceil((velocityEvents * ((Math.pow(velocityExp, (velocity / velocityEvents)) - 1.0) / (velocityExp - 1.0))) / binsize);
+};
+
+const midiMessageToToken = (msg: MidiMessage) => {
+  if (msg.messageType === 'NoteOn') {
+    const instrument = InstrumentTypeTokenMap[commonStore.instrumentType];
+    const note = msg.note.toString(16);
+    const velocity = velocityToBin(msg.velocity).toString(16);
+    return `${instrument}:${note}:${velocity} `;
+  } else if (msg.messageType === 'ElapsedTime') {
+    let time = Math.round(msg.value / minimalMoveTime);
+    const num = Math.floor(time / 125); // wait_events=125
+    time -= num * 125;
+    let ret = '';
+    for (let i = 0; i < num; i++) {
+      ret += 't125 ';
+    }
+    if (time > 0)
+      ret += `t${time} `;
+    return ret;
+  } else
+    return '';
+};
+
+let dropRecordingTime = false;
+
+export const midiMessageHandler = (data: MidiMessage) => {
+  if (data.messageType === 'NoteOff')
+    return;
+  if (data.messageType === 'ControlChange') {
+    commonStore.setInstrumentType(Math.round(data.value / 127 * (InstrumentTypeNameMap.length - 1)));
+    displayCurrentInstrumentType();
+    return;
+  }
+  if (commonStore.recordingTrackId) {
+    if (dropRecordingTime && data.messageType === 'ElapsedTime') {
+      dropRecordingTime = false;
+      return;
+    }
+    commonStore.setRecordingRawContent([...commonStore.recordingRawContent, data]);
+    commonStore.setRecordingContent(commonStore.recordingContent + midiMessageToToken(data));
+  }
+};
+
 const Track: React.FC<TrackProps> = observer(({
  id,
  right,
@@ -66,18 +162,19 @@ const Track: React.FC<TrackProps> = observer(({
        const tracks = commonStore.tracks.slice();
        tracks[trackIndex].offsetTime += offsetTime;
        commonStore.setTracks(tracks);
+        refreshTracksTotalTime();
      }}
    >
      <div
        className={`p-1 cursor-move rounded whitespace-nowrap overflow-hidden ${trackClass}`}
        style={{
-          width: `${Math.max(80,
+          width: `${Math.max(minimalTrackWidth,
            track.contentTime / (baseMoveTime * scale) * snapValue
          )}px`
        }}
        onClick={() => onSelect(id)}
      >
-        <span className="text-white">{t('Track') + ' ' + id}</span>
+        <span className="text-white">{t('Track') + ' ' + (track.content || id)}</span>
      </div>
    </Draggable>
  );
@@ -86,12 +183,15 @@ const Track: React.FC<TrackProps> = observer(({
 const AudiotrackEditor: FC = observer(() => {
  const { t } = useTranslation();

+  const viewControlsContainerRef = useRef<HTMLDivElement>(null);
  const currentTimeControlRef = useRef<HTMLDivElement>(null);
  const playStartTimeControlRef = useRef<HTMLDivElement>(null);
+  const tracksEndLineRef = useRef<HTMLDivElement>(null);
  const tracksRef = useRef<HTMLDivElement>(null);
  const toolbarRef = useRef<HTMLDivElement>(null);
  const toolbarButtonRef = useRef<HTMLDivElement>(null);
  const toolbarSliderRef = useRef<HTMLInputElement>(null);
+  const contentPreviewRef = useRef<HTMLDivElement>(null);

  const [refreshRef, setRefreshRef] = useState(false);

@@ -102,6 +202,20 @@ const AudiotrackEditor: FC = observer(() => {
  const playStartTimeControlX = useRef(0);
  const selectedTrack = selectedTrackId ? commonStore.tracks.find(t => t.id === selectedTrackId) : undefined;

+  useEffect(() => {
+    if (toolbarSliderRef.current && toolbarSliderRef.current.parentElement)
+      toolbarSliderRef.current.parentElement.style.removeProperty('--fui-Slider--steps-percent');
+  }, []);
+
+  const scrollContentToBottom = () => {
+    if (contentPreviewRef.current)
+      contentPreviewRef.current.scrollTop = contentPreviewRef.current.scrollHeight;
+  };
+
+  useEffect(() => {
+    scrollContentToBottom();
+  }, [commonStore.recordingContent]);
+
  useEffect(() => {
    setRefreshRef(!refreshRef);
  }, [windowSize, commonStore.tracks]);
@@ -115,10 +229,12 @@ const AudiotrackEditor: FC = observer(() => {
  const currentTimeControlWidth = (timeOfTracksWidth < commonStore.trackTotalTime)
    ? timeOfTracksWidth / commonStore.trackTotalTime * viewControlsContainerWidth
    : 0;
-  const playStartTimeControlPosition = {
-    x: (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue,
-    y: 0
-  };
+  const playStartTimeControlPosition = (commonStore.trackPlayStartTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue;
+  const tracksEndPosition = (commonStore.trackTotalTime - commonStore.trackCurrentTime) / (baseMoveTime * scale) * snapValue;
+  const moveableTracksWidth = (tracksEndLineRef.current && viewControlsContainerRef.current &&
+    ((tracksEndLineRef.current.getBoundingClientRect().left - (viewControlsContainerRef.current.getBoundingClientRect().left + trackInitOffsetPx)) > 0))
+    ? tracksEndLineRef.current.getBoundingClientRect().left - (viewControlsContainerRef.current.getBoundingClientRect().left + trackInitOffsetPx)
+    : Infinity;

  return (
    <div className="flex flex-col gap-2 overflow-hidden" style={{ width: '80vw', height: '80vh' }}>
@@ -130,10 +246,28 @@ const AudiotrackEditor: FC = observer(() => {
          <ToolTipButton desc={t('Play All')} icon={<Play16Regular />} />
          <ToolTipButton desc={t('Clear All')} icon={<Delete16Regular />} onClick={() => {
            commonStore.setTracks([]);
+            commonStore.setTrackScale(1);
+            commonStore.setTrackTotalTime(tracksMinimalTotalTime);
+            commonStore.setTrackCurrentTime(0);
+            commonStore.setTrackPlayStartTime(0);
          }} />
        </div>
        <div className="grow">
-          <div className="flex flex-col ml-2 mr-2">
+          <div className="flex flex-col ml-2 mr-2" ref={viewControlsContainerRef}>
+            <div className="relative">
+              <Tooltip content={`${commonStore.trackTotalTime} ms`} showDelay={0} hideDelay={0}
+                relationship="description">
+                <div className="border-l absolute"
+                  ref={tracksEndLineRef}
+                  style={{
+                    height: (tracksRef.current && commonStore.tracks.length > 0)
+                      ? tracksRef.current.clientHeight - arrowIconToTracks
+                      : 0,
+                    top: `${topToArrowIcon + arrowIconToTracks}px`,
+                    left: `${tracksEndPosition + trackInitOffsetPx - pixelFix}px`
+                  }} />
+              </Tooltip>
+            </div>
            <Draggable axis="x" bounds={{
              left: 0,
              right: viewControlsContainerWidth - currentTimeControlWidth
@@ -160,17 +294,17 @@ const AudiotrackEditor: FC = observer(() => {
            </Draggable>
            <div className={classnames(
              'flex',
-              (playStartTimeControlPosition.x < 0 || playStartTimeControlPosition.x > viewControlsContainerWidth)
+              (playStartTimeControlPosition < 0 || playStartTimeControlPosition > viewControlsContainerWidth)
              && 'hidden'
            )}>
              <Draggable axis="x" bounds={{
                left: 0,
                right: (playStartTimeControlRef.current)
-                  ? viewControlsContainerWidth - playStartTimeControlRef.current.clientWidth
+                  ? Math.min(viewControlsContainerWidth - playStartTimeControlRef.current.clientWidth, moveableTracksWidth)
                  : 0
              }}
                grid={[snapValue, snapValue]}
-                position={playStartTimeControlPosition}
+                position={{ x: playStartTimeControlPosition, y: 0 }}
                onStart={(e, data) => {
                  playStartTimeControlX.current = data.lastX;
                }}
@@ -192,14 +326,15 @@ const AudiotrackEditor: FC = observer(() => {
                        ? tracksRef.current.clientHeight
                        : 0,
                      top: '50%',
-                      left: 'calc(50% - 0.5px)'
+                      left: `calc(50% - ${pixelFix}px)`
                    }} />
                </div>
              </Draggable>
            </div>
          </div>
        </div>
-        <Tooltip content={t('Scale View')!} showDelay={0} hideDelay={0} relationship="label">
+        <Tooltip content={t('Scale View')! + ': ' + commonStore.trackScale} showDelay={0} hideDelay={0}
+          relationship="description">
          <Slider ref={toolbarSliderRef} value={commonStore.trackScale} step={scaleMin} max={scaleMax} min={scaleMin}
            onChange={(e, data) => {
              commonStore.setTrackScale(data.value);
@@ -211,8 +346,23 @@ const AudiotrackEditor: FC = observer(() => {
        {commonStore.tracks.map(track =>
          <div key={track.id} className="flex gap-2 pb-1 border-b">
            <div className="flex gap-1 border-r h-7">
-              <ToolTipButton desc={t('Record')} icon={<Record16Regular />} size="small" shape="circular"
-                appearance="subtle" />
+              <ToolTipButton desc={commonStore.recordingTrackId === track.id ? t('Stop') : t('Record')}
+                icon={commonStore.recordingTrackId === track.id ? <Stop16Filled /> : <Record16Regular />}
+                size="small" shape="circular"
+                appearance="subtle" onClick={() => {
+                flushMidiRecordingContent();
+
+                if (commonStore.recordingTrackId === track.id) {
+                  commonStore.setRecordingTrackId('');
+                } else {
+                  dropRecordingTime = true;
+                  setSelectedTrackId(track.id);
+
+                  commonStore.setRecordingTrackId(track.id);
+                  commonStore.setRecordingContent(track.content);
+                  commonStore.setRecordingRawContent(track.rawContent.slice());
+                }
+              }} />
              <ToolTipButton desc={t('Play')} icon={<Play16Regular />} size="small" shape="circular"
                appearance="subtle" />
              <ToolTipButton desc={t('Delete')} icon={<Delete16Regular />} size="small" shape="circular"
@@ -226,7 +376,7 @@ const AudiotrackEditor: FC = observer(() => {
                <Track
                  id={track.id}
                  scale={scale}
-                  right={tracksWidth}
+                  right={Math.min(tracksWidth, moveableTracksWidth)}
                  isSelected={selectedTrackId === track.id}
                  onSelect={setSelectedTrackId}
                />
@@ -240,6 +390,7 @@ const AudiotrackEditor: FC = observer(() => {
              commonStore.setTracks([...commonStore.tracks, {
                id: uuid(),
                content: '',
+                rawContent: [],
                offsetTime: 0,
                contentTime: 0
              }]);
@@ -253,12 +404,14 @@ const AudiotrackEditor: FC = observer(() => {
      </div>
      <div className="grow"></div>
      {selectedTrack &&
-        <Card size="small" appearance="outline" style={{ minHeight: '150px' }}>
+        <Card size="small" appearance="outline" style={{ minHeight: '150px', maxHeight: '200px' }}>
          <div className="flex flex-col gap-1 overflow-hidden">
            <Text size={100}>{`${t('Start Time')}: ${selectedTrack.offsetTime} ms`}</Text>
            <Text size={100}>{`${t('Content Time')}: ${selectedTrack.contentTime} ms`}</Text>
-            <div className="overflow-y-auto overflow-x-hidden">
-              {selectedTrack.content}
+            <div className="overflow-y-auto overflow-x-hidden" ref={contentPreviewRef}>
+              {selectedTrackId === commonStore.recordingTrackId
+                ? commonStore.recordingContent
+                : selectedTrack.content}
            </div>
          </div>
        </Card>
--- a/frontend/src/pages/Composition.tsx
+++ b/frontend/src/pages/Composition.tsx
@@ -16,7 +16,13 @@ import { PlayerElement, VisualizerElement } from 'html-midi-player';
 import * as mm from '@magenta/music/esm/core.js';
 import { NoteSequence } from '@magenta/music/esm/protobuf.js';
 import { defaultCompositionPrompt } from './defaultConfigs';
-import { FileExists, OpenFileFolder, OpenSaveFileDialogBytes } from '../../wailsjs/go/backend_golang/App';
+import {
+  CloseMidiPort,
+  FileExists,
+  OpenFileFolder,
+  OpenMidiPort,
+  OpenSaveFileDialogBytes
+} from '../../wailsjs/go/backend_golang/App';
 import { getServerRoot, toastWithButton } from '../utils';
 import { CompositionParams } from '../types/composition';
 import { useMediaQuery } from 'usehooks-ts';
@@ -273,8 +279,26 @@ const CompositionPanel: FC = observer(() => {
                desc={t('Select the MIDI input device to be used.')}
                content={
                  <div className="flex flex-col gap-1">
-                    <Dropdown style={{ minWidth: 0 }}>
-                      <Option>{t('None')!}</Option>
+                    <Dropdown style={{ minWidth: 0 }}
+                      value={commonStore.activeMidiDeviceIndex === -1 ? t('None')! : commonStore.midiPorts[commonStore.activeMidiDeviceIndex].name}
+                      selectedOptions={[commonStore.activeMidiDeviceIndex.toString()]}
+                      onOptionSelect={(_, data) => {
+                        if (data.optionValue) {
+                          const index = Number(data.optionValue);
+                          let action = (index === -1)
+                            ? () => CloseMidiPort()
+                            : () => OpenMidiPort(index);
+                          action().then(() => {
+                            commonStore.setActiveMidiDeviceIndex(index);
+                          }).catch((e) => {
+                            toast(t('Error') + ' - ' + (e.message || e), { type: 'error', autoClose: 2500 });
+                          });
+                        }
+                      }}>
+                      <Option value={'-1'}>{t('None')!}</Option>
+                      {commonStore.midiPorts.map((p, i) =>
+                        <Option key={i} value={i.toString()}>{p.name}</Option>)
+                      }
                    </Dropdown>
                    <AudiotrackButton />
                  </div>