import 'html-midi-player'; import React, { FC, useEffect, useRef } from 'react'; import { observer } from 'mobx-react-lite'; import { WorkHeader } from '../components/WorkHeader'; import { Button, Checkbox, Dropdown, Option, Textarea } from '@fluentui/react-components'; import { Labeled } from '../components/Labeled'; import { ValuedSlider } from '../components/ValuedSlider'; import { useTranslation } from 'react-i18next'; import commonStore, { ModelStatus } from '../stores/commonStore'; import { fetchEventSource } from '@microsoft/fetch-event-source'; import { toast } from 'react-toastify'; import { DialogButton } from '../components/DialogButton'; import { ToolTipButton } from '../components/ToolTipButton'; import { ArrowSync20Regular, Save28Regular } from '@fluentui/react-icons'; import { PlayerElement, VisualizerElement } from 'html-midi-player'; import * as mm from '@magenta/music/esm/core.js'; import { NoteSequence } from '@magenta/music/esm/protobuf.js'; import { defaultCompositionABCPrompt, defaultCompositionPrompt } from './defaultConfigs'; import { CloseMidiPort, FileExists, OpenFileFolder, OpenMidiPort, OpenSaveFileDialogBytes, SaveFile, StartFile } from '../../wailsjs/go/backend_golang/App'; import { getServerRoot, getSoundFont, toastWithButton } from '../utils'; import { CompositionParams } from '../types/composition'; import { useMediaQuery } from 'usehooks-ts'; import { AudiotrackButton } from './AudiotrackManager/AudiotrackButton'; let compositionSseController: AbortController | null = null; const CompositionPanel: FC = observer(() => { const { t } = useTranslation(); const mq = useMediaQuery('(min-width: 640px)'); const inputRef = useRef(null); const port = commonStore.getCurrentModelConfig().apiParameters.apiPort; const visualizerRef = useRef(null); const playerRef = useRef(null); const scrollToBottom = () => { if (inputRef.current) inputRef.current.scrollTop = inputRef.current.scrollHeight; }; const params = commonStore.compositionParams; const setParams = (newParams: Partial) => { commonStore.setCompositionParams({ ...commonStore.compositionParams, ...newParams }); }; const setPrompt = (prompt: string) => { setParams({ prompt }); if (!commonStore.compositionGenerating) generateNs(false); }; const updateNs = (ns: NoteSequence | null) => { if (playerRef.current) { playerRef.current.noteSequence = ns; playerRef.current.reload(); } if (visualizerRef.current) { visualizerRef.current.noteSequence = ns; visualizerRef.current.reload(); } }; const setSoundFont = async () => { if (playerRef.current) { playerRef.current.soundFont = await getSoundFont(); } }; useEffect(() => { if (inputRef.current) { inputRef.current.style.height = '100%'; inputRef.current.style.maxHeight = '100%'; } scrollToBottom(); if (playerRef.current && visualizerRef.current) { playerRef.current.addVisualizer(visualizerRef.current); playerRef.current.addEventListener('start', () => { visualizerRef.current?.reload(); }); setSoundFont().then(() => { updateNs(params.ns); }); const button = playerRef.current.shadowRoot?.querySelector('.controls .play') as HTMLElement | null; if (button) button.style.background = '#f2f5f6'; } }, []); const externalPlayListener = () => { const params = commonStore.compositionParams; const saveAndPlay = async (midi: ArrayBuffer, path: string) => { await SaveFile(path, Array.from(new Uint8Array(midi))); StartFile(path); }; if (params.externalPlay) { if (params.midi) { setTimeout(() => { playerRef.current?.stop(); }); saveAndPlay(params.midi, './midi/last.mid').catch((e: string) => { if (e.includes('being used')) saveAndPlay(params.midi!, './midi/last-2.mid'); }); } } }; useEffect(() => { playerRef.current?.addEventListener('start', externalPlayListener); return () => { playerRef.current?.removeEventListener('start', externalPlayListener); }; }, [params.externalPlay]); useEffect(() => { if (!(commonStore.activeMidiDeviceIndex in commonStore.midiPorts)) { commonStore.setActiveMidiDeviceIndex(-1); CloseMidiPort(); } }, [commonStore.midiPorts]); const generateNs = (autoPlay: boolean) => { fetch(getServerRoot(port) + '/text-to-midi', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ 'text': commonStore.compositionParams.prompt.replaceAll(/||/g, '').replaceAll(' ', ' ').trim() }) }).then(r => { r.arrayBuffer().then(midi => { const ns = mm.midiToSequenceProto(midi); setParams({ midi, ns }); updateNs(ns); if (autoPlay) { if (commonStore.compositionParams.externalPlay) externalPlayListener(); else { if (commonStore.compositionParams.playOnlyGeneratedContent && playerRef.current) { playerRef.current.currentTime = Math.max(commonStore.compositionParams.generationStartTime - 1, 0); } setTimeout(() => { playerRef.current?.start(); }); } } }); }); }; const onSubmit = (prompt: string) => { commonStore.setCompositionSubmittedPrompt(prompt); if (commonStore.status.status === ModelStatus.Offline && !commonStore.settings.apiUrl && commonStore.platform !== 'web') { toast(t('Please click the button in the top right corner to start the model'), { type: 'warning' }); commonStore.setCompositionGenerating(false); return; } let answer = ''; compositionSseController = new AbortController(); fetchEventSource( // https://api.openai.com/v1/completions || http://127.0.0.1:${port}/v1/completions getServerRoot(port, true) + '/v1/completions', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${commonStore.settings.apiKey}` }, body: JSON.stringify({ prompt, stream: true, model: commonStore.settings.apiCompletionModelName, // 'text-davinci-003' max_tokens: params.maxResponseToken, temperature: params.temperature, top_p: params.topP }), signal: compositionSseController?.signal, onmessage(e) { scrollToBottom(); if (e.data.trim() === '[DONE]') { commonStore.setCompositionGenerating(false); generateNs(commonStore.compositionParams.autoPlay); return; } let data; try { data = JSON.parse(e.data); } catch (error) { console.debug('json error', error); return; } if (data.model) commonStore.setLastModelName(data.model); if (data.choices && Array.isArray(data.choices) && data.choices.length > 0) { answer += data.choices[0]?.text || data.choices[0]?.delta?.content || ''; setPrompt(prompt + answer.replace(/\s+$/, '')); } }, async onopen(response) { if (response.status !== 200) { toast(response.statusText + '\n' + (await response.text()), { type: 'error' }); } }, onclose() { console.log('Connection closed'); }, onerror(err) { err = err.message || err; if (err && !err.includes('ReadableStreamDefaultReader')) toast(err, { type: 'error' }); commonStore.setCompositionGenerating(false); throw err; } }); }; return (