import { useEffect, useMemo, useRef, useState } from "react"; import { Alert, Avatar, Badge, Button, Card, Col, Empty, Row, Space, Statistic, Tag, Typography, message, } from "antd"; import { AudioOutlined, ClockCircleOutlined, PauseCircleOutlined, PlayCircleOutlined, SoundOutlined, SyncOutlined, UserOutlined, } from "@ant-design/icons"; import { useNavigate, useParams } from "react-router-dom"; import dayjs from "dayjs"; import PageHeader from "../../components/shared/PageHeader"; import { appendRealtimeTranscripts, completeRealtimeMeeting, getMeetingDetail, getRealtimeMeetingSessionStatus, getTranscripts, openRealtimeMeetingSocketSession, pauseRealtimeMeeting, type MeetingTranscriptVO, type MeetingVO, type RealtimeMeetingSessionStatus, type RealtimeTranscriptItemDTO, type RealtimeSocketSessionVO, } from "../../api/business/meeting"; const { Text, Title } = Typography; const SAMPLE_RATE = 16000; const CHUNK_SIZE = 1280; type WsSpeaker = string | { name?: string; user_id?: string | number } | undefined; type WsMessage = { type?: string; code?: number | string; message?: string; data?: { text?: string; is_final?: boolean; start?: number; end?: number; speaker_id?: string; speaker_name?: string; user_id?: string | number | null; }; text?: string; is_final?: boolean; speaker?: WsSpeaker; timestamp?: number[][]; }; type TranscriptCard = { id: string; speakerName: string; userId?: string | number; text: string; startTime?: number; endTime?: number; final: boolean; }; type RealtimeMeetingSessionDraft = { meetingId: number; meetingTitle: string; asrModelName: string; summaryModelName: string; asrModelId: number; mode: string; language: string; useSpkId: number; enablePunctuation: boolean; enableItn: boolean; enableTextRefine: boolean; saveAudio: boolean; hotwords: Array<{ hotword: string; weight: number }>; }; function getSessionKey(meetingId: number) { return `realtimeMeetingSession:${meetingId}`; } function buildDraftFromStatus(meetingId: number, meeting: MeetingVO | null, status?: RealtimeMeetingSessionStatus | null): RealtimeMeetingSessionDraft | null { const config = status?.resumeConfig; if (!config?.asrModelId) { return null; } return { meetingId, meetingTitle: meeting?.title || `实时会议 ${meetingId}`, asrModelName: "ASR", summaryModelName: "LLM", asrModelId: config.asrModelId, mode: config.mode || "2pass", language: config.language || "auto", useSpkId: config.useSpkId ? 1 : 0, enablePunctuation: config.enablePunctuation !== false, enableItn: config.enableItn !== false, enableTextRefine: !!config.enableTextRefine, saveAudio: !!config.saveAudio, hotwords: config.hotwords || [], }; } function floatTo16BitPCM(input: Float32Array) { const buffer = new ArrayBuffer(input.length * 2); const view = new DataView(buffer); for (let i = 0; i < input.length; i += 1) { const value = Math.max(-1, Math.min(1, input[i])); view.setInt16(i * 2, value < 0 ? value * 0x8000 : value * 0x7fff, true); } return buffer; } function resolveSpeaker(speaker?: WsSpeaker) { if (!speaker) { return { speakerId: "spk_0", speakerName: "Unknown", userId: undefined }; } if (typeof speaker === "string") { return { speakerId: speaker, speakerName: speaker, userId: undefined }; } return { speakerId: speaker.user_id ? String(speaker.user_id) : "spk_0", speakerName: speaker.name || (speaker.user_id ? String(speaker.user_id) : "Unknown"), userId: speaker.user_id, }; } function formatClock(totalSeconds: number) { const hours = Math.floor(totalSeconds / 3600); const minutes = Math.floor((totalSeconds % 3600) / 60); const seconds = totalSeconds % 60; if (hours > 0) { return `${hours.toString().padStart(2, "0")}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`; } return `${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`; } function formatTranscriptTime(ms?: number) { if (ms === undefined || ms === null) { return "--:--"; } const totalSeconds = Math.floor(ms / 1000); const minutes = Math.floor(totalSeconds / 60); const seconds = totalSeconds % 60; return `${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`; } function toMs(value?: number) { if (value === undefined || value === null || Number.isNaN(value)) { return undefined; } return Math.round(value * 1000); } function buildRealtimeProxyWsUrl(socketSession: RealtimeSocketSessionVO) { const protocol = window.location.protocol === "https:" ? "wss" : "ws"; return `${protocol}://${window.location.host}${socketSession.path}?sessionToken=${encodeURIComponent(socketSession.sessionToken)}`; } function normalizeWsMessage(payload: WsMessage) { if (payload.type === "partial" || payload.type === "segment") { const data = payload.data || {}; return { text: data.text || "", isFinal: payload.type === "segment" || !!data.is_final, speaker: { name: data.speaker_name, user_id: data.user_id ?? data.speaker_id, } as WsSpeaker, startTime: toMs(data.start), endTime: toMs(data.end), }; } if (!payload.text) { return null; } return { text: payload.text, isFinal: !!payload.is_final, speaker: payload.speaker, startTime: payload.timestamp?.[0]?.[0], endTime: payload.timestamp?.[payload.timestamp.length - 1]?.[1], }; } export function RealtimeAsrSession() { const navigate = useNavigate(); const { id } = useParams<{ id: string }>(); const meetingId = Number(id); const [meeting, setMeeting] = useState(null); const [sessionDraft, setSessionDraft] = useState(null); const [loading, setLoading] = useState(true); const [recording, setRecording] = useState(false); const [connecting, setConnecting] = useState(false); const [finishing, setFinishing] = useState(false); const [pausing, setPausing] = useState(false); const [statusText, setStatusText] = useState("待开始"); const [streamingText, setStreamingText] = useState(""); const [streamingSpeaker, setStreamingSpeaker] = useState("Unknown"); const [transcripts, setTranscripts] = useState([]); const [audioLevel, setAudioLevel] = useState(0); const [elapsedSeconds, setElapsedSeconds] = useState(0); const [sessionStatus, setSessionStatus] = useState(null); const transcriptRef = useRef(null); const wsRef = useRef(null); const audioContextRef = useRef(null); const processorRef = useRef(null); const audioSourceRef = useRef(null); const streamRef = useRef(null); const audioBufferRef = useRef([]); const completeOnceRef = useRef(false); const startedAtRef = useRef(null); const sessionStartedRef = useRef(false); const finalTranscriptCount = transcripts.length; const totalTranscriptChars = useMemo( () => transcripts.reduce((sum, item) => sum + item.text.length, 0) + streamingText.length, [streamingText, transcripts], ); const statusColor = recording ? "#1677ff" : connecting || finishing ? "#faad14" : "#94a3b8"; const hasRemoteActiveConnection = Boolean(sessionStatus?.activeConnection) && !recording && !connecting; useEffect(() => { if (!meetingId || Number.isNaN(meetingId)) { return; } const loadData = async () => { setLoading(true); try { const stored = sessionStorage.getItem(getSessionKey(meetingId)); const parsedDraft = stored ? JSON.parse(stored) : null; const [detailRes, transcriptRes, statusRes] = await Promise.all([ getMeetingDetail(meetingId), getTranscripts(meetingId), getRealtimeMeetingSessionStatus(meetingId), ]); const detail = detailRes.data.data; const realtimeStatus = statusRes.data.data; setMeeting(detail); setSessionStatus(realtimeStatus); const fallbackDraft = buildDraftFromStatus(meetingId, detail, realtimeStatus); const resolvedDraft = parsedDraft || fallbackDraft; setSessionDraft(resolvedDraft); if (resolvedDraft) { sessionStorage.setItem(getSessionKey(meetingId), JSON.stringify(resolvedDraft)); } if (realtimeStatus?.status === "PAUSED_RESUMABLE") { setStatusText(`已暂停,可在 ${Math.max(1, Math.ceil((realtimeStatus.remainingSeconds || 0) / 60))} 分钟内继续`); } else if (realtimeStatus?.status === "PAUSED_EMPTY") { setStatusText("已暂停,可继续识别"); } else if (realtimeStatus?.status === "ACTIVE" && realtimeStatus?.activeConnection) { setStatusText("当前会议已有活跃实时连接"); } else if (realtimeStatus?.status === "COMPLETING") { setStatusText("正在生成总结"); } setTranscripts( (transcriptRes.data.data || []).map((item: MeetingTranscriptVO) => ({ id: String(item.id), speakerName: item.speakerName || item.speakerId || "发言人", text: item.content, startTime: item.startTime, endTime: item.endTime, final: true, })), ); } catch { message.error("加载实时会议失败"); } finally { setLoading(false); } }; void loadData(); }, [meetingId]); useEffect(() => { if (!recording) { setElapsedSeconds(0); return; } const timer = window.setInterval(() => { if (startedAtRef.current) { setElapsedSeconds(Math.floor((Date.now() - startedAtRef.current) / 1000)); } }, 1000); return () => window.clearInterval(timer); }, [recording]); useEffect(() => { if (!transcriptRef.current) { return; } transcriptRef.current.scrollTop = transcriptRef.current.scrollHeight; }, [streamingText, transcripts]); useEffect(() => { const handlePageHide = () => { if (!meetingId || completeOnceRef.current) { return; } const token = localStorage.getItem("accessToken"); if (wsRef.current?.readyState === WebSocket.OPEN) { wsRef.current.send(JSON.stringify({ is_speaking: false })); } fetch(`/api/biz/meeting/${meetingId}/realtime/pause`, { method: "POST", keepalive: true, headers: { "Content-Type": "application/json", ...(token ? { Authorization: `Bearer ${token}` } : {}), }, body: JSON.stringify({}), }).catch(() => undefined); }; window.addEventListener("pagehide", handlePageHide); return () => window.removeEventListener("pagehide", handlePageHide); }, [meetingId]); const shutdownAudioPipeline = async () => { processorRef.current?.disconnect(); audioSourceRef.current?.disconnect(); if (streamRef.current) { streamRef.current.getTracks().forEach((track) => track.stop()); } if (audioContextRef.current && audioContextRef.current.state !== "closed") { await audioContextRef.current.close(); } streamRef.current = null; processorRef.current = null; audioSourceRef.current = null; audioContextRef.current = null; audioBufferRef.current = []; setAudioLevel(0); }; const handleFatalRealtimeError = async (errorMessage: string) => { setConnecting(false); setRecording(false); setStatusText("连接失败"); sessionStartedRef.current = false; wsRef.current?.close(); wsRef.current = null; await shutdownAudioPipeline(); startedAtRef.current = null; message.error(errorMessage); }; const startAudioPipeline = async () => { if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) { throw new Error("当前浏览器环境不支持麦克风访问。请使用 localhost 或 HTTPS 域名访问系统。"); } const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, }, }); const audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); const source = audioContext.createMediaStreamSource(stream); const processor = audioContext.createScriptProcessor(4096, 1, 1); streamRef.current = stream; audioContextRef.current = audioContext; audioSourceRef.current = source; processorRef.current = processor; processor.onaudioprocess = (event) => { const input = event.inputBuffer.getChannelData(0); let maxAmplitude = 0; for (let i = 0; i < input.length; i += 1) { const amplitude = Math.abs(input[i]); if (amplitude > maxAmplitude) { maxAmplitude = amplitude; } audioBufferRef.current.push(input[i]); } setAudioLevel(Math.min(100, Math.round(maxAmplitude * 180))); while (audioBufferRef.current.length >= CHUNK_SIZE) { const chunk = audioBufferRef.current.slice(0, CHUNK_SIZE); audioBufferRef.current = audioBufferRef.current.slice(CHUNK_SIZE); if (wsRef.current?.readyState === WebSocket.OPEN) { wsRef.current.send(floatTo16BitPCM(new Float32Array(chunk))); } } }; source.connect(processor); processor.connect(audioContext.destination); }; const saveFinalTranscript = async (normalized: { text: string; speaker?: WsSpeaker; startTime?: number; endTime?: number; }) => { if (!normalized.text || !meetingId) { return; } const speaker = resolveSpeaker(normalized.speaker); const item: RealtimeTranscriptItemDTO = { speakerId: speaker.speakerId, speakerName: speaker.speakerName, content: normalized.text, startTime: normalized.startTime, endTime: normalized.endTime, }; await appendRealtimeTranscripts(meetingId, [item]); }; const handlePause = async () => { if (!meetingId || pausing || finishing || (!recording && !connecting)) { return; } setPausing(true); setStatusText("暂停识别中..."); try { if (wsRef.current?.readyState === WebSocket.OPEN) { wsRef.current.send(JSON.stringify({ is_speaking: false })); } wsRef.current?.close(); wsRef.current = null; sessionStartedRef.current = false; await shutdownAudioPipeline(); const pauseRes = await pauseRealtimeMeeting(meetingId); setSessionStatus(pauseRes.data.data); setRecording(false); setConnecting(false); startedAtRef.current = null; setStatusText(pauseRes.data.data?.hasTranscript ? "已暂停,可继续识别" : "已暂停,当前还没有转录内容"); message.success("实时识别已暂停"); } catch (error) { setStatusText("暂停失败"); message.error(error instanceof Error ? error.message : "暂停实时识别失败"); } finally { setPausing(false); } }; const handleStart = async () => { if (!sessionDraft?.asrModelId) { message.error("未找到实时识别配置,请返回创建页重新进入"); return; } if (recording || connecting) { return; } setConnecting(true); setStatusText("连接识别服务..."); sessionStartedRef.current = false; try { const socketSessionRes = await openRealtimeMeetingSocketSession(meetingId, { asrModelId: sessionDraft.asrModelId, mode: sessionDraft.mode || "2pass", language: sessionDraft.language || "auto", useSpkId: sessionDraft.useSpkId, enablePunctuation: sessionDraft.enablePunctuation !== false, enableItn: sessionDraft.enableItn !== false, enableTextRefine: !!sessionDraft.enableTextRefine, saveAudio: !!sessionDraft.saveAudio, hotwords: sessionDraft.hotwords || [], }); const socketSession = socketSessionRes.data.data; const socket = new WebSocket(buildRealtimeProxyWsUrl(socketSession)); socket.binaryType = "arraybuffer"; wsRef.current = socket; socket.onopen = () => { setStatusText("识别服务连接中,等待第三方服务就绪..."); }; socket.onmessage = (event) => { try { const payload = JSON.parse(event.data) as WsMessage; if (payload.type === "proxy_ready") { if (sessionStartedRef.current) { return; } sessionStartedRef.current = true; setStatusText("启动音频采集中..."); socket.send(JSON.stringify(socketSession.startMessage || {})); void startAudioPipeline() .then(() => { startedAtRef.current = Date.now(); setConnecting(false); setRecording(true); setSessionStatus((prev) => prev ? { ...prev, status: "ACTIVE", activeConnection: true } : prev); setStatusText("实时识别中"); }) .catch((error) => { void handleFatalRealtimeError(error instanceof Error ? error.message : "启动麦克风失败"); }); return; } if ((payload.code || payload.type === "error") && payload.message) { setStatusText(payload.message); void handleFatalRealtimeError(payload.message); return; } const normalized = normalizeWsMessage(payload); if (!normalized) { return; } const speaker = resolveSpeaker(normalized.speaker); if (normalized.isFinal) { setTranscripts((prev) => [ ...prev, { id: `${Date.now()}-${Math.random()}`, speakerName: speaker.speakerName, userId: speaker.userId, text: normalized.text, startTime: normalized.startTime, endTime: normalized.endTime, final: true, }, ]); setStreamingText(""); setStreamingSpeaker("Unknown"); void saveFinalTranscript(normalized); } else { setStreamingText(normalized.text); setStreamingSpeaker(speaker.speakerName); } } catch { // ignore invalid payload } }; socket.onerror = () => { void handleFatalRealtimeError("实时识别 WebSocket 连接失败"); }; socket.onclose = () => { setConnecting(false); setRecording(false); sessionStartedRef.current = false; setSessionStatus((prev) => prev ? { ...prev, activeConnection: false } : prev); }; } catch (error) { setConnecting(false); setStatusText("启动失败"); sessionStartedRef.current = false; message.error(error instanceof Error ? error.message : "启动实时识别失败"); } }; const handleStop = async (navigateAfterStop = true) => { if (!meetingId || completeOnceRef.current) { return; } completeOnceRef.current = true; setFinishing(true); setStatusText("结束会议中..."); if (wsRef.current?.readyState === WebSocket.OPEN) { wsRef.current.send(JSON.stringify({ is_speaking: false })); } wsRef.current?.close(); wsRef.current = null; sessionStartedRef.current = false; await shutdownAudioPipeline(); try { await completeRealtimeMeeting(meetingId, {}); sessionStorage.removeItem(getSessionKey(meetingId)); setSessionStatus((prev) => prev ? { ...prev, status: "COMPLETING", canResume: false, activeConnection: false } : prev); setStatusText("已提交总结任务"); message.success("实时会议已结束,正在生成总结"); if (navigateAfterStop) { navigate(`/meetings/${meetingId}`); } } catch (error) { completeOnceRef.current = false; const errorMessage = error instanceof Error ? error.message : "结束会议失败"; if (errorMessage.includes("当前还没有转录内容")) { try { const statusRes = await getRealtimeMeetingSessionStatus(meetingId); setSessionStatus(statusRes.data.data); } catch { // ignore status refresh failure } setStatusText("当前还没有转录内容,可继续识别"); } else { setStatusText("结束失败"); } } finally { setRecording(false); setFinishing(false); startedAtRef.current = null; sessionStartedRef.current = false; } }; if (loading) { return (
); } if (!meeting) { return (
); } return (
} />
{!sessionDraft ? ( navigate("/meeting-live-create")}>返回创建页} /> ) : (
LIVE SESSION 会中实时识别 会中页面只保留控制区和实时转写流。
} />
ASR 模型{sessionDraft.asrModelName}
总结模型{sessionDraft.summaryModelName}
识别模式{sessionDraft.mode}
热词数量{sessionDraft.hotwords.length}
麦克风输入
实时转写流 优先展示最终片段,流式草稿保留在底部作为当前正在识别的内容。
} color={recording ? "processing" : sessionStatus?.status === "ACTIVE" && hasRemoteActiveConnection ? "processing" : sessionStatus?.status === "PAUSED_RESUMABLE" || sessionStatus?.status === "PAUSED_EMPTY" ? "warning" : "default"}>{recording ? "采集中" : connecting ? "连接中" : sessionStatus?.status === "ACTIVE" && hasRemoteActiveConnection ? "连接占用中" : sessionStatus?.status === "PAUSED_RESUMABLE" || sessionStatus?.status === "PAUSED_EMPTY" ? "已暂停" : "待命"} {sessionDraft.asrModelName}
{transcripts.length === 0 && !streamingText ? (
) : ( {transcripts.map((item) => (
{formatTranscriptTime(item.startTime)}
} className="transcript-avatar" /> {item.speakerName} {item.userId ? UID: {item.userId} : null} {formatTranscriptTime(item.startTime)} - {formatTranscriptTime(item.endTime)}
{item.text}
))} {streamingText ? (
--:--
} className="transcript-avatar" /> {streamingSpeaker} 流式草稿
{streamingText}
) : null}
)}
)}
); } export default RealtimeAsrSession;