diff --git a/backend/src/main/java/com/imeeting/common/MeetingConstants.java b/backend/src/main/java/com/imeeting/common/MeetingConstants.java new file mode 100644 index 0000000..cce19c6 --- /dev/null +++ b/backend/src/main/java/com/imeeting/common/MeetingConstants.java @@ -0,0 +1,13 @@ +package com.imeeting.common; + +public final class MeetingConstants { + + public static final String TYPE_OFFLINE = "OFFLINE"; + public static final String TYPE_REALTIME = "REALTIME"; + + public static final String SOURCE_WEB = "WEB"; + public static final String SOURCE_ANDROID = "ANDROID"; + + private MeetingConstants() { + } +} diff --git a/backend/src/main/java/com/imeeting/dto/biz/MeetingVO.java b/backend/src/main/java/com/imeeting/dto/biz/MeetingVO.java index fd17b58..b851359 100644 --- a/backend/src/main/java/com/imeeting/dto/biz/MeetingVO.java +++ b/backend/src/main/java/com/imeeting/dto/biz/MeetingVO.java @@ -37,6 +37,8 @@ public class MeetingVO { private String tags; @Schema(description = "音频地址") private String audioUrl; + @Schema(description = "浏览器播放音频地址") + private String playbackAudioUrl; @Schema(description = "会议类型") private String meetingType; @Schema(description = "会议来源") diff --git a/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java b/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java index 55c1194..c8a74e8 100644 --- a/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java @@ -86,7 +86,7 @@ public class LegacyMeetingAdapterServiceImpl implements LegacyMeetingAdapterServ meetingService.save(meeting); MeetingVO vo = new MeetingVO(); - meetingDomainSupport.fillMeetingVO(meeting, vo, false); + meetingDomainSupport.fillMeetingVO(meeting, vo, false, false); return vo; } diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java index 0f9392b..e5de779 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java @@ -103,10 +103,11 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { ); meeting.setAudioUrl(meetingDomainSupport.relocateAudioUrl(meeting.getId(), command.getAudioUrl())); meetingService.updateById(meeting); + meetingDomainSupport.prewarmPlaybackAudioAfterCommit(meeting.getAudioUrl()); meetingDomainSupport.publishMeetingCreated(meeting.getId(), meeting.getTenantId(), meeting.getCreatorId()); MeetingVO vo = new MeetingVO(); - meetingDomainSupport.fillMeetingVO(meeting, vo, false); + meetingDomainSupport.fillMeetingVO(meeting, vo, false, false); return vo; } @@ -129,7 +130,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { realtimeMeetingSessionStateService.rememberResumeConfig(meeting.getId(), buildRealtimeResumeConfig(command, tenantId, runtimeProfile)); MeetingVO vo = new MeetingVO(); - meetingDomainSupport.fillMeetingVO(meeting, vo, false); + meetingDomainSupport.fillMeetingVO(meeting, vo, false, false); return vo; } @@ -262,6 +263,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { meeting.setAudioUrl(meetingDomainSupport.relocateAudioUrl(meetingId, audioUrl)); markAudioSaveSuccess(meeting); meetingService.updateById(meeting); + meetingDomainSupport.prewarmPlaybackAudioAfterCommit(meeting.getAudioUrl()); prepareOfflineReprocessTasks(meetingId, currentStatus); realtimeMeetingSessionStateService.clear(meetingId); updateMeetingProgress(meetingId, 0, "正在转入离线音频识别流程...", 0); @@ -292,6 +294,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { meeting.setStatus(2); meetingService.updateById(meeting); updateMeetingProgress(meetingId, 90, "正在生成会议总结...", 0); + meetingDomainSupport.prewarmPlaybackAudioAfterCommit(meeting.getAudioUrl()); aiTaskService.dispatchSummaryTask(meetingId, meeting.getTenantId(), meeting.getCreatorId()); } diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingDomainSupport.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingDomainSupport.java index bb2fd41..985be8d 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingDomainSupport.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingDomainSupport.java @@ -44,6 +44,7 @@ public class MeetingDomainSupport { private final SysUserMapper sysUserMapper; private final ApplicationEventPublisher eventPublisher; private final MeetingSummaryFileService meetingSummaryFileService; + private final MeetingPlaybackAudioResolver meetingPlaybackAudioResolver; @Value("${unisbase.app.upload-path}") private String uploadPath; @@ -135,6 +136,22 @@ public class MeetingDomainSupport { } } + public void prewarmPlaybackAudioAfterCommit(String audioUrl) { + if (audioUrl == null || audioUrl.isBlank()) { + return; + } + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + meetingPlaybackAudioResolver.prewarmBrowserPlaybackAudio(audioUrl); + return; + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + meetingPlaybackAudioResolver.prewarmBrowserPlaybackAudio(audioUrl); + } + }); + } + private AudioRelocationPlan buildAudioRelocationPlan(Long meetingId, Path sourcePath) { String fileName = sourcePath.getFileName().toString(); String ext = ""; @@ -257,7 +274,8 @@ public class MeetingDomainSupport { return null; } - public void fillMeetingVO(Meeting meeting, com.imeeting.dto.biz.MeetingVO vo, boolean includeSummary) { + public void fillMeetingVO(Meeting meeting, com.imeeting.dto.biz.MeetingVO vo, boolean includeSummary, + boolean includePlaybackAudio) { vo.setId(meeting.getId()); vo.setTenantId(meeting.getTenantId()); vo.setCreatorId(meeting.getCreatorId()); @@ -268,6 +286,9 @@ public class MeetingDomainSupport { vo.setMeetingTime(meeting.getMeetingTime()); vo.setTags(meeting.getTags()); vo.setAudioUrl(meeting.getAudioUrl()); + if (includePlaybackAudio) { + vo.setPlaybackAudioUrl(meetingPlaybackAudioResolver.resolveBrowserPlaybackAudioUrl(meeting.getAudioUrl())); + } vo.setMeetingType(meeting.getMeetingType()); vo.setMeetingSource(meeting.getMeetingSource()); vo.setAudioSaveStatus(meeting.getAudioSaveStatus()); diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingPlaybackAudioResolver.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingPlaybackAudioResolver.java new file mode 100644 index 0000000..729980d --- /dev/null +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingPlaybackAudioResolver.java @@ -0,0 +1,615 @@ +package com.imeeting.service.biz.impl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Component +public class MeetingPlaybackAudioResolver { + + private static final int BROWSER_SAMPLE_RATE = 48_000; + private static final int DIRECT_PLAY_SAMPLE_RATE_44K = 44_100; + private static final int SOURCE_SAMPLE_RATE_16K = 16_000; + private static final int PCM_AUDIO_FORMAT = 1; + private static final int PCM_16_BITS = 16; + private static final int RESAMPLE_MULTIPLIER = 3; + private static final long MAX_WAVE_DATA_SIZE = 0xffff_ffffL - 36; + private static final String PLAYBACK_FILE_SUFFIX = "_browser_48000"; + private static final Set PLAYABLE_M4A_SAMPLE_ENTRY_TYPES = Set.of("mp4a"); + private static final Set MP4_CONTAINER_TYPES = Set.of( + "moov", "trak", "mdia", "minf", "stbl", "edts", "dinf", "udta", "meta", "ilst" + ); + + private final ConcurrentMap conversionLocks = new ConcurrentHashMap<>(); + + @Value("${unisbase.app.upload-path}") + private String uploadPath; + + @Value("${unisbase.app.resource-prefix:/api/static/}") + private String resourcePrefix; + + @Value("${imeeting.audio.ffmpeg-path:ffmpeg}") + private String ffmpegPath; + + @Async + public void prewarmBrowserPlaybackAudio(String audioUrl) { + try { + resolveBrowserPlaybackAudioUrl(audioUrl); + } catch (Exception ex) { + log.warn("Failed to prewarm browser playback audio, audioUrl={}", audioUrl, ex); + } + } + + public String resolveBrowserPlaybackAudioUrl(String audioUrl) { + if (audioUrl == null || audioUrl.isBlank()) { + return audioUrl; + } + + AudioResource audioResource = resolveAudioResource(audioUrl); + if (audioResource == null || !Files.exists(audioResource.path())) { + return audioUrl; + } + if (isConvertedPlaybackFile(audioResource.path())) { + return audioResource.publicUrl(); + } + + String extension = resolveExtension(audioResource.path()); + if ("wav".equals(extension)) { + return resolveWavPlaybackAudioUrl(audioResource, audioUrl); + } + if ("m4a".equals(extension)) { + return resolveM4aPlaybackAudioUrl(audioResource, audioUrl); + } + return audioUrl; + } + + private String resolveWavPlaybackAudioUrl(AudioResource audioResource, String fallbackAudioUrl) { + WavMetadata metadata = readWavMetadata(audioResource.path()); + if (metadata == null) { + return fallbackAudioUrl; + } + if (metadata.sampleRate() == DIRECT_PLAY_SAMPLE_RATE_44K || metadata.sampleRate() == BROWSER_SAMPLE_RATE) { + return fallbackAudioUrl; + } + if (metadata.sampleRate() != SOURCE_SAMPLE_RATE_16K + || metadata.audioFormat() != PCM_AUDIO_FORMAT + || metadata.bitsPerSample() != PCM_16_BITS) { + return fallbackAudioUrl; + } + + Path convertedPath = audioResource.path().resolveSibling(buildConvertedFileName(audioResource.path())); + String convertedPublicUrl = resolvePublicUrl(convertedPath); + if (convertedPublicUrl == null) { + return fallbackAudioUrl; + } + if (isUsableConvertedFile(audioResource.path(), convertedPath)) { + return convertedPublicUrl; + } + + Object lock = conversionLocks.computeIfAbsent(audioResource.path().toAbsolutePath().normalize().toString(), ignored -> new Object()); + synchronized (lock) { + if (isUsableConvertedFile(audioResource.path(), convertedPath)) { + return convertedPublicUrl; + } + try { + Files.createDirectories(convertedPath.getParent()); + Path tempPath = buildTemporaryOutputPath(convertedPath); + convertToBrowserWave(audioResource.path(), tempPath, metadata); + moveReplacing(tempPath, convertedPath); + return convertedPublicUrl; + } catch (Exception ex) { + log.warn("Failed to generate browser playback wav, source={}", audioResource.path(), ex); + return fallbackAudioUrl; + } + } + } + + private String resolveM4aPlaybackAudioUrl(AudioResource audioResource, String fallbackAudioUrl) { + M4aMetadata metadata = readM4aMetadata(audioResource.path()); + if (metadata == null || !StringUtils.hasText(metadata.sampleEntryType())) { + return fallbackAudioUrl; + } + if (!PLAYABLE_M4A_SAMPLE_ENTRY_TYPES.contains(metadata.sampleEntryType())) { + return fallbackAudioUrl; + } + if (metadata.sampleRate() == DIRECT_PLAY_SAMPLE_RATE_44K || metadata.sampleRate() == BROWSER_SAMPLE_RATE) { + return fallbackAudioUrl; + } + if (metadata.sampleRate() != SOURCE_SAMPLE_RATE_16K) { + return fallbackAudioUrl; + } + + Path convertedPath = audioResource.path().resolveSibling(buildConvertedFileName(audioResource.path())); + String convertedPublicUrl = resolvePublicUrl(convertedPath); + if (convertedPublicUrl == null) { + return fallbackAudioUrl; + } + if (isUsableConvertedFile(audioResource.path(), convertedPath)) { + return convertedPublicUrl; + } + + Object lock = conversionLocks.computeIfAbsent(audioResource.path().toAbsolutePath().normalize().toString(), ignored -> new Object()); + synchronized (lock) { + if (isUsableConvertedFile(audioResource.path(), convertedPath)) { + return convertedPublicUrl; + } + try { + Files.createDirectories(convertedPath.getParent()); + Path tempPath = buildTemporaryOutputPath(convertedPath); + convertToBrowserM4a(audioResource.path(), tempPath); + moveReplacing(tempPath, convertedPath); + return convertedPublicUrl; + } catch (Exception ex) { + log.warn("Failed to generate browser playback m4a, source={}", audioResource.path(), ex); + return fallbackAudioUrl; + } + } + } + + private AudioResource resolveAudioResource(String audioUrl) { + String normalizedUrl = stripQueryAndFragment(audioUrl); + String prefix = normalizedResourcePrefix(); + if (!normalizedUrl.startsWith(prefix)) { + return null; + } + + String relativePath = normalizedUrl.substring(prefix.length()); + if (relativePath.isBlank()) { + return null; + } + + Path uploadRoot = uploadRoot(); + Path resolvedPath = uploadRoot.resolve(relativePath.replace('/', java.io.File.separatorChar)).normalize(); + if (!resolvedPath.startsWith(uploadRoot)) { + return null; + } + return new AudioResource(resolvedPath, normalizedUrl); + } + + private String resolvePublicUrl(Path path) { + Path uploadRoot = uploadRoot(); + Path normalizedPath = path.toAbsolutePath().normalize(); + if (!normalizedPath.startsWith(uploadRoot)) { + return null; + } + String relativePath = uploadRoot.relativize(normalizedPath).toString().replace('\\', '/'); + return normalizedResourcePrefix() + relativePath; + } + + private Path uploadRoot() { + return Paths.get(normalizedUploadPath()).toAbsolutePath().normalize(); + } + + private String normalizedUploadPath() { + return uploadPath.endsWith("/") || uploadPath.endsWith("\\") + ? uploadPath.substring(0, uploadPath.length() - 1) + : uploadPath; + } + + private String normalizedResourcePrefix() { + return resourcePrefix.endsWith("/") ? resourcePrefix : resourcePrefix + "/"; + } + + private String stripQueryAndFragment(String audioUrl) { + int queryIndex = audioUrl.indexOf('?'); + int fragmentIndex = audioUrl.indexOf('#'); + int endIndex = audioUrl.length(); + if (queryIndex >= 0) { + endIndex = Math.min(endIndex, queryIndex); + } + if (fragmentIndex >= 0) { + endIndex = Math.min(endIndex, fragmentIndex); + } + return audioUrl.substring(0, endIndex); + } + + private boolean isUsableConvertedFile(Path sourcePath, Path convertedPath) { + try { + return Files.exists(convertedPath) + && Files.size(convertedPath) > 0 + && Files.getLastModifiedTime(convertedPath).compareTo(Files.getLastModifiedTime(sourcePath)) >= 0; + } catch (IOException ex) { + return false; + } + } + + private boolean isConvertedPlaybackFile(Path path) { + return path.getFileName().toString().toLowerCase(Locale.ROOT).contains(PLAYBACK_FILE_SUFFIX + "."); + } + + private String buildConvertedFileName(Path sourcePath) { + String fileName = sourcePath.getFileName().toString(); + int dotIndex = fileName.lastIndexOf('.'); + String baseName = dotIndex >= 0 ? fileName.substring(0, dotIndex) : fileName; + String extension = dotIndex >= 0 ? fileName.substring(dotIndex) : ""; + return baseName + PLAYBACK_FILE_SUFFIX + extension; + } + + private Path buildTemporaryOutputPath(Path targetPath) { + String fileName = targetPath.getFileName().toString(); + int dotIndex = fileName.lastIndexOf('.'); + if (dotIndex < 0) { + return targetPath.resolveSibling(fileName + ".tmp"); + } + String baseName = fileName.substring(0, dotIndex); + String extension = fileName.substring(dotIndex); + return targetPath.resolveSibling(baseName + ".tmp" + extension); + } + + private String resolveExtension(Path path) { + String fileName = path.getFileName().toString().toLowerCase(Locale.ROOT); + int dotIndex = fileName.lastIndexOf('.'); + return dotIndex >= 0 ? fileName.substring(dotIndex + 1) : ""; + } + + private WavMetadata readWavMetadata(Path path) { + try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) { + if (raf.length() < 44) { + return null; + } + if (!"RIFF".equals(readAscii(raf, 4))) { + return null; + } + readUnsignedIntLe(raf); + if (!"WAVE".equals(readAscii(raf, 4))) { + return null; + } + + Integer audioFormat = null; + Integer channels = null; + Integer bitsPerSample = null; + Long sampleRate = null; + Long dataOffset = null; + Long dataSize = null; + + while (raf.getFilePointer() + 8 <= raf.length()) { + String chunkId = readAscii(raf, 4); + long chunkSize = readUnsignedIntLe(raf); + long chunkDataStart = raf.getFilePointer(); + long nextChunkStart = chunkDataStart + chunkSize; + if (nextChunkStart > raf.length()) { + return null; + } + + if ("fmt ".equals(chunkId)) { + if (chunkSize < 16) { + return null; + } + audioFormat = readUnsignedShortLe(raf); + channels = readUnsignedShortLe(raf); + sampleRate = readUnsignedIntLe(raf); + readUnsignedIntLe(raf); + readUnsignedShortLe(raf); + bitsPerSample = readUnsignedShortLe(raf); + } else if ("data".equals(chunkId)) { + dataOffset = raf.getFilePointer(); + dataSize = chunkSize; + } + + raf.seek(nextChunkStart); + if ((chunkSize & 1) == 1 && raf.getFilePointer() < raf.length()) { + raf.seek(raf.getFilePointer() + 1); + } + + if (audioFormat != null && channels != null && bitsPerSample != null && sampleRate != null && dataOffset != null && dataSize != null) { + return new WavMetadata(audioFormat, channels, sampleRate.intValue(), bitsPerSample, dataOffset, dataSize); + } + } + return null; + } catch (Exception ex) { + log.debug("Failed to parse wav metadata, path={}", path, ex); + return null; + } + } + + private M4aMetadata readM4aMetadata(Path path) { + try (SeekableByteChannel channel = Files.newByteChannel(path, StandardOpenOption.READ)) { + return findM4aMetadata(channel, 0, channel.size()); + } catch (Exception ex) { + log.debug("Failed to parse m4a metadata, path={}", path, ex); + return null; + } + } + + private M4aMetadata findM4aMetadata(SeekableByteChannel channel, long start, long end) throws IOException { + long position = start; + while (position + 8 <= end) { + Mp4AtomHeader header = readAtomHeader(channel, position, end); + if (header == null || header.endPosition() <= position) { + return null; + } + if ("stsd".equals(header.type())) { + return readM4aMetadataFromStsd(channel, header.payloadPosition(), header.endPosition()); + } + if (MP4_CONTAINER_TYPES.contains(header.type())) { + M4aMetadata nested = findM4aMetadata(channel, header.payloadPosition(), header.endPosition()); + if (nested != null && StringUtils.hasText(nested.sampleEntryType())) { + return nested; + } + } + position = header.endPosition(); + } + return null; + } + + private M4aMetadata readM4aMetadataFromStsd(SeekableByteChannel channel, long payloadStart, long atomEnd) throws IOException { + if (payloadStart + 8 > atomEnd) { + return null; + } + ByteBuffer stsdHeader = ByteBuffer.allocate(8); + if (!readFully(channel, stsdHeader, payloadStart, 8)) { + return null; + } + long entryCount = Integer.toUnsignedLong(stsdHeader.getInt(4)); + long entryPosition = payloadStart + 8; + for (long index = 0; index < entryCount && entryPosition + 8 <= atomEnd; index++) { + ByteBuffer entryPrefix = ByteBuffer.allocate(36); + if (!readFully(channel, entryPrefix, entryPosition, 36)) { + return null; + } + long entrySize = Integer.toUnsignedLong(entryPrefix.getInt()); + String entryType = readFourCc(entryPrefix); + if (entrySize < 8) { + return null; + } + Integer sampleRate = null; + if ("mp4a".equals(entryType) && entrySize >= 36) { + sampleRate = entryPrefix.getInt(32) >>> 16; + } + if (StringUtils.hasText(entryType)) { + return new M4aMetadata(entryType, sampleRate); + } + entryPosition += entrySize; + } + return null; + } + + private Mp4AtomHeader readAtomHeader(SeekableByteChannel channel, long position, long parentEnd) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(16); + if (!readFully(channel, buffer, position, 8)) { + return null; + } + long size = Integer.toUnsignedLong(buffer.getInt()); + String type = readFourCc(buffer); + long headerLength = 8; + if (size == 1) { + if (!readFully(channel, buffer, position + 8, 8)) { + return null; + } + size = buffer.getLong(); + headerLength = 16; + } else if (size == 0) { + size = parentEnd - position; + } + if (size < headerLength) { + return null; + } + long endPosition = position + size; + if (endPosition > parentEnd) { + return null; + } + return new Mp4AtomHeader(type, position + headerLength, endPosition); + } + + private boolean readFully(SeekableByteChannel channel, ByteBuffer buffer, long position, int length) throws IOException { + buffer.clear(); + buffer.limit(length); + channel.position(position); + while (buffer.hasRemaining()) { + if (channel.read(buffer) < 0) { + return false; + } + } + buffer.flip(); + return true; + } + + private String readFourCc(ByteBuffer buffer) { + byte[] typeBytes = new byte[4]; + buffer.get(typeBytes); + return new String(typeBytes, StandardCharsets.US_ASCII); + } + + private void convertToBrowserWave(Path sourcePath, Path targetPath, WavMetadata metadata) throws IOException { + int frameSize = metadata.frameSize(); + if (frameSize <= 0 || metadata.dataSize() <= 0 || metadata.dataSize() % frameSize != 0) { + throw new IOException("Invalid wav frame layout"); + } + + long convertedDataSize = Math.multiplyExact(metadata.dataSize(), RESAMPLE_MULTIPLIER); + if (convertedDataSize > MAX_WAVE_DATA_SIZE) { + throw new IOException("Converted wav exceeds RIFF size limit"); + } + + try (InputStream input = new BufferedInputStream(Files.newInputStream(sourcePath)); + OutputStream output = new BufferedOutputStream(Files.newOutputStream( + targetPath, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE + ))) { + skipFully(input, metadata.dataOffset()); + writeWavHeader(output, convertedDataSize, metadata.channels(), BROWSER_SAMPLE_RATE, metadata.bitsPerSample()); + + byte[] sourceBuffer = new byte[frameSize * 4096]; + byte[] targetBuffer = new byte[sourceBuffer.length * RESAMPLE_MULTIPLIER]; + long remaining = metadata.dataSize(); + while (remaining > 0) { + int chunkSize = (int) Math.min(sourceBuffer.length, remaining); + readFully(input, sourceBuffer, chunkSize); + + int targetOffset = 0; + for (int offset = 0; offset < chunkSize; offset += frameSize) { + for (int i = 0; i < RESAMPLE_MULTIPLIER; i++) { + System.arraycopy(sourceBuffer, offset, targetBuffer, targetOffset, frameSize); + targetOffset += frameSize; + } + } + output.write(targetBuffer, 0, targetOffset); + remaining -= chunkSize; + } + } + } + + private void convertToBrowserM4a(Path sourcePath, Path targetPath) throws IOException, InterruptedException { + List command = List.of( + ffmpegPath, + "-v", "error", + "-y", + "-i", sourcePath.toString(), + "-vn", + "-ar", String.valueOf(BROWSER_SAMPLE_RATE), + "-c:a", "aac", + "-f", "mp4", + targetPath.toString() + ); + executeCommand(command, targetPath); + } + + private void executeCommand(List command, Path expectedOutput) throws IOException, InterruptedException { + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.redirectErrorStream(true); + Process process = processBuilder.start(); + byte[] output; + try (InputStream processStream = process.getInputStream()) { + output = processStream.readAllBytes(); + } + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new IOException("Audio conversion timed out"); + } + if (process.exitValue() != 0) { + throw new IOException("Audio conversion failed: " + new String(output, StandardCharsets.UTF_8)); + } + if (!Files.exists(expectedOutput) || Files.size(expectedOutput) <= 0) { + throw new IOException("Audio conversion produced empty output"); + } + } + + private void writeWavHeader(OutputStream output, long dataSize, int channels, int sampleRate, int bitsPerSample) throws IOException { + int blockAlign = channels * bitsPerSample / 8; + long byteRate = (long) sampleRate * blockAlign; + + output.write(new byte[]{'R', 'I', 'F', 'F'}); + writeIntLe(output, 36 + dataSize); + output.write(new byte[]{'W', 'A', 'V', 'E'}); + output.write(new byte[]{'f', 'm', 't', ' '}); + writeIntLe(output, 16); + writeShortLe(output, PCM_AUDIO_FORMAT); + writeShortLe(output, channels); + writeIntLe(output, sampleRate); + writeIntLe(output, byteRate); + writeShortLe(output, blockAlign); + writeShortLe(output, bitsPerSample); + output.write(new byte[]{'d', 'a', 't', 'a'}); + writeIntLe(output, dataSize); + } + + private void moveReplacing(Path source, Path target) throws IOException { + try { + Files.move(source, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException ex) { + Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); + } + } + + private void readFully(InputStream input, byte[] buffer, int length) throws IOException { + int offset = 0; + while (offset < length) { + int read = input.read(buffer, offset, length - offset); + if (read < 0) { + throw new IOException("Unexpected EOF while reading wav data"); + } + offset += read; + } + } + + private void skipFully(InputStream input, long bytesToSkip) throws IOException { + long remaining = bytesToSkip; + while (remaining > 0) { + long skipped = input.skip(remaining); + if (skipped > 0) { + remaining -= skipped; + continue; + } + if (input.read() < 0) { + throw new IOException("Unexpected EOF while skipping wav data"); + } + remaining--; + } + } + + private String readAscii(RandomAccessFile raf, int length) throws IOException { + byte[] bytes = new byte[length]; + raf.readFully(bytes); + return new String(bytes, StandardCharsets.US_ASCII); + } + + private int readUnsignedShortLe(RandomAccessFile raf) throws IOException { + int b1 = raf.readUnsignedByte(); + int b2 = raf.readUnsignedByte(); + return b1 | (b2 << 8); + } + + private long readUnsignedIntLe(RandomAccessFile raf) throws IOException { + long b1 = raf.readUnsignedByte(); + long b2 = raf.readUnsignedByte(); + long b3 = raf.readUnsignedByte(); + long b4 = raf.readUnsignedByte(); + return b1 | (b2 << 8) | (b3 << 16) | (b4 << 24); + } + + private void writeShortLe(OutputStream output, int value) throws IOException { + output.write(value & 0xff); + output.write((value >> 8) & 0xff); + } + + private void writeIntLe(OutputStream output, long value) throws IOException { + output.write((int) (value & 0xff)); + output.write((int) ((value >> 8) & 0xff)); + output.write((int) ((value >> 16) & 0xff)); + output.write((int) ((value >> 24) & 0xff)); + } + + private record AudioResource(Path path, String publicUrl) { + } + + private record WavMetadata(int audioFormat, int channels, int sampleRate, int bitsPerSample, long dataOffset, + long dataSize) { + + private int frameSize() { + return channels * bitsPerSample / 8; + } + } + + private record M4aMetadata(String sampleEntryType, Integer sampleRate) { + } + + private record Mp4AtomHeader(String type, long payloadPosition, long endPosition) { + } +} diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingQueryServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingQueryServiceImpl.java index b9a37e5..047dcb3 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingQueryServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingQueryServiceImpl.java @@ -131,7 +131,7 @@ public class MeetingQueryServiceImpl implements MeetingQueryService { private MeetingVO toVO(Meeting meeting, boolean includeSummary) { MeetingVO vo = new MeetingVO(); - meetingDomainSupport.fillMeetingVO(meeting, vo, includeSummary); + meetingDomainSupport.fillMeetingVO(meeting, vo, includeSummary, includeSummary); return vo; } } diff --git a/backend/src/main/resources/application-dev.yml b/backend/src/main/resources/application-dev.yml index b3c2821..e6e723c 100644 --- a/backend/src/main/resources/application-dev.yml +++ b/backend/src/main/resources/application-dev.yml @@ -33,3 +33,6 @@ unisbase: app: server-base-url: ${APP_SERVER_BASE_URL:http://127.0.0.1:${server.port}} upload-path: ${APP_UPLOAD_PATH:D:/data/imeeting/uploads/} +imeeting: + audio: + ffmpeg-path: D:\tools\exe\ffmpeg-master-latest-win64-gpl-shared\bin\ffmpeg diff --git a/backend/src/main/resources/application-prod.yml b/backend/src/main/resources/application-prod.yml index a7442a3..f84c5e6 100644 --- a/backend/src/main/resources/application-prod.yml +++ b/backend/src/main/resources/application-prod.yml @@ -21,3 +21,6 @@ unisbase: app: server-base-url: ${APP_SERVER_BASE_URL:http://127.0.0.1:${server.port}} upload-path: ${APP_UPLOAD_PATH:/data/imeeting/uploads/} +imeeting: + audio: + ffmpeg-path: ${IMEETING_AUDIO_FFMPEG_PATH:ffmpeg} diff --git a/backend/src/main/resources/application-test.yml b/backend/src/main/resources/application-test.yml index d952ec7..bc69d9e 100644 --- a/backend/src/main/resources/application-test.yml +++ b/backend/src/main/resources/application-test.yml @@ -25,3 +25,6 @@ unisbase: app: server-base-url: ${APP_SERVER_BASE_URL:http://10.100.53.199:${server.port}} upload-path: ${APP_UPLOAD_PATH:D:/data/imeeting-test/uploads/} +imeeting: + audio: + ffmpeg-path: ${IMEETING_AUDIO_FFMPEG_PATH:ffmpeg} diff --git a/backend/src/test/java/com/imeeting/service/biz/impl/MeetingDomainSupportTest.java b/backend/src/test/java/com/imeeting/service/biz/impl/MeetingDomainSupportTest.java index 68a44ea..053d54d 100644 --- a/backend/src/test/java/com/imeeting/service/biz/impl/MeetingDomainSupportTest.java +++ b/backend/src/test/java/com/imeeting/service/biz/impl/MeetingDomainSupportTest.java @@ -27,6 +27,8 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class MeetingDomainSupportTest { @@ -116,6 +118,19 @@ class MeetingDomainSupportTest { assertFalse(Files.exists(tempDir.resolve("uploads/meetings/301"))); } + @Test + void shouldPrewarmPlaybackAudioAfterTransactionCommit() { + MeetingPlaybackAudioResolver playbackAudioResolver = mock(MeetingPlaybackAudioResolver.class); + MeetingDomainSupport support = newSupport(mock(AiTaskService.class), mock(MeetingSummaryPromptAssembler.class), playbackAudioResolver); + + TransactionSynchronizationManager.initSynchronization(); + support.prewarmPlaybackAudioAfterCommit("/api/static/meetings/401/source_audio.m4a"); + + verify(playbackAudioResolver, never()).prewarmBrowserPlaybackAudio(any()); + triggerAfterCompletion(TransactionSynchronization.STATUS_COMMITTED); + verify(playbackAudioResolver).prewarmBrowserPlaybackAudio("/api/static/meetings/401/source_audio.m4a"); + } + @Test void shouldPreferLatestSummaryTaskIdWhenResolvingLastUserPrompt() { AiTaskService aiTaskService = mock(AiTaskService.class); @@ -214,13 +229,20 @@ class MeetingDomainSupportTest { } private MeetingDomainSupport newSupport(AiTaskService aiTaskService, MeetingSummaryPromptAssembler assembler) { + return newSupport(aiTaskService, assembler, mock(MeetingPlaybackAudioResolver.class)); + } + + private MeetingDomainSupport newSupport(AiTaskService aiTaskService, + MeetingSummaryPromptAssembler assembler, + MeetingPlaybackAudioResolver playbackAudioResolver) { MeetingDomainSupport support = new MeetingDomainSupport( assembler, aiTaskService, mock(MeetingTranscriptMapper.class), mock(SysUserMapper.class), mock(ApplicationEventPublisher.class), - mock(MeetingSummaryFileService.class) + mock(MeetingSummaryFileService.class), + playbackAudioResolver ); ReflectionTestUtils.setField(support, "uploadPath", tempDir.resolve("uploads").toString()); return support; @@ -228,6 +250,9 @@ class MeetingDomainSupportTest { private void triggerAfterCompletion(int status) { for (TransactionSynchronization synchronization : TransactionSynchronizationManager.getSynchronizations()) { + if (status == TransactionSynchronization.STATUS_COMMITTED) { + synchronization.afterCommit(); + } synchronization.afterCompletion(status); } TransactionSynchronizationManager.clearSynchronization(); diff --git a/frontend/src/api/business/meeting.ts b/frontend/src/api/business/meeting.ts index ecbd3e0..1d003e0 100644 --- a/frontend/src/api/business/meeting.ts +++ b/frontend/src/api/business/meeting.ts @@ -2,6 +2,7 @@ import http from "../http"; import axios from "axios"; const MEETING_UPLOAD_FLOW_TIMEOUT = 600000; +const MEETING_DETAIL_TIMEOUT = 120000; export interface MeetingVO { id: number; @@ -16,6 +17,7 @@ export interface MeetingVO { participantIds?: number[]; tags: string; audioUrl: string; + playbackAudioUrl?: string; meetingType?: "OFFLINE" | "REALTIME"; meetingSource?: "WEB" | "ANDROID"; audioSaveStatus?: "NONE" | "SUCCESS" | "FAILED"; @@ -54,6 +56,10 @@ export const resolveAudioMimeType = (audioUrl?: string) => { return extension ? AUDIO_MIME_TYPE_BY_EXTENSION[extension] : undefined; }; +export const resolveMeetingPlaybackAudioUrl = (meeting?: Pick | null) => { + return meeting?.playbackAudioUrl || meeting?.audioUrl; +}; + export interface CreateMeetingCommand { id?: number; title: string; @@ -252,7 +258,10 @@ export interface PublicMeetingPreviewVO { export const getMeetingDetail = (id: number) => { return http.get<{ code: string; data: MeetingVO; msg: string }>( - `/api/biz/meeting/${id}` + `/api/biz/meeting/${id}`, + { + timeout: MEETING_DETAIL_TIMEOUT, + } ); }; @@ -272,6 +281,7 @@ export const getPublicMeetingPreview = (id: number, accessPassword?: string) => return http.get<{ code: string; data: PublicMeetingPreviewVO; msg: string }>( `/api/public/meetings/${id}/preview`, { + timeout: MEETING_DETAIL_TIMEOUT, params: accessPassword ? { accessPassword } : undefined, } ); diff --git a/frontend/src/pages/business/MeetingDetail.tsx b/frontend/src/pages/business/MeetingDetail.tsx index d29c7ec..d00f993 100644 --- a/frontend/src/pages/business/MeetingDetail.tsx +++ b/frontend/src/pages/business/MeetingDetail.tsx @@ -36,6 +36,7 @@ import { MeetingVO, reSummary, resolveAudioMimeType, + resolveMeetingPlaybackAudioUrl, retryMeetingTranscription, updateMeetingBasic, updateMeetingTranscript, @@ -113,6 +114,22 @@ const generateAccessPassword = () => const normalizeAccessPasswordInput = (value?: string | null) => (value || '').replace(/[^A-Za-z0-9]/g, '').slice(0, 4); +const sanitizeDownloadFileName = (value?: string | null, fallback = 'meeting-recording') => { + const normalized = (value || '').replace(/[\\/:*?"<>|\r\n]/g, '_').trim(); + return normalized || fallback; +}; + +const resolveAudioExtension = (audioUrl?: string) => { + const normalizedUrl = audioUrl?.split('#')[0]?.split('?')[0] || ''; + return normalizedUrl.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase() || 'mp3'; +}; + +const getMeetingAudioDownloadName = (meeting?: Pick | null) => { + const audioUrl = resolveMeetingPlaybackAudioUrl(meeting); + const extension = resolveAudioExtension(audioUrl); + return `${sanitizeDownloadFileName(meeting?.title)}-录音.${extension}`; +}; + const buildMeetingPreviewUrl = (meetingId?: number, accessPassword?: string) => { if (!meetingId || Number.isNaN(meetingId) || typeof window === 'undefined') { return ''; @@ -698,6 +715,12 @@ const MeetingDetail: React.FC = () => { [speakerLabels], ); const previewAccessPassword = useMemo(() => (meeting?.accessPassword || '').trim(), [meeting?.accessPassword]); + const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]); + const audioDownloadFileName = useMemo(() => getMeetingAudioDownloadName(meeting), [meeting]); + const audioDownloadFormatLabel = useMemo( + () => resolveAudioExtension(playbackAudioUrl).toUpperCase(), + [playbackAudioUrl], + ); const sharePreviewUrl = useMemo(() => { const meetingId = meeting?.id ?? (id ? Number(id) : NaN); return buildMeetingPreviewUrl(meetingId); @@ -734,7 +757,7 @@ const MeetingDetail: React.FC = () => { }, [canRetryTranscription, meeting, transcripts.length]); useEffect(() => { - if (!meeting?.audioUrl) { + if (!playbackAudioUrl) { setShowFloatingTranscriptPlayer(false); setFloatingTranscriptPlayerLayout(null); return undefined; @@ -785,7 +808,7 @@ const MeetingDetail: React.FC = () => { root?.removeEventListener('scroll', updateFloatingPlayerState); resizeObserver?.disconnect(); }; - }, [meeting?.audioUrl, meeting?.status]); + }, [meeting?.status, playbackAudioUrl]); useEffect(() => { if (!id) return; @@ -1077,7 +1100,7 @@ const MeetingDetail: React.FC = () => { }, []); const handleAudioPlaybackError = useCallback(() => { - const currentAudioUrl = meeting?.audioUrl || ''; + const currentAudioUrl = playbackAudioUrl || ''; if (!currentAudioUrl || audioPlaybackErrorShownRef.current === currentAudioUrl) { return; } @@ -1090,7 +1113,7 @@ const MeetingDetail: React.FC = () => { ); audioPlaybackErrorShownRef.current = currentAudioUrl; setAudioPlaying(false); - }, [meeting?.audioUrl, message]); + }, [message, playbackAudioUrl]); useEffect(() => { const audio = audioRef.current; @@ -1130,7 +1153,7 @@ const MeetingDetail: React.FC = () => { audio.removeEventListener('ended', handleEnded); audio.removeEventListener('error', handleError); }; - }, [meeting?.audioUrl, audioPlaybackRate, meeting?.status, handleAudioPlaybackError]); + }, [audioPlaybackRate, meeting?.status, playbackAudioUrl, handleAudioPlaybackError]); const toggleAudioPlayback = () => { if (!audioRef.current) return; @@ -1213,6 +1236,22 @@ const MeetingDetail: React.FC = () => { } }; + const handleDownloadAudio = () => { + if (!playbackAudioUrl) { + message.warning('当前暂无可下载的会议录音'); + return; + } + + const anchor = document.createElement('a'); + anchor.href = playbackAudioUrl; + anchor.download = audioDownloadFileName; + anchor.target = '_blank'; + anchor.rel = 'noopener noreferrer'; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + }; + const handleCopyPreviewLink = async () => { if (!sharePreviewUrl) { message.error('预览链接暂不可用'); @@ -1421,24 +1460,32 @@ const MeetingDetail: React.FC = () => { 正在总结 )} - {meeting.status === 3 && !!meeting.summaryContent && ( + {(playbackAudioUrl || (meeting.status === 3 && !!meeting.summaryContent)) && ( , - onClick: () => handleDownloadSummary('pdf'), - disabled: downloadLoading === 'pdf' - }, - { - key: 'word', - label: '下载 Word', - icon: , - onClick: () => handleDownloadSummary('word'), - disabled: downloadLoading === 'word' - } + ...(playbackAudioUrl ? [{ + key: 'audio', + label: `下载录音 (${audioDownloadFormatLabel})`, + icon: , + onClick: handleDownloadAudio, + }] : []), + ...(meeting.status === 3 && !!meeting.summaryContent ? [ + { + key: 'pdf', + label: '下载 PDF', + icon: , + onClick: () => handleDownloadSummary('pdf'), + disabled: downloadLoading === 'pdf' + }, + { + key: 'word', + label: '下载 Word', + icon: , + onClick: () => handleDownloadSummary('word'), + disabled: downloadLoading === 'word' + } + ] : []) ] }} placement="bottomRight" @@ -1668,9 +1715,9 @@ const MeetingDetail: React.FC = () => {
原文}> - {meeting.audioUrl && ( + {playbackAudioUrl && ( )} {emptyTranscriptFailureNotice && ( @@ -1690,7 +1737,7 @@ const MeetingDetail: React.FC = () => { { const nextStartTime = transcripts[index + 1]?.startTime || Infinity; const isActive = (audioCurrentTime * 1000) >= item.startTime && (audioCurrentTime * 1000) < nextStartTime; @@ -1789,7 +1836,7 @@ const MeetingDetail: React.FC = () => { )}
- {meeting?.audioUrl && showFloatingTranscriptPlayer && floatingTranscriptPlayerLayout && ( + {playbackAudioUrl && showFloatingTranscriptPlayer && floatingTranscriptPlayerLayout && (
splitDisplayItems(meeting?.tags), [meeting?.tags]); const keywords = useMemo(() => analysis.keywords || [], [analysis.keywords]); + const playbackAudioUrl = useMemo(() => resolveMeetingPlaybackAudioUrl(meeting), [meeting]); const statusMeta = STATUS_META[meeting?.status || 0] || { label: TEXT.statusPending, className: "is-warning", @@ -414,7 +416,7 @@ export default function MeetingPreview() { }; const handleAudioError = () => { - const currentAudioUrl = meeting?.audioUrl || ""; + const currentAudioUrl = playbackAudioUrl || ""; if (!currentAudioUrl || audioPlaybackErrorShownRef.current === currentAudioUrl) { return; } @@ -900,7 +902,7 @@ export default function MeetingPreview() {
- {meeting.audioUrl && ( + {playbackAudioUrl && ( )} - {meeting.audioUrl && pageTab === 'transcript' ? ( + {playbackAudioUrl && pageTab === 'transcript' ? ( <>