From 6600d377578222eec73b96fd5c5912fea0bd868e Mon Sep 17 00:00:00 2001 From: chenhao Date: Mon, 27 Apr 2026 10:39:34 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20M4A=20=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E9=AA=8C=E8=AF=81=E5=92=8C=E9=9F=B3=E9=A2=91=E6=92=AD?= =?UTF-8?q?=E6=94=BE=E9=94=99=E8=AF=AF=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 `MeetingAudioUploadSupport` 中添加 M4A 文件验证逻辑，确保文件可播放 - 更新前端 `MeetingPreview.tsx` 和 `MeetingDetail.tsx` 以处理音频播放错误，并显示相应的警告信息 - 在 `WebMvcConfig` 中配置 M4A 媒 --- .../com/imeeting/config/WebMvcConfig.java | 7 + .../AndroidMeetingRealtimeController.java | 1 + .../dto/biz/CreateMeetingCommand.java | 2 + .../dto/biz/CreateRealtimeMeetingCommand.java | 2 + .../impl/LegacyMeetingAdapterServiceImpl.java | 1 + .../biz/MeetingRuntimeProfileResolver.java | 1 + .../biz/impl/MeetingAudioUploadSupport.java | 132 +++++++++- .../biz/impl/MeetingCommandServiceImpl.java | 2 + .../MeetingRuntimeProfileResolverImpl.java | 41 +++- .../impl/MeetingAccessServiceImplTest.java | 20 ++ ...MeetingRuntimeProfileResolverImplTest.java | 63 +++++ frontend/src/api/business/meeting.ts | 19 ++ .../business/MeetingCreateDrawer.tsx | 108 ++++++++- frontend/src/pages/business/MeetingDetail.tsx | 227 +++++++++++++++--- .../src/pages/business/MeetingPreview.tsx | 26 +- 15 files changed, 598 insertions(+), 54 deletions(-) diff --git a/backend/src/main/java/com/imeeting/config/WebMvcConfig.java b/backend/src/main/java/com/imeeting/config/WebMvcConfig.java index 6254a61..37a8201 100644 --- a/backend/src/main/java/com/imeeting/config/WebMvcConfig.java +++ b/backend/src/main/java/com/imeeting/config/WebMvcConfig.java @@ -2,6 +2,8 @@ package com.imeeting.config; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; +import org.springframework.http.MediaType; +import org.springframework.web.servlet.config.annotation.ContentNegotiationConfigurer; import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @@ -16,6 +18,11 @@ public class WebMvcConfig implements WebMvcConfigurer { @Value("${unisbase.app.resource-prefix}") private String resourcePrefix; + @Override + public void configureContentNegotiation(ContentNegotiationConfigurer configurer) { + configurer.mediaType("m4a", MediaType.parseMediaType("audio/mp4")); + } + @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { // 确保目录存在 diff --git a/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingRealtimeController.java b/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingRealtimeController.java index a6c7415..46f58f8 100644 --- a/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingRealtimeController.java +++ b/backend/src/main/java/com/imeeting/controller/android/AndroidMeetingRealtimeController.java @@ -85,6 +85,7 @@ public class AndroidMeetingRealtimeController { command == null ? null : command.getEnableItn(), command == null ? null : command.getEnableTextRefine(), command == null ? null : command.getSaveAudio(), + null, command == null ? null : command.getHotWords() ); CreateRealtimeMeetingCommand createCommand = buildCreateCommand(command, authContext, runtimeProfile); diff --git a/backend/src/main/java/com/imeeting/dto/biz/CreateMeetingCommand.java b/backend/src/main/java/com/imeeting/dto/biz/CreateMeetingCommand.java index c253a84..0b8f8ee 100644 --- a/backend/src/main/java/com/imeeting/dto/biz/CreateMeetingCommand.java +++ b/backend/src/main/java/com/imeeting/dto/biz/CreateMeetingCommand.java @@ -35,6 +35,8 @@ public class CreateMeetingCommand { @NotNull(message = "promptId must not be null") private Long promptId; + private Long hotWordGroupId; + @Size(max = 2000, message = "userPrompt length must be <= 2000") private String userPrompt; diff --git a/backend/src/main/java/com/imeeting/dto/biz/CreateRealtimeMeetingCommand.java b/backend/src/main/java/com/imeeting/dto/biz/CreateRealtimeMeetingCommand.java index 2283a76..6b04861 100644 --- a/backend/src/main/java/com/imeeting/dto/biz/CreateRealtimeMeetingCommand.java +++ b/backend/src/main/java/com/imeeting/dto/biz/CreateRealtimeMeetingCommand.java @@ -32,6 +32,8 @@ public class CreateRealtimeMeetingCommand { @NotNull(message = "promptId must not be null") private Long promptId; + private Long hotWordGroupId; + @Size(max = 2000, message = "userPrompt length must be <= 2000") private String userPrompt; diff --git a/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java b/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java index 3253f51..55c1194 100644 --- a/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/android/legacy/impl/LegacyMeetingAdapterServiceImpl.java @@ -137,6 +137,7 @@ public class LegacyMeetingAdapterServiceImpl implements LegacyMeetingAdapterServ null, null, null, + null, List.of() ); diff --git a/backend/src/main/java/com/imeeting/service/biz/MeetingRuntimeProfileResolver.java b/backend/src/main/java/com/imeeting/service/biz/MeetingRuntimeProfileResolver.java index 668cb53..6def540 100644 --- a/backend/src/main/java/com/imeeting/service/biz/MeetingRuntimeProfileResolver.java +++ b/backend/src/main/java/com/imeeting/service/biz/MeetingRuntimeProfileResolver.java @@ -16,5 +16,6 @@ public interface MeetingRuntimeProfileResolver { Boolean enableItn, Boolean enableTextRefine, Boolean saveAudio, + Long hotWordGroupId, List hotWords); } diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingAudioUploadSupport.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingAudioUploadSupport.java index 1796ac2..0d279f0 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingAudioUploadSupport.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingAudioUploadSupport.java @@ -10,11 +10,14 @@ import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.Locale; import java.util.Set; import java.util.UUID; @@ -31,6 +34,8 @@ public class MeetingAudioUploadSupport { private static final Set WAV_MIME_TYPES = Set.of("audio/wav", "audio/x-wav", "audio/wave", "audio/vnd.wave"); private static final Set MP3_MIME_TYPES = Set.of("audio/mpeg", "audio/mp3", "audio/x-mp3", "audio/x-mpeg"); private static final Set M4A_MIME_TYPES = Set.of("audio/mp4", "audio/m4a", "audio/x-m4a", "audio/aac", "video/mp4"); + private static final Set PLAYABLE_M4A_SAMPLE_ENTRY_TYPES = Set.of("mp4a"); + private static final Set MP4_CONTAINER_TYPES = Set.of("moov", "trak", "mdia", "minf", "stbl"); @Value("${unisbase.app.upload-path}") private String uploadPath; @@ -52,8 +57,14 @@ public class MeetingAudioUploadSupport { String storedFileName = UUID.randomUUID() + "." + extension; Path targetPath = stagingDir.resolve(storedFileName); - try (InputStream inputStream = file.getInputStream()) { - Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); + try { + try (InputStream inputStream = file.getInputStream()) { + Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); + } + validateStoredAudio(targetPath, extension); + } catch (Exception ex) { + Files.deleteIfExists(targetPath); + throw ex; } return buildStagingAudioToken(storedFileName); } @@ -198,4 +209,121 @@ public class MeetingAudioUploadSupport { } return new String(header, startInclusive, endExclusive - startInclusive, StandardCharsets.US_ASCII); } + + private void validateStoredAudio(Path audioPath, String extension) throws IOException { + if (!"m4a".equals(extension)) { + return; + } + String sampleEntryType = resolveM4aSampleEntryType(audioPath); + if (!StringUtils.hasText(sampleEntryType)) { + throw new RuntimeException("当前 m4a 文件未找到可识别的音频轨道，无法在网页中播放，请转为 mp3、wav 或 AAC 编码的 m4a 后重试"); + } + if (!PLAYABLE_M4A_SAMPLE_ENTRY_TYPES.contains(sampleEntryType)) { + throw new RuntimeException("当前 m4a 文件音频编码为 " + sampleEntryType + "，浏览器通常只支持 AAC（mp4a）编码，请转为 mp3、wav 或 AAC 编码的 m4a 后重试"); + } + } + + private String resolveM4aSampleEntryType(Path audioPath) throws IOException { + try (SeekableByteChannel channel = Files.newByteChannel(audioPath, StandardOpenOption.READ)) { + return findM4aSampleEntryType(channel, 0, channel.size()); + } + } + + private String findM4aSampleEntryType(SeekableByteChannel channel, long start, long end) throws IOException { + long position = start; + while (position + 8 <= end) { + Mp4AtomHeader header = readAtomHeader(channel, position, end); + if (header == null || header.endPosition() <= position) { + return null; + } + if ("stsd".equals(header.type())) { + return readSampleEntryType(channel, header.payloadPosition(), header.endPosition()); + } + if (MP4_CONTAINER_TYPES.contains(header.type())) { + String nestedType = findM4aSampleEntryType(channel, header.payloadPosition(), header.endPosition()); + if (StringUtils.hasText(nestedType)) { + return nestedType; + } + } + position = header.endPosition(); + } + return null; + } + + private Mp4AtomHeader readAtomHeader(SeekableByteChannel channel, long position, long parentEnd) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(16); + if (!readFully(channel, buffer, position, 8)) { + return null; + } + long size = Integer.toUnsignedLong(buffer.getInt()); + String type = readFourCc(buffer); + long headerLength = 8; + if (size == 1) { + if (!readFully(channel, buffer, position + 8, 8)) { + return null; + } + size = buffer.getLong(); + headerLength = 16; + } else if (size == 0) { + size = parentEnd - position; + } + if (size < headerLength) { + return null; + } + long endPosition = position + size; + if (endPosition > parentEnd) { + return null; + } + return new Mp4AtomHeader(type, position + headerLength, endPosition); + } + + private String readSampleEntryType(SeekableByteChannel channel, long payloadStart, long atomEnd) throws IOException { + if (payloadStart + 8 > atomEnd) { + return null; + } + ByteBuffer stsdHeader = ByteBuffer.allocate(8); + if (!readFully(channel, stsdHeader, payloadStart, 8)) { + return null; + } + long entryCount = Integer.toUnsignedLong(stsdHeader.getInt(4)); + long entryPosition = payloadStart + 8; + for (long index = 0; index < entryCount && entryPosition + 8 <= atomEnd; index++) { + ByteBuffer entryHeader = ByteBuffer.allocate(8); + if (!readFully(channel, entryHeader, entryPosition, 8)) { + return null; + } + long entrySize = Integer.toUnsignedLong(entryHeader.getInt()); + String entryType = readFourCc(entryHeader); + if (entrySize < 8) { + return null; + } + if (StringUtils.hasText(entryType)) { + return entryType; + } + entryPosition += entrySize; + } + return null; + } + + private boolean readFully(SeekableByteChannel channel, ByteBuffer buffer, long position, int length) throws IOException { + buffer.clear(); + buffer.limit(length); + channel.position(position); + while (buffer.hasRemaining()) { + if (channel.read(buffer) < 0) { + return false; + } + } + buffer.flip(); + return true; + } + + private String readFourCc(ByteBuffer buffer) { + byte[] typeBytes = new byte[4]; + buffer.get(typeBytes); + return new String(typeBytes, StandardCharsets.US_ASCII); + } + + private record Mp4AtomHeader(String type, long payloadPosition, long endPosition) { + } } diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java index d5859c8..0f9392b 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingCommandServiceImpl.java @@ -627,6 +627,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { null, command.getEnableTextRefine(), null, + command.getHotWordGroupId(), command.getHotWords() ); } @@ -644,6 +645,7 @@ public class MeetingCommandServiceImpl implements MeetingCommandService { command.getEnableItn(), command.getEnableTextRefine(), command.getSaveAudio(), + command.getHotWordGroupId(), command.getHotWords() ); } diff --git a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingRuntimeProfileResolverImpl.java b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingRuntimeProfileResolverImpl.java index 5f0b872..ed6f69d 100644 --- a/backend/src/main/java/com/imeeting/service/biz/impl/MeetingRuntimeProfileResolverImpl.java +++ b/backend/src/main/java/com/imeeting/service/biz/impl/MeetingRuntimeProfileResolverImpl.java @@ -2,6 +2,7 @@ package com.imeeting.service.biz.impl; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.imeeting.dto.biz.AiModelVO; +import com.imeeting.dto.biz.HotWordGroupVO; import com.imeeting.dto.biz.RealtimeMeetingRuntimeProfile; import com.imeeting.entity.biz.HotWord; import com.imeeting.entity.biz.AsrModel; @@ -10,6 +11,7 @@ import com.imeeting.entity.biz.PromptTemplate; import com.imeeting.mapper.biz.AsrModelMapper; import com.imeeting.mapper.biz.LlmModelMapper; import com.imeeting.service.biz.AiModelService; +import com.imeeting.service.biz.HotWordGroupService; import com.imeeting.service.biz.HotWordService; import com.imeeting.service.biz.MeetingRuntimeProfileResolver; import com.imeeting.service.biz.PromptTemplateService; @@ -25,6 +27,7 @@ public class MeetingRuntimeProfileResolverImpl implements MeetingRuntimeProfileR private final AiModelService aiModelService; private final PromptTemplateService promptTemplateService; + private final HotWordGroupService hotWordGroupService; private final HotWordService hotWordService; private final AsrModelMapper asrModelMapper; private final LlmModelMapper llmModelMapper; @@ -41,6 +44,7 @@ public class MeetingRuntimeProfileResolverImpl implements MeetingRuntimeProfileR Boolean enableItn, Boolean enableTextRefine, Boolean saveAudio, + Long hotWordGroupId, List hotWords) { long resolvedTenantId = tenantId == null ? 0L : tenantId; AiModelVO asrModel = resolveModel("ASR", asrModelId, resolvedTenantId); @@ -54,7 +58,7 @@ public class MeetingRuntimeProfileResolverImpl implements MeetingRuntimeProfileR profile.setResolvedSummaryModelName(summaryModel.getModelName()); profile.setResolvedPromptId(promptTemplate.getId()); profile.setResolvedPromptName(promptTemplate.getTemplateName()); - profile.setResolvedHotWordGroupId(resolveHotWordGroupId(promptTemplate, hotWords)); + profile.setResolvedHotWordGroupId(resolveHotWordGroupId(resolvedTenantId, promptTemplate, hotWordGroupId, hotWords)); profile.setResolvedMode(nonBlank(mode, "2pass")); profile.setResolvedLanguage(nonBlank(language, "auto")); profile.setResolvedUseSpkId(useSpkId != null ? useSpkId : 1); @@ -62,11 +66,18 @@ public class MeetingRuntimeProfileResolverImpl implements MeetingRuntimeProfileR profile.setResolvedEnableItn(enableItn != null ? enableItn : Boolean.TRUE); profile.setResolvedEnableTextRefine(Boolean.TRUE.equals(enableTextRefine)); profile.setResolvedSaveAudio(Boolean.TRUE.equals(saveAudio)); - profile.setResolvedHotWords(resolveHotWords(promptTemplate, hotWords)); + profile.setResolvedHotWords(resolveHotWords(resolvedTenantId, promptTemplate, hotWordGroupId, hotWords)); return profile; } - private Long resolveHotWordGroupId(PromptTemplate promptTemplate, List hotWords) { + private Long resolveHotWordGroupId(Long tenantId, PromptTemplate promptTemplate, Long hotWordGroupId, List hotWords) { + if (hotWordGroupId != null) { + if (hotWordGroupId <= 0) { + return null; + } + assertHotWordGroupAvailable(hotWordGroupId, tenantId); + return hotWordGroupId; + } List normalized = normalizeHotWords(hotWords); if (!normalized.isEmpty()) { return null; @@ -74,7 +85,20 @@ public class MeetingRuntimeProfileResolverImpl implements MeetingRuntimeProfileR return promptTemplate == null ? null : promptTemplate.getHotWordGroupId(); } - private List resolveHotWords(PromptTemplate promptTemplate, List hotWords) { + private List resolveHotWords(Long tenantId, PromptTemplate promptTemplate, Long hotWordGroupId, List hotWords) { + if (hotWordGroupId != null) { + if (hotWordGroupId <= 0) { + return List.of(); + } + assertHotWordGroupAvailable(hotWordGroupId, tenantId); + return hotWordService.listEnabledByGroupIdIgnoreTenant(hotWordGroupId).stream() + .map(HotWord::getWord) + .filter(Objects::nonNull) + .map(String::trim) + .filter(item -> !item.isEmpty()) + .distinct() + .toList(); + } List normalized = normalizeHotWords(hotWords); if (!normalized.isEmpty()) { return normalized; @@ -92,6 +116,15 @@ public class MeetingRuntimeProfileResolverImpl implements MeetingRuntimeProfileR .toList(); } + private void assertHotWordGroupAvailable(Long hotWordGroupId, Long tenantId) { + boolean visible = hotWordGroupService.listVisibleOptions(tenantId).stream() + .map(HotWordGroupVO::getId) + .anyMatch(id -> Objects.equals(id, hotWordGroupId)); + if (!visible) { + throw new RuntimeException("热词组不存在或不可用"); + } + } + private List normalizeHotWords(List hotWords) { return hotWords == null ? List.of() : hotWords.stream() .filter(Objects::nonNull) diff --git a/backend/src/test/java/com/imeeting/service/biz/impl/MeetingAccessServiceImplTest.java b/backend/src/test/java/com/imeeting/service/biz/impl/MeetingAccessServiceImplTest.java index 8dce5a0..7df5833 100644 --- a/backend/src/test/java/com/imeeting/service/biz/impl/MeetingAccessServiceImplTest.java +++ b/backend/src/test/java/com/imeeting/service/biz/impl/MeetingAccessServiceImplTest.java @@ -40,6 +40,26 @@ class MeetingAccessServiceImplTest { () -> service.assertCanControlRealtimeMeeting(meeting, loginUser, MeetingConstants.SOURCE_WEB)); } + @Test + void allowsParticipantToViewAndExportButNotEdit() { + Meeting meeting = buildMeeting(MeetingConstants.TYPE_OFFLINE, MeetingConstants.SOURCE_WEB); + meeting.setParticipants("201,202,203"); + LoginUser participant = new LoginUser(202L, 100L, "participant", false, false, null); + + assertDoesNotThrow(() -> service.assertCanViewMeeting(meeting, participant)); + assertDoesNotThrow(() -> service.assertCanExportMeeting(meeting, participant)); + assertThrows(RuntimeException.class, () -> service.assertCanEditMeeting(meeting, participant)); + assertThrows(RuntimeException.class, () -> service.assertCanManageRealtimeMeeting(meeting, participant)); + } + + @Test + void allowsTenantAdminToEditMeeting() { + Meeting meeting = buildMeeting(MeetingConstants.TYPE_OFFLINE, MeetingConstants.SOURCE_WEB); + LoginUser tenantAdmin = new LoginUser(300L, 100L, "tenant-admin", false, true, null); + + assertDoesNotThrow(() -> service.assertCanEditMeeting(meeting, tenantAdmin)); + } + private Meeting buildMeeting(String meetingType, String meetingSource) { Meeting meeting = new Meeting(); meeting.setTenantId(100L); diff --git a/backend/src/test/java/com/imeeting/service/biz/impl/MeetingRuntimeProfileResolverImplTest.java b/backend/src/test/java/com/imeeting/service/biz/impl/MeetingRuntimeProfileResolverImplTest.java index e96d9d7..de53411 100644 --- a/backend/src/test/java/com/imeeting/service/biz/impl/MeetingRuntimeProfileResolverImplTest.java +++ b/backend/src/test/java/com/imeeting/service/biz/impl/MeetingRuntimeProfileResolverImplTest.java @@ -2,6 +2,7 @@ package com.imeeting.service.biz.impl; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.imeeting.dto.biz.AiModelVO; +import com.imeeting.dto.biz.HotWordGroupVO; import com.imeeting.dto.biz.RealtimeMeetingRuntimeProfile; import com.imeeting.entity.biz.AsrModel; import com.imeeting.entity.biz.HotWord; @@ -10,6 +11,7 @@ import com.imeeting.entity.biz.PromptTemplate; import com.imeeting.mapper.biz.AsrModelMapper; import com.imeeting.mapper.biz.LlmModelMapper; import com.imeeting.service.biz.AiModelService; +import com.imeeting.service.biz.HotWordGroupService; import com.imeeting.service.biz.HotWordService; import com.imeeting.service.biz.PromptTemplateService; import org.junit.jupiter.api.Test; @@ -30,10 +32,12 @@ class MeetingRuntimeProfileResolverImplTest { void resolveShouldUseRequestedResourcesAndNormalizeHotWords() { AiModelService aiModelService = mock(AiModelService.class); PromptTemplateService promptTemplateService = mock(PromptTemplateService.class); + HotWordGroupService hotWordGroupService = mock(HotWordGroupService.class); HotWordService hotWordService = mock(HotWordService.class); MeetingRuntimeProfileResolverImpl resolver = new MeetingRuntimeProfileResolverImpl( aiModelService, promptTemplateService, + hotWordGroupService, hotWordService, mock(AsrModelMapper.class), mock(LlmModelMapper.class) @@ -55,6 +59,7 @@ class MeetingRuntimeProfileResolverImplTest { null, Boolean.TRUE, Boolean.TRUE, + null, Arrays.asList(" alpha ", "", "alpha", "beta", null) ); @@ -78,10 +83,12 @@ class MeetingRuntimeProfileResolverImplTest { void resolveShouldRejectCrossTenantModel() { AiModelService aiModelService = mock(AiModelService.class); PromptTemplateService promptTemplateService = mock(PromptTemplateService.class); + HotWordGroupService hotWordGroupService = mock(HotWordGroupService.class); HotWordService hotWordService = mock(HotWordService.class); MeetingRuntimeProfileResolverImpl resolver = new MeetingRuntimeProfileResolverImpl( aiModelService, promptTemplateService, + hotWordGroupService, hotWordService, mock(AsrModelMapper.class), mock(LlmModelMapper.class) @@ -101,6 +108,7 @@ class MeetingRuntimeProfileResolverImplTest { null, null, null, + null, List.of() )); } @@ -109,10 +117,12 @@ class MeetingRuntimeProfileResolverImplTest { void resolveShouldUseTemplateBoundGroupWhenNoExplicitHotWords() { AiModelService aiModelService = mock(AiModelService.class); PromptTemplateService promptTemplateService = mock(PromptTemplateService.class); + HotWordGroupService hotWordGroupService = mock(HotWordGroupService.class); HotWordService hotWordService = mock(HotWordService.class); MeetingRuntimeProfileResolverImpl resolver = new MeetingRuntimeProfileResolverImpl( aiModelService, promptTemplateService, + hotWordGroupService, hotWordService, mock(AsrModelMapper.class), mock(LlmModelMapper.class) @@ -142,6 +152,7 @@ class MeetingRuntimeProfileResolverImplTest { null, Boolean.FALSE, Boolean.FALSE, + null, null ); @@ -153,12 +164,14 @@ class MeetingRuntimeProfileResolverImplTest { void resolveShouldFallbackToFirstEnabledModelUsingSortOrder() { AiModelService aiModelService = mock(AiModelService.class); PromptTemplateService promptTemplateService = mock(PromptTemplateService.class); + HotWordGroupService hotWordGroupService = mock(HotWordGroupService.class); HotWordService hotWordService = mock(HotWordService.class); AsrModelMapper asrModelMapper = mock(AsrModelMapper.class); LlmModelMapper llmModelMapper = mock(LlmModelMapper.class); MeetingRuntimeProfileResolverImpl resolver = new MeetingRuntimeProfileResolverImpl( aiModelService, promptTemplateService, + hotWordGroupService, hotWordService, asrModelMapper, llmModelMapper @@ -184,6 +197,7 @@ class MeetingRuntimeProfileResolverImplTest { null, null, null, + null, List.of() ); @@ -191,6 +205,55 @@ class MeetingRuntimeProfileResolverImplTest { assertEquals(22L, profile.getResolvedSummaryModelId()); } + @Test + void resolveShouldPreferExplicitHotWordGroupOverTemplateBinding() { + AiModelService aiModelService = mock(AiModelService.class); + PromptTemplateService promptTemplateService = mock(PromptTemplateService.class); + HotWordGroupService hotWordGroupService = mock(HotWordGroupService.class); + HotWordService hotWordService = mock(HotWordService.class); + MeetingRuntimeProfileResolverImpl resolver = new MeetingRuntimeProfileResolverImpl( + aiModelService, + promptTemplateService, + hotWordGroupService, + hotWordService, + mock(AsrModelMapper.class), + mock(LlmModelMapper.class) + ); + + when(aiModelService.getModelById(11L, "ASR")).thenReturn(enabledModel(11L, 1L, "ASR-Model")); + when(aiModelService.getModelById(22L, "LLM")).thenReturn(enabledModel(22L, 1L, "LLM-Model")); + PromptTemplate template = enabledPrompt(33L, 1L, "Summary Prompt"); + template.setHotWordGroupId(99L); + when(promptTemplateService.getById(33L)).thenReturn(template); + + HotWordGroupVO explicitGroup = new HotWordGroupVO(); + explicitGroup.setId(88L); + when(hotWordGroupService.listVisibleOptions(1L)).thenReturn(List.of(explicitGroup)); + + HotWord hotWord = new HotWord(); + hotWord.setWord("override"); + when(hotWordService.listEnabledByGroupIdIgnoreTenant(88L)).thenReturn(List.of(hotWord)); + + RealtimeMeetingRuntimeProfile profile = resolver.resolve( + 1L, + 11L, + 22L, + 33L, + null, + null, + null, + null, + null, + null, + null, + 88L, + List.of() + ); + + assertEquals(88L, profile.getResolvedHotWordGroupId()); + assertIterableEquals(List.of("override"), profile.getResolvedHotWords()); + } + private AiModelVO enabledModel(Long id, Long tenantId, String name) { AiModelVO model = new AiModelVO(); model.setId(id); diff --git a/frontend/src/api/business/meeting.ts b/frontend/src/api/business/meeting.ts index e0ad2cd..ecbd3e0 100644 --- a/frontend/src/api/business/meeting.ts +++ b/frontend/src/api/business/meeting.ts @@ -37,6 +37,23 @@ export interface MeetingVO { createdAt: string; } +const AUDIO_MIME_TYPE_BY_EXTENSION: Record = { + mp3: "audio/mpeg", + wav: "audio/wav", + m4a: "audio/mp4", + mp4: "audio/mp4", + aac: "audio/aac", +}; + +export const resolveAudioMimeType = (audioUrl?: string) => { + if (!audioUrl) { + return undefined; + } + const normalizedUrl = audioUrl.split("#")[0]?.split("?")[0] || ""; + const extension = normalizedUrl.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase(); + return extension ? AUDIO_MIME_TYPE_BY_EXTENSION[extension] : undefined; +}; + export interface CreateMeetingCommand { id?: number; title: string; @@ -49,6 +66,7 @@ export interface CreateMeetingCommand { asrModelId: number; summaryModelId?: number; promptId: number; + hotWordGroupId?: number; userPrompt?: string; useSpkId?: number; enableTextRefine?: boolean; @@ -67,6 +85,7 @@ export interface CreateRealtimeMeetingCommand { asrModelId: number; summaryModelId?: number; promptId: number; + hotWordGroupId?: number; userPrompt?: string; mode?: string; language?: string; diff --git a/frontend/src/components/business/MeetingCreateDrawer.tsx b/frontend/src/components/business/MeetingCreateDrawer.tsx index b7bc526..fd82677 100644 --- a/frontend/src/components/business/MeetingCreateDrawer.tsx +++ b/frontend/src/components/business/MeetingCreateDrawer.tsx @@ -7,6 +7,7 @@ import { useNavigate } from 'react-router-dom'; import { getAiModelPage, getAiModelDefault, AiModelVO } from '../../api/business/aimodel'; import { getPromptPage, PromptTemplateVO } from '../../api/business/prompt'; import { getHotWordPage, HotWordVO } from '../../api/business/hotword'; +import { getHotWordGroupOptions, HotWordGroupVO } from '../../api/business/hotwordGroup'; import { listUsers, pageParams } from '../../api'; import { createMeeting, createRealtimeMeeting, uploadAudio, CreateRealtimeMeetingCommand } from '../../api/business/meeting'; import { SysUser } from '../../types'; @@ -71,7 +72,9 @@ export const MeetingCreateDrawer: React.FC = ({ open, const [llmModels, setLlmModels] = useState([]); const [prompts, setPrompts] = useState([]); const [hotwordList, setHotwordList] = useState([]); + const [hotWordGroups, setHotWordGroups] = useState([]); const [userList, setUserList] = useState([]); + const [hotWordGroupTouched, setHotWordGroupTouched] = useState(false); const [audioUrl, setAudioUrl] = useState(''); const [uploadProgress, setUploadProgress] = useState(0); @@ -80,15 +83,18 @@ export const MeetingCreateDrawer: React.FC = ({ open, const watchedAsrModelId = Form.useWatch("asrModelId", form); const watchedPromptId = Form.useWatch("promptId", form); + const watchedHotWordGroupId = Form.useWatch("hotWordGroupId", form); const watchedSummaryModelId = Form.useWatch("summaryModelId", form); const selectedAsrModel = useMemo(() => asrModels.find((item) => item.id === watchedAsrModelId) || null, [asrModels, watchedAsrModelId]); const selectedSummaryModel = useMemo(() => llmModels.find((item) => item.id === watchedSummaryModelId) || null, [llmModels, watchedSummaryModelId]); + const selectedPrompt = useMemo(() => prompts.find((item) => item.id === watchedPromptId) || null, [prompts, watchedPromptId]); const offlineAudioMaxSizeBytes = useMemo(() => offlineAudioMaxSizeMb * 1024 * 1024, [offlineAudioMaxSizeMb]); useEffect(() => { if (open) { setType(initialType); + setHotWordGroupTouched(false); loadInitialData(); setAudioUrl(''); setUploadProgress(0); @@ -96,14 +102,22 @@ export const MeetingCreateDrawer: React.FC = ({ open, } }, [open, initialType]); + useEffect(() => { + if (!open || hotWordGroupTouched) { + return; + } + form.setFieldValue('hotWordGroupId', selectedPrompt?.hotWordGroupId); + }, [form, hotWordGroupTouched, open, selectedPrompt]); + const loadInitialData = async () => { setLoading(true); try { - const [asrRes, llmRes, promptRes, hotwordRes, users, defaultAsr, defaultLlm] = await Promise.all([ + const [asrRes, llmRes, promptRes, hotwordRes, hotWordGroupRes, users, defaultAsr, defaultLlm] = await Promise.all([ getAiModelPage({ current: 1, size: 100, type: 'ASR' }), getAiModelPage({ current: 1, size: 100, type: 'LLM' }), getPromptPage({ current: 1, size: 100 }), getHotWordPage({ current: 1, size: 1000 }), + getHotWordGroupOptions(), listUsers(), getAiModelDefault("ASR"), getAiModelDefault("LLM"), @@ -118,15 +132,18 @@ export const MeetingCreateDrawer: React.FC = ({ open, setLlmModels(activeLlmModels); setPrompts(activePrompts); setHotwordList(activeHotwords); + setHotWordGroups((hotWordGroupRes.data.data || []).filter((item: HotWordGroupVO) => item.status === 1)); setUserList(users || []); setOfflineAudioMaxSizeMb(await loadOfflineAudioMaxSizeMb()); + const defaultPrompt = activePrompts.length > 0 ? activePrompts[0] : undefined; form.setFieldsValue({ title: type === 'upload' ? `文件会议 ${dayjs().format("MM-DD HH:mm")}` : `实时会议 ${dayjs().format("MM-DD HH:mm")}`, meetingTime: dayjs(), asrModelId: defaultAsr.data.data?.id, summaryModelId: defaultLlm.data.data?.id, - promptId: activePrompts.length > 0 ? activePrompts[0].id : undefined, + promptId: defaultPrompt?.id, + hotWordGroupId: defaultPrompt?.hotWordGroupId, useSpkId: 1, enableTextRefine: false, mode: "2pass", @@ -223,6 +240,12 @@ export const MeetingCreateDrawer: React.FC = ({ open, setSubmitting(true); try { const { hostUserId, ...meetingValues } = values; + const selectedHotWords = meetingValues.hotWordGroupId == null || meetingValues.hotWordGroupId === 0 + ? undefined + : hotwordList + .filter((item) => item.hotWordGroupId === meetingValues.hotWordGroupId) + .map((item) => item.word) + .filter((word) => !!word?.trim()); if (type === 'upload') { await createMeeting({ @@ -231,16 +254,19 @@ export const MeetingCreateDrawer: React.FC = ({ open, meetingTime: meetingValues.meetingTime.format('YYYY-MM-DD HH:mm:ss'), audioUrl, participants: meetingValues.participants?.join(','), - tags: meetingValues.tags?.join(',') + tags: meetingValues.tags?.join(','), + hotWords: selectedHotWords }); message.success('会议发起成功'); onSuccess(); onCancel(); } else { - const selectedHotwords = hotwordList.map((item) => ({ - hotword: item.word, - weight: Number(item.weight || 2) / 10, - })); + const selectedHotwords = hotwordList + .filter((item) => item.hotWordGroupId === meetingValues.hotWordGroupId && meetingValues.hotWordGroupId !== 0) + .map((item) => ({ + hotword: item.word, + weight: Number(item.weight || 2) / 10, + })); const payload: CreateRealtimeMeetingCommand = { ...meetingValues, @@ -255,6 +281,7 @@ export const MeetingCreateDrawer: React.FC = ({ open, enableItn: meetingValues.enableItn !== false, enableTextRefine: !!meetingValues.enableTextRefine, saveAudio: !!meetingValues.saveAudio, + hotWords: selectedHotWords, }; const res = await createRealtimeMeeting(payload); @@ -426,6 +453,71 @@ export const MeetingCreateDrawer: React.FC = ({ open, )} + {/* + + ({ + label: `${item.groupName} (${item.hotWordCount}/200)`, + value: item.id, + })), + ]} + onChange={() => setHotWordGroupTouched(true)} + /> + + */} + + +