Skip to content

Commit de4ee0f

Browse files
nickvergessennickvergessen
authored
Merge pull request #13822 from nextcloud/backport/13807/stable30
[stable30] feat(AI-call-summary): Automatically summarize call transcript
2 parents b643e70 + 4c2d477 commit de4ee0f

File tree

9 files changed

+284
-74
lines changed

9 files changed

+284
-74
lines changed

‎docs/settings.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@ Legend:
103103
| `calls_start_without_media` | string<br>`yes` or `no` | `no` | Yes | | Whether participants start with enabled or disabled audio and video by default |
104104
| `breakout_rooms` | string<br>`yes` or `no` | `yes` | Yes | | Whether or not breakout rooms are allowed (Will only prevent creating new breakout rooms. Existing conversations are not modified.) |
105105
| `call_recording` | string<br>`yes` or `no` | `yes` | Yes | | Enable call recording |
106-
| `call_recording_transcription` | string<br>`yes` or `no` | `no` | No | | Whether call recordings should automatically be transcripted when a transcription provider is enabled. |
106+
| `call_recording_summary` | string<br>`yes` or `no` | `no` | No | | Whether call recordings should automatically be summarized when a transcription and summary provider is enabled. |
107+
| `call_recording_transcription` | string<br>`yes` or `no` | `no` | No | | Whether call recordings should automatically be transcribed when a transcription provider is enabled. |
107108
| `sip_dialout` | string<br>`yes` or `no` | `no` | Yes | | SIP dial-out is allowed when a SIP bridge is configured |
108109
| `federation_enabled` | string<br>`yes` or `no` | `no` | Yes | | 🏗️ *Work in progress:* Whether or not federation with this instance is allowed |
109110
| `federation_incoming_enabled` | string<br>`1` or `0` | `1` | Yes | | 🏗️ *Work in progress:* Whether users of this instance can be invited to federated conversations |

‎lib/AppInfo/Application.php

+4-4
Original file line numberDiff line numberDiff line change
@@ -148,8 +148,8 @@
148148
use OCP\Share\Events\BeforeShareCreatedEvent;
149149
use OCP\Share\Events\ShareCreatedEvent;
150150
use OCP\Share\Events\VerifyMountPointEvent;
151-
use OCP\SpeechToText\Events\TranscriptionFailedEvent;
152-
use OCP\SpeechToText\Events\TranscriptionSuccessfulEvent;
151+
use OCP\TaskProcessing\Events\TaskFailedEvent;
152+
use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
153153
use OCP\User\Events\BeforeUserLoggedOutEvent;
154154
use OCP\User\Events\UserChangedEvent;
155155
use OCP\User\Events\UserDeletedEvent;
@@ -273,8 +273,8 @@ public function register(IRegistrationContext $context): void {
273273
$context->registerEventListener(RoomDeletedEvent::class, RecordingListener::class);
274274
$context->registerEventListener(CallEndedEvent::class, RecordingListener::class);
275275
$context->registerEventListener(CallEndedForEveryoneEvent::class, RecordingListener::class);
276-
$context->registerEventListener(TranscriptionSuccessfulEvent::class, RecordingListener::class);
277-
$context->registerEventListener(TranscriptionFailedEvent::class, RecordingListener::class);
276+
$context->registerEventListener(TaskSuccessfulEvent::class, RecordingListener::class);
277+
$context->registerEventListener(TaskFailedEvent::class, RecordingListener::class);
278278

279279
// Federation listeners
280280
$context->registerEventListener(BeforeRoomDeletedEvent::class, TalkV1BeforeRoomDeletedListener::class);

‎lib/Controller/RecordingController.php

+3-2
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,8 @@ public function notificationDismiss(int $timestamp): DataResponse {
418418
$this->recordingService->notificationDismiss(
419419
$this->getRoom(),
420420
$this->participant,
421-
$timestamp
421+
$timestamp,
422+
null, // FIXME we would/should extend the URL, but the iOS app is crafting it manually atm due to OS limitations
422423
);
423424
} catch (InvalidArgumentException $e) {
424425
return new DataResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
@@ -446,7 +447,7 @@ public function shareToChat(int $fileId, int $timestamp): DataResponse {
446447
$this->getRoom(),
447448
$this->participant,
448449
$fileId,
449-
$timestamp
450+
$timestamp,
450451
);
451452
} catch (InvalidArgumentException $e) {
452453
return new DataResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);

‎lib/Notification/Notifier.php

+8-2
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ public function prepare(INotification $notification, string $languageCode): INot
240240
->setLink($this->url->linkToRouteAbsolute('spreed.Page.showCall', ['token' => $room->getToken()]));
241241

242242
$subject = $notification->getSubject();
243-
if ($subject === 'record_file_stored' || $subject === 'transcript_file_stored' || $subject === 'transcript_failed') {
243+
if ($subject === 'record_file_stored' || $subject === 'transcript_file_stored' || $subject === 'transcript_failed' || $subject === 'summary_file_stored' || $subject === 'summary_failed') {
244244
return $this->parseStoredRecording($notification, $room, $participant, $l);
245245
}
246246
if ($subject === 'record_file_store_fail') {
@@ -363,9 +363,15 @@ protected function parseStoredRecording(
363363
} elseif ($notification->getSubject() === 'transcript_file_stored') {
364364
$subject = $l->t('Transcript now available');
365365
$message = $l->t('The transcript for the call in {call} was uploaded to {file}.');
366-
} else {
366+
} elseif ($notification->getSubject() === 'transcript_failed') {
367367
$subject = $l->t('Failed to transcript call recording');
368368
$message = $l->t('The server failed to transcript the recording at {file} for the call in {call}. Please reach out to the administration.');
369+
} elseif ($notification->getSubject() === 'summary_file_stored') {
370+
$subject = $l->t('Call summary now available');
371+
$message = $l->t('The summary for the call in {call} was uploaded to {file}.');
372+
} else {
373+
$subject = $l->t('Failed to summarize call recording');
374+
$message = $l->t('The server failed to summarize the recording at {file} for the call in {call}. Please reach out to the administration.');
369375
}
370376

371377
$notification

‎lib/Recording/Listener.php

+32-27
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@
2020
use OCA\Talk\Service\RecordingService;
2121
use OCP\EventDispatcher\Event;
2222
use OCP\EventDispatcher\IEventListener;
23-
use OCP\Files\File;
2423
use OCP\Files\IRootFolder;
25-
use OCP\SpeechToText\Events\AbstractTranscriptionEvent;
26-
use OCP\SpeechToText\Events\TranscriptionFailedEvent;
27-
use OCP\SpeechToText\Events\TranscriptionSuccessfulEvent;
24+
use OCP\TaskProcessing\Events\AbstractTaskProcessingEvent;
25+
use OCP\TaskProcessing\Events\TaskFailedEvent;
26+
use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
27+
use Psr\Log\LoggerInterface;
2828

2929
/**
3030
* @template-implements IEventListener<Event>
@@ -34,12 +34,17 @@ public function __construct(
3434
protected RecordingService $recordingService,
3535
protected ConsentService $consentService,
3636
protected IRootFolder $rootFolder,
37+
protected LoggerInterface $logger,
3738
) {
3839
}
3940

4041
public function handle(Event $event): void {
41-
if ($event instanceof AbstractTranscriptionEvent) {
42-
$this->handleTranscriptionEvents($event);
42+
if ($event instanceof AbstractTaskProcessingEvent) {
43+
try {
44+
$this->handleTranscriptionEvents($event);
45+
} catch (\Throwable $e) {
46+
$this->logger->error('An error occurred while processing recording AI follow-up task', ['exception' => $e]);
47+
}
4348
return;
4449
}
4550

@@ -54,40 +59,40 @@ public function handle(Event $event): void {
5459
};
5560
}
5661

57-
public function handleTranscriptionEvents(AbstractTranscriptionEvent $event): void {
58-
if ($event->getAppId() !== Application::APP_ID) {
62+
public function handleTranscriptionEvents(AbstractTaskProcessingEvent $event): void {
63+
$task = $event->getTask();
64+
if ($task->getAppId() !== Application::APP_ID) {
5965
return;
6066
}
6167

62-
if ($event instanceof TranscriptionSuccessfulEvent) {
63-
$this->successfulTranscript($event->getUserId(), $event->getFile(), $event->getTranscript());
64-
} elseif ($event instanceof TranscriptionFailedEvent) {
65-
$this->failedTranscript($event->getUserId(), $event->getFile());
66-
}
67-
}
68-
69-
protected function successfulTranscript(?string $owner, ?File $fileNode, string $transcript): void {
70-
if (!$fileNode instanceof File) {
68+
// 'call/transcription/' . $room->getToken()
69+
$customId = $task->getCustomId();
70+
if (str_starts_with($customId, 'call/transcription/')) {
71+
$aiType = 'transcript';
72+
$roomToken = substr($customId, strlen('call/transcription/'));
73+
74+
$fileId = (int)($task->getInput()['input'] ?? null);
75+
} elseif (str_starts_with($customId, 'call/summary/')) {
76+
$aiType = 'summary';
77+
[$roomToken, $fileId] = explode('/', substr($customId, strlen('call/summary/')));
78+
$fileId = (int)$fileId;
79+
} else {
7180
return;
7281
}
7382

74-
if ($owner === null) {
83+
if ($fileId === 0) {
7584
return;
7685
}
7786

78-
$this->recordingService->storeTranscript($owner, $fileNode, $transcript);
79-
}
80-
81-
protected function failedTranscript(?string $owner, ?File $fileNode): void {
82-
if (!$fileNode instanceof File) {
87+
if ($task->getUserId() === null) {
8388
return;
8489
}
8590

86-
if ($owner === null) {
87-
return;
91+
if ($event instanceof TaskSuccessfulEvent) {
92+
$this->recordingService->storeTranscript($task->getUserId(), $roomToken, $fileId, $task->getOutput()['output'] ?? '', $aiType);
93+
} elseif ($event instanceof TaskFailedEvent) {
94+
$this->recordingService->notifyAboutFailedTranscript($task->getUserId(), $roomToken, $fileId, $aiType);
8895
}
89-
90-
$this->recordingService->notifyAboutFailedTranscript($owner, $fileNode);
9196
}
9297

9398
protected function roomDeleted(RoomDeletedEvent $event): void {

0 commit comments

Comments
 (0)