feat(audio-chat): store and use potential remote_audio_expires_at

Signed-off-by: Julien Veyssier <julien-nc@posteo.net>

Author: julien-nc

lib/Controller/ChattyLLMController.php

@@ -571,6 +571,7 @@ public function generateForSession(int $sessionId, int $agencyConfirm = 0): JSON
 	private function getAudioHistory(array $history): array {
 		// history is a list of JSON strings
 		// the content is the remote audio ID (or the transcription as fallback)
+		// we only use the audio ID for assistant messages, if we have one and if it's not expired
 		return array_map(static function (Message $message) {
 			$entry = [
 				'role' => $message->getRole(),
@@ -581,10 +582,15 @@ private function getAudioHistory(array $history): array {
 				&& $attachments[0]['type'] === 'Audio'
 				&& isset($attachments[0]['remote_audio_id'])
 			) {
-				$entry['audio'] = ['id' => $attachments[0]['remote_audio_id']];
-			} else {
-				$entry['content'] = $message->getContent();
+				if (!isset($attachments[0]['remote_audio_expires_at'])
+					|| time() < $attachments[0]['remote_audio_expires_at']
+				) {
+					$entry['audio'] = ['id' => $attachments[0]['remote_audio_id']];
+					return json_encode($entry);
+				}
 			}
+
+			$entry['content'] = $message->getContent();
 			return json_encode($entry);
 		}, $history);
 	}

lib/Listener/ChattyLLMTaskListener.php

@@ -66,34 +66,39 @@ public function handle(Event $event): void {
 			$isAgencyAudioChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentAudioInteraction')
 				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID;
 
+			$taskOutput = $task->getOutput();
+
 			$message = new Message();
 			$message->setSessionId($sessionId);
 			$message->setOcpTaskId($task->getId());
 			$message->setRole('assistant');
 			$message->setTimestamp(time());
-			$sources = json_encode($task->getOutput()['sources'] ?? []);
+			$sources = json_encode($taskOutput['sources'] ?? []);
 			$message->setSources($sources ?: '[]');
 			$message->setAttachments('[]');
 			if ($isAudioChat || $isAgencyAudioChat) {
-				$outputTranscript = trim($task->getOutput()['output_transcript'] ?? '');
+				$outputTranscript = trim($taskOutput['output_transcript'] ?? '');
 				$message->setContent($outputTranscript);
 				// agency might not return any output but just ask for confirmation
 				if ($outputTranscript !== '') {
-					$attachment = ['type' => 'Audio', 'file_id' => $task->getOutput()['output']];
-					if (isset($task->getOutput()['audio_id'])) {
-						$attachment['remote_audio_id'] = $task->getOutput()['audio_id'];
+					$attachment = ['type' => 'Audio', 'file_id' => $taskOutput['output']];
+					if (isset($taskOutput['audio_id'])) {
+						$attachment['remote_audio_id'] = $taskOutput['audio_id'];
+						if (isset($taskOutput['audio_expires_at'])) {
+							$attachment['remote_audio_expires_at'] = $taskOutput['audio_expires_at'];
+						}
 					}
 					$message->setAttachments(json_encode([$attachment]));
 				}
 				// now we have the transcription of the user audio input
 				if (preg_match('/^chatty-llm:\d+:(\d+)$/', $customId, $matches)) {
 					$queryMessageId = (int)$matches[1];
 					$queryMessage = $this->messageMapper->getMessageById($queryMessageId);
-					$queryMessage->setContent(trim($task->getOutput()['input_transcript'] ?? ''));
+					$queryMessage->setContent(trim($taskOutput['input_transcript'] ?? ''));
 					$this->messageMapper->update($queryMessage);
 				}
 			} else {
-				$content = trim($task->getOutput()['output'] ?? '');
+				$content = trim($taskOutput['output'] ?? '');
 				$message->setContent($content);
 				$this->runTtsIfNeeded($sessionId, $message, $taskTypeId, $task->getUserId());
 			}
@@ -106,8 +111,8 @@ public function handle(Event $event): void {
 			// store the conversation token and the actions if we are using the agency feature
 			if ($isAgency || $isAgencyAudioChat) {
 				$session = $this->sessionMapper->getUserSession($task->getUserId(), $sessionId);
-				$conversationToken = ($task->getOutput()['conversation_token'] ?? null) ?: null;
-				$pendingActions = ($task->getOutput()['actions'] ?? null) ?: null;
+				$conversationToken = ($taskOutput['conversation_token'] ?? null) ?: null;
+				$pendingActions = ($taskOutput['actions'] ?? null) ?: null;
 				$session->setAgencyConversationToken($conversationToken);
 				$session->setAgencyPendingActions($pendingActions);
 				$this->sessionMapper->update($session);