feat(audio-chat): add support for optional remote_audio_id which we can use in the history instead of the transcripts

Signed-off-by: Julien Veyssier <julien-nc@posteo.net>

Author: julien-nc

lib/Controller/ChattyLLMController.php

@@ -500,15 +500,6 @@ public function generateForSession(int $sessionId, int $agencyConfirm = 0): JSON
 			do {
 				$lastUserMessage = array_pop($history);
 			} while ($lastUserMessage->getRole() !== 'human');
-			// history is a list of JSON strings
-			// we ignore audio attachments here because they are supposed to have been transcribed, the content is the transcription
-			// this makes the history smaller
-			$history = array_map(static function (Message $message) {
-				return json_encode([
-					'role' => $message->getRole(),
-					'content' => $message->getContent(),
-				]);
-			}, $history);
 
 			$lastAttachments = $lastUserMessage->jsonSerialize()['attachments'];
 			$audioAttachment = $lastAttachments[0] ?? null;
@@ -522,13 +513,22 @@ public function generateForSession(int $sessionId, int $agencyConfirm = 0): JSON
 				&& class_exists('OCP\\TaskProcessing\\TaskTypes\\AudioToAudioChat')
 				&& isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\AudioToAudioChat::ID])
 			) {
+				// for an audio chat task, let's try to get the remote audio IDs for all the previous audio messages
+				$history = $this->getAudioHistory($history);
 				$fileId = $audioAttachment['file_id'];
 				try {
 					$taskId = $this->scheduleAudioChatTask($fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId());
 				} catch (\Exception $e) {
 					return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
 				}
 			} else {
+				// for a text chat task, let's only use text in the history
+				$history = array_map(static function (Message $message) {
+					return json_encode([
+						'role' => $message->getRole(),
+						'content' => $message->getContent(),
+					]);
+				}, $history);
 				try {
 					$taskId = $this->scheduleLLMChatTask($lastUserMessage->getContent(), $systemPrompt, $history, $sessionId);
 				} catch (\Exception $e) {
@@ -540,6 +540,27 @@ public function generateForSession(int $sessionId, int $agencyConfirm = 0): JSON
 		return new JSONResponse(['taskId' => $taskId]);
 	}
 
+	private function getAudioHistory(array $history): array {
+		// history is a list of JSON strings
+		// the content is the remote audio ID (or the transcription as fallback)
+		return array_map(static function (Message $message) {
+			$entry = [
+				'role' => $message->getRole(),
+			];
+			$attachments = $message->jsonSerialize()['attachments'];
+			if ($message->getRole() === 'assistant'
+				&& count($attachments) > 0
+				&& $attachments[0]['type'] === 'Audio'
+				&& isset($attachments[0]['remote_audio_id'])
+			) {
+				$entry['audio'] = ['id' => $attachments[0]['remote_audio_id']];
+			} else {
+				$entry['content'] = $message->getContent();
+			}
+			return json_encode($entry);
+		}, $history);
+	}
+
 	/**
 	 * Regenerate response for a message
 	 *

lib/Listener/ChattyLLMTaskListener.php

@@ -79,7 +79,11 @@ public function handle(Event $event): void {
 				$message->setContent($outputTranscript);
 				// agency might not return any output but just ask for confirmation
 				if ($outputTranscript !== '') {
-					$message->setAttachments('[{"type":"Audio","file_id":' . $task->getOutput()['output'] . '}]');
+					$attachment = ['type' => 'Audio', 'file_id' => $task->getOutput()['output']];
+					if (isset($task->getOutput()['audio_id'])) {
+						$attachment['remote_audio_id'] = $task->getOutput()['audio_id'];
+					}
+					$message->setAttachments(json_encode([$attachment]));
 				}
 				// now we have the transcription of the user audio input
 				if (preg_match('/^chatty-llm:\d+:(\d+)$/', $customId, $matches)) {
@@ -152,6 +156,11 @@ private function runTtsTask(Message $message, ?string $userId): void {
 		$speechFileId = $ttsTaskOutput['speech'];
 		// we need to set "ocp_task_id" here because the file is not an output of the task that produced the message
 		// and we need the task ID + the file ID to load the audio file in the frontend
-		$message->setAttachments('[{"type":"Audio","file_id":' . $speechFileId . ',"ocp_task_id":' . $task->getId() . '}]');
+		$attachment = [
+			'type' => 'Audio',
+			'file_id' => $speechFileId,
+			'ocp_task_id' => $task->getId(),
+		];
+		$message->setAttachments(json_encode([$attachment]));
 	}
 }