New features + bug fixes

- Improved Options page
- Added Whisper API transcription logic for voice commands
- Added custom question box
- Multiple bug fixes with model selection and defaults.

Author: TiesdeKok

package-lock.json

@@ -26,7 +26,8 @@
     "rehype-highlight": "^6.0.0",
     "swr": "^2.0.0",
     "uuid": "^9.0.0",
-    "marked": "^4.2.12"
+    "marked": "^4.2.12",
+    "react-rnd": "10.4.1"
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.13",
@@ -36,6 +37,7 @@
     "@typescript-eslint/eslint-plugin": "^5.47.0",
     "@typescript-eslint/parser": "^5.47.0",
     "@types/marked": "^4.0.8",
+    "@primer/octicons-react": "^17.9.0",
     "archiver": "^5.3.1",
     "autoprefixer": "^10.4.13",
     "chokidar-cli": "^3.0.0",

package.json

@@ -3,6 +3,7 @@ import { getProviderConfigs, ProviderType } from '../config'
 import { ChatGPTProvider, getChatGPTAccessToken, sendMessageFeedback } from './providers/chatgpt'
 import { OpenAIProvider } from './providers/openai'
 import { Provider } from './types'
+import { Buffer } from 'buffer'
 
 async function generateAnswers(port: Browser.Runtime.Port, question: string) {
     const providerConfigs = await getProviderConfigs()
@@ -29,40 +30,89 @@ async function generateAnswers(port: Browser.Runtime.Port, question: string) {
       prompt: question,
       signal: controller.signal,
       onEvent(event) {
+        console.log("debug")
         if (event.type === 'done') {
           port.postMessage({ event: 'DONE' })
           return
         }
         port.postMessage(event.data)
       },
     })
+}
+
+function dataURLtoBlob(dataUrl: string): Blob {
+    const base64Regex = /^data:.+\/(.+);base64,(.*)$/;
+    const match = dataUrl.match(base64Regex);
+    if (!match) {
+      throw new Error('Invalid data URL format');
+    }
+  
+    const mimeString = match[1];
+    const base64Data = match[2];
+    const byteString = Buffer.from(base64Data, 'base64').toString('binary');
+    const ab = new ArrayBuffer(byteString.length);
+    const ia = new Uint8Array(ab);
+    for (let i = 0; i < byteString.length; i++) {
+      ia[i] = byteString.charCodeAt(i);
+    }
+    return new Blob([ab], { type: mimeString });
   }
 
-Browser.runtime.onConnect.addListener((port) => {
-  port.onMessage.addListener(async (msg) => {
-    console.debug('received msg', msg)
+async function transcribeAudio(port: Browser.Runtime.Port, dataUrl: string) {
+    const providerConfigs = await getProviderConfigs();
+
+    const { apiKey } = providerConfigs.configs[ProviderType.API]!;
+    if (!apiKey) {
+        throw new Error('No API token available. Please add it in the config.');
+    }
+
+    const provider = new OpenAIProvider(apiKey, 'whisper-1');
+    const audioBlob = dataURLtoBlob(dataUrl);
+    const audioFile = new File([audioBlob], 'recorded_audio.wav', { type: 'audio/wav' });
+
     try {
-      await generateAnswers(port, msg.question)
+        const data = await provider.transcribeAudio(audioFile);
+        port.postMessage({ event: 'DONE', data });
     } catch (err: any) {
-      console.error(err)
-      port.postMessage({ error: err.message })
+        console.error(err);
+        port.postMessage({ error: err.message });
     }
-  })
-})
+}
 
-Browser.runtime.onMessage.addListener(async (message) => {
-    if (message.type === 'FEEDBACK') {
-      const token = await getChatGPTAccessToken()
-      await sendMessageFeedback(token, message.data)
-    } else if (message.type === 'OPEN_OPTIONS_PAGE') {
-      Browser.runtime.openOptionsPage()
-    } else if (message.type === 'GET_ACCESS_TOKEN') {
-      return getChatGPTAccessToken()
+Browser.runtime.onConnect.addListener((port) => {
+port.onMessage.addListener(async (msg) => {
+    console.debug('received msg', msg);
+    try {
+    if (msg.type === 'GENERATE_ANSWERS') {
+        await generateAnswers(port, msg.question);
+    } else if (msg.type === 'TRANSCRIBE_AUDIO') {
+        console.log("received", msg.dataUrl)
+        await transcribeAudio(port, msg.dataUrl);
     }
-  })
-  
-  Browser.runtime.onInstalled.addListener((details) => {
-    if (details.reason === 'install') {
-      Browser.runtime.openOptionsPage()
+    } catch (err: any) {
+    console.error(err);
+    port.postMessage({ error: err.message });
     }
-  })
+});
+});
+
+Browser.runtime.onMessage.addListener(async (message) => {
+if (message.type === 'FEEDBACK') {
+    const token = await getChatGPTAccessToken()
+    await sendMessageFeedback(token, message.data)
+} else if (message.type === 'OPEN_OPTIONS_PAGE') {
+    Browser.runtime.openOptionsPage()
+} else if (message.type === 'GET_ACCESS_TOKEN') {
+    return getChatGPTAccessToken()
+}  else if (message.type === 'CHECK_API_KEY') {
+    const providerConfigs = await getProviderConfigs();
+    const { apiKey } = providerConfigs.configs[ProviderType.API]!;
+    return !!apiKey;
+}
+})
+
+Browser.runtime.onInstalled.addListener((details) => {
+if (details.reason === 'install') {
+    Browser.runtime.openOptionsPage()
+}
+})

src/background/index.ts

@@ -92,8 +92,6 @@ export class ChatGPTProvider implements Provider {
       }
     }
 
-    
-
     //const modelName = await this.getModelName()
     console.log('Using model:', this.model)
 

src/background/providers/chatgpt.ts

@@ -91,4 +91,32 @@ export class OpenAIProvider implements Provider {
     })
     return {}
   }
+
+  async transcribeAudio(file: File, prompt?: string): Promise<any> {
+    const endpoint = 'https://api.openai.com/v1/audio/transcriptions';
+
+    const formData = new FormData();
+    formData.append("file", file);
+    formData.append("model", "whisper-1");
+    formData.append("response_format", "json");
+    formData.append("temperature", "0");
+  
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.token}`,
+      },
+      body: formData,
+    });
+  
+    if (!response.ok) {
+        const errorData = await response.json();
+        throw new Error(`Error: ${response.status} ${response.statusText} - ${JSON.stringify(errorData)}`);
+      }
+  
+    const data = await response.json();
+    
+    console.log(data)
+    return data;
+  }
 }